diff --git a/crates/rustc_codegen_spirv/src/attr.rs b/crates/rustc_codegen_spirv/src/attr.rs index 0194b5ad79..fe444fb8a5 100644 --- a/crates/rustc_codegen_spirv/src/attr.rs +++ b/crates/rustc_codegen_spirv/src/attr.rs @@ -98,6 +98,8 @@ pub enum SpirvAttribute { // `fn`/closure attributes: UnrollLoops, + InternalBufferLoad, + InternalBufferStore, } // HACK(eddyb) this is similar to `rustc_span::Spanned` but with `value` as the @@ -130,6 +132,8 @@ pub struct AggregatedSpirvAttributes { // `fn`/closure attributes: pub unroll_loops: Option>, + pub internal_buffer_load: Option>, + pub internal_buffer_store: Option>, } struct MultipleAttrs { @@ -211,6 +215,18 @@ impl AggregatedSpirvAttributes { Flat => try_insert(&mut self.flat, (), span, "#[spirv(flat)]"), Invariant => try_insert(&mut self.invariant, (), span, "#[spirv(invariant)]"), UnrollLoops => try_insert(&mut self.unroll_loops, (), span, "#[spirv(unroll_loops)]"), + InternalBufferLoad => try_insert( + &mut self.internal_buffer_load, + (), + span, + "#[spirv(internal_buffer_load)]", + ), + InternalBufferStore => try_insert( + &mut self.internal_buffer_store, + (), + span, + "#[spirv(internal_buffer_store)]", + ), } } } @@ -334,8 +350,9 @@ impl CheckSpirvAttrVisitor<'_> { _ => Err(Expected("function parameter")), }, - - SpirvAttribute::UnrollLoops => match target { + SpirvAttribute::InternalBufferLoad + | SpirvAttribute::InternalBufferStore + | SpirvAttribute::UnrollLoops => match target { Target::Fn | Target::Closure | Target::Method(MethodKind::Trait { body: true } | MethodKind::Inherent) => { diff --git a/crates/rustc_codegen_spirv/src/builder/builder_methods.rs b/crates/rustc_codegen_spirv/src/builder/builder_methods.rs index 7cde901df7..0cae7dd362 100644 --- a/crates/rustc_codegen_spirv/src/builder/builder_methods.rs +++ b/crates/rustc_codegen_spirv/src/builder/builder_methods.rs @@ -1448,12 +1448,13 @@ impl<'a, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'tcx> { SpirvType::Pointer { .. } => match op { IntEQ => { if self.emit().version().unwrap() > (1, 3) { - self.emit() - .ptr_equal(b, None, lhs.def(self), rhs.def(self)) - .map(|result| { - self.zombie_ptr_equal(result, "OpPtrEqual"); - result - }) + let ptr_equal = + self.emit().ptr_equal(b, None, lhs.def(self), rhs.def(self)); + + ptr_equal.map(|result| { + self.zombie_ptr_equal(result, "OpPtrEqual"); + result + }) } else { let int_ty = self.type_usize(); let lhs = self @@ -1831,6 +1832,7 @@ impl<'a, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'tcx> { fn extract_value(&mut self, agg_val: Self::Value, idx: u64) -> Self::Value { let result_type = match self.lookup_type(agg_val.ty) { SpirvType::Adt { field_types, .. } => field_types[idx as usize], + SpirvType::Array { element, .. } | SpirvType::Vector { element, .. } => element, other => self.fatal(&format!( "extract_value not implemented on type {:?}", other @@ -2176,6 +2178,16 @@ impl<'a, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'tcx> { // needing to materialize `&core::panic::Location` or `format_args!`. self.abort(); self.undef(result_type) + } else if self.internal_buffer_load_id.borrow().contains(&callee_val) { + self.codegen_internal_buffer_load(result_type, args) + } else if self.internal_buffer_store_id.borrow().contains(&callee_val) { + self.codegen_internal_buffer_store(args); + + let void_ty = SpirvType::Void.def(rustc_span::DUMMY_SP, self); + SpirvValue { + kind: SpirvValueKind::IllegalTypeUsed(void_ty), + ty: void_ty, + } } else { let args = args.iter().map(|arg| arg.def(self)).collect::>(); self.emit() diff --git a/crates/rustc_codegen_spirv/src/builder/load_store.rs b/crates/rustc_codegen_spirv/src/builder/load_store.rs new file mode 100644 index 0000000000..4ff96a1a6b --- /dev/null +++ b/crates/rustc_codegen_spirv/src/builder/load_store.rs @@ -0,0 +1,598 @@ +use super::Builder; +use crate::builder_spirv::{SpirvValue, SpirvValueExt}; +use crate::codegen_cx::BindlessDescriptorSets; +use crate::rustc_codegen_ssa::traits::BuilderMethods; +use crate::spirv_type::SpirvType; +use rspirv::spirv::Word; +use rustc_target::abi::Align; +use std::convert::TryInto; + +impl<'a, 'tcx> Builder<'a, 'tcx> { + // walk down every member in the ADT recursively and load their values as uints + // this will break up larger data types into uint sized sections, for + // each load, this also has an offset in dwords. + fn recurse_adt_for_stores( + &mut self, + uint_ty: u32, + val: SpirvValue, + base_offset: u32, + uint_values_and_offsets: &mut Vec<(u32, SpirvValue)>, + ) { + let ty = self.lookup_type(val.ty); + + match ty { + SpirvType::Adt { + ref field_types, + ref field_offsets, + ref field_names, + .. + } => { + for (element_idx, (_ty, offset)) in + field_types.iter().zip(field_offsets.iter()).enumerate() + { + let load_res = self.extract_value(val, element_idx as u64); + + if offset.bytes() as u32 % 4 != 0 { + let adt_name = self.type_cache.lookup_name(val.ty); + let field_name = if let Some(field_names) = field_names { + &field_names[element_idx] + } else { + "" + }; + + self.err(&format!( + "Trying to store to unaligned field: `{}::{}`. Field must be aligned to multiple of 4 bytes, but has offset {}", + adt_name, + field_name, + offset.bytes() as u32)); + } + + let offset = offset.bytes() as u32 / 4; + + self.recurse_adt_for_stores( + uint_ty, + load_res, + base_offset + offset, + uint_values_and_offsets, + ); + } + } + SpirvType::Vector { count, element: _ } => { + for offset in 0..count { + let load_res = self.extract_value(val, offset as u64); + + self.recurse_adt_for_stores( + uint_ty, + load_res, + base_offset + offset, + uint_values_and_offsets, + ); + } + } + SpirvType::Array { element: _, count } => { + let count = self + .cx + .builder + .lookup_const_u64(count) + .expect("Array type has invalid count value"); + + for offset in 0..count { + let load_res = self.extract_value(val, offset); + let offset : u32 = offset.try_into().expect("Array count needs to fit in u32"); + + self.recurse_adt_for_stores( + uint_ty, + load_res, + base_offset + offset, + uint_values_and_offsets, + ); + } + } + SpirvType::Float(bits) => { + let unsigned_ty = SpirvType::Integer(bits, false).def(rustc_span::DUMMY_SP, self); + let val_def = val.def(self); + + let bitcast_res = self + .emit() + .bitcast(unsigned_ty, None, val_def) + .unwrap() + .with_type(unsigned_ty); + + self.store_as_u32( + bits, + false, + uint_ty, + bitcast_res, + base_offset, + uint_values_and_offsets, + ); + } + SpirvType::Integer(bits, signed) => { + self.store_as_u32( + bits, + signed, + uint_ty, + val, + base_offset, + uint_values_and_offsets, + ); + } + SpirvType::Void => self.err("Type () unsupported for bindless buffer stores"), + SpirvType::Bool => self.err("Type bool unsupported for bindless buffer stores"), + SpirvType::Opaque { ref name } => self.err(&format!("Opaque type {} unsupported for bindless buffer stores", name)), + SpirvType::RuntimeArray { element: _ } => + self.err("Type `RuntimeArray` unsupported for bindless buffer stores"), + SpirvType::Pointer { pointee: _ } => + self.err("Pointer type unsupported for bindless buffer stores"), + SpirvType::Function { + return_type: _, + arguments: _, + } => self.err("Function type unsupported for bindless buffer stores"), + SpirvType::Image { + sampled_type: _, + dim: _, + depth: _, + arrayed: _, + multisampled: _, + sampled: _, + image_format: _, + access_qualifier: _, + } => self.err("Image type unsupported for bindless buffer stores (use a bindless Texture type instead)"), + SpirvType::Sampler => self.err("Sampler type unsupported for bindless buffer stores"), + SpirvType::SampledImage { image_type: _ } => self.err("SampledImage type unsupported for bindless buffer stores"), + SpirvType::InterfaceBlock { inner_type: _ } => self.err("InterfaceBlock type unsupported for bindless buffer stores"), + SpirvType::AccelerationStructureKhr => self.fatal("AccelerationStructureKhr type unsupported for bindless buffer stores"), + SpirvType::RayQueryKhr => self.fatal("RayQueryKhr type unsupported for bindless buffer stores"), + } + } + + fn store_as_u32( + &mut self, + bits: u32, + signed: bool, + uint_ty: u32, + val: SpirvValue, + base_offset: u32, + uint_values_and_offsets: &mut Vec<(u32, SpirvValue)>, + ) { + let val_def = val.def(self); + + match (bits, signed) { + (32, false) => uint_values_and_offsets.push((base_offset, val)), + (32, true) => { + // need a bitcast to go from signed to unsigned + let bitcast_res = self + .emit() + .bitcast(uint_ty, None, val_def) + .unwrap() + .with_type(uint_ty); + + uint_values_and_offsets.push((base_offset, bitcast_res)); + } + (64, _) => { + let (ulong_ty, ulong_data) = if signed { + // bitcast from i64 into a u64 first, then proceed + let ulong_ty = SpirvType::Integer(64, false).def(rustc_span::DUMMY_SP, self); + + let bitcast_res = self.emit().bitcast(ulong_ty, None, val_def).unwrap(); + + (ulong_ty, bitcast_res) + } else { + (val.ty, val_def) + }; + + // note: assumes little endian + // [base] => uint(ulong_data) + // [base + 1] => uint(ulong_data >> 32) + let lower = self + .emit() + .u_convert(uint_ty, None, ulong_data) + .unwrap() + .with_type(uint_ty); + uint_values_and_offsets.push((base_offset, lower)); + + let const_32 = self.constant_int(uint_ty, 32).def(self); + let shifted = self + .emit() + .shift_right_logical(ulong_ty, None, ulong_data, const_32) + .unwrap(); + let upper = self + .emit() + .u_convert(uint_ty, None, shifted) + .unwrap() + .with_type(uint_ty); + uint_values_and_offsets.push((base_offset + 1, upper)); + } + _ => { + let mut err = self + .tcx + .sess + .struct_err("Unsupported integer type for `codegen_internal_buffer_store`"); + err.note(&format!("bits: `{:?}`", bits)); + err.note(&format!("signed: `{:?}`", signed)); + err.emit(); + } + } + } + + pub(crate) fn codegen_internal_buffer_store(&mut self, args: &[SpirvValue]) { + if !self.bindless() { + self.fatal("Need to run the compiler with -Ctarget-feature=+bindless to be able to use the bindless features"); + } + + let uint_ty = SpirvType::Integer(32, false).def(rustc_span::DUMMY_SP, self); + + let uniform_uint_ptr = + SpirvType::Pointer { pointee: uint_ty }.def(rustc_span::DUMMY_SP, self); + + let zero = self.constant_int(uint_ty, 0).def(self); + + let sets = self.bindless_descriptor_sets.borrow().unwrap(); + + let bindless_idx = args[0].def(self); + let offset_arg = args[1].def(self); + + let two = self.constant_int(uint_ty, 2).def(self); + + let dword_offset = self + .emit() + .shift_right_arithmetic(uint_ty, None, offset_arg, two) + .unwrap(); + + let mut uint_values_and_offsets = vec![]; + self.recurse_adt_for_stores(uint_ty, args[2], 0, &mut uint_values_and_offsets); + + for (offset, uint_value) in uint_values_and_offsets { + let offset = if offset > 0 { + let element_offset = self.constant_int(uint_ty, offset as u64).def(self); + + self.emit() + .i_add(uint_ty, None, dword_offset, element_offset) + .unwrap() + } else { + dword_offset + }; + + let indices = vec![bindless_idx, zero, offset]; + + let access_chain = self + .emit() + .access_chain(uniform_uint_ptr, None, sets.buffers, indices) + .unwrap() + .with_type(uniform_uint_ptr); + + self.store(uint_value, access_chain, Align::from_bytes(0).unwrap()); + } + } + + pub(crate) fn codegen_internal_buffer_load( + &mut self, + result_type: Word, + args: &[SpirvValue], + ) -> SpirvValue { + if !self.bindless() { + self.fatal("Need to run the compiler with -Ctarget-feature=+bindless to be able to use the bindless features"); + } + + let uint_ty = SpirvType::Integer(32, false).def(rustc_span::DUMMY_SP, self); + + let uniform_uint_ptr = + SpirvType::Pointer { pointee: uint_ty }.def(rustc_span::DUMMY_SP, self); + + let two = self.constant_int(uint_ty, 2).def(self); + + let offset_arg = args[1].def(self); + + let base_offset_var = self + .emit() + .shift_right_arithmetic(uint_ty, None, offset_arg, two) + .unwrap(); + + let bindless_idx = args[0].def(self); + + let sets = self.bindless_descriptor_sets.borrow().unwrap(); + + self.recurse_adt_for_loads( + uint_ty, + uniform_uint_ptr, + bindless_idx, + base_offset_var, + 0, + result_type, + &sets, + ) + } + + #[allow(clippy::too_many_arguments)] + fn load_from_u32( + &mut self, + bits: u32, + signed: bool, + target_ty: Word, + uint_ty: u32, + uniform_uint_ptr: u32, + bindless_idx: u32, + base_offset_var: Word, + element_offset_literal: u32, + sets: &BindlessDescriptorSets, + ) -> SpirvValue { + let zero = self.constant_int(uint_ty, 0).def(self); + + let offset = if element_offset_literal > 0 { + let element_offset = self + .constant_int(uint_ty, element_offset_literal as u64) + .def(self); + + self.emit() + .i_add(uint_ty, None, base_offset_var, element_offset) + .unwrap() + } else { + base_offset_var + }; + + let indices = vec![bindless_idx, zero, offset]; + + let result = self + .emit() + .access_chain(uniform_uint_ptr, None, sets.buffers, indices) + .unwrap(); + + match (bits, signed) { + (32, false) => self + .emit() + .load(uint_ty, None, result, None, std::iter::empty()) + .unwrap() + .with_type(uint_ty), + (32, true) => { + let load_res = self + .emit() + .load(uint_ty, None, result, None, std::iter::empty()) + .unwrap(); + + self.emit() + .bitcast(target_ty, None, load_res) + .unwrap() + .with_type(target_ty) + } + (64, _) => { + // note: assumes little endian + // lower = u64(base[0]) + // upper = u64(base[1]) + // result = lower | (upper << 32) + let ulong_ty = SpirvType::Integer(64, false).def(rustc_span::DUMMY_SP, self); + + let lower = self + .emit() + .load(uint_ty, None, result, None, std::iter::empty()) + .unwrap(); + + let lower = self.emit().u_convert(ulong_ty, None, lower).unwrap(); + + let const_one = self.constant_int(uint_ty, 1u64).def(self); + + let upper_offset = self.emit().i_add(uint_ty, None, offset, const_one).unwrap(); + + let indices = vec![bindless_idx, zero, upper_offset]; + + let upper_chain = self + .emit() + .access_chain(uniform_uint_ptr, None, sets.buffers, indices) + .unwrap(); + + let upper = self + .emit() + .load(uint_ty, None, upper_chain, None, std::iter::empty()) + .unwrap(); + + let upper = self.emit().u_convert(ulong_ty, None, upper).unwrap(); + + let thirty_two = self.constant_int(uint_ty, 32).def(self); + + let upper_shifted = self + .emit() + .shift_left_logical(ulong_ty, None, upper, thirty_two) + .unwrap(); + + let value = self + .emit() + .bitwise_or(ulong_ty, None, upper_shifted, lower) + .unwrap(); + + if signed { + self.emit() + .bitcast(target_ty, None, value) + .unwrap() + .with_type(target_ty) + } else { + value.with_type(ulong_ty) + } + } + _ => self.fatal(&format!( + "Trying to load invalid data type: {}{}", + if signed { "i" } else { "u" }, + bits + )), + } + } + + #[allow(clippy::too_many_arguments)] + fn recurse_adt_for_loads( + &mut self, + uint_ty: u32, + uniform_uint_ptr: u32, + bindless_idx: u32, + base_offset_var: Word, + element_offset_literal: u32, + result_type: u32, + sets: &BindlessDescriptorSets, + ) -> SpirvValue { + let data = self.lookup_type(result_type); + + match data { + SpirvType::Adt { + ref field_types, + ref field_offsets, + ref field_names, + def_id: _, + .. + } => { + let mut composite_components = vec![]; + + for (idx, (ty, offset)) in field_types.iter().zip(field_offsets.iter()).enumerate() + { + if offset.bytes() as u32 % 4 != 0 { + let adt_name = self.type_cache.lookup_name(result_type); + let field_name = if let Some(field_names) = field_names { + &field_names[idx] + } else { + "" + }; + + self.fatal(&format!( + "Trying to load from unaligned field: `{}::{}`. Field must be aligned to multiple of 4 bytes, but has offset {}", + adt_name, + field_name, + offset.bytes() as u32)); + } + + let offset = offset.bytes() as u32 / 4; + + composite_components.push( + self.recurse_adt_for_loads( + uint_ty, + uniform_uint_ptr, + bindless_idx, + base_offset_var, + element_offset_literal + offset, + *ty, + sets, + ) + .def(self), + ); + } + + let adt = data.def(rustc_span::DUMMY_SP, self); + + self.emit() + .composite_construct(adt, None, composite_components) + .unwrap() + .with_type(adt) + } + SpirvType::Vector { count, element } => { + let mut composite_components = vec![]; + + for offset in 0..count { + composite_components.push( + self.recurse_adt_for_loads( + uint_ty, + uniform_uint_ptr, + bindless_idx, + base_offset_var, + element_offset_literal + offset, + element, + sets, + ) + .def(self), + ); + } + + let adt = data.def(rustc_span::DUMMY_SP, self); + + self.emit() + .composite_construct(adt, None, composite_components) + .unwrap() + .with_type(adt) + } + SpirvType::Float(bits) => { + let loaded_as_int = self + .load_from_u32( + bits, + false, + uint_ty, + uint_ty, + uniform_uint_ptr, + bindless_idx, + base_offset_var, + element_offset_literal, + sets, + ) + .def(self); + + self.emit() + .bitcast(result_type, None, loaded_as_int) + .unwrap() + .with_type(result_type) + } + SpirvType::Integer(bits, signed) => self.load_from_u32( + bits, + signed, + result_type, + uint_ty, + uniform_uint_ptr, + bindless_idx, + base_offset_var, + element_offset_literal, + sets, + ), + SpirvType::Array { element, count } => { + let count = self + .cx + .builder + .lookup_const_u64(count) + .expect("Array type has invalid count value"); + + let mut composite_components = vec![]; + + for offset in 0..count { + let offset : u32 = offset.try_into().expect("Array count needs to fit in u32"); + + composite_components.push( + self.recurse_adt_for_loads( + uint_ty, + uniform_uint_ptr, + bindless_idx, + base_offset_var, + element_offset_literal + offset, + element, + sets, + ) + .def(self), + ); + } + + let adt = data.def(rustc_span::DUMMY_SP, self); + + self.emit() + .composite_construct(adt, None, composite_components) + .unwrap() + .with_type(adt) + } + SpirvType::Void => self.fatal("Type () unsupported for bindless buffer loads"), + SpirvType::Bool => self.fatal("Type bool unsupported for bindless buffer loads"), + SpirvType::Opaque { ref name } => self.fatal(&format!("Opaque type {} unsupported for bindless buffer loads", name)), + SpirvType::RuntimeArray { element: _ } => + self.fatal("Type `RuntimeArray` unsupported for bindless buffer loads"), + SpirvType::Pointer { pointee: _ } => + self.fatal("Pointer type unsupported for bindless buffer loads"), + SpirvType::Function { + return_type: _, + arguments: _, + } => self.fatal("Function type unsupported for bindless buffer loads"), + SpirvType::Image { + sampled_type: _, + dim: _, + depth: _, + arrayed: _, + multisampled: _, + sampled: _, + image_format: _, + access_qualifier: _, + } => self.fatal("Image type unsupported for bindless buffer loads (use a bindless Texture type instead)"), + SpirvType::Sampler => self.fatal("Sampler type unsupported for bindless buffer loads"), + SpirvType::SampledImage { image_type: _ } => self.fatal("SampledImage type unsupported for bindless buffer loads"), + SpirvType::InterfaceBlock { inner_type: _ } => self.fatal("InterfaceBlock type unsupported for bindless buffer loads"), + SpirvType::AccelerationStructureKhr => self.fatal("AccelerationStructureKhr type unsupported for bindless buffer loads"), + SpirvType::RayQueryKhr => self.fatal("RayQueryKhr type unsupported for bindless buffer loads"), + } + } +} diff --git a/crates/rustc_codegen_spirv/src/builder/mod.rs b/crates/rustc_codegen_spirv/src/builder/mod.rs index d7a84941ad..dbbbe24a5f 100644 --- a/crates/rustc_codegen_spirv/src/builder/mod.rs +++ b/crates/rustc_codegen_spirv/src/builder/mod.rs @@ -2,6 +2,7 @@ mod builder_methods; mod ext_inst; mod intrinsics; pub mod libm_intrinsics; +mod load_store; mod spirv_asm; pub use ext_inst::ExtInst; diff --git a/crates/rustc_codegen_spirv/src/builder/spirv_asm.rs b/crates/rustc_codegen_spirv/src/builder/spirv_asm.rs index a0ce960bb2..810aa0678a 100644 --- a/crates/rustc_codegen_spirv/src/builder/spirv_asm.rs +++ b/crates/rustc_codegen_spirv/src/builder/spirv_asm.rs @@ -1,8 +1,7 @@ -use crate::builder_spirv::SpirvValue; -use crate::spirv_type::SpirvType; - use super::Builder; +use crate::builder_spirv::{BuilderCursor, SpirvValue}; use crate::codegen_cx::CodegenCx; +use crate::spirv_type::SpirvType; use rspirv::dr; use rspirv::grammar::{LogicalOperand, OperandKind, OperandQuantifier}; use rspirv::spirv::{ @@ -313,11 +312,20 @@ impl<'cx, 'tcx> Builder<'cx, 'tcx> { } .def(self.span(), self), Op::TypeRayQueryKHR => SpirvType::RayQueryKhr.def(self.span(), self), - Op::Variable if inst.operands[0].unwrap_storage_class() != StorageClass::Function => { + Op::Variable => { // OpVariable with Function storage class should be emitted inside the function, // however, all other OpVariables should appear in the global scope instead. - self.emit_global() - .insert_types_global_values(dr::InsertPoint::End, inst); + if inst.operands[0].unwrap_storage_class() == StorageClass::Function { + self.emit_with_cursor(BuilderCursor { + block: Some(0), + ..self.cursor + }) + .insert_into_block(dr::InsertPoint::Begin, inst) + .unwrap(); + } else { + self.emit_global() + .insert_types_global_values(dr::InsertPoint::End, inst); + } return; } _ => { diff --git a/crates/rustc_codegen_spirv/src/builder_spirv.rs b/crates/rustc_codegen_spirv/src/builder_spirv.rs index dee4205cdc..dc5dfd80d6 100644 --- a/crates/rustc_codegen_spirv/src/builder_spirv.rs +++ b/crates/rustc_codegen_spirv/src/builder_spirv.rs @@ -22,6 +22,13 @@ pub enum SpirvValueKind { /// of such constants, instead of where they're generated (and cached). IllegalConst(Word), + /// This can only happen in one specific case - which is as a result of + /// `codegen_internal_buffer_store`, that function is supposed to return + /// OpTypeVoid, however because it gets inline by the compiler it can't. + /// Instead we return this, and trigger an error if we ever end up using + /// the result of this function call (which we can't). + IllegalTypeUsed(Word), + // FIXME(eddyb) this shouldn't be needed, but `rustc_codegen_ssa` still relies // on converting `Function`s to `Value`s even for direct calls, the `Builder` // should just have direct and indirect `call` variants (or a `Callee` enum). @@ -129,6 +136,16 @@ impl SpirvValue { id } + SpirvValueKind::IllegalTypeUsed(id) => { + cx.tcx + .sess + .struct_span_err(span, "Can't use type as a value") + .note(&format!("Type: *{}", cx.debug_type(id))) + .emit(); + + id + } + SpirvValueKind::FnAddr { .. } => { if cx.is_system_crate() { cx.builder diff --git a/crates/rustc_codegen_spirv/src/codegen_cx/declare.rs b/crates/rustc_codegen_spirv/src/codegen_cx/declare.rs index 0644336c8a..67cb44e77d 100644 --- a/crates/rustc_codegen_spirv/src/codegen_cx/declare.rs +++ b/crates/rustc_codegen_spirv/src/codegen_cx/declare.rs @@ -94,7 +94,7 @@ impl<'tcx> CodegenCx<'tcx> { // HACK(eddyb) this is a bit roundabout, but the easiest way to get a // fully absolute path that contains at least as much information as // `instance.to_string()` (at least with `-Z symbol-mangling-version=v0`). - // While we could use the mangled symbol insyead, like we do for linkage, + // While we could use the mangled symbol instead, like we do for linkage, // `OpName` is more of a debugging aid, so not having to separately // demangle the SPIR-V can help. However, if some tools assume `OpName` // is always a valid identifier, we may have to offer the mangled name @@ -124,8 +124,15 @@ impl<'tcx> CodegenCx<'tcx> { .borrow_mut() .insert(fn_id, UnrollLoopsDecoration {}); } + if attrs.internal_buffer_load.is_some() { + self.internal_buffer_load_id.borrow_mut().insert(fn_id); + } + if attrs.internal_buffer_store.is_some() { + self.internal_buffer_store_id.borrow_mut().insert(fn_id); + } let instance_def_id = instance.def_id(); + if self.tcx.crate_name(instance_def_id.krate) == self.sym.libm { let item_name = self.tcx.item_name(instance_def_id); let intrinsic = self.sym.libm_intrinsics.get(&item_name); diff --git a/crates/rustc_codegen_spirv/src/codegen_cx/entry.rs b/crates/rustc_codegen_spirv/src/codegen_cx/entry.rs index 6e6d00c6a6..0ff1b7d9c2 100644 --- a/crates/rustc_codegen_spirv/src/codegen_cx/entry.rs +++ b/crates/rustc_codegen_spirv/src/codegen_cx/entry.rs @@ -3,9 +3,10 @@ use crate::abi::ConvSpirvType; use crate::attr::{AggregatedSpirvAttributes, Entry}; use crate::builder::Builder; use crate::builder_spirv::{SpirvValue, SpirvValueExt}; +use crate::codegen_cx::BindlessDescriptorSets; use crate::spirv_type::SpirvType; use rspirv::dr::Operand; -use rspirv::spirv::{Decoration, ExecutionModel, FunctionControl, StorageClass, Word}; +use rspirv::spirv::{Capability, Decoration, ExecutionModel, FunctionControl, StorageClass, Word}; use rustc_codegen_ssa::traits::{BaseTypeMethods, BuilderMethods}; use rustc_data_structures::fx::FxHashMap; use rustc_hir as hir; @@ -14,7 +15,7 @@ use rustc_middle::ty::{Instance, Ty, TyKind}; use rustc_span::Span; use rustc_target::abi::{ call::{ArgAbi, ArgAttribute, ArgAttributes, FnAbi, PassMode}, - LayoutOf, Size, + Align, LayoutOf, Size, }; impl<'tcx> CodegenCx<'tcx> { @@ -104,6 +105,167 @@ impl<'tcx> CodegenCx<'tcx> { }); } + pub fn lazy_add_bindless_descriptor_sets(&self) { + self.bindless_descriptor_sets + .replace(Some(BindlessDescriptorSets { + // all storage buffers are compatible and go in set 0 + buffers: self.buffer_descriptor_set(0), + + // sampled images are all compatible in vulkan, so we can overlap them + sampled_image_1d: self.texture_bindless_descriptor_set( + 1, + rspirv::spirv::Dim::Dim1D, + true, + ), + sampled_image_2d: self.texture_bindless_descriptor_set( + 1, + rspirv::spirv::Dim::Dim2D, + true, + ), + sampled_image_3d: self.texture_bindless_descriptor_set( + 1, + rspirv::spirv::Dim::Dim3D, + true, + ), + // jb-todo: storage images are all compatible so they can live in the same descriptor set too + })); + } + + fn buffer_descriptor_set(&self, descriptor_set: u32) -> Word { + let uint_ty = SpirvType::Integer(32, false).def(rustc_span::DUMMY_SP, self); + + let runtime_array_uint = + SpirvType::RuntimeArray { element: uint_ty }.def(rustc_span::DUMMY_SP, self); + + let buffer_struct = SpirvType::Adt { + def_id: None, + size: Some(Size::from_bytes(4)), + align: Align::from_bytes(4).unwrap(), + field_types: vec![runtime_array_uint], + field_offsets: vec![], + field_names: None, + } + .def(rustc_span::DUMMY_SP, self); + + let runtime_array_struct = SpirvType::RuntimeArray { + element: buffer_struct, + } + .def(rustc_span::DUMMY_SP, self); + + let uniform_ptr_runtime_array = SpirvType::Pointer { + pointee: runtime_array_struct, + } + .def(rustc_span::DUMMY_SP, self); + + let mut emit_global = self.emit_global(); + let buffer = emit_global + .variable( + uniform_ptr_runtime_array, + None, + if self.target.spirv_version() <= (1, 3) { + StorageClass::Uniform + } else { + StorageClass::StorageBuffer + }, + None, + ) + .with_type(uniform_ptr_runtime_array) + .def_cx(self); + + emit_global.decorate( + buffer, + rspirv::spirv::Decoration::DescriptorSet, + std::iter::once(Operand::LiteralInt32(descriptor_set)), + ); + emit_global.decorate( + buffer, + rspirv::spirv::Decoration::Binding, + std::iter::once(Operand::LiteralInt32(0)), + ); + + if self.target.spirv_version() <= (1, 3) { + emit_global.decorate( + buffer_struct, + rspirv::spirv::Decoration::BufferBlock, + std::iter::empty(), + ); + } else { + emit_global.decorate( + buffer_struct, + rspirv::spirv::Decoration::Block, + std::iter::empty(), + ); + } + + emit_global.decorate( + runtime_array_uint, + rspirv::spirv::Decoration::ArrayStride, + std::iter::once(Operand::LiteralInt32(4)), + ); + + emit_global.member_decorate( + buffer_struct, + 0, + rspirv::spirv::Decoration::Offset, + std::iter::once(Operand::LiteralInt32(0)), + ); + + buffer + } + + fn texture_bindless_descriptor_set( + &self, + descriptor_set: u32, + dim: rspirv::spirv::Dim, + sampled: bool, + ) -> Word { + let float_ty = SpirvType::Float(32).def(rustc_span::DUMMY_SP, self); + + let image = SpirvType::Image { + sampled_type: float_ty, + dim, + depth: 0, + arrayed: 0, + multisampled: 0, + sampled: if sampled { 1 } else { 0 }, + image_format: rspirv::spirv::ImageFormat::Unknown, + access_qualifier: None, + } + .def(rustc_span::DUMMY_SP, self); + + let sampled_image = + SpirvType::SampledImage { image_type: image }.def(rustc_span::DUMMY_SP, self); + + let runtime_array_image = SpirvType::RuntimeArray { + element: sampled_image, + } + .def(rustc_span::DUMMY_SP, self); + + let uniform_ptr_runtime_array = SpirvType::Pointer { + pointee: runtime_array_image, + } + .def(rustc_span::DUMMY_SP, self); + + let mut emit_global = self.emit_global(); + let image_array = emit_global + .variable(uniform_ptr_runtime_array, None, StorageClass::Uniform, None) + .with_type(uniform_ptr_runtime_array) + .def_cx(self); + + emit_global.decorate( + image_array, + rspirv::spirv::Decoration::DescriptorSet, + std::iter::once(Operand::LiteralInt32(descriptor_set)), + ); + emit_global.decorate( + image_array, + rspirv::spirv::Decoration::Binding, + std::iter::once(Operand::LiteralInt32(0)), + ); + + image_array + } + fn shader_entry_stub( &self, span: Span, @@ -148,6 +310,25 @@ impl<'tcx> CodegenCx<'tcx> { bx.call(entry_func, &call_args, None); bx.ret_void(); + if self.bindless() { + self.emit_global().extension("SPV_EXT_descriptor_indexing"); + self.emit_global() + .capability(Capability::RuntimeDescriptorArray); + + if self.target.spirv_version() > (1, 3) { + let sets = self.bindless_descriptor_sets.borrow().unwrap(); + + op_entry_point_interface_operands.push(sets.buffers); + + //op_entry_point_interface_operands + // .push(sets.sampled_image_1d); + // op_entry_point_interface_operands + // .push(sets.sampled_image_2d); + //op_entry_point_interface_operands + //.push(sets.sampled_image_3d); + } + } + let stub_fn_id = stub_fn.def_cx(self); self.emit_global().entry_point( execution_model, @@ -385,6 +566,13 @@ impl<'tcx> CodegenCx<'tcx> { decoration_supersedes_location = true; } if let Some(index) = attrs.descriptor_set.map(|attr| attr.value) { + if self.bindless() { + self.tcx.sess.span_fatal( + attrs.descriptor_set.unwrap().span, + "Can't use #[spirv(descriptor_set)] attribute in bindless mode", + ); + } + self.emit_global().decorate( var, Decoration::DescriptorSet, @@ -393,6 +581,12 @@ impl<'tcx> CodegenCx<'tcx> { decoration_supersedes_location = true; } if let Some(index) = attrs.binding.map(|attr| attr.value) { + if self.bindless() { + self.tcx.sess.span_fatal( + attrs.binding.unwrap().span, + "Can't use #[spirv(binding)] attribute in bindless mode", + ); + } self.emit_global().decorate( var, Decoration::Binding, diff --git a/crates/rustc_codegen_spirv/src/codegen_cx/mod.rs b/crates/rustc_codegen_spirv/src/codegen_cx/mod.rs index 656122d257..6d4ee919a1 100644 --- a/crates/rustc_codegen_spirv/src/codegen_cx/mod.rs +++ b/crates/rustc_codegen_spirv/src/codegen_cx/mod.rs @@ -18,7 +18,7 @@ use rustc_codegen_ssa::mir::debuginfo::{FunctionDebugContext, VariableKind}; use rustc_codegen_ssa::traits::{ AsmMethods, BackendTypes, CoverageInfoMethods, DebugInfoMethods, MiscMethods, }; -use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_hir::GlobalAsm; use rustc_middle::mir::mono::CodegenUnit; use rustc_middle::mir::Body; @@ -36,6 +36,14 @@ use std::iter::once; use std::rc::Rc; use std::str::FromStr; +#[derive(Copy, Clone, Debug)] +pub struct BindlessDescriptorSets { + pub buffers: Word, + pub sampled_image_1d: Word, + pub sampled_image_2d: Word, + pub sampled_image_3d: Word, +} + pub struct CodegenCx<'tcx> { pub tcx: TyCtxt<'tcx>, pub codegen_unit: &'tcx CodegenUnit<'tcx>, @@ -64,6 +72,8 @@ pub struct CodegenCx<'tcx> { /// Simple `panic!("...")` and builtin panics (from MIR `Assert`s) call `#[lang = "panic"]`. pub panic_fn_id: Cell>, + pub internal_buffer_load_id: RefCell>, + pub internal_buffer_store_id: RefCell>, /// Builtin bounds-checking panics (from MIR `Assert`s) call `#[lang = "panic_bounds_check"]`. pub panic_bounds_check_fn_id: Cell>, @@ -71,6 +81,9 @@ pub struct CodegenCx<'tcx> { /// This enables/disables them. pub i8_i16_atomics_allowed: bool, + /// If bindless is enable, this contains the information about the global + /// descriptor sets that are always bound. + pub bindless_descriptor_sets: RefCell>, pub codegen_args: CodegenArgs, /// Information about the SPIR-V target. @@ -80,10 +93,12 @@ pub struct CodegenCx<'tcx> { impl<'tcx> CodegenCx<'tcx> { pub fn new(tcx: TyCtxt<'tcx>, codegen_unit: &'tcx CodegenUnit<'tcx>) -> Self { let sym = Symbols::get(); + let features = tcx .sess .target_features .iter() + .filter(|s| *s != &sym.bindless) .map(|s| s.as_str().parse()) .collect::>() .unwrap_or_else(|error| { @@ -91,10 +106,18 @@ impl<'tcx> CodegenCx<'tcx> { Vec::new() }); + let mut bindless = false; + for &feature in &tcx.sess.target_features { + if feature == sym.bindless { + bindless = true; + break; + } + } + let codegen_args = CodegenArgs::from_session(tcx.sess); let target = tcx.sess.target.llvm_target.parse().unwrap(); - Self { + let result = Self { tcx, codegen_unit, builder: BuilderSpirv::new(&target, &features), @@ -110,10 +133,25 @@ impl<'tcx> CodegenCx<'tcx> { instruction_table: InstructionTable::new(), libm_intrinsics: Default::default(), panic_fn_id: Default::default(), + internal_buffer_load_id: Default::default(), + internal_buffer_store_id: Default::default(), panic_bounds_check_fn_id: Default::default(), i8_i16_atomics_allowed: false, codegen_args, + bindless_descriptor_sets: Default::default(), + }; + + if bindless { + result.lazy_add_bindless_descriptor_sets(); } + + result + } + + /// Temporary toggle to see if bindless has been enabled in the compiler, should + /// be removed longer term when we use bindless as the default model + pub fn bindless(&self) -> bool { + self.bindless_descriptor_sets.borrow().is_some() } /// See comment on `BuilderCursor` diff --git a/crates/rustc_codegen_spirv/src/spirv_type.rs b/crates/rustc_codegen_spirv/src/spirv_type.rs index fe98b54bc2..ab99587358 100644 --- a/crates/rustc_codegen_spirv/src/spirv_type.rs +++ b/crates/rustc_codegen_spirv/src/spirv_type.rs @@ -223,7 +223,7 @@ impl SpirvType { result, def_span, "function pointer types are not allowed", - ) + ); } result } @@ -728,4 +728,13 @@ impl TypeCache<'_> { .insert_no_overwrite(word, ty) .unwrap(); } + + pub fn lookup_name(&self, word: Word) -> String { + let type_names = self.type_names.borrow(); + type_names + .get(&word) + .and_then(|names| names.iter().next().copied()) + .map(|v| v.to_string()) + .unwrap_or_else(|| "".to_string()) + } } diff --git a/crates/rustc_codegen_spirv/src/symbols.rs b/crates/rustc_codegen_spirv/src/symbols.rs index 3cf817bea3..9e4fb81258 100644 --- a/crates/rustc_codegen_spirv/src/symbols.rs +++ b/crates/rustc_codegen_spirv/src/symbols.rs @@ -31,6 +31,7 @@ pub struct Symbols { sampled: Symbol, image_format: Symbol, access_qualifier: Symbol, + pub bindless: Symbol, attributes: FxHashMap, execution_modes: FxHashMap, pub libm_intrinsics: FxHashMap, @@ -337,6 +338,8 @@ impl Symbols { SpirvAttribute::IntrinsicType(IntrinsicType::SampledImage), ), ("unroll_loops", SpirvAttribute::UnrollLoops), + ("internal_buffer_load", SpirvAttribute::InternalBufferLoad), + ("internal_buffer_store", SpirvAttribute::InternalBufferStore), ] .iter() .cloned(); @@ -381,6 +384,7 @@ impl Symbols { sampled: Symbol::intern("sampled"), image_format: Symbol::intern("image_format"), access_qualifier: Symbol::intern("access_qualifier"), + bindless: Symbol::intern("bindless"), attributes, execution_modes, libm_intrinsics, diff --git a/crates/spirv-builder/src/lib.rs b/crates/spirv-builder/src/lib.rs index 7733f10acf..b8b475918d 100644 --- a/crates/spirv-builder/src/lib.rs +++ b/crates/spirv-builder/src/lib.rs @@ -104,6 +104,7 @@ pub struct SpirvBuilder { print_metadata: bool, release: bool, target: String, + bindless: bool, } impl SpirvBuilder { @@ -112,6 +113,7 @@ impl SpirvBuilder { path_to_crate: path_to_crate.as_ref().to_owned(), print_metadata: true, release: true, + bindless: false, target: target.into(), } } @@ -122,6 +124,13 @@ impl SpirvBuilder { self } + /// Run the compiler in bindless mode, this flag is in preparation for the full feature + /// and it's expected to be the default mode going forward + pub fn bindless(mut self, v: bool) -> Self { + self.bindless = v; + self + } + /// Build in release. Defaults to true. pub fn release(mut self, v: bool) -> Self { self.release = v; @@ -200,11 +209,25 @@ fn invoke_rustc(builder: &SpirvBuilder, multimodule: bool) -> Result Result TokenStream { let fn_name = sig.ident.clone(); + let sig = syn::Signature { abi: None, ..sig }; + let output = quote::quote! { // Don't warn on unused arguments on the CPU side. #[cfg_attr(not(target_arch = "spirv"), allow(unused_variables))] diff --git a/crates/spirv-std/src/bindless.rs b/crates/spirv-std/src/bindless.rs new file mode 100644 index 0000000000..9008d27c02 --- /dev/null +++ b/crates/spirv-std/src/bindless.rs @@ -0,0 +1,281 @@ +use crate::vector::Vector; + +/// A handle that points to a rendering related resource (TLAS, Sampler, Buffer, Texture etc) +/// this handle can be uploaded directly to the GPU to refer to our resources in a bindless +/// fashion and can be plainly stored in buffers directly - even without the help of a `DescriptorSet` +/// the handle isn't guaranteed to live as long as the resource it's associated with so it's up to +/// the user to ensure that their data lives long enough. The handle is versioned to prevent +/// use-after-free bugs however. +/// +/// This handle is expected to be used engine-side to refer to descriptors within a descriptor set. +/// To be able to use the bindless system in rust-gpu, an engine is expected to have created +/// four `DescriptorSets`, each containing a large table of max 1 << 23 elements for each type. +/// And to sub-allocate descriptors from those tables. It must use `RenderResourceHandle` to +/// refer to slots within this table, and it's then expected that these `RenderResourceHandle`'s +/// are freely copied to the GPU to refer to resources there. +/// +/// | Buffer Type | Set | +/// |------------------|-----| +/// | Buffers | 0 | +/// | Textures | 1 | +/// | Storage textures | 2 | +/// | Tlas | 3 | +#[derive(Copy, Clone, Eq, PartialEq, Hash)] +#[repr(transparent)] +pub struct RenderResourceHandle(u32); + +#[repr(u8)] +#[derive(Debug, Copy, Clone, Eq, PartialEq)] +pub enum RenderResourceTag { + Sampler, + Tlas, + Buffer, + Texture, +} + +impl core::fmt::Debug for RenderResourceHandle { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("RenderResourceHandle") + .field("version", &self.version()) + .field("tag", &self.tag()) + .field("index", unsafe { &self.index() }) + .finish() + } +} + +impl RenderResourceHandle { + pub fn new(version: u8, tag: RenderResourceTag, index: u32) -> Self { + let version = version as u32; + let tag = tag as u32; + let index = index as u32; + + assert!(version < 64); // version wraps around, it's just to make sure invalid resources don't get another version + assert!(tag < 8); + assert!(index < (1 << 23)); + + Self(version << 26 | tag << 23 | index) + } + + pub fn invalid() -> Self { + Self(!0) + } + + pub fn is_valid(self) -> bool { + self.0 != !0 + } + + pub fn version(self) -> u32 { + self.0 >> 26 + } + + pub fn tag(self) -> RenderResourceTag { + match (self.0 >> 23) & 7 { + 0 => RenderResourceTag::Sampler, + 1 => RenderResourceTag::Tlas, + 2 => RenderResourceTag::Buffer, + 3 => RenderResourceTag::Texture, + invalid_tag => panic!( + "RenderResourceHandle corrupt: invalid tag ({})", + invalid_tag + ), + } + } + + /// # Safety + /// This method can only safely refer to a resource if that resource + /// is guaranteed to exist by the caller. `RenderResourceHandle` can't + /// track lifetimes or keep ref-counts between GPU and CPU and thus + /// requires extra caution from the user. + #[inline] + pub unsafe fn index(self) -> u32 { + self.0 & ((1 << 23) - 1) + } + + /// This function is primarily intended for use in a slot allocator, where the slot + /// needs to get re-used and it's data updated. This bumps the `version` of the + /// `RenderResourceHandle` and updates the `tag`. + pub fn bump_version_and_update_tag(self, tag: RenderResourceTag) -> Self { + let mut version = self.0 >> 26; + version = ((version + 1) % 64) << 26; + let tag = (tag as u32) << 23; + Self(version | tag | (self.0 & ((1 << 23) - 1))) + } +} + +#[derive(Copy, Clone)] +#[repr(transparent)] +pub struct Buffer(RenderResourceHandle); + +mod internal { + #[spirv(internal_buffer_load)] + #[spirv_std_macros::gpu_only] + pub extern "unadjusted" fn internal_buffer_load(_buffer: u32, _offset: u32) -> T { + unimplemented!() + } // actually implemented in the compiler + + #[spirv(internal_buffer_store)] + #[spirv_std_macros::gpu_only] + pub unsafe extern "unadjusted" fn internal_buffer_store( + _buffer: u32, + _offset: u32, + _value: T, + ) { + unimplemented!() + } // actually implemented in the compiler +} + +impl Buffer { + #[spirv_std_macros::gpu_only] + #[inline] + pub extern "unadjusted" fn load(self, dword_aligned_byte_offset: u32) -> T { + // jb-todo: figure out why this assert breaks with complaints about pointers + // assert!(self.0.tag() == RenderResourceTag::Buffer); + // assert!(std::mem::sizeof::() % 4 == 0); + // assert!(dword_aligned_byte_offset % 4 == 0); + + unsafe { internal::internal_buffer_load(self.0.index(), dword_aligned_byte_offset) } + } + + #[spirv_std_macros::gpu_only] + pub unsafe extern "unadjusted" fn store(self, dword_aligned_byte_offset: u32, value: T) { + // jb-todo: figure out why this assert breaks with complaints about pointers + // assert!(self.0.tag() == RenderResourceTag::Buffer); + + internal::internal_buffer_store(self.0.index(), dword_aligned_byte_offset, value) + } +} + +#[derive(Copy, Clone)] +#[repr(transparent)] +pub struct SimpleBuffer(RenderResourceHandle, core::marker::PhantomData); + +impl SimpleBuffer { + #[spirv_std_macros::gpu_only] + #[inline] + pub extern "unadjusted" fn load(self) -> T { + unsafe { internal::internal_buffer_load(self.0.index(), 0) } + } + + #[spirv_std_macros::gpu_only] + pub unsafe extern "unadjusted" fn store(self, value: T) { + internal::internal_buffer_store(self.0.index(), 0, value) + } +} + +#[derive(Copy, Clone)] +#[repr(transparent)] +pub struct ArrayBuffer(RenderResourceHandle, core::marker::PhantomData); + +impl ArrayBuffer { + #[spirv_std_macros::gpu_only] + #[inline] + pub extern "unadjusted" fn load(self, index: u32) -> T { + unsafe { + internal::internal_buffer_load(self.0.index(), index * core::mem::size_of::() as u32) + } + } + + #[spirv_std_macros::gpu_only] + pub unsafe extern "unadjusted" fn store(self, index: u32, value: T) { + internal::internal_buffer_store( + self.0.index(), + index * core::mem::size_of::() as u32, + value, + ) + } +} + +#[derive(Copy, Clone)] +#[repr(transparent)] +pub struct Texture2d(RenderResourceHandle); + +// #[derive(Copy, Clone)] +// #[repr(transparent)] +// struct SamplerState(RenderResourceHandle); + +impl Texture2d { + #[spirv_std_macros::gpu_only] + pub fn sample>(self, coord: impl Vector) -> V { + // jb-todo: also do a bindless fetch of the sampler + unsafe { + let mut result = Default::default(); + asm!( + "OpExtension \"SPV_EXT_descriptor_indexing\"", + "OpCapability RuntimeDescriptorArray", + "OpDecorate %image_2d_var DescriptorSet 1", + "OpDecorate %image_2d_var Binding 0", + "%uint = OpTypeInt 32 0", + "%float = OpTypeFloat 32", + "%image_2d = OpTypeImage %float Dim2D 0 0 0 1 Unknown", + "%sampled_image_2d = OpTypeSampledImage %image_2d", + "%image_array = OpTypeRuntimeArray %sampled_image_2d", + "%ptr_image_array = OpTypePointer Generic %image_array", + "%image_2d_var = OpVariable %ptr_image_array UniformConstant", + "%ptr_sampled_image_2d = OpTypePointer Generic %sampled_image_2d", + "", // ^^ type preamble + "%offset = OpLoad _ {1}", + "%24 = OpAccessChain %ptr_sampled_image_2d %image_2d_var %offset", + "%25 = OpLoad %sampled_image_2d %24", + "%coord = OpLoad _ {0}", + "%result = OpImageSampleImplicitLod _ %25 %coord", + "OpStore {2} %result", + in(reg) &coord, + in(reg) &self.0.index(), + in(reg) &mut result, + ); + result + } + } + + #[spirv_std_macros::gpu_only] + pub fn sample_proj_lod>( + self, + coord: impl Vector, + ddx: impl Vector, + ddy: impl Vector, + offset_x: i32, + offset_y: i32, + ) -> V { + // jb-todo: also do a bindless fetch of the sampler + unsafe { + let mut result = Default::default(); + asm!( + "OpExtension \"SPV_EXT_descriptor_indexing\"", + "OpCapability RuntimeDescriptorArray", + "OpDecorate %image_2d_var DescriptorSet 1", + "OpDecorate %image_2d_var Binding 0", + "%uint = OpTypeInt 32 0", + "%int = OpTypeInt 32 1", + "%float = OpTypeFloat 32", + "%v2int = OpTypeVector %int 2", + "%int_0 = OpConstant %int 0", + "%image_2d = OpTypeImage %float Dim2D 0 0 0 1 Unknown", + "%sampled_image_2d = OpTypeSampledImage %image_2d", + "%image_array = OpTypeRuntimeArray %sampled_image_2d", + "%ptr_image_array = OpTypePointer Generic %image_array", + "%image_2d_var = OpVariable %ptr_image_array UniformConstant", + "%ptr_sampled_image_2d = OpTypePointer Generic %sampled_image_2d", + "", // ^^ type preamble + "%offset = OpLoad _ {1}", + "%24 = OpAccessChain %ptr_sampled_image_2d %image_2d_var %offset", + "%25 = OpLoad %sampled_image_2d %24", + "%coord = OpLoad _ {0}", + "%ddx = OpLoad _ {3}", + "%ddy = OpLoad _ {4}", + "%offset_x = OpLoad _ {5}", + "%offset_y = OpLoad _ {6}", + "%const_offset = OpConstantComposite %v2int %int_0 %int_0", + "%result = OpImageSampleProjExplicitLod _ %25 %coord Grad|ConstOffset %ddx %ddy %const_offset", + "OpStore {2} %result", + in(reg) &coord, + in(reg) &self.0.index(), + in(reg) &mut result, + in(reg) &ddx, + in(reg) &ddy, + in(reg) &offset_x, + in(reg) &offset_y, + ); + result + } + } +} diff --git a/crates/spirv-std/src/lib.rs b/crates/spirv-std/src/lib.rs index 9337c126a9..0d727beb27 100644 --- a/crates/spirv-std/src/lib.rs +++ b/crates/spirv-std/src/lib.rs @@ -1,7 +1,14 @@ #![no_std] #![cfg_attr( target_arch = "spirv", - feature(asm, register_attr, repr_simd, core_intrinsics, lang_items), + feature( + asm, + register_attr, + repr_simd, + core_intrinsics, + lang_items, + abi_unadjusted + ), register_attr(spirv) )] #![cfg_attr( @@ -76,6 +83,7 @@ pub extern crate spirv_std_macros as macros; pub mod arch; +pub mod bindless; pub mod float; #[cfg(feature = "const-generics")] pub mod image;