diff --git a/crates/bevy_ecs/macros/src/fetch.rs b/crates/bevy_ecs/macros/src/fetch.rs index 734057b7bf463f..002a1605bd9f83 100644 --- a/crates/bevy_ecs/macros/src/fetch.rs +++ b/crates/bevy_ecs/macros/src/fetch.rs @@ -261,9 +261,9 @@ pub fn derive_world_query_impl(ast: DeriveInput) -> TokenStream { _fetch: &mut >::Fetch, _state: &Self::State, _archetype: &'__w #path::archetype::Archetype, - _tables: &'__w #path::storage::Tables + _table: &'__w #path::storage::Table ) { - #(<#field_types>::set_archetype(&mut _fetch.#field_idents, &_state.#field_idents, _archetype, _tables);)* + #(<#field_types>::set_archetype(&mut _fetch.#field_idents, &_state.#field_idents, _archetype, _table);)* } /// SAFETY: we call `set_table` for each member that implements `Fetch` @@ -276,40 +276,27 @@ pub fn derive_world_query_impl(ast: DeriveInput) -> TokenStream { #(<#field_types>::set_table(&mut _fetch.#field_idents, &_state.#field_idents, _table);)* } - /// SAFETY: we call `table_fetch` for each member that implements `Fetch`. - #[inline] - unsafe fn table_fetch<'__w>( + /// SAFETY: we call `fetch` for each member that implements `Fetch`. + #[inline(always)] + unsafe fn fetch<'__w>( _fetch: &mut >::Fetch, + _entity: Entity, _table_row: usize ) -> >::Item { Self::Item { - #(#field_idents: <#field_types>::table_fetch(&mut _fetch.#field_idents, _table_row),)* + #(#field_idents: <#field_types>::fetch(&mut _fetch.#field_idents, _entity, _table_row),)* #(#ignored_field_idents: Default::default(),)* } } - /// SAFETY: we call `archetype_fetch` for each member that implements `Fetch`. - #[inline] - unsafe fn archetype_fetch<'__w>( - _fetch: &mut >::Fetch, - _archetype_index: usize - ) -> >::Item { - Self::Item { - #(#field_idents: <#field_types>::archetype_fetch(&mut _fetch.#field_idents, _archetype_index),)* - #(#ignored_field_idents: Default::default(),)* - } - } - - #[allow(unused_variables)] - #[inline] - unsafe fn table_filter_fetch<'__w>(_fetch: &mut >::Fetch, _table_row: usize) -> bool { - true #(&& <#field_types>::table_filter_fetch(&mut _fetch.#field_idents, _table_row))* - } - #[allow(unused_variables)] - #[inline] - unsafe fn archetype_filter_fetch<'__w>(_fetch: &mut >::Fetch, _archetype_index: usize) -> bool { - true #(&& <#field_types>::archetype_filter_fetch(&mut _fetch.#field_idents, _archetype_index))* + #[inline(always)] + unsafe fn filter_fetch<'__w>( + _fetch: &mut >::Fetch, + _entity: Entity, + _table_row: usize + ) -> bool { + true #(&& <#field_types>::filter_fetch(&mut _fetch.#field_idents, _entity, _table_row))* } fn update_component_access(state: &Self::State, _access: &mut #path::query::FilteredAccess<#path::component::ComponentId>) { diff --git a/crates/bevy_ecs/src/archetype.rs b/crates/bevy_ecs/src/archetype.rs index b52d4baad44a30..5725b3c6c7c402 100644 --- a/crates/bevy_ecs/src/archetype.rs +++ b/crates/bevy_ecs/src/archetype.rs @@ -121,9 +121,19 @@ impl Edges { } } -struct TableInfo { - id: TableId, - entity_rows: Vec, +pub struct ArchetypeEntity { + pub(crate) entity: Entity, + pub(crate) table_row: usize, +} + +impl ArchetypeEntity { + pub fn entity(&self) -> Entity { + self.entity + } + + pub fn table_row(&self) -> usize { + self.table_row + } } pub(crate) struct ArchetypeSwapRemoveResult { @@ -138,9 +148,9 @@ pub(crate) struct ArchetypeComponentInfo { pub struct Archetype { id: ArchetypeId, - entities: Vec, + table_id: TableId, edges: Edges, - table_info: TableInfo, + entities: Vec, table_components: Box<[ComponentId]>, sparse_set_components: Box<[ComponentId]>, components: SparseSet, @@ -183,14 +193,11 @@ impl Archetype { } Self { id, - table_info: TableInfo { - id: table_id, - entity_rows: Default::default(), - }, + table_id, + entities: Vec::new(), components, table_components, sparse_set_components, - entities: Default::default(), edges: Default::default(), } } @@ -202,19 +209,14 @@ impl Archetype { #[inline] pub fn table_id(&self) -> TableId { - self.table_info.id + self.table_id } #[inline] - pub fn entities(&self) -> &[Entity] { + pub fn entities(&self) -> &[ArchetypeEntity] { &self.entities } - #[inline] - pub fn entity_table_rows(&self) -> &[usize] { - &self.table_info.entity_rows - } - #[inline] pub fn table_components(&self) -> &[ComponentId] { &self.table_components @@ -242,20 +244,19 @@ impl Archetype { #[inline] pub fn entity_table_row(&self, index: usize) -> usize { - self.table_info.entity_rows[index] + self.entities[index].table_row } #[inline] pub(crate) fn set_entity_table_row(&mut self, index: usize, table_row: usize) { - self.table_info.entity_rows[index] = table_row; + self.entities[index].table_row = table_row; } /// # Safety /// valid component values must be immediately written to the relevant storages /// `table_row` must be valid pub(crate) unsafe fn allocate(&mut self, entity: Entity, table_row: usize) -> EntityLocation { - self.entities.push(entity); - self.table_info.entity_rows.push(table_row); + self.entities.push(ArchetypeEntity { entity, table_row }); EntityLocation { archetype_id: self.id, @@ -265,21 +266,20 @@ impl Archetype { pub(crate) fn reserve(&mut self, additional: usize) { self.entities.reserve(additional); - self.table_info.entity_rows.reserve(additional); } /// Removes the entity at `index` by swapping it out. Returns the table row the entity is stored /// in. pub(crate) fn swap_remove(&mut self, index: usize) -> ArchetypeSwapRemoveResult { let is_last = index == self.entities.len() - 1; - self.entities.swap_remove(index); + let entity = self.entities.swap_remove(index); ArchetypeSwapRemoveResult { swapped_entity: if is_last { None } else { - Some(self.entities[index]) + Some(self.entities[index].entity) }, - table_row: self.table_info.entity_rows.swap_remove(index), + table_row: entity.table_row, } } @@ -317,7 +317,6 @@ impl Archetype { pub(crate) fn clear_entities(&mut self) { self.entities.clear(); - self.table_info.entity_rows.clear(); } } diff --git a/crates/bevy_ecs/src/lib.rs b/crates/bevy_ecs/src/lib.rs index 3c34c1e836132d..197e1368943127 100644 --- a/crates/bevy_ecs/src/lib.rs +++ b/crates/bevy_ecs/src/lib.rs @@ -540,8 +540,8 @@ mod tests { let f = world .spawn((TableStored("def"), A(456), SparseStored(1))) .id(); - // // this should be skipped - // SparseStored(1).spawn("abc"); + // this should be skipped + // world.spawn(SparseStored(1)); let ents = world .query::<(Entity, Option<&SparseStored>, &A)>() .iter(&world) diff --git a/crates/bevy_ecs/src/query/fetch.rs b/crates/bevy_ecs/src/query/fetch.rs index 510420aa4600ca..52886817e20e04 100644 --- a/crates/bevy_ecs/src/query/fetch.rs +++ b/crates/bevy_ecs/src/query/fetch.rs @@ -4,7 +4,7 @@ use crate::{ component::{Component, ComponentId, ComponentStorage, ComponentTicks, StorageType}, entity::Entity, query::{debug_checked_unreachable, Access, FilteredAccess}, - storage::{ComponentSparseSet, Table, Tables}, + storage::{ComponentSparseSet, Table}, world::{Mut, World}, }; use bevy_ecs_macros::all_tuples; @@ -285,11 +285,10 @@ use std::{cell::UnsafeCell, marker::PhantomData}; /// exactly reflects the results of the following methods: /// /// - [`matches_component_set`] -/// - [`archetype_fetch`] -/// - [`table_fetch`] +/// - [`fetch`] /// /// [`Added`]: crate::query::Added -/// [`archetype_fetch`]: Self::archetype_fetch +/// [`fetch`]: Self::fetch /// [`Changed`]: crate::query::Changed /// [`Fetch`]: crate::query::WorldQueryGats::Fetch /// [`matches_component_set`]: Self::matches_component_set @@ -297,7 +296,6 @@ use std::{cell::UnsafeCell, marker::PhantomData}; /// [`Query`]: crate::system::Query /// [`ReadOnly`]: Self::ReadOnly /// [`State`]: Self::State -/// [`table_fetch`]: Self::table_fetch /// [`update_archetype_component_access`]: Self::update_archetype_component_access /// [`update_component_access`]: Self::update_component_access /// [`With`]: crate::query::With @@ -339,9 +337,10 @@ pub unsafe trait WorldQuery: for<'w> WorldQueryGats<'w> { /// Returns true if (and only if) every table of every archetype matched by this fetch contains /// all of the matched components. This is used to select a more efficient "table iterator" - /// for "dense" queries. If this returns true, [`WorldQuery::set_table`] and [`WorldQuery::table_fetch`] - /// will be called for iterators. If this returns false, [`WorldQuery::set_archetype`] and - /// [`WorldQuery::archetype_fetch`] will be called for iterators. + /// for "dense" queries. If this returns true, [`WorldQuery::set_table`] must be used before + /// [`WorldQuery::fetch`] can be called for iterators. If this returns false, + /// [`WorldQuery::set_archetype`] must be used before [`WorldQuery::fetch`] can be called for + /// iterators. const IS_DENSE: bool; /// Returns true if (and only if) this Fetch relies strictly on archetypes to limit which @@ -362,7 +361,7 @@ pub unsafe trait WorldQuery: for<'w> WorldQueryGats<'w> { fetch: &mut >::Fetch, state: &Self::State, archetype: &'w Archetype, - tables: &'w Tables, + table: &'w Table, ); /// Adjusts internal state to account for the next [`Table`]. This will always be called on tables @@ -378,51 +377,30 @@ pub unsafe trait WorldQuery: for<'w> WorldQueryGats<'w> { table: &'w Table, ); - /// Fetch [`Self::Item`](`WorldQueryGats::Item`) for the given `archetype_index` in the current [`Archetype`]. This must - /// always be called after [`WorldQuery::set_archetype`] with an `archetype_index` in the range of - /// the current [`Archetype`] - /// - /// # Safety - /// Must always be called _after_ [`WorldQuery::set_archetype`]. `archetype_index` must be in the range - /// of the current archetype - unsafe fn archetype_fetch<'w>( - fetch: &mut >::Fetch, - archetype_index: usize, - ) -> >::Item; - - /// Fetch [`Self::Item`](`WorldQueryGats::Item`) for the given `table_row` in the current [`Table`]. This must always be - /// called after [`WorldQuery::set_table`] with a `table_row` in the range of the current [`Table`] + /// Fetch [`Self::Item`](`WorldQueryGats::Item`) for either the given `entity` in the current [`Table`], + /// or for the given `entity` in the current [`Archetype`]. This must always be called after + /// [`WorldQuery::set_table`] with a `table_row` in the range of the current [`Table`] or after + /// [`WorldQuery::set_archetype`] with a `entity` in the current archetype. /// /// # Safety /// - /// Must always be called _after_ [`WorldQuery::set_table`]. `table_row` must be in the range of the - /// current table - unsafe fn table_fetch<'w>( + /// Must always be called _after_ [`WorldQuery::set_table`] or [`WorldQuery::set_archetype`]. `entity` and + /// `table_row` must be in the range of the current table and archetype. + unsafe fn fetch<'w>( fetch: &mut >::Fetch, + entity: Entity, table_row: usize, ) -> >::Item; /// # Safety /// - /// Must always be called _after_ [`WorldQuery::set_archetype`]. `archetype_index` must be in the range - /// of the current archetype. + /// Must always be called _after_ [`WorldQuery::set_table`] or [`WorldQuery::set_archetype`]. `entity` and + /// `table_row` must be in the range of the current table and archetype. #[allow(unused_variables)] - #[inline] - unsafe fn archetype_filter_fetch( - fetch: &mut >::Fetch, - archetype_index: usize, - ) -> bool { - true - } - - /// # Safety - /// - /// Must always be called _after_ [`WorldQuery::set_table`]. `table_row` must be in the range of the - /// current table. - #[allow(unused_variables)] - #[inline] - unsafe fn table_filter_fetch( + #[inline(always)] + unsafe fn filter_fetch( fetch: &mut >::Fetch, + entity: Entity, table_row: usize, ) -> bool { true @@ -470,11 +448,6 @@ pub type ROQueryFetch<'w, Q> = QueryFetch<'w, ::ReadOnly>; /// The read-only variant of the item type returned when a [`WorldQuery`] is iterated over immutably pub type ROQueryItem<'w, Q> = QueryItem<'w, ::ReadOnly>; -#[doc(hidden)] -pub struct EntityFetch<'w> { - entities: Option>, -} - /// SAFETY: no component or archetype access unsafe impl WorldQuery for Entity { type ReadOnly = Self; @@ -490,56 +463,41 @@ unsafe impl WorldQuery for Entity { unsafe fn init_fetch<'w>( _world: &'w World, - _state: &(), + _state: &Self::State, _last_change_tick: u32, _change_tick: u32, - ) -> EntityFetch<'w> { - EntityFetch { entities: None } + ) -> >::Fetch { } unsafe fn clone_fetch<'w>( - fetch: &>::Fetch, + _fetch: &>::Fetch, ) -> >::Fetch { - EntityFetch { - entities: fetch.entities, - } } #[inline] unsafe fn set_archetype<'w>( - fetch: &mut EntityFetch<'w>, - _state: &(), - archetype: &'w Archetype, - _tables: &Tables, + _fetch: &mut >::Fetch, + _state: &Self::State, + _archetype: &'w Archetype, + _table: &Table, ) { - fetch.entities = Some(archetype.entities().into()); - } - - #[inline] - unsafe fn set_table<'w>(fetch: &mut EntityFetch<'w>, _state: &(), table: &'w Table) { - fetch.entities = Some(table.entities().into()); } #[inline] - unsafe fn table_fetch<'w>( - fetch: &mut >::Fetch, - table_row: usize, - ) -> QueryItem<'w, Self> { - let entities = fetch - .entities - .unwrap_or_else(|| debug_checked_unreachable()); - *entities.get(table_row) + unsafe fn set_table<'w>( + _fetch: &mut >::Fetch, + _state: &Self::State, + _table: &'w Table, + ) { } - #[inline] - unsafe fn archetype_fetch<'w>( - fetch: &mut >::Fetch, - archetype_index: usize, + #[inline(always)] + unsafe fn fetch<'w>( + _fetch: &mut >::Fetch, + entity: Entity, + _table_row: usize, ) -> >::Item { - let entities = fetch - .entities - .unwrap_or_else(|| debug_checked_unreachable()); - *entities.get(archetype_index) + entity } fn update_component_access(_state: &Self::State, _access: &mut FilteredAccess) {} @@ -562,7 +520,7 @@ unsafe impl WorldQuery for Entity { } impl<'w> WorldQueryGats<'w> for Entity { - type Fetch = EntityFetch<'w>; + type Fetch = (); type Item = Entity; } @@ -573,9 +531,7 @@ unsafe impl ReadOnlyWorldQuery for Entity {} pub struct ReadFetch<'w, T> { // T::Storage = TableStorage table_components: Option>>, - entity_table_rows: Option>, // T::Storage = SparseStorage - entities: Option>, sparse_set: Option<&'w ComponentSparseSet>, } @@ -605,10 +561,13 @@ unsafe impl WorldQuery for &T { ) -> ReadFetch<'w, T> { ReadFetch { table_components: None, - entity_table_rows: None, - entities: None, - sparse_set: (T::Storage::STORAGE_TYPE == StorageType::SparseSet) - .then(|| world.storages().sparse_sets.get(component_id).unwrap()), + sparse_set: (T::Storage::STORAGE_TYPE == StorageType::SparseSet).then(|| { + world + .storages() + .sparse_sets + .get(component_id) + .unwrap_or_else(|| debug_checked_unreachable()) + }), } } @@ -617,8 +576,6 @@ unsafe impl WorldQuery for &T { ) -> >::Fetch { ReadFetch { table_components: fetch.table_components, - entity_table_rows: fetch.entity_table_rows, - entities: fetch.entities, sparse_set: fetch.sparse_set, } } @@ -626,66 +583,51 @@ unsafe impl WorldQuery for &T { #[inline] unsafe fn set_archetype<'w>( fetch: &mut ReadFetch<'w, T>, - &component_id: &ComponentId, - archetype: &'w Archetype, - tables: &'w Tables, + component_id: &ComponentId, + _archetype: &'w Archetype, + table: &'w Table, ) { - match T::Storage::STORAGE_TYPE { - StorageType::Table => { - fetch.entity_table_rows = Some(archetype.entity_table_rows().into()); - let column = tables[archetype.table_id()] - .get_column(component_id) - .unwrap(); - fetch.table_components = Some(column.get_data_slice().into()); - } - StorageType::SparseSet => fetch.entities = Some(archetype.entities().into()), + if Self::IS_DENSE { + Self::set_table(fetch, component_id, table); } } #[inline] - unsafe fn set_table<'w>(fetch: &mut ReadFetch<'w, T>, &id: &ComponentId, table: &'w Table) { - fetch.table_components = Some(table.get_column(id).unwrap().get_data_slice().into()); + unsafe fn set_table<'w>( + fetch: &mut ReadFetch<'w, T>, + &component_id: &ComponentId, + table: &'w Table, + ) { + fetch.table_components = Some( + table + .get_column(component_id) + .unwrap_or_else(|| debug_checked_unreachable()) + .get_data_slice() + .into(), + ); } - #[inline] - unsafe fn archetype_fetch<'w>( + #[inline(always)] + unsafe fn fetch<'w>( fetch: &mut >::Fetch, - archetype_index: usize, + entity: Entity, + table_row: usize, ) -> >::Item { match T::Storage::STORAGE_TYPE { - StorageType::Table => { - let (entity_table_rows, table_components) = fetch - .entity_table_rows - .zip(fetch.table_components) - .unwrap_or_else(|| debug_checked_unreachable()); - let table_row = *entity_table_rows.get(archetype_index); - table_components.get(table_row).deref() - } - StorageType::SparseSet => { - let (entities, sparse_set) = fetch - .entities - .zip(fetch.sparse_set) - .unwrap_or_else(|| debug_checked_unreachable()); - let entity = *entities.get(archetype_index); - sparse_set - .get(entity) - .unwrap_or_else(|| debug_checked_unreachable()) - .deref::() - } + StorageType::Table => fetch + .table_components + .unwrap_or_else(|| debug_checked_unreachable()) + .get(table_row) + .deref(), + StorageType::SparseSet => fetch + .sparse_set + .unwrap_or_else(|| debug_checked_unreachable()) + .get(entity) + .unwrap_or_else(|| debug_checked_unreachable()) + .deref(), } } - #[inline] - unsafe fn table_fetch<'w>( - fetch: &mut >::Fetch, - table_row: usize, - ) -> >::Item { - let components = fetch - .table_components - .unwrap_or_else(|| debug_checked_unreachable()); - components.get(table_row).deref() - } - fn update_component_access( &component_id: &ComponentId, access: &mut FilteredAccess, @@ -731,11 +673,11 @@ impl<'w, T: Component> WorldQueryGats<'w> for &T { #[doc(hidden)] pub struct WriteFetch<'w, T> { // T::Storage = TableStorage - table_components: Option>>, - table_ticks: Option>>, - entity_table_rows: Option>, + table_data: Option<( + ThinSlicePtr<'w, UnsafeCell>, + ThinSlicePtr<'w, UnsafeCell>, + )>, // T::Storage = SparseStorage - entities: Option>, sparse_set: Option<&'w ComponentSparseSet>, last_change_tick: u32, @@ -767,12 +709,14 @@ unsafe impl<'__w, T: Component> WorldQuery for &'__w mut T { change_tick: u32, ) -> WriteFetch<'w, T> { WriteFetch { - table_components: None, - entities: None, - entity_table_rows: None, - sparse_set: (T::Storage::STORAGE_TYPE == StorageType::SparseSet) - .then(|| world.storages().sparse_sets.get(component_id).unwrap()), - table_ticks: None, + table_data: None, + sparse_set: (T::Storage::STORAGE_TYPE == StorageType::SparseSet).then(|| { + world + .storages() + .sparse_sets + .get(component_id) + .unwrap_or_else(|| debug_checked_unreachable()) + }), last_change_tick, change_tick, } @@ -782,10 +726,7 @@ unsafe impl<'__w, T: Component> WorldQuery for &'__w mut T { fetch: &>::Fetch, ) -> >::Fetch { WriteFetch { - table_components: fetch.table_components, - table_ticks: fetch.table_ticks, - entities: fetch.entities, - entity_table_rows: fetch.entity_table_rows, + table_data: fetch.table_data, sparse_set: fetch.sparse_set, last_change_tick: fetch.last_change_tick, change_tick: fetch.change_tick, @@ -795,20 +736,12 @@ unsafe impl<'__w, T: Component> WorldQuery for &'__w mut T { #[inline] unsafe fn set_archetype<'w>( fetch: &mut WriteFetch<'w, T>, - &component_id: &ComponentId, - archetype: &'w Archetype, - tables: &'w Tables, + component_id: &ComponentId, + _archetype: &'w Archetype, + table: &'w Table, ) { - match T::Storage::STORAGE_TYPE { - StorageType::Table => { - fetch.entity_table_rows = Some(archetype.entity_table_rows().into()); - let column = tables[archetype.table_id()] - .get_column(component_id) - .unwrap(); - fetch.table_components = Some(column.get_data_slice().into()); - fetch.table_ticks = Some(column.get_ticks_slice().into()); - } - StorageType::SparseSet => fetch.entities = Some(archetype.entities().into()), + if Self::IS_DENSE { + Self::set_table(fetch, component_id, table); } } @@ -818,23 +751,26 @@ unsafe impl<'__w, T: Component> WorldQuery for &'__w mut T { &component_id: &ComponentId, table: &'w Table, ) { - let column = table.get_column(component_id).unwrap(); - fetch.table_components = Some(column.get_data_slice().into()); - fetch.table_ticks = Some(column.get_ticks_slice().into()); + let column = table + .get_column(component_id) + .unwrap_or_else(|| debug_checked_unreachable()); + fetch.table_data = Some(( + column.get_data_slice().into(), + column.get_ticks_slice().into(), + )); } - #[inline] - unsafe fn archetype_fetch<'w>( + #[inline(always)] + unsafe fn fetch<'w>( fetch: &mut >::Fetch, - archetype_index: usize, + entity: Entity, + table_row: usize, ) -> >::Item { match T::Storage::STORAGE_TYPE { StorageType::Table => { - let (entity_table_rows, (table_components, table_ticks)) = fetch - .entity_table_rows - .zip(fetch.table_components.zip(fetch.table_ticks)) + let (table_components, table_ticks) = fetch + .table_data .unwrap_or_else(|| debug_checked_unreachable()); - let table_row = *entity_table_rows.get(archetype_index); Mut { value: table_components.get(table_row).deref_mut(), ticks: Ticks { @@ -845,12 +781,9 @@ unsafe impl<'__w, T: Component> WorldQuery for &'__w mut T { } } StorageType::SparseSet => { - let (entities, sparse_set) = fetch - .entities - .zip(fetch.sparse_set) - .unwrap_or_else(|| debug_checked_unreachable()); - let entity = *entities.get(archetype_index); - let (component, component_ticks) = sparse_set + let (component, component_ticks) = fetch + .sparse_set + .unwrap_or_else(|| debug_checked_unreachable()) .get_with_ticks(entity) .unwrap_or_else(|| debug_checked_unreachable()); Mut { @@ -865,25 +798,6 @@ unsafe impl<'__w, T: Component> WorldQuery for &'__w mut T { } } - #[inline] - unsafe fn table_fetch<'w>( - fetch: &mut >::Fetch, - table_row: usize, - ) -> >::Item { - let (table_components, table_ticks) = fetch - .table_components - .zip(fetch.table_ticks) - .unwrap_or_else(|| debug_checked_unreachable()); - Mut { - value: table_components.get(table_row).deref_mut(), - ticks: Ticks { - component_ticks: table_ticks.get(table_row).deref_mut(), - change_tick: fetch.change_tick, - last_change_tick: fetch.last_change_tick, - }, - } - } - fn update_component_access( &component_id: &ComponentId, access: &mut FilteredAccess, @@ -968,11 +882,11 @@ unsafe impl WorldQuery for Option { fetch: &mut OptionFetch<'w, T>, state: &T::State, archetype: &'w Archetype, - tables: &'w Tables, + table: &'w Table, ) { fetch.matches = T::matches_component_set(state, &|id| archetype.contains(id)); if fetch.matches { - T::set_archetype(&mut fetch.fetch, state, archetype, tables); + T::set_archetype(&mut fetch.fetch, state, archetype, table); } } @@ -984,28 +898,15 @@ unsafe impl WorldQuery for Option { } } - #[inline] - unsafe fn archetype_fetch<'w>( - fetch: &mut >::Fetch, - archetype_index: usize, - ) -> >::Item { - if fetch.matches { - Some(T::archetype_fetch(&mut fetch.fetch, archetype_index)) - } else { - None - } - } - - #[inline] - unsafe fn table_fetch<'w>( + #[inline(always)] + unsafe fn fetch<'w>( fetch: &mut >::Fetch, + entity: Entity, table_row: usize, ) -> >::Item { - if fetch.matches { - Some(T::table_fetch(&mut fetch.fetch, table_row)) - } else { - None - } + fetch + .matches + .then(|| T::fetch(&mut fetch.fetch, entity, table_row)) } fn update_component_access(state: &T::State, access: &mut FilteredAccess) { @@ -1127,9 +1028,7 @@ impl ChangeTrackers { pub struct ChangeTrackersFetch<'w, T> { // T::Storage = TableStorage table_ticks: Option>>, - entity_table_rows: Option>, // T::Storage = SparseStorage - entities: Option>, sparse_set: Option<&'w ComponentSparseSet>, marker: PhantomData, @@ -1157,16 +1056,19 @@ unsafe impl WorldQuery for ChangeTrackers { unsafe fn init_fetch<'w>( world: &'w World, - &id: &ComponentId, + &component_id: &ComponentId, last_change_tick: u32, change_tick: u32, ) -> ChangeTrackersFetch<'w, T> { ChangeTrackersFetch { table_ticks: None, - entities: None, - entity_table_rows: None, - sparse_set: (T::Storage::STORAGE_TYPE == StorageType::SparseSet) - .then(|| world.storages().sparse_sets.get(id).unwrap()), + sparse_set: (T::Storage::STORAGE_TYPE == StorageType::SparseSet).then(|| { + world + .storages() + .sparse_sets + .get(component_id) + .unwrap_or_else(|| debug_checked_unreachable()) + }), marker: PhantomData, last_change_tick, change_tick, @@ -1178,8 +1080,6 @@ unsafe impl WorldQuery for ChangeTrackers { ) -> >::Fetch { ChangeTrackersFetch { table_ticks: fetch.table_ticks, - entity_table_rows: fetch.entity_table_rows, - entities: fetch.entities, sparse_set: fetch.sparse_set, marker: fetch.marker, last_change_tick: fetch.last_change_tick, @@ -1190,17 +1090,12 @@ unsafe impl WorldQuery for ChangeTrackers { #[inline] unsafe fn set_archetype<'w>( fetch: &mut ChangeTrackersFetch<'w, T>, - &id: &ComponentId, - archetype: &'w Archetype, - tables: &'w Tables, + component_id: &ComponentId, + _archetype: &'w Archetype, + table: &'w Table, ) { - match T::Storage::STORAGE_TYPE { - StorageType::Table => { - fetch.entity_table_rows = Some(archetype.entity_table_rows().into()); - let column = tables[archetype.table_id()].get_column(id).unwrap(); - fetch.table_ticks = Some(column.get_ticks_slice().into()); - } - StorageType::SparseSet => fetch.entities = Some(archetype.entities().into()), + if Self::IS_DENSE { + Self::set_table(fetch, component_id, table); } } @@ -1210,68 +1105,44 @@ unsafe impl WorldQuery for ChangeTrackers { &id: &ComponentId, table: &'w Table, ) { - fetch.table_ticks = Some(table.get_column(id).unwrap().get_ticks_slice().into()); - } - - #[inline] - unsafe fn archetype_fetch<'w>( - fetch: &mut >::Fetch, - archetype_index: usize, - ) -> >::Item { - match T::Storage::STORAGE_TYPE { - StorageType::Table => { - let entity_table_rows = fetch - .entity_table_rows - .unwrap_or_else(|| debug_checked_unreachable()); - let table_row = *entity_table_rows.get(archetype_index); - ChangeTrackers { - component_ticks: { - let table_ticks = fetch - .table_ticks - .unwrap_or_else(|| debug_checked_unreachable()); - table_ticks.get(table_row).read() - }, - marker: PhantomData, - last_change_tick: fetch.last_change_tick, - change_tick: fetch.change_tick, - } - } - StorageType::SparseSet => { - let entities = fetch - .entities - .unwrap_or_else(|| debug_checked_unreachable()); - let entity = *entities.get(archetype_index); - ChangeTrackers { - component_ticks: fetch - .sparse_set - .unwrap_or_else(|| debug_checked_unreachable()) - .get_ticks(entity) - .map(|ticks| &*ticks.get()) - .cloned() - .unwrap_or_else(|| debug_checked_unreachable()), - marker: PhantomData, - last_change_tick: fetch.last_change_tick, - change_tick: fetch.change_tick, - } - } - } + fetch.table_ticks = Some( + table + .get_column(id) + .unwrap_or_else(|| debug_checked_unreachable()) + .get_ticks_slice() + .into(), + ); } - #[inline] - unsafe fn table_fetch<'w>( + #[inline(always)] + unsafe fn fetch<'w>( fetch: &mut >::Fetch, + entity: Entity, table_row: usize, ) -> >::Item { - ChangeTrackers { - component_ticks: { - let table_ticks = fetch - .table_ticks - .unwrap_or_else(|| debug_checked_unreachable()); - table_ticks.get(table_row).read() + match T::Storage::STORAGE_TYPE { + StorageType::Table => ChangeTrackers { + component_ticks: { + let table_ticks = fetch + .table_ticks + .unwrap_or_else(|| debug_checked_unreachable()); + table_ticks.get(table_row).read() + }, + marker: PhantomData, + last_change_tick: fetch.last_change_tick, + change_tick: fetch.change_tick, + }, + StorageType::SparseSet => ChangeTrackers { + component_ticks: *fetch + .sparse_set + .unwrap_or_else(|| debug_checked_unreachable()) + .get_ticks(entity) + .unwrap_or_else(|| debug_checked_unreachable()) + .get(), + marker: PhantomData, + last_change_tick: fetch.last_change_tick, + change_tick: fetch.change_tick, }, - marker: PhantomData, - last_change_tick: fetch.last_change_tick, - change_tick: fetch.change_tick, } } @@ -1355,10 +1226,15 @@ macro_rules! impl_tuple_fetch { const IS_ARCHETYPAL: bool = true $(&& $name::IS_ARCHETYPAL)*; #[inline] - unsafe fn set_archetype<'w>(_fetch: &mut >::Fetch, _state: &Self::State, _archetype: &'w Archetype, _tables: &'w Tables) { + unsafe fn set_archetype<'w>( + _fetch: &mut >::Fetch, + _state: &Self::State, + _archetype: &'w Archetype, + _table: &'w Table + ) { let ($($name,)*) = _fetch; let ($($state,)*) = _state; - $($name::set_archetype($name, $state, _archetype, _tables);)* + $($name::set_archetype($name, $state, _archetype, _table);)* } #[inline] @@ -1368,32 +1244,25 @@ macro_rules! impl_tuple_fetch { $($name::set_table($name, $state, _table);)* } - #[inline] + #[inline(always)] #[allow(clippy::unused_unit)] - unsafe fn table_fetch<'w>(_fetch: &mut >::Fetch, _table_row: usize) -> QueryItem<'w, Self> { + unsafe fn fetch<'w>( + _fetch: &mut >::Fetch, + _entity: Entity, + _table_row: usize + ) -> >::Item { let ($($name,)*) = _fetch; - ($($name::table_fetch($name, _table_row),)*) + ($($name::fetch($name, _entity, _table_row),)*) } - #[inline] - #[allow(clippy::unused_unit)] - unsafe fn archetype_fetch<'w>(_fetch: &mut >::Fetch, _archetype_index: usize) -> QueryItem<'w, Self> { - let ($($name,)*) = _fetch; - ($($name::archetype_fetch($name, _archetype_index),)*) - } - - #[allow(unused_variables)] - #[inline] - unsafe fn table_filter_fetch(_fetch: &mut QueryFetch<'_, Self>, table_row: usize) -> bool { + #[inline(always)] + unsafe fn filter_fetch<'w>( + _fetch: &mut >::Fetch, + _entity: Entity, + _table_row: usize + ) -> bool { let ($($name,)*) = _fetch; - true $(&& $name::table_filter_fetch($name, table_row))* - } - - #[allow(unused_variables)] - #[inline] - unsafe fn archetype_filter_fetch(_fetch: &mut QueryFetch<'_, Self>, archetype_index: usize) -> bool { - let ($($name,)*) = _fetch; - true $(&& $name::archetype_filter_fetch($name, archetype_index))* + true $(&& $name::filter_fetch($name, _entity, _table_row))* } fn update_component_access(state: &Self::State, _access: &mut FilteredAccess) { @@ -1471,13 +1340,18 @@ macro_rules! impl_anytuple_fetch { const IS_ARCHETYPAL: bool = true $(&& $name::IS_ARCHETYPAL)*; #[inline] - unsafe fn set_archetype<'w>(_fetch: &mut >::Fetch, _state: &Self::State, _archetype: &'w Archetype, _tables: &'w Tables) { + unsafe fn set_archetype<'w>( + _fetch: &mut >::Fetch, + _state: &Self::State, + _archetype: &'w Archetype, + _table: &'w Table + ) { let ($($name,)*) = _fetch; let ($($state,)*) = _state; $( $name.1 = $name::matches_component_set($state, &|id| _archetype.contains(id)); if $name.1 { - $name::set_archetype(&mut $name.0, $state, _archetype, _tables); + $name::set_archetype(&mut $name.0, $state, _archetype, _table); } )* } @@ -1494,21 +1368,16 @@ macro_rules! impl_anytuple_fetch { )* } - #[inline] - #[allow(clippy::unused_unit)] - unsafe fn table_fetch<'w>(_fetch: &mut >::Fetch, _table_row: usize) -> QueryItem<'w, Self> { - let ($($name,)*) = _fetch; - ($( - $name.1.then(|| $name::table_fetch(&mut $name.0, _table_row)), - )*) - } - - #[inline] + #[inline(always)] #[allow(clippy::unused_unit)] - unsafe fn archetype_fetch<'w>(_fetch: &mut >::Fetch, _archetype_index: usize) -> QueryItem<'w, Self> { + unsafe fn fetch<'w>( + _fetch: &mut >::Fetch, + _entity: Entity, + _table_row: usize + ) -> >::Item { let ($($name,)*) = _fetch; ($( - $name.1.then(|| $name::archetype_fetch(&mut $name.0, _archetype_index)), + $name.1.then(|| $name::fetch(&mut $name.0, _entity, _table_row)), )*) } @@ -1608,7 +1477,7 @@ unsafe impl WorldQuery for NopWorldQuery { _fetch: &mut (), _state: &Q::State, _archetype: &Archetype, - _tables: &Tables, + _tables: &Table, ) { } @@ -1616,15 +1485,9 @@ unsafe impl WorldQuery for NopWorldQuery { unsafe fn set_table<'w>(_fetch: &mut (), _state: &Q::State, _table: &Table) {} #[inline(always)] - unsafe fn archetype_fetch<'w>( + unsafe fn fetch<'w>( _fetch: &mut >::Fetch, - _archetype_index: usize, - ) -> >::Item { - } - - #[inline(always)] - unsafe fn table_fetch<'w>( - _fetch: &mut (), + _entity: Entity, _table_row: usize, ) -> >::Item { } diff --git a/crates/bevy_ecs/src/query/filter.rs b/crates/bevy_ecs/src/query/filter.rs index 238964aaa909b8..1571907f567a6d 100644 --- a/crates/bevy_ecs/src/query/filter.rs +++ b/crates/bevy_ecs/src/query/filter.rs @@ -5,7 +5,7 @@ use crate::{ query::{ debug_checked_unreachable, Access, FilteredAccess, QueryFetch, WorldQuery, WorldQueryGats, }, - storage::{ComponentSparseSet, Table, Tables}, + storage::{ComponentSparseSet, Table}, world::World, }; use bevy_ecs_macros::all_tuples; @@ -88,20 +88,14 @@ unsafe impl WorldQuery for With { _fetch: &mut (), _state: &ComponentId, _archetype: &Archetype, - _tables: &Tables, + _table: &Table, ) { } - #[inline] - unsafe fn archetype_fetch<'w>( - _fetch: &mut >::Fetch, - _archetype_index: usize, - ) -> >::Item { - } - - #[inline] - unsafe fn table_fetch<'w>( + #[inline(always)] + unsafe fn fetch<'w>( _fetch: &mut >::Fetch, + _entity: Entity, _table_row: usize, ) -> >::Item { } @@ -200,20 +194,14 @@ unsafe impl WorldQuery for Without { _fetch: &mut (), _state: &ComponentId, _archetype: &Archetype, - _tables: &Tables, + _table: &Table, ) { } - #[inline] - unsafe fn archetype_fetch<'w>( - _fetch: &mut >::Fetch, - _archetype_index: usize, - ) -> >::Item { - } - - #[inline] - unsafe fn table_fetch<'w>( + #[inline(always)] + unsafe fn fetch<'w>( _fetch: &mut >::Fetch, + _entity: Entity, _table_row: usize, ) -> >::Item { } @@ -348,37 +336,39 @@ macro_rules! impl_query_filter_tuple { } #[inline] - unsafe fn set_archetype<'w>(fetch: &mut >::Fetch, state: &Self::State, archetype: &'w Archetype, tables: &'w Tables) { + unsafe fn set_archetype<'w>( + fetch: &mut >::Fetch, + state: & Self::State, + archetype: &'w Archetype, + table: &'w Table + ) { let ($($filter,)*) = fetch; - let ($($state,)*) = state; + let ($($state,)*) = &state; $( $filter.matches = $filter::matches_component_set($state, &|id| archetype.contains(id)); if $filter.matches { - $filter::set_archetype(&mut $filter.fetch, $state, archetype, tables); + $filter::set_archetype(&mut $filter.fetch, $state, archetype, table); } )* } - #[inline] - unsafe fn table_fetch<'w>(fetch: &mut >::Fetch, table_row: usize) -> >::Item { - let ($($filter,)*) = fetch; - false $(|| ($filter.matches && $filter::table_filter_fetch(&mut $filter.fetch, table_row)))* - } - - #[inline] - unsafe fn archetype_fetch<'w>(fetch: &mut >::Fetch, archetype_index: usize) -> >::Item { + #[inline(always)] + unsafe fn fetch<'w>( + fetch: &mut >::Fetch, + _entity: Entity, + _table_row: usize + ) -> >::Item { let ($($filter,)*) = fetch; - false $(|| ($filter.matches && $filter::archetype_filter_fetch(&mut $filter.fetch, archetype_index)))* + false $(|| ($filter.matches && $filter::filter_fetch(&mut $filter.fetch, _entity, _table_row)))* } - #[inline] - unsafe fn table_filter_fetch(fetch: &mut QueryFetch<'_, Self>, table_row: usize) -> bool { - Self::table_fetch(fetch, table_row) - } - - #[inline] - unsafe fn archetype_filter_fetch(fetch: &mut QueryFetch<'_, Self>, archetype_index: usize) -> bool { - Self::archetype_fetch(fetch, archetype_index) + #[inline(always)] + unsafe fn filter_fetch<'w>( + fetch: &mut >::Fetch, + entity: Entity, + table_row: usize + ) -> bool { + Self::fetch(fetch, entity, table_row) } fn update_component_access(state: &Self::State, access: &mut FilteredAccess) { @@ -450,9 +440,7 @@ macro_rules! impl_tick_filter { $(#[$fetch_meta])* pub struct $fetch_name<'w, T> { table_ticks: Option>>, - entity_table_rows: Option>, marker: PhantomData, - entities: Option>, sparse_set: Option<&'w ComponentSparseSet>, last_change_tick: u32, change_tick: u32, @@ -470,10 +458,13 @@ macro_rules! impl_tick_filter { unsafe fn init_fetch<'w>(world: &'w World, &id: &ComponentId, last_change_tick: u32, change_tick: u32) -> >::Fetch { QueryFetch::<'w, Self> { table_ticks: None, - entities: None, - entity_table_rows: None, sparse_set: (T::Storage::STORAGE_TYPE == StorageType::SparseSet) - .then(|| world.storages().sparse_sets.get(id).unwrap()), + .then(|| { + world.storages() + .sparse_sets + .get(id) + .unwrap_or_else(|| debug_checked_unreachable()) + }), marker: PhantomData, last_change_tick, change_tick, @@ -485,8 +476,6 @@ macro_rules! impl_tick_filter { ) -> >::Fetch { $fetch_name { table_ticks: fetch.table_ticks, - entity_table_rows: fetch.entity_table_rows, - entities: fetch.entities, sparse_set: fetch.sparse_set, last_change_tick: fetch.last_change_tick, change_tick: fetch.change_tick, @@ -503,53 +492,68 @@ macro_rules! impl_tick_filter { const IS_ARCHETYPAL: bool = false; - unsafe fn set_table<'w>(fetch: &mut >::Fetch, &id: &ComponentId, table: &'w Table) { - fetch.table_ticks = Some(table.get_column(id).unwrap().get_ticks_slice().into()); + #[inline] + unsafe fn set_table<'w>( + fetch: &mut >::Fetch, + &component_id: &ComponentId, + table: &'w Table + ) { + fetch.table_ticks = Some( + table.get_column(component_id) + .unwrap_or_else(|| debug_checked_unreachable()) + .get_ticks_slice() + .into() + ); } - unsafe fn set_archetype<'w>(fetch: &mut >::Fetch, &id: &ComponentId, archetype: &'w Archetype, tables: &'w Tables) { - match T::Storage::STORAGE_TYPE { - StorageType::Table => { - fetch.entity_table_rows = Some(archetype.entity_table_rows().into()); - let table = &tables[archetype.table_id()]; - fetch.table_ticks = Some(table.get_column(id).unwrap().get_ticks_slice().into()); - } - StorageType::SparseSet => fetch.entities = Some(archetype.entities().into()), + #[inline] + unsafe fn set_archetype<'w>( + fetch: &mut >::Fetch, + component_id: &ComponentId, + _archetype: &'w Archetype, + table: &'w Table + ) { + if Self::IS_DENSE { + Self::set_table(fetch, component_id, table); } } - unsafe fn table_fetch<'w>(fetch: &mut >::Fetch, table_row: usize) -> >::Item { - $is_detected(&*(fetch.table_ticks.unwrap_or_else(|| debug_checked_unreachable()).get(table_row)).deref(), fetch.last_change_tick, fetch.change_tick) - } - - unsafe fn archetype_fetch<'w>(fetch: &mut >::Fetch, archetype_index: usize) -> >::Item { + #[inline(always)] + unsafe fn fetch<'w>( + fetch: &mut >::Fetch, + entity: Entity, + table_row: usize + ) -> >::Item { match T::Storage::STORAGE_TYPE { StorageType::Table => { - let table_row = *fetch.entity_table_rows.unwrap_or_else(|| debug_checked_unreachable()).get(archetype_index); - $is_detected(&*(fetch.table_ticks.unwrap_or_else(|| debug_checked_unreachable()).get(table_row)).deref(), fetch.last_change_tick, fetch.change_tick) + $is_detected(&*( + fetch.table_ticks + .unwrap_or_else(|| debug_checked_unreachable()) + .get(table_row)) + .deref(), + fetch.last_change_tick, + fetch.change_tick + ) } StorageType::SparseSet => { - let entity = *fetch.entities.unwrap_or_else(|| debug_checked_unreachable()).get(archetype_index); - let ticks = fetch + let ticks = &*fetch .sparse_set .unwrap_or_else(|| debug_checked_unreachable()) .get_ticks(entity) - .map(|ticks| &*ticks.get()) - .cloned() - .unwrap(); - $is_detected(&ticks, fetch.last_change_tick, fetch.change_tick) + .unwrap_or_else(|| debug_checked_unreachable()) + .get(); + $is_detected(ticks, fetch.last_change_tick, fetch.change_tick) } } } - #[inline] - unsafe fn table_filter_fetch(fetch: &mut QueryFetch<'_, Self>, table_row: usize) -> bool { - Self::table_fetch(fetch, table_row) - } - - #[inline] - unsafe fn archetype_filter_fetch(fetch: &mut QueryFetch<'_, Self>, archetype_index: usize) -> bool { - Self::archetype_fetch(fetch, archetype_index) + #[inline(always)] + unsafe fn filter_fetch<'w>( + fetch: &mut QueryFetch<'w, Self>, + entity: Entity, + table_row: usize + ) -> bool { + Self::fetch(fetch, entity, table_row) } #[inline] @@ -625,7 +629,7 @@ impl_tick_filter!( impl_tick_filter!( /// A filter on a component that only retains results added or mutably dereferenced after the system last ran. - /// + /// /// A common use for this filter is avoiding redundant work when values have not changed. /// /// **Note** that simply *mutably dereferencing* a component is considered a change ([`DerefMut`](std::ops::DerefMut)). diff --git a/crates/bevy_ecs/src/query/iter.rs b/crates/bevy_ecs/src/query/iter.rs index 174048f3b40b7e..8a693eabb02c43 100644 --- a/crates/bevy_ecs/src/query/iter.rs +++ b/crates/bevy_ecs/src/query/iter.rs @@ -1,5 +1,5 @@ use crate::{ - archetype::{ArchetypeId, Archetypes}, + archetype::{ArchetypeEntity, ArchetypeId, Archetypes}, entity::{Entities, Entity}, prelude::World, query::{ArchetypeFilter, QueryState, WorldQuery}, @@ -139,7 +139,8 @@ where #[inline(always)] unsafe fn fetch_next_aliased_unchecked(&mut self) -> Option> { for entity in self.entity_iter.by_ref() { - let location = match self.entities.get(*entity.borrow()) { + let entity = *entity.borrow(); + let location = match self.entities.get(entity) { Some(location) => location, None => continue, }; @@ -153,6 +154,7 @@ where } let archetype = &self.archetypes[location.archetype_id]; + let table = &self.tables[archetype.table_id()]; // SAFETY: `archetype` is from the world that `fetch/filter` were created for, // `fetch_state`/`filter_state` are the states that `fetch/filter` were initialized with @@ -160,7 +162,7 @@ where &mut self.fetch, &self.query_state.fetch_state, archetype, - self.tables, + table, ); // SAFETY: `table` is from the world that `fetch/filter` were created for, // `fetch_state`/`filter_state` are the states that `fetch/filter` were initialized with @@ -168,13 +170,15 @@ where &mut self.filter, &self.query_state.filter_state, archetype, - self.tables, + table, ); + + let table_row = archetype.entity_table_row(location.index); // SAFETY: set_archetype was called prior. // `location.index` is an archetype index row in range of the current archetype, because if it was not, the match above would have `continue`d - if F::archetype_filter_fetch(&mut self.filter, location.index) { + if F::filter_fetch(&mut self.filter, entity, table_row) { // SAFETY: set_archetype was called prior, `location.index` is an archetype index in range of the current archetype - return Some(Q::archetype_fetch(&mut self.fetch, location.index)); + return Some(Q::fetch(&mut self.fetch, entity, table_row)); } } None @@ -462,6 +466,8 @@ impl<'w, 's, Q: ReadOnlyWorldQuery, F: ReadOnlyWorldQuery, const K: usize> Fused struct QueryIterationCursor<'w, 's, Q: WorldQuery, F: ReadOnlyWorldQuery> { table_id_iter: std::slice::Iter<'s, TableId>, archetype_id_iter: std::slice::Iter<'s, ArchetypeId>, + table_entities: &'w [Entity], + archetype_entities: &'w [ArchetypeEntity], fetch: QueryFetch<'w, Q>, filter: QueryFetch<'w, F>, // length of the table table or length of the archetype, depending on whether both `Q`'s and `F`'s fetches are dense @@ -482,6 +488,8 @@ impl<'w, 's, Q: WorldQuery, F: ReadOnlyWorldQuery> QueryIterationCursor<'w, 's, Self { table_id_iter: self.table_id_iter.clone(), archetype_id_iter: self.archetype_id_iter.clone(), + table_entities: self.table_entities, + archetype_entities: self.archetype_entities, // SAFETY: upheld by caller invariants fetch: Q::clone_fetch(&self.fetch), filter: F::clone_fetch(&self.filter), @@ -529,6 +537,8 @@ impl<'w, 's, Q: WorldQuery, F: ReadOnlyWorldQuery> QueryIterationCursor<'w, 's, QueryIterationCursor { fetch, filter, + table_entities: &[], + archetype_entities: &[], table_id_iter: query_state.matched_table_ids.iter(), archetype_id_iter: query_state.matched_archetype_ids.iter(), current_len: 0, @@ -541,10 +551,17 @@ impl<'w, 's, Q: WorldQuery, F: ReadOnlyWorldQuery> QueryIterationCursor<'w, 's, #[inline] unsafe fn peek_last(&mut self) -> Option> { if self.current_index > 0 { + let index = self.current_index - 1; if Self::IS_DENSE { - Some(Q::table_fetch(&mut self.fetch, self.current_index - 1)) + let entity = self.table_entities.get_unchecked(index); + Some(Q::fetch(&mut self.fetch, *entity, index)) } else { - Some(Q::archetype_fetch(&mut self.fetch, self.current_index - 1)) + let archetype_entity = self.archetype_entities.get_unchecked(index); + Some(Q::fetch( + &mut self.fetch, + archetype_entity.entity, + archetype_entity.table_row, + )) } } else { None @@ -574,6 +591,7 @@ impl<'w, 's, Q: WorldQuery, F: ReadOnlyWorldQuery> QueryIterationCursor<'w, 's, // `fetch_state`/`filter_state` are the states that `fetch/filter` were initialized with Q::set_table(&mut self.fetch, &query_state.fetch_state, table); F::set_table(&mut self.filter, &query_state.filter_state, table); + self.table_entities = table.entities(); self.current_len = table.entity_count(); self.current_index = 0; continue; @@ -581,14 +599,15 @@ impl<'w, 's, Q: WorldQuery, F: ReadOnlyWorldQuery> QueryIterationCursor<'w, 's, // SAFETY: set_table was called prior. // `current_index` is a table row in range of the current table, because if it was not, then the if above would have been executed. - if !F::table_filter_fetch(&mut self.filter, self.current_index) { + let entity = self.table_entities.get_unchecked(self.current_index); + if !F::filter_fetch(&mut self.filter, *entity, self.current_index) { self.current_index += 1; continue; } // SAFETY: set_table was called prior. // `current_index` is a table row in range of the current table, because if it was not, then the if above would have been executed. - let item = Q::table_fetch(&mut self.fetch, self.current_index); + let item = Q::fetch(&mut self.fetch, *entity, self.current_index); self.current_index += 1; return Some(item); @@ -600,13 +619,15 @@ impl<'w, 's, Q: WorldQuery, F: ReadOnlyWorldQuery> QueryIterationCursor<'w, 's, let archetype = &archetypes[*archetype_id]; // SAFETY: `archetype` and `tables` are from the world that `fetch/filter` were created for, // `fetch_state`/`filter_state` are the states that `fetch/filter` were initialized with - Q::set_archetype(&mut self.fetch, &query_state.fetch_state, archetype, tables); + let table = &tables[archetype.table_id()]; + Q::set_archetype(&mut self.fetch, &query_state.fetch_state, archetype, table); F::set_archetype( &mut self.filter, &query_state.filter_state, archetype, - tables, + table, ); + self.archetype_entities = archetype.entities(); self.current_len = archetype.len(); self.current_index = 0; continue; @@ -614,14 +635,23 @@ impl<'w, 's, Q: WorldQuery, F: ReadOnlyWorldQuery> QueryIterationCursor<'w, 's, // SAFETY: set_archetype was called prior. // `current_index` is an archetype index row in range of the current archetype, because if it was not, then the if above would have been executed. - if !F::archetype_filter_fetch(&mut self.filter, self.current_index) { + let archetype_entity = self.archetype_entities.get_unchecked(self.current_index); + if !F::filter_fetch( + &mut self.filter, + archetype_entity.entity, + archetype_entity.table_row, + ) { self.current_index += 1; continue; } // SAFETY: set_archetype was called prior, `current_index` is an archetype index in range of the current archetype // `current_index` is an archetype index row in range of the current archetype, because if it was not, then the if above would have been executed. - let item = Q::archetype_fetch(&mut self.fetch, self.current_index); + let item = Q::fetch( + &mut self.fetch, + archetype_entity.entity, + archetype_entity.table_row, + ); self.current_index += 1; return Some(item); } diff --git a/crates/bevy_ecs/src/query/state.rs b/crates/bevy_ecs/src/query/state.rs index dccdc217960eef..7bbbe3dad266c6 100644 --- a/crates/bevy_ecs/src/query/state.rs +++ b/crates/bevy_ecs/src/query/state.rs @@ -413,20 +413,12 @@ impl QueryState { let mut fetch = Q::init_fetch(world, &self.fetch_state, last_change_tick, change_tick); let mut filter = F::init_fetch(world, &self.filter_state, last_change_tick, change_tick); - Q::set_archetype( - &mut fetch, - &self.fetch_state, - archetype, - &world.storages().tables, - ); - F::set_archetype( - &mut filter, - &self.filter_state, - archetype, - &world.storages().tables, - ); - if F::archetype_filter_fetch(&mut filter, location.index) { - Ok(Q::archetype_fetch(&mut fetch, location.index)) + let table = &world.storages().tables[archetype.table_id()]; + Q::set_archetype(&mut fetch, &self.fetch_state, archetype, table); + F::set_archetype(&mut filter, &self.filter_state, archetype, table); + + if F::filter_fetch(&mut filter, entity, location.index) { + Ok(Q::fetch(&mut fetch, entity, location.index)) } else { Err(QueryEntityError::QueryDoesNotMatch(entity)) } @@ -945,34 +937,45 @@ impl QueryState { let mut fetch = Q::init_fetch(world, &self.fetch_state, last_change_tick, change_tick); let mut filter = F::init_fetch(world, &self.filter_state, last_change_tick, change_tick); + let tables = &world.storages().tables; if Q::IS_DENSE && F::IS_DENSE { - let tables = &world.storages().tables; for table_id in &self.matched_table_ids { let table = &tables[*table_id]; Q::set_table(&mut fetch, &self.fetch_state, table); F::set_table(&mut filter, &self.filter_state, table); - for table_index in 0..table.entity_count() { - if !F::table_filter_fetch(&mut filter, table_index) { + let entities = table.entities(); + for row in 0..table.entity_count() { + let entity = entities.get_unchecked(row); + if !F::filter_fetch(&mut filter, *entity, row) { continue; } - let item = Q::table_fetch(&mut fetch, table_index); - func(item); + func(Q::fetch(&mut fetch, *entity, row)); } } } else { let archetypes = &world.archetypes; - let tables = &world.storages().tables; for archetype_id in &self.matched_archetype_ids { let archetype = &archetypes[*archetype_id]; - Q::set_archetype(&mut fetch, &self.fetch_state, archetype, tables); - F::set_archetype(&mut filter, &self.filter_state, archetype, tables); - - for archetype_index in 0..archetype.len() { - if !F::archetype_filter_fetch(&mut filter, archetype_index) { + let table = &tables[archetype.table_id()]; + Q::set_archetype(&mut fetch, &self.fetch_state, archetype, table); + F::set_archetype(&mut filter, &self.filter_state, archetype, table); + + let entities = archetype.entities(); + for idx in 0..archetype.len() { + let archetype_entity = entities.get_unchecked(idx); + if !F::filter_fetch( + &mut filter, + archetype_entity.entity, + archetype_entity.table_row, + ) { continue; } - func(Q::archetype_fetch(&mut fetch, archetype_index)); + func(Q::fetch( + &mut fetch, + archetype_entity.entity, + archetype_entity.table_row, + )); } } } @@ -1033,14 +1036,15 @@ impl QueryState { ); let tables = &world.storages().tables; let table = &tables[*table_id]; + let entities = table.entities(); Q::set_table(&mut fetch, &self.fetch_state, table); F::set_table(&mut filter, &self.filter_state, table); - for table_index in offset..offset + len { - if !F::table_filter_fetch(&mut filter, table_index) { + for row in offset..offset + len { + let entity = entities.get_unchecked(row); + if !F::filter_fetch(&mut filter, *entity, row) { continue; } - let item = Q::table_fetch(&mut fetch, table_index); - func(item); + func(Q::fetch(&mut fetch, *entity, row)); } }; #[cfg(feature = "trace")] @@ -1083,14 +1087,25 @@ impl QueryState { ); let tables = &world.storages().tables; let archetype = &world.archetypes[*archetype_id]; - Q::set_archetype(&mut fetch, &self.fetch_state, archetype, tables); - F::set_archetype(&mut filter, &self.filter_state, archetype, tables); + let table = &tables[archetype.table_id()]; + Q::set_archetype(&mut fetch, &self.fetch_state, archetype, table); + F::set_archetype(&mut filter, &self.filter_state, archetype, table); + let entities = archetype.entities(); for archetype_index in offset..offset + len { - if !F::archetype_filter_fetch(&mut filter, archetype_index) { + let archetype_entity = entities.get_unchecked(archetype_index); + if !F::filter_fetch( + &mut filter, + archetype_entity.entity, + archetype_entity.table_row, + ) { continue; } - func(Q::archetype_fetch(&mut fetch, archetype_index)); + func(Q::fetch( + &mut fetch, + archetype_entity.entity, + archetype_entity.table_row, + )); } }; diff --git a/crates/bevy_ecs/src/world/mod.rs b/crates/bevy_ecs/src/world/mod.rs index bb7299195d57f3..c184615317063f 100644 --- a/crates/bevy_ecs/src/world/mod.rs +++ b/crates/bevy_ecs/src/world/mod.rs @@ -327,7 +327,8 @@ impl World { pub fn iter_entities(&self) -> impl Iterator + '_ { self.archetypes .iter() - .flat_map(|archetype| archetype.entities().iter().copied()) + .flat_map(|archetype| archetype.entities().iter()) + .map(|archetype_entity| archetype_entity.entity) } /// Retrieves an [`EntityMut`] that exposes read and write operations for the given `entity`. diff --git a/crates/bevy_scene/src/scene.rs b/crates/bevy_scene/src/scene.rs index e90d34b736955f..4df757aa5838f7 100644 --- a/crates/bevy_scene/src/scene.rs +++ b/crates/bevy_scene/src/scene.rs @@ -65,7 +65,7 @@ impl Scene { for scene_entity in archetype.entities() { let entity = *instance_info .entity_map - .entry(*scene_entity) + .entry(scene_entity.entity()) .or_insert_with(|| world.spawn_empty().id()); for component_id in archetype.components() { let component_info = self @@ -86,7 +86,7 @@ impl Scene { } }) })?; - reflect_component.copy(&self.world, world, *scene_entity, entity); + reflect_component.copy(&self.world, world, scene_entity.entity(), entity); } } }