diff --git a/Cargo.lock b/Cargo.lock index 495bdf8817..1e2ae3c4c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -16,9 +16,14 @@ checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" [[package]] name = "ahash" -version = "0.4.7" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "739f4a8db6605981345c5654f3a85b056ce52f37a39d34da03f25bf2151ea16e" +checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" +dependencies = [ + "getrandom 0.2.6", + "once_cell", + "version_check", +] [[package]] name = "aho-corasick" @@ -37,15 +42,15 @@ checksum = "000444226fcff248f2bc4c7625be32c63caccfecc2723a2b9f78a7487a49c407" [[package]] name = "anyhow" -version = "1.0.52" +version = "1.0.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84450d0b4a8bd1ba4144ce8ce718fbc5d071358b1e5384bace6536b3d1f2d5b3" +checksum = "08f9b8508dccb7687a1d6c4ce66b2b0ecef467c94667de27d8d7fe1f8d2a9cdc" [[package]] name = "arrayvec" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4dc07131ffa69b8072d35f5007352af944213cde02545e2103680baed38fcd" +checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" dependencies = [ "serde", ] @@ -75,9 +80,9 @@ dependencies = [ [[package]] name = "async-task" -version = "4.0.3" +version = "4.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91831deabf0d6d7ec49552e489aed63b7456a7a3c46cff62adad428110b0af0" +checksum = "30696a84d817107fc028e049980e09d5e140e8da8f1caeb17e8e950658a3cea9" [[package]] name = "atty" @@ -151,24 +156,24 @@ checksum = "0d8c1fef690941d3e7788d328517591fecc684c084084702d6ff1641e993699a" [[package]] name = "bumpalo" -version = "3.7.1" +version = "3.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9df67f7bf9ef8498769f994239c45613ef0c5899415fb58e9add412d2c1a538" +checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899" [[package]] name = "bytemuck" -version = "1.7.2" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72957246c41db82b8ef88a5486143830adeb8227ef9837740bdec67724cf2c5b" +checksum = "cdead85bdec19c194affaeeb670c0e41fe23de31459efd1c174d049269cf02cc" dependencies = [ "bytemuck_derive", ] [[package]] name = "bytemuck_derive" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e215f8c2f9f79cb53c8335e687ffd07d5bfcb6fe5fc80723762d0be46e7cc54" +checksum = "562e382481975bc61d11275ac5e62a19abd00b0547d99516a415336f183dcd0e" dependencies = [ "proc-macro2", "quote", @@ -183,15 +188,15 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "cache-padded" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "631ae5198c9be5e753e5cc215e1bd73c2b466a3565173db433f52bb9d3e66dba" +checksum = "c1db59621ec70f09c5e9b597b220c7a2b43611f4710dc03ceb8748637775692c" [[package]] name = "calloop" -version = "0.9.1" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42dcfbd723aa6eff9f024cfd5ad08b11144d79b2d8d37b4a31a006ceab255c77" +checksum = "bf2eec61efe56aa1e813f5126959296933cf0700030e4314786c48779a66ab82" dependencies = [ "log", "nix", @@ -210,9 +215,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.70" +version = "1.0.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d26a6ce4b6a484fa3edb70f7efa6fc430fd2b87285fe8b84304fd0936faa0dc0" +checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" [[package]] name = "cfg-if" @@ -250,8 +255,8 @@ dependencies = [ "bitflags", "block", "cocoa-foundation", - "core-foundation 0.9.1", - "core-graphics 0.22.2", + "core-foundation 0.9.3", + "core-graphics 0.22.3", "foreign-types", "libc", "objc", @@ -265,7 +270,7 @@ checksum = "7ade49b65d560ca58c403a479bb396592b155c0185eada742ee323d1d68d6318" dependencies = [ "bitflags", "block", - "core-foundation 0.9.1", + "core-foundation 0.9.3", "core-graphics-types", "foreign-types", "libc", @@ -293,11 +298,11 @@ dependencies = [ [[package]] name = "console_error_panic_hook" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8d976903543e0c48546a91908f21588a680a8c8f984df9a5d69feccb2b2a211" +checksum = "a06aeb73f470f66dcdbf7223caeebb85984942f22f1adb2a088cf9668146bbbc" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", "wasm-bindgen", ] @@ -329,11 +334,11 @@ dependencies = [ [[package]] name = "core-foundation" -version = "0.9.1" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a89e2ae426ea83155dccf10c0fa6b1463ef6d5fcb44cee0b224a408fa640a62" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" dependencies = [ - "core-foundation-sys 0.8.2", + "core-foundation-sys 0.8.3", "libc", ] @@ -345,9 +350,9 @@ checksum = "b3a71ab494c0b5b860bdc8407ae08978052417070c2ced38573a9157ad75b8ac" [[package]] name = "core-foundation-sys" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b" +checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" [[package]] name = "core-graphics" @@ -363,12 +368,12 @@ dependencies = [ [[package]] name = "core-graphics" -version = "0.22.2" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "269f35f69b542b80e736a20a89a05215c0ce80c2c03c514abb2e318b78379d86" +checksum = "2581bbab3b8ffc6fcbd550bf46c355135d16e9ff2a6ea032ad6b9bf1d7efe4fb" dependencies = [ "bitflags", - "core-foundation 0.9.1", + "core-foundation 0.9.3", "core-graphics-types", "foreign-types", "libc", @@ -381,7 +386,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a68b68b3446082644c91ac778bf50cd4104bfb002b5a6a7c44cca5a2c70788b" dependencies = [ "bitflags", - "core-foundation 0.9.1", + "core-foundation 0.9.3", "foreign-types", "libc", ] @@ -401,9 +406,9 @@ dependencies = [ [[package]] name = "crc32fast" -version = "1.2.1" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" dependencies = [ "cfg-if 1.0.0", ] @@ -426,9 +431,9 @@ dependencies = [ [[package]] name = "darling" -version = "0.13.0" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "757c0ded2af11d8e739c4daea1ac623dd1624b06c844cf3f5a39f1bdbd99bb12" +checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c" dependencies = [ "darling_core", "darling_macro", @@ -436,9 +441,9 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.13.0" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c34d8efb62d0c2d7f60ece80f75e5c63c1588ba68032740494b0b9a996466e3" +checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610" dependencies = [ "fnv", "ident_case", @@ -450,9 +455,9 @@ dependencies = [ [[package]] name = "darling_macro" -version = "0.13.0" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ade7bff147130fe5e6d39f089c6bd49ec0250f35d70b2eebf72afdfc919f15cc" +checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" dependencies = [ "darling_core", "quote", @@ -461,9 +466,9 @@ dependencies = [ [[package]] name = "ddsfile" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ac3f07973d5b85a0ad9e01fbec4a2a45fdf79ea54287ade3a47ebc8e2f82cf0" +checksum = "594ecd967c2f40db5dde8da4c356975fc1fe030e951c7c3962f6dc2e80042e87" dependencies = [ "bitflags", "byteorder", @@ -479,17 +484,6 @@ dependencies = [ "adler32", ] -[[package]] -name = "derivative" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "devserver_lib" version = "0.4.1" @@ -548,18 +542,18 @@ dependencies = [ [[package]] name = "fastrand" -version = "1.5.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b394ed3d285a429378d3b384b9eb1285267e7df4b166df24b7a6939a04dc392e" +checksum = "c3fcf0cee53519c866c09b5de1f6c56ff9d647101f81c1964fa632e148896cdf" dependencies = [ "instant", ] [[package]] name = "fixedbitset" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "398ea4fabe40b9b0d885340a2a991a44c8a645624075ad966d21f88688e2b69e" +checksum = "279fb028e20b3c4c320317955b77c5e0c9701f05a1d309905d6fc702cdc5053e" [[package]] name = "fnv" @@ -584,15 +578,15 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" [[package]] name = "futures-core" -version = "0.3.17" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88d1c26957f23603395cd326b0ffe64124b818f4449552f960d815cfba83a53d" +checksum = "0c09fd04b7e4073ac7156a9539b57a484a8ea920f79c7c675d05d289ab6110d3" [[package]] name = "futures-io" -version = "0.3.17" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "522de2a0fe3e380f1bc577ba0474108faf3f6b18321dbf60b3b9c39a75073377" +checksum = "fc4045962a5a5e935ee2fdedaa4e08284547402885ab326734432bed5d12966b" [[package]] name = "futures-lite" @@ -633,9 +627,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.3" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753" +checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad" dependencies = [ "cfg-if 1.0.0", "libc", @@ -661,9 +655,9 @@ checksum = "f43e957e744be03f5801a55472f593d43fabdebf25a4585db250f04d86b1675f" [[package]] name = "glow" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c07210904884e8e2e6a2d7f36f39040a9cefe3b379b721969b5275e9f5b464a" +checksum = "d8bd5877156a19b8ac83a29b2306fe20537429d318f3ff0a1a2119f8d9c61919" dependencies = [ "js-sys", "slotmap", @@ -680,7 +674,7 @@ dependencies = [ "android_glue", "cgl", "cocoa", - "core-foundation 0.9.1", + "core-foundation 0.9.3", "glutin_egl_sys", "glutin_emscripten_sys", "glutin_gles2_sys", @@ -691,7 +685,7 @@ dependencies = [ "log", "objc", "osmesa-sys", - "parking_lot", + "parking_lot 0.11.2", "wayland-client", "wayland-egl", "winapi", @@ -745,9 +739,9 @@ dependencies = [ [[package]] name = "gpu-alloc" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e64cbb8d36508d3e19da95e56e196a84f674fc190881f2cc010000798838aa6" +checksum = "7fc59e5f710e310e76e6707f86c561dd646f69a8876da9131703b2f717de818d" dependencies = [ "bitflags", "gpu-alloc-types", @@ -764,9 +758,9 @@ dependencies = [ [[package]] name = "gpu-descriptor" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7a237f0419ab10d17006d55c62ac4f689a6bf52c75d3f38b8361d249e8d4b0b" +checksum = "a538f217be4d405ff4719a283ca68323cc2384003eca5baaa87501e821c81dda" dependencies = [ "bitflags", "gpu-descriptor-types", @@ -784,9 +778,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.9.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" +checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" dependencies = [ "ahash", ] @@ -835,9 +829,9 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "indexmap" -version = "1.6.2" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "824845a0bf897a9042383849b02c1bc219c2383772efcd5c6f9766fa4b81aef3" +checksum = "bc633605454125dec4b66843673f01c7df2b89479b32e0ed634e43a91cff62a5" dependencies = [ "autocfg", "hashbrown", @@ -852,9 +846,9 @@ checksum = "90953f308a79fe6d62a4643e51f848fbfddcd05975a38e69fdf4ab86a7baf7ca" [[package]] name = "instant" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "716d3d89f35ac6a34fd0eed635395f4c3b76fa889338a4632e5231a8684216bd" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -864,9 +858,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" +checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d" [[package]] name = "jni-sys" @@ -914,15 +908,15 @@ checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" [[package]] name = "libc" -version = "0.2.102" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2a5ac8f984bfcf3a823267e5fde638acc3325f6496633a5da6bb6eb2171e103" +checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836" [[package]] name = "libloading" -version = "0.7.0" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f84d96438c15fcd6c3f244c8fce01d1e2b9c6b5623e9c711dc9286d8fc92d6a" +checksum = "efbc0f03f9a775e9f6aed295c6a1ba2253c5757a9e03d55c6caa46a681abcddd" dependencies = [ "cfg-if 1.0.0", "winapi", @@ -940,9 +934,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.14" +version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" dependencies = [ "cfg-if 1.0.0", ] @@ -958,9 +952,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "memmap2" @@ -973,9 +967,9 @@ dependencies = [ [[package]] name = "memoffset" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59accc507f1338036a0477ef61afdae33cde60840f4dfe481319ce3ad116ddf9" +checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" dependencies = [ "autocfg", ] @@ -995,9 +989,9 @@ dependencies = [ [[package]] name = "minimal-lexical" -version = "0.1.3" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c835948974f68e0bd58636fc6c5b1fbff7b297e3046f11b3b3c18bbac012c6d" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" @@ -1010,24 +1004,14 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.0" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba272f85fa0b41fc91872be579b3bbe0f56b792aa361a380eb669469f68dafb2" +checksum = "713d550d9b44d89174e066b7a6217ae06234c10cb47819a88290d2b353c31799" dependencies = [ "libc", "log", - "miow", - "ntapi", - "winapi", -] - -[[package]] -name = "miow" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" -dependencies = [ - "winapi", + "wasi 0.11.0+wasi-snapshot-preview1", + "windows-sys", ] [[package]] @@ -1041,7 +1025,7 @@ dependencies = [ "hexf-parse", "indexmap", "log", - "num-traits 0.2.14", + "num-traits 0.2.15", "petgraph", "pp-rs", "rustc-hash", @@ -1065,16 +1049,23 @@ dependencies = [ "thiserror", ] +[[package]] +name = "ndk-context" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" + [[package]] name = "ndk-glue" -version = "0.5.0" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc291b8de2095cba8dab7cf381bf582ff4c17a09acf854c32e46545b08085d28" +checksum = "c71bee8ea72d685477e28bd004cfe1bf99c754d688cd78cad139eae4089484d4" dependencies = [ "lazy_static", "libc", "log", "ndk", + "ndk-context", "ndk-macro", "ndk-sys", ] @@ -1100,9 +1091,9 @@ checksum = "e1bcdd74c20ad5d95aacd60ef9ba40fdf77f767051040541df557b7a9b2a2121" [[package]] name = "nix" -version = "0.22.0" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf1e25ee6b412c2a1e3fcb6a4499a5c1bfe7f43e014bdce9a6b6666e5aa2d187" +checksum = "e4916f159ed8e5de0082076562152a76b7a1f64a01fd9d1e0fea002c37624faf" dependencies = [ "bitflags", "cc", @@ -1117,28 +1108,18 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "82051dd6745d5184c6efb7bc8be14892a7f6d4f3ad6dbf754d1c7d7d5fe24b43" dependencies = [ - "rand 0.7.3", + "rand", "rand_xorshift", ] [[package]] name = "nom" -version = "7.0.0" +version = "7.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ffd9d26838a953b4af82cbeb9f1592c6798916983959be223a7124e992742c1" +checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36" dependencies = [ "memchr", "minimal-lexical", - "version_check", -] - -[[package]] -name = "ntapi" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44" -dependencies = [ - "winapi", ] [[package]] @@ -1147,33 +1128,32 @@ version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92e5113e9fd4cc14ded8e499429f396a20f98c772a47cc8622a736e1ec843c31" dependencies = [ - "num-traits 0.2.14", + "num-traits 0.2.15", ] [[package]] name = "num-traits" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" dependencies = [ "autocfg", ] [[package]] name = "num_enum" -version = "0.5.4" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9bd055fb730c4f8f4f57d45d35cd6b3f0980535b056dc7ff119cee6a66ed6f" +checksum = "cf5395665662ef45796a4ff5486c5d41d29e0c09640af4c5f17fd94ee2c119c9" dependencies = [ - "derivative", "num_enum_derive", ] [[package]] name = "num_enum_derive" -version = "0.5.4" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "486ea01961c4a818096de679a8b740b26d9033146ac5291b1c98557658f8cdd9" +checksum = "3b0498641e53dd6ac1a4f22547548caa6864cc4933784319cd1775271c5a46ce" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1208,9 +1188,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.8.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56" +checksum = "7709cef83f0c1f58f666e746a08b21e0085f7440fa6a29cc194d68aac97a4225" [[package]] name = "osmesa-sys" @@ -1235,7 +1215,17 @@ checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" dependencies = [ "instant", "lock_api", - "parking_lot_core", + "parking_lot_core 0.8.5", +] + +[[package]] +name = "parking_lot" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f5ec2493a61ac0506c0f4199f99070cbe83857b0337006a30f3e6719b8ef58" +dependencies = [ + "lock_api", + "parking_lot_core 0.9.3", ] [[package]] @@ -1252,6 +1242,19 @@ dependencies = [ "winapi", ] +[[package]] +name = "parking_lot_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "redox_syscall", + "smallvec", + "windows-sys", +] + [[package]] name = "percent-encoding" version = "2.1.0" @@ -1260,9 +1263,9 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" [[package]] name = "petgraph" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a13a2fa9d0b63e5f22328828741e523766fff0ee9e779316902290dff3f824f" +checksum = "51b305cc4569dd4e8765bab46261f67ef5d4d11a4b6e745100ee5dad8948b46c" dependencies = [ "fixedbitset", "indexmap", @@ -1276,15 +1279,15 @@ checksum = "db8bcd96cb740d03149cbad5518db9fd87126a10ab519c011893b1754134c468" [[package]] name = "pin-project-lite" -version = "0.2.7" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d31d11c69a6b52a174b42bdc0c30e5e11670f90788b2c471c31c1d17d449443" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" [[package]] name = "pkg-config" -version = "0.3.20" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c9b1041b4387893b91ee6746cddfc28516aff326a3519fb2adf820932c5e6cb" +checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" [[package]] name = "player" @@ -1302,9 +1305,9 @@ dependencies = [ [[package]] name = "png" -version = "0.17.3" +version = "0.17.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e8f1882177b17c98ec33a51f5910ecbf4db92ca0def706781a1f8d0c661f393" +checksum = "dc38c0ad57efb786dd57b9864e5b18bae478c00c824dc55a38bbc9da95dde3ba" dependencies = [ "bitflags", "crc32fast", @@ -1314,9 +1317,9 @@ dependencies = [ [[package]] name = "pollster" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb20dcc30536a1508e75d47dd0e399bb2fe7354dcf35cda9127f2bf1ed92e30e" +checksum = "5da3b0203fd7ee5720aa0b5e790b591aa5d3f41c3ed2c34a3a393382198af2f7" [[package]] name = "pp-rs" @@ -1329,15 +1332,15 @@ dependencies = [ [[package]] name = "ppv-lite86" -version = "0.2.10" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" +checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" [[package]] name = "proc-macro-crate" -version = "1.1.0" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ebace6889caf889b4d3f76becee12e90353f2b8c7d875534a71e5742f8f6f83" +checksum = "e17d47ce914bf4de440332250b0edd23ce48c005f59fab39d3335866b114f11a" dependencies = [ "thiserror", "toml", @@ -1345,24 +1348,24 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.29" +version = "1.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9f5105d4fdaab20335ca9565e106a5d9b82b6219b5ba735731124ac6711d23d" +checksum = "c54b25569025b7fc9651de43004ae593a75ad88543b17178aa5e1b9c4f15f56f" dependencies = [ - "unicode-xid", + "unicode-ident", ] [[package]] name = "profiling" -version = "1.0.3" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87dfd5592a8eed7e74f56ad7b125f8234763b805c30f0c7c95c486920026a6ec" +checksum = "9145ac0af1d93c638c98c40cf7d25665f427b2a44ad0a99b1dccf3e2f25bb987" [[package]] name = "quote" -version = "1.0.9" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" +checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1" dependencies = [ "proc-macro2", ] @@ -1375,21 +1378,9 @@ checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" dependencies = [ "getrandom 0.1.16", "libc", - "rand_chacha 0.2.2", - "rand_core 0.5.1", - "rand_hc 0.2.0", -] - -[[package]] -name = "rand" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8" -dependencies = [ - "libc", - "rand_chacha 0.3.1", - "rand_core 0.6.3", - "rand_hc 0.3.1", + "rand_chacha", + "rand_core", + "rand_hc", ] [[package]] @@ -1399,17 +1390,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" dependencies = [ "ppv-lite86", - "rand_core 0.5.1", -] - -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core 0.6.3", + "rand_core", ] [[package]] @@ -1421,31 +1402,13 @@ dependencies = [ "getrandom 0.1.16", ] -[[package]] -name = "rand_core" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" -dependencies = [ - "getrandom 0.2.3", -] - [[package]] name = "rand_hc" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" dependencies = [ - "rand_core 0.5.1", -] - -[[package]] -name = "rand_hc" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d51e9f596de227fda2ea6c84607f5558e196eeaf43c986b724ba4fb8fdf497e7" -dependencies = [ - "rand_core 0.6.3", + "rand_core", ] [[package]] @@ -1454,7 +1417,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77d416b86801d23dde1aa643023b775c3a462efc0ed96443add11546cdf1dca8" dependencies = [ - "rand_core 0.5.1", + "rand_core", ] [[package]] @@ -1465,27 +1428,27 @@ checksum = "63e935c45e09cc6dcf00d2f0b2d630a58f4095320223d47fc68918722f0538b6" [[package]] name = "raw-window-handle" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fba75eee94a9d5273a68c9e1e105d9cffe1ef700532325788389e5a83e2522b7" +checksum = "b800beb9b6e7d2df1fe337c9e3d04e3af22a124460fb4c30fcc22c9117cefb41" dependencies = [ "cty", ] [[package]] name = "redox_syscall" -version = "0.2.10" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff" +checksum = "62f25bc4c7e55e0b0b7a1d43fb893f4fa1361d0abe38b9ce4f323c2adfe6ef42" dependencies = [ "bitflags", ] [[package]] name = "regex" -version = "1.5.4" +version = "1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461" +checksum = "d83f127d94bdbcda4c8cc2e50f6f84f4b611f69c902699ca385a39c3a75f9ff1" dependencies = [ "aho-corasick", "memchr", @@ -1494,9 +1457,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.25" +version = "0.6.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" +checksum = "49b3de9ec5dc0a3417da371aab17d729997c15010e7fd24ff707773a33bddb64" [[package]] name = "remove_dir_all" @@ -1545,9 +1508,9 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "ryu" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f" +checksum = "f3f6f92acf49d1b98f7a81226834412ada05458b7364277387724a237f062695" [[package]] name = "safemem" @@ -1569,18 +1532,18 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] name = "serde" -version = "1.0.130" +version = "1.0.137" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f12d06de37cf59146fbdecab66aa99f9fe4f78722e3607577a5375d66bd0c913" +checksum = "61ea8d54c77f8315140a05f4c7237403bf38b72704d031543aa1d16abbf517d1" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.130" +version = "1.0.137" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7bc1a1ab1961464eae040d96713baa5a724a8152c1222492465b54322ec508b" +checksum = "1f26faba0c3959972377d3b2d306ee9f71faee9714294e41bb777f83f88578be" dependencies = [ "proc-macro2", "quote", @@ -1589,9 +1552,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.74" +version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee2bb9cd061c5865d345bb02ca49fcef1391741b672b54a0bf7b679badec3142" +checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c" dependencies = [ "itoa", "ryu", @@ -1610,9 +1573,9 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.4" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c307a32c1c5c437f38c7fd45d753050587732ba8628319fbdf12a7e289ccc590" +checksum = "eb703cfe953bccee95685111adeedb76fabe4e97549a58d16f03ea7b9367bb32" [[package]] name = "slotmap" @@ -1625,15 +1588,15 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.6.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" +checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83" [[package]] name = "smithay-client-toolkit" -version = "0.15.2" +version = "0.15.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "210cf40de565aaaa085face1d860b17f6aee9f76f9d2816307ea2cc45eeb64f3" +checksum = "8a28f16a97fa0e8ce563b2774d1e732dd5d4025d2772c5dba0a41a0f90a29da3" dependencies = [ "bitflags", "calloop", @@ -1655,7 +1618,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "246bfa38fe3db3f1dfc8ca5a2cdeb7348c78be2112740cc0ec8ef18b6d94f830" dependencies = [ "bitflags", - "num-traits 0.2.14", + "num-traits 0.2.15", ] [[package]] @@ -1666,24 +1629,24 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "syn" -version = "1.0.77" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5239bc68e0fef57495900cfea4e8dc75596d9a319d7e16b1e0a440d24e6fe0a0" +checksum = "fbaf6116ab8924f39d52792136fb74fd60a80194cf1b1c6ffa6453eef1c3f942" dependencies = [ "proc-macro2", "quote", - "unicode-xid", + "unicode-ident", ] [[package]] name = "tempfile" -version = "3.2.0" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" +checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" dependencies = [ "cfg-if 1.0.0", + "fastrand", "libc", - "rand 0.8.4", "redox_syscall", "remove_dir_all", "winapi", @@ -1691,27 +1654,27 @@ dependencies = [ [[package]] name = "termcolor" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4" +checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" dependencies = [ "winapi-util", ] [[package]] name = "thiserror" -version = "1.0.29" +version = "1.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "602eca064b2d83369e2b2f34b09c70b605402801927c65c11071ac911d299b88" +checksum = "bd829fe32373d27f76265620b5309d0340cb8550f523c1dda251d6298069069a" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.29" +version = "1.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bad553cc2c78e8de258400763a647e80e6d1b31ee237275d756f6836d204494c" +checksum = "0396bc89e626244658bef819e22d0cc459e795a5ebe878e6ec336d1674a8d79a" dependencies = [ "proc-macro2", "quote", @@ -1720,18 +1683,24 @@ dependencies = [ [[package]] name = "toml" -version = "0.5.8" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa" +checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7" dependencies = [ "serde", ] +[[package]] +name = "unicode-ident" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d22af068fba1eb5edcb4aea19d382b2a3deb4c8f9d475c589b6ada9e0fd493ee" + [[package]] name = "unicode-segmentation" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8895849a949e7845e06bd6dc1aa51731a103c42707010a5b591c0038fb73385b" +checksum = "7e8820f5d777f6224dc4be3632222971ac30164d4a258d595640799554ebfd99" [[package]] name = "unicode-width" @@ -1747,9 +1716,9 @@ checksum = "957e51f3646910546462e67d5f7599b9e4fb8acdd304b087a6494730f9eebf04" [[package]] name = "version_check" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "waker-fn" @@ -1795,6 +1764,12 @@ version = "0.10.2+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + [[package]] name = "wasm-bindgen" version = "0.2.80" @@ -1961,9 +1936,9 @@ dependencies = [ [[package]] name = "wayland-client" -version = "0.29.1" +version = "0.29.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9108ec1c37f4774d0c2937ba1a6c23d1786b2152c4a13bd9fdb20e42d16e8841" +checksum = "91223460e73257f697d9e23d401279123d36039a3f7a449e983f123292d4458f" dependencies = [ "bitflags", "downcast-rs", @@ -1977,9 +1952,9 @@ dependencies = [ [[package]] name = "wayland-commons" -version = "0.29.1" +version = "0.29.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "265ef51b3b3e5c9ef098f10425c39624663f459c3821dcaacc4748be975f1beb" +checksum = "94f6e5e340d7c13490eca867898c4cec5af56c27a5ffe5c80c6fc4708e22d33e" dependencies = [ "nix", "once_cell", @@ -1989,9 +1964,9 @@ dependencies = [ [[package]] name = "wayland-cursor" -version = "0.29.1" +version = "0.29.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c19bb6628daf4097e58b7911481e8371e13318d5a60894779901bd3267407a7" +checksum = "c52758f13d5e7861fc83d942d3d99bf270c83269575e52ac29e5b73cb956a6bd" dependencies = [ "nix", "wayland-client", @@ -2000,9 +1975,9 @@ dependencies = [ [[package]] name = "wayland-egl" -version = "0.29.1" +version = "0.29.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accf27d1e5e1f64ba30b683fd926c2c916cc1014bea3376fb258e80abf622e40" +checksum = "83281d69ee162b59031c666385e93bde4039ec553b90c4191cdb128ceea29a3a" dependencies = [ "wayland-client", "wayland-sys", @@ -2010,9 +1985,9 @@ dependencies = [ [[package]] name = "wayland-protocols" -version = "0.29.1" +version = "0.29.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b3b6f1dc0193072ef4eadcb144da30d58c1f2895516c063804d213310703c8e" +checksum = "60147ae23303402e41fe034f74fb2c35ad0780ee88a1c40ac09a3be1e7465741" dependencies = [ "bitflags", "wayland-client", @@ -2022,9 +1997,9 @@ dependencies = [ [[package]] name = "wayland-scanner" -version = "0.29.1" +version = "0.29.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaaf2bc85e7b9143159af96bd23d954a5abe391c4376db712320643280fdc6f4" +checksum = "39a1ed3143f7a143187156a2ab52742e89dac33245ba505c17224df48939f9e0" dependencies = [ "proc-macro2", "quote", @@ -2033,9 +2008,9 @@ dependencies = [ [[package]] name = "wayland-sys" -version = "0.29.1" +version = "0.29.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba9e06acb775b3007f8d3094438306979e572d1d3b844d7a71557a84b055d959" +checksum = "d9341df79a8975679188e37dab3889bfa57c44ac2cb6da166f519a81cbe452d4" dependencies = [ "dlib", "lazy_static", @@ -2070,10 +2045,10 @@ dependencies = [ "naga", "noise", "obj", - "parking_lot", + "parking_lot 0.12.0", "png", "pollster", - "rand 0.7.3", + "rand", "raw-window-handle", "serde", "smallvec", @@ -2091,6 +2066,7 @@ name = "wgpu-core" version = "0.12.0" dependencies = [ "arrayvec", + "bit-vec", "bitflags", "cfg_aliases", "codespan-reporting", @@ -2098,7 +2074,7 @@ dependencies = [ "fxhash", "log", "naga", - "parking_lot", + "parking_lot 0.12.0", "profiling", "raw-window-handle", "ron", @@ -2135,7 +2111,7 @@ dependencies = [ "metal", "naga", "objc", - "parking_lot", + "parking_lot 0.12.0", "profiling", "range-alloc", "raw-window-handle", @@ -2196,6 +2172,49 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-sys" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" +dependencies = [ + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" + +[[package]] +name = "windows_i686_gnu" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" + +[[package]] +name = "windows_i686_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" + [[package]] name = "winit" version = "0.26.1" @@ -2204,8 +2223,8 @@ checksum = "9b43cc931d58b99461188607efd7acb2a093e65fc621f54cad78517a6063e73a" dependencies = [ "bitflags", "cocoa", - "core-foundation 0.9.1", - "core-graphics 0.22.2", + "core-foundation 0.9.3", + "core-graphics 0.22.3", "core-video-sys", "dispatch", "instant", @@ -2217,7 +2236,7 @@ dependencies = [ "ndk-glue", "ndk-sys", "objc", - "parking_lot", + "parking_lot 0.11.2", "percent-encoding", "raw-window-handle", "smithay-client-toolkit", diff --git a/wgpu-core/Cargo.toml b/wgpu-core/Cargo.toml index 58d9723166..aa13091282 100644 --- a/wgpu-core/Cargo.toml +++ b/wgpu-core/Cargo.toml @@ -26,6 +26,7 @@ vulkan-portability = ["hal/vulkan"] [dependencies] arrayvec = "0.7" bitflags = "1.0" +bit-vec = "0.6" codespan-reporting = "0.11" copyless = "0.1" fxhash = "0.2" diff --git a/wgpu-core/src/binding_model.rs b/wgpu-core/src/binding_model.rs index 030574842a..ef97af6d91 100644 --- a/wgpu-core/src/binding_model.rs +++ b/wgpu-core/src/binding_model.rs @@ -1,10 +1,10 @@ use crate::{ device::{DeviceError, MissingDownlevelFlags, MissingFeatures, SHADER_STAGE_COUNT}, error::{ErrorFormatter, PrettyError}, - hub::Resource, - id::{BindGroupLayoutId, BufferId, DeviceId, SamplerId, TextureViewId, Valid}, + hub::{HalApi, Resource}, + id::{BindGroupLayoutId, BufferId, DeviceId, SamplerId, TextureId, TextureViewId, Valid}, init_tracker::{BufferInitTrackerAction, TextureInitTrackerAction}, - track::{TrackerSet, UsageConflict, DUMMY_SELECTOR}, + track::{BindGroupStates, UsageConflict}, validation::{MissingBufferUsageError, MissingTextureUsageError}, FastHashMap, Label, LifeGuard, MultiRefCount, Stored, }; @@ -16,10 +16,7 @@ use serde::Deserialize; #[cfg(feature = "trace")] use serde::Serialize; -use std::{ - borrow::{Borrow, Cow}, - ops::Range, -}; +use std::{borrow::Cow, ops::Range}; use thiserror::Error; @@ -63,6 +60,8 @@ pub enum CreateBindGroupError { InvalidBuffer(BufferId), #[error("texture view {0:?} is invalid")] InvalidTextureView(TextureViewId), + #[error("texture {0:?} is invalid")] + InvalidTexture(TextureId), #[error("sampler {0:?} is invalid")] InvalidSampler(SamplerId), #[error( @@ -709,13 +708,12 @@ pub(crate) fn buffer_binding_type_alignment( } } -#[derive(Debug)] -pub struct BindGroup { +pub struct BindGroup { pub(crate) raw: A::BindGroup, pub(crate) device_id: Stored, pub(crate) layout_id: Valid, pub(crate) life_guard: LifeGuard, - pub(crate) used: TrackerSet, + pub(crate) used: BindGroupStates, pub(crate) used_buffer_ranges: Vec, pub(crate) used_texture_ranges: Vec, pub(crate) dynamic_binding_info: Vec, @@ -724,7 +722,7 @@ pub struct BindGroup { pub(crate) late_buffer_binding_sizes: Vec, } -impl BindGroup { +impl BindGroup { pub(crate) fn validate_dynamic_bindings( &self, offsets: &[wgt::DynamicOffset], @@ -766,13 +764,7 @@ impl BindGroup { } } -impl Borrow<()> for BindGroup { - fn borrow(&self) -> &() { - &DUMMY_SELECTOR - } -} - -impl Resource for BindGroup { +impl Resource for BindGroup { const TYPE: &'static str = "BindGroup"; fn life_guard(&self) -> &LifeGuard { diff --git a/wgpu-core/src/command/bundle.rs b/wgpu-core/src/command/bundle.rs index dc0eaf733e..9ab9a75f3d 100644 --- a/wgpu-core/src/command/bundle.rs +++ b/wgpu-core/src/command/bundle.rs @@ -34,7 +34,7 @@ invalidations or index format changes. #![allow(clippy::reversed_empty_ranges)] use crate::{ - binding_model::buffer_binding_type_alignment, + binding_model::{self, buffer_binding_type_alignment}, command::{ BasePass, BindGroupStateChange, DrawError, MapPassErr, PassErrorScope, RenderCommand, RenderCommandError, StateChange, @@ -48,8 +48,9 @@ use crate::{ hub::{GlobalIdentityHandlerFactory, HalApi, Hub, Resource, Storage, Token}, id, init_tracker::{BufferInitTrackerAction, MemoryInitKind, TextureInitTrackerAction}, - pipeline::PipelineFlags, - track::{TrackerSet, UsageConflict}, + pipeline::{self, PipelineFlags}, + resource, + track::RenderBundleScope, validation::check_buffer_usage, Label, LabelHelpers, LifeGuard, Stored, }; @@ -117,7 +118,7 @@ impl RenderBundleEncoder { }, sample_count: { let sc = desc.sample_count; - if sc == 0 || sc > 32 || !conv::is_power_of_two(sc) { + if sc == 0 || sc > 32 || !conv::is_power_of_two_u32(sc) { return Err(CreateRenderBundleError::InvalidSampleCount(sc)); } sc @@ -177,20 +178,28 @@ impl RenderBundleEncoder { /// and accumulate buffer and texture initialization actions. /// /// [`ExecuteBundle`]: RenderCommand::ExecuteBundle - pub(crate) fn finish( + pub(crate) fn finish( self, desc: &RenderBundleDescriptor, device: &Device, hub: &Hub, token: &mut Token>, - ) -> Result { + ) -> Result, RenderBundleError> { let (pipeline_layout_guard, mut token) = hub.pipeline_layouts.read(token); let (bind_group_guard, mut token) = hub.bind_groups.read(&mut token); let (pipeline_guard, mut token) = hub.render_pipelines.read(&mut token); - let (buffer_guard, _) = hub.buffers.read(&mut token); + let (query_set_guard, mut token) = hub.query_sets.read(&mut token); + let (buffer_guard, mut token) = hub.buffers.read(&mut token); + let (texture_guard, _) = hub.textures.read(&mut token); let mut state = State { - trackers: TrackerSet::new(self.parent_id.backend()), + trackers: RenderBundleScope::new( + &*buffer_guard, + &*texture_guard, + &*bind_group_guard, + &*pipeline_guard, + &*query_set_guard, + ), index: IndexState::new(), vertex: (0..hal::MAX_VERTEX_BUFFERS) .map(|_| VertexState::new()) @@ -234,11 +243,11 @@ impl RenderBundleEncoder { next_dynamic_offset = offsets_range.end; let offsets = &base.dynamic_offsets[offsets_range.clone()]; - let bind_group = state + let bind_group: &binding_model::BindGroup = state .trackers .bind_groups - .use_extend(&*bind_group_guard, bind_group_id, (), ()) - .map_err(|_| RenderCommandError::InvalidBindGroup(bind_group_id)) + .add_single(&*bind_group_guard, bind_group_id) + .ok_or(RenderCommandError::InvalidBindGroup(bind_group_id)) .map_pass_err(scope)?; if bind_group.dynamic_binding_info.len() != offsets.len() { return Err(RenderCommandError::InvalidDynamicOffsetCount { @@ -268,10 +277,12 @@ impl RenderBundleEncoder { texture_memory_init_actions.extend_from_slice(&bind_group.used_texture_ranges); state.set_bind_group(index, bind_group_id, bind_group.layout_id, offsets_range); - state - .trackers - .merge_extend_stateful(&bind_group.used) - .map_pass_err(scope)?; + unsafe { + state + .trackers + .merge_bind_group(&*texture_guard, &bind_group.used) + .map_pass_err(scope)? + }; //Note: stateless trackers are not merged: the lifetime reference // is held to the bind group itself. } @@ -280,11 +291,11 @@ impl RenderBundleEncoder { state.pipeline = Some(pipeline_id); - let pipeline = state + let pipeline: &pipeline::RenderPipeline = state .trackers - .render_pipes - .use_extend(&*pipeline_guard, pipeline_id, (), ()) - .map_err(|_| RenderCommandError::InvalidPipeline(pipeline_id)) + .render_pipelines + .add_single(&*pipeline_guard, pipeline_id) + .ok_or(RenderCommandError::InvalidPipeline(pipeline_id)) .map_pass_err(scope)?; self.context @@ -320,11 +331,11 @@ impl RenderBundleEncoder { size, } => { let scope = PassErrorScope::SetIndexBuffer(buffer_id); - let buffer = state + let buffer: &resource::Buffer = state .trackers .buffers - .use_extend(&*buffer_guard, buffer_id, (), hal::BufferUses::INDEX) - .unwrap(); + .merge_single(&*buffer_guard, buffer_id, hal::BufferUses::INDEX) + .map_pass_err(scope)?; check_buffer_usage(buffer.usage, wgt::BufferUsages::INDEX) .map_pass_err(scope)?; @@ -347,11 +358,11 @@ impl RenderBundleEncoder { size, } => { let scope = PassErrorScope::SetVertexBuffer(buffer_id); - let buffer = state + let buffer: &resource::Buffer = state .trackers .buffers - .use_extend(&*buffer_guard, buffer_id, (), hal::BufferUses::VERTEX) - .unwrap(); + .merge_single(&*buffer_guard, buffer_id, hal::BufferUses::VERTEX) + .map_pass_err(scope)?; check_buffer_usage(buffer.usage, wgt::BufferUsages::VERTEX) .map_pass_err(scope)?; @@ -472,11 +483,11 @@ impl RenderBundleEncoder { .require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION) .map_pass_err(scope)?; - let buffer = state + let buffer: &resource::Buffer = state .trackers .buffers - .use_extend(&*buffer_guard, buffer_id, (), hal::BufferUses::INDIRECT) - .unwrap(); + .merge_single(&*buffer_guard, buffer_id, hal::BufferUses::INDIRECT) + .map_pass_err(scope)?; check_buffer_usage(buffer.usage, wgt::BufferUsages::INDIRECT) .map_pass_err(scope)?; @@ -505,11 +516,10 @@ impl RenderBundleEncoder { .require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION) .map_pass_err(scope)?; - let buffer = state + let buffer: &resource::Buffer = state .trackers .buffers - .use_extend(&*buffer_guard, buffer_id, (), hal::BufferUses::INDIRECT) - .map_err(|err| RenderCommandError::Buffer(buffer_id, err)) + .merge_single(&*buffer_guard, buffer_id, hal::BufferUses::INDIRECT) .map_pass_err(scope)?; check_buffer_usage(buffer.usage, wgt::BufferUsages::INDIRECT) .map_pass_err(scope)?; @@ -612,24 +622,23 @@ pub type RenderBundleDescriptor<'a> = wgt::RenderBundleDescriptor>; //Note: here, `RenderBundle` is just wrapping a raw stream of render commands. // The plan is to back it by an actual Vulkan secondary buffer, D3D12 Bundle, // or Metal indirect command buffer. -#[derive(Debug)] -pub struct RenderBundle { +pub struct RenderBundle { // Normalized command stream. It can be executed verbatim, // without re-binding anything on the pipeline change. base: BasePass, pub(super) is_ds_read_only: bool, pub(crate) device_id: Stored, - pub(crate) used: TrackerSet, + pub(crate) used: RenderBundleScope, pub(super) buffer_memory_init_actions: Vec, pub(super) texture_memory_init_actions: Vec, pub(super) context: RenderPassContext, pub(crate) life_guard: LifeGuard, } -unsafe impl Send for RenderBundle {} -unsafe impl Sync for RenderBundle {} +unsafe impl Send for RenderBundle {} +unsafe impl Sync for RenderBundle {} -impl RenderBundle { +impl RenderBundle { /// Actually encode the contents into a native command buffer. /// /// This is partially duplicating the logic of `command_encoder_run_render_pass`. @@ -639,7 +648,7 @@ impl RenderBundle { /// Note that the function isn't expected to fail, generally. /// All the validation has already been done by this point. /// The only failure condition is if some of the used buffers are destroyed. - pub(super) unsafe fn execute( + pub(super) unsafe fn execute( &self, raw: &mut A::CommandEncoder, pipeline_layout_guard: &Storage< @@ -828,7 +837,7 @@ impl RenderBundle { } } -impl Resource for RenderBundle { +impl Resource for RenderBundle { const TYPE: &'static str = "RenderBundle"; fn life_guard(&self) -> &LifeGuard { @@ -1027,10 +1036,9 @@ struct VertexLimitState { /// [`SetIndexBuffer`] to the simulated state stored here, and then /// calls the `flush_foo` methods before draw calls to produce the /// update commands we actually need. -#[derive(Debug)] -struct State { +struct State { /// Resources used by this bundle. This will become [`RenderBundle::used`]. - trackers: TrackerSet, + trackers: RenderBundleScope, /// The current index buffer. We flush this state before indexed /// draw commands. @@ -1056,7 +1064,7 @@ struct State { pipeline: Option, } -impl State { +impl State { fn vertex_limits(&self) -> VertexLimitState { let mut vert_state = VertexLimitState { vertex_limit: u32::MAX, @@ -1235,8 +1243,6 @@ pub(super) enum RenderBundleErrorInner { #[error(transparent)] RenderCommand(RenderCommandError), #[error(transparent)] - ResourceUsageConflict(#[from] UsageConflict), - #[error(transparent)] Draw(#[from] DrawError), #[error(transparent)] MissingDownlevelFlags(#[from] MissingDownlevelFlags), diff --git a/wgpu-core/src/command/clear.rs b/wgpu-core/src/command/clear.rs index 0bd909c76e..3e94843798 100644 --- a/wgpu-core/src/command/clear.rs +++ b/wgpu-core/src/command/clear.rs @@ -4,13 +4,12 @@ use std::{num::NonZeroU32, ops::Range}; use crate::device::trace::Command as TraceCommand; use crate::{ command::CommandBuffer, - device::Device, get_lowest_common_denom, - hub::{Global, GlobalIdentityHandlerFactory, HalApi, Resource, Token}, + hub::{self, Global, GlobalIdentityHandlerFactory, HalApi, Token}, id::{BufferId, CommandEncoderId, DeviceId, TextureId, Valid}, init_tracker::{MemoryInitKind, TextureInitRange}, resource::{Texture, TextureClearMode}, - track::{ResourceTracker, TextureSelector, TextureState}, + track::{TextureSelector, TextureTracker}, }; use hal::{auxil::align_to, CommandEncoder as _}; @@ -90,8 +89,8 @@ impl Global { let (dst_buffer, dst_pending) = cmd_buf .trackers .buffers - .use_replace(&*buffer_guard, dst, (), hal::BufferUses::COPY_DST) - .map_err(ClearError::InvalidBuffer)?; + .set_single(&*buffer_guard, dst, hal::BufferUses::COPY_DST) + .ok_or(ClearError::InvalidBuffer(dst))?; let dst_raw = dst_buffer .raw .as_ref() @@ -139,7 +138,7 @@ impl Global { let dst_barrier = dst_pending.map(|pending| pending.into_hal(dst_buffer)); let cmd_buf_raw = cmd_buf.encoder.open(); unsafe { - cmd_buf_raw.transition_buffers(dst_barrier); + cmd_buf_raw.transition_buffers(dst_barrier.into_iter()); cmd_buf_raw.clear_buffer(dst_raw, offset..end); } Ok(()) @@ -191,13 +190,13 @@ impl Global { // Check if subresource level range is valid let subresource_level_end = match subresource_range.mip_level_count { Some(count) => subresource_range.base_mip_level + count.get(), - None => dst_texture.full_range.levels.end, + None => dst_texture.full_range.mips.end, }; - if dst_texture.full_range.levels.start > subresource_range.base_mip_level - || dst_texture.full_range.levels.end < subresource_level_end + if dst_texture.full_range.mips.start > subresource_range.base_mip_level + || dst_texture.full_range.mips.end < subresource_level_end { return Err(ClearError::InvalidTextureLevelRange { - texture_level_range: dst_texture.full_range.levels.clone(), + texture_level_range: dst_texture.full_range.mips.clone(), subresource_base_mip_level: subresource_range.base_mip_level, subresource_mip_level_count: subresource_range.mip_level_count, }); @@ -217,48 +216,34 @@ impl Global { }); } + let device = &device_guard[cmd_buf.device_id.value]; + clear_texture( + &*texture_guard, Valid(dst), - dst_texture, TextureInitRange { mip_range: subresource_range.base_mip_level..subresource_level_end, layer_range: subresource_range.base_array_layer..subresource_layer_end, }, cmd_buf.encoder.open(), &mut cmd_buf.trackers.textures, - &device_guard[cmd_buf.device_id.value], + &device.alignments, + &device.zero_buffer, ) } } -pub(crate) fn clear_texture( - dst_texture_id: Valid, - dst_texture: &Texture, - range: TextureInitRange, - encoder: &mut A::CommandEncoder, - texture_tracker: &mut ResourceTracker, - device: &Device, -) -> Result<(), ClearError> { - clear_texture_no_device( - dst_texture_id, - dst_texture, - range, - encoder, - texture_tracker, - &device.alignments, - &device.zero_buffer, - ) -} - -pub(crate) fn clear_texture_no_device( +pub(crate) fn clear_texture( + storage: &hub::Storage, TextureId>, dst_texture_id: Valid, - dst_texture: &Texture, range: TextureInitRange, encoder: &mut A::CommandEncoder, - texture_tracker: &mut ResourceTracker, + texture_tracker: &mut TextureTracker, alignments: &hal::Alignments, zero_buffer: &A::Buffer, ) -> Result<(), ClearError> { + let dst_texture = &storage[dst_texture_id]; + let dst_raw = dst_texture .inner .as_raw() @@ -277,7 +262,7 @@ pub(crate) fn clear_texture_no_device( }; let selector = TextureSelector { - levels: range.mip_range.clone(), + mips: range.mip_range.clone(), layers: range.layer_range.clone(), }; @@ -287,14 +272,13 @@ pub(crate) fn clear_texture_no_device( // On the other hand, when coming via command_encoder_clear_texture, the life_guard is still there since in order to call it a texture object is needed. // // We could in theory distinguish these two scenarios in the internal clear_texture api in order to remove this check and call the cheaper change_replace_tracked whenever possible. - let dst_barrier = if let Some(ref_count) = dst_texture.life_guard().ref_count.as_ref() { - texture_tracker.change_replace(dst_texture_id, ref_count, selector, clear_usage) - } else { - texture_tracker.change_replace_tracked(dst_texture_id, selector, clear_usage) - } - .map(|pending| pending.into_hal(dst_texture)); + let dst_barrier = texture_tracker + .set_single(storage, dst_texture_id.0, selector, clear_usage) + .unwrap() + .1 + .map(|pending| pending.into_hal(dst_texture)); unsafe { - encoder.transition_textures(dst_barrier); + encoder.transition_textures(dst_barrier.into_iter()); } // Record actual clearing diff --git a/wgpu-core/src/command/compute.rs b/wgpu-core/src/command/compute.rs index 7edb147617..d81c726ad2 100644 --- a/wgpu-core/src/command/compute.rs +++ b/wgpu-core/src/command/compute.rs @@ -14,8 +14,9 @@ use crate::{ hub::{Global, GlobalIdentityHandlerFactory, HalApi, Storage, Token}, id, init_tracker::MemoryInitKind, - resource::{Buffer, Texture}, - track::{StatefulTrackerSubset, TrackerSet, UsageConflict, UseExtendError}, + pipeline, + resource::{self, Buffer, Texture}, + track::{Tracker, UsageConflict, UsageScope}, validation::{check_buffer_usage, MissingBufferUsageError}, Label, }; @@ -228,15 +229,14 @@ where } } -#[derive(Debug)] -struct State { +struct State { binder: Binder, pipeline: Option, - trackers: StatefulTrackerSubset, + scope: UsageScope, debug_scope_depth: u32, } -impl State { +impl State { fn is_ready(&self) -> Result<(), DispatchError> { let bind_mask = self.binder.invalid_mask(); if bind_mask != 0 { @@ -253,32 +253,36 @@ impl State { Ok(()) } - fn flush_states( + fn flush_states( &mut self, raw_encoder: &mut A::CommandEncoder, - base_trackers: &mut TrackerSet, + base_trackers: &mut Tracker, bind_group_guard: &Storage, id::BindGroupId>, buffer_guard: &Storage, id::BufferId>, texture_guard: &Storage, id::TextureId>, ) -> Result<(), UsageConflict> { for id in self.binder.list_active() { - self.trackers.merge_extend(&bind_group_guard[id].used)?; - //Note: stateless trackers are not merged: the lifetime reference + unsafe { + self.scope + .merge_bind_group(texture_guard, &bind_group_guard[id].used)? + }; + // Note: stateless trackers are not merged: the lifetime reference // is held to the bind group itself. } - log::trace!("Encoding dispatch barriers"); + for id in self.binder.list_active() { + unsafe { + base_trackers.set_and_remove_from_usage_scope_sparse( + texture_guard, + &mut self.scope, + &bind_group_guard[id].used, + ) + } + } - CommandBuffer::insert_barriers( - raw_encoder, - base_trackers, - &self.trackers.buffers, - &self.trackers.textures, - buffer_guard, - texture_guard, - ); + log::trace!("Encoding dispatch barriers"); - self.trackers.clear(); + CommandBuffer::drain_barriers(raw_encoder, base_trackers, buffer_guard, texture_guard); Ok(()) } } @@ -338,7 +342,7 @@ impl Global { let mut state = State { binder: Binder::new(), pipeline: None, - trackers: StatefulTrackerSubset::new(A::VARIANT), + scope: UsageScope::new(&*buffer_guard, &*texture_guard), debug_scope_depth: 0, }; let mut temp_offsets = Vec::new(); @@ -346,6 +350,18 @@ impl Global { let mut string_offset = 0; let mut active_query = None; + cmd_buf.trackers.set_size( + Some(&*buffer_guard), + Some(&*texture_guard), + None, + None, + Some(&*bind_group_guard), + Some(&*pipeline_guard), + None, + None, + Some(&*query_set_guard), + ); + let hal_desc = hal::ComputePassDescriptor { label: base.label }; unsafe { raw.begin_compute_pass(&hal_desc); @@ -379,11 +395,11 @@ impl Global { ); dynamic_offset_count += num_dynamic_offsets as usize; - let bind_group = cmd_buf + let bind_group: &BindGroup = cmd_buf .trackers .bind_groups - .use_extend(&*bind_group_guard, bind_group_id, (), ()) - .map_err(|_| ComputePassErrorInner::InvalidBindGroup(bind_group_id)) + .add_single(&*bind_group_guard, bind_group_id) + .ok_or(ComputePassErrorInner::InvalidBindGroup(bind_group_id)) .map_pass_err(scope)?; bind_group .validate_dynamic_bindings(&temp_offsets, &cmd_buf.limits) @@ -434,11 +450,11 @@ impl Global { state.pipeline = Some(pipeline_id); - let pipeline = cmd_buf + let pipeline: &pipeline::ComputePipeline = cmd_buf .trackers - .compute_pipes - .use_extend(&*pipeline_guard, pipeline_id, (), ()) - .map_err(|_| ComputePassErrorInner::InvalidPipeline(pipeline_id)) + .compute_pipelines + .add_single(&*pipeline_guard, pipeline_id) + .ok_or(ComputePassErrorInner::InvalidPipeline(pipeline_id)) .map_pass_err(scope)?; unsafe { @@ -587,11 +603,10 @@ impl Global { .require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION) .map_pass_err(scope)?; - let indirect_buffer = state - .trackers + let indirect_buffer: &Buffer = state + .scope .buffers - .use_extend(&*buffer_guard, buffer_id, (), hal::BufferUses::INDIRECT) - .map_err(|_| ComputePassErrorInner::InvalidIndirectBuffer(buffer_id)) + .merge_single(&*buffer_guard, buffer_id, hal::BufferUses::INDIRECT) .map_pass_err(scope)?; check_buffer_usage(indirect_buffer.usage, wgt::BufferUsages::INDIRECT) .map_pass_err(scope)?; @@ -670,16 +685,11 @@ impl Global { } => { let scope = PassErrorScope::WriteTimestamp; - let query_set = cmd_buf + let query_set: &resource::QuerySet = cmd_buf .trackers .query_sets - .use_extend(&*query_set_guard, query_set_id, (), ()) - .map_err(|e| match e { - UseExtendError::InvalidResource => { - ComputePassErrorInner::InvalidQuerySet(query_set_id) - } - _ => unreachable!(), - }) + .add_single(&*query_set_guard, query_set_id) + .ok_or(ComputePassErrorInner::InvalidQuerySet(query_set_id)) .map_pass_err(scope)?; query_set @@ -692,16 +702,11 @@ impl Global { } => { let scope = PassErrorScope::BeginPipelineStatisticsQuery; - let query_set = cmd_buf + let query_set: &resource::QuerySet = cmd_buf .trackers .query_sets - .use_extend(&*query_set_guard, query_set_id, (), ()) - .map_err(|e| match e { - UseExtendError::InvalidResource => { - ComputePassErrorInner::InvalidQuerySet(query_set_id) - } - _ => unreachable!(), - }) + .add_single(&*query_set_guard, query_set_id) + .ok_or(ComputePassErrorInner::InvalidQuerySet(query_set_id)) .map_pass_err(scope)?; query_set diff --git a/wgpu-core/src/command/draw.rs b/wgpu-core/src/command/draw.rs index e8e53e3ade..2206f3f204 100644 --- a/wgpu-core/src/command/draw.rs +++ b/wgpu-core/src/command/draw.rs @@ -5,7 +5,7 @@ use crate::{ binding_model::{LateMinBufferBindingSizeMismatch, PushConstantUploadError}, error::ErrorFormatter, id, - track::UseExtendError, + track::UsageConflict, validation::{MissingBufferUsageError, MissingTextureUsageError}, }; use wgt::{BufferAddress, BufferSize, Color}; @@ -13,8 +13,6 @@ use wgt::{BufferAddress, BufferSize, Color}; use std::num::NonZeroU32; use thiserror::Error; -pub type BufferError = UseExtendError; - /// Error validating a draw call. #[derive(Clone, Debug, Error, PartialEq)] pub enum DrawError { @@ -79,8 +77,8 @@ pub enum RenderCommandError { IncompatiblePipelineTargets(#[from] crate::device::RenderPassCompatibilityError), #[error("pipeline writes to depth/stencil, while the pass has read-only depth/stencil")] IncompatiblePipelineRods, - #[error("buffer {0:?} is in error {1:?}")] - Buffer(id::BufferId, BufferError), + #[error(transparent)] + UsageConflict(#[from] UsageConflict), #[error("buffer {0:?} is destroyed")] DestroyedBuffer(id::BufferId), #[error(transparent)] @@ -106,7 +104,11 @@ impl crate::error::PrettyError for RenderCommandError { Self::InvalidPipeline(id) => { fmt.render_pipeline_label(&id); } - Self::Buffer(id, ..) | Self::DestroyedBuffer(id) => { + Self::UsageConflict(UsageConflict::TextureInvalid { id }) => { + fmt.texture_label(&id); + } + Self::UsageConflict(UsageConflict::BufferInvalid { id }) + | Self::DestroyedBuffer(id) => { fmt.buffer_label(&id); } _ => {} diff --git a/wgpu-core/src/command/memory_init.rs b/wgpu-core/src/command/memory_init.rs index ea0b303609..d974575f91 100644 --- a/wgpu-core/src/command/memory_init.rs +++ b/wgpu-core/src/command/memory_init.rs @@ -4,11 +4,11 @@ use hal::CommandEncoder; use crate::{ device::Device, - hub::Storage, + hub::{HalApi, Storage}, id::{self, TextureId}, init_tracker::*, resource::{Buffer, Texture}, - track::{ResourceTracker, TextureState, TrackerSet}, + track::{TextureTracker, Tracker}, FastHashMap, }; @@ -121,36 +121,37 @@ impl CommandBufferTextureMemoryActions { // Utility function that takes discarded surfaces from (several calls to) register_init_action and initializes them on the spot. // Takes care of barriers as well! pub(crate) fn fixup_discarded_surfaces< - A: hal::Api, + A: HalApi, InitIter: Iterator, >( inits: InitIter, encoder: &mut A::CommandEncoder, texture_guard: &Storage, TextureId>, - texture_tracker: &mut ResourceTracker, + texture_tracker: &mut TextureTracker, device: &Device, ) { for init in inits { clear_texture( + texture_guard, id::Valid(init.texture), - texture_guard.get(init.texture).unwrap(), TextureInitRange { mip_range: init.mip_level..(init.mip_level + 1), layer_range: init.layer..(init.layer + 1), }, encoder, texture_tracker, - device, + &device.alignments, + &device.zero_buffer, ) .unwrap(); } } -impl BakedCommands { +impl BakedCommands { // inserts all buffer initializations that are going to be needed for executing the commands and updates resource init states accordingly pub(crate) fn initialize_buffer_memory( &mut self, - device_tracker: &mut TrackerSet, + device_tracker: &mut Tracker, buffer_guard: &mut Storage, id::BufferId>, ) -> Result<(), DestroyedBufferError> { // Gather init ranges for each buffer so we can collapse them. @@ -202,11 +203,11 @@ impl BakedCommands { // Don't do use_replace since the buffer may already no longer have a ref_count. // However, we *know* that it is currently in use, so the tracker must already know about it. - let transition = device_tracker.buffers.change_replace_tracked( - id::Valid(buffer_id), - (), - hal::BufferUses::COPY_DST, - ); + let transition = device_tracker + .buffers + .set_single(buffer_guard, buffer_id, hal::BufferUses::COPY_DST) + .unwrap() + .1; let buffer = buffer_guard .get_mut(buffer_id) @@ -214,8 +215,11 @@ impl BakedCommands { let raw_buf = buffer.raw.as_ref().ok_or(DestroyedBufferError(buffer_id))?; unsafe { - self.encoder - .transition_buffers(transition.map(|pending| pending.into_hal(buffer))); + self.encoder.transition_buffers( + transition + .map(|pending| pending.into_hal(buffer)) + .into_iter(), + ); } for range in ranges.iter() { @@ -234,7 +238,7 @@ impl BakedCommands { // any textures that are left discarded by this command buffer will be marked as uninitialized pub(crate) fn initialize_texture_memory( &mut self, - device_tracker: &mut TrackerSet, + device_tracker: &mut Tracker, texture_guard: &mut Storage, TextureId>, device: &Device, ) -> Result<(), DestroyedTextureError> { @@ -274,12 +278,13 @@ impl BakedCommands { // TODO: Could we attempt some range collapsing here? for range in ranges.drain(..) { clear_texture( + texture_guard, id::Valid(texture_use.id), - &*texture, range, &mut self.encoder, &mut device_tracker.textures, - device, + &device.alignments, + &device.zero_buffer, ) .unwrap(); } diff --git a/wgpu-core/src/command/mod.rs b/wgpu-core/src/command/mod.rs index f16704c53f..47ee9c0ecc 100644 --- a/wgpu-core/src/command/mod.rs +++ b/wgpu-core/src/command/mod.rs @@ -10,7 +10,7 @@ mod transfer; use std::slice; -pub(crate) use self::clear::clear_texture_no_device; +pub(crate) use self::clear::clear_texture; pub use self::{ bundle::*, clear::ClearError, compute::*, draw::*, query::*, render::*, transfer::*, }; @@ -19,11 +19,11 @@ use self::memory_init::CommandBufferTextureMemoryActions; use crate::error::{ErrorFormatter, PrettyError}; use crate::init_tracker::BufferInitTrackerAction; +use crate::track::{Tracker, UsageScope}; use crate::{ hub::{Global, GlobalIdentityHandlerFactory, HalApi, Storage, Token}, id, resource::{Buffer, Texture}, - track::{BufferState, ResourceTracker, TextureState, TrackerSet}, Label, Stored, }; @@ -81,10 +81,10 @@ impl CommandEncoder { } } -pub struct BakedCommands { +pub struct BakedCommands { pub(crate) encoder: A::CommandEncoder, pub(crate) list: Vec, - pub(crate) trackers: TrackerSet, + pub(crate) trackers: Tracker, buffer_memory_init_actions: Vec, texture_memory_actions: CommandBufferTextureMemoryActions, } @@ -92,11 +92,11 @@ pub struct BakedCommands { pub(crate) struct DestroyedBufferError(pub id::BufferId); pub(crate) struct DestroyedTextureError(pub id::TextureId); -pub struct CommandBuffer { +pub struct CommandBuffer { encoder: CommandEncoder, status: CommandEncoderStatus, pub(crate) device_id: Stored, - pub(crate) trackers: TrackerSet, + pub(crate) trackers: Tracker, buffer_memory_init_actions: Vec, texture_memory_actions: CommandBufferTextureMemoryActions, limits: wgt::Limits, @@ -124,7 +124,7 @@ impl CommandBuffer { }, status: CommandEncoderStatus::Recording, device_id, - trackers: TrackerSet::new(A::VARIANT), + trackers: Tracker::new(), buffer_memory_init_actions: Default::default(), texture_memory_actions: Default::default(), limits, @@ -138,23 +138,52 @@ impl CommandBuffer { } } - pub(crate) fn insert_barriers( + pub(crate) fn insert_barriers_from_tracker( raw: &mut A::CommandEncoder, - base: &mut TrackerSet, - head_buffers: &ResourceTracker, - head_textures: &ResourceTracker, + base: &mut Tracker, + head: &Tracker, buffer_guard: &Storage, id::BufferId>, texture_guard: &Storage, id::TextureId>, ) { profiling::scope!("insert_barriers"); - debug_assert_eq!(A::VARIANT, base.backend()); - let buffer_barriers = base.buffers.merge_replace(head_buffers).map(|pending| { - let buf = &buffer_guard[pending.id]; + base.buffers.set_from_tracker(&head.buffers); + base.textures + .set_from_tracker(&*texture_guard, &head.textures); + + Self::drain_barriers(raw, base, buffer_guard, texture_guard); + } + + pub(crate) fn insert_barriers_from_scope( + raw: &mut A::CommandEncoder, + base: &mut Tracker, + head: &UsageScope, + buffer_guard: &Storage, id::BufferId>, + texture_guard: &Storage, id::TextureId>, + ) { + profiling::scope!("insert_barriers"); + + base.buffers.set_from_usage_scope(&head.buffers); + base.textures + .set_from_usage_scope(&*texture_guard, &head.textures); + + Self::drain_barriers(raw, base, buffer_guard, texture_guard); + } + + pub(crate) fn drain_barriers( + raw: &mut A::CommandEncoder, + base: &mut Tracker, + buffer_guard: &Storage, id::BufferId>, + texture_guard: &Storage, id::TextureId>, + ) { + profiling::scope!("drain_barriers"); + + let buffer_barriers = base.buffers.drain().map(|pending| { + let buf = unsafe { &buffer_guard.get_unchecked(pending.id) }; pending.into_hal(buf) }); - let texture_barriers = base.textures.merge_replace(head_textures).map(|pending| { - let tex = &texture_guard[pending.id]; + let texture_barriers = base.textures.drain().map(|pending| { + let tex = unsafe { texture_guard.get_unchecked(pending.id) }; pending.into_hal(tex) }); @@ -165,7 +194,7 @@ impl CommandBuffer { } } -impl CommandBuffer { +impl CommandBuffer { fn get_encoder_mut( storage: &mut Storage, id: id::CommandEncoderId, @@ -198,7 +227,7 @@ impl CommandBuffer { } } -impl crate::hub::Resource for CommandBuffer { +impl crate::hub::Resource for CommandBuffer { const TYPE: &'static str = "CommandBuffer"; fn life_guard(&self) -> &crate::LifeGuard { @@ -321,7 +350,7 @@ impl Global { cmd_buf.status = CommandEncoderStatus::Finished; //Note: if we want to stop tracking the swapchain texture view, // this is the place to do it. - log::trace!("Command buffer {:?} {:#?}", encoder_id, cmd_buf.trackers); + log::trace!("Command buffer {:?}", encoder_id); None } CommandEncoderStatus::Finished => Some(CommandEncoderError::NotRecording), diff --git a/wgpu-core/src/command/query.rs b/wgpu-core/src/command/query.rs index 42a0dd905b..873a3031ac 100644 --- a/wgpu-core/src/command/query.rs +++ b/wgpu-core/src/command/query.rs @@ -8,7 +8,6 @@ use crate::{ id::{self, Id, TypedId}, init_tracker::MemoryInitKind, resource::QuerySet, - track::UseExtendError, Epoch, FastHashMap, Index, }; use std::{iter, marker::PhantomData}; @@ -300,11 +299,8 @@ impl Global { let query_set = cmd_buf .trackers .query_sets - .use_extend(&*query_set_guard, query_set_id, (), ()) - .map_err(|e| match e { - UseExtendError::InvalidResource => QueryError::InvalidQuerySet(query_set_id), - _ => unreachable!(), - })?; + .add_single(&*query_set_guard, query_set_id) + .ok_or(QueryError::InvalidQuerySet(query_set_id))?; query_set.validate_and_write_timestamp(raw_encoder, query_set_id, query_index, None)?; @@ -348,17 +344,14 @@ impl Global { let query_set = cmd_buf .trackers .query_sets - .use_extend(&*query_set_guard, query_set_id, (), ()) - .map_err(|e| match e { - UseExtendError::InvalidResource => QueryError::InvalidQuerySet(query_set_id), - _ => unreachable!(), - })?; + .add_single(&*query_set_guard, query_set_id) + .ok_or(QueryError::InvalidQuerySet(query_set_id))?; let (dst_buffer, dst_pending) = cmd_buf .trackers .buffers - .use_replace(&*buffer_guard, destination, (), hal::BufferUses::COPY_DST) - .map_err(QueryError::InvalidBuffer)?; + .set_single(&*buffer_guard, destination, hal::BufferUses::COPY_DST) + .ok_or(QueryError::InvalidBuffer(destination))?; let dst_barrier = dst_pending.map(|pending| pending.into_hal(dst_buffer)); if !dst_buffer.usage.contains(wgt::BufferUsages::COPY_DST) { @@ -407,7 +400,7 @@ impl Global { )); unsafe { - raw_encoder.transition_buffers(dst_barrier); + raw_encoder.transition_buffers(dst_barrier.into_iter()); raw_encoder.copy_query_results( &query_set.raw, start_query..end_query, diff --git a/wgpu-core/src/command/render.rs b/wgpu-core/src/command/render.rs index bbbffa5a13..7ef45aa0a6 100644 --- a/wgpu-core/src/command/render.rs +++ b/wgpu-core/src/command/render.rs @@ -1,6 +1,7 @@ use crate::{ binding_model::BindError, command::{ + self, bind::Binder, end_pipeline_statistics_query, memory_init::{fixup_discarded_surfaces, SurfacesInDiscardState}, @@ -17,8 +18,8 @@ use crate::{ id, init_tracker::{MemoryInitKind, TextureInitRange, TextureInitTrackerAction}, pipeline::PipelineFlags, - resource::{Texture, TextureView}, - track::{StatefulTrackerSubset, TextureSelector, UsageConflict}, + resource::{self, Buffer, Texture, TextureView}, + track::{TextureSelector, UsageConflict, UsageScope}, validation::{ check_buffer_usage, check_texture_usage, MissingBufferUsageError, MissingTextureUsageError, }, @@ -38,7 +39,6 @@ use serde::Deserialize; #[cfg(any(feature = "serial-pass", feature = "trace"))] use serde::Serialize; -use crate::track::UseExtendError; use std::{borrow::Cow, fmt, iter, marker::PhantomData, mem, num::NonZeroU32, ops::Range, str}; use super::{memory_init::TextureSurfaceDiscard, CommandBufferTextureMemoryActions}; @@ -561,9 +561,9 @@ impl TextureView { const MAX_TOTAL_ATTACHMENTS: usize = hal::MAX_COLOR_TARGETS + hal::MAX_COLOR_TARGETS + 1; type AttachmentDataVec = ArrayVec; -struct RenderPassInfo<'a, A: hal::Api> { +struct RenderPassInfo<'a, A: HalApi> { context: RenderPassContext, - trackers: StatefulTrackerSubset, + usage_scope: UsageScope, render_attachments: AttachmentDataVec>, // All render attachments, including depth/stencil is_ds_read_only: bool, extent: wgt::Extent3d, @@ -605,7 +605,7 @@ impl<'a, A: HalApi> RenderPassInfo<'a, A> { // but recording the discard right away be alright since the texture can't be used during the pass anyways texture_memory_actions.discard(TextureSurfaceDiscard { texture: view.parent_id.value.0, - mip_level: view.selector.levels.start, + mip_level: view.selector.mips.start, layer: view.selector.layers.start, }); } @@ -618,6 +618,7 @@ impl<'a, A: HalApi> RenderPassInfo<'a, A> { depth_stencil_attachment: Option<&RenderPassDepthStencilAttachment>, cmd_buf: &mut CommandBuffer, view_guard: &'a Storage, id::TextureViewId>, + buffer_guard: &'a Storage, id::BufferId>, texture_guard: &'a Storage, id::TextureId>, ) -> Result { profiling::scope!("start", "RenderPassInfo"); @@ -699,8 +700,8 @@ impl<'a, A: HalApi> RenderPassInfo<'a, A> { let view = cmd_buf .trackers .views - .use_extend(&*view_guard, at.view, (), ()) - .map_err(|_| RenderPassErrorInner::InvalidAttachment(at.view))?; + .add_single(&*view_guard, at.view) + .ok_or(RenderPassErrorInner::InvalidAttachment(at.view))?; check_multiview(view)?; add_view(view, "depth")?; @@ -779,7 +780,7 @@ impl<'a, A: HalApi> RenderPassInfo<'a, A> { // Both are discarded using the regular path. discarded_surfaces.push(TextureSurfaceDiscard { texture: view.parent_id.value.0, - mip_level: view.selector.levels.start, + mip_level: view.selector.mips.start, layer: view.selector.layers.start, }); } @@ -808,8 +809,8 @@ impl<'a, A: HalApi> RenderPassInfo<'a, A> { let color_view = cmd_buf .trackers .views - .use_extend(&*view_guard, at.view, (), ()) - .map_err(|_| RenderPassErrorInner::InvalidAttachment(at.view))?; + .add_single(&*view_guard, at.view) + .ok_or(RenderPassErrorInner::InvalidAttachment(at.view))?; check_multiview(color_view)?; add_view(color_view, "color")?; @@ -838,8 +839,8 @@ impl<'a, A: HalApi> RenderPassInfo<'a, A> { let resolve_view = cmd_buf .trackers .views - .use_extend(&*view_guard, resolve_target, (), ()) - .map_err(|_| RenderPassErrorInner::InvalidAttachment(resolve_target))?; + .add_single(&*view_guard, resolve_target) + .ok_or(RenderPassErrorInner::InvalidAttachment(resolve_target))?; check_multiview(resolve_view)?; if color_view.extent != resolve_view.extent { @@ -934,7 +935,7 @@ impl<'a, A: HalApi> RenderPassInfo<'a, A> { Ok(Self { context, - trackers: StatefulTrackerSubset::new(A::VARIANT), + usage_scope: UsageScope::new(buffer_guard, texture_guard), render_attachments, is_ds_read_only, extent, @@ -949,7 +950,7 @@ impl<'a, A: HalApi> RenderPassInfo<'a, A> { mut self, raw: &mut A::CommandEncoder, texture_guard: &Storage, id::TextureId>, - ) -> Result<(StatefulTrackerSubset, SurfacesInDiscardState), RenderPassErrorInner> { + ) -> Result<(UsageScope, SurfacesInDiscardState), RenderPassErrorInner> { profiling::scope!("finish", "RenderPassInfo"); unsafe { raw.end_render_pass(); @@ -963,15 +964,18 @@ impl<'a, A: HalApi> RenderPassInfo<'a, A> { check_texture_usage(texture.desc.usage, TextureUsages::RENDER_ATTACHMENT)?; // the tracker set of the pass is always in "extend" mode - self.trackers - .textures - .change_extend( - ra.texture_id.value, - &ra.texture_id.ref_count, - ra.selector.clone(), - ra.usage, - ) - .map_err(UsageConflict::from)?; + unsafe { + self.usage_scope + .textures + .merge_single( + &*texture_guard, + ra.texture_id.value, + Some(ra.selector.clone()), + &ra.texture_id.ref_count, + ra.usage, + ) + .map_err(UsageConflict::from)? + }; } // If either only stencil or depth was discarded, we put in a special clear pass to keep the init status of the aspects in sync. @@ -1012,7 +1016,7 @@ impl<'a, A: HalApi> RenderPassInfo<'a, A> { } } - Ok((self.trackers, self.pending_discard_init_fixups)) + Ok((self.usage_scope, self.pending_discard_init_fixups)) } } @@ -1047,7 +1051,7 @@ impl Global { let mut token = Token::root(); let (device_guard, mut token) = hub.devices.read(&mut token); - let (trackers, query_reset_state, pending_discard_init_fixups) = { + let (scope, query_reset_state, pending_discard_init_fixups) = { let (mut cmb_guard, mut token) = hub.command_buffers.write(&mut token); // Spell out the type, to placate rust-analyzer. @@ -1075,7 +1079,7 @@ impl Global { let (bundle_guard, mut token) = hub.render_bundles.read(&mut token); let (pipeline_layout_guard, mut token) = hub.pipeline_layouts.read(&mut token); let (bind_group_guard, mut token) = hub.bind_groups.read(&mut token); - let (pipeline_guard, mut token) = hub.render_pipelines.read(&mut token); + let (render_pipeline_guard, mut token) = hub.render_pipelines.read(&mut token); let (query_set_guard, mut token) = hub.query_sets.read(&mut token); let (buffer_guard, mut token) = hub.buffers.read(&mut token); let (texture_guard, mut token) = hub.textures.read(&mut token); @@ -1093,10 +1097,23 @@ impl Global { depth_stencil_attachment, cmd_buf, &*view_guard, + &*buffer_guard, &*texture_guard, ) .map_pass_err(init_scope)?; + cmd_buf.trackers.set_size( + Some(&*buffer_guard), + Some(&*texture_guard), + Some(&*view_guard), + None, + Some(&*bind_group_guard), + None, + Some(&*render_pipeline_guard), + Some(&*bundle_guard), + Some(&*query_set_guard), + ); + let raw = &mut cmd_buf.encoder.raw; let mut state = State { @@ -1142,17 +1159,19 @@ impl Global { let bind_group = cmd_buf .trackers .bind_groups - .use_extend(&*bind_group_guard, bind_group_id, (), ()) - .map_err(|_| RenderCommandError::InvalidBindGroup(bind_group_id)) + .add_single(&*bind_group_guard, bind_group_id) + .ok_or(RenderCommandError::InvalidBindGroup(bind_group_id)) .map_pass_err(scope)?; bind_group .validate_dynamic_bindings(&temp_offsets, &cmd_buf.limits) .map_pass_err(scope)?; // merge the resource tracker in - info.trackers - .merge_extend(&bind_group.used) - .map_pass_err(scope)?; + unsafe { + info.usage_scope + .merge_bind_group(&*texture_guard, &bind_group.used) + .map_pass_err(scope)?; + } //Note: stateless trackers are not merged: the lifetime reference // is held to the bind group itself. @@ -1203,9 +1222,9 @@ impl Global { let pipeline = cmd_buf .trackers - .render_pipes - .use_extend(&*pipeline_guard, pipeline_id, (), ()) - .map_err(|_| RenderCommandError::InvalidPipeline(pipeline_id)) + .render_pipelines + .add_single(&*render_pipeline_guard, pipeline_id) + .ok_or(RenderCommandError::InvalidPipeline(pipeline_id)) .map_pass_err(scope)?; info.context @@ -1312,11 +1331,10 @@ impl Global { size, } => { let scope = PassErrorScope::SetIndexBuffer(buffer_id); - let buffer = info - .trackers + let buffer: &Buffer = info + .usage_scope .buffers - .use_extend(&*buffer_guard, buffer_id, (), hal::BufferUses::INDEX) - .map_err(|e| RenderCommandError::Buffer(buffer_id, e)) + .merge_single(&*buffer_guard, buffer_id, hal::BufferUses::INDEX) .map_pass_err(scope)?; check_buffer_usage(buffer.usage, BufferUsages::INDEX) .map_pass_err(scope)?; @@ -1359,11 +1377,10 @@ impl Global { size, } => { let scope = PassErrorScope::SetVertexBuffer(buffer_id); - let buffer = info - .trackers + let buffer: &Buffer = info + .usage_scope .buffers - .use_extend(&*buffer_guard, buffer_id, (), hal::BufferUses::VERTEX) - .map_err(|e| RenderCommandError::Buffer(buffer_id, e)) + .merge_single(&*buffer_guard, buffer_id, hal::BufferUses::VERTEX) .map_pass_err(scope)?; check_buffer_usage(buffer.usage, BufferUsages::VERTEX) .map_pass_err(scope)?; @@ -1617,11 +1634,10 @@ impl Global { .require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION) .map_pass_err(scope)?; - let indirect_buffer = info - .trackers + let indirect_buffer: &Buffer = info + .usage_scope .buffers - .use_extend(&*buffer_guard, buffer_id, (), hal::BufferUses::INDIRECT) - .map_err(|e| RenderCommandError::Buffer(buffer_id, e)) + .merge_single(&*buffer_guard, buffer_id, hal::BufferUses::INDIRECT) .map_pass_err(scope)?; check_buffer_usage(indirect_buffer.usage, BufferUsages::INDIRECT) .map_pass_err(scope)?; @@ -1688,11 +1704,10 @@ impl Global { .require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION) .map_pass_err(scope)?; - let indirect_buffer = info - .trackers + let indirect_buffer: &Buffer = info + .usage_scope .buffers - .use_extend(&*buffer_guard, buffer_id, (), hal::BufferUses::INDIRECT) - .map_err(|e| RenderCommandError::Buffer(buffer_id, e)) + .merge_single(&*buffer_guard, buffer_id, hal::BufferUses::INDIRECT) .map_pass_err(scope)?; check_buffer_usage(indirect_buffer.usage, BufferUsages::INDIRECT) .map_pass_err(scope)?; @@ -1702,16 +1717,14 @@ impl Global { .ok_or(RenderCommandError::DestroyedBuffer(buffer_id)) .map_pass_err(scope)?; - let count_buffer = info - .trackers + let count_buffer: &Buffer = info + .usage_scope .buffers - .use_extend( + .merge_single( &*buffer_guard, count_buffer_id, - (), hal::BufferUses::INDIRECT, ) - .map_err(|e| RenderCommandError::Buffer(count_buffer_id, e)) .map_pass_err(scope)?; check_buffer_usage(count_buffer.usage, BufferUsages::INDIRECT) .map_pass_err(scope)?; @@ -1814,16 +1827,11 @@ impl Global { } => { let scope = PassErrorScope::WriteTimestamp; - let query_set = cmd_buf + let query_set: &resource::QuerySet = cmd_buf .trackers .query_sets - .use_extend(&*query_set_guard, query_set_id, (), ()) - .map_err(|e| match e { - UseExtendError::InvalidResource => { - RenderCommandError::InvalidQuerySet(query_set_id) - } - _ => unreachable!(), - }) + .add_single(&*query_set_guard, query_set_id) + .ok_or(RenderCommandError::InvalidQuerySet(query_set_id)) .map_pass_err(scope)?; query_set @@ -1841,16 +1849,11 @@ impl Global { } => { let scope = PassErrorScope::BeginPipelineStatisticsQuery; - let query_set = cmd_buf + let query_set: &resource::QuerySet = cmd_buf .trackers .query_sets - .use_extend(&*query_set_guard, query_set_id, (), ()) - .map_err(|e| match e { - UseExtendError::InvalidResource => { - RenderCommandError::InvalidQuerySet(query_set_id) - } - _ => unreachable!(), - }) + .add_single(&*query_set_guard, query_set_id) + .ok_or(RenderCommandError::InvalidQuerySet(query_set_id)) .map_pass_err(scope)?; query_set @@ -1871,11 +1874,11 @@ impl Global { } RenderCommand::ExecuteBundle(bundle_id) => { let scope = PassErrorScope::ExecuteBundle; - let bundle = cmd_buf + let bundle: &command::RenderBundle = cmd_buf .trackers .bundles - .use_extend(&*bundle_guard, bundle_id, (), ()) - .map_err(|_| RenderCommandError::InvalidRenderBundle(bundle_id)) + .add_single(&*bundle_guard, bundle_id) + .ok_or(RenderCommandError::InvalidRenderBundle(bundle_id)) .map_pass_err(scope)?; info.context @@ -1913,7 +1916,7 @@ impl Global { raw, &*pipeline_layout_guard, &*bind_group_guard, - &*pipeline_guard, + &*render_pipeline_guard, &*buffer_guard, ) } @@ -1927,23 +1930,21 @@ impl Global { }) .map_pass_err(scope)?; - info.trackers - .merge_extend(&bundle.used) - .map_pass_err(scope)?; - // Start tracking the bind groups specifically, as they are the only - // compound resources, to make it easier to update submission indices - // later at submission time. - cmd_buf - .trackers - .bind_groups - .merge_extend(&bundle.used.bind_groups) - .unwrap(); + unsafe { + info.usage_scope + .merge_render_bundle(&*texture_guard, &bundle.used) + .map_pass_err(scope)?; + cmd_buf + .trackers + .add_from_render_bundle(&bundle.used) + .map_pass_err(scope)?; + }; state.reset_bundle(); } } } - log::trace!("Merging {:?} with the render pass", encoder_id); + log::trace!("Merging renderpass into cmd_buf {:?}", encoder_id); let (trackers, pending_discard_init_fixups) = info.finish(raw, &*texture_guard).map_pass_err(init_scope)?; @@ -1977,11 +1978,10 @@ impl Global { .map_err(RenderCommandError::InvalidQuerySet) .map_pass_err(PassErrorScope::QueryReset)?; - super::CommandBuffer::insert_barriers( + super::CommandBuffer::insert_barriers_from_scope( transit, &mut cmd_buf.trackers, - &trackers.buffers, - &trackers.textures, + &scope, &*buffer_guard, &*texture_guard, ); diff --git a/wgpu-core/src/command/transfer.rs b/wgpu-core/src/command/transfer.rs index d676bc1d3c..998263b44e 100644 --- a/wgpu-core/src/command/transfer.rs +++ b/wgpu-core/src/command/transfer.rs @@ -1,12 +1,12 @@ #[cfg(feature = "trace")] use crate::device::trace::Command as TraceCommand; use crate::{ - command::{CommandBuffer, CommandEncoderError}, + command::{clear_texture, CommandBuffer, CommandEncoderError}, conv, device::{Device, MissingDownlevelFlags}, error::{ErrorFormatter, PrettyError}, hub::{Global, GlobalIdentityHandlerFactory, HalApi, Storage, Token}, - id::{BufferId, CommandEncoderId, Id, TextureId, Valid}, + id::{BufferId, CommandEncoderId, TextureId, Valid}, init_tracker::{ has_copy_partial_init_tracker_coverage, MemoryInitKind, TextureInitRange, TextureInitTrackerAction, @@ -15,14 +15,13 @@ use crate::{ track::TextureSelector, }; +use arrayvec::ArrayVec; use hal::CommandEncoder as _; use thiserror::Error; use wgt::{BufferAddress, BufferUsages, Extent3d, TextureUsages}; use std::iter; -use super::clear::clear_texture; - pub type ImageCopyBuffer = wgt::ImageCopyBuffer; pub type ImageCopyTexture = wgt::ImageCopyTexture; @@ -191,7 +190,7 @@ pub(crate) fn extract_texture_selector( aspect: copy_aspect, }; let selector = TextureSelector { - levels: copy_texture.mip_level..copy_texture.mip_level + 1, + mips: copy_texture.mip_level..copy_texture.mip_level + 1, layers, }; @@ -381,14 +380,13 @@ pub(crate) fn validate_texture_copy_range( Ok((copy_extent, array_layer_count)) } -fn handle_texture_init( +fn handle_texture_init( init_kind: MemoryInitKind, cmd_buf: &mut CommandBuffer, device: &Device, copy_texture: &ImageCopyTexture, copy_size: &Extent3d, - texture_guard: &Storage, Id>>, - texture: &Texture, + texture_guard: &Storage, TextureId>, ) { let init_action = TextureInitTrackerAction { id: copy_texture.texture, @@ -410,15 +408,16 @@ fn handle_texture_init( let cmd_buf_raw = cmd_buf.encoder.open(); for init in immediate_inits { clear_texture( + texture_guard, Valid(init.texture), - texture, TextureInitRange { mip_range: init.mip_level..(init.mip_level + 1), layer_range: init.layer..(init.layer + 1), }, cmd_buf_raw, &mut cmd_buf.trackers.textures, - device, + &device.alignments, + &device.zero_buffer, ) .unwrap(); } @@ -426,14 +425,14 @@ fn handle_texture_init( } // Ensures the source texture of a transfer is in the right initialization state and records the state for after the transfer operation. -fn handle_src_texture_init( +fn handle_src_texture_init( cmd_buf: &mut CommandBuffer, device: &Device, source: &ImageCopyTexture, copy_size: &Extent3d, texture_guard: &Storage, TextureId>, ) -> Result<(), TransferError> { - let texture = texture_guard + let _ = texture_guard .get(source.texture) .map_err(|_| TransferError::InvalidTexture(source.texture))?; @@ -444,13 +443,12 @@ fn handle_src_texture_init( source, copy_size, texture_guard, - texture, ); Ok(()) } // Ensures the destination texture of a transfer is in the right initialization state and records the state for after the transfer operation. -fn handle_dst_texture_init( +fn handle_dst_texture_init( cmd_buf: &mut CommandBuffer, device: &Device, destination: &ImageCopyTexture, @@ -480,7 +478,6 @@ fn handle_dst_texture_init( destination, copy_size, texture_guard, - texture, ); Ok(()) } @@ -521,8 +518,8 @@ impl Global { let (src_buffer, src_pending) = cmd_buf .trackers .buffers - .use_replace(&*buffer_guard, source, (), hal::BufferUses::COPY_SRC) - .map_err(TransferError::InvalidBuffer)?; + .set_single(&*buffer_guard, source, hal::BufferUses::COPY_SRC) + .ok_or(TransferError::InvalidBuffer(source))?; let src_raw = src_buffer .raw .as_ref() @@ -531,15 +528,13 @@ impl Global { return Err(TransferError::MissingCopySrcUsageFlag.into()); } // expecting only a single barrier - let src_barrier = src_pending - .map(|pending| pending.into_hal(src_buffer)) - .next(); + let src_barrier = src_pending.map(|pending| pending.into_hal(src_buffer)); let (dst_buffer, dst_pending) = cmd_buf .trackers .buffers - .use_replace(&*buffer_guard, destination, (), hal::BufferUses::COPY_DST) - .map_err(TransferError::InvalidBuffer)?; + .set_single(&*buffer_guard, destination, hal::BufferUses::COPY_DST) + .ok_or(TransferError::InvalidBuffer(destination))?; let dst_raw = dst_buffer .raw .as_ref() @@ -547,9 +542,7 @@ impl Global { if !dst_buffer.usage.contains(BufferUsages::COPY_DST) { return Err(TransferError::MissingCopyDstUsageFlag(Some(destination), None).into()); } - let dst_barrier = dst_pending - .map(|pending| pending.into_hal(dst_buffer)) - .next(); + let dst_barrier = dst_pending.map(|pending| pending.into_hal(dst_buffer)); if size % wgt::COPY_BUFFER_ALIGNMENT != 0 { return Err(TransferError::UnalignedCopySize(size).into()); @@ -659,8 +652,8 @@ impl Global { let (src_buffer, src_pending) = cmd_buf .trackers .buffers - .use_replace(&*buffer_guard, source.buffer, (), hal::BufferUses::COPY_SRC) - .map_err(TransferError::InvalidBuffer)?; + .set_single(&*buffer_guard, source.buffer, hal::BufferUses::COPY_SRC) + .ok_or(TransferError::InvalidBuffer(source.buffer))?; let src_raw = src_buffer .raw .as_ref() @@ -668,18 +661,18 @@ impl Global { if !src_buffer.usage.contains(BufferUsages::COPY_SRC) { return Err(TransferError::MissingCopySrcUsageFlag.into()); } - let src_barriers = src_pending.map(|pending| pending.into_hal(src_buffer)); + let src_barrier = src_pending.map(|pending| pending.into_hal(src_buffer)); let (dst_texture, dst_pending) = cmd_buf .trackers .textures - .use_replace( + .set_single( &*texture_guard, destination.texture, dst_range, hal::TextureUses::COPY_DST, ) - .unwrap(); + .ok_or(TransferError::InvalidTexture(destination.texture))?; let dst_raw = dst_texture .inner .as_raw() @@ -689,7 +682,7 @@ impl Global { TransferError::MissingCopyDstUsageFlag(None, Some(destination.texture)).into(), ); } - let dst_barriers = dst_pending.map(|pending| pending.into_hal(dst_texture)); + let dst_barrier = dst_pending.map(|pending| pending.into_hal(dst_texture)); let format_desc = dst_texture.desc.format.describe(); let (hal_copy_size, array_layer_count) = validate_texture_copy_range( @@ -736,8 +729,8 @@ impl Global { let cmd_buf_raw = cmd_buf.encoder.open(); unsafe { - cmd_buf_raw.transition_textures(dst_barriers); - cmd_buf_raw.transition_buffers(src_barriers); + cmd_buf_raw.transition_textures(dst_barrier.into_iter()); + cmd_buf_raw.transition_buffers(src_barrier.into_iter()); cmd_buf_raw.copy_buffer_to_texture(src_raw, dst_raw, regions); } Ok(()) @@ -786,13 +779,13 @@ impl Global { let (src_texture, src_pending) = cmd_buf .trackers .textures - .use_replace( + .set_single( &*texture_guard, source.texture, src_range, hal::TextureUses::COPY_SRC, ) - .unwrap(); + .ok_or(TransferError::InvalidTexture(source.texture))?; let src_raw = src_texture .inner .as_raw() @@ -800,18 +793,17 @@ impl Global { if !src_texture.desc.usage.contains(TextureUsages::COPY_SRC) { return Err(TransferError::MissingCopySrcUsageFlag.into()); } - let src_barriers = src_pending.map(|pending| pending.into_hal(src_texture)); + let src_barrier = src_pending.map(|pending| pending.into_hal(src_texture)); let (dst_buffer, dst_pending) = cmd_buf .trackers .buffers - .use_replace( + .set_single( &*buffer_guard, destination.buffer, - (), hal::BufferUses::COPY_DST, ) - .map_err(TransferError::InvalidBuffer)?; + .ok_or(TransferError::InvalidBuffer(destination.buffer))?; let dst_raw = dst_buffer .raw .as_ref() @@ -821,7 +813,7 @@ impl Global { TransferError::MissingCopyDstUsageFlag(Some(destination.buffer), None).into(), ); } - let dst_barriers = dst_pending.map(|pending| pending.into_hal(dst_buffer)); + let dst_barrier = dst_pending.map(|pending| pending.into_hal(dst_buffer)); let format_desc = src_texture.desc.format.describe(); let (hal_copy_size, array_layer_count) = @@ -876,8 +868,8 @@ impl Global { }); let cmd_buf_raw = cmd_buf.encoder.open(); unsafe { - cmd_buf_raw.transition_buffers(dst_barriers); - cmd_buf_raw.transition_textures(src_barriers); + cmd_buf_raw.transition_buffers(dst_barrier.into_iter()); + cmd_buf_raw.transition_textures(src_barrier.into_iter()); cmd_buf_raw.copy_texture_to_buffer( src_raw, hal::TextureUses::COPY_SRC, @@ -937,13 +929,13 @@ impl Global { let (src_texture, src_pending) = cmd_buf .trackers .textures - .use_replace( + .set_single( &*texture_guard, source.texture, src_range, hal::TextureUses::COPY_SRC, ) - .unwrap(); + .ok_or(TransferError::InvalidTexture(source.texture))?; let src_raw = src_texture .inner .as_raw() @@ -954,20 +946,21 @@ impl Global { //TODO: try to avoid this the collection. It's needed because both // `src_pending` and `dst_pending` try to hold `trackers.textures` mutably. - let mut barriers = src_pending + let mut barriers: ArrayVec<_, 2> = src_pending .map(|pending| pending.into_hal(src_texture)) - .collect::>(); + .into_iter() + .collect(); let (dst_texture, dst_pending) = cmd_buf .trackers .textures - .use_replace( + .set_single( &*texture_guard, destination.texture, dst_range, hal::TextureUses::COPY_DST, ) - .unwrap(); + .ok_or(TransferError::InvalidTexture(destination.texture))?; let dst_raw = dst_texture .inner .as_raw() diff --git a/wgpu-core/src/conv.rs b/wgpu-core/src/conv.rs index 7982390b55..9ffa956aa4 100644 --- a/wgpu-core/src/conv.rs +++ b/wgpu-core/src/conv.rs @@ -1,6 +1,10 @@ use crate::resource; -pub fn is_power_of_two(val: u32) -> bool { +pub fn is_power_of_two_u16(val: u16) -> bool { + val != 0 && (val & (val - 1)) == 0 +} + +pub fn is_power_of_two_u32(val: u32) -> bool { val != 0 && (val & (val - 1)) == 0 } @@ -53,7 +57,7 @@ pub fn map_buffer_usage(usage: wgt::BufferUsages) -> hal::BufferUses { usage.contains(wgt::BufferUsages::UNIFORM), ); u.set( - hal::BufferUses::STORAGE_READ | hal::BufferUses::STORAGE_WRITE, + hal::BufferUses::STORAGE_READ | hal::BufferUses::STORAGE_READ_WRITE, usage.contains(wgt::BufferUsages::STORAGE), ); u.set( @@ -81,7 +85,7 @@ pub fn map_texture_usage( usage.contains(wgt::TextureUsages::TEXTURE_BINDING), ); u.set( - hal::TextureUses::STORAGE_READ | hal::TextureUses::STORAGE_WRITE, + hal::TextureUses::STORAGE_READ | hal::TextureUses::STORAGE_READ_WRITE, usage.contains(wgt::TextureUsages::STORAGE_BINDING), ); let is_color = aspect.contains(hal::FormatAspects::COLOR); @@ -148,7 +152,7 @@ pub fn check_texture_dimension_size( return Err(Tde::LimitExceeded { dim, given, limit }); } } - if sample_size == 0 || sample_size > sample_limit || !is_power_of_two(sample_size) { + if sample_size == 0 || sample_size > sample_limit || !is_power_of_two_u32(sample_size) { return Err(Tde::InvalidSampleCount(sample_size)); } diff --git a/wgpu-core/src/device/life.rs b/wgpu-core/src/device/life.rs index 9d0c8d96e3..8c6705276f 100644 --- a/wgpu-core/src/device/life.rs +++ b/wgpu-core/src/device/life.rs @@ -7,7 +7,7 @@ use crate::{ }, hub::{GlobalIdentityHandlerFactory, HalApi, Hub, Token}, id, resource, - track::TrackerSet, + track::{BindGroupStates, RenderBundleScope, Tracker}, RefCount, Stored, SubmissionIndex, }; use smallvec::SmallVec; @@ -68,17 +68,21 @@ impl SuspectedResources { self.query_sets.extend_from_slice(&other.query_sets); } - pub(super) fn add_trackers(&mut self, trackers: &TrackerSet) { + pub(super) fn add_render_bundle_scope(&mut self, trackers: &RenderBundleScope) { self.buffers.extend(trackers.buffers.used()); self.textures.extend(trackers.textures.used()); - self.texture_views.extend(trackers.views.used()); - self.samplers.extend(trackers.samplers.used()); self.bind_groups.extend(trackers.bind_groups.used()); - self.compute_pipelines.extend(trackers.compute_pipes.used()); - self.render_pipelines.extend(trackers.render_pipes.used()); - self.render_bundles.extend(trackers.bundles.used()); + self.render_pipelines + .extend(trackers.render_pipelines.used()); self.query_sets.extend(trackers.query_sets.used()); } + + pub(super) fn add_bind_group_states(&mut self, trackers: &BindGroupStates) { + self.buffers.extend(trackers.buffers.used()); + self.textures.extend(trackers.textures.used()); + self.texture_views.extend(trackers.views.used()); + self.samplers.extend(trackers.samplers.used()); + } } /// Raw backend resources that should be freed shortly. @@ -273,7 +277,8 @@ pub(super) struct LifetimeTracker { /// Textures can be used in the upcoming submission by `write_texture`. pub future_suspected_textures: Vec>, - /// Resources that are suspected for destruction. + /// Resources whose user handle has died (i.e. drop/destroy has been called) + /// and will likely be ready for destruction soon. pub suspected_resources: SuspectedResources, /// Resources used by queue submissions still in flight. One entry per @@ -491,7 +496,7 @@ impl LifetimeTracker { pub(super) fn triage_suspected( &mut self, hub: &Hub, - trackers: &Mutex, + trackers: &Mutex>, #[cfg(feature = "trace")] trace: Option<&Mutex>, token: &mut Token>, ) { @@ -510,7 +515,7 @@ impl LifetimeTracker { } if let Some(res) = hub.render_bundles.unregister_locked(id.0, &mut *guard) { - self.suspected_resources.add_trackers(&res.used); + self.suspected_resources.add_render_bundle_scope(&res.used); } } } @@ -529,7 +534,7 @@ impl LifetimeTracker { } if let Some(res) = hub.bind_groups.unregister_locked(id.0, &mut *guard) { - self.suspected_resources.add_trackers(&res.used); + self.suspected_resources.add_bind_group_states(&res.used); self.suspected_resources .bind_group_layouts @@ -670,7 +675,7 @@ impl LifetimeTracker { let mut trackers = trackers.lock(); for id in self.suspected_resources.compute_pipelines.drain(..) { - if trackers.compute_pipes.remove_abandoned(id) { + if trackers.compute_pipelines.remove_abandoned(id) { log::debug!("Compute pipeline {:?} will be destroyed", id); #[cfg(feature = "trace")] if let Some(t) = trace { @@ -695,7 +700,7 @@ impl LifetimeTracker { let mut trackers = trackers.lock(); for id in self.suspected_resources.render_pipelines.drain(..) { - if trackers.render_pipes.remove_abandoned(id) { + if trackers.render_pipelines.remove_abandoned(id) { log::debug!("Render pipeline {:?} will be destroyed", id); #[cfg(feature = "trace")] if let Some(t) = trace { @@ -829,7 +834,7 @@ impl LifetimeTracker { &mut self, hub: &Hub, raw: &A::Device, - trackers: &Mutex, + trackers: &Mutex>, token: &mut Token>, ) -> Vec { if self.ready_to_map.is_empty() { diff --git a/wgpu-core/src/device/mod.rs b/wgpu-core/src/device/mod.rs index 890e5a95bc..24fa72e571 100644 --- a/wgpu-core/src/device/mod.rs +++ b/wgpu-core/src/device/mod.rs @@ -8,7 +8,7 @@ use crate::{ TextureInitTracker, TextureInitTrackerAction, }, instance, pipeline, present, resource, - track::{BufferState, TextureSelector, TextureState, TrackerSet, UsageConflict}, + track::{BindGroupStates, TextureSelector, Tracker}, validation::{self, check_buffer_usage, check_texture_usage}, FastHashMap, Label, LabelHelpers as _, LifeGuard, MultiRefCount, RefCount, Stored, SubmissionIndex, DOWNLEVEL_ERROR_MESSAGE, @@ -22,7 +22,7 @@ use smallvec::SmallVec; use thiserror::Error; use wgt::{BufferAddress, TextureFormat, TextureViewDimension}; -use std::{borrow::Cow, iter, marker::PhantomData, mem, num::NonZeroU32, ops::Range, ptr}; +use std::{borrow::Cow, iter, mem, num::NonZeroU32, ops::Range, ptr}; mod life; pub mod queue; @@ -256,7 +256,7 @@ impl CommandAllocator { /// 1. `life_tracker` is locked after `hub.devices`, enforced by the type system /// 1. `self.trackers` is locked last (unenforced) /// 1. `self.trace` is locked last (unenforced) -pub struct Device { +pub struct Device { pub(crate) raw: A::Device, pub(crate) adapter_id: Stored, pub(crate) queue: A::Queue, @@ -281,7 +281,7 @@ pub struct Device { /// All live resources allocated with this [`Device`]. /// /// Has to be locked temporarily only (locked last) - pub(crate) trackers: Mutex, + pub(crate) trackers: Mutex>, // Life tracker should be locked right after the device and before anything else. life_tracker: Mutex>, /// Temporary storage for resource management functions. Cleared at the end @@ -306,7 +306,7 @@ pub enum CreateDeviceError { FailedToCreateZeroBuffer(#[from] DeviceError), } -impl Device { +impl Device { pub(crate) fn require_features(&self, feature: wgt::Features) -> Result<(), MissingFeatures> { if self.features.contains(feature) { Ok(()) @@ -328,7 +328,6 @@ impl Device { } impl Device { - #[allow(clippy::too_many_arguments)] pub(crate) fn new( open: hal::OpenDevice, adapter_id: Stored, @@ -394,7 +393,7 @@ impl Device { command_allocator: Mutex::new(com_alloc), active_submission_index: 0, fence, - trackers: Mutex::new(TrackerSet::new(A::VARIANT)), + trackers: Mutex::new(Tracker::new()), life_tracker: Mutex::new(life::LifetimeTracker::new()), temp_suspected: life::SuspectedResources::default(), #[cfg(feature = "trace")] @@ -495,7 +494,7 @@ impl Device { fn untrack<'this, 'token: 'this, G: GlobalIdentityHandlerFactory>( &'this mut self, hub: &Hub, - trackers: &TrackerSet, + trackers: &Tracker, token: &mut Token<'token, Self>, ) { self.temp_suspected.clear(); @@ -536,12 +535,12 @@ impl Device { self.temp_suspected.samplers.push(id); } } - for id in trackers.compute_pipes.used() { + for id in trackers.compute_pipelines.used() { if compute_pipe_guard[id].life_guard.ref_count.is_none() { self.temp_suspected.compute_pipelines.push(id); } } - for id in trackers.render_pipes.used() { + for id in trackers.render_pipelines.used() { if render_pipe_guard[id].life_guard.ref_count.is_none() { self.temp_suspected.render_pipelines.push(id); } @@ -655,7 +654,7 @@ impl Device { desc.array_layer_count(), ), full_range: TextureSelector { - levels: 0..desc.mip_level_count, + mips: 0..desc.mip_level_count, layers: 0..desc.array_layer_count(), }, life_guard: LifeGuard::new(desc.label.borrow_or_default()), @@ -876,7 +875,7 @@ impl Device { _ => texture.desc.array_layer_count(), }, }; - let level_end = texture.full_range.levels.end; + let level_end = texture.full_range.mips.end; let layer_end = texture.full_range.layers.end; if required_level_count > level_end { return Err(resource::CreateTextureViewError::TooManyMipLevels { @@ -927,7 +926,7 @@ impl Device { .array_layer_count .map_or(layer_end, |_| required_layer_count); let selector = TextureSelector { - levels: desc.range.base_mip_level..end_level, + mips: desc.range.base_mip_level..end_level, layers: desc.range.base_array_layer..end_layer, }; @@ -972,11 +971,11 @@ impl Device { wgt::TextureViewDimension::D3 => { hal::TextureUses::RESOURCE | hal::TextureUses::STORAGE_READ - | hal::TextureUses::STORAGE_WRITE + | hal::TextureUses::STORAGE_READ_WRITE } _ => hal::TextureUses::all(), }; - let mask_mip_level = if selector.levels.end - selector.levels.start != 1 { + let mask_mip_level = if selector.mips.end - selector.mips.start != 1 { hal::TextureUses::RESOURCE } else { hal::TextureUses::all() @@ -1058,7 +1057,8 @@ impl Device { let anisotropy_clamp = if let Some(clamp) = desc.anisotropy_clamp { let clamp = clamp.get(); - let valid_clamp = clamp <= hal::MAX_ANISOTROPY && conv::is_power_of_two(clamp as u32); + let valid_clamp = + clamp <= hal::MAX_ANISOTROPY && conv::is_power_of_two_u32(clamp as u32); if !valid_clamp { return Err(resource::CreateSamplerError::InvalidClamp(clamp)); } @@ -1486,7 +1486,6 @@ impl Device { }) } - #[allow(clippy::too_many_arguments)] fn create_buffer_binding<'a>( bb: &binding_model::BufferBinding, binding: u32, @@ -1494,7 +1493,7 @@ impl Device { used_buffer_ranges: &mut Vec, dynamic_binding_info: &mut Vec, late_buffer_binding_sizes: &mut FastHashMap, - used: &mut TrackerSet, + used: &mut BindGroupStates, storage: &'a Storage, id::BufferId>, limits: &wgt::Limits, ) -> Result, binding_model::CreateBindGroupError> { @@ -1525,7 +1524,7 @@ impl Device { if read_only { hal::BufferUses::STORAGE_READ } else { - hal::BufferUses::STORAGE_READ | hal::BufferUses::STORAGE_WRITE + hal::BufferUses::STORAGE_READ_WRITE }, limits.max_storage_buffer_binding_size, ), @@ -1543,8 +1542,8 @@ impl Device { let buffer = used .buffers - .use_extend(storage, bb.buffer_id, (), internal_use) - .map_err(|_| Error::InvalidBuffer(bb.buffer_id))?; + .add_single(storage, bb.buffer_id, internal_use) + .ok_or(Error::InvalidBuffer(bb.buffer_id))?; check_buffer_usage(buffer.usage, pub_usage)?; let raw_buffer = buffer .raw @@ -1613,26 +1612,26 @@ impl Device { fn create_texture_binding( view: &resource::TextureView, - texture_guard: &parking_lot::lock_api::RwLockReadGuard< - parking_lot::RawRwLock, - Storage, id::Id>>, - >, + texture_guard: &Storage, id::TextureId>, internal_use: hal::TextureUses, pub_usage: wgt::TextureUsages, - used: &mut TrackerSet, + used: &mut BindGroupStates, used_texture_ranges: &mut Vec, ) -> Result<(), binding_model::CreateBindGroupError> { // Careful here: the texture may no longer have its own ref count, // if it was deleted by the user. - let texture = &texture_guard[view.parent_id.value]; - used.textures - .change_extend( - view.parent_id.value, - &view.parent_id.ref_count, - view.selector.clone(), + let texture = used + .textures + .add_single( + texture_guard, + view.parent_id.value.0, + view.parent_id.ref_count.clone(), + Some(view.selector.clone()), internal_use, ) - .map_err(UsageConflict::from)?; + .ok_or(binding_model::CreateBindGroupError::InvalidTexture( + view.parent_id.value.0, + ))?; check_texture_usage(texture.desc.usage, pub_usage)?; used_texture_ranges.push(TextureInitTrackerAction { @@ -1674,7 +1673,7 @@ impl Device { // it needs to be in BGL iteration order, not BG entry order. let mut late_buffer_binding_sizes = FastHashMap::default(); // fill out the descriptors - let mut used = TrackerSet::new(A::VARIANT); + let mut used = BindGroupStates::new(); let (buffer_guard, mut token) = hub.buffers.read(token); let (texture_guard, mut token) = hub.textures.read(&mut token); //skip token @@ -1738,8 +1737,8 @@ impl Device { wgt::BindingType::Sampler(ty) => { let sampler = used .samplers - .use_extend(&*sampler_guard, id, (), ()) - .map_err(|_| Error::InvalidSampler(id))?; + .add_single(&*sampler_guard, id) + .ok_or(Error::InvalidSampler(id))?; // Allowed sampler values for filtering and comparison let (allowed_filtering, allowed_comparison) = match ty { @@ -1787,8 +1786,8 @@ impl Device { for &id in bindings_array.iter() { let sampler = used .samplers - .use_extend(&*sampler_guard, id, (), ()) - .map_err(|_| Error::InvalidSampler(id))?; + .add_single(&*sampler_guard, id) + .ok_or(Error::InvalidSampler(id))?; hal_samplers.push(&sampler.raw); } @@ -1797,8 +1796,8 @@ impl Device { Br::TextureView(id) => { let view = used .views - .use_extend(&*texture_view_guard, id, (), ()) - .map_err(|_| Error::InvalidTextureView(id))?; + .add_single(&*texture_view_guard, id) + .ok_or(Error::InvalidTextureView(id))?; let (pub_usage, internal_use) = Self::texture_use_parameters( binding, decl, @@ -1828,8 +1827,8 @@ impl Device { for &id in bindings_array.iter() { let view = used .views - .use_extend(&*texture_view_guard, id, (), ()) - .map_err(|_| Error::InvalidTextureView(id))?; + .add_single(&*texture_view_guard, id) + .ok_or(Error::InvalidTextureView(id))?; let (pub_usage, internal_use) = Self::texture_use_parameters(binding, decl, view, "SampledTextureArray, ReadonlyStorageTextureArray or WriteonlyStorageTextureArray")?; @@ -1858,6 +1857,8 @@ impl Device { }); } + used.optimize(); + hal_entries.sort_by_key(|entry| entry.binding); for (a, b) in hal_entries.iter().zip(hal_entries.iter().skip(1)) { if a.binding == b.binding { @@ -2018,7 +2019,7 @@ impl Device { }); } - let mip_level_count = view.selector.levels.end - view.selector.levels.start; + let mip_level_count = view.selector.mips.end - view.selector.mips.start; if mip_level_count != 1 { return Err(Error::InvalidStorageTextureMipLevelCount { binding, @@ -2027,7 +2028,7 @@ impl Device { } let internal_use = match access { - wgt::StorageTextureAccess::WriteOnly => hal::TextureUses::STORAGE_WRITE, + wgt::StorageTextureAccess::WriteOnly => hal::TextureUses::STORAGE_READ_WRITE, wgt::StorageTextureAccess::ReadOnly => { if !view .format_features @@ -2047,7 +2048,7 @@ impl Device { return Err(Error::StorageReadNotSupported(view.desc.format)); } - hal::TextureUses::STORAGE_WRITE | hal::TextureUses::STORAGE_READ + hal::TextureUses::STORAGE_READ_WRITE } }; Ok((wgt::TextureUsages::STORAGE_BINDING, internal_use)) @@ -2551,7 +2552,7 @@ impl Device { let samples = { let sc = desc.multisample.count; - if sc == 0 || sc > 32 || !conv::is_power_of_two(sc) { + if sc == 0 || sc > 32 || !conv::is_power_of_two_u32(sc) { return Err(pipeline::CreateRenderPipelineError::InvalidSampleCount(sc)); } sc @@ -2879,7 +2880,7 @@ impl Device { } } -impl Device { +impl Device { pub(crate) fn destroy_buffer(&self, buffer: resource::Buffer) { if let Some(raw) = buffer.raw { unsafe { @@ -2925,7 +2926,7 @@ impl Device { } } -impl crate::hub::Resource for Device { +impl crate::hub::Resource for Device { const TYPE: &'static str = "Device"; fn life_guard(&self) -> &LifeGuard { @@ -2981,7 +2982,7 @@ pub struct ImplicitPipelineIds<'a, G: GlobalIdentityHandlerFactory> { } impl ImplicitPipelineIds<'_, G> { - fn prepare(self, hub: &Hub) -> ImplicitPipelineContext { + fn prepare(self, hub: &Hub) -> ImplicitPipelineContext { ImplicitPipelineContext { root_id: hub.pipeline_layouts.prepare(self.root_id).into_id(), group_ids: self @@ -3184,8 +3185,8 @@ impl Global { .trackers .lock() .buffers - .init(id, ref_count, BufferState::with_usage(buffer_use)) - .unwrap(); + .insert_single(id, ref_count, buffer_use); + return (id.0, None); }; @@ -3483,19 +3484,17 @@ impl Global { Ok(texture) => texture, Err(error) => break error, }; - let num_levels = texture.full_range.levels.end; - let num_layers = texture.full_range.layers.end; let ref_count = texture.life_guard.add_ref(); let id = fid.assign(texture, &mut token); log::info!("Created texture {:?} with {:?}", id, desc); - device - .trackers - .lock() - .textures - .init(id, ref_count, TextureState::new(num_levels, num_layers)) - .unwrap(); + device.trackers.lock().textures.insert_single( + id.0, + ref_count, + hal::TextureUses::UNINITIALIZED, + ); + return (id.0, None); }; @@ -3561,19 +3560,17 @@ impl Global { texture.initialization_status = TextureInitTracker::new(desc.mip_level_count, 0); - let num_levels = texture.full_range.levels.end; - let num_layers = texture.full_range.layers.end; let ref_count = texture.life_guard.add_ref(); let id = fid.assign(texture, &mut token); log::info!("Created texture {:?} with {:?}", id, desc); - device - .trackers - .lock() - .textures - .init(id, ref_count, TextureState::new(num_levels, num_layers)) - .unwrap(); + device.trackers.lock().textures.insert_single( + id.0, + ref_count, + hal::TextureUses::UNINITIALIZED, + ); + return (id.0, None); }; @@ -3731,12 +3728,7 @@ impl Global { let ref_count = view.life_guard.add_ref(); let id = fid.assign(view, &mut token); - device - .trackers - .lock() - .views - .init(id, ref_count, PhantomData) - .unwrap(); + device.trackers.lock().views.insert_single(id, ref_count); return (id.0, None); }; @@ -3829,12 +3821,8 @@ impl Global { let ref_count = sampler.life_guard.add_ref(); let id = fid.assign(sampler, &mut token); - device - .trackers - .lock() - .samplers - .init(id, ref_count, PhantomData) - .unwrap(); + device.trackers.lock().samplers.insert_single(id, ref_count); + return (id.0, None); }; @@ -4092,18 +4080,13 @@ impl Global { let ref_count = bind_group.life_guard.add_ref(); let id = fid.assign(bind_group, &mut token); - log::debug!( - "Bind group {:?} {:#?}", - id, - hub.bind_groups.read(&mut token).0[id].used - ); + log::debug!("Bind group {:?}", id,); device .trackers .lock() .bind_groups - .init(id, ref_count, PhantomData) - .unwrap(); + .insert_single(id, ref_count); return (id.0, None); }; @@ -4406,16 +4389,11 @@ impl Global { Err(e) => break e, }; - log::debug!("Render bundle {:#?}", render_bundle.used); + log::debug!("Render bundle"); let ref_count = render_bundle.life_guard.add_ref(); let id = fid.assign(render_bundle, &mut token); - device - .trackers - .lock() - .bundles - .init(id, ref_count, PhantomData) - .unwrap(); + device.trackers.lock().bundles.insert_single(id, ref_count); return (id.0, None); }; @@ -4494,8 +4472,7 @@ impl Global { .trackers .lock() .query_sets - .init(id, ref_count, PhantomData) - .unwrap(); + .insert_single(id, ref_count); return (id.0, None); }; @@ -4589,9 +4566,9 @@ impl Global { device .trackers .lock() - .render_pipes - .init(id, ref_count, PhantomData) - .unwrap(); + .render_pipelines + .insert_single(id, ref_count); + return (id.0, None); }; @@ -4730,9 +4707,8 @@ impl Global { device .trackers .lock() - .compute_pipes - .init(id, ref_count, PhantomData) - .unwrap(); + .compute_pipelines + .insert_single(id, ref_count); return (id.0, None); }; @@ -5204,16 +5180,20 @@ impl Global { }; log::debug!("Buffer {:?} map state -> Waiting", buffer_id); - (buffer.device_id.value, buffer.life_guard.add_ref()) + let device = &device_guard[buffer.device_id.value]; + + let ret = (buffer.device_id.value, buffer.life_guard.add_ref()); + + let mut trackers = device.trackers.lock(); + trackers + .buffers + .set_single(&*buffer_guard, buffer_id, internal_use); + trackers.buffers.drain(); + + ret }; let device = &device_guard[device_id]; - device.trackers.lock().buffers.change_replace( - id::Valid(buffer_id), - &ref_count, - (), - internal_use, - ); device .lock_life(&mut token) diff --git a/wgpu-core/src/device/queue.rs b/wgpu-core/src/device/queue.rs index 572f49e5dc..e76255f772 100644 --- a/wgpu-core/src/device/queue.rs +++ b/wgpu-core/src/device/queue.rs @@ -192,7 +192,7 @@ impl PendingWrites { } } -impl super::Device { +impl super::Device { fn prepare_stage(&mut self, size: wgt::BufferAddress) -> Result, DeviceError> { profiling::scope!("prepare_stage"); let stage_desc = hal::BufferDescriptor { @@ -286,8 +286,8 @@ impl Global { let mut trackers = device.trackers.lock(); let (dst, transition) = trackers .buffers - .use_replace(&*buffer_guard, buffer_id, (), hal::BufferUses::COPY_DST) - .map_err(TransferError::InvalidBuffer)?; + .set_single(&*buffer_guard, buffer_id, hal::BufferUses::COPY_DST) + .ok_or(TransferError::InvalidBuffer(buffer_id))?; let dst_raw = dst .raw .as_ref() @@ -451,9 +451,9 @@ impl Global { .drain(init_layer_range) .collect::>>() { - crate::command::clear_texture_no_device( + crate::command::clear_texture( + &*texture_guard, id::Valid(destination.texture), - &*dst, TextureInitRange { mip_range: destination.mip_level..(destination.mip_level + 1), layer_range, @@ -473,13 +473,13 @@ impl Global { let (dst, transition) = trackers .textures - .use_replace( + .set_single( &*texture_guard, destination.texture, selector, hal::TextureUses::COPY_DST, ) - .unwrap(); + .ok_or(TransferError::InvalidTexture(destination.texture))?; let (hal_copy_size, array_layer_count) = validate_texture_copy_range(destination, &dst.desc, CopySide::Destination, size)?; @@ -561,7 +561,8 @@ impl Global { .ok_or(TransferError::InvalidTexture(destination.texture))?; unsafe { - encoder.transition_textures(transition.map(|pending| pending.into_hal(dst))); + encoder + .transition_textures(transition.map(|pending| pending.into_hal(dst)).into_iter()); encoder.transition_buffers(iter::once(barrier)); encoder.copy_buffer_to_texture(&stage.buffer, dst_raw, regions); } @@ -594,7 +595,7 @@ impl Global { device.active_submission_index += 1; let submit_index = device.active_submission_index; let mut active_executions = Vec::new(); - let mut used_surface_textures = track::ResourceTracker::new(A::VARIANT); + let mut used_surface_textures = track::TextureUsageScope::new(); { let (mut command_buffer_guard, mut token) = hub.command_buffers.write(&mut token); @@ -616,12 +617,15 @@ impl Global { //Note: locking the trackers has to be done after the storages let mut trackers = device.trackers.lock(); + used_surface_textures.set_size(texture_guard.len()); + //TODO: if multiple command buffers are submitted, we can re-use the last // native command buffer of the previous chain instead of always creating // a temporary one, since the chains are not finished. // finish all the command buffers first for &cmb_id in command_buffer_ids { + #[allow(unused_mut)] let mut cmdbuf = match hub .command_buffers .unregister_locked(cmb_id, &mut *command_buffer_guard) @@ -642,7 +646,7 @@ impl Global { } // optimize the tracked states - cmdbuf.trackers.optimize(); + // cmdbuf.trackers.optimize(); // update submission IDs for id in cmdbuf.trackers.buffers.used() { @@ -669,33 +673,35 @@ impl Global { } for id in cmdbuf.trackers.textures.used() { let texture = &mut texture_guard[id]; - match texture.inner { + let should_extend = match texture.inner { TextureInner::Native { raw: None } => { return Err(QueueSubmitError::DestroyedTexture(id.0)); } - TextureInner::Native { raw: Some(_) } => {} + TextureInner::Native { raw: Some(_) } => false, TextureInner::Surface { ref mut has_work, .. } => { - use track::ResourceState as _; - *has_work = true; - let ref_count = cmdbuf.trackers.textures.get_ref_count(id); - //TODO: better error handling here? - // register it in the temporary tracker. - let mut ts = track::TextureState::default(); - let _ = ts.change( - id, - texture.full_range.clone(), - hal::TextureUses::empty(), //present - None, - ); - let _ = used_surface_textures.init(id, ref_count.clone(), ts); + true } - } + }; if !texture.life_guard.use_at(submit_index) { device.temp_suspected.textures.push(id); } + if should_extend { + unsafe { + let ref_count = cmdbuf.trackers.textures.get_ref_count(id); + used_surface_textures + .merge_single( + &*texture_guard, + id, + None, + ref_count, + hal::TextureUses::PRESENT, + ) + .unwrap(); + }; + } } for id in cmdbuf.trackers.views.used() { if !texture_view_guard[id].life_guard.use_at(submit_index) { @@ -717,13 +723,13 @@ impl Global { sampler_guard[sub_id].life_guard.use_at(submit_index); } } - assert!(cmdbuf.trackers.samplers.is_empty()); - for id in cmdbuf.trackers.compute_pipes.used() { + // assert!(cmdbuf.trackers.samplers.is_empty()); + for id in cmdbuf.trackers.compute_pipelines.used() { if !compute_pipe_guard[id].life_guard.use_at(submit_index) { device.temp_suspected.compute_pipelines.push(id); } } - for id in cmdbuf.trackers.render_pipes.used() { + for id in cmdbuf.trackers.render_pipelines.used() { if !render_pipe_guard[id].life_guard.use_at(submit_index) { device.temp_suspected.render_pipelines.push(id); } @@ -741,12 +747,12 @@ impl Global { // We need to update the submission indices for the contained // state-less (!) resources as well, excluding the bind groups. // They don't get deleted too early if the bundle goes out of scope. - for sub_id in bundle.used.compute_pipes.used() { - compute_pipe_guard[sub_id].life_guard.use_at(submit_index); - } - for sub_id in bundle.used.render_pipes.used() { + for sub_id in bundle.used.render_pipelines.used() { render_pipe_guard[sub_id].life_guard.use_at(submit_index); } + for sub_id in bundle.used.query_sets.used() { + query_set_guard[sub_id].life_guard.use_at(submit_index); + } } let mut baked = cmdbuf.into_baked(); @@ -766,11 +772,10 @@ impl Global { .map_err(|err| QueueSubmitError::DestroyedTexture(err.0))?; //Note: stateless trackers are not merged: // device already knows these resources exist. - CommandBuffer::insert_barriers( + CommandBuffer::insert_barriers_from_tracker( &mut baked.encoder, &mut *trackers, - &baked.trackers.buffers, - &baked.trackers.textures, + &baked.trackers, &*buffer_guard, &*texture_guard, ); @@ -788,19 +793,19 @@ impl Global { .begin_encoding(Some("(wgpu internal) Present")) .map_err(DeviceError::from)? }; - let texture_barriers = trackers + trackers .textures - .merge_replace(&used_surface_textures) - .map(|pending| { - let tex = &texture_guard[pending.id]; - pending.into_hal(tex) - }); + .set_from_usage_scope(&*texture_guard, &used_surface_textures); + let texture_barriers = trackers.textures.drain().map(|pending| { + let tex = unsafe { texture_guard.get_unchecked(pending.id) }; + pending.into_hal(tex) + }); let present = unsafe { baked.encoder.transition_textures(texture_barriers); baked.encoder.end_encoding().unwrap() }; baked.list.push(present); - used_surface_textures.clear(); + used_surface_textures = track::TextureUsageScope::new(); } // done @@ -810,7 +815,7 @@ impl Global { }); } - log::trace!("Device after submission {}: {:#?}", submit_index, trackers); + log::trace!("Device after submission {}", submit_index); } let super::Device { @@ -830,6 +835,8 @@ impl Global { let (_, mut token) = hub.buffers.read(&mut token); // skip token let (mut texture_guard, _) = hub.textures.write(&mut token); + used_surface_textures.set_size(texture_guard.len()); + for &id in pending_writes.dst_textures.iter() { let texture = texture_guard.get_mut(id).unwrap(); match texture.inner { @@ -840,39 +847,39 @@ impl Global { TextureInner::Surface { ref mut has_work, .. } => { - use track::ResourceState as _; - *has_work = true; let ref_count = texture.life_guard.add_ref(); - //TODO: better error handling here? - // register it in the temporary tracker. - let mut ts = track::TextureState::default(); - let _ = ts.change( - id::Valid(id), - texture.full_range.clone(), - hal::TextureUses::empty(), //present - None, - ); - let _ = used_surface_textures.init(id::Valid(id), ref_count, ts); + unsafe { + used_surface_textures + .merge_single( + &*texture_guard, + id::Valid(id), + None, + &ref_count, + hal::TextureUses::PRESENT, + ) + .unwrap() + }; } } } if !used_surface_textures.is_empty() { let mut trackers = device.trackers.lock(); - let texture_barriers = trackers + + trackers .textures - .merge_replace(&used_surface_textures) - .map(|pending| { - let tex = &texture_guard[pending.id]; - pending.into_hal(tex) - }); + .set_from_usage_scope(&*texture_guard, &used_surface_textures); + let texture_barriers = trackers.textures.drain().map(|pending| { + let tex = unsafe { texture_guard.get_unchecked(pending.id) }; + pending.into_hal(tex) + }); + unsafe { pending_writes .command_encoder .transition_textures(texture_barriers); }; - used_surface_textures.clear(); } } diff --git a/wgpu-core/src/hub.rs b/wgpu-core/src/hub.rs index feef06e2e4..4e8ddf78f6 100644 --- a/wgpu-core/src/hub.rs +++ b/wgpu-core/src/hub.rs @@ -194,6 +194,14 @@ impl Storage { result } + pub(crate) unsafe fn get_unchecked(&self, id: u32) -> &T { + match self.map[id as usize] { + Element::Occupied(ref v, _) => v, + Element::Vacant => panic!("{}[{}] does not exist", self.kind, id), + Element::Error(_, _) => panic!(""), + } + } + pub(crate) fn label_for_invalid_id(&self, id: I) -> &str { let (index, _, _) = id.unzip(); match self.map.get(index as usize) { @@ -266,6 +274,10 @@ impl Storage { }) } + pub(crate) fn len(&self) -> usize { + self.map.len() + } + fn generate_report(&self) -> StorageReport { let mut report = StorageReport { element_size: mem::size_of::(), @@ -298,56 +310,56 @@ pub enum Root {} impl Access for Root {} impl Access for Root {} impl Access for Instance {} -impl Access> for Root {} -impl Access> for Surface {} -impl Access> for Root {} -impl Access> for Surface {} -impl Access> for Adapter {} -impl Access> for Root {} -impl Access> for Device {} -impl Access> for RenderBundle {} -impl Access> for Root {} -impl Access> for Device {} -impl Access> for PipelineLayout {} -impl Access> for Root {} -impl Access> for Device {} -impl Access> for BindGroupLayout {} -impl Access> for PipelineLayout {} -impl Access> for CommandBuffer {} -impl Access> for Root {} -impl Access> for Device {} -impl Access for Device {} -impl Access for CommandBuffer {} -impl Access> for Device {} -impl Access> for BindGroup {} -impl Access> for Device {} -impl Access> for BindGroup {} -impl Access> for ComputePipeline {} -impl Access> for Root {} -impl Access> for Device {} -impl Access> for CommandBuffer {} -impl Access> for RenderPipeline {} -impl Access> for ComputePipeline {} -impl Access> for Sampler {} -impl Access> for Device {} -impl Access> for BindGroupLayout {} -impl Access> for Root {} -impl Access> for Device {} -impl Access> for BindGroupLayout {} -impl Access> for BindGroup {} -impl Access> for CommandBuffer {} -impl Access> for ComputePipeline {} -impl Access> for RenderPipeline {} -impl Access> for QuerySet {} -impl Access> for Root {} -impl Access> for Device {} -impl Access> for Buffer {} -impl Access> for Root {} -impl Access> for Device {} -impl Access> for Texture {} -impl Access> for Root {} -impl Access> for Device {} -impl Access> for TextureView {} +impl Access> for Root {} +impl Access> for Surface {} +impl Access> for Root {} +impl Access> for Surface {} +impl Access> for Adapter {} +impl Access> for Root {} +impl Access> for Device {} +impl Access> for RenderBundle {} +impl Access> for Root {} +impl Access> for Device {} +impl Access> for PipelineLayout {} +impl Access> for Root {} +impl Access> for Device {} +impl Access> for BindGroupLayout {} +impl Access> for PipelineLayout {} +impl Access> for CommandBuffer {} +impl Access> for Root {} +impl Access> for Device {} +impl Access> for Device {} +impl Access> for CommandBuffer {} +impl Access> for Device {} +impl Access> for BindGroup {} +impl Access> for Device {} +impl Access> for BindGroup {} +impl Access> for ComputePipeline {} +impl Access> for Root {} +impl Access> for Device {} +impl Access> for CommandBuffer {} +impl Access> for RenderPipeline {} +impl Access> for ComputePipeline {} +impl Access> for Sampler {} +impl Access> for Device {} +impl Access> for BindGroupLayout {} +impl Access> for Root {} +impl Access> for Device {} +impl Access> for BindGroupLayout {} +impl Access> for BindGroup {} +impl Access> for CommandBuffer {} +impl Access> for ComputePipeline {} +impl Access> for RenderPipeline {} +impl Access> for QuerySet {} +impl Access> for Root {} +impl Access> for Device {} +impl Access> for Buffer {} +impl Access> for Root {} +impl Access> for Device {} +impl Access> for Texture {} +impl Access> for Root {} +impl Access> for Device {} +impl Access> for TextureView {} #[cfg(debug_assertions)] thread_local! { @@ -614,7 +626,7 @@ impl HubReport { } } -pub struct Hub { +pub struct Hub { pub adapters: Registry, id::AdapterId, F>, pub devices: Registry, id::DeviceId, F>, pub pipeline_layouts: Registry, id::PipelineLayoutId, F>, @@ -622,7 +634,7 @@ pub struct Hub { pub bind_group_layouts: Registry, id::BindGroupLayoutId, F>, pub bind_groups: Registry, id::BindGroupId, F>, pub command_buffers: Registry, id::CommandBufferId, F>, - pub render_bundles: Registry, + pub render_bundles: Registry, id::RenderBundleId, F>, pub render_pipelines: Registry, id::RenderPipelineId, F>, pub compute_pipelines: Registry, id::ComputePipelineId, F>, pub query_sets: Registry, id::QuerySetId, F>, @@ -1008,6 +1020,25 @@ pub trait HalApi: hal::Api { fn get_surface_mut(surface: &mut Surface) -> &mut HalSurface; } +impl HalApi for hal::api::Empty { + const VARIANT: Backend = Backend::Empty; + fn create_instance_from_hal(_: &str, _: Self::Instance) -> Instance { + unimplemented!("called empty api") + } + fn instance_as_hal(_: &Instance) -> Option<&Self::Instance> { + unimplemented!("called empty api") + } + fn hub(_: &Global) -> &Hub { + unimplemented!("called empty api") + } + fn get_surface(_: &Surface) -> &HalSurface { + unimplemented!("called empty api") + } + fn get_surface_mut(_: &mut Surface) -> &mut HalSurface { + unimplemented!("called empty api") + } +} + #[cfg(vulkan)] impl HalApi for hal::api::Vulkan { const VARIANT: Backend = Backend::Vulkan; diff --git a/wgpu-core/src/id.rs b/wgpu-core/src/id.rs index cbd7669e02..0955857235 100644 --- a/wgpu-core/src/id.rs +++ b/wgpu-core/src/id.rs @@ -64,9 +64,9 @@ impl From for Id { } impl Id { - #[cfg(test)] - pub(crate) fn dummy() -> Valid { - Valid(Id(NonZeroId::new(1).unwrap(), PhantomData)) + #[allow(dead_code)] + pub(crate) fn dummy(index: u32) -> Valid { + Valid(Id::zip(index, 1, Backend::Empty)) } pub fn backend(self) -> Backend { @@ -135,7 +135,7 @@ pub(crate) struct Valid(pub I); /// Most `wgpu-core` clients should not use this trait. Unusual clients that /// need to construct `Id` values directly, or access their components, like the /// WGPU recording player, may use this trait to do so. -pub trait TypedId { +pub trait TypedId: Copy { fn zip(index: Index, epoch: Epoch, backend: Backend) -> Self; fn unzip(self) -> (Index, Epoch, Backend); } @@ -184,7 +184,7 @@ pub type CommandBufferId = Id>; pub type RenderPassEncoderId = *mut crate::command::RenderPass; pub type ComputePassEncoderId = *mut crate::command::ComputePass; pub type RenderBundleEncoderId = *mut crate::command::RenderBundleEncoder; -pub type RenderBundleId = Id; +pub type RenderBundleId = Id>; pub type QuerySetId = Id>; #[test] diff --git a/wgpu-core/src/init_tracker/texture.rs b/wgpu-core/src/init_tracker/texture.rs index a2913ea3d4..87bfcdc887 100644 --- a/wgpu-core/src/init_tracker/texture.rs +++ b/wgpu-core/src/init_tracker/texture.rs @@ -26,7 +26,7 @@ pub(crate) fn has_copy_partial_init_tracker_coverage( impl From for TextureInitRange { fn from(selector: TextureSelector) -> Self { TextureInitRange { - mip_range: selector.levels, + mip_range: selector.mips, layer_range: selector.layers, } } diff --git a/wgpu-core/src/instance.rs b/wgpu-core/src/instance.rs index 315caa4544..907c65d08c 100644 --- a/wgpu-core/src/instance.rs +++ b/wgpu-core/src/instance.rs @@ -494,6 +494,9 @@ impl Global { } #[cfg(dx12)] + /// # Safety + /// + /// The visual must be valid and able to be used to make a swapchain with. pub unsafe fn instance_create_surface_from_visual( &self, visual: *mut std::ffi::c_void, diff --git a/wgpu-core/src/lib.rs b/wgpu-core/src/lib.rs index e3036ad0f2..8d28a3df26 100644 --- a/wgpu-core/src/lib.rs +++ b/wgpu-core/src/lib.rs @@ -4,6 +4,8 @@ */ #![allow( + // It is much clearer to assert negative conditions with eq! false + clippy::bool_assert_comparison, // We use loops for getting early-out of scope without closures. clippy::never_loop, // We don't use syntax sugar where it's not necessary. @@ -16,6 +18,8 @@ clippy::new_without_default, // Needless updates are more scaleable, easier to play with features. clippy::needless_update, + // Need many arguments for some core functions to be able to re-use code in many situations. + clippy::too_many_arguments, // For some reason `rustc` can warn about these in const generics even // though they are required. unused_braces, diff --git a/wgpu-core/src/present.rs b/wgpu-core/src/present.rs index 833e65d6be..fa7a34bf9d 100644 --- a/wgpu-core/src/present.rs +++ b/wgpu-core/src/present.rs @@ -174,7 +174,7 @@ impl Global { initialization_status: TextureInitTracker::new(1, 1), full_range: track::TextureSelector { layers: 0..1, - levels: 0..1, + mips: 0..1, }, life_guard: LifeGuard::new(""), clear_mode: resource::TextureClearMode::RenderPass { @@ -187,20 +187,13 @@ impl Global { let id = fid.assign(texture, &mut token); { - use track::ResourceState as _; // register it in the device tracker as uninitialized let mut trackers = device.trackers.lock(); - let mut ts = track::TextureState::default(); - let _ = ts.change( - id, - track::TextureSelector { - layers: 0..1, - levels: 0..1, - }, + trackers.textures.insert_single( + id.0, + ref_count.clone(), hal::TextureUses::UNINITIALIZED, - None, ); - let _ = trackers.textures.init(id, ref_count.clone(), ts); } if present.acquired_texture.is_some() { @@ -273,6 +266,10 @@ impl Global { // The texture ID got added to the device tracker by `submit()`, // and now we are moving it away. + log::debug!( + "Removing swapchain texture {:?} from the device tracker", + texture_id.value + ); device.trackers.lock().textures.remove(texture_id.value); let (texture, _) = hub.textures.unregister(texture_id.value.0, &mut token); diff --git a/wgpu-core/src/resource.rs b/wgpu-core/src/resource.rs index 1f0af1b388..a47e064f44 100644 --- a/wgpu-core/src/resource.rs +++ b/wgpu-core/src/resource.rs @@ -3,7 +3,7 @@ use crate::{ hub::{Global, GlobalIdentityHandlerFactory, HalApi, Resource, Token}, id::{AdapterId, DeviceId, SurfaceId, TextureId, Valid}, init_tracker::{BufferInitTracker, TextureInitTracker}, - track::{TextureSelector, DUMMY_SELECTOR}, + track::TextureSelector, validation::MissingBufferUsageError, Label, LifeGuard, RefCount, Stored, }; @@ -149,12 +149,6 @@ impl Resource for Buffer { } } -impl Borrow<()> for Buffer { - fn borrow(&self) -> &() { - &DUMMY_SELECTOR - } -} - pub type TextureDescriptor<'a> = wgt::TextureDescriptor>; #[derive(Debug)] @@ -448,12 +442,6 @@ impl Resource for TextureView { } } -impl Borrow<()> for TextureView { - fn borrow(&self) -> &() { - &DUMMY_SELECTOR - } -} - /// Describes a [`Sampler`] #[derive(Clone, Debug, PartialEq)] #[cfg_attr(feature = "trace", derive(serde::Serialize))] @@ -530,11 +518,6 @@ impl Resource for Sampler { } } -impl Borrow<()> for Sampler { - fn borrow(&self) -> &() { - &DUMMY_SELECTOR - } -} #[derive(Clone, Debug, Error)] pub enum CreateQuerySetError { #[error(transparent)] @@ -565,12 +548,6 @@ impl Resource for QuerySet { } } -impl Borrow<()> for QuerySet { - fn borrow(&self) -> &() { - &DUMMY_SELECTOR - } -} - #[derive(Clone, Debug, Error)] pub enum DestroyError { #[error("resource is invalid")] diff --git a/wgpu-core/src/track/buffer.rs b/wgpu-core/src/track/buffer.rs index 01e3a79798..7770b42308 100644 --- a/wgpu-core/src/track/buffer.rs +++ b/wgpu-core/src/track/buffer.rs @@ -1,236 +1,778 @@ -use super::{PendingTransition, ResourceState, Unit}; -use crate::id::{BufferId, Valid}; +/*! Buffer Trackers + * + * Buffers are represented by a single state for the whole resource, + * a 16 bit bitflag of buffer usages. Because there is only ever + * one subresource, they have no selector. +!*/ + +use std::{borrow::Cow, marker::PhantomData, vec::Drain}; + +use super::PendingTransition; +use crate::{ + hub, + id::{BufferId, TypedId, Valid}, + resource::Buffer, + track::{ + invalid_resource_state, iterate_bitvec_indices, skip_barrier, ResourceMetadata, + ResourceMetadataProvider, ResourceUses, UsageConflict, + }, + LifeGuard, RefCount, +}; use hal::BufferUses; -pub(crate) type BufferState = Unit; +impl ResourceUses for BufferUses { + const EXCLUSIVE: Self = Self::EXCLUSIVE; -impl PendingTransition { - fn collapse(self) -> Result { - if self.usage.start.is_empty() - || self.usage.start == self.usage.end - || !BufferUses::EXCLUSIVE.intersects(self.usage.start | self.usage.end) - { - Ok(self.usage.start | self.usage.end) - } else { - Err(self) - } + type Id = BufferId; + type Selector = (); + + fn bits(self) -> u16 { + Self::bits(&self) + } + + fn all_ordered(self) -> bool { + Self::ORDERED.contains(self) + } + + fn any_exclusive(self) -> bool { + self.intersects(Self::EXCLUSIVE) } } -impl Default for BufferState { - fn default() -> Self { +/// Stores all the buffers that a bind group stores. +pub(crate) struct BufferBindGroupState { + buffers: Vec<(Valid, RefCount, BufferUses)>, + + _phantom: PhantomData, +} +impl BufferBindGroupState { + pub fn new() -> Self { Self { - first: None, - last: BufferUses::empty(), + buffers: Vec::new(), + + _phantom: PhantomData, } } -} -impl BufferState { - pub fn with_usage(usage: BufferUses) -> Self { - Unit::new(usage) + /// Optimize the buffer bind group state by sorting it by ID. + /// + /// When this list of states is merged into a tracker, the memory + /// accesses will be in a constant assending order. + pub(crate) fn optimize(&mut self) { + self.buffers + .sort_unstable_by_key(|&(id, _, _)| id.0.unzip().0); + } + + /// Returns a list of all buffers tracked. May contain duplicates. + pub fn used(&self) -> impl Iterator> + '_ { + self.buffers.iter().map(|&(id, _, _)| id) + } + + /// Adds the given resource with the given state. + pub fn add_single<'a>( + &mut self, + storage: &'a hub::Storage, BufferId>, + id: BufferId, + state: BufferUses, + ) -> Option<&'a Buffer> { + let buffer = storage.get(id).ok()?; + + self.buffers + .push((Valid(id), buffer.life_guard.add_ref(), state)); + + Some(buffer) } } -impl ResourceState for BufferState { - type Id = BufferId; - type Selector = (); - type Usage = BufferUses; +/// Stores all buffer state within a single usage scope. +#[derive(Debug)] +pub(crate) struct BufferUsageScope { + state: Vec, + + metadata: ResourceMetadata, +} + +impl BufferUsageScope { + pub fn new() -> Self { + Self { + state: Vec::new(), + + metadata: ResourceMetadata::new(), + } + } + + fn debug_assert_in_bounds(&self, index: usize) { + debug_assert!(index < self.state.len()); + self.metadata.debug_assert_in_bounds(index); + } + + /// Sets the size of all the vectors inside the tracker. + /// + /// Must be called with the highest possible Buffer ID before + /// all unsafe functions are called. + pub fn set_size(&mut self, size: usize) { + self.state.resize(size, BufferUses::empty()); + self.metadata.set_size(size); + } + + /// Extend the vectors to let the given index be valid. + fn allow_index(&mut self, index: usize) { + if index >= self.state.len() { + self.set_size(index + 1); + } + } - fn query(&self, _selector: Self::Selector) -> Option { - Some(self.last) + /// Returns a list of all buffers tracked. + pub fn used(&self) -> impl Iterator> + '_ { + self.metadata.used() } - fn change( + /// Merge the list of buffer states in the given bind group into this usage scope. + /// + /// If any of the resulting states is invalid, stops the merge and returns a usage + /// conflict with the details of the invalid state. + /// + /// Because bind groups do not check if the union of all their states is valid, + /// this method is allowed to return Err on the first bind group bound. + /// + /// # Safety + /// + /// [`Self::set_size`] must be called with the maximum possible Buffer ID before this + /// method is called. + pub unsafe fn merge_bind_group( &mut self, - id: Valid, - _selector: Self::Selector, - usage: Self::Usage, - output: Option<&mut Vec>>, - ) -> Result<(), PendingTransition> { - let old = self.last; - if old != usage || !BufferUses::ORDERED.contains(usage) { - let pending = PendingTransition { - id, - selector: (), - usage: old..usage, - }; - *self = match output { - None => { - assert_eq!( - self.first, None, - "extending a state that is already a transition" - ); - Unit::new(pending.collapse()?) - } - Some(transitions) => { - transitions.push(pending); - Unit { - first: self.first.or(Some(old)), - last: usage, - } - } - }; + bind_group: &BufferBindGroupState, + ) -> Result<(), UsageConflict> { + for &(id, ref ref_count, state) in &bind_group.buffers { + let (index32, epoch, _) = id.0.unzip(); + let index = index32 as usize; + + insert_or_merge( + None, + None, + &mut self.state, + &mut self.metadata, + index32, + index, + BufferStateProvider::Direct { state }, + ResourceMetadataProvider::Direct { + epoch, + ref_count: Cow::Borrowed(ref_count), + }, + )?; } + Ok(()) } - fn merge( - &mut self, - id: Valid, - other: &Self, - output: Option<&mut Vec>>, - ) -> Result<(), PendingTransition> { - let old = self.last; - let new = other.port(); - if old == new && BufferUses::ORDERED.contains(new) { - if output.is_some() && self.first.is_none() { - *self = Unit { - first: Some(old), - last: other.last, - }; - } - } else { - let pending = PendingTransition { - id, - selector: (), - usage: old..new, - }; - *self = match output { - None => { - assert_eq!( - self.first, None, - "extending a state that is already a transition" - ); - Unit::new(pending.collapse()?) - } - Some(transitions) => { - transitions.push(pending); - Unit { - first: self.first.or(Some(old)), - last: other.last, - } - } + /// Merge the list of buffer states in the given usage scope into this UsageScope. + /// + /// If any of the resulting states is invalid, stops the merge and returns a usage + /// conflict with the details of the invalid state. + /// + /// If the given tracker uses IDs higher than the length of internal vectors, + /// the vectors will be extended. A call to set_size is not needed. + pub fn merge_usage_scope(&mut self, scope: &Self) -> Result<(), UsageConflict> { + let incoming_size = scope.state.len(); + if incoming_size > self.state.len() { + self.set_size(incoming_size); + } + + for index in iterate_bitvec_indices(&scope.metadata.owned) { + self.debug_assert_in_bounds(index); + scope.debug_assert_in_bounds(index); + + unsafe { + insert_or_merge( + None, + None, + &mut self.state, + &mut self.metadata, + index as u32, + index, + BufferStateProvider::Indirect { + state: &scope.state, + }, + ResourceMetadataProvider::Indirect { + metadata: &scope.metadata, + }, + )?; }; } + Ok(()) } - fn optimize(&mut self) {} + /// Merge a single state into the UsageScope. + /// + /// If the resulting state is invalid, returns a usage + /// conflict with the details of the invalid state. + /// + /// If the ID is higher than the length of internal vectors, + /// the vectors will be extended. A call to set_size is not needed. + pub fn merge_single<'a>( + &mut self, + storage: &'a hub::Storage, BufferId>, + id: BufferId, + new_state: BufferUses, + ) -> Result<&'a Buffer, UsageConflict> { + let buffer = storage + .get(id) + .map_err(|_| UsageConflict::BufferInvalid { id })?; + + let (index32, epoch, _) = id.unzip(); + let index = index32 as usize; + + self.allow_index(index); + + self.debug_assert_in_bounds(index); + + unsafe { + insert_or_merge( + Some(&buffer.life_guard), + None, + &mut self.state, + &mut self.metadata, + index32, + index, + BufferStateProvider::Direct { state: new_state }, + ResourceMetadataProvider::Resource { epoch }, + )?; + } + + Ok(buffer) + } } -#[cfg(test)] -mod test { - use super::*; - use crate::id::Id; +/// Stores all buffer state within a command buffer or device. +pub(crate) struct BufferTracker { + start: Vec, + end: Vec, - #[test] - fn change_extend() { - let mut bs = Unit { - first: None, - last: BufferUses::INDEX, - }; - let id = Id::dummy(); - assert_eq!( - bs.change(id, (), BufferUses::STORAGE_WRITE, None), - Err(PendingTransition { - id, - selector: (), - usage: BufferUses::INDEX..BufferUses::STORAGE_WRITE, - }), - ); - bs.change(id, (), BufferUses::VERTEX, None).unwrap(); - bs.change(id, (), BufferUses::INDEX, None).unwrap(); - assert_eq!(bs, Unit::new(BufferUses::VERTEX | BufferUses::INDEX)); + metadata: ResourceMetadata, + + temp: Vec>, +} +impl BufferTracker { + pub fn new() -> Self { + Self { + start: Vec::new(), + end: Vec::new(), + + metadata: ResourceMetadata::new(), + + temp: Vec::new(), + } + } + + fn debug_assert_in_bounds(&self, index: usize) { + debug_assert!(index < self.start.len()); + debug_assert!(index < self.end.len()); + self.metadata.debug_assert_in_bounds(index); + } + + /// Sets the size of all the vectors inside the tracker. + /// + /// Must be called with the highest possible Buffer ID before + /// all unsafe functions are called. + pub fn set_size(&mut self, size: usize) { + self.start.resize(size, BufferUses::empty()); + self.end.resize(size, BufferUses::empty()); + + self.metadata.set_size(size); + } + + /// Extend the vectors to let the given index be valid. + fn allow_index(&mut self, index: usize) { + if index >= self.start.len() { + self.set_size(index + 1); + } + } + + /// Returns a list of all buffers tracked. + pub fn used(&self) -> impl Iterator> + '_ { + self.metadata.used() + } + + /// Drains all currently pending transitions. + pub fn drain(&mut self) -> Drain<'_, PendingTransition> { + self.temp.drain(..) + } + + /// Inserts a single buffer and its state into the resource tracker. + /// + /// If the resource already exists in the tracker, this will panic. + /// + /// If the ID is higher than the length of internal vectors, + /// the vectors will be extended. A call to set_size is not needed. + pub fn insert_single(&mut self, id: Valid, ref_count: RefCount, state: BufferUses) { + let (index32, epoch, _) = id.0.unzip(); + let index = index32 as usize; + + self.allow_index(index); + + self.debug_assert_in_bounds(index); + + unsafe { + let currently_owned = self.metadata.owned.get(index).unwrap_unchecked(); + + if currently_owned { + panic!("Tried to insert buffer already tracked"); + } + + insert( + None, + Some(&mut self.start), + &mut self.end, + &mut self.metadata, + index, + BufferStateProvider::Direct { state }, + None, + ResourceMetadataProvider::Direct { + epoch, + ref_count: Cow::Owned(ref_count), + }, + ) + } } - #[test] - fn change_replace() { - let mut bs = Unit { - first: None, - last: BufferUses::STORAGE_WRITE, + /// Sets the state of a single buffer. + /// + /// If a transition is needed to get the buffer into the given state, that transition + /// is returned. No more than one transition is needed. + /// + /// If the ID is higher than the length of internal vectors, + /// the vectors will be extended. A call to set_size is not needed. + pub fn set_single<'a>( + &mut self, + storage: &'a hub::Storage, BufferId>, + id: BufferId, + state: BufferUses, + ) -> Option<(&'a Buffer, Option>)> { + let value = storage.get(id).ok()?; + + let (index32, epoch, _) = id.unzip(); + let index = index32 as usize; + + self.allow_index(index); + + self.debug_assert_in_bounds(index); + + unsafe { + insert_or_barrier_update( + Some(&value.life_guard), + Some(&mut self.start), + &mut self.end, + &mut self.metadata, + index32, + index, + BufferStateProvider::Direct { state }, + None, + ResourceMetadataProvider::Resource { epoch }, + &mut self.temp, + ) }; - let id = Id::dummy(); - let mut list = Vec::new(); - bs.change(id, (), BufferUses::VERTEX, Some(&mut list)) - .unwrap(); - assert_eq!( - &list, - &[PendingTransition { - id, - selector: (), - usage: BufferUses::STORAGE_WRITE..BufferUses::VERTEX, - }], - ); - assert_eq!( - bs, - Unit { - first: Some(BufferUses::STORAGE_WRITE), - last: BufferUses::VERTEX, + + debug_assert!(self.temp.len() <= 1); + + Some((value, self.temp.pop())) + } + + /// Sets the given state for all buffers in the given tracker. + /// + /// If a transition is needed to get the buffers into the needed state, + /// those transitions are stored within the tracker. A subsequent + /// call to [`Self::drain`] is needed to get those transitions. + /// + /// If the ID is higher than the length of internal vectors, + /// the vectors will be extended. A call to set_size is not needed. + pub fn set_from_tracker(&mut self, tracker: &Self) { + let incoming_size = tracker.start.len(); + if incoming_size > self.start.len() { + self.set_size(incoming_size); + } + + for index in iterate_bitvec_indices(&tracker.metadata.owned) { + self.debug_assert_in_bounds(index); + tracker.debug_assert_in_bounds(index); + unsafe { + insert_or_barrier_update( + None, + Some(&mut self.start), + &mut self.end, + &mut self.metadata, + index as u32, + index, + BufferStateProvider::Indirect { + state: &tracker.start, + }, + Some(BufferStateProvider::Indirect { + state: &tracker.end, + }), + ResourceMetadataProvider::Indirect { + metadata: &tracker.metadata, + }, + &mut self.temp, + ) } - ); + } + } - list.clear(); - bs.change(id, (), BufferUses::STORAGE_WRITE, Some(&mut list)) - .unwrap(); - assert_eq!( - &list, - &[PendingTransition { - id, - selector: (), - usage: BufferUses::VERTEX..BufferUses::STORAGE_WRITE, - }], - ); - assert_eq!( - bs, - Unit { - first: Some(BufferUses::STORAGE_WRITE), - last: BufferUses::STORAGE_WRITE, + /// Sets the given state for all buffers in the given UsageScope. + /// + /// If a transition is needed to get the buffers into the needed state, + /// those transitions are stored within the tracker. A subsequent + /// call to [`Self::drain`] is needed to get those transitions. + /// + /// If the ID is higher than the length of internal vectors, + /// the vectors will be extended. A call to set_size is not needed. + pub fn set_from_usage_scope(&mut self, scope: &BufferUsageScope) { + let incoming_size = scope.state.len(); + if incoming_size > self.start.len() { + self.set_size(incoming_size); + } + + for index in iterate_bitvec_indices(&scope.metadata.owned) { + self.debug_assert_in_bounds(index); + scope.debug_assert_in_bounds(index); + unsafe { + insert_or_barrier_update( + None, + Some(&mut self.start), + &mut self.end, + &mut self.metadata, + index as u32, + index, + BufferStateProvider::Indirect { + state: &scope.state, + }, + None, + ResourceMetadataProvider::Indirect { + metadata: &scope.metadata, + }, + &mut self.temp, + ) } - ); + } } - #[test] - fn merge_replace() { - let mut bs = Unit { - first: None, - last: BufferUses::empty(), - }; - let other_smooth = Unit { - first: Some(BufferUses::empty()), - last: BufferUses::COPY_DST, - }; - let id = Id::dummy(); - let mut list = Vec::new(); - bs.merge(id, &other_smooth, Some(&mut list)).unwrap(); - assert!(list.is_empty()); - assert_eq!( - bs, - Unit { - first: Some(BufferUses::empty()), - last: BufferUses::COPY_DST, + /// Iterates through all buffers in the given bind group and adopts + /// the state given for those buffers in the UsageScope. It also + /// removes all touched buffers from the usage scope. + /// + /// If a transition is needed to get the buffers into the needed state, + /// those transitions are stored within the tracker. A subsequent + /// call to [`Self::drain`] is needed to get those transitions. + /// + /// This is a really funky method used by Compute Passes to generate + /// barriers after a call to dispatch without needing to iterate + /// over all elements in the usage scope. We use each the + /// bind group as a source of which IDs to look at. The bind groups + /// must have first been added to the usage scope. + /// + /// # Safety + /// + /// [`Self::set_size`] must be called with the maximum possible Buffer ID before this + /// method is called. + pub unsafe fn set_and_remove_from_usage_scope_sparse( + &mut self, + scope: &mut BufferUsageScope, + bind_group_state: &BufferBindGroupState, + ) { + let incoming_size = scope.state.len(); + if incoming_size > self.start.len() { + self.set_size(incoming_size); + } + + for &(id, ref ref_count, _) in bind_group_state.buffers.iter() { + let (index32, epoch, _) = id.0.unzip(); + let index = index32 as usize; + + scope.debug_assert_in_bounds(index); + + if !scope.metadata.owned.get(index).unwrap_unchecked() { + continue; } - ); + insert_or_barrier_update( + None, + Some(&mut self.start), + &mut self.end, + &mut self.metadata, + index as u32, + index, + BufferStateProvider::Indirect { + state: &scope.state, + }, + None, + ResourceMetadataProvider::Direct { + epoch, + ref_count: Cow::Borrowed(ref_count), + }, + &mut self.temp, + ); - let other_rough = Unit { - first: Some(BufferUses::empty()), - last: BufferUses::UNIFORM, - }; - bs.merge(id, &other_rough, Some(&mut list)).unwrap(); - assert_eq!( - &list, - &[PendingTransition { - id, - selector: (), - usage: BufferUses::COPY_DST..BufferUses::empty(), - }], - ); - assert_eq!( - bs, - Unit { - first: Some(BufferUses::empty()), - last: BufferUses::UNIFORM, + scope.metadata.reset(index); + } + } + + /// Removes the given resource from the tracker iff we have the last reference to the + /// resource and the epoch matches. + /// + /// Returns true if the resource was removed. + /// + /// If the ID is higher than the length of internal vectors, + /// false will be returned. + pub fn remove_abandoned(&mut self, id: Valid) -> bool { + let (index32, epoch, _) = id.0.unzip(); + let index = index32 as usize; + + if index > self.metadata.owned.len() { + return false; + } + + self.debug_assert_in_bounds(index); + + unsafe { + if self.metadata.owned.get(index).unwrap_unchecked() { + let existing_epoch = self.metadata.epochs.get_unchecked_mut(index); + let existing_ref_count = self.metadata.ref_counts.get_unchecked_mut(index); + + if *existing_epoch == epoch + && existing_ref_count.as_mut().unwrap_unchecked().load() == 1 + { + self.metadata.reset(index); + + return true; + } } + } + + false + } +} + +/// Source of Buffer State. +#[derive(Debug, Clone)] +enum BufferStateProvider<'a> { + /// Get a state that was provided directly. + Direct { state: BufferUses }, + /// Get a state from an an array of states. + Indirect { state: &'a [BufferUses] }, +} +impl BufferStateProvider<'_> { + /// Gets the state from the provider, given a resource ID index. + /// + /// # Safety + /// + /// Index must be in bounds for the indirect source iff this is in the indirect state. + #[inline(always)] + unsafe fn get_state(&self, index: usize) -> BufferUses { + match *self { + BufferStateProvider::Direct { state } => state, + BufferStateProvider::Indirect { state } => { + debug_assert!(index < state.len()); + *state.get_unchecked(index) + } + } + } +} + +/// Does an insertion operation if the index isn't tracked +/// in the current metadata, otherwise merges the given state +/// with the current state. If the merging would cause +/// a conflict, returns that usage conflict. +/// +/// # Safety +/// +/// Indexes must be valid indexes into all arrays passed in +/// to this function, either directly or via metadata or provider structs. +#[inline(always)] +unsafe fn insert_or_merge( + life_guard: Option<&LifeGuard>, + start_states: Option<&mut [BufferUses]>, + current_states: &mut [BufferUses], + resource_metadata: &mut ResourceMetadata, + index32: u32, + index: usize, + state_provider: BufferStateProvider<'_>, + metadata_provider: ResourceMetadataProvider<'_, A>, +) -> Result<(), UsageConflict> { + let currently_owned = resource_metadata.owned.get(index).unwrap_unchecked(); + + if !currently_owned { + insert( + life_guard, + start_states, + current_states, + resource_metadata, + index, + state_provider, + None, + metadata_provider, + ); + return Ok(()); + } + + merge( + current_states, + index32, + index, + state_provider, + metadata_provider, + ) +} + +/// If the resource isn't tracked +/// - Inserts the given resource. +/// - Uses the `start_state_provider` to populate `start_states` +/// - Uses either `end_state_provider` or `start_state_provider` +/// to populate `current_states`. +/// If the resource is tracked +/// - Inserts barriers from the state in `current_states` +/// to the state provided by `start_state_provider`. +/// - Updates the `current_states` with either the state from +/// `end_state_provider` or `start_state_provider`. +/// +/// Any barriers are added to the barrier vector. +/// +/// # Safety +/// +/// Indexes must be valid indexes into all arrays passed in +/// to this function, either directly or via metadata or provider structs. +#[inline(always)] +unsafe fn insert_or_barrier_update( + life_guard: Option<&LifeGuard>, + start_states: Option<&mut [BufferUses]>, + current_states: &mut [BufferUses], + resource_metadata: &mut ResourceMetadata, + index32: u32, + index: usize, + start_state_provider: BufferStateProvider<'_>, + end_state_provider: Option>, + metadata_provider: ResourceMetadataProvider<'_, A>, + barriers: &mut Vec>, +) { + let currently_owned = resource_metadata.owned.get(index).unwrap_unchecked(); + + if !currently_owned { + insert( + life_guard, + start_states, + current_states, + resource_metadata, + index, + start_state_provider, + end_state_provider, + metadata_provider, ); + return; } + + let update_state_provider = end_state_provider.unwrap_or_else(|| start_state_provider.clone()); + barrier( + current_states, + index32, + index, + start_state_provider, + barriers, + ); + + update(current_states, index, update_state_provider); +} + +#[inline(always)] +unsafe fn insert( + life_guard: Option<&LifeGuard>, + start_states: Option<&mut [BufferUses]>, + current_states: &mut [BufferUses], + resource_metadata: &mut ResourceMetadata, + index: usize, + start_state_provider: BufferStateProvider<'_>, + end_state_provider: Option>, + metadata_provider: ResourceMetadataProvider<'_, A>, +) { + let new_start_state = start_state_provider.get_state(index); + let new_end_state = end_state_provider.map_or(new_start_state, |p| p.get_state(index)); + + // This should only ever happen with a wgpu bug, but let's just double + // check that resource states don't have any conflicts. + debug_assert_eq!(invalid_resource_state(new_start_state), false); + debug_assert_eq!(invalid_resource_state(new_end_state), false); + + log::trace!("\tbuf {index}: insert {new_start_state:?}..{new_end_state:?}"); + + if let Some(&mut ref mut start_state) = start_states { + *start_state.get_unchecked_mut(index) = new_start_state; + } + *current_states.get_unchecked_mut(index) = new_end_state; + + let (epoch, ref_count) = metadata_provider.get_own(life_guard, index); + + resource_metadata.owned.set(index, true); + *resource_metadata.epochs.get_unchecked_mut(index) = epoch; + *resource_metadata.ref_counts.get_unchecked_mut(index) = Some(ref_count); +} + +#[inline(always)] +unsafe fn merge( + current_states: &mut [BufferUses], + index32: u32, + index: usize, + state_provider: BufferStateProvider<'_>, + metadata_provider: ResourceMetadataProvider<'_, A>, +) -> Result<(), UsageConflict> { + let current_state = current_states.get_unchecked_mut(index); + let new_state = state_provider.get_state(index); + + let merged_state = *current_state | new_state; + + if invalid_resource_state(merged_state) { + return Err(UsageConflict::from_buffer( + BufferId::zip(index32, metadata_provider.get_epoch(index), A::VARIANT), + *current_state, + new_state, + )); + } + + log::trace!("\tbuf {index32}: merge {current_state:?} + {new_state:?}"); + + *current_state = merged_state; + + Ok(()) +} + +#[inline(always)] +unsafe fn barrier( + current_states: &mut [BufferUses], + index32: u32, + index: usize, + state_provider: BufferStateProvider<'_>, + barriers: &mut Vec>, +) { + let current_state = *current_states.get_unchecked(index); + let new_state = state_provider.get_state(index); + + if skip_barrier(current_state, new_state) { + return; + } + + barriers.push(PendingTransition { + id: index32, + selector: (), + usage: current_state..new_state, + }); + + log::trace!("\tbuf {index32}: transition {current_state:?} -> {new_state:?}"); +} + +#[inline(always)] +unsafe fn update( + current_states: &mut [BufferUses], + index: usize, + state_provider: BufferStateProvider<'_>, +) { + let current_state = current_states.get_unchecked_mut(index); + let new_state = state_provider.get_state(index); + + *current_state = new_state; } diff --git a/wgpu-core/src/track/mod.rs b/wgpu-core/src/track/mod.rs index bcf5eb4a4b..8d38ff8e72 100644 --- a/wgpu-core/src/track/mod.rs +++ b/wgpu-core/src/track/mod.rs @@ -1,126 +1,131 @@ +/*! Resource State and Lifetime Trackers + * + * These structures are responsible for keeping track of resource state, + * generating barriers where needed, and making sure resources are kept + * alive until the trackers die. + * + * ## General Architecture + * + * Tracking is some of the hottest code in the entire codebase, so the trackers + * are designed to be as cache efficient as possible. They store resource state + * in flat vectors, storing metadata SOA style, one vector per type of metadata. + * + * A lot of the tracker code is deeply unsafe, using unchecked accesses all over + * to make performance as good as possible. However, for all unsafe accesses, there + * is a corresponding debug assert the checks if that access is valid. This helps + * get bugs caught fast, while still letting users not need to pay for the bounds + * checks. + * + * In wgpu, resource IDs are allocated and re-used, so will always be as low + * as reasonably possible. This allows us to use the ID as an index into an array. + * + * ## Statefulness + * + * There are two main types of trackers, stateful and stateless. + * + * Stateful trackers are for buffers and textures. They both have + * resource state attached to them which needs to be used to generate + * automatic synchronization. Because of the different requirements of + * buffers and textures, they have two separate tracking structures. + * + * Stateless trackers only store metadata and own the given resource. + * + * ## Use Case + * + * Within each type of tracker, the trackers are further split into 3 different + * use cases, Bind Group, Usage Scope, and a full Tracker. + * + * Bind Group trackers are just a list of different resources, their refcount, + * and how they are used. Textures are used via a selector and a usage type. + * Buffers by just a usage type. Stateless resources don't have a usage type. + * + * Usage Scope trackers are only for stateful resources. These trackers represent + * a single [`UsageScope`] in the spec. When a use is added to a usage scope, + * it is merged with all other uses of that resource in that scope. If there + * is a usage conflict, merging will fail and an error will be reported. + * + * Full trackers represent a before and after state of a resource. These + * are used for tracking on the device and on command buffers. The before + * state represents the state the resource is first used as in the command buffer, + * the after state is the state the command buffer leaves the resource in. + * These double ended buffers can then be used to generate the needed transitions + * between command buffers. + * + * ## Dense Datastructure with Sparse Data + * + * This tracking system is based on having completely dense data, but trackers do + * not always contain every resource. Some resources (or even most resources) go + * unused in any given command buffer. So to help speed up the process of iterating + * through possibly thousands of resources, we use a bit vector to represent if + * a resource is in the buffer or not. This allows us extremely efficient memory + * utilization, as well as being able to bail out of whole blocks of 32-64 resources + * with a single usize comparison with zero. In practice this means that merging + * partially resident buffers is extremely quick. + * + * The main advantage of this dense datastructure is that we can do merging + * of trackers in an extremely efficient fashion that results in us doing linear + * scans down a couple of buffers. CPUs and their caches absolutely eat this up. + * + * ## Stateful Resource Operations + * + * All operations on stateful trackers boil down to one of four operations: + * - `insert(tracker, new_state)` adds a resource with a given state to the tracker + * for the first time. + * - `merge(tracker, new_state)` merges this new state with the previous state, checking + * for usage conflicts. + * - `barrier(tracker, new_state)` compares the given state to the existing state and + * generates the needed barriers. + * - `update(tracker, new_state)` takes the given new state and overrides the old state. + * + * This allows us to compose the operations to form the various kinds of tracker merges + * that need to happen in the codebase. For each resource in the given merger, the following + * operation applies: + * + * UsageScope <- Resource = insert(scope, usage) OR merge(scope, usage) + * UsageScope <- UsageScope = insert(scope, scope) OR merge(scope, scope) + * CommandBuffer <- UsageScope = insert(buffer.start, buffer.end, scope) OR barrier(buffer.end, scope) + update(buffer.end, scope) + * Deivce <- CommandBuffer = insert(device.start, device.end, buffer.start, buffer.end) OR barrier(device.end, buffer.start) + update(device.end, buffer.end) + * + * [`UsageScope`]: https://gpuweb.github.io/gpuweb/#programming-model-synchronization +!*/ + mod buffer; mod range; +mod stateless; mod texture; use crate::{ - hub, - id::{self, TypedId, Valid}, - resource, Epoch, FastHashMap, Index, RefCount, + binding_model, command, conv, hub, + id::{self, TypedId}, + pipeline, resource, Epoch, LifeGuard, RefCount, }; -use std::{ - collections::hash_map::Entry, fmt, marker::PhantomData, num::NonZeroU32, ops, vec::Drain, -}; +use bit_vec::BitVec; +use std::{borrow::Cow, fmt, marker::PhantomData, mem, num::NonZeroU32, ops}; use thiserror::Error; -pub(crate) use buffer::BufferState; -pub(crate) use texture::{TextureSelector, TextureState}; - -/// A single unit of state tracking. It keeps an initial -/// usage as well as the last/current one, similar to `Range`. -#[derive(Clone, Copy, Debug, PartialEq)] -pub struct Unit { - first: Option, - last: U, -} - -impl Unit { - /// Create a new unit from a given usage. - fn new(usage: U) -> Self { - Self { - first: None, - last: usage, - } - } - - /// Return a usage to link to. - fn port(&self) -> U { - self.first.unwrap_or(self.last) - } -} - -/// The main trait that abstracts away the tracking logic of -/// a particular resource type, like a buffer or a texture. -pub(crate) trait ResourceState: Clone + Default { - /// Corresponding `HUB` identifier. - type Id: Copy + fmt::Debug + TypedId; - /// A type specifying the sub-resources. - type Selector: fmt::Debug; - /// Usage type for a `Unit` of a sub-resource. - type Usage: fmt::Debug; - - /// Check if all the selected sub-resources have the same - /// usage, and return it. - /// - /// Returns `None` if no sub-resources - /// are intersecting with the selector, or their usage - /// isn't consistent. - fn query(&self, selector: Self::Selector) -> Option; - - /// Change the last usage of the selected sub-resources. - /// - /// If `output` is specified, it's filled with the - /// `PendingTransition` objects corresponding to smaller - /// sub-resource transitions. The old usage is replaced by - /// the new one. - /// - /// If `output` is `None`, the old usage is extended with - /// the new usage. The error is returned if it's not possible, - /// specifying the conflicting transition. Extension can only - /// be done for read-only usages. - fn change( - &mut self, - id: Valid, - selector: Self::Selector, - usage: Self::Usage, - output: Option<&mut Vec>>, - ) -> Result<(), PendingTransition>; - - /// Merge the state of this resource tracked by a different instance - /// with the current one. - /// - /// Same rules for `output` apply as with `change()`: last usage state - /// is either replaced (when `output` is provided) with a - /// `PendingTransition` pushed to this vector, or extended with the - /// other read-only usage, unless there is a usage conflict, and - /// the error is generated (returning the conflict). - fn merge( - &mut self, - id: Valid, - other: &Self, - output: Option<&mut Vec>>, - ) -> Result<(), PendingTransition>; - - /// Try to optimize the internal representation. - fn optimize(&mut self); -} - -/// Structure wrapping the abstract tracking state with the relevant resource -/// data, such as the reference count and the epoch. -#[derive(Clone)] -struct Resource { - ref_count: RefCount, - state: S, - epoch: Epoch, -} +pub(crate) use buffer::{BufferBindGroupState, BufferTracker, BufferUsageScope}; +pub(crate) use stateless::{StatelessBindGroupSate, StatelessTracker}; +pub(crate) use texture::{ + TextureBindGroupState, TextureSelector, TextureTracker, TextureUsageScope, +}; /// A structure containing all the information about a particular resource /// transition. User code should be able to generate a pipeline barrier /// based on the contents. #[derive(Debug, PartialEq)] -pub(crate) struct PendingTransition { - pub id: Valid, +pub(crate) struct PendingTransition { + pub id: u32, pub selector: S::Selector, - pub usage: ops::Range, + pub usage: ops::Range, } -impl PendingTransition { +impl PendingTransition { /// Produce the hal barrier corresponding to the transition. pub fn into_hal<'a, A: hal::Api>( self, buf: &'a resource::Buffer, ) -> hal::BufferBarrier<'a, A> { - log::trace!("\tbuffer -> {:?}", self); let buffer = buf.raw.as_ref().expect("Buffer is destroyed"); hal::BufferBarrier { buffer, @@ -129,549 +134,611 @@ impl PendingTransition { } } -impl From> for UsageConflict { - fn from(e: PendingTransition) -> Self { - Self::Buffer { - id: e.id.0, - combined_use: e.usage.end, - } - } -} - -impl PendingTransition { +impl PendingTransition { /// Produce the hal barrier corresponding to the transition. pub fn into_hal<'a, A: hal::Api>( self, tex: &'a resource::Texture, ) -> hal::TextureBarrier<'a, A> { - log::trace!("\ttexture -> {:?}", self); let texture = tex.inner.as_raw().expect("Texture is destroyed"); + + // These showing up in a barrier is always a bug + debug_assert_ne!(self.usage.start, hal::TextureUses::UNKNOWN); + debug_assert_ne!(self.usage.end, hal::TextureUses::UNKNOWN); + + let mip_count = self.selector.mips.end - self.selector.mips.start; + debug_assert_ne!(mip_count, 0); + let layer_count = self.selector.layers.end - self.selector.layers.start; + debug_assert_ne!(layer_count, 0); + hal::TextureBarrier { texture, range: wgt::ImageSubresourceRange { aspect: wgt::TextureAspect::All, - base_mip_level: self.selector.levels.start, - mip_level_count: NonZeroU32::new( - self.selector.levels.end - self.selector.levels.start, - ), + base_mip_level: self.selector.mips.start, + mip_level_count: unsafe { Some(NonZeroU32::new_unchecked(mip_count)) }, base_array_layer: self.selector.layers.start, - array_layer_count: NonZeroU32::new( - self.selector.layers.end - self.selector.layers.start, - ), + array_layer_count: unsafe { Some(NonZeroU32::new_unchecked(layer_count)) }, }, usage: self.usage, } } } -impl From> for UsageConflict { - fn from(e: PendingTransition) -> Self { - Self::Texture { - id: e.id.0, - mip_levels: e.selector.levels.start..e.selector.levels.end, - array_layers: e.selector.layers.start..e.selector.layers.end, - combined_use: e.usage.end, - } - } -} +/// The uses that a resource or subresource can be in. +pub(crate) trait ResourceUses: + fmt::Debug + ops::BitAnd + ops::BitOr + PartialEq + Sized + Copy +{ + /// All flags that are exclusive. + const EXCLUSIVE: Self; -#[derive(Clone, Debug, Error)] -pub enum UseExtendError { - #[error("resource is invalid")] - InvalidResource, - #[error("total usage {0:?} is not valid")] - Conflict(U), + /// The relevant resource ID type. + type Id: Copy + fmt::Debug + TypedId; + /// The selector used by this resource. + type Selector: fmt::Debug; + + /// Turn the resource into a pile of bits. + fn bits(self) -> u16; + /// Returns true if the all the uses are ordered. + fn all_ordered(self) -> bool; + /// Returns true if any of the uses are exclusive. + fn any_exclusive(self) -> bool; } -/// A tracker for all resources of a given type. -pub(crate) struct ResourceTracker { - /// An association of known resource indices with their tracked states. - map: FastHashMap>, - /// Temporary storage for collecting transitions. - temp: Vec>, - /// The backend variant for all the tracked resources. - backend: wgt::Backend, +/// Returns true if the given states violates the usage scope rule +/// of any(inclusive) XOR one(exclusive) +fn invalid_resource_state(state: T) -> bool { + // Is power of two also means "is one bit set". We check for this as if + // we're in any exclusive state, we must only be in a single state. + state.any_exclusive() && !conv::is_power_of_two_u16(state.bits()) } -impl fmt::Debug for ResourceTracker { - fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - self.map - .iter() - .map(|(&index, res)| ((index, res.epoch), &res.state)) - .collect::>() - .fmt(formatter) - } +/// Returns true if the transition from one state to another does not require +/// a barrier. +fn skip_barrier(old_state: T, new_state: T) -> bool { + // If the state didn't change and all the usages are ordered, the hardware + // will guarentee the order of accesses, so we do not need to issue a barrier at all + old_state == new_state && old_state.all_ordered() } -#[allow( - // Explicit lifetimes are easier to reason about here. - clippy::needless_lifetimes, -)] -impl ResourceTracker { - /// Create a new empty tracker. - pub fn new(backend: wgt::Backend) -> Self { - Self { - map: FastHashMap::default(), - temp: Vec::new(), - backend, +/// Resizes the given bitvec to the given size. I'm not sure why this is hard to do but it is. +fn resize_bitvec(vec: &mut BitVec, size: usize) { + let owned_size_to_grow = size.checked_sub(vec.len()); + if let Some(delta) = owned_size_to_grow { + if delta != 0 { + vec.grow(delta, false); } + } else { + vec.truncate(size); } +} - /// Remove an id from the tracked map. - pub(crate) fn remove(&mut self, id: Valid) -> bool { - let (index, epoch, backend) = id.0.unzip(); - debug_assert_eq!(backend, self.backend); - match self.map.remove(&index) { - Some(resource) => { - assert_eq!(resource.epoch, epoch); - true - } - None => false, - } - } +/// Produces an iterator that yields the indexes of all bits that are set in the bitvec. +/// +/// Will skip entire usize's worth of bits if they are all false. +fn iterate_bitvec_indices(ownership: &BitVec) -> impl Iterator + '_ { + const BITS_PER_BLOCK: usize = mem::size_of::() * 8; + + let size = ownership.len(); + + ownership + .blocks() + .enumerate() + .filter(|&(_, word)| word != 0) + .flat_map(move |(word_index, mut word)| { + let bit_start = word_index * BITS_PER_BLOCK; + let bit_end = (bit_start + BITS_PER_BLOCK).min(size); + + (bit_start..bit_end).filter(move |_| { + let active = word & 0b1 != 0; + word >>= 1; + + active + }) + }) +} - /// Remove the resource `id`, only if `self` is holding the last reference to it. - /// - /// Return `true` if we did remove the resource; the underlying hal resource - /// is ready to be freed. - /// - /// This is generally only meaningful to apply to members of - /// [`Device::trackers`], which holds all resources allocated with that - /// [`Device`]. Other trackers should never be the final reference. - /// - /// [`Device`]: crate::device::Device - /// [`Device::trackers`]: crate::device::Device::trackers - pub(crate) fn remove_abandoned(&mut self, id: Valid) -> bool { - let (index, epoch, backend) = id.0.unzip(); - debug_assert_eq!(backend, self.backend); - match self.map.entry(index) { - // This code explicitly ignores requests for IDs that are no longer valid, - // i.e. corresponding to removed entries, or entries that got re-filled - // with new elements (having different epochs). - // This is needed because of the asynchronous nature of the device internals. - // As such, by the time a resource is added to the suspected list, it may - // already be fully removed from all the trackers (and be a stale ID). - // see https://github.com/gfx-rs/wgpu/issues/1996 - Entry::Occupied(e) => { - // see https://github.com/gfx-rs/wgpu/issues/1996 - if e.get().epoch == epoch && e.get().ref_count.load() == 1 { - e.remove(); - true - } else { - false - } - } - _ => false, +#[derive(Clone, Debug, Error, PartialEq)] +pub enum UsageConflict { + #[error("Attempted to use buffer {id:?} which is invalid.")] + BufferInvalid { id: id::BufferId }, + #[error("Attempted to use texture {id:?} which is invalid.")] + TextureInvalid { id: id::TextureId }, + #[error("Attempted to use buffer {id:?} with {invalid_use}.")] + Buffer { + id: id::BufferId, + invalid_use: InvalidUse, + }, + #[error("Attempted to use a texture {id:?} mips {mip_levels:?} layers {array_layers:?} with {invalid_use}.")] + Texture { + id: id::TextureId, + mip_levels: ops::Range, + array_layers: ops::Range, + invalid_use: InvalidUse, + }, +} +impl UsageConflict { + fn from_buffer( + id: id::BufferId, + current_state: hal::BufferUses, + new_state: hal::BufferUses, + ) -> Self { + Self::Buffer { + id, + invalid_use: InvalidUse { + current_state, + new_state, + }, } } - /// Try to optimize the internal representation. - pub(crate) fn optimize(&mut self) { - for resource in self.map.values_mut() { - resource.state.optimize(); + fn from_texture( + id: id::TextureId, + selector: TextureSelector, + current_state: hal::TextureUses, + new_state: hal::TextureUses, + ) -> Self { + Self::Texture { + id, + mip_levels: selector.mips, + array_layers: selector.layers, + invalid_use: InvalidUse { + current_state, + new_state, + }, } } +} - /// Return an iterator over used resources keys. - pub fn used<'a>(&'a self) -> impl 'a + Iterator> { - let backend = self.backend; - self.map - .iter() - .map(move |(&index, resource)| Valid(S::Id::zip(index, resource.epoch, backend))) - } +/// Pretty print helper that shows helpful descriptions of a conflicting usage. +#[derive(Clone, Debug, PartialEq)] +pub struct InvalidUse { + current_state: T, + new_state: T, +} - pub fn get_ref_count(&self, id: Valid) -> &RefCount { - let (index, _, _) = id.0.unzip(); - &self.map[&index].ref_count - } +impl fmt::Display for InvalidUse { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let current = self.current_state; + let new = self.new_state; - /// Return true if there is nothing here. - pub fn is_empty(&self) -> bool { - self.map.is_empty() - } + let current_exclusive = current & T::EXCLUSIVE; + let new_exclusive = new & T::EXCLUSIVE; - /// Clear the tracked contents. - pub fn clear(&mut self) { - self.map.clear(); + let exclusive = current_exclusive | new_exclusive; + + // The text starts with "tried to use X resource with {self}" + write!( + f, + "conflicting usages. Current usage {current:?} and new usage {new:?}. \ + {exclusive:?} is an exclusive usage and cannot be used with any other\ + usages within the usage scope (renderpass or compute dispatch)" + ) } +} - /// Begin tracking a new resource `id` in state `state`. - /// - /// Hold `ref_count` in the tracker. - /// - /// Returns false if the resource is already registered. - pub(crate) fn init( - &mut self, - id: Valid, - ref_count: RefCount, - state: S, - ) -> Result<(), &S> { - let (index, epoch, backend) = id.0.unzip(); - debug_assert_eq!(backend, self.backend); - match self.map.entry(index) { - Entry::Vacant(e) => { - e.insert(Resource { - ref_count, - state, - epoch, - }); - Ok(()) - } - Entry::Occupied(e) => Err(&e.into_mut().state), +/// SOA container for storing metadata of a resource. +/// +/// This contins the ownership bitvec, the refcount of +/// the resource, and the epoch of the object's full ID. +#[derive(Debug)] +pub(crate) struct ResourceMetadata { + owned: BitVec, + ref_counts: Vec>, + epochs: Vec, + + _phantom: PhantomData, +} +impl ResourceMetadata { + pub fn new() -> Self { + Self { + owned: BitVec::default(), + ref_counts: Vec::new(), + epochs: Vec::new(), + + _phantom: PhantomData, } } - /// Query the usage of a resource selector. - /// - /// Returns `Some(Usage)` only if this usage is consistent - /// across the given selector. - #[allow(unused)] // TODO: figure out if this needs to be removed - pub fn query(&self, id: Valid, selector: S::Selector) -> Option { - let (index, epoch, backend) = id.0.unzip(); - debug_assert_eq!(backend, self.backend); - let res = self.map.get(&index)?; - assert_eq!(res.epoch, epoch); - res.state.query(selector) + pub fn set_size(&mut self, size: usize) { + self.ref_counts.resize(size, None); + self.epochs.resize(size, u32::MAX); + + resize_bitvec(&mut self.owned, size); } - /// Make sure that a resource is tracked, and return a mutable reference to it. + /// Ensures a given index is in bounds for all arrays and does + /// sanity checks of the presence of a refcount. /// - /// If the resource isn't tracked, start it in the default state, and take a - /// clone of `ref_count`. - /// - /// The `self_backend` and `map` arguments should be the `backend` and `map` - /// fields of a `ResourceTracker`. Ideally this function would just take - /// `&mut self` and access those from there, but that would upset the borrow - /// checker in some callers, who want to borrow `ResourceTracker::temp` - /// alongside our return value. The approach taken here has the caller - /// borrow both `map` and `temp`, so the borrow checker can see that they - /// don't alias. - fn get_or_insert<'a>( - self_backend: wgt::Backend, - map: &'a mut FastHashMap>, - id: Valid, - ref_count: &RefCount, - ) -> &'a mut Resource { - let (index, epoch, backend) = id.0.unzip(); - debug_assert_eq!(self_backend, backend); - match map.entry(index) { - Entry::Vacant(e) => e.insert(Resource { - ref_count: ref_count.clone(), - state: S::default(), - epoch, - }), - Entry::Occupied(e) => { - assert_eq!(e.get().epoch, epoch); - e.into_mut() - } - } - } + /// In release mode this function is completely empty and is removed. + fn debug_assert_in_bounds(&self, index: usize) { + debug_assert!(index < self.owned.len()); + debug_assert!(index < self.ref_counts.len()); + debug_assert!(index < self.epochs.len()); - /// Return a mutable reference to `id`'s state. - fn get<'a>( - self_backend: wgt::Backend, - map: &'a mut FastHashMap>, - id: Valid, - ) -> &'a mut Resource { - let (index, epoch, backend) = id.0.unzip(); - debug_assert_eq!(self_backend, backend); - let e = map.get_mut(&index).unwrap(); - assert_eq!(e.epoch, epoch); - e + debug_assert!(if self.owned.get(index).unwrap() { + self.ref_counts[index].is_some() + } else { + true + }); } - /// Extend the usage of `id`, tracking it if necessary. + /// Returns true if the tracker owns no resources. /// - /// Returns conflicting transition as an error. - pub(crate) fn change_extend( - &mut self, - id: Valid, - ref_count: &RefCount, - selector: S::Selector, - usage: S::Usage, - ) -> Result<(), PendingTransition> { - Self::get_or_insert(self.backend, &mut self.map, id, ref_count) - .state - .change(id, selector, usage, None) + /// This is a O(n) operation. + fn is_empty(&self) -> bool { + !self.owned.any() } - /// Replace the usage of a specified resource. - pub(crate) fn change_replace( - &mut self, - id: Valid, - ref_count: &RefCount, - selector: S::Selector, - usage: S::Usage, - ) -> Drain> { - let res = Self::get_or_insert(self.backend, &mut self.map, id, ref_count); - res.state - .change(id, selector, usage, Some(&mut self.temp)) - .ok(); //TODO: unwrap? - self.temp.drain(..) + /// Returns ids for all resources we own. + fn used(&self) -> impl Iterator> + '_ { + if !self.owned.is_empty() { + self.debug_assert_in_bounds(self.owned.len() - 1) + }; + iterate_bitvec_indices(&self.owned).map(move |index| { + let epoch = unsafe { *self.epochs.get_unchecked(index) }; + id::Valid(Id::zip(index as u32, epoch, A::VARIANT)) + }) } - /// Replace the usage of a specified already tracked resource. - /// (panics if the resource is not yet tracked) - pub(crate) fn change_replace_tracked( - &mut self, - id: Valid, - selector: S::Selector, - usage: S::Usage, - ) -> Drain> { - let res = Self::get(self.backend, &mut self.map, id); - res.state - .change(id, selector, usage, Some(&mut self.temp)) - .ok(); - self.temp.drain(..) + /// Resets the metadata for a given index to sane "invalid" values. + unsafe fn reset(&mut self, index: usize) { + *self.ref_counts.get_unchecked_mut(index) = None; + *self.epochs.get_unchecked_mut(index) = u32::MAX; + self.owned.set(index, false); } +} - /// Merge another tracker into `self` by extending the current states - /// without any transitions. - pub(crate) fn merge_extend(&mut self, other: &Self) -> Result<(), PendingTransition> { - debug_assert_eq!(self.backend, other.backend); - for (&index, new) in other.map.iter() { - match self.map.entry(index) { - Entry::Vacant(e) => { - e.insert(new.clone()); - } - Entry::Occupied(e) => { - assert_eq!( - e.get().epoch, - new.epoch, - "ID {:?} wasn't properly removed", - S::Id::zip(index, e.get().epoch, self.backend) - ); - let id = Valid(S::Id::zip(index, new.epoch, self.backend)); - e.into_mut().state.merge(id, &new.state, None)?; - } +/// A source of resource metadata. +/// +/// This is used to abstract over the various places +/// trackers can get new resource metadata from. +enum ResourceMetadataProvider<'a, A: hub::HalApi> { + /// Comes directly from explicit values. + Direct { + epoch: Epoch, + ref_count: Cow<'a, RefCount>, + }, + /// Comes from another metadata tracker. + Indirect { metadata: &'a ResourceMetadata }, + /// The epoch is given directly, but the life count comes from the resource itself. + Resource { epoch: Epoch }, +} +impl ResourceMetadataProvider<'_, A> { + /// Get the epoch and an owned refcount from this. + /// + /// # Safety + /// + /// - The index must be in bounds of the metadata tracker if this uses an indirect source. + /// - life_guard must be Some if this uses a Resource source. + #[inline(always)] + unsafe fn get_own(self, life_guard: Option<&LifeGuard>, index: usize) -> (Epoch, RefCount) { + match self { + ResourceMetadataProvider::Direct { epoch, ref_count } => { + (epoch, ref_count.into_owned()) + } + ResourceMetadataProvider::Indirect { metadata } => { + metadata.debug_assert_in_bounds(index); + ( + *metadata.epochs.get_unchecked(index), + metadata + .ref_counts + .get_unchecked(index) + .clone() + .unwrap_unchecked(), + ) + } + ResourceMetadataProvider::Resource { epoch } => { + debug_assert!(life_guard.is_some()); + (epoch, life_guard.unwrap_unchecked().add_ref()) } } - Ok(()) } - - /// Merge another tracker, adding it's transitions to `self`. - /// Transitions the current usage to the new one. - pub(crate) fn merge_replace<'a>(&'a mut self, other: &'a Self) -> Drain> { - for (&index, new) in other.map.iter() { - match self.map.entry(index) { - Entry::Vacant(e) => { - e.insert(new.clone()); - } - Entry::Occupied(e) => { - assert_eq!( - e.get().epoch, - new.epoch, - "ID {:?} wasn't properly removed", - S::Id::zip(index, e.get().epoch, self.backend) - ); - let id = Valid(S::Id::zip(index, new.epoch, self.backend)); - e.into_mut() - .state - .merge(id, &new.state, Some(&mut self.temp)) - .ok(); //TODO: unwrap? - } + /// Get the epoch from this. + /// + /// # Safety + /// + /// - The index must be in bounds of the metadata tracker if this uses an indirect source. + #[inline(always)] + unsafe fn get_epoch(self, index: usize) -> Epoch { + match self { + ResourceMetadataProvider::Direct { epoch, .. } + | ResourceMetadataProvider::Resource { epoch, .. } => epoch, + ResourceMetadataProvider::Indirect { metadata } => { + metadata.debug_assert_in_bounds(index); + *metadata.epochs.get_unchecked(index) } } - self.temp.drain(..) } +} - /// Use a given resource provided by an `Id` with the specified usage. - /// Combines storage access by 'Id' with the transition that extends - /// the last read-only usage, if possible. - /// - /// Returns the old usage as an error if there is a conflict. - pub(crate) fn use_extend<'a, T: 'a + hub::Resource>( - &mut self, - storage: &'a hub::Storage, - id: S::Id, - selector: S::Selector, - usage: S::Usage, - ) -> Result<&'a T, UseExtendError> { - let item = storage - .get(id) - .map_err(|_| UseExtendError::InvalidResource)?; - self.change_extend( - Valid(id), - item.life_guard().ref_count.as_ref().unwrap(), - selector, - usage, - ) - .map(|()| item) - .map_err(|pending| UseExtendError::Conflict(pending.usage.end)) +/// All the usages that a bind group contains. The uses are not deduplicated in any way +/// and may include conflicting uses. This is fully compliant by the WebGPU spec. +/// +/// All bind group states are sorted by their ID so that when adding to a tracker, +/// they are added in the most efficient order possible (assending order). +pub(crate) struct BindGroupStates { + pub buffers: BufferBindGroupState, + pub textures: TextureBindGroupState, + pub views: StatelessBindGroupSate, id::TextureViewId>, + pub samplers: StatelessBindGroupSate, id::SamplerId>, +} + +impl BindGroupStates { + pub fn new() -> Self { + Self { + buffers: BufferBindGroupState::new(), + textures: TextureBindGroupState::new(), + views: StatelessBindGroupSate::new(), + samplers: StatelessBindGroupSate::new(), + } } - /// Use a given resource provided by an `Id` with the specified usage. - /// Combines storage access by 'Id' with the transition that replaces - /// the last usage with a new one, returning an iterator over these - /// transitions. - pub(crate) fn use_replace<'a, T: 'a + hub::Resource>( - &mut self, - storage: &'a hub::Storage, - id: S::Id, - selector: S::Selector, - usage: S::Usage, - ) -> Result<(&'a T, Drain>), S::Id> { - let item = storage.get(id).map_err(|_| id)?; - let drain = self.change_replace( - Valid(id), - item.life_guard().ref_count.as_ref().unwrap(), - selector, - usage, - ); - Ok((item, drain)) + /// Optimize the bind group states by sorting them by ID. + /// + /// When this list of states is merged into a tracker, the memory + /// accesses will be in a constant assending order. + pub fn optimize(&mut self) { + self.buffers.optimize(); + self.textures.optimize(); + self.views.optimize(); + self.samplers.optimize(); } } -impl ResourceState for PhantomData { - type Id = I; - type Selector = (); - type Usage = (); - - fn query(&self, _selector: Self::Selector) -> Option { - Some(()) - } +/// This is a render bundle specific usage scope. It includes stateless resources +/// that are not normally included in a usage scope, but are used by render bundles +/// and need to be owned by the render bundles. +pub(crate) struct RenderBundleScope { + pub buffers: BufferUsageScope, + pub textures: TextureUsageScope, + // Don't need to track views and samplers, they are never used directly, only by bind groups. + pub bind_groups: StatelessTracker, id::BindGroupId>, + pub render_pipelines: StatelessTracker, id::RenderPipelineId>, + pub query_sets: StatelessTracker, id::QuerySetId>, +} - fn change( +impl RenderBundleScope { + /// Create the render bundle scope and pull the maximum IDs from the hubs. + pub fn new( + buffers: &hub::Storage, id::BufferId>, + textures: &hub::Storage, id::TextureId>, + bind_groups: &hub::Storage, id::BindGroupId>, + render_pipelines: &hub::Storage, id::RenderPipelineId>, + query_sets: &hub::Storage, id::QuerySetId>, + ) -> Self { + let mut value = Self { + buffers: BufferUsageScope::new(), + textures: TextureUsageScope::new(), + bind_groups: StatelessTracker::new(), + render_pipelines: StatelessTracker::new(), + query_sets: StatelessTracker::new(), + }; + + value.buffers.set_size(buffers.len()); + value.textures.set_size(textures.len()); + value.bind_groups.set_size(bind_groups.len()); + value.render_pipelines.set_size(render_pipelines.len()); + value.query_sets.set_size(query_sets.len()); + + value + } + + /// Merge the inner contents of a bind group into the render bundle tracker. + /// + /// Only stateful things are merged in here, all other resources are owned + /// indirectly by the bind group. + /// + /// # Safety + /// + /// The maximum ID given by each bind group resource must be less than the + /// length of the storage given at the call to `new`. + pub unsafe fn merge_bind_group( &mut self, - _id: Valid, - _selector: Self::Selector, - _usage: Self::Usage, - _output: Option<&mut Vec>>, - ) -> Result<(), PendingTransition> { - Ok(()) - } + textures: &hub::Storage, id::TextureId>, + bind_group: &BindGroupStates, + ) -> Result<(), UsageConflict> { + self.buffers.merge_bind_group(&bind_group.buffers)?; + self.textures + .merge_bind_group(textures, &bind_group.textures)?; - fn merge( - &mut self, - _id: Valid, - _other: &Self, - _output: Option<&mut Vec>>, - ) -> Result<(), PendingTransition> { Ok(()) } - - fn optimize(&mut self) {} } -pub const DUMMY_SELECTOR: () = (); - -#[derive(Clone, Debug, Error)] -pub enum UsageConflict { - #[error( - "Attempted to use buffer {id:?} as a combination of {combined_use:?} within a usage scope." - )] - Buffer { - id: id::BufferId, - combined_use: hal::BufferUses, - }, - #[error("Attempted to use texture {id:?} mips {mip_levels:?} layers {array_layers:?} as a combination of {combined_use:?} within a usage scope.")] - Texture { - id: id::TextureId, - mip_levels: ops::Range, - array_layers: ops::Range, - combined_use: hal::TextureUses, - }, -} - -/// A set of trackers for all relevant resources. -/// -/// `Device` uses this to track all resources allocated from that device. -/// Resources like `BindGroup`, `CommandBuffer`, and so on that may own a -/// variety of other resources also use a value of this type to keep track of -/// everything they're depending on. +/// A usage scope tracker. Only needs to store stateful resources as stateless +/// resources cannot possibly have a usage conflict. #[derive(Debug)] -pub(crate) struct TrackerSet { - pub buffers: ResourceTracker, - pub textures: ResourceTracker, - pub views: ResourceTracker>, - pub bind_groups: ResourceTracker>, - pub samplers: ResourceTracker>, - pub compute_pipes: ResourceTracker>, - pub render_pipes: ResourceTracker>, - pub bundles: ResourceTracker>, - pub query_sets: ResourceTracker>, +pub(crate) struct UsageScope { + pub buffers: BufferUsageScope, + pub textures: TextureUsageScope, } -impl TrackerSet { - /// Create an empty set. - pub fn new(backend: wgt::Backend) -> Self { - Self { - buffers: ResourceTracker::new(backend), - textures: ResourceTracker::new(backend), - views: ResourceTracker::new(backend), - bind_groups: ResourceTracker::new(backend), - samplers: ResourceTracker::new(backend), - compute_pipes: ResourceTracker::new(backend), - render_pipes: ResourceTracker::new(backend), - bundles: ResourceTracker::new(backend), - query_sets: ResourceTracker::new(backend), - } - } +impl UsageScope { + /// Create the render bundle scope and pull the maximum IDs from the hubs. + pub fn new( + buffers: &hub::Storage, id::BufferId>, + textures: &hub::Storage, id::TextureId>, + ) -> Self { + let mut value = Self { + buffers: BufferUsageScope::new(), + textures: TextureUsageScope::new(), + }; - /// Clear all the trackers. - pub fn _clear(&mut self) { - self.buffers.clear(); - self.textures.clear(); - self.views.clear(); - self.bind_groups.clear(); - self.samplers.clear(); - self.compute_pipes.clear(); - self.render_pipes.clear(); - self.bundles.clear(); - self.query_sets.clear(); - } + value.buffers.set_size(buffers.len()); + value.textures.set_size(textures.len()); - /// Try to optimize the tracking representation. - pub fn optimize(&mut self) { - self.buffers.optimize(); - self.textures.optimize(); - self.views.optimize(); - self.bind_groups.optimize(); - self.samplers.optimize(); - self.compute_pipes.optimize(); - self.render_pipes.optimize(); - self.bundles.optimize(); - self.query_sets.optimize(); + value } - /// Merge only the stateful trackers of another instance by extending - /// the usage. Returns a conflict if any. - pub fn merge_extend_stateful(&mut self, other: &Self) -> Result<(), UsageConflict> { - self.buffers.merge_extend(&other.buffers)?; - self.textures.merge_extend(&other.textures)?; + /// Merge the inner contents of a bind group into the usage scope. + /// + /// Only stateful things are merged in here, all other resources are owned + /// indirectly by the bind group. + /// + /// # Safety + /// + /// The maximum ID given by each bind group resource must be less than the + /// length of the storage given at the call to `new`. + pub unsafe fn merge_bind_group( + &mut self, + textures: &hub::Storage, id::TextureId>, + bind_group: &BindGroupStates, + ) -> Result<(), UsageConflict> { + self.buffers.merge_bind_group(&bind_group.buffers)?; + self.textures + .merge_bind_group(textures, &bind_group.textures)?; + Ok(()) } - pub fn backend(&self) -> wgt::Backend { - self.buffers.backend + /// Merge the inner contents of a bind group into the usage scope. + /// + /// Only stateful things are merged in here, all other resources are owned + /// indirectly by a bind group or are merged directly into the command buffer tracker. + /// + /// # Safety + /// + /// The maximum ID given by each bind group resource must be less than the + /// length of the storage given at the call to `new`. + pub unsafe fn merge_render_bundle( + &mut self, + textures: &hub::Storage, id::TextureId>, + render_bundle: &RenderBundleScope, + ) -> Result<(), UsageConflict> { + self.buffers.merge_usage_scope(&render_bundle.buffers)?; + self.textures + .merge_usage_scope(textures, &render_bundle.textures)?; + + Ok(()) } } -#[derive(Debug)] -pub(crate) struct StatefulTrackerSubset { - pub buffers: ResourceTracker, - pub textures: ResourceTracker, +/// A full double sided tracker used by CommandBuffers and the Device. +pub(crate) struct Tracker { + pub buffers: BufferTracker, + pub textures: TextureTracker, + pub views: StatelessTracker, id::TextureViewId>, + pub samplers: StatelessTracker, id::SamplerId>, + pub bind_groups: StatelessTracker, id::BindGroupId>, + pub compute_pipelines: StatelessTracker, id::ComputePipelineId>, + pub render_pipelines: StatelessTracker, id::RenderPipelineId>, + pub bundles: StatelessTracker, id::RenderBundleId>, + pub query_sets: StatelessTracker, id::QuerySetId>, } -impl StatefulTrackerSubset { - /// Create an empty set. - pub fn new(backend: wgt::Backend) -> Self { +impl Tracker { + pub fn new() -> Self { Self { - buffers: ResourceTracker::new(backend), - textures: ResourceTracker::new(backend), + buffers: BufferTracker::new(), + textures: TextureTracker::new(), + views: StatelessTracker::new(), + samplers: StatelessTracker::new(), + bind_groups: StatelessTracker::new(), + compute_pipelines: StatelessTracker::new(), + render_pipelines: StatelessTracker::new(), + bundles: StatelessTracker::new(), + query_sets: StatelessTracker::new(), } } - /// Clear all the trackers. - pub fn clear(&mut self) { - self.buffers.clear(); - self.textures.clear(); + /// Pull the maximum IDs from the hubs. + pub fn set_size( + &mut self, + buffers: Option<&hub::Storage, id::BufferId>>, + textures: Option<&hub::Storage, id::TextureId>>, + views: Option<&hub::Storage, id::TextureViewId>>, + samplers: Option<&hub::Storage, id::SamplerId>>, + bind_groups: Option<&hub::Storage, id::BindGroupId>>, + compute_pipelines: Option< + &hub::Storage, id::ComputePipelineId>, + >, + render_pipelines: Option<&hub::Storage, id::RenderPipelineId>>, + bundles: Option<&hub::Storage, id::RenderBundleId>>, + query_sets: Option<&hub::Storage, id::QuerySetId>>, + ) { + if let Some(buffers) = buffers { + self.buffers.set_size(buffers.len()); + }; + if let Some(textures) = textures { + self.textures.set_size(textures.len()); + }; + if let Some(views) = views { + self.views.set_size(views.len()); + }; + if let Some(samplers) = samplers { + self.samplers.set_size(samplers.len()); + }; + if let Some(bind_groups) = bind_groups { + self.bind_groups.set_size(bind_groups.len()); + }; + if let Some(compute_pipelines) = compute_pipelines { + self.compute_pipelines.set_size(compute_pipelines.len()); + } + if let Some(render_pipelines) = render_pipelines { + self.render_pipelines.set_size(render_pipelines.len()); + }; + if let Some(bundles) = bundles { + self.bundles.set_size(bundles.len()); + }; + if let Some(query_sets) = query_sets { + self.query_sets.set_size(query_sets.len()); + }; + } + + /// Iterates through all resources in the given bind group and adopts + /// the state given for those resources in the UsageScope. It also + /// removes all touched resources from the usage scope. + /// + /// If a transition is needed to get the resources into the needed state, + /// those transitions are stored within the tracker. A subsequent + /// call to [`Self::drain`] is needed to get those transitions. + /// + /// This is a really funky method used by Compute Passes to generate + /// barriers after a call to dispatch without needing to iterate + /// over all elements in the usage scope. We use each the + /// bind group as a source of which IDs to look at. The bind groups + /// must have first been added to the usage scope. + /// + /// Only stateful things are merged in here, all other resources are owned + /// indirectly by the bind group. + /// + /// # Safety + /// + /// The maximum ID given by each bind group resource must be less than the + /// value given to `set_size` + pub unsafe fn set_and_remove_from_usage_scope_sparse( + &mut self, + textures: &hub::Storage, id::TextureId>, + scope: &mut UsageScope, + bind_group: &BindGroupStates, + ) { + self.buffers + .set_and_remove_from_usage_scope_sparse(&mut scope.buffers, &bind_group.buffers); + self.textures.set_and_remove_from_usage_scope_sparse( + textures, + &mut scope.textures, + &bind_group.textures, + ); } - /// Merge all the trackers of another tracker the usage. - pub fn merge_extend(&mut self, other: &TrackerSet) -> Result<(), UsageConflict> { - self.buffers.merge_extend(&other.buffers)?; - self.textures.merge_extend(&other.textures)?; + /// Tracks the stateless resources from the given renderbundle. It is expected + /// that the stateful resources will get merged into a usage scope first. + /// + /// # Safety + /// + /// The maximum ID given by each bind group resource must be less than the + /// value given to `set_size` + pub unsafe fn add_from_render_bundle( + &mut self, + render_bundle: &RenderBundleScope, + ) -> Result<(), UsageConflict> { + self.bind_groups + .add_from_tracker(&render_bundle.bind_groups); + self.render_pipelines + .add_from_tracker(&render_bundle.render_pipelines); + self.query_sets.add_from_tracker(&render_bundle.query_sets); + Ok(()) } } diff --git a/wgpu-core/src/track/range.rs b/wgpu-core/src/track/range.rs index c2238439db..12c527fb2a 100644 --- a/wgpu-core/src/track/range.rs +++ b/wgpu-core/src/track/range.rs @@ -2,25 +2,19 @@ //TODO: consider getting rid of it. use smallvec::SmallVec; -use std::{cmp::Ordering, fmt::Debug, iter, ops::Range, slice::Iter}; +use std::{fmt::Debug, iter, ops::Range}; /// Structure that keeps track of a I -> T mapping, /// optimized for a case where keys of the same values /// are often grouped together linearly. #[derive(Clone, Debug, PartialEq)] -pub struct RangedStates { +pub(crate) struct RangedStates { /// List of ranges, each associated with a singe value. /// Ranges of keys have to be non-intersecting and ordered. ranges: SmallVec<[(Range, T); 1]>, } -impl RangedStates { - pub fn empty() -> Self { - Self { - ranges: SmallVec::new(), - } - } - +impl RangedStates { pub fn from_range(range: Range, value: T) -> Self { Self { ranges: iter::once((range, value)).collect(), @@ -35,20 +29,12 @@ impl RangedStates { } } - /// Clear all the ranges. - pub fn clear(&mut self) { - self.ranges.clear(); + pub fn iter(&self) -> impl Iterator, T)> + Clone { + self.ranges.iter() } - /// Append a range. - /// - /// Assumes that the object is being constructed from a set of - /// ranges, and they are given in the ascending order of their keys. - pub fn append(&mut self, index: Range, value: T) { - if let Some(last) = self.ranges.last() { - debug_assert!(last.0.end <= index.start); - } - self.ranges.push((index, value)); + pub fn iter_mut(&mut self) -> impl Iterator, T)> { + self.ranges.iter_mut() } /// Check that all the ranges are non-intersecting and ordered. @@ -64,7 +50,6 @@ impl RangedStates { } /// Merge the neighboring ranges together, where possible. - #[allow(clippy::suspicious_operation_groupings)] pub fn coalesce(&mut self) { let mut num_removed = 0; let mut iter = self.ranges.iter_mut(); @@ -86,25 +71,18 @@ impl RangedStates { } } - /// Check if all intersecting ranges have the same value, which is returned. - /// - /// Returns `None` if no intersections are detected. - /// Returns `Some(Err)` if the intersected values are inconsistent. - pub fn query( - &self, - index: &Range, - fun: impl Fn(&T) -> U, - ) -> Option> { - let mut result = None; - for &(ref range, ref value) in self.ranges.iter() { - if range.end > index.start && range.start < index.end { - let old = result.replace(fun(value)); - if old.is_some() && old != result { - return Some(Err(())); - } - } - } - result.map(Ok) + pub fn iter_filter<'a>( + &'a self, + range: &'a Range, + ) -> impl Iterator, &T)> + 'a { + self.ranges + .iter() + .filter(move |&&(ref inner, ..)| inner.end > range.start && inner.start < range.end) + .map(move |&(ref inner, ref v)| { + let new_range = inner.start.max(range.start)..inner.end.min(range.end); + + (new_range, v) + }) } /// Split the storage ranges in such a way that there is a linear subset of @@ -176,112 +154,12 @@ impl RangedStates { clone.check_sanity(); result } - - /// Produce an iterator that merges two instances together. - /// - /// Each range in the returned iterator is a subset of a range in either - /// `self` or `other`, and the value returned as a `Range` from `self` to `other`. - pub fn merge<'a>(&'a self, other: &'a Self, base: I) -> Merge<'a, I, T> { - Merge { - base, - sa: self.ranges.iter().peekable(), - sb: other.ranges.iter().peekable(), - } - } -} - -/// A custom iterator that goes through two `RangedStates` and process a merge. -#[derive(Debug)] -pub struct Merge<'a, I, T> { - base: I, - sa: iter::Peekable, T)>>, - sb: iter::Peekable, T)>>, -} - -impl<'a, I: Copy + Debug + Ord, T: Copy + Debug> Iterator for Merge<'a, I, T> { - type Item = (Range, Range>); - fn next(&mut self) -> Option { - match (self.sa.peek(), self.sb.peek()) { - // we have both streams - (Some(&&(ref ra, va)), Some(&&(ref rb, vb))) => { - let (range, usage) = if ra.start < self.base { - // in the middle of the left stream - let (end, end_value) = if self.base == rb.start { - // right stream is starting - debug_assert!(self.base < ra.end); - (rb.end, Some(vb)) - } else { - // right hasn't started yet - debug_assert!(self.base < rb.start); - (rb.start, None) - }; - (self.base..ra.end.min(end), Some(va)..end_value) - } else if rb.start < self.base { - // in the middle of the right stream - let (end, start_value) = if self.base == ra.start { - // left stream is starting - debug_assert!(self.base < rb.end); - (ra.end, Some(va)) - } else { - // left hasn't started yet - debug_assert!(self.base < ra.start); - (ra.start, None) - }; - (self.base..rb.end.min(end), start_value..Some(vb)) - } else { - // no active streams - match ra.start.cmp(&rb.start) { - // both are starting - Ordering::Equal => (ra.start..ra.end.min(rb.end), Some(va)..Some(vb)), - // only left is starting - Ordering::Less => (ra.start..rb.start.min(ra.end), Some(va)..None), - // only right is starting - Ordering::Greater => (rb.start..ra.start.min(rb.end), None..Some(vb)), - } - }; - self.base = range.end; - if ra.end == range.end { - let _ = self.sa.next(); - } - if rb.end == range.end { - let _ = self.sb.next(); - } - Some((range, usage)) - } - // only right stream - (None, Some(&&(ref rb, vb))) => { - let range = self.base.max(rb.start)..rb.end; - self.base = rb.end; - let _ = self.sb.next(); - Some((range, None..Some(vb))) - } - // only left stream - (Some(&&(ref ra, va)), None) => { - let range = self.base.max(ra.start)..ra.end; - self.base = ra.end; - let _ = self.sa.next(); - Some((range, Some(va)..None)) - } - // done - (None, None) => None, - } - } } #[cfg(test)] mod test { //TODO: randomized/fuzzy testing use super::RangedStates; - use std::{fmt::Debug, ops::Range}; - - fn easy_merge( - ra: &[(Range, T)], - rb: &[(Range, T)], - ) -> Vec<(Range, Range>)> { - RangedStates::from_slice(ra) - .merge(&RangedStates::from_slice(rb), 0) - .collect() - } #[test] fn sane_good() { @@ -311,14 +189,6 @@ mod test { assert_eq!(rs.ranges.as_slice(), &[(1..5, 9), (5..7, 1), (8..9, 1),]); } - #[test] - fn query() { - let rs = RangedStates::from_slice(&[(1..4, 1u8), (5..7, 2)]); - assert_eq!(rs.query(&(0..1), |v| *v), None); - assert_eq!(rs.query(&(1..3), |v| *v), Some(Ok(1))); - assert_eq!(rs.query(&(1..6), |v| *v), Some(Err(()))); - } - #[test] fn isolate() { let rs = RangedStates::from_slice(&[(1..4, 9u8), (4..5, 9), (5..7, 1), (8..9, 1)]); @@ -333,104 +203,4 @@ mod test { &[(6..7, 1), (7..8, 0), (8..9, 1),] ); } - - #[test] - fn merge_same() { - assert_eq!( - &easy_merge(&[(1..4, 0u8),], &[(1..4, 2u8),],), - &[(1..4, Some(0)..Some(2)),] - ); - } - - #[test] - fn merge_empty() { - assert_eq!( - &easy_merge(&[(1..2, 0u8),], &[],), - &[(1..2, Some(0)..None),] - ); - assert_eq!( - &easy_merge(&[], &[(3..4, 1u8),],), - &[(3..4, None..Some(1)),] - ); - } - - #[test] - fn merge_separate() { - assert_eq!( - &easy_merge(&[(1..2, 0u8), (5..6, 1u8),], &[(2..4, 2u8),],), - &[ - (1..2, Some(0)..None), - (2..4, None..Some(2)), - (5..6, Some(1)..None), - ] - ); - } - - #[test] - fn merge_subset() { - assert_eq!( - &easy_merge(&[(1..6, 0u8),], &[(2..4, 2u8),],), - &[ - (1..2, Some(0)..None), - (2..4, Some(0)..Some(2)), - (4..6, Some(0)..None), - ] - ); - assert_eq!( - &easy_merge(&[(2..4, 0u8),], &[(1..4, 2u8),],), - &[(1..2, None..Some(2)), (2..4, Some(0)..Some(2)),] - ); - } - - #[test] - fn merge_all() { - assert_eq!( - &easy_merge(&[(1..4, 0u8), (5..8, 1u8),], &[(2..6, 2u8), (7..9, 3u8),],), - &[ - (1..2, Some(0)..None), - (2..4, Some(0)..Some(2)), - (4..5, None..Some(2)), - (5..6, Some(1)..Some(2)), - (6..7, Some(1)..None), - (7..8, Some(1)..Some(3)), - (8..9, None..Some(3)), - ] - ); - } - - #[test] - fn merge_complex() { - assert_eq!( - &easy_merge( - &[ - (0..8, 0u8), - (8..9, 1), - (9..16, 2), - (16..17, 3), - (17..118, 4), - (118..119, 5), - (119..124, 6), - (124..125, 7), - (125..512, 8), - ], - &[(15..16, 10u8), (51..52, 11), (126..127, 12),], - ), - &[ - (0..8, Some(0)..None), - (8..9, Some(1)..None), - (9..15, Some(2)..None), - (15..16, Some(2)..Some(10)), - (16..17, Some(3)..None), - (17..51, Some(4)..None), - (51..52, Some(4)..Some(11)), - (52..118, Some(4)..None), - (118..119, Some(5)..None), - (119..124, Some(6)..None), - (124..125, Some(7)..None), - (125..126, Some(8)..None), - (126..127, Some(8)..Some(12)), - (127..512, Some(8)..None), - ] - ); - } } diff --git a/wgpu-core/src/track/stateless.rs b/wgpu-core/src/track/stateless.rs new file mode 100644 index 0000000000..a8051bb3ce --- /dev/null +++ b/wgpu-core/src/track/stateless.rs @@ -0,0 +1,209 @@ +/*! Stateless Trackers + * + * Stateless trackers don't have any state, so make no + * distinction between a usage scope and a full tracker. +!*/ + +use std::marker::PhantomData; + +use crate::{ + hub, + id::{TypedId, Valid}, + track::{iterate_bitvec_indices, ResourceMetadata}, + RefCount, +}; + +/// Stores all the resources that a bind group stores. +pub(crate) struct StatelessBindGroupSate { + resources: Vec<(Valid, RefCount)>, + + _phantom: PhantomData, +} + +impl StatelessBindGroupSate { + pub fn new() -> Self { + Self { + resources: Vec::new(), + + _phantom: PhantomData, + } + } + + /// Optimize the buffer bind group state by sorting it by ID. + /// + /// When this list of states is merged into a tracker, the memory + /// accesses will be in a constant assending order. + pub(crate) fn optimize(&mut self) { + self.resources + .sort_unstable_by_key(|&(id, _)| id.0.unzip().0); + } + + /// Returns a list of all resources tracked. May contain duplicates. + pub fn used(&self) -> impl Iterator> + '_ { + self.resources.iter().map(|&(id, _)| id) + } + + /// Adds the given resource. + pub fn add_single<'a>(&mut self, storage: &'a hub::Storage, id: Id) -> Option<&'a T> { + let resource = storage.get(id).ok()?; + + self.resources + .push((Valid(id), resource.life_guard().add_ref())); + + Some(resource) + } +} + +/// Stores all resource state within a command buffer or device. +pub(crate) struct StatelessTracker { + metadata: ResourceMetadata, + + _phantom: PhantomData<(T, Id)>, +} + +impl StatelessTracker { + pub fn new() -> Self { + Self { + metadata: ResourceMetadata::new(), + + _phantom: PhantomData, + } + } + + fn debug_assert_in_bounds(&self, index: usize) { + self.metadata.debug_assert_in_bounds(index); + } + + /// Sets the size of all the vectors inside the tracker. + /// + /// Must be called with the highest possible Resource ID of this type + /// before all unsafe functions are called. + pub fn set_size(&mut self, size: usize) { + self.metadata.set_size(size); + } + + /// Extend the vectors to let the given index be valid. + fn allow_index(&mut self, index: usize) { + if index >= self.metadata.owned.len() { + self.set_size(index + 1); + } + } + + /// Returns a list of all resources tracked. + pub fn used(&self) -> impl Iterator> + '_ { + self.metadata.used() + } + + /// Inserts a single resource into the resource tracker. + /// + /// If the resource already exists in the tracker, it will be overwritten. + /// + /// If the ID is higher than the length of internal vectors, + /// the vectors will be extended. A call to set_size is not needed. + pub fn insert_single(&mut self, id: Valid, ref_count: RefCount) { + let (index32, epoch, _) = id.0.unzip(); + let index = index32 as usize; + + self.allow_index(index); + + self.debug_assert_in_bounds(index); + + unsafe { + *self.metadata.epochs.get_unchecked_mut(index) = epoch; + *self.metadata.ref_counts.get_unchecked_mut(index) = Some(ref_count); + self.metadata.owned.set(index, true); + } + } + + /// Adds the given resource to the tracker. + /// + /// If the ID is higher than the length of internal vectors, + /// the vectors will be extended. A call to set_size is not needed. + pub fn add_single<'a>(&mut self, storage: &'a hub::Storage, id: Id) -> Option<&'a T> { + let item = storage.get(id).ok()?; + + let (index32, epoch, _) = id.unzip(); + let index = index32 as usize; + + self.allow_index(index); + + self.debug_assert_in_bounds(index); + + unsafe { + *self.metadata.epochs.get_unchecked_mut(index) = epoch; + *self.metadata.ref_counts.get_unchecked_mut(index) = Some(item.life_guard().add_ref()); + self.metadata.owned.set(index, true); + } + + Some(item) + } + + /// Adds the given resources from the given tracker. + /// + /// If the ID is higher than the length of internal vectors, + /// the vectors will be extended. A call to set_size is not needed. + pub fn add_from_tracker(&mut self, other: &Self) { + let incoming_size = other.metadata.owned.len(); + if incoming_size > self.metadata.owned.len() { + self.set_size(incoming_size); + } + + for index in iterate_bitvec_indices(&other.metadata.owned) { + self.debug_assert_in_bounds(index); + other.debug_assert_in_bounds(index); + unsafe { + let previously_owned = self.metadata.owned.get(index).unwrap_unchecked(); + + if !previously_owned { + self.metadata.owned.set(index, true); + + let other_ref_count = other + .metadata + .ref_counts + .get_unchecked(index) + .clone() + .unwrap_unchecked(); + *self.metadata.ref_counts.get_unchecked_mut(index) = Some(other_ref_count); + + let epoch = *other.metadata.epochs.get_unchecked(index); + *self.metadata.epochs.get_unchecked_mut(index) = epoch; + } + } + } + } + + /// Removes the given resource from the tracker iff we have the last reference to the + /// resource and the epoch matches. + /// + /// Returns true if the resource was removed. + /// + /// If the ID is higher than the length of internal vectors, + /// false will be returned. + pub fn remove_abandoned(&mut self, id: Valid) -> bool { + let (index32, epoch, _) = id.0.unzip(); + let index = index32 as usize; + + if index > self.metadata.owned.len() { + return false; + } + + self.debug_assert_in_bounds(index); + + unsafe { + if self.metadata.owned.get(index).unwrap_unchecked() { + let existing_epoch = self.metadata.epochs.get_unchecked_mut(index); + let existing_ref_count = self.metadata.ref_counts.get_unchecked_mut(index); + + if *existing_epoch == epoch + && existing_ref_count.as_mut().unwrap_unchecked().load() == 1 + { + self.metadata.reset(index); + + return true; + } + } + } + + false + } +} diff --git a/wgpu-core/src/track/texture.rs b/wgpu-core/src/track/texture.rs index c519ee07a5..0f157c8f15 100644 --- a/wgpu-core/src/track/texture.rs +++ b/wgpu-core/src/track/texture.rs @@ -1,425 +1,1513 @@ -use super::{range::RangedStates, PendingTransition, ResourceState, Unit}; -use crate::id::{TextureId, Valid}; +/*! Texture Trackers + * + * Texture trackers are signifigantly more complicated than + * the buffer trackers because textures can be in a "complex" + * state where each individual subresource can potentially be + * in a different state from every other subtresource. These + * complex states are stored seperately from the simple states + * because they are signifignatly more difficult to track and + * most resources spend the vast majority of their lives in + * simple states. + * + * There are two special texture usages: `UNKNOWN` and `UNINITIALIZED`. + * - `UNKNOWN` is only used in complex states and is used to signify + * that the complex state does not know anything about those subresources. + * It cannot leak into transitions, it is invalid to transition into UNKNOWN + * state. + * - `UNINITIALIZED` is used in both simple and complex states to mean the texture + * is known to be in some undefined state. Any transition away from UNINITIALIZED + * will treat the contents as junk. +!*/ + +use super::{range::RangedStates, PendingTransition}; +use crate::{ + hub, + id::{TextureId, TypedId, Valid}, + resource::Texture, + track::{ + invalid_resource_state, iterate_bitvec_indices, skip_barrier, ResourceMetadata, + ResourceMetadataProvider, ResourceUses, UsageConflict, + }, + LifeGuard, RefCount, +}; use hal::TextureUses; use arrayvec::ArrayVec; +use naga::FastHashMap; -use std::{iter, ops::Range}; - -type PlaneStates = RangedStates>; +use std::{borrow::Cow, iter, marker::PhantomData, ops::Range, vec::Drain}; +/// Specifies a particular set of subresources in a texture. #[derive(Clone, Debug, PartialEq, Eq)] pub struct TextureSelector { - //TODO: rename to `mip_levels` and `array_layers` for consistency - //pub aspects: hal::FormatAspects, - pub levels: Range, + pub mips: Range, pub layers: Range, } -#[derive(Clone, Debug, Default, PartialEq)] -pub(crate) struct TextureState { - mips: ArrayVec, - /// True if we have the information about all the subresources here - full: bool, -} +impl ResourceUses for TextureUses { + const EXCLUSIVE: Self = Self::EXCLUSIVE; -impl PendingTransition { - fn collapse(self) -> Result { - if self.usage.start.is_empty() - || self.usage.start == self.usage.end - || !TextureUses::EXCLUSIVE.intersects(self.usage.start | self.usage.end) - { - Ok(self.usage.start | self.usage.end) - } else { - Err(self) - } + type Id = TextureId; + type Selector = TextureSelector; + + fn bits(self) -> u16 { + Self::bits(&self) + } + + fn all_ordered(self) -> bool { + Self::ORDERED.contains(self) + } + + fn any_exclusive(self) -> bool { + self.intersects(Self::EXCLUSIVE) } } -impl TextureState { - pub fn new(mip_level_count: u32, array_layer_count: u32) -> Self { +/// Represents the complex state of textures where every subresource is potentially +/// in a different state. +#[derive(Clone, Debug, Default, PartialEq)] +struct ComplexTextureState { + mips: ArrayVec, { hal::MAX_MIP_LEVELS as usize }>, +} + +impl ComplexTextureState { + /// Creates complex texture state for the given sizes. + /// + /// This state will be initialized with the UNKNOWN state, a special state + /// which means the trakcer knows nothing about the state. + fn new(mip_level_count: u32, array_layer_count: u32) -> Self { Self { mips: iter::repeat_with(|| { - PlaneStates::from_range(0..array_layer_count, Unit::new(TextureUses::UNINITIALIZED)) + RangedStates::from_range(0..array_layer_count, TextureUses::UNKNOWN) }) .take(mip_level_count as usize) .collect(), - full: true, } } -} -impl ResourceState for TextureState { - type Id = TextureId; - type Selector = TextureSelector; - type Usage = TextureUses; - - fn query(&self, selector: Self::Selector) -> Option { - let mut result = None; - // Note: we only consider the subresources tracked by `self`. - // If some are not known to `self`, it means the can assume the - // initial state to whatever we need, which we can always make - // to be the same as the query result for the known subresources. - let num_levels = self.mips.len(); - if self.full { - assert!(num_levels >= selector.levels.end as usize); - } - let mip_start = num_levels.min(selector.levels.start as usize); - let mip_end = num_levels.min(selector.levels.end as usize); - for mip in self.mips[mip_start..mip_end].iter() { - match mip.query(&selector.layers, |unit| unit.last) { - None => {} - Some(Ok(usage)) if result == Some(usage) => {} - Some(Ok(usage)) if result.is_none() => { - result = Some(usage); + /// Initialize a complex state from a selector representing the full size of the texture + /// and an iterator of a selector and a texture use, specifying a usage for a specific + /// set of subresources. + /// + /// [`Self::to_selector_state_iter`] can be used to create such an iterator. + /// + /// # Safety + /// + /// All selectors in the iterator must be inside of the full_range selector. + /// + /// The full range selector must have mips and layers start at 0. + unsafe fn from_selector_state_iter( + full_range: TextureSelector, + state_iter: impl Iterator, + ) -> Self { + debug_assert_eq!(full_range.layers.start, 0); + debug_assert_eq!(full_range.mips.start, 0); + + let mut complex = + ComplexTextureState::new(full_range.mips.len() as u32, full_range.layers.len() as u32); + for (selector, desired_state) in state_iter { + debug_assert!(selector.layers.end <= full_range.layers.end); + debug_assert!(selector.mips.end <= full_range.mips.end); + + // This should only ever happen with a wgpu bug, but let's just double + // check that resource states don't have any conflicts. + debug_assert_eq!(invalid_resource_state(desired_state), false); + + let mips = selector.mips.start as usize..selector.mips.end as usize; + for mip in complex.mips.get_unchecked_mut(mips) { + for &mut (_, ref mut state) in mip.isolate(&selector.layers, TextureUses::UNKNOWN) { + *state = desired_state; } - Some(Ok(_)) | Some(Err(())) => return None, } } - result + complex } - fn change( - &mut self, - id: Valid, - selector: Self::Selector, - usage: Self::Usage, - mut output: Option<&mut Vec>>, - ) -> Result<(), PendingTransition> { - if self.full { - assert!(self.mips.len() >= selector.levels.end as usize); - } else { - while self.mips.len() < selector.levels.end as usize { - self.mips.push(PlaneStates::empty()); + /// Convert a complex state into an iterator over all states stored. + /// + /// [`Self::from_selector_state_iter`] can be used to consume such an iterator. + fn to_selector_state_iter( + &self, + ) -> impl Iterator + Clone + '_ { + self.mips.iter().enumerate().flat_map(|(mip, inner)| { + let mip = mip as u32; + { + inner.iter().map(move |&(ref layers, inner)| { + ( + TextureSelector { + mips: mip..mip + 1, + layers: layers.clone(), + }, + inner, + ) + }) } + }) + } +} + +/// Stores all the textures that a bind group stores. +pub(crate) struct TextureBindGroupState { + textures: Vec<( + Valid, + Option, + RefCount, + TextureUses, + )>, + + _phantom: PhantomData, +} +impl TextureBindGroupState { + pub fn new() -> Self { + Self { + textures: Vec::new(), + + _phantom: PhantomData, + } + } + + /// Optimize the texture bind group state by sorting it by ID. + /// + /// When this list of states is merged into a tracker, the memory + /// accesses will be in a constant assending order. + pub(crate) fn optimize(&mut self) { + self.textures + .sort_unstable_by_key(|&(id, _, _, _)| id.0.unzip().0); + } + + /// Returns a list of all buffers tracked. May contain duplicates. + pub fn used(&self) -> impl Iterator> + '_ { + self.textures.iter().map(|&(id, _, _, _)| id) + } + + /// Adds the given resource with the given state. + pub fn add_single<'a>( + &mut self, + storage: &'a hub::Storage, TextureId>, + id: TextureId, + ref_count: RefCount, + selector: Option, + state: TextureUses, + ) -> Option<&'a Texture> { + let value = storage.get(id).ok()?; + + self.textures.push((Valid(id), selector, ref_count, state)); + + Some(value) + } +} + +/// Container for corresponding simple and complex texture states. +#[derive(Debug)] +pub(crate) struct TextureStateSet { + simple: Vec, + complex: FastHashMap, +} +impl TextureStateSet { + fn new() -> Self { + Self { + simple: Vec::new(), + complex: FastHashMap::default(), + } + } + + fn set_size(&mut self, size: usize) { + self.simple.resize(size, TextureUses::UNINITIALIZED); + } +} + +/// Stores all texture state within a single usage scope. +#[derive(Debug)] +pub(crate) struct TextureUsageScope { + set: TextureStateSet, + + metadata: ResourceMetadata, +} + +impl TextureUsageScope { + pub fn new() -> Self { + Self { + set: TextureStateSet::new(), + + metadata: ResourceMetadata::new(), } - for (mip_id, mip) in self.mips[selector.levels.start as usize..selector.levels.end as usize] - .iter_mut() - .enumerate() + } + + fn debug_assert_in_bounds(&self, index: usize) { + self.metadata.debug_assert_in_bounds(index); + + debug_assert!(index < self.set.simple.len()); + + debug_assert!(if self.metadata.owned.get(index).unwrap() + && self.set.simple[index] == TextureUses::COMPLEX { - let level = selector.levels.start + mip_id as u32; - let layers = mip.isolate(&selector.layers, Unit::new(usage)); - for &mut (ref range, ref mut unit) in layers { - if unit.last == usage && TextureUses::ORDERED.contains(usage) { - continue; - } - // TODO: Can't satisfy clippy here unless we modify - // `TextureSelector` to use `std::ops::RangeBounds`. - #[allow(clippy::range_plus_one)] - let pending = PendingTransition { - id, - selector: TextureSelector { - levels: level..level + 1, - layers: range.clone(), + self.set.complex.contains_key(&(index as u32)) + } else { + true + }); + } + + /// Sets the size of all the vectors inside the tracker. + /// + /// Must be called with the highest possible Texture ID before + /// all unsafe functions are called. + pub fn set_size(&mut self, size: usize) { + self.set.set_size(size); + self.metadata.set_size(size); + } + + /// Returns a list of all textures tracked. + pub fn used(&self) -> impl Iterator> + '_ { + self.metadata.used() + } + + /// Returns true if the tracker owns no resources. + /// + /// This is a O(n) operation. + pub(crate) fn is_empty(&self) -> bool { + self.metadata.is_empty() + } + + /// Merge the list of texture states in the given usage scope into this UsageScope. + /// + /// If any of the resulting states is invalid, stops the merge and returns a usage + /// conflict with the details of the invalid state. + /// + /// If the given tracker uses IDs higher than the length of internal vectors, + /// the vectors will be extended. A call to set_size is not needed. + pub fn merge_usage_scope( + &mut self, + storage: &hub::Storage, TextureId>, + scope: &Self, + ) -> Result<(), UsageConflict> { + let incoming_size = scope.set.simple.len(); + if incoming_size > self.set.simple.len() { + self.set_size(incoming_size); + } + + for index in iterate_bitvec_indices(&scope.metadata.owned) { + let index32 = index as u32; + + self.debug_assert_in_bounds(index); + scope.debug_assert_in_bounds(index); + + unsafe { + insert_or_merge( + texture_data_from_texture(storage, index32), + &mut self.set, + &mut self.metadata, + index32, + index, + TextureStateProvider::TextureSet { set: &scope.set }, + ResourceMetadataProvider::Indirect { + metadata: &scope.metadata, }, - usage: unit.last..usage, - }; - - *unit = match output { - None => { - assert_eq!( - unit.first, None, - "extending a state that is already a transition" - ); - Unit::new(pending.collapse()?) - } - Some(ref mut out) => { - out.push(pending); - Unit { - first: unit.first.or(Some(unit.last)), - last: usage, - } - } - }; - } + )? + }; } + Ok(()) } - fn merge( + /// Merge the list of texture states in the given bind group into this usage scope. + /// + /// If any of the resulting states is invalid, stops the merge and returns a usage + /// conflict with the details of the invalid state. + /// + /// Because bind groups do not check if the union of all their states is valid, + /// this method is allowed to return Err on the first bind group bound. + /// + /// # Safety + /// + /// [`Self::set_size`] must be called with the maximum possible Buffer ID before this + /// method is called. + pub unsafe fn merge_bind_group( &mut self, - id: Valid, - other: &Self, - mut output: Option<&mut Vec>>, - ) -> Result<(), PendingTransition> { - let mut temp = Vec::new(); - if self.full { - assert!(self.mips.len() >= other.mips.len()); - } else { - while self.mips.len() < other.mips.len() { - self.mips.push(PlaneStates::empty()); - } + storage: &hub::Storage, TextureId>, + bind_group: &TextureBindGroupState, + ) -> Result<(), UsageConflict> { + for &(id, ref selector, ref ref_count, state) in &bind_group.textures { + self.merge_single(storage, id, selector.clone(), ref_count, state)?; } - for (mip_id, (mip_self, mip_other)) in self.mips.iter_mut().zip(&other.mips).enumerate() { - let level = mip_id as u32; - temp.extend(mip_self.merge(mip_other, 0)); - mip_self.clear(); - - for (layers, states) in temp.drain(..) { - let unit = match states { - Range { - start: None, - end: None, - } => unreachable!(), - Range { - start: Some(start), - end: None, - } => start, - Range { - start: None, - end: Some(end), - } => end, - Range { - start: Some(start), - end: Some(end), - } => { - let to_usage = end.port(); - if start.last == to_usage && TextureUses::ORDERED.contains(to_usage) { - Unit { - first: match output { - None => start.first, - Some(_) => start.first.or(Some(start.last)), - }, - last: end.last, - } - } else { - // TODO: Can't satisfy clippy here unless we modify - // `TextureSelector` to use `std::ops::RangeBounds`. - #[allow(clippy::range_plus_one)] - let pending = PendingTransition { - id, - selector: TextureSelector { - levels: level..level + 1, - layers: layers.clone(), - }, - usage: start.last..to_usage, - }; - - match output { - None => { - assert_eq!( - start.first, None, - "extending a state that is already a transition" - ); - Unit::new(pending.collapse()?) - } - Some(ref mut out) => { - out.push(pending); - Unit { - // this has to leave a valid `first` state - first: start.first.or(Some(start.last)), - last: end.last, - } - } - } - } - } - }; - mip_self.append(layers, unit); - } - } + Ok(()) + } + + /// Merge a single state into the UsageScope. + /// + /// If the resulting state is invalid, returns a usage + /// conflict with the details of the invalid state. + /// + /// # Safety + /// + /// Unlike other trackers whose merge_single is safe, this method is only + /// called where there is already other unsafe tracking functions active, + /// so we can prove this unsafe "for free". + /// + /// [`Self::set_size`] must be called with the maximum possible Buffer ID before this + /// method is called. + pub unsafe fn merge_single( + &mut self, + storage: &hub::Storage, TextureId>, + id: Valid, + selector: Option, + ref_count: &RefCount, + new_state: TextureUses, + ) -> Result<(), UsageConflict> { + let (index32, epoch, _) = id.0.unzip(); + let index = index32 as usize; + + self.debug_assert_in_bounds(index); + + insert_or_merge( + texture_data_from_texture(storage, index32), + &mut self.set, + &mut self.metadata, + index32, + index, + TextureStateProvider::from_option(selector, new_state), + ResourceMetadataProvider::Direct { + epoch, + ref_count: Cow::Borrowed(ref_count), + }, + )?; Ok(()) } +} + +/// Stores all texture state within a command buffer or device. +pub(crate) struct TextureTracker { + start_set: TextureStateSet, + end_set: TextureStateSet, + + metadata: ResourceMetadata, + + temp: Vec>, + + _phantom: PhantomData, +} +impl TextureTracker { + pub fn new() -> Self { + Self { + start_set: TextureStateSet::new(), + end_set: TextureStateSet::new(), - fn optimize(&mut self) { - for mip in self.mips.iter_mut() { - mip.coalesce(); + metadata: ResourceMetadata::new(), + + temp: Vec::new(), + + _phantom: PhantomData, } } -} -#[cfg(test)] -mod test { - //TODO: change() tests - use super::*; - use crate::id::Id; - - #[test] - fn query() { - let mut ts = TextureState::default(); - ts.mips.push(PlaneStates::empty()); - ts.mips.push(PlaneStates::from_slice(&[ - (1..3, Unit::new(TextureUses::RESOURCE)), - (3..5, Unit::new(TextureUses::RESOURCE)), - (5..6, Unit::new(TextureUses::STORAGE_READ)), - ])); - - assert_eq!( - ts.query(TextureSelector { - levels: 1..2, - layers: 2..5, - }), - // level 1 matches - Some(TextureUses::RESOURCE), - ); - assert_eq!( - ts.query(TextureSelector { - levels: 0..2, - layers: 2..5, - }), - // level 0 is empty, level 1 matches - Some(TextureUses::RESOURCE), - ); - assert_eq!( - ts.query(TextureSelector { - levels: 1..2, - layers: 1..5, - }), - // level 1 matches with gaps - Some(TextureUses::RESOURCE), - ); - assert_eq!( - ts.query(TextureSelector { - levels: 1..2, - layers: 4..6, - }), - // level 1 doesn't match - None, - ); + fn debug_assert_in_bounds(&self, index: usize) { + self.metadata.debug_assert_in_bounds(index); + + debug_assert!(index < self.start_set.simple.len()); + debug_assert!(index < self.end_set.simple.len()); + + debug_assert!(if self.metadata.owned.get(index).unwrap() + && self.start_set.simple[index] == TextureUses::COMPLEX + { + self.start_set.complex.contains_key(&(index as u32)) + } else { + true + }); + debug_assert!(if self.metadata.owned.get(index).unwrap() + && self.end_set.simple[index] == TextureUses::COMPLEX + { + self.end_set.complex.contains_key(&(index as u32)) + } else { + true + }); } - #[test] - fn merge() { - let id = Id::dummy(); - let mut ts1 = TextureState::default(); - ts1.mips.push(PlaneStates::from_slice(&[( - 1..3, - Unit::new(TextureUses::RESOURCE), - )])); - let mut ts2 = TextureState::default(); - assert_eq!( - ts1.merge(id, &ts2, None), - Ok(()), - "failed to merge with an empty" - ); + /// Sets the size of all the vectors inside the tracker. + /// + /// Must be called with the highest possible Texture ID before + /// all unsafe functions are called. + pub fn set_size(&mut self, size: usize) { + self.start_set.set_size(size); + self.end_set.set_size(size); - ts2.mips.push(PlaneStates::from_slice(&[( - 1..2, - Unit::new(TextureUses::COPY_SRC), - )])); - assert_eq!( - ts1.merge(Id::dummy(), &ts2, None), - Ok(()), - "failed to extend a compatible state" - ); - assert_eq!( - ts1.mips[0].query(&(1..2), |&v| v), - Some(Ok(Unit { - first: None, - last: TextureUses::RESOURCE | TextureUses::COPY_SRC, - })), - "wrong extension result" - ); + self.metadata.set_size(size); + } - ts2.mips[0] = PlaneStates::from_slice(&[(1..2, Unit::new(TextureUses::COPY_DST))]); - assert_eq!( - ts1.clone().merge(Id::dummy(), &ts2, None), - Err(PendingTransition { - id, - selector: TextureSelector { - levels: 0..1, - layers: 1..2, + /// Extend the vectors to let the given index be valid. + fn allow_index(&mut self, index: usize) { + if index >= self.start_set.simple.len() { + self.set_size(index + 1); + } + } + + /// Returns a list of all textures tracked. + pub fn used(&self) -> impl Iterator> + '_ { + self.metadata.used() + } + + /// Drains all currently pending transitions. + pub fn drain(&mut self) -> Drain> { + self.temp.drain(..) + } + + /// Get the refcount of the given resource. + /// + /// # Safety + /// + /// [`Self::set_size`] must be called with the maximum possible Buffer ID before this + /// method is called. + /// + /// The resource must be tracked by this tracker. + pub unsafe fn get_ref_count(&self, id: Valid) -> &RefCount { + let (index32, _, _) = id.0.unzip(); + let index = index32 as usize; + + self.debug_assert_in_bounds(index); + + self.metadata + .ref_counts + .get_unchecked(index) + .as_ref() + .unwrap_unchecked() + } + + /// Inserts a single texture and a state into the resource tracker. + /// + /// If the resource already exists in the tracker, this will panic. + /// + /// If the ID is higher than the length of internal vectors, + /// the vectors will be extended. A call to set_size is not needed. + pub fn insert_single(&mut self, id: TextureId, ref_count: RefCount, usage: TextureUses) { + let (index32, epoch, _) = id.unzip(); + let index = index32 as usize; + + self.allow_index(index); + + self.debug_assert_in_bounds(index); + + unsafe { + let currently_owned = self.metadata.owned.get(index).unwrap_unchecked(); + + if currently_owned { + panic!("Tried to insert texture already tracked"); + } + + insert( + None, + Some(&mut self.start_set), + &mut self.end_set, + &mut self.metadata, + index32, + index, + TextureStateProvider::KnownSingle { state: usage }, + None, + ResourceMetadataProvider::Direct { + epoch, + ref_count: Cow::Owned(ref_count), }, - usage: TextureUses::RESOURCE | TextureUses::COPY_SRC..TextureUses::COPY_DST, - }), - "wrong error on extending with incompatible state" - ); + ) + }; + } + + /// Sets the state of a single texture. + /// + /// If a transition is needed to get the texture into the given state, that transition + /// is returned. + /// + /// If the ID is higher than the length of internal vectors, + /// the vectors will be extended. A call to set_size is not needed. + pub fn set_single<'a>( + &mut self, + storage: &'a hub::Storage, TextureId>, + id: TextureId, + selector: TextureSelector, + new_state: TextureUses, + ) -> Option<(&'a Texture, Drain<'_, PendingTransition>)> { + let texture = storage.get(id).ok()?; + + let (index32, epoch, _) = id.unzip(); + let index = index32 as usize; + + self.allow_index(index); - let mut list = Vec::new(); - ts2.mips[0] = PlaneStates::from_slice(&[ - (1..2, Unit::new(TextureUses::COPY_DST)), - ( - 2..3, - Unit { - first: Some(TextureUses::COPY_SRC), - last: TextureUses::COLOR_TARGET, + self.debug_assert_in_bounds(index); + + unsafe { + insert_or_barrier_update( + texture_data_from_texture(storage, index32), + Some(&mut self.start_set), + &mut self.end_set, + &mut self.metadata, + index32, + index, + TextureStateProvider::Selector { + selector, + state: new_state, }, - ), - ]); - ts1.merge(Id::dummy(), &ts2, Some(&mut list)).unwrap(); - assert_eq!( - &list, - &[ - PendingTransition { - id, - selector: TextureSelector { - levels: 0..1, - layers: 1..2, + None, + ResourceMetadataProvider::Resource { epoch }, + &mut self.temp, + ) + } + + Some((texture, self.temp.drain(..))) + } + + /// Sets the given state for all texture in the given tracker. + /// + /// If a transition is needed to get the texture into the needed state, + /// those transitions are stored within the tracker. A subsequent + /// call to [`Self::drain`] is needed to get those transitions. + /// + /// If the ID is higher than the length of internal vectors, + /// the vectors will be extended. A call to set_size is not needed. + pub fn set_from_tracker( + &mut self, + storage: &hub::Storage, TextureId>, + tracker: &Self, + ) { + let incoming_size = tracker.start_set.simple.len(); + if incoming_size > self.start_set.simple.len() { + self.set_size(incoming_size); + } + + for index in iterate_bitvec_indices(&tracker.metadata.owned) { + let index32 = index as u32; + + self.debug_assert_in_bounds(index); + tracker.debug_assert_in_bounds(index); + unsafe { + insert_or_barrier_update( + texture_data_from_texture(storage, index32), + Some(&mut self.start_set), + &mut self.end_set, + &mut self.metadata, + index32, + index, + TextureStateProvider::TextureSet { + set: &tracker.start_set, }, - usage: TextureUses::RESOURCE | TextureUses::COPY_SRC..TextureUses::COPY_DST, - }, - PendingTransition { - id, - selector: TextureSelector { - levels: 0..1, - layers: 2..3, + Some(TextureStateProvider::TextureSet { + set: &tracker.end_set, + }), + ResourceMetadataProvider::Indirect { + metadata: &tracker.metadata, }, - // the transition links the end of the base rage (..SAMPLED) - // with the start of the next range (COPY_SRC..) - usage: TextureUses::RESOURCE..TextureUses::COPY_SRC, - }, - ], - "replacing produced wrong transitions" - ); - assert_eq!( - ts1.mips[0].query(&(1..2), |&v| v), - Some(Ok(Unit { - first: Some(TextureUses::RESOURCE | TextureUses::COPY_SRC), - last: TextureUses::COPY_DST, - })), - "wrong final layer 1 state" - ); - assert_eq!( - ts1.mips[0].query(&(2..3), |&v| v), - Some(Ok(Unit { - first: Some(TextureUses::RESOURCE), - last: TextureUses::COLOR_TARGET, - })), - "wrong final layer 2 state" - ); + &mut self.temp, + ); + } + } + } - list.clear(); - ts2.mips[0] = PlaneStates::from_slice(&[( - 2..3, - Unit { - first: Some(TextureUses::COLOR_TARGET), - last: TextureUses::COPY_SRC, - }, - )]); - ts1.merge(Id::dummy(), &ts2, Some(&mut list)).unwrap(); - assert_eq!(&list, &[], "unexpected replacing transition"); - - list.clear(); - ts2.mips[0] = PlaneStates::from_slice(&[( - 2..3, - Unit { - first: Some(TextureUses::COPY_DST), - last: TextureUses::COPY_DST, - }, - )]); - ts1.merge(Id::dummy(), &ts2, Some(&mut list)).unwrap(); - assert_eq!( - &list, - &[PendingTransition { - id, - selector: TextureSelector { - levels: 0..1, - layers: 2..3, + /// Sets the given state for all textures in the given UsageScope. + /// + /// If a transition is needed to get the textures into the needed state, + /// those transitions are stored within the tracker. A subsequent + /// call to [`Self::drain`] is needed to get those transitions. + /// + /// If the ID is higher than the length of internal vectors, + /// the vectors will be extended. A call to set_size is not needed. + pub fn set_from_usage_scope( + &mut self, + storage: &hub::Storage, TextureId>, + scope: &TextureUsageScope, + ) { + let incoming_size = scope.set.simple.len(); + if incoming_size > self.start_set.simple.len() { + self.set_size(incoming_size); + } + + for index in iterate_bitvec_indices(&scope.metadata.owned) { + let index32 = index as u32; + + self.debug_assert_in_bounds(index); + scope.debug_assert_in_bounds(index); + unsafe { + insert_or_barrier_update( + texture_data_from_texture(storage, index32), + Some(&mut self.start_set), + &mut self.end_set, + &mut self.metadata, + index32, + index, + TextureStateProvider::TextureSet { set: &scope.set }, + None, + ResourceMetadataProvider::Indirect { + metadata: &scope.metadata, + }, + &mut self.temp, + ); + } + } + } + + /// Iterates through all textures in the given bind group and adopts + /// the state given for those textures in the UsageScope. It also + /// removes all touched textures from the usage scope. + /// + /// If a transition is needed to get the textures into the needed state, + /// those transitions are stored within the tracker. A subsequent + /// call to [`Self::drain`] is needed to get those transitions. + /// + /// This is a really funky method used by Compute Passes to generate + /// barriers after a call to dispatch without needing to iterate + /// over all elements in the usage scope. We use each the + /// bind group as a source of which IDs to look at. The bind groups + /// must have first been added to the usage scope. + /// + /// # Safety + /// + /// [`Self::set_size`] must be called with the maximum possible Buffer ID before this + /// method is called. + pub unsafe fn set_and_remove_from_usage_scope_sparse( + &mut self, + storage: &hub::Storage, TextureId>, + scope: &mut TextureUsageScope, + bind_group_state: &TextureBindGroupState, + ) { + let incoming_size = scope.set.simple.len(); + if incoming_size > self.start_set.simple.len() { + self.set_size(incoming_size); + } + + for &(id, _, _, _) in bind_group_state.textures.iter() { + let (index32, _, _) = id.0.unzip(); + let index = index32 as usize; + scope.debug_assert_in_bounds(index); + + if !scope.metadata.owned.get(index).unwrap_unchecked() { + continue; + } + insert_or_barrier_update( + texture_data_from_texture(storage, index32), + Some(&mut self.start_set), + &mut self.end_set, + &mut self.metadata, + index32, + index, + TextureStateProvider::TextureSet { set: &scope.set }, + None, + ResourceMetadataProvider::Indirect { + metadata: &scope.metadata, }, - usage: TextureUses::COPY_SRC..TextureUses::COPY_DST, - },], - "invalid replacing transition" + &mut self.temp, + ); + + scope.metadata.reset(index); + } + } + + /// Unconditionally removes the given resource from the tracker. + /// + /// Returns true if the resource was removed. + /// + /// If the ID is higher than the length of internal vectors, + /// false will be returned. + pub fn remove(&mut self, id: Valid) -> bool { + let (index32, epoch, _) = id.0.unzip(); + let index = index32 as usize; + + if index > self.metadata.owned.len() { + return false; + } + + self.debug_assert_in_bounds(index); + + unsafe { + if self.metadata.owned.get(index).unwrap_unchecked() { + let existing_epoch = *self.metadata.epochs.get_unchecked_mut(index); + assert_eq!(existing_epoch, epoch); + + self.start_set.complex.remove(&index32); + self.end_set.complex.remove(&index32); + + self.metadata.reset(index); + + return true; + } + } + + false + } + + /// Removes the given resource from the tracker iff we have the last reference to the + /// resource and the epoch matches. + /// + /// Returns true if the resource was removed. + /// + /// If the ID is higher than the length of internal vectors, + /// false will be returned. + pub fn remove_abandoned(&mut self, id: Valid) -> bool { + let (index32, epoch, _) = id.0.unzip(); + let index = index32 as usize; + + if index > self.metadata.owned.len() { + return false; + } + + self.debug_assert_in_bounds(index); + + unsafe { + if self.metadata.owned.get(index).unwrap_unchecked() { + let existing_epoch = self.metadata.epochs.get_unchecked_mut(index); + let existing_ref_count = self.metadata.ref_counts.get_unchecked_mut(index); + + if *existing_epoch == epoch + && existing_ref_count.as_mut().unwrap_unchecked().load() == 1 + { + self.start_set.complex.remove(&index32); + self.end_set.complex.remove(&index32); + + self.metadata.reset(index); + + return true; + } + } + } + + false + } +} + +/// An iterator adapter that can store two different iterator types. +#[derive(Clone)] +enum EitherIter { + Left(L), + Right(R), +} + +impl Iterator for EitherIter +where + L: Iterator, + R: Iterator, +{ + type Item = D; + + fn next(&mut self) -> Option { + match *self { + EitherIter::Left(ref mut inner) => inner.next(), + EitherIter::Right(ref mut inner) => inner.next(), + } + } +} + +/// Container that signifies storing both different things +/// if there is a single state or many different states +/// involved in the operation. +#[derive(Debug, Clone)] +enum SingleOrManyStates { + Single(S), + Many(M), +} + +/// A source of texture state. +#[derive(Clone)] +enum TextureStateProvider<'a> { + /// Comes directly from a single state. + KnownSingle { state: TextureUses }, + /// Comes from a selector and a single state. + Selector { + selector: TextureSelector, + state: TextureUses, + }, + /// Comes from another texture set. + TextureSet { set: &'a TextureStateSet }, +} +impl<'a> TextureStateProvider<'a> { + /// Convenience function turning Option into this enum. + fn from_option(selector: Option, state: TextureUses) -> Self { + match selector { + Some(selector) => Self::Selector { selector, state }, + None => Self::KnownSingle { state }, + } + } + + /// Get the state provided by this. + /// + /// # Panics + /// + /// Panics if texture_data is None and this uses a Selector source. + /// + /// # Safety + /// + /// - The index must be in bounds of the state set if this uses an TextureSet source. + #[inline(always)] + unsafe fn get_state( + self, + texture_data: Option<(&LifeGuard, &TextureSelector)>, + index32: u32, + index: usize, + ) -> SingleOrManyStates< + TextureUses, + impl Iterator + Clone + 'a, + > { + match self { + TextureStateProvider::KnownSingle { state } => SingleOrManyStates::Single(state), + TextureStateProvider::Selector { selector, state } => { + // We check if the selector given is actually for the full resource, + // and if it is we promote to a simple state. This allows upstream + // code to specify selectors willy nilly, and all that are really + // single states are promoted here. + if *texture_data.unwrap().1 == selector { + SingleOrManyStates::Single(state) + } else { + SingleOrManyStates::Many(EitherIter::Left(iter::once((selector, state)))) + } + } + TextureStateProvider::TextureSet { set } => { + let new_state = *set.simple.get_unchecked(index); + + if new_state == TextureUses::COMPLEX { + let new_complex = set.complex.get(&index32).unwrap_unchecked(); + + SingleOrManyStates::Many(EitherIter::Right( + new_complex.to_selector_state_iter(), + )) + } else { + SingleOrManyStates::Single(new_state) + } + } + } + } +} + +/// Helper function that gets what is needed from the texture storage +/// out of the texture storage. +#[inline(always)] +unsafe fn texture_data_from_texture( + storage: &hub::Storage, TextureId>, + index32: u32, +) -> (&LifeGuard, &TextureSelector) { + let texture = storage.get_unchecked(index32); + (&texture.life_guard, &texture.full_range) +} + +/// Does an insertion operation if the index isn't tracked +/// in the current metadata, otherwise merges the given state +/// with the current state. If the merging would cause +/// a conflict, returns that usage conflict. +/// +/// # Safety +/// +/// Indexes must be valid indexes into all arrays passed in +/// to this function, either directly or via metadata or provider structs. +#[inline(always)] +unsafe fn insert_or_merge( + texture_data: (&LifeGuard, &TextureSelector), + current_state_set: &mut TextureStateSet, + resource_metadata: &mut ResourceMetadata, + index32: u32, + index: usize, + state_provider: TextureStateProvider<'_>, + metadata_provider: ResourceMetadataProvider<'_, A>, +) -> Result<(), UsageConflict> { + let currently_owned = resource_metadata.owned.get(index).unwrap_unchecked(); + + if !currently_owned { + insert( + Some(texture_data), + None, + current_state_set, + resource_metadata, + index32, + index, + state_provider, + None, + metadata_provider, ); - assert_eq!( - ts1.mips[0].query(&(2..3), |&v| v), - Some(Ok(Unit { - // the initial state here is never expected to change - first: Some(TextureUses::RESOURCE), - last: TextureUses::COPY_DST, - })), - "wrong final layer 2 state" + return Ok(()); + } + + merge( + texture_data, + current_state_set, + index32, + index, + state_provider, + metadata_provider, + ) +} + +/// If the resource isn't tracked +/// - Inserts the given resource. +/// - Uses the `start_state_provider` to populate `start_states` +/// - Uses either `end_state_provider` or `start_state_provider` +/// to populate `current_states`. +/// If the resource is tracked +/// - Inserts barriers from the state in `current_states` +/// to the state provided by `start_state_provider`. +/// - Updates the `current_states` with either the state from +/// `end_state_provider` or `start_state_provider`. +/// +/// Any barriers are added to the barrier vector. +/// +/// # Safety +/// +/// Indexes must be valid indexes into all arrays passed in +/// to this function, either directly or via metadata or provider structs. +#[inline(always)] +unsafe fn insert_or_barrier_update( + texture_data: (&LifeGuard, &TextureSelector), + start_state: Option<&mut TextureStateSet>, + current_state_set: &mut TextureStateSet, + resource_metadata: &mut ResourceMetadata, + index32: u32, + index: usize, + start_state_provider: TextureStateProvider<'_>, + end_state_provider: Option>, + metadata_provider: ResourceMetadataProvider<'_, A>, + barriers: &mut Vec>, +) { + let currently_owned = resource_metadata.owned.get(index).unwrap_unchecked(); + + if !currently_owned { + insert( + Some(texture_data), + start_state, + current_state_set, + resource_metadata, + index32, + index, + start_state_provider, + end_state_provider, + metadata_provider, ); + return; + } + + let update_state_provider = end_state_provider.unwrap_or_else(|| start_state_provider.clone()); + barrier( + texture_data, + current_state_set, + index32, + index, + start_state_provider, + barriers, + ); + + let start_state_set = start_state.unwrap(); + update( + texture_data, + start_state_set, + current_state_set, + index32, + index, + update_state_provider, + ); +} + +#[inline(always)] +unsafe fn insert( + texture_data: Option<(&LifeGuard, &TextureSelector)>, + start_state: Option<&mut TextureStateSet>, + end_state: &mut TextureStateSet, + resource_metadata: &mut ResourceMetadata, + index32: u32, + index: usize, + start_state_provider: TextureStateProvider<'_>, + end_state_provider: Option>, + metadata_provider: ResourceMetadataProvider<'_, A>, +) { + let start_layers = start_state_provider.get_state(texture_data, index32, index); + match start_layers { + SingleOrManyStates::Single(state) => { + // This should only ever happen with a wgpu bug, but let's just double + // check that resource states don't have any conflicts. + debug_assert_eq!(invalid_resource_state(state), false); + + log::trace!("\ttex {index32}: insert start {state:?}"); + + if let Some(start_state) = start_state { + *start_state.simple.get_unchecked_mut(index) = state; + } + + // We only need to insert ourselves the end state if there is no end state provider. + if end_state_provider.is_none() { + *end_state.simple.get_unchecked_mut(index) = state; + } + } + SingleOrManyStates::Many(state_iter) => { + let full_range = texture_data.unwrap().1.clone(); + + let complex = ComplexTextureState::from_selector_state_iter(full_range, state_iter); + + log::trace!("\ttex {index32}: insert start {complex:?}"); + + if let Some(start_state) = start_state { + *start_state.simple.get_unchecked_mut(index) = TextureUses::COMPLEX; + start_state.complex.insert(index32, complex.clone()); + } + + // We only need to insert ourselves the end state if there is no end state provider. + if end_state_provider.is_none() { + *end_state.simple.get_unchecked_mut(index) = TextureUses::COMPLEX; + end_state.complex.insert(index32, complex); + } + } + } + + if let Some(end_state_provider) = end_state_provider { + match end_state_provider.get_state(texture_data, index32, index) { + SingleOrManyStates::Single(state) => { + // This should only ever happen with a wgpu bug, but let's just double + // check that resource states don't have any conflicts. + debug_assert_eq!(invalid_resource_state(state), false); + + log::trace!("\ttex {index32}: insert end {state:?}"); + + // We only need to insert into the end, as there is guarenteed to be + // a start state provider. + *end_state.simple.get_unchecked_mut(index) = state; + } + SingleOrManyStates::Many(state_iter) => { + let full_range = texture_data.unwrap().1.clone(); + + let complex = ComplexTextureState::from_selector_state_iter(full_range, state_iter); + + log::trace!("\ttex {index32}: insert end {complex:?}"); + + // We only need to insert into the end, as there is guarenteed to be + // a start state provider. + *end_state.simple.get_unchecked_mut(index) = TextureUses::COMPLEX; + end_state.complex.insert(index32, complex); + } + } + } + + let (epoch, ref_count) = + metadata_provider.get_own(texture_data.map(|(life_guard, _)| life_guard), index); + + resource_metadata.owned.set(index, true); + *resource_metadata.epochs.get_unchecked_mut(index) = epoch; + *resource_metadata.ref_counts.get_unchecked_mut(index) = Some(ref_count); +} + +#[inline(always)] +unsafe fn merge( + texture_data: (&LifeGuard, &TextureSelector), + current_state_set: &mut TextureStateSet, + index32: u32, + index: usize, + state_provider: TextureStateProvider<'_>, + metadata_provider: ResourceMetadataProvider<'_, A>, +) -> Result<(), UsageConflict> { + let current_simple = current_state_set.simple.get_unchecked_mut(index); + let current_state = if *current_simple == TextureUses::COMPLEX { + SingleOrManyStates::Many( + current_state_set + .complex + .get_mut(&index32) + .unwrap_unchecked(), + ) + } else { + SingleOrManyStates::Single(current_simple) + }; + + let new_state = state_provider.get_state(Some(texture_data), index32, index); + + match (current_state, new_state) { + (SingleOrManyStates::Single(current_simple), SingleOrManyStates::Single(new_simple)) => { + let merged_state = *current_simple | new_simple; + + log::trace!("\ttex {index32}: merge simple {current_simple:?} + {new_simple:?}"); + + if invalid_resource_state(merged_state) { + return Err(UsageConflict::from_texture( + TextureId::zip(index32, metadata_provider.get_epoch(index), A::VARIANT), + texture_data.1.clone(), + *current_simple, + new_simple, + )); + } + + *current_simple = merged_state; + } + (SingleOrManyStates::Single(current_simple), SingleOrManyStates::Many(new_many)) => { + // Because we are now demoting this simple state to a complex state, we actually need to make a whole + // new complex state for us to use as there wasn't one before. + let mut new_complex = ComplexTextureState::from_selector_state_iter( + texture_data.1.clone(), + iter::once((texture_data.1.clone(), *current_simple)), + ); + + for (selector, new_state) in new_many { + let merged_state = *current_simple | new_state; + + log::trace!( + "\ttex {index32}: merge {selector:?} {current_simple:?} + {new_state:?}" + ); + + if invalid_resource_state(merged_state) { + return Err(UsageConflict::from_texture( + TextureId::zip(index32, metadata_provider.get_epoch(index), A::VARIANT), + selector, + *current_simple, + new_state, + )); + } + + for mip in + &mut new_complex.mips[selector.mips.start as usize..selector.mips.end as usize] + { + for &mut (_, ref mut current_layer_state) in + mip.isolate(&selector.layers, TextureUses::UNKNOWN) + { + *current_layer_state = merged_state; + } + + mip.coalesce(); + } + } + + *current_simple = TextureUses::COMPLEX; + current_state_set.complex.insert(index32, new_complex); + } + (SingleOrManyStates::Many(current_complex), SingleOrManyStates::Single(new_simple)) => { + for (mip_id, mip) in current_complex.mips.iter_mut().enumerate() { + let mip_id = mip_id as u32; + + for &mut (ref layers, ref mut current_layer_state) in mip.iter_mut() { + let merged_state = *current_layer_state | new_simple; + + // Once we remove unknown, this will never be empty, as simple states are never unknown. + let merged_state = merged_state - TextureUses::UNKNOWN; + + log::trace!( + "\ttex {index32}: merge mip {mip_id} layers {layers:?} \ + {current_layer_state:?} + {new_simple:?}" + ); + + if invalid_resource_state(merged_state) { + return Err(UsageConflict::from_texture( + TextureId::zip(index32, metadata_provider.get_epoch(index), A::VARIANT), + TextureSelector { + mips: mip_id..mip_id + 1, + layers: layers.clone(), + }, + *current_layer_state, + new_simple, + )); + } + + *current_layer_state = merged_state; + } + + mip.coalesce(); + } + } + (SingleOrManyStates::Many(current_complex), SingleOrManyStates::Many(new_many)) => { + for (selector, new_state) in new_many { + for mip_id in selector.mips { + debug_assert!((mip_id as usize) < current_complex.mips.len()); + + let mip = current_complex.mips.get_unchecked_mut(mip_id as usize); + + for &mut (ref layers, ref mut current_layer_state) in + mip.isolate(&selector.layers, TextureUses::UNKNOWN) + { + let merged_state = *current_layer_state | new_state; + let merged_state = merged_state - TextureUses::UNKNOWN; + + if merged_state.is_empty() { + // We know nothing about this state, lets just move on. + continue; + } + + log::trace!( + "\ttex {index32}: merge mip {mip_id} layers {layers:?} \ + {current_layer_state:?} + {new_state:?}" + ); + + if invalid_resource_state(merged_state) { + return Err(UsageConflict::from_texture( + TextureId::zip( + index32, + metadata_provider.get_epoch(index), + A::VARIANT, + ), + TextureSelector { + mips: mip_id..mip_id + 1, + layers: layers.clone(), + }, + *current_layer_state, + new_state, + )); + } + *current_layer_state = merged_state; + } + + mip.coalesce(); + } + } + } + } + Ok(()) +} + +#[inline(always)] +unsafe fn barrier( + texture_data: (&LifeGuard, &TextureSelector), + current_state_set: &TextureStateSet, + index32: u32, + index: usize, + state_provider: TextureStateProvider<'_>, + barriers: &mut Vec>, +) { + let current_simple = *current_state_set.simple.get_unchecked(index); + let current_state = if current_simple == TextureUses::COMPLEX { + SingleOrManyStates::Many(current_state_set.complex.get(&index32).unwrap_unchecked()) + } else { + SingleOrManyStates::Single(current_simple) + }; + + let new_state = state_provider.get_state(Some(texture_data), index32, index); + + match (current_state, new_state) { + (SingleOrManyStates::Single(current_simple), SingleOrManyStates::Single(new_simple)) => { + if skip_barrier(current_simple, new_simple) { + return; + } + + log::trace!("\ttex {index32}: transition simple {current_simple:?} -> {new_simple:?}"); + + barriers.push(PendingTransition { + id: index32, + selector: texture_data.1.clone(), + usage: current_simple..new_simple, + }); + } + (SingleOrManyStates::Single(current_simple), SingleOrManyStates::Many(new_many)) => { + for (selector, new_state) in new_many { + if new_state == TextureUses::UNKNOWN { + continue; + } + + if skip_barrier(current_simple, new_state) { + continue; + } + + log::trace!( + "\ttex {index32}: transition {selector:?} {current_simple:?} -> {new_state:?}" + ); + + barriers.push(PendingTransition { + id: index32, + selector, + usage: current_simple..new_state, + }); + } + } + (SingleOrManyStates::Many(current_complex), SingleOrManyStates::Single(new_simple)) => { + for (mip_id, mip) in current_complex.mips.iter().enumerate() { + let mip_id = mip_id as u32; + + for &(ref layers, current_layer_state) in mip.iter() { + if current_layer_state == TextureUses::UNKNOWN { + continue; + } + + if skip_barrier(current_layer_state, new_simple) { + continue; + } + + log::trace!( + "\ttex {index32}: transition mip {mip_id} layers {layers:?} \ + {current_layer_state:?} -> {new_simple:?}" + ); + + barriers.push(PendingTransition { + id: index32, + selector: TextureSelector { + mips: mip_id..mip_id + 1, + layers: layers.clone(), + }, + usage: current_layer_state..new_simple, + }); + } + } + } + (SingleOrManyStates::Many(current_complex), SingleOrManyStates::Many(new_many)) => { + for (selector, new_state) in new_many { + for mip_id in selector.mips { + debug_assert!((mip_id as usize) < current_complex.mips.len()); + + let mip = current_complex.mips.get_unchecked(mip_id as usize); + + for (layers, current_layer_state) in mip.iter_filter(&selector.layers) { + if *current_layer_state == TextureUses::UNKNOWN + || new_state == TextureUses::UNKNOWN + { + continue; + } + + if skip_barrier(*current_layer_state, new_state) { + continue; + } + + log::trace!( + "\ttex {index32}: transition mip {mip_id} layers {layers:?} \ + {current_layer_state:?} -> {new_state:?}" + ); + + barriers.push(PendingTransition { + id: index32, + selector: TextureSelector { + mips: mip_id..mip_id + 1, + layers, + }, + usage: *current_layer_state..new_state, + }); + } + } + } + } + } +} + +#[allow(clippy::needless_option_as_deref)] // we use this for reborrowing Option<&mut T> +#[inline(always)] +unsafe fn update( + texture_data: (&LifeGuard, &TextureSelector), + start_state_set: &mut TextureStateSet, + current_state_set: &mut TextureStateSet, + index32: u32, + index: usize, + state_provider: TextureStateProvider<'_>, +) { + let start_simple = *start_state_set.simple.get_unchecked(index); + + // We only ever need to update the start state here if the state is complex. + // + // If the state is simple, the first insert to the tracker would cover it. + let mut start_complex = None; + if start_simple == TextureUses::COMPLEX { + start_complex = Some(start_state_set.complex.get_mut(&index32).unwrap_unchecked()); + } + + let current_simple = current_state_set.simple.get_unchecked_mut(index); + let current_state = if *current_simple == TextureUses::COMPLEX { + SingleOrManyStates::Many( + current_state_set + .complex + .get_mut(&index32) + .unwrap_unchecked(), + ) + } else { + SingleOrManyStates::Single(current_simple) + }; + + let new_state = state_provider.get_state(Some(texture_data), index32, index); + + match (current_state, new_state) { + (SingleOrManyStates::Single(current_simple), SingleOrManyStates::Single(new_simple)) => { + *current_simple = new_simple; + } + (SingleOrManyStates::Single(current_simple), SingleOrManyStates::Many(new_many)) => { + // Because we are now demoting this simple state to a complex state, we actually need to make a whole + // new complex state for us to use as there wasn't one before. + let mut new_complex = ComplexTextureState::from_selector_state_iter( + texture_data.1.clone(), + iter::once((texture_data.1.clone(), *current_simple)), + ); + + for (selector, mut new_state) in new_many { + if new_state == TextureUses::UNKNOWN { + new_state = *current_simple; + } + for mip in + &mut new_complex.mips[selector.mips.start as usize..selector.mips.end as usize] + { + for &mut (_, ref mut current_layer_state) in + mip.isolate(&selector.layers, TextureUses::UNKNOWN) + { + *current_layer_state = new_state; + } + + mip.coalesce(); + } + } + + *current_simple = TextureUses::COMPLEX; + current_state_set.complex.insert(index32, new_complex); + } + (SingleOrManyStates::Many(current_complex), SingleOrManyStates::Single(new_single)) => { + for (mip_id, mip) in current_complex.mips.iter().enumerate() { + for &(ref layers, current_layer_state) in mip.iter() { + // If this state is unknown, that means that the start is _also_ unknown. + if current_layer_state == TextureUses::UNKNOWN { + if let Some(&mut ref mut start_complex) = start_complex { + debug_assert!(mip_id < start_complex.mips.len()); + + let start_mip = start_complex.mips.get_unchecked_mut(mip_id); + + for &mut (_, ref mut current_start_state) in + start_mip.isolate(layers, TextureUses::UNKNOWN) + { + debug_assert_eq!(*current_start_state, TextureUses::UNKNOWN); + *current_start_state = new_single; + } + + start_mip.coalesce(); + } + } + } + } + + *current_state_set.simple.get_unchecked_mut(index) = new_single; + current_state_set + .complex + .remove(&index32) + .unwrap_unchecked(); + } + (SingleOrManyStates::Many(current_complex), SingleOrManyStates::Many(new_many)) => { + for (selector, new_state) in new_many { + if new_state == TextureUses::UNKNOWN { + // We know nothing new + continue; + } + + for mip_id in selector.mips { + let mip_id = mip_id as usize; + debug_assert!(mip_id < current_complex.mips.len()); + + let mip = current_complex.mips.get_unchecked_mut(mip_id); + + for &mut (ref layers, ref mut current_layer_state) in + mip.isolate(&selector.layers, TextureUses::UNKNOWN) + { + if *current_layer_state == TextureUses::UNKNOWN + && new_state != TextureUses::UNKNOWN + { + // We now know something about this subresource that we didn't before + // so we should go back and update the start state. + + // We know we must have starter state be complex, otherwise we would know + // about this state. + debug_assert!(start_complex.is_some()); + + let start_complex = start_complex.as_deref_mut().unwrap_unchecked(); + + debug_assert!(mip_id < start_complex.mips.len()); + + let start_mip = start_complex.mips.get_unchecked_mut(mip_id); + + for &mut (_, ref mut current_start_state) in + start_mip.isolate(layers, TextureUses::UNKNOWN) + { + debug_assert_eq!(*current_start_state, TextureUses::UNKNOWN); + *current_start_state = new_state; + } + + start_mip.coalesce(); + } + + *current_layer_state = new_state; + } + + mip.coalesce(); + } + } + } } } diff --git a/wgpu-hal/examples/halmark/main.rs b/wgpu-hal/examples/halmark/main.rs index 1ed59325c1..ff021a889d 100644 --- a/wgpu-hal/examples/halmark/main.rs +++ b/wgpu-hal/examples/halmark/main.rs @@ -685,7 +685,7 @@ impl Example { let target_barrier1 = hal::TextureBarrier { texture: surface_tex.borrow(), range: wgt::ImageSubresourceRange::default(), - usage: hal::TextureUses::COLOR_TARGET..hal::TextureUses::empty(), + usage: hal::TextureUses::COLOR_TARGET..hal::TextureUses::PRESENT, }; unsafe { ctx.encoder.end_render_pass(); diff --git a/wgpu-hal/src/dx12/command.rs b/wgpu-hal/src/dx12/command.rs index 95546d213a..dc653911c5 100644 --- a/wgpu-hal/src/dx12/command.rs +++ b/wgpu-hal/src/dx12/command.rs @@ -287,7 +287,7 @@ impl crate::CommandEncoder for super::CommandEncoder { StateAfter: s1, }; self.temp.barriers.push(raw); - } else if barrier.usage.start == crate::BufferUses::STORAGE_WRITE { + } else if barrier.usage.start == crate::BufferUses::STORAGE_READ_WRITE { let mut raw = d3d12::D3D12_RESOURCE_BARRIER { Type: d3d12::D3D12_RESOURCE_BARRIER_TYPE_UAV, Flags: d3d12::D3D12_RESOURCE_BARRIER_FLAG_NONE, @@ -382,7 +382,7 @@ impl crate::CommandEncoder for super::CommandEncoder { } } } - } else if barrier.usage.start == crate::TextureUses::STORAGE_WRITE { + } else if barrier.usage.start == crate::TextureUses::STORAGE_READ_WRITE { let mut raw = d3d12::D3D12_RESOURCE_BARRIER { Type: d3d12::D3D12_RESOURCE_BARRIER_TYPE_UAV, Flags: d3d12::D3D12_RESOURCE_BARRIER_FLAG_NONE, diff --git a/wgpu-hal/src/dx12/conv.rs b/wgpu-hal/src/dx12/conv.rs index 6ea0339afb..8dfd40c92b 100644 --- a/wgpu-hal/src/dx12/conv.rs +++ b/wgpu-hal/src/dx12/conv.rs @@ -3,7 +3,7 @@ use winapi::um::{d3d12, d3dcommon}; pub fn map_buffer_usage_to_resource_flags(usage: crate::BufferUses) -> d3d12::D3D12_RESOURCE_FLAGS { let mut flags = 0; - if usage.contains(crate::BufferUses::STORAGE_WRITE) { + if usage.contains(crate::BufferUses::STORAGE_READ_WRITE) { flags |= d3d12::D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS; } flags @@ -33,7 +33,7 @@ pub fn map_texture_usage_to_resource_flags( flags |= d3d12::D3D12_RESOURCE_FLAG_DENY_SHADER_RESOURCE; } } - if usage.contains(crate::TextureUses::STORAGE_WRITE) { + if usage.contains(crate::TextureUses::STORAGE_READ_WRITE) { flags |= d3d12::D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS; } @@ -130,7 +130,7 @@ pub fn map_buffer_usage_to_state(usage: crate::BufferUses) -> d3d12::D3D12_RESOU if usage.intersects(Bu::VERTEX | Bu::UNIFORM) { state |= d3d12::D3D12_RESOURCE_STATE_VERTEX_AND_CONSTANT_BUFFER; } - if usage.intersects(Bu::STORAGE_WRITE) { + if usage.intersects(Bu::STORAGE_READ_WRITE) { state |= d3d12::D3D12_RESOURCE_STATE_UNORDERED_ACCESS; } else if usage.intersects(Bu::STORAGE_READ) { state |= d3d12::D3D12_RESOURCE_STATE_PIXEL_SHADER_RESOURCE @@ -170,7 +170,7 @@ pub fn map_texture_usage_to_state(usage: crate::TextureUses) -> d3d12::D3D12_RES if usage.intersects(Tu::DEPTH_STENCIL_WRITE) { state |= d3d12::D3D12_RESOURCE_STATE_DEPTH_WRITE; } - if usage.intersects(Tu::STORAGE_READ | Tu::STORAGE_WRITE) { + if usage.intersects(Tu::STORAGE_READ | Tu::STORAGE_READ_WRITE) { state |= d3d12::D3D12_RESOURCE_STATE_UNORDERED_ACCESS; } state diff --git a/wgpu-hal/src/dx12/device.rs b/wgpu-hal/src/dx12/device.rs index 2f92cab44c..c25f58c767 100644 --- a/wgpu-hal/src/dx12/device.rs +++ b/wgpu-hal/src/dx12/device.rs @@ -428,7 +428,7 @@ impl crate::Device for super::Device { || !desc.usage.intersects( crate::TextureUses::RESOURCE | crate::TextureUses::STORAGE_READ - | crate::TextureUses::STORAGE_WRITE, + | crate::TextureUses::STORAGE_READ_WRITE, ) { auxil::dxgi::conv::map_texture_format(desc.format) } else { @@ -516,10 +516,9 @@ impl crate::Device for super::Device { } else { None }, - handle_uav: if desc - .usage - .intersects(crate::TextureUses::STORAGE_READ | crate::TextureUses::STORAGE_WRITE) - { + handle_uav: if desc.usage.intersects( + crate::TextureUses::STORAGE_READ | crate::TextureUses::STORAGE_READ_WRITE, + ) { let raw_desc = view_desc.to_uav(); let handle = self.srv_uav_pool.lock().alloc_handle(); self.raw.CreateUnorderedAccessView( diff --git a/wgpu-hal/src/gles/command.rs b/wgpu-hal/src/gles/command.rs index f0f6028c62..307a2865c3 100644 --- a/wgpu-hal/src/gles/command.rs +++ b/wgpu-hal/src/gles/command.rs @@ -230,7 +230,11 @@ impl crate::CommandEncoder for super::CommandEncoder { } for bar in barriers { // GLES only synchronizes storage -> anything explicitly - if !bar.usage.start.contains(crate::BufferUses::STORAGE_WRITE) { + if !bar + .usage + .start + .contains(crate::BufferUses::STORAGE_READ_WRITE) + { continue; } self.cmd_buffer @@ -253,7 +257,11 @@ impl crate::CommandEncoder for super::CommandEncoder { let mut combined_usage = crate::TextureUses::empty(); for bar in barriers { // GLES only synchronizes storage -> anything explicitly - if !bar.usage.start.contains(crate::TextureUses::STORAGE_WRITE) { + if !bar + .usage + .start + .contains(crate::TextureUses::STORAGE_READ_WRITE) + { continue; } // unlike buffers, there is no need for a concrete texture diff --git a/wgpu-hal/src/gles/queue.rs b/wgpu-hal/src/gles/queue.rs index fdc9268676..2af9e72f21 100644 --- a/wgpu-hal/src/gles/queue.rs +++ b/wgpu-hal/src/gles/queue.rs @@ -790,9 +790,9 @@ impl super::Queue { if usage.intersects(crate::BufferUses::MAP_READ | crate::BufferUses::MAP_WRITE) { flags |= glow::BUFFER_UPDATE_BARRIER_BIT; } - if usage - .intersects(crate::BufferUses::STORAGE_READ | crate::BufferUses::STORAGE_WRITE) - { + if usage.intersects( + crate::BufferUses::STORAGE_READ | crate::BufferUses::STORAGE_READ_WRITE, + ) { flags |= glow::SHADER_STORAGE_BARRIER_BIT; } gl.memory_barrier(flags); @@ -803,7 +803,7 @@ impl super::Queue { flags |= glow::TEXTURE_FETCH_BARRIER_BIT; } if usage.intersects( - crate::TextureUses::STORAGE_READ | crate::TextureUses::STORAGE_WRITE, + crate::TextureUses::STORAGE_READ | crate::TextureUses::STORAGE_READ_WRITE, ) { flags |= glow::SHADER_IMAGE_ACCESS_BARRIER_BIT; } diff --git a/wgpu-hal/src/lib.rs b/wgpu-hal/src/lib.rs index 6eb297019a..82a3243986 100644 --- a/wgpu-hal/src/lib.rs +++ b/wgpu-hal/src/lib.rs @@ -628,53 +628,77 @@ bitflags!( bitflags::bitflags! { /// Similar to `wgt::BufferUsages` but for internal use. - pub struct BufferUses: u32 { + pub struct BufferUses: u16 { + /// The argument to a read-only mapping. const MAP_READ = 1 << 0; + /// The argument to a write-only mapping. const MAP_WRITE = 1 << 1; + /// The source of a hardware copy. const COPY_SRC = 1 << 2; + /// The destination of a hardware copy. const COPY_DST = 1 << 3; + /// The index buffer used for drawing. const INDEX = 1 << 4; + /// A vertex buffer used for drawing. const VERTEX = 1 << 5; + /// A uniform buffer bound in a bind group. const UNIFORM = 1 << 6; + /// A read-only storage buffer used in a bind group. const STORAGE_READ = 1 << 7; - const STORAGE_WRITE = 1 << 8; + /// A read-write or write-only buffer used in a bind group. + const STORAGE_READ_WRITE = 1 << 8; + /// The indirect or count buffer in a indirect draw or dispatch. const INDIRECT = 1 << 9; - /// The combination of usages that can be used together (read-only). + /// The combination of states that a buffer may be in _at the same time_. const INCLUSIVE = Self::MAP_READ.bits | Self::COPY_SRC.bits | Self::INDEX.bits | Self::VERTEX.bits | Self::UNIFORM.bits | Self::STORAGE_READ.bits | Self::INDIRECT.bits; - /// The combination of exclusive usages (write-only and read-write). - /// These usages may still show up with others, but can't automatically be combined. - const EXCLUSIVE = Self::MAP_WRITE.bits | Self::COPY_DST.bits | Self::STORAGE_WRITE.bits; + /// The combination of states that a buffer must exclusively be in. + const EXCLUSIVE = Self::MAP_WRITE.bits | Self::COPY_DST.bits | Self::STORAGE_READ_WRITE.bits; /// The combination of all usages that the are guaranteed to be be ordered by the hardware. - /// If a usage is not ordered, then even if it doesn't change between draw calls, there - /// still need to be pipeline barriers inserted for synchronization. + /// If a usage is ordered, then if the buffer state doesn't change between draw calls, there + /// are no barriers needed for synchronization. const ORDERED = Self::INCLUSIVE.bits | Self::MAP_WRITE.bits; } } bitflags::bitflags! { /// Similar to `wgt::TextureUsages` but for internal use. - pub struct TextureUses: u32 { - const COPY_SRC = 1 << 0; - const COPY_DST = 1 << 1; - const RESOURCE = 1 << 2; - const COLOR_TARGET = 1 << 3; - const DEPTH_STENCIL_READ = 1 << 4; - const DEPTH_STENCIL_WRITE = 1 << 5; - const STORAGE_READ = 1 << 6; - const STORAGE_WRITE = 1 << 7; - /// The combination of usages that can be used together (read-only). + pub struct TextureUses: u16 { + /// The texture is in unknown state. + const UNINITIALIZED = 1 << 0; + /// Ready to present image to the surface. + const PRESENT = 1 << 1; + /// The source of a hardware copy. + const COPY_SRC = 1 << 2; + /// The destination of a hardware copy. + const COPY_DST = 1 << 3; + /// Read-only sampled or fetched resource. + const RESOURCE = 1 << 4; + /// The color target of a renderpass. + const COLOR_TARGET = 1 << 5; + /// Read-only depth stencil usage. + const DEPTH_STENCIL_READ = 1 << 6; + /// Read-write depth stencil usage + const DEPTH_STENCIL_WRITE = 1 << 7; + /// Read-only storage buffer usage. Corresponds to a UAV in d3d, so is exclusive, despite being read only. + const STORAGE_READ = 1 << 8; + /// Read-write or write-only storage buffer usage. + const STORAGE_READ_WRITE = 1 << 9; + /// The combination of states that a texture may be in _at the same time_. const INCLUSIVE = Self::COPY_SRC.bits | Self::RESOURCE.bits | Self::DEPTH_STENCIL_READ.bits; - /// The combination of exclusive usages (write-only and read-write). - /// These usages may still show up with others, but can't automatically be combined. - const EXCLUSIVE = Self::COPY_DST.bits | Self::COLOR_TARGET.bits | Self::DEPTH_STENCIL_WRITE.bits | Self::STORAGE_READ.bits | Self::STORAGE_WRITE.bits; + /// The combination of states that a texture must exclusively be in. + const EXCLUSIVE = Self::COPY_DST.bits | Self::COLOR_TARGET.bits | Self::DEPTH_STENCIL_WRITE.bits | Self::STORAGE_READ.bits | Self::STORAGE_READ_WRITE.bits | Self::PRESENT.bits; /// The combination of all usages that the are guaranteed to be be ordered by the hardware. - /// If a usage is not ordered, then even if it doesn't change between draw calls, there - /// still need to be pipeline barriers inserted for synchronization. + /// If a usage is ordered, then if the texture state doesn't change between draw calls, there + /// are no barriers needed for synchronization. const ORDERED = Self::INCLUSIVE.bits | Self::COLOR_TARGET.bits | Self::DEPTH_STENCIL_WRITE.bits | Self::STORAGE_READ.bits; - //TODO: remove this - const UNINITIALIZED = 0xFFFF; + + /// Flag used by the wgpu-core texture tracker to say a texture is in different states for every sub-resource + const COMPLEX = 1 << 10; + /// Flag used by the wgpu-core texture tracker to say that the tracker does not know the state of the sub-resource. + /// This is different from UNINITIALIZED as that says the tracker does know, but the texture has not been initialized. + const UNKNOWN = 1 << 11; } } diff --git a/wgpu-hal/src/metal/conv.rs b/wgpu-hal/src/metal/conv.rs index f275a74ec9..75ed58df24 100644 --- a/wgpu-hal/src/metal/conv.rs +++ b/wgpu-hal/src/metal/conv.rs @@ -9,11 +9,13 @@ pub fn map_texture_usage(usage: crate::TextureUses) -> mtl::MTLTextureUsage { ); mtl_usage.set( mtl::MTLTextureUsage::ShaderRead, - usage.intersects(Tu::RESOURCE | Tu::DEPTH_STENCIL_READ | Tu::STORAGE_READ), + usage.intersects( + Tu::RESOURCE | Tu::DEPTH_STENCIL_READ | Tu::STORAGE_READ | Tu::STORAGE_READ_WRITE, + ), ); mtl_usage.set( mtl::MTLTextureUsage::ShaderWrite, - usage.intersects(Tu::STORAGE_WRITE), + usage.intersects(Tu::STORAGE_READ_WRITE), ); mtl_usage diff --git a/wgpu-hal/src/vulkan/conv.rs b/wgpu-hal/src/vulkan/conv.rs index 602dedb95a..83fc318486 100644 --- a/wgpu-hal/src/vulkan/conv.rs +++ b/wgpu-hal/src/vulkan/conv.rs @@ -200,7 +200,7 @@ pub fn derive_image_layout( vk::ImageLayout::DEPTH_STENCIL_ATTACHMENT_OPTIMAL } _ => { - if usage.is_empty() { + if usage == crate::TextureUses::PRESENT { vk::ImageLayout::PRESENT_SRC_KHR } else if is_color { vk::ImageLayout::GENERAL @@ -230,7 +230,7 @@ pub fn map_texture_usage(usage: crate::TextureUses) -> vk::ImageUsageFlags { ) { flags |= vk::ImageUsageFlags::DEPTH_STENCIL_ATTACHMENT; } - if usage.intersects(crate::TextureUses::STORAGE_READ | crate::TextureUses::STORAGE_WRITE) { + if usage.intersects(crate::TextureUses::STORAGE_READ | crate::TextureUses::STORAGE_READ_WRITE) { flags |= vk::ImageUsageFlags::STORAGE; } flags @@ -276,12 +276,12 @@ pub fn map_texture_usage_to_barrier( stages |= shader_stages; access |= vk::AccessFlags::SHADER_READ; } - if usage.contains(crate::TextureUses::STORAGE_WRITE) { + if usage.contains(crate::TextureUses::STORAGE_READ_WRITE) { stages |= shader_stages; - access |= vk::AccessFlags::SHADER_WRITE; + access |= vk::AccessFlags::SHADER_READ | vk::AccessFlags::SHADER_WRITE; } - if usage == crate::TextureUses::UNINITIALIZED || usage.is_empty() { + if usage == crate::TextureUses::UNINITIALIZED || usage == crate::TextureUses::PRESENT { ( vk::PipelineStageFlags::TOP_OF_PIPE, vk::AccessFlags::empty(), @@ -309,7 +309,7 @@ pub fn map_vk_image_usage(usage: vk::ImageUsageFlags) -> crate::TextureUses { bits |= crate::TextureUses::DEPTH_STENCIL_READ | crate::TextureUses::DEPTH_STENCIL_WRITE; } if usage.contains(vk::ImageUsageFlags::STORAGE) { - bits |= crate::TextureUses::STORAGE_READ | crate::TextureUses::STORAGE_WRITE; + bits |= crate::TextureUses::STORAGE_READ | crate::TextureUses::STORAGE_READ_WRITE; } bits } @@ -457,7 +457,7 @@ pub fn map_buffer_usage(usage: crate::BufferUses) -> vk::BufferUsageFlags { if usage.contains(crate::BufferUses::UNIFORM) { flags |= vk::BufferUsageFlags::UNIFORM_BUFFER; } - if usage.intersects(crate::BufferUses::STORAGE_READ | crate::BufferUses::STORAGE_WRITE) { + if usage.intersects(crate::BufferUses::STORAGE_READ | crate::BufferUses::STORAGE_READ_WRITE) { flags |= vk::BufferUsageFlags::STORAGE_BUFFER; } if usage.contains(crate::BufferUses::INDEX) { @@ -505,9 +505,9 @@ pub fn map_buffer_usage_to_barrier( stages |= shader_stages; access |= vk::AccessFlags::SHADER_READ; } - if usage.intersects(crate::BufferUses::STORAGE_WRITE) { + if usage.intersects(crate::BufferUses::STORAGE_READ_WRITE) { stages |= shader_stages; - access |= vk::AccessFlags::SHADER_WRITE; + access |= vk::AccessFlags::SHADER_READ | vk::AccessFlags::SHADER_WRITE; } if usage.contains(crate::BufferUses::INDEX) { stages |= vk::PipelineStageFlags::VERTEX_INPUT; diff --git a/wgpu-hal/src/vulkan/device.rs b/wgpu-hal/src/vulkan/device.rs index 24f992953f..3dd6b8b31b 100644 --- a/wgpu-hal/src/vulkan/device.rs +++ b/wgpu-hal/src/vulkan/device.rs @@ -1413,7 +1413,7 @@ impl crate::Device for super::Device { unsafe fn destroy_shader_module(&self, module: super::ShaderModule) { match module { super::ShaderModule::Raw(raw) => { - let _ = self.shared.raw.destroy_shader_module(raw, None); + self.shared.raw.destroy_shader_module(raw, None); } super::ShaderModule::Intermediate { .. } => {} }