diff --git a/ci/licenses_golden/excluded_files b/ci/licenses_golden/excluded_files index d525a6154aae0..db8bcd971146e 100644 --- a/ci/licenses_golden/excluded_files +++ b/ci/licenses_golden/excluded_files @@ -133,6 +133,7 @@ ../../../flutter/impeller/base/base_unittests.cc ../../../flutter/impeller/compiler/README.md ../../../flutter/impeller/compiler/compiler_unittests.cc +../../../flutter/impeller/compiler/shader_bundle_unittests.cc ../../../flutter/impeller/compiler/switches_unittests.cc ../../../flutter/impeller/core/allocator_unittests.cc ../../../flutter/impeller/display_list/dl_unittests.cc diff --git a/ci/licenses_golden/licenses_flutter b/ci/licenses_golden/licenses_flutter index 99c8122d0d994..5c379ed01c12e 100644 --- a/ci/licenses_golden/licenses_flutter +++ b/ci/licenses_golden/licenses_flutter @@ -4933,6 +4933,8 @@ ORIGIN: ../../../flutter/impeller/compiler/reflector.cc + ../../../flutter/LICEN ORIGIN: ../../../flutter/impeller/compiler/reflector.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/compiler/runtime_stage_data.cc + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/compiler/runtime_stage_data.h + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/impeller/compiler/shader_bundle.cc + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/impeller/compiler/shader_bundle.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/compiler/shader_lib/flutter/runtime_effect.glsl + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/compiler/shader_lib/impeller/blending.glsl + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/compiler/shader_lib/impeller/branching.glsl + ../../../flutter/LICENSE @@ -5473,6 +5475,7 @@ ORIGIN: ../../../flutter/impeller/runtime_stage/runtime_stage.fbs + ../../../flu ORIGIN: ../../../flutter/impeller/runtime_stage/runtime_stage.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/runtime_stage/runtime_stage_playground.cc + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/runtime_stage/runtime_stage_playground.h + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/impeller/runtime_stage/runtime_stage_types.fbs + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/scene/animation/animation.cc + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/scene/animation/animation.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/scene/animation/animation_clip.cc + ../../../flutter/LICENSE @@ -5527,6 +5530,7 @@ ORIGIN: ../../../flutter/impeller/shader_archive/shader_archive_main.cc + ../../ ORIGIN: ../../../flutter/impeller/shader_archive/shader_archive_types.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/shader_archive/shader_archive_writer.cc + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/shader_archive/shader_archive_writer.h + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/impeller/shader_bundle/shader_bundle.fbs + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/tessellator/c/tessellator.cc + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/tessellator/c/tessellator.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/tessellator/dart/lib/tessellator.dart + ../../../flutter/LICENSE @@ -7734,6 +7738,8 @@ FILE: ../../../flutter/impeller/compiler/reflector.cc FILE: ../../../flutter/impeller/compiler/reflector.h FILE: ../../../flutter/impeller/compiler/runtime_stage_data.cc FILE: ../../../flutter/impeller/compiler/runtime_stage_data.h +FILE: ../../../flutter/impeller/compiler/shader_bundle.cc +FILE: ../../../flutter/impeller/compiler/shader_bundle.h FILE: ../../../flutter/impeller/compiler/shader_lib/flutter/runtime_effect.glsl FILE: ../../../flutter/impeller/compiler/shader_lib/impeller/blending.glsl FILE: ../../../flutter/impeller/compiler/shader_lib/impeller/branching.glsl @@ -8275,6 +8281,7 @@ FILE: ../../../flutter/impeller/runtime_stage/runtime_stage.fbs FILE: ../../../flutter/impeller/runtime_stage/runtime_stage.h FILE: ../../../flutter/impeller/runtime_stage/runtime_stage_playground.cc FILE: ../../../flutter/impeller/runtime_stage/runtime_stage_playground.h +FILE: ../../../flutter/impeller/runtime_stage/runtime_stage_types.fbs FILE: ../../../flutter/impeller/scene/animation/animation.cc FILE: ../../../flutter/impeller/scene/animation/animation.h FILE: ../../../flutter/impeller/scene/animation/animation_clip.cc @@ -8329,6 +8336,7 @@ FILE: ../../../flutter/impeller/shader_archive/shader_archive_main.cc FILE: ../../../flutter/impeller/shader_archive/shader_archive_types.h FILE: ../../../flutter/impeller/shader_archive/shader_archive_writer.cc FILE: ../../../flutter/impeller/shader_archive/shader_archive_writer.h +FILE: ../../../flutter/impeller/shader_bundle/shader_bundle.fbs FILE: ../../../flutter/impeller/tessellator/c/tessellator.cc FILE: ../../../flutter/impeller/tessellator/c/tessellator.h FILE: ../../../flutter/impeller/tessellator/dart/lib/tessellator.dart diff --git a/impeller/compiler/BUILD.gn b/impeller/compiler/BUILD.gn index b4ff92ee3cb15..3f508fe605f1a 100644 --- a/impeller/compiler/BUILD.gn +++ b/impeller/compiler/BUILD.gn @@ -46,6 +46,8 @@ impeller_component("compiler_lib") { "reflector.h", "runtime_stage_data.cc", "runtime_stage_data.h", + "shader_bundle.cc", + "shader_bundle.h", "source_options.cc", "source_options.h", "spirv_compiler.cc", @@ -66,6 +68,7 @@ impeller_component("compiler_lib") { "../geometry", "../runtime_stage", "//flutter/fml", + "//flutter/impeller/shader_bundle:shader_bundle_flatbuffers", # All third_party deps must be included by the global license script. "//third_party/inja", @@ -102,6 +105,7 @@ impeller_component("compiler_unittests") { "compiler_test.cc", "compiler_test.h", "compiler_unittests.cc", + "shader_bundle_unittests.cc", "switches_unittests.cc", ] diff --git a/impeller/compiler/impellerc_main.cc b/impeller/compiler/impellerc_main.cc index 9c7b58b6d760b..48c7e9cc5fc14 100644 --- a/impeller/compiler/impellerc_main.cc +++ b/impeller/compiler/impellerc_main.cc @@ -10,6 +10,7 @@ #include "flutter/fml/file.h" #include "flutter/fml/mapping.h" #include "impeller/compiler/compiler.h" +#include "impeller/compiler/shader_bundle.h" #include "impeller/compiler/source_options.h" #include "impeller/compiler/switches.h" #include "impeller/compiler/types.h" @@ -18,107 +19,52 @@ namespace impeller { namespace compiler { -// Sets the file access mode of the file at path 'p' to 0644. -static bool SetPermissiveAccess(const std::filesystem::path& p) { - auto permissions = - std::filesystem::perms::owner_read | std::filesystem::perms::owner_write | - std::filesystem::perms::group_read | std::filesystem::perms::others_read; - std::error_code error; - std::filesystem::permissions(p, permissions, error); - if (error) { - std::cerr << "Failed to set access on file '" << p - << "': " << error.message() << std::endl; - return false; - } - return true; -} - -bool Main(const fml::CommandLine& command_line) { - fml::InstallCrashHandler(); - if (command_line.HasOption("help")) { - Switches::PrintHelp(std::cout); - return true; - } - - Switches switches(command_line); - if (!switches.AreValid(std::cerr)) { - std::cerr << "Invalid flags specified." << std::endl; - Switches::PrintHelp(std::cerr); - return false; - } +/// Run the shader compiler to geneate SkSL. +/// If there is an error, prints error text and returns `nullptr`. +static std::shared_ptr CompileSkSL( + std::shared_ptr source_file_mapping, + SourceOptions& options, + Reflector::Options& reflector_options) { + SourceOptions sksl_options = options; + sksl_options.target_platform = TargetPlatform::kSkSL; - std::shared_ptr source_file_mapping = - fml::FileMapping::CreateReadOnly(switches.source_file_name); - if (!source_file_mapping) { - std::cerr << "Could not open input file." << std::endl; - return false; - } + Reflector::Options sksl_reflector_options = reflector_options; + sksl_reflector_options.target_platform = TargetPlatform::kSkSL; - SourceOptions options; - options.target_platform = switches.target_platform; - options.source_language = switches.source_language; - if (switches.input_type == SourceType::kUnknown) { - options.type = SourceTypeFromFileName(switches.source_file_name); - } else { - options.type = switches.input_type; + Compiler sksl_compiler = Compiler(std::move(source_file_mapping), + sksl_options, sksl_reflector_options); + if (!sksl_compiler.IsValid()) { + std::cerr << "Compilation to SkSL failed." << std::endl; + std::cerr << sksl_compiler.GetErrorMessages() << std::endl; + return nullptr; } - options.working_directory = switches.working_directory; - options.file_name = switches.source_file_name; - options.include_dirs = switches.include_directories; - options.defines = switches.defines; - options.entry_point_name = EntryPointFunctionNameFromSourceName( - switches.source_file_name, options.type, options.source_language, - switches.entry_point); - options.json_format = switches.json_format; - options.gles_language_version = switches.gles_language_version; - options.metal_version = switches.metal_version; - options.use_half_textures = switches.use_half_textures; - options.require_framebuffer_fetch = switches.require_framebuffer_fetch; + return sksl_compiler.GetSLShaderSource(); +} - Reflector::Options reflector_options; - reflector_options.target_platform = switches.target_platform; - reflector_options.entry_point_name = options.entry_point_name; - reflector_options.shader_name = - InferShaderNameFromPath(switches.source_file_name); - reflector_options.header_file_name = Utf8FromPath( - std::filesystem::path{switches.reflection_header_name}.filename()); +/// Outputs artifacts for a single compiler invocation and option configuration. +/// If there is an error, prints error text and returns `false`. +static bool OutputArtifacts(Compiler& compiler, + Switches& switches, + std::shared_ptr source_file_mapping, + SourceOptions& options, + Reflector::Options& reflector_options) { + // -------------------------------------------------------------------------- + /// 1. Invoke the compiler to generate SkSL if needed. + /// - // Generate SkSL if needed. std::shared_ptr sksl_mapping; - if (switches.iplr && TargetPlatformBundlesSkSL(switches.target_platform) && - switches.iplr_bundle.empty()) { - SourceOptions sksl_options = options; - sksl_options.target_platform = TargetPlatform::kSkSL; - - Reflector::Options sksl_reflector_options = reflector_options; - sksl_reflector_options.target_platform = TargetPlatform::kSkSL; - - Compiler sksl_compiler = - Compiler(source_file_mapping, sksl_options, sksl_reflector_options); - if (!sksl_compiler.IsValid()) { - std::cerr << "Compilation to SkSL failed." << std::endl; - std::cerr << sksl_compiler.GetErrorMessages() << std::endl; + if (switches.iplr && TargetPlatformBundlesSkSL(switches.target_platform)) { + sksl_mapping = + CompileSkSL(std::move(source_file_mapping), options, reflector_options); + if (!sksl_mapping) { return false; } - sksl_mapping = sksl_compiler.GetSLShaderSource(); - } - - Compiler compiler(source_file_mapping, options, reflector_options); - if (!compiler.IsValid()) { - std::cerr << "Compilation failed." << std::endl; - std::cerr << compiler.GetErrorMessages() << std::endl; - return false; } - auto spriv_file_name = std::filesystem::absolute( - std::filesystem::current_path() / switches.spirv_file_name); - if (!fml::WriteAtomically(*switches.working_directory, - Utf8FromPath(spriv_file_name).c_str(), - *compiler.GetSPIRVAssembly())) { - std::cerr << "Could not write file to " << switches.spirv_file_name - << std::endl; - return false; - } + // -------------------------------------------------------------------------- + /// 2. Output the source file. When in IPLR/RuntimeStage mode, output the + /// serialized IPLR flatbuffer. + /// auto sl_file_name = std::filesystem::absolute( std::filesystem::current_path() / switches.sl_file_name); @@ -134,7 +80,7 @@ bool Main(const fml::CommandLine& command_line) { return false; } if (sksl_mapping) { - stage_data->SetSkSLData(sksl_mapping); + stage_data->SetSkSLData(std::move(sksl_mapping)); } auto stage_data_mapping = options.json_format ? stage_data->CreateJsonMapping() @@ -166,6 +112,11 @@ bool Main(const fml::CommandLine& command_line) { } } + // -------------------------------------------------------------------------- + /// 3. Output shader reflection data. + /// May include a JSON file, a C++ header, and/or a C++ TU. + /// + if (TargetPlatformNeedsReflection(options.target_platform)) { if (!switches.reflection_json_name.empty()) { auto reflection_json_name = std::filesystem::absolute( @@ -208,6 +159,10 @@ bool Main(const fml::CommandLine& command_line) { } } + // -------------------------------------------------------------------------- + /// 4. Output a depfile. + /// + if (!switches.depfile_path.empty()) { std::string result_file; switch (switches.target_platform) { @@ -240,6 +195,89 @@ bool Main(const fml::CommandLine& command_line) { return true; } +bool Main(const fml::CommandLine& command_line) { + fml::InstallCrashHandler(); + if (command_line.HasOption("help")) { + Switches::PrintHelp(std::cout); + return true; + } + + Switches switches(command_line); + if (!switches.AreValid(std::cerr)) { + std::cerr << "Invalid flags specified." << std::endl; + Switches::PrintHelp(std::cerr); + return false; + } + SourceOptions options; + options.target_platform = switches.target_platform; + options.source_language = switches.source_language; + if (switches.input_type == SourceType::kUnknown) { + options.type = SourceTypeFromFileName(switches.source_file_name); + } else { + options.type = switches.input_type; + } + options.working_directory = switches.working_directory; + options.file_name = switches.source_file_name; + options.include_dirs = switches.include_directories; + options.defines = switches.defines; + options.entry_point_name = EntryPointFunctionNameFromSourceName( + switches.source_file_name, options.type, options.source_language, + switches.entry_point); + options.json_format = switches.json_format; + options.gles_language_version = switches.gles_language_version; + options.metal_version = switches.metal_version; + options.use_half_textures = switches.use_half_textures; + options.require_framebuffer_fetch = switches.require_framebuffer_fetch; + + if (!switches.shader_bundle.empty()) { + // Invoke the compiler multiple times to build a shader bundle with the + // given shader_bundle spec. + return GenerateShaderBundle(switches, options); + } + + std::shared_ptr source_file_mapping = + fml::FileMapping::CreateReadOnly(switches.source_file_name); + if (!source_file_mapping) { + std::cerr << "Could not open input file." << std::endl; + return false; + } + + // Invoke the compiler and generate reflection data for a single shader or + // runtime stage IPLR. + + Reflector::Options reflector_options; + reflector_options.target_platform = switches.target_platform; + reflector_options.entry_point_name = options.entry_point_name; + reflector_options.shader_name = + InferShaderNameFromPath(switches.source_file_name); + reflector_options.header_file_name = Utf8FromPath( + std::filesystem::path{switches.reflection_header_name}.filename()); + + Compiler compiler(source_file_mapping, options, reflector_options); + if (!compiler.IsValid()) { + std::cerr << "Compilation failed." << std::endl; + std::cerr << compiler.GetErrorMessages() << std::endl; + return false; + } + + auto spriv_file_name = std::filesystem::absolute( + std::filesystem::current_path() / switches.spirv_file_name); + if (!fml::WriteAtomically(*switches.working_directory, + Utf8FromPath(spriv_file_name).c_str(), + *compiler.GetSPIRVAssembly())) { + std::cerr << "Could not write file to " << switches.spirv_file_name + << std::endl; + return false; + } + + if (!OutputArtifacts(compiler, switches, std::move(source_file_mapping), + options, reflector_options)) { + return false; + } + + return true; +} + } // namespace compiler } // namespace impeller diff --git a/impeller/compiler/reflector.cc b/impeller/compiler/reflector.cc index 50903b1b5ce97..05827e341f3ef 100644 --- a/impeller/compiler/reflector.cc +++ b/impeller/compiler/reflector.cc @@ -11,12 +11,10 @@ #include #include -#include "flutter/fml/closure.h" #include "flutter/fml/logging.h" #include "impeller/base/strings.h" #include "impeller/base/validation.h" #include "impeller/compiler/code_gen_template.h" -#include "impeller/compiler/types.h" #include "impeller/compiler/uniform_sorter.h" #include "impeller/compiler/utilities.h" #include "impeller/geometry/half.h" @@ -353,6 +351,35 @@ std::shared_ptr Reflector::GenerateRuntimeStageData() const { uniform_description.array_elements = GetArrayElements(spir_type); data->AddUniformDescription(std::move(uniform_description)); } + + // We only need to worry about storing vertex attributes. + if (entrypoints.front().execution_model == spv::ExecutionModelVertex) { + const auto inputs = compiler_->get_shader_resources().stage_inputs; + auto input_offsets = ComputeOffsets(inputs); + for (const auto& input : inputs) { + auto location = compiler_->get_decoration( + input.id, spv::Decoration::DecorationLocation); + std::optional offset = input_offsets[location]; + + const auto type = compiler_->get_type(input.type_id); + + InputDescription input_description; + input_description.name = input.name; + input_description.location = compiler_->get_decoration( + input.id, spv::Decoration::DecorationLocation); + input_description.set = compiler_->get_decoration( + input.id, spv::Decoration::DecorationDescriptorSet); + input_description.binding = compiler_->get_decoration( + input.id, spv::Decoration::DecorationBinding); + input_description.type = type.basetype; + input_description.bit_width = type.width; + input_description.vec_size = type.vecsize; + input_description.columns = type.columns; + input_description.offset = offset.value_or(0u); + data->AddInputDescription(std::move(input_description)); + } + } + return data; } @@ -387,7 +414,7 @@ std::shared_ptr Reflector::InflateTemplate( env.set_lstrip_blocks(true); env.add_callback("camel_case", 1u, [](inja::Arguments& args) { - return ConvertToCamelCase(args.at(0u)->get()); + return ToCamelCase(args.at(0u)->get()); }); env.add_callback("to_shader_stage", 1u, [](inja::Arguments& args) { @@ -1128,7 +1155,7 @@ std::vector Reflector::ReflectBindPrototypes( for (const auto& uniform_buffer : resources.uniform_buffers) { auto& proto = prototypes.emplace_back(BindPrototype{}); proto.return_type = "bool"; - proto.name = ConvertToCamelCase(uniform_buffer.name); + proto.name = ToCamelCase(uniform_buffer.name); { std::stringstream stream; stream << "Bind uniform buffer for resource named " << uniform_buffer.name @@ -1147,7 +1174,7 @@ std::vector Reflector::ReflectBindPrototypes( for (const auto& storage_buffer : resources.storage_buffers) { auto& proto = prototypes.emplace_back(BindPrototype{}); proto.return_type = "bool"; - proto.name = ConvertToCamelCase(storage_buffer.name); + proto.name = ToCamelCase(storage_buffer.name); { std::stringstream stream; stream << "Bind storage buffer for resource named " << storage_buffer.name @@ -1166,7 +1193,7 @@ std::vector Reflector::ReflectBindPrototypes( for (const auto& sampled_image : resources.sampled_images) { auto& proto = prototypes.emplace_back(BindPrototype{}); proto.return_type = "bool"; - proto.name = ConvertToCamelCase(sampled_image.name); + proto.name = ToCamelCase(sampled_image.name); { std::stringstream stream; stream << "Bind combined image sampler for resource named " @@ -1189,7 +1216,7 @@ std::vector Reflector::ReflectBindPrototypes( for (const auto& separate_image : resources.separate_images) { auto& proto = prototypes.emplace_back(BindPrototype{}); proto.return_type = "bool"; - proto.name = ConvertToCamelCase(separate_image.name); + proto.name = ToCamelCase(separate_image.name); { std::stringstream stream; stream << "Bind separate image for resource named " << separate_image.name @@ -1208,7 +1235,7 @@ std::vector Reflector::ReflectBindPrototypes( for (const auto& separate_sampler : resources.separate_samplers) { auto& proto = prototypes.emplace_back(BindPrototype{}); proto.return_type = "bool"; - proto.name = ConvertToCamelCase(separate_sampler.name); + proto.name = ToCamelCase(separate_sampler.name); { std::stringstream stream; stream << "Bind separate sampler for resource named " diff --git a/impeller/compiler/runtime_stage_data.cc b/impeller/compiler/runtime_stage_data.cc index 95b35bcbbc1db..a27517c2369e8 100644 --- a/impeller/compiler/runtime_stage_data.cc +++ b/impeller/compiler/runtime_stage_data.cc @@ -29,6 +29,10 @@ void RuntimeStageData::AddUniformDescription(UniformDescription uniform) { uniforms_.emplace_back(std::move(uniform)); } +void RuntimeStageData::AddInputDescription(InputDescription input) { + inputs_.emplace_back(std::move(input)); +} + void RuntimeStageData::SetShaderData(std::shared_ptr shader) { shader_ = std::move(shader); } @@ -108,7 +112,7 @@ static std::optional ToJsonTargetPlatform(TargetPlatform platform) { FML_UNREACHABLE(); } -static std::optional ToType( +static std::optional ToUniformType( spirv_cross::SPIRType::BaseType type) { switch (type) { case spirv_cross::SPIRType::Boolean: @@ -152,6 +156,48 @@ static std::optional ToType( } FML_UNREACHABLE(); } +static std::optional ToInputType( + spirv_cross::SPIRType::BaseType type) { + switch (type) { + case spirv_cross::SPIRType::Boolean: + return fb::InputDataType::kBoolean; + case spirv_cross::SPIRType::SByte: + return fb::InputDataType::kSignedByte; + case spirv_cross::SPIRType::UByte: + return fb::InputDataType::kUnsignedByte; + case spirv_cross::SPIRType::Short: + return fb::InputDataType::kSignedShort; + case spirv_cross::SPIRType::UShort: + return fb::InputDataType::kUnsignedShort; + case spirv_cross::SPIRType::Int: + return fb::InputDataType::kSignedInt; + case spirv_cross::SPIRType::UInt: + return fb::InputDataType::kUnsignedInt; + case spirv_cross::SPIRType::Int64: + return fb::InputDataType::kSignedInt64; + case spirv_cross::SPIRType::UInt64: + return fb::InputDataType::kUnsignedInt64; + case spirv_cross::SPIRType::Float: + return fb::InputDataType::kFloat; + case spirv_cross::SPIRType::Double: + return fb::InputDataType::kDouble; + case spirv_cross::SPIRType::Unknown: + case spirv_cross::SPIRType::Void: + case spirv_cross::SPIRType::Half: + case spirv_cross::SPIRType::AtomicCounter: + case spirv_cross::SPIRType::Struct: + case spirv_cross::SPIRType::Image: + case spirv_cross::SPIRType::SampledImage: + case spirv_cross::SPIRType::Sampler: + case spirv_cross::SPIRType::AccelerationStructure: + case spirv_cross::SPIRType::RayQuery: + case spirv_cross::SPIRType::ControlPointArray: + case spirv_cross::SPIRType::Interpolant: + case spirv_cross::SPIRType::Char: + return std::nullopt; + } + FML_UNREACHABLE(); +} static std::optional ToJsonType( spirv_cross::SPIRType::BaseType type) { @@ -285,35 +331,36 @@ std::shared_ptr RuntimeStageData::CreateJsonMapping() const { json_string->size(), [json_string](auto, auto) {}); } -std::shared_ptr RuntimeStageData::CreateMapping() const { +std::unique_ptr RuntimeStageData::CreateFlatbuffer() const { + auto runtime_stage = std::make_unique(); + // The high level object API is used here for writing to the buffer. This is // just a convenience. - fb::RuntimeStageT runtime_stage; - runtime_stage.entrypoint = entrypoint_; + runtime_stage->entrypoint = entrypoint_; const auto stage = ToStage(stage_); if (!stage.has_value()) { VALIDATION_LOG << "Invalid runtime stage."; return nullptr; } - runtime_stage.stage = stage.value(); + runtime_stage->stage = stage.value(); const auto target_platform = ToTargetPlatform(target_platform_); if (!target_platform.has_value()) { VALIDATION_LOG << "Invalid target platform for runtime stage."; return nullptr; } - runtime_stage.target_platform = target_platform.value(); + runtime_stage->target_platform = target_platform.value(); if (!shader_) { VALIDATION_LOG << "No shader specified for runtime stage."; return nullptr; } if (shader_->GetSize() > 0u) { - runtime_stage.shader = {shader_->GetMapping(), - shader_->GetMapping() + shader_->GetSize()}; + runtime_stage->shader = {shader_->GetMapping(), + shader_->GetMapping() + shader_->GetSize()}; } // It is not an error for the SkSL to be ommitted. if (sksl_ && sksl_->GetSize() > 0u) { - runtime_stage.sksl = {sksl_->GetMapping(), - sksl_->GetMapping() + sksl_->GetSize()}; + runtime_stage->sksl = {sksl_->GetMapping(), + sksl_->GetMapping() + sksl_->GetSize()}; } for (const auto& uniform : uniforms_) { auto desc = std::make_unique(); @@ -326,7 +373,7 @@ std::shared_ptr RuntimeStageData::CreateMapping() const { desc->location = uniform.location; desc->rows = uniform.rows; desc->columns = uniform.columns; - auto uniform_type = ToType(uniform.type); + auto uniform_type = ToUniformType(uniform.type); if (!uniform_type.has_value()) { VALIDATION_LOG << "Invalid uniform type for runtime stage."; return nullptr; @@ -337,10 +384,46 @@ std::shared_ptr RuntimeStageData::CreateMapping() const { desc->array_elements = uniform.array_elements.value(); } - runtime_stage.uniforms.emplace_back(std::move(desc)); + runtime_stage->uniforms.emplace_back(std::move(desc)); + } + + for (const auto& input : inputs_) { + auto desc = std::make_unique(); + + desc->name = input.name; + + if (desc->name.empty()) { + VALIDATION_LOG << "Stage input name cannot be empty."; + return nullptr; + } + desc->location = input.location; + desc->set = input.set; + desc->binding = input.binding; + auto input_type = ToInputType(input.type); + if (!input_type.has_value()) { + VALIDATION_LOG << "Invalid uniform type for runtime stage."; + return nullptr; + } + desc->type = input_type.value(); + desc->bit_width = input.bit_width; + desc->vec_size = input.vec_size; + desc->columns = input.columns; + desc->offset = input.offset; + + runtime_stage->inputs.emplace_back(std::move(desc)); } + + return runtime_stage; +} + +std::shared_ptr RuntimeStageData::CreateMapping() const { + auto runtime_stage = CreateFlatbuffer(); + if (!runtime_stage) { + return nullptr; + } + auto builder = std::make_shared(); - builder->Finish(fb::RuntimeStage::Pack(*builder.get(), &runtime_stage), + builder->Finish(fb::RuntimeStage::Pack(*builder.get(), runtime_stage.get()), fb::RuntimeStageIdentifier()); return std::make_shared(builder->GetBufferPointer(), builder->GetSize(), diff --git a/impeller/compiler/runtime_stage_data.h b/impeller/compiler/runtime_stage_data.h index 45e3476fc84f5..b012d70ff9e00 100644 --- a/impeller/compiler/runtime_stage_data.h +++ b/impeller/compiler/runtime_stage_data.h @@ -10,6 +10,7 @@ #include "flutter/fml/macros.h" #include "flutter/fml/mapping.h" #include "impeller/compiler/types.h" +#include "runtime_stage_types_flatbuffers.h" #include "spirv_parser.hpp" namespace impeller { @@ -25,6 +26,19 @@ struct UniformDescription { std::optional array_elements = std::nullopt; }; +struct InputDescription { + std::string name; + size_t location; + size_t set; + size_t binding; + spirv_cross::SPIRType::BaseType type = + spirv_cross::SPIRType::BaseType::Unknown; + size_t bit_width; + size_t vec_size; + size_t columns; + size_t offset; +}; + class RuntimeStageData { public: RuntimeStageData(std::string entrypoint, @@ -35,10 +49,14 @@ class RuntimeStageData { void AddUniformDescription(UniformDescription uniform); + void AddInputDescription(InputDescription input); + void SetShaderData(std::shared_ptr shader); void SetSkSLData(std::shared_ptr sksl); + std::unique_ptr CreateFlatbuffer() const; + std::shared_ptr CreateMapping() const; std::shared_ptr CreateJsonMapping() const; @@ -48,6 +66,7 @@ class RuntimeStageData { const spv::ExecutionModel stage_; const TargetPlatform target_platform_; std::vector uniforms_; + std::vector inputs_; std::shared_ptr shader_; std::shared_ptr sksl_; diff --git a/impeller/compiler/shader_bundle.cc b/impeller/compiler/shader_bundle.cc new file mode 100644 index 0000000000000..d71d3bfa3c3a1 --- /dev/null +++ b/impeller/compiler/shader_bundle.cc @@ -0,0 +1,219 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "impeller/compiler/shader_bundle.h" +#include "impeller/compiler/compiler.h" +#include "impeller/compiler/reflector.h" +#include "impeller/compiler/source_options.h" +#include "impeller/compiler/types.h" + +#include "impeller/compiler/utilities.h" +#include "impeller/shader_bundle/shader_bundle_flatbuffers.h" +#include "third_party/json/include/nlohmann/json.hpp" + +namespace impeller { +namespace compiler { + +std::optional ParseShaderBundleConfig( + const std::string& bundle_config_json, + std::ostream& error_stream) { + auto json = nlohmann::json::parse(bundle_config_json, nullptr, false); + if (json.is_discarded() || !json.is_object()) { + error_stream << "The shader bundle is not a valid JSON object." + << std::endl; + return std::nullopt; + } + + ShaderBundleConfig bundle; + for (auto& [shader_name, shader_value] : json.items()) { + if (bundle.find(shader_name) != bundle.end()) { + error_stream << "Duplicate shader \"" << shader_name << "\"." + << std::endl; + return std::nullopt; + } + if (!shader_value.is_object()) { + error_stream << "Invalid shader entry \"" << shader_name + << "\": Entry is not a JSON object." << std::endl; + return std::nullopt; + } + + ShaderConfig shader; + + if (!shader_value.contains("file")) { + error_stream << "Invalid shader entry \"" << shader_name + << "\": Missing required \"file\" field." << std::endl; + return std::nullopt; + } + shader.source_file_name = shader_value["file"]; + + if (!shader_value.contains("type")) { + error_stream << "Invalid shader entry \"" << shader_name + << "\": Missing required \"type\" field." << std::endl; + return std::nullopt; + } + shader.type = SourceTypeFromString(shader_value["type"]); + if (shader.type == SourceType::kUnknown) { + error_stream << "Invalid shader entry \"" << shader_name + << "\": Shader type " << shader_value["type"] + << " is unknown." << std::endl; + return std::nullopt; + } + + shader.language = shader_value.contains("language") + ? ToSourceLanguage(shader_value["language"]) + : SourceLanguage::kGLSL; + if (shader.language == SourceLanguage::kUnknown) { + error_stream << "Invalid shader entry \"" << shader_name + << "\": Unknown language type " << shader_value["language"] + << "." << std::endl; + return std::nullopt; + } + + shader.entry_point = shader_value.contains("entry_point") + ? shader_value["entry_point"] + : "main"; + + bundle[shader_name] = shader; + } + + return bundle; +} + +static std::unique_ptr GenerateShaderFB( + SourceOptions& options, + const std::string& shader_name, + const ShaderConfig& shader_config) { + auto result = std::make_unique(); + result->name = shader_name; + + std::shared_ptr source_file_mapping = + fml::FileMapping::CreateReadOnly(shader_config.source_file_name); + if (!source_file_mapping) { + std::cerr << "Could not open file for bundled shader \"" << shader_name + << "\"." << std::endl; + return nullptr; + } + + /// Override options. + options.type = shader_config.type; + options.source_language = shader_config.language; + options.entry_point_name = EntryPointFunctionNameFromSourceName( + shader_config.source_file_name, options.type, options.source_language, + shader_config.entry_point); + + Reflector::Options reflector_options; + reflector_options.target_platform = options.target_platform; + reflector_options.entry_point_name = options.entry_point_name; + reflector_options.shader_name = shader_name; + + Compiler compiler(source_file_mapping, options, reflector_options); + if (!compiler.IsValid()) { + std::cerr << "Compilation failed for bundled shader \"" << shader_name + << "\"." << std::endl; + std::cerr << compiler.GetErrorMessages() << std::endl; + return nullptr; + } + + auto reflector = compiler.GetReflector(); + if (reflector == nullptr) { + std::cerr << "Could not create reflector for bundled shader \"" + << shader_name << "\"." << std::endl; + return nullptr; + } + + auto stage_data = reflector->GetRuntimeStageData(); + if (!stage_data) { + std::cerr << "Runtime stage information was nil for bundled shader \"" + << shader_name << "\"." << std::endl; + return nullptr; + } + + result->shader = stage_data->CreateFlatbuffer(); + if (!result->shader) { + std::cerr << "Failed to create flatbuffer for bundled shader \"" + << shader_name << "\"." << std::endl; + return nullptr; + } + + return result; +} + +std::optional GenerateShaderBundleFlatbuffer( + const std::string& bundle_config_json, + SourceOptions& options) { + // -------------------------------------------------------------------------- + /// 1. Parse the bundle configuration. + /// + + std::optional bundle_config = + ParseShaderBundleConfig(bundle_config_json, std::cerr); + if (!bundle_config) { + return std::nullopt; + } + + // -------------------------------------------------------------------------- + /// 2. Build the deserialized shader bundle. + /// + + fb::ShaderBundleT shader_bundle; + + for (const auto& [shader_name, shader_config] : bundle_config.value()) { + std::unique_ptr shader = + GenerateShaderFB(options, shader_name, shader_config); + if (!shader) { + return std::nullopt; + } + shader_bundle.shaders.push_back(std::move(shader)); + } + + return shader_bundle; +} + +bool GenerateShaderBundle(Switches& switches, SourceOptions& options) { + // -------------------------------------------------------------------------- + /// 1. Parse the shader bundle and generate the flatbuffer result. + /// + + auto shader_bundle = + GenerateShaderBundleFlatbuffer(switches.shader_bundle, options); + if (!shader_bundle.has_value()) { + // Specific error messages are already handled by + // GenerateShaderBundleFlatbuffer. + return false; + } + + // -------------------------------------------------------------------------- + /// 2. Serialize the shader bundle and write to disk. + /// + + auto builder = std::make_shared(); + builder->Finish( + fb::ShaderBundle::Pack(*builder.get(), &shader_bundle.value()), + fb::ShaderBundleIdentifier()); + auto mapping = std::make_shared( + builder->GetBufferPointer(), builder->GetSize(), + [builder](auto, auto) {}); + + auto sl_file_name = std::filesystem::absolute( + std::filesystem::current_path() / switches.sl_file_name); + + if (!fml::WriteAtomically(*switches.working_directory, // + Utf8FromPath(sl_file_name).c_str(), // + *mapping // + )) { + std::cerr << "Could not write file to " << switches.sl_file_name + << std::endl; + return false; + } + // Tools that consume the runtime stage data expect the access mode to + // be 0644. + if (!SetPermissiveAccess(sl_file_name)) { + return false; + } + + return true; +} + +} // namespace compiler +} // namespace impeller diff --git a/impeller/compiler/shader_bundle.h b/impeller/compiler/shader_bundle.h new file mode 100644 index 0000000000000..438e5a56d754f --- /dev/null +++ b/impeller/compiler/shader_bundle.h @@ -0,0 +1,35 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "impeller/compiler/source_options.h" +#include "impeller/compiler/switches.h" +#include "impeller/shader_bundle/shader_bundle_flatbuffers.h" + +namespace impeller { +namespace compiler { + +/// @brief Parse a shader bundle configuration from a given JSON string. +/// +/// @note Exposed only for testing purposes. Use `GenerateShaderBundle` +/// directly. +std::optional ParseShaderBundleConfig( + const std::string& bundle_config_json, + std::ostream& error_stream); + +/// @brief Parses the JSON shader bundle configuration and invokes the +/// compiler multiple times to produce a shader bundle flatbuffer. +/// +/// @note Exposed only for testing purposes. Use `GenerateShaderBundle` +/// directly. +std::optional GenerateShaderBundleFlatbuffer( + const std::string& bundle_config_json, + SourceOptions& options); + +/// @brief Parses the JSON shader bundle configuration and invokes the +/// compiler multiple times to produce a shader bundle flatbuffer, which +/// is then output to the `sl` file. +bool GenerateShaderBundle(Switches& switches, SourceOptions& options); + +} // namespace compiler +} // namespace impeller diff --git a/impeller/compiler/shader_bundle_unittests.cc b/impeller/compiler/shader_bundle_unittests.cc new file mode 100644 index 0000000000000..375f366bf64a4 --- /dev/null +++ b/impeller/compiler/shader_bundle_unittests.cc @@ -0,0 +1,214 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "gtest/gtest.h" +#include "impeller/compiler/shader_bundle.h" + +#include "flutter/testing/testing.h" +#include "impeller/compiler/source_options.h" +#include "impeller/compiler/types.h" +#include "impeller/runtime_stage/runtime_stage_types_flatbuffers.h" +#include "impeller/shader_bundle/shader_bundle_flatbuffers.h" + +namespace impeller { +namespace compiler { +namespace testing { + +const std::string kUnlitFragmentBundleConfig = + "\"UnlitFragment\": {\"type\": \"fragment\", \"file\": " + "\"shaders/flutter_gpu_unlit.frag\"}"; +const std::string kUnlitVertexBundleConfig = + "\"UnlitVertex\": {\"type\": \"vertex\", \"file\": " + "\"shaders/flutter_gpu_unlit.vert\"}"; + +TEST(ShaderBundleTest, ParseShaderBundleConfigFailsForInvalidJSON) { + std::string bundle = ""; + std::stringstream error; + auto result = ParseShaderBundleConfig(bundle, error); + ASSERT_FALSE(result.has_value()); + ASSERT_STREQ(error.str().c_str(), + "The shader bundle is not a valid JSON object.\n"); +} + +TEST(ShaderBundleTest, ParseShaderBundleConfigFailsWhenEntryNotObject) { + std::string bundle = "{\"UnlitVertex\": []}"; + std::stringstream error; + auto result = ParseShaderBundleConfig(bundle, error); + ASSERT_FALSE(result.has_value()); + ASSERT_STREQ( + error.str().c_str(), + "Invalid shader entry \"UnlitVertex\": Entry is not a JSON object.\n"); +} + +TEST(ShaderBundleTest, ParseShaderBundleConfigFailsWhenMissingFile) { + std::string bundle = "{\"UnlitVertex\": {\"type\": \"vertex\"}}"; + std::stringstream error; + auto result = ParseShaderBundleConfig(bundle, error); + ASSERT_FALSE(result.has_value()); + ASSERT_STREQ(error.str().c_str(), + "Invalid shader entry \"UnlitVertex\": Missing required " + "\"file\" field.\n"); +} + +TEST(ShaderBundleTest, ParseShaderBundleConfigFailsWhenMissingType) { + std::string bundle = + "{\"UnlitVertex\": {\"file\": \"shaders/flutter_gpu_unlit.vert\"}}"; + std::stringstream error; + auto result = ParseShaderBundleConfig(bundle, error); + ASSERT_FALSE(result.has_value()); + ASSERT_STREQ(error.str().c_str(), + "Invalid shader entry \"UnlitVertex\": Missing required " + "\"type\" field.\n"); +} + +TEST(ShaderBundleTest, ParseShaderBundleConfigFailsForInvalidType) { + std::string bundle = + "{\"UnlitVertex\": {\"type\": \"invalid\", \"file\": " + "\"shaders/flutter_gpu_unlit.vert\"}}"; + std::stringstream error; + auto result = ParseShaderBundleConfig(bundle, error); + ASSERT_FALSE(result.has_value()); + ASSERT_STREQ(error.str().c_str(), + "Invalid shader entry \"UnlitVertex\": Shader type " + "\"invalid\" is unknown.\n"); +} + +TEST(ShaderBundleTest, ParseShaderBundleConfigFailsForInvalidLanguage) { + std::string bundle = + "{\"UnlitVertex\": {\"type\": \"vertex\", \"language\": \"invalid\", " + "\"file\": \"shaders/flutter_gpu_unlit.vert\"}}"; + std::stringstream error; + auto result = ParseShaderBundleConfig(bundle, error); + ASSERT_FALSE(result.has_value()); + ASSERT_STREQ(error.str().c_str(), + "Invalid shader entry \"UnlitVertex\": Unknown language type " + "\"invalid\".\n"); +} + +TEST(ShaderBundleTest, ParseShaderBundleConfigReturnsExpectedConfig) { + std::string bundle = + "{" + kUnlitVertexBundleConfig + ", " + kUnlitFragmentBundleConfig + "}"; + std::stringstream error; + auto result = ParseShaderBundleConfig(bundle, error); + ASSERT_TRUE(result.has_value()); + ASSERT_STREQ(error.str().c_str(), ""); + + // NOLINTBEGIN(bugprone-unchecked-optional-access) + auto maybe_vertex = result->find("UnlitVertex"); + auto maybe_fragment = result->find("UnlitFragment"); + ASSERT_TRUE(maybe_vertex != result->end()); + ASSERT_TRUE(maybe_fragment != result->end()); + auto vertex = maybe_vertex->second; + auto fragment = maybe_fragment->second; + // NOLINTEND(bugprone-unchecked-optional-access) + + EXPECT_EQ(vertex.type, SourceType::kVertexShader); + EXPECT_EQ(vertex.language, SourceLanguage::kGLSL); + EXPECT_STREQ(vertex.entry_point.c_str(), "main"); + EXPECT_STREQ(vertex.source_file_name.c_str(), + "shaders/flutter_gpu_unlit.vert"); + + EXPECT_EQ(fragment.type, SourceType::kFragmentShader); + EXPECT_EQ(fragment.language, SourceLanguage::kGLSL); + EXPECT_STREQ(fragment.entry_point.c_str(), "main"); + EXPECT_STREQ(fragment.source_file_name.c_str(), + "shaders/flutter_gpu_unlit.frag"); +} + +template +const T* FindByName(const std::vector>& collection, + const std::string& name) { + const auto maybe = std::find_if( + collection.begin(), collection.end(), + [&name](const std::unique_ptr& value) { return value->name == name; }); + if (maybe == collection.end()) { + return nullptr; + } + return maybe->get(); +} + +TEST(ShaderBundleTest, GenerateShaderBundleFlatbufferProducesCorrectResult) { + std::string fixtures_path = flutter::testing::GetFixturesPath(); + std::string config = + "{\"UnlitFragment\": {\"type\": \"fragment\", \"file\": \"" + + fixtures_path + + "/flutter_gpu_unlit.frag\"}, \"UnlitVertex\": {\"type\": " + "\"vertex\", \"file\": \"" + + fixtures_path + "/flutter_gpu_unlit.vert\"}}"; + + SourceOptions options; + options.target_platform = TargetPlatform::kRuntimeStageMetal; + options.source_language = SourceLanguage::kGLSL; + + std::optional bundle = + GenerateShaderBundleFlatbuffer(config, options); + ASSERT_TRUE(bundle.has_value()); + + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + const auto& shaders = bundle->shaders; + const auto* vertex = FindByName(shaders, "UnlitVertex"); + const auto* fragment = FindByName(shaders, "UnlitFragment"); + ASSERT_NE(vertex, nullptr); + ASSERT_NE(fragment, nullptr); + + // -------------------------------------------------------------------------- + /// Verify vertex shader. + /// + + EXPECT_STREQ(vertex->shader->entrypoint.c_str(), + "flutter_gpu_unlit_vertex_main"); + EXPECT_EQ(vertex->shader->stage, fb::Stage::kVertex); + EXPECT_EQ(vertex->shader->target_platform, fb::TargetPlatform::kMetal); + + // Inputs. + ASSERT_EQ(vertex->shader->inputs.size(), 1u); + const auto& v_in_position = vertex->shader->inputs[0]; + EXPECT_STREQ(v_in_position->name.c_str(), "position"); + EXPECT_EQ(v_in_position->location, 0u); + EXPECT_EQ(v_in_position->set, 0u); + EXPECT_EQ(v_in_position->binding, 0u); + EXPECT_EQ(v_in_position->type, fb::InputDataType::kFloat); + EXPECT_EQ(v_in_position->bit_width, 32u); + EXPECT_EQ(v_in_position->vec_size, 2u); + EXPECT_EQ(v_in_position->columns, 1u); + EXPECT_EQ(v_in_position->offset, 0u); + + // Uniforms. + ASSERT_EQ(vertex->shader->uniforms.size(), 2u); + const auto* v_mvp = FindByName(vertex->shader->uniforms, "mvp"); + ASSERT_NE(v_mvp, nullptr); + EXPECT_EQ(v_mvp->location, 0u); + EXPECT_EQ(v_mvp->type, fb::UniformDataType::kFloat); + EXPECT_EQ(v_mvp->bit_width, 32u); + EXPECT_EQ(v_mvp->rows, 4u); + EXPECT_EQ(v_mvp->columns, 4u); + EXPECT_EQ(v_mvp->array_elements, 0u); + const auto* v_color = FindByName(vertex->shader->uniforms, "color"); + ASSERT_NE(v_color, nullptr); + EXPECT_EQ(v_color->location, 1u); + EXPECT_EQ(v_color->type, fb::UniformDataType::kFloat); + EXPECT_EQ(v_color->bit_width, 32u); + EXPECT_EQ(v_color->rows, 4u); + EXPECT_EQ(v_color->columns, 1u); + EXPECT_EQ(v_color->array_elements, 0u); + + // -------------------------------------------------------------------------- + /// Verify fragment shader. + /// + + EXPECT_STREQ(fragment->shader->entrypoint.c_str(), + "flutter_gpu_unlit_fragment_main"); + EXPECT_EQ(fragment->shader->stage, fb::Stage::kFragment); + EXPECT_EQ(fragment->shader->target_platform, fb::TargetPlatform::kMetal); + + // Inputs (not recorded for fragment shaders). + ASSERT_EQ(fragment->shader->inputs.size(), 0u); + + // Uniforms. + ASSERT_EQ(fragment->shader->inputs.size(), 0u); +} + +} // namespace testing +} // namespace compiler +} // namespace impeller diff --git a/impeller/compiler/switches.cc b/impeller/compiler/switches.cc index 0e590a5c5c89a..46a65c246f294 100644 --- a/impeller/compiler/switches.cc +++ b/impeller/compiler/switches.cc @@ -53,7 +53,8 @@ void Switches::PrintHelp(std::ostream& stream) { } stream << "}" << std::endl; stream << "--sl=" << std::endl; - stream << "--spirv=" << std::endl; + stream << "--spirv= (ignored for --shader-bundle)" + << std::endl; stream << "[optional] --source-language=glsl|hlsl (default: glsl)" << std::endl; stream << "[optional] --entry-point= (default: main; " @@ -61,8 +62,8 @@ void Switches::PrintHelp(std::ostream& stream) { << std::endl; stream << "[optional] --iplr (causes --sl file to be emitted in iplr format)" << std::endl; - stream << "[optional] --iplr-bundle= (causes --sl file to be " - "emitted in the iplr bundle format)" + stream << "[optional] --shader-bundle= (causes --sl file to be " + "emitted in Flutter GPU's shader bundle format)" << std::endl; stream << "[optional] --reflection-json=" << std::endl; stream << "[optional] --reflection-header=" @@ -122,7 +123,8 @@ Switches::Switches(const fml::CommandLine& command_line) input_type(SourceTypeFromCommandLine(command_line)), sl_file_name(command_line.GetOptionValueWithDefault("sl", "")), iplr(command_line.HasOption("iplr")), - iplr_bundle(command_line.GetOptionValueWithDefault("iplr-bundle", "")), + shader_bundle( + command_line.GetOptionValueWithDefault("shader-bundle", "")), spirv_file_name(command_line.GetOptionValueWithDefault("spirv", "")), reflection_json_name( command_line.GetOptionValueWithDefault("reflection-json", "")), @@ -142,15 +144,10 @@ Switches::Switches(const fml::CommandLine& command_line) use_half_textures(command_line.HasOption("use-half-textures")), require_framebuffer_fetch( command_line.HasOption("require-framebuffer-fetch")) { - auto language = - command_line.GetOptionValueWithDefault("source-language", "glsl"); - std::transform(language.begin(), language.end(), language.begin(), - [](char x) { return std::tolower(x); }); - if (language == "glsl") { - source_language = SourceLanguage::kGLSL; - } else if (language == "hlsl") { - source_language = SourceLanguage::kHLSL; - } + auto language = ToLowerCase( + command_line.GetOptionValueWithDefault("source-language", "glsl")); + + source_language = ToSourceLanguage(language); if (!working_directory || !working_directory->is_valid()) { return; @@ -196,6 +193,12 @@ Switches::Switches(const fml::CommandLine& command_line) } bool Switches::AreValid(std::ostream& explain) const { + // When producing a shader bundle, all flags related to single shader inputs + // and outputs such as `--input` and `--spirv-file-name` are ignored. Instead, + // input files are read from the shader bundle spec and a single flatbuffer + // containing all compiled shaders and reflection state is output to `--sl`. + const bool shader_bundle_mode = !shader_bundle.empty(); + bool valid = true; if (target_platform == TargetPlatform::kUnknown) { explain << "The target platform (only one) was not specified." << std::endl; @@ -214,7 +217,7 @@ bool Switches::AreValid(std::ostream& explain) const { valid = false; } - if (source_file_name.empty()) { + if (source_file_name.empty() && !shader_bundle_mode) { explain << "Input file name was empty." << std::endl; valid = false; } @@ -224,10 +227,18 @@ bool Switches::AreValid(std::ostream& explain) const { valid = false; } - if (spirv_file_name.empty()) { + if (spirv_file_name.empty() && !shader_bundle_mode) { explain << "Spirv file name was empty." << std::endl; valid = false; } + + if (iplr && shader_bundle_mode) { + explain << "--iplr and --shader-bundle flag cannot be specified at the " + "same time" + << std::endl; + valid = false; + } + return valid; } diff --git a/impeller/compiler/switches.h b/impeller/compiler/switches.h index c02e344fa4a88..7ac6e0c540a7d 100644 --- a/impeller/compiler/switches.h +++ b/impeller/compiler/switches.h @@ -22,9 +22,12 @@ struct Switches { std::vector include_directories = {}; std::string source_file_name = ""; SourceType input_type = SourceType::kUnknown; + /// The raw shader file output by the compiler. For --iplr and + /// --shader-bundle modes, this is used as the filename for the output + /// flatbuffer output. std::string sl_file_name = ""; bool iplr = false; - std::string iplr_bundle = ""; + std::string shader_bundle = ""; std::string spirv_file_name = ""; std::string reflection_json_name = ""; std::string reflection_header_name = ""; diff --git a/impeller/compiler/switches_unittests.cc b/impeller/compiler/switches_unittests.cc index 92b03ecc2a1fe..fd1591e193f0d 100644 --- a/impeller/compiler/switches_unittests.cc +++ b/impeller/compiler/switches_unittests.cc @@ -80,6 +80,19 @@ TEST(SwitchesTEst, ConvertToEntrypointName) { ASSERT_EQ(ConvertToEntrypointName(""), ""); } +TEST(SwitchesTest, ShaderBundleModeValid) { + // Shader bundles process multiple shaders, and so the single-file input/spirv + // flags are not required. + std::vector options = { + "--shader-bundle={}", "--sl=test.shaderbundle", "--runtime-stage-metal"}; + + auto cl = fml::CommandLineFromIteratorsWithArgv0("impellerc", options.begin(), + options.end()); + Switches switches(cl); + ASSERT_TRUE(switches.AreValid(std::cout)); + ASSERT_EQ(switches.shader_bundle, "{}"); +} + } // namespace testing } // namespace compiler } // namespace impeller diff --git a/impeller/compiler/types.cc b/impeller/compiler/types.cc index 77b40e9b6652e..2368bcd20fbec 100644 --- a/impeller/compiler/types.cc +++ b/impeller/compiler/types.cc @@ -4,6 +4,7 @@ #include "impeller/compiler/types.h" +#include #include #include @@ -42,6 +43,34 @@ SourceType SourceTypeFromFileName(const std::string& file_name) { return SourceType::kUnknown; } +SourceType SourceTypeFromString(std::string name) { + name = ToLowerCase(name); + + if (name == "vertex") { + return SourceType::kVertexShader; + } + + if (name == "fragment") { + return SourceType::kFragmentShader; + } + + if (name == "compute") { + return SourceType::kComputeShader; + } + + return SourceType::kUnknown; +} + +SourceLanguage ToSourceLanguage(const std::string& source_language) { + if (source_language == "glsl") { + return SourceLanguage::kGLSL; + } + if (source_language == "hlsl") { + return SourceLanguage::kHLSL; + } + return SourceLanguage::kUnknown; +} + std::string TargetPlatformToString(TargetPlatform platform) { switch (platform) { case TargetPlatform::kUnknown: diff --git a/impeller/compiler/types.h b/impeller/compiler/types.h index 0af2a5c403352..dee14a5cdfcd8 100644 --- a/impeller/compiler/types.h +++ b/impeller/compiler/types.h @@ -6,6 +6,7 @@ #include #include +#include #include #include "flutter/fml/macros.h" @@ -42,6 +43,16 @@ enum class SourceLanguage { kHLSL, }; +/// A shader config parsed as part of a ShaderBundleConfig. +struct ShaderConfig { + std::string source_file_name; + SourceType type; + SourceLanguage language; + std::string entry_point; +}; + +using ShaderBundleConfig = std::unordered_map; + bool TargetPlatformIsMetal(TargetPlatform platform); bool TargetPlatformIsOpenGL(TargetPlatform platform); @@ -50,10 +61,14 @@ bool TargetPlatformIsVulkan(TargetPlatform platform); SourceType SourceTypeFromFileName(const std::string& file_name); +SourceType SourceTypeFromString(std::string name); + std::string SourceTypeToString(SourceType type); std::string TargetPlatformToString(TargetPlatform platform); +SourceLanguage ToSourceLanguage(const std::string& source_language); + std::string SourceLanguageToString(SourceLanguage source_language); std::string TargetPlatformSLExtension(TargetPlatform platform); diff --git a/impeller/compiler/utilities.cc b/impeller/compiler/utilities.cc index 75a535a0c7092..ef24dc1681a5d 100644 --- a/impeller/compiler/utilities.cc +++ b/impeller/compiler/utilities.cc @@ -4,13 +4,29 @@ #include "impeller/compiler/utilities.h" +#include #include #include +#include #include namespace impeller { namespace compiler { +bool SetPermissiveAccess(const std::filesystem::path& p) { + auto permissions = + std::filesystem::perms::owner_read | std::filesystem::perms::owner_write | + std::filesystem::perms::group_read | std::filesystem::perms::others_read; + std::error_code error; + std::filesystem::permissions(p, permissions, error); + if (error) { + std::cerr << "Failed to set access on file '" << p + << "': " << error.message() << std::endl; + return false; + } + return true; +} + std::string Utf8FromPath(const std::filesystem::path& path) { return reinterpret_cast(path.u8string().c_str()); } @@ -20,7 +36,7 @@ std::string InferShaderNameFromPath(std::string_view path) { return Utf8FromPath(p); } -std::string ConvertToCamelCase(std::string_view string) { +std::string ToCamelCase(std::string_view string) { if (string.empty()) { return ""; } @@ -43,6 +59,13 @@ std::string ConvertToCamelCase(std::string_view string) { return stream.str(); } +std::string ToLowerCase(std::string_view string) { + std::string result = std::string(string); + std::transform(result.begin(), result.end(), result.begin(), + [](char x) { return std::tolower(x); }); + return result; +} + std::string ConvertToEntrypointName(std::string_view string) { if (string.empty()) { return ""; diff --git a/impeller/compiler/utilities.h b/impeller/compiler/utilities.h index d1d27436ac08e..2aff1530fa2fd 100644 --- a/impeller/compiler/utilities.h +++ b/impeller/compiler/utilities.h @@ -11,6 +11,9 @@ namespace impeller { namespace compiler { +/// @brief Sets the file access mode of the file at path 'p' to 0644. +bool SetPermissiveAccess(const std::filesystem::path& p); + /// @brief Converts a native format path to a utf8 string. /// /// This utility uses `path::u8string()` to convert native paths to @@ -21,7 +24,9 @@ std::string Utf8FromPath(const std::filesystem::path& path); std::string InferShaderNameFromPath(std::string_view path); -std::string ConvertToCamelCase(std::string_view string); +std::string ToCamelCase(std::string_view string); + +std::string ToLowerCase(std::string_view string); /// @brief Ensure that the entrypoint name is a valid identifier in the target /// language. diff --git a/impeller/fixtures/BUILD.gn b/impeller/fixtures/BUILD.gn index 073e3bee59f32..1867b65862e0d 100644 --- a/impeller/fixtures/BUILD.gn +++ b/impeller/fixtures/BUILD.gn @@ -76,6 +76,10 @@ test_fixtures("file_fixtures") { "blue_noise.png", "boston.jpg", "embarcadero.jpg", + "flutter_gpu_texture.frag", + "flutter_gpu_texture.vert", + "flutter_gpu_unlit.frag", + "flutter_gpu_unlit.vert", "flutter_logo_baked.glb", "kalimba.jpg", "multiple_stages.hlsl", @@ -121,17 +125,19 @@ impellerc("flutter_gpu_shaders") { "flutter_gpu_texture.frag", "flutter_gpu_texture.vert", ] - sl_file_extension = "iplr" shader_target_flag = "--runtime-stage-metal" - iplr = true - iplr_bundle = "temporary" + shader_bundle = "{\"UnlitFragment\": {\"type\": \"fragment\", \"file\": \"../../flutter/impeller/fixtures/flutter_gpu_unlit.frag\"}, \"UnlitVertex\": {\"type\": \"vertex\", \"file\": \"../../flutter/impeller/fixtures/flutter_gpu_unlit.vert\"}, \"TextureFragment\": {\"type\": \"fragment\", \"file\": \"../../flutter/impeller/fixtures/flutter_gpu_texture.frag\"}, \"TextureVertex\": {\"type\": \"vertex\", \"file\": \"../../flutter/impeller/fixtures/flutter_gpu_texture.vert\"}}" + shader_bundle_output = "playground.shaderbundle" } test_fixtures("flutter_gpu_fixtures") { dart_main = "dart_tests.dart" - fixtures = - filter_include(get_target_outputs(":flutter_gpu_shaders"), [ "*.iplr" ]) + fixtures = filter_include(get_target_outputs(":flutter_gpu_shaders"), + [ + "*.iplr", + "*.shaderbundle", + ]) deps = [ ":flutter_gpu_shaders" ] } diff --git a/impeller/renderer/renderer_dart_unittests.cc b/impeller/renderer/renderer_dart_unittests.cc index 7e413346cc9a0..ae9a7804d82e6 100644 --- a/impeller/renderer/renderer_dart_unittests.cc +++ b/impeller/renderer/renderer_dart_unittests.cc @@ -18,7 +18,6 @@ #include "flutter/testing/testing.h" #include "fml/memory/ref_ptr.h" #include "impeller/core/shader_types.h" -#include "impeller/fixtures/flutter_gpu_unlit.vert.h" #include "impeller/playground/playground_test.h" #include "impeller/renderer/render_pass.h" #include "impeller/renderer/vertex_descriptor.h" @@ -30,37 +29,11 @@ namespace impeller { namespace testing { -// This helper is for piggybacking on the RuntimeStage infrastructure for -// testing shaders/pipelines before the full shader bundle importer is finished. -static fml::RefPtr OpenRuntimeStageAsShader( - const std::string& fixture_name, - std::shared_ptr vertex_desc) { - auto fixture = flutter::testing::OpenFixtureAsMapping(fixture_name); - assert(fixture); - RuntimeStage stage(std::move(fixture)); - return flutter::gpu::Shader::Make( - stage.GetEntrypoint(), ToShaderStage(stage.GetShaderStage()), - stage.GetCodeMapping(), stage.GetUniforms(), std::move(vertex_desc)); -} - static void InstantiateTestShaderLibrary() { - flutter::gpu::ShaderLibrary::ShaderMap shaders; - auto vertex_desc = std::make_shared(); - vertex_desc->SetStageInputs( - // TODO(bdero): The stage inputs need to be packed into the flatbuffer. - FlutterGpuUnlitVertexShader::kAllShaderStageInputs, - // TODO(bdero): Make the vertex attribute layout fully configurable. - // When encoding commands, allow for specifying a stride, - // type, and vertex buffer slot for each attribute. - // Provide a way to lookup vertex attribute slot locations by - // name from the shader. - FlutterGpuUnlitVertexShader::kInterleavedBufferLayout); - shaders["UnlitVertex"] = OpenRuntimeStageAsShader( - "flutter_gpu_unlit.vert.iplr", std::move(vertex_desc)); - shaders["UnlitFragment"] = - OpenRuntimeStageAsShader("flutter_gpu_unlit.frag.iplr", nullptr); + auto fixture = + flutter::testing::OpenFixtureAsMapping("playground.shaderbundle"); auto library = - flutter::gpu::ShaderLibrary::MakeFromShaders(std::move(shaders)); + flutter::gpu::ShaderLibrary::MakeFromFlatbuffer(std::move(fixture)); flutter::gpu::ShaderLibrary::SetOverride(library); } diff --git a/impeller/renderer/vertex_descriptor.cc b/impeller/renderer/vertex_descriptor.cc index a16ff1309e94a..1c912308859b4 100644 --- a/impeller/renderer/vertex_descriptor.cc +++ b/impeller/renderer/vertex_descriptor.cc @@ -25,6 +25,13 @@ void VertexDescriptor::SetStageInputs( } } +void VertexDescriptor::SetStageInputs( + const std::vector& inputs, + const std::vector& layout) { + inputs_.insert(inputs_.end(), inputs.begin(), inputs.end()); + layouts_.insert(layouts_.end(), layout.begin(), layout.end()); +} + void VertexDescriptor::RegisterDescriptorSetLayouts( const DescriptorSetLayout desc_set_layout[], size_t count) { diff --git a/impeller/renderer/vertex_descriptor.h b/impeller/renderer/vertex_descriptor.h index 72d12e11df61e..db7cb7aae61e3 100644 --- a/impeller/renderer/vertex_descriptor.h +++ b/impeller/renderer/vertex_descriptor.h @@ -6,7 +6,6 @@ #include -#include "flutter/fml/macros.h" #include "impeller/base/comparable.h" #include "impeller/core/shader_types.h" @@ -38,6 +37,9 @@ class VertexDescriptor final : public Comparable { layout.size()); } + void SetStageInputs(const std::vector& inputs, + const std::vector& layout); + template void RegisterDescriptorSetLayouts( const std::array& inputs) { diff --git a/impeller/runtime_stage/BUILD.gn b/impeller/runtime_stage/BUILD.gn index e8f287132c627..8cc4f3b0a2235 100644 --- a/impeller/runtime_stage/BUILD.gn +++ b/impeller/runtime_stage/BUILD.gn @@ -7,13 +7,25 @@ import("../tools/impeller.gni") config("runtime_stage_config") { configs = [ "//flutter/impeller:impeller_public_config" ] - include_dirs = [ "$root_gen_dir/flutter" ] + include_dirs = [ + "$root_gen_dir/flutter", + "$root_gen_dir/flutter/impeller/runtime_stage", + ] +} + +flatbuffers("runtime_stage_types_flatbuffers") { + flatbuffers = [ "runtime_stage_types.fbs" ] + public_configs = [ ":runtime_stage_config" ] + public_deps = [ "//flutter/third_party/flatbuffers" ] } flatbuffers("runtime_stage_flatbuffers") { flatbuffers = [ "runtime_stage.fbs" ] public_configs = [ ":runtime_stage_config" ] - public_deps = [ "//flutter/third_party/flatbuffers" ] + public_deps = [ + ":runtime_stage_types_flatbuffers", + "//flutter/third_party/flatbuffers", + ] } impeller_component("runtime_stage") { diff --git a/impeller/runtime_stage/runtime_stage.cc b/impeller/runtime_stage/runtime_stage.cc index aed0e01509fcc..cbdee53517ad4 100644 --- a/impeller/runtime_stage/runtime_stage.cc +++ b/impeller/runtime_stage/runtime_stage.cc @@ -63,7 +63,14 @@ RuntimeStage::RuntimeStage(std::shared_ptr payload) if (!fb::RuntimeStageBufferHasIdentifier(payload_->GetMapping())) { return; } - auto runtime_stage = fb::GetRuntimeStage(payload_->GetMapping()); + Setup(fb::GetRuntimeStage(payload_->GetMapping())); +} + +RuntimeStage::RuntimeStage(const fb::RuntimeStage* runtime_stage) { + Setup(runtime_stage); +} + +void RuntimeStage::Setup(const fb::RuntimeStage* runtime_stage) { if (!runtime_stage) { return; } diff --git a/impeller/runtime_stage/runtime_stage.fbs b/impeller/runtime_stage/runtime_stage.fbs index fdb67c71917b5..afb2589cbb18b 100644 --- a/impeller/runtime_stage/runtime_stage.fbs +++ b/impeller/runtime_stage/runtime_stage.fbs @@ -2,55 +2,9 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -namespace impeller.fb; - -enum Stage:byte { - kVertex, - kFragment, - kCompute, -} - -enum TargetPlatform:byte { - kMetal, - kOpenGLES, - kSkSL, - kVulkan, -} +include "runtime_stage_types.fbs"; -enum UniformDataType:uint32 { - kBoolean, - kSignedByte, - kUnsignedByte, - kSignedShort, - kUnsignedShort, - kSignedInt, - kUnsignedInt, - kSignedInt64, - kUnsignedInt64, - kHalfFloat, - kFloat, - kDouble, - kSampledImage, -} - -table UniformDescription { - name: string; - location: uint64; - type: UniformDataType; - bit_width: uint64; - rows: uint64; - columns: uint64; - array_elements: uint64; -} - -table RuntimeStage { - stage: Stage; - target_platform: TargetPlatform; - entrypoint: string; - uniforms: [UniformDescription]; - shader: [ubyte]; - sksl: [ubyte]; -} +namespace impeller.fb; root_type RuntimeStage; file_identifier "IPLR"; diff --git a/impeller/runtime_stage/runtime_stage.h b/impeller/runtime_stage/runtime_stage.h index 8ee501bbdb650..989b4fab2ac19 100644 --- a/impeller/runtime_stage/runtime_stage.h +++ b/impeller/runtime_stage/runtime_stage.h @@ -11,6 +11,7 @@ #include "flutter/fml/mapping.h" #include "flutter/impeller/core/runtime_types.h" +#include "runtime_stage_types_flatbuffers.h" namespace impeller { @@ -18,6 +19,8 @@ class RuntimeStage { public: explicit RuntimeStage(std::shared_ptr payload); + explicit RuntimeStage(const fb::RuntimeStage* runtime_stage); + ~RuntimeStage(); RuntimeStage(RuntimeStage&&); RuntimeStage& operator=(RuntimeStage&&); @@ -50,6 +53,8 @@ class RuntimeStage { bool is_valid_ = false; bool is_dirty_ = true; + void Setup(const fb::RuntimeStage* runtime_stage); + RuntimeStage(const RuntimeStage&) = delete; RuntimeStage& operator=(const RuntimeStage&) = delete; diff --git a/impeller/runtime_stage/runtime_stage_types.fbs b/impeller/runtime_stage/runtime_stage_types.fbs new file mode 100644 index 0000000000000..b0f7baf152957 --- /dev/null +++ b/impeller/runtime_stage/runtime_stage_types.fbs @@ -0,0 +1,84 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +namespace impeller.fb; + +enum Stage:byte { + kVertex, + kFragment, + kCompute, +} + +enum TargetPlatform:byte { + kMetal, + kOpenGLES, + kSkSL, + kVulkan, +} + +// The subset of impeller::ShaderType that may be used for uniform bindings. +enum UniformDataType:uint32 { + kBoolean, + kSignedByte, + kUnsignedByte, + kSignedShort, + kUnsignedShort, + kSignedInt, + kUnsignedInt, + kSignedInt64, + kUnsignedInt64, + kHalfFloat, + kFloat, + kDouble, + kSampledImage, +} + +table UniformDescription { + name: string; + location: uint64; + type: UniformDataType; + bit_width: uint64; + rows: uint64; + columns: uint64; + array_elements: uint64; +} + +// The subset of impeller::ShaderType that may be used for vertex attributes. +enum InputDataType:uint32 { + kBoolean, + kSignedByte, + kUnsignedByte, + kSignedShort, + kUnsignedShort, + kSignedInt, + kUnsignedInt, + kSignedInt64, + kUnsignedInt64, + kFloat, + kDouble, +} + +// This contains the same attribute reflection data as +// impeller::ShaderStageIOSlot. +table StageInput { + name: string; + location: uint64; + set: uint64; + binding: uint64; + type: InputDataType; + bit_width: uint64; + vec_size: uint64; + columns: uint64; + offset: uint64; +} + +table RuntimeStage { + stage: Stage; + target_platform: TargetPlatform; + entrypoint: string; + inputs: [StageInput]; + uniforms: [UniformDescription]; + shader: [ubyte]; + sksl: [ubyte]; +} diff --git a/impeller/shader_bundle/BUILD.gn b/impeller/shader_bundle/BUILD.gn new file mode 100644 index 0000000000000..fd58931d755ef --- /dev/null +++ b/impeller/shader_bundle/BUILD.gn @@ -0,0 +1,16 @@ +# Copyright 2013 The Flutter Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import("//flutter/third_party/flatbuffers/flatbuffers.gni") +import("../tools/impeller.gni") + +flatbuffers("shader_bundle_flatbuffers") { + flatbuffers = [ "shader_bundle.fbs" ] + public_configs = [ "//flutter/impeller/runtime_stage:runtime_stage_config" ] + public_deps = [ + "//flutter/impeller/runtime_stage:runtime_stage_flatbuffers", + "//flutter/impeller/runtime_stage:runtime_stage_types_flatbuffers", + "//flutter/third_party/flatbuffers", + ] +} diff --git a/impeller/shader_bundle/shader_bundle.fbs b/impeller/shader_bundle/shader_bundle.fbs new file mode 100644 index 0000000000000..1c0f4ce038967 --- /dev/null +++ b/impeller/shader_bundle/shader_bundle.fbs @@ -0,0 +1,19 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +include "../runtime_stage/runtime_stage_types.fbs"; + +namespace impeller.fb; + +table Shader { + name: string; + shader: RuntimeStage; +} + +table ShaderBundle { + shaders: [Shader]; +} + +root_type ShaderBundle; +file_identifier "IPSB"; diff --git a/impeller/tools/impeller.gni b/impeller/tools/impeller.gni index 4d48704493975..a07963ee380b3 100644 --- a/impeller/tools/impeller.gni +++ b/impeller/tools/impeller.gni @@ -239,31 +239,81 @@ template("embed_blob") { } # Dispatches to the build or prebuilt impellerc depending on the value of -# the impeller_use_prebuilt_impellerc argument. Forwards all variables to -# compiled_action_foreach or action_foreach as appropriate. +# the `impeller_use_prebuilt_impellerc` argument. +# * When the `single_invocation` argument is false, all variables are +# forwarded to `compiled_action_foreach` or `action_foreach`, which will +# invoke `impellerc` separately for each source. +# * When the `single_invocation` argument is true, `impellerc` is only +# invoked once via `compiled_action` or `action`. template("_impellerc") { - if (impeller_use_prebuilt_impellerc == "") { - compiled_action_foreach(target_name) { - forward_variables_from(invoker, "*") - tool = "//flutter/impeller/compiler:impellerc" + if (invoker.single_invocation) { + if (impeller_use_prebuilt_impellerc == "") { + compiled_action(target_name) { + forward_variables_from(invoker, "*") + tool = "//flutter/impeller/compiler:impellerc" + } + } else { + action(target_name) { + forward_variables_from(invoker, "*", [ "args" ]) + script = "//build/gn_run_binary.py" + impellerc_path = + rebase_path(impeller_use_prebuilt_impellerc, root_build_dir) + args = [ impellerc_path ] + invoker.args + } } } else { - action_foreach(target_name) { - forward_variables_from(invoker, "*", [ "args" ]) - script = "//build/gn_run_binary.py" - impellerc_path = - rebase_path(impeller_use_prebuilt_impellerc, root_build_dir) - args = [ impellerc_path ] + invoker.args + if (impeller_use_prebuilt_impellerc == "") { + compiled_action_foreach(target_name) { + forward_variables_from(invoker, "*") + tool = "//flutter/impeller/compiler:impellerc" + } + } else { + action_foreach(target_name) { + forward_variables_from(invoker, "*", [ "args" ]) + script = "//build/gn_run_binary.py" + impellerc_path = + rebase_path(impeller_use_prebuilt_impellerc, root_build_dir) + args = [ impellerc_path ] + invoker.args + } } } } +# Required: shaders The list of shaders inputs to compile. +# Required: shader_target_flag The target flag to append. Valid options: +# --sksl +# --metal-ios +# --metal-desktop +# --opengl-es +# --opengl-desktop +# --vulkan +# --runtime-stage-metal +# Required: sl_file_extension The file extension to use for output files. +# Not required for --shader_bundle mode. +# Optional: iplr Causes --sl output to be in iplr/runtime +# stage flatbuffer format. +# Optional: shader_bundle Specifies a Flutter GPU shader bundle +# configuration. +# Required: shader_bundle_output Specifies the output filename of the shader +# bundle. This is only required if +# shader_bundle is supplied. +# Optional: defines Specifies a list of valueless macro +# definitions. +# Optional: intermediates_subdir Specifies the subdirectory in which to put +# intermediates. +# Optional: json Causes output format to be JSON instead of +# flatbuffer. template("impellerc") { assert(defined(invoker.shaders), "Impeller shaders must be specified.") assert(defined(invoker.shader_target_flag), "The flag to impellerc for target selection must be specified.") - assert(defined(invoker.sl_file_extension), + assert(defined(invoker.sl_file_extension) || defined(invoker.shader_bundle), "The extension of the SL file must be specified (metal, glsl, etc..).") + if (defined(invoker.shader_bundle)) { + assert( + defined(invoker.shader_bundle_output), + "When shader_bundle is specified, shader_output_bundle must also be specified.") + } sksl = invoker.shader_target_flag == "--sksl" iplr = false @@ -279,17 +329,17 @@ template("impellerc") { not_needed([ "iplr", "sksl", + "shader_bundle", + "shader_bundle_output", ]) - # Optional: invoker.iplr Causes --sl output to be in iplr format. - # Optional: invoker.iplr_bundle specifies a Flutter GPU shader bundle configuration. - # Optional: invoker.defines specifies a list of valueless macro definitions. - # Optional: invoker.intermediates_subdir specifies the subdirectory in which - # to put intermediates. - # Optional: invoker.json Causes output format to be JSON instead of flatbuffer. - _impellerc(target_name) { - sources = invoker.shaders + shader_bundle = defined(invoker.shader_bundle) + + # When single_invocation is true, impellerc will be invoked exactly once. When it's + # false, impellerc be invoked for each of the source file entries (invoker.shaders). + single_invocation = shader_bundle + if (defined(invoker.intermediates_subdir)) { subdir = invoker.intermediates_subdir generated_dir = "$target_gen_dir/$subdir" @@ -299,22 +349,25 @@ template("impellerc") { shader_target_flag = invoker.shader_target_flag - spirv_intermediate = "$generated_dir/{{source_file_part}}.spirv" - spirv_intermediate_path = rebase_path(spirv_intermediate, root_build_dir) - depfile_path = "$generated_dir/{{source_file_part}}.d" depfile_intermediate_path = rebase_path(depfile_path, root_build_dir) depfile = depfile_path shader_lib_dir = rebase_path("//flutter/impeller/compiler/shader_lib") args = [ - "--input={{source}}", - "--include={{source_dir}}", "--include=$shader_lib_dir", "--depfile=$depfile_intermediate_path", "$shader_target_flag", ] + # When we're in single invocation mode, we can't use source enumeration. + if (!single_invocation) { + args += [ + "--input={{source}}", + "--include={{source_dir}}", + ] + } + if (defined(invoker.gles_language_version)) { gles_language_version = invoker.gles_language_version args += [ "--gles-language-version=$gles_language_version" ] @@ -338,27 +391,60 @@ template("impellerc") { args += [ "--json" ] } - if (sksl) { - sl_intermediate = + if (iplr) { + # When building in IPLR mode, the compiler may be executed twice + args += [ "--iplr" ] + } + + # The `sl_output` is the raw shader file output by the compiler. For --iplr + # and --shader-bundle, this is used as the filename for the flatbuffer to + # output. + + if (shader_bundle) { + # When a shader bundle is specified, don't bother supplying flags for + # the reflection state as these are ignored. In this mode, the compiler + # is invoked multiple times and the reflection state for each shader is + # written to the output flatbuffer. + sl_output = "$generated_dir/${invoker.shader_bundle_output}" + sl_output_path = rebase_path(sl_output, root_build_dir) + + args += [ + "--sl=$sl_output_path", + "--shader-bundle=${invoker.shader_bundle}", + ] + + outputs = [ sl_output ] + } else if (sksl) { + # When SkSL is selected as the `shader_target_flag`, don't generate + # C++ reflection state. Nothing needs to use it and it's likely invalid + # given the special cases when generating SkSL. + # Note that this configuration is orthogonal to the "--iplr" flag + sl_output = "$generated_dir/{{source_file_part}}.${invoker.sl_file_extension}" - sl_intermediate_path = rebase_path(sl_intermediate, root_build_dir) + sl_output_path = rebase_path(sl_output, root_build_dir) + + spirv_intermediate = "$generated_dir/{{source_file_part}}.spirv" + spirv_intermediate_path = rebase_path(spirv_intermediate, root_build_dir) args += [ - "--sl=$sl_intermediate_path", + "--sl=$sl_output_path", "--spirv=$spirv_intermediate_path", ] - if (iplr) { - args += [ "--iplr" ] - } - outputs = [ sl_intermediate ] + outputs = [ sl_output ] } else { - sl_intermediate = + # The default branch. Here we just generate one shader along with all of + # its C++ reflection state. + + sl_output = "$generated_dir/{{source_file_part}}.${invoker.sl_file_extension}" + sl_output_path = rebase_path(sl_output, root_build_dir) + reflection_json_intermediate = "$generated_dir/{{source_file_part}}.json" reflection_header_intermediate = "$generated_dir/{{source_file_part}}.h" reflection_cc_intermediate = "$generated_dir/{{source_file_part}}.cc" - sl_intermediate_path = rebase_path(sl_intermediate, root_build_dir) + spirv_intermediate = "$generated_dir/{{source_file_part}}.spirv" + spirv_intermediate_path = rebase_path(spirv_intermediate, root_build_dir) reflection_json_path = rebase_path(reflection_json_intermediate, root_build_dir) reflection_header_path = @@ -367,21 +453,15 @@ template("impellerc") { rebase_path(reflection_cc_intermediate, root_build_dir) args += [ - "--sl=$sl_intermediate_path", + "--sl=$sl_output_path", "--spirv=$spirv_intermediate_path", "--reflection-json=$reflection_json_path", "--reflection-header=$reflection_header_path", "--reflection-cc=$reflection_cc_path", ] - if (iplr) { - args += [ "--iplr" ] - } - if (defined(invoker.iplr_bundle)) { - args += [ "--iplr-bundle=${invoker.iplr_bundle}" ] - } outputs = [ - sl_intermediate, + sl_output, reflection_header_intermediate, reflection_cc_intermediate, ] @@ -392,6 +472,12 @@ template("impellerc") { args += [ "--define=$def" ] } } + + if (single_invocation) { + inputs = invoker.shaders + } else { + sources = invoker.shaders + } } } diff --git a/lib/gpu/BUILD.gn b/lib/gpu/BUILD.gn index b26dc419dc864..abaad37932437 100644 --- a/lib/gpu/BUILD.gn +++ b/lib/gpu/BUILD.gn @@ -61,8 +61,10 @@ source_set("gpu") { ] } deps = [ + "//flutter/assets", "//flutter/impeller", "//flutter/impeller/display_list:skia_conversions", + "//flutter/impeller/shader_bundle:shader_bundle_flatbuffers", "//flutter/lib/ui", "//flutter/third_party/tonic", ] diff --git a/lib/gpu/lib/src/shader_library.dart b/lib/gpu/lib/src/shader_library.dart index 551c20bc589ee..859ba02d38f29 100644 --- a/lib/gpu/lib/src/shader_library.dart +++ b/lib/gpu/lib/src/shader_library.dart @@ -18,6 +18,11 @@ base class ShaderLibrary extends NativeFieldWrapperClass1 { ShaderLibrary._(); + // Hold a Dart-side reference to shaders in the library as they're wrapped for + // the first time. This prevents the wrapper from getting prematurely + // destroyed. + final Map shaders_ = {}; + Shader? operator [](String shaderName) { // This `flutter_gpu` library isn't always registered as part of the builtin // DartClassLibrary, and so we can't instantiate the Dart classes on the @@ -25,7 +30,8 @@ base class ShaderLibrary extends NativeFieldWrapperClass1 { // Providing a new wrapper to [_getShader] for wrapping the native // counterpart (if it hasn't been wrapped already) is a hack to work around // this. - return _getShader(shaderName, Shader._()); + return shaders_.putIfAbsent( + shaderName, () => _getShader(shaderName, Shader._())); } @Native( diff --git a/lib/gpu/shader_library.cc b/lib/gpu/shader_library.cc index c82fbcc5862f3..72a39a138824e 100644 --- a/lib/gpu/shader_library.cc +++ b/lib/gpu/shader_library.cc @@ -6,61 +6,23 @@ #include +#include "flutter/assets/asset_manager.h" +#include "flutter/impeller/runtime_stage/runtime_stage_types_flatbuffers.h" #include "flutter/lib/gpu/fixtures.h" #include "flutter/lib/gpu/shader.h" +#include "flutter/lib/ui/ui_dart_state.h" +#include "flutter/lib/ui/window/platform_configuration.h" #include "fml/mapping.h" #include "fml/memory/ref_ptr.h" +#include "impeller/core/runtime_types.h" +#include "impeller/core/shader_types.h" #include "impeller/renderer/vertex_descriptor.h" #include "impeller/runtime_stage/runtime_stage.h" +#include "impeller/shader_bundle/shader_bundle_flatbuffers.h" namespace flutter { namespace gpu { -// ===[ BEGIN MEMES ]=========================================================== -static fml::RefPtr OpenRuntimeStageAsShader( - std::shared_ptr payload, - const std::shared_ptr& vertex_desc) { - impeller::RuntimeStage stage(std::move(payload)); - return Shader::Make(stage.GetEntrypoint(), - ToShaderStage(stage.GetShaderStage()), - stage.GetCodeMapping(), stage.GetUniforms(), vertex_desc); -} - -static fml::RefPtr InstantiateTestShaderLibrary() { - ShaderLibrary::ShaderMap shaders; - { - auto vertex_desc = std::make_shared(); - vertex_desc->SetStageInputs( - FlutterGPUUnlitVertexShader::kAllShaderStageInputs, - FlutterGPUUnlitVertexShader::kInterleavedBufferLayout); - shaders["UnlitVertex"] = OpenRuntimeStageAsShader( - std::make_shared(kFlutterGPUUnlitVertIPLR, - kFlutterGPUUnlitVertIPLRLength), - vertex_desc); - shaders["UnlitFragment"] = OpenRuntimeStageAsShader( - std::make_shared(kFlutterGPUUnlitFragIPLR, - kFlutterGPUUnlitFragIPLRLength), - nullptr); - } - { - auto vertex_desc = std::make_shared(); - vertex_desc->SetStageInputs( - FlutterGPUTextureVertexShader::kAllShaderStageInputs, - FlutterGPUTextureVertexShader::kInterleavedBufferLayout); - shaders["TextureVertex"] = OpenRuntimeStageAsShader( - std::make_shared( - kFlutterGPUTextureVertIPLR, kFlutterGPUTextureVertIPLRLength), - vertex_desc); - shaders["TextureFragment"] = OpenRuntimeStageAsShader( - std::make_shared( - kFlutterGPUTextureFragIPLR, kFlutterGPUTextureFragIPLRLength), - nullptr); - } - auto library = ShaderLibrary::MakeFromShaders(std::move(shaders)); - return library; -} -// ===[ END MEMES ]============================================================= - IMPLEMENT_WRAPPERTYPEINFO(flutter_gpu, ShaderLibrary); fml::RefPtr ShaderLibrary::override_shader_library_; @@ -68,26 +30,144 @@ fml::RefPtr ShaderLibrary::override_shader_library_; fml::RefPtr ShaderLibrary::MakeFromAsset( const std::string& name, std::string& out_error) { - // =========================================================================== - // This is a temporary hack to get the shader library populated in the - // framework before the shader bundle format is landed! - if (!override_shader_library_) { - return InstantiateTestShaderLibrary(); - } - // =========================================================================== - if (override_shader_library_) { return override_shader_library_; } - // TODO(bdero): Load the ShaderLibrary asset. - out_error = "Shader bundle asset unimplemented"; - return nullptr; + + auto dart_state = UIDartState::Current(); + std::shared_ptr asset_manager = + dart_state->platform_configuration()->client()->GetAssetManager(); + + std::unique_ptr data = asset_manager->GetAsMapping(name); + if (data == nullptr) { + out_error = std::string("Asset '") + name + std::string("' not found."); + return nullptr; + } + + return MakeFromFlatbuffer(std::move(data)); } fml::RefPtr ShaderLibrary::MakeFromShaders(ShaderMap shaders) { - auto res = - fml::MakeRefCounted(std::move(shaders)); - return res; + return fml::MakeRefCounted(nullptr, + std::move(shaders)); +} + +static impeller::ShaderType FromInputType( + impeller::fb::InputDataType input_type) { + switch (input_type) { + case impeller::fb::InputDataType::kBoolean: + return impeller::ShaderType::kBoolean; + case impeller::fb::InputDataType::kSignedByte: + return impeller::ShaderType::kSignedByte; + case impeller::fb::InputDataType::kUnsignedByte: + return impeller::ShaderType::kUnsignedByte; + case impeller::fb::InputDataType::kSignedShort: + return impeller::ShaderType::kSignedShort; + case impeller::fb::InputDataType::kUnsignedShort: + return impeller::ShaderType::kUnsignedShort; + case impeller::fb::InputDataType::kSignedInt: + return impeller::ShaderType::kSignedInt; + case impeller::fb::InputDataType::kUnsignedInt: + return impeller::ShaderType::kUnsignedInt; + case impeller::fb::InputDataType::kSignedInt64: + return impeller::ShaderType::kSignedInt64; + case impeller::fb::InputDataType::kUnsignedInt64: + return impeller::ShaderType::kUnsignedInt64; + case impeller::fb::InputDataType::kFloat: + return impeller::ShaderType::kFloat; + case impeller::fb::InputDataType::kDouble: + return impeller::ShaderType::kDouble; + } +} + +static size_t SizeOfInputType(impeller::fb::InputDataType input_type) { + switch (input_type) { + case impeller::fb::InputDataType::kBoolean: + return 1; + case impeller::fb::InputDataType::kSignedByte: + return 1; + case impeller::fb::InputDataType::kUnsignedByte: + return 1; + case impeller::fb::InputDataType::kSignedShort: + return 2; + case impeller::fb::InputDataType::kUnsignedShort: + return 2; + case impeller::fb::InputDataType::kSignedInt: + return 4; + case impeller::fb::InputDataType::kUnsignedInt: + return 4; + case impeller::fb::InputDataType::kSignedInt64: + return 8; + case impeller::fb::InputDataType::kUnsignedInt64: + return 8; + case impeller::fb::InputDataType::kFloat: + return 4; + case impeller::fb::InputDataType::kDouble: + return 8; + } +} + +fml::RefPtr ShaderLibrary::MakeFromFlatbuffer( + std::shared_ptr payload) { + if (payload == nullptr || !payload->GetMapping()) { + return nullptr; + } + if (!impeller::fb::ShaderBundleBufferHasIdentifier(payload->GetMapping())) { + return nullptr; + } + auto* bundle = impeller::fb::GetShaderBundle(payload->GetMapping()); + if (!bundle) { + return nullptr; + } + + ShaderLibrary::ShaderMap shader_map; + + for (const auto* bundled_shader : *bundle->shaders()) { + const impeller::fb::RuntimeStage* runtime_stage = bundled_shader->shader(); + impeller::RuntimeStage stage(runtime_stage); + + std::shared_ptr vertex_descriptor = nullptr; + if (stage.GetShaderStage() == impeller::RuntimeShaderStage::kVertex) { + vertex_descriptor = std::make_shared(); + auto inputs_fb = runtime_stage->inputs(); + + std::vector inputs; + inputs.reserve(inputs_fb->size()); + size_t default_stride = 0; + for (const auto& input : *inputs_fb) { + impeller::ShaderStageIOSlot slot; + slot.name = input->name()->c_str(); + slot.location = input->location(); + slot.set = input->set(); + slot.binding = input->binding(); + slot.type = FromInputType(input->type()); + slot.bit_width = input->bit_width(); + slot.vec_size = input->vec_size(); + slot.columns = input->columns(); + slot.offset = input->offset(); + inputs.emplace_back(slot); + + default_stride += + SizeOfInputType(input->type()) * slot.vec_size * slot.columns; + } + std::vector layouts = { + impeller::ShaderStageBufferLayout{ + .stride = default_stride, + .binding = 0u, + }}; + + vertex_descriptor->SetStageInputs(inputs, layouts); + } + + auto shader = flutter::gpu::Shader::Make( + stage.GetEntrypoint(), ToShaderStage(stage.GetShaderStage()), + stage.GetCodeMapping(), stage.GetUniforms(), + std::move(vertex_descriptor)); + shader_map[bundled_shader->name()->str()] = std::move(shader); + } + + return fml::MakeRefCounted( + std::move(payload), std::move(shader_map)); } void ShaderLibrary::SetOverride( @@ -109,8 +189,9 @@ fml::RefPtr ShaderLibrary::GetShader(const std::string& shader_name, return shader; } -ShaderLibrary::ShaderLibrary(ShaderMap shaders) - : shaders_(std::move(shaders)) {} +ShaderLibrary::ShaderLibrary(std::shared_ptr payload, + ShaderMap shaders) + : payload_(std::move(payload)), shaders_(std::move(shaders)) {} ShaderLibrary::~ShaderLibrary() = default; diff --git a/lib/gpu/shader_library.h b/lib/gpu/shader_library.h index 35fd737f7a7a1..92d847b559a8a 100644 --- a/lib/gpu/shader_library.h +++ b/lib/gpu/shader_library.h @@ -4,6 +4,7 @@ #pragma once +#include #include #include @@ -11,6 +12,7 @@ #include "flutter/lib/gpu/shader.h" #include "flutter/lib/ui/dart_wrapper.h" #include "fml/memory/ref_ptr.h" +#include "impeller/shader_bundle/shader_bundle_flatbuffers.h" namespace flutter { namespace gpu { @@ -28,6 +30,9 @@ class ShaderLibrary : public RefCountedDartWrappable { static fml::RefPtr MakeFromShaders(ShaderMap shaders); + static fml::RefPtr MakeFromFlatbuffer( + std::shared_ptr payload); + /// Sets a return override for `MakeFromAsset` for testing purposes. static void SetOverride(fml::RefPtr override_shader_library); @@ -42,9 +47,11 @@ class ShaderLibrary : public RefCountedDartWrappable { /// this library. static fml::RefPtr override_shader_library_; + std::shared_ptr payload_; ShaderMap shaders_; - explicit ShaderLibrary(ShaderMap shaders); + explicit ShaderLibrary(std::shared_ptr payload, + ShaderMap shaders); FML_DISALLOW_COPY_AND_ASSIGN(ShaderLibrary); };