diff --git a/.ci.yaml b/.ci.yaml index 9e5b32cf6e8b4..f9a936f0e898a 100644 --- a/.ci.yaml +++ b/.ci.yaml @@ -380,6 +380,10 @@ targets: properties: release_build: "true" config_name: mac_host_engine + dependencies: >- + [ + {"dependency": "goldctl", "version": "git_revision:3a77d0b12c697a840ca0c7705208e8622dc94603"} + ] $flutter/osx_sdk : >- { "sdk_version": "14a5294e" } diff --git a/BUILD.gn b/BUILD.gn index 0b7cca847deb4..d00ac7c9e234e 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -181,9 +181,9 @@ group("unittests") { } if (is_mac) { - public_deps += - [ "//flutter/shell/platform/darwin:flutter_channels_unittests" ] public_deps += [ + "//flutter/impeller/golden_tests:impeller_golden_tests", + "//flutter/shell/platform/darwin:flutter_channels_unittests", "//flutter/third_party/spring_animation:spring_animation_unittests", ] } diff --git a/ci/builders/mac_host_engine.json b/ci/builders/mac_host_engine.json index a5957064929d0..a5e72ade6b8e2 100644 --- a/ci/builders/mac_host_engine.json +++ b/ci/builders/mac_host_engine.json @@ -110,6 +110,9 @@ "os=Mac-12", "cpu=x86" ], + "dependencies": [ + {"dependency": "goldctl", "version": "git_revision:3a77d0b12c697a840ca0c7705208e8622dc94603"} + ], "gclient_custom_vars": { "download_android_deps": false }, @@ -124,13 +127,27 @@ "ninja": { "config": "host_release", "targets": [ - "flutter/shell/platform/darwin/macos:zip_macos_flutter_framework", "flutter/build/archives:archive_gen_snapshot", "flutter/build/archives:artifacts", + "flutter/impeller/golden_tests:impeller_golden_tests", + "flutter/shell/platform/darwin/macos:zip_macos_flutter_framework", "flutter/tools/font-subset" ] }, - "tests": [] + "tests": [ + { + "language": "python3", + "name": "Impeller golden Tests for host_release", + "parameters": [ + "--variant", + "host_release", + "--type", + "impeller-golden" + ], + "script": "flutter/testing/run_tests.py", + "type": "local" + } + ] }, { "archives": [ diff --git a/ci/licenses_golden/excluded_files b/ci/licenses_golden/excluded_files index ac858e2a3fca7..adee33e61a433 100644 --- a/ci/licenses_golden/excluded_files +++ b/ci/licenses_golden/excluded_files @@ -135,6 +135,14 @@ ../../../flutter/impeller/geometry/README.md ../../../flutter/impeller/geometry/geometry_unittests.cc ../../../flutter/impeller/geometry/geometry_unittests.h +../../../flutter/impeller/golden_tests/README.md +../../../flutter/impeller/golden_tests_harvester/.dart_tool +../../../flutter/impeller/golden_tests_harvester/.gitignore +../../../flutter/impeller/golden_tests_harvester/README.md +../../../flutter/impeller/golden_tests_harvester/analysis_options.yaml +../../../flutter/impeller/golden_tests_harvester/pubspec.lock +../../../flutter/impeller/golden_tests_harvester/pubspec.yaml +../../../flutter/impeller/golden_tests_harvester/test ../../../flutter/impeller/image/README.md ../../../flutter/impeller/playground ../../../flutter/impeller/renderer/compute_subgroup_unittests.cc diff --git a/ci/licenses_golden/licenses_flutter b/ci/licenses_golden/licenses_flutter index bec4b4a6e7e75..eab793f77ceb4 100644 --- a/ci/licenses_golden/licenses_flutter +++ b/ci/licenses_golden/licenses_flutter @@ -1315,6 +1315,19 @@ ORIGIN: ../../../flutter/impeller/geometry/type_traits.cc + ../../../flutter/LIC ORIGIN: ../../../flutter/impeller/geometry/type_traits.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/geometry/vector.cc + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/geometry/vector.h + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/impeller/golden_tests/golden_digest.cc + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/impeller/golden_tests/golden_digest.h + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/impeller/golden_tests/golden_tests.cc + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/impeller/golden_tests/main.cc + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/impeller/golden_tests/metal_screenshot.h + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/impeller/golden_tests/metal_screenshot.mm + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/impeller/golden_tests/metal_screenshoter.h + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/impeller/golden_tests/metal_screenshoter.mm + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/impeller/golden_tests/working_directory.cc + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/impeller/golden_tests/working_directory.h + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/impeller/golden_tests_harvester/bin/golden_tests_harvester.dart + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/impeller/golden_tests_harvester/lib/golden_tests_harvester.dart + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/impeller/golden_tests_harvester/lib/logger.dart + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/image/backends/skia/compressed_image_skia.cc + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/image/backends/skia/compressed_image_skia.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/image/compressed_image.cc + ../../../flutter/LICENSE @@ -3853,6 +3866,19 @@ FILE: ../../../flutter/impeller/geometry/type_traits.cc FILE: ../../../flutter/impeller/geometry/type_traits.h FILE: ../../../flutter/impeller/geometry/vector.cc FILE: ../../../flutter/impeller/geometry/vector.h +FILE: ../../../flutter/impeller/golden_tests/golden_digest.cc +FILE: ../../../flutter/impeller/golden_tests/golden_digest.h +FILE: ../../../flutter/impeller/golden_tests/golden_tests.cc +FILE: ../../../flutter/impeller/golden_tests/main.cc +FILE: ../../../flutter/impeller/golden_tests/metal_screenshot.h +FILE: ../../../flutter/impeller/golden_tests/metal_screenshot.mm +FILE: ../../../flutter/impeller/golden_tests/metal_screenshoter.h +FILE: ../../../flutter/impeller/golden_tests/metal_screenshoter.mm +FILE: ../../../flutter/impeller/golden_tests/working_directory.cc +FILE: ../../../flutter/impeller/golden_tests/working_directory.h +FILE: ../../../flutter/impeller/golden_tests_harvester/bin/golden_tests_harvester.dart +FILE: ../../../flutter/impeller/golden_tests_harvester/lib/golden_tests_harvester.dart +FILE: ../../../flutter/impeller/golden_tests_harvester/lib/logger.dart FILE: ../../../flutter/impeller/image/backends/skia/compressed_image_skia.cc FILE: ../../../flutter/impeller/image/backends/skia/compressed_image_skia.h FILE: ../../../flutter/impeller/image/compressed_image.cc diff --git a/impeller/golden_tests/BUILD.gn b/impeller/golden_tests/BUILD.gn new file mode 100644 index 0000000000000..c723725729dbf --- /dev/null +++ b/impeller/golden_tests/BUILD.gn @@ -0,0 +1,39 @@ +# Copyright 2013 The Flutter Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import("//flutter/common/config.gni") +import("//flutter/impeller/tools/impeller.gni") + +if (is_mac) { + test_fixtures("impeller_golden_tests_fixtures") { + fixtures = [] + } + + impeller_component("impeller_golden_tests") { + target_type = "executable" + + testonly = true + + sources = [ + "golden_digest.cc", + "golden_digest.h", + "golden_tests.cc", + "main.cc", + "metal_screenshot.h", + "metal_screenshot.mm", + "metal_screenshoter.h", + "metal_screenshoter.mm", + "working_directory.cc", + "working_directory.h", + ] + + deps = [ + ":impeller_golden_tests_fixtures", + "//flutter/impeller/aiks", + "//flutter/impeller/playground", + "//flutter/impeller/renderer/backend/metal:metal", + "//third_party/googletest:gtest", + ] + } +} diff --git a/impeller/golden_tests/README.md b/impeller/golden_tests/README.md new file mode 100644 index 0000000000000..857349f44ed0d --- /dev/null +++ b/impeller/golden_tests/README.md @@ -0,0 +1,20 @@ +# Impeller Golden Tests + +This is the executable that will generate the golden image results that can then +be sent to Skia Gold vial the +[golden_tests_harvester]("../golden_tests_harvester"). + +Running these tests should happen from +[//flutter/testing/run_tests.py](../../testing/run_tests.py). That will do all +the steps to generate the golden images and transmit them to Skia Gold. If you +run the tests locally it will not actually upload anything. That only happens if +the script is executed from LUCI. + +Example invocation: + +```sh +./run_tests.py --variant="host_debug_unopt_arm64" --type="impeller-golden" +``` + +Currently these tests are only supported on macOS and only test the Metal +backend to Impeller. diff --git a/impeller/golden_tests/golden_digest.cc b/impeller/golden_tests/golden_digest.cc new file mode 100644 index 0000000000000..3de06521b9dbb --- /dev/null +++ b/impeller/golden_tests/golden_digest.cc @@ -0,0 +1,58 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "impeller/golden_tests/golden_digest.h" + +#include + +namespace impeller { +namespace testing { + +GoldenDigest* GoldenDigest::instance_ = nullptr; + +GoldenDigest* GoldenDigest::Instance() { + if (!instance_) { + instance_ = new GoldenDigest(); + } + return instance_; +} + +GoldenDigest::GoldenDigest() {} + +void GoldenDigest::AddImage(const std::string& test_name, + const std::string& filename, + int32_t width, + int32_t height) { + entries_.push_back({test_name, filename, width, height}); +} + +bool GoldenDigest::Write(WorkingDirectory* working_directory) { + std::ofstream fout; + fout.open(working_directory->GetFilenamePath("digest.json")); + if (!fout.good()) { + return false; + } + + fout << "[" << std::endl; + bool is_first = true; + for (const auto& entry : entries_) { + if (!is_first) { + fout << "," << std::endl; + is_first = false; + } + fout << " { " + << "\"testName\" : \"" << entry.test_name << "\", " + << "\"filename\" : \"" << entry.filename << "\", " + << "\"width\" : " << entry.width << ", " + << "\"height\" : " << entry.height << " " + << "}"; + } + fout << std::endl << "]" << std::endl; + + fout.close(); + return true; +} + +} // namespace testing +} // namespace impeller diff --git a/impeller/golden_tests/golden_digest.h b/impeller/golden_tests/golden_digest.h new file mode 100644 index 0000000000000..011607ee599ae --- /dev/null +++ b/impeller/golden_tests/golden_digest.h @@ -0,0 +1,45 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#pragma once + +#include +#include + +#include "flutter/fml/macros.h" +#include "flutter/impeller/golden_tests/working_directory.h" + +namespace impeller { +namespace testing { + +/// Manages a global variable for tracking instances of golden images. +class GoldenDigest { + public: + static GoldenDigest* Instance(); + + void AddImage(const std::string& test_name, + const std::string& filename, + int32_t width, + int32_t height); + + /// Writes a "digest.json" file to `working_directory`. + /// + /// Returns `true` on success. + bool Write(WorkingDirectory* working_directory); + + private: + FML_DISALLOW_COPY_AND_ASSIGN(GoldenDigest); + GoldenDigest(); + struct Entry { + std::string test_name; + std::string filename; + int32_t width; + int32_t height; + }; + + static GoldenDigest* instance_; + std::vector entries_; +}; +} // namespace testing +} // namespace impeller diff --git a/impeller/golden_tests/golden_tests.cc b/impeller/golden_tests/golden_tests.cc new file mode 100644 index 0000000000000..3bcb76e279720 --- /dev/null +++ b/impeller/golden_tests/golden_tests.cc @@ -0,0 +1,79 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "gtest/gtest.h" + +#include + +#include "impeller/aiks/canvas.h" +#include "impeller/entity/contents/conical_gradient_contents.h" +#include "impeller/geometry/path_builder.h" +#include "impeller/golden_tests/golden_digest.h" +#include "impeller/golden_tests/metal_screenshot.h" +#include "impeller/golden_tests/metal_screenshoter.h" +#include "impeller/golden_tests/working_directory.h" + +namespace impeller { +namespace testing { + +namespace { +std::string GetTestName() { + std::string suite_name = + ::testing::UnitTest::GetInstance()->current_test_suite()->name(); + std::string test_name = + ::testing::UnitTest::GetInstance()->current_test_info()->name(); + std::stringstream ss; + ss << "impeller_" << suite_name << "_" << test_name; + return ss.str(); +} + +std::string GetGoldenFilename() { + return GetTestName() + ".png"; +} + +bool SaveScreenshot(std::unique_ptr screenshot) { + if (!screenshot || !screenshot->GetBytes()) { + return false; + } + std::string test_name = GetTestName(); + std::string filename = GetGoldenFilename(); + GoldenDigest::Instance()->AddImage( + test_name, filename, screenshot->GetWidth(), screenshot->GetHeight()); + return screenshot->WriteToPNG( + WorkingDirectory::Instance()->GetFilenamePath(filename)); +} +} // namespace + +class GoldenTests : public ::testing::Test { + public: + GoldenTests() : screenshoter_(new MetalScreenshoter()) {} + + MetalScreenshoter& Screenshoter() { return *screenshoter_; } + + private: + std::unique_ptr screenshoter_; +}; + +TEST_F(GoldenTests, ConicalGradient) { + Canvas canvas; + Paint paint; + paint.color_source_type = Paint::ColorSourceType::kConicalGradient; + paint.color_source = []() { + auto result = std::make_shared(); + result->SetCenterAndRadius(Point(125, 125), 125); + result->SetColors({Color(1.0, 0.0, 0.0, 1.0), Color(0.0, 0.0, 1.0, 1.0)}); + result->SetStops({0, 1}); + result->SetFocus(Point(180, 180), 0); + result->SetTileMode(Entity::TileMode::kClamp); + return result; + }; + paint.stroke_width = 0.0; + paint.style = Paint::Style::kFill; + canvas.DrawRect(Rect(10, 10, 250, 250), paint); + Picture picture = canvas.EndRecordingAsPicture(); + auto screenshot = Screenshoter().MakeScreenshot(std::move(picture)); + ASSERT_TRUE(SaveScreenshot(std::move(screenshot))); +} +} // namespace testing +} // namespace impeller diff --git a/impeller/golden_tests/main.cc b/impeller/golden_tests/main.cc new file mode 100644 index 0000000000000..e1ed2a8c4eccc --- /dev/null +++ b/impeller/golden_tests/main.cc @@ -0,0 +1,61 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include + +#include "flutter/fml/backtrace.h" +#include "flutter/fml/build_config.h" +#include "flutter/fml/command_line.h" +#include "flutter/fml/logging.h" +#include "flutter/impeller/golden_tests/golden_digest.h" +#include "flutter/impeller/golden_tests/working_directory.h" +#include "gtest/gtest.h" + +namespace { +void print_usage() { + std::cout << "usage: impeller_golden_tests --working_dir=" + << std::endl + << std::endl; + std::cout << "flags:" << std::endl; + std::cout << " working_dir: Where the golden images will be generated and " + "uploaded to Skia Gold from." + << std::endl; +} +} // namespace + +int main(int argc, char** argv) { + fml::InstallCrashHandler(); + testing::InitGoogleTest(&argc, argv); + fml::CommandLine cmd = fml::CommandLineFromPlatformOrArgcArgv(argc, argv); + + std::optional working_dir; + for (const auto& option : cmd.options()) { + if (option.name == "working_dir") { + wordexp_t wordexp_result; + int code = wordexp(option.value.c_str(), &wordexp_result, 0); + FML_CHECK(code == 0); + FML_CHECK(wordexp_result.we_wordc != 0); + working_dir = wordexp_result.we_wordv[0]; + wordfree(&wordexp_result); + } + } + if (!working_dir) { + std::cout << "required argument \"working_dir\" is missing." << std::endl + << std::endl; + print_usage(); + return 1; + } + + impeller::testing::WorkingDirectory::Instance()->SetPath(working_dir.value()); + std::cout << "working directory: " + << impeller::testing::WorkingDirectory::Instance()->GetPath() + << std::endl; + + int return_code = RUN_ALL_TESTS(); + if (0 == return_code) { + impeller::testing::GoldenDigest::Instance()->Write( + impeller::testing::WorkingDirectory::Instance()); + } + return return_code; +} diff --git a/impeller/golden_tests/metal_screenshot.h b/impeller/golden_tests/metal_screenshot.h new file mode 100644 index 0000000000000..0d2df7fc44309 --- /dev/null +++ b/impeller/golden_tests/metal_screenshot.h @@ -0,0 +1,39 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#pragma once + +#include +#include +#include + +#include "flutter/fml/macros.h" + +namespace impeller { +namespace testing { + +class MetalScreenshoter; + +/// A screenshot that was produced from `MetalScreenshoter`. +class MetalScreenshot { + public: + ~MetalScreenshot(); + + const UInt8* GetBytes() const; + + size_t GetHeight() const; + + size_t GetWidth() const; + + bool WriteToPNG(const std::string& path) const; + + private: + friend class MetalScreenshoter; + MetalScreenshot(CGImageRef cgImage); + FML_DISALLOW_COPY_AND_ASSIGN(MetalScreenshot); + CGImageRef cgImage_; + CFDataRef pixel_data_; +}; +} // namespace testing +} // namespace impeller diff --git a/impeller/golden_tests/metal_screenshot.mm b/impeller/golden_tests/metal_screenshot.mm new file mode 100644 index 0000000000000..db274a1de81e2 --- /dev/null +++ b/impeller/golden_tests/metal_screenshot.mm @@ -0,0 +1,52 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "impeller/golden_tests/metal_screenshot.h" + +namespace impeller { +namespace testing { + +MetalScreenshot::MetalScreenshot(CGImageRef cgImage) : cgImage_(cgImage) { + CGDataProviderRef data_provider = CGImageGetDataProvider(cgImage); + pixel_data_ = CGDataProviderCopyData(data_provider); +} + +MetalScreenshot::~MetalScreenshot() { + CFRelease(pixel_data_); + CGImageRelease(cgImage_); +} + +const UInt8* MetalScreenshot::GetBytes() const { + return CFDataGetBytePtr(pixel_data_); +} + +size_t MetalScreenshot::GetHeight() const { + return CGImageGetHeight(cgImage_); +} + +size_t MetalScreenshot::GetWidth() const { + return CGImageGetWidth(cgImage_); +} + +bool MetalScreenshot::WriteToPNG(const std::string& path) const { + bool result = false; + NSURL* output_url = + [NSURL fileURLWithPath:[NSString stringWithUTF8String:path.c_str()]]; + CGImageDestinationRef destination = CGImageDestinationCreateWithURL( + (__bridge CFURLRef)output_url, kUTTypePNG, 1, nullptr); + if (destination != nullptr) { + CGImageDestinationAddImage(destination, cgImage_, + (__bridge CFDictionaryRef) @{}); + + if (CGImageDestinationFinalize(destination)) { + result = true; + } + + CFRelease(destination); + } + return result; +} + +} // namespace testing +} // namespace impeller diff --git a/impeller/golden_tests/metal_screenshoter.h b/impeller/golden_tests/metal_screenshoter.h new file mode 100644 index 0000000000000..673f7f7091f0f --- /dev/null +++ b/impeller/golden_tests/metal_screenshoter.h @@ -0,0 +1,30 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#pragma once + +#include "flutter/fml/macros.h" +#include "flutter/impeller/aiks/picture.h" +#include "flutter/impeller/golden_tests/metal_screenshot.h" +#include "flutter/impeller/playground/playground_impl.h" + +namespace impeller { +namespace testing { + +/// Converts `Picture`'s to `MetalScreenshot`'s with the playground backend. +class MetalScreenshoter { + public: + MetalScreenshoter(); + + std::unique_ptr MakeScreenshot(Picture&& picture, + const ISize& size = {300, + 300}); + + private: + std::unique_ptr playground_; + std::unique_ptr aiks_context_; +}; + +} // namespace testing +} // namespace impeller diff --git a/impeller/golden_tests/metal_screenshoter.mm b/impeller/golden_tests/metal_screenshoter.mm new file mode 100644 index 0000000000000..cd1a8bbd536ea --- /dev/null +++ b/impeller/golden_tests/metal_screenshoter.mm @@ -0,0 +1,55 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "flutter/impeller/golden_tests/metal_screenshoter.h" + +#include +#include "impeller/renderer/backend/metal/context_mtl.h" +#include "impeller/renderer/backend/metal/texture_mtl.h" +#define GLFW_INCLUDE_NONE +#include "third_party/glfw/include/GLFW/glfw3.h" + +namespace impeller { +namespace testing { + +MetalScreenshoter::MetalScreenshoter() { + FML_CHECK(::glfwInit() == GLFW_TRUE); + playground_ = PlaygroundImpl::Create(PlaygroundBackend::kMetal); + aiks_context_.reset(new AiksContext(playground_->GetContext())); +} + +std::unique_ptr MetalScreenshoter::MakeScreenshot( + Picture&& picture, + const ISize& size) { + std::shared_ptr image = picture.ToImage(*aiks_context_, size); + std::shared_ptr texture = image->GetTexture(); + id metal_texture = + std::static_pointer_cast(texture)->GetMTLTexture(); + + if (metal_texture.pixelFormat != MTLPixelFormatBGRA8Unorm) { + return {}; + } + + CIImage* ciImage = [[CIImage alloc] initWithMTLTexture:metal_texture + options:@{}]; + FML_CHECK(ciImage); + + std::shared_ptr context = playground_->GetContext(); + std::shared_ptr context_mtl = + std::static_pointer_cast(context); + CIContext* cicontext = + [CIContext contextWithMTLDevice:context_mtl->GetMTLDevice()]; + FML_CHECK(context); + + CIImage* flipped = [ciImage + imageByApplyingOrientation:kCGImagePropertyOrientationDownMirrored]; + + CGImageRef cgImage = [cicontext createCGImage:flipped + fromRect:[ciImage extent]]; + + return std::unique_ptr(new MetalScreenshot(cgImage)); +} + +} // namespace testing +} // namespace impeller diff --git a/impeller/golden_tests/working_directory.cc b/impeller/golden_tests/working_directory.cc new file mode 100644 index 0000000000000..ac9e608cf98ad --- /dev/null +++ b/impeller/golden_tests/working_directory.cc @@ -0,0 +1,35 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "impeller/golden_tests/working_directory.h" + +#include "flutter/fml/paths.h" + +namespace impeller { +namespace testing { + +WorkingDirectory* WorkingDirectory::instance_ = nullptr; + +WorkingDirectory::WorkingDirectory() {} + +WorkingDirectory* WorkingDirectory::Instance() { + if (!instance_) { + instance_ = new WorkingDirectory(); + } + return instance_; +} + +std::string WorkingDirectory::GetFilenamePath( + const std::string& filename) const { + return fml::paths::JoinPaths({path_, filename}); +} + +void WorkingDirectory::SetPath(const std::string& path) { + FML_CHECK(did_set_ == false); + path_ = path; + did_set_ = true; +} + +} // namespace testing +} // namespace impeller diff --git a/impeller/golden_tests/working_directory.h b/impeller/golden_tests/working_directory.h new file mode 100644 index 0000000000000..b3a3d74317fa6 --- /dev/null +++ b/impeller/golden_tests/working_directory.h @@ -0,0 +1,35 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#pragma once + +#include + +#include "flutter/fml/macros.h" + +namespace impeller { +namespace testing { + +/// Keeps track of the global variable for the specified working +/// directory. +class WorkingDirectory { + public: + static WorkingDirectory* Instance(); + + std::string GetFilenamePath(const std::string& filename) const; + + void SetPath(const std::string& path); + + const std::string& GetPath() const { return path_; } + + private: + FML_DISALLOW_COPY_AND_ASSIGN(WorkingDirectory); + WorkingDirectory(); + static WorkingDirectory* instance_; + std::string path_; + bool did_set_ = false; +}; + +} // namespace testing +} // namespace impeller diff --git a/impeller/golden_tests_harvester/.gitignore b/impeller/golden_tests_harvester/.gitignore new file mode 100644 index 0000000000000..3a85790408401 --- /dev/null +++ b/impeller/golden_tests_harvester/.gitignore @@ -0,0 +1,3 @@ +# https://dart.dev/guides/libraries/private-files +# Created by `dart pub` +.dart_tool/ diff --git a/impeller/golden_tests_harvester/README.md b/impeller/golden_tests_harvester/README.md new file mode 100644 index 0000000000000..9920093f3ecb7 --- /dev/null +++ b/impeller/golden_tests_harvester/README.md @@ -0,0 +1,14 @@ +# Golden Tests Harvester + +Reaps the output of impeller's golden image tests and sends it to Skia gold. + +## Usage + +```sh +cd $SRC +./out/host_debug_unopt_arm64/impeller_golden_tests --working_dir=~/Desktop/temp +cd flutter/impeller/golden_tests_harvester +dart run ./bin/golden_tests_harvester.dart ~/Desktop/temp +``` + +See also [golden_tests](../golden_tests/). diff --git a/impeller/golden_tests_harvester/analysis_options.yaml b/impeller/golden_tests_harvester/analysis_options.yaml new file mode 100644 index 0000000000000..f04c6cf0f30d4 --- /dev/null +++ b/impeller/golden_tests_harvester/analysis_options.yaml @@ -0,0 +1 @@ +include: ../../analysis_options.yaml diff --git a/impeller/golden_tests_harvester/bin/golden_tests_harvester.dart b/impeller/golden_tests_harvester/bin/golden_tests_harvester.dart new file mode 100644 index 0000000000000..7aa1f84d128be --- /dev/null +++ b/impeller/golden_tests_harvester/bin/golden_tests_harvester.dart @@ -0,0 +1,88 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:io'; + +import 'package:golden_tests_harvester/golden_tests_harvester.dart'; +import 'package:process/src/interface/process_manager.dart'; +import 'package:skia_gold_client/skia_gold_client.dart'; + +const String _kLuciEnvName = 'LUCI_CONTEXT'; + +bool get isLuciEnv => Platform.environment.containsKey(_kLuciEnvName); + +/// Fake SkiaGoldClient that is used if the harvester is run outside of Luci. +class FakeSkiaGoldClient implements SkiaGoldClient { + FakeSkiaGoldClient(this._workingDirectory); + + final Directory _workingDirectory; + + @override + Future addImg(String testName, File goldenFile, + {double differentPixelsRate = 0.01, + int pixelColorDelta = 0, + required int screenshotSize}) async { + Logger.instance.log('addImg $testName ${goldenFile.path} $screenshotSize'); + } + + @override + Future auth() async { + Logger.instance.log('auth'); + } + + @override + String cleanTestName(String fileName) { + throw UnimplementedError(); + } + + @override + Map? get dimensions => throw UnimplementedError(); + + @override + List getCIArguments() { + throw UnimplementedError(); + } + + @override + Future getExpectationForTest(String testName) { + throw UnimplementedError(); + } + + @override + Future> getImageBytes(String imageHash) { + throw UnimplementedError(); + } + + @override + String getTraceID(String testName) { + throw UnimplementedError(); + } + + @override + HttpClient get httpClient => throw UnimplementedError(); + + @override + ProcessManager get process => throw UnimplementedError(); + + @override + Directory get workDirectory => _workingDirectory; +} + +void _printUsage() { + Logger.instance + .log('dart run ./bin/golden_tests_harvester.dart '); +} + +Future main(List arguments) async { + if (arguments.length != 1) { + return _printUsage(); + } + + final Directory workDirectory = Directory(arguments[0]); + final SkiaGoldClient skiaGoldClient = isLuciEnv + ? SkiaGoldClient(workDirectory) + : FakeSkiaGoldClient(workDirectory); + + await harvest(skiaGoldClient, workDirectory); +} diff --git a/impeller/golden_tests_harvester/lib/golden_tests_harvester.dart b/impeller/golden_tests_harvester/lib/golden_tests_harvester.dart new file mode 100644 index 0000000000000..85fb43b60cafe --- /dev/null +++ b/impeller/golden_tests_harvester/lib/golden_tests_harvester.dart @@ -0,0 +1,46 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:convert'; +import 'dart:io'; + +import 'package:path/path.dart' as p; +import 'package:skia_gold_client/skia_gold_client.dart'; + +import 'logger.dart'; + +export 'logger.dart'; + +/// Reads the digest inside of [workDirectory], sending tests to +/// [skiaGoldClient]. +Future harvest( + SkiaGoldClient skiaGoldClient, Directory workDirectory) async { + await skiaGoldClient.auth(); + + final File digest = File(p.join(workDirectory.path, 'digest.json')); + if (!digest.existsSync()) { + Logger.instance + .log('Error: digest.json does not exist in ${workDirectory.path}.'); + return; + } + final Object? decoded = jsonDecode(digest.readAsStringSync()); + final List entries = (decoded as List?)!; + final List> pendingComparisons = >[]; + for (final Object? entry in entries) { + final Map map = (entry as Map?)!; + final String filename = (map['filename'] as String?)!; + final int width = (map['width'] as int?)!; + final int height = (map['height'] as int?)!; + final File goldenImage = File(p.join(workDirectory.path, filename)); + final Future future = skiaGoldClient + .addImg(filename, goldenImage, screenshotSize: width * height) + .catchError((dynamic err) { + Logger.instance.log('skia gold comparison failed: $err'); + throw Exception('Failed comparison: $filename'); + }); + pendingComparisons.add(future); + } + + await Future.wait(pendingComparisons); +} diff --git a/impeller/golden_tests_harvester/lib/logger.dart b/impeller/golden_tests_harvester/lib/logger.dart new file mode 100644 index 0000000000000..b9f5818b37692 --- /dev/null +++ b/impeller/golden_tests_harvester/lib/logger.dart @@ -0,0 +1,20 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +/// Logs messages to the appropriate console. +class Logger { + Logger._(); + + /// Singleton accessor for the [Logger]. + static Logger get instance { + _instance ??= Logger._(); + return _instance!; + } + + static Logger? _instance; + + /// Log [message] to the console. + // ignore: avoid_print + void log(String message) => print(message); +} diff --git a/impeller/golden_tests_harvester/pubspec.yaml b/impeller/golden_tests_harvester/pubspec.yaml new file mode 100644 index 0000000000000..04e93ab85e450 --- /dev/null +++ b/impeller/golden_tests_harvester/pubspec.yaml @@ -0,0 +1,40 @@ +# Copyright 2013 The Flutter Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +name: golden_tests_harvester +publish_to: none +environment: + sdk: '>=3.0.0-0 <4.0.0' + +# Do not add any dependencies that require more than what is provided in +# //third_party/dart/pkg, //third_party/dart/third_party/pkg, or +# //third_party/pkg. In particular, package:test is not usable here. + +# If you do add packages here, make sure you can run `pub get --offline`, and +# check the .packages and .package_config to make sure all the paths are +# relative to this directory into //third_party/dart, or //third_party/pkg +dependencies: + crypto: any + path: any + process: any + skia_gold_client: + path: ../../testing/skia_gold_client + +dependency_overrides: + collection: + path: ../../../third_party/dart/third_party/pkg/collection + crypto: + path: ../../../third_party/dart/third_party/pkg/crypto + file: + path: ../../../third_party/pkg/file/packages/file + meta: + path: ../../../third_party/dart/pkg/meta + path: + path: ../../../third_party/dart/third_party/pkg/path + platform: + path: ../../../third_party/pkg/platform + process: + path: ../../../third_party/pkg/process + typed_data: + path: ../../../third_party/dart/third_party/pkg/typed_data diff --git a/impeller/golden_tests_harvester/test/golden_tests_harvester_test.dart b/impeller/golden_tests_harvester/test/golden_tests_harvester_test.dart new file mode 100644 index 0000000000000..f9b0dd79fed95 --- /dev/null +++ b/impeller/golden_tests_harvester/test/golden_tests_harvester_test.dart @@ -0,0 +1,5 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +void main() {} diff --git a/testing/run_tests.py b/testing/run_tests.py index 23be9205b1d45..98509be2b153a 100755 --- a/testing/run_tests.py +++ b/testing/run_tests.py @@ -8,16 +8,19 @@ A top level harness to run all unit-tests in a specific engine build. """ +from pathlib import Path + import argparse -import glob +import csv import errno +import glob import multiprocessing import os import re import subprocess import sys +import tempfile import time -import csv import xvfb SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) @@ -972,6 +975,46 @@ def run_engine_tasks_in_parallel(tasks): raise Exception() +class DirectoryChange(): + """ + A scoped change in the CWD. + """ + old_cwd: str = '' + new_cwd: str = '' + + def __init__(self, new_cwd: str): + self.new_cwd = new_cwd + + def __enter__(self): + self.old_cwd = os.getcwd() + os.chdir(self.new_cwd) + + def __exit__(self, exception_type, exception_value, exception_traceback): + os.chdir(self.old_cwd) + + +def run_impeller_golden_tests(build_dir: str): + """ + Executes the impeller golden image tests from in the `variant` build. + """ + tests_path: str = os.path.join(build_dir, 'impeller_golden_tests') + if not os.path.exists(tests_path): + raise Exception( + 'Cannot find the "impeller_golden_tests" executable in "%s". You may need to build it.' + % (build_dir) + ) + harvester_path: Path = Path(SCRIPT_DIR).parent.joinpath('impeller').joinpath( + 'golden_tests_harvester' + ) + with tempfile.TemporaryDirectory(prefix='impeller_golden') as temp_dir: + run_cmd([tests_path, '--working_dir=%s' % temp_dir]) + with DirectoryChange(harvester_path): + run_cmd(['dart', 'pub', 'get']) + bin_path = Path('.').joinpath('bin' + ).joinpath('golden_tests_harvester.dart') + run_cmd(['dart', 'run', str(bin_path), temp_dir]) + + def main(): parser = argparse.ArgumentParser( description=""" @@ -980,7 +1023,14 @@ def main(): """ ) all_types = [ - 'engine', 'dart', 'benchmarks', 'java', 'android', 'objc', 'font-subset' + 'engine', + 'dart', + 'benchmarks', + 'java', + 'android', + 'objc', + 'font-subset', + 'impeller-golden', ] parser.add_argument( @@ -1171,6 +1221,9 @@ def main(): 'font-subset' in types) and args.variant not in variants_to_skip: run_cmd(['python3', 'test.py'], cwd=FONT_SUBSET_DIR) + if 'impeller-golden' in types: + run_impeller_golden_tests(build_dir) + if __name__ == '__main__': sys.exit(main()) diff --git a/tools/pub_get_offline.py b/tools/pub_get_offline.py index e1807ebb0cc1a..36790b281f57f 100644 --- a/tools/pub_get_offline.py +++ b/tools/pub_get_offline.py @@ -21,6 +21,7 @@ ALL_PACKAGES = [ os.path.join(ENGINE_DIR, "ci"), + os.path.join(ENGINE_DIR, "impeller", "golden_tests_harvester"), os.path.join(ENGINE_DIR, "flutter_frontend_server"), os.path.join(ENGINE_DIR, "shell", "vmservice"), os.path.join(ENGINE_DIR, "testing", "benchmark"),