diff --git a/.circleci/config.yml b/.circleci/config.yml index 2131193f88..226c4346f2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -104,28 +104,19 @@ commands: # It is OK to generically restore boost, even if it is not used. - restore_cache: keys: - - boost-1-71-0-v5 - - restore_cache: - keys: - - protobuf3-v3 + - boost-1-71-0-v1 - run: name: Setup - environment: - TOOLCHAIN_TMP: /tmp/toolchain command: | - sudo -E ./setup_oss_toolchain.sh << parameters.setup_toolchain_extra >> + sudo ./setup_oss_toolchain.sh << parameters.setup_toolchain_extra >> # Only save the cache when asked, e.g., hopefully when it was populated. - when: condition: << parameters.save_boost_cache >> steps: - save_cache: paths: - - /tmp/toolchain/dl_cache/boost_cache - key: boost-1-71-0-v5 - - save_cache: - paths: - - /tmp/toolchain/dl_cache/protobuf - key: protobuf3-v3 + - boost_cache + key: boost-1-71-0-v1 # For testing, need additional dependencies not provided by the image. @@ -387,7 +378,6 @@ jobs: steps: - build_debian: use_sdk: false - save_boost_cache: true build-20_04: docker: @@ -395,7 +385,8 @@ jobs: # GCC is hungry. resource_class: large steps: - - build_debian + - build_debian: + save_boost_cache: true build-22_04: docker: diff --git a/Makefile.am b/Makefile.am index c72f2d348e..ff926cc301 100644 --- a/Makefile.am +++ b/Makefile.am @@ -22,7 +22,6 @@ DISABLED_WARNINGS = \ $(DISABLED_WARNINGS_HARD) \ -Wno-array-bounds \ -Wno-class-memaccess \ - -Wno-deprecated-copy \ -Wno-deprecated-declarations \ -Wno-enum-constexpr-conversion \ -Wno-format-truncation \ @@ -75,16 +74,6 @@ proto_res_header_out = $(addprefix protores/, $(notdir ${proto_res_in:.proto=.pb proto_res_body_out = $(addprefix protores/, $(notdir ${proto_res_in:.proto=.pb.cc})) BUILT_SOURCES = $(proto_res_header_out) -proto_cfg_in = \ - proto/config.proto - -protocfg/%.pb.cc protocfg/%.pb.h: proto/config.proto | mdir ; $(PROTOC) --proto_path=$(dir $^) --cpp_out=$(top_srcdir)/protocfg $^ -mdir: ; mkdir -p $(top_srcdir)/protocfg - -proto_cfg_header_out = $(addprefix protocfg/, $(notdir ${proto_cfg_in:.proto=.pb.h})) -proto_cfg_body_out = $(addprefix protocfg/, $(notdir ${proto_cfg_in:.proto=.pb.cc})) -BUILT_SOURCES += $(proto_cfg_header_out) - endif # @@ -151,6 +140,7 @@ libredex_la_SOURCES = \ libredex/GlobalConfig.cpp \ libredex/GraphVisualizer.cpp \ libredex/HierarchyUtil.cpp \ + libredex/InitCollisionFinder.cpp \ libredex/InitClassesWithSideEffects.cpp \ libredex/InlinerConfig.cpp \ libredex/InstructionLowering.cpp \ @@ -208,7 +198,6 @@ libredex_la_SOURCES = \ libredex/RedexMappedFile.cpp \ libredex/ReadMaybeMapped.cpp \ libredex/RedexContext.cpp \ - libredex/RedexProperties.cpp \ libredex/RedexPropertiesManager.cpp \ libredex/RedexPropertyChecker.cpp \ libredex/RedexPropertyCheckerRegistry.cpp \ @@ -225,7 +214,6 @@ libredex_la_SOURCES = \ libredex/SourceBlocks.cpp \ libredex/StringTreeSet.cpp \ libredex/Timer.cpp \ - libredex/ThreadPool.cpp \ libredex/ThrowPropagationImpl.cpp \ libredex/Trace.cpp \ libredex/Transform.cpp \ @@ -321,6 +309,7 @@ libredex_la_SOURCES = \ service/shrinker/Shrinker.cpp \ service/switch-dispatch/SwitchDispatch.cpp \ service/switch-partitioning/SwitchEquivFinder.cpp \ + service/switch-partitioning/SwitchMethodPartitioning.cpp \ service/type-analysis/GlobalTypeAnalyzer.cpp \ service/type-analysis/LocalTypeAnalyzer.cpp \ service/type-analysis/WholeProgramState.cpp \ @@ -346,8 +335,7 @@ libredex_la_LIBADD = \ if SET_PROTOBUF libredex_la_SOURCES += \ protores/Resources.pb.cc \ - protores/Configuration.pb.cc \ - protocfg/config.pb.cc + protores/Configuration.pb.cc libredex_la_LIBADD += \ $(LIBPROTOBUF_LIBS) @@ -363,7 +351,6 @@ endif libopt_la_SOURCES = \ checkers/HasSourceBlocksChecker.cpp \ - checkers/InitialRenameClassCheker.cpp \ checkers/InjectionIdInstructionsChecker.cpp \ checkers/NoInitClassInstructionsChecker.cpp \ checkers/NoUnreachableInstructionsChecker.cpp \ @@ -411,7 +398,6 @@ libopt_la_SOURCES = \ opt/int_type_patcher/IntTypePatcher.cpp \ opt/interdex/InterDex.cpp \ opt/interdex/InterDexPass.cpp \ - opt/interdex/InterDexReshuffleImpl.cpp \ opt/interdex/InterDexReshufflePass.cpp \ opt/interdex/SortRemainingClassesPass.cpp \ opt/kotlin-lambda/RewriteKotlinSingletonInstance.cpp \ @@ -499,7 +485,6 @@ libopt_la_SOURCES = \ opt/split_huge_switches/SplitHugeSwitchPass.cpp \ opt/split_resource_tables/SplitResourceTables.cpp \ opt/object-escape-analysis/ExpandableMethodParams.cpp \ - opt/object-escape-analysis/ObjectEscapeAnalysisImpl.cpp \ opt/object-escape-analysis/ObjectEscapeAnalysis.cpp \ opt/staticrelo/StaticReloV2.cpp \ opt/string_concatenator/StringConcatenator.cpp \ diff --git a/RELEASE b/RELEASE index a8ca530807..d89abe1804 100644 --- a/RELEASE +++ b/RELEASE @@ -1 +1 @@ -2024-02-27 +2023-12-15 diff --git a/analysis/ip-reflection-analysis/IPReflectionAnalysis.cpp b/analysis/ip-reflection-analysis/IPReflectionAnalysis.cpp index 7cc684c8da..b3cbeb8660 100644 --- a/analysis/ip-reflection-analysis/IPReflectionAnalysis.cpp +++ b/analysis/ip-reflection-analysis/IPReflectionAnalysis.cpp @@ -33,7 +33,8 @@ struct Caller { using Domain = PatriciaTreeMapAbstractPartition; - Domain analyze_edge(const call_graph::EdgeId edge, const Domain& original) { + Domain analyze_edge(const std::shared_ptr& edge, + const Domain& original) { auto callee = edge->callee()->method(); if (!callee) { return Domain::bottom(); @@ -148,8 +149,8 @@ class ReflectionAnalyzer : public Base { reflection::SummaryQueryFn query_fn = [&](const IRInstruction* insn) -> reflection::AbstractObjectDomain { - auto callees = - call_graph::resolve_callees_in_graph(*this->get_call_graph(), insn); + auto callees = call_graph::resolve_callees_in_graph( + *this->get_call_graph(), m_method, insn); reflection::AbstractObjectDomain ret = reflection::AbstractObjectDomain::bottom(); @@ -180,8 +181,8 @@ class ReflectionAnalyzer : public Base { auto op = insn->opcode(); always_assert(opcode::is_an_invoke(op)); - auto callees = - call_graph::resolve_callees_in_graph(*this->get_call_graph(), insn); + auto callees = call_graph::resolve_callees_in_graph( + *this->get_call_graph(), m_method, insn); for (const DexMethod* method : callees) { this->get_caller_context()->update( @@ -235,7 +236,7 @@ void IPReflectionAnalysisPass::run_pass(DexStoresVector& stores, AnalysisParameters param; auto analysis = Analysis(scope, m_max_iteration, ¶m); analysis.run(); - const auto& summaries = analysis.registry.get_map(); + auto summaries = analysis.registry.get_map(); m_result = std::make_shared(); for (const auto& entry : summaries) { (*m_result)[entry.first] = entry.second.get_reflection_sites(); diff --git a/analysis/ip-reflection-analysis/IPReflectionAnalysis.h b/analysis/ip-reflection-analysis/IPReflectionAnalysis.h index decf61dfe3..16f9be8e9a 100644 --- a/analysis/ip-reflection-analysis/IPReflectionAnalysis.h +++ b/analysis/ip-reflection-analysis/IPReflectionAnalysis.h @@ -20,7 +20,9 @@ class IPReflectionAnalysisPass : public Pass { const override { using namespace redex_properties::interactions; using namespace redex_properties::names; - return {}; + return { + {HasSourceBlocks, Preserves}, + }; } void bind_config() override { diff --git a/analysis/max-depth/MaxDepthAnalysis.h b/analysis/max-depth/MaxDepthAnalysis.h index 9eea6f295e..681e0baaad 100644 --- a/analysis/max-depth/MaxDepthAnalysis.h +++ b/analysis/max-depth/MaxDepthAnalysis.h @@ -20,7 +20,9 @@ class MaxDepthAnalysisPass : public Pass { const override { using namespace redex_properties::interactions; using namespace redex_properties::names; - return {}; + return { + {HasSourceBlocks, Preserves}, + }; } void bind_config() override { bind("max_iteration", 20U, m_max_iteration); } diff --git a/checkers/InitialRenameClassCheker.cpp b/checkers/InitialRenameClassCheker.cpp deleted file mode 100644 index 1bab5e73bc..0000000000 --- a/checkers/InitialRenameClassCheker.cpp +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * - * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. - */ - -#include "InitialRenameClassCheker.h" - -#include "Debug.h" -#include "DexClass.h" -#include "Show.h" -#include "Walkers.h" - -namespace redex_properties { - -void InitialRenameClassCheker::run_checker(DexStoresVector& stores, - ConfigFiles& /* conf */, - PassManager& /* mgr */, - bool established) { - if (!established) { - return; - } - const auto& scope = build_class_scope(stores); - walk::classes(scope, [&](const DexClass* cls) { - always_assert_log(cls->rstate.is_renamable_initialized() || - cls->rstate.is_generated(), - "[%s] is not generated by Redex, and its renamable " - "status has not been initialized yet!\n", - SHOW(cls)); - }); -} - -} // namespace redex_properties - -namespace { -static redex_properties::InitialRenameClassCheker s_checker; -} // namespace diff --git a/checkers/InitialRenameClassCheker.h b/checkers/InitialRenameClassCheker.h deleted file mode 100644 index 4ebc35c3b9..0000000000 --- a/checkers/InitialRenameClassCheker.h +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * - * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. - */ - -#pragma once - -#include "RedexPropertyChecker.h" - -namespace redex_properties { - -class InitialRenameClassCheker : public PropertyChecker { - public: - InitialRenameClassCheker() : PropertyChecker(names::InitialRenameClass) {} - - void run_checker(DexStoresVector&, ConfigFiles&, PassManager&, bool) override; -}; - -} // namespace redex_properties diff --git a/checkers/InjectionIdInstructionsChecker.cpp b/checkers/InjectionIdInstructionsChecker.cpp index 49fb91725a..52d1b2506c 100644 --- a/checkers/InjectionIdInstructionsChecker.cpp +++ b/checkers/InjectionIdInstructionsChecker.cpp @@ -25,7 +25,7 @@ void InjectionIdInstructionsChecker::run_checker(DexStoresVector& stores, walk::parallel::opcodes(scope, [&](DexMethod* method, IRInstruction* insn) { always_assert_log(!opcode::is_injection_id(insn->opcode()), "[%s] %s contains injection id instruction!\n {%s}", - get_name(get_property()), SHOW(method), SHOW(insn)); + get_property_name().c_str(), SHOW(method), SHOW(insn)); }); } diff --git a/checkers/MethodRegisterChecker.cpp b/checkers/MethodRegisterChecker.cpp deleted file mode 100644 index 97f64f19d3..0000000000 --- a/checkers/MethodRegisterChecker.cpp +++ /dev/null @@ -1,155 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * - * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. - */ - -#include "MethodRegisterChecker.h" - -#include "DexClass.h" -#include "DexOpcode.h" -#include "IRInstruction.h" -#include "IROpcode.h" -#include "Interference.h" -#include "ScopedCFG.h" -#include "Show.h" -#include "Walkers.h" - -namespace redex_properties { - -/** - * Return the end of param register frame, to check there is no other - * instructions using register bigger than param registers. - * Also check in this method that param registers are continuous and - * crash program if not. - */ -reg_t get_param_end(const char* property_name, - const cfg::ControlFlowGraph& cfg, - DexMethod* method) { - auto params = cfg.get_param_instructions(); - if (params.empty()) { - return regalloc::max_unsigned_value(16); - } - auto param_ops = InstructionIterable(params); - auto it = param_ops.begin(); - reg_t prev_reg = it->insn->dest(); - reg_t spacing = it->insn->dest_is_wide() ? 2 : 1; - ++it; - for (; it != param_ops.end(); ++it) { - // check that the param registers are contiguous - always_assert_log( - (prev_reg + spacing) == it->insn->dest(), - "[%s] Param registers are not contiguous for method %s:\n%s", - property_name, - SHOW(method), - SHOW(params)); - spacing = it->insn->dest_is_wide() ? 2 : 1; - prev_reg = it->insn->dest(); - } - return prev_reg + spacing - 1; -} - -void MethodRegisterChecker::run_checker(DexStoresVector& stores, - ConfigFiles& /* conf */, - PassManager& /* mgr */, - bool established) { - if (!established) { - return; - } - const auto& scope = build_class_scope(stores); - walk::parallel::code(scope, [&](DexMethod* method, IRCode& code) { - cfg::ScopedCFG cfg(&code); - // 1. Load param's registers are at the end of register frames. - reg_t max_param_reg = - get_param_end(get_name(get_property()), code.cfg(), method); - auto ii = cfg::InstructionIterable(*cfg); - for (auto it = ii.begin(); it != ii.end(); ++it) { - // Checking several things for each method: - auto insn = it->insn; - - // 2. dest register is below max param reg and register limit. - if (insn->has_dest()) { - always_assert_log( - insn->dest() <= max_param_reg, - "[%s] Instruction %s refers to a register (v%u) > param" - " registers (%u) in method %s\n", - get_name(get_property()), - SHOW(insn), - insn->dest(), - max_param_reg, - SHOW(method)); - size_t max_dest_reg = regalloc::max_unsigned_value( - regalloc::interference::dest_bit_width(it)); - always_assert_log( - insn->dest() <= max_dest_reg, - "[%s] Instruction %s refers to a register (v%u) > max dest" - " register (%zu) in method %s\n", - get_name(get_property()), - SHOW(insn), - insn->dest(), - max_dest_reg, - SHOW(method)); - } - bool is_range = false; - if (opcode::has_range_form(insn->opcode())) { - insn->denormalize_registers(); - is_range = needs_range_conversion(insn); - if (is_range) { - // 3. invoke-range's registers are continuous - always_assert_log(insn->has_contiguous_range_srcs_denormalized(), - "[%s] Instruction %s has non-contiguous srcs in " - "method %s.\n", - get_name(get_property()), - SHOW(insn), - SHOW(method)); - - // 4. No overly large range instructions. - auto size = insn->srcs_size(); - // From DexInstruction::set_range_size; - always_assert_log( - dex_opcode::format(opcode::range_version(insn->opcode())) == - FMT_f5rc || - size == (size & 0xff), - "[%s] Range instruction %s takes too much src size in method " - "%s.\n", - get_name(get_property()), - SHOW(insn), - SHOW(method)); - } - insn->normalize_registers(); - } - // 5. All src registers are below max param reg and register limits. - for (size_t i = 0; i < insn->srcs_size(); ++i) { - always_assert_log( - insn->src(i) <= max_param_reg, - "[%s] Instruction %s refers to a register (v%u) > param" - " registers (%u) in method %s\n", - get_name(get_property()), - SHOW(insn), - insn->src(i), - max_param_reg, - SHOW(method)); - if (!is_range) { - auto max_src_reg = regalloc::interference::max_value_for_src( - insn, i, insn->src_is_wide(i)); - always_assert_log( - insn->src(i) <= max_src_reg, - "[%s] Instruction %s refers to a register (v%u) > max src" - " registers (%u) in method %s\n", - get_name(get_property()), - SHOW(insn), - insn->src(i), - max_src_reg, - SHOW(method)); - } - } - } - }); -} - -} // namespace redex_properties - -namespace { -static redex_properties::MethodRegisterChecker s_checker; -} // namespace diff --git a/checkers/MethodRegisterChecker.h b/checkers/MethodRegisterChecker.h deleted file mode 100644 index ac5e52901a..0000000000 --- a/checkers/MethodRegisterChecker.h +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * - * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. - */ - -#pragma once - -#include "RedexPropertyChecker.h" - -namespace redex_properties { - -class MethodRegisterChecker : public PropertyChecker { - public: - MethodRegisterChecker() : PropertyChecker(names::MethodRegister) {} - - void run_checker(DexStoresVector&, ConfigFiles&, PassManager&, bool) override; -}; - -} // namespace redex_properties diff --git a/checkers/NoInitClassInstructionsChecker.cpp b/checkers/NoInitClassInstructionsChecker.cpp index 6ee4677fd5..abde177969 100644 --- a/checkers/NoInitClassInstructionsChecker.cpp +++ b/checkers/NoInitClassInstructionsChecker.cpp @@ -25,7 +25,7 @@ void NoInitClassInstructionsChecker::run_checker(DexStoresVector& stores, walk::parallel::opcodes(scope, [&](DexMethod* method, IRInstruction* insn) { always_assert_log(!opcode::is_init_class(insn->opcode()), "[%s] %s contains init-class instruction!\n {%s}", - get_name(get_property()), SHOW(method), SHOW(insn)); + get_property_name().c_str(), SHOW(method), SHOW(insn)); }); } diff --git a/checkers/NoResolvablePureRefsChecker.cpp b/checkers/NoResolvablePureRefsChecker.cpp index e134f3e52d..0fa1a91ae2 100644 --- a/checkers/NoResolvablePureRefsChecker.cpp +++ b/checkers/NoResolvablePureRefsChecker.cpp @@ -133,7 +133,7 @@ void NoResolvablePureRefsChecker::run_checker(DexStoresVector& stores, } always_assert_log(mref->is_def(), "[%s] %s contains pure method ref!\n {%s}", - get_name(get_property()), SHOW(method), SHOW(insn)); + get_property_name().c_str(), SHOW(method), SHOW(insn)); } else if (insn->has_field()) { DexFieldRef* fref = insn->get_field(); if (fref->is_def()) { @@ -170,7 +170,7 @@ void NoResolvablePureRefsChecker::run_checker(DexStoresVector& stores, } always_assert_log(fref->is_def(), "[%s] %s contains pure field ref!\n {%s}", - get_name(get_property()), SHOW(method), SHOW(insn)); + get_property_name().c_str(), SHOW(method), SHOW(insn)); } }); } diff --git a/checkers/NoSpuriousGetClassCallsChecker.cpp b/checkers/NoSpuriousGetClassCallsChecker.cpp index d2a488811f..4f4ad41996 100644 --- a/checkers/NoSpuriousGetClassCallsChecker.cpp +++ b/checkers/NoSpuriousGetClassCallsChecker.cpp @@ -37,7 +37,7 @@ void NoSpuriousGetClassCallsChecker::check_spurious_getClass( always_assert_log( !move_result.is_end(), "[%s] %s contains spurious Object.getClass() instruction!\n {%s}", - get_name(get_property()), SHOW(method), SHOW(insn)); + get_property_name().c_str(), SHOW(method), SHOW(insn)); } } } diff --git a/checkers/NoUnreachableInstructionsChecker.cpp b/checkers/NoUnreachableInstructionsChecker.cpp index 468d5d2074..944b181ccc 100644 --- a/checkers/NoUnreachableInstructionsChecker.cpp +++ b/checkers/NoUnreachableInstructionsChecker.cpp @@ -25,7 +25,7 @@ void NoUnreachableInstructionsChecker::run_checker(DexStoresVector& stores, walk::parallel::opcodes(scope, [&](DexMethod* method, IRInstruction* insn) { always_assert_log(!opcode::is_unreachable(insn->opcode()), "[%s] %s contains unreachable instruction!\n {%s}", - get_name(get_property()), SHOW(method), SHOW(insn)); + get_property_name().c_str(), SHOW(method), SHOW(insn)); }); } diff --git a/checkers/RenameClassChecker.cpp b/checkers/RenameClassChecker.cpp index 368d1936bd..c389f986d9 100644 --- a/checkers/RenameClassChecker.cpp +++ b/checkers/RenameClassChecker.cpp @@ -33,7 +33,7 @@ void RenameClassChecker::run_checker(DexStoresVector& stores, always_assert_log( sequence_nr == global_cls_nr, "[%s] invalid class number, expected %u, got %u, class %s!\n", - get_name(get_property()), sequence_nr, global_cls_nr, cls_name); + get_property_name().c_str(), sequence_nr, global_cls_nr, cls_name); } ++sequence_nr; }); diff --git a/get_boost.sh b/get_boost.sh index 6f0b1aff03..8c0d97c986 100755 --- a/get_boost.sh +++ b/get_boost.sh @@ -9,7 +9,7 @@ BOOST_VERSION="1.71.0" BOOST_VERSION_UNDERSCORE="${BOOST_VERSION//./_}" BOOST_FILE="boost_${BOOST_VERSION_UNDERSCORE}.tar.bz2" BOOST_TAR_URL="https://boostorg.jfrog.io/artifactory/main/release/${BOOST_VERSION}/source/${BOOST_FILE}" -BOOST_CACHE_DIR="dl_cache/boost_cache" +BOOST_CACHE_DIR="boost_cache" BOOST_TAR_LOCAL="${BOOST_CACHE_DIR}/${BOOST_FILE}" BOOST_DIR="boost_${BOOST_VERSION_UNDERSCORE}" @@ -17,17 +17,13 @@ set -e # Check for cached artifacts. if [ ! -d "$BOOST_CACHE_DIR" ] ; then - mkdir -p "$BOOST_CACHE_DIR" + mkdir boost_cache fi if [ ! -f "$BOOST_TAR_LOCAL" ] ; then - echo "Downloading Boost 1.71.0" wget "$BOOST_TAR_URL" -O "$BOOST_TAR_LOCAL" fi -mkdir -p toolchain_install/boost -pushd toolchain_install/boost - -tar --bzip2 -xf "../../$BOOST_TAR_LOCAL" +tar --bzip2 -xf "$BOOST_TAR_LOCAL" cd "$BOOST_DIR" ./bootstrap.sh --with-libraries=filesystem,iostreams,program_options,regex,system,thread diff --git a/libredex/ApkResources.cpp b/libredex/ApkResources.cpp index 15820c881c..86faf94c8b 100644 --- a/libredex/ApkResources.cpp +++ b/libredex/ApkResources.cpp @@ -673,7 +673,7 @@ ManifestClassInfo extract_classes_from_manifest(const char* data, size_t size) { } } else if (tag == instrumentation) { std::string classname = get_string_attribute_value(parser, name); - always_assert(!classname.empty()); + always_assert(classname.size()); manifest_classes.instrumentation_classes.emplace( java_names::external_to_internal(classname)); } else if (tag == queries) { @@ -683,7 +683,7 @@ ManifestClassInfo extract_classes_from_manifest(const char* data, size_t size) { } else if (string_to_tag.count(tag)) { std::string classname = get_string_attribute_value( parser, tag != activity_alias ? name : target_activity); - always_assert(!classname.empty()); + always_assert(classname.size()); bool has_exported_attribute = has_bool_attribute(parser, exported); android::Res_value ignore_output; @@ -2417,7 +2417,6 @@ size_t ResourcesArscFile::serialize() { // Refer to the re-parsed data at initial step to get full details of the // entries, flags and values for the new type. std::vector flags; - flags.reserve(type_def.source_res_ids.size()); for (auto& id : type_def.source_res_ids) { flags.emplace_back(table_parser.m_res_id_to_flags.at(id)); } @@ -2566,7 +2565,7 @@ std::unordered_set ResourcesArscFile::get_types_by_name_prefixes( const auto& type_name = all_types.at(i); if (std::find_if(type_name_prefixes.begin(), type_name_prefixes.end(), [&](const std::string& prefix) { - return type_name.find(prefix) == 0; + return type_name.find(prefix) != std::string::npos; }) != type_name_prefixes.end()) { type_ids.emplace((i + 1) << TYPE_INDEX_BIT_SHIFT); } diff --git a/libredex/BundleResources.cpp b/libredex/BundleResources.cpp index 413d5b3d0e..3bbdc06451 100644 --- a/libredex/BundleResources.cpp +++ b/libredex/BundleResources.cpp @@ -28,7 +28,6 @@ #include #include "Debug.h" -#include "DetectBundle.h" #include "DexUtil.h" #include "ReadMaybeMapped.h" #include "RedexMappedFile.h" @@ -300,14 +299,14 @@ void read_single_manifest(const std::string& manifest, } } else if (tag == "instrumentation") { auto classname = get_string_attribute_value(element, "name"); - always_assert(!classname.empty()); + always_assert(classname.size()); manifest_classes->instrumentation_classes.emplace( fully_qualified_external(package_name, classname)); } else if (string_to_tag.count(tag)) { std::string classname = get_string_attribute_value( element, tag != "activity-alias" ? "name" : "targetActivity"); - always_assert(!classname.empty()); + always_assert(classname.size()); bool has_exported_attribute = has_primitive_attribute( element, "exported", aapt::pb::Primitive::kBooleanValue); @@ -1909,7 +1908,7 @@ std::unordered_set ResourcesPbFile::get_types_by_name_prefixes( const auto& type_name = pair.second; if (std::find_if(type_name_prefixes.begin(), type_name_prefixes.end(), [&](const std::string& prefix) { - return type_name.find(prefix) == 0; + return type_name.find(prefix) != std::string::npos; }) != type_name_prefixes.end()) { type_ids.emplace((pair.first) << TYPE_INDEX_BIT_SHIFT); } @@ -2301,28 +2300,6 @@ void BundleResources::obfuscate_xml_files( } } -void BundleResources::finalize_bundle_config(const ResourceConfig& config) { - if (!config.canonical_entry_types.empty() && has_bundle_config(m_directory)) { - std::string bundle_config_path = - (boost::filesystem::path(m_directory) / "BundleConfig.pb").string(); - read_protobuf_file_contents( - bundle_config_path, - [&](google::protobuf::io::CodedInputStream& input, size_t size) { - android::bundle::BundleConfig bundle_config; - always_assert_log(bundle_config.ParseFromCodedStream(&input), - "BundleResource failed to read %s", - bundle_config_path.c_str()); - auto pb_resource_optimizations = - bundle_config.mutable_optimizations() - ->mutable_resource_optimizations(); - pb_resource_optimizations->mutable_collapsed_resource_names() - ->set_deduplicate_resource_entries(true); - std::ofstream out(bundle_config_path, std::ofstream::binary); - always_assert(bundle_config.SerializeToOstream(&out)); - }); - } -} - ResourcesPbFile::~ResourcesPbFile() {} #endif // HAS_PROTOBUF diff --git a/libredex/BundleResources.h b/libredex/BundleResources.h index 5cde79a439..e434b96ef5 100644 --- a/libredex/BundleResources.h +++ b/libredex/BundleResources.h @@ -18,9 +18,7 @@ #include #include -#include "GlobalConfig.h" #include "androidfw/ResourceTypes.h" -#include "protocfg/config.pb.h" #include "protores/Resources.pb.h" #include @@ -127,7 +125,6 @@ class BundleResources : public AndroidResources { void obfuscate_xml_files(const std::unordered_set& allowed_types, const std::unordered_set& do_not_obfuscate_elements) override; - void finalize_bundle_config(const ResourceConfig& config) override; protected: std::vector find_res_directories() override; diff --git a/libredex/CFGMutation.cpp b/libredex/CFGMutation.cpp index f8f97f976b..cc0963e71c 100644 --- a/libredex/CFGMutation.cpp +++ b/libredex/CFGMutation.cpp @@ -15,7 +15,9 @@ namespace cfg { -static AccumulatingTimer s_timer("CFGMutation"); +static AccumulatingTimer s_timer; + +double CFGMutation::get_seconds() { return s_timer.get_seconds(); } void CFGMutation::clear() { for (auto& [block, changes] : m_changes) { diff --git a/libredex/CFGMutation.h b/libredex/CFGMutation.h index 44c9bdffba..c04fc572c5 100644 --- a/libredex/CFGMutation.h +++ b/libredex/CFGMutation.h @@ -26,6 +26,7 @@ namespace cfg { /// TODO(T59235117) Flush mutation in the destructor. class CFGMutation { public: + static double get_seconds(); /// Create a new mutation to apply to \p cfg. explicit CFGMutation(ControlFlowGraph& cfg); diff --git a/libredex/CallGraph.cpp b/libredex/CallGraph.cpp index d6db5014da..7d657e3fe1 100644 --- a/libredex/CallGraph.cpp +++ b/libredex/CallGraph.cpp @@ -19,12 +19,6 @@ namespace mog = method_override_graph; -namespace { - -AccumulatingTimer s_timer("CallGraph"); - -} // namespace - namespace call_graph { Graph single_callee_graph(const mog::Graph& method_override_graph, @@ -46,8 +40,11 @@ Graph multiple_callee_graph(const mog::Graph& method_override_graph, SingleCalleeStrategy::SingleCalleeStrategy( const mog::Graph& method_override_graph, const Scope& scope) - : m_scope(scope), - m_non_virtual(mog::get_non_true_virtuals(method_override_graph, scope)) {} + : m_scope(scope) { + auto non_virtual_vec = + mog::get_non_true_virtuals(method_override_graph, scope); + m_non_virtual.insert(non_virtual_vec.begin(), non_virtual_vec.end()); +} CallSites SingleCalleeStrategy::get_callsites(const DexMethod* method) const { CallSites callsites; @@ -59,15 +56,17 @@ CallSites SingleCalleeStrategy::get_callsites(const DexMethod* method) const { code, [&](const IRList::iterator& it) { auto insn = it->insn; if (opcode::is_an_invoke(insn->opcode())) { - auto callee = resolve_invoke_method(insn, method); + auto callee = this->resolve_callee(method, insn); if (callee == nullptr || is_definitely_virtual(callee)) { return editable_cfg_adapter::LOOP_CONTINUE; + ; } if (callee->is_concrete()) { callsites.emplace_back(callee, insn); } } return editable_cfg_adapter::LOOP_CONTINUE; + ; }); return callsites; } @@ -78,14 +77,19 @@ RootAndDynamic SingleCalleeStrategy::get_roots() const { walk::code(m_scope, [&](DexMethod* method, IRCode& /* code */) { if (is_definitely_virtual(method) || root(method) || method::is_clinit(method) || method::is_argless_init(method)) { - roots.insert(method); + roots.emplace_back(method); } }); return root_and_dynamic; } bool SingleCalleeStrategy::is_definitely_virtual(DexMethod* method) const { - return method->is_virtual() && m_non_virtual.count_unsafe(method) == 0; + return method->is_virtual() && m_non_virtual.count(method) == 0; +} + +DexMethod* SingleCalleeStrategy::resolve_callee(const DexMethod* caller, + IRInstruction* invoke) const { + return resolve_method(invoke->get_method(), opcode_to_search(invoke), caller); } MultipleCalleeBaseStrategy::MultipleCalleeBaseStrategy( @@ -94,33 +98,32 @@ MultipleCalleeBaseStrategy::MultipleCalleeBaseStrategy( m_method_override_graph(method_override_graph) {} const std::vector& -MultipleCalleeBaseStrategy::get_ordered_overriding_methods_with_code_or_native( +MultipleCalleeBaseStrategy::get_ordered_overriding_methods_with_code( const DexMethod* method) const { - auto res = m_overriding_methods_cache.get(method); - if (res) { - return *res; + auto res = m_overriding_methods_cache.get(method, nullptr); + if (!res) { + auto overriding_methods = + mog::get_overriding_methods(m_method_override_graph, method); + std20::erase_if(overriding_methods, [](auto* m) { return !m->get_code(); }); + std::sort(overriding_methods.begin(), overriding_methods.end(), + compare_dexmethods); + m_overriding_methods_cache.update( + method, [&overriding_methods, &res](auto, auto& p, bool exists) { + if (exists) { + always_assert(*p == overriding_methods); + } else { + p = std::make_shared>( + std::move(overriding_methods)); + } + res = p; + }); } - return init_ordered_overriding_methods_with_code_or_native( - method, mog::get_overriding_methods(m_method_override_graph, method)); -} - -const std::vector& -MultipleCalleeBaseStrategy::init_ordered_overriding_methods_with_code_or_native( - const DexMethod* method, - std::vector overriding_methods) const { - std20::erase_if(overriding_methods, - [](auto* m) { return !m->get_code() && !is_native(m); }); - std::sort(overriding_methods.begin(), overriding_methods.end(), - compare_dexmethods); - return *m_overriding_methods_cache - .get_or_emplace_and_assert_equal(method, - std::move(overriding_methods)) - .first; + return *res; } RootAndDynamic MultipleCalleeBaseStrategy::get_roots() const { - Timer t("get_roots"); RootAndDynamic root_and_dynamic; + MethodSet emplaced_methods; auto& roots = root_and_dynamic.roots; auto& dynamic_methods = root_and_dynamic.dynamic_methods; // Gather clinits and root methods, and the methods that override or @@ -130,13 +133,21 @@ RootAndDynamic MultipleCalleeBaseStrategy::get_roots() const { // No need to add root methods, they will be added anyway. return; } - roots.insert(method); + if (emplaced_methods.emplace(method).second) { + roots.emplace_back(method); + } }; walk::methods(m_scope, [&](DexMethod* method) { if (method::is_clinit(method)) { - roots.insert(method); + roots.emplace_back(method); + emplaced_methods.emplace(method); return; } + // For methods marked with DoNotInline, we also add to dynamic methods set + // to avoid propagating return value. + if (method->rstate.dont_inline()) { + dynamic_methods.emplace(method); + } if (!root(method) && !method::is_argless_init(method) && !(method->is_virtual() && is_interface(type_class(method->get_class())) && @@ -150,14 +161,19 @@ RootAndDynamic MultipleCalleeBaseStrategy::get_roots() const { !can_rename(method)) { dynamic_methods.emplace(method); } - if (method->get_code()) { - roots.insert(method); + if (method->get_code() && emplaced_methods.emplace(method).second) { + roots.emplace_back(method); } const auto& overriding_methods = mog::get_overriding_methods(m_method_override_graph, method); for (auto overriding_method : overriding_methods) { add_root_method_overrides(overriding_method); } + const auto& overiden_methods = + mog::get_overridden_methods(m_method_override_graph, method); + for (auto overiden_method : overiden_methods) { + add_root_method_overrides(overiden_method); + } }); // Gather methods that override or implement external or native methods // as well. @@ -168,26 +184,39 @@ RootAndDynamic MultipleCalleeBaseStrategy::get_roots() const { const auto& overriding_methods = mog::get_overriding_methods(m_method_override_graph, method); for (auto* overriding : overriding_methods) { - // We don't need to add overriding to dynamic_methods here, as that will - // happen anyway. - if (!overriding->is_external() && overriding->get_code()) { - roots.insert(overriding); + if (overriding->is_external()) { + dynamic_methods.emplace(overriding); + } + if (!overriding->is_external() && overriding->get_code() && + emplaced_methods.emplace(overriding).second) { + roots.emplace_back(overriding); } } - // We don't need to add overridden external methods to dynamic_methods, as - // that will happen anyway. Internal interface methods can be overridden - // by external methods as well. - const auto& overridden_methods = + // Internal methods might be overriden by external methods. Add such + // methods to dynamic methods to avoid return value propagation as well. + const auto& overiden_methods = mog::get_overridden_methods(m_method_override_graph, method, true); - for (auto m : overridden_methods) { - if (!m->is_external()) { - auto* cls = type_class(m->get_class()); - always_assert(is_interface(cls) || is_abstract(cls)); - dynamic_methods.emplace(m); - } + for (auto m : overiden_methods) { + dynamic_methods.emplace(m); + } + } + if (is_native(method)) { + dynamic_methods.emplace(method); + const auto& overriding_methods = + mog::get_overriding_methods(m_method_override_graph, method, true); + for (auto m : overriding_methods) { + dynamic_methods.emplace(m); + } + const auto& overiden_methods = + mog::get_overridden_methods(m_method_override_graph, method, true); + for (auto m : overiden_methods) { + dynamic_methods.emplace(m); } } } + // Add additional roots if needed. + auto additional_roots = get_additional_roots(emplaced_methods); + roots.insert(roots.end(), additional_roots.begin(), additional_roots.end()); return root_and_dynamic; } @@ -195,6 +224,23 @@ CompleteCallGraphStrategy::CompleteCallGraphStrategy( const mog::Graph& method_override_graph, const Scope& scope) : MultipleCalleeBaseStrategy(method_override_graph, scope) {} +static DexMethod* resolve_interface_virtual_callee(const IRInstruction* insn, + const DexMethod* caller) { + DexMethod* callee = nullptr; + if (opcode_to_search(insn) == MethodSearch::Virtual) { + callee = resolve_method(insn->get_method(), MethodSearch::InterfaceVirtual, + caller); + if (callee == nullptr) { + auto insn_method_cls = type_class(insn->get_method()->get_class()); + if (insn_method_cls != nullptr && !insn_method_cls->is_external()) { + TRACE(CALLGRAPH, 1, "Unexpected unresolved insn %s in %s", SHOW(insn), + SHOW(caller)); + } + } + } + return callee; +} + CallSites CompleteCallGraphStrategy::get_callsites( const DexMethod* method) const { CallSites callsites; @@ -206,40 +252,48 @@ CallSites CompleteCallGraphStrategy::get_callsites( code, [&](const IRList::iterator& it) { auto insn = it->insn; if (opcode::is_an_invoke(insn->opcode())) { - auto callee = resolve_invoke_method(insn, method); + auto callee = this->resolve_callee(method, insn); if (callee == nullptr) { - return editable_cfg_adapter::LOOP_CONTINUE; + callee = resolve_interface_virtual_callee(insn, method); + if (callee == nullptr) { + return editable_cfg_adapter::LOOP_CONTINUE; + } } - if (callee->get_code() || is_native(callee)) { + if (callee->is_concrete()) { callsites.emplace_back(callee, insn); } if (opcode::is_invoke_virtual(insn->opcode()) || opcode::is_invoke_interface(insn->opcode())) { const auto& overriding_methods = - get_ordered_overriding_methods_with_code_or_native(callee); - for (auto overriding_method : overriding_methods) { - callsites.emplace_back(overriding_method, insn); + get_ordered_overriding_methods_with_code(callee); + + for (auto m : overriding_methods) { + callsites.emplace_back(m, insn); } } } return editable_cfg_adapter::LOOP_CONTINUE; + ; }); return callsites; } RootAndDynamic CompleteCallGraphStrategy::get_roots() const { RootAndDynamic root_and_dynamic; + MethodSet emplaced_methods; auto& roots = root_and_dynamic.roots; auto add_root_method_overrides = [&](const DexMethod* method) { - if (!root(method)) { + if (!root(method) && emplaced_methods.emplace(method).second) { // No need to add root methods, they will be added anyway. - roots.insert(method); + roots.emplace_back(method); } }; walk::methods(m_scope, [&](DexMethod* method) { if (root(method) || method::is_clinit(method) || method::is_argless_init(method)) { - roots.insert(method); + if (emplaced_methods.emplace(method).second) { + roots.emplace_back(method); + } } if (!root(method) && !(method->is_virtual() && is_interface(type_class(method->get_class())) && @@ -254,6 +308,11 @@ RootAndDynamic CompleteCallGraphStrategy::get_roots() const { for (auto overriding_method : overriding_methods) { add_root_method_overrides(overriding_method); } + const auto& overiden_methods = + mog::get_overridden_methods(m_method_override_graph, method); + for (auto overiden_method : overiden_methods) { + add_root_method_overrides(overiden_method); + } }); // Gather methods that override or implement external methods for (auto& pair : m_method_override_graph.nodes()) { @@ -262,7 +321,9 @@ RootAndDynamic CompleteCallGraphStrategy::get_roots() const { const auto& overriding_methods = mog::get_overriding_methods(m_method_override_graph, method, true); for (auto* overriding : overriding_methods) { - roots.insert(overriding); + if (emplaced_methods.emplace(overriding).second) { + roots.emplace_back(overriding); + } } } } @@ -276,37 +337,41 @@ MultipleCalleeStrategy::MultipleCalleeStrategy( : MultipleCalleeBaseStrategy(method_override_graph, scope) { // Gather big overrides true virtual methods. ConcurrentSet concurrent_callees; - walk::parallel::opcodes( - scope, [&](const DexMethod* method, IRInstruction* insn) { - if (opcode::is_an_invoke(insn->opcode())) { - auto callee = resolve_invoke_method(insn, method); - if (callee == nullptr) { - return; - } - if (!callee->is_virtual() || insn->opcode() == OPCODE_INVOKE_SUPER) { - return; - } - if (!concurrent_callees.insert(callee)) { - return; - } - auto overriding_methods = - mog::get_overriding_methods(m_method_override_graph, callee); - uint32_t num_override = 0; - for (auto overriding_method : overriding_methods) { - if (overriding_method->get_code()) { - ++num_override; - } - } - if (num_override <= big_override_threshold) { - init_ordered_overriding_methods_with_code_or_native( - callee, std::move(overriding_methods)); - } else { - m_big_virtuals.emplace(callee); - m_big_virtual_overrides.insert(overriding_methods.begin(), - overriding_methods.end()); - } + ConcurrentSet concurrent_big_overrides; + walk::parallel::opcodes(scope, [&](const DexMethod* method, + IRInstruction* insn) { + if (opcode::is_an_invoke(insn->opcode())) { + auto callee = + resolve_method(insn->get_method(), opcode_to_search(insn), method); + if (callee == nullptr) { + callee = resolve_interface_virtual_callee(insn, method); + if (callee == nullptr) { + return; } - }); + } + if (!callee->is_virtual()) { + return; + } + if (!concurrent_callees.insert(callee)) { + return; + } + const auto& overriding_methods = + mog::get_overriding_methods(m_method_override_graph, callee); + uint32_t num_override = 0; + for (auto overriding_method : overriding_methods) { + if (overriding_method->get_code()) { + ++num_override; + } + } + if (num_override > big_override_threshold) { + concurrent_big_overrides.emplace(callee); + for (auto overriding_method : overriding_methods) { + concurrent_big_overrides.emplace(overriding_method); + } + } + } + }); + m_big_override = concurrent_big_overrides.move_to_container(); } CallSites MultipleCalleeStrategy::get_callsites(const DexMethod* method) const { @@ -319,208 +384,247 @@ CallSites MultipleCalleeStrategy::get_callsites(const DexMethod* method) const { code, [&](const IRList::iterator& it) { auto insn = it->insn; if (opcode::is_an_invoke(insn->opcode())) { - auto callee = resolve_invoke_method(insn, method); + auto callee = this->resolve_callee(method, insn); if (callee == nullptr) { - return editable_cfg_adapter::LOOP_CONTINUE; + callee = resolve_interface_virtual_callee(insn, method); + if (callee == nullptr) { + return editable_cfg_adapter::LOOP_CONTINUE; + ; + } } - if (is_definitely_virtual(callee) && - insn->opcode() != OPCODE_INVOKE_SUPER) { + if (is_definitely_virtual(callee)) { // For true virtual callees, add the callee itself and all of its - // overrides if they are not in big virtuals. - if (m_big_virtuals.count_unsafe(callee)) { + // overrides if they are not in big overrides. + if (m_big_override.count(callee)) { return editable_cfg_adapter::LOOP_CONTINUE; } - if (callee->get_code() || is_native(callee)) { + if (callee->get_code()) { callsites.emplace_back(callee, insn); } - const auto& overriding_methods = - get_ordered_overriding_methods_with_code_or_native(callee); - for (auto overriding_method : overriding_methods) { - callsites.emplace_back(overriding_method, insn); + if (insn->opcode() != OPCODE_INVOKE_SUPER) { + const auto& overriding_methods = + get_ordered_overriding_methods_with_code(callee); + for (auto overriding_method : overriding_methods) { + callsites.emplace_back(overriding_method, insn); + } } } else if (callee->is_concrete()) { callsites.emplace_back(callee, insn); } } return editable_cfg_adapter::LOOP_CONTINUE; + ; }); return callsites; } // Add big override methods to root as well. -RootAndDynamic MultipleCalleeStrategy::get_roots() const { - auto root_and_dynamic = MultipleCalleeBaseStrategy::get_roots(); - auto add_root = [&](auto* method) { - if (!method->is_external() && method->get_code()) { - root_and_dynamic.roots.insert(method); +std::vector MultipleCalleeStrategy::get_additional_roots( + const MethodSet& existing_roots) const { + std::vector additional_roots; + for (auto method : m_big_override) { + if (!method->is_external() && !existing_roots.count(method) && + method->get_code()) { + additional_roots.emplace_back(method); } - }; - std::for_each(m_big_virtuals.begin(), m_big_virtuals.end(), add_root); - std::for_each(m_big_virtual_overrides.begin(), m_big_virtual_overrides.end(), - add_root); - return root_and_dynamic; + } + return additional_roots; } Edge::Edge(NodeId caller, NodeId callee, IRInstruction* invoke_insn) : m_caller(caller), m_callee(callee), m_invoke_insn(invoke_insn) {} Graph::Graph(const BuildStrategy& strat) - : m_entry(std::make_unique(Node::GHOST_ENTRY)), - m_exit(std::make_unique(Node::GHOST_EXIT)) { - auto timer_scope = s_timer.scope(); - Timer t("Graph::Graph"); - - auto root_and_dynamic = strat.get_roots(); - m_dynamic_methods = std::move(root_and_dynamic.dynamic_methods); - std::vector root_nodes; - + : m_entry(std::make_shared(Node::GHOST_ENTRY)), + m_exit(std::make_shared(Node::GHOST_EXIT)) { // Obtain the callsites of each method recursively, building the graph in the // process. - ConcurrentMap>> concurrent_preds; - std::mutex predecessors_wq_mutex; - auto predecessors_wq = workqueue_foreach([&](Node* callee_node) { - auto& preds = concurrent_preds.at_unsafe(callee_node); - std::vector> callee_edges; - callee_edges.reserve(preds.size()); - size_t size = 0; - for (auto& edges : preds) { - size += edges.size(); - callee_edges.emplace_back(std::move(edges)); - } - std::sort(callee_edges.begin(), callee_edges.end(), [](auto& p, auto& q) { - return compare_dexmethods(p.front()->caller()->method(), - q.front()->caller()->method()); + ConcurrentMap concurrent_nodes; + ConcurrentMap> + concurrent_insn_to_callee; + std::mutex nodes_mutex; + struct MethodEdges { + const DexMethod* method; + std::vector> edges; + }; + ConcurrentMap> concurrent_preds; + ConcurrentMap> concurrent_succs; + auto record_trivial_edge = [&](auto caller_node, auto callee_node) { + auto edge = std::make_shared(caller_node, callee_node, + /* invoke_insn */ nullptr); + concurrent_preds.update(callee_node, [&](auto, auto& v, bool) { + v.emplace_back((MethodEdges){caller_node->method(), {edge}}); }); - auto& callee_predecessors = callee_node->m_predecessors; - callee_predecessors.reserve(size); - for (auto& edges : callee_edges) { - callee_predecessors.insert(callee_predecessors.end(), edges.begin(), - edges.end()); - } - }); - + concurrent_succs.update(caller_node, [&](auto, auto& w, bool) { + w.emplace_back((MethodEdges){callee_node->method(), {std::move(edge)}}); + }); + }; struct WorkItem { const DexMethod* caller; - Node* caller_node; + NodeId caller_node; + bool caller_is_root; }; - constexpr IRInstruction* no_insn = nullptr; - auto successors_wq = workqueue_foreach( + auto wq = workqueue_foreach( [&](sparta::WorkerState* worker_state, const WorkItem& work_item) { - auto get_node = [&](const DexMethod* method) -> Node* { - auto [const_node, node_created] = - m_nodes.get_or_emplace_and_assert_equal(method, method); - Node* node = const_cast(const_node); - if (node_created) { - worker_state->push_task((WorkItem){method, node}); - } - return node; - }; + auto caller = work_item.caller; + auto caller_node = work_item.caller_node; + auto caller_is_root = work_item.caller_is_root; - using Insns = std::vector; + if (caller_is_root) { + // Add edges from the single "ghost" entry node to all the "real" + // entry nodes in the graph. + record_trivial_edge(this->entry(), caller_node); + } + + auto callsites = strat.get_callsites(caller); + if (callsites.empty()) { + // Add edges from the single "ghost" exit node to all the "real" exit + // nodes in the graph. + record_trivial_edge(caller_node, this->exit()); + return; + } + + // Gather and create all callee nodes, and kick off new concurrent work + std::unordered_map callee_indices; struct CalleePartition { - Node* callee_node; - Insns invoke_insns; - CalleePartition(Node* callee_node, Insns invoke_insns) - : callee_node(callee_node), - invoke_insns(std::move(invoke_insns)) {} + const DexMethod* callee; + std::vector invoke_insns{}; }; std::vector callee_partitions; std::unordered_map> insn_to_callee; - size_t caller_successors_size; - - auto* caller = work_item.caller; - if (caller == nullptr) { - // Add edges from the single "ghost" entry node to all the "real" root - // entry nodes in the graph. - callee_partitions.reserve(root_nodes.size()); - for (auto root_node : root_nodes) { - callee_partitions.emplace_back(root_node, Insns{no_insn}); - } - caller_successors_size = root_nodes.size(); - } else { - auto callsites = strat.get_callsites(caller); - if (callsites.empty()) { - // Add edges from the single "ghost" exit node to all the "real" - // exit nodes in the graph. - callee_partitions.emplace_back(m_exit.get(), Insns{no_insn}); - caller_successors_size = 1; - } else { - // Gather and create all "real" callee nodes, and kick off new - // concurrent work - std::unordered_map callee_indices; - for (const auto& callsite : callsites) { - auto callee = callsite.callee; - auto [it, emplaced] = - callee_indices.emplace(callee, callee_indices.size()); - if (emplaced) { - callee_partitions.emplace_back(get_node(callee), Insns()); - } - auto& callee_partition = callee_partitions[it->second]; - callee_partition.invoke_insns.push_back(callsite.invoke_insn); - insn_to_callee[callsite.invoke_insn].emplace(callee); - } - caller_successors_size = callsites.size(); + for (const auto& callsite : callsites) { + auto callee = callsite.callee; + auto [it, emplaced] = + callee_indices.emplace(callee, callee_indices.size()); + if (emplaced) { + callee_partitions.push_back(CalleePartition{callee}); } + auto& callee_partition = callee_partitions[it->second]; + callee_partition.invoke_insns.push_back(callsite.invoke_insn); + insn_to_callee[callsite.invoke_insn].emplace(callee); } - // Record all edges - auto* caller_node = work_item.caller_node; - auto& caller_successors = caller_node->m_successors; - caller_successors.reserve(caller_successors_size); - std::sort(callee_partitions.begin(), callee_partitions.end(), - [](auto& p, auto& q) { - return compare_dexmethods(p.callee_node->method(), - q.callee_node->method()); - }); - std::vector added_preds; - for (auto&& [callee_node, callee_invoke_insns] : callee_partitions) { - std::vector callee_edges; - callee_edges.reserve(callee_invoke_insns.size()); - for (auto* invoke_insn : callee_invoke_insns) { - caller_successors.emplace_back(caller_node, callee_node, - invoke_insn); - callee_edges.push_back(&caller_successors.back()); - } - bool preds_added; - concurrent_preds.update(callee_node, - [&](auto, auto& preds, bool exists) { - preds_added = !exists; - preds.emplace_back(std::move(callee_edges)); - }); - if (preds_added) { - added_preds.push_back(callee_node); + // Record all edges (we actually add them in a deterministic way later) + std::vector w; + w.reserve(callee_partitions.size()); + for (auto& callee_partition : callee_partitions) { + auto callee = callee_partition.callee; + auto& callee_invoke_insns = callee_partition.invoke_insns; + NodeId callee_node{}; + bool added{false}; + concurrent_nodes.update(callee, [&](auto, auto& n, bool exists) { + if (!exists) { + added = true; + std::lock_guard lock_guard(nodes_mutex); + n = this->make_node(callee); + } + callee_node = n; + }); + if (added) { + worker_state->push_task( + (WorkItem){callee, callee_node, /* is_root */ false}); } - } - // Schedule postprocessing of predecessors of newly-added preds. - if (!added_preds.empty()) { - std::lock_guard lock_guard(predecessors_wq_mutex); - for (auto* node : added_preds) { - predecessors_wq.add_item(node); + std::vector> edges; + edges.reserve(callee_invoke_insns.size()); + for (auto* invoke_insn : callee_invoke_insns) { + edges.push_back( + std::make_shared(caller_node, callee_node, invoke_insn)); } + concurrent_preds.update(callee_node, [&](auto, auto& v, bool) { + v.push_back((MethodEdges){caller, edges}); + }); + w.push_back((MethodEdges){callee, std::move(edges)}); } + concurrent_succs.emplace(caller_node, std::move(w)); - // Populate insn-to-callee map + // Populate concurrent_insn_to_callee for (auto&& [invoke_insn, callees] : insn_to_callee) { - m_insn_to_callee.emplace(invoke_insn, std::move(callees)); + concurrent_insn_to_callee.emplace(invoke_insn, std::move(callees)); } }, redex_parallel::default_num_threads(), /*push_tasks_while_running=*/true); - successors_wq.add_item((WorkItem){/* caller */ nullptr, m_entry.get()}); - root_nodes.reserve(root_and_dynamic.roots.size()); - for (const DexMethod* root : root_and_dynamic.roots) { - auto [root_node, emplaced] = m_nodes.emplace_unsafe(root, root); + auto root_and_dynamic = strat.get_roots(); + const auto& roots = root_and_dynamic.roots; + m_dynamic_methods = std::move(root_and_dynamic.dynamic_methods); + for (const DexMethod* root : roots) { + auto root_node = make_node(root); + auto emplaced = concurrent_nodes.emplace_unsafe(root, root_node); always_assert(emplaced); - successors_wq.add_item((WorkItem){root, root_node}); - root_nodes.emplace_back(root_node); + wq.add_item((WorkItem){root, root_node, /* is_root */ true}); + } + wq.run_all(); + + // Fill in all predecessors and successors, and sort them + auto wq2 = workqueue_foreach([&](NodeId node) { + auto linearize = [node](auto& m, auto& res) { + auto it = m.find(node); + if (it == m.end()) { + return; + } + auto& v = it->second; + std::sort(v.begin(), v.end(), [](auto& p, auto& q) { + return compare_dexmethods(p.method, q.method); + }); + for (auto& me : v) { + res.insert(res.end(), std::make_move_iterator(me.edges.begin()), + std::make_move_iterator(me.edges.end())); + } + }; + linearize(concurrent_succs, node->m_successors); + linearize(concurrent_preds, node->m_predecessors); + }); + wq2.add_item(m_entry.get()); + wq2.add_item(m_exit.get()); + for (auto&& [_, node] : m_nodes) { + wq2.add_item(node.get()); + } + wq2.run_all(); + + m_insn_to_callee = concurrent_insn_to_callee.move_to_container(); +} + +NodeId Graph::make_node(const DexMethod* m) { + auto [it, inserted] = m_nodes.emplace(m, nullptr); + if (inserted) { + it->second = std::make_shared(m); } - successors_wq.run_all(); - predecessors_wq.run_all(); + + return it->second.get(); +} + +void Graph::add_edge(const NodeId& caller, + const NodeId& callee, + IRInstruction* invoke_insn) { + auto edge = std::make_shared(caller, callee, invoke_insn); + caller->m_successors.emplace_back(edge); + callee->m_predecessors.emplace_back(std::move(edge)); +} + +MethodSet resolve_callees_in_graph(const Graph& graph, + const DexMethod* method, + const IRInstruction* insn) { + + always_assert(insn); + MethodSet ret; + for (const auto& edge_id : graph.node(method)->callees()) { + auto invoke_insn = edge_id->invoke_insn(); + if (invoke_insn == insn) { + auto callee_node_id = edge_id->callee(); + if (callee_node_id) { + auto callee = callee_node_id->method(); + if (callee) { + ret.emplace(callee); + } + } + } + } + return ret; } const MethodSet& resolve_callees_in_graph(const Graph& graph, @@ -534,21 +638,8 @@ const MethodSet& resolve_callees_in_graph(const Graph& graph, return no_methods; } -bool invoke_is_dynamic(const Graph& graph, const IRInstruction* insn) { - auto* callee = resolve_invoke_method(insn); - if (callee == nullptr) { - return true; - } - // For methods marked with DoNotInline, we also treat them like dynamic - // methods to avoid propagating return value. - if (callee->rstate.dont_inline()) { - return true; - } - if (insn->opcode() != OPCODE_INVOKE_VIRTUAL && - insn->opcode() != OPCODE_INVOKE_INTERFACE) { - return false; - } - return graph.get_dynamic_methods().count(callee); +bool method_is_dynamic(const Graph& graph, const DexMethod* method) { + return graph.get_dynamic_methods().count(method); } CallgraphStats get_num_nodes_edges(const Graph& graph) { @@ -576,21 +667,4 @@ CallgraphStats get_num_nodes_edges(const Graph& graph) { return CallgraphStats(visited_node.size(), num_edge, num_callsites); } -const MethodVector& Graph::get_callers(const DexMethod* callee) const { - return *m_callee_to_callers - .get_or_create_and_assert_equal( - callee, - [&](const DexMethod*) { - std::unordered_set set; - if (has_node(callee)) { - for (const auto& edge : node(callee)->callers()) { - set.insert(edge->caller()->method()); - } - set.erase(nullptr); - } - return MethodVector(set.begin(), set.end()); - }) - .first; -} - } // namespace call_graph diff --git a/libredex/CallGraph.h b/libredex/CallGraph.h index 14b15a43d6..5afaa0e52f 100644 --- a/libredex/CallGraph.h +++ b/libredex/CallGraph.h @@ -53,18 +53,16 @@ struct CallSite { const DexMethod* callee; IRInstruction* invoke_insn; - CallSite() { not_reached(); } CallSite(const DexMethod* callee, IRInstruction* invoke_insn) : callee(callee), invoke_insn(invoke_insn) {} }; using CallSites = std::vector; using MethodSet = std::unordered_set; -using MethodVector = std::vector; struct RootAndDynamic { - MethodSet roots; - MethodSet dynamic_methods; + std::vector roots; + std::unordered_set dynamic_methods; }; /* @@ -84,56 +82,10 @@ class BuildStrategy { }; class Edge; -using EdgeId = const Edge*; +using EdgeId = std::shared_ptr; +using Edges = std::vector>; -// This exposes a `EdgesVector` as a iterable of `const Edge*`. -class EdgesAdapter { - public: - explicit EdgesAdapter(const std::vector* edges) : m_edges(edges) {} - - class iterator { - public: - using value_type = const Edge*; - using difference_type = std::ptrdiff_t; - using pointer = const value_type*; - using reference = const value_type&; - using iterator_category = std::input_iterator_tag; - - explicit iterator(value_type current) : m_current(current) {} - - reference operator*() const { return m_current; } - - pointer operator->() const { return &(this->operator*()); } - - bool operator==(const iterator& other) const { - return m_current == other.m_current; - } - - bool operator!=(iterator& other) const { return !(*this == other); } - - iterator operator++(int) { - auto result = *this; - ++(*this); - return result; - } - - iterator& operator++(); - - private: - value_type m_current; - }; - - iterator begin() const { return iterator(&*m_edges->begin()); } - - iterator end() const { return iterator(&*m_edges->end()); } - - size_t size() const { return m_edges->size(); } - - private: - const std::vector* m_edges; -}; - -class Node final { +class Node { enum NodeType { GHOST_ENTRY, GHOST_EXIT, @@ -145,26 +97,22 @@ class Node final { explicit Node(NodeType type) : m_method(nullptr), m_type(type) {} const DexMethod* method() const { return m_method; } - const std::vector& callers() const { return m_predecessors; } - EdgesAdapter callees() const { return EdgesAdapter(&m_successors); } - - bool is_entry() const { return m_type == GHOST_ENTRY; } - bool is_exit() const { return m_type == GHOST_EXIT; } + const Edges& callers() const { return m_predecessors; } + const Edges& callees() const { return m_successors; } - bool operator==(const Node& other) const { - return m_method == other.m_method && m_type == other.m_type; - } + bool is_entry() { return m_type == GHOST_ENTRY; } + bool is_exit() { return m_type == GHOST_EXIT; } private: const DexMethod* m_method; - std::vector m_predecessors; - std::vector m_successors; + Edges m_predecessors; + Edges m_successors; NodeType m_type; friend class Graph; }; -using NodeId = const Node*; +using NodeId = Node*; class Edge { public: @@ -179,38 +127,26 @@ class Edge { IRInstruction* m_invoke_insn; }; -inline EdgesAdapter::iterator& EdgesAdapter::iterator::operator++() { - ++m_current; - return *this; -} - class Graph final { public: explicit Graph(const BuildStrategy&); - Graph(Graph&&) = default; - Graph& operator=(Graph&&) = default; - - // The call graph cannot be copied. - Graph(const Graph&) = delete; - Graph& operator=(const Graph&) = delete; - NodeId entry() const { return m_entry.get(); } NodeId exit() const { return m_exit.get(); } bool has_node(const DexMethod* m) const { - return m_nodes.count_unsafe(const_cast(m)) != 0; + return m_nodes.count(const_cast(m)) != 0; } NodeId node(const DexMethod* m) const { if (m == nullptr) { return this->entry(); } - return m_nodes.get_unsafe(m); + return m_nodes.at(m).get(); } - const InsertOnlyConcurrentMap>& + const std::unordered_map>& get_insn_to_callee() const { return m_insn_to_callee; } @@ -219,18 +155,18 @@ class Graph final { return m_dynamic_methods; } - const MethodVector& get_callers(const DexMethod* callee) const; - private: - std::unique_ptr m_entry; - std::unique_ptr m_exit; - InsertOnlyConcurrentMap m_nodes; - InsertOnlyConcurrentMap> - m_insn_to_callee; - mutable InsertOnlyConcurrentMap - m_callee_to_callers; + NodeId make_node(const DexMethod*); + + void add_edge(const NodeId& caller, + const NodeId& callee, + IRInstruction* invoke_insn); + std::shared_ptr m_entry; + std::shared_ptr m_exit; + std::unordered_map> m_nodes; + std::unordered_map> + m_insn_to_callee; // Methods that might have unknown inputs/outputs that we need special handle. // Like external methods with internal overrides, they might have external // implementation that we don't know about. Or methods that might have @@ -251,9 +187,11 @@ class SingleCalleeStrategy : public BuildStrategy { protected: bool is_definitely_virtual(DexMethod* method) const; + virtual DexMethod* resolve_callee(const DexMethod* caller, + IRInstruction* invoke) const; const Scope& m_scope; - InsertOnlyConcurrentSet m_non_virtual; + std::unordered_set m_non_virtual; }; class MultipleCalleeBaseStrategy : public SingleCalleeStrategy { @@ -265,19 +203,20 @@ class MultipleCalleeBaseStrategy : public SingleCalleeStrategy { RootAndDynamic get_roots() const override; protected: - const std::vector& - get_ordered_overriding_methods_with_code_or_native( - const DexMethod* method) const; + virtual std::vector get_additional_roots( + const MethodSet& /* existing_roots */) const { + // No additional roots by default. + return std::vector(); + } - const std::vector& - init_ordered_overriding_methods_with_code_or_native( - const DexMethod* method, std::vector) const; + const std::vector& get_ordered_overriding_methods_with_code( + const DexMethod* method) const; const method_override_graph::Graph& m_method_override_graph; private: - mutable InsertOnlyConcurrentMap> + mutable ConcurrentMap>> m_overriding_methods_cache; }; @@ -295,40 +234,44 @@ class MultipleCalleeStrategy : public MultipleCalleeBaseStrategy { const Scope& scope, uint32_t big_override_threshold); CallSites get_callsites(const DexMethod* method) const override; - RootAndDynamic get_roots() const override; protected: - ConcurrentSet m_big_virtuals; - ConcurrentSet m_big_virtual_overrides; + std::vector get_additional_roots( + const MethodSet& existing_roots) const override; + MethodSet m_big_override; }; // A static-method-only API for use with the monotonic fixpoint iterator. class GraphInterface { public: using Graph = call_graph::Graph; - using NodeId = const Node*; - using EdgeId = const Edge*; + using NodeId = Node*; + using EdgeId = std::shared_ptr; static NodeId entry(const Graph& graph) { return graph.entry(); } static NodeId exit(const Graph& graph) { return graph.exit(); } - static const std::vector& predecessors(const Graph&, - const NodeId& m) { + static const Edges& predecessors(const Graph& graph, const NodeId& m) { return m->callers(); } - static EdgesAdapter successors(const Graph&, const NodeId& m) { + static const Edges& successors(const Graph& graph, const NodeId& m) { return m->callees(); } - static NodeId source(const Graph&, const EdgeId& e) { return e->caller(); } - static NodeId target(const Graph&, const EdgeId& e) { return e->callee(); } + static NodeId source(const Graph& graph, const EdgeId& e) { + return e->caller(); + } + static NodeId target(const Graph& graph, const EdgeId& e) { + return e->callee(); + } }; +MethodSet resolve_callees_in_graph(const Graph& graph, + const DexMethod* method, + const IRInstruction* insn); + const MethodSet& resolve_callees_in_graph(const Graph& graph, const IRInstruction* insn); -const MethodVector& get_callee_to_callers(const Graph& graph, - const DexMethod* callee); - -bool invoke_is_dynamic(const Graph& graph, const IRInstruction* insn); +bool method_is_dynamic(const Graph& graph, const DexMethod* method); struct CallgraphStats { uint32_t num_nodes; diff --git a/libredex/ClassReferencesCache.cpp b/libredex/ClassReferencesCache.cpp index 973a2a5562..5cf06091a4 100644 --- a/libredex/ClassReferencesCache.cpp +++ b/libredex/ClassReferencesCache.cpp @@ -36,22 +36,26 @@ ClassReferences::ClassReferences(const DexClass* cls) { std::sort(init_types.begin(), init_types.end(), compare_dextypes); } -bool ClassReferences::operator==(const ClassReferences& other) const { - return method_refs == other.method_refs && field_refs == other.field_refs && - types == other.types && strings == other.strings && - init_types == other.init_types; -} - ClassReferencesCache::ClassReferencesCache( const std::vector& classes) { workqueue_run( - [&](DexClass* cls) { m_cache.emplace(cls, ClassReferences(cls)); }, + [&](DexClass* cls) { + m_cache.emplace(cls, std::make_shared(cls)); + }, classes); } -const ClassReferences& ClassReferencesCache::get(const DexClass* cls) const { - return *m_cache - .get_or_create_and_assert_equal( - cls, [](const auto* cls) { return ClassReferences(cls); }) - .first; +std::shared_ptr ClassReferencesCache::get( + const DexClass* cls) const { + auto res = m_cache.get(cls, nullptr); + if (res) { + return res; + } + res = std::make_shared(cls); + if (m_cache.emplace(cls, res)) { + return res; + } + res = m_cache.get(cls, nullptr); + always_assert(res); + return res; } diff --git a/libredex/ClassReferencesCache.h b/libredex/ClassReferencesCache.h index 5f05557354..bd68cad4c6 100644 --- a/libredex/ClassReferencesCache.h +++ b/libredex/ClassReferencesCache.h @@ -15,9 +15,6 @@ struct ClassReferences { explicit ClassReferences(const DexClass* cls); - - bool operator==(const ClassReferences& other) const; - std::vector method_refs; std::vector field_refs; std::vector types; @@ -28,8 +25,9 @@ struct ClassReferences { class ClassReferencesCache { public: explicit ClassReferencesCache(const std::vector& classes); - const ClassReferences& get(const DexClass* cls) const; + std::shared_ptr get(const DexClass* cls) const; private: - mutable InsertOnlyConcurrentMap m_cache; + mutable ConcurrentMap> + m_cache; }; diff --git a/libredex/ClassUtil.cpp b/libredex/ClassUtil.cpp index 7877e6f341..08718cb5c9 100644 --- a/libredex/ClassUtil.cpp +++ b/libredex/ClassUtil.cpp @@ -34,7 +34,7 @@ Serdes get_serdes(const DexClass* cls) { return Serdes(deser, flatbuf_deser, ser, flatbuf_ser); } -DexMethod* get_or_create_clinit(DexClass* cls, bool need_editable_cfg) { +DexMethod* get_or_create_clinit(DexClass* cls) { using namespace dex_asm; DexMethod* clinit = cls->get_clinit(); @@ -57,9 +57,6 @@ DexMethod* get_or_create_clinit(DexClass* cls, bool need_editable_cfg) { auto ir_code = std::make_unique(clinit, 1); ir_code->push_back(dasm(OPCODE_RETURN_VOID)); clinit->set_code(std::move(ir_code)); - if (need_editable_cfg) { - clinit->get_code()->build_cfg(); - } cls->add_method(clinit); return clinit; } diff --git a/libredex/ClassUtil.h b/libredex/ClassUtil.h index c57f05da9f..0ee4055820 100644 --- a/libredex/ClassUtil.h +++ b/libredex/ClassUtil.h @@ -39,7 +39,7 @@ Serdes get_serdes(const DexClass* cls); * Looks for a method for the given class, creates a new one if it * does not exist */ -DexMethod* get_or_create_clinit(DexClass* cls, bool need_editable_cfg = false); +DexMethod* get_or_create_clinit(DexClass* cls); /** * Return true if the parent chain leads to known classes. diff --git a/libredex/ConcurrentContainers.cpp b/libredex/ConcurrentContainers.cpp index 4d38acbea7..40546a2d40 100644 --- a/libredex/ConcurrentContainers.cpp +++ b/libredex/ConcurrentContainers.cpp @@ -11,37 +11,10 @@ namespace cc_impl { -bool is_thread_pool_active() { - return redex_thread_pool::ThreadPool::get_instance() != nullptr; -} - void workqueue_run_for(size_t start, size_t end, const std::function& fn) { ::workqueue_run_for(start, end, fn); } -size_t get_prime_number_greater_or_equal_to(size_t value) { - // We'll choose prime numbers that are roughly doubling in size, but are also - // always at least 3 less than the next power of two. This allows fitting the - // allocated Storage into a memory chunk that's just under the next power of - // two. - static std::array primes = { - 13, 29, 61, 113, 251, 509, 1021, - 2039, 4093, 8179, 16381, 32749, 65521, 131063, - 262139, 524269, 1048573, 2097143, 4194301, 8388593, 16777213, - 33554393, 67108859, 134217689, 268435399, 536870909, 1073741789, - }; - value >>= 3; - size_t idx = 0; - while (value >>= 1) { - idx++; - } - if (idx >= primes.size()) { - // Not necessarily a prime number, too bad. - return ((size_t)1 << (idx + 4)) - 1; - } - return primes[idx]; -}; - } // namespace cc_impl diff --git a/libredex/ConcurrentContainers.h b/libredex/ConcurrentContainers.h index 50ccc8e3f3..82d9ee520f 100644 --- a/libredex/ConcurrentContainers.h +++ b/libredex/ConcurrentContainers.h @@ -7,14 +7,10 @@ #pragma once -#include -#include #include #include #include -#include #include -#include #include #include #include @@ -22,910 +18,23 @@ #include "Debug.h" #include "Timer.h" +// Forward declaration. namespace cc_impl { -constexpr size_t kDefaultSlots = 83; - template class ConcurrentContainerIterator; -inline AccumulatingTimer s_destructor("cc_impl::destructor_seconds"); -inline AccumulatingTimer s_reserving("cc_impl::reserving_seconds"); +inline AccumulatingTimer s_destructor{}; + +inline double get_destructor_seconds() { return s_destructor.get_seconds(); } inline size_t s_concurrent_destruction_threshold{ std::numeric_limits::max()}; -bool is_thread_pool_active(); - void workqueue_run_for(size_t start, size_t end, const std::function& fn); -template -class ConcurrentHashtableIterator; - -template -class ConcurrentHashtableInsertionResult; - -size_t get_prime_number_greater_or_equal_to(size_t); - -/* - * This ConcurrentHashtable supports inserting (and "emplacing"), getting (the - * address of inserted key-value pairs), and erasing key-value pairs. There is - * no built-in support for mutation of previously inserted elements; however, - * once inserted, a key-value is assigned a fixed storage location that will - * remain valid until the concurrent hashtable is destroyed, or a destructive - * NOT thread-safe function such as `compact` is called. - * - * Some guiding principles for concurrency are: - * - All insertions/erasures performed on the current thread are reflected when - * calling get on the current thread. - * - Insertions/erasures performed by other threads will become visible - * eventually, but with no ordering guarantees. - * - * The concurrent hashtable has the following performance characteristics: - * - getting, inserting and erasing is O(1) on average, lock-free (*), and not - * blocked by resizing - * - resizing is O(n) on the current thread, and acquires a table-wide mutex - * - * (*) While implemented without locks, there is effectively some spinning on - * individual buckets when competing operations are in progress on that bucket. - * - * Resizing is automatically triggered when an insertion causes the table to - * exceed the (hard-coded) load factor, and then this insertion blocks the - * current thread while it is busy resizing. However, concurrent gets, - * insertions and erasures can proceed; new insertions will go into the enlarged - * table version, possibly (temporarily) exceeding the load factor. - * - * All key-value pairs are stored in a fixed memory location, and are not moved - * during resizing, similar to how std::unordered_set/map manages memory. - * Erasing a key does not immediately destroy the key-value pair, but keeps (a - * reference to) it until the concurrent hashtable is destroyed, copied, moved, - * or `compact` is called. This ensures that get always returns a valid - * reference, even in the face of concurrent erasing. - * - * TODO: Right now, we use the (default) std::memory_order_seq_cst everywhere. - * Acquire/release semantics should be sufficient. - */ -template -class ConcurrentHashtable final { - public: - using key_type = Key; - using value_type = Value; - using pointer = Value*; - using const_pointer = const Value*; - using reference = Value&; - using const_reference = const Value&; - using iterator = ConcurrentHashtableIterator< - ConcurrentHashtable>; - using const_iterator = ConcurrentHashtableIterator< - const ConcurrentHashtable>; - using hasher = Hash; - using key_equal = KeyEqual; - using insertion_result = ConcurrentHashtableInsertionResult< - ConcurrentHashtable>; - - struct const_key_projection { - template >> - const key_type2& operator()(const key_type2& key) { - return key; - } - - template >> - const key_type2& operator()(const value_type& key) { - return key.first; - } - }; - - /* - * This operation by itself is always thread-safe. However, any mutating - * operations (concurrent or synchronous) invalidate all iterators. - */ - iterator begin() { - auto* storage = m_storage.load(); - auto* ptr = storage->ptrs[0].load(); - return iterator(storage, 0, get_node(ptr)); - } - - /* - * This operation by itself is always thread-safe. However, any mutating - * operations (concurrent or synchronous) invalidate all iterators. - */ - iterator end() { - auto* storage = m_storage.load(); - return iterator(storage, storage->size, nullptr); - } - - /* - * This operation by itself is always thread-safe. However, any mutating - * operations (concurrent or synchronous) invalidate all iterators. - */ - const_iterator begin() const { - auto* storage = m_storage.load(); - auto* ptr = storage->ptrs[0].load(); - return const_iterator(storage, 0, get_node(ptr)); - } - - /* - * This operation by itself is always thread-safe. However, any mutating - * operations (concurrent or synchronous) invalidate all iterators. - */ - const_iterator end() const { - auto* storage = m_storage.load(); - return const_iterator(storage, storage->size, nullptr); - } - - /* - * This operation by itself is always thread-safe. However, any mutating - * operations (concurrent or synchronous) invalidate all iterators. - */ - iterator find(const key_type& key) { - auto hash = hasher()(key); - auto* storage = m_storage.load(); - auto* ptrs = storage->ptrs; - size_t i = hash % storage->size; - auto* root_loc = &ptrs[i]; - auto* root = root_loc->load(); - for (auto* ptr = root; ptr;) { - auto* node = get_node(ptr); - if (key_equal()(const_key_projection()(node->value), key)) { - return iterator(storage, i, node); - } - ptr = node->prev.load(); - } - return end(); - } - - /* - * This operation by itself is always thread-safe. However, any mutating - * operations (concurrent or synchronous) invalidate all iterators. - */ - const_iterator find(const key_type& key) const { - auto hash = hasher()(key); - auto* storage = m_storage.load(); - auto* ptrs = storage->ptrs; - size_t i = hash % storage->size; - auto* root_loc = &ptrs[i]; - auto* root = root_loc->load(); - for (auto* ptr = root; ptr;) { - auto* node = get_node(ptr); - if (key_equal()(const_key_projection()(node->value), key)) { - return const_iterator(storage, i, node); - } - ptr = node->prev.load(); - } - return end(); - } - - /* - * This operation is always thread-safe. - */ - size_t size() const { return m_count.load(); } - - /* - * This operation is always thread-safe. - */ - bool empty() const { return size() == 0; } - - /* - * This operation is NOT thread-safe. - */ - void clear(size_t size = INITIAL_SIZE) { - if (m_count.load() > 0) { - Storage::destroy(m_storage.exchange(Storage::create(size, nullptr))); - m_count.store(0); - } - compact(); - } - - ConcurrentHashtable() noexcept - : m_storage(Storage::create()), m_count(0), m_erased(nullptr) {} - - /* - * This operation is NOT thread-safe. - */ - ConcurrentHashtable(const ConcurrentHashtable& container) noexcept - : m_storage(Storage::create(container.size() / LOAD_FACTOR + 1, nullptr)), - m_count(0), - m_erased(nullptr) { - for (const auto& p : container) { - try_insert(p); - } - } - - /* - * This operation is NOT thread-safe. - */ - ConcurrentHashtable(ConcurrentHashtable&& container) noexcept - : m_storage(container.m_storage.exchange(Storage::create())), - m_count(container.m_count.exchange(0)), - m_erased(container.m_erased.exchange(nullptr)) { - compact(); - } - - /* - * This operation is NOT thread-safe. - */ - ConcurrentHashtable& operator=(ConcurrentHashtable&& container) noexcept { - clear(); - container.compact(); - m_storage.store(container.m_storage.exchange(m_storage.load())); - m_count.store(container.m_count.exchange(0)); - return *this; - } - - /* - * This operation is NOT thread-safe. - */ - ConcurrentHashtable& operator=( - const ConcurrentHashtable& container) noexcept { - if (this != &container) { - clear(container.size() / LOAD_FACTOR + 1); - for (const auto& p : container) { - try_insert(p); - } - } - return *this; - } - - /* - * This operation releases all memory and leaves behind the object in an - * uninitialized state. - */ - void destroy() { - Storage::destroy(m_storage.exchange(nullptr)); - m_count.store(0); - process_erased(); - } - - ~ConcurrentHashtable() { destroy(); } - - /* - * This operation is always thread-safe. - */ - value_type* get(const key_type& key) { - auto hash = hasher()(key); - auto* storage = m_storage.load(); - do { - auto* ptrs = storage->ptrs; - auto* root_loc = &ptrs[hash % storage->size]; - auto* root = root_loc->load(); - for (auto* node = get_node(root); node; - node = get_node(node->prev.load())) { - if (key_equal()(const_key_projection()(node->value), key)) { - return &node->value; - } - } - storage = storage->next.load(); - } while (storage); - return nullptr; - } - - /* - * This operation is always thread-safe. - */ - const value_type* get(const key_type& key) const { - auto hash = hasher()(key); - auto* storage = m_storage.load(); - do { - auto* ptrs = storage->ptrs; - auto* root_loc = &ptrs[hash % storage->size]; - auto* root = root_loc->load(); - for (auto* node = get_node(root); node; - node = get_node(node->prev.load())) { - if (key_equal()(const_key_projection()(node->value), key)) { - return &node->value; - } - } - storage = storage->next.load(); - } while (storage); - return nullptr; - } - - /* - * This operation is always thread-safe. - */ - template - insertion_result try_emplace(const key_type& key, Args&&... args) { - Node* new_node = nullptr; - auto hash = hasher()(key); - auto* storage = m_storage.load(); - while (true) { - auto* ptrs = storage->ptrs; - auto* root_loc = &ptrs[hash % storage->size]; - auto* root = root_loc->load(); - for (auto* node = get_node(root); node; - node = get_node(node->prev.load())) { - if (key_equal()(const_key_projection()(node->value), key)) { - return insertion_result(&node->value, new_node); - } - } - if (is_moved_or_locked(root)) { - if (auto* next_storage = storage->next.load()) { - storage = next_storage; - continue; - } - // We are racing with an erasure; assume it's not affecting us. - root = get_node(root); - } - if (load_factor_exceeded(storage) && reserve(storage->size * 2)) { - storage = m_storage.load(); - continue; - } - if (!new_node) { - new_node = - new Node(ConstRefKeyArgsTag(), key, std::forward(args)...); - } - new_node->prev = root; - if (root_loc->compare_exchange_strong(root, new_node)) { - m_count.fetch_add(1); - return insertion_result(&new_node->value); - } - // We lost a race with another insertion - } - } - - /* - * This operation is always thread-safe. - */ - template - insertion_result try_emplace(key_type&& key, Args&&... args) { - Node* new_node = nullptr; - auto hash = hasher()(key); - auto* storage = m_storage.load(); - const key_type* key_ptr = &key; - while (true) { - auto* ptrs = storage->ptrs; - auto* root_loc = &ptrs[hash % storage->size]; - auto* root = root_loc->load(); - for (auto* node = get_node(root); node; - node = get_node(node->prev.load())) { - if (key_equal()(const_key_projection()(node->value), *key_ptr)) { - return insertion_result(&node->value, new_node); - } - } - if (is_moved_or_locked(root)) { - if (auto* next_storage = storage->next.load()) { - storage = next_storage; - continue; - } - // We are racing with an erasure; assume it's not affecting us. - root = get_node(root); - } - if (load_factor_exceeded(storage) && reserve(storage->size * 2)) { - storage = m_storage.load(); - continue; - } - if (!new_node) { - new_node = new Node(RvalueRefKeyArgsTag(), std::forward(key), - std::forward(args)...); - key_ptr = &const_key_projection()(new_node->value); - } - new_node->prev = root; - if (root_loc->compare_exchange_strong(root, new_node)) { - m_count.fetch_add(1); - return insertion_result(&new_node->value); - } - // We lost a race with another insertion - } - } - - /* - * This operation is always thread-safe. - */ - insertion_result try_insert(const value_type& value) { - Node* new_node = nullptr; - auto hash = hasher()(const_key_projection()(value)); - auto* storage = m_storage.load(); - while (true) { - auto* ptrs = storage->ptrs; - auto* root_loc = &ptrs[hash % storage->size]; - auto* root = root_loc->load(); - for (auto* node = get_node(root); node; - node = get_node(node->prev.load())) { - if (key_equal()(const_key_projection()(node->value), - const_key_projection()(value))) { - return insertion_result(&node->value, new_node); - } - } - if (is_moved_or_locked(root)) { - if (auto* next_storage = storage->next.load()) { - storage = next_storage; - continue; - } - // We are racing with an erasure; assume it's not affecting us. - root = get_node(root); - } - if (load_factor_exceeded(storage) && reserve(storage->size * 2)) { - storage = m_storage.load(); - continue; - } - if (!new_node) { - new_node = new Node(ConstRefValueTag(), value); - } - new_node->prev = root; - if (root_loc->compare_exchange_strong(root, new_node)) { - m_count.fetch_add(1); - return insertion_result(&new_node->value); - } - // We lost a race with another insertion - } - } - - /* - * This operation is always thread-safe. - */ - insertion_result try_insert(value_type&& value) { - Node* new_node = nullptr; - auto hash = hasher()(const_key_projection()(value)); - auto* storage = m_storage.load(); - auto* value_ptr = &value; - while (true) { - auto* ptrs = storage->ptrs; - auto* root_loc = &ptrs[hash % storage->size]; - auto* root = root_loc->load(); - for (auto* node = get_node(root); node; - node = get_node(node->prev.load())) { - if (key_equal()(const_key_projection()(node->value), - const_key_projection()(*value_ptr))) { - // We lost a race with an equivalent insertion - return insertion_result(&node->value, new_node); - } - } - if (is_moved_or_locked(root)) { - if (auto* next_storage = storage->next.load()) { - storage = next_storage; - continue; - } - // We are racing with an erasure; assume it's not affecting us. - root = get_node(root); - } - if (load_factor_exceeded(storage) && reserve(storage->size * 2)) { - storage = m_storage.load(); - continue; - } - if (!new_node) { - new_node = - new Node(RvalueRefValueTag(), std::forward(value)); - value_ptr = &new_node->value; - } - new_node->prev = root; - if (root_loc->compare_exchange_strong(root, new_node)) { - m_count.fetch_add(1); - return insertion_result(&new_node->value); - } - // We lost a race with another insertion - } - } - - /* - * This operation is always thread-safe. - */ - bool reserve(size_t capacity) { - bool resizing = false; - if (!m_resizing.compare_exchange_strong(resizing, true)) { - return false; - } - auto* storage = m_storage.load(); - if (storage->size >= capacity) { - m_resizing.store(false); - return true; - } - auto timer_scope = s_reserving.scope(); - auto new_capacity = get_prime_number_greater_or_equal_to(capacity); - auto* ptrs = storage->ptrs; - auto* new_storage = Storage::create(new_capacity, storage); - storage->next.store(new_storage); - std::stack*> locs; - for (size_t i = 0; i < storage->size; ++i) { - std::atomic* loc = &ptrs[i]; - // Lock the bucket (or mark the bucket as moved if its empty). This might - // fail due to a race with an insertion or erasure - Ptr ptr = nullptr; - Node* node = nullptr; - while (!loc->compare_exchange_strong(ptr, moved_or_lock(node))) { - node = get_node(ptr); - ptr = node; - } - if (node == nullptr) { - continue; - } - // Lets rewire the nodes from the back to the new storage version. - locs.push(loc); - auto* prev_loc = &node->prev; - auto* prev_ptr = prev_loc->load(); - while (prev_ptr) { - loc = prev_loc; - locs.push(loc); - ptr = prev_ptr; - node = get_node(ptr); - prev_loc = &node->prev; - prev_ptr = prev_loc->load(); - } - while (!locs.empty()) { - loc = locs.top(); - locs.pop(); - ptr = loc->load(); - node = get_node(ptr); - prev_loc = &node->prev; - prev_ptr = prev_loc->load(); - always_assert(prev_ptr == nullptr || is_moved_or_locked(prev_ptr)); - auto new_hash = hasher()(const_key_projection()(node->value)); - auto* new_loc = &new_storage->ptrs[new_hash % new_storage->size]; - auto* new_ptr = new_loc->load(); - // Rewiring the node happens in three steps: - do { - // Assume there is no race with an erasure. - new_ptr = get_node(new_ptr); - // 1. Set the (null) prev node pointer to the first chain element in - // the new storage version. This is ultimately what we want it to be; - // it might allow a racing read operation to scan irrelevant nodes, - // but that is not a problem for correctness. - prev_loc->store(new_ptr); - // 2. Wire up the current node pointer to be the first chain element - // in the new storage version. This may fail due to a race with - // another thread inserting into or erasing from the same chain. But - // then we'll just retry. - } while (!new_loc->compare_exchange_strong(new_ptr, node)); - // 3. Detach the current node pointer from the end of the old chain. - loc->store(moved()); - } - } - auto* old_storage = m_storage.exchange(new_storage); - always_assert(old_storage == storage); - m_resizing.store(false); - return true; - } - - /* - * This operation is always thread-safe. - */ - value_type* erase(const key_type& key) { - auto hash = hasher()(key); - auto* storage = m_storage.load(); - while (true) { - auto* ptrs = storage->ptrs; - auto* root_loc = &ptrs[hash % storage->size]; - auto* root = root_loc->load(); - if (root == nullptr) { - return nullptr; - } - if (root == moved()) { - storage = storage->next.load(); - continue; - } - // The chain is not empty. Try to lock the bucket. This might fail due - // to a race with an insertion, erasure, or resizing. - auto* node = get_node(root); - always_assert(node); - root = node; - if (!root_loc->compare_exchange_strong(root, lock(node))) { - continue; - } - auto* loc = root_loc; - for (; node && !key_equal()(const_key_projection()(node->value), key); - loc = &node->prev, node = get_node(loc->load())) { - } - if (node) { - // Erase node. - loc->store(node->prev.load()); - m_count.fetch_sub(1); - // Store erased node for later actual deletion. - auto* erased = new Erased{node, nullptr}; - while (!m_erased.compare_exchange_strong(erased->prev, erased)) { - } - } - if (loc != root_loc) { - // Unlock root node (as we didn't erase it). - root_loc->store(root); - } - return node ? &node->value : nullptr; - } - } - - /* - * This operation is NOT thread-safe. - */ - void compact() { - process_erased(); - auto* storage = m_storage.load(); - always_assert(storage->next.load() == nullptr); - Storage* prev_storage = nullptr; - std::swap(storage->prev, prev_storage); - Storage::destroy(prev_storage); - } - - private: - static constexpr float LOAD_FACTOR = 0.75; - static constexpr size_t INITIAL_SIZE = 5; - - // We store Node pointers as tagged values, to indicate, and be able to - // atomically update, whether a location where a node pointer is stored is - // currently involved in an erasure or resizing operation. - using Ptr = void*; - static const size_t MOVED_OR_LOCKED = 1; - using PtrPlusBits = boost::intrusive::pointer_plus_bits; - - struct ConstRefValueTag {}; - struct RvalueRefValueTag {}; - struct ConstRefKeyArgsTag {}; - struct RvalueRefKeyArgsTag {}; - - struct Node { - value_type value; - std::atomic prev{nullptr}; - - explicit Node(ConstRefValueTag, const value_type& value) : value(value) {} - - explicit Node(RvalueRefValueTag, value_type&& value) - : value(std::forward(value)) {} - - template >> - explicit Node(ConstRefKeyArgsTag, const key_type2& key) : value(key) {} - - template >> - explicit Node(RvalueRefKeyArgsTag, key_type2&& key) - : value(std::forward(key)) {} - - template >, - typename... Args> - explicit Node(ConstRefKeyArgsTag, const key_type2& key, Args&&... args) - : value(std::piecewise_construct, - std::forward_as_tuple(key), - std::forward_as_tuple(std::forward(args)...)) {} - - template >, - typename... Args> - explicit Node(RvalueRefKeyArgsTag, key_type2&& key, Args&&... args) - : value(std::piecewise_construct, - std::forward_as_tuple(std::forward(key)), - std::forward_as_tuple(std::forward(args)...)) {} - }; - - // Initially, and every time we resize, a new Storage version is created. - struct Storage { - size_t size; - Storage* prev; - std::atomic next; - std::atomic ptrs[1]; - - // Only create instances via `create`. - Storage() = delete; - - static Storage* create(size_t size, Storage* prev) { - always_assert(size > 0); - size_t bytes = sizeof(Storage) + sizeof(std::atomic) * (size - 1); - always_assert(bytes % sizeof(size_t) == 0); - auto* storage = (Storage*)calloc(bytes / sizeof(size_t), sizeof(size_t)); - always_assert(storage); - always_assert(storage->prev == nullptr); - storage->size = size; - storage->prev = prev; - return storage; - } - - static Storage* create() { return create(INITIAL_SIZE, nullptr); } - - static void destroy(Storage* t) { - for (auto* s = t; s; s = t) { - if (s->next.load() == nullptr) { - for (size_t i = 0; i < s->size; i++) { - auto* loc = &s->ptrs[i]; - auto* ptr = loc->load(); - for (auto* node = get_node(ptr); node; node = get_node(ptr)) { - ptr = node->prev.load(); - delete node; - } - } - } - t = s->prev; - free(s); - } - } - }; - - std::atomic m_storage; - std::atomic m_count; - std::atomic m_resizing{false}; - struct Erased { - Node* node; - Erased* prev; - }; - std::atomic m_erased; - - bool load_factor_exceeded(const Storage* storage) const { - return m_count.load() > storage->size * LOAD_FACTOR; - } - - // Whether more elements can be found in the next Storage version, or if an - // erasure is ongoing. - static bool is_moved_or_locked(Ptr ptr) { - return PtrPlusBits::get_bits(ptr) != 0; - } - - static Node* get_node(Ptr ptr) { - return static_cast(PtrPlusBits::get_pointer(ptr)); - } - - // Creates a tagged `Ptr` indicating that this is a sentinel due to resizing; - // if so, additional nodes may be found in the next Storage version. - static Ptr moved() { - Ptr ptr = nullptr; - PtrPlusBits::set_bits(ptr, MOVED_OR_LOCKED); - return ptr; - } - - // Creates a tagged root node indicating that the node chain is in the process - // of being resized or part of it is being erased. If there is a next Storage - // version, any additional nodes must go to it. - static Ptr lock(Node* node) { - always_assert(node); - Ptr ptr = node; - PtrPlusBits::set_bits(ptr, MOVED_OR_LOCKED); - return ptr; - } - - // Creates a tagged `Ptr` either indicating that the bucket moved if the node - // is absent, or locking the given node. - static Ptr moved_or_lock(Node* node) { - Ptr ptr = node; - PtrPlusBits::set_bits(ptr, MOVED_OR_LOCKED); - return ptr; - } - - void process_erased() { - for (auto* erased = m_erased.load(); erased != nullptr;) { - delete erased->node; - auto* prev = erased->prev; - delete erased; - erased = prev; - } - m_erased.store(nullptr); - } - - friend class ConcurrentHashtableIterator< - ConcurrentHashtable>; - friend class ConcurrentHashtableIterator< - const ConcurrentHashtable>; - friend class ConcurrentHashtableInsertionResult< - ConcurrentHashtable>; -}; - -/* - * Helper class to represent result of an (attmpted) insertion. What's - * interesting is that even when insertion fails, because a value with the same - * key is already present in the hashtable, a new value might have been - * incidentally constructed, possibly moving the supplied arguments in the - * process. This result value captures such an incidentally created value, and - * allows checking for equality with the stored value. - */ -template -class ConcurrentHashtableInsertionResult final { - using value_type = typename ConcurrentHashtable::value_type; - using Node = typename ConcurrentHashtable::Node; - std::unique_ptr m_node; - explicit ConcurrentHashtableInsertionResult(value_type* stored_value_ptr) - : stored_value_ptr(stored_value_ptr), success(true) {} - ConcurrentHashtableInsertionResult(value_type* stored_value_ptr, Node* node) - : m_node(node), stored_value_ptr(stored_value_ptr), success(false) {} - - public: - value_type* stored_value_ptr; - bool success; - - value_type* incidentally_constructed_value() const { - return m_node ? &m_node->value : nullptr; - } - - friend ConcurrentHashtable; -}; - -template -class ConcurrentHashtableIterator final { - public: - using difference_type = std::ptrdiff_t; - using value_type = typename ConcurrentHashtable::value_type; - using pointer = - std::conditional_t::value, - typename ConcurrentHashtable::const_pointer, - typename ConcurrentHashtable::pointer>; - using const_pointer = typename ConcurrentHashtable::const_pointer; - using reference = - std::conditional_t::value, - typename ConcurrentHashtable::const_reference, - typename ConcurrentHashtable::reference>; - using const_reference = typename ConcurrentHashtable::const_reference; - using iterator_category = std::forward_iterator_tag; - - private: - using Storage = typename ConcurrentHashtable::Storage; - using Node = typename ConcurrentHashtable::Node; - - Storage* m_storage; - size_t m_index; - Node* m_node; - - bool is_end() const { return m_index == m_storage->size; } - - void advance() { - if (m_node) { - m_node = ConcurrentHashtable::get_node(m_node->prev.load()); - if (m_node) { - return; - } - } - do { - if (++m_index == m_storage->size) { - return; - } - m_node = ConcurrentHashtable::get_node(m_storage->ptrs[m_index].load()); - } while (!m_node); - } - - ConcurrentHashtableIterator(Storage* storage, size_t index, Node* node) - : m_storage(storage), m_index(index), m_node(node) { - if (!node && index < storage->size) { - advance(); - } - } - - public: - ConcurrentHashtableIterator& operator++() { - always_assert(!is_end()); - advance(); - return *this; - } - - ConcurrentHashtableIterator& operator++(int) { - ConcurrentHashtableIterator retval = *this; - ++(*this); - return retval; - } - - bool operator==(const ConcurrentHashtableIterator& other) const { - return m_storage == other.m_storage && m_index == other.m_index && - m_node == other.m_node; - } - - bool operator!=(const ConcurrentHashtableIterator& other) const { - return !(*this == other); - } - - reference operator*() { - always_assert(!is_end()); - return m_node->value; - } - - pointer operator->() { - always_assert(!is_end()); - return &m_node->value; - } - - const_reference operator*() const { - always_assert(!is_end()); - return m_node->value; - } - - const_pointer operator->() const { - always_assert(!is_end()); - return &m_node->value; - } - - friend ConcurrentHashtable; -}; - } // namespace cc_impl // Use this scope at the top-level function of your application to allow for @@ -949,14 +58,15 @@ class ConcurrentContainerConcurrentDestructionScope { /* * This class implements the common functionalities of concurrent sets and maps. - * A concurrent container is a collection of a ConcurrentHashtable - * (providing functionality similar yo unordered_map/unordered_set) arranged in - * slots. Whenever a thread performs a concurrent operation on an element, the - * slot is uniquely determined by the hash code of the element. A sharded lock - * is obtained if the operation in question cannot be performed lock-free. A - * high number of slots may help reduce thread contention at the expense of a - * larger memory footprint. It is advised to use a prime number for `n_slots`, - * so as to ensure a more even spread of elements across slots. + * A concurrent container is just a collection of STL hash maps/sets + * (unordered_map/unordered_set) arranged in slots. Whenever a thread performs a + * concurrent operation on an element, the slot uniquely determined by the hash + * code of the element is locked and the corresponding operation is performed on + * the underlying STL container. This is a very simple design, which offers + * reasonable performance in practice. A high number of slots may help reduce + * thread contention at the expense of a larger memory footprint. It is advised + * to use a prime number for `n_slots`, so as to ensure a more even spread of + * elements across slots. * * There are two major modes in which a concurrent container is thread-safe: * - Read only: multiple threads access the contents of the container but do @@ -966,36 +76,26 @@ class ConcurrentContainerConcurrentDestructionScope { * The few operations that are thread-safe regardless of the access mode are * documented as such. */ -template +template class ConcurrentContainer { public: static_assert(n_slots > 0, "The concurrent container has no slots"); - using Key = typename Container::key_type; - using Hash = typename Container::hasher; - using KeyEqual = typename Container::key_equal; - using Value = typename Container::value_type; - - using ConcurrentHashtable = - cc_impl::ConcurrentHashtable; - - using iterator = - cc_impl::ConcurrentContainerIterator; + using iterator = cc_impl::ConcurrentContainerIterator; using const_iterator = - cc_impl::ConcurrentContainerIterator; + cc_impl::ConcurrentContainerIterator; virtual ~ConcurrentContainer() { auto timer_scope = cc_impl::s_destructor.scope(); - if (!cc_impl::is_thread_pool_active() || - size() <= cc_impl::s_concurrent_destruction_threshold) { + if (size() <= cc_impl::s_concurrent_destruction_threshold) { for (size_t slot = 0; slot < n_slots; ++slot) { - m_slots[slot].destroy(); + m_slots[slot] = Container(); } return; } cc_impl::workqueue_run_for( - 0, n_slots, [this](size_t slot) { m_slots[slot].destroy(); }); + 0, n_slots, [this](size_t slot) { m_slots[slot] = Container(); }); } /* @@ -1023,980 +123,387 @@ class ConcurrentContainer { if (it == m_slots[slot].end()) { return end(); } - return iterator(&m_slots[0], slot, it); - } - - const_iterator find(const Key& key) const { - size_t slot = Hash()(key) % n_slots; - const auto& it = m_slots[slot].find(key); - if (it == m_slots[slot].end()) { - return end(); - } - return const_iterator(&m_slots[0], slot, it); - } - - /* - * This operation is always thread-safe. - */ - size_t size() const { - size_t s = 0; - for (size_t slot = 0; slot < n_slots; ++slot) { - s += m_slots[slot].size(); - } - return s; - } - - /* - * This operation is always thread-safe. - */ - bool empty() const { - for (size_t slot = 0; slot < n_slots; ++slot) { - if (!m_slots[slot].empty()) { - return false; - } - } - return true; - } - - /* - * This operation is always thread-safe. - */ - void reserve(size_t capacity) { - size_t slot_capacity = capacity / n_slots; - if (slot_capacity > 0) { - for (size_t i = 0; i < n_slots; ++i) { - m_slots[i].reserve(slot_capacity); - } - } - } - - void clear() { - for (size_t slot = 0; slot < n_slots; ++slot) { - m_slots[slot].clear(); - } - } - - void compact() { - for (size_t slot = 0; slot < n_slots; ++slot) { - m_slots[slot].compact(); - } - } - - /* - * This operation is always thread-safe. - */ - size_t count(const Key& key) const { - size_t slot = Hash()(key) % n_slots; - return m_slots[slot].get(key) != nullptr; - } - - size_t count_unsafe(const Key& key) const { return count(key); } - - /* - * This operation is always thread-safe. - */ - size_t erase(const Key& key) { - size_t slot = Hash()(key) % n_slots; - return m_slots[slot].erase(key) ? 1 : 0; - } - - size_t erase_unsafe(const Key& key) { return erase(key); } - - protected: - // Only derived classes may be instantiated or copied. - ConcurrentContainer() = default; - - ConcurrentContainer(const ConcurrentContainer& container) { - for (size_t i = 0; i < n_slots; ++i) { - m_slots[i] = container.m_slots[i]; - } - } - - ConcurrentContainer(ConcurrentContainer&& container) noexcept { - for (size_t i = 0; i < n_slots; ++i) { - m_slots[i] = std::move(container.m_slots[i]); - } - } - - ConcurrentContainer& operator=(ConcurrentContainer&& container) noexcept { - for (size_t i = 0; i < n_slots; ++i) { - m_slots[i] = std::move(container.m_slots[i]); - } - return *this; - } - - ConcurrentContainer& operator=( - const ConcurrentContainer& container) noexcept { - if (this != &container) { - for (size_t i = 0; i < n_slots; ++i) { - m_slots[i] = container.m_slots[i]; - } - } - return *this; - } - - ConcurrentHashtable& get_container(size_t slot) { return m_slots[slot]; } - - const ConcurrentHashtable& get_container(size_t slot) const { - return m_slots[slot]; - } - - ConcurrentHashtable m_slots[n_slots]; -}; - -/* - * A concurrent container with map semantics, also allowing erasing and updating - * values. - * - * Compared to other concurrent datatypes, the ConcurrentMap has additional - * overhead: It maintains another set mutexes, one per slot, and even reading a - * value safely requires aquiring a lock. - * - * Prefer using an InsertOnlyConcurrentMap or an AtomicMap when possibly: - * - InsertOnlyConcurrentMap more clearly conveys the possible intent of an - * insertion-only map whose elements cannot be mutated, and it allows safely - * reading values without requiring copying them under a lock. - * - AtomicMap typically allows for lock free reads and writes, and even - * supports all common lock-free atomic mutations typically found on - * std::atomic<>. - */ -template , - typename KeyEqual = std::equal_to, - size_t n_slots = cc_impl::kDefaultSlots> -class ConcurrentMap final - : public ConcurrentContainer, - n_slots> { - public: - using Base = - ConcurrentContainer, - n_slots>; - - using typename Base::const_iterator; - using typename Base::iterator; - - using Base::end; - using Base::m_slots; - - using KeyValuePair = typename Base::Value; - - ConcurrentMap() = default; - - ConcurrentMap(const ConcurrentMap& container) noexcept : Base(container) {} - - ConcurrentMap(ConcurrentMap&& container) noexcept - : Base(std::move(container)) {} - - ConcurrentMap& operator=(ConcurrentMap&& container) noexcept { - Base::operator=(std::move(container)); - return *this; - } - - ConcurrentMap& operator=(const ConcurrentMap& container) noexcept { - if (this != &container) { - Base::operator=(container); - } - return *this; - } - - template - ConcurrentMap(InputIt first, InputIt last) { - insert(first, last); - } - - /* - * This operation is always thread-safe. Note that it returns a copy of Value - * rather than a reference since erasing or updating from other threads may - * cause a stored value to become invalid. If you are reading from a - * ConcurrentMap that is not being concurrently modified, it will probably be - * faster to use `find()` or `at_unsafe()` to avoid the copy. - */ - Value at(const Key& key) const { - size_t slot = Hash()(key) % n_slots; - const auto& map = this->get_container(slot); - const auto* ptr = map.get(key); - if (ptr == nullptr) { - throw std::out_of_range("at"); - } - std::unique_lock lock(this->get_lock_by_slot(slot)); - return ptr->second; - } - - const Value& at_unsafe(const Key& key) const { - size_t slot = Hash()(key) % n_slots; - const auto& map = this->get_container(slot); - const auto* ptr = map.get(key); - if (ptr == nullptr) { - throw std::out_of_range("at_unsafe"); - } - return ptr->second; - } - - Value& at_unsafe(const Key& key) { - size_t slot = Hash()(key) % n_slots; - auto& map = this->get_container(slot); - auto* ptr = map.get(key); - if (ptr == nullptr) { - throw std::out_of_range("at_unsafe"); - } - return ptr->second; - } - - /* - * This operation is always thread-safe. - */ - Value get(const Key& key, Value default_value) const { - size_t slot = Hash()(key) % n_slots; - const auto& map = this->get_container(slot); - const auto* ptr = map.get(key); - if (!ptr) { - return default_value; - } - std::unique_lock lock(this->get_lock_by_slot(slot)); - return ptr->second; - } - - Value* get_unsafe(const Key& key) { - size_t slot = Hash()(key) % n_slots; - auto& map = this->get_container(slot); - auto* ptr = map.get(key); - return ptr ? &ptr->second : nullptr; - } - - const Value* get_unsafe(const Key& key) const { - size_t slot = Hash()(key) % n_slots; - const auto& map = this->get_container(slot); - const auto* ptr = map.get(key); - return ptr ? &ptr->second : nullptr; - } - - /* - * The Boolean return value denotes whether the insertion took place. - * This operation is always thread-safe. - * - * Note that while the STL containers' insert() methods return both an - * iterator and a boolean success value, we only return the boolean value - * here as any operations on a returned iterator are not guaranteed to be - * thread-safe. - */ - bool insert(const KeyValuePair& entry) { - size_t slot = Hash()(entry.first) % n_slots; - auto& map = this->get_container(slot); - return map.try_insert(entry).success; - } - - bool insert(KeyValuePair&& entry) { - size_t slot = Hash()(entry.first) % n_slots; - auto& map = this->get_container(slot); - return map.try_insert(std::forward(entry)).success; - } - - /* - * This operation is always thread-safe. - */ - void insert(std::initializer_list l) { - for (const auto& entry : l) { - insert(entry); - } - } - - /* - * This operation is always thread-safe. - */ - template - void insert(InputIt first, InputIt last) { - for (; first != last; ++first) { - insert(*first); - } - } - - /* - * This operation is always thread-safe. - */ - void insert_or_assign(const KeyValuePair& entry) { - size_t slot = Hash()(entry.first) % n_slots; - auto& map = this->get_container(slot); - auto insertion_result = map.try_insert(entry); - if (insertion_result.success) { - return; - } - auto* constructed_value = insertion_result.incidentally_constructed_value(); - std::unique_lock lock(this->get_lock_by_slot(slot)); - if (constructed_value) { - insertion_result.stored_value_ptr->second = - std::move(constructed_value->second); - } else { - insertion_result.stored_value_ptr->second = entry.second; - } - } - - void insert_or_assign(KeyValuePair&& entry) { - size_t slot = Hash()(entry.first) % n_slots; - auto& map = this->get_container(slot); - auto insertion_result = map.try_insert(std::forward(entry)); - if (insertion_result.success) { - return; - } - auto* constructed_value = insertion_result.incidentally_constructed_value(); - std::unique_lock lock(this->get_lock_by_slot(slot)); - if (constructed_value) { - insertion_result.stored_value_ptr->second = - std::move(constructed_value->second); - } else { - insertion_result.stored_value_ptr->second = - std::forward(entry.second); - } - } - - /* - * This operation is always thread-safe. - */ - template - bool emplace(const Key& key, Args&&... args) { - size_t slot = Hash()(key) % n_slots; - auto& map = this->get_container(slot); - return map.try_emplace(key, std::forward(args)...).success; - } - - template - bool emplace(Key&& key, Args&&... args) { - size_t slot = Hash()(key) % n_slots; - auto& map = this->get_container(slot); - return map.try_emplace(std::forward(key), std::forward(args)...) - .success; - } - - template - std::pair emplace_unsafe(const Key& key, Args&&... args) { - size_t slot = Hash()(key) % n_slots; - auto& map = this->get_container(slot); - auto insertion_result = map.try_emplace(key, std::forward(args)...); - return std::make_pair(&insertion_result.stored_value_ptr->second, - insertion_result.success); - } - - template - std::pair emplace_unsafe(Key&& key, Args&&... args) { - size_t slot = Hash()(key) % n_slots; - auto& map = this->get_container(slot); - auto insertion_result = - map.try_emplace(std::forward(key), std::forward(args)...); - return std::make_pair(&insertion_result.stored_value_ptr->second, - insertion_result.success); - } - - /* - * This operation atomically observes an entry in the map if it exists. - */ - template < - typename ObserveFn = const std::function&> - bool observe(const Key& key, ObserveFn observer) const { - size_t slot = Hash()(key) % n_slots; - const auto& map = this->get_container(slot); - const auto* ptr = map.get(key); - if (!ptr) { - return false; - } - std::unique_lock lock(this->get_lock_by_slot(slot)); - observer(ptr->first, ptr->second); - return true; - } - - /* - * This operation atomically modifies an entry in the map. If the entry - * doesn't exist, it is created. The third argument of the updater function is - * a Boolean flag denoting whether the entry exists or not. - */ - template < - typename UpdateFn = const std::function&> - void update(const Key& key, UpdateFn updater) { - size_t slot = Hash()(key) % n_slots; - auto& map = this->get_container(slot); - std::unique_lock lock(this->get_lock_by_slot(slot)); - auto insertion_result = map.try_emplace(key); - auto* ptr = insertion_result.stored_value_ptr; - updater(ptr->first, ptr->second, !insertion_result.success); - } - - template < - typename UpdateFn = const std::function&> - void update(Key&& key, UpdateFn updater) { - size_t slot = Hash()(key) % n_slots; - auto& map = this->get_container(slot); - std::unique_lock lock(this->get_lock_by_slot(slot)); - auto insertion_result = map.try_emplace(std::forward(key)); - auto* ptr = insertion_result.stored_value_ptr; - updater(ptr->first, ptr->second, !insertion_result.success); - } - - template < - typename UpdateFn = const std::function&> - void update_unsafe(const Key& key, UpdateFn updater) { - size_t slot = Hash()(key) % n_slots; - auto& map = this->get_container(slot); - auto insertion_result = map.try_emplace(key); - auto* ptr = insertion_result.stored_value_ptr; - updater(ptr->first, ptr->second, !insertion_result.success); - } - - template < - typename UpdateFn = const std::function&> - void update_unsafe(Key&& key, UpdateFn updater) { - size_t slot = Hash()(key) % n_slots; - auto& map = this->get_container(slot); - auto insertion_result = map.try_emplace(std::forward(key)); - auto* ptr = insertion_result.stored_value_ptr; - updater(ptr->first, ptr->second, !insertion_result.success); - } - - /* - * This operation is always thread-safe. If the key exists, a non-null pointer - * to the corresponding value is returned. This pointer is valid until this - * container is destroyed, or the compact function is called. Note that there - * may be other threads that are also reading or even updating the value. If - * the key is added to the map again, its new value would be stored in a new - * independent location. - */ - Value* get_and_erase(const Key& key) { - size_t slot = Hash()(key) % n_slots; - auto ptr = m_slots[slot].erase(key); - return ptr ? &ptr->second : nullptr; - } - - private: - std::mutex& get_lock_by_slot(size_t slot) const { return m_locks[slot]; } - - mutable std::mutex m_locks[n_slots]; -}; - -/** - * A concurrent container with map semantics that only accepts insertions. - * - * This allows accessing constant references on values safely and lock-free. - */ -template , - typename KeyEqual = std::equal_to, - size_t n_slots = cc_impl::kDefaultSlots> -class InsertOnlyConcurrentMap final - : public ConcurrentContainer, - n_slots> { - public: - using Base = - ConcurrentContainer, - n_slots>; - using typename Base::const_iterator; - using typename Base::iterator; - - using Base::end; - using Base::m_slots; - - using KeyValuePair = typename Base::Value; - - InsertOnlyConcurrentMap() = default; - - InsertOnlyConcurrentMap(const InsertOnlyConcurrentMap& container) noexcept - : Base(container) {} - - InsertOnlyConcurrentMap(InsertOnlyConcurrentMap&& container) noexcept - : Base(std::move(container)) {} - - InsertOnlyConcurrentMap& operator=( - InsertOnlyConcurrentMap&& container) noexcept { - Base::operator=(std::move(container)); - return *this; - } - - InsertOnlyConcurrentMap& operator=( - const InsertOnlyConcurrentMap& container) noexcept { - if (this != &container) { - Base::operator=(container); - } - return *this; - } - - template - InsertOnlyConcurrentMap(InputIt first, InputIt last) { - insert(first, last); - } - - /* - * This operation is always thread-safe. - */ - const Value* get(const Key& key) const { - size_t slot = Hash()(key) % n_slots; - const auto& map = this->get_container(slot); - const auto* ptr = map.get(key); - return ptr ? &ptr->second : nullptr; - } - - Value* get_unsafe(const Key& key) { - size_t slot = Hash()(key) % n_slots; - auto& map = this->get_container(slot); - auto* ptr = map.get(key); - return ptr ? &ptr->second : nullptr; - } - - const Value* get_unsafe(const Key& key) const { return get(key); } - - /* - * This operation is always thread-safe. If you are reading from a - * ConcurrentMap that is not being concurrently modified, it will probably be - * faster to use `find()` or `at_unsafe()` to avoid locking. - */ - const Value& at(const Key& key) const { - size_t slot = Hash()(key) % n_slots; - const auto& map = this->get_container(slot); - const auto* ptr = map.get(key); - if (ptr == nullptr) { - throw std::out_of_range("at"); - } - return ptr->second; - } - - Value& at_unsafe(const Key& key) { - size_t slot = Hash()(key) % n_slots; - auto& map = this->get_container(slot); - auto* ptr = map.get(key); - if (ptr == nullptr) { - throw std::out_of_range("at_unsafe"); - } - return ptr->second; - } - - const Value& at_unsafe(const Key& key) const { return at(key); } - - /* - * Returns a pair consisting of a pointer on the inserted element (or the - * element that prevented the insertion) and a boolean denoting whether the - * insertion took place. This operation is always thread-safe. - */ - std::pair insert(const KeyValuePair& entry) { - size_t slot = Hash()(entry.first) % n_slots; - auto& map = this->get_container(slot); - auto insertion_result = map.try_insert(entry); - return std::make_pair(&insertion_result.stored_value_ptr->second, - insertion_result.success); - } - - /* - * Returns a pair consisting of a pointer on the inserted element (or the - * element that prevented the insertion) and a boolean denoting whether the - * insertion took place. This operation is always thread-safe. - */ - std::pair insert(KeyValuePair&& entry) { - size_t slot = Hash()(entry.first) % n_slots; - auto& map = this->get_container(slot); - auto insertion_result = map.try_insert(std::move(entry)); - return std::make_pair(&insertion_result.stored_value_ptr->second, - insertion_result.success); - } - - std::pair insert_unsafe(const KeyValuePair& entry) { - size_t slot = Hash()(entry.first) % n_slots; - auto& map = this->get_container(slot); - auto insertion_result = map.try_insert(entry); - return std::make_pair(&insertion_result.stored_value_ptr->second, - insertion_result.success); + return iterator(&m_slots[0], slot, it); } - std::pair insert_unsafe(KeyValuePair entry) { - size_t slot = Hash()(entry.first) % n_slots; - auto& map = this->get_container(slot); - auto insertion_result = map.try_insert(std::move(entry)); - return std::make_pair(&insertion_result.stored_value_ptr->second, - insertion_result.success); + const_iterator find(const Key& key) const { + size_t slot = Hash()(key) % n_slots; + const auto& it = m_slots[slot].find(key); + if (it == m_slots[slot].end()) { + return end(); + } + return const_iterator(&m_slots[0], slot, it); } - /* - * This operation is always thread-safe. - */ - void insert(std::initializer_list l) { - for (const auto& entry : l) { - insert(entry); + size_t size() const { + size_t s = 0; + for (size_t slot = 0; slot < n_slots; ++slot) { + s += m_slots[slot].size(); } + return s; } - /* - * This operation is always thread-safe. - */ - template - void insert(InputIt first, InputIt last) { - for (; first != last; ++first) { - insert(*first); + bool empty() const { + for (size_t slot = 0; slot < n_slots; ++slot) { + if (!m_slots[slot].empty()) { + return false; + } } + return true; } - std::pair insert_or_assign_unsafe(KeyValuePair&& entry) { - size_t slot = Hash()(entry.first) % n_slots; - auto& map = this->get_container(slot); - auto insertion_result = map.try_emplace(std::forward(entry)); - if (insertion_result.success) { - return std::make_pair(&insertion_result.stored_value_ptr->second, true); + void reserve(size_t capacity) { + size_t slot_capacity = capacity / n_slots; + if (slot_capacity > 0) { + for (size_t i = 0; i < n_slots; ++i) { + m_slots[i].reserve(slot_capacity); + } } - auto* constructed_value = insertion_result.incidentally_constructed_value(); - if (constructed_value) { - insertion_result.stored_value_ptr->second = - std::move(constructed_value->second); - } else { - insertion_result.stored_value_ptr->second = - std::forward(entry.second); + } + + void clear() { + for (size_t slot = 0; slot < n_slots; ++slot) { + m_slots[slot].clear(); } - return std::make_pair(&insertion_result.stored_value_ptr->second, false); } /* * This operation is always thread-safe. */ - template - std::pair emplace(Args&&... args) { - KeyValuePair entry(std::forward(args)...); - size_t slot = Hash()(entry.first) % n_slots; - auto& map = this->get_container(slot); - auto insertion_result = map.try_insert(std::move(entry)); - return std::make_pair(&insertion_result.stored_value_ptr->second, - insertion_result.success); + size_t count(const Key& key) const { + size_t slot = Hash()(key) % n_slots; + std::unique_lock lock{m_locks[slot]}; + return m_slots[slot].count(key); } - template - std::pair emplace_unsafe(Args&&... args) { - KeyValuePair entry(std::forward(args)...); - size_t slot = Hash()(entry.first) % n_slots; - auto& map = this->get_container(slot); - auto insertion_result = map.try_insert(std::move(entry)); - return std::make_pair(&insertion_result.stored_value_ptr->second, - insertion_result.success); + size_t count_unsafe(const Key& key) const { + size_t slot = Hash()(key) % n_slots; + return m_slots[slot].count(key); } /* * This operation is always thread-safe. */ - template , typename... Args> - std::pair get_or_emplace_and_assert_equal( - Key&& key, Args&&... args) { + size_t erase(const Key& key) { size_t slot = Hash()(key) % n_slots; - auto& map = this->get_container(slot); - auto insertion_result = - map.try_emplace(std::forward(key), std::forward(args)...); - always_assert( - insertion_result.success || - (insertion_result.incidentally_constructed_value() - ? ValueEqual()( - insertion_result.stored_value_ptr->second, - insertion_result.incidentally_constructed_value()->second) - : ValueEqual()(insertion_result.stored_value_ptr->second, - Value(std::forward(args)...)))); - auto* ptr = insertion_result.stored_value_ptr; - return {&ptr->second, insertion_result.success}; + std::unique_lock lock{m_locks[slot]}; + return m_slots[slot].erase(key); } /* - * This operation is always thread-safe. + * This operation is not thread-safe. */ - template , typename... Args> - std::pair get_or_emplace_and_assert_equal( - const Key& key, Args&&... args) { - size_t slot = Hash()(key) % n_slots; - auto& map = this->get_container(slot); - auto insertion_result = map.try_emplace(key, std::forward(args)...); - always_assert( - insertion_result.success || - (insertion_result.incidentally_constructed_value() - ? ValueEqual()( - insertion_result.stored_value_ptr->second, - insertion_result.incidentally_constructed_value()->second) - : ValueEqual()(insertion_result.stored_value_ptr->second, - Value(std::forward(args)...)))); - auto* ptr = insertion_result.stored_value_ptr; - return {&ptr->second, insertion_result.success}; + size_t bucket_size(size_t i) const { + always_assert(i < n_slots); + return m_slots[i].size(); } /* - * This operation is always thread-safe. + * WARNING: Only use with unsafe functions, or risk deadlock or undefined + * behavior! */ - template , - typename Creator, - typename... Args> - std::pair get_or_create_and_assert_equal( - const Key& key, const Creator& creator, Args&&... args) { - auto* ptr = get(key); - if (ptr) { - return {ptr, false}; - } - return get_or_emplace_and_assert_equal( - key, creator(key, std::forward(args)...)); + std::mutex& get_lock(const Key& key) const { + size_t slot = Hash()(key) % n_slots; + return get_lock_by_slot(slot); } /* - * This operation is always thread-safe. + * This operation is not thread-safe. */ - template , - typename Creator, - typename... Args> - std::pair get_or_create_and_assert_equal( - Key&& key, const Creator& creator, Args&&... args) { - auto* ptr = get(key); - if (ptr) { - return {ptr, false}; + Container move_to_container() { + Container res; + res.reserve(size()); + for (size_t slot = 0; slot < n_slots; ++slot) { + auto& c = m_slots[slot]; + res.insert(std::make_move_iterator(c.begin()), + std::make_move_iterator(c.end())); + c.clear(); } - return get_or_emplace_and_assert_equal( - std::forward(key), - creator(std::forward(key), std::forward(args)...)); + return res; } - template < - typename UpdateFn = const std::function&> - void update_unsafe(const Key& key, UpdateFn updater) { - size_t slot = Hash()(key) % n_slots; - auto& map = this->get_container(slot); - auto [ptr, emplaced] = map.try_emplace(key); - updater(ptr->first, ptr->second, !emplaced); + protected: + // Only derived classes may be instantiated or copied. + ConcurrentContainer() = default; + + ConcurrentContainer(const ConcurrentContainer& container) { + for (size_t i = 0; i < n_slots; ++i) { + m_slots[i] = container.m_slots[i]; + } } - template < - typename UpdateFn = const std::function&> - void update_unsafe(Key&& key, UpdateFn updater) { - size_t slot = Hash()(key) % n_slots; - auto& map = this->get_container(slot); - auto [ptr, emplaced] = map.try_emplace(std::forward(key)); - updater(ptr->first, ptr->second, !emplaced); + ConcurrentContainer(ConcurrentContainer&& container) noexcept { + for (size_t i = 0; i < n_slots; ++i) { + m_slots[i] = std::move(container.m_slots[i]); + } } - size_t erase(const Key& key) = delete; + Container& get_container(size_t slot) { return m_slots[slot]; } + + const Container& get_container(size_t slot) const { return m_slots[slot]; } + + std::mutex& get_lock_by_slot(size_t slot) const { return m_locks[slot]; } + + mutable std::mutex m_locks[n_slots]; + Container m_slots[n_slots]; }; -/** - * A concurrent container with map semantics that holds atomic values. - * - * This typically allows for lock free reads and writes (if the underlying - * std::atomic<> data types allows), and even supports all common lock-free - * atomic mutations typically found on std::atomic<>. - */ -template + T&& operator()(T&& t) const { + return std::forward(t); + } +}; + +// A concurrent container with map semantics. +// +// Note: `KeyProjection` allows to use a different key for the +// `ConcurrentMapContainer` vs the internal sharded storage. +// +// This is in general possible without, but may have storage overhead. +// An example is a `pair` key, where the first component is +// used for sharding, and the second component for the internal +// map. This could be simulated by a `pair` key in the internal map +// and corresponding hash/compare/equals functions, + +template , - typename KeyEqual = std::equal_to, - size_t n_slots = cc_impl::kDefaultSlots> -class AtomicMap final - : public ConcurrentContainer< - std::unordered_map, Hash, KeyEqual>, - n_slots> { + typename KeyProjection = Identity, + size_t n_slots = 31> +class ConcurrentMapContainer + : public ConcurrentContainer { public: - using Base = ConcurrentContainer< - std::unordered_map, Hash, KeyEqual>, - n_slots>; - using typename Base::const_iterator; - using typename Base::iterator; + using typename ConcurrentContainer:: + const_iterator; + using + typename ConcurrentContainer::iterator; - using Base::end; - using Base::m_slots; + using ConcurrentContainer::m_slots; + using ConcurrentContainer::end; - AtomicMap() = default; + ConcurrentMapContainer() = default; - AtomicMap(const AtomicMap& container) noexcept : Base(container) {} + ConcurrentMapContainer(const ConcurrentMapContainer& container) + : ConcurrentContainer(container) {} - AtomicMap(AtomicMap&& container) noexcept : Base(std::move(container)) {} + ConcurrentMapContainer(ConcurrentMapContainer&& container) noexcept + : ConcurrentContainer( + std::move(container)) {} - AtomicMap& operator=(AtomicMap&& container) noexcept { - Base::operator=(std::move(container)); - return *this; - } + ConcurrentMapContainer& operator=(ConcurrentMapContainer&&) noexcept = + default; - AtomicMap& operator=(const AtomicMap& container) noexcept { - if (this != &container) { - Base::operator=(container); - } - return *this; + ConcurrentMapContainer& operator=(const ConcurrentMapContainer&) noexcept = + default; + + template + ConcurrentMapContainer(InputIt first, InputIt last) { + insert(first, last); } /* - * This operation is always thread-safe. + * This operation is always thread-safe. Note that it returns a copy of Value + * rather than a reference since insertions from other threads may cause the + * hashtables to be resized. If you are reading from a ConcurrentMap that is + * not being concurrently modified, it will probably be faster to use + * `find()` or `at_unsafe()` to avoid the copy. */ - Value load(const Key& key, Value default_value = Value()) const { + Value at(const Key& key) const { size_t slot = Hash()(key) % n_slots; - const auto& map = this->get_container(slot); - const auto* ptr = map.get(key); - return ptr ? ptr->second.load() : default_value; + std::unique_lock lock(this->get_lock_by_slot(slot)); + return this->get_container(slot).at(KeyProjection()(key)); } - /* - * This operation is always thread-safe. - */ - std::atomic* store(const Key& key, const Value& arg) { + const Value& at_unsafe(const Key& key) const { size_t slot = Hash()(key) % n_slots; - auto& map = this->get_container(slot); - auto insertion_result = map.try_emplace(key, arg); - if (!insertion_result.success) { - insertion_result.stored_value_ptr->second.store(arg); - } - return &insertion_result.stored_value_ptr->second; + return this->get_container(slot).at(KeyProjection()(key)); } - /* - * This operation is always thread-safe. - */ - std::atomic* store(Key&& key, Value&& arg) { + Value& at_unsafe(const Key& key) { size_t slot = Hash()(key) % n_slots; - auto& map = this->get_container(slot); - auto insertion_result = - map.try_emplace(std::forward(key), std::forward(arg)); - if (!insertion_result->success) { - auto* constructed_value = - insertion_result.incidentally_constructed_value(); - if (constructed_value) { - insertion_result.stored_value_ptr->second.store( - std::move(constructed_value->second)); - } else { - insertion_result.stored_value_ptr->second = std::forward(arg); - } - } - return &insertion_result.stored_value_ptr->second; + return this->get_container(slot).at(KeyProjection()(key)); } - /* - * This operation is always thread-safe. - */ - template - std::pair*, bool> emplace(const Key& key, Args&&... args) { + iterator find(const Key& key) { size_t slot = Hash()(key) % n_slots; - auto& map = this->get_container(slot); - auto insertion_result = map.try_emplace(key, std::forward(args)...); - return std::make_pair(&insertion_result.stored_value_ptr->second, - insertion_result.success); + const auto& it = m_slots[slot].find(KeyProjection()(key)); + if (it == m_slots[slot].end()) { + return end(); + } + return iterator(&m_slots[0], slot, it); } - template - std::pair*, bool> emplace(Key&& key, Args&&... args) { + const_iterator find(const Key& key) const { size_t slot = Hash()(key) % n_slots; - auto& map = this->get_container(slot); - auto insertion_result = - map.try_emplace(std::forward(key), std::forward(args)...); - return std::make_pair(&insertion_result.stored_value_ptr->second, - insertion_result.success); + const auto& it = m_slots[slot].find(KeyProjection()(key)); + if (it == m_slots[slot].end()) { + return end(); + } + return const_iterator(&m_slots[0], slot, it); } /* * This operation is always thread-safe. */ - Value exchange(const Key& key, Value desired, Value default_value = Value()) { + Value get(const Key& key, Value default_value) const { size_t slot = Hash()(key) % n_slots; - auto& map = this->get_container(slot); - auto insertion_result = map.try_emplace(key, desired); - if (insertion_result.success) { + std::unique_lock lock(this->get_lock_by_slot(slot)); + const auto& map = this->get_container(slot); + const auto& it = map.find(KeyProjection()(key)); + if (it == map.end()) { return default_value; } - return insertion_result.stored_value_ptr->second.exchange(desired); + return it->second; } /* + * The Boolean return value denotes whether the insertion took place. * This operation is always thread-safe. + * + * Note that while the STL containers' insert() methods return both an + * iterator and a boolean success value, we only return the boolean value + * here as any operations on a returned iterator are not guaranteed to be + * thread-safe. */ - bool compare_exchange(const Key& key, - Value& expected, - Value desired, - Value default_value = Value()) { - size_t slot = Hash()(key) % n_slots; + bool insert(const std::pair& entry) { + size_t slot = Hash()(entry.first) % n_slots; + std::unique_lock lock(this->get_lock_by_slot(slot)); auto& map = this->get_container(slot); - if (expected == default_value) { - auto insertion_result = map.try_emplace(key, desired); - if (insertion_result.success) { - return true; - } - return insertion_result.stored_value_ptr->second.compare_exchange_strong( - expected, desired); - } - auto ptr = map.get(key); - if (ptr == nullptr) { - expected = default_value; - return false; - } - return ptr->second.compare_exchange_strong(expected, desired); + return map + .insert(std::make_pair(KeyProjection()(entry.first), entry.second)) + .second; } /* * This operation is always thread-safe. */ - Value fetch_add(const Key& key, Value arg, Value default_value = 0) { - size_t slot = Hash()(key) % n_slots; - auto& map = this->get_container(slot); - auto insertion_result = map.try_emplace(key, default_value); - return insertion_result.stored_value_ptr->second.fetch_add(arg); + void insert(std::initializer_list> l) { + for (const auto& entry : l) { + insert(entry); + } } /* * This operation is always thread-safe. */ - Value fetch_sub(const Key& key, Value arg, Value default_value = 0) { - size_t slot = Hash()(key) % n_slots; - auto& map = this->get_container(slot); - auto insertion_result = map.try_insert(key, default_value); - return insertion_result.stored_value_ptr->second.fetch_sub(arg); + template + void insert(InputIt first, InputIt last) { + for (; first != last; ++first) { + insert(*first); + } } /* * This operation is always thread-safe. */ - Value fetch_and(const Key& key, Value arg, Value default_value = 0) { - size_t slot = Hash()(key) % n_slots; + void insert_or_assign(const std::pair& entry) { + size_t slot = Hash()(entry.first) % n_slots; + std::unique_lock lock(this->get_lock_by_slot(slot)); auto& map = this->get_container(slot); - auto insertion_result = map.try_insert(key, default_value); - return insertion_result.stored_value_ptr->second.fetch_and(arg); + map[KeyProjection()(entry.first)] = entry.second; } /* * This operation is always thread-safe. */ - Value fetch_or(const Key& key, Value arg, Value default_value = 0) { - size_t slot = Hash()(key) % n_slots; + template + bool emplace(Args&&... args) { + std::pair entry(std::forward(args)...); + size_t slot = Hash()(entry.first) % n_slots; + std::unique_lock lock(this->get_lock_by_slot(slot)); + auto& map = this->get_container(slot); + return map + .emplace(KeyProjection()(std::move(entry.first)), + std::move(entry.second)) + .second; + } + + template + bool emplace_unsafe(Args&&... args) { + std::pair entry(std::forward(args)...); + size_t slot = Hash()(entry.first) % n_slots; auto& map = this->get_container(slot); - auto insertion_result = map.try_insert(key, default_value); - return insertion_result.stored_value_ptr->second.fetch_or(arg); + return map + .emplace(KeyProjection()(std::move(entry.first)), + std::move(entry.second)) + .second; } /* - * This operation is always thread-safe. + * This operation atomically modifies an entry in the map. If the entry + * doesn't exist, it is created. The third argument of the updater function is + * a Boolean flag denoting whether the entry exists or not. */ - Value fetch_xor(const Key& key, Value arg, Value default_value = 0) { + template < + typename UpdateFn = const std::function&> + void update(const Key& key, UpdateFn updater) { + size_t slot = Hash()(key) % n_slots; + std::unique_lock lock(this->get_lock_by_slot(slot)); + auto& map = this->get_container(slot); + auto it = map.find(KeyProjection()(key)); + if (it == map.end()) { + updater(KeyProjection()(key), map[KeyProjection()(key)], false); + } else { + updater(it->first, it->second, true); + } + } + + template < + typename UpdateFn = const std::function&> + void update_unsafe(const Key& key, UpdateFn updater) { size_t slot = Hash()(key) % n_slots; auto& map = this->get_container(slot); - auto insertion_result = map.try_insert(key, default_value); - return insertion_result.stored_value_ptr->second.fetch_xor(arg); + auto it = map.find(KeyProjection()(key)); + if (it == map.end()) { + updater(KeyProjection()(key), map[KeyProjection()(key)], false); + } else { + updater(it->first, it->second, true); + } } }; -/* - * A concurrent container with set semantics, also allowing erasing values. - */ +template , + typename Equal = std::equal_to, + size_t n_slots = 31> +using ConcurrentMap = + ConcurrentMapContainer, + Key, + Value, + Hash, + Identity, + n_slots>; + template , - typename KeyEqual = std::equal_to, - size_t n_slots = cc_impl::kDefaultSlots> + typename Equal = std::equal_to, + size_t n_slots = 31> class ConcurrentSet final - : public ConcurrentContainer, + : public ConcurrentContainer, + Key, + Hash, n_slots> { public: - using Base = - ConcurrentContainer, n_slots>; - ConcurrentSet() = default; - ConcurrentSet(const ConcurrentSet& set) noexcept : Base(set) {} + ConcurrentSet(const ConcurrentSet& set) + : ConcurrentContainer, + Key, + Hash, + n_slots>(set) {} - ConcurrentSet(ConcurrentSet&& set) noexcept : Base(std::move(set)) {} + ConcurrentSet(ConcurrentSet&& set) noexcept + : ConcurrentContainer, + Key, + Hash, + n_slots>(std::move(set)) {} - ConcurrentSet& operator=(ConcurrentSet&& container) noexcept { - Base::operator=(std::move(container)); - return *this; - } + ConcurrentSet& operator=(ConcurrentSet&&) noexcept = default; - ConcurrentSet& operator=(const ConcurrentSet& container) noexcept { - if (this != &container) { - Base::operator=(container); - } - return *this; - } + ConcurrentSet& operator=(const ConcurrentSet&) noexcept = default; /* * The Boolean return value denotes whether the insertion took place. @@ -2004,14 +511,9 @@ class ConcurrentSet final */ bool insert(const Key& key) { size_t slot = Hash()(key) % n_slots; + std::unique_lock lock(this->get_lock_by_slot(slot)); auto& set = this->get_container(slot); - return set.try_insert(key).success; - } - - bool insert(Key&& key) { - size_t slot = Hash()(key) % n_slots; - auto& set = this->get_container(slot); - return set.try_insert(std::forward(key)).success; + return set.insert(key).second; } /* @@ -2040,8 +542,9 @@ class ConcurrentSet final bool emplace(Args&&... args) { Key key(std::forward(args)...); size_t slot = Hash()(key) % n_slots; + std::unique_lock lock(this->get_lock_by_slot(slot)); auto& set = this->get_container(slot); - return set.try_insert(std::move(key)).success; + return set.emplace(std::move(key)).second; } }; @@ -2050,38 +553,27 @@ class ConcurrentSet final * * This allows accessing constant references on elements safely. */ -template , - typename KeyEqual = std::equal_to, - size_t n_slots = cc_impl::kDefaultSlots> -class InsertOnlyConcurrentSet final - : public ConcurrentContainer, - n_slots> { + size_t n_slots = 31> +class InsertOnlyConcurrentSetContainer final + : public ConcurrentContainer { public: - using Base = - ConcurrentContainer, n_slots>; + InsertOnlyConcurrentSetContainer() = default; - InsertOnlyConcurrentSet() = default; + InsertOnlyConcurrentSetContainer(const InsertOnlyConcurrentSetContainer& set) + : ConcurrentContainer(set) {} - InsertOnlyConcurrentSet(const InsertOnlyConcurrentSet& set) noexcept - : Base(set) {} + InsertOnlyConcurrentSetContainer( + InsertOnlyConcurrentSetContainer&& set) noexcept + : ConcurrentContainer(std::move(set)) {} - InsertOnlyConcurrentSet(InsertOnlyConcurrentSet&& set) noexcept - : Base(std::move(set)) {} - - InsertOnlyConcurrentSet& operator=( - InsertOnlyConcurrentSet&& container) noexcept { - Base::operator=(std::move(container)); - return *this; - } + InsertOnlyConcurrentSetContainer& operator=( + InsertOnlyConcurrentSetContainer&&) noexcept = default; - InsertOnlyConcurrentSet& operator=( - const InsertOnlyConcurrentSet& container) noexcept { - if (this != &container) { - Base::operator=(container); - } - return *this; - } + InsertOnlyConcurrentSetContainer& operator=( + const InsertOnlyConcurrentSetContainer&) noexcept = default; /* * Returns a pair consisting of a pointer on the inserted element (or the @@ -2090,35 +582,12 @@ class InsertOnlyConcurrentSet final */ std::pair insert(const Key& key) { size_t slot = Hash()(key) % n_slots; + std::unique_lock lock(this->get_lock_by_slot(slot)); auto& set = this->get_container(slot); - auto insertion_result = set.try_insert(key); - return {insertion_result.stored_value_ptr, insertion_result.success}; - } - - /* - * Returns a pair consisting of a pointer on the inserted element (or the - * element that prevented the insertion) and a boolean denoting whether the - * insertion took place. This operation is always thread-safe. - */ - std::pair insert(Key&& key) { - size_t slot = Hash()(key) % n_slots; - auto& set = this->get_container(slot); - auto insertion_result = set.try_insert(std::forward(key)); - return {insertion_result.stored_value_ptr, insertion_result.success}; - } - - std::pair insert_unsafe(const Key& key) { - size_t slot = Hash()(key) % n_slots; - auto& set = this->get_container(slot); - auto insertion_result = set.try_insert(key); - return {insertion_result.stored_value_ptr, insertion_result.success}; - } - - std::pair insert_unsafe(Key&& key) { - size_t slot = Hash()(key) % n_slots; - auto& set = this->get_container(slot); - auto insertion_result = set.try_insert(std::forward(key)); - return {insertion_result.stored_value_ptr, insertion_result.success}; + // `std::unordered_set::insert` does not invalidate references, + // thus it is safe to return a reference on the object. + auto result = set.insert(key); + return {&*result.first, result.second}; } /* @@ -2127,13 +596,34 @@ class InsertOnlyConcurrentSet final */ const Key* get(const Key& key) const { size_t slot = Hash()(key) % n_slots; + std::unique_lock lock(this->get_lock_by_slot(slot)); const auto& set = this->get_container(slot); - return set.get(key); + auto result = set.find(key); + if (result == set.end()) { + return nullptr; + } else { + return &*result; + } } size_t erase(const Key& key) = delete; }; +/** + * A concurrent set that only accepts insertions. + * + * This allows accessing constant references on elements safely. + */ +template , + typename Equal = std::equal_to, + size_t n_slots = 31> +using InsertOnlyConcurrentSet = + InsertOnlyConcurrentSetContainer, + Key, + Hash, + n_slots>; + namespace cc_impl { template diff --git a/libredex/Configurable.cpp b/libredex/Configurable.cpp index 93628a93c0..1b891a0087 100644 --- a/libredex/Configurable.cpp +++ b/libredex/Configurable.cpp @@ -564,6 +564,6 @@ IMPLEMENT_REFLECTOR_EX(TypeMap, "dict") IMPLEMENT_TRAIT_REFLECTOR(bool) IMPLEMENT_TRAIT_REFLECTOR(int) -IMPLEMENT_TRAIT_REFLECTOR(const std::string&) +IMPLEMENT_TRAIT_REFLECTOR(std::string) #undef error_or_warn diff --git a/libredex/ControlFlow.cpp b/libredex/ControlFlow.cpp index 1e1dda2994..fa40b3a968 100644 --- a/libredex/ControlFlow.cpp +++ b/libredex/ControlFlow.cpp @@ -10,7 +10,6 @@ #include #include #include -#include #include #include @@ -30,33 +29,6 @@ std::atomic build_cfg_counter{0}; namespace { -bool edge_type_structural_equals(std::vector e1, - std::vector e2) { - if (e1.empty() && e2.empty()) { - return true; - } - - if (e1.empty() || e2.empty()) { - return false; - } - - if (e1.size() != e2.size()) { - return false; - } - std::unordered_map edge_types; - for (size_t i = 0; i < e1.size(); i++) { - edge_types[e1[i]->type()] += 1; - edge_types[e2[i]->type()] -= 1; - } - - for (auto pair : edge_types) { - if (pair.second != 0) { - return false; - } - } - return true; -} - // return true if `it` should be the last instruction of this block bool end_of_block(const IRList* ir, const IRList::iterator& it, bool in_try) { auto next = std::next(it); @@ -644,17 +616,6 @@ std::vector Block::get_outgoing_throws_in_order() const { return result; } -std::vector Block::get_outgoing_branches_in_order() const { - std::vector result = - m_parent->get_succ_edges_of_type(this, EDGE_BRANCH); - if (result.size() > 1) { - std::sort(result.begin(), result.end(), [](const Edge* e1, const Edge* e2) { - return e1->case_key() < e2->case_key(); - }); - } - return result; -} - // These assume that the iterator is inside this block cfg::InstructionIterator Block::to_cfg_instruction_iterator( const ir_list::InstructionIterator& list_it, bool next_on_end) { @@ -721,12 +682,8 @@ bool Block::push_back(const std::vector& insns) { bool Block::push_back(IRInstruction* insn) { return m_parent->push_back(this, insn); } -bool Block::structural_equals(const Block* other) const { - return this->structural_equals(other, std::equal_to()); -} -bool Block::structural_equals( - const Block* other, const InstructionEquality& instruction_equals) const { +bool Block::structural_equals(const Block* other) const { auto iterable1 = ir_list::ConstInstructionIterable(this); auto iterable2 = ir_list::ConstInstructionIterable(other); auto it1 = iterable1.begin(); @@ -736,7 +693,7 @@ bool Block::structural_equals( auto& mie1 = *it1; auto& mie2 = *it2; - if (!instruction_equals(*mie1.insn, *mie2.insn)) { + if (*mie1.insn != *mie2.insn) { return false; } } @@ -744,15 +701,6 @@ bool Block::structural_equals( return it1 == iterable1.end() && it2 == iterable2.end(); } -bool Block::extended_structural_equals( - const Block* other, const InstructionEquality& instruction_equals) const { - if (!edge_type_structural_equals(this->preds(), other->preds()) || - !edge_type_structural_equals(this->succs(), other->succs())) { - return false; - } - return this->structural_equals(other, instruction_equals); -} - std::ostream& operator<<(std::ostream& os, const Edge& e) { switch (e.type()) { case EDGE_GOTO: @@ -1653,26 +1601,6 @@ void ControlFlowGraph::gather_methodhandles( } } -cfg::InstructionIterator -ControlFlowGraph::primary_instruction_of_move_result_for_type_check( - const cfg::InstructionIterator& it) { - auto move_result_insn = it->insn; - always_assert(opcode::is_move_result_any(move_result_insn->opcode())); - auto block = const_cast(it.block()); - if (block->get_first_insn()->insn == move_result_insn) { - auto& preds = block->preds(); - always_assert(preds.size() == 1); - auto previous_block = preds.front()->src(); - auto res = previous_block->to_cfg_instruction_iterator( - previous_block->get_last_insn()); - return res; - } else { - auto res = std::prev(it); - always_assert(res.block() == it.block()); - return res; - } -} - cfg::InstructionIterator ControlFlowGraph::primary_instruction_of_move_result( const cfg::InstructionIterator& it) { auto move_result_insn = it->insn; @@ -1868,7 +1796,7 @@ std::vector ControlFlowGraph::order( m_blocks.size()); // The entry block must always be first. - redex_assert(m_entry_block == std::as_const(result).at(0)); + redex_assert(m_entry_block == result.at(0)); return result; } @@ -1916,7 +1844,7 @@ void ControlFlowGraph::build_chains( // throw block in the middle. TRACE(CFG, 5, "Need to collapse goto chain with move result!"); auto* goto_chain = block_to_chain->at(goto_block); - redex_assert(CONSTP(goto_chain)->at(0) == goto_block); + redex_assert(goto_chain->at(0) == goto_block); for (auto* gcb : *goto_chain) { chain->push_back(gcb); (*block_to_chain)[gcb] = chain; @@ -1924,7 +1852,7 @@ void ControlFlowGraph::build_chains( auto it = std::find_if( chains->begin(), chains->end(), [&](const auto& uptr) { return uptr.get() == goto_chain; }); - redex_assert(it != chains->cend()); + redex_assert(it != chains->end()); chains->erase(it); } break; @@ -1942,7 +1870,7 @@ void ControlFlowGraph::build_chains( redex_assert(m_entry_block != nullptr); if (DEBUG) { auto it = m_blocks.find(m_entry_block->id()); - redex_assert(it != m_blocks.cend()); + redex_assert(it != m_blocks.end()); redex_assert(it->second == m_entry_block); } handle_block(m_entry_block); @@ -2273,7 +2201,7 @@ std::vector ControlFlowGraph::blocks_reverse_post_deprecated() const { return true; }(); if (all_succs_visited) { - redex_assert(curr == std::as_const(stack).top()); + redex_assert(curr == stack.top()); postorder.push_back(curr); stack.pop(); } @@ -2584,11 +2512,9 @@ void ControlFlowGraph::cleanup_deleted_edges(const EdgeSet& edges) { if (last_it != pred_block->end()) { auto last_insn = last_it->insn; auto op = last_insn->opcode(); - if (!opcode::is_a_conditional_branch(op) && !opcode::is_switch(op)) { - continue; - } - auto* fwd_edge = get_singleton_normal_forward_edge(pred_block); - if (fwd_edge != nullptr) { + Edge* fwd_edge; + if ((opcode::is_a_conditional_branch(op) || opcode::is_switch(op)) && + (fwd_edge = get_singleton_normal_forward_edge(pred_block))) { m_removed_insns.push_back(last_insn); pred_block->m_entries.erase_and_dispose(last_it); fwd_edge->set_type(EDGE_GOTO); @@ -3151,104 +3077,6 @@ ControlFlowGraph::EdgeSet ControlFlowGraph::remove_pred_edges(Block* b, cleanup); } -bool ControlFlowGraph::structural_equals(const ControlFlowGraph& other) const { - return this->structural_equals(other, std::equal_to()); -} - -bool ControlFlowGraph::structural_equals( - const ControlFlowGraph& other, - const InstructionEquality& instruction_equals) const { - if (this->num_blocks() != other.num_blocks() || - this->num_edges() != other.num_edges()) { - return false; - } - - std::queue this_blocks; - std::queue other_blocks; - std::unordered_set block_visited; - // Check entry block. - if (!this->entry_block()->extended_structural_equals(other.entry_block(), - instruction_equals)) { - return false; - } - - // Check exit_block. Then no need to check EDGE_GHOST later. - if (this->exit_block()) { - if (!this->exit_block()->extended_structural_equals(other.exit_block(), - instruction_equals)) { - return false; - } - } else if (other.exit_block()) { - return false; - } - - this_blocks.push(this->entry_block()); - other_blocks.push(other.entry_block()); - block_visited.emplace(this->entry_block()->id()); - while (!this_blocks.empty()) { - always_assert(!other_blocks.empty()); - auto b1 = this_blocks.front(); - auto b2 = other_blocks.front(); - this_blocks.pop(); - other_blocks.pop(); - // Push b1, b2's GOTO succes into queue; - auto goto1 = b1->goes_to(); - auto goto2 = b2->goes_to(); - if (goto1) { - if (!goto1->extended_structural_equals(goto2, instruction_equals)) { - return false; - } - if (block_visited.count(goto1->id()) == 0) { - this_blocks.push(goto1); - other_blocks.push(goto2); - block_visited.emplace(goto1->id()); - } - } else if (goto2) { - return false; - } - - // Push b1, b2's THROW succes into queue; - auto throw1_edges = b1->get_outgoing_throws_in_order(); - auto throw2_edges = b2->get_outgoing_throws_in_order(); - if (throw1_edges.size() != throw2_edges.size()) { - return false; - } - for (size_t i = 0; i < throw1_edges.size(); i++) { - auto throw1 = throw1_edges[i]->target(); - auto throw2 = throw2_edges[i]->target(); - if (!throw1->extended_structural_equals(throw2, instruction_equals)) { - return false; - } - if (block_visited.count(throw1->id()) == 0) { - this_blocks.push(throw1); - other_blocks.push(throw2); - block_visited.emplace(throw1->id()); - } - } - - // Push b1, b2's BRANCH into queue; - auto branch1_edges = b1->get_outgoing_branches_in_order(); - auto branch2_edges = b2->get_outgoing_branches_in_order(); - if (branch1_edges.size() != branch2_edges.size()) { - return false; - } - for (size_t i = 0; i < branch1_edges.size(); i++) { - auto branch1 = branch1_edges[i]->target(); - auto branch2 = branch2_edges[i]->target(); - if (!branch1->extended_structural_equals(branch2, instruction_equals)) { - return false; - } - if (block_visited.count(branch1->id()) == 0) { - this_blocks.push(branch1); - other_blocks.push(branch2); - block_visited.emplace(branch1->id()); - } - } - } - - return true; -} - DexPosition* ControlFlowGraph::get_dbg_pos(const cfg::InstructionIterator& it) { always_assert(&it.cfg() == this); auto search_block = [](Block* b, diff --git a/libredex/ControlFlow.h b/libredex/ControlFlow.h index c0b15aaa3b..401701d07b 100644 --- a/libredex/ControlFlow.h +++ b/libredex/ControlFlow.h @@ -384,8 +384,6 @@ class Block final { // TODO?: Should we just always store the throws in index order? std::vector get_outgoing_throws_in_order() const; - std::vector get_outgoing_branches_in_order() const; - // These assume that the iterator is inside this block InstructionIterator to_cfg_instruction_iterator( const ir_list::InstructionIterator& list_it, bool next_on_end = false); @@ -430,11 +428,6 @@ class Block final { std::unique_ptr sb); bool structural_equals(const Block* other) const; - bool structural_equals(const Block* other, - const InstructionEquality& instruction_equals) const; - - bool extended_structural_equals( - const Block* other, const InstructionEquality& instruction_equals) const; private: friend class ControlFlowGraph; @@ -983,11 +976,6 @@ class ControlFlowGraph { void gather_callsites(std::vector& callsites) const; void gather_methodhandles(std::vector& methodhandles) const; - // Different from primary_instruction_of_move_result(), this method is only - // used for IR type check. - cfg::InstructionIterator primary_instruction_of_move_result_for_type_check( - const cfg::InstructionIterator& it); - cfg::InstructionIterator primary_instruction_of_move_result( const cfg::InstructionIterator& it); @@ -1047,11 +1035,6 @@ class ControlFlowGraph { return std::move(m_removed_insns); } - bool structural_equals(const ControlFlowGraph& other) const; - - bool structural_equals(const ControlFlowGraph& other, - const InstructionEquality& instruction_equals) const; - private: friend class Block; diff --git a/libredex/CppUtil.h b/libredex/CppUtil.h index 1e941dd385..e232f0d1a8 100644 --- a/libredex/CppUtil.h +++ b/libredex/CppUtil.h @@ -16,7 +16,7 @@ // // Example: // self_recursive_fn([](auto self, int i) { -// return i == 0 ? 1 : i == 1 ? 1 : self(self, i - 1) + self(self, i-2); +// return i == 0 ? 1 : i == 1 ? 1 : self(i - 1) + self(i-2); // }, 3); template auto self_recursive_fn(const Fn& fn, Args&&... args) diff --git a/libredex/Creators.cpp b/libredex/Creators.cpp index 97d80629f6..d789e3d909 100644 --- a/libredex/Creators.cpp +++ b/libredex/Creators.cpp @@ -48,9 +48,8 @@ DexClass* ClassCreator::create() { } } m_cls->m_interfaces = DexTypeList::make_type_list(std::move(m_interfaces)); - DexClass* cls = m_cls.release(); - g_redex->publish_class(cls); - return cls; + g_redex->publish_class(m_cls); + return m_cls; } MethodBlock::MethodBlock(const IRList::iterator& iterator, @@ -508,7 +507,7 @@ MethodBlock* MethodBlock::switch_op(Location test, // Copy initialized case blocks back. for (const auto& it : indices_cases) { SwitchIndices indices = it.first; - always_assert(!indices.empty()); + always_assert(indices.size()); int idx = *indices.begin(); cases[idx] = it.second; } diff --git a/libredex/Creators.h b/libredex/Creators.h index e3c328a539..da4a2ef4b4 100644 --- a/libredex/Creators.h +++ b/libredex/Creators.h @@ -7,7 +7,6 @@ #pragma once -#include #include #include @@ -507,13 +506,13 @@ struct ClassCreator { if (location == nullptr) { location = DexLocation::make_location("", ""); } - m_cls = std::unique_ptr(new DexClass(type, location)); + m_cls = new DexClass(type, location); } /** * Return the DexClass associated with this creator. */ - DexClass* get_class() const { return m_cls.get(); } + DexClass* get_class() const { return m_cls; } /** * Return the DexType associated with this creator. @@ -539,7 +538,7 @@ struct ClassCreator { * Set the external bit for the DexClass. */ void set_external() { - m_cls->set_deobfuscated_name(show_cls(m_cls.get())); + m_cls->set_deobfuscated_name(show_cls(m_cls)); m_cls->m_external = true; } @@ -573,6 +572,6 @@ struct ClassCreator { static std::string show_cls(const DexClass* cls); static std::string show_type(const DexType* type); - std::unique_ptr m_cls; + DexClass* m_cls; DexTypeList::ContainerType m_interfaces; }; diff --git a/libredex/Debug.cpp b/libredex/Debug.cpp index c418ea170a..4880b35ab7 100644 --- a/libredex/Debug.cpp +++ b/libredex/Debug.cpp @@ -6,7 +6,6 @@ */ #include "Debug.h" -#include "DebugUtils.h" #include #include diff --git a/libredex/Debug.h b/libredex/Debug.h index 142e1ee04b..3719030f4d 100644 --- a/libredex/Debug.h +++ b/libredex/Debug.h @@ -11,7 +11,8 @@ #include "RedexException.h" #include -#include +#include +#include constexpr bool debug = #ifdef NDEBUG @@ -23,29 +24,47 @@ constexpr bool debug = extern bool slow_invariants_debug; +namespace redex_debug { + +void set_exc_type_as_abort(RedexError type); +void disable_stack_trace_for_exc_type(RedexError type); + +} // namespace redex_debug + #ifdef _MSC_VER #define DEBUG_ONLY -#define UNREACHABLE() __assume(false) -#define PRETTY_FUNC() __func__ -#else -#define DEBUG_ONLY __attribute__((unused)) -#define UNREACHABLE() __builtin_unreachable() -#define PRETTY_FUNC() __PRETTY_FUNCTION__ -#endif // _MSC_VER #define not_reached() \ do { \ redex_assert(false); \ - UNREACHABLE(); \ + __assume(false); \ } while (true) #define not_reached_log(msg, ...) \ do { \ assert_log(false, msg, ##__VA_ARGS__); \ - UNREACHABLE(); \ + __assume(false); \ } while (true) #define assert_fail_impl(e, type, msg, ...) \ - assert_fail(#e, __FILE__, __LINE__, PRETTY_FUNC(), type, msg, ##__VA_ARGS__) + assert_fail(#e, __FILE__, __LINE__, __func__, type, msg, ##__VA_ARGS__) +#else +#define DEBUG_ONLY __attribute__((unused)) + +#define not_reached() \ + do { \ + redex_assert(false); \ + __builtin_unreachable(); \ + } while (true) +#define not_reached_log(msg, ...) \ + do { \ + assert_log(false, msg, ##__VA_ARGS__); \ + __builtin_unreachable(); \ + } while (true) + +#define assert_fail_impl(e, type, msg, ...) \ + assert_fail( \ + #e, __FILE__, __LINE__, __PRETTY_FUNCTION__, type, msg, ##__VA_ARGS__) +#endif [[noreturn]] void assert_fail(const char* expr, const char* file, @@ -78,8 +97,25 @@ extern bool slow_invariants_debug; #define assert_type_log(e, type, msg, ...) \ always_assert_type_log(!debug || e, type, msg, ##__VA_ARGS__) -// Helper for const assertions. -#define CONSTP(e) \ - static_cast::type>::type>::type>::type>(e) +void print_stack_trace(std::ostream& os, const std::exception& e); + +void crash_backtrace_handler(int sig); +void debug_backtrace_handler(int sig); + +// If `block` is true, only a single assert will be logged. All following +// asserts will sleep forever. +void block_multi_asserts(bool block); + +// If called, assertions on threads other than the caller may immediately abort +// instead of raising an exception. Currently only implemented for Linux. +// Note: this is a workaround for libstdc++ from GCC < 8. +void set_abort_if_not_this_thread(); + +// Stats from /proc. See http://man7.org/linux/man-pages/man5/proc.5.html. +struct VmStats { + uint64_t vm_peak = 0; // "Peak virtual memory size." + uint64_t vm_hwm = 0; // "Peak resident set size ("high water mark")." + uint64_t vm_rss = 0; // "Resident set size" +}; +VmStats get_mem_stats(); +bool try_reset_hwm_mem_stat(); // Attempt to reset the vm_hwm value. diff --git a/libredex/DebugUtils.h b/libredex/DebugUtils.h deleted file mode 100644 index 760b86bc37..0000000000 --- a/libredex/DebugUtils.h +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * - * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. - */ - -#pragma once - -#include "RedexException.h" - -#include -#include -#include - -namespace redex_debug { - -void set_exc_type_as_abort(RedexError type); -void disable_stack_trace_for_exc_type(RedexError type); - -} // namespace redex_debug - -void print_stack_trace(std::ostream& os, const std::exception& e); - -void crash_backtrace_handler(int sig); -void debug_backtrace_handler(int sig); - -// If `block` is true, only a single assert will be logged. All following -// asserts will sleep forever. -void block_multi_asserts(bool block); - -// If called, assertions on threads other than the caller may immediately abort -// instead of raising an exception. Currently only implemented for Linux. -// Note: this is a workaround for libstdc++ from GCC < 8. -void set_abort_if_not_this_thread(); - -// Stats from /proc. See http://man7.org/linux/man-pages/man5/proc.5.html. -struct VmStats { - uint64_t vm_peak = 0; // "Peak virtual memory size." - uint64_t vm_hwm = 0; // "Peak resident set size ("high water mark")." - uint64_t vm_rss = 0; // "Resident set size" -}; -VmStats get_mem_stats(); -bool try_reset_hwm_mem_stat(); // Attempt to reset the vm_hwm value. diff --git a/libredex/DexAnnotation.h b/libredex/DexAnnotation.h index 669c32dda4..6eb45c70e9 100644 --- a/libredex/DexAnnotation.h +++ b/libredex/DexAnnotation.h @@ -87,6 +87,7 @@ class DexEncodedValue : public Gatherable { uint64_t value() const { return m_val.m_value; } static std::unique_ptr get_encoded_value( DexIdx* idx, const uint8_t*& encdata); + DexEncodedValueTypes evtype() { return m_evtype; } virtual void encode(DexOutputIdx* dodx, uint8_t*& encdata); void vencode(DexOutputIdx* dodx, std::vector& bytes); diff --git a/libredex/DexClass.cpp b/libredex/DexClass.cpp index 290b853b77..dcb51eb606 100644 --- a/libredex/DexClass.cpp +++ b/libredex/DexClass.cpp @@ -110,7 +110,7 @@ std::string get_simple_deobf_name(const T* ref) { const auto& full_name = ref->get_deobfuscated_name_or_empty(); if (full_name.empty()) { // This comes up for redex-created methods/fields. - return ref->str_copy(); + return std::string(ref->c_str()); } auto dot_pos = full_name.find('.'); auto colon_pos = full_name.find(':'); @@ -121,8 +121,6 @@ std::string get_simple_deobf_name(const T* ref) { } } // namespace -static_assert(sizeof(DexString) == sizeof(DexStringRepr)); - const std::string DexString::EMPTY; const DexString* DexString::make_string(std::string_view nstr) { @@ -814,7 +812,7 @@ DexMethod* DexMethod::make_method_from(DexMethod* that, always_assert_log(that->get_code() != nullptr, "%s", vshow(that).c_str()); m->set_code(std::make_unique(*that->get_code())); } else { - redex_assert(CONSTP(that)->get_code() == nullptr); + redex_assert(that->get_code() == nullptr); } m->m_access = that->m_access; @@ -947,9 +945,8 @@ void DexMethod::attach_annotation_set(std::unique_ptr aset) { } void DexMethod::attach_param_annotation_set( int paramno, std::unique_ptr aset) { - always_assert_type_log(!m_concrete || is_synthetic(get_access()), - RedexError::BAD_ANNOTATION, "method %s is concrete\n", - self_show().c_str()); + always_assert_type_log(!m_concrete, RedexError::BAD_ANNOTATION, + "method %s is concrete\n", self_show().c_str()); always_assert_type_log( m_param_anno == nullptr || m_param_anno->count(paramno) == 0, RedexError::BAD_ANNOTATION, "param %d annotation to method %s exists\n", @@ -1434,7 +1431,7 @@ void DexClass::load_class_annotations(DexIdx* idx, uint32_t anno_off) { auto aset = DexAnnotationSet::get_annotation_set(idx, off); if (aset != nullptr) { method->attach_param_annotation_set(j, std::move(aset)); - redex_assert(CONSTP(method)->get_param_anno()); + redex_assert(method->get_param_anno()); } } } @@ -1572,8 +1569,7 @@ DexClass::DexClass(DexType* type, const DexLocation* location) m_location(location), m_access_flags((DexAccessFlags)0), m_external(false), - m_perf_sensitive(PerfSensitiveGroup::NONE), - m_dynamically_dead(false) { + m_perf_sensitive(PerfSensitiveGroup::NONE) { always_assert(type != nullptr); always_assert_log(type_class(type) == nullptr, "class already exists for %s\n", type->c_str()); @@ -1591,8 +1587,7 @@ DexClass::DexClass(DexIdx* idx, m_location(location), m_access_flags((DexAccessFlags)cdef->access_flags), m_external(false), - m_perf_sensitive(PerfSensitiveGroup::NONE), - m_dynamically_dead(false) { + m_perf_sensitive(PerfSensitiveGroup::NONE) { always_assert(m_self != nullptr); } diff --git a/libredex/DexClass.h b/libredex/DexClass.h index 43df411b97..d112b7558b 100644 --- a/libredex/DexClass.h +++ b/libredex/DexClass.h @@ -84,28 +84,26 @@ using Scope = std::vector; extern "C" bool strcmp_less(const char* str1, const char* str2); #endif -// Internal representation of a DexString; used by RedexContext to construct -// DexString instances. -struct DexStringRepr { - const char* storage; - const uint32_t length; - const uint32_t utfsize; -}; - class DexString { + friend struct RedexContext; + + const char* m_storage; + const uint32_t m_length; + const uint32_t m_utfsize; + // See UNIQUENESS above for the rationale for the private constructor pattern. explicit DexString(const char* storage, uint32_t length, uint32_t utfsize) - : m_repr({storage, length, utfsize}) {} + : m_storage(storage), m_length(length), m_utfsize(utfsize) {} public: DexString() = delete; DexString(DexString&&) = delete; DexString(const DexString&) = delete; - uint32_t size() const { return m_repr.length; } + uint32_t size() const { return m_length; } // UTF-aware length - uint32_t length() const { return m_repr.utfsize; } + uint32_t length() const { return m_utfsize; } int32_t java_hashcode() const; @@ -120,32 +118,23 @@ class DexString { static const std::string EMPTY; - bool is_simple() const { return size() == m_repr.utfsize; } + bool is_simple() const { return size() == m_utfsize; } - const char* c_str() const { return m_repr.storage; } - std::string_view str() const { - return std::string_view(m_repr.storage, m_repr.length); - } - std::string str_copy() const { - return std::string(m_repr.storage, m_repr.length); - } + const char* c_str() const { return m_storage; } + std::string_view str() const { return std::string_view(m_storage, m_length); } + std::string str_copy() const { return std::string(m_storage, m_length); } uint32_t get_entry_size() const { - uint32_t len = uleb128_encoding_size(m_repr.utfsize); + uint32_t len = uleb128_encoding_size(m_utfsize); len += size(); len++; // NULL byte return len; } void encode(uint8_t* output) const { - output = write_uleb128(output, m_repr.utfsize); + output = write_uleb128(output, m_utfsize); strcpy((char*)output, c_str()); } - - private: - DexStringRepr m_repr; - - friend struct RedexContext; }; /* Non-optimizing DexSpec compliant ordering */ @@ -628,12 +617,11 @@ struct DebugLineItem { */ enum class DexDebugEntryType { Instruction, Position }; -enum class PerfSensitiveGroup : uint8_t { +enum class PerfSensitiveGroup { NONE, BETAMAP_ORDERED, OUTLINED, - STRINGS_LOOKUP, - UNREACHABLE, + STRINGS_LOOKUP }; struct DexDebugEntry final { @@ -1161,7 +1149,6 @@ class DexClass { DexAccessFlags m_access_flags; bool m_external; PerfSensitiveGroup m_perf_sensitive; - bool m_dynamically_dead; DexClass(DexType* type, const DexLocation* location); void load_class_annotations(DexIdx* idx, uint32_t anno_off); @@ -1359,9 +1346,6 @@ class DexClass { } PerfSensitiveGroup get_perf_sensitive() { return m_perf_sensitive; } - bool is_dynamically_dead() const { return m_dynamically_dead; } - void set_dynamically_dead() { m_dynamically_dead = true; } - // Find methods and fields from a class using its obfuscated name. DexField* find_field_from_simple_deobfuscated_name( const std::string& field_name); diff --git a/libredex/DexHasher.cpp b/libredex/DexHasher.cpp index 579fbfc40b..9b31b644d0 100644 --- a/libredex/DexHasher.cpp +++ b/libredex/DexHasher.cpp @@ -51,13 +51,13 @@ class Impl final { void hash(const IRCode* c); void hash(const cfg::ControlFlowGraph& cfg); void hash_code_init( - const IRList::const_iterator& begin, - const IRList::const_iterator& end, + IRList::const_iterator begin, + IRList::const_iterator end, std::unordered_map* mie_ids, std::unordered_map* pos_ids); void hash_code_flush( - const IRList::const_iterator& begin, - const IRList::const_iterator& end, + IRList::const_iterator begin, + IRList::const_iterator end, const std::unordered_map& mie_ids, const std::unordered_map& pos_ids); void hash(const IRInstruction* insn); @@ -263,8 +263,8 @@ void Impl::hash(const cfg::ControlFlowGraph& cfg) { } void Impl::hash_code_init( - const IRList::const_iterator& begin, - const IRList::const_iterator& end, + IRList::const_iterator begin, + IRList::const_iterator end, std::unordered_map* mie_ids, std::unordered_map* pos_ids) { auto get_mie_id = [mie_ids](const MethodItemEntry* mie) { @@ -347,8 +347,8 @@ void Impl::hash_code_init( } void Impl::hash_code_flush( - const IRList::const_iterator& begin, - const IRList::const_iterator& end, + IRList::const_iterator begin, + IRList::const_iterator end, const std::unordered_map& mie_ids, const std::unordered_map& pos_ids) { uint32_t mie_index = 0; diff --git a/libredex/DexIdx.cpp b/libredex/DexIdx.cpp index 726639c3be..349a2016d8 100644 --- a/libredex/DexIdx.cpp +++ b/libredex/DexIdx.cpp @@ -92,12 +92,8 @@ DexCallSite* DexIdx::get_callsiteidx_fromdex(uint32_t csidx) { DexMethodHandle* DexIdx::get_methodhandleidx_fromdex(uint32_t mhidx) { redex_assert(mhidx < m_methodhandle_ids_size); - auto type_uint16 = m_methodhandle_ids[mhidx].method_handle_type; - always_assert_log( - type_uint16 >= MethodHandleType::METHOD_HANDLE_TYPE_STATIC_PUT && - type_uint16 <= MethodHandleType::METHOD_HANDLE_TYPE_INVOKE_INTERFACE, - "Invalid MethodHandle type"); - MethodHandleType method_handle_type = (MethodHandleType)type_uint16; + MethodHandleType method_handle_type = + (MethodHandleType)m_methodhandle_ids[mhidx].method_handle_type; if (DexMethodHandle::isInvokeType(method_handle_type)) { DexMethodRef* methodref = get_methodidx(m_methodhandle_ids[mhidx].field_or_method_id); diff --git a/libredex/DexLoader.cpp b/libredex/DexLoader.cpp index 22ef1890da..3772abde85 100644 --- a/libredex/DexLoader.cpp +++ b/libredex/DexLoader.cpp @@ -213,17 +213,18 @@ void DexLoader::gather_input_stats(dex_stats_t* stats, const dex_header* dh) { stats->num_fields += clz->get_ifields().size() + clz->get_sfields().size(); stats->num_methods += clz->get_vmethods().size() + clz->get_dmethods().size(); - auto process_methods = [&](const auto& methods) { - for (auto* meth : methods) { - DexCode* code = meth->get_dex_code(); - if (code) { - stats->num_instructions += code->get_instructions().size(); - stats->num_tries += code->get_tries().size(); - } + for (auto* meth : clz->get_vmethods()) { + DexCode* code = meth->get_dex_code(); + if (code) { + stats->num_instructions += code->get_instructions().size(); + } + } + for (auto* meth : clz->get_dmethods()) { + DexCode* code = meth->get_dex_code(); + if (code) { + stats->num_instructions += code->get_instructions().size(); } - }; - process_methods(clz->get_vmethods()); - process_methods(clz->get_dmethods()); + } } for (uint32_t meth_idx = 0; meth_idx < dh->method_ids_size; ++meth_idx) { auto* meth = m_idx->get_methodidx(meth_idx); @@ -622,7 +623,7 @@ DexClasses DexLoader::load_dex(const dex_header* dh, dex_stats_t* stats) { } static void balloon_all(const Scope& scope, bool throw_on_error) { - InsertOnlyConcurrentMap ir_balloon_errors; + ConcurrentMap ir_balloon_errors; walk::parallel::methods(scope, [&](DexMethod* m) { if (m->get_dex_code()) { try { diff --git a/libredex/DexOutput.cpp b/libredex/DexOutput.cpp index 9c6e855b47..7e52d454c0 100644 --- a/libredex/DexOutput.cpp +++ b/libredex/DexOutput.cpp @@ -78,24 +78,23 @@ class CustomSort { } bool operator()(const T* a, const T* b) const { - const auto end_it = m_map.end(); - const auto a_it = m_map.find(a); - const auto b_it = m_map.find(b); - if (a_it == end_it && b_it == end_it) { + bool a_in = m_map.count(a); + bool b_in = m_map.count(b); + if (!a_in && !b_in) { return m_cmp(a, b); - } - if (a_it != end_it && b_it != end_it) { - const auto a_idx = a_it->second; - const auto b_idx = b_it->second; + } else if (a_in && b_in) { + auto const a_idx = m_map.at(a); + auto const b_idx = m_map.at(b); if (a_idx != b_idx) { return a_idx < b_idx; + } else { + return m_cmp(a, b); } - return m_cmp(a, b); - } - if (a_it != end_it) { + } else if (a_in) { return true; + } else { + return false; } - return false; } }; @@ -346,7 +345,7 @@ void GatheredTypes::sort_dexmethod_emitlist_clinit_order( }); } -DexOutputIdx GatheredTypes::get_dodx(const uint8_t* base) { +DexOutputIdx* GatheredTypes::get_dodx(const uint8_t* base) { /* * These are symbol table indices. Symbols which are used * should be bunched together. We will pass a different @@ -357,55 +356,60 @@ DexOutputIdx GatheredTypes::get_dodx(const uint8_t* base) { * methods and fields, only dexes with annotations have a * dependency on ordering. */ - auto proto = get_proto_index(); - auto typelist = get_typelist_list(&proto); - return DexOutputIdx(get_string_index(), - get_type_index(), - std::move(proto), - get_field_index(), - get_method_index(), - std::move(typelist), - get_callsite_index(), - get_methodhandle_index(), - base); -} - -namespace { - -template -Cout create_index(Cin& in, const Cmp& cmp) { - std::sort(in.begin(), in.end(), cmp); - Cout sidx{}; - sidx.reserve(in.size()); + dexstring_to_idx* string = get_string_index(); + dextype_to_idx* type = get_type_index(); + dexproto_to_idx* proto = get_proto_index(); + dexfield_to_idx* field = get_field_index(); + dexmethod_to_idx* method = get_method_index(); + std::vector* typelist = get_typelist_list(proto); + dexcallsite_to_idx* callsite = get_callsite_index(); + dexmethodhandle_to_idx* methodhandle = get_methodhandle_index(); + return new DexOutputIdx(string, type, proto, field, method, typelist, + callsite, methodhandle, base); +} + +dexstring_to_idx* GatheredTypes::get_string_index(cmp_dstring cmp) { + std::sort(m_lstring.begin(), m_lstring.end(), cmp); + dexstring_to_idx* sidx = new dexstring_to_idx(); uint32_t idx = 0; - for (auto it = in.begin(); it != in.end(); it++) { - sidx.insert(std::make_pair(*it, idx++)); + for (auto it = m_lstring.begin(); it != m_lstring.end(); it++) { + sidx->insert(std::make_pair(*it, idx++)); } return sidx; } -} // namespace - -dexstring_to_idx GatheredTypes::get_string_index(cmp_dstring cmp) { - return create_index(m_lstring, cmp); -} - -dextype_to_idx GatheredTypes::get_type_index(cmp_dtype cmp) { - return create_index(m_ltype, cmp); +dextype_to_idx* GatheredTypes::get_type_index(cmp_dtype cmp) { + std::sort(m_ltype.begin(), m_ltype.end(), cmp); + dextype_to_idx* sidx = new dextype_to_idx(); + uint32_t idx = 0; + for (auto it = m_ltype.begin(); it != m_ltype.end(); it++) { + sidx->insert(std::make_pair(*it, idx++)); + } + return sidx; } -dexfield_to_idx GatheredTypes::get_field_index(cmp_dfield cmp) { - return create_index(m_lfield, cmp); +dexfield_to_idx* GatheredTypes::get_field_index(cmp_dfield cmp) { + std::sort(m_lfield.begin(), m_lfield.end(), cmp); + dexfield_to_idx* sidx = new dexfield_to_idx(); + uint32_t idx = 0; + for (auto it = m_lfield.begin(); it != m_lfield.end(); it++) { + sidx->insert(std::make_pair(*it, idx++)); + } + return sidx; } -dexmethod_to_idx GatheredTypes::get_method_index(cmp_dmethod cmp) { - return create_index(m_lmethod, cmp); +dexmethod_to_idx* GatheredTypes::get_method_index(cmp_dmethod cmp) { + std::sort(m_lmethod.begin(), m_lmethod.end(), cmp); + dexmethod_to_idx* sidx = new dexmethod_to_idx(); + uint32_t idx = 0; + for (auto it = m_lmethod.begin(); it != m_lmethod.end(); it++) { + sidx->insert(std::make_pair(*it, idx++)); + } + return sidx; } -dexproto_to_idx GatheredTypes::get_proto_index(cmp_dproto cmp) { - // TODO: Profile and see whether a set first would be more efficient. +dexproto_to_idx* GatheredTypes::get_proto_index(cmp_dproto cmp) { std::vector protos; - protos.reserve(m_lmethod.size() + m_lcallsite.size()); for (auto const& m : m_lmethod) { protos.push_back(m->get_proto()); } @@ -421,34 +425,52 @@ dexproto_to_idx GatheredTypes::get_proto_index(cmp_dproto cmp) { } std::sort(protos.begin(), protos.end()); protos.erase(std::unique(protos.begin(), protos.end()), protos.end()); - return create_index(protos, cmp); + std::sort(protos.begin(), protos.end(), cmp); + dexproto_to_idx* sidx = new dexproto_to_idx(); + uint32_t idx = 0; + for (auto const& proto : protos) { + sidx->insert(std::make_pair(proto, idx++)); + } + return sidx; } -std::vector GatheredTypes::get_typelist_list( +std::vector* GatheredTypes::get_typelist_list( dexproto_to_idx* protos, cmp_dtypelist cmp) { - std::vector typel{}; + std::vector* typel = new std::vector(); auto class_defs_size = (uint32_t)m_classes->size(); - typel.reserve(protos->size() + class_defs_size); + typel->reserve(protos->size() + class_defs_size); for (auto& it : *protos) { auto proto = it.first; - typel.push_back(proto->get_args()); + typel->push_back(proto->get_args()); } for (uint32_t i = 0; i < class_defs_size; i++) { DexClass* clz = m_classes->at(i); - typel.push_back(clz->get_interfaces()); + typel->push_back(clz->get_interfaces()); } - sort_unique(typel, compare_dextypelists); + sort_unique(*typel, compare_dextypelists); return typel; } -dexcallsite_to_idx GatheredTypes::get_callsite_index(cmp_callsite cmp) { - return create_index(m_lcallsite, cmp); +dexcallsite_to_idx* GatheredTypes::get_callsite_index(cmp_callsite cmp) { + std::sort(m_lcallsite.begin(), m_lcallsite.end(), cmp); + dexcallsite_to_idx* csidx = new dexcallsite_to_idx(); + uint32_t idx = 0; + for (auto it = m_lcallsite.begin(); it != m_lcallsite.end(); it++) { + csidx->insert(std::make_pair(*it, idx++)); + } + return csidx; } -dexmethodhandle_to_idx GatheredTypes::get_methodhandle_index( +dexmethodhandle_to_idx* GatheredTypes::get_methodhandle_index( cmp_methodhandle cmp) { - return create_index(m_lmethodhandle, cmp); + std::sort(m_lmethodhandle.begin(), m_lmethodhandle.end(), cmp); + dexmethodhandle_to_idx* mhidx = new dexmethodhandle_to_idx(); + uint32_t idx = 0; + for (auto it = m_lmethodhandle.begin(); it != m_lmethodhandle.end(); it++) { + mhidx->insert(std::make_pair(*it, idx++)); + } + return mhidx; } void GatheredTypes::build_cls_load_map() { @@ -583,15 +605,14 @@ DexOutput::DexOutput( const DexOutputConfig& dex_output_config, int min_sdk) : m_classes(classes), + m_gtypes(std::move(gtypes)), + // Required because the BytecodeDebugger setting creates huge amounts + // of debug information (multiple dex debug entries per instruction) m_output_size((debug_info_kind == DebugInfoKind::BytecodeDebugger ? get_dex_output_size(config_files) * 2 : get_dex_output_size(config_files)) + k_output_red_zone), m_output(std::make_unique(m_output_size)), - m_gtypes(std::move(gtypes)), - m_dodx(m_gtypes->get_dodx(m_output.get())), - // Required because the BytecodeDebugger setting creates huge amounts - // of debug information (multiple dex debug entries per instruction) m_offset(0), m_iodi_metadata(iodi_metadata), m_config_files(config_files), @@ -600,19 +621,21 @@ DexOutput::DexOutput( // Ensure a clean slate. memset(m_output.get(), 0, m_output_size); + m_dodx = std::make_unique(*m_gtypes->get_dodx(m_output.get())); + always_assert_log( - m_dodx.method_to_idx().size() <= kMaxMethodRefs, + m_dodx->method_to_idx().size() <= kMaxMethodRefs, "Trying to encode too many method refs in dex %s: %zu (limit: %zu). Run " "with check_properties_deep turned on.", boost::filesystem::path(path).filename().c_str(), - m_dodx.method_to_idx().size(), + m_dodx->method_to_idx().size(), kMaxMethodRefs); always_assert_log( - m_dodx.field_to_idx().size() <= kMaxFieldRefs, + m_dodx->field_to_idx().size() <= kMaxFieldRefs, "Trying to encode too many field refs in dex %s: %zu (limit: %zu). Run " "with check_properties_deep turned on.", boost::filesystem::path(path).filename().c_str(), - m_dodx.field_to_idx().size(), + m_dodx->field_to_idx().size(), kMaxFieldRefs); m_filename = path; @@ -820,7 +843,7 @@ void DexOutput::generate_string_data(SortMode mode) { if (m_locator_index != nullptr) { locators += 3; - always_assert(m_dodx.stringidx(DexString::make_string("")) == 0); + always_assert(m_dodx->stringidx(DexString::make_string("")) == 0); } size_t nrstr = string_order.size() + locators; @@ -837,7 +860,7 @@ void DexOutput::generate_string_data(SortMode mode) { // Emit name-based lookup acceleration information for string with index 0 // if requested - uint32_t idx = m_dodx.stringidx(str); + uint32_t idx = m_dodx->stringidx(str); if (idx == 0 && m_locator_index != nullptr) { always_assert(!locator); unsigned orig_offset = m_offset; @@ -924,26 +947,26 @@ void DexOutput::emit_magic_locators() { void DexOutput::generate_type_data() { always_assert_log( - m_dodx.type_to_idx().size() < get_max_type_refs(m_min_sdk), + m_dodx->type_to_idx().size() < get_max_type_refs(m_min_sdk), "Trying to encode too many type refs in dex %zu: %zu (limit: %zu).\n" "NOTE: Please check InterDexPass config flags and set: " "`reserved_trefs: %zu` (or larger, until the issue goes away)", m_dex_number, - m_dodx.type_to_idx().size(), + m_dodx->type_to_idx().size(), get_max_type_refs(m_min_sdk), - m_dodx.type_to_idx().size() - get_max_type_refs(m_min_sdk)); + m_dodx->type_to_idx().size() - get_max_type_refs(m_min_sdk)); dex_type_id* typeids = (dex_type_id*)(m_output.get() + hdr.type_ids_off); - for (auto& p : m_dodx.type_to_idx()) { + for (auto& p : m_dodx->type_to_idx()) { auto t = p.first; auto idx = p.second; - typeids[idx].string_idx = m_dodx.stringidx(t->get_name()); + typeids[idx].string_idx = m_dodx->stringidx(t->get_name()); m_stats.num_types++; } } void DexOutput::generate_typelist_data() { - const auto& typel = m_dodx.typelist_list(); + std::vector& typel = m_dodx->typelist_list(); uint32_t tl_start = align(m_offset); size_t num_tls = 0; for (DexTypeList* tl : typel) { @@ -954,7 +977,7 @@ void DexOutput::generate_typelist_data() { ++num_tls; align_output(); m_tl_emit_offsets[tl] = m_offset; - int size = tl->encode(&m_dodx, (uint32_t*)(m_output.get() + m_offset)); + int size = tl->encode(m_dodx.get(), (uint32_t*)(m_output.get() + m_offset)); inc_offset(size); m_stats.num_type_lists++; } @@ -965,11 +988,11 @@ void DexOutput::generate_typelist_data() { void DexOutput::generate_proto_data() { auto protoids = (dex_proto_id*)(m_output.get() + hdr.proto_ids_off); - for (auto& it : m_dodx.proto_to_idx()) { + for (auto& it : m_dodx->proto_to_idx()) { auto proto = it.first; auto idx = it.second; - protoids[idx].shortyidx = m_dodx.stringidx(proto->get_shorty()); - protoids[idx].rtypeidx = m_dodx.typeidx(proto->get_rtype()); + protoids[idx].shortyidx = m_dodx->stringidx(proto->get_shorty()); + protoids[idx].rtypeidx = m_dodx->typeidx(proto->get_rtype()); protoids[idx].param_off = m_tl_emit_offsets.at(proto->get_args()); m_stats.num_protos++; } @@ -977,24 +1000,24 @@ void DexOutput::generate_proto_data() { void DexOutput::generate_field_data() { auto fieldids = (dex_field_id*)(m_output.get() + hdr.field_ids_off); - for (auto& it : m_dodx.field_to_idx()) { + for (auto& it : m_dodx->field_to_idx()) { auto field = it.first; auto idx = it.second; - fieldids[idx].classidx = m_dodx.typeidx(field->get_class()); - fieldids[idx].typeidx = m_dodx.typeidx(field->get_type()); - fieldids[idx].nameidx = m_dodx.stringidx(field->get_name()); + fieldids[idx].classidx = m_dodx->typeidx(field->get_class()); + fieldids[idx].typeidx = m_dodx->typeidx(field->get_type()); + fieldids[idx].nameidx = m_dodx->stringidx(field->get_name()); m_stats.num_field_refs++; } } void DexOutput::generate_method_data() { auto methodids = (dex_method_id*)(m_output.get() + hdr.method_ids_off); - for (auto& it : m_dodx.method_to_idx()) { + for (auto& it : m_dodx->method_to_idx()) { auto method = it.first; auto idx = it.second; - methodids[idx].classidx = m_dodx.typeidx(method->get_class()); - methodids[idx].protoidx = m_dodx.protoidx(method->get_proto()); - methodids[idx].nameidx = m_dodx.stringidx(method->get_name()); + methodids[idx].classidx = m_dodx->typeidx(method->get_class()); + methodids[idx].protoidx = m_dodx->protoidx(method->get_proto()); + methodids[idx].nameidx = m_dodx->stringidx(method->get_name()); m_stats.num_method_refs++; } } @@ -1004,15 +1027,15 @@ void DexOutput::generate_class_data() { for (uint32_t i = 0; i < hdr.class_defs_size; i++) { m_stats.num_classes++; DexClass* clz = m_classes->at(i); - cdefs[i].typeidx = m_dodx.typeidx(clz->get_type()); + cdefs[i].typeidx = m_dodx->typeidx(clz->get_type()); cdefs[i].access_flags = clz->get_access(); - cdefs[i].super_idx = m_dodx.typeidx(clz->get_super_class()); + cdefs[i].super_idx = m_dodx->typeidx(clz->get_super_class()); cdefs[i].interfaces_off = 0; cdefs[i].annotations_off = 0; cdefs[i].interfaces_off = m_tl_emit_offsets[clz->get_interfaces()]; auto source_file = m_pos_mapper->get_source_file(clz); if (source_file != nullptr) { - cdefs[i].source_file_idx = m_dodx.stringidx(source_file); + cdefs[i].source_file_idx = m_dodx->stringidx(source_file); } else { cdefs[i].source_file_idx = DEX_NO_INDEX; } @@ -1044,7 +1067,7 @@ void DexOutput::generate_class_data_items() { DexClass* clz = m_classes->at(i); if (!clz->has_class_data()) continue; /* No alignment constraints for this data */ - int size = clz->encode(&m_dodx, dco, m_output.get() + m_offset); + int size = clz->encode(m_dodx.get(), dco, m_output.get() + m_offset); if (m_dex_output_config.write_class_sizes) { m_stats.class_size[clz] = size; } @@ -1131,7 +1154,8 @@ void DexOutput::generate_code_items(const std::vector& mode) { "Undefined method in generate_code_items()\n\t prototype: %s\n", SHOW(meth)); align_output(); - int size = code->encode(&m_dodx, (uint32_t*)(m_output.get() + m_offset)); + int size = + code->encode(m_dodx.get(), (uint32_t*)(m_output.get() + m_offset)); check_method_instruction_size_limit(m_config_files, size, SHOW(meth)); m_method_bytecode_offsets.emplace_back(meth->get_name()->c_str(), m_offset); m_code_item_emits.emplace_back(meth, code, @@ -1140,7 +1164,6 @@ void DexOutput::generate_code_items(const std::vector& mode) { ((const dex_code_item*)(m_output.get() + m_offset))->insns_size; inc_offset(size); m_stats.num_instructions += code->get_instructions().size(); - m_stats.num_tries += code->get_tries().size(); m_stats.instruction_bytes += insns_size * 2; } /// insert_map_item returns early if m_code_item_emits is empty @@ -1163,23 +1186,23 @@ void DexOutput::generate_callsite_data() { void DexOutput::generate_methodhandle_data() { uint32_t total_callsite_size = - m_dodx.callsitesize() * sizeof(dex_callsite_id); + m_dodx->callsitesize() * sizeof(dex_callsite_id); uint32_t offset = hdr.class_defs_off + hdr.class_defs_size * sizeof(dex_class_def) + total_callsite_size; dex_methodhandle_id* dexmethodhandles = (dex_methodhandle_id*)(m_output.get() + offset); - for (auto it : m_dodx.methodhandle_to_idx()) { + for (auto it : m_dodx->methodhandle_to_idx()) { m_stats.num_methodhandles++; DexMethodHandle* methodhandle = it.first; uint32_t idx = it.second; dexmethodhandles[idx].method_handle_type = methodhandle->type(); if (DexMethodHandle::isInvokeType(methodhandle->type())) { dexmethodhandles[idx].field_or_method_id = - m_dodx.methodidx(methodhandle->methodref()); + m_dodx->methodidx(methodhandle->methodref()); } else { dexmethodhandles[idx].field_or_method_id = - m_dodx.fieldidx(methodhandle->fieldref()); + m_dodx->fieldidx(methodhandle->fieldref()); } dexmethodhandles[idx].unused1 = 0; dexmethodhandles[idx].unused2 = 0; @@ -1224,8 +1247,8 @@ void DexOutput::generate_static_values() { uint8_t* output = m_output.get() + m_offset; uint8_t* outputsv = output; /* No alignment requirements */ - deva->encode(&m_dodx, output); - enc_arrays.emplace(std::move(*deva), m_offset); + deva->encode(m_dodx.get(), output); + enc_arrays.emplace(std::move(*deva.release()), m_offset); m_static_values[clz] = m_offset; inc_offset(output - outputsv); m_stats.num_static_values++; @@ -1241,7 +1264,7 @@ void DexOutput::generate_static_values() { } else { uint8_t* output = m_output.get() + m_offset; uint8_t* outputsv = output; - eva.encode(&m_dodx, output); + eva.encode(m_dodx.get(), output); enc_arrays.emplace(std::move(eva), m_offset); m_call_site_items[callsite] = m_offset; inc_offset(output - outputsv); @@ -1268,7 +1291,7 @@ void DexOutput::unique_annotations(annomap_t& annomap, for (auto anno : annolist) { if (annomap.count(anno)) continue; std::vector annotation_bytes; - anno->vencode(&m_dodx, annotation_bytes); + anno->vencode(m_dodx.get(), annotation_bytes); if (annotation_byte_offsets.count(annotation_bytes)) { annomap[anno] = annotation_byte_offsets[annotation_bytes]; continue; @@ -1298,7 +1321,7 @@ void DexOutput::unique_asets(annomap_t& annomap, for (auto aset : asetlist) { if (asetmap.count(aset)) continue; std::vector aset_bytes; - aset->vencode(&m_dodx, aset_bytes, annomap); + aset->vencode(m_dodx.get(), aset_bytes, annomap); if (aset_offsets.count(aset_bytes)) { asetmap[aset] = aset_offsets[aset_bytes]; continue; @@ -1365,7 +1388,7 @@ void DexOutput::unique_adirs(asetmap_t& asetmap, for (auto adir : adirlist) { if (adirmap.count(adir)) continue; std::vector adir_bytes; - adir->vencode(&m_dodx, adir_bytes, xrefmap, asetmap); + adir->vencode(m_dodx.get(), adir_bytes, xrefmap, asetmap); if (adir_offsets.count(adir_bytes)) { adirmap[adir] = adir_offsets[adir_bytes]; continue; @@ -2367,7 +2390,7 @@ void DexOutput::generate_debug_items() { bool use_iodi = is_iodi(m_debug_info_kind); if (use_iodi && m_iodi_metadata) { inc_offset(emit_instruction_offset_debug_info( - &m_dodx, + m_dodx.get(), m_pos_mapper, m_code_item_emits, *m_iodi_metadata, @@ -2391,7 +2414,7 @@ void DexOutput::generate_debug_items() { if (dbg == nullptr) continue; dbgcount++; size_t num_params = it.method->get_proto()->get_args()->size(); - inc_offset(emit_debug_info(&m_dodx, emit_positions, dbg, dc, dci, + inc_offset(emit_debug_info(m_dodx.get(), emit_positions, dbg, dc, dci, m_pos_mapper, m_output.get(), m_offset, num_params, m_code_debug_lines)); } @@ -2465,38 +2488,38 @@ void DexOutput::init_header_offsets(const std::string& dex_magic) { hdr.endian_tag = ENDIAN_CONSTANT; /* Link section was never used */ hdr.link_size = hdr.link_off = 0; - hdr.string_ids_size = (uint32_t)m_dodx.stringsize(); + hdr.string_ids_size = (uint32_t)m_dodx->stringsize(); hdr.string_ids_off = hdr.string_ids_size ? m_offset : 0; - uint32_t total_string_size = m_dodx.stringsize() * sizeof(dex_string_id); - insert_map_item(TYPE_STRING_ID_ITEM, (uint32_t)m_dodx.stringsize(), m_offset, + uint32_t total_string_size = m_dodx->stringsize() * sizeof(dex_string_id); + insert_map_item(TYPE_STRING_ID_ITEM, (uint32_t)m_dodx->stringsize(), m_offset, total_string_size); inc_offset(total_string_size); - hdr.type_ids_size = (uint32_t)m_dodx.typesize(); + hdr.type_ids_size = (uint32_t)m_dodx->typesize(); hdr.type_ids_off = hdr.type_ids_size ? m_offset : 0; - uint32_t total_type_size = m_dodx.typesize() * sizeof(dex_type_id); - insert_map_item(TYPE_TYPE_ID_ITEM, (uint32_t)m_dodx.typesize(), m_offset, + uint32_t total_type_size = m_dodx->typesize() * sizeof(dex_type_id); + insert_map_item(TYPE_TYPE_ID_ITEM, (uint32_t)m_dodx->typesize(), m_offset, total_type_size); inc_offset(total_type_size); - hdr.proto_ids_size = (uint32_t)m_dodx.protosize(); + hdr.proto_ids_size = (uint32_t)m_dodx->protosize(); hdr.proto_ids_off = hdr.proto_ids_size ? m_offset : 0; - uint32_t total_proto_size = m_dodx.protosize() * sizeof(dex_proto_id); - insert_map_item(TYPE_PROTO_ID_ITEM, (uint32_t)m_dodx.protosize(), m_offset, + uint32_t total_proto_size = m_dodx->protosize() * sizeof(dex_proto_id); + insert_map_item(TYPE_PROTO_ID_ITEM, (uint32_t)m_dodx->protosize(), m_offset, total_proto_size); inc_offset(total_proto_size); - hdr.field_ids_size = (uint32_t)m_dodx.fieldsize(); + hdr.field_ids_size = (uint32_t)m_dodx->fieldsize(); hdr.field_ids_off = hdr.field_ids_size ? m_offset : 0; - uint32_t total_field_size = m_dodx.fieldsize() * sizeof(dex_field_id); - insert_map_item(TYPE_FIELD_ID_ITEM, (uint32_t)m_dodx.fieldsize(), m_offset, + uint32_t total_field_size = m_dodx->fieldsize() * sizeof(dex_field_id); + insert_map_item(TYPE_FIELD_ID_ITEM, (uint32_t)m_dodx->fieldsize(), m_offset, total_field_size); inc_offset(total_field_size); - hdr.method_ids_size = (uint32_t)m_dodx.methodsize(); + hdr.method_ids_size = (uint32_t)m_dodx->methodsize(); hdr.method_ids_off = hdr.method_ids_size ? m_offset : 0; - uint32_t total_method_size = m_dodx.methodsize() * sizeof(dex_method_id); - insert_map_item(TYPE_METHOD_ID_ITEM, (uint32_t)m_dodx.methodsize(), m_offset, + uint32_t total_method_size = m_dodx->methodsize() * sizeof(dex_method_id); + insert_map_item(TYPE_METHOD_ID_ITEM, (uint32_t)m_dodx->methodsize(), m_offset, total_method_size); inc_offset(total_method_size); @@ -2509,14 +2532,14 @@ void DexOutput::init_header_offsets(const std::string& dex_magic) { inc_offset(total_class_size); uint32_t total_callsite_size = - m_dodx.callsitesize() * sizeof(dex_callsite_id); - insert_map_item(TYPE_CALL_SITE_ID_ITEM, (uint32_t)m_dodx.callsitesize(), + m_dodx->callsitesize() * sizeof(dex_callsite_id); + insert_map_item(TYPE_CALL_SITE_ID_ITEM, (uint32_t)m_dodx->callsitesize(), m_offset, total_callsite_size); inc_offset(total_callsite_size); uint32_t total_methodhandle_size = - m_dodx.methodhandlesize() * sizeof(dex_methodhandle_id); - insert_map_item(TYPE_METHOD_HANDLE_ITEM, (uint32_t)m_dodx.methodhandlesize(), + m_dodx->methodhandlesize() * sizeof(dex_methodhandle_id); + insert_map_item(TYPE_METHOD_HANDLE_ITEM, (uint32_t)m_dodx->methodhandlesize(), m_offset, total_methodhandle_size); inc_offset(total_methodhandle_size); @@ -2907,7 +2930,7 @@ void write_bytecode_offset_mapping( void DexOutput::write_symbol_files() { if (m_debug_info_kind != DebugInfoKind::NoCustomSymbolication) { - write_method_mapping(m_method_mapping_filename, &m_dodx, m_classes, + write_method_mapping(m_method_mapping_filename, m_dodx.get(), m_classes, hdr.signature); write_class_mapping(m_class_mapping_filename, m_classes, hdr.class_defs_size, hdr.signature); @@ -2927,7 +2950,7 @@ void DexOutput::prepare(SortMode string_mode, const std::string& dex_magic) { m_gtypes->set_config(&conf); - fix_jumbos(m_classes, &m_dodx); + fix_jumbos(m_classes, m_dodx.get()); init_header_offsets(dex_magic); generate_static_values(); generate_typelist_data(); @@ -2945,7 +2968,8 @@ void DexOutput::prepare(SortMode string_mode, generate_debug_items(); generate_map(); finalize_header(); - compute_method_to_id_map(&m_dodx, m_classes, hdr.signature, m_method_to_id); + compute_method_to_id_map(m_dodx.get(), m_classes, hdr.signature, + m_method_to_id); } void DexOutput::write() { @@ -2996,39 +3020,39 @@ void DexOutput::metrics() { } memcpy(m_stats.signature, hdr.signature, 20); - for (auto& p : m_dodx.string_to_idx()) { + for (auto& p : m_dodx->string_to_idx()) { s_unique_references.strings.insert(p.first); } m_stats.num_unique_strings = s_unique_references.strings.size(); - s_unique_references.total_strings_size += m_dodx.string_to_idx().size(); + s_unique_references.total_strings_size += m_dodx->string_to_idx().size(); m_stats.strings_total_size = s_unique_references.total_strings_size; - for (auto& p : m_dodx.type_to_idx()) { + for (auto& p : m_dodx->type_to_idx()) { s_unique_references.types.insert(p.first); } m_stats.num_unique_types = s_unique_references.types.size(); - s_unique_references.total_types_size += m_dodx.type_to_idx().size(); + s_unique_references.total_types_size += m_dodx->type_to_idx().size(); m_stats.types_total_size = s_unique_references.total_types_size; - for (auto& p : m_dodx.proto_to_idx()) { + for (auto& p : m_dodx->proto_to_idx()) { s_unique_references.protos.insert(p.first); } m_stats.num_unique_protos = s_unique_references.protos.size(); - s_unique_references.total_protos_size += m_dodx.proto_to_idx().size(); + s_unique_references.total_protos_size += m_dodx->proto_to_idx().size(); m_stats.protos_total_size = s_unique_references.total_protos_size; - for (auto& p : m_dodx.field_to_idx()) { + for (auto& p : m_dodx->field_to_idx()) { s_unique_references.fields.insert(p.first); } m_stats.num_unique_field_refs = s_unique_references.fields.size(); - s_unique_references.total_fields_size += m_dodx.field_to_idx().size(); + s_unique_references.total_fields_size += m_dodx->field_to_idx().size(); m_stats.field_refs_total_size = s_unique_references.total_fields_size; - for (auto& p : m_dodx.method_to_idx()) { + for (auto& p : m_dodx->method_to_idx()) { s_unique_references.methods.insert(p.first); } m_stats.num_unique_method_refs = s_unique_references.methods.size(); - s_unique_references.total_methods_size += m_dodx.method_to_idx().size(); + s_unique_references.total_methods_size += m_dodx->method_to_idx().size(); m_stats.method_refs_total_size = s_unique_references.total_methods_size; } diff --git a/libredex/DexOutput.h b/libredex/DexOutput.h index 2eb18033f8..193c501af0 100644 --- a/libredex/DexOutput.h +++ b/libredex/DexOutput.h @@ -54,70 +54,76 @@ enum class SortMode { class DexOutputIdx { private: - dexstring_to_idx m_string; - dextype_to_idx m_type; - dexproto_to_idx m_proto; - dexfield_to_idx m_field; - dexmethod_to_idx m_method; - std::vector m_typelist; - dexcallsite_to_idx m_callsite; - dexmethodhandle_to_idx m_methodhandle; + dexstring_to_idx* m_string; + dextype_to_idx* m_type; + dexproto_to_idx* m_proto; + dexfield_to_idx* m_field; + dexmethod_to_idx* m_method; + std::vector* m_typelist; + dexcallsite_to_idx* m_callsite; + dexmethodhandle_to_idx* m_methodhandle; const uint8_t* m_base; public: - DexOutputIdx(dexstring_to_idx string, - dextype_to_idx type, - dexproto_to_idx proto, - dexfield_to_idx field, - dexmethod_to_idx method, - std::vector typelist, - dexcallsite_to_idx callsite, - dexmethodhandle_to_idx methodhandle, - const uint8_t* base) - : m_string(std::move(string)), - m_type(std::move(type)), - m_proto(std::move(proto)), - m_field(std::move(field)), - m_method(std::move(method)), - m_typelist(std::move(typelist)), - m_callsite(std::move(callsite)), - m_methodhandle(std::move(methodhandle)), - m_base(base) {} - - DexOutputIdx(const DexOutputIdx&) = delete; - DexOutputIdx& operator=(const DexOutputIdx&) = delete; - - DexOutputIdx(DexOutputIdx&&) = default; - DexOutputIdx& operator=(DexOutputIdx&&) = default; - - const dexstring_to_idx& string_to_idx() const { return m_string; } - const dextype_to_idx& type_to_idx() const { return m_type; } - const dexproto_to_idx& proto_to_idx() const { return m_proto; } - const dexfield_to_idx& field_to_idx() const { return m_field; } - const dexmethod_to_idx& method_to_idx() const { return m_method; } - const std::vector& typelist_list() const { return m_typelist; } - const dexcallsite_to_idx& callsite_to_idx() const { return m_callsite; } - const dexmethodhandle_to_idx& methodhandle_to_idx() const { - return m_methodhandle; + DexOutputIdx(dexstring_to_idx* string, + dextype_to_idx* type, + dexproto_to_idx* proto, + dexfield_to_idx* field, + dexmethod_to_idx* method, + std::vector* typelist, + dexcallsite_to_idx* callsite, + dexmethodhandle_to_idx* methodhandle, + const uint8_t* base) { + m_string = string; + m_type = type; + m_proto = proto; + m_field = field; + m_method = method; + m_typelist = typelist; + m_callsite = callsite; + m_methodhandle = methodhandle; + m_base = base; } - uint32_t stringidx(const DexString* s) const { return m_string.at(s); } - uint16_t typeidx(DexType* t) const { return m_type.at(t); } - uint16_t protoidx(DexProto* p) const { return m_proto.at(p); } - uint32_t fieldidx(DexFieldRef* f) const { return m_field.at(f); } - uint32_t methodidx(DexMethodRef* m) const { return m_method.at(m); } - uint32_t callsiteidx(DexCallSite* c) const { return m_callsite.at(c); } + ~DexOutputIdx() { + delete m_string; + delete m_type; + delete m_proto; + delete m_field; + delete m_method; + delete m_typelist; + delete m_callsite; + delete m_methodhandle; + } + + dexstring_to_idx& string_to_idx() const { return *m_string; } + dextype_to_idx& type_to_idx() const { return *m_type; } + dexproto_to_idx& proto_to_idx() const { return *m_proto; } + dexfield_to_idx& field_to_idx() const { return *m_field; } + dexmethod_to_idx& method_to_idx() const { return *m_method; } + std::vector& typelist_list() const { return *m_typelist; } + dexcallsite_to_idx& callsite_to_idx() const { return *m_callsite; } + dexmethodhandle_to_idx& methodhandle_to_idx() const { + return *m_methodhandle; + } + + uint32_t stringidx(const DexString* s) const { return m_string->at(s); } + uint16_t typeidx(DexType* t) const { return m_type->at(t); } + uint16_t protoidx(DexProto* p) const { return m_proto->at(p); } + uint32_t fieldidx(DexFieldRef* f) const { return m_field->at(f); } + uint32_t methodidx(DexMethodRef* m) const { return m_method->at(m); } + uint32_t callsiteidx(DexCallSite* c) const { return m_callsite->at(c); } uint32_t methodhandleidx(DexMethodHandle* c) const { - return m_methodhandle.at(c); + return m_methodhandle->at(c); } - size_t stringsize() const { return m_string.size(); } - size_t typesize() const { return m_type.size(); } - size_t protosize() const { return m_proto.size(); } - size_t fieldsize() const { return m_field.size(); } - size_t methodsize() const { return m_method.size(); } - size_t callsitesize() const { return m_callsite.size(); } - size_t methodhandlesize() const { return m_methodhandle.size(); } + size_t stringsize() const { return m_string->size(); } + size_t typesize() const { return m_type->size(); } + size_t protosize() const { return m_proto->size(); } + size_t fieldsize() const { return m_field->size(); } + size_t methodsize() const { return m_method->size(); } + size_t callsitesize() const { return m_callsite->size(); } + size_t methodhandlesize() const { return m_methodhandle->size(); } uint32_t get_offset(uint8_t* ptr) { return (uint32_t)(ptr - m_base); } @@ -241,16 +247,16 @@ class GatheredTypes { std::unordered_map m_methods_in_cls_order; ConfigFiles* m_config{nullptr}; - dexstring_to_idx get_string_index(cmp_dstring cmp = compare_dexstrings); - dextype_to_idx get_type_index(cmp_dtype cmp = compare_dextypes); - dexproto_to_idx get_proto_index(cmp_dproto cmp = compare_dexprotos); - dexfield_to_idx get_field_index(cmp_dfield cmp = compare_dexfields); - dexmethod_to_idx get_method_index(cmp_dmethod cmp = compare_dexmethods); - std::vector get_typelist_list( + dexstring_to_idx* get_string_index(cmp_dstring cmp = compare_dexstrings); + dextype_to_idx* get_type_index(cmp_dtype cmp = compare_dextypes); + dexproto_to_idx* get_proto_index(cmp_dproto cmp = compare_dexprotos); + dexfield_to_idx* get_field_index(cmp_dfield cmp = compare_dexfields); + dexmethod_to_idx* get_method_index(cmp_dmethod cmp = compare_dexmethods); + std::vector* get_typelist_list( dexproto_to_idx* protos, cmp_dtypelist cmp = compare_dextypelists); - dexcallsite_to_idx get_callsite_index( + dexcallsite_to_idx* get_callsite_index( cmp_callsite cmp = compare_dexcallsites); - dexmethodhandle_to_idx get_methodhandle_index( + dexmethodhandle_to_idx* get_methodhandle_index( cmp_methodhandle cmp = compare_dexmethodhandles); void build_cls_load_map(); @@ -260,7 +266,7 @@ class GatheredTypes { public: explicit GatheredTypes(DexClasses* classes); - DexOutputIdx get_dodx(const uint8_t* base); + DexOutputIdx* get_dodx(const uint8_t* base); template std::vector get_dexstring_emitlist( T cmp = compare_dexstrings); @@ -322,10 +328,10 @@ class DexOutput { private: DexClasses* m_classes; + std::unique_ptr m_dodx; + std::shared_ptr m_gtypes; const size_t m_output_size; std::unique_ptr m_output; - std::shared_ptr m_gtypes; - DexOutputIdx m_dodx; uint32_t m_offset; const char* m_filename; size_t m_store_number; diff --git a/libredex/DexPosition.cpp b/libredex/DexPosition.cpp index 6c7356a609..ef55fa405a 100644 --- a/libredex/DexPosition.cpp +++ b/libredex/DexPosition.cpp @@ -7,16 +7,13 @@ #include #include -#include -#include "ConcurrentContainers.h" #include "DexClass.h" #include "DexPosition.h" #include "DexUtil.h" #include "RedexContext.h" #include "Show.h" #include "Trace.h" -#include "WorkQueue.h" DexPosition::DexPosition(const DexString* file, uint32_t line) : file(file), line(line) { @@ -128,19 +125,7 @@ std::unique_ptr PositionPatternSwitchManager::make_switch_position( void RealPositionMapper::register_position(DexPosition* pos) { always_assert(pos->file); - auto [_, emplaced] = m_pos_line_map.emplace(pos, -1); - if (emplaced) { - m_possibly_incomplete_positions.push(pos); - } -} - -int64_t RealPositionMapper::add_position(DexPosition* pos) { - auto [it, _] = m_pos_line_map.emplace(pos, -1); - if (it->second == -1) { - it->second = m_positions.size(); - m_positions.push_back(pos); - } - return it->second; + m_pos_line_map[pos] = -1; } uint32_t RealPositionMapper::get_line(DexPosition* pos) { @@ -148,7 +133,10 @@ uint32_t RealPositionMapper::get_line(DexPosition* pos) { } uint32_t RealPositionMapper::position_to_line(DexPosition* pos) { - return add_position(pos) + 1; + auto idx = m_positions.size(); + m_positions.emplace_back(pos); + m_pos_line_map[pos] = idx; + return get_line(pos); } void RealPositionMapper::write_map() { @@ -165,46 +153,40 @@ void RealPositionMapper::process_pattern_switch_positions() { // First. we find all reachable patterns, switches and cases. auto switches = manager->get_switches(); - ConcurrentMap> reachable_patterns; - InsertOnlyConcurrentSet reachable_switches; - InsertOnlyConcurrentSet visited; + std::unordered_set reachable_patterns; + std::unordered_set reachable_switches; + std::unordered_set visited; + std::unordered_map> pending; std::function visit; visit = [&](DexPosition* pos) { always_assert(pos); for (; pos && visited.insert(pos).second; pos = pos->parent) { if (manager->is_pattern_position(pos)) { - std::vector cases; - reachable_patterns.update(pos->line, - [&](auto, auto& pending, bool exists) { - if (exists) { - cases = std::move(pending); - } - }); - for (auto& c : cases) { - visit(c.position); + if (reachable_patterns.insert(pos->line).second) { + auto it = pending.find(pos->line); + if (it != pending.end()) { + for (auto c : it->second) { + visit(c.position); + } + pending.erase(pos->line); + } } } else if (manager->is_switch_position(pos)) { if (reachable_switches.insert(pos->line).second) { for (auto& c : switches.at(pos->line)) { - bool pattern_reachable = false; - reachable_patterns.update(c.pattern_id, - [&](auto, auto& pending, bool exists) { - if (exists && pending.empty()) { - pattern_reachable = true; - return; - } - pending.push_back(c); - }); - if (pattern_reachable) { + if (reachable_patterns.count(c.pattern_id)) { visit(c.position); + } else { + pending[c.pattern_id].push_back(c); } } } } } }; - - workqueue_run(visit, m_positions); + for (auto pos : m_positions) { + visit(pos); + } auto count_string = DexString::make_string("Lredex/$Position;.count:()V"); auto case_string = DexString::make_string("Lredex/$Position;.case:()V"); @@ -236,12 +218,14 @@ void RealPositionMapper::process_pattern_switch_positions() { continue; } for (auto pos = c.position; pos && pos->file; pos = pos->parent) { - auto [it, emplaced] = m_pos_line_map.emplace(pos, m_positions.size()); - if (emplaced) { - m_positions.push_back(pos); - } else { + auto it = m_pos_line_map.find(pos); + if (it != m_pos_line_map.end()) { always_assert(it->second != -1); + break; } + auto idx = m_positions.size(); + m_positions.emplace_back(pos); + m_pos_line_map.emplace(pos, idx); } reachable_cases.push_back(c); } @@ -257,7 +241,9 @@ void RealPositionMapper::process_pattern_switch_positions() { auto count_pos = new DexPosition(count_string, unknown_source_string, reachable_cases.size()); m_owned_auxiliary_positions.emplace_back(count_pos); - add_position(count_pos); + auto idx = m_positions.size(); + m_positions.emplace_back(count_pos); + m_pos_line_map[count_pos] = idx; } // Then we emit consecutive list of cases for (auto& c : reachable_cases) { @@ -267,7 +253,9 @@ void RealPositionMapper::process_pattern_switch_positions() { always_assert(c.position); always_assert(c.position->file); case_pos->parent = c.position; - add_position(case_pos); + auto idx = m_positions.size(); + m_positions.emplace_back(case_pos); + m_pos_line_map[case_pos] = idx; } } @@ -295,10 +283,9 @@ uint32_t RealPositionMapper::size() const { return m_positions.size(); } void RealPositionMapper::write_map_v2() { // to ensure that the line numbers in the Dex are as compact as possible, // we put the emitted positions at the start of the list and rest at the end - while (!m_possibly_incomplete_positions.empty()) { - auto* pos = m_possibly_incomplete_positions.front(); - m_possibly_incomplete_positions.pop(); - auto& line = m_pos_line_map[pos]; + for (auto& p : m_pos_line_map) { + auto pos = p.first; + auto& line = p.second; if (line == -1) { auto idx = m_positions.size(); m_positions.emplace_back(pos); @@ -321,104 +308,55 @@ void RealPositionMapper::write_map_v2() { * string_length (4 bytes) * char[string_length] */ - - // We initially build a somewhat dense mapping of strings to ids; the exact - // ids are not deterministic, and there might be some skipped ids due to - // races. - InsertOnlyConcurrentMap semi_dense_string_ids; - InsertOnlyConcurrentMap> string_pool; - std::atomic next_semi_dense_string_id{0}; - auto semi_dense_id_of_string = [&](std::string_view s) -> uint32_t { - // Fast path - const uint32_t* opt_id = semi_dense_string_ids.get(s); - if (opt_id) { - return *opt_id; + std::ostringstream pos_out; + std::unordered_map string_ids; + std::vector> string_pool; + + auto id_of_string = [&](const std::string_view s) -> uint32_t { + auto it = string_ids.find(s); + if (it == string_ids.end()) { + auto p = std::make_unique(s); + it = string_ids.emplace(*p, string_pool.size()).first; + string_pool.push_back(std::move(p)); } - - // Slow path - auto id = next_semi_dense_string_id.fetch_add(1); - always_assert(id < std::numeric_limits::max()); - auto p = std::make_unique(s); - auto [id_ptr, emplaced] = semi_dense_string_ids.emplace(*p, id); - if (emplaced) { - string_pool.emplace(id, std::move(p)); - } // else, we wasted a string-id. Oh well... We'll renumber densely later. - return *id_ptr; + return it->second; }; - // Many DexPositions refer to the same method. We cache its class and method - // string ids. - struct MethodInfo { - uint32_t class_id; - uint32_t method_id; - bool operator==(const MethodInfo& other) const { - return class_id == other.class_id && method_id == other.method_id; - } - }; - InsertOnlyConcurrentMap method_infos; - auto get_method_info = [&](const DexString* method) -> const MethodInfo& { - return *method_infos - .get_or_create_and_assert_equal( - method, - [&semi_dense_id_of_string](auto* m) { - // of the form - // "class_name.method_name:(arg_types)return_type" - const auto full_method_name = m->str(); - // strip out the args and return type - const auto qualified_method_name = - full_method_name.substr(0, - full_method_name.find(':')); - auto class_name = java_names::internal_to_external( - qualified_method_name.substr( - 0, qualified_method_name.rfind('.'))); - auto method_name = qualified_method_name.substr( - qualified_method_name.rfind('.') + 1); - auto class_id = semi_dense_id_of_string(class_name); - auto method_id = semi_dense_id_of_string(method_name); - return MethodInfo{class_id, method_id}; - }) - .first; - }; + size_t unregistered_parent_positions{0}; - std::atomic unregistered_parent_positions{0}; - - static_assert(std::is_sameline), uint32_t>::value); - static_assert( - std::is_same::value); - std::vector pos_data(5 * m_positions.size()); - - // We process positions in parallel in batches to benefit from cache locality. - const size_t BATCH_SIZE = 100; - workqueue_run_for( - 0, (m_positions.size() + BATCH_SIZE - 1) / BATCH_SIZE, [&](size_t batch) { - auto end = std::min(m_positions.size(), (batch + 1) * BATCH_SIZE); - for (size_t idx = batch * BATCH_SIZE; idx < end; ++idx) { - auto* pos = m_positions[idx]; - uint32_t parent_line = 0; - try { - parent_line = pos->parent == nullptr ? 0 : get_line(pos->parent); - } catch (std::out_of_range& e) { - ++unregistered_parent_positions; - TRACE(OPUT, 1, "Parent position %s of %s was not registered", - SHOW(pos->parent), SHOW(pos)); - } - auto [class_id, method_id] = get_method_info(pos->method); - auto file_id = semi_dense_id_of_string(pos->file->str()); - pos_data[5 * idx + 0] = class_id; - pos_data[5 * idx + 1] = method_id; - pos_data[5 * idx + 2] = file_id; - pos_data[5 * idx + 3] = pos->line; - pos_data[5 * idx + 4] = parent_line; - } - }); - always_assert(pos_data.size() == 5 * m_positions.size()); - always_assert(semi_dense_string_ids.size() == string_pool.size()); + for (auto pos : m_positions) { + uint32_t parent_line = 0; + try { + parent_line = pos->parent == nullptr ? 0 : get_line(pos->parent); + } catch (std::out_of_range& e) { + ++unregistered_parent_positions; + TRACE(OPUT, 1, "Parent position %s of %s was not registered", + SHOW(pos->parent), SHOW(pos)); + } + // of the form "class_name.method_name:(arg_types)return_type" + const auto full_method_name = pos->method->str(); + // strip out the args and return type + const auto qualified_method_name = + full_method_name.substr(0, full_method_name.find(':')); + auto class_name = java_names::internal_to_external( + qualified_method_name.substr(0, qualified_method_name.rfind('.'))); + auto method_name = + qualified_method_name.substr(qualified_method_name.rfind('.') + 1); + auto class_id = id_of_string(class_name); + auto method_id = id_of_string(method_name); + auto file_id = id_of_string(pos->file->str()); + pos_out.write((const char*)&class_id, sizeof(class_id)); + pos_out.write((const char*)&method_id, sizeof(method_id)); + pos_out.write((const char*)&file_id, sizeof(file_id)); + pos_out.write((const char*)&pos->line, sizeof(pos->line)); + pos_out.write((const char*)&parent_line, sizeof(parent_line)); + } - if (unregistered_parent_positions.load() > 0 && !traceEnabled(OPUT, 1)) { + if (unregistered_parent_positions > 0 && !traceEnabled(OPUT, 1)) { TRACE(OPUT, 0, "%zu parent positions had not been registered. Run with TRACE=OPUT:1 " "to list them.", - unregistered_parent_positions.load()); + unregistered_parent_positions); } std::ofstream ofs(m_filename_v2.c_str(), @@ -429,39 +367,14 @@ void RealPositionMapper::write_map_v2() { ofs.write((const char*)&version, sizeof(version)); uint32_t spool_count = string_pool.size(); ofs.write((const char*)&spool_count, sizeof(spool_count)); - always_assert(string_pool.size() < std::numeric_limits::max()); - - // Finally, rewrite the string-ids following the deterministic ordering of the - // positions. - auto map = std::make_unique(next_semi_dense_string_id.load()); - const uint32_t unmapped = 0; - const uint32_t first_mapped = 1; - uint32_t next_mapped = first_mapped; - auto order = [&](uint32_t& string_id) { - auto& mapped = map[string_id]; - if (mapped == unmapped) { - const auto& s = string_pool.at(string_id); - uint32_t ssize = s->size(); - ofs.write((const char*)&ssize, sizeof(ssize)); - ofs.write(s->data(), ssize * sizeof(char)); - mapped = next_mapped++; - } - string_id = mapped - first_mapped; - }; - for (size_t idx = 0; idx < m_positions.size(); ++idx) { - order(pos_data[5 * idx + 0]); // class_id - order(pos_data[5 * idx + 1]); // method_id - order(pos_data[5 * idx + 2]); // file_id + for (const auto& s : string_pool) { + uint32_t ssize = s->size(); + ofs.write((const char*)&ssize, sizeof(ssize)); + ofs << *s; } - always_assert(next_mapped - first_mapped == string_pool.size()); uint32_t pos_count = m_positions.size(); ofs.write((const char*)&pos_count, sizeof(pos_count)); - ofs.write((const char*)pos_data.data(), sizeof(uint32_t) * pos_data.size()); - - TRACE(OPUT, 2, - "positions: %zu, string pool size: %zu, semi-dense string ids: %u", - m_positions.size(), string_pool.size(), - next_semi_dense_string_id.load()); + ofs << pos_out.str(); } PositionMapper* PositionMapper::make(const std::string& map_filename_v2) { diff --git a/libredex/DexPosition.h b/libredex/DexPosition.h index 7be775a82e..40d2f5e3b9 100644 --- a/libredex/DexPosition.h +++ b/libredex/DexPosition.h @@ -12,7 +12,6 @@ #include #include #include -#include #include #include #include @@ -168,13 +167,11 @@ class RealPositionMapper : public PositionMapper { std::string m_filename_v2; std::vector m_positions; std::unordered_map m_pos_line_map; - std::queue m_possibly_incomplete_positions; std::vector> m_owned_auxiliary_positions; void process_pattern_switch_positions(); protected: - int64_t add_position(DexPosition* pos); uint32_t get_line(DexPosition*); void write_map_v2(); diff --git a/libredex/DexStats.cpp b/libredex/DexStats.cpp index 68bc655b5f..5f77c49eca 100644 --- a/libredex/DexStats.cpp +++ b/libredex/DexStats.cpp @@ -21,7 +21,6 @@ dex_stats_t& dex_stats_t::operator+=(const dex_stats_t& rhs) { num_type_lists += rhs.num_type_lists; num_bytes += rhs.num_bytes; num_instructions += rhs.num_instructions; - num_tries += rhs.num_tries; num_unique_types += rhs.num_unique_types; num_unique_protos += rhs.num_unique_protos; num_unique_strings += rhs.num_unique_strings; diff --git a/libredex/DexStats.h b/libredex/DexStats.h index de23821992..0750fd963e 100644 --- a/libredex/DexStats.h +++ b/libredex/DexStats.h @@ -27,7 +27,6 @@ struct dex_stats_t { int num_instructions = 0; int num_callsites = 0; int num_methodhandles = 0; - int num_tries = 0; int num_unique_strings = 0; int num_unique_types = 0; diff --git a/libredex/DexStore.cpp b/libredex/DexStore.cpp index 9700af6a4d..eb78a4cc6c 100644 --- a/libredex/DexStore.cpp +++ b/libredex/DexStore.cpp @@ -239,32 +239,31 @@ XStoreRefs::XStoreRefs(const DexStoresVector& stores, m_reverse_dependencies(build_reverse_dependencies(stores)), m_shared_module_prefix(shared_module_prefix) { - std::vector> dexes; - + m_xstores.push_back(std::unordered_set()); m_stores.push_back(&stores[0]); - dexes.emplace_back(&stores[0].get_dexen()[0], 0); + for (const auto& cls : stores[0].get_dexen()[0]) { + m_xstores.back().insert(cls->get_type()); + } m_root_stores = 1; if (stores[0].get_dexen().size() > 1) { m_root_stores++; + m_xstores.push_back(std::unordered_set()); m_stores.push_back(&stores[0]); for (size_t i = 1; i < stores[0].get_dexen().size(); i++) { - dexes.emplace_back(&stores[0].get_dexen()[i], 1); + for (const auto& cls : stores[0].get_dexen()[i]) { + m_xstores.back().insert(cls->get_type()); + } } } for (size_t i = 1; i < stores.size(); i++) { + m_xstores.push_back(std::unordered_set()); m_stores.push_back(&stores[i]); - size_t store_idx = m_stores.size() - 1; for (const auto& classes : stores[i].get_dexen()) { - dexes.emplace_back(&classes, store_idx); + for (const auto& cls : classes) { + m_xstores.back().insert(cls->get_type()); + } } } - - workqueue_run_for(0, dexes.size(), [&](size_t i) { - auto [dex, store_idx] = dexes[i]; - for (auto* cls : *dex) { - m_xstores.emplace(cls->get_type(), store_idx); - } - }); } bool XStoreRefs::illegal_ref_load_types(const DexType* location, diff --git a/libredex/DexStore.h b/libredex/DexStore.h index 6c12ececc3..8cc6647d03 100644 --- a/libredex/DexStore.h +++ b/libredex/DexStore.h @@ -14,7 +14,6 @@ #include #include -#include "ConcurrentContainers.h" #include "Debug.h" #include "DexClass.h" @@ -66,6 +65,7 @@ class DexStore { const std::vector& get_dexen() const; std::vector get_dependencies() const; bool is_root_store() const; + bool is_longtail_store() const; void set_generated() { m_generated = true; } bool is_generated() const { return m_generated; } @@ -169,10 +169,10 @@ std::unordered_set get_root_store_types( class XStoreRefs { private: /** - * Map of classes to their logical store index. A primary DEX goes in its own + * Set of classes in each logical store. A primary DEX goes in its own * bucket (first element in the array). */ - InsertOnlyConcurrentMap m_xstores; + std::vector> m_xstores; /** * Pointers to original stores in the same order as used to populate @@ -239,9 +239,8 @@ class XStoreRefs { * api. */ size_t get_store_idx(const DexType* type) const { - auto* res = m_xstores.get(type); - if (res) { - return *res; + for (size_t store_idx = 0; store_idx < m_xstores.size(); store_idx++) { + if (m_xstores[store_idx].count(type) > 0) return store_idx; } not_reached_log("type %s not in the current APK", show_type(type).c_str()); } @@ -253,13 +252,17 @@ class XStoreRefs { * the current scope. */ bool is_in_root_store(const DexType* type) const { - auto* res = m_xstores.get(type); - return res && *res < m_root_stores; + for (size_t store_idx = 0; store_idx < m_root_stores; store_idx++) { + if (m_xstores[store_idx].count(type) > 0) { + return true; + } + } + + return false; } bool is_in_primary_dex(const DexType* type) const { - auto* res = m_xstores.get(type); - return res && *res == 0; + return !m_xstores.empty() && m_xstores[0].count(type); } const DexStore* get_store(size_t idx) const { return m_stores[idx]; } @@ -295,14 +298,15 @@ class XStoreRefs { if (type_class_internal(type) == nullptr) return false; // Temporary HACK: optimizations may leave references to dead classes and // if we just call get_store_idx() - as we should - the assert will fire... - if (store_idx >= m_xstores.size()) { - return false; + size_t type_store_idx = 0; + for (; type_store_idx < m_xstores.size(); type_store_idx++) { + if (m_xstores[type_store_idx].count(type) > 0) break; } - auto* res = m_xstores.get(type); - if (!res) { - return true; + if ((store_idx >= m_xstores.size()) || + (type_store_idx >= m_xstores.size())) { + return type_store_idx > store_idx; } - return illegal_ref_between_stores(store_idx, *res); + return illegal_ref_between_stores(store_idx, type_store_idx); } bool illegal_ref_between_stores(size_t caller_store_idx, diff --git a/libredex/DexStoreUtil.cpp b/libredex/DexStoreUtil.cpp index 5ec7f9de50..b81597ccc5 100644 --- a/libredex/DexStoreUtil.cpp +++ b/libredex/DexStoreUtil.cpp @@ -42,7 +42,6 @@ DexClass* create_canary(int dexnum, const DexString* store_name) { canary_cls = cc.create(); // Don't rename the Canary we've created canary_cls->rstate.set_keepnames(); - canary_cls->rstate.set_generated(); } return canary_cls; } @@ -81,7 +80,7 @@ std::unordered_set get_non_root_store_types( const TypeSet& types, bool include_primary_dex) { - always_assert(!stores.empty()); + always_assert(stores.size()); XStoreRefs xstores(stores); return get_non_root_store_types(stores, xstores, types, include_primary_dex); } diff --git a/libredex/DexStructure.h b/libredex/DexStructure.h index 6aced46426..58ffe3a731 100644 --- a/libredex/DexStructure.h +++ b/libredex/DexStructure.h @@ -28,13 +28,6 @@ struct ReserveRefsInfo { ReserveRefsInfo(size_t frefs, size_t trefs, size_t mrefs) : frefs(frefs), trefs(trefs), mrefs(mrefs) {} - - ReserveRefsInfo& operator+=(const ReserveRefsInfo& rhs) { - frefs += rhs.frefs; - trefs += rhs.trefs; - mrefs += rhs.mrefs; - return *this; - } }; using MethodRefs = std::unordered_set; diff --git a/libredex/DexTypeEnvironment.h b/libredex/DexTypeEnvironment.h index e781d2ff43..2991d18716 100644 --- a/libredex/DexTypeEnvironment.h +++ b/libredex/DexTypeEnvironment.h @@ -12,7 +12,6 @@ #include #include -#include #include #include #include @@ -256,7 +255,7 @@ using TypedefAnnotationDomain = SingletonDexTypeDomain; /* * - * NullnessDomain X SingletonDexTypeDomain X SmallSetDexTypeDomain + * ArrayConstNullnessDomain X SingletonDexTypeDomain X SmallSetDexTypeDomain * * * When the SmallSetDexTypeDomain has elements, then they represent an exact set @@ -264,7 +263,7 @@ using TypedefAnnotationDomain = SingletonDexTypeDomain; */ class DexTypeDomain final : public sparta::ReducedProductAbstractDomain { @@ -273,7 +272,7 @@ class DexTypeDomain final using BaseType = sparta::ReducedProductAbstractDomain; @@ -284,13 +283,36 @@ class DexTypeDomain final // catch this. So we insert a redundant '= default'. DexTypeDomain() = default; - private: + explicit DexTypeDomain(int64_t v) + : ReducedProductAbstractDomain( + std::make_tuple(ConstNullnessDomain(v), + SingletonDexTypeDomain(), + SmallSetDexTypeDomain::top(), + TypedefAnnotationDomain())) {} + + explicit DexTypeDomain(const DexType* array_type, uint32_t array_length) + : ReducedProductAbstractDomain( + std::make_tuple(ArrayNullnessDomain(array_length), + SingletonDexTypeDomain(array_type), + SmallSetDexTypeDomain(array_type), + TypedefAnnotationDomain())) {} + + explicit DexTypeDomain(const DexType* dex_type, + const DexAnnoType* annotation = nullptr) + : ReducedProductAbstractDomain( + std::make_tuple(ConstNullnessDomain(NOT_NULL), + SingletonDexTypeDomain(dex_type), + SmallSetDexTypeDomain(dex_type), + (annotation && annotation->m_type) + ? TypedefAnnotationDomain(annotation->m_type) + : TypedefAnnotationDomain())) {} + explicit DexTypeDomain(const DexType* dex_type, const Nullness nullness, bool is_dex_type_exact, const DexAnnoType* annotation = nullptr) : ReducedProductAbstractDomain( - std::make_tuple(NullnessDomain(nullness), + std::make_tuple(ConstNullnessDomain(nullness), SingletonDexTypeDomain(dex_type), is_dex_type_exact ? SmallSetDexTypeDomain(dex_type) : SmallSetDexTypeDomain::top(), @@ -300,44 +322,28 @@ class DexTypeDomain final explicit DexTypeDomain(const DexAnnoType* annotation) : ReducedProductAbstractDomain(std::make_tuple( - NullnessDomain(), + ConstNullnessDomain(), SingletonDexTypeDomain(), SmallSetDexTypeDomain::top(), annotation->m_type ? TypedefAnnotationDomain(annotation->m_type) : TypedefAnnotationDomain())) {} - public: static void reduce_product( - std::tuple& /* product */) {} - static DexTypeDomain create_nullable( - const DexType* dex_type, const DexAnnoType* annotation = nullptr) { - return DexTypeDomain(dex_type, NN_TOP, false, annotation); - } - - /* - * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - * Only create not_null domain when you are absolutely sure about the nullness - * of the value!!! - * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - * Usually it means the value is created via new-instance or alike. If the - * nullness info is incorrect from the source, it could lead to incorrect - * analysis result. - */ - static DexTypeDomain create_not_null(const DexType* dex_type) { - return DexTypeDomain(dex_type, NOT_NULL, true); - } - - static DexTypeDomain create_for_anno(const DexAnnoType* annotation) { - return DexTypeDomain(annotation); - } - static DexTypeDomain null() { return DexTypeDomain(IS_NULL); } - NullnessDomain get_nullness() const { return get<0>(); } + NullnessDomain get_nullness() const { + auto domain = get<0>(); + if (domain.which() == 0) { + return domain.get().get_nullness(); + } else { + return domain.get().get_nullness(); + } + } bool is_null() const { return get_nullness().element() == IS_NULL; } @@ -345,6 +351,45 @@ class DexTypeDomain final bool is_nullable() const { return get_nullness().is_top(); } + boost::optional get_constant() const { + if (auto const_nullness = get<0>().maybe_get()) { + return const_nullness->const_domain().get_constant(); + } + return boost::none; + } + + ArrayNullnessDomain get_array_nullness() const { + if (auto array_nullness = get<0>().maybe_get()) { + return *array_nullness; + } + return ArrayNullnessDomain::top(); + } + + NullnessDomain get_array_element_nullness( + boost::optional idx) const { + if (!ArrayNullnessDomain::is_valid_array_idx(idx)) { + return NullnessDomain::top(); + } + return get_array_nullness().get_element(*idx); + } + + void set_array_element_nullness(boost::optional idx, + const NullnessDomain& nullness) { + if (!ArrayNullnessDomain::is_valid_array_idx(idx)) { + apply<0>([&](ArrayConstNullnessDomain* d) { + d->apply([&](ArrayNullnessDomain* array_nullness) { + array_nullness->reset_elements(); + }); + }); + return; + } + apply<0>([&](ArrayConstNullnessDomain* d) { + d->apply([&](ArrayNullnessDomain* array_nullness) { + array_nullness->set_element(*idx, nullness); + }); + }); + } + const SingletonDexTypeDomain& get_single_domain() const { return get<1>(); } const TypedefAnnotationDomain& get_annotation_domain() const { @@ -371,14 +416,13 @@ class DexTypeDomain final const SmallSetDexTypeDomain& get_set_domain() const { return get<2>(); } const sparta::PatriciaTreeSet& get_type_set() const { - always_assert(!get<2>().is_top() && !get<2>().is_bottom()); return get<2>().get_types(); } private: explicit DexTypeDomain(const Nullness nullness) : ReducedProductAbstractDomain( - std::make_tuple(NullnessDomain(nullness), + std::make_tuple(ConstNullnessDomain(nullness), SingletonDexTypeDomain::none(), SmallSetDexTypeDomain(), TypedefAnnotationDomain::none())) {} diff --git a/libredex/DexUtil.cpp b/libredex/DexUtil.cpp index 9a620b200a..e43ddf78d1 100644 --- a/libredex/DexUtil.cpp +++ b/libredex/DexUtil.cpp @@ -174,7 +174,6 @@ void load_root_dexen(DexStore& store, int support_dex_version) { namespace fs = boost::filesystem; fs::path dexen_dir_path(dexen_dir_str); - // NOLINTNEXTLINE(bugprone-assert-side-effect) redex_assert(fs::is_directory(dexen_dir_path)); // Discover dex files @@ -249,6 +248,7 @@ void create_store(const std::string& store_name, } void relocate_field(DexField* field, DexType* to_type) { + // change_visibility(field, to_type); auto from_cls = type_class(field->get_class()); auto to_cls = type_class(to_type); from_cls->remove_field(field); @@ -259,6 +259,7 @@ void relocate_field(DexField* field, DexType* to_type) { } void relocate_method(DexMethod* method, DexType* to_type) { + change_visibility(method, to_type); auto from_cls = type_class(method->get_class()); auto to_cls = type_class(to_type); from_cls->remove_method(method); @@ -440,8 +441,8 @@ bool relocate_method_if_no_changes(DexMethod* method, DexType* to_type) { } set_public(method); - change_visibility(method, to_type); relocate_method(method, to_type); + change_visibility(method); return true; } diff --git a/libredex/GlobalConfig.cpp b/libredex/GlobalConfig.cpp index b68bb05dab..abfa3024b3 100644 --- a/libredex/GlobalConfig.cpp +++ b/libredex/GlobalConfig.cpp @@ -191,7 +191,6 @@ void GlobalConfig::bind_config() { bind("create_init_class_insns", true, bool_param); bind("finalize_resource_table", false, bool_param); bind("check_required_resources", {}, string_vector_param); - bind("update_method_profiles_stats", false, bool_param); bind("recognize_betamap_coldstart_pct_marker", false, bool_param); for (const auto& entry : m_registry) { diff --git a/libredex/HierarchyUtil.cpp b/libredex/HierarchyUtil.cpp index 35fc2e8267..d87bb695df 100644 --- a/libredex/HierarchyUtil.cpp +++ b/libredex/HierarchyUtil.cpp @@ -8,28 +8,32 @@ #include "HierarchyUtil.h" #include "RedexContext.h" -#include "Walkers.h" namespace mog = method_override_graph; namespace hierarchy_util { -NonOverriddenVirtuals::NonOverriddenVirtuals( - const Scope& scope, const method_override_graph::Graph& override_graph) { - walk::parallel::classes(scope, [&](DexClass* cls) { - for (auto* method : cls->get_vmethods()) { - if (!mog::any_overriding_methods(override_graph, method)) { - m_non_overridden_virtuals.emplace(method); +std::unordered_set find_non_overridden_virtuals( + const mog::Graph& override_graph) { + std::unordered_set non_overridden_virtuals; + g_redex->walk_type_class([&](const DexType*, const DexClass* cls) { + if (!cls->is_external()) { + for (auto* method : cls->get_vmethods()) { + const auto& overrides = + mog::get_overriding_methods(override_graph, method); + if (overrides.empty()) { + non_overridden_virtuals.emplace(method); + } + } + } else { + for (auto* method : cls->get_vmethods()) { + if (is_final(cls) || is_final(method)) { + non_overridden_virtuals.emplace(method); + } } } }); -} - -size_t NonOverriddenVirtuals::count(const DexMethod* method) const { - if (method->is_external()) { - return is_final(method) || is_final(type_class(method->get_class())); - } - return m_non_overridden_virtuals.count_unsafe(method); + return non_overridden_virtuals; } } // namespace hierarchy_util diff --git a/libredex/HierarchyUtil.h b/libredex/HierarchyUtil.h index 765b7d1dec..0508b36ad3 100644 --- a/libredex/HierarchyUtil.h +++ b/libredex/HierarchyUtil.h @@ -9,29 +9,29 @@ #include -#include "ConcurrentContainers.h" #include "DexClass.h" #include "MethodOverrideGraph.h" namespace hierarchy_util { /* - * Identifies all non-overridden virtual methods in scope, plus methods from + * Returns all non-overridden virtual methods in scope, plus methods from * external classes. The external classes will be included even if they are not * in the input Scope parameter. */ -class NonOverriddenVirtuals { - public: - NonOverriddenVirtuals(const Scope& scope, - const method_override_graph::Graph& override_graph); - explicit NonOverriddenVirtuals(const Scope& scope) - : NonOverriddenVirtuals(scope, - *method_override_graph::build_graph(scope)) {} +std::unordered_set find_non_overridden_virtuals( + const method_override_graph::Graph& override_graph); - size_t count(const DexMethod*) const; - - private: - ConcurrentSet m_non_overridden_virtuals; -}; +/* + * Returns all non-overridden virtual methods in scope, plus methods from + * external classes. The external classes will be included even if they are not + * in the input Scope parameter. + */ +inline std::unordered_set find_non_overridden_virtuals( + const Scope& scope) { + std::unique_ptr override_graph = + method_override_graph::build_graph(scope); + return find_non_overridden_virtuals(*override_graph); +} } // namespace hierarchy_util diff --git a/libredex/IRAssembler.cpp b/libredex/IRAssembler.cpp index 04cda5dd09..d2b1600219 100644 --- a/libredex/IRAssembler.cpp +++ b/libredex/IRAssembler.cpp @@ -534,8 +534,7 @@ std::unordered_map get_catch_name_map( } // Can we merge this target into the same label as the previous target? -bool can_merge(const IRList::const_iterator& prev, - const IRList::const_iterator& it) { +bool can_merge(IRList::const_iterator prev, IRList::const_iterator it) { always_assert(it->type == MFLOW_TARGET); return prev->type == MFLOW_TARGET && // can't merge if/goto targets with switch targets @@ -1006,8 +1005,8 @@ DexClass* class_with_methods(const std::string& class_name, for (const auto& method : methods) { class_creator.add_method(method); } - auto cls = class_creator.create(); - return cls; + class_creator.create(); + return class_creator.get_class(); } } // namespace assembler diff --git a/libredex/IRCode.cpp b/libredex/IRCode.cpp index 5141950748..031cb7c194 100644 --- a/libredex/IRCode.cpp +++ b/libredex/IRCode.cpp @@ -19,7 +19,6 @@ #include "ControlFlow.h" #include "Debug.h" -#include "DebugUtils.h" #include "DexClass.h" #include "DexDebugInstruction.h" #include "DexInstruction.h" @@ -1030,8 +1029,7 @@ bool IRCode::try_sync(DexCode* code) { packed_payload[1] = size; uint32_t* psdata = (uint32_t*)&packed_payload[2]; int32_t next_key = *psdata++ = targets.front()->case_key; - redex_assert(std::as_const(targets).front()->case_key <= - std::as_const(targets).back()->case_key); + redex_assert(targets.front()->case_key <= targets.back()->case_key); for (BranchTarget* target : targets) { // Fill in holes with relative offsets that are falling through to the // instruction after the switch instruction diff --git a/libredex/IRInstruction.h b/libredex/IRInstruction.h index 597d988a75..4c50adfdce 100644 --- a/libredex/IRInstruction.h +++ b/libredex/IRInstruction.h @@ -208,20 +208,6 @@ class IRInstruction final { return has_move_result() || has_move_result_pseudo(); } - bool has_contiguous_range_srcs_denormalized() const { - if (srcs_size() == 0) { - return true; - } - auto last = src(0); - for (size_t i = 1; i < srcs_size(); ++i) { - if (src(i) - last != 1) { - return false; - } - last = src(i); - } - return true; - } - /* * Information about operands. */ diff --git a/libredex/IRList.h b/libredex/IRList.h index c11c228d53..3c259531cd 100644 --- a/libredex/IRList.h +++ b/libredex/IRList.h @@ -551,16 +551,16 @@ class IRList { // transfer all of `other` into `this` starting at `pos` // memory ownership is also transferred - void splice(const IRList::const_iterator& pos, IRList& other) { + void splice(IRList::const_iterator pos, IRList& other) { m_list.splice(pos, other.m_list); } // transfer `other[begin]` to `other[end]` into `this` starting at `pos` // memory ownership is also transferred - void splice_selection(const IRList::const_iterator& pos, + void splice_selection(IRList::const_iterator pos, IRList& other, - const IRList::const_iterator& begin, - const IRList::const_iterator& end) { + IRList::const_iterator begin, + IRList::const_iterator end) { m_list.splice(pos, other.m_list, begin, end); } diff --git a/libredex/IRTypeChecker.cpp b/libredex/IRTypeChecker.cpp index e20e07ca13..e0a391860f 100644 --- a/libredex/IRTypeChecker.cpp +++ b/libredex/IRTypeChecker.cpp @@ -9,7 +9,6 @@ #include -#include "BigBlocks.h" #include "Debug.h" #include "DexPosition.h" #include "DexUtil.h" @@ -353,6 +352,15 @@ class Result final { Result() = default; }; +static bool has_move_result_pseudo(const MethodItemEntry& mie) { + return mie.type == MFLOW_OPCODE && mie.insn->has_move_result_pseudo(); +} + +static bool is_move_result_pseudo(const MethodItemEntry& mie) { + return mie.type == MFLOW_OPCODE && + opcode::is_a_move_result_pseudo(mie.insn->opcode()); +} + Result check_load_params(const DexMethod* method) { bool is_static_method = is_static(method); const auto* signature = method->get_proto()->get_args(); @@ -437,133 +445,117 @@ Result check_load_params(const DexMethod* method) { // Every variable created by a new-instance call should be initialized by a // proper invoke-direct . Here, we perform simple check to find some // missing calls resulting in use of uninitialized variables. We correctly track -// variables in a "big block", the most common form of allocation+init. +// variables in a basic block, the most common form of allocation+init. Result check_uninitialized(const DexMethod* method) { - auto code = (const_cast(method))->get_code(); - always_assert(code->editable_cfg_built()); - auto& cfg = code->cfg(); - - std::unordered_set block_visited; - auto ordered_blocks = cfg.order(); + auto* code = method->get_code(); + std::map uninitialized_regs; + std::map> uninitialized_regs_rev; + auto remove_from_uninitialized_list = [&](uint16_t reg) { + auto it = uninitialized_regs.find(reg); + if (it != uninitialized_regs.end()) { + uninitialized_regs_rev[it->second].erase(reg); + uninitialized_regs.erase(reg); + } + }; - for (cfg::Block* block : ordered_blocks) { - if (block_visited.count(block->id())) { + for (auto it = code->begin(); it != code->end(); ++it) { + if (it->type != MFLOW_OPCODE) { continue; } - auto big_block = big_blocks::get_big_block(block); - if (!big_block) { + auto* insn = it->insn; + auto op = insn->opcode(); + + if (op == OPCODE_NEW_INSTANCE) { + ++it; + while (it != code->end() && it->type != MFLOW_OPCODE) + ++it; + if (it == code->end() || + it->insn->opcode() != IOPCODE_MOVE_RESULT_PSEUDO_OBJECT) { + auto prev = it; + prev--; + return Result::make_error("No opcode-move-result after new-instance " + + show(*prev) + " in \n" + show(code->cfg())); + } + + auto reg_dest = it->insn->dest(); + remove_from_uninitialized_list(reg_dest); + + uninitialized_regs[reg_dest] = insn; + uninitialized_regs_rev[insn].insert(reg_dest); continue; } - // Find a big block starting from current block. - for (auto b : big_block->get_blocks()) { - block_visited.emplace(b->id()); - } - std::map uninitialized_regs; - std::map> uninitialized_regs_rev; - auto remove_from_uninitialized_list = [&](uint16_t reg) { - auto it = uninitialized_regs.find(reg); - if (it != uninitialized_regs.end()) { - uninitialized_regs_rev[it->second].erase(reg); - uninitialized_regs.erase(reg); + + if (opcode::is_a_move(op) && !opcode::is_move_result_any(op)) { + assert(insn->srcs().size() > 0); + auto src = insn->srcs()[0]; + auto dest = insn->dest(); + if (src == dest) continue; + + auto it_src = uninitialized_regs.find(src); + // We no longer care about the old dest + remove_from_uninitialized_list(dest); + // But if src was uninitialized, dest is now too + if (it_src != uninitialized_regs.end()) { + uninitialized_regs[dest] = it_src->second; + uninitialized_regs_rev[it_src->second].insert(dest); } + continue; + } + + auto create_error = [&](const IRInstruction* instruction, + const IRCode* code) { + return Result::make_error("Use of uninitialized variable " + + show(instruction) + " detected at " + + show(*it) + " in \n" + show(code->cfg())); }; - auto current_block = big_block->get_first_block(); - while (current_block) { - auto ii = InstructionIterable(current_block); - for (auto it = ii.begin(); it != ii.end(); it++) { - auto* insn = it->insn; - auto op = insn->opcode(); - if (op == OPCODE_NEW_INSTANCE) { - auto cfg_it = current_block->to_cfg_instruction_iterator(it); - auto move_result = cfg.move_result_of(cfg_it); - if (move_result.is_end()) { - return Result::make_error( - "No opcode-move-result after new-instance " + show(*cfg_it) + - " in \n" + show(cfg)); - } - - auto reg_dest = move_result->insn->dest(); - remove_from_uninitialized_list(reg_dest); - - uninitialized_regs[reg_dest] = insn; - uninitialized_regs_rev[insn].insert(reg_dest); - // skip the move_result - it++; - if (it == ii.end()) { - break; - } - continue; - } + if (op == OPCODE_INVOKE_DIRECT) { + auto const& sources = insn->srcs(); + auto object = sources[0]; - if (opcode::is_a_move(op) && !opcode::is_move_result_any(op)) { - assert(insn->srcs().size() > 0); - auto src = insn->srcs()[0]; - auto dest = insn->dest(); - if (src == dest) continue; - - auto it_src = uninitialized_regs.find(src); - // We no longer care about the old dest - remove_from_uninitialized_list(dest); - // But if src was uninitialized, dest is now too - if (it_src != uninitialized_regs.end()) { - uninitialized_regs[dest] = it_src->second; - uninitialized_regs_rev[it_src->second].insert(dest); - } - continue; + auto object_it = uninitialized_regs.find(object); + if (object_it != uninitialized_regs.end()) { + auto* object_ir = object_it->second; + if (insn->get_method()->get_name()->str() != "") { + return create_error(object_ir, code); } - - auto create_error = [&](const IRInstruction* instruction, - const cfg::ControlFlowGraph& cfg) { - return Result::make_error("Use of uninitialized variable " + - show(instruction) + " detected at " + - show(*it) + " in \n" + show(cfg)); - }; - - if (op == OPCODE_INVOKE_DIRECT) { - auto const& sources = insn->srcs(); - auto object = sources[0]; - - auto object_it = uninitialized_regs.find(object); - if (object_it != uninitialized_regs.end()) { - auto* object_ir = object_it->second; - if (insn->get_method()->get_name()->str() != "") { - return create_error(object_ir, cfg); - } - if (insn->get_method()->get_class()->str() != - object_ir->get_type()->str()) { - return Result::make_error("Variable " + show(object_ir) + - "initialized with the wrong type at " + - show(*it) + " in \n" + show(cfg)); - } - for (auto reg : uninitialized_regs_rev[object_ir]) { - uninitialized_regs.erase(reg); - } - uninitialized_regs_rev.erase(object_ir); - } - - for (unsigned int i = 1; i < sources.size(); i++) { - auto u_it = uninitialized_regs.find(sources[i]); - if (u_it != uninitialized_regs.end()) - return create_error(u_it->second, cfg); - } - continue; + if (insn->get_method()->get_class()->str() != + object_ir->get_type()->str()) { + return Result::make_error("Variable " + show(object_ir) + + "initialized with the wrong type at " + + show(*it) + " in \n" + show(code->cfg())); } - - auto const& sources = insn->srcs(); - for (auto reg : sources) { - auto u_it = uninitialized_regs.find(reg); - if (u_it != uninitialized_regs.end()) - return create_error(u_it->second, cfg); + for (auto reg : uninitialized_regs_rev[object_ir]) { + uninitialized_regs.erase(reg); } - - if (insn->has_dest()) remove_from_uninitialized_list(insn->dest()); + uninitialized_regs_rev.erase(object_ir); } - // get the the next block. - if (current_block == big_block->get_last_block()) { - break; + + for (unsigned int i = 1; i < sources.size(); i++) { + auto u_it = uninitialized_regs.find(sources[i]); + if (u_it != uninitialized_regs.end()) + return create_error(u_it->second, code); } - current_block = current_block->goes_to(); + continue; + } + + auto const& sources = insn->srcs(); + for (auto reg : sources) { + auto u_it = uninitialized_regs.find(reg); + if (u_it != uninitialized_regs.end()) + return create_error(u_it->second, code); + } + + if (insn->has_dest()) remove_from_uninitialized_list(insn->dest()); + + // We clear the structures after any branch, this doesn't cover all the + // possible issues, but is simple + auto branchingness = opcode::branchingness(op); + if (op == OPCODE_THROW || + (branchingness != opcode::Branchingness::BRANCH_NONE && + branchingness != opcode::Branchingness::BRANCH_THROW)) { + uninitialized_regs.clear(); + uninitialized_regs_rev.clear(); } } return Result::Ok(); @@ -572,92 +564,81 @@ Result check_uninitialized(const DexMethod* method) { /* * Do a linear pass to sanity-check the structure of the bytecode. */ -Result check_structure(const DexMethod* method, - cfg::ControlFlowGraph& cfg, - bool check_no_overwrite_this) { +Result check_structure(const DexMethod* method, bool check_no_overwrite_this) { check_no_overwrite_this &= !is_static(method); + auto* code = method->get_code(); IRInstruction* this_insn = nullptr; - auto entry_block = cfg.entry_block(); - for (cfg::Block* block : cfg.blocks()) { - bool has_seen_non_load_param_opcode{false}; - auto ii = InstructionIterable(block); - for (auto it = ii.begin(); it != ii.end(); it++) { - auto* insn = it->insn; - auto op = insn->opcode(); - auto cfg_it = block->to_cfg_instruction_iterator(it); - if ((block != entry_block || has_seen_non_load_param_opcode) && - opcode::is_a_load_param(op)) { - return Result::make_error("Encountered " + show(*it) + - " not at the start of the method"); - } - has_seen_non_load_param_opcode = !opcode::is_a_load_param(op); - - if (check_no_overwrite_this) { - if (op == IOPCODE_LOAD_PARAM_OBJECT && this_insn == nullptr) { - this_insn = insn; - } else if (insn->has_dest() && insn->dest() == this_insn->dest()) { - return Result::make_error( - "Encountered overwrite of `this` register by " + show(insn)); - } - } + bool has_seen_non_load_param_opcode{false}; + for (auto it = code->begin(); it != code->end(); ++it) { + // XXX we are using IRList::iterator instead of InstructionIterator here + // because the latter does not support reverse iteration + if (it->type != MFLOW_OPCODE) { + continue; + } + auto* insn = it->insn; + auto op = insn->opcode(); - if (opcode::is_move_result_any(op)) { - if (block == cfg.entry_block() && it == ii.begin()) { - return Result::make_error("Encountered " + show(*it) + - " at start of the method"); - } - auto prev = - cfg.primary_instruction_of_move_result_for_type_check(cfg_it); - // The instruction immediately before a move-result instruction must be - // either an invoke-* or a filled-new-array instruction. - if (opcode::is_a_move_result(op)) { - if (prev->type != MFLOW_OPCODE) { - return Result::make_error("Encountered " + show(*it) + - " at start of the method"); - } - auto prev_op = prev->insn->opcode(); - if (!(opcode::is_an_invoke(prev_op) || - opcode::is_filled_new_array(prev_op))) { - return Result::make_error( - "Encountered " + show(*it) + - " without appropriate prefix " - "instruction. Expected invoke or filled-new-array, got " + - show(prev->insn)); - } - if (!prev->insn->has_move_result()) { - return Result::make_error("Encountered " + show(*it) + - " without appropriate prefix " - "instruction"); - } - } + if (has_seen_non_load_param_opcode && opcode::is_a_load_param(op)) { + return Result::make_error("Encountered " + show(*it) + + " not at the start of the method"); + } + has_seen_non_load_param_opcode = !opcode::is_a_load_param(op); + + if (check_no_overwrite_this) { + if (op == IOPCODE_LOAD_PARAM_OBJECT && this_insn == nullptr) { + this_insn = insn; + } else if (insn->has_dest() && insn->dest() == this_insn->dest()) { + return Result::make_error( + "Encountered overwrite of `this` register by " + show(insn)); + } + } - if (opcode::is_a_move_result_pseudo(insn->opcode()) && - (!prev->insn->has_move_result_pseudo())) { - return Result::make_error("Encountered " + show(*it) + - " without appropriate prefix " - "instruction"); + // The instruction immediately before a move-result instruction must be + // either an invoke-* or a filled-new-array instruction. + if (opcode::is_a_move_result(op)) { + auto prev = it; + while (prev != code->begin()) { + --prev; + if (prev->type == MFLOW_OPCODE) { + break; } } - if (insn->has_move_result_pseudo()) { - auto move_result = cfg.move_result_of(cfg_it); - if (move_result.is_end() || - !opcode::is_a_move_result_pseudo(move_result->insn->opcode())) { - return Result::make_error("Did not find move-result-pseudo after " + - show(*it) + " in \n" + show(cfg)); - } + if (it == code->begin() || prev->type != MFLOW_OPCODE) { + return Result::make_error("Encountered " + show(*it) + + " at start of the method"); } + auto prev_op = prev->insn->opcode(); + if (!(opcode::is_an_invoke(prev_op) || + opcode::is_filled_new_array(prev_op))) { + return Result::make_error( + "Encountered " + show(*it) + + " without appropriate prefix " + "instruction. Expected invoke or filled-new-array, got " + + show(prev->insn)); + } + } else if (opcode::is_a_move_result_pseudo(insn->opcode()) && + (it == code->begin() || + !has_move_result_pseudo(*std::prev(it)))) { + return Result::make_error("Encountered " + show(*it) + + " without appropriate prefix " + "instruction"); + } else if (insn->has_move_result_pseudo() && + (it == code->end() || std::next(it) == code->end() || + !is_move_result_pseudo(*std::next(it)))) { + return Result::make_error("Did not find move-result-pseudo after " + + show(*it) + " in \n" + show(code)); } } return check_uninitialized(method); } /* - * Sanity-check the structure of the positions for editable cfg format. + * Do a linear pass to sanity-check the structure of the positions. */ -Result check_positions_cfg(cfg::ControlFlowGraph& cfg) { +Result check_positions(const DexMethod* method) { + auto code = method->get_code(); std::unordered_set positions; - auto iterable = cfg::InstructionIterable(cfg); - for (auto it = iterable.begin(); it != iterable.end(); ++it) { + for (auto it = code->begin(); it != code->end(); ++it) { if (it->type != MFLOW_POSITION) { continue; } @@ -666,7 +647,6 @@ Result check_positions_cfg(cfg::ControlFlowGraph& cfg) { return Result::make_error("Duplicate position " + show(pos)); } } - std::unordered_set visited_parents; for (auto pos : positions) { if (!pos->parent) { @@ -825,16 +805,6 @@ void validate_invoke_virtual(const DexMethod* caller, } if (callee->as_def()->is_virtual()) { - // Make sure the callee is not known to be an interface. - auto callee_type = callee->as_def()->get_class(); - auto callee_cls = type_class(callee_type); - if (callee_cls != nullptr && is_interface(callee_cls)) { - std::ostringstream out; - out << "\nillegal invoke-virtual to interface type " - << show_deobfuscated(callee) << " in " << show_deobfuscated(caller); - throw TypeCheckingException(out.str()); - } - // Otherwise okay. return; } @@ -927,9 +897,8 @@ void IRTypeChecker::run() { return; } - cfg::ScopedCFG cfg(code); - - auto result = check_structure(m_dex_method, *cfg, m_check_no_overwrite_this); + code->build_cfg(/* editable */ false); + auto result = check_structure(m_dex_method, m_check_no_overwrite_this); if (result != Result::Ok()) { m_complete = true; m_good = false; @@ -938,6 +907,8 @@ void IRTypeChecker::run() { } // We then infer types for all the registers used in the method. + const cfg::ControlFlowGraph& cfg = code->cfg(); + // Check that the load-params match the signature. auto params_result = check_load_params(m_dex_method); if (params_result != Result::Ok()) { @@ -947,18 +918,18 @@ void IRTypeChecker::run() { return; } - m_type_inference = std::make_unique(*cfg); + m_type_inference = std::make_unique(cfg); m_type_inference->run(m_dex_method); // Finally, we use the inferred types to type-check each instruction in the // method. We stop at the first type error encountered. auto& type_envs = m_type_inference->get_type_environments(); - for (const MethodItemEntry& mie : InstructionIterable(*cfg)) { + for (const MethodItemEntry& mie : InstructionIterable(code)) { IRInstruction* insn = mie.insn; try { auto it = type_envs.find(insn); always_assert_log( - it != type_envs.end(), "%s in:\n%s", SHOW(mie), SHOW(*cfg)); + it != type_envs.end(), "%s in:\n%s", SHOW(mie), SHOW(code)); check_instruction(insn, &it->second); } catch (const TypeCheckingException& e) { m_good = false; @@ -973,8 +944,7 @@ void IRTypeChecker::run() { } } - auto positions_result = check_positions_cfg(*cfg); - + auto positions_result = check_positions(m_dex_method); if (positions_result != Result::Ok()) { m_complete = true; m_good = false; diff --git a/libredex/IRTypeChecker.h b/libredex/IRTypeChecker.h index adc11f8e70..f0c298a0c1 100644 --- a/libredex/IRTypeChecker.h +++ b/libredex/IRTypeChecker.h @@ -153,7 +153,6 @@ class IRTypeChecker final { bool m_verify_moves; bool m_check_no_overwrite_this; bool m_good; - bool m_orig_editable_cfg; std::string m_what; std::unique_ptr m_type_inference; diff --git a/libredex/InitClassesWithSideEffects.cpp b/libredex/InitClassesWithSideEffects.cpp index 385e04e247..7e9819cd6e 100644 --- a/libredex/InitClassesWithSideEffects.cpp +++ b/libredex/InitClassesWithSideEffects.cpp @@ -17,34 +17,46 @@ namespace init_classes { const InitClasses* InitClassesWithSideEffects::compute( const DexClass* cls, const method::ClInitHasNoSideEffectsPredicate& clinit_has_no_side_effects, - const InsertOnlyConcurrentSet* non_true_virtuals) { - const DexType* key = cls->get_type(); - auto [ptr, emplaced] = - m_init_classes.get_or_create_and_assert_equal(key, [&](const auto*) { - InitClasses classes; - const auto* refined_cls = method::clinit_may_have_side_effects( - cls, /* allow_benign_method_invocations */ true, - &clinit_has_no_side_effects, non_true_virtuals); - if (refined_cls == nullptr) { - } else if (refined_cls != cls) { - classes = *compute(refined_cls, clinit_has_no_side_effects, - non_true_virtuals); + const std::unordered_set* non_true_virtuals) { + auto res = + m_init_classes.get(cls->get_type(), std::shared_ptr()); + if (res) { + return res.get(); + } + + InitClasses classes; + const auto* refined_cls = method::clinit_may_have_side_effects( + cls, /* allow_benign_method_invocations */ true, + &clinit_has_no_side_effects, non_true_virtuals); + if (refined_cls == nullptr) { + } else if (refined_cls != cls) { + classes = + *compute(refined_cls, clinit_has_no_side_effects, non_true_virtuals); + } else { + classes.push_back(cls); + auto super_cls = type_class(cls->get_super_class()); + if (super_cls) { + const auto super_classes = + compute(super_cls, clinit_has_no_side_effects, non_true_virtuals); + classes.insert(classes.end(), super_classes->begin(), + super_classes->end()); + } + } + m_init_classes.update( + cls->get_type(), + [&res, &classes, this](const DexType*, + std::shared_ptr& value, bool exist) { + if (exist) { + always_assert(classes == *value); } else { - classes.push_back(cls); - auto super_cls = type_class(cls->get_super_class()); - if (super_cls) { - const auto super_classes = compute( - super_cls, clinit_has_no_side_effects, non_true_virtuals); - classes.insert(classes.end(), super_classes->begin(), - super_classes->end()); + if (classes.empty()) { + m_trivial_init_classes++; } + value = std::make_shared(std::move(classes)); } - return classes; + res = value; }); - if (emplaced && ptr->empty()) { - m_trivial_init_classes++; - } - return ptr; + return res.get(); } InitClassesWithSideEffects::InitClassesWithSideEffects( @@ -53,9 +65,9 @@ InitClassesWithSideEffects::InitClassesWithSideEffects( const method_override_graph::Graph* method_override_graph) : m_create_init_class_insns(create_init_class_insns) { Timer t("InitClassesWithSideEffects"); - std::unique_ptr> non_true_virtuals; + std::unique_ptr> non_true_virtuals; if (method_override_graph) { - non_true_virtuals = std::make_unique>( + non_true_virtuals = std::make_unique>( method_override_graph::get_non_true_virtuals(*method_override_graph, scope)); } @@ -67,7 +79,7 @@ InitClassesWithSideEffects::InitClassesWithSideEffects( [&](const DexType* type) { auto it = prev_init_classes.find(type); if (it != prev_init_classes.end()) { - return it->second.empty(); + return it->second->empty(); } auto cls = type_class(type); return cls && (cls->is_external() || @@ -94,7 +106,7 @@ InitClassesWithSideEffects::InitClassesWithSideEffects( const InitClasses* InitClassesWithSideEffects::get(const DexType* type) const { auto it = m_init_classes.find(type); - return it == m_init_classes.end() ? &m_empty_init_classes : &it->second; + return it == m_init_classes.end() ? &m_empty_init_classes : it->second.get(); } const DexType* InitClassesWithSideEffects::refine(const DexType* type) const { diff --git a/libredex/InitClassesWithSideEffects.h b/libredex/InitClassesWithSideEffects.h index a6d8828ff4..0f06c253fe 100644 --- a/libredex/InitClassesWithSideEffects.h +++ b/libredex/InitClassesWithSideEffects.h @@ -26,7 +26,7 @@ using InitClasses = std::vector; */ class InitClassesWithSideEffects { private: - InsertOnlyConcurrentMap m_init_classes; + ConcurrentMap> m_init_classes; std::atomic m_trivial_init_classes{0}; InitClasses m_empty_init_classes; bool m_create_init_class_insns; @@ -34,7 +34,7 @@ class InitClassesWithSideEffects { const InitClasses* compute( const DexClass* cls, const method::ClInitHasNoSideEffectsPredicate& clinit_has_no_side_effects, - const InsertOnlyConcurrentSet* non_true_virtuals); + const std::unordered_set* non_true_virtuals); public: InitClassesWithSideEffects( diff --git a/libredex/InitCollisionFinder.cpp b/libredex/InitCollisionFinder.cpp new file mode 100644 index 0000000000..3c6972a122 --- /dev/null +++ b/libredex/InitCollisionFinder.cpp @@ -0,0 +1,64 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "InitCollisionFinder.h" + +#include "DexUtil.h" + +/** + * Some optimizations want to change the prototypes of many methods. Sometimes, + * changing those method prototypes will collide with another method. For most + * method collisions we rename the new method to avoid the collision. But we + * cannot rename methods. + * + * This utility works around the init collision problem by finding the types + * that cause the init collision. This allows an optimization to exclude these + * types before it makes any changes. + */ + +namespace init_collision_finder { + +/** + * Given a method, return the new DexMethodSpec that the optimization wants to + * change this method to (returning boost::none if it does not want to make a + * change). In the process, this method should fill the vector argument with any + * DexTypes that were replaced in the method's prototype. + * + * This function is supplied by the user of the Init Collision Finder. + */ +using GetNewSpec = std::function( + const DexMethod*, std::vector*)>; + +std::vector find(const Scope& scope, const GetNewSpec& get_new_spec) { + // Compute what the new prototypes will be after we convert a method. Check + // the prototypes against existing methods and other prototypes created by + // this method. + std::vector result; + for (const DexClass* cls : scope) { + std::unordered_set new_specs; + for (const DexMethod* m : cls->get_dmethods()) { + if (method::is_init(m)) { + std::vector unsafe_refs; + const auto& new_spec = get_new_spec(m, &unsafe_refs); + if (new_spec) { + const auto& pair = new_specs.emplace(*new_spec); + bool already_there = !pair.second; + if (already_there || DexMethod::get_method(*new_spec)) { + always_assert_log( + !unsafe_refs.empty(), + "unsafe_refs should be filled with the types that will be " + "replaced on this method's prototype"); + result.insert(result.end(), unsafe_refs.begin(), unsafe_refs.end()); + } + } + } + } + } + return result; +} + +} // namespace init_collision_finder diff --git a/libredex/InitCollisionFinder.h b/libredex/InitCollisionFinder.h new file mode 100644 index 0000000000..a6813333d3 --- /dev/null +++ b/libredex/InitCollisionFinder.h @@ -0,0 +1,21 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include + +#include "DexClass.h" + +namespace init_collision_finder { + +using GetNewSpec = std::function( + const DexMethod*, std::vector*)>; + +std::vector find(const Scope& scope, const GetNewSpec& get_new_spec); + +} // namespace init_collision_finder diff --git a/libredex/InstructionLowering.cpp b/libredex/InstructionLowering.cpp index a54e8b599e..9987b73870 100644 --- a/libredex/InstructionLowering.cpp +++ b/libredex/InstructionLowering.cpp @@ -365,13 +365,30 @@ void lower_fill_array_data(DexMethod*, IRCode* code, IRList::iterator* it_) { it->replace_ir_with_dex(dex_insn); } +/* + * Necessary condition for an instruction to be converted to /range form + */ +bool has_contiguous_srcs(const IRInstruction* insn) { + if (insn->srcs_size() == 0) { + return true; + } + auto last = insn->src(0); + for (size_t i = 1; i < insn->srcs_size(); ++i) { + if (insn->src(i) - last != 1) { + return false; + } + last = insn->src(i); + } + return true; +} + void lower_to_range_instruction(DexMethod* method, IRCode* code, IRList::iterator* it_) { auto& it = *it_; const auto* insn = it->insn; always_assert_log( - insn->has_contiguous_range_srcs_denormalized(), + has_contiguous_srcs(insn), "Instruction %s has non-contiguous srcs in method %s.\nContext:\n%s\n", SHOW(insn), SHOW(method), @@ -466,7 +483,7 @@ Stats lower(DexMethod* method, bool lower_with_cfg, ConfigFiles* conf) { // Remove any source blocks. They are no longer necessary and slow down // iteration. if (it->type == MFLOW_SOURCE_BLOCK) { - redex_assert(it != code->cbegin()); + redex_assert(it != code->begin()); auto prev = std::prev(it); code->erase_and_dispose(it); it = prev; diff --git a/libredex/JarLoader.cpp b/libredex/JarLoader.cpp index 50f80a7e4e..8598e1ddb6 100644 --- a/libredex/JarLoader.cpp +++ b/libredex/JarLoader.cpp @@ -40,7 +40,7 @@ namespace JarLoaderUtil { uint32_t read32(uint8_t*& buffer, uint8_t* buffer_end) { uint32_t rv; auto next = buffer + sizeof(uint32_t); - if (next > buffer_end) { + if (next >= buffer_end) { throw RedexException(RedexError::BUFFER_END_EXCEEDED); } memcpy(&rv, buffer, sizeof(uint32_t)); @@ -48,23 +48,16 @@ uint32_t read32(uint8_t*& buffer, uint8_t* buffer_end) { return htonl(rv); } -uint16_t read16(uint8_t*& buffer, uint8_t* buffer_end) { +uint32_t read16(uint8_t*& buffer, uint8_t* buffer_end) { uint16_t rv; auto next = buffer + sizeof(uint16_t); - if (next > buffer_end) { + if (next >= buffer_end) { throw RedexException(RedexError::BUFFER_END_EXCEEDED); } memcpy(&rv, buffer, sizeof(uint16_t)); buffer = next; return htons(rv); } - -uint8_t read8(uint8_t*& buffer, uint8_t* buffer_end) { - if (buffer >= buffer_end) { - throw RedexException(RedexError::BUFFER_END_EXCEEDED); - } - return *buffer++; -} } // namespace JarLoaderUtil using namespace JarLoaderUtil; @@ -129,7 +122,7 @@ constexpr size_t CP_CONST_PACKAGE = 20; /* clang-format on */ bool parse_cp_entry(uint8_t*& buffer, uint8_t* buffer_end, cp_entry& cpe) { - cpe.tag = read8(buffer, buffer_end); + cpe.tag = *buffer++; switch (cpe.tag) { case CP_CONST_CLASS: case CP_CONST_STRING: @@ -146,7 +139,7 @@ bool parse_cp_entry(uint8_t*& buffer, uint8_t* buffer_end, cp_entry& cpe) { cpe.s1 = read16(buffer, buffer_end); return true; case CP_CONST_METHHANDLE: - cpe.s0 = read8(buffer, buffer_end); + cpe.s0 = *buffer++; cpe.s1 = read16(buffer, buffer_end); return true; case CP_CONST_INT: @@ -162,9 +155,6 @@ bool parse_cp_entry(uint8_t*& buffer, uint8_t* buffer_end, cp_entry& cpe) { cpe.len = read16(buffer, buffer_end); cpe.data = buffer; buffer += cpe.len; - if (buffer > buffer_end) { - throw RedexException(RedexError::BUFFER_END_EXCEEDED); - } return true; case CP_CONST_INVOKEDYN: std::cerr << "INVOKEDYN constant unsupported, Bailing\n"; @@ -192,10 +182,6 @@ constexpr size_t MAX_CLASS_NAMELEN = 8 * 1024; DexType* make_dextype_from_cref(std::vector& cpool, uint16_t cref) { char nbuffer[MAX_CLASS_NAMELEN]; - if (cref >= cpool.size()) { - std::cerr << "Illegal cref, Bailing\n"; - return nullptr; - } if (cpool[cref].tag != CP_CONST_CLASS) { std::cerr << "Non-class ref in get_class_name, Bailing\n"; return nullptr; @@ -210,24 +196,16 @@ DexType* make_dextype_from_cref(std::vector& cpool, uint16_t cref) { std::cerr << "classname is greater than max, bailing"; return nullptr; } - try { - nbuffer[0] = 'L'; - memcpy(nbuffer + 1, utf8cpe.data, utf8cpe.len); - nbuffer[1 + utf8cpe.len] = ';'; - nbuffer[2 + utf8cpe.len] = '\0'; - return DexType::make_type(nbuffer); - } catch (std::invalid_argument&) { - return nullptr; - } + nbuffer[0] = 'L'; + memcpy(nbuffer + 1, utf8cpe.data, utf8cpe.len); + nbuffer[1 + utf8cpe.len] = ';'; + nbuffer[2 + utf8cpe.len] = '\0'; + return DexType::make_type(nbuffer); } bool extract_utf8(std::vector& cpool, uint16_t utf8ref, std::string_view* out) { - if (utf8ref >= cpool.size()) { - std::cerr << "utf8 ref out of bound, bailing\n"; - return false; - } const cp_entry& utf8cpe = cpool[utf8ref]; if (utf8cpe.tag != CP_CONST_UTF8) { std::cerr << "Non-utf8 ref in get_utf8, bailing\n"; @@ -448,10 +426,6 @@ bool parse_class(uint8_t* buffer, } DexType* self = make_dextype_from_cref(cpool, clazz); - if (self == nullptr) { - std::cerr << "Bad class cpool index " << clazz << ", Bailing\n"; - return false; - } DexClass* cls = type_class(self); if (cls) { // We are seeing duplicate classes when parsing jar file @@ -487,10 +461,6 @@ bool parse_class(uint8_t* buffer, cc.set_external(); if (super != 0) { DexType* sclazz = make_dextype_from_cref(cpool, super); - if (sclazz == nullptr) { - std::cerr << "Bad super class cpool index " << super << ", Bailing\n"; - return false; - } cc.set_super(sclazz); } cc.set_access((DexAccessFlags)aflags); @@ -498,10 +468,6 @@ bool parse_class(uint8_t* buffer, for (int i = 0; i < ifcount; i++) { uint16_t iface = read16(buffer, buffer_end); DexType* iftype = make_dextype_from_cref(cpool, iface); - if (iftype == nullptr) { - std::cerr << "Bad interface cpool index " << super << ", Bailing\n"; - return false; - } cc.add_interface(iftype); } } diff --git a/libredex/JarLoader.h b/libredex/JarLoader.h index b6d313566d..4fdf8fb4b1 100644 --- a/libredex/JarLoader.h +++ b/libredex/JarLoader.h @@ -21,7 +21,7 @@ class DexMethod; namespace JarLoaderUtil { uint32_t read32(uint8_t*& buffer, uint8_t* buffer_end); -uint16_t read16(uint8_t*& buffer, uint8_t* buffer_end); +uint32_t read16(uint8_t*& buffer, uint8_t* buffer_end); }; // namespace JarLoaderUtil using attribute_hook_t = diff --git a/libredex/KeepReason.cpp b/libredex/KeepReason.cpp index b331c6b2dd..8e39765e9e 100644 --- a/libredex/KeepReason.cpp +++ b/libredex/KeepReason.cpp @@ -53,9 +53,10 @@ namespace { // Lint will complain about this, but it is better than having to // forward-declare all of concurrent containers. -std::unique_ptr> +std::unique_ptr> s_keep_reasons{nullptr}; } // namespace @@ -65,18 +66,17 @@ bool Reason::s_record_keep_reasons = false; void Reason::set_record_keep_reasons(bool v) { s_record_keep_reasons = v; if (v && s_keep_reasons == nullptr) { - s_keep_reasons = std::make_unique>(); } } Reason* Reason::try_insert(std::unique_ptr to_insert) { - auto [reason_ptr, emplaced] = s_keep_reasons->insert(to_insert.get()); - if (emplaced) { + if (s_keep_reasons->emplace(to_insert.get(), to_insert.get())) { return to_insert.release(); } - return const_cast(*reason_ptr); + return s_keep_reasons->at(to_insert.get()); } void Reason::release_keep_reasons() { s_keep_reasons.reset(); } diff --git a/libredex/Lazy.h b/libredex/Lazy.h index 4d46aedef4..d2792a06ab 100644 --- a/libredex/Lazy.h +++ b/libredex/Lazy.h @@ -11,8 +11,6 @@ #include #include -#include "Debug.h" - template // A convenient helper class for lazy initialization. // This class is not thread-safe. @@ -26,7 +24,7 @@ class Lazy { explicit Lazy(const std::function()>& creator) : m_creator(creator) {} // NOLINTNEXTLINE(google-explicit-constructor,hicpp-explicit-conversions) - operator bool() const { return !!m_value; } + operator bool() { return !!m_value; } T& operator*() { init(); return *m_value; @@ -35,10 +33,6 @@ class Lazy { init(); return m_value.get(); } - T* operator->() const { - redex_assert(m_value); - return m_value.get(); - } private: std::function()> m_creator; diff --git a/libredex/MethodDevirtualizer.cpp b/libredex/MethodDevirtualizer.cpp index 48a80b030f..1424428b04 100644 --- a/libredex/MethodDevirtualizer.cpp +++ b/libredex/MethodDevirtualizer.cpp @@ -65,9 +65,7 @@ void fix_call_sites(const std::vector& scope, return call_counter; } - always_assert(code->editable_cfg_built()); - auto& cfg = code->cfg(); - for (const MethodItemEntry& mie : InstructionIterable(cfg)) { + for (const MethodItemEntry& mie : InstructionIterable(code)) { IRInstruction* insn = mie.insn; if (!insn->has_method()) { continue; @@ -119,17 +117,14 @@ void make_methods_static(const std::unordered_set& methods, } bool uses_this(const DexMethod* method) { - auto code = (const_cast(method))->get_code(); + auto const* code = method->get_code(); always_assert_log(!is_static(method) && code != nullptr, "%s", SHOW(method)); - always_assert(code->editable_cfg_built()); - auto& cfg = code->cfg(); - auto first = cfg.entry_block()->get_first_insn(); - always_assert(first != cfg.entry_block()->end()); - auto this_insn = first->insn; + auto iterable = InstructionIterable(code); + auto const this_insn = iterable.begin()->insn; always_assert(this_insn->opcode() == IOPCODE_LOAD_PARAM_OBJECT); auto const this_reg = this_insn->dest(); - for (const auto& mie : cfg::InstructionIterable(cfg)) { + for (const auto& mie : iterable) { auto insn = mie.insn; for (unsigned i = 0; i < insn->srcs_size(); i++) { if (this_reg == insn->src(i)) { @@ -194,10 +189,6 @@ void MethodDevirtualizer::verify_and_split( TRACE(VIRT, 2, "failed to devirt method %s: keep", SHOW(m)); continue; } - if (m->rstate.no_optimizations()) { - TRACE(VIRT, 2, "failed to devirt method %s: no_optimizations", SHOW(m)); - continue; - } if (m->is_external() || is_abstract(m) || is_native(m)) { TRACE(VIRT, 2, diff --git a/libredex/MethodOverrideGraph.cpp b/libredex/MethodOverrideGraph.cpp index 74bfb0ce44..92b930b62f 100644 --- a/libredex/MethodOverrideGraph.cpp +++ b/libredex/MethodOverrideGraph.cpp @@ -13,7 +13,6 @@ #include #include "BinarySerialization.h" -#include "CppUtil.h" #include "Show.h" #include "Timer.h" #include "Walkers.h" @@ -46,14 +45,12 @@ struct ClassSignatureMap { SignatureMap unimplemented; }; -using ClassSignatureMaps = - InsertOnlyConcurrentMap; +using ClassSignatureMaps = ConcurrentMap; -using InterfaceSignatureMaps = - InsertOnlyConcurrentMap; +using InterfaceSignatureMaps = ConcurrentMap; using UnifiedInterfacesSignatureMaps = - InsertOnlyConcurrentMap; + ConcurrentMap; void update_signature_map(const DexMethod* method, MethodSet value, @@ -96,11 +93,10 @@ class GraphBuilder { } private: - const ClassSignatureMap& analyze_non_interface(const DexClass* cls) { + ClassSignatureMap analyze_non_interface(const DexClass* cls) { always_assert(!is_interface(cls)); - auto* res = m_class_signature_maps.get(cls); - if (res) { - return *res; + if (m_class_signature_maps.count(cls) != 0) { + return m_class_signature_maps.at(cls); } // Initialize the signature maps from those of the superclass. @@ -155,9 +151,7 @@ class GraphBuilder { &class_signatures.unimplemented); } - auto [map_ptr, emplaced] = - m_class_signature_maps.emplace(cls, class_signatures); - if (emplaced) { + if (m_class_signature_maps.emplace(cls, class_signatures)) { // Mark all overriding methods as reachable via their parent method ref. for (auto* method : cls->get_vmethods()) { const auto& overridden_set = @@ -183,16 +177,15 @@ class GraphBuilder { } } } + return class_signatures; } - - return *map_ptr; + return m_class_signature_maps.at(cls); } - const SignatureMap& analyze_interface(const DexClass* cls) { + SignatureMap analyze_interface(const DexClass* cls) { always_assert(is_interface(cls)); - auto* res = m_interface_signature_maps.get(cls); - if (res) { - return *res; + if (m_interface_signature_maps.count(cls) != 0) { + return m_interface_signature_maps.at(cls); } SignatureMap interface_signatures = unify_super_interface_signatures(cls); @@ -201,9 +194,7 @@ class GraphBuilder { update_signature_map(method, MethodSet{method}, &interface_signatures); } - auto [map_ptr, emplaced] = - m_interface_signature_maps.emplace(cls, interface_signatures); - if (emplaced) { + if (m_interface_signature_maps.emplace(cls, interface_signatures)) { for (auto* method : cls->get_vmethods()) { const auto& overridden_set = inherited_interface_signatures.at(method->get_name()) @@ -230,16 +221,16 @@ class GraphBuilder { method, /* overriding_is_interface */ true); } } - } - return *map_ptr; + return interface_signatures; + } + return m_interface_signature_maps.at(cls); } - const SignatureMap& unify_super_interface_signatures(const DexClass* cls) { + SignatureMap unify_super_interface_signatures(const DexClass* cls) { auto* type_list = cls->get_interfaces(); - auto* res = m_unified_interfaces_signature_maps.get(type_list); - if (res) { - return *res; + if (m_unified_interfaces_signature_maps.count(type_list)) { + return m_unified_interfaces_signature_maps.at(type_list); } SignatureMap super_interface_signatures; @@ -251,9 +242,11 @@ class GraphBuilder { } } - auto [map_ptr, _] = m_unified_interfaces_signature_maps.emplace( - type_list, super_interface_signatures); - return *map_ptr; + if (m_unified_interfaces_signature_maps.emplace( + type_list, super_interface_signatures)) { + return super_interface_signatures; + } + return m_unified_interfaces_signature_maps.at(type_list); } std::unique_ptr m_graph; @@ -370,171 +363,113 @@ std::vector get_overriding_methods(const Graph& graph, const DexMethod* method, bool include_interfaces, const DexType* base_type) { - std::vector res; - all_overriding_methods( - graph, method, - [&](const DexMethod* method) { - res.push_back(method); - return true; - }, - include_interfaces, base_type); - return res; -} - -std::vector get_overridden_methods(const Graph& graph, - const DexMethod* method, - bool include_interfaces) { - std::vector res; - all_overridden_methods( - graph, method, - [&](const DexMethod* method) { - res.push_back(method); - return true; - }, - include_interfaces); - return res; -} - -bool is_true_virtual(const Graph& graph, const DexMethod* method) { - if (is_abstract(method)) { - return true; - } - const auto& node = graph.get_node(method); - return !node.parents.empty() || !node.children.empty(); -} - -InsertOnlyConcurrentSet get_non_true_virtuals(const Graph& graph, - const Scope& scope) { - InsertOnlyConcurrentSet non_true_virtuals; - workqueue_run( - [&](DexClass* cls) { - for (auto* method : cls->get_vmethods()) { - if (!is_true_virtual(graph, method)) { - non_true_virtuals.insert(method); - } - } - }, - scope); - return non_true_virtuals; -} - -bool all_overriding_methods(const Graph& graph, - const DexMethod* method, - const std::function& f, - bool include_interfaces, - const DexType* base_type) { + std::vector overrides; const Node& root = graph.get_node(method); if (base_type && method->get_class() == base_type) { base_type = nullptr; } if (root.is_interface) { std::unordered_set visited{method}; - return self_recursive_fn( - [&](auto self, const auto& children) -> bool { - for (const auto* current : children) { - if (!visited.emplace(current).second) { - continue; - } - const Node& node = graph.get_node(current); - if (!self(self, node.children)) { - return false; - } - if ((include_interfaces || !node.is_interface) && - (!base_type || node.overrides(current, base_type)) && - !f(current)) { - return false; - } + std::function visit = + [&](const DexMethod* current) { + if (!visited.emplace(current).second) { + return; } - return true; - }, - root.children); - } - // optimized code path - return self_recursive_fn( - [&](auto self, const auto& children) -> bool { - for (const auto* current : children) { const Node& node = graph.get_node(current); - if (!self(self, node.children)) { - return false; + for (const auto* child : node.children) { + visit(child); } - if ((!base_type || node.overrides(current, base_type)) && - !f(current)) { - return false; + if ((include_interfaces || !node.is_interface) && + (!base_type || node.overrides(current, base_type))) { + overrides.push_back(current); } - } - return true; - }, - root.children); -} - -bool any_overriding_methods(const Graph& graph, - const DexMethod* method, - const std::function& f, - bool include_interfaces, - const DexType* base_type) { - return !all_overriding_methods( - graph, method, [&](const DexMethod* m) { return !f(m); }, - include_interfaces, base_type); + }; + for (const auto* child : root.children) { + visit(child); + } + return overrides; + } + // optimized code path + std::function visit = [&](const DexMethod* current) { + const Node& node = graph.get_node(current); + for (const auto* child : node.children) { + visit(child); + } + if (!base_type || node.overrides(current, base_type)) { + overrides.push_back(current); + } + }; + for (const auto* child : root.children) { + visit(child); + } + return overrides; } -bool all_overridden_methods(const Graph& graph, - const DexMethod* method, - const std::function& f, - bool include_interfaces) { +std::vector get_overridden_methods(const Graph& graph, + const DexMethod* method, + bool include_interfaces) { + std::vector overridden; const Node& root = graph.get_node(method); if (include_interfaces) { std::unordered_set visited{method}; - return self_recursive_fn( - [&](auto self, const auto& children) -> bool { - for (const auto* current : children) { - if (!visited.emplace(current).second) { - continue; - } - const Node& node = graph.get_node(current); - if (!include_interfaces && node.is_interface) { - continue; - } - if (!self(self, node.parents)) { - return false; - } - if (!f(current)) { - return false; - } + std::function visit = + [&](const DexMethod* current) { + if (!visited.emplace(current).second) { + return; + } + const Node& node = graph.get_node(current); + if (!include_interfaces && node.is_interface) { + return; } - return true; - }, - root.parents); + for (const auto* parent : node.parents) { + visit(parent); + } + overridden.push_back(current); + }; + for (const auto* parent : root.parents) { + visit(parent); + } + return overridden; } if (root.is_interface) { - return true; + return overridden; } // optimized code path - return self_recursive_fn( - [&](auto self, const auto& children) -> bool { - for (const auto* current : children) { - const Node& node = graph.get_node(current); - if (node.is_interface) { - continue; - } - if (!self(self, node.parents)) { - return false; - } - if (!f(current)) { - return false; - } - } - return true; - }, - root.parents); + std::function visit = [&](const DexMethod* current) { + const Node& node = graph.get_node(current); + if (!include_interfaces && node.is_interface) { + return; + } + for (const auto* parent : node.parents) { + visit(parent); + } + overridden.push_back(current); + }; + for (const auto* parent : root.parents) { + visit(parent); + } + return overridden; } -bool any_overridden_methods(const Graph& graph, - const DexMethod* method, - const std::function& f, - bool include_interfaces) { - return !all_overridden_methods( - graph, method, [&](const DexMethod* m) { return !f(m); }, - include_interfaces); +bool is_true_virtual(const Graph& graph, const DexMethod* method) { + if (is_abstract(method)) { + return true; + } + const auto& node = graph.get_node(method); + return !node.parents.empty() || !node.children.empty(); +} + +std::unordered_set get_non_true_virtuals(const Graph& graph, + const Scope& scope) { + std::unordered_set non_true_virtuals; + for (const auto* cls : scope) { + for (auto* method : cls->get_vmethods()) { + if (!is_true_virtual(graph, method)) { + non_true_virtuals.emplace(method); + } + } + } + return non_true_virtuals; } } // namespace method_override_graph diff --git a/libredex/MethodOverrideGraph.h b/libredex/MethodOverrideGraph.h index 62f2ae4c3a..4dceba10aa 100644 --- a/libredex/MethodOverrideGraph.h +++ b/libredex/MethodOverrideGraph.h @@ -57,8 +57,8 @@ bool is_true_virtual(const Graph& graph, const DexMethod* method); /* * Return all non-true-virtuals in scope. */ -InsertOnlyConcurrentSet get_non_true_virtuals(const Graph& graph, - const Scope& scope); +std::unordered_set get_non_true_virtuals(const Graph& graph, + const Scope& scope); /* * When a class method implements interface methods only in a subclass of the @@ -117,28 +117,4 @@ class Graph { ConcurrentMap m_nodes; }; -bool all_overriding_methods(const Graph& graph, - const DexMethod* method, - const std::function& f, - bool include_interfaces = false, - const DexType* base_type = nullptr); - -bool any_overriding_methods( - const Graph& graph, - const DexMethod* method, - const std::function& f = [](auto*) { return true; }, - bool include_interfaces = false, - const DexType* base_type = nullptr); - -bool all_overridden_methods(const Graph& graph, - const DexMethod* method, - const std::function& f, - bool include_interfaces); - -bool any_overridden_methods( - const Graph& graph, - const DexMethod* method, - const std::function& f = [](auto*) { return true; }, - bool include_interfaces = false); - } // namespace method_override_graph diff --git a/libredex/MethodProfiles.cpp b/libredex/MethodProfiles.cpp index c7dcb6a784..9594f4578e 100644 --- a/libredex/MethodProfiles.cpp +++ b/libredex/MethodProfiles.cpp @@ -47,8 +47,7 @@ bool empty_column(std::string_view sv) { return sv.empty() || sv == "\n"; } } // namespace -AccumulatingTimer MethodProfiles::s_process_unresolved_lines_timer( - "MethodProfiles::process_unresolved_lines"); +AccumulatingTimer MethodProfiles::s_process_unresolved_lines_timer; const StatsMap& MethodProfiles::method_stats( const std::string& interaction_id) const { @@ -312,48 +311,6 @@ size_t MethodProfiles::derive_stats(DexMethod* target, return res; } -size_t MethodProfiles::substitute_stats( - DexMethod* target, const std::vector& sources) { - size_t res = 0; - for (auto& [interaction_id, method_stats] : m_method_stats) { - std::optional stats; - for (auto* src : sources) { - auto it = method_stats.find(src); - if (it == method_stats.end()) { - continue; - } - if (!stats) { - stats = it->second; - continue; - } - stats->appear_percent = - std::max(stats->appear_percent, it->second.appear_percent); - stats->call_count += it->second.call_count; - stats->order_percent = - std::min(stats->order_percent, it->second.order_percent); - stats->min_api_level = - std::min(stats->min_api_level, it->second.min_api_level); - } - - if (stats) { - auto target_it = method_stats.find(target); - if (target_it != method_stats.end()) { - const auto& target_stats = target_it->second; - if (target_stats.min_api_level == stats->min_api_level && - target_stats.appear_percent == stats->appear_percent && - target_stats.call_count == stats->call_count) { - // Target method has stats and is same as the stats to be substituted. - // Do not change. - return 0; - } - } - method_stats.emplace(target, *stats); - res++; - } - } - return res; -} - bool MethodProfiles::parse_main(const std::string& line, std::string* interaction_id) { auto result = parse_main_internal(line); @@ -384,6 +341,10 @@ boost::optional MethodProfiles::get_interaction_count( } } +double MethodProfiles::get_process_unresolved_lines_seconds() { + return s_process_unresolved_lines_timer.get_seconds(); +} + void MethodProfiles::process_unresolved_lines() { if (m_unresolved_lines.empty()) { return; diff --git a/libredex/MethodProfiles.h b/libredex/MethodProfiles.h index 0a109f6e59..af1dad6293 100644 --- a/libredex/MethodProfiles.h +++ b/libredex/MethodProfiles.h @@ -125,6 +125,8 @@ class MethodProfiles { // Try to resolve previously unresolved lines void process_unresolved_lines(); + static double get_process_unresolved_lines_seconds(); + std::unordered_set get_unresolved_method_descriptor_tokens() const; @@ -137,10 +139,6 @@ class MethodProfiles { size_t derive_stats(DexMethod* target, const std::vector& sources); - // Substitute target method's stat with derived stats from given sources. - size_t substitute_stats(DexMethod* target, - const std::vector& sources); - private: static AccumulatingTimer s_process_unresolved_lines_timer; AllInteractions m_method_stats; diff --git a/libredex/MethodSimilarityCompressionConsciousOrderer.cpp b/libredex/MethodSimilarityCompressionConsciousOrderer.cpp index 5cd96438c5..c106e0c8f2 100644 --- a/libredex/MethodSimilarityCompressionConsciousOrderer.cpp +++ b/libredex/MethodSimilarityCompressionConsciousOrderer.cpp @@ -202,7 +202,9 @@ std::vector create_kmers(const std::vector& content) { std::vector MethodSimilarityCompressionConsciousOrderer::get_encoded_method_content( - DexMethod* meth, DexOutputIdx& dodx, std::unique_ptr& output) { + DexMethod* meth, + std::unique_ptr& dodx, + std::unique_ptr& output) { // Get the code DexCode* code = meth->get_dex_code(); always_assert_log(code != nullptr, "Empty code for method %s", SHOW(meth)); @@ -211,7 +213,7 @@ MethodSimilarityCompressionConsciousOrderer::get_encoded_method_content( memset(output.get(), 0, METHOD_MAX_OUTPUT_SIZE); // Encode - size_t size = code->encode(&dodx, (uint32_t*)(output.get())); + size_t size = code->encode(dodx.get(), (uint32_t*)(output.get())); always_assert_log(size <= METHOD_MAX_OUTPUT_SIZE, "Encoded code size limit exceeded %zu versus %zu", size, METHOD_MAX_OUTPUT_SIZE); @@ -234,7 +236,7 @@ void MethodSimilarityCompressionConsciousOrderer::order( // We assume no method takes more than 512KB auto output = std::make_unique(METHOD_MAX_OUTPUT_SIZE); - auto dodx = m_gtypes->get_dodx(output.get()); + auto dodx = std::make_unique(*m_gtypes->get_dodx(output.get())); // Collect binary functions in the original order std::vector functions; diff --git a/libredex/MethodSimilarityCompressionConsciousOrderer.h b/libredex/MethodSimilarityCompressionConsciousOrderer.h index 6a8db9ffe3..20fd20fa23 100644 --- a/libredex/MethodSimilarityCompressionConsciousOrderer.h +++ b/libredex/MethodSimilarityCompressionConsciousOrderer.h @@ -36,7 +36,9 @@ class MethodSimilarityCompressionConsciousOrderer { private: // The content of the method (a sequence of bytes representing the method). std::vector get_encoded_method_content( - DexMethod* meth, DexOutputIdx& dodx, std::unique_ptr& output); + DexMethod* meth, + std::unique_ptr& dodx, + std::unique_ptr& output); public: void order(std::vector& methods, GatheredTypes* m_gtypes); diff --git a/libredex/MethodUtil.cpp b/libredex/MethodUtil.cpp index fe6e312996..6134552b81 100644 --- a/libredex/MethodUtil.cpp +++ b/libredex/MethodUtil.cpp @@ -19,7 +19,7 @@ class ClInitSideEffectsAnalysis { private: bool m_allow_benign_method_invocations; const method::ClInitHasNoSideEffectsPredicate* m_clinit_has_no_side_effects; - const InsertOnlyConcurrentSet* m_non_true_virtuals; + const std::unordered_set* m_non_true_virtuals; std::unordered_set m_active; std::unordered_set m_initialized; @@ -27,7 +27,7 @@ class ClInitSideEffectsAnalysis { explicit ClInitSideEffectsAnalysis( bool allow_benign_method_invocations, const method::ClInitHasNoSideEffectsPredicate* clinit_has_no_side_effects, - const InsertOnlyConcurrentSet* non_true_virtuals) + const std::unordered_set* non_true_virtuals) : m_allow_benign_method_invocations(allow_benign_method_invocations), m_clinit_has_no_side_effects(clinit_has_no_side_effects), m_non_true_virtuals(non_true_virtuals) {} @@ -109,7 +109,7 @@ class ClInitSideEffectsAnalysis { return true; } if (opcode::is_invoke_virtual(insn->opcode()) && - (!m_non_true_virtuals || !m_non_true_virtuals->count_unsafe(method))) { + (!m_non_true_virtuals || !m_non_true_virtuals->count(method))) { return true; } if (opcode::is_invoke_static(insn->opcode()) && @@ -498,7 +498,7 @@ const DexClass* clinit_may_have_side_effects( const DexClass* cls, bool allow_benign_method_invocations, const ClInitHasNoSideEffectsPredicate* clinit_has_no_side_effects, - const InsertOnlyConcurrentSet* non_true_virtuals) { + const std::unordered_set* non_true_virtuals) { ClInitSideEffectsAnalysis analysis(allow_benign_method_invocations, clinit_has_no_side_effects, non_true_virtuals); diff --git a/libredex/MethodUtil.h b/libredex/MethodUtil.h index 4ff52a8776..622ab3302c 100644 --- a/libredex/MethodUtil.h +++ b/libredex/MethodUtil.h @@ -7,7 +7,6 @@ #pragma once -#include "ConcurrentContainers.h" #include "ControlFlow.h" #include "DexClass.h" #include "IROpcode.h" @@ -66,7 +65,7 @@ const DexClass* clinit_may_have_side_effects( const DexClass* cls, bool allow_benign_method_invocations, const ClInitHasNoSideEffectsPredicate* clinit_has_no_side_effects = nullptr, - const InsertOnlyConcurrentSet* non_true_virtuals = nullptr); + const std::unordered_set* non_true_virtuals = nullptr); /** * Check that the method contains no invoke-super instruction; this is a diff --git a/libredex/MonitorCount.cpp b/libredex/MonitorCount.cpp index ff0d64b02c..74e1ae85cb 100644 --- a/libredex/MonitorCount.cpp +++ b/libredex/MonitorCount.cpp @@ -104,8 +104,7 @@ std::vector Analyzer::get_monitor_mismatches() { // Dead block continue; } - if (count.is_top() && - !m_cfg.get_pred_edge_of_type(block, cfg::EDGE_GHOST)) { + if (count.is_top()) { blocks.push_back(block); } } diff --git a/libredex/NullnessDomain.h b/libredex/NullnessDomain.h index a554a165bb..addcbe0165 100644 --- a/libredex/NullnessDomain.h +++ b/libredex/NullnessDomain.h @@ -7,7 +7,11 @@ #pragma once +#include +#include #include +#include +#include #include "DexUtil.h" @@ -46,3 +50,165 @@ using NullnessDomain = sparta::FiniteAbstractDomain; std::ostream& operator<<(std::ostream& output, const Nullness& nullness); + +/* + * Constant domain + * + * Simple domain that tracks the value of integer constants. + */ +using ConstantDomain = sparta::ConstantAbstractDomain; + +/* + * ConstNullness domain + * + * A const integer value can have nullness. E.g., const 0 -> NULL. + */ +class ConstNullnessDomain final + : public sparta::ReducedProductAbstractDomain { + public: + using ReducedProductAbstractDomain::ReducedProductAbstractDomain; + + // Some older compilers complain that the class is not default + // constructible. We intended to use the default constructors of the base + // class (via the `using` declaration above), but some compilers fail to + // catch this. So we insert a redundant '= default'. + ConstNullnessDomain() = default; + + explicit ConstNullnessDomain(Nullness nullness) + : ConstNullnessDomain(std::make_tuple( + nullness == IS_NULL ? ConstantDomain(0) : ConstantDomain::top(), + NullnessDomain(nullness))) {} + + explicit ConstNullnessDomain(int64_t v) + : ConstNullnessDomain(std::make_tuple( + ConstantDomain(v), NullnessDomain(v == 0 ? IS_NULL : NOT_NULL))) {} + + ConstNullnessDomain null() { return ConstNullnessDomain(IS_NULL); } + + static void reduce_product( + std::tuple& /* domains */) {} + + ConstantDomain const_domain() const { return get<0>(); } + + boost::optional get_constant() const { + return const_domain().get_constant(); + } + + NullnessDomain get_nullness() const { return get<1>(); } +}; + +/* + * Spec wise the max size of a Java array is + * std::numeric_limits::max() - 8. Reference: + * http://hg.openjdk.java.net/jdk7/jdk7/jdk/rev/ec45423a4700#l5.12 + * + * However, for performance reasons, we don't want to allocate a Domain this + * large. We cap the size of the array elements at 1000. + */ +constexpr int64_t JAVA_ARRAY_SIZE_MAX = 1000; + +class ArrayNullnessDomain final + : public sparta::ReducedProductAbstractDomain< + ArrayNullnessDomain, + NullnessDomain /* nullness of the array object */, + sparta::ConstantAbstractDomain /* array length */, + sparta::PatriciaTreeMapAbstractEnvironment< + uint32_t, + NullnessDomain> /* array elements */> { + + public: + using ArrayLengthDomain = sparta::ConstantAbstractDomain; + using ElementsNullness = + sparta::PatriciaTreeMapAbstractEnvironment; + using BaseType = sparta::ReducedProductAbstractDomain; + using ReducedProductAbstractDomain::ReducedProductAbstractDomain; + + // Some older compilers complain that the class is not default constructible. + // We intended to use the default constructors of the base class (via the + // `using` declaration above), but some compilers fail to catch this. So we + // insert a redundant '= default'. + ArrayNullnessDomain() = default; + + static void reduce_product( + std::tuple& + product) { + if (std::get<1>(product).is_top()) { + std::get<2>(product).set_to_top(); + } + } + + explicit ArrayNullnessDomain(uint32_t length) + : ReducedProductAbstractDomain(std::make_tuple(NullnessDomain(NOT_NULL), + ArrayLengthDomain(length), + ElementsNullness())) { + mutate_elements([length](ElementsNullness* elements) { + for (size_t i = 0; i < length; ++i) { + elements->set(i, NullnessDomain(UNINITIALIZED)); + } + }); + reduce(); + } + + NullnessDomain get_nullness() const { return get<0>(); } + + boost::optional get_length() const { + return get<1>().get_constant(); + } + + ElementsNullness get_elements() const { return get<2>(); } + + void reset_elements() { + apply<2>([](auto elements) { elements->set_to_top(); }); + } + + NullnessDomain get_element(uint32_t idx) const { + return get_elements().get(idx); + } + + ArrayNullnessDomain& set_element(uint32_t idx, const NullnessDomain& domain) { + if (is_top() || is_bottom()) { + return *this; + } + auto length = get_length(); + if (!length || idx >= *length) { + return *this; + } + return this->mutate_elements([idx, domain](ElementsNullness* elements) { + elements->set(idx, domain); + }); + } + + void join_with(const ArrayNullnessDomain& other) { + BaseType::join_with(other); + reduce(); + } + + void widen_with(const ArrayNullnessDomain& other) { + BaseType::widen_with(other); + reduce(); + } + + static bool is_valid_array_size(boost::optional& val) { + return val && *val >= 0 && *val <= JAVA_ARRAY_SIZE_MAX; + } + + static bool is_valid_array_idx(boost::optional& val) { + return val && *val >= 0 && *val < JAVA_ARRAY_SIZE_MAX; + } + + private: + ArrayNullnessDomain& mutate_elements( + std::function f) { + this->template apply<2>(std::move(f)); + return *this; + } +}; + +using ArrayConstNullnessDomain = + sparta::DisjointUnionAbstractDomain; diff --git a/libredex/PassManager.cpp b/libredex/PassManager.cpp index 5b883bd7c6..78a05e1df7 100644 --- a/libredex/PassManager.cpp +++ b/libredex/PassManager.cpp @@ -34,7 +34,6 @@ #include "ApiLevelChecker.h" #include "AssetManager.h" #include "CFGMutation.h" -#include "CallGraph.h" #include "ClassChecker.h" #include "CommandProfiling.h" #include "ConfigFiles.h" @@ -42,7 +41,6 @@ #include "DexClass.h" #include "DexLoader.h" #include "DexOutput.h" -#include "DexStructure.h" #include "DexUtil.h" #include "GlobalConfig.h" #include "GraphVisualizer.h" @@ -66,23 +64,17 @@ #include "ScopedMetrics.h" #include "Show.h" #include "SourceBlocks.h" -#include "ThreadPool.h" #include "Timer.h" #include "Walkers.h" namespace { -AccumulatingTimer m_hashers_timer{"PassManager.Hashers"}; -AccumulatingTimer m_check_unique_deobfuscateds_timer{ - "PassManager.CheckUniqueDeobfuscateds"}; - constexpr const char* INCOMING_HASHES = "incoming_hashes.txt"; constexpr const char* OUTGOING_HASHES = "outgoing_hashes.txt"; constexpr const char* REMOVABLE_NATIVES = "redex-removable-natives.txt"; const std::string PASS_ORDER_KEY = "pass_order"; const Pass* get_profiled_pass(const PassManager& mgr) { - // NOLINTNEXTLINE(bugprone-assert-side-effect) redex_assert(getenv("PROFILE_PASS") != nullptr); // Resolve the pass in the constructor so that any typos / references to // nonexistent passes are caught as early as possible @@ -234,8 +226,7 @@ class CheckerConfig { } }); } - auto* code = dex_method->get_code(); - return code->editable_cfg_built() ? show(code->cfg()) : show(code); + return show(dex_method->get_code()); }; auto res = @@ -688,8 +679,6 @@ class AfterPassSizes { tmp_dir = dir_name; } - redex_thread_pool::ThreadPool::get_instance()->join(); - pid_t p = fork(); if (p < 0) { @@ -806,14 +795,8 @@ class AfterPassSizes { } // Better run MakePublicPass. maybe_run("MakePublicPass"); - // Run ReBindRefsPass to not get a 'trying to encode too many method refs in - // dex' error - maybe_run("ReBindRefsPass"); - // May need register allocation. Run InjectionIdLoweringPass too so - // RegAllocPass doesn't fail on injection-id opcodes + // May need register allocation. if (!m_mgr->regalloc_has_run()) { - maybe_run("IntrinsifyInjectionIdsPass"); - maybe_run("InjectionIdLoweringPass"); maybe_run("RegAllocPass"); } @@ -889,14 +872,8 @@ class TraceClassAfterEachPass { } for (auto* v : methods) { fprintf(fd, "Method %s\n", SHOW(v)); - auto code = v->get_code(); - if (code != nullptr) { - if (code->editable_cfg_built()) { - auto& cfg = code->cfg(); - fprintf(fd, "%s\n", SHOW(cfg)); - } else { - fprintf(fd, "%s\n", SHOW(code)); - } + if (v->get_code()) { + fprintf(fd, "%s\n", SHOW(v->get_code())); } } } @@ -1144,7 +1121,7 @@ void PassManager::init_property_interactions(ConfigFiles& conf) { always_assert_log(property_interaction.is_valid(), "%s has an invalid property interaction for %s", - pass->name().c_str(), redex_properties::get_name(name)); + pass->name().c_str(), name.c_str()); ++it; } pass_info->property_interactions = std::move(m); @@ -1270,7 +1247,6 @@ void PassManager::run_passes(DexStoresVector& stores, ConfigFiles& conf) { IRCode& code) { if (is_editable_cfg_friendly) { always_assert_log(code.editable_cfg_built(), "%s has a cfg!", SHOW(m)); - code.cfg().reset_exit_block(); } if (slow_invariants_debug) { std::vector methods; @@ -1347,7 +1323,6 @@ void PassManager::run_passes(DexStoresVector& stores, ConfigFiles& conf) { ///////////////////// // MAIN PASS LOOP. // ///////////////////// - bool handled_child = false; for (size_t i = 0; i < m_activated_passes.size(); ++i) { Pass* pass = m_activated_passes[i]; const size_t pass_run = ++runs[pass]; @@ -1407,8 +1382,6 @@ void PassManager::run_passes(DexStoresVector& stores, ConfigFiles& conf) { }); } - g_redex->compact(); - trace_cls.dump(pass->name()); cpu_time = cpu_time_end - cpu_time_start; @@ -1430,8 +1403,7 @@ void PassManager::run_passes(DexStoresVector& stores, ConfigFiles& conf) { process_method_profiles(*this, conf); process_secondary_method_profiles(*this, conf); - handled_child = after_pass_size.handle(m_current_pass_info, &stores, &conf); - if (handled_child) { + if (after_pass_size.handle(m_current_pass_info, &stores, &conf)) { // Measuring child. Return to write things out. break; } @@ -1464,9 +1436,6 @@ void PassManager::run_passes(DexStoresVector& stores, ConfigFiles& conf) { jni_native_context_helper.post_passes(scope, conf); check_unique_deobfuscated.run_finally(scope); - if (!handled_child) { - check_unreleased_reserved_refs(); - } graph_visualizer.finalize(); @@ -1475,9 +1444,17 @@ void PassManager::run_passes(DexStoresVector& stores, ConfigFiles& conf) { sanitizers::lsan_do_recoverable_leak_check(); - for (auto& [name, seconds] : AccumulatingTimer::get_times()) { - Timer::add_timer(std::move(name), seconds); - } + Timer::add_timer("PassManager.Hashers", m_hashers_timer.get_seconds()); + Timer::add_timer("PassManager.CheckUniqueDeobfuscateds", + m_check_unique_deobfuscateds_timer.get_seconds()); + Timer::add_timer("CFGMutation", cfg::CFGMutation::get_seconds()); + Timer::add_timer( + "MethodProfiles::process_unresolved_lines", + method_profiles::MethodProfiles::get_process_unresolved_lines_seconds()); + Timer::add_timer("compute_locations_closure_wto", + get_compute_locations_closure_wto_seconds()); + Timer::add_timer("cc_impl::destructor_second", + cc_impl::get_destructor_seconds()); } PassManager::ActivatedPasses PassManager::compute_activated_passes( @@ -1611,29 +1588,3 @@ PassManager::get_interdex_metrics() { static std::unordered_map empty; return empty; } - -ReserveRefsInfoHandle PassManager::reserve_refs(const std::string& name, - const ReserveRefsInfo& info) { - return m_reserved_ref_infos.insert(m_reserved_ref_infos.end(), - std::make_pair(name, info)); -} - -void PassManager::release_reserved_refs(ReserveRefsInfoHandle handle) { - m_reserved_ref_infos.erase(handle); -} - -ReserveRefsInfo PassManager::get_reserved_refs() const { - ReserveRefsInfo res; - for (const auto& [_, info] : m_reserved_ref_infos) { - res += info; - } - return res; -} - -void PassManager::check_unreleased_reserved_refs() { - for (const auto& [name, info] : m_reserved_ref_infos) { - fprintf(stderr, "ABORT! Unreleased reserved refs: %s(%zu, %zu, %zu)\n", - name.c_str(), info.frefs, info.trefs, info.mrefs); - exit(EXIT_FAILURE); - } -} diff --git a/libredex/PassManager.h b/libredex/PassManager.h index da2bfb90fa..b4ee563e31 100644 --- a/libredex/PassManager.h +++ b/libredex/PassManager.h @@ -8,7 +8,6 @@ #pragma once #include -#include #include #include #include @@ -23,6 +22,7 @@ #include "RedexOptions.h" #include "RedexProperties.h" #include "RedexPropertyCheckerRegistry.h" +#include "Timer.h" struct ConfigFiles; class DexStore; @@ -44,10 +44,6 @@ namespace redex_properties { class Manager; } // namespace redex_properties -struct ReserveRefsInfo; -using ReserveRefsInfoList = std::list>; -using ReserveRefsInfoHandle = ReserveRefsInfoList::iterator; - class PassManager { public: explicit PassManager(const std::vector& passes); @@ -146,16 +142,6 @@ class PassManager { const ConfigFiles& config, PassManagerConfig* pm_config_override = nullptr); - // Reserves refs in every dex, effectively lowering the capacity of each dex. - // This is applied uniformly (e.g., cannot be a per-dex value). - // All reserved refs must eventually released. - ReserveRefsInfoHandle reserve_refs(const std::string& name, - const ReserveRefsInfo& info); - - void release_reserved_refs(ReserveRefsInfoHandle handle); - - ReserveRefsInfo get_reserved_refs() const; - private: void init(const ConfigFiles& config); @@ -165,8 +151,6 @@ class PassManager { void init_property_interactions(ConfigFiles& conf); - void check_unreleased_reserved_refs(); - AssetManager m_asset_mgr; std::vector m_registered_passes; std::vector m_activated_passes; @@ -184,10 +168,11 @@ class PassManager { bool m_materialize_nullchecks_has_run{false}; bool m_interdex_has_run{false}; bool m_unreliable_virtual_scopes{false}; - ReserveRefsInfoList m_reserved_ref_infos; Pass* m_malloc_profile_pass{nullptr}; boost::optional m_initial_hash; + AccumulatingTimer m_hashers_timer; + AccumulatingTimer m_check_unique_deobfuscateds_timer; std::vector> m_cloned_passes; diff --git a/libredex/PluginRegistry.h b/libredex/PluginRegistry.h index e1f9b31b9e..4d37c63572 100644 --- a/libredex/PluginRegistry.h +++ b/libredex/PluginRegistry.h @@ -47,7 +47,6 @@ class PluginEntry : public Plugin { } std::vector> create_plugins() { std::vector> res; - res.reserve(m_ordered_creator_names.size()); for (const auto& name : m_ordered_creator_names) { res.emplace_back(std::move(create(name))); } diff --git a/libredex/PriorityThreadPool.h b/libredex/PriorityThreadPool.h index b807481db1..4777a1206c 100644 --- a/libredex/PriorityThreadPool.h +++ b/libredex/PriorityThreadPool.h @@ -8,9 +8,14 @@ #pragma once #include +#include #include // For `default_num_threads`. +// We for now need a larger stack size than the default, and on Mac OS +// this is the only way (or pthreads directly), as `ulimit -s` does not +// apply to non-main threads. +#include #include #include #include @@ -19,7 +24,6 @@ #include #include "Debug.h" -#include "ThreadPool.h" #include "WorkQueue.h" // For redex_queue_exception_handler. /* @@ -34,13 +38,10 @@ class PriorityThreadPool { private: std::vector m_pool; - size_t m_threads{0}; // The following data structures are guarded by this mutex. std::mutex m_mutex; - size_t m_running{0}; std::condition_variable m_work_condition; std::condition_variable m_done_condition; - std::condition_variable m_not_running_condition; std::map>> m_pending_work_items; std::atomic m_running_work_items{0}; std::chrono::duration m_waited_time{0}; @@ -59,7 +60,7 @@ class PriorityThreadPool { // If the pool was created (>0 threads), `join` must be manually called // before the executor may be destroyed. always_assert(m_pending_work_items.empty()); - if (m_threads > 0) { + if (!m_pool.empty()) { always_assert(m_shutdown); always_assert(m_running_work_items == 0); } @@ -72,38 +73,24 @@ class PriorityThreadPool { // The number of threads may be set at most once to a positive number void set_num_threads(int num_threads) { - always_assert(m_threads == 0); + always_assert(m_pool.empty()); always_assert(!m_shutdown); - m_threads = num_threads; - { - std::unique_lock lock{m_mutex}; - m_running = num_threads; - } + if (num_threads > 0) { + // std::thread cannot be copied, so need to do this in a loop instead of + // `resize`. - if (num_threads == 0) { - return; - } + boost::thread::attributes attrs; + attrs.set_stack_size(8 * 1024 * 1024); // 8MB stack. - sparta::AsyncRunner* async_runner = - redex_thread_pool::ThreadPool::get_instance(); - if (async_runner) { - for (int i = 0; i < num_threads; ++i) { - async_runner->run_async(&PriorityThreadPool::run, this); + for (size_t i = 0; i != (size_t)num_threads; ++i) { + m_pool.emplace_back(attrs, [this]() { this->run(); }); } - return; - } - - boost::thread::attributes attrs; - attrs.set_stack_size(8 * 1024 * 1024); // 8MB stack. - - for (int i = 0; i < num_threads; ++i) { - m_pool.emplace_back(attrs, [this]() { this->run(); }); } } // Post a work item with a priority. This method is thread safe. void post(int priority, const std::function& f) { - always_assert(m_threads > 0); + always_assert(!m_pool.empty()); std::unique_lock lock{m_mutex}; always_assert(!m_shutdown); m_pending_work_items[priority].push(f); @@ -112,7 +99,7 @@ class PriorityThreadPool { // Wait for all work items to be processed. void wait(bool init_shutdown = false) { - always_assert(m_threads > 0); + always_assert(!m_pool.empty()); auto start = std::chrono::system_clock::now(); { // We wait until *all* work is done, i.e. nothing is running or pending. @@ -130,7 +117,7 @@ class PriorityThreadPool { } void join(bool allow_new_work = true) { - always_assert(m_threads > 0); + always_assert(!m_pool.empty()); always_assert(!m_shutdown); if (!allow_new_work) { std::unique_lock lock{m_mutex}; @@ -138,12 +125,6 @@ class PriorityThreadPool { m_work_condition.notify_all(); } wait(/*init_shutdown=*/allow_new_work); - { - std::unique_lock lock{m_mutex}; - while (m_running > 0) { - m_not_running_condition.wait(lock); - } - } for (auto& thread : m_pool) { thread.join(); } @@ -151,15 +132,9 @@ class PriorityThreadPool { private: void run() { - auto not_running = [&]() { - std::unique_lock lock{m_mutex}; - if (--m_running == 0) { - m_not_running_condition.notify_one(); - } - }; - for (bool first = true;; first = false) { - auto highest_priority_f = [&]() -> std::optional> { + auto highest_priority_f = + [&]() -> boost::optional> { std::unique_lock lock{m_mutex}; // Notify when *all* work is done, i.e. nothing is running or pending. @@ -177,7 +152,7 @@ class PriorityThreadPool { }); if (m_pending_work_items.empty()) { redex_assert(m_shutdown); - return std::nullopt; + return boost::none; } m_running_work_items++; @@ -193,7 +168,6 @@ class PriorityThreadPool { return f; }(); if (!highest_priority_f) { - not_running(); return; } @@ -202,7 +176,6 @@ class PriorityThreadPool { (*highest_priority_f)(); } catch (std::exception& e) { redex_workqueue_impl::redex_queue_exception_handler(e); - not_running(); throw; } diff --git a/libredex/PriorityThreadPoolDAGScheduler.h b/libredex/PriorityThreadPoolDAGScheduler.h index 30e3728304..fdbecd1c06 100644 --- a/libredex/PriorityThreadPoolDAGScheduler.h +++ b/libredex/PriorityThreadPoolDAGScheduler.h @@ -20,53 +20,71 @@ class PriorityThreadPoolDAGScheduler { PriorityThreadPool m_priority_thread_pool; Executor m_executor; std::unordered_map> m_waiting_for; - std::unordered_map> m_wait_counts; + std::unordered_map m_wait_counts; std::unique_ptr> m_priorities; int m_max_priority{-1}; - using Continuations = std::vector>; - std::unique_ptr> - m_concurrent_continuations; + struct ConcurrentState { + uint32_t wait_count{0}; + std::vector> continuations{}; + }; + std::unique_ptr> m_concurrent_states; int compute_priority(Task task) { - auto [it, success] = m_priorities->emplace(task, 0); - auto& priority = it->second; - if (!success) { - return priority; + auto it = m_priorities->find(task); + if (it != m_priorities->end()) { + return it->second; } + auto value = 0; auto it2 = m_waiting_for.find(task); if (it2 != m_waiting_for.end()) { for (auto other_task : it2->second) { - priority = std::max(priority, compute_priority(other_task) + 1); + value = std::max(value, compute_priority(other_task) + 1); } } - m_max_priority = std::max(m_max_priority, priority); - return priority; + m_priorities->emplace(task, value); + m_max_priority = std::max(m_max_priority, value); + return value; } uint32_t increment_wait_count(Task task, uint32_t count = 1) { - return m_wait_counts.at(task).fetch_add(count); + uint32_t res = 0; + m_concurrent_states->update( + task, [&res, count](Task, ConcurrentState& state, bool) { + res = state.wait_count; + state.wait_count += count; + }); + return res; } - void push_back_continuation(Task task, std::function f) { - m_concurrent_continuations->update( - task, [f = std::move(f)](Task, Continuations& continuations, bool) { - continuations.push_back(std::move(f)); - }); + uint32_t push_back_continuation(Task task, std::function f) { + uint32_t res = 0; + m_concurrent_states->update(task, + [f, &res](Task, ConcurrentState& state, bool) { + state.continuations.push_back(f); + res = state.wait_count; + }); + return res; } void decrement_wait_count(Task task) { - if (m_wait_counts.at(task).fetch_sub(1) != 1) { + bool task_ready = false; + std::vector> continuations; + m_concurrent_states->update( + task, + [&task_ready, &continuations](Task, ConcurrentState& state, bool) { + if (--state.wait_count == 0) { + task_ready = true; + continuations = std::move(state.continuations); + } + }); + if (!task_ready) { return; } - auto* continuations = m_concurrent_continuations->get_and_erase(task); - if (continuations) { - // Since the current wait-count is 0, there are not other threads that may - // read from or append to the continuations of this task. - always_assert(!continuations->empty()); + + if (!continuations.empty()) { auto priority = m_priorities->at(task); - auto wait_count = increment_wait_count(task, continuations->size()); - always_assert(wait_count == 0); - for (auto& f : *continuations) { + increment_wait_count(task, continuations.size()); + for (auto& f : continuations) { m_priority_thread_pool.post(priority, [this, task, f = std::move(f)] { f(); decrement_wait_count(task); @@ -78,7 +96,15 @@ class PriorityThreadPoolDAGScheduler { auto it = m_waiting_for.find(task); if (it != m_waiting_for.end()) { for (auto waiting_task : m_waiting_for.at(task)) { - if (m_wait_counts.at(waiting_task).fetch_sub(1) == 1) { + bool waiting_task_ready = false; + m_concurrent_states->update( + waiting_task, + [&waiting_task_ready](Task, ConcurrentState& state, bool) { + if (--state.wait_count == 0) { + waiting_task_ready = true; + } + }); + if (waiting_task_ready) { schedule(waiting_task); } } @@ -87,8 +113,7 @@ class PriorityThreadPoolDAGScheduler { void schedule(Task task) { auto priority = m_priorities->at(task); - auto wait_count = increment_wait_count(task); - always_assert(wait_count == 0); + increment_wait_count(task); m_priority_thread_pool.post(priority, [this, task] { m_executor(task); decrement_wait_count(task); @@ -107,10 +132,9 @@ class PriorityThreadPoolDAGScheduler { // The dependency must be scheduled before the task void add_dependency(Task task, Task dependency) { - always_assert(!m_concurrent_continuations); + always_assert(!m_concurrent_states); m_waiting_for[dependency].insert(task); - auto& wait_count = m_wait_counts.emplace(task, 0).first->second; - wait_count.fetch_add(1, std::memory_order_relaxed); + ++m_wait_counts[task]; } // While the given task is running, register another function that needs to @@ -119,12 +143,13 @@ class PriorityThreadPoolDAGScheduler { // associated with this task have finished running. void augment(Task task, std::function f, bool continuation = false) { if (continuation) { - push_back_continuation(task, std::move(f)); + auto active = push_back_continuation(task, std::move(f)); + always_assert(active); return; } + auto active = increment_wait_count(task); + always_assert(active); auto priority = m_priorities->at(task); - auto wait_count = increment_wait_count(task); - always_assert(wait_count > 0); m_priority_thread_pool.post(priority, [this, task, f = std::move(f)] { f(); decrement_wait_count(task); @@ -133,35 +158,40 @@ class PriorityThreadPoolDAGScheduler { template uint32_t run(const ForwardIt& begin, const ForwardIt& end) { - always_assert(!m_concurrent_continuations); + always_assert(!m_concurrent_states); m_priorities = std::make_unique>(); - std::vector> ready_tasks; for (auto it = begin; it != end; it++) { - int priority = compute_priority(*it); - auto& wait_count = m_wait_counts.emplace(*it, 0).first->second; - if (wait_count.load(std::memory_order_relaxed) != 0) { - continue; - } - always_assert(priority >= 0); - if ((uint32_t)priority >= ready_tasks.size()) { - ready_tasks.resize(priority * 2 + 1); - } - ready_tasks[priority].push_back(*it); + compute_priority(*it); + } + for (auto& p : *m_priorities) { + auto it = m_wait_counts.find(p.first); + p.second = + (p.second << 16) + (it == m_wait_counts.end() ? 0 : it->second); } - m_concurrent_continuations = - std::make_unique>(); - for (size_t i = ready_tasks.size(); i > 0; --i) { - for (auto task : ready_tasks[i - 1]) { + + m_concurrent_states = + std::make_unique>(); + for (auto& p : m_wait_counts) { + m_concurrent_states->emplace(p.first, (ConcurrentState){p.second}); + } + std::vector tasks(begin, end); + std::stable_sort(tasks.begin(), + tasks.end(), + [priorities = m_priorities.get()](Task a, Task b) { + return priorities->at(a) > priorities->at(b); + }); + for (auto task : tasks) { + if (!m_wait_counts.count(task)) { schedule(task); } } + m_wait_counts.clear(); m_priority_thread_pool.join(); - always_assert(m_concurrent_continuations->empty()); - m_concurrent_continuations = nullptr; - for (auto& p : m_wait_counts) { - always_assert(p.second.load(std::memory_order_relaxed) == 0); + for (auto& p : *m_concurrent_states) { + always_assert(p.second.wait_count == 0); + always_assert(p.second.continuations.empty()); } - m_wait_counts.clear(); + m_concurrent_states = nullptr; m_waiting_for.clear(); m_priorities = nullptr; auto max_priority = m_max_priority; diff --git a/libredex/Purity.cpp b/libredex/Purity.cpp index afbc4b64ef..3331a36a0a 100644 --- a/libredex/Purity.cpp +++ b/libredex/Purity.cpp @@ -18,7 +18,6 @@ #include "Resolver.h" #include "Show.h" #include "StlUtil.h" -#include "Timer.h" #include "Trace.h" #include "Walkers.h" #include "WorkQueue.h" @@ -400,22 +399,25 @@ bool process_base_and_overriding_methods( } // Okay, let's process all overridden methods just like the base method. - return method_override_graph::all_overriding_methods( - *method_override_graph, method, [&](const DexMethod* overriding_method) { - action = get_base_or_overriding_method_action( - overriding_method, methods_to_ignore, - ignore_methods_with_assumenosideeffects); - if (action == MethodOverrideAction::UNKNOWN || - (action == MethodOverrideAction::INCLUDE && - !handler_func(const_cast(overriding_method)))) { - return false; - } - return true; - }); + auto overriding_methods = method_override_graph::get_overriding_methods( + *method_override_graph, method); + for (auto overriding_method : overriding_methods) { + action = get_base_or_overriding_method_action( + overriding_method, methods_to_ignore, + ignore_methods_with_assumenosideeffects); + if (action == MethodOverrideAction::UNKNOWN || + (action == MethodOverrideAction::INCLUDE && + !handler_func(const_cast(overriding_method)))) { + return false; + } + } return true; } -static AccumulatingTimer s_wto_timer("compute_locations_closure_wto"); +static AccumulatingTimer s_wto_timer; +double get_compute_locations_closure_wto_seconds() { + return s_wto_timer.get_seconds(); +} static constexpr const DexMethod* WTO_ROOT = nullptr; static std::function&(const DexMethod*)> @@ -476,8 +478,8 @@ get_wto_successors( // In subsequent iteration, besides computing the sorted root successors // again, we also filter all previously sorted inverse_dependencies entries. - auto concurrent_cache = std::make_shared>>(); + auto concurrent_cache = std::make_shared< + ConcurrentMap>>(); workqueue_run( [&impacted_methods, &get_sorted_impacted_methods, &inverse_dependencies, concurrent_cache](const DexMethod* m) { @@ -495,8 +497,7 @@ get_wto_successors( } } } - auto [_, emplaced] = - concurrent_cache->emplace(m, std::move(successors)); + auto emplaced = concurrent_cache->emplace(m, std::move(successors)); always_assert(emplaced); }, wto_nodes); @@ -515,18 +516,21 @@ size_t compute_locations_closure( std::unordered_map* result) { // 1. Let's initialize known method read locations and dependencies by // scanning method bodies - InsertOnlyConcurrentMap - method_lads; + ConcurrentMap + concurrent_method_lads; { Timer t{"Initialize LADS"}; walk::parallel::methods(scope, [&](DexMethod* method) { auto lads = init_func(method); if (lads) { - method_lads.emplace(method, std::move(*lads)); + concurrent_method_lads.emplace(method, std::move(*lads)); } }); } + std::unordered_map method_lads = + concurrent_method_lads.move_to_container(); + // 2. Compute inverse dependencies so that we know what needs to be recomputed // during the fixpoint computation, and determine set of methods that are // initially "impacted" in the sense that they have dependencies. @@ -581,7 +585,7 @@ size_t compute_locations_closure( std::vector changed_methods; for (const DexMethod* method : ordered_impacted_methods) { - auto& lads = method_lads.at_unsafe(method); + auto& lads = method_lads.at(method); bool unknown = false; size_t lads_locations_size = lads.locations.size(); for (const DexMethod* d : lads.dependencies) { @@ -600,7 +604,7 @@ size_t compute_locations_closure( // something changed changed_methods.push_back(method); if (unknown) { - method_lads.erase_unsafe(method); + method_lads.erase(method); } } } @@ -615,8 +619,7 @@ size_t compute_locations_closure( // remove inverse dependency entries as appropriate auto& entries = it->second; - std20::erase_if(entries, - [&](auto* m) { return !method_lads.count_unsafe(m); }); + std20::erase_if(entries, [&](auto* m) { return !method_lads.count(m); }); if (entries.empty()) { // remove inverse dependency diff --git a/libredex/Purity.h b/libredex/Purity.h index 9f6d0442e7..02c0ae979b 100644 --- a/libredex/Purity.h +++ b/libredex/Purity.h @@ -142,6 +142,10 @@ bool process_base_and_overriding_methods( bool ignore_methods_with_assumenosideeffects, const std::function& handler_func); +// Accumulated time of all internal wto computations of +// compute_locations_closure. +double get_compute_locations_closure_wto_seconds(); + // Given initial locations and dependencies for each method, compute the closure // (union) of all such locations over all the stated dependencies, taking into // account all overriding methods. diff --git a/libredex/Reachability.cpp b/libredex/Reachability.cpp index 6d93285616..ca6a836d5e 100644 --- a/libredex/Reachability.cpp +++ b/libredex/Reachability.cpp @@ -797,20 +797,17 @@ MethodReferencesGatherer::get_returning_dependency( std::unordered_set unique_methods; auto identity = [](const auto* x) { return x; }; auto select_first = [](const auto& p) { return p.first; }; - auto is = [&](const auto& item, const auto& p) { - auto* m = p(item); - if (!unique_methods.insert(m).second) { - return false; - } - always_assert(m->is_virtual()); - if (is_abstract(m)) { - return false; - } - return f(m); - }; auto any_of = [&](const auto& collection, const auto& p) { for (auto&& item : collection) { - if (is(item, p)) { + auto* m = p(item); + if (!unique_methods.insert(m).second) { + continue; + } + always_assert(m->is_virtual()); + if (is_abstract(m)) { + continue; + } + if (f(m)) { return true; } } @@ -826,12 +823,10 @@ MethodReferencesGatherer::get_returning_dependency( refs->base_invoke_virtual_targets_if_class_instantiable) { for (auto* base_type : base_types) { always_assert(!type_class(base_type)->is_external()); - if (mog::any_overriding_methods( - *m_shared_state->method_override_graph, base_method, - [&](const DexMethod* overriding_method) { - return is(overriding_method, identity); - }, - /* include_interfaces*/ false, base_type)) { + auto overriding_methods = mog::get_overriding_methods( + *m_shared_state->method_override_graph, base_method, + /* include_interfaces*/ false, base_type); + if (any_of(overriding_methods, identity)) { return true; } } @@ -893,14 +888,20 @@ void MethodReferencesGatherer::default_gather_mie(const MethodItemEntry& mie, } } else if (gather_methods && (opcode::is_invoke_virtual(op) || opcode::is_invoke_interface(op))) { - auto resolved_callee = resolve_invoke_method(insn, m_method); + auto method_ref = insn->get_method(); + auto resolved_callee = resolve_method(method_ref, opcode_to_search(insn)); + if (resolved_callee == nullptr && opcode::is_invoke_virtual(op)) { + // There are some invoke-virtual call on methods whose def are + // actually in interface. + resolved_callee = + resolve_method(method_ref, MethodSearch::InterfaceVirtual); + } if (!resolved_callee) { // Typically clone() on an array, or other obscure external references TRACE(REACH, 2, "Unresolved virtual callee at %s", SHOW(insn)); refs->unknown_invoke_virtual_targets = true; return; } - auto method_ref = insn->get_method(); auto base_type = method_ref->get_class(); refs->base_invoke_virtual_targets_if_class_instantiable[resolved_callee] .insert(base_type); @@ -1876,10 +1877,8 @@ void ReachableAspects::finish(const ConditionallyMarked& cond_marked, add(map); } for (auto&& [method, mrefs_gatherer] : remaining_mrefs_gatherers) { - auto set = mrefs_gatherer->get_non_returning_insns(); - if (!set.empty()) { - non_returning_insns.emplace(method, std::move(set)); - } + non_returning_insns.emplace(method, + mrefs_gatherer->get_non_returning_insns()); } std::atomic concurrent_instructions_unvisited{0}; workqueue_run>( @@ -1913,8 +1912,7 @@ void ReachableAspects::finish(const ConditionallyMarked& cond_marked, } std::unique_ptr compute_reachable_objects( - const Scope& scope, - const method_override_graph::Graph& method_override_graph, + const DexStoresVector& stores, const IgnoreSets& ignore_sets, int* num_ignore_check_strings, ReachableAspects* reachable_aspects, @@ -1925,14 +1923,17 @@ std::unique_ptr compute_reachable_objects( bool cfg_gathering_check_instance_callable, bool cfg_gathering_check_returning, bool should_mark_all_as_seed, + std::unique_ptr* out_method_override_graph, bool remove_no_argument_constructors) { Timer t("Marking"); + auto scope = build_class_scope(stores); std::unordered_set scope_set(scope.begin(), scope.end()); auto reachable_objects = std::make_unique(); ConditionallyMarked cond_marked; + auto method_override_graph = mog::build_graph(scope); ConcurrentSet root_set; - RootSetMarker root_set_marker(method_override_graph, record_reachability, + RootSetMarker root_set_marker(*method_override_graph, record_reachability, relaxed_keep_class_members, remove_no_argument_constructors, &cond_marked, reachable_objects.get(), &root_set); @@ -1948,7 +1949,7 @@ std::unique_ptr compute_reachable_objects( TransitiveClosureMarkerSharedState shared_state{ std::move(scope_set), &ignore_sets, - &method_override_graph, + method_override_graph.get(), record_reachability, relaxed_keep_class_members, relaxed_keep_interfaces, @@ -1969,13 +1970,17 @@ std::unique_ptr compute_reachable_objects( }, root_set, num_threads, /*push_tasks_while_running=*/true); - compute_zombie_methods(method_override_graph, *reachable_objects, + compute_zombie_methods(*method_override_graph, *reachable_objects, *reachable_aspects); if (num_ignore_check_strings != nullptr) { *num_ignore_check_strings = (int)stats.num_ignore_check_strings; } + if (out_method_override_graph) { + *out_method_override_graph = std::move(method_override_graph); + } + reachable_aspects->finish(cond_marked, *reachable_objects); return reachable_objects; @@ -2181,14 +2186,11 @@ void reanimate_zombie_methods(const ReachableAspects& reachable_aspects) { } } -void sweep_code( +std::pair sweep_code( DexStoresVector& stores, bool prune_uncallable_instance_method_bodies, bool skip_uncallable_virtual_methods, - const ReachableAspects& reachable_aspects, - remove_uninstantiables_impl::Stats* remove_uninstantiables_stats, - std::atomic* throws_inserted, - InsertOnlyConcurrentSet* affected_methods) { + const ReachableAspects& reachable_aspects) { Timer t("Sweep Code"); auto scope = build_class_scope(stores); std::unordered_set uninstantiable_types; @@ -2212,7 +2214,8 @@ void sweep_code( } } uninstantiable_types.insert(type::java_lang_Void()); - *remove_uninstantiables_stats = walk::parallel::methods< + std::atomic throws_inserted{0}; + auto res = walk::parallel::methods< remove_uninstantiables_impl::Stats>(scope, [&](DexMethod* method) { auto code = method->get_code(); if (!code || method->rstate.no_optimizations()) { @@ -2223,7 +2226,7 @@ void sweep_code( auto non_returning_it = reachable_aspects.non_returning_insns.find(method); if (non_returning_it != reachable_aspects.non_returning_insns.end()) { auto& non_returning_insns = non_returning_it->second; - throw_propagation_impl::ThrowPropagator impl(cfg); + throw_propagation_impl::ThrowPropagator impl(cfg, /* debug */ false); for (auto block : cfg.blocks()) { auto ii = InstructionIterable(block); for (auto it = ii.begin(); it != ii.end(); it++) { @@ -2231,31 +2234,27 @@ void sweep_code( continue; } if (impl.try_apply(block->to_cfg_instruction_iterator(it))) { - (*throws_inserted)++; + throws_inserted++; } // Stop processing more instructions in this block break; } } cfg.remove_unreachable_blocks(); - affected_methods->insert(method); } if (uncallable_instance_methods.count(method)) { if (skip_uncallable_virtual_methods && method->is_virtual()) { return remove_uninstantiables_impl::Stats(); } - affected_methods->insert(method); return remove_uninstantiables_impl::replace_all_with_unreachable_throw( cfg); } auto stats = remove_uninstantiables_impl::replace_uninstantiable_refs( uninstantiable_types, cfg); - if (stats.sum()) { - cfg.remove_unreachable_blocks(); - affected_methods->insert(method); - } + cfg.remove_unreachable_blocks(); return stats; }); + return std::make_pair(res, (size_t)throws_inserted); } remove_uninstantiables_impl::Stats sweep_uncallable_virtual_methods( @@ -2290,7 +2289,7 @@ remove_uninstantiables_impl::Stats sweep_uncallable_virtual_methods( } }, reachable_aspects.directly_instantiable_types); - ConcurrentSet uncallable_instance_methods; + std::unordered_set uncallable_instance_methods; for (auto* cls : scope) { if (is_interface(cls)) { // TODO: Is this needed? @@ -2343,6 +2342,8 @@ void report(PassManager& pm, pm.incr_metric("zombie_implementation_methods", reachable_aspects.zombie_implementation_methods.size()); pm.incr_metric("zombie_methods", reachable_aspects.zombie_methods.size()); + pm.incr_metric("non_returning_dependencies", + reachable_aspects.non_returning_dependencies.size()); pm.incr_metric("returning_methods", reachable_aspects.returning_methods.size()); } @@ -2414,7 +2415,7 @@ void dump_graph(std::ostream& os, const ReachableObjectGraph& retainers_of) { if (!retainers_of.count(obj)) { return {}; } - const auto& preds = retainers_of.at_unsafe(obj); + const auto& preds = retainers_of.at(obj); std::vector preds_vec(preds.begin(), preds.end()); // Gotta sort the reachables or the output is nondeterministic. std::sort(preds_vec.begin(), preds_vec.end(), compare); diff --git a/libredex/Reachability.h b/libredex/Reachability.h index 741073359f..a71250c319 100644 --- a/libredex/Reachability.h +++ b/libredex/Reachability.h @@ -10,7 +10,7 @@ #include #include -#include +#include #include "ConcurrentContainers.h" #include "ControlFlow.h" @@ -691,8 +691,7 @@ class TransitiveClosureMarkerWorker { * (e.g. proguard rules). */ std::unique_ptr compute_reachable_objects( - const Scope& scope, - const method_override_graph::Graph& method_override_graph, + const DexStoresVector& stores, const IgnoreSets& ignore_sets, int* num_ignore_check_strings, ReachableAspects* reachable_aspects, @@ -703,6 +702,8 @@ std::unique_ptr compute_reachable_objects( bool cfg_gathering_check_instance_callable = false, bool cfg_gathering_check_returning = false, bool should_mark_all_as_seed = false, + std::unique_ptr* + out_method_override_graph = nullptr, bool remove_no_argument_constructors = false); void compute_zombie_methods( @@ -722,14 +723,11 @@ void sweep(DexStoresVector& stores, void reanimate_zombie_methods(const ReachableAspects& reachable_aspects); -void sweep_code( +std::pair sweep_code( DexStoresVector& stores, bool prune_uncallable_instance_method_bodies, bool skip_uncallable_virtual_methods, - const ReachableAspects& reachable_aspects, - remove_uninstantiables_impl::Stats* remove_uninstantiables_stats, - std::atomic* throws_inserted, - InsertOnlyConcurrentSet* affected_methods); + const ReachableAspects& reachable_aspects); remove_uninstantiables_impl::Stats sweep_uncallable_virtual_methods( DexStoresVector& stores, const ReachableAspects& reachable_aspects); diff --git a/libredex/ReachableClasses.cpp b/libredex/ReachableClasses.cpp index 95f0763da2..b51fe35849 100644 --- a/libredex/ReachableClasses.cpp +++ b/libredex/ReachableClasses.cpp @@ -180,8 +180,7 @@ void analyze_reflection(const Scope& scope) { return DexString::get_string(""); } int arg_str_idx = refl_type == ReflectionType::REF_UPDATER ? 2 : 1; - const auto& arg_str = - analysis.get_abstract_object(insn->src(arg_str_idx), insn); + auto arg_str = analysis.get_abstract_object(insn->src(arg_str_idx), insn); if (arg_str && arg_str->obj_kind == AbstractObjectKind::STRING) { return arg_str->dex_string; } else { @@ -228,7 +227,7 @@ void analyze_reflection(const Scope& scope) { /* metadata_cache */ &refl_metadata_cache); } - const auto& arg_cls = analysis->get_abstract_object(insn->src(0), insn); + auto arg_cls = analysis->get_abstract_object(insn->src(0), insn); if (!arg_cls || arg_cls->obj_kind != AbstractObjectKind::CLASS) { continue; } diff --git a/libredex/RedexContext.cpp b/libredex/RedexContext.cpp index 7da37f8ad8..966e91cf81 100644 --- a/libredex/RedexContext.cpp +++ b/libredex/RedexContext.cpp @@ -37,13 +37,7 @@ RedexContext::RedexContext(bool allow_class_duplicates) s_medium_string_storage{65536, 2000, boost::thread::hardware_concurrency() / 4}, s_large_string_storage{0, 0, boost::thread::hardware_concurrency()}, - m_allow_class_duplicates(allow_class_duplicates) { - for (size_t i = 0; i < s_small_string_set.size(); ++i) { - s_small_string_set[i] = - new InsertOnlyConcurrentSet(); - } -} + m_allow_class_duplicates(allow_class_duplicates) {} RedexContext::~RedexContext() { // We parallelize destruction for efficiency. @@ -59,9 +53,6 @@ RedexContext::~RedexContext() { } }; - size_t small_strings_size = 0; - size_t large_strings_size = 0; - parallel_run({[&] { Timer timer("Delete DexTypes", /* indent */ false); // NB: This table intentionally contains aliases (multiple @@ -76,14 +67,14 @@ RedexContext::~RedexContext() { s_type_map.clear(); }, [&] { - Timer timer("Delete DexTypeLists", /* indent */ false); + Timer timer("DexTypeLists", /* indent */ false); for (auto const& p : s_typelist_map) { delete p.second; } s_typelist_map.clear(); }, [&] { - Timer timer("Delete DexProtos", /* indent */ false); + Timer timer("Delete DexProtos.", /* indent */ false); for (auto* proto : s_proto_set) { delete proto; } @@ -139,8 +130,8 @@ RedexContext::~RedexContext() { fns.push_back([bucket, this]() { // Delete DexMethods. Use set to prevent double freeing aliases std::unordered_set delete_methods; - for (auto&& [_, loc] : s_method_map) { - auto method = static_cast(loc.load()); + for (auto const& it : s_method_map) { + auto method = static_cast(it.second); if ((reinterpret_cast(method) >> 16) % method_buckets_count == bucket && @@ -163,8 +154,8 @@ RedexContext::~RedexContext() { fns.push_back([bucket, this]() { // Delete DexFields. Use set to prevent double freeing aliases std::unordered_set delete_fields; - for (auto&& [_, loc] : s_field_map) { - auto field = static_cast(loc.load()); + for (auto const& it : s_field_map) { + auto field = static_cast(it.second); if ((reinterpret_cast(field) >> 16) % field_buckets_count == bucket && @@ -180,20 +171,17 @@ RedexContext::~RedexContext() { parallel_run( [&]() { + size_t segment_index{0}; std::vector> fns; - fns.reserve(s_small_string_set.size() + s_large_string_set.slots()); - for (size_t i = 0; i < s_small_string_set.size(); ++i) { - auto* small_string_set = s_small_string_set[i]; - small_strings_size += small_string_set->size(); - fns.push_back([small_string_set]() { - small_string_set->clear(); - delete small_string_set; + fns.reserve(s_string_set.slots()); + for (auto& segment : s_string_set) { + fns.push_back([&segment, index = segment_index++]() { + for (auto* v : segment) { + delete v; + } + segment.clear(); }); } - for (auto& segment : s_large_string_set) { - large_strings_size += segment.size(); - fns.push_back([&segment]() { segment.release(); }); - } return fns; }(), "Delete DexStrings"); @@ -213,9 +201,7 @@ RedexContext::~RedexContext() { log_stats("small", s_small_string_storage); log_stats("medium", s_medium_string_storage); log_stats("large", s_large_string_storage); - TRACE(PM, 1, - "String storage of %zu + %zu strings @ %u hardware concurrency:%s", - small_strings_size, large_strings_size, + TRACE(PM, 1, "String storage @ %u hardware concurrency:%s", boost::thread::hardware_concurrency(), oss.str().c_str()); } @@ -251,11 +237,10 @@ template value, Container* container) { - auto [ptr, success] = container->emplace(key, value.get()); - if (success) { + if (container->emplace(key, value.get())) { return value.release(); } - return ptr->load(); + return container->at(key); } RedexContext::ConcurrentStringStorage::Container::~Container() { @@ -365,7 +350,20 @@ RedexContext::ConcurrentStringStorage::Context::~Context() { } } -char* RedexContext::store_string(std::string_view str) { +const DexString* RedexContext::make_string(std::string_view str) { + // We are creating a DexString key that is just "defined enough" to be used as + // a key into our string set. The provided string does not have to be zero + // terminated, and we won't compute the utf size, as neither is needed for + // this purpose. + uint32_t dummy_utfsize{0}; + const DexString key(str.data(), str.size(), dummy_utfsize); + auto& segment = s_string_set.at(&key); + + auto rv_ptr = segment.get(&key); + if (rv_ptr != nullptr) { + return *rv_ptr; + } + ConcurrentStringStorage& concurrent_string_storage = str.length() < s_small_string_storage.max_allocation ? s_small_string_storage @@ -383,37 +381,7 @@ char* RedexContext::store_string(std::string_view str) { } memcpy(storage, str.data(), str.length()); storage[str.length()] = 0; - return storage; -} -const DexString* RedexContext::make_string(std::string_view str) { - // We are creating a DexString key that is just "defined enough" to be used as - // a key into our string set. The provided string does not have to be zero - // terminated, and we won't compute the utf size, as neither is needed for - // this purpose. - uint32_t dummy_utfsize{0}; - DexStringRepr repr{str.data(), (uint32_t)str.size(), dummy_utfsize}; - if (str.size() < s_small_string_set.size()) { - auto* rv_ptr = s_small_string_set[str.size()]->get(repr); - if (rv_ptr != nullptr) { - return reinterpret_cast(rv_ptr); - } - char* storage = store_string(str); - uint32_t utfsize = length_of_utf8_string(storage); - return reinterpret_cast( - s_small_string_set[str.size()] - ->insert(DexStringRepr{storage, (uint32_t)str.length(), utfsize}) - .first); - // If unsuccessful, we have wasted a bit of string storage. Oh well... - } - - auto* key = reinterpret_cast(&repr); - auto& segment = s_large_string_set.at(key); - auto rv_ptr = segment.get(key); - if (rv_ptr != nullptr) { - return *rv_ptr; - } - char* storage = store_string(str); uint32_t utfsize = length_of_utf8_string(storage); std::unique_ptr string( new DexString(storage, str.length(), utfsize)); @@ -443,36 +411,17 @@ size_t RedexContext::TruncatedStringHash::operator()(StringSetKey k) { return boost::hash_range(s + start, s + len); } -size_t RedexContext::DexStringReprHash::operator()( - const DexStringRepr& k) const { - return boost::hash_range(k.storage, k.storage + k.length); -} - -bool RedexContext::DexStringReprEqual::operator()( - const DexStringRepr& a, const DexStringRepr& b) const { - if (a.length != b.length) { - return false; - } - return memcmp(a.storage, b.storage, a.length) == 0; -} - const DexString* RedexContext::get_string(std::string_view str) { uint32_t dummy_utfsize{0}; - DexStringRepr repr{str.data(), (uint32_t)str.size(), dummy_utfsize}; - if (str.size() < s_small_string_set.size()) { - return reinterpret_cast( - s_small_string_set[str.size()]->get(repr)); - } - - auto* key = reinterpret_cast(&repr); - const auto& segment = s_large_string_set.at(key); - auto rv_ptr = segment.get(key); + const DexString key(str.data(), str.size(), dummy_utfsize); + const auto& segment = s_string_set.at(&key); + auto rv_ptr = segment.get(&key); return rv_ptr == nullptr ? nullptr : *rv_ptr; } DexType* RedexContext::make_type(const DexString* dstring) { always_assert(dstring != nullptr); - auto rv = s_type_map.load(dstring, nullptr); + auto rv = s_type_map.get(dstring, nullptr); if (rv != nullptr) { return rv; } @@ -484,7 +433,7 @@ DexType* RedexContext::get_type(const DexString* dstring) { if (dstring == nullptr) { return nullptr; } - return s_type_map.load(dstring, nullptr); + return s_type_map.get(dstring, nullptr); } void RedexContext::set_type_name(DexType* type, const DexString* new_name) { @@ -510,7 +459,7 @@ DexFieldRef* RedexContext::make_field(const DexType* container, always_assert(container != nullptr && name != nullptr && type != nullptr); DexFieldSpec r(const_cast(container), name, const_cast(type)); - auto rv = s_field_map.load(r, nullptr); + auto rv = s_field_map.get(r, nullptr); if (rv != nullptr) { return rv; } @@ -527,7 +476,7 @@ DexFieldRef* RedexContext::get_field(const DexType* container, } DexFieldSpec r(const_cast(container), name, const_cast(type)); - return s_field_map.load(r, nullptr); + return s_field_map.get(r, nullptr); } void RedexContext::alias_field_name(DexFieldRef* field, @@ -563,24 +512,24 @@ void RedexContext::mutate_field(DexFieldRef* field, r.type = ref.type != nullptr ? ref.type : field->m_spec.type; field->m_spec = r; - if (rename_on_collision && s_field_map.count(r)) { + if (rename_on_collision && s_field_map.find(r) != s_field_map.end()) { uint32_t i = 0; while (true) { - r.name = DexString::make_string("f$" + std::to_string(i++)); - if (!s_field_map.count(r)) { + r.name = DexString::make_string(("f$" + std::to_string(i++)).c_str()); + if (s_field_map.find(r) == s_field_map.end()) { break; } } } - always_assert_log(!s_field_map.count(r), + always_assert_log(s_field_map.find(r) == s_field_map.end(), "Another field with the same signature already exists %s", - SHOW(s_field_map.load(r))); + SHOW(s_field_map.at(r))); s_field_map.emplace(r, field); } DexTypeList* RedexContext::make_type_list( RedexContext::DexTypeListContainerType&& p) { - auto rv = s_typelist_map.load(&p, nullptr); + auto rv = s_typelist_map.get(&p, nullptr); if (rv != nullptr) { return rv; } @@ -591,7 +540,7 @@ DexTypeList* RedexContext::make_type_list( DexTypeList* RedexContext::get_type_list( const RedexContext::DexTypeListContainerType& p) { - return s_typelist_map.load(&p, nullptr); + return s_typelist_map.get(&p, nullptr); } size_t RedexContext::DexProtoKeyHash::operator()(DexProto* k) const { @@ -639,7 +588,7 @@ DexMethodRef* RedexContext::make_method(const DexType* type_, auto proto = const_cast(proto_); always_assert(type != nullptr && name != nullptr && proto != nullptr); DexMethodSpec r(type, name, proto); - auto rv = s_method_map.load(r, nullptr); + auto rv = s_method_map.get(r, nullptr); if (rv != nullptr) { return rv; } @@ -657,7 +606,7 @@ DexMethodRef* RedexContext::get_method(const DexType* type, } DexMethodSpec r(const_cast(type), name, const_cast(proto)); - return s_method_map.load(r, nullptr); + return s_method_map.get(r, nullptr); } void RedexContext::alias_method_name(DexMethodRef* method, @@ -731,7 +680,7 @@ void RedexContext::mutate_method(DexMethodRef* method, prefix = r.name->str() + "$"; } do { - r.name = DexString::make_string(prefix + std::to_string(i++)); + r.name = DexString::make_string((prefix + std::to_string(i++)).c_str()); } while (s_method_map.count(r)); } else { // We are about to change its class. Use a better name to remember its @@ -778,7 +727,7 @@ void RedexContext::mutate_method(DexMethodRef* method, DexLocation* RedexContext::make_location(std::string_view store_name, std::string_view file_name) { auto key = std::make_pair(store_name, file_name); - auto rv = s_location_map.load(key, nullptr); + auto rv = s_location_map.get(key, nullptr); if (rv != nullptr) { return rv; } @@ -794,7 +743,7 @@ DexLocation* RedexContext::make_location(std::string_view store_name, DexLocation* RedexContext::get_location(std::string_view store_name, std::string_view file_name) { auto key = std::make_pair(store_name, file_name); - return s_location_map.load(key, nullptr); + return s_location_map.get(key, nullptr); } PositionPatternSwitchManager* @@ -906,22 +855,3 @@ void RedexContext::set_sb_interaction_index( const std::unordered_map& input) { m_sb_interaction_indices = input; } - -void RedexContext::compact() { - // We parallelize destruction for efficiency. - auto parallel_run = [](const std::vector>& fns) { - workqueue_run>( - [](const std::function& fn) { fn(); }, fns); - }; - - parallel_run({ - [&] { s_type_map.compact(); }, - [&] { s_field_map.compact(); }, - [&] { s_typelist_map.compact(); }, - [&] { s_proto_set.compact(); }, - [&] { s_method_map.compact(); }, - [&] { s_location_map.compact(); }, - [&] { field_values.compact(); }, - [&] { method_return_values.compact(); }, - }); -} diff --git a/libredex/RedexContext.h b/libredex/RedexContext.h index 35e1a26a45..177fa3f400 100644 --- a/libredex/RedexContext.h +++ b/libredex/RedexContext.h @@ -39,7 +39,6 @@ class DexMethodHandle; class DexMethodRef; class DexProto; class DexString; -struct DexStringRepr; class DexType; class DexTypeList; class PositionPatternSwitchManager; @@ -220,10 +219,6 @@ struct RedexContext { ConcurrentSet blanket_native_root_classes; ConcurrentSet blanket_native_root_methods; - // Release memory used by erased items and old ConcurrentHashtable storage - // versions. - void compact(); - private: struct Strcmp; struct TruncatedStringHash; @@ -333,44 +328,11 @@ struct RedexContext { }; template - class ConcurrentProjectedStringSet { - std::array, n_slots> m_slots; - mutable std::array m_locks; - - public: - const StringSetKey* get(StringSetKey str) const { - size_t i = StringSetKeyHash()(str) % n_slots; - auto& map = m_slots[i]; - std::lock_guard lock(m_locks[i]); - auto it = map.find(str); - return it == map.end() ? nullptr : &*it; - } - - std::pair insert(StringSetKey str) { - size_t i = StringSetKeyHash()(str) % n_slots; - auto& map = m_slots[i]; - std::lock_guard lock(m_locks[i]); - auto [it, emplaced] = map.emplace(str); - return std::make_pair(&*it, emplaced); - } - - size_t size() const { - size_t res = 0; - for (auto& set : m_slots) { - res += set.size(); - } - return res; - } - - void release() { - for (auto& set : m_slots) { - for (auto* s : set) { - delete s; - } - set.clear(); - } - } - }; + using ConcurrentProjectedStringSet = InsertOnlyConcurrentSetContainer< + std::set, + StringSetKey, + StringSetKeyHash, + n_slots>; template struct LargeStringSet { @@ -402,33 +364,19 @@ struct RedexContext { size_t operator()(StringSetKey k); }; - struct DexStringReprHash { - size_t operator()(const DexStringRepr& k) const; - }; - struct DexStringReprEqual { - bool operator()(const DexStringRepr& a, const DexStringRepr& b) const; - }; - // DexString - LargeStringSet<31, 127> s_large_string_set; - std::array*, - 128> - s_small_string_set; + LargeStringSet<31, 127> s_string_set; // We maintain three kinds of raw string storage ConcurrentStringStorage s_small_string_storage; ConcurrentStringStorage s_medium_string_storage; ConcurrentStringStorage s_large_string_storage; - char* store_string(std::string_view); - // DexType - AtomicMap s_type_map; + ConcurrentMap s_type_map; // DexFieldRef - AtomicMap s_field_map; + ConcurrentMap s_field_map; std::mutex s_field_lock; // DexTypeList @@ -443,10 +391,10 @@ struct RedexContext { return lhs == rhs || *lhs == *rhs; } }; - AtomicMap + ConcurrentMap s_typelist_map; // DexProto @@ -460,7 +408,7 @@ struct RedexContext { s_proto_set; // DexMethod - AtomicMap s_method_map; + ConcurrentMap s_method_map; std::mutex s_method_lock; // DexLocation @@ -470,7 +418,7 @@ struct RedexContext { return std::hash()(k.second); } }; - AtomicMap + ConcurrentMap s_location_map; // DexPositionSwitch and DexPositionPattern diff --git a/libredex/RedexProperties.cpp b/libredex/RedexProperties.cpp deleted file mode 100644 index 0bf5e6435a..0000000000 --- a/libredex/RedexProperties.cpp +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * - * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. - */ - -#include "RedexProperties.h" - -#include - -namespace redex_properties { - -bool is_negative(Property property) { - switch (property) { -#define REDEX_PROPS(name, neg, _init, _final, _def_pres) \ - case Property::name: \ - return neg; -#include "RedexProperties.def" -#undef REDEX_PROPS - } - return false; -} - -bool is_default_preserving(Property property) { - switch (property) { -#define REDEX_PROPS(name, _neg, _init, _final, def_pres) \ - case Property::name: \ - return def_pres; -#include "RedexProperties.def" -#undef REDEX_PROPS - } - return false; -} - -std::vector get_all_properties() { - return { -#define REDEX_PROPS(name, _neg, _init, _final, _def_pres) Property::name, -#include "RedexProperties.def" -#undef REDEX_PROPS - }; -} - -const char* get_name(Property property) { - switch (property) { -#define REDEX_PROPS(name, _neg, _init, _final, _def_pres) \ - case Property::name: \ - return #name; -#include "RedexProperties.def" -#undef REDEX_PROPS - } - return ""; -} - -std::ostream& operator<<(std::ostream& os, const Property& property) { - os << get_name(property); - return os; -} - -} // namespace redex_properties diff --git a/libredex/RedexProperties.def b/libredex/RedexProperties.def deleted file mode 100644 index eed208c32f..0000000000 --- a/libredex/RedexProperties.def +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * - * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. - */ - - -#ifndef REDEX_PROPS -#error "You must define REDEX_PROPS!" -#endif - -/** - * Negative: This property is by default preserved, and need to be explicitly destroyed - * before final state. - * DefaultInitial: This property will be established at beginning of all Redex passes - * (unless disabled explicitly) by default. - * DefaultFinal: This property will be required at end of all Redex passes (unless - * disabled explicitly) by default. - * DefaultPreserve: This property will be preserved by default after establised. - */ -// REDEX_PROPS(Name, Negative, DefaultInitial, DefaultFinal, DefaultPreserve) -REDEX_PROPS(DexLimitsObeyed, false, false, true, false) -REDEX_PROPS(HasSourceBlocks, false, false, false, true) -REDEX_PROPS(InitialRenameClass, false, false, false, false) -REDEX_PROPS(NeedsEverythingPublic, true, false, false, false) -REDEX_PROPS(NeedsInjectionIdLowering, true, false, false, false) -REDEX_PROPS(NoInitClassInstructions, false, true, true, false) -REDEX_PROPS(NoResolvablePureRefs, false, false, false, false) -REDEX_PROPS(NoSpuriousGetClassCalls, false, false, false, false) -REDEX_PROPS(NoUnreachableInstructions, false, true, true, false) -REDEX_PROPS(RenameClass, false, false, false, false) -REDEX_PROPS(MethodRegister, false, false, true, true) -// This is different from above because it is only a marker that signals interning happened, -// but the property is not checked. New spurious getClass calls may be produced, but were -// cannot have been input-code null-checks. -REDEX_PROPS(SpuriousGetClassCallsInterned, true, false, false, false) -REDEX_PROPS(UltralightCodePatterns, false, true, false, false) diff --git a/libredex/RedexProperties.h b/libredex/RedexProperties.h index 548011b89a..4ed2d918ea 100644 --- a/libredex/RedexProperties.h +++ b/libredex/RedexProperties.h @@ -7,10 +7,8 @@ #pragma once -#include #include #include -#include namespace redex_properties { @@ -40,57 +38,38 @@ struct PropertyInteraction { }; namespace interactions { -// Not specified property will have Destroys interaction for -// passes by default unless specified with Negative or DefaultPreserve inline const PropertyInteraction Destroys = // default PropertyInteraction(false, false, false, false); -// Preserve established property for passes. -// DefaultPreserve will preserve the property by default. inline const PropertyInteraction Preserves = PropertyInteraction(false, false, true, false); -// Requires property for passes will be checked if they have -// already been established. inline const PropertyInteraction Requires = PropertyInteraction(false, true, false, false); -// Establishes a property for passes. DefaultInitial property -// will be established at beginning by default. -// In deep check mode, after each pass eastablished property -// will be running their own checks. inline const PropertyInteraction Establishes = PropertyInteraction(true, false, false, false); inline const PropertyInteraction RequiresAndEstablishes = PropertyInteraction(true, true, true, false); -inline const PropertyInteraction RequiresAndPreserves = - PropertyInteraction(false, true, true, false); -// Establish a property and add it to final require list with other -// default finals. inline const PropertyInteraction EstablishesAndRequiresFinally = PropertyInteraction(true, false, false, true); } // namespace interactions -enum class Property { -#define REDEX_PROPS(name, _neg, _init, _final, _def_pres) name, -#include "RedexProperties.def" -#undef REDEX_PROPS -}; - -bool is_negative(Property property); -bool is_default_preserving(Property property); -std::vector get_all_properties(); - -const char* get_name(Property property); -std::ostream& operator<<(std::ostream& os, const Property& property); - -using PropertyInteractions = std::unordered_map; - -// Legacy naming scheme. May update references at some point. +using PropertyName = std::string; +using PropertyInteractions = + std::unordered_map; namespace names { -#define REDEX_PROPS(name, _neg, _init, _final, _def_pres) \ - constexpr Property name = Property::name; -#include "RedexProperties.def" -#undef REDEX_PROPS +inline const PropertyName NoInitClassInstructions("NoInitClassInstructions"); +inline const PropertyName NoUnreachableInstructions( + "NoUnreachableInstructions"); +inline const PropertyName DexLimitsObeyed("DexLimitsObeyed"); +// Stand-in for fixing up passes. +inline const PropertyName NeedsEverythingPublic("NeedsEverythingPublic"); +inline const PropertyName NeedsInjectionIdLowering("NeedsInjectionIdLowering"); +inline const PropertyName HasSourceBlocks("HasSourceBlocks"); +inline const PropertyName NoResolvablePureRefs("NoResolvablePureRefs"); +inline const PropertyName NoSpuriousGetClassCalls("NoSpuriousGetClassCalls"); +inline const PropertyName RenameClass("RenameClass"); +inline const PropertyName UltralightCodePatterns("UltralightCodePatterns"); } // namespace names @@ -100,11 +79,18 @@ namespace simple { // explicit. inline PropertyInteractions preserves_all() { using namespace redex_properties::interactions; + using namespace redex_properties::names; return { -#define REDEX_PROPS(name, _neg, _init, _final, _def_pres) \ - {Property::name, Preserves}, -#include "RedexProperties.def" -#undef REDEX_PROPS + {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, + {NeedsEverythingPublic, Preserves}, + {NeedsInjectionIdLowering, Preserves}, + {NoInitClassInstructions, Preserves}, + {NoUnreachableInstructions, Preserves}, + {NoResolvablePureRefs, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, + {RenameClass, Preserves}, + {UltralightCodePatterns, Preserves}, }; } diff --git a/libredex/RedexPropertiesManager.cpp b/libredex/RedexPropertiesManager.cpp index b2be7b6663..8e5a21f57c 100644 --- a/libredex/RedexPropertiesManager.cpp +++ b/libredex/RedexPropertiesManager.cpp @@ -41,48 +41,55 @@ CollectionT filter_out_disabled_properties(const CollectionT& c, } // namespace -const std::unordered_set& Manager::get_default_initial() { - static const auto default_initial_properties = []() { - std::unordered_set set; -#define REDEX_PROPS(name, _neg, is_init, _final, _def_pres) \ - if (is_init) { \ - set.insert(Property::name); \ - } -#include "RedexProperties.def" -#undef REDEX_PROPS - return set; - }(); +const std::unordered_set& Manager::get_default_initial() { + using namespace names; + static const std::unordered_set default_initial_properties{ + UltralightCodePatterns}; return default_initial_properties; } -std::unordered_set Manager::get_initial() const { +std::unordered_set Manager::get_initial() const { return filter_out_disabled_properties(get_default_initial(), *this); } -const std::unordered_set& Manager::get_default_final() { - static const auto default_final_properties = []() { - std::unordered_set set; -#define REDEX_PROPS(name, _neg, _init, is_final, _def_pres) \ - if (is_final) { \ - set.insert(Property::name); \ - } -#include "RedexProperties.def" -#undef REDEX_PROPS - return set; - }(); +const std::unordered_set& Manager::get_default_final() { + using namespace names; + static const std::unordered_set default_final_properties{ + NoInitClassInstructions, NoUnreachableInstructions, DexLimitsObeyed}; return default_final_properties; } -std::unordered_set Manager::get_final() const { +std::unordered_set Manager::get_final() const { return filter_out_disabled_properties(get_default_final(), *this); } -std::unordered_set Manager::get_required( +// TODO: Figure out a way to keep this complete. +// +// We could move to an enum with the typical .def file to autogenerate. +std::vector Manager::get_all_properties() { + using namespace names; + return { + NoInitClassInstructions, NoUnreachableInstructions, + DexLimitsObeyed, NeedsEverythingPublic, + NeedsInjectionIdLowering, HasSourceBlocks, + NoSpuriousGetClassCalls, RenameClass, + UltralightCodePatterns, + }; +} + +// TODO: This should really be with the RedexProperties definitions. +bool Manager::is_negative(const PropertyName& property) { + using namespace names; + return property == NeedsEverythingPublic || + property == NeedsInjectionIdLowering; +} + +std::unordered_set Manager::get_required( const PropertyInteractions& interactions) const { - std::unordered_set res; - for (const auto& [property, interaction] : interactions) { + std::unordered_set res; + for (const auto& [property_name, interaction] : interactions) { if (interaction.requires_) { - res.insert(property); + res.insert(property_name); } } return res; @@ -90,31 +97,31 @@ std::unordered_set Manager::get_required( void Manager::check(DexStoresVector& stores, PassManager& mgr) { for (auto* checker : m_checkers) { - TRACE(PM, 3, "Checking for %s...", get_name(checker->get_property())); + TRACE(PM, 3, "Checking for %s...", checker->get_property_name().c_str()); checker->run_checker(stores, m_conf, mgr, - m_established.count(checker->get_property())); + m_established.count(checker->get_property_name())); } } -const std::unordered_set& Manager::apply( +const std::unordered_set& Manager::apply( const PropertyInteractions& interactions) { - std20::erase_if(m_established, [&](const auto& property) { - auto it = interactions.find(property); + std20::erase_if(m_established, [&](const auto& property_name) { + auto it = interactions.find(property_name); if (it == interactions.end()) { - return !is_negative(property) && !is_default_preserving(property); + return !is_negative(property_name); } const auto& interaction = it->second; return !interaction.preserves; }); - for (const auto& [property, interaction] : interactions) { + for (const auto& [property_name, interaction] : interactions) { if (interaction.establishes) { - m_established.insert(property); + m_established.insert(property_name); } } return m_established; } -const std::unordered_set& Manager::apply_and_check( +const std::unordered_set& Manager::apply_and_check( const PropertyInteractions& interactions, DexStoresVector& stores, PassManager& mgr) { @@ -135,18 +142,18 @@ std::optional Manager::verify_pass_interactions( auto log_established_properties = [&](const std::string& title) { oss << " " << title << ": "; - for (auto& property : m.m_established) { - oss << property << ", "; + for (auto& property_name : m.m_established) { + oss << property_name << ", "; } oss << "\n"; }; log_established_properties("initial state establishes"); auto check = [&](const auto& properties) { - for (auto& property : properties) { - if (!m.m_established.count(property)) { + for (auto& property_name : properties) { + if (!m.m_established.count(property_name)) { oss << " *** REQUIRED PROPERTY NOT CURRENTLY ESTABLISHED ***: " - << property << "\n"; + << property_name << "\n"; failed = true; } } @@ -161,22 +168,22 @@ std::optional Manager::verify_pass_interactions( oss << pass_name << "\n"; m.apply(interactions); log_established_properties("establishes"); - for (const auto& [property, interaction] : interactions) { + for (const auto& [property_name, interaction] : interactions) { if (interaction.requires_finally) { - final_properties.insert(property); + final_properties.insert(property_name); } } } log_established_properties("final state requires"); check(final_properties); - for (auto& property : get_all_properties()) { - if (!is_negative(property)) { + for (auto& property_name : get_all_properties()) { + if (!is_negative(property_name)) { continue; } - if (m.m_established.count(property)) { + if (m.m_established.count(property_name)) { oss << " *** MUST-NOT PROPERTY IS ESTABLISHED IN FINAL STATE ***: " - << property << "\n"; + << property_name << "\n"; failed = true; } } @@ -212,18 +219,18 @@ static bool pass_is_enabled(const std::string& pass_name, return true; } -bool Manager::property_is_enabled(const Property& property) const { +bool Manager::property_is_enabled(const PropertyName& property_name) const { // If we had c++20, we could use a constexpr sorted std::array with // std::lower_bound for fast lookups... - static const std::unordered_map + static const std::unordered_map enable_check_funcs{{ - Property::HasSourceBlocks, + names::HasSourceBlocks, [](const ConfigFiles& conf) { return pass_is_enabled("InsertSourceBlocksPass", conf); }, }}; - auto it = enable_check_funcs.find(property); + auto it = enable_check_funcs.find(property_name); if (it == enable_check_funcs.end()) { return true; } diff --git a/libredex/RedexPropertiesManager.h b/libredex/RedexPropertiesManager.h index 3f9d595de2..c81bbddc37 100644 --- a/libredex/RedexPropertiesManager.h +++ b/libredex/RedexPropertiesManager.h @@ -30,22 +30,22 @@ class Manager { Manager(ConfigFiles& conf, std::vector checkers); - bool property_is_enabled(const Property& property) const; + bool property_is_enabled(const PropertyName& property_name) const; - std::unordered_set get_initial() const; - static const std::unordered_set& get_default_initial(); + std::unordered_set get_initial() const; + static const std::unordered_set& get_default_initial(); - std::unordered_set get_final() const; - static const std::unordered_set& get_default_final(); + std::unordered_set get_final() const; + static const std::unordered_set& get_default_final(); - std::unordered_set get_required( + std::unordered_set get_required( const PropertyInteractions& interactions) const; void check(DexStoresVector& stores, PassManager& mgr); - const std::unordered_set& apply( + const std::unordered_set& apply( const PropertyInteractions& interactions); - const std::unordered_set& apply_and_check( + const std::unordered_set& apply_and_check( const PropertyInteractions& interactions, DexStoresVector& stores, PassManager& mgr); @@ -55,15 +55,19 @@ class Manager { pass_interactions, ConfigFiles& conf); - const std::unordered_set& get_established() const { + const std::unordered_set& get_established() const { return m_established; } + static std::vector get_all_properties(); + + static bool is_negative(const PropertyName& property); + private: // TODO: See whether we can make checkers use const. ConfigFiles& m_conf; - std::unordered_set m_established; + std::unordered_set m_established; std::vector m_checkers; }; diff --git a/libredex/RedexPropertyChecker.cpp b/libredex/RedexPropertyChecker.cpp index a046ea2088..b4db7bead3 100644 --- a/libredex/RedexPropertyChecker.cpp +++ b/libredex/RedexPropertyChecker.cpp @@ -10,7 +10,8 @@ namespace redex_properties { -PropertyChecker::PropertyChecker(Property property) : m_property(property) { +PropertyChecker::PropertyChecker(std::string property_name) + : m_property_name(std::move(property_name)) { PropertyCheckerRegistry::get().register_checker(this); } diff --git a/libredex/RedexPropertyChecker.h b/libredex/RedexPropertyChecker.h index 5db219de45..0f70b487c7 100644 --- a/libredex/RedexPropertyChecker.h +++ b/libredex/RedexPropertyChecker.h @@ -17,12 +17,12 @@ class PassManager; namespace redex_properties { class PropertyChecker { - const Property m_property; + PropertyName m_property_name; public: - explicit PropertyChecker(Property property); + explicit PropertyChecker(PropertyName property_name); virtual ~PropertyChecker(); - const Property& get_property() const { return m_property; } + const PropertyName& get_property_name() { return m_property_name; } virtual void run_checker(DexStoresVector& stores, ConfigFiles& conf, PassManager& mgr, diff --git a/libredex/RedexResources.cpp b/libredex/RedexResources.cpp index dc60ce7249..85bf36a8e5 100644 --- a/libredex/RedexResources.cpp +++ b/libredex/RedexResources.cpp @@ -449,10 +449,6 @@ bool AndroidResources::can_obfuscate_xml_file( return false; } -void AndroidResources::finalize_bundle_config(const ResourceConfig& config) { - // Do nothing in super implementation, sub class will override if relevant. -} - void ResourceTableFile::finalize_resource_table(const ResourceConfig& config) { // Intentionally left empty, proto resource table will not contain a relevant // structure to clean up. diff --git a/libredex/RedexResources.h b/libredex/RedexResources.h index 80a1cedfe5..bad1990167 100644 --- a/libredex/RedexResources.h +++ b/libredex/RedexResources.h @@ -331,9 +331,6 @@ class AndroidResources { const std::string& dirname); // Classnames present in native libraries (lib/*/*.so) std::unordered_set get_native_classes(); - // Sets up BundleConfig.pb file with relevant options for resource - // optimizations that need to executed by bundletool/aapt2. - virtual void finalize_bundle_config(const ResourceConfig& config); const std::string& get_directory() { return m_directory; } diff --git a/libredex/RefChecker.cpp b/libredex/RefChecker.cpp index 13ccc159d4..c1d8ba06a0 100644 --- a/libredex/RefChecker.cpp +++ b/libredex/RefChecker.cpp @@ -27,12 +27,17 @@ CodeRefs::CodeRefs(const DexMethod* method) { always_assert(insn->get_type()); types_set.insert(insn->get_type()); } else if (insn->has_method()) { - auto callee = resolve_invoke_method(insn, method); + auto callee_ref = insn->get_method(); + auto callee = + resolve_method(callee_ref, opcode_to_search(insn), method); + if (!callee && opcode_to_search(insn) == MethodSearch::Virtual) { + callee = resolve_method(callee_ref, MethodSearch::InterfaceVirtual, + method); + } if (!callee) { invalid_refs = true; return editable_cfg_adapter::LOOP_BREAK; } - auto callee_ref = insn->get_method(); if (callee != callee_ref) { types_set.insert(callee_ref->get_class()); } @@ -74,27 +79,44 @@ CodeRefs::CodeRefs(const DexMethod* method) { } bool RefChecker::check_type(const DexType* type) const { - return *m_type_cache - .get_or_create_and_assert_equal( - type, - [this](const auto* _) { return check_type_internal(_); }) - .first; + auto res = m_type_cache.get(type, boost::none); + if (res == boost::none) { + res = check_type_internal(type); + m_type_cache.update( + type, [res](const DexType*, boost::optional& value, bool exists) { + always_assert(!exists || value == res); + value = res; + }); + } + return *res; } bool RefChecker::check_method(const DexMethod* method) const { - return *m_method_cache - .get_or_create_and_assert_equal( - method, - [this](const auto* _) { return check_method_internal(_); }) - .first; + auto res = m_method_cache.get(method, boost::none); + if (res == boost::none) { + res = check_method_internal(method); + m_method_cache.update( + method, + [res](const DexMethod*, boost::optional& value, bool exists) { + always_assert(!exists || value == res); + value = res; + }); + } + return *res; } bool RefChecker::check_field(const DexField* field) const { - return *m_field_cache - .get_or_create_and_assert_equal( - field, - [this](const auto* _) { return check_field_internal(_); }) - .first; + auto res = m_field_cache.get(field, boost::none); + if (res == boost::none) { + res = check_field_internal(field); + m_field_cache.update( + field, + [res](const DexField*, boost::optional& value, bool exists) { + always_assert(!exists || value == res); + value = res; + }); + } + return *res; } bool RefChecker::check_class( @@ -114,21 +136,17 @@ bool RefChecker::check_class( return false; } if (mog && method->is_virtual()) { - if (method_override_graph::any_overridden_methods( - *mog, method, - [&](const auto* m) { - if (!m->is_external()) { - return false; - } - if (m_min_sdk_api && !m_min_sdk_api->has_method(m)) { - TRACE(REFC, 4, "Risky external method override %s -> %s", - SHOW(method), SHOW(m)); - return true; - } - return false; - }, - true)) { - return false; + const auto& overriddens = + method_override_graph::get_overridden_methods(*mog, method, true); + for (const auto* m : overriddens) { + if (!m->is_external()) { + continue; + } + if (m_min_sdk_api && !m_min_sdk_api->has_method(m)) { + TRACE(REFC, 4, "Risky external method override %s -> %s", + SHOW(method), SHOW(m)); + return false; + } } } } diff --git a/libredex/RefChecker.h b/libredex/RefChecker.h index 818e112ef8..df6301f759 100644 --- a/libredex/RefChecker.h +++ b/libredex/RefChecker.h @@ -79,9 +79,9 @@ class RefChecker { size_t m_store_idx; const api::AndroidSDK* m_min_sdk_api; - mutable InsertOnlyConcurrentMap m_type_cache; - mutable InsertOnlyConcurrentMap m_method_cache; - mutable InsertOnlyConcurrentMap m_field_cache; + mutable ConcurrentMap> m_type_cache; + mutable ConcurrentMap> m_method_cache; + mutable ConcurrentMap> m_field_cache; bool check_type_internal(const DexType* type) const; diff --git a/libredex/ReflectionAnalysis.cpp b/libredex/ReflectionAnalysis.cpp index e48e255cf9..f931a7b4c7 100644 --- a/libredex/ReflectionAnalysis.cpp +++ b/libredex/ReflectionAnalysis.cpp @@ -7,7 +7,6 @@ #include "ReflectionAnalysis.h" -#include #include #include #include @@ -350,34 +349,30 @@ class AbstractObjectEnvironment final ReturnValueDomain, CallingContextMap>& /* product */) {} - const AbstractObjectDomain& get_abstract_obj(reg_t reg) const { + AbstractObjectDomain get_abstract_obj(reg_t reg) const { return get<0>().get(reg); } - const BasicAbstractObjectEnvironment& get_basic_abstract_obj_env() const { - return get<0>(); - } - - void set_abstract_obj(reg_t reg, const AbstractObjectDomain& aobj) { - apply<0>([&](auto env) { env->set(reg, aobj); }, true); + void set_abstract_obj(reg_t reg, const AbstractObjectDomain aobj) { + apply<0>([=](auto env) { env->set(reg, aobj); }, true); } void update_abstract_obj( reg_t reg, const std::function& operation) { - apply<0>([&](auto env) { env->update(reg, operation); }, true); + apply<0>([=](auto env) { env->update(reg, operation); }, true); } - const ClassObjectSourceDomain& get_class_source(reg_t reg) const { + ClassObjectSourceDomain get_class_source(reg_t reg) const { return get<1>().get(reg); } - void set_class_source(reg_t reg, const ClassObjectSourceDomain& cls_src) { + void set_class_source(reg_t reg, const ClassObjectSourceDomain cls_src) { apply<1>([=](auto env) { env->set(reg, cls_src); }, true); } - const ConstantAbstractDomain>& get_heap_class_array( + ConstantAbstractDomain> get_heap_class_array( AbstractHeapAddress addr) const { return get<2>().get(addr); } @@ -394,7 +389,7 @@ class AbstractObjectEnvironment final set_heap_class_array(addr, domain); } - const ReturnValueDomain& get_return_value() const { return get<3>(); } + ReturnValueDomain get_return_value() const { return get<3>(); } void join_return_value(const ReturnValueDomain& domain) { apply<3>([=](auto original) { original->join_with(domain); }, true); @@ -420,20 +415,6 @@ class Analyzer final : public BaseIRAnalyzer { m_summary_query_fn(summary_query_fn), m_cache(cache) {} - static std::optional> - maybe_get_value_type_parameter(CallingContext* context, - param_index_t param_position) { - if (context == nullptr) { - return std::nullopt; - } - // This is a regular parameter of the method. - auto& param_abstract_obj = context->get(param_position); - if (param_abstract_obj.is_value()) { - return param_abstract_obj; - } - return std::nullopt; - } - void run(CallingContext* context) { // We need to compute the initial environment by assigning the parameter // registers their correct abstract object derived from the method's @@ -461,13 +442,16 @@ class Analyzer final : public BaseIRAnalyzer { // `this`. update_non_string_input(&init_state, insn, m_dex_method->get_class()); } else { + // This is a regular parameter of the method. + AbstractObjectDomain param_abstract_obj; + DexType* type = *sig_it; always_assert(sig_it++ != signature->end()); - auto maybe_value_param = - maybe_get_value_type_parameter(context, param_position); - if (maybe_value_param) { + if (context && (param_abstract_obj = context->get(param_position), + param_abstract_obj.is_value())) { // Parameter domain is provided with the calling context. - init_state.set_abstract_obj(insn->dest(), maybe_value_param->get()); + init_state.set_abstract_obj(insn->dest(), + context->get(param_position)); } else { update_non_string_input(&init_state, insn, type); } @@ -500,7 +484,7 @@ class Analyzer final : public BaseIRAnalyzer { auto srcs = insn->srcs(); for (param_index_t i = 0; i < srcs.size(); i++) { reg_t src = insn->src(i); - const auto& aobj = current_state->get_abstract_obj(src); + auto aobj = current_state->get_abstract_obj(src); cc.set(i, aobj); } if (!cc.is_bottom()) { @@ -522,14 +506,14 @@ class Analyzer final : public BaseIRAnalyzer { } case OPCODE_MOVE: case OPCODE_MOVE_OBJECT: { - const auto& aobj = current_state->get_abstract_obj(insn->src(0)); + const auto aobj = current_state->get_abstract_obj(insn->src(0)); current_state->set_abstract_obj(insn->dest(), aobj); set_class_source(insn->src(0), insn->dest(), aobj, current_state); break; } case IOPCODE_MOVE_RESULT_PSEUDO_OBJECT: case OPCODE_MOVE_RESULT_OBJECT: { - const auto& aobj = current_state->get_abstract_obj(RESULT_REGISTER); + const auto aobj = current_state->get_abstract_obj(RESULT_REGISTER); current_state->set_abstract_obj(insn->dest(), aobj); set_class_source(RESULT_REGISTER, insn->dest(), aobj, current_state); break; @@ -551,7 +535,7 @@ class Analyzer final : public BaseIRAnalyzer { break; } case OPCODE_CHECK_CAST: { - const auto& aobj = current_state->get_abstract_obj(insn->src(0)); + const auto aobj = current_state->get_abstract_obj(insn->src(0)); set_abstract_obj(RESULT_REGISTER, AbstractObjectKind::OBJECT, insn->get_type(), current_state); set_class_source(insn->src(0), RESULT_REGISTER, aobj, current_state); @@ -563,7 +547,7 @@ class Analyzer final : public BaseIRAnalyzer { break; } case OPCODE_INSTANCE_OF: { - const auto& aobj = current_state->get_abstract_obj(insn->src(0)); + const auto aobj = current_state->get_abstract_obj(insn->src(0)); auto obj = aobj.get_object(); // Append the referenced type here to the potential dex types list. // Doing this increases the type information we have at the reflection @@ -776,16 +760,6 @@ class Analyzer final : public BaseIRAnalyzer { return it->second.get_abstract_obj(reg).get_object(); } - const AbstractObjectEnvironment& get_abstract_object_env( - IRInstruction* insn) const { - auto it = m_environments.find(insn); - if (it == m_environments.end()) { - static const AbstractObjectEnvironment empty_environment; - return empty_environment; - } - return it->second; - } - boost::optional get_class_source( size_t reg, IRInstruction* insn) const { auto it = m_environments.find(insn); @@ -795,9 +769,7 @@ class Analyzer final : public BaseIRAnalyzer { return it->second.get_class_source(reg).get_constant(); } - const AbstractObjectDomain& get_return_value() const { - return m_return_value; - } + AbstractObjectDomain get_return_value() const { return m_return_value; } const AbstractObjectEnvironment& get_exit_state() const { return get_exit_state_at(m_cfg.exit_block()); @@ -1124,39 +1096,32 @@ ReflectionAnalysis::ReflectionAnalysis(DexMethod* dex_method, m_analyzer->run(context); } -void ReflectionAnalysis::gather_reflection_sites( +void ReflectionAnalysis::get_reflection_site( + const reg_t reg, IRInstruction* insn, std::map* abstract_objects) const { - const auto& env = - m_analyzer->get_abstract_object_env(insn).get_basic_abstract_obj_env(); - if (env.kind() != AbstractValueKind::Value) { + auto aobj = m_analyzer->get_abstract_object(reg, insn); + if (!aobj) { return; } - for (auto&& [reg, domain] : env.bindings()) { - auto aobj = domain.get_object(); - if (!aobj) { - continue; - } - if (is_not_reflection_output(*aobj)) { - continue; - } - boost::optional cls_src = - aobj->is_class() ? m_analyzer->get_class_source(reg, insn) - : boost::none; - if (aobj->is_class() && cls_src == ClassObjectSource::NON_REFLECTION) { - continue; - } - if (traceEnabled(REFL, 5)) { - std::ostringstream out; - out << "reg " << reg << " " << *aobj << " "; - if (cls_src) { - out << *cls_src; - } - out << std::endl; - TRACE(REFL, 5, " reflection site: %s", out.str().c_str()); + if (is_not_reflection_output(*aobj)) { + return; + } + boost::optional cls_src = + aobj->is_class() ? m_analyzer->get_class_source(reg, insn) : boost::none; + if (aobj->is_class() && cls_src == ClassObjectSource::NON_REFLECTION) { + return; + } + if (traceEnabled(REFL, 5)) { + std::ostringstream out; + out << "reg " << reg << " " << *aobj << " "; + if (cls_src) { + out << *cls_src; } - (*abstract_objects)[reg] = ReflectionAbstractObject(*aobj, cls_src); + out << std::endl; + TRACE(REFL, 5, " reflection site: %s", out.str().c_str()); } + (*abstract_objects)[reg] = ReflectionAbstractObject(*aobj, cls_src); } ReflectionSites ReflectionAnalysis::get_reflection_sites() const { @@ -1171,10 +1136,13 @@ ReflectionSites ReflectionAnalysis::get_reflection_sites() const { for (auto& mie : InstructionIterable(cfg)) { IRInstruction* insn = mie.insn; std::map abstract_objects; - gather_reflection_sites(insn, &abstract_objects); + for (size_t i = 0; i < reg_size; i++) { + get_reflection_site(i, insn, &abstract_objects); + } + get_reflection_site(RESULT_REGISTER, insn, &abstract_objects); if (!abstract_objects.empty()) { - reflection_sites.emplace_back(insn, std::move(abstract_objects)); + reflection_sites.push_back(std::make_pair(insn, abstract_objects)); } } return reflection_sites; diff --git a/libredex/ReflectionAnalysis.h b/libredex/ReflectionAnalysis.h index 7d4d5e7c8a..7792ebe78c 100644 --- a/libredex/ReflectionAnalysis.h +++ b/libredex/ReflectionAnalysis.h @@ -389,7 +389,8 @@ class ReflectionAnalysis final { std::unique_ptr m_analyzer; MetadataCache* m_fallback_cache = nullptr; - void gather_reflection_sites( + void get_reflection_site( + const reg_t reg, IRInstruction* insn, std::map* abstract_objects) const; }; diff --git a/libredex/RemoveUninstantiablesImpl.cpp b/libredex/RemoveUninstantiablesImpl.cpp index d153f5e09c..cd6f04080b 100644 --- a/libredex/RemoveUninstantiablesImpl.cpp +++ b/libredex/RemoveUninstantiablesImpl.cpp @@ -219,7 +219,7 @@ Stats replace_all_with_unreachable_throw(cfg::ControlFlowGraph& cfg) { Stats reduce_uncallable_instance_methods( const Scope& scope, - const ConcurrentSet& uncallable_instance_methods, + const std::unordered_set& uncallable_instance_methods, const std::function& is_implementation_method) { // We perform structural changes, i.e. whether a method has a body and // removal, as a post-processing step, to streamline the main operations diff --git a/libredex/RemoveUninstantiablesImpl.h b/libredex/RemoveUninstantiablesImpl.h index 0b06f2448f..8dca50df3c 100644 --- a/libredex/RemoveUninstantiablesImpl.h +++ b/libredex/RemoveUninstantiablesImpl.h @@ -7,7 +7,6 @@ #pragma once -#include "ConcurrentContainers.h" #include "DexStore.h" #include "Pass.h" @@ -30,13 +29,6 @@ struct Stats { int invoke_uninstantiables = 0; int check_casts = 0; - int sum() const { - return instance_ofs + invokes + field_accesses_on_uninstantiable + - throw_null_methods + abstracted_classes + abstracted_vmethods + - removed_vmethods + get_uninstantiables + invoke_uninstantiables + - check_casts; - } - Stats& operator+=(const Stats&); Stats operator+(const Stats&) const; @@ -61,6 +53,6 @@ Stats replace_all_with_unreachable_throw(cfg::ControlFlowGraph& cfg); /// either making them abstract, removing their body, or deleting them. Stats reduce_uncallable_instance_methods( const Scope& scope, - const ConcurrentSet& uncallable_instance_methods, + const std::unordered_set& uncallable_instance_methods, const std::function& is_implementation_methods); } // namespace remove_uninstantiables_impl diff --git a/libredex/Resolver.h b/libredex/Resolver.h index 4d94900133..fc9780899d 100644 --- a/libredex/Resolver.h +++ b/libredex/Resolver.h @@ -74,9 +74,8 @@ struct MethodRefCacheKeyHash { using MethodRefCache = std::unordered_map; -using ConcurrentMethodRefCache = InsertOnlyConcurrentMap; +using ConcurrentMethodRefCache = + ConcurrentMap; /** * Helper to map an opcode to a MethodSearch rule. @@ -274,9 +273,10 @@ inline DexMethod* resolve_method(DexMethodRef* method, if (m) { return m; } - auto res = concurrent_ref_cache.get(MethodRefCacheKey{method, search}); - if (res) { - return *res; + auto def = + concurrent_ref_cache.get(MethodRefCacheKey{method, search}, nullptr); + if (def != nullptr) { + return def; } auto mdef = resolve_method(method, search, caller); if (mdef != nullptr) { @@ -285,50 +285,6 @@ inline DexMethod* resolve_method(DexMethodRef* method, return mdef; } -/** - * Resolve the method of an invoke instruction. Note that there are some - * invoke-virtual call on methods whose def are actually in interface. Thus, for - * an invoke-virtual, it first tries MethodSearch::Virtual, and then - * MethodSearch::InterfaceVirtual. - */ -inline DexMethod* resolve_invoke_method( - const IRInstruction* insn, - const DexMethod* caller = nullptr, - bool* resolved_virtual_to_interface = nullptr) { - auto callee_ref = insn->get_method(); - auto search = opcode_to_search(insn); - auto callee = resolve_method(callee_ref, search, caller); - if (!callee && search == MethodSearch::Virtual) { - callee = resolve_method(callee_ref, MethodSearch::InterfaceVirtual, caller); - if (resolved_virtual_to_interface) { - *resolved_virtual_to_interface = callee != nullptr; - } - } else if (resolved_virtual_to_interface) { - *resolved_virtual_to_interface = false; - } - return callee; -} - -inline DexMethod* resolve_invoke_method( - const IRInstruction* insn, - MethodRefCache& ref_cache, - const DexMethod* caller = nullptr, - bool* resolved_virtual_to_interface = nullptr) { - auto callee_ref = insn->get_method(); - auto search = opcode_to_search(insn); - auto callee = resolve_method(callee_ref, search, ref_cache, caller); - if (!callee && search == MethodSearch::Virtual) { - callee = resolve_method(callee_ref, MethodSearch::InterfaceVirtual, - ref_cache, caller); - if (resolved_virtual_to_interface) { - *resolved_virtual_to_interface = callee != nullptr; - } - } else if (resolved_virtual_to_interface) { - *resolved_virtual_to_interface = false; - } - return callee; -} - /** * Given a scope defined by DexClass, a name and a proto look for the vmethod * on the top ancestor. Essentially finds where the method was introduced. diff --git a/libredex/SanitizersConfig.h b/libredex/SanitizersConfig.h index 54923df06b..6e7738ca88 100644 --- a/libredex/SanitizersConfig.h +++ b/libredex/SanitizersConfig.h @@ -28,9 +28,7 @@ const char* kAsanDefaultOptions = ":" "print_suppressions=0" ":" - "strict_init_order=1" - ":" - "detect_odr_violation=0"; + "strict_init_order=1"; #if defined(__clang__) #define NO_SANITIZE \ diff --git a/libredex/ScopedMemStats.h b/libredex/ScopedMemStats.h index 6f2e91a83d..38e2a3e491 100644 --- a/libredex/ScopedMemStats.h +++ b/libredex/ScopedMemStats.h @@ -7,7 +7,7 @@ #pragma once -#include "DebugUtils.h" +#include "Debug.h" #include "Pass.h" #include "PassManager.h" #include "Show.h" diff --git a/libredex/ShrinkerConfig.h b/libredex/ShrinkerConfig.h index 2a2f7297ec..92675e253d 100644 --- a/libredex/ShrinkerConfig.h +++ b/libredex/ShrinkerConfig.h @@ -23,8 +23,6 @@ struct ShrinkerConfig { bool run_fast_reg_alloc{false}; bool run_dedup_blocks{false}; - bool normalize_new_instances{true}; - // Internally used option that decides whether to compute pure methods with a // relatively expensive analysis over the scope bool compute_pure_methods{true}; diff --git a/libredex/SingletonIterable.h b/libredex/SingletonIterable.h index 3e382ab130..ac7e99ca3f 100644 --- a/libredex/SingletonIterable.h +++ b/libredex/SingletonIterable.h @@ -7,7 +7,6 @@ #pragma once -#include #include template diff --git a/libredex/SourceBlocks.cpp b/libredex/SourceBlocks.cpp index 5c60c8251e..3bb8e847c6 100644 --- a/libredex/SourceBlocks.cpp +++ b/libredex/SourceBlocks.cpp @@ -1365,18 +1365,4 @@ void fill_source_block(SourceBlock& sb, } } -void fill_source_block(SourceBlock& sb, - DexMethod* ref, - uint32_t id, - const std::vector& many) { - sb.src = ref->get_deobfuscated_name_or_null(); - sb.id = id; - for (size_t i = 0; i < sb.vals_size; i++) { - sb.vals[i] = SourceBlock::Val::none(); - } - for (auto& other : many) { - sb.max(*other); - } -} - } // namespace source_blocks diff --git a/libredex/SourceBlocks.h b/libredex/SourceBlocks.h index 11801b03fb..d8fe3d6ba3 100644 --- a/libredex/SourceBlocks.h +++ b/libredex/SourceBlocks.h @@ -194,10 +194,7 @@ void visit_in_order(const ControlFlowGraph* cfg, } } - assert_log(visited.size() == cfg->num_blocks(), - "%zu vs %zu", - visited.size(), - cfg->num_blocks()); + assert_log(visited.size() == cfg->num_blocks(), "%zu vs %zu", visited.size(), cfg->num_blocks()); } } // namespace impl @@ -522,11 +519,4 @@ void fill_source_block( uint32_t id, const std::optional& opt_val = std::nullopt); -// Fill a source block with the maximum values of the provided many -// source-blocks. -void fill_source_block(SourceBlock& sb, - DexMethod* ref, - uint32_t id, - const std::vector& many); - } // namespace source_blocks diff --git a/libredex/StringBuilder.h b/libredex/StringBuilder.h index 7988ea24f7..5f656f80ec 100644 --- a/libredex/StringBuilder.h +++ b/libredex/StringBuilder.h @@ -10,8 +10,6 @@ #include #include -#include "Debug.h" - /* * Quickly concatenate std::strings * diff --git a/libredex/ThreadPool.cpp b/libredex/ThreadPool.cpp deleted file mode 100644 index d9f1f8833c..0000000000 --- a/libredex/ThreadPool.cpp +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * - * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. - */ - -#include "ThreadPool.h" - -namespace { - -redex_thread_pool::ThreadPool* s_threadpool{nullptr}; - -} // anonymous namespace - -namespace redex_thread_pool { - -ThreadPool* ThreadPool::get_instance() { return s_threadpool; } - -boost::thread ThreadPool::create_thread(std::function bound_f) { - boost::thread::attributes attrs; - attrs.set_stack_size(8 * 1024 * 1024); // 8MB stack. - auto bound_run = std::bind(&ThreadPool::run, this, std::move(bound_f)); - return boost::thread(attrs, std::move(bound_run)); -} - -void ThreadPool::create() { s_threadpool = new ThreadPool(); } - -void ThreadPool::destroy() { - delete s_threadpool; - s_threadpool = nullptr; -} - -} // namespace redex_thread_pool diff --git a/libredex/ThreadPool.h b/libredex/ThreadPool.h deleted file mode 100644 index 501ca328c9..0000000000 --- a/libredex/ThreadPool.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * - * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. - */ - -#pragma once - -// We for now need a larger stack size than the default, and on Mac OS -// this is the only way (or pthreads directly), as `ulimit -s` does not -// apply to non-main threads. -#include - -#include - -namespace redex_thread_pool { - -class ThreadPool : public sparta::ThreadPool { - public: - static ThreadPool* get_instance(); - - static void create(); - - static void destroy(); - - protected: - boost::thread create_thread(std::function bound_f) override; -}; - -} // namespace redex_thread_pool diff --git a/libredex/ThrowPropagationImpl.cpp b/libredex/ThrowPropagationImpl.cpp index 86138eb1ae..d5bbc21be2 100644 --- a/libredex/ThrowPropagationImpl.cpp +++ b/libredex/ThrowPropagationImpl.cpp @@ -17,12 +17,11 @@ bool ThrowPropagator::try_apply(const cfg::InstructionIterator& cfg_it) { if (!check_if_dead_code_present_and_prepare_block(cfg_it)) { return false; } - insert_unreachable(cfg_it); + insert_throw(cfg_it); return true; } -bool ThrowPropagator::will_throw_or_not_terminate_or_unreachable( - cfg::InstructionIterator it) { +bool ThrowPropagator::will_throw_or_not_terminate(cfg::InstructionIterator it) { std::unordered_set visited{it->insn}; while (true) { it = m_cfg.next_following_gotos(it); @@ -48,7 +47,6 @@ bool ThrowPropagator::will_throw_or_not_terminate_or_unreachable( break; } case OPCODE_THROW: - case IOPCODE_UNREACHABLE: return true; default: return false; @@ -67,7 +65,7 @@ bool ThrowPropagator::check_if_dead_code_present_and_prepare_block( auto insn = it->insn; TRACE(TP, 4, "no return: %s", SHOW(insn)); if (insn == block->get_last_insn()->insn) { - if (will_throw_or_not_terminate_or_unreachable(cfg_it)) { + if (will_throw_or_not_terminate(cfg_it)) { // There's already code in place that will immediately and // unconditionally throw an exception, and thus we don't need to // bother rewriting the code into a throw. The main reason we are @@ -81,7 +79,7 @@ bool ThrowPropagator::check_if_dead_code_present_and_prepare_block( // causes complications due to the possibly following and then dangling // move-result instruction, so we'll explicitly split the block here // in order to keep all invariant happy.) - if (will_throw_or_not_terminate_or_unreachable(cfg_it)) { + if (will_throw_or_not_terminate(cfg_it)) { // As above, nothing to do, since an exception will be thrown anyway. return false; } @@ -93,19 +91,53 @@ bool ThrowPropagator::check_if_dead_code_present_and_prepare_block( return true; }; -void ThrowPropagator::insert_unreachable( - const cfg::InstructionIterator& cfg_it) { +void ThrowPropagator::insert_throw(const cfg::InstructionIterator& cfg_it) { const auto block = cfg_it.block(); IRInstruction* insn = cfg_it->insn; - if (!m_reg) { - m_reg = m_cfg.allocate_temp(); + std::string message{"Redex: Unreachable code after no-return invoke"}; + if (m_debug) { + message += ":"; + message += SHOW(insn); + } + if (!m_regs) { + m_regs = std::make_pair(m_cfg.allocate_temp(), m_cfg.allocate_temp()); } + auto exception_reg = m_regs->first; + auto string_reg = m_regs->second; cfg::Block* new_block = m_cfg.create_block(); - std::vector insns{ - (new IRInstruction(OPCODE_CONST))->set_literal(0)->set_dest(*m_reg), - (new IRInstruction(OPCODE_THROW))->set_src(0, *m_reg), - }; + std::vector insns; + auto new_instance_insn = new IRInstruction(OPCODE_NEW_INSTANCE); + auto exception_type = type::java_lang_RuntimeException(); + always_assert(exception_type != nullptr); + new_instance_insn->set_type(exception_type); + insns.push_back(new_instance_insn); + + auto move_result_pseudo_exception_insn = + new IRInstruction(IOPCODE_MOVE_RESULT_PSEUDO_OBJECT); + move_result_pseudo_exception_insn->set_dest(exception_reg); + insns.push_back(move_result_pseudo_exception_insn); + + auto const_string_insn = new IRInstruction(OPCODE_CONST_STRING); + const_string_insn->set_string(DexString::make_string(message)); + insns.push_back(const_string_insn); + + auto move_result_pseudo_string_insn = + new IRInstruction(IOPCODE_MOVE_RESULT_PSEUDO_OBJECT); + move_result_pseudo_string_insn->set_dest(string_reg); + insns.push_back(move_result_pseudo_string_insn); + + auto invoke_direct_insn = new IRInstruction(OPCODE_INVOKE_DIRECT); + auto init_method = method::java_lang_RuntimeException_init_String(); + always_assert(init_method != nullptr); + invoke_direct_insn->set_method(init_method) + ->set_srcs_size(2) + ->set_src(0, exception_reg) + ->set_src(1, string_reg); + insns.push_back(invoke_direct_insn); + auto throw_insn = new IRInstruction(OPCODE_THROW); + throw_insn->set_src(0, exception_reg); + insns.push_back(throw_insn); new_block->push_back(insns); m_cfg.copy_succ_edges_of_type(block, new_block, cfg::EDGE_THROW); auto existing_goto_edge = m_cfg.get_succ_edge_of_type(block, cfg::EDGE_GOTO); diff --git a/libredex/ThrowPropagationImpl.h b/libredex/ThrowPropagationImpl.h index 39bfacbe47..03c2d028ac 100644 --- a/libredex/ThrowPropagationImpl.h +++ b/libredex/ThrowPropagationImpl.h @@ -13,18 +13,20 @@ namespace throw_propagation_impl { class ThrowPropagator { public: - explicit ThrowPropagator(cfg::ControlFlowGraph& cfg) : m_cfg(cfg) {} + ThrowPropagator(cfg::ControlFlowGraph& cfg, bool debug) + : m_cfg(cfg), m_debug(debug) {} bool try_apply(const cfg::InstructionIterator& cfg_it); private: - bool will_throw_or_not_terminate_or_unreachable(cfg::InstructionIterator it); + bool will_throw_or_not_terminate(cfg::InstructionIterator it); bool check_if_dead_code_present_and_prepare_block( const cfg::InstructionIterator& cfg_it); - void insert_unreachable(const cfg::InstructionIterator& cfg_it); + void insert_throw(const cfg::InstructionIterator& cfg_it); cfg::ControlFlowGraph& m_cfg; - boost::optional m_reg; + bool m_debug; + boost::optional> m_regs; }; } // namespace throw_propagation_impl diff --git a/libredex/Timer.cpp b/libredex/Timer.cpp index 84e55a6158..415f45202d 100644 --- a/libredex/Timer.cpp +++ b/libredex/Timer.cpp @@ -7,15 +7,11 @@ #include "Timer.h" -#include - #include "Trace.h" unsigned Timer::s_indent = 0; std::mutex Timer::s_lock; Timer::times_t Timer::s_times; -std::mutex AccumulatingTimer::s_lock; -AccumulatingTimer::times_impl_t* AccumulatingTimer::s_times{nullptr}; Timer::Timer(const std::string& msg, bool indent) : m_msg(msg), @@ -38,32 +34,7 @@ Timer::~Timer() { Timer::add_timer(std::move(m_msg), duration_s); } -void Timer::add_timer(std::string msg, double dur_s) { +void Timer::add_timer(std::string&& msg, double dur_s) { std::lock_guard guard(s_lock); s_times.emplace_back(std::move(msg), dur_s); } - -AccumulatingTimer::AccumulatingTimer(std::string msg) { - add_timer(std::move(msg), m_microseconds); -} - -AccumulatingTimer::times_t AccumulatingTimer::get_times() { - std::lock_guard guard(s_lock); - AccumulatingTimer::times_t res; - if (s_times) { - res.reserve(s_times->size()); - for (auto& [str, microseconds] : *s_times) { - res.emplace_back(str, ((double)microseconds->load()) / 1000000); - } - } - return res; -} - -void AccumulatingTimer::add_timer( - std::string msg, std::shared_ptr> microseconds) { - std::lock_guard guard(s_lock); - if (!s_times) { - s_times = new AccumulatingTimer::times_impl_t(); - } - s_times->emplace_back(std::move(msg), std::move(microseconds)); -} diff --git a/libredex/Timer.h b/libredex/Timer.h index e27dbdc5c1..b3ebb6172c 100644 --- a/libredex/Timer.h +++ b/libredex/Timer.h @@ -9,8 +9,6 @@ #include #include -#include -#include #include #include #include @@ -21,11 +19,10 @@ struct Timer { ~Timer(); using times_t = std::vector>; - // there should be no currently running Timers when this function is called static const times_t& get_times() { return s_times; } - static void add_timer(std::string msg, double dur_s); + static void add_timer(std::string&& msg, double dur_s); private: static std::mutex s_lock; @@ -49,7 +46,7 @@ class AccumulatingTimer { auto end = std::chrono::high_resolution_clock::now(); auto dur_in_mus = std::chrono::duration_cast(end - m_start); - m_context->m_microseconds->fetch_add((uint64_t)dur_in_mus.count()); + m_context->m_microseconds += (uint64_t)dur_in_mus.count(); } // Disallow copying. @@ -65,33 +62,15 @@ class AccumulatingTimer { std::chrono::high_resolution_clock::time_point m_start; }; - AccumulatingTimer() {} - - explicit AccumulatingTimer(std::string msg); + AccumulatingTimer() : m_microseconds(0) {} AccumulatingTimerScope scope() { return AccumulatingTimerScope(this); } - uint64_t get_microseconds() const { return m_microseconds->load(); } + uint64_t get_microseconds() const { return m_microseconds.load(); } double get_seconds() const { - return ((double)m_microseconds->load()) / 1000000; + return ((double)m_microseconds.load()) / 1000000; } - using times_t = std::vector>; - - // there should be no currently running Timers when this function is called - static times_t get_times(); - - static void add_timer(std::string msg, - std::shared_ptr> microseconds); - private: - using times_impl_t = - std::list>>>; - - static std::mutex s_lock; - // We dynamically allocate the static list of times to avoid problems with the - // static initialization / destruction order. - static times_impl_t* s_times; - std::shared_ptr> m_microseconds{ - std::make_shared>(0)}; + std::atomic m_microseconds; }; diff --git a/libredex/Trace.cpp b/libredex/Trace.cpp index ef36259fd3..668c6d4632 100644 --- a/libredex/Trace.cpp +++ b/libredex/Trace.cpp @@ -194,6 +194,7 @@ struct Tracer { } } + private: FILE* m_file{nullptr}; long m_level{0}; std::array m_traces; diff --git a/libredex/Trace.h b/libredex/Trace.h index 8eb61d9036..4177802a58 100644 --- a/libredex/Trace.h +++ b/libredex/Trace.h @@ -23,7 +23,6 @@ class DexType; TM(API_UTILS) \ TM(APP_MOD_USE) \ TM(ARGS) \ - TM(ARSC) \ TM(ASSESSOR) \ TM(BBPROFILE) \ TM(BBREORDERING) \ diff --git a/libredex/TypeInference.cpp b/libredex/TypeInference.cpp index a9bc3f91a0..69b449fe11 100644 --- a/libredex/TypeInference.cpp +++ b/libredex/TypeInference.cpp @@ -186,7 +186,7 @@ void set_integral( const boost::optional& annotation = boost::none) { state->set_type(reg, TypeDomain(IRType::INT)); const auto anno = DexAnnoType(annotation); - DexTypeDomain dex_type_domain = DexTypeDomain::create_for_anno(&anno); + DexTypeDomain dex_type_domain = DexTypeDomain(&anno); state->set_dex_type(reg, dex_type_domain); } @@ -200,30 +200,15 @@ void set_scalar(TypeEnvironment* state, reg_t reg) { state->reset_dex_type(reg); } -void set_reference(TypeEnvironment* state, - reg_t reg, - const boost::optional& dex_type_opt, - bool is_not_null = false) { - state->set_type(reg, TypeDomain(IRType::REFERENCE)); - auto* dex_type = dex_type_opt ? *dex_type_opt : nullptr; - always_assert(!is_not_null || dex_type != nullptr); - const DexTypeDomain dex_type_domain = - is_not_null ? DexTypeDomain::create_not_null(dex_type) - : DexTypeDomain::create_nullable(dex_type); - state->set_dex_type(reg, dex_type_domain); -} - -// The presence of DexAnnoType implies Nullable -void set_reference_with_anno( +void set_reference( TypeEnvironment* state, reg_t reg, const boost::optional& dex_type_opt, - const boost::optional& annotation) { + const boost::optional& annotation = boost::none) { state->set_type(reg, TypeDomain(IRType::REFERENCE)); - auto* dex_type = dex_type_opt ? *dex_type_opt : nullptr; + auto dex_type = dex_type_opt ? *dex_type_opt : nullptr; const auto anno = DexAnnoType(annotation); - const DexTypeDomain dex_type_domain = - DexTypeDomain::create_nullable(dex_type, &anno); + const DexTypeDomain dex_type_domain = DexTypeDomain(dex_type, &anno); state->set_dex_type(reg, dex_type_domain); } @@ -353,8 +338,8 @@ const DexType* merge_dex_types(const DexTypeIt& begin, return t1; } - DexTypeDomain d1 = DexTypeDomain::create_nullable(t1); - DexTypeDomain d2 = DexTypeDomain::create_nullable(t2); + DexTypeDomain d1(t1); + DexTypeDomain d2(t2); d1.join_with(d2); auto maybe_dextype = d1.get_dex_type(); @@ -454,7 +439,7 @@ void TypeInference::refine_scalar(TypeEnvironment* state, reg_t reg) const { /* expected */ IRType::SCALAR); const boost::optional annotation = state->get_annotation(reg); const auto anno = DexAnnoType(annotation); - DexTypeDomain dex_type_domain = DexTypeDomain::create_for_anno(&anno); + DexTypeDomain dex_type_domain = DexTypeDomain(&anno); state->set_dex_type(reg, dex_type_domain); } @@ -462,7 +447,7 @@ void TypeInference::refine_integral(TypeEnvironment* state, reg_t reg) const { refine_type(state, reg, /* expected */ IRType::INT); const boost::optional annotation = state->get_annotation(reg); const auto anno = DexAnnoType(annotation); - DexTypeDomain dex_type_domain = DexTypeDomain::create_for_anno(&anno); + DexTypeDomain dex_type_domain = DexTypeDomain(&anno); state->set_dex_type(reg, dex_type_domain); } @@ -555,13 +540,12 @@ void TypeInference::run(bool is_static, // If the method is not static, the first parameter corresponds to // `this`. first_param = false; - set_reference_with_anno(&init_state, insn->dest(), declaring_type, - annotation); + set_reference(&init_state, insn->dest(), declaring_type, annotation); } else { // This is a regular parameter of the method. always_assert(sig_it != args->end()); const DexType* type = *sig_it; - set_reference_with_anno(&init_state, insn->dest(), type, annotation); + set_reference(&init_state, insn->dest(), type, annotation); ++sig_it; } break; @@ -829,14 +813,12 @@ void TypeInference::analyze_instruction(const IRInstruction* insn, break; } case OPCODE_NEW_INSTANCE: { - set_reference(current_state, RESULT_REGISTER, insn->get_type(), - /* is_not_null */ true); + set_reference(current_state, RESULT_REGISTER, insn->get_type()); break; } case OPCODE_NEW_ARRAY: { refine_int(current_state, insn->src(0)); - set_reference(current_state, RESULT_REGISTER, insn->get_type(), - /* is_not_null */ true); + set_reference(current_state, RESULT_REGISTER, insn->get_type()); break; } case OPCODE_FILLED_NEW_ARRAY: { @@ -853,8 +835,7 @@ void TypeInference::analyze_instruction(const IRInstruction* insn, refine_scalar(current_state, insn->src(i)); } } - set_reference(current_state, RESULT_REGISTER, insn->get_type(), - /* is_not_null */ true); + set_reference(current_state, RESULT_REGISTER, insn->get_type()); break; } case OPCODE_FILL_ARRAY_DATA: { @@ -1054,8 +1035,8 @@ void TypeInference::analyze_instruction(const IRInstruction* insn, get_typedef_anno_from_member(insn->get_field()); always_assert(insn->has_field()); const auto field = insn->get_field(); - set_reference_with_anno(current_state, RESULT_REGISTER, field->get_type(), - annotation); + set_reference(current_state, RESULT_REGISTER, field->get_type(), + annotation); break; } case OPCODE_IPUT: { @@ -1063,8 +1044,7 @@ void TypeInference::analyze_instruction(const IRInstruction* insn, if (!m_annotations.empty()) { auto annotation = current_state->get_annotation(insn->src(0)); const DexAnnoType anno = DexAnnoType(annotation); - const DexTypeDomain dex_type_domain = - DexTypeDomain::create_nullable(type, &anno); + const DexTypeDomain dex_type_domain = DexTypeDomain(type, &anno); current_state->set_dex_type(insn->src(1), dex_type_domain); } if (type::is_float(type)) { @@ -1106,8 +1086,7 @@ void TypeInference::analyze_instruction(const IRInstruction* insn, const auto anno = DexAnnoType(annotation); auto type = current_state->get_dex_type(insn->src(1)); auto dex_type = type ? *type : nullptr; - const DexTypeDomain dex_type_domain = - DexTypeDomain::create_nullable(dex_type, &anno); + const DexTypeDomain dex_type_domain = DexTypeDomain(dex_type, &anno); current_state->set_dex_type(insn->src(1), dex_type_domain); } refine_reference(current_state, insn->src(0)); @@ -1257,8 +1236,7 @@ void TypeInference::analyze_instruction(const IRInstruction* insn, if (type::is_object(return_type)) { boost::optional annotation = get_typedef_anno_from_member(dex_method); - set_reference_with_anno(current_state, RESULT_REGISTER, return_type, - annotation); + set_reference(current_state, RESULT_REGISTER, return_type, annotation); break; } if (type::is_integral(return_type)) { diff --git a/libredex/Vinfo.cpp b/libredex/Vinfo.cpp index e020dda50f..07bbcbdab9 100644 --- a/libredex/Vinfo.cpp +++ b/libredex/Vinfo.cpp @@ -54,26 +54,26 @@ Vinfo::Vinfo(const std::vector& scope) { } const DexMethod* Vinfo::get_decl(const DexMethod* meth) { - redex_assert(std::as_const(m_vinfos).find(meth) != m_vinfos.cend()); + redex_assert(m_vinfos.find(meth) != m_vinfos.end()); return m_vinfos[meth].decl; } bool Vinfo::is_override(const DexMethod* meth) { - redex_assert(std::as_const(m_vinfos).find(meth) != m_vinfos.cend()); + redex_assert(m_vinfos.find(meth) != m_vinfos.end()); return m_vinfos[meth].override_of; } const DexMethod* Vinfo::get_overriden_method(const DexMethod* meth) { - redex_assert(std::as_const(m_vinfos).find(meth) != m_vinfos.cend()); + redex_assert(m_vinfos.find(meth) != m_vinfos.end()); return m_vinfos[meth].override_of; } bool Vinfo::is_overriden(const DexMethod* meth) { - redex_assert(std::as_const(m_vinfos).find(meth) != m_vinfos.cend()); + redex_assert(m_vinfos.find(meth) != m_vinfos.end()); return m_vinfos[meth].is_overriden; } const methods_t& Vinfo::get_override_methods(const DexMethod* meth) { - redex_assert(std::as_const(m_vinfos).find(meth) != m_vinfos.cend()); + redex_assert(m_vinfos.find(meth) != m_vinfos.end()); return m_vinfos[meth].overriden_by; } diff --git a/libredex/Walkers.h b/libredex/Walkers.h index 6c379d6864..4ca7430d64 100644 --- a/libredex/Walkers.h +++ b/libredex/Walkers.h @@ -361,7 +361,7 @@ class walk { const WalkerFn& walker) { std::vector>> block_matches; - always_assert(ir_code.editable_cfg_built()); + ir_code.build_cfg(/* editable */ false); for (cfg::Block* block : ir_code.cfg().blocks()) { std::vector> method_matches; std::vector insns; diff --git a/libredex/WellKnownTypes.h b/libredex/WellKnownTypes.h index a5755a37e0..80cc29ee12 100644 --- a/libredex/WellKnownTypes.h +++ b/libredex/WellKnownTypes.h @@ -48,7 +48,6 @@ #define WELL_KNOWN_METHODS \ FOR_EACH(java_lang_Object_ctor, "Ljava/lang/Object;.:()V") \ - FOR_EACH(java_lang_Object_finalize, "Ljava/lang/Object;.finalize:()V") \ FOR_EACH(java_lang_Enum_ctor, \ "Ljava/lang/Enum;.:(Ljava/lang/String;I)V") \ FOR_EACH(java_lang_Enum_ordinal, "Ljava/lang/Enum;.ordinal:()I") \ diff --git a/libredex/WorkQueue.cpp b/libredex/WorkQueue.cpp index cf6ed9623f..475da498a3 100644 --- a/libredex/WorkQueue.cpp +++ b/libredex/WorkQueue.cpp @@ -9,7 +9,7 @@ #include -#include "DebugUtils.h" +#include "Debug.h" namespace redex_workqueue_impl { diff --git a/libredex/WorkQueue.h b/libredex/WorkQueue.h index 445588e659..aec63ad578 100644 --- a/libredex/WorkQueue.h +++ b/libredex/WorkQueue.h @@ -12,8 +12,6 @@ #include -#include "ThreadPool.h" - namespace redex_workqueue_impl { void redex_queue_exception_handler(std::exception& e); @@ -65,10 +63,11 @@ workqueue_foreach( const Fn& fn, unsigned int num_threads = redex_parallel::default_num_threads(), bool push_tasks_while_running = false) { - return sparta::WorkQueue< - Input, redex_workqueue_impl::NoStateWorkQueueHelper>( - redex_workqueue_impl::NoStateWorkQueueHelper{fn}, num_threads, - push_tasks_while_running, redex_thread_pool::ThreadPool::get_instance()); + return sparta:: + WorkQueue>( + redex_workqueue_impl::NoStateWorkQueueHelper{fn}, + num_threads, + push_tasks_while_running); } template >( + Input, + redex_workqueue_impl::WithStateWorkQueueHelper>( redex_workqueue_impl::WithStateWorkQueueHelper{fn}, - num_threads, push_tasks_while_running, - redex_thread_pool::ThreadPool::get_instance()); + num_threads, + push_tasks_while_running); } template >( - redex_workqueue_impl::NoStateWorkQueueHelper{fn}, num_threads, - push_tasks_while_running, redex_thread_pool::ThreadPool::get_instance()); + auto wq = sparta:: + WorkQueue>( + redex_workqueue_impl::NoStateWorkQueueHelper{fn}, + num_threads, + push_tasks_while_running); for (Input item : items) { wq.add_item(std::move(item)); } @@ -113,10 +114,11 @@ void workqueue_run( const Items& items, unsigned int num_threads = redex_parallel::default_num_threads(), bool push_tasks_while_running = false) { - auto wq = sparta::WorkQueue< - Input, redex_workqueue_impl::NoStateWorkQueueHelper>( - redex_workqueue_impl::NoStateWorkQueueHelper{fn}, num_threads, - push_tasks_while_running, redex_thread_pool::ThreadPool::get_instance()); + auto wq = sparta:: + WorkQueue>( + redex_workqueue_impl::NoStateWorkQueueHelper{fn}, + num_threads, + push_tasks_while_running); for (Input item : items) { wq.add_item(std::move(item)); } @@ -132,10 +134,11 @@ void workqueue_run( unsigned int num_threads = redex_parallel::default_num_threads(), bool push_tasks_while_running = false) { auto wq = sparta::WorkQueue< - Input, redex_workqueue_impl::WithStateWorkQueueHelper>( + Input, + redex_workqueue_impl::WithStateWorkQueueHelper>( redex_workqueue_impl::WithStateWorkQueueHelper{fn}, - num_threads, push_tasks_while_running, - redex_thread_pool::ThreadPool::get_instance()); + num_threads, + push_tasks_while_running); for (Input item : items) { wq.add_item(std::move(item)); } @@ -151,10 +154,11 @@ void workqueue_run( unsigned int num_threads = redex_parallel::default_num_threads(), bool push_tasks_while_running = false) { auto wq = sparta::WorkQueue< - Input, redex_workqueue_impl::WithStateWorkQueueHelper>( + Input, + redex_workqueue_impl::WithStateWorkQueueHelper>( redex_workqueue_impl::WithStateWorkQueueHelper{fn}, - num_threads, push_tasks_while_running, - redex_thread_pool::ThreadPool::get_instance()); + num_threads, + push_tasks_while_running); for (Input item : items) { wq.add_item(std::move(item)); } @@ -171,8 +175,7 @@ void workqueue_run_for( redex_workqueue_impl::NoStateWorkQueueHelper>( redex_workqueue_impl::NoStateWorkQueueHelper{fn}, num_threads, - /* push_tasks_while_running */ false, - redex_thread_pool::ThreadPool::get_instance()); + /* push_tasks_while_running */ false); for (InteralType i = start; i < end; i++) { wq.add_item(i); } diff --git a/libresource/Serialize.cpp b/libresource/Serialize.cpp index 9181fe3524..7b002e88c4 100644 --- a/libresource/Serialize.cpp +++ b/libresource/Serialize.cpp @@ -129,7 +129,7 @@ void push_chunk(android::ResChunk_header* header, android::Vector* out) { } void push_vec(android::Vector& vec, android::Vector* out) { - if (!vec.empty()) { + if (vec.size() > 0) { out->appendVector(vec); } } diff --git a/opt/access-marking/AccessMarking.cpp b/opt/access-marking/AccessMarking.cpp index acc65d8cbc..25471e0262 100644 --- a/opt/access-marking/AccessMarking.cpp +++ b/opt/access-marking/AccessMarking.cpp @@ -101,30 +101,30 @@ size_t mark_fields_final(const Scope& scope, return n_fields_finalized; } -ConcurrentSet find_private_methods( +std::vector direct_methods(const std::vector& scope) { + std::vector ret; + for (auto cls : scope) { + for (auto m : cls->get_dmethods()) { + ret.push_back(m); + } + } + return ret; +} + +std::unordered_set find_private_methods( const std::vector& scope, const mog::Graph& override_graph) { - ConcurrentSet candidates; - auto is_excluded = [](DexMethod* m) { + auto candidates = mog::get_non_true_virtuals(override_graph, scope); + auto dmethods = direct_methods(scope); + for (auto* dmethod : dmethods) { + candidates.emplace(dmethod); + } + std20::erase_if(candidates, [](auto* m) { TRACE(ACCESS, 3, "Considering for privatization: %s", SHOW(m)); return method::is_clinit(m) || !can_rename(m) || is_abstract(m) || is_private(m); - }; - workqueue_run( - [&](DexClass* cls) { - for (auto m : cls->get_dmethods()) { - if (!is_excluded(m)) { - candidates.insert(m); - } - } - for (auto m : cls->get_vmethods()) { - if (!is_excluded(m) && - !method_override_graph::is_true_virtual(override_graph, m)) { - candidates.insert(m); - } - } - }, - scope); + }); + ConcurrentSet externally_referenced; walk::parallel::opcodes( scope, [](DexMethod*) { return true; }, @@ -137,24 +137,25 @@ ConcurrentSet find_private_methods( if (callee == nullptr || callee->get_class() == caller->get_class()) { return; } - candidates.erase(callee); + externally_referenced.emplace(callee); }); + for (auto* m : externally_referenced) { + candidates.erase(m); + } return candidates; } void fix_call_sites_private(const std::vector& scope, - const ConcurrentSet& privates) { + const std::unordered_set& privates) { walk::parallel::code(scope, [&](DexMethod* caller, IRCode& code) { - always_assert(code.editable_cfg_built()); - auto& cfg = code.cfg(); - for (const MethodItemEntry& mie : cfg::InstructionIterable(cfg)) { + for (const MethodItemEntry& mie : InstructionIterable(code)) { IRInstruction* insn = mie.insn; if (!insn->has_method()) continue; auto callee = resolve_method(insn->get_method(), opcode_to_search(insn), caller); // should be safe to read `privates` here because there are no writers - if (callee != nullptr && privates.count_unsafe(callee)) { + if (callee != nullptr && privates.count(callee)) { insn->set_method(callee); if (!is_static(callee)) { insn->set_opcode(OPCODE_INVOKE_DIRECT); @@ -164,7 +165,7 @@ void fix_call_sites_private(const std::vector& scope, }); } -void mark_methods_private(const ConcurrentSet& privates) { +void mark_methods_private(const std::unordered_set& privates) { // Compute an ordered representation of the methods. This matters, as // the dmethods and vmethods are not necessarily sorted, but add_method does // a best-effort of inserting in an ordered matter. diff --git a/opt/access-marking/AccessMarking.h b/opt/access-marking/AccessMarking.h index e8a2821d2c..ac9220369f 100644 --- a/opt/access-marking/AccessMarking.h +++ b/opt/access-marking/AccessMarking.h @@ -18,8 +18,8 @@ class AccessMarkingPass : public Pass { using namespace redex_properties::interactions; using namespace redex_properties::names; return { - {DexLimitsObeyed, Preserves}, - {NoResolvablePureRefs, Preserves}, + {DexLimitsObeyed, Preserves}, {HasSourceBlocks, Preserves}, + {NoResolvablePureRefs, Preserves}, {NoSpuriousGetClassCalls, Preserves}, {RenameClass, Preserves}, }; } @@ -43,6 +43,8 @@ class AccessMarkingPass : public Pass { "Mark every eligible method as private."); } + bool is_cfg_legacy() override { return true; } + void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; private: diff --git a/opt/add-secondary-dex/AddSecondaryDexPass.h b/opt/add-secondary-dex/AddSecondaryDexPass.h index 462be9cf60..db24965f9b 100644 --- a/opt/add-secondary-dex/AddSecondaryDexPass.h +++ b/opt/add-secondary-dex/AddSecondaryDexPass.h @@ -19,9 +19,11 @@ class AddSecondaryDexPass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoInitClassInstructions, Preserves}, {NoResolvablePureRefs, Preserves}, {NoUnreachableInstructions, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, {RenameClass, Preserves}, }; } diff --git a/opt/analyze-pure-method/PureMethods.cpp b/opt/analyze-pure-method/PureMethods.cpp index 98af91e296..9a1d0ad9ea 100644 --- a/opt/analyze-pure-method/PureMethods.cpp +++ b/opt/analyze-pure-method/PureMethods.cpp @@ -36,7 +36,7 @@ bool AnalyzePureMethodsPass::analyze_and_check_pure_method_helper( /* analyze_external_reads */ true) .build(); - return !side_effect_summary.has_side_effects(); + return side_effect_summary.is_pure(); } void AnalyzePureMethodsPass::run_pass(DexStoresVector& stores, diff --git a/opt/analyze-pure-method/PureMethods.h b/opt/analyze-pure-method/PureMethods.h index 8cf2b04275..8fdeb3cbbe 100644 --- a/opt/analyze-pure-method/PureMethods.h +++ b/opt/analyze-pure-method/PureMethods.h @@ -34,7 +34,9 @@ class AnalyzePureMethodsPass : public Pass { const override { using namespace redex_properties::interactions; using namespace redex_properties::names; - return {}; + return { + {HasSourceBlocks, Preserves}, + }; } void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; diff --git a/opt/annokill/AnnoKill.cpp b/opt/annokill/AnnoKill.cpp index 273bb531e4..c72df5be49 100644 --- a/opt/annokill/AnnoKill.cpp +++ b/opt/annokill/AnnoKill.cpp @@ -69,7 +69,7 @@ AnnoKill::AnnoKill( // Load annotations we know and want dead. for (auto const& anno_name : kill) { - DexType* anno = DexType::get_type(anno_name); + DexType* anno = DexType::get_type(anno_name.c_str()); TRACE(ANNO, 2, "Kill annotation type string %s", anno_name.c_str()); if (anno) { TRACE(ANNO, 2, "Kill anno: %s", SHOW(anno)); @@ -94,7 +94,7 @@ AnnoKill::AnnoKill( // Populate class hierarchy keep map auto ch = build_type_hierarchy(m_scope); for (const auto& it : class_hierarchy_keep_annos) { - auto* type = DexType::get_type(it.first); + auto* type = DexType::get_type(it.first.c_str()); auto* type_cls = type ? type_class(type) : nullptr; if (type_cls == nullptr) { continue; @@ -360,19 +360,20 @@ AnnoKill::AnnoSet AnnoKill::get_removable_annotation_instances() { void AnnoKill::count_annotation(const DexAnnotation* da, AnnoKillStats& stats) const { + auto inc_counter = [](auto&, auto& c, auto) { c++; }; if (da->system_visible()) { if (traceEnabled(ANNO, 3)) { - m_system_anno_map.fetch_add(da->type()->get_name()->str(), 1); + m_system_anno_map.update(da->type()->get_name()->str(), inc_counter); } stats.visibility_system_count++; } else if (da->runtime_visible()) { if (traceEnabled(ANNO, 3)) { - m_runtime_anno_map.fetch_add(da->type()->get_name()->str(), 1); + m_runtime_anno_map.update(da->type()->get_name()->str(), inc_counter); } stats.visibility_runtime_count++; } else if (da->build_visible()) { if (traceEnabled(ANNO, 3)) { - m_build_anno_map.fetch_add(da->type()->get_name()->str(), 1); + m_build_anno_map.update(da->type()->get_name()->str(), inc_counter); } stats.visibility_build_count++; } @@ -634,27 +635,21 @@ bool AnnoKill::kill_annotations() { if (traceEnabled(ANNO, 3)) { for (const auto& p : m_build_anno_map) { - TRACE(ANNO, - 3, - "Build anno: %zu, %s", - p.second.load(), - str_copy(p.first).c_str()); + TRACE( + ANNO, 3, "Build anno: %zu, %s", p.second, str_copy(p.first).c_str()); } for (const auto& p : m_runtime_anno_map) { TRACE(ANNO, 3, "Runtime anno: %zu, %s", - p.second.load(), + p.second, str_copy(p.first).c_str()); } for (const auto& p : m_system_anno_map) { - TRACE(ANNO, - 3, - "System anno: %zu, %s", - p.second.load(), - str_copy(p.first).c_str()); + TRACE( + ANNO, 3, "System anno: %zu, %s", p.second, str_copy(p.first).c_str()); } } diff --git a/opt/annokill/AnnoKill.h b/opt/annokill/AnnoKill.h index 2b10d52c1e..d41c3a2917 100644 --- a/opt/annokill/AnnoKill.h +++ b/opt/annokill/AnnoKill.h @@ -104,9 +104,9 @@ class AnnoKill { AnnoSet m_keep; AnnoKillStats m_stats; - mutable AtomicMap m_build_anno_map; - mutable AtomicMap m_runtime_anno_map; - mutable AtomicMap m_system_anno_map; + mutable ConcurrentMap m_build_anno_map; + mutable ConcurrentMap m_runtime_anno_map; + mutable ConcurrentMap m_system_anno_map; std::unordered_map> m_anno_class_hierarchy_keep; std::unordered_map> @@ -123,8 +123,9 @@ class AnnoKillPass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, - {InitialRenameClass, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } @@ -140,6 +141,8 @@ class AnnoKillPass : public Pass { bind("only_force_kill", false, m_only_force_kill); } + bool is_cfg_legacy() override { return true; } + void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; std::unique_ptr clone(const std::string& new_name) const override { diff --git a/opt/app_module_usage/AppModuleUsage.cpp b/opt/app_module_usage/AppModuleUsage.cpp index b4773faf5d..3ea5f03cdf 100644 --- a/opt/app_module_usage/AppModuleUsage.cpp +++ b/opt/app_module_usage/AppModuleUsage.cpp @@ -50,7 +50,7 @@ void write_violations_to_file(const app_module_usage::Violations& violations, void write_method_module_usages_to_file( const app_module_usage::MethodStoresReferenced& method_store_refs, - const InsertOnlyConcurrentMap& type_store_map, + const ConcurrentMap& type_store_map, const std::string& path) { TRACE(APP_MOD_USE, 4, "Outputting module usages at %s", path.c_str()); @@ -200,7 +200,7 @@ void AppModuleUsagePass::load_preexisting_violations(DexStoresVector& stores) { ifs.close(); } -app_module_usage::MethodStoresReferenced +ConcurrentMap AppModuleUsagePass::analyze_method_xstore_references(const Scope& scope) { auto get_type_ref_for_insn = [](IRInstruction* insn) -> DexType* { @@ -214,7 +214,8 @@ AppModuleUsagePass::analyze_method_xstore_references(const Scope& scope) { return nullptr; }; - app_module_usage::MethodStoresReferenced method_store_refs; + ConcurrentMap + method_store_refs; reflection::MetadataCache refl_metadata_cache; walk::parallel::code(scope, [&](DexMethod* method, IRCode& code) { @@ -291,9 +292,9 @@ AppModuleUsagePass::analyze_method_xstore_references(const Scope& scope) { return method_store_refs; } -InsertOnlyConcurrentMap +ConcurrentMap AppModuleUsagePass::analyze_field_xstore_references(const Scope& scope) { - InsertOnlyConcurrentMap ret; + ConcurrentMap ret; walk::parallel::fields(scope, [&](DexField* field) { auto field_store = m_type_store_map.at(field->get_class()); @@ -313,7 +314,7 @@ AppModuleUsagePass::analyze_field_xstore_references(const Scope& scope) { unsigned AppModuleUsagePass::gather_violations( const app_module_usage::MethodStoresReferenced& method_store_refs, - const InsertOnlyConcurrentMap& field_store_refs, + const ConcurrentMap& field_store_refs, app_module_usage::Violations& violations) const { int trace_level = m_crash_with_violations ? 0 : 1; diff --git a/opt/app_module_usage/AppModuleUsage.h b/opt/app_module_usage/AppModuleUsage.h index bf916d1729..8fc36ef56c 100644 --- a/opt/app_module_usage/AppModuleUsage.h +++ b/opt/app_module_usage/AppModuleUsage.h @@ -18,8 +18,7 @@ namespace app_module_usage { using StoresReferenced = std::unordered_map; -using MethodStoresReferenced = - InsertOnlyConcurrentMap; +using MethodStoresReferenced = ConcurrentMap; using Violations = std::map>; @@ -52,6 +51,8 @@ class AppModuleUsagePass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, {UltralightCodePatterns, Preserves}, }; } @@ -80,13 +81,13 @@ class AppModuleUsagePass : public Pass { app_module_usage::MethodStoresReferenced analyze_method_xstore_references( const Scope& scope); - InsertOnlyConcurrentMap analyze_field_xstore_references( + ConcurrentMap analyze_field_xstore_references( const Scope& scope); // Returns number of violations. unsigned gather_violations( const app_module_usage::MethodStoresReferenced& method_store_refs, - const InsertOnlyConcurrentMap& field_store_refs, + const ConcurrentMap& field_store_refs, app_module_usage::Violations& violations) const; // returns true if the given entrypoint name is allowed to use the given store @@ -102,7 +103,7 @@ class AppModuleUsagePass : public Pass { m_preexisting_violations; // To quickly look up wich DexStore ("module") a DexType is from - InsertOnlyConcurrentMap m_type_store_map; + ConcurrentMap m_type_store_map; bool m_output_module_use; bool m_crash_with_violations; diff --git a/opt/art-profile-writer/ArtProfileWriterPass.h b/opt/art-profile-writer/ArtProfileWriterPass.h index 266f233859..af99aeeafd 100644 --- a/opt/art-profile-writer/ArtProfileWriterPass.h +++ b/opt/art-profile-writer/ArtProfileWriterPass.h @@ -21,7 +21,7 @@ class ArtProfileWriterPass : public Pass { void bind_config() override; void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; - private: +private: struct PerfConfig { float appear100_threshold; float call_count_threshold; diff --git a/opt/branch-prefix-hoisting/BranchPrefixHoisting.cpp b/opt/branch-prefix-hoisting/BranchPrefixHoisting.cpp index 00a7e25921..fc29ee4c20 100644 --- a/opt/branch-prefix-hoisting/BranchPrefixHoisting.cpp +++ b/opt/branch-prefix-hoisting/BranchPrefixHoisting.cpp @@ -259,7 +259,7 @@ std::vector get_insns_to_hoist( // all conditions satisfied insns_to_hoist.push_back(*common_insn); for (size_t i = 0; i != block_iters.size(); ++i) { - redex_assert(block_iters[i] != CONSTP(succ_blocks[i])->end()); + redex_assert(block_iters[i] != succ_blocks[i]->end()); redex_assert(block_iters[i]->type == MFLOW_OPCODE); redex_assert(*block_iters[i]->insn == *common_insn); ++block_iters[i]; @@ -391,7 +391,7 @@ size_t hoist_insns_for_block( for (auto& p : succs) { auto* b = p.first; auto& it = p.second; - redex_assert(it != CONSTP(b)->end()); // Should have instructions. + redex_assert(it != b->end()); // Should have instructions. IRList::iterator next; for (; it != b->end(); it = next) { next = std::next(it); @@ -441,7 +441,7 @@ size_t hoist_insns_for_block( not_reached(); } } - redex_assert(it != CONSTP(b)->end()); + redex_assert(it != b->end()); } } @@ -465,7 +465,7 @@ size_t hoist_insns_for_block( for (auto& p : succs) { auto* b = p.first; auto& it = p.second; - redex_assert(it != CONSTP(b)->end()); + redex_assert(it != b->end()); redex_assert(it->type == MFLOW_OPCODE); redex_assert(*it->insn == insn); @@ -594,14 +594,14 @@ size_t process_hoisting_for_block( size_t BranchPrefixHoistingPass::process_code(IRCode* code, DexMethod* method, bool can_allocate_regs) { - auto& cfg = code->cfg(); - TRACE(BPH, 5, "%s", SHOW(cfg)); + cfg::ScopedCFG cfg(code); + TRACE(BPH, 5, "%s", SHOW(*cfg)); Lazy constant_uses([&] { return std::make_unique( - cfg, method, + *cfg, method, /* force_type_inference */ true); }); - size_t ret = process_cfg(cfg, constant_uses, can_allocate_regs); + size_t ret = process_cfg(*cfg, constant_uses, can_allocate_regs); return ret; } @@ -639,7 +639,7 @@ void BranchPrefixHoistingPass::run_pass(DexStoresVector& stores, int total_insns_hoisted = walk::parallel::methods( scope, [can_allocate_regs](DexMethod* method) -> int { const auto code = method->get_code(); - if (!code || method->rstate.no_optimizations()) { + if (!code) { return 0; } TraceContext context{method}; diff --git a/opt/branch-prefix-hoisting/BranchPrefixHoisting.h b/opt/branch-prefix-hoisting/BranchPrefixHoisting.h index 4b92aa9b4c..ef80b2f3dd 100644 --- a/opt/branch-prefix-hoisting/BranchPrefixHoisting.h +++ b/opt/branch-prefix-hoisting/BranchPrefixHoisting.h @@ -33,13 +33,17 @@ class BranchPrefixHoistingPass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoInitClassInstructions, Preserves}, {NoUnreachableInstructions, Preserves}, {NoResolvablePureRefs, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, {RenameClass, Preserves}, }; } + bool is_cfg_legacy() override { return true; } + void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; static size_t process_code(IRCode*, diff --git a/opt/builder_pattern/RemoveBuilderPattern.cpp b/opt/builder_pattern/RemoveBuilderPattern.cpp index c692b5d950..2c1dc94a95 100644 --- a/opt/builder_pattern/RemoveBuilderPattern.cpp +++ b/opt/builder_pattern/RemoveBuilderPattern.cpp @@ -189,7 +189,7 @@ class RemoveClasses { std::vector methods; walk::parallel::methods(m_scope, [&](DexMethod* method) { - if (!method || !method->get_code() || method->rstate.no_optimizations()) { + if (!method || !method->get_code()) { return; } diff --git a/opt/builder_pattern/RemoveBuilderPattern.h b/opt/builder_pattern/RemoveBuilderPattern.h index 5b6392432d..4d9bd177ed 100644 --- a/opt/builder_pattern/RemoveBuilderPattern.h +++ b/opt/builder_pattern/RemoveBuilderPattern.h @@ -22,8 +22,8 @@ class RemoveBuilderPatternPass : public Pass { using namespace redex_properties::interactions; using namespace redex_properties::names; return { + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, - {SpuriousGetClassCallsInterned, RequiresAndPreserves}, }; } diff --git a/opt/check-recursion/CheckRecursion.cpp b/opt/check-recursion/CheckRecursion.cpp index 0086a204b0..982328deee 100644 --- a/opt/check-recursion/CheckRecursion.cpp +++ b/opt/check-recursion/CheckRecursion.cpp @@ -123,6 +123,7 @@ void CheckRecursionPass::run_pass(DexStoresVector& stores, scope, [this, &num_methods_detected, &num_methods_patched](DexMethod* method, IRCode& code) { + code.build_cfg(/* editable */ true); switch (do_check_recursion(method, code, bad_recursion_count)) { case CheckRecursionResult::SafeRecursion: ++num_methods_detected; @@ -136,6 +137,7 @@ void CheckRecursionPass::run_pass(DexStoresVector& stores, case CheckRecursionResult::NotFound: break; } + code.clear_cfg(); }); mgr.incr_metric(METRIC_METHODS_DETECTED, num_methods_detected); diff --git a/opt/check-recursion/CheckRecursion.h b/opt/check-recursion/CheckRecursion.h index 5af185040d..58487ca297 100644 --- a/opt/check-recursion/CheckRecursion.h +++ b/opt/check-recursion/CheckRecursion.h @@ -19,8 +19,9 @@ class CheckRecursionPass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, - {InitialRenameClass, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } @@ -30,5 +31,7 @@ class CheckRecursionPass : public Pass { bind("bad_recursion_count", 4, bad_recursion_count); } + bool is_cfg_legacy() override { return true; } + void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; }; diff --git a/opt/check_breadcrumbs/CheckBreadcrumbs.cpp b/opt/check_breadcrumbs/CheckBreadcrumbs.cpp index 3f1876ecd5..37785ff9bd 100644 --- a/opt/check_breadcrumbs/CheckBreadcrumbs.cpp +++ b/opt/check_breadcrumbs/CheckBreadcrumbs.cpp @@ -597,8 +597,7 @@ bool Breadcrumbs::has_illegal_access(const DexMethod* input_method) { if (input_method->get_code() == nullptr) { return false; } - auto& cfg = (const_cast(input_method))->get_code()->cfg(); - for (const auto& mie : cfg::InstructionIterable(cfg)) { + for (const auto& mie : InstructionIterable(input_method->get_code())) { auto* insn = mie.insn; if (insn->has_field()) { auto res_field = resolve_field(insn->get_field()); diff --git a/opt/check_breadcrumbs/CheckBreadcrumbs.h b/opt/check_breadcrumbs/CheckBreadcrumbs.h index fa8380783f..f9aa9f348e 100644 --- a/opt/check_breadcrumbs/CheckBreadcrumbs.h +++ b/opt/check_breadcrumbs/CheckBreadcrumbs.h @@ -51,6 +51,8 @@ class CheckBreadcrumbsPass : public Pass { trait(Traits::Pass::unique, true); } + bool is_cfg_legacy() override { return true; } + void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; private: diff --git a/opt/class-merging/AnonymousClassMergingPass.h b/opt/class-merging/AnonymousClassMergingPass.h index 4c2b4ab222..7872e372cf 100644 --- a/opt/class-merging/AnonymousClassMergingPass.h +++ b/opt/class-merging/AnonymousClassMergingPass.h @@ -7,11 +7,12 @@ #pragma once -#include "Model.h" #include "Pass.h" namespace class_merging { +struct ModelSpec; + class AnonymousClassMergingPass : public Pass { public: AnonymousClassMergingPass() : Pass("AnonymousClassMergingPass") {} @@ -21,7 +22,9 @@ class AnonymousClassMergingPass : public Pass { using namespace redex_properties::interactions; using namespace redex_properties::names; return { + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } diff --git a/opt/class-merging/ClassMergingPass.h b/opt/class-merging/ClassMergingPass.h index 6edd8399a6..c9897c21b5 100644 --- a/opt/class-merging/ClassMergingPass.h +++ b/opt/class-merging/ClassMergingPass.h @@ -27,7 +27,9 @@ class ClassMergingPass : public Pass { using namespace redex_properties::interactions; using namespace redex_properties::names; return { + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } diff --git a/opt/class-merging/IntraDexClassMergingPass.h b/opt/class-merging/IntraDexClassMergingPass.h index 7e89a1aa5e..431d6a8eac 100644 --- a/opt/class-merging/IntraDexClassMergingPass.h +++ b/opt/class-merging/IntraDexClassMergingPass.h @@ -14,11 +14,12 @@ #pragma once -#include "Model.h" #include "Pass.h" namespace class_merging { +struct ModelSpec; + class IntraDexClassMergingPass : public Pass { public: IntraDexClassMergingPass() : Pass("IntraDexClassMergingPass") {} @@ -29,8 +30,9 @@ class IntraDexClassMergingPass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, - {InitialRenameClass, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } diff --git a/opt/class-splitting/ClassSplittingPass.h b/opt/class-splitting/ClassSplittingPass.h index 4916c5712e..251e6d49c8 100644 --- a/opt/class-splitting/ClassSplittingPass.h +++ b/opt/class-splitting/ClassSplittingPass.h @@ -23,6 +23,7 @@ class ClassSplittingPass : public Pass { return { {HasSourceBlocks, RequiresAndEstablishes}, {NoResolvablePureRefs, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } diff --git a/opt/const-class-branches/TransformConstClassBranches.cpp b/opt/const-class-branches/TransformConstClassBranches.cpp index 0d82c526e3..ce62e82ebc 100644 --- a/opt/const-class-branches/TransformConstClassBranches.cpp +++ b/opt/const-class-branches/TransformConstClassBranches.cpp @@ -30,7 +30,6 @@ constexpr const char* METRIC_CONST_CLASS_INSTRUCTIONS_REMOVED = "num_const_class_instructions_removed"; constexpr const char* METRIC_TOTAL_STRING_SIZE = "total_string_size"; -// Holder for the pass's configuration options. struct PassState { DexMethodRef* lookup_method; bool consider_external_classes; @@ -39,21 +38,20 @@ struct PassState { std::mutex& transforms_mutex; }; -// Denotes a branch and successor blocks within a method that can be -// successfully represented/transformed. -struct BranchTransform { - cfg::Block* block; +// Denotes a branch within a method that can be successfully +// represented/transformed. Used to gather up possibilities and execute them in +// order of biggest gain. This pass may be configured to run late in the pass +// list, and thus should be mindful for number of fields to create. +struct PendingTransform { + DexClass* cls; + DexMethod* method; + cfg::Block* last_prologue_block; IRInstruction* insn; + size_t insn_idx; reg_t determining_reg; - std::unique_ptr switch_equiv; -}; - -// Denotes a method that will have one or many transforms. -struct MethodTransform { - DexMethod* method; std::unique_ptr code_copy; std::unique_ptr scoped_cfg; - std::vector transforms; + std::unique_ptr switch_equiv; }; struct Stats { @@ -68,275 +66,261 @@ struct Stats { } }; -size_t num_const_class_opcodes(const cfg::ControlFlowGraph* cfg) { - size_t result{0}; - for (auto& mie : InstructionIterable(*cfg)) { - if (mie.insn->opcode() == OPCODE_CONST_CLASS) { - result++; +bool should_consider_method(DexMethod* method) { + auto proto = method->get_proto(); + auto args = proto->get_args(); + for (size_t i = 0; i < args->size(); i++) { + if (args->at(i) == type::java_lang_Class()) { + return method->get_code() != nullptr; } } - return result; -} - -// This pass cares about comparing objects, so only eq, ne are relevant at the -// end of a block. -bool ends_in_if_statment(const cfg::Block* b) { - auto it = b->get_last_insn(); - if (it == b->end()) { - return false; - } - auto op = it->insn->opcode(); - return opcode::is_if_eq(op) || opcode::is_if_ne(op); + return false; } -// Meant to be a quick guess, to skip some of the preliminary work in deciding -// for real if the method should be operated upon if nothing looks relevant. -bool should_consider_method(const PassState& pass_state, DexMethod* method) { - if (method->rstate.no_optimizations()) { - return false; - } - auto code = method->get_code(); - if (code == nullptr) { - return false; - } - auto& cfg = code->cfg(); - bool found_branch = false; - for (const auto* b : cfg.blocks()) { - // Note: SwitchEquivFinder assumes the non-leaf blocks (the blocks that - // perform equals checks) have no throw edges. Avoid considering such a - // method early on. - if (b->is_catch()) { - return false; - } - if (ends_in_if_statment(b)) { - found_branch = true; - break; +// Like ControlFlowGraph::find_insn, return an iterator to the instruction but +// also spit out the index of the instruction (used for compares if needed). +cfg::InstructionIterator find_insn_and_idx(cfg::ControlFlowGraph& cfg, + IRInstruction* insn, + size_t* insn_idx) { + size_t idx = 0; + auto iterable = InstructionIterable(cfg); + for (auto it = iterable.begin(); it != iterable.end(); ++it) { + if (it->insn == insn) { + *insn_idx = idx; + return it; } + idx++; } - return found_branch && num_const_class_opcodes(&cfg) >= pass_state.min_cases; + not_reached(); } -// True if the finder is successful, has a default block and does not have some -// edge cases we don't wanna deal with right now. -bool finder_results_are_supported(SwitchEquivFinder* finder) { - return finder->success() && - finder->are_keys_uniform(SwitchEquivFinder::KeyKind::CLASS) && - finder->extra_loads().empty() && finder->default_case(); -} - -// Rather than looping over the cfg blocks, explicitly start from the entry -// block and bfs through the graph. Makes sure that even if the cfg got -// manipulated such that entry block is not the smallest id, we will start -// looking for eligible transforms roughly from that point. -void order_blocks(const cfg::ControlFlowGraph& cfg, - std::vector* out) { - std::stack to_visit; - std::unordered_set visited; - to_visit.push(cfg.entry_block()); - while (!to_visit.empty()) { - cfg::Block* b = to_visit.top(); - to_visit.pop(); - - if (visited.count(b->id()) > 0) { - continue; - } - visited.emplace(b->id()); - out->emplace_back(b); - for (cfg::Edge* e : b->succs()) { - to_visit.push(e->target()); +size_t num_const_class_opcodes(const cfg::ControlFlowGraph* cfg) { + size_t result{0}; + for (auto& mie : InstructionIterable(*cfg)) { + if (mie.insn->opcode() == OPCODE_CONST_CLASS) { + result++; } } + return result; } namespace cp = constant_propagation; void gather_possible_transformations( const PassState& pass_state, + DexClass* cls, DexMethod* method, - std::vector* method_transforms) { + std::vector* pending_transforms) { // First step is to operate on a simplified copy of the code. If the transform // is applicable, this copy will take effect. auto code_copy = std::make_unique(*method->get_code()); SwitchEquivEditor::simplify_moves(code_copy.get()); auto scoped_cfg = std::make_unique(code_copy.get()); auto& cfg = **scoped_cfg; - - std::vector transforms; + // Many checks to see if this method conforms to the types of patterns that + // are able to be easily represented without losing information. At the time + // of writing this just looks for 1 branching point in the beginning of the + // method, but this should get addressed to be applicable anywhere in the + // method. + std::vector prologue_blocks; + if (!gather_linear_prologue_blocks(&cfg, &prologue_blocks)) { + return; + } + auto last_prologue_block = prologue_blocks.back(); + auto last_prologue_insn = last_prologue_block->get_last_insn(); + if (last_prologue_insn->insn->opcode() == OPCODE_SWITCH) { + // Not the expected form + return; + } TRACE(CCB, 3, "Checking for const-class branching in %s", SHOW(method)); auto fixpoint = std::make_shared( cfg, SwitchEquivFinder::Analyzer()); fixpoint->run(ConstantEnvironment()); + reg_t determining_reg; + if (!find_determining_reg(*fixpoint, prologue_blocks.back(), + &determining_reg)) { + TRACE(CCB, 2, "Cannot find determining_reg; bailing."); + return; + } + TRACE(CCB, 2, "determining_reg is %d", determining_reg); + + size_t insn_idx{0}; + auto finder = std::make_unique( + &cfg, find_insn_and_idx(cfg, last_prologue_insn->insn, &insn_idx), + determining_reg, SwitchEquivFinder::NO_LEAF_DUPLICATION, fixpoint, + SwitchEquivFinder::EXECUTION_ORDER); + if (!finder->success() || + !finder->are_keys_uniform(SwitchEquivFinder::KeyKind::CLASS)) { + TRACE(CCB, 2, "SwitchEquivFinder failed!"); + return; + } + TRACE(CCB, 2, "SwitchEquivFinder succeeded for branch at: %s", + SHOW(last_prologue_insn->insn)); + if (!finder->extra_loads().empty()) { + TRACE(CCB, 2, + "Not supporting extra const-class loads during switch; bailing."); + return; + } - std::vector blocks; - order_blocks(cfg, &blocks); - std::unordered_set blocks_considered; - for (const auto& b : blocks) { - if (blocks_considered.count(b) > 0) { - continue; - } - blocks_considered.emplace(b); - reg_t determining_reg; - if (ends_in_if_statment(b) && - find_determining_reg(*fixpoint, b, &determining_reg)) { - // Keep going, maybe this block is a useful starting point. - TRACE(CCB, 2, "determining_reg is %d for B%zu", determining_reg, b->id()); - auto last_insn = b->get_last_insn()->insn; - auto root_branch = cfg.find_insn(last_insn); - auto finder = std::make_unique( - &cfg, root_branch, determining_reg, - SwitchEquivFinder::NO_LEAF_DUPLICATION, fixpoint, - SwitchEquivFinder::EXECUTION_ORDER); - if (finder_results_are_supported(finder.get())) { - TRACE(CCB, 2, "SwitchEquivFinder succeeded on B%zu for branch at: %s", - b->id(), SHOW(last_insn)); - auto visited = finder->visited_blocks(); - std::copy(visited.begin(), visited.end(), - std::inserter(blocks_considered, blocks_considered.end())); - const auto& key_to_case = finder->key_to_case(); - size_t relevant_case_count{0}; - for (auto&& [key, leaf] : key_to_case) { - if (!SwitchEquivFinder::is_default_case(key)) { - auto dtype = boost::get(key); - auto case_class = type_class(dtype); - if (pass_state.consider_external_classes || - (case_class != nullptr && !case_class->is_external())) { - relevant_case_count++; - } - } - } - if (relevant_case_count > pass_state.max_cases || - relevant_case_count < pass_state.min_cases) { - TRACE(CCB, 2, "Not considering branch due to number of cases."); - continue; - } - // Part of this method should conform to expectations, note this. - BranchTransform transform{b, last_insn, determining_reg, - std::move(finder)}; - transforms.emplace_back(std::move(transform)); + const auto& key_to_case = finder->key_to_case(); + size_t relevant_case_count{0}; + for (auto&& [key, block] : key_to_case) { + if (!SwitchEquivFinder::is_default_case(key)) { + auto dtype = boost::get(key); + auto case_class = type_class(dtype); + if (pass_state.consider_external_classes || + (case_class != nullptr && !case_class->is_external())) { + relevant_case_count++; } } } - if (!transforms.empty()) { - std::lock_guard lock(pass_state.transforms_mutex); - MethodTransform mt{method, std::move(code_copy), std::move(scoped_cfg), - std::move(transforms)}; - method_transforms->emplace_back(std::move(mt)); + if (finder->default_case() == boost::none) { + TRACE(CCB, 2, "Default block not found; bailing."); + return; + } + if (relevant_case_count > pass_state.max_cases || + relevant_case_count < pass_state.min_cases) { + TRACE(CCB, 2, "Not operating on method due to size."); + return; } + // Method should conform to expectations! + std::lock_guard lock(pass_state.transforms_mutex); + PendingTransform t{cls, + method, + last_prologue_block, + last_prologue_insn->insn, + insn_idx, + determining_reg, + std::move(code_copy), + std::move(scoped_cfg), + std::move(finder)}; + pending_transforms->emplace_back(std::move(t)); } -Stats apply_transform(const PassState& pass_state, MethodTransform& mt) { +Stats apply_transform(const PassState& pass_state, + PendingTransform& transform) { Stats result; - auto method = mt.method; - auto& cfg = **mt.scoped_cfg; - auto before_const_class_count = num_const_class_opcodes(&cfg); - TRACE(CCB, 3, - "Processing const-class branching in %s (transform size = %zu) %s", - SHOW(method), mt.transforms.size(), SHOW(cfg)); - - for (auto& transform : mt.transforms) { - // Determine stable order of the types that are being switched on. - std::set ordered_types; - const auto& key_to_case = transform.switch_equiv->key_to_case(); - cfg::Block* default_case{nullptr}; - for (auto&& [key, block] : key_to_case) { - if (!SwitchEquivFinder::is_default_case(key)) { - auto dtype = boost::get(key); - ordered_types.emplace(dtype); - } else { - TRACE(CCB, 3, "DEFAULT -> B%zu\n%s", block->id(), SHOW(block)); - default_case = block; - } - } - // Create ordinals for each type being switched on, reserving zero to denote - // an explicit default case. - std::map string_tree_items; - std::vector> new_edges; - constexpr int16_t STRING_TREE_NO_ENTRY = 0; - int16_t counter = STRING_TREE_NO_ENTRY + 1; - for (const auto& type : ordered_types) { - auto string_name = java_names::internal_to_external(type->str_copy()); - int16_t ordinal = counter++; - string_tree_items.emplace(string_name, ordinal); - auto block = key_to_case.at(type); - new_edges.emplace_back(ordinal, block); - TRACE(CCB, 3, "%s (%s) -> B%zu\n%s", SHOW(type), string_name.c_str(), - block->id(), SHOW(block)); + auto method = transform.method; + auto& cfg = **transform.scoped_cfg; + TRACE(CCB, 3, "Transforming const-class branching in %s %s", SHOW(method), + SHOW(cfg)); + auto finder = transform.switch_equiv.get(); + + // Determine stable order of the types that are being switched on. + std::set ordered_types; + const auto& key_to_case = finder->key_to_case(); + cfg::Block* default_case{nullptr}; + for (auto&& [key, block] : key_to_case) { + if (!SwitchEquivFinder::is_default_case(key)) { + auto dtype = boost::get(key); + ordered_types.emplace(dtype); + } else { + TRACE(CCB, 3, "DEFAULT -> B%zu\n%s", block->id(), SHOW(block)); + default_case = block; } + } + // Create ordinals for each type being switched on, reserving zero to denote + // an explicit default case. + auto before_const_class_count = num_const_class_opcodes(&cfg); + std::map string_tree_items; + std::vector> new_edges; + constexpr int16_t STRING_TREE_NO_ENTRY = 0; + int16_t counter = STRING_TREE_NO_ENTRY + 1; + for (const auto& type : ordered_types) { + auto string_name = java_names::internal_to_external(type->str_copy()); + int16_t ordinal = counter++; + string_tree_items.emplace(string_name, ordinal); + auto block = key_to_case.at(type); + new_edges.emplace_back(ordinal, block); + TRACE(CCB, 3, "%s (%s) -> B%zu\n%s", SHOW(type), string_name.c_str(), + block->id(), SHOW(block)); + } - auto encoded_str = - StringTreeMap::encode_string_tree_map(string_tree_items); - result.string_tree_size = encoded_str.size(); - auto encoded_dex_str = DexString::make_string(encoded_str); - - // Fiddle with the block's last instruction and install an actual switch - TRACE(CCB, 2, "Removing B%zu's last instruction: %s", transform.block->id(), - SHOW(transform.insn)); - - std::vector replacements; - auto encoded_str_reg = cfg.allocate_temp(); - auto const_string_insn = - (new IRInstruction(OPCODE_CONST_STRING))->set_string(encoded_dex_str); - replacements.push_back(const_string_insn); - auto move_string_insn = - (new IRInstruction(IOPCODE_MOVE_RESULT_PSEUDO_OBJECT)) - ->set_dest(encoded_str_reg); - replacements.push_back(move_string_insn); - - auto default_value_reg = cfg.allocate_temp(); - auto default_value_const = new IRInstruction(OPCODE_CONST); - default_value_const->set_literal(STRING_TREE_NO_ENTRY); - default_value_const->set_dest(default_value_reg); - replacements.push_back(default_value_const); - - auto invoke_string_tree = new IRInstruction(OPCODE_INVOKE_STATIC); - invoke_string_tree->set_method(pass_state.lookup_method); - invoke_string_tree->set_srcs_size(3); - invoke_string_tree->set_src(0, transform.determining_reg); - invoke_string_tree->set_src(1, encoded_str_reg); - invoke_string_tree->set_src(2, default_value_reg); - replacements.push_back(invoke_string_tree); - - // Just reuse a reg we don't need anymore - auto switch_result_reg = default_value_reg; - auto move_lookup_result = new IRInstruction(OPCODE_MOVE_RESULT); - move_lookup_result->set_dest(switch_result_reg); - replacements.push_back(move_lookup_result); - - auto new_switch = new IRInstruction(OPCODE_SWITCH); - new_switch->set_src(0, switch_result_reg); - // Note: it seems instruction "new_switch" gets appended via create_branch; - // no need to push to replacements - - cfg.replace_insns(cfg.find_insn(transform.insn), replacements); - // We are explicitly covering the default block via the default return value - // from the string tree. Not needed here. - cfg.create_branch(transform.block, new_switch, nullptr, new_edges); - - // Reset successor of last prologue block to implement the default case. - for (auto& edge : transform.block->succs()) { - if (edge->type() == cfg::EDGE_GOTO) { - cfg.set_edge_target(edge, default_case); - } + // Install a new static string field for the encoded types and their ordinals. + auto encoded_str = + StringTreeMap::encode_string_tree_map(string_tree_items); + result.string_tree_size = encoded_str.size(); + auto encoded_dex_str = DexString::make_string(encoded_str); + + // Fiddle with the prologue block and install an actual switch + TRACE(CCB, 2, "Removing last prologue instruction: %s", SHOW(transform.insn)); + + std::vector replacements; + auto encoded_str_reg = cfg.allocate_temp(); + auto const_string_insn = + (new IRInstruction(OPCODE_CONST_STRING))->set_string(encoded_dex_str); + replacements.push_back(const_string_insn); + auto move_string_insn = (new IRInstruction(IOPCODE_MOVE_RESULT_PSEUDO_OBJECT)) + ->set_dest(encoded_str_reg); + replacements.push_back(move_string_insn); + + auto default_value_reg = cfg.allocate_temp(); + auto default_value_const = new IRInstruction(OPCODE_CONST); + default_value_const->set_literal(STRING_TREE_NO_ENTRY); + default_value_const->set_dest(default_value_reg); + replacements.push_back(default_value_const); + + auto invoke_string_tree = new IRInstruction(OPCODE_INVOKE_STATIC); + invoke_string_tree->set_method(pass_state.lookup_method); + invoke_string_tree->set_srcs_size(3); + invoke_string_tree->set_src(0, transform.determining_reg); + invoke_string_tree->set_src(1, encoded_str_reg); + invoke_string_tree->set_src(2, default_value_reg); + replacements.push_back(invoke_string_tree); + + // Just reuse a reg we don't need anymore + auto switch_result_reg = default_value_reg; + auto move_lookup_result = new IRInstruction(OPCODE_MOVE_RESULT); + move_lookup_result->set_dest(switch_result_reg); + replacements.push_back(move_lookup_result); + + auto new_switch = new IRInstruction(OPCODE_SWITCH); + new_switch->set_src(0, switch_result_reg); + // Note: it seems instruction "new_switch" gets appended via create_branch; no + // need to push to replacements + + cfg.replace_insns(cfg.find_insn(transform.insn), replacements); + // We are explicitly covering the default block via the default return value + // from the string tree. Not needed here. + cfg.create_branch(transform.last_prologue_block, new_switch, nullptr, + new_edges); + + // Reset successor of last prologue block to implement the default case. + for (auto& edge : transform.last_prologue_block->succs()) { + if (edge->type() == cfg::EDGE_GOTO) { + cfg.set_edge_target(edge, default_case); } } - // Last step is to prune leaf blocks which are now unreachable. Do this - // before computing metrics (so we know if this pass is doing anything - // useful) but be sure to not dereference any Block ptrs from here on out! + + // Last step is to prune leaf blocks which are now unreachable. Do this before + // computing metrics (so we know if this pass is doing anything useful) but + // be sure to not dereference any Block ptrs from here on out! cfg.remove_unreachable_blocks(); TRACE(CCB, 3, "POST EDIT %s", SHOW(cfg)); result.methods_transformed = 1; - // Metric is not entirely accurate as we don't do dce on the first block that - // starts the if chain (eehhh close enough). + // Metric is not entirely accurate as we don't do dce on the prologue block + // (eehhh close enough). result.const_class_instructions_removed = before_const_class_count - num_const_class_opcodes(&cfg); always_assert(result.const_class_instructions_removed >= 0); // Make the copy take effect. - method->set_code(std::move(mt.code_copy)); + transform.method->set_code(std::move(transform.code_copy)); return result; } +class TransformConstClassBranchesInterDexPlugin + : public interdex::InterDexPassPlugin { + public: + explicit TransformConstClassBranchesInterDexPlugin() {} + + ReserveRefsInfo reserve_refs() override { + return ReserveRefsInfo(/* frefs */ 0, + /* trefs */ 0, + /* mrefs */ 2); + } +}; } // namespace void TransformConstClassBranchesPass::bind_config() { @@ -348,24 +332,23 @@ void TransformConstClassBranchesPass::bind_config() { bind("max_cases", 2000, m_max_cases); bind("string_tree_lookup_method", "", m_string_tree_lookup_method); trait(Traits::Pass::unique, true); -} -void TransformConstClassBranchesPass::eval_pass(DexStoresVector&, - ConfigFiles&, - PassManager& mgr) { - m_reserved_refs_handle = mgr.reserve_refs(name(), - ReserveRefsInfo(/* frefs */ 0, - /* trefs */ 0, - /* mrefs */ 1)); + after_configuration([] { + interdex::InterDexRegistry* registry = + static_cast( + PluginRegistry::get().pass_registry(interdex::INTERDEX_PASS_NAME)); + std::function fn = + []() -> interdex::InterDexPassPlugin* { + return new TransformConstClassBranchesInterDexPlugin(); + }; + registry->register_plugin("TRANSFORM_CONST_CLASS_BRANCHES_PLUGIN", + std::move(fn)); + }); } void TransformConstClassBranchesPass::run_pass(DexStoresVector& stores, ConfigFiles& /* unused */, PassManager& mgr) { - always_assert(m_reserved_refs_handle); - mgr.release_reserved_refs(*m_reserved_refs_handle); - m_reserved_refs_handle = std::nullopt; - auto scope = build_class_scope(stores); if (m_string_tree_lookup_method.empty()) { TRACE(CCB, 1, "Pass not configured; returning."); @@ -378,19 +361,20 @@ void TransformConstClassBranchesPass::run_pass(DexStoresVector& stores, return; } - std::vector method_transforms; + std::vector transforms; std::mutex transforms_mutex; PassState pass_state{string_tree_lookup_method, m_consider_external_classes, m_min_cases, m_max_cases, transforms_mutex}; walk::parallel::methods(scope, [&](DexMethod* method) { - if (should_consider_method(pass_state, method)) { - gather_possible_transformations(pass_state, method, &method_transforms); + if (should_consider_method(method)) { + gather_possible_transformations( + pass_state, type_class(method->get_class()), method, &transforms); } }); Stats stats; - for (auto& mt : method_transforms) { - stats += apply_transform(pass_state, mt); + for (auto& transform : transforms) { + stats += apply_transform(pass_state, transform); } mgr.incr_metric(METRIC_METHODS_TRANSFORMED, stats.methods_transformed); mgr.incr_metric(METRIC_CONST_CLASS_INSTRUCTIONS_REMOVED, diff --git a/opt/const-class-branches/TransformConstClassBranches.h b/opt/const-class-branches/TransformConstClassBranches.h index 15b4fb0425..d48f9bb396 100644 --- a/opt/const-class-branches/TransformConstClassBranches.h +++ b/opt/const-class-branches/TransformConstClassBranches.h @@ -9,7 +9,6 @@ #include "InterDexPass.h" #include "Pass.h" -#include "PassManager.h" #include "PluginRegistry.h" class TransformConstClassBranchesPass : public Pass { @@ -22,14 +21,12 @@ class TransformConstClassBranchesPass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, {RenameClass, Preserves}, }; } void bind_config() override; - - void eval_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; - void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; private: @@ -42,5 +39,4 @@ class TransformConstClassBranchesPass : public Pass { // StringTree returning the payload or "notFound" if o is not in the given // data structure. std::string m_string_tree_lookup_method; - std::optional m_reserved_refs_handle; }; diff --git a/opt/constant-propagation/ConstantPropagationPass.h b/opt/constant-propagation/ConstantPropagationPass.h index 57fcaedad4..1099d86b99 100644 --- a/opt/constant-propagation/ConstantPropagationPass.h +++ b/opt/constant-propagation/ConstantPropagationPass.h @@ -21,6 +21,7 @@ class ConstantPropagationPass : public Pass { using namespace redex_properties::interactions; using namespace redex_properties::names; return { + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, }; } diff --git a/opt/constant-propagation/IPConstantPropagation.h b/opt/constant-propagation/IPConstantPropagation.h index 309ac9e07c..820efd9ebc 100644 --- a/opt/constant-propagation/IPConstantPropagation.h +++ b/opt/constant-propagation/IPConstantPropagation.h @@ -43,8 +43,9 @@ class PassImpl : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, - {InitialRenameClass, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } diff --git a/opt/copy-propagation/CopyPropagationPass.h b/opt/copy-propagation/CopyPropagationPass.h index dc303c6b19..5fd44e32a2 100644 --- a/opt/copy-propagation/CopyPropagationPass.h +++ b/opt/copy-propagation/CopyPropagationPass.h @@ -20,14 +20,16 @@ class CopyPropagationPass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoInitClassInstructions, Preserves}, {NoUnreachableInstructions, Preserves}, {NoResolvablePureRefs, Preserves}, - {InitialRenameClass, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, {RenameClass, Preserves}, }; } + bool is_cfg_legacy() override { return true; } void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; void bind_config() override { diff --git a/opt/cse/CommonSubexpressionEliminationPass.cpp b/opt/cse/CommonSubexpressionEliminationPass.cpp index 5d7b5b2094..9b34c6051f 100644 --- a/opt/cse/CommonSubexpressionEliminationPass.cpp +++ b/opt/cse/CommonSubexpressionEliminationPass.cpp @@ -54,6 +54,10 @@ void CommonSubexpressionEliminationPass::run_pass(DexStoresVector& stores, init_classes::InitClassesWithSideEffects init_classes_with_side_effects( scope, conf.create_init_class_insns()); + walk::parallel::code(scope, [&](DexMethod*, IRCode& code) { + code.build_cfg(/* editable */ true); + }); + auto pure_methods = /* Android framework */ get_pure_methods(); auto configured_pure_methods = conf.get_pure_methods(); pure_methods.insert(configured_pure_methods.begin(), @@ -85,6 +89,7 @@ void CommonSubexpressionEliminationPass::run_pass(DexStoresVector& stores, } if (method->rstate.no_optimizations()) { + code->clear_cfg(); return Stats(); } @@ -101,6 +106,7 @@ void CommonSubexpressionEliminationPass::run_pass(DexStoresVector& stores, stats += cse.get_stats(); if (!any_changes) { + code->clear_cfg(); return stats; } diff --git a/opt/cse/CommonSubexpressionEliminationPass.h b/opt/cse/CommonSubexpressionEliminationPass.h index 8743a1d2d1..05248e6bea 100644 --- a/opt/cse/CommonSubexpressionEliminationPass.h +++ b/opt/cse/CommonSubexpressionEliminationPass.h @@ -21,12 +21,16 @@ class CommonSubexpressionEliminationPass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, - {InitialRenameClass, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } void bind_config() override; + + bool is_cfg_legacy() override { return true; } + void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; private: diff --git a/opt/dedup-strings/DedupStrings.cpp b/opt/dedup-strings/DedupStrings.cpp index 61cd2ad779..cfffddc88c 100644 --- a/opt/dedup-strings/DedupStrings.cpp +++ b/opt/dedup-strings/DedupStrings.cpp @@ -89,8 +89,7 @@ bool is_hot(cfg::Block* b) { // initialization code paths, or often. bool treat_all_blocks_as_hot(size_t dexnr, DexMethod* method) { return dexnr == 0 || method::is_clinit(method) || - type_class(method->get_class())->rstate.outlined() || - method->rstate.no_optimizations(); + type_class(method->get_class())->rstate.outlined(); } } // namespace @@ -721,6 +720,24 @@ void DedupStrings::rewrite_const_string_instructions( }); } +// In each dex, we might introduce as many new method refs and type refs as we +// might add factory methods. This makes sure that the inter-dex pass keeps +// space for that many method refs and type refs. +class DedupStringsInterDexPlugin : public interdex::InterDexPassPlugin { + public: + explicit DedupStringsInterDexPlugin(size_t max_factory_methods) + : m_max_factory_methods(max_factory_methods) {} + + ReserveRefsInfo reserve_refs() override { + return ReserveRefsInfo(/* frefs */ 0, + /* trefs */ m_max_factory_methods, + /* mrefs */ m_max_factory_methods); + } + + private: + size_t m_max_factory_methods; +}; + void DedupStringsPass::bind_config() { // The dedup-strings transformation introduces new method refs to refer // to factory methods. Factory methods are currently only placed into @@ -745,32 +762,23 @@ void DedupStringsPass::bind_config() { trait(Traits::Pass::unique, true); after_configuration([this, perf_mode_str = std::move(perf_mode_str)] { + always_assert(m_max_factory_methods > 0); + interdex::InterDexRegistry* registry = + static_cast( + PluginRegistry::get().pass_registry(interdex::INTERDEX_PASS_NAME)); + std::function fn = + [this]() -> interdex::InterDexPassPlugin* { + return new DedupStringsInterDexPlugin(m_max_factory_methods); + }; + registry->register_plugin("DEDUP_STRINGS_PLUGIN", std::move(fn)); always_assert(!perf_mode_str.empty()); m_perf_mode = parse_perf_mode(perf_mode_str); }); } -void DedupStringsPass::eval_pass(DexStoresVector&, - ConfigFiles&, - PassManager& mgr) { - // In each dex, we might introduce as many new method refs and type refs as we - // might add factory methods. This makes sure that the inter-dex pass keeps - // space for that many method refs and type refs. - always_assert(m_max_factory_methods > 0); - m_reserved_refs_handle = - mgr.reserve_refs(name(), - ReserveRefsInfo(/* frefs */ 0, - /* trefs */ m_max_factory_methods, - /* mrefs */ m_max_factory_methods)); -} - void DedupStringsPass::run_pass(DexStoresVector& stores, ConfigFiles& conf, PassManager& mgr) { - always_assert(m_reserved_refs_handle); - mgr.release_reserved_refs(*m_reserved_refs_handle); - m_reserved_refs_handle = std::nullopt; - DedupStrings ds(m_max_factory_methods, m_method_profiles_appear_percent_threshold, m_perf_mode, conf.get_method_profiles()); diff --git a/opt/dedup-strings/DedupStrings.h b/opt/dedup-strings/DedupStrings.h index 33ca156a30..8cc5f44355 100644 --- a/opt/dedup-strings/DedupStrings.h +++ b/opt/dedup-strings/DedupStrings.h @@ -9,7 +9,6 @@ #include "InterDexPass.h" #include "Pass.h" -#include "PassManager.h" #include "PluginRegistry.h" namespace method_profiles { @@ -182,19 +181,14 @@ class DedupStringsPass : public Pass { {DexLimitsObeyed, Preserves}, {HasSourceBlocks, RequiresAndEstablishes}, {NoResolvablePureRefs, Preserves}, - {InitialRenameClass, Preserves}, }; } void bind_config() override; - - void eval_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; - void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; private: int64_t m_max_factory_methods; float m_method_profiles_appear_percent_threshold{1.f}; DedupStringsPerfMode m_perf_mode; - std::optional m_reserved_refs_handle; }; diff --git a/opt/dedup_blocks/DedupBlocksPass.cpp b/opt/dedup_blocks/DedupBlocksPass.cpp index 79b3b8dad7..0bd7493d75 100644 --- a/opt/dedup_blocks/DedupBlocksPass.cpp +++ b/opt/dedup_blocks/DedupBlocksPass.cpp @@ -29,13 +29,13 @@ void DedupBlocksPass::run_pass(DexStoresVector& stores, scope, [&](DexMethod* method) { const auto code = method->get_code(); - if (code == nullptr || m_config.method_blocklist.count(method) != 0 || - method->rstate.no_optimizations()) { + if (code == nullptr || m_config.method_blocklist.count(method) != 0) { return dedup_blocks_impl::Stats(); } TRACE(DEDUP_BLOCKS, 3, "[dedup blocks] method %s", SHOW(method)); - always_assert(code->editable_cfg_built()); + + code->build_cfg(/* editable */ true); auto& cfg = code->cfg(); TRACE(DEDUP_BLOCKS, 5, "[dedup blocks] method %s before:\n%s", @@ -44,6 +44,7 @@ void DedupBlocksPass::run_pass(DexStoresVector& stores, dedup_blocks_impl::DedupBlocks impl(&m_config, method); impl.run(); + code->clear_cfg(); return impl.get_stats(); }, m_config.debug ? 1 : redex_parallel::default_num_threads()); diff --git a/opt/dedup_blocks/DedupBlocksPass.h b/opt/dedup_blocks/DedupBlocksPass.h index f89ea336c2..bafe83bdb5 100644 --- a/opt/dedup_blocks/DedupBlocksPass.h +++ b/opt/dedup_blocks/DedupBlocksPass.h @@ -20,13 +20,17 @@ class DedupBlocksPass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoInitClassInstructions, Preserves}, {NoUnreachableInstructions, Preserves}, {NoResolvablePureRefs, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, {RenameClass, Preserves}, }; } + bool is_cfg_legacy() override { return true; } + void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; void bind_config() override { diff --git a/opt/dedup_resources/DedupResources.cpp b/opt/dedup_resources/DedupResources.cpp index f34f13a8a3..332ac2171e 100644 --- a/opt/dedup_resources/DedupResources.cpp +++ b/opt/dedup_resources/DedupResources.cpp @@ -416,7 +416,7 @@ void DedupResourcesPass::run_pass(DexStoresVector& stores, PassManager& mgr) { std::string apk_dir; conf.get_json_config().get("apk_dir", "", apk_dir); - always_assert(!apk_dir.empty()); + always_assert(apk_dir.size()); // 1. Basic information about what shoudln't be operate on. std::unordered_set disallowed_types; diff --git a/opt/dedup_resources/DedupResources.h b/opt/dedup_resources/DedupResources.h index 34417d86d7..c0209015c5 100644 --- a/opt/dedup_resources/DedupResources.h +++ b/opt/dedup_resources/DedupResources.h @@ -57,7 +57,9 @@ class DedupResourcesPass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } @@ -66,6 +68,8 @@ class DedupResourcesPass : public Pass { bind("disallowed_resources", {}, m_disallowed_resources); } + bool is_cfg_legacy() override { return true; } + void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; private: diff --git a/opt/delinit/DelInit.cpp b/opt/delinit/DelInit.cpp index 89ca552bcb..2fd9063699 100644 --- a/opt/delinit/DelInit.cpp +++ b/opt/delinit/DelInit.cpp @@ -39,6 +39,10 @@ constexpr const char* METRIC_VMETHODS_REMOVED = "num_vmethods_removed"; constexpr const char* METRIC_IFIELDS_REMOVED = "num_ifields_removed"; constexpr const char* METRIC_DMETHODS_REMOVED = "num_dmethods_removed"; +static ConcurrentSet referenced_classes; +// List of packages on the white list +static std::vector package_filter; + // Note: this method will return nullptr if the dotname refers to an unknown // type. DexType* get_dextype_from_dotname(const char* dotname) { @@ -56,8 +60,7 @@ DexType* get_dextype_from_dotname(const char* dotname) { // Search a class name in a list of package names, return true if there is a // match -bool find_package(const char* name, - const std::vector& package_filter) { +bool find_package(const char* name) { // If there's no allowed package, optimize every package by default if (package_filter.empty()) { return true; @@ -71,8 +74,7 @@ bool find_package(const char* name, return false; }; -ConcurrentSet find_referenced_classes(const Scope& scope) { - ConcurrentSet referenced_classes; +void find_referenced_classes(const Scope& scope) { DexType* dalviksig = type::dalvik_annotation_Signature(); walk::parallel::annotations(scope, [&](DexAnnotation* anno) { // Signature annotations contain strings that Jackson uses @@ -123,21 +125,15 @@ ConcurrentSet find_referenced_classes(const Scope& scope) { } } }); - - return referenced_classes; } -bool can_remove(const DexClass* cls, - const ConcurrentSet& referenced_classes) { +bool can_remove(const DexClass* cls) { return !root_or_string(cls) && !referenced_classes.count_unsafe(cls); } -bool can_remove(const DexMethod* m, - const ConcurrentSet& callers, - const ConcurrentSet& referenced_classes) { +bool can_remove(const DexMethod* m, const ConcurrentSet& callers) { return callers.count_unsafe(const_cast(m)) == 0 && - (can_remove(type_class(m->get_class()), referenced_classes) || - !root_or_string(m)); + (can_remove(type_class(m->get_class())) || !root_or_string(m)); } /** @@ -148,10 +144,9 @@ bool can_remove(const DexMethod* m, * - there is another constructor for the class that is used. */ bool can_remove_init(const DexMethod* m, - const ConcurrentSet& called, - const ConcurrentSet& referenced_classes) { + const ConcurrentSet& called) { DexClass* clazz = type_class(m->get_class()); - if (can_remove(clazz, referenced_classes)) { + if (can_remove(clazz)) { return true; } else if (m->get_proto()->get_args()->empty()) { // If the class is kept, we should probably keep the no argument constructor @@ -176,19 +171,16 @@ bool can_remove_init(const DexMethod* m, return false; } -bool can_remove(const DexField* f, - const ConcurrentSet& referenced_classes) { - return can_remove(type_class(f->get_class()), referenced_classes) || - !root_or_string(f); +bool can_remove(const DexField* f) { + return can_remove(type_class(f->get_class())) || !root_or_string(f); } /** * Return true for classes that should not be processed by the optimization. */ -bool filter_class(DexClass* clazz, - const std::vector& package_filter) { +bool filter_class(DexClass* clazz) { always_assert(!clazz->is_external()); - if (!find_package(clazz->get_name()->c_str(), package_filter)) { + if (!find_package(clazz->get_name()->c_str())) { return true; } return is_interface(clazz) || is_annotation(clazz); @@ -236,14 +228,6 @@ struct DeadRefs { size_t deleted_dmeths{0}; } del_init_res; - ConcurrentSet referenced_classes; - std::vector package_filter; - - explicit DeadRefs(ConcurrentSet referenced_classes, - std::vector package_filter) - : referenced_classes(std::move(referenced_classes)), - package_filter(std::move(package_filter)) {} - void delinit(Scope& scope); int find_new_unreachable(Scope& scope); void find_unreachable(Scope& scope); @@ -293,7 +277,7 @@ int DeadRefs::find_new_unreachable(Scope& scope) { stats.init_called++; continue; } - if (!can_remove_init(init, called, referenced_classes)) { + if (!can_remove_init(init, called)) { stats.init_cant_delete++; continue; } @@ -328,7 +312,7 @@ void DeadRefs::find_unreachable(Scope& scope) { auto& ci = class_infos.at(clazz); ci.vmethods.clear(); ci.ifields.clear(); - if (filter_class(clazz, package_filter)) return; + if (filter_class(clazz)) return; auto const& dmeths = clazz->get_dmethods(); bool hasInit = false; @@ -360,12 +344,12 @@ void DeadRefs::find_unreachable_data(DexClass* clazz) { ClassInfo& ci = class_infos.at(clazz); for (const auto& meth : clazz->get_vmethods()) { - if (!can_remove(meth, called, referenced_classes)) continue; + if (!can_remove(meth, called)) continue; ci.vmethods.insert(meth); } for (const auto& field : clazz->get_ifields()) { - if (!can_remove(field, referenced_classes)) continue; + if (!can_remove(field)) continue; ci.ifields.insert(field); } @@ -385,7 +369,7 @@ void DeadRefs::collect_dmethods(Scope& scope) { auto& ci = class_infos.at(clazz); ci.initmethods.clear(); ci.dmethods.clear(); - if (filter_class(clazz, package_filter)) return; + if (filter_class(clazz)) return; auto const& dmeths = clazz->get_dmethods(); for (auto meth : dmeths) { @@ -529,7 +513,7 @@ int DeadRefs::remove_unreachable(Scope& scope) { stats.called_dmeths++; continue; } - if (!can_remove(meth, called, referenced_classes)) { + if (!can_remove(meth, called)) { stats.dont_delete_dmeths++; continue; } @@ -571,8 +555,10 @@ int DeadRefs::remove_unreachable(Scope& scope) { void DelInitPass::run_pass(DexStoresVector& stores, ConfigFiles& /* conf */, PassManager& mgr) { + package_filter = m_package_filter; auto scope = build_class_scope(stores); - DeadRefs drefs{find_referenced_classes(scope), m_package_filter}; + find_referenced_classes(scope); + DeadRefs drefs; drefs.delinit(scope); TRACE(DELINIT, 1, "Removed %zu methods", drefs.del_init_res.deleted_inits); diff --git a/opt/delinit/DelInit.h b/opt/delinit/DelInit.h index 75615ec896..6e76fe752a 100644 --- a/opt/delinit/DelInit.h +++ b/opt/delinit/DelInit.h @@ -23,7 +23,9 @@ class DelInitPass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } diff --git a/opt/delsuper/DelSuper.cpp b/opt/delsuper/DelSuper.cpp index e2c6d650b2..ba1cf60b06 100644 --- a/opt/delsuper/DelSuper.cpp +++ b/opt/delsuper/DelSuper.cpp @@ -101,8 +101,8 @@ class DelSuper { const IRInstruction* insn) { redex_assert(insn->opcode() == OPCODE_INVOKE_SUPER); size_t src_idx{0}; - for (const auto& mie : InstructionIterable( - meth->get_code()->cfg().get_param_instructions())) { + for (const auto& mie : + InstructionIterable(meth->get_code()->get_param_instructions())) { auto load_param = mie.insn; if (load_param->dest() != insn->src(src_idx++)) { return false; @@ -141,7 +141,7 @@ class DelSuper { * super. */ DexMethod* get_trivial_return_invoke_super(const DexMethod* meth) { - auto* code = (const_cast(meth))->get_code(); + const auto* code = meth->get_code(); // Must have code if (!code) { @@ -150,10 +150,8 @@ class DelSuper { // TODO: rewrite the following code to not require a random-access // container of instructions - always_assert(code->editable_cfg_built()); - auto& cfg = code->cfg(); std::vector insns; - for (const auto& mie : cfg::InstructionIterable(cfg)) { + for (const auto& mie : InstructionIterable(meth->get_code())) { if (opcode::is_a_load_param(mie.insn->opcode())) { continue; } diff --git a/opt/delsuper/DelSuper.h b/opt/delsuper/DelSuper.h index b26b422932..65b81709b6 100644 --- a/opt/delsuper/DelSuper.h +++ b/opt/delsuper/DelSuper.h @@ -19,9 +19,13 @@ class DelSuperPass : public Pass { using namespace redex_properties::interactions; using namespace redex_properties::names; return { + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } + bool is_cfg_legacy() override { return true; } + void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; }; diff --git a/opt/evaluate_type_checks/EvaluateTypeChecks.cpp b/opt/evaluate_type_checks/EvaluateTypeChecks.cpp index b3f08aa377..82f1b3ac60 100644 --- a/opt/evaluate_type_checks/EvaluateTypeChecks.cpp +++ b/opt/evaluate_type_checks/EvaluateTypeChecks.cpp @@ -151,7 +151,6 @@ void analyze_true_instance_ofs( return true; } // Just an integrity check. - // NOLINTNEXTLINE(bugprone-assert-side-effect) Can't figure this out. redex_assert(*it->second.begin() == mie->insn); return false; }); @@ -219,15 +218,15 @@ void analyze_true_instance_ofs( } RemoveResult analyze_and_evaluate_instance_of(DexMethod* method) { - auto& cfg = method->get_code()->cfg(); - CFGMutation mutation(cfg); + ScopedCFG cfg(method->get_code()); + CFGMutation mutation(*cfg); RemoveResult res; std::vector true_modulo_nulls; // Figure out types. Find guaranteed-false checks. { - TypeInference type_inf(cfg); + TypeInference type_inf(*cfg); type_inf.run(method); auto& type_envs = type_inf.get_type_environments(); @@ -240,7 +239,7 @@ RemoveResult analyze_and_evaluate_instance_of(DexMethod* method) { return &it->second; }; - for (const MethodItemEntry& mie : cfg::InstructionIterable(cfg)) { + for (const MethodItemEntry& mie : cfg::InstructionIterable(*cfg)) { auto insn = mie.insn; if (insn->opcode() != OPCODE_INSTANCE_OF) { continue; @@ -283,8 +282,8 @@ RemoveResult analyze_and_evaluate_instance_of(DexMethod* method) { } redex_assert(*eval == 0); - auto def_it = cfg.find_insn(insn); - auto move_it = cfg.move_result_of(def_it); + auto def_it = cfg->find_insn(insn); + auto move_it = cfg->move_result_of(def_it); if (move_it.is_end()) { // Should not happen. continue; } @@ -304,7 +303,7 @@ RemoveResult analyze_and_evaluate_instance_of(DexMethod* method) { // See whether the checks that will succeed if the value is not null // can be turned into a null check. If the result is used for more // than a branch, transformation is likely not beneficial at the moment. - analyze_true_instance_ofs(cfg, mutation, res, true_modulo_nulls); + analyze_true_instance_ofs(*cfg, mutation, res, true_modulo_nulls); mutation.flush(); return res; @@ -348,15 +347,14 @@ void handle_false_case(IRInstruction* insn, } RemoveResult analyze_and_evaluate(DexMethod* method) { - always_assert(method->get_code()->editable_cfg_built()); - auto& cfg = method->get_code()->cfg(); - CFGMutation mutation(cfg); + ScopedCFG cfg(method->get_code()); + CFGMutation mutation(*cfg); RemoveResult res; // Figure out types. { - TypeInference type_inf(cfg); + TypeInference type_inf(*cfg); type_inf.run(method); auto& type_envs = type_inf.get_type_environments(); @@ -369,7 +367,7 @@ RemoveResult analyze_and_evaluate(DexMethod* method) { return &it->second; }; - for (const MethodItemEntry& mie : cfg::InstructionIterable(cfg)) { + for (const MethodItemEntry& mie : cfg::InstructionIterable(*cfg)) { auto insn = mie.insn; if (insn->opcode() != OPCODE_CHECK_CAST) { continue; @@ -406,15 +404,15 @@ RemoveResult analyze_and_evaluate(DexMethod* method) { } if (*eval == 0) { - handle_false_case(insn, cfg, mutation, res); + handle_false_case(insn, *cfg, mutation, res); continue; } redex_assert(*eval == 1); // Successful check, can be eliminated. reg_t src_reg = insn->src(0); - auto def_it = cfg.find_insn(insn); - auto move_it = cfg.move_result_of(def_it); + auto def_it = cfg->find_insn(insn); + auto move_it = cfg->move_result_of(def_it); if (move_it.is_end()) { // Should not happen. continue; } @@ -513,9 +511,7 @@ void EvaluateTypeChecksPass::run_pass(DexStoresVector& stores, } auto has_instance_of_check_cast = [&code]() { std::pair res; - always_assert(code->editable_cfg_built()); - auto& cfg = code->cfg(); - for (const auto& mie : cfg::InstructionIterable(cfg)) { + for (const auto& mie : *code) { if (mie.type != MFLOW_OPCODE) { continue; } diff --git a/opt/evaluate_type_checks/EvaluateTypeChecks.h b/opt/evaluate_type_checks/EvaluateTypeChecks.h index 5cca26f63c..54e2ad568b 100644 --- a/opt/evaluate_type_checks/EvaluateTypeChecks.h +++ b/opt/evaluate_type_checks/EvaluateTypeChecks.h @@ -28,12 +28,14 @@ class EvaluateTypeChecksPass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, - {SpuriousGetClassCallsInterned, RequiresAndPreserves}, - {InitialRenameClass, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } + bool is_cfg_legacy() override { return true; } + void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; // Exposed for testing. diff --git a/opt/final_inline/FinalInline.h b/opt/final_inline/FinalInline.h index 48faeb619b..98e4779af6 100644 --- a/opt/final_inline/FinalInline.h +++ b/opt/final_inline/FinalInline.h @@ -32,7 +32,7 @@ class FinalInlinePass : public Pass { using namespace redex_properties::interactions; using namespace redex_properties::names; return { - {SpuriousGetClassCallsInterned, RequiresAndPreserves}, + {HasSourceBlocks, Preserves}, }; } diff --git a/opt/final_inline/FinalInlineV2.cpp b/opt/final_inline/FinalInlineV2.cpp index b7d3c07b1c..2e08837f26 100644 --- a/opt/final_inline/FinalInlineV2.cpp +++ b/opt/final_inline/FinalInlineV2.cpp @@ -60,7 +60,7 @@ std::ostream& operator<<(std::ostream& o, auto compute_deps(const Scope& scope, const std::unordered_set& scope_set) { - InsertOnlyConcurrentMap> deps_parallel; + ConcurrentMap> deps_parallel; ConcurrentMap> reverse_deps_parallel; ConcurrentSet is_target; ConcurrentSet maybe_roots; @@ -362,7 +362,7 @@ class ClassInitStrategy final : public call_graph::SingleCalleeStrategy { walk::methods(m_scope, [&](DexMethod* method) { if (method::is_clinit(method)) { - roots.insert(method); + roots.emplace_back(method); } }); return root_and_dynamic; @@ -490,7 +490,7 @@ StaticFieldReadAnalysis::Result StaticFieldReadAnalysis::analyze( m_allowed_opaque_callees.count(callee_method_def)) { return editable_cfg_adapter::LOOP_CONTINUE; } - auto callees = resolve_callees_in_graph(m_graph, insn); + auto callees = resolve_callees_in_graph(m_graph, method, insn); if (callees.empty()) { TRACE(FINALINLINE, 2, "%s has opaque callees %s", SHOW(method), SHOW(insn->get_method())); @@ -552,16 +552,10 @@ cp::WholeProgramState analyze_and_simplify_clinits( cp::Transform::RuntimeCache runtime_cache{}; for (DexClass* cls : reverse_tsort_by_clinit_deps(scope, init_cycles)) { - auto clinit = cls->get_clinit(); - if (clinit != nullptr && clinit->get_code() == nullptr) { - continue; - } - if (clinit != nullptr && clinit->rstate.no_optimizations()) { - continue; - } ConstantEnvironment env; cp::set_encoded_values(cls, &env); - if (clinit != nullptr) { + auto clinit = cls->get_clinit(); + if (clinit != nullptr && clinit->get_code() != nullptr) { auto* code = clinit->get_code(); { auto& cfg = code->cfg(); @@ -633,6 +627,7 @@ cp::WholeProgramState analyze_and_simplify_inits( if (cls->is_external()) { continue; } + ConstantEnvironment env; auto ctors = cls->get_ctors(); if (ctors.size() > 1) { continue; @@ -649,38 +644,31 @@ cp::WholeProgramState analyze_and_simplify_inits( continue; } } - ConstantEnvironment env; cp::set_ifield_values(cls, eligible_ifields, &env); - always_assert(ctors.size() <= 1); if (ctors.size() == 1) { auto ctor = ctors[0]; - if (ctor->get_code() == nullptr) { - continue; - } - if (ctor->rstate.no_optimizations()) { - continue; + if (ctor->get_code() != nullptr) { + auto* code = ctor->get_code(); + auto& cfg = code->cfg(); + cfg.calculate_exit_block(); + constant_propagation::WholeProgramStateAccessor wps_accessor(wps); + cp::intraprocedural::FixpointIterator intra_cp( + cfg, + CombinedInitAnalyzer(cls->get_type(), &wps_accessor, nullptr, + nullptr, nullptr)); + intra_cp.run(env); + env = intra_cp.get_exit_state_at(cfg.exit_block()); + + // Remove redundant iputs in inits + cp::Transform::Config transform_config; + transform_config.class_under_init = cls->get_type(); + cp::Transform(transform_config) + .legacy_apply_constants_and_prune_unreachable( + intra_cp, wps, cfg, xstores, cls->get_type()); + // Delete the instructions rendered dead by the removal of those iputs. + LocalDce(&init_classes_with_side_effects, pure_methods) + .dce(cfg, /* normalize_new_instances */ true, ctor->get_class()); } - cp::set_ifield_values(cls, eligible_ifields, &env); - auto* code = ctor->get_code(); - auto& cfg = code->cfg(); - cfg.calculate_exit_block(); - constant_propagation::WholeProgramStateAccessor wps_accessor(wps); - cp::intraprocedural::FixpointIterator intra_cp( - cfg, - CombinedInitAnalyzer(cls->get_type(), &wps_accessor, nullptr, nullptr, - nullptr)); - intra_cp.run(env); - env = intra_cp.get_exit_state_at(cfg.exit_block()); - - // Remove redundant iputs in inits - cp::Transform::Config transform_config; - transform_config.class_under_init = cls->get_type(); - cp::Transform(transform_config) - .legacy_apply_constants_and_prune_unreachable( - intra_cp, wps, cfg, xstores, cls->get_type()); - // Delete the instructions rendered dead by the removal of those iputs. - LocalDce(&init_classes_with_side_effects, pure_methods) - .dce(cfg, /* normalize_new_instances */ true, ctor->get_class()); } wps.collect_instance_finals(cls, eligible_ifields, env.get_field_environment()); @@ -723,9 +711,6 @@ FinalInlinePassV2::Stats inline_final_gets( if (field_type == cp::FieldType::STATIC && method::is_clinit(method)) { return; } - if (method->rstate.no_optimizations()) { - return; - } cfg::CFGMutation mutation(code.cfg()); size_t replacements = 0; for (auto block : code.cfg().blocks()) { diff --git a/opt/final_inline/FinalInlineV2.h b/opt/final_inline/FinalInlineV2.h index ff866dbf9a..c618bd561e 100644 --- a/opt/final_inline/FinalInlineV2.h +++ b/opt/final_inline/FinalInlineV2.h @@ -33,8 +33,9 @@ class FinalInlinePassV2 : public Pass { using namespace redex_properties::interactions; using namespace redex_properties::names; return { + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, - {SpuriousGetClassCallsInterned, RequiresAndPreserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } diff --git a/opt/fully-qualify-layouts/FullyQualifyLayouts.cpp b/opt/fully-qualify-layouts/FullyQualifyLayouts.cpp index 25637dfbfa..5f789d04c5 100644 --- a/opt/fully-qualify-layouts/FullyQualifyLayouts.cpp +++ b/opt/fully-qualify-layouts/FullyQualifyLayouts.cpp @@ -31,7 +31,7 @@ void FullyQualifyLayouts::run_pass(DexStoresVector& /* unused */, PassManager& mgr) { std::string zip_dir; conf.get_json_config().get("apk_dir", "", zip_dir); - always_assert(!zip_dir.empty()); + always_assert(zip_dir.size()); auto resources = create_resource_reader(zip_dir); auto res_table = resources->load_res_table(); diff --git a/opt/fully-qualify-layouts/FullyQualifyLayouts.h b/opt/fully-qualify-layouts/FullyQualifyLayouts.h index 98a55eb34f..7c40b03ecf 100644 --- a/opt/fully-qualify-layouts/FullyQualifyLayouts.h +++ b/opt/fully-qualify-layouts/FullyQualifyLayouts.h @@ -25,8 +25,9 @@ class FullyQualifyLayouts : Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, - {InitialRenameClass, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } diff --git a/opt/init-classes/InitClassLoweringPass.cpp b/opt/init-classes/InitClassLoweringPass.cpp index 6e6e75a42e..e384745c1c 100644 --- a/opt/init-classes/InitClassLoweringPass.cpp +++ b/opt/init-classes/InitClassLoweringPass.cpp @@ -51,18 +51,20 @@ class InitClassFields { } } m_dex_referenced_sfields.resize(dex_to_classes.size()); + ConcurrentMap concurrent_class_dex_indices; workqueue_run>( [&](std::pair p) { auto& dex_referenced_sfields = m_dex_referenced_sfields.at(p.first); for (auto* cls : *p.second) { cls->gather_fields(dex_referenced_sfields); - m_class_dex_indices.emplace(cls->get_type(), p.first); + concurrent_class_dex_indices.emplace(cls->get_type(), p.first); } std20::erase_if(dex_referenced_sfields, [](DexFieldRef* f) { return !f->is_def() || !is_static(f->as_def()); }); }, dex_to_classes); + m_class_dex_indices = concurrent_class_dex_indices.move_to_container(); } std::vector get_replacements( @@ -70,7 +72,7 @@ class InitClassFields { DexMethod* caller, const std::function& reg_getter) const { std::vector insns; - auto caller_dex_idx = m_class_dex_indices.at_unsafe(caller->get_class()); + auto caller_dex_idx = m_class_dex_indices.at(caller->get_class()); auto field = get(type, caller_dex_idx); auto reg = reg_getter(field); auto sget_insn = (new IRInstruction(opcode::sget_opcode_for_field(field))) @@ -118,7 +120,7 @@ class InitClassFields { private: std::vector> m_dex_referenced_sfields; - InsertOnlyConcurrentMap m_class_dex_indices; + std::unordered_map m_class_dex_indices; const DexString* m_field_name = DexString::make_string(redex_field_name); mutable std::atomic m_fields_added{0}; struct InitClassField { @@ -364,16 +366,15 @@ void InitClassLoweringPass::run_pass(DexStoresVector& stores, if (!code) { return Stats(); } - always_assert(code->editable_cfg_built()); - auto& cfg = code->cfg(); if (method::is_clinit(method)) { clinits.insert(method); } - if (!method::count_opcode_of_types(cfg, {IOPCODE_INIT_CLASS})) { + if (!method::count_opcode_of_types(code, {IOPCODE_INIT_CLASS})) { return Stats(); } + cfg::ScopedCFG cfg(code); InitClassPruner pruner(init_classes_with_side_effects, - method->get_class(), cfg); + method->get_class(), *cfg); pruner.apply(); auto local_stats = pruner.get_stats(); if (local_stats.init_class_instructions == 0) { @@ -384,25 +385,25 @@ void InitClassLoweringPass::run_pass(DexStoresVector& stores, "[InitClassLowering] method %s with %zu init-classes:\n%s", SHOW(method), local_stats.init_class_instructions, - SHOW(cfg)); + SHOW(*cfg)); methods_with_init_class++; boost::optional tmp_reg; boost::optional wide_tmp_reg; auto get_reg = [&](DexField* field) { if (type::is_wide_type(field->get_type())) { if (!wide_tmp_reg) { - wide_tmp_reg = cfg.allocate_wide_temp(); + wide_tmp_reg = cfg->allocate_wide_temp(); } return *wide_tmp_reg; } if (!tmp_reg) { - tmp_reg = cfg.allocate_temp(); + tmp_reg = cfg->allocate_temp(); } return *tmp_reg; }; - cfg::CFGMutation mutation(cfg); + cfg::CFGMutation mutation(*cfg); size_t local_sget_instructions_added = 0; - for (auto block : cfg.blocks()) { + for (auto block : cfg->blocks()) { auto ii = InstructionIterable(block); for (auto it = ii.begin(); it != ii.end(); it++) { if (it->insn->opcode() != IOPCODE_INIT_CLASS) { @@ -419,7 +420,7 @@ void InitClassLoweringPass::run_pass(DexStoresVector& stores, auto cfg_it = block->to_cfg_instruction_iterator(it); if (tag_str) { auto message = get_init_class_message(method, type, cfg_it); - auto log_insns = log_creator.get_insns(cfg, tag_str, message); + auto log_insns = log_creator.get_insns(*cfg, tag_str, message); replacements.insert(replacements.begin(), log_insns.begin(), log_insns.end()); } @@ -455,11 +456,11 @@ void InitClassLoweringPass::run_pass(DexStoresVector& stores, auto count = p.second; auto clinit = cls->get_clinit(); always_assert(clinit); - auto& cfg = clinit->get_code()->cfg(); + cfg::ScopedCFG cfg(clinit->get_code()); TRACE(ICL, 5, "[InitClassLowering] clinit of %s referenced by %zu init-class " "instructions:\n%s", - SHOW(cls), count, SHOW(cfg)); + SHOW(cls), count, SHOW(*cfg)); } } diff --git a/opt/init-classes/InitClassLoweringPass.h b/opt/init-classes/InitClassLoweringPass.h index 103a141ce3..3d50a22b0b 100644 --- a/opt/init-classes/InitClassLoweringPass.h +++ b/opt/init-classes/InitClassLoweringPass.h @@ -20,6 +20,7 @@ class InitClassLoweringPass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoInitClassInstructions, Establishes}, {NoResolvablePureRefs, Preserves}, {RenameClass, Preserves}, @@ -28,6 +29,8 @@ class InitClassLoweringPass : public Pass { void bind_config() override; + bool is_cfg_legacy() override { return true; } + void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; private: diff --git a/opt/insert-source-blocks/InsertSourceBlocks.cpp b/opt/insert-source-blocks/InsertSourceBlocks.cpp index b17cde044d..4559c08eea 100644 --- a/opt/insert-source-blocks/InsertSourceBlocks.cpp +++ b/opt/insert-source-blocks/InsertSourceBlocks.cpp @@ -33,6 +33,7 @@ #include "PassManager.h" #include "RedexContext.h" #include "RedexMappedFile.h" +#include "ScopedCFG.h" #include "Show.h" #include "SourceBlockConsistencyCheck.h" #include "SourceBlocks.h" @@ -645,13 +646,13 @@ struct Injector { std::string_view access_method_name; std::string access_method_hash_name; - always_assert(code->editable_cfg_built()); - auto& cfg = code->cfg(); + ScopedCFG cfg(code); + if (access_method) { access_method_type = access_method->first; access_method_name = access_method->second; - auto hash_value = hasher::stable_hash(cfg); + auto hash_value = hasher::stable_hash(*cfg); access_method_hash_name = hasher::hashed_name(hash_value, access_method_name); @@ -679,7 +680,7 @@ struct Injector { }(); auto res = source_blocks::insert_source_blocks( - sb_name, &cfg, profiles.first, serialize, exc_inject); + sb_name, cfg.get(), profiles.first, serialize, exc_inject); smi.add({sb_name, std::move(res.serialized), std::move(res.serialized_idom_map)}); diff --git a/opt/insert-source-blocks/InsertSourceBlocks.h b/opt/insert-source-blocks/InsertSourceBlocks.h index e8fbd2ae26..2491663871 100644 --- a/opt/insert-source-blocks/InsertSourceBlocks.h +++ b/opt/insert-source-blocks/InsertSourceBlocks.h @@ -29,12 +29,15 @@ class InsertSourceBlocksPass : public Pass { return { {DexLimitsObeyed, Preserves}, {HasSourceBlocks, Establishes}, + {NoSpuriousGetClassCalls, Preserves}, {UltralightCodePatterns, Preserves}, }; } void bind_config() override; + bool is_cfg_legacy() override { return true; } + void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; private: diff --git a/opt/instrument/BlockInstrument.cpp b/opt/instrument/BlockInstrument.cpp index 098afca838..4580db7cd1 100644 --- a/opt/instrument/BlockInstrument.cpp +++ b/opt/instrument/BlockInstrument.cpp @@ -642,7 +642,6 @@ auto insert_prologue_insts(cfg::ControlFlowGraph& cfg, } return false; }; - // NOLINTNEXTLINE(bugprone-assert-side-effect) assert_log(b.block->end() == b.it || is_in_block(), "%s\n%s", SHOW(b.block), SHOW(**b.it)); } @@ -683,7 +682,6 @@ std::tuple> insert_onMethodExit_calls( } return *reg; } - // NOLINTNEXTLINE(google-explicit-constructor) operator bool() const { return reg.has_value(); } }; @@ -1131,9 +1129,8 @@ auto get_blocks_to_instrument(const DexMethod* m, info->bit_id = id++; } } - // NOLINTNEXTLINE(bugprone-assert-side-effect) redex_assert(std::all_of( - block_info_list.cbegin(), block_info_list.cend(), + block_info_list.begin(), block_info_list.end(), [](const auto& bi) { return bi.type != BlockType::Unspecified; })); return std::make_tuple(block_info_list, id, hit_id, false); @@ -1228,7 +1225,7 @@ MethodInfo instrument_basic_blocks( using namespace cfg; - always_assert(code.editable_cfg_built()); + code.build_cfg(/*editable*/ true); ControlFlowGraph& cfg = code.cfg(); std::string before_cfg = @@ -1406,10 +1403,9 @@ MethodInfo instrument_basic_blocks( info.num_instrumented_blocks = num_to_instrument; always_assert(count(BlockType::Instrumentable) == num_to_instrument); - // NOLINTNEXTLINE(bugprone-assert-side-effect) - redex_assert(std::none_of(blocks.cbegin(), blocks.cend(), [](const auto& b) { - return std::find(b.merge_in.cbegin(), b.merge_in.cend(), b.block) != - b.merge_in.cend(); + redex_assert(std::none_of(blocks.begin(), blocks.end(), [](const auto& b) { + return std::find(b.merge_in.begin(), b.merge_in.end(), b.block) != + b.merge_in.end(); })); info.num_merged = std::accumulate( blocks.begin(), blocks.end(), 0, @@ -1454,6 +1450,7 @@ MethodInfo instrument_basic_blocks( TRACE(INSTRUMENT, 7, "%s", SHOW(cfg)); } + code.clear_cfg(); return info; } @@ -1862,12 +1859,12 @@ void BlockInstrumentHelper::do_basic_block_tracing( // Create Buildable CFG so we can inline functions correctly. if (options.inline_onBlockHit) { IRCode* blockHit_code = onBlockHit->get_code(); - always_assert(blockHit_code->editable_cfg_built()); + blockHit_code->build_cfg(true); } for (auto& en : onNonLoopBlockHit_map) { IRCode* nonLoopBlockHit_code = en.second->get_code(); - always_assert(nonLoopBlockHit_code->editable_cfg_built()); + nonLoopBlockHit_code->build_cfg(true); } DexMethod* binaryIncrementer; @@ -1888,7 +1885,7 @@ void BlockInstrumentHelper::do_basic_block_tracing( break; } IRCode* binaryIncrementer_code = binaryIncrementer->get_code(); - always_assert(binaryIncrementer_code->editable_cfg_built()); + binaryIncrementer_code->build_cfg(true); // This method_offset is used in sMethodStats[] to locate a method profile. // We have a small header in the beginning of sMethodStats. @@ -2018,6 +2015,19 @@ void BlockInstrumentHelper::do_basic_block_tracing( hit_offset += method_info.num_hit_blocks; }); + // Destroy the CFG because we are done Instrumenting + if (options.inline_onBlockHit) { + IRCode* blockHit_code = onBlockHit->get_code(); + blockHit_code->clear_cfg(); + } + + for (auto& en : onNonLoopBlockHit_map) { + IRCode* nonLoopBlockHit_code = en.second->get_code(); + nonLoopBlockHit_code->clear_cfg(); + } + + binaryIncrementer_code->clear_cfg(); + // Patch static fields. const auto field_name = array_fields.at(1)->get_name()->str(); InstrumentPass::patch_array_size(analysis_cls, field_name, method_offset); diff --git a/opt/instrument/Instrument.cpp b/opt/instrument/Instrument.cpp index d23b6d4f34..efd2853781 100644 --- a/opt/instrument/Instrument.cpp +++ b/opt/instrument/Instrument.cpp @@ -54,6 +54,21 @@ constexpr const char* BASIC_BLOCK_TRACING = "basic_block_tracing"; constexpr const char* BASIC_BLOCK_HIT_COUNT = "basic_block_hit_count"; constexpr const char* METHOD_REPLACEMENT = "methods_replacement"; +class InstrumentInterDexPlugin : public interdex::InterDexPassPlugin { + public: + InstrumentInterDexPlugin(size_t frefs, size_t trefs, size_t mrefs) + : m_frefs(frefs), m_trefs(trefs), m_mrefs(mrefs) {} + + ReserveRefsInfo reserve_refs() override { + return ReserveRefsInfo(m_frefs, m_trefs, m_mrefs); + } + + private: + const size_t m_frefs; + const size_t m_trefs; + const size_t m_mrefs; +}; + // For example, say that "Lcom/facebook/debug/" is in the set. We match either // "^Lcom/facebook/debug/*" or "^Lcom/facebook/debug;". bool match_class_name(std::string cls_name, @@ -79,12 +94,10 @@ void instrument_onMethodBegin(DexMethod* method, DexMethod* method_onMethodBegin) { IRCode* code = method->get_code(); assert(code != nullptr); - always_assert(code->editable_cfg_built()); - auto& cfg = code->cfg(); IRInstruction* const_inst = new IRInstruction(OPCODE_CONST); const_inst->set_literal(index); - const auto reg_dest = cfg.allocate_temp(); + const auto reg_dest = code->allocate_temp(); const_inst->set_dest(reg_dest); IRInstruction* invoke_inst = new IRInstruction(OPCODE_INVOKE_STATIC); @@ -92,25 +105,47 @@ void instrument_onMethodBegin(DexMethod* method, invoke_inst->set_srcs_size(1); invoke_inst->set_src(0, reg_dest); + // TODO(minjang): Consider using get_param_instructions. // Try to find a right insertion point: the entry point of the method. // We skip any fall throughs and IOPCODE_LOAD_PARRM*. - auto entry_block = cfg.entry_block(); - auto insert_point = entry_block->get_first_non_param_loading_insn(); - auto cfg_insert_point = - entry_block->to_cfg_instruction_iterator(insert_point); - cfg.insert_before(cfg_insert_point, {const_inst, invoke_inst}); + auto insert_point = std::find_if_not( + code->begin(), code->end(), [&](const MethodItemEntry& mie) { + return mie.type == MFLOW_FALLTHROUGH || + (mie.type == MFLOW_OPCODE && + opcode::is_a_load_param(mie.insn->opcode())); + }); + + if (insert_point == code->end()) { + // No load params. So just insert before the head. + insert_point = code->begin(); + } else if (insert_point->type == MFLOW_DEBUG) { + // Right after the load params, there could be DBG_SET_PROLOGUE_END. + // Skip if there is a following POSITION, too. For example: + // 1: OPCODE: IOPCODE_LOAD_PARAM_OBJECT v1 + // 2: OPCODE: IOPCODE_LOAD_PARAM_OBJECT v2 + // 3: DEBUG: DBG_SET_PROLOGUE_END + // 4: POSITION: foo.java:42 (this might be optional.) + // <== Instrumentation code will be inserted here. + // + std::advance(insert_point, + std::next(insert_point)->type != MFLOW_POSITION ? 1 : 2); + } else { + // Otherwise, insert_point can be used directly. + } + + code->insert_before(code->insert_before(insert_point, invoke_inst), + const_inst); if (instr_debug) { - auto ii = cfg::InstructionIterable(cfg); - for (auto it = ii.begin(); it != ii.end(); ++it) { - if (it == cfg_insert_point) { + for (auto it = code->begin(); it != code->end(); ++it) { + if (it == insert_point) { TRACE(INSTRUMENT, 9, "<==== insertion"); TRACE(INSTRUMENT, 9, "%s", SHOW(*it)); ++it; - if (it != ii.end()) { + if (it != code->end()) { TRACE(INSTRUMENT, 9, "%s", SHOW(*it)); ++it; - if (it != ii.end()) { + if (it != code->end()) { TRACE(INSTRUMENT, 9, "%s", SHOW(*it)); } } @@ -170,8 +205,7 @@ void do_simple_method_tracing(DexClass* analysis_cls, return 0; } - const size_t sum_opcode_sizes = - method->get_code()->cfg().sum_opcode_sizes(); + const size_t sum_opcode_sizes = method->get_code()->sum_opcode_sizes(); total_size += sum_opcode_sizes; // Excluding analysis methods myselves. @@ -362,32 +396,28 @@ void count_source_block_chain_length(DexStoresVector& stores, PassManager& pm) { return; } boost::optional last_known = boost::none; - always_assert(code->editable_cfg_built()); - auto& cfg = code->cfg(); - for (const auto* b : cfg.blocks()) { - for (const auto& mie : *b) { - if (mie.type == MFLOW_SOURCE_BLOCK) { - size_t len = 0; - for (auto* sb = mie.src_block.get(); sb != nullptr; - sb = sb->next.get()) { - ++len; - } - count.fetch_add(1); - sum.fetch_add(len); + for (auto& mie : *code) { + if (mie.type == MFLOW_SOURCE_BLOCK) { + size_t len = 0; + for (auto* sb = mie.src_block.get(); sb != nullptr; + sb = sb->next.get()) { + ++len; + } + count.fetch_add(1); + sum.fetch_add(len); - if (last_known && *last_known >= len) { - continue; + if (last_known && *last_known >= len) { + continue; + } + for (;;) { + auto cur = longest_list.load(); + if (cur >= len) { + last_known = cur; + break; } - for (;;) { - auto cur = longest_list.load(); - if (cur >= len) { - last_known = cur; - break; - } - if (longest_list.compare_exchange_strong(cur, len)) { - last_known = len; - break; - } + if (longest_list.compare_exchange_strong(cur, len)) { + last_known = len; + break; } } } @@ -411,8 +441,6 @@ void InstrumentPass::patch_array_size(DexClass* analysis_cls, always_assert(clinit != nullptr); auto* code = clinit->get_code(); - always_assert(code->editable_cfg_built()); - auto& cfg = code->cfg(); bool patched = false; walk::matching_opcodes_in_block( *clinit, @@ -434,13 +462,12 @@ void InstrumentPass::patch_array_size(DexClass* analysis_cls, IRInstruction* const_inst = new IRInstruction(OPCODE_CONST); const_inst->set_literal(array_size); - const auto reg_dest = cfg.allocate_temp(); + const auto reg_dest = code->allocate_temp(); const_inst->set_dest(reg_dest); insts[0]->set_src(0, reg_dest); - auto ii = cfg::InstructionIterable(cfg); - for (auto it = ii.begin(); it != ii.end(); ++it) { - if (it->insn == insts[0]) { - cfg.insert_before(it, const_inst); + for (auto& mie : InstructionIterable(code)) { + if (mie.insn == insts[0]) { + code->insert_before(code->iterator_to(mie), const_inst); patched = true; return; } @@ -450,7 +477,7 @@ void InstrumentPass::patch_array_size(DexClass* analysis_cls, if (!patched) { std::cerr << "[InstrumentPass] error: cannot patch array size." << std::endl; - std::cerr << show(clinit->get_code()->cfg()) << std::endl; + std::cerr << show(clinit->get_code()) << std::endl; exit(1); } @@ -465,51 +492,40 @@ void InstrumentPass::patch_static_field(DexClass* analysis_cls, always_assert(clinit != nullptr); // Find the sput with the given field name. - auto code = clinit->get_code(); - always_assert(code->editable_cfg_built()); - auto& cfg = code->cfg(); - auto ii = cfg::InstructionIterable(cfg); - for (auto it = ii.begin(); it != ii.end(); ++it) { - auto* sput_inst = it->insn; - if (sput_inst->opcode() != OPCODE_SPUT || - sput_inst->get_field()->get_name()->str() != field_name) { - continue; + auto* code = clinit->get_code(); + IRInstruction* sput_inst = nullptr; + IRList::iterator insert_point; + for (auto& mie : InstructionIterable(code)) { + auto* insn = mie.insn; + if (insn->opcode() == OPCODE_SPUT && + insn->get_field()->get_name()->str() == field_name) { + sput_inst = insn; + insert_point = code->iterator_to(mie); + break; } - // Find the SPUT. - // Create a new const instruction just like patch_stat_array_size. - IRInstruction* const_inst = new IRInstruction(OPCODE_CONST); - const_inst->set_literal(new_number); - const auto reg_dest = cfg.allocate_temp(); - const_inst->set_dest(reg_dest); - sput_inst->set_src(0, reg_dest); - cfg.insert_before(it, const_inst); - TRACE(INSTRUMENT, 2, "%s was patched: %d", SHOW(field_name), new_number); - return; } + // SPUT can be null if the original field value was encoded in the // static_values_off array. And consider simplifying using make_concrete. - TRACE(INSTRUMENT, 2, "sput %s was deleted; creating it", SHOW(field_name)); - auto sput_inst = new IRInstruction(OPCODE_SPUT); - sput_inst->set_field( - DexField::make_field(DexType::make_type(analysis_cls->get_name()), - DexString::make_string(field_name), - DexType::make_type("I"))); + if (sput_inst == nullptr) { + TRACE(INSTRUMENT, 2, "sput %s was deleted; creating it", SHOW(field_name)); + sput_inst = new IRInstruction(OPCODE_SPUT); + sput_inst->set_field( + DexField::make_field(DexType::make_type(analysis_cls->get_name()), + DexString::make_string(field_name), + DexType::make_type("I"))); + insert_point = + code->insert_after(code->get_param_instructions().end(), sput_inst); + } + + // Create a new const instruction just like patch_stat_array_size. IRInstruction* const_inst = new IRInstruction(OPCODE_CONST); const_inst->set_literal(new_number); - const auto reg_dest = cfg.allocate_temp(); + const auto reg_dest = code->allocate_temp(); const_inst->set_dest(reg_dest); + sput_inst->set_src(0, reg_dest); - auto entry_block = cfg.entry_block(); - auto last_param = entry_block->get_last_param_loading_insn(); - // always_assert(last_param != entry_block->end()); - if (last_param != entry_block->end()) { - auto cfg_last_param = entry_block->to_cfg_instruction_iterator(last_param); - cfg.insert_after(cfg_last_param, {const_inst, sput_inst}); - } else { - auto first_insn = entry_block->get_first_non_param_loading_insn(); - auto cfg_first_insn = entry_block->to_cfg_instruction_iterator(first_insn); - cfg.insert_before(cfg_first_insn, {const_inst, sput_inst}); - } + code->insert_before(insert_point, const_inst); TRACE(INSTRUMENT, 2, "%s was patched: %d", SHOW(field_name), new_number); } @@ -540,7 +556,6 @@ void InstrumentPass::bind_config() { bind("inline_onBlockHit", false, m_options.inline_onBlockHit); bind("inline_onNonLoopBlockHit", false, m_options.inline_onNonLoopBlockHit); bind("apply_CSE_CopyProp", false, m_options.apply_CSE_CopyProp); - trait(Traits::Pass::unique, true); after_configuration([this] { // Currently we only support instance call to static call. @@ -657,11 +672,13 @@ void InstrumentPass::eval_pass(DexStoresVector& stores, max_analysis_methods = 1; } - m_reserved_refs_handle = - mgr.reserve_refs(name(), - ReserveRefsInfo(/* frefs */ 1, - /* trefs */ 1, - /* mrefs */ max_analysis_methods)); + m_plugin = std::unique_ptr( + new InstrumentInterDexPlugin(1, 1, max_analysis_methods)); + + registry->register_plugin("INSTRUMENT_PASS_PLUGIN", [this]() { + return new InstrumentInterDexPlugin( + *(InstrumentInterDexPlugin*)m_plugin.get()); + }); } // Check for inclusion in allow/block lists of methods/classes. It supports: @@ -753,7 +770,6 @@ InstrumentPass::generate_sharded_analysis_methods( always_assert_log(patched, "Failed to patch sMethodStats1 in %s\n", SHOW(new_method)); method_names.insert(new_name); - new_method->get_code()->build_cfg(); new_analysis_methods[i] = new_method; TRACE(INSTRUMENT, 2, "Created %s with %s", SHOW(new_method), SHOW(array_fields.at(i))); @@ -786,8 +802,6 @@ InstrumentPass::patch_sharded_arrays( always_assert(num_shards > 0); DexMethod* clinit = cls->get_clinit(); IRCode* code = clinit->get_code(); - always_assert(code->editable_cfg_built()); - auto& cfg = code->cfg(); std::unordered_map fields; bool patched = false; walk::matching_opcodes_in_block( @@ -834,7 +848,6 @@ InstrumentPass::patch_sharded_arrays( // Clone the matched three instructions, but with new field names. for (size_t i = num_shards; i >= 1; --i) { - auto pos_it = cfg.find_insn(insts[2]); auto new_insts = { (new IRInstruction(OPCODE_NEW_ARRAY)) ->set_type(insts[0]->get_type()) @@ -845,9 +858,9 @@ InstrumentPass::patch_sharded_arrays( ->set_src(0, insts[2]->src(0)) ->set_field(fields.at(i))}; if (i == 1) { - cfg.replace_insns(pos_it, new_insts); + code->replace_opcode(insts[2], new_insts); } else { - cfg.insert_after(pos_it, new_insts); + code->insert_after(insts[2], new_insts); } } patched = true; @@ -855,7 +868,7 @@ InstrumentPass::patch_sharded_arrays( }); always_assert_log(patched, "Failed to insert sMethodStatsN:\n%s", - SHOW(clinit->get_code()->cfg())); + SHOW(clinit->get_code())); // static short[][] sMethodStatsArray = new short[][] { // sMethodStats1, <== Add @@ -890,12 +903,11 @@ InstrumentPass::patch_sharded_arrays( } const reg_t vX = insts[1]->dest(); - const reg_t vY = cfg.allocate_temp(); - const reg_t vN = cfg.allocate_temp(); + const reg_t vY = code->allocate_temp(); + const reg_t vN = code->allocate_temp(); for (size_t i = num_shards; i >= 1; --i) { - auto pos_it = cfg.find_insn(insts[2]); - cfg.insert_after( - pos_it, + code->insert_after( + insts[2], {(new IRInstruction(OPCODE_SGET_OBJECT))->set_field(fields.at(i)), (new IRInstruction(IOPCODE_MOVE_RESULT_PSEUDO_OBJECT)) ->set_dest(vY), @@ -913,7 +925,7 @@ InstrumentPass::patch_sharded_arrays( always_assert_log(patched, "Failed to insert sMethodStatsN to sMethodStatsArray:\n%s", - SHOW(clinit->get_code()->cfg())); + SHOW(clinit->get_code())); return fields; } @@ -921,10 +933,6 @@ InstrumentPass::patch_sharded_arrays( void InstrumentPass::run_pass(DexStoresVector& stores, ConfigFiles& cfg, PassManager& pm) { - if (m_reserved_refs_handle) { - pm.release_reserved_refs(*m_reserved_refs_handle); - } - // TODO(fengliu): We may need change this but leave it here for local test. if (m_options.instrumentation_strategy == METHOD_REPLACEMENT) { bool exclude_primary_dex = @@ -933,7 +941,6 @@ void InstrumentPass::run_pass(DexStoresVector& stores, method_reference::wrap_instance_call_with_static( stores, m_options.methods_replacement, exclude_primary_dex); pm.set_metric("wrapped_invocations", num_wrapped_invocations); - m_reserved_refs_handle = std::nullopt; return; } @@ -948,9 +955,6 @@ void InstrumentPass::run_pass(DexStoresVector& stores, return; } - always_assert(m_reserved_refs_handle); - m_reserved_refs_handle = std::nullopt; - // Append block listed classes from the file, if exists. if (!m_options.blocklist_file_name.empty()) { for (const auto& e : load_blocklist_file(m_options.blocklist_file_name)) { @@ -966,8 +970,8 @@ void InstrumentPass::run_pass(DexStoresVector& stores, } // Get the analysis class. - DexType* analysis_class_type = - g_redex->get_type(DexString::get_string(m_options.analysis_class_name)); + DexType* analysis_class_type = g_redex->get_type( + DexString::get_string(m_options.analysis_class_name.c_str())); if (analysis_class_type == nullptr) { std::cerr << "[InstrumentPass] error: cannot find analysis class: " << m_options.analysis_class_name << std::endl; diff --git a/opt/instrument/Instrument.h b/opt/instrument/Instrument.h index 08a7594217..b33732e6cf 100644 --- a/opt/instrument/Instrument.h +++ b/opt/instrument/Instrument.h @@ -12,7 +12,6 @@ #include #include "Pass.h" -#include "PassManager.h" class DexMethod; @@ -46,7 +45,6 @@ class InstrumentPass : public Pass { {DexLimitsObeyed, Preserves}, {HasSourceBlocks, Requires}, {NoResolvablePureRefs, Preserves}, - {SpuriousGetClassCallsInterned, RequiresAndPreserves}, {RenameClass, Preserves}, }; } @@ -55,6 +53,7 @@ class InstrumentPass : public Pass { void eval_pass(DexStoresVector& stores, ConfigFiles& conf, PassManager& mgr) override; + bool is_cfg_legacy() override { return true; } void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; // Helper functions for both method and block instrumentations. @@ -110,7 +109,6 @@ class InstrumentPass : public Pass { private: Options m_options; std::unique_ptr m_plugin; - std::optional m_reserved_refs_handle; }; } // namespace instrument diff --git a/opt/int_type_patcher/IntTypePatcher.cpp b/opt/int_type_patcher/IntTypePatcher.cpp index 4183d6f973..cc49ed8683 100644 --- a/opt/int_type_patcher/IntTypePatcher.cpp +++ b/opt/int_type_patcher/IntTypePatcher.cpp @@ -47,7 +47,7 @@ void IntTypePatcherPass::run(DexMethod* m) { } IRCode* code = m->get_code(); - if (!code || m->rstate.no_optimizations()) { + if (!code) { return; } always_assert(code->editable_cfg_built()); diff --git a/opt/int_type_patcher/IntTypePatcher.h b/opt/int_type_patcher/IntTypePatcher.h index 07bfc4c6e4..83cec15242 100644 --- a/opt/int_type_patcher/IntTypePatcher.h +++ b/opt/int_type_patcher/IntTypePatcher.h @@ -21,8 +21,12 @@ class IntTypePatcherPass : public Pass { using namespace redex_properties::interactions; using namespace redex_properties::names; return {{DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {UltralightCodePatterns, Preserves}, {NoInitClassInstructions, Preserves}, + {NeedsEverythingPublic, Preserves}, + {NeedsInjectionIdLowering, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, {RenameClass, Preserves}}; } diff --git a/opt/interdex/InterDex.cpp b/opt/interdex/InterDex.cpp index 89b5de541b..4ec0630641 100644 --- a/opt/interdex/InterDex.cpp +++ b/opt/interdex/InterDex.cpp @@ -81,8 +81,7 @@ std::unordered_set find_unrefenced_coldstart_classes( }, [&](DexMethod* meth, const IRCode& code) { auto base_cls = meth->get_class(); - always_assert(meth->get_code()->editable_cfg_built()); - for (auto& mie : cfg::InstructionIterable(meth->get_code()->cfg())) { + for (auto& mie : InstructionIterable(meth->get_code())) { auto inst = mie.insn; DexType* called_cls = nullptr; if (inst->has_method()) { @@ -110,8 +109,8 @@ std::unordered_set find_unrefenced_coldstart_classes( // opcodes directly. for (const auto& cls : input_scope) { if (cold_cold_references.count(cls->get_type())) { - const auto& refs = class_references_cache.get(cls); - for (const auto& type : refs.types) { + auto refs = class_references_cache.get(cls); + for (const auto& type : refs->types) { cold_cold_references.insert(type); } } @@ -142,11 +141,11 @@ void gather_refs( FieldRefs* frefs, TypeRefs* trefs, TypeRefs* itrefs) { - const auto& refs = class_references_cache.get(cls); - mrefs->insert(refs.method_refs.begin(), refs.method_refs.end()); - frefs->insert(refs.field_refs.begin(), refs.field_refs.end()); - trefs->insert(refs.types.begin(), refs.types.end()); - itrefs->insert(refs.init_types.begin(), refs.init_types.end()); + auto refs = class_references_cache.get(cls); + mrefs->insert(refs->method_refs.begin(), refs->method_refs.end()); + frefs->insert(refs->field_refs.begin(), refs->field_refs.end()); + trefs->insert(refs->types.begin(), refs->types.end()); + itrefs->insert(refs->init_types.begin(), refs->init_types.end()); std::vector method_refs; std::vector field_refs; @@ -717,21 +716,17 @@ void InterDex::init_cross_dex_ref_minimizer() { TRACE(IDEX, 2, "[dex ordering] Cross-dex-ref-minimizer active with method ref weight " "%" PRIu64 ", field ref weight %" PRIu64 ", type ref weight %" PRIu64 - ", large string ref weight %" PRIu64 - ", small string ref weight %" PRIu64 ", method seed weight %" PRIu64 + ", string ref weight %" PRIu64 ", method seed weight %" PRIu64 ", field seed weight %" PRIu64 ", type seed weight %" PRIu64 - ", large string seed weight %" PRIu64 - ", small string seed weight %" PRIu64 ".", + ", string seed weight %" PRIu64 ".", m_cross_dex_ref_minimizer.get_config().method_ref_weight, m_cross_dex_ref_minimizer.get_config().field_ref_weight, m_cross_dex_ref_minimizer.get_config().type_ref_weight, - m_cross_dex_ref_minimizer.get_config().large_string_ref_weight, - m_cross_dex_ref_minimizer.get_config().small_string_ref_weight, + m_cross_dex_ref_minimizer.get_config().string_ref_weight, m_cross_dex_ref_minimizer.get_config().method_seed_weight, m_cross_dex_ref_minimizer.get_config().field_seed_weight, m_cross_dex_ref_minimizer.get_config().type_seed_weight, - m_cross_dex_ref_minimizer.get_config().large_string_seed_weight, - m_cross_dex_ref_minimizer.get_config().small_string_seed_weight); + m_cross_dex_ref_minimizer.get_config().string_seed_weight); std::vector classes_to_insert; // Emit classes using some algorithm to group together classes which @@ -1144,6 +1139,65 @@ void InterDex::add_dexes_from_store(const DexStore& store) { post_process_dex(m_emitting_state, fodr); } +void InterDex::set_clinit_methods_if_needed(DexClass* cls) const { + using namespace dex_asm; + + if (m_methods_for_canary_clinit_reference.empty()) { + // No methods to call from clinit; don't create clinit. + return; + } + + if (cls->get_clinit()) { + // We already created and added a clinit. + return; + } + + // Create a clinit static method. + auto proto = + DexProto::make_proto(type::_void(), DexTypeList::make_type_list({})); + DexMethod* clinit = + DexMethod::make_method(cls->get_type(), + DexString::make_string(""), proto) + ->make_concrete(ACC_STATIC | ACC_CONSTRUCTOR, false); + clinit->set_code(std::make_unique()); + cls->add_method(clinit); + clinit->set_deobfuscated_name(show_deobfuscated(clinit)); + + // Add code to clinit to call the other methods. + auto code = clinit->get_code(); + size_t max_size = 0; + for (const auto& method_name : m_methods_for_canary_clinit_reference) { + // No need to do anything if this method isn't present in the build. + if (DexMethodRef* method = DexMethod::get_method(method_name)) { + std::vector reg_operands; + int64_t reg = 0; + for (auto* dex_type : *method->get_proto()->get_args()) { + Operand reg_operand = {VREG, reg}; + switch (dex_type->get_name()->c_str()[0]) { + case 'J': + case 'D': + // 8 bytes + code->push_back(dasm(OPCODE_CONST_WIDE, {reg_operand, 0_L})); + reg_operands.push_back(reg_operand); + reg += 2; + break; + default: + // 4 or fewer bytes + code->push_back(dasm(OPCODE_CONST, {reg_operand, 0_L})); + reg_operands.push_back(reg_operand); + ++reg; + break; + } + } + max_size = std::max(max_size, (size_t)reg); + code->push_back(dasm(OPCODE_INVOKE_STATIC, method, reg_operands.begin(), + reg_operands.end())); + } + } + code->set_registers_size(max_size); + code->push_back(dasm(OPCODE_RETURN_VOID)); +} + // Creates a canary class if necessary. (In particular, the primary dex never // has a canary class.) This method should be called after flush_out_dex when // beginning a new dex. As canary classes are added in the end without checks, @@ -1160,6 +1214,7 @@ DexClass* InterDex::get_canary_cls(EmittingState& emitting_state, static std::mutex canary_mutex; std::lock_guard lock_guard(canary_mutex); canary_cls = create_canary(dexnum); + set_clinit_methods_if_needed(canary_cls); } MethodRefs clazz_mrefs; FieldRefs clazz_frefs; @@ -1255,14 +1310,6 @@ void InterDex::post_process_dex(EmittingState& emitting_state, TRACE(IDEX, 4, "IDEX: Emitting %s-plugin-generated class :: %s", plugin->name().c_str(), SHOW(cls)); classes.push_back(cls); - // For the plugin cls, make sure all methods are editable cfg built. - for (auto* m : cls->get_all_methods()) { - if (m->get_code() == nullptr) { - continue; - } - m->get_code()->build_cfg(); - } - // If this is the primary dex, or if there are any betamap-ordered // classes in this dex, then we treat the additional classes as // perf-sensitive, to be conservative. diff --git a/opt/interdex/InterDex.h b/opt/interdex/InterDex.h index 2ea15f6efd..843cd0f778 100644 --- a/opt/interdex/InterDex.h +++ b/opt/interdex/InterDex.h @@ -54,6 +54,7 @@ class InterDex { const ReserveRefsInfo& reserve_refs, const XStoreRefs* xstore_refs, int min_sdk, + std::vector methods_for_canary_clinit_reference, const init_classes::InitClassesWithSideEffects& init_classes_with_side_effects, bool transitively_close_interdex_order, @@ -82,6 +83,8 @@ class InterDex { m_original_scope(original_scope), m_scope(build_class_scope(m_dexen)), m_xstore_refs(xstore_refs), + m_methods_for_canary_clinit_reference( + std::move(methods_for_canary_clinit_reference)), m_transitively_close_interdex_order(transitively_close_interdex_order), m_minimize_cross_dex_refs_explore_alternatives( minimize_cross_dex_refs_explore_alternatives), @@ -191,6 +194,8 @@ class InterDex { void post_process_dex(EmittingState& emitting_state, const FlushOutDexResult&) const; + void set_clinit_methods_if_needed(DexClass* cls) const; + /** * Stores in m_interdex_order a list of coldstart types. It will only contain: * * classes that still exist in the current scope @@ -251,6 +256,7 @@ class InterDex { std::vector m_interdex_types; const XStoreRefs* m_xstore_refs; size_t m_current_classes_when_emitting_remaining{0}; + std::vector m_methods_for_canary_clinit_reference; size_t m_transitive_closure_added{0}; size_t m_transitive_closure_moved{0}; diff --git a/opt/interdex/InterDexPass.cpp b/opt/interdex/InterDexPass.cpp index 95ef85cdce..dedba21fed 100644 --- a/opt/interdex/InterDexPass.cpp +++ b/opt/interdex/InterDexPass.cpp @@ -139,30 +139,21 @@ void InterDexPass::bind_config() { bind("minimize_cross_dex_refs_type_ref_weight", m_minimize_cross_dex_refs_config.type_ref_weight, m_minimize_cross_dex_refs_config.type_ref_weight); - bind("minimize_cross_dex_refs_large_string_ref_weight", - m_minimize_cross_dex_refs_config.large_string_ref_weight, - m_minimize_cross_dex_refs_config.large_string_ref_weight); - bind("minimize_cross_dex_refs_small_string_ref_weight", - m_minimize_cross_dex_refs_config.small_string_ref_weight, - m_minimize_cross_dex_refs_config.small_string_ref_weight); - bind("minimize_cross_dex_refs_min_large_string_size", - m_minimize_cross_dex_refs_config.min_large_string_size, - m_minimize_cross_dex_refs_config.min_large_string_size); + bind("minimize_cross_dex_refs_string_ref_weight", + m_minimize_cross_dex_refs_config.string_ref_weight, + m_minimize_cross_dex_refs_config.string_ref_weight); bind("minimize_cross_dex_refs_method_seed_weight", m_minimize_cross_dex_refs_config.method_seed_weight, m_minimize_cross_dex_refs_config.method_seed_weight); bind("minimize_cross_dex_refs_field_seed_weight", m_minimize_cross_dex_refs_config.field_seed_weight, m_minimize_cross_dex_refs_config.field_seed_weight); - bind("minimize_cross_dex_refs_type_seed_weight", + bind("minimize_cross_dex_refs_type_ref_weight", m_minimize_cross_dex_refs_config.type_seed_weight, m_minimize_cross_dex_refs_config.type_seed_weight); - bind("minimize_cross_dex_refs_large_string_seed_weight", - m_minimize_cross_dex_refs_config.large_string_seed_weight, - m_minimize_cross_dex_refs_config.large_string_seed_weight); - bind("minimize_cross_dex_refs_small_string_seed_weight", - m_minimize_cross_dex_refs_config.small_string_seed_weight, - m_minimize_cross_dex_refs_config.small_string_seed_weight); + bind("minimize_cross_dex_refs_string_ref_weight", + m_minimize_cross_dex_refs_config.string_seed_weight, + m_minimize_cross_dex_refs_config.string_seed_weight); bind("minimize_cross_dex_refs_emit_json", false, m_minimize_cross_dex_refs_config.emit_json); bind("minimize_cross_dex_refs_explore_alternatives", 1, @@ -175,6 +166,10 @@ void InterDexPass::bind_config() { bind("can_touch_coldstart_extended_cls", false, m_can_touch_coldstart_extended_cls); bind("expect_order_list", false, m_expect_order_list); + bind("methods_for_canary_clinit_reference", {}, + m_methods_for_canary_clinit_reference, + "If set, canary classes will have a clinit generated which call the " + "specified methods, if they exist"); bind("transitively_close_interdex_order", m_transitively_close_interdex_order, m_transitively_close_interdex_order); @@ -227,8 +222,8 @@ void InterDexPass::run_pass( m_keep_primary_order, force_single_dex, m_order_interdex, m_emit_canaries, m_minimize_cross_dex_refs, m_fill_last_coldstart_dex, m_minimize_cross_dex_refs_config, refs_info, &xstore_refs, - mgr.get_redex_options().min_sdk, init_classes_with_side_effects, - m_transitively_close_interdex_order, + mgr.get_redex_options().min_sdk, m_methods_for_canary_clinit_reference, + init_classes_with_side_effects, m_transitively_close_interdex_order, m_minimize_cross_dex_refs_explore_alternatives, cache, m_exclude_baseline_profile_classes, std::move(m_baseline_profile_config)); @@ -320,7 +315,8 @@ void InterDexPass::run_pass_on_nonroot_store( false /* emit canaries */, false /* minimize_cross_dex_refs */, /* fill_last_coldstart_dex=*/false, cross_dex_refs_config, refs_info, &xstore_refs, mgr.get_redex_options().min_sdk, - init_classes_with_side_effects, m_transitively_close_interdex_order, + m_methods_for_canary_clinit_reference, init_classes_with_side_effects, + m_transitively_close_interdex_order, m_minimize_cross_dex_refs_explore_alternatives, cache, m_exclude_baseline_profile_classes, std::move(m_baseline_profile_config)); @@ -342,13 +338,15 @@ void InterDexPass::run_pass(DexStoresVector& stores, PluginRegistry::get().pass_registry(INTERDEX_PASS_NAME)); auto plugins = registry->create_plugins(); + ReserveRefsInfo refs_info = m_reserve_refs; for (const auto& plugin : plugins) { plugin->configure(original_scope, conf); + const auto plugin_reserve_refs = plugin->reserve_refs(); + refs_info.frefs += plugin_reserve_refs.frefs; + refs_info.trefs += plugin_reserve_refs.trefs; + refs_info.mrefs += plugin_reserve_refs.mrefs; } - ReserveRefsInfo refs_info = m_reserve_refs; - refs_info += mgr.get_reserved_refs(); - ClassReferencesCache cache(original_scope); std::vector parallel_stores; diff --git a/opt/interdex/InterDexPass.h b/opt/interdex/InterDexPass.h index e0312ba833..fdc5d68b4f 100644 --- a/opt/interdex/InterDexPass.h +++ b/opt/interdex/InterDexPass.h @@ -72,8 +72,9 @@ class InterDexPass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Establishes}, + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, - {InitialRenameClass, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } @@ -87,6 +88,8 @@ class InterDexPass : public Pass { ++m_eval; } + bool is_cfg_legacy() override { return true; } + void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; bool minimize_cross_dex_refs() const { return m_minimize_cross_dex_refs; } diff --git a/opt/interdex/InterDexPassPlugin.h b/opt/interdex/InterDexPassPlugin.h index dd0873eb79..c4e545f585 100644 --- a/opt/interdex/InterDexPassPlugin.h +++ b/opt/interdex/InterDexPassPlugin.h @@ -33,6 +33,11 @@ class InterDexPassPlugin { std::vector&, std::vector&) {} + // Reserves refs in every dex, effectively lowering the capacity of each dex. + // + // This is applied uniformly (e.g., cannot be a per-dex value). + virtual ReserveRefsInfo reserve_refs() { return ReserveRefsInfo{}; } + // Return any new codegened classes that should be added to the current dex. virtual DexClasses additional_classes(size_t dex_count, const DexClasses&) { DexClasses empty; diff --git a/opt/interdex/InterDexReshuffleImpl.cpp b/opt/interdex/InterDexReshuffleImpl.cpp deleted file mode 100644 index c038369093..0000000000 --- a/opt/interdex/InterDexReshuffleImpl.cpp +++ /dev/null @@ -1,271 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * - * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. - */ - -#include "InterDexReshuffleImpl.h" -#include "InterDexPass.h" -#include "Show.h" -#include "Trace.h" -#include - -#include -#include -#include -#include - -/// populate_reshuffable_classes_types collects type of classes that are in cold -/// start but below 20pct appear count. These classes can be considered not as -/// perf sensitive, thus should be movable. -void populate_reshuffable_classes_types( - std::unordered_set& reshuffable_classes, ConfigFiles& conf) { - const auto& betamap_classes = conf.get_coldstart_classes(); - bool seen_coldstart_20pct_end = false; - bool seen_coldstart_1pct_end = false; - constexpr const char* coldstart_20pct_end = "ColdStart20PctEnd"; - constexpr const char* coldstart_1pct_end = "ColdStart1PctEnd"; - for (const auto& cls_name : betamap_classes) { - if (cls_name.find(coldstart_20pct_end) != std::string::npos) { - seen_coldstart_20pct_end = true; - } - if (cls_name.find(coldstart_1pct_end) != std::string::npos) { - seen_coldstart_1pct_end = true; - } - if (seen_coldstart_20pct_end) { - if (!seen_coldstart_1pct_end) { - reshuffable_classes.insert(cls_name); - } else { - // if a class appears after ColdStart marker, then it is loaded - // by another interaction which might be perf sensitive. - reshuffable_classes.erase(cls_name); - } - } - } -} - -bool is_reshuffable_class( - const DexClass* cls, - const std::unordered_set& reshuffable_classes) { - return reshuffable_classes.find(cls->get_type()->str_copy()) != - reshuffable_classes.end(); -} - -InterDexReshuffleImpl::InterDexReshuffleImpl(ConfigFiles& conf, - PassManager& mgr, - ReshuffleConfig& config, - DexClasses& original_scope, - DexClassesVector& dexen) - : m_conf(conf), - m_mgr(mgr), - m_config(config), - m_init_classes_with_side_effects(original_scope, - conf.create_init_class_insns()), - m_dexen(dexen) { - m_dexes_structure.set_min_sdk(mgr.get_redex_options().min_sdk); - const auto& interdex_metrics = mgr.get_interdex_metrics(); - auto it = interdex_metrics.find(interdex::METRIC_LINEAR_ALLOC_LIMIT); - m_linear_alloc_limit = (it != interdex_metrics.end() ? it->second : 0) + - m_config.extra_linear_alloc_limit; - it = interdex_metrics.find(interdex::METRIC_ORDER_INTERDEX); - m_order_interdex = it == interdex_metrics.end() || it->second; - auto refs_info = mgr.get_reserved_refs(); - m_dexes_structure.set_reserve_frefs(refs_info.frefs + - m_config.reserved_extra_frefs); - m_dexes_structure.set_reserve_trefs(refs_info.trefs + - m_config.reserved_extra_trefs); - m_dexes_structure.set_reserve_mrefs(refs_info.mrefs + - m_config.reserved_extra_mrefs); - - m_mutable_dexen.resize(dexen.size()); - m_mutable_dexen_strings.resize(dexen.size()); - - Timer t("init"); - DexClasses classes; - std::unordered_set reshuffable_classes; - if (config.exclude_below20pct_coldstart_classes) { - populate_reshuffable_classes_types(reshuffable_classes, conf); - } - for (size_t dex_index = m_first_dex_index; dex_index < dexen.size(); - dex_index++) { - auto& dex = dexen.at(dex_index); - if (dex_index == m_first_dex_index && - !std::any_of(dex.begin(), dex.end(), - [this, &reshuffable_classes](auto* cls) { - return can_move(cls) || - is_reshuffable_class(cls, reshuffable_classes); - })) { - m_first_dex_index++; - continue; - } - for (auto cls : dex) { - classes.push_back(cls); - m_class_refs.emplace(cls, Refs()); - if (!can_move(cls) && !is_reshuffable_class(cls, reshuffable_classes)) { - continue; - } - m_movable_classes.push_back(cls); - m_class_dex_indices.emplace(cls, dex_index); - } - } - walk::parallel::classes(classes, [&](DexClass* cls) { - always_assert(m_class_refs.count(cls)); - auto& refs = m_class_refs.at(cls); - cls->gather_methods(refs.mrefs); - cls->gather_fields(refs.frefs); - cls->gather_types(refs.trefs); - std::vector itrefs; - cls->gather_init_classes(itrefs); - refs.itrefs.insert(itrefs.begin(), itrefs.end()); - cls->gather_strings(refs.srefs); - }); - workqueue_run_for( - m_first_dex_index, dexen.size(), [&](size_t dex_idx) { - auto& dex = dexen.at(dex_idx); - auto& mutable_dex = m_mutable_dexen.at(dex_idx); - auto& mutable_dex_strings = m_mutable_dexen_strings.at(dex_idx); - for (auto cls : dex) { - always_assert(m_class_refs.count(cls)); - const auto& refs = m_class_refs.at(cls); - TypeRefs pending_init_class_fields; - TypeRefs pending_init_class_types; - mutable_dex.resolve_init_classes(&m_init_classes_with_side_effects, - refs.frefs, refs.trefs, refs.itrefs, - &pending_init_class_fields, - &pending_init_class_types); - auto laclazz = estimate_linear_alloc(cls); - mutable_dex.add_class_no_checks( - refs.mrefs, refs.frefs, refs.trefs, pending_init_class_fields, - pending_init_class_types, laclazz, cls); - for (auto* sref : refs.srefs) { - mutable_dex_strings[sref]++; - } - } - }); -} - -void InterDexReshuffleImpl::compute_plan() { - Timer t("compute_plan"); - MoveGains move_gains(m_first_dex_index, m_movable_classes, - m_class_dex_indices, m_class_refs, m_mutable_dexen, - m_mutable_dexen_strings); - size_t batches{0}; - size_t total_moves{0}; - size_t max_move_gains{0}; - for (; batches < m_config.max_batches; batches++) { - Timer u("batch"); - move_gains.recompute_gains(); - max_move_gains = std::max(max_move_gains, move_gains.size()); - - while (move_gains.moves_this_epoch() < m_config.max_batch_size) { - std::optional move_opt = move_gains.pop_max_gain(); - if (!move_opt) { - break; - } - - const Move& move = *move_opt; - auto recomputed_gain = - move_gains.compute_move_gain(move.cls, move.target_dex_index); - if (recomputed_gain <= 0) { - continue; - } - - // Check if it is a valid move. - if (!try_plan_move(move)) { - continue; - } - if (traceEnabled(IDEXR, 5)) { - print_stats(); - } - move_gains.moved_class(move); - } - total_moves += move_gains.moves_this_epoch(); - TRACE(IDEXR, 2, "executed %zu moves in epoch %zu", - move_gains.moves_this_epoch(), batches); - if (move_gains.should_stop()) { - break; - } - } - - m_mgr.incr_metric("max_move_gains", max_move_gains); - m_mgr.incr_metric("total_moves", total_moves); - m_mgr.incr_metric("batches", batches); - m_mgr.incr_metric("first_dex_index", m_first_dex_index); - TRACE(IDEXR, 1, "executed %zu moves in %zu batches", total_moves, batches); -} - -void InterDexReshuffleImpl::apply_plan() { - Timer t("finish"); - workqueue_run_for( - m_first_dex_index, m_mutable_dexen.size(), [&](size_t dex_idx) { - auto& dex = m_dexen.at(dex_idx); - const auto& mutable_dex = m_mutable_dexen.at(dex_idx); - auto classes = mutable_dex.get_classes(/* perf_based */ true); - TRACE(IDEXR, 2, "dex %zu: %zu => %zu classes", dex_idx, dex.size(), - classes.size()); - dex = std::move(classes); - }); -} - -void InterDexReshuffleImpl::print_stats() { - size_t n_classes = 0; - size_t n_mrefs = 0; - size_t n_frefs = 0; - for (size_t idx = 0; idx < m_mutable_dexen.size(); ++idx) { - auto& mutable_dex = m_mutable_dexen.at(idx); - n_classes += mutable_dex.get_num_classes(); - n_mrefs += mutable_dex.get_num_mrefs(); - n_frefs += mutable_dex.get_num_frefs(); - } - - TRACE(IDEXR, 5, "Global stats:"); - TRACE(IDEXR, 5, "\t %zu classes", n_classes); - TRACE(IDEXR, 5, "\t %zu mrefs", n_mrefs); - TRACE(IDEXR, 5, "\t %zu frefs", n_frefs); -} - -bool InterDexReshuffleImpl::try_plan_move(const Move& move) { - auto& target_dex = m_mutable_dexen.at(move.target_dex_index); - always_assert(m_class_refs.count(move.cls)); - const auto& refs = m_class_refs.at(move.cls); - TypeRefs pending_init_class_fields; - TypeRefs pending_init_class_types; - target_dex.resolve_init_classes( - &m_init_classes_with_side_effects, refs.frefs, refs.trefs, refs.itrefs, - &pending_init_class_fields, &pending_init_class_types); - auto laclazz = estimate_linear_alloc(move.cls); - if (!target_dex.add_class_if_fits( - refs.mrefs, refs.frefs, refs.trefs, pending_init_class_fields, - pending_init_class_types, m_linear_alloc_limit, - m_dexes_structure.get_frefs_limit(), - m_dexes_structure.get_mrefs_limit(), - m_dexes_structure.get_trefs_limit(), move.cls)) { - return false; - } - auto& target_dex_strings = m_mutable_dexen_strings.at(move.target_dex_index); - for (auto* sref : refs.srefs) { - target_dex_strings[sref]++; - } - always_assert(m_class_dex_indices.count(move.cls)); - auto& dex_index = m_class_dex_indices.at(move.cls); - auto& source_dex = m_mutable_dexen.at(dex_index); - source_dex.remove_class(&m_init_classes_with_side_effects, refs.mrefs, - refs.frefs, refs.trefs, pending_init_class_fields, - pending_init_class_types, laclazz, move.cls); - auto& source_dex_strings = m_mutable_dexen_strings.at(dex_index); - for (auto* sref : refs.srefs) { - auto it = source_dex_strings.find(sref); - if (--it->second == 0) { - source_dex_strings.erase(it); - } - } - dex_index = move.target_dex_index; - return true; -} - -bool InterDexReshuffleImpl::can_move(DexClass* cls) { - return (!m_order_interdex || - cls->get_perf_sensitive() != PerfSensitiveGroup::BETAMAP_ORDERED) && - !is_canary(cls); -} diff --git a/opt/interdex/InterDexReshuffleImpl.h b/opt/interdex/InterDexReshuffleImpl.h deleted file mode 100644 index 99346e15e3..0000000000 --- a/opt/interdex/InterDexReshuffleImpl.h +++ /dev/null @@ -1,272 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * - * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. - */ - -#pragma once - -#include "ConfigFiles.h" -#include "DexClass.h" -#include "DexStructure.h" -#include "PassManager.h" -#include "StlUtil.h" -#include "Walkers.h" - -struct Refs { - MethodRefs mrefs; - FieldRefs frefs; - TypeRefs trefs; - TypeRefs itrefs; - std::unordered_set srefs; -}; - -struct ReshuffleConfig { - size_t reserved_extra_frefs{0}; - size_t reserved_extra_trefs{0}; - size_t reserved_extra_mrefs{0}; - size_t extra_linear_alloc_limit{0}; - size_t max_batches{20}; - size_t max_batch_size{200000}; - bool exclude_below20pct_coldstart_classes{false}; -}; - -// Compute gain powers by reference occurrences. We don't use the upper 20 (19, -// actually, because sign bit) bits to allow for adding all gains of a class. -// TODO: While this integer-based representation allows for a fast and -// deterministic algorithm, it's precision ends at more than 11 occurrences, -// where the gain goes to 0. Based on current experiments, increasing 11 may -// increase the size gain a little bit, but come with a cost of -// “non-determinism” (due to rounding errors or space complexity if we compute -// these differently). -using gain_t = int64_t; -static constexpr gain_t power_value_for(size_t occurrences) { - return occurrences > 11 ? 0 : (((gain_t)1) << 44) >> (occurrences * 4); -} - -// A suggested move of a class from one dex to another. -struct Move { - DexClass* cls; - gain_t gain; - size_t target_dex_index; -}; - -// All move gains for all classes. -class MoveGains { - auto compare_indices_by_gains() { - return [&](size_t first_gain_index, size_t second_gain_index) { - auto& first = m_gains.at(first_gain_index); - auto& second = m_gains.at(second_gain_index); - if (first.gain != second.gain) { - return first.gain < second.gain; - } - // tie breakers for determinism - if (first.target_dex_index != second.target_dex_index) { - return first.target_dex_index < second.target_dex_index; - } - return compare_dexclasses(first.cls, second.cls); - }; - } - - public: - MoveGains(size_t first_dex_index, - const std::vector& movable_classes, - const std::unordered_map& class_dex_indices, - const std::unordered_map& class_refs, - const std::vector& dexen, - const std::vector>& - dexen_strings) - : m_first_dex_index(first_dex_index), - m_movable_classes(movable_classes), - m_class_dex_indices(class_dex_indices), - m_class_refs(class_refs), - m_dexen(dexen), - m_dexen_strings(dexen_strings) {} - - void recompute_gains() { - Timer t("recompute_gains"); - m_gains_size = 0; - std::mutex mutex; - walk::parallel::classes(m_movable_classes, [&](DexClass* cls) { - for (size_t dex_index = m_first_dex_index; dex_index < m_dexen.size(); - ++dex_index) { - auto gain = compute_move_gain(cls, dex_index); - if (gain > 0) { - std::lock_guard lock_guard(mutex); - if (m_gains_size == m_gains.size()) { - m_gains.resize(std::max((size_t)1024, m_gains_size * 2)); - } - m_gains.at(m_gains_size++) = (Move){cls, gain, dex_index}; - } - } - }); - - m_gains_heap_size = m_gains_size; - if (m_gains_heap.size() < m_gains_heap_size) { - m_gains_heap.resize(std::max((size_t)1024, m_gains_heap_size * 2)); - } - std::iota(m_gains_heap.begin(), m_gains_heap.begin() + m_gains_heap_size, - 0); - std::make_heap(m_gains_heap.begin(), - m_gains_heap.begin() + m_gains_heap_size, - compare_indices_by_gains()); - - m_epoch += 1; - m_moves_last_epoch = m_moves_this_epoch; - m_moves_this_epoch = 0; - m_also_moved_in_last_epoch = 0; - } - - std::optional pop_max_gain() { - while (m_gains_heap_size > 0) { - std::pop_heap(m_gains_heap.begin(), - m_gains_heap.begin() + m_gains_heap_size, - compare_indices_by_gains()); - m_gains_heap_size -= 1; - - const size_t gain_index = m_gains_heap.at(m_gains_heap_size); - const auto& move = m_gains.at(gain_index); - - auto it = m_move_epoch.find(move.cls); - if (it != m_move_epoch.end() && it->second >= m_epoch) { - // Class already moved. - continue; - } - - return move; - } - - return std::nullopt; - } - - void moved_class(const Move& move) { - size_t& class_epoch = m_move_epoch[move.cls]; - const bool was_moved_last_epoch = class_epoch == m_epoch - 1; - class_epoch = m_epoch; - - m_moves_this_epoch += 1; - m_also_moved_in_last_epoch += was_moved_last_epoch; - } - - size_t moves_this_epoch() const { return m_moves_this_epoch; } - - bool should_stop() const { - return m_moves_this_epoch == 0 || - (static_cast(m_also_moved_in_last_epoch) / - static_cast(m_moves_last_epoch) > - 0.9f); - } - - size_t size() const { return m_gains_heap_size; } - - gain_t compute_move_gain(DexClass* cls, size_t target_index) const { - gain_t gain = 0; - always_assert(m_class_dex_indices.count(cls)); - auto source_index = m_class_dex_indices.at(cls); - if (source_index != target_index) { - always_assert(m_class_refs.count(cls)); - const auto& refs = m_class_refs.at(cls); - auto& source = m_dexen.at(source_index); - auto& target = m_dexen.at(target_index); - for (auto* fref : refs.frefs) { - auto source_occurrences = source.get_fref_occurrences(fref); - auto target_occurrences = target.get_fref_occurrences(fref); - gain += compute_gain(source_occurrences, target_occurrences); - } - for (auto* mref : refs.mrefs) { - auto source_occurrences = source.get_mref_occurrences(mref); - auto target_occurrences = target.get_mref_occurrences(mref); - gain += compute_gain(source_occurrences, target_occurrences); - } - for (auto* tref : refs.trefs) { - auto source_occurrences = source.get_tref_occurrences(tref); - auto target_occurrences = target.get_tref_occurrences(tref); - gain += compute_gain(source_occurrences, target_occurrences); - } - auto& source_strings = m_dexen_strings.at(source_index); - auto& target_strings = m_dexen_strings.at(target_index); - for (auto* sref : refs.srefs) { - auto it = source_strings.find(sref); - auto source_occurrences = it == source_strings.end() ? 0 : it->second; - it = target_strings.find(sref); - auto target_occurrences = it == target_strings.end() ? 0 : it->second; - gain += compute_gain(source_occurrences, target_occurrences); - } - } - - return gain; - } - - gain_t compute_gain(size_t source_occurrences, - size_t target_occurrences) const { - return source_occurrences == 0 ? 0 - : power_value_for(source_occurrences - 1) - - power_value_for(target_occurrences); - } - - // The gains improvement values; the class + target dex are a function - // of the index. It is indices into the gains that are heapified. - std::vector m_gains; - size_t m_gains_size{0}; - std::vector m_gains_heap; - size_t m_gains_heap_size{0}; - - // Tracks when a class was last moved. - // - // Epoch advances when gains are recomputed. It starts at 1, so that we - // are in a state as if every class was moved in epoch 0, and none were - // moved in epoch 1. Then the first recomputation moves us to epoch 2, - // so that the stopping criteria doesn't think every class was moved. - std::unordered_map m_move_epoch; - size_t m_epoch{1}; - - // Tracks epoch move counts and inter-epoch move differences. - size_t m_moves_this_epoch{0}; - size_t m_moves_last_epoch{0}; - size_t m_also_moved_in_last_epoch{0}; - - const size_t m_first_dex_index; - const std::vector& m_movable_classes; - const std::unordered_map& m_class_dex_indices; - const std::unordered_map& m_class_refs; - const std::vector& m_dexen; - const std::vector>& - m_dexen_strings; -}; - -class InterDexReshuffleImpl { - public: - InterDexReshuffleImpl(ConfigFiles& conf, - PassManager& mgr, - ReshuffleConfig& config, - DexClasses& original_scope, - DexClassesVector& dexen); - - void compute_plan(); - - void apply_plan(); - - private: - void print_stats(); - - bool try_plan_move(const Move& move); - - bool can_move(DexClass* cls); - - ConfigFiles& m_conf; - PassManager& m_mgr; - ReshuffleConfig& m_config; - init_classes::InitClassesWithSideEffects m_init_classes_with_side_effects; - DexClassesVector& m_dexen; - size_t m_linear_alloc_limit; - DexesStructure m_dexes_structure; - std::vector m_movable_classes; - std::unordered_map m_class_dex_indices; - std::unordered_map m_class_refs; - std::vector m_mutable_dexen; - std::vector> - m_mutable_dexen_strings; - size_t m_first_dex_index{1}; // skip primary dex - bool m_order_interdex; -}; diff --git a/opt/interdex/InterDexReshufflePass.cpp b/opt/interdex/InterDexReshufflePass.cpp index 08a018ac0d..df2e8e5709 100644 --- a/opt/interdex/InterDexReshufflePass.cpp +++ b/opt/interdex/InterDexReshufflePass.cpp @@ -18,14 +18,505 @@ #include "StlUtil.h" #include "Trace.h" #include "Walkers.h" +#include +#include namespace { +struct Refs { + MethodRefs mrefs; + FieldRefs frefs; + TypeRefs trefs; + TypeRefs itrefs; + std::unordered_set srefs; +}; + const interdex::InterDexPass* get_interdex_pass(const PassManager& mgr) { const auto* pass = static_cast(mgr.find_pass("InterDexPass")); always_assert_log(pass, "InterDexPass missing"); return pass; } + +/// populate_reshuffable_classes_types collects type of classes that are in cold +/// start but below 20pct appear count. These classes can be considered not as +/// perf sensitive, thus should be movable. +void populate_reshuffable_classes_types( + std::unordered_set& reshuffable_classes, ConfigFiles& conf) { + const auto& betamap_classes = conf.get_coldstart_classes(); + bool seen_coldstart_20pct_end = false; + bool seen_coldstart_1pct_end = false; + constexpr const char* coldstart_20pct_end = "ColdStart20PctEnd"; + constexpr const char* coldstart_1pct_end = "ColdStart1PctEnd"; + for (const auto& cls_name : betamap_classes) { + if (cls_name.find(coldstart_20pct_end) != std::string::npos) { + seen_coldstart_20pct_end = true; + } + if (cls_name.find(coldstart_1pct_end) != std::string::npos) { + seen_coldstart_1pct_end = true; + } + if (seen_coldstart_20pct_end) { + if (!seen_coldstart_1pct_end) { + reshuffable_classes.insert(cls_name); + } else { + // if a class appears after ColdStart marker, then it is loaded + // by another interaction which might be perf sensitive. + reshuffable_classes.erase(cls_name); + } + } + } +} + +bool is_reshuffable_class( + const DexClass* cls, + const std::unordered_set& reshuffable_classes) { + return reshuffable_classes.find(cls->get_type()->str_copy()) != + reshuffable_classes.end(); +} + +// Compute gain powers by reference occurrences. We don't use the upper 20 (19, +// actually, because sign bit) bits to allow for adding all gains of a class. +// TODO: While this integer-based representation allows for a fast and +// deterministic algorithm, it's precision ends at more than 11 occurrences, +// where the gain goes to 0. Based on current experiments, increasing 11 may +// increase the size gain a little bit, but come with a cost of +// “non-determinism” (due to rounding errors or space complexity if we compute +// these differently). +using gain_t = int64_t; +static constexpr gain_t power_value_for(size_t occurrences) { + return occurrences > 11 ? 0 : (((gain_t)1) << 44) >> (occurrences * 4); +} + +// A suggested move of a class from one dex to another. +struct Move { + DexClass* cls; + gain_t gain; + size_t target_dex_index; +}; + +// All move gains for all classes. +class MoveGains { + auto compare_indices_by_gains() { + return [&](size_t first_gain_index, size_t second_gain_index) { + auto& first = m_gains.at(first_gain_index); + auto& second = m_gains.at(second_gain_index); + if (first.gain != second.gain) { + return first.gain < second.gain; + } + // tie breakers for determinism + if (first.target_dex_index != second.target_dex_index) { + return first.target_dex_index < second.target_dex_index; + } + return compare_dexclasses(first.cls, second.cls); + }; + } + + public: + MoveGains(size_t first_dex_index, + const std::vector& movable_classes, + const std::unordered_map& class_dex_indices, + const std::unordered_map& class_refs, + const std::vector& dexen, + const std::vector>& + dexen_strings) + : m_first_dex_index(first_dex_index), + m_movable_classes(movable_classes), + m_class_dex_indices(class_dex_indices), + m_class_refs(class_refs), + m_dexen(dexen), + m_dexen_strings(dexen_strings) {} + + void recompute_gains() { + Timer t("recompute_gains"); + m_gains_size = 0; + std::mutex mutex; + walk::parallel::classes(m_movable_classes, [&](DexClass* cls) { + for (size_t dex_index = m_first_dex_index; dex_index < m_dexen.size(); + ++dex_index) { + auto gain = compute_move_gain(cls, dex_index); + if (gain > 0) { + std::lock_guard lock_guard(mutex); + if (m_gains_size == m_gains.size()) { + m_gains.resize(std::max((size_t)1024, m_gains_size * 2)); + } + m_gains.at(m_gains_size++) = (Move){cls, gain, dex_index}; + } + } + }); + + m_gains_heap_size = m_gains_size; + if (m_gains_heap.size() < m_gains_heap_size) { + m_gains_heap.resize(std::max((size_t)1024, m_gains_heap_size * 2)); + } + std::iota(m_gains_heap.begin(), m_gains_heap.begin() + m_gains_heap_size, + 0); + std::make_heap(m_gains_heap.begin(), + m_gains_heap.begin() + m_gains_heap_size, + compare_indices_by_gains()); + + m_epoch += 1; + m_moves_last_epoch = m_moves_this_epoch; + m_moves_this_epoch = 0; + m_also_moved_in_last_epoch = 0; + } + + std::optional pop_max_gain() { + while (m_gains_heap_size > 0) { + std::pop_heap(m_gains_heap.begin(), + m_gains_heap.begin() + m_gains_heap_size, + compare_indices_by_gains()); + m_gains_heap_size -= 1; + + const size_t gain_index = m_gains_heap.at(m_gains_heap_size); + const auto& move = m_gains.at(gain_index); + + auto it = m_move_epoch.find(move.cls); + if (it != m_move_epoch.end() && it->second >= m_epoch) { + // Class already moved. + continue; + } + + return move; + } + + return std::nullopt; + } + + void moved_class(const Move& move) { + size_t& class_epoch = m_move_epoch[move.cls]; + const bool was_moved_last_epoch = class_epoch == m_epoch - 1; + class_epoch = m_epoch; + + m_moves_this_epoch += 1; + m_also_moved_in_last_epoch += was_moved_last_epoch; + } + + size_t moves_this_epoch() const { return m_moves_this_epoch; } + + bool should_stop() const { + return m_moves_this_epoch == 0 || + (static_cast(m_also_moved_in_last_epoch) / + static_cast(m_moves_last_epoch) > + 0.9f); + } + + size_t size() const { return m_gains_heap_size; } + + gain_t compute_move_gain(DexClass* cls, size_t target_index) const { + gain_t gain = 0; + always_assert(m_class_dex_indices.count(cls)); + auto source_index = m_class_dex_indices.at(cls); + if (source_index != target_index) { + always_assert(m_class_refs.count(cls)); + const auto& refs = m_class_refs.at(cls); + auto& source = m_dexen.at(source_index); + auto& target = m_dexen.at(target_index); + for (auto* fref : refs.frefs) { + auto source_occurrences = source.get_fref_occurrences(fref); + auto target_occurrences = target.get_fref_occurrences(fref); + gain += compute_gain(source_occurrences, target_occurrences); + } + for (auto* mref : refs.mrefs) { + auto source_occurrences = source.get_mref_occurrences(mref); + auto target_occurrences = target.get_mref_occurrences(mref); + gain += compute_gain(source_occurrences, target_occurrences); + } + for (auto* tref : refs.trefs) { + auto source_occurrences = source.get_tref_occurrences(tref); + auto target_occurrences = target.get_tref_occurrences(tref); + gain += compute_gain(source_occurrences, target_occurrences); + } + auto& source_strings = m_dexen_strings.at(source_index); + auto& target_strings = m_dexen_strings.at(target_index); + for (auto* sref : refs.srefs) { + auto it = source_strings.find(sref); + auto source_occurrences = it == source_strings.end() ? 0 : it->second; + it = target_strings.find(sref); + auto target_occurrences = it == target_strings.end() ? 0 : it->second; + gain += compute_gain(source_occurrences, target_occurrences); + } + } + + return gain; + } + + gain_t compute_gain(size_t source_occurrences, + size_t target_occurrences) const { + return source_occurrences == 0 ? 0 + : power_value_for(source_occurrences - 1) - + power_value_for(target_occurrences); + } + + // The gains improvement values; the class + target dex are a function + // of the index. It is indices into the gains that are heapified. + std::vector m_gains; + size_t m_gains_size{0}; + std::vector m_gains_heap; + size_t m_gains_heap_size{0}; + + // Tracks when a class was last moved. + // + // Epoch advances when gains are recomputed. It starts at 1, so that we + // are in a state as if every class was moved in epoch 0, and none were + // moved in epoch 1. Then the first recomputation moves us to epoch 2, + // so that the stopping criteria doesn't think every class was moved. + std::unordered_map m_move_epoch; + size_t m_epoch{1}; + + // Tracks epoch move counts and inter-epoch move differences. + size_t m_moves_this_epoch{0}; + size_t m_moves_last_epoch{0}; + size_t m_also_moved_in_last_epoch{0}; + + const size_t m_first_dex_index; + const std::vector& m_movable_classes; + const std::unordered_map& m_class_dex_indices; + const std::unordered_map& m_class_refs; + const std::vector& m_dexen; + const std::vector>& + m_dexen_strings; +}; + +class Impl { + public: + Impl(ConfigFiles& conf, + PassManager& mgr, + InterDexReshufflePass::Config& config, + DexClasses& original_scope, + DexClassesVector& dexen) + : m_conf(conf), + m_mgr(mgr), + m_config(config), + m_init_classes_with_side_effects(original_scope, + conf.create_init_class_insns()), + m_dexen(dexen) { + m_dexes_structure.set_min_sdk(mgr.get_redex_options().min_sdk); + const auto& interdex_metrics = mgr.get_interdex_metrics(); + auto it = interdex_metrics.find(interdex::METRIC_LINEAR_ALLOC_LIMIT); + m_linear_alloc_limit = (it != interdex_metrics.end() ? it->second : 0) + + m_config.extra_linear_alloc_limit; + it = interdex_metrics.find(interdex::METRIC_RESERVED_FREFS); + m_dexes_structure.set_reserve_frefs( + (it != interdex_metrics.end() ? it->second : 0) + + m_config.reserved_extra_frefs); + it = interdex_metrics.find(interdex::METRIC_RESERVED_TREFS); + m_dexes_structure.set_reserve_trefs( + (it != interdex_metrics.end() ? it->second : 0) + + m_config.reserved_extra_trefs); + it = interdex_metrics.find(interdex::METRIC_RESERVED_MREFS); + m_dexes_structure.set_reserve_mrefs( + (it != interdex_metrics.end() ? it->second : 0) + + m_config.reserved_extra_mrefs); + it = interdex_metrics.find(interdex::METRIC_ORDER_INTERDEX); + m_order_interdex = it == interdex_metrics.end() || it->second; + + m_mutable_dexen.resize(dexen.size()); + m_mutable_dexen_strings.resize(dexen.size()); + + Timer t("init"); + DexClasses classes; + std::unordered_set reshuffable_classes; + if (config.exclude_below20pct_coldstart_classes) { + populate_reshuffable_classes_types(reshuffable_classes, conf); + } + for (size_t dex_index = m_first_dex_index; dex_index < dexen.size(); + dex_index++) { + auto& dex = dexen.at(dex_index); + if (dex_index == m_first_dex_index && + !std::any_of(dex.begin(), dex.end(), + [this, &reshuffable_classes](auto* cls) { return can_move(cls)|| + is_reshuffable_class(cls, reshuffable_classes); })) { + m_first_dex_index++; + continue; + } + for (auto cls : dex) { + classes.push_back(cls); + m_class_refs.emplace(cls, Refs()); + if (!can_move(cls) && !is_reshuffable_class(cls, reshuffable_classes)) { + continue; + } + m_movable_classes.push_back(cls); + m_class_dex_indices.emplace(cls, dex_index); + } + } + walk::parallel::classes(classes, [&](DexClass* cls) { + always_assert(m_class_refs.count(cls)); + auto& refs = m_class_refs.at(cls); + cls->gather_methods(refs.mrefs); + cls->gather_fields(refs.frefs); + cls->gather_types(refs.trefs); + std::vector itrefs; + cls->gather_init_classes(itrefs); + refs.itrefs.insert(itrefs.begin(), itrefs.end()); + cls->gather_strings(refs.srefs); + }); + workqueue_run_for( + m_first_dex_index, dexen.size(), [&](size_t dex_idx) { + auto& dex = dexen.at(dex_idx); + auto& mutable_dex = m_mutable_dexen.at(dex_idx); + auto& mutable_dex_strings = m_mutable_dexen_strings.at(dex_idx); + for (auto cls : dex) { + always_assert(m_class_refs.count(cls)); + const auto& refs = m_class_refs.at(cls); + TypeRefs pending_init_class_fields; + TypeRefs pending_init_class_types; + mutable_dex.resolve_init_classes( + &m_init_classes_with_side_effects, refs.frefs, refs.trefs, + refs.itrefs, &pending_init_class_fields, + &pending_init_class_types); + auto laclazz = estimate_linear_alloc(cls); + mutable_dex.add_class_no_checks( + refs.mrefs, refs.frefs, refs.trefs, pending_init_class_fields, + pending_init_class_types, laclazz, cls); + for (auto* sref : refs.srefs) { + mutable_dex_strings[sref]++; + } + } + }); + } + + void compute_plan() { + Timer t("compute_plan"); + MoveGains move_gains(m_first_dex_index, m_movable_classes, + m_class_dex_indices, m_class_refs, m_mutable_dexen, + m_mutable_dexen_strings); + size_t batches{0}; + size_t total_moves{0}; + size_t max_move_gains{0}; + for (; batches < m_config.max_batches; batches++) { + Timer u("batch"); + move_gains.recompute_gains(); + max_move_gains = std::max(max_move_gains, move_gains.size()); + + while (move_gains.moves_this_epoch() < m_config.max_batch_size) { + std::optional move_opt = move_gains.pop_max_gain(); + if (!move_opt) { + break; + } + + const Move& move = *move_opt; + auto recomputed_gain = + move_gains.compute_move_gain(move.cls, move.target_dex_index); + if (recomputed_gain <= 0) { + continue; + } + + // Check if it is a valid move. + if (!try_plan_move(move)) { + continue; + } + if (traceEnabled(IDEXR, 5)) { + print_stats(); + } + move_gains.moved_class(move); + } + total_moves += move_gains.moves_this_epoch(); + TRACE(IDEXR, 2, "executed %zu moves in epoch %zu", + move_gains.moves_this_epoch(), batches); + if (move_gains.should_stop()) { + break; + } + } + + m_mgr.incr_metric("max_move_gains", max_move_gains); + m_mgr.incr_metric("total_moves", total_moves); + m_mgr.incr_metric("batches", batches); + m_mgr.incr_metric("first_dex_index", m_first_dex_index); + TRACE(IDEXR, 1, "executed %zu moves in %zu batches", total_moves, batches); + } + + void apply_plan() { + Timer t("finish"); + workqueue_run_for( + m_first_dex_index, m_mutable_dexen.size(), [&](size_t dex_idx) { + auto& dex = m_dexen.at(dex_idx); + const auto& mutable_dex = m_mutable_dexen.at(dex_idx); + auto classes = mutable_dex.get_classes(/* perf_based */ true); + TRACE(IDEXR, 2, "dex %zu: %zu => %zu classes", dex_idx, dex.size(), + classes.size()); + dex = std::move(classes); + }); + } + + private: + void print_stats() { + size_t n_classes = 0; + size_t n_mrefs = 0; + size_t n_frefs = 0; + for (size_t idx = 0; idx < m_mutable_dexen.size(); ++idx) { + auto& mutable_dex = m_mutable_dexen.at(idx); + n_classes += mutable_dex.get_num_classes(); + n_mrefs += mutable_dex.get_num_mrefs(); + n_frefs += mutable_dex.get_num_frefs(); + } + + TRACE(IDEXR, 5, "Global stats:"); + TRACE(IDEXR, 5, "\t %zu classes", n_classes); + TRACE(IDEXR, 5, "\t %zu mrefs", n_mrefs); + TRACE(IDEXR, 5, "\t %zu frefs", n_frefs); + } + + bool try_plan_move(const Move& move) { + auto& target_dex = m_mutable_dexen.at(move.target_dex_index); + always_assert(m_class_refs.count(move.cls)); + const auto& refs = m_class_refs.at(move.cls); + TypeRefs pending_init_class_fields; + TypeRefs pending_init_class_types; + target_dex.resolve_init_classes( + &m_init_classes_with_side_effects, refs.frefs, refs.trefs, refs.itrefs, + &pending_init_class_fields, &pending_init_class_types); + auto laclazz = estimate_linear_alloc(move.cls); + if (!target_dex.add_class_if_fits( + refs.mrefs, refs.frefs, refs.trefs, pending_init_class_fields, + pending_init_class_types, m_linear_alloc_limit, + m_dexes_structure.get_frefs_limit(), + m_dexes_structure.get_mrefs_limit(), + m_dexes_structure.get_trefs_limit(), move.cls)) { + return false; + } + auto& target_dex_strings = + m_mutable_dexen_strings.at(move.target_dex_index); + for (auto* sref : refs.srefs) { + target_dex_strings[sref]++; + } + always_assert(m_class_dex_indices.count(move.cls)); + auto& dex_index = m_class_dex_indices.at(move.cls); + auto& source_dex = m_mutable_dexen.at(dex_index); + source_dex.remove_class(&m_init_classes_with_side_effects, refs.mrefs, + refs.frefs, refs.trefs, pending_init_class_fields, + pending_init_class_types, laclazz, move.cls); + auto& source_dex_strings = m_mutable_dexen_strings.at(dex_index); + for (auto* sref : refs.srefs) { + auto it = source_dex_strings.find(sref); + if (--it->second == 0) { + source_dex_strings.erase(it); + } + } + dex_index = move.target_dex_index; + return true; + } + + bool can_move(DexClass* cls) { + return (!m_order_interdex || + cls->get_perf_sensitive() != PerfSensitiveGroup::BETAMAP_ORDERED) && + !is_canary(cls); + } + + ConfigFiles& m_conf; + PassManager& m_mgr; + InterDexReshufflePass::Config& m_config; + init_classes::InitClassesWithSideEffects m_init_classes_with_side_effects; + DexClassesVector& m_dexen; + size_t m_linear_alloc_limit; + DexesStructure m_dexes_structure; + std::vector m_movable_classes; + std::unordered_map m_class_dex_indices; + std::unordered_map m_class_refs; + std::vector m_mutable_dexen; + std::vector> + m_mutable_dexen_strings; + size_t m_first_dex_index{1}; // skip primary dex + bool m_order_interdex; +}; } // namespace void InterDexReshufflePass::run_pass(DexStoresVector& stores, @@ -50,7 +541,7 @@ void InterDexReshufflePass::run_pass(DexStoresVector& stores, return; } - InterDexReshuffleImpl impl(conf, mgr, m_config, original_scope, root_dexen); + Impl impl(conf, mgr, m_config, original_scope, root_dexen); impl.compute_plan(); impl.apply_plan(); diff --git a/opt/interdex/InterDexReshufflePass.h b/opt/interdex/InterDexReshufflePass.h index 35af0668cc..979440dbb1 100644 --- a/opt/interdex/InterDexReshufflePass.h +++ b/opt/interdex/InterDexReshufflePass.h @@ -9,7 +9,6 @@ #include "DexClass.h" #include "InterDex.h" -#include "InterDexReshuffleImpl.h" #include "Pass.h" /* This pass impls Local Search Algotithm to minize cross-dex refs by @@ -43,6 +42,21 @@ for batch_idx in 1, ..., num_batches: */ class InterDexReshufflePass : public Pass { public: + struct Config { + size_t reserved_extra_frefs{0}; + + size_t reserved_extra_trefs{0}; + + size_t reserved_extra_mrefs{0}; + + size_t extra_linear_alloc_limit{0}; + + size_t max_batches{20}; + + size_t max_batch_size{200000}; + + bool exclude_below20pct_coldstart_classes{false}; + }; explicit InterDexReshufflePass() : Pass("InterDexReshufflePass") {} redex_properties::PropertyInteractions get_property_interactions() @@ -51,11 +65,14 @@ class InterDexReshufflePass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, - {InitialRenameClass, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } + bool is_cfg_legacy() override { return true; } + void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; void bind_config() override { @@ -98,5 +115,5 @@ class InterDexReshufflePass : public Pass { } private: - ReshuffleConfig m_config; + Config m_config; }; diff --git a/opt/interdex/SortRemainingClassesPass.h b/opt/interdex/SortRemainingClassesPass.h index 5f9b9f0ddf..3255095326 100644 --- a/opt/interdex/SortRemainingClassesPass.h +++ b/opt/interdex/SortRemainingClassesPass.h @@ -19,8 +19,8 @@ class SortRemainingClassesPass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, RequiresAndEstablishes}, + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, - {InitialRenameClass, Preserves}, }; } @@ -31,6 +31,8 @@ class SortRemainingClassesPass : public Pass { "Whether to sort classes in primary dex."); } + bool is_cfg_legacy() override { return true; } + void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; private: diff --git a/opt/kotlin-lambda/KotlinObjectInliner.cpp b/opt/kotlin-lambda/KotlinObjectInliner.cpp index ff32517107..bfb6999639 100644 --- a/opt/kotlin-lambda/KotlinObjectInliner.cpp +++ b/opt/kotlin-lambda/KotlinObjectInliner.cpp @@ -74,7 +74,7 @@ bool uses_this(const DexMethod* method, bool strict = false) { } always_assert(!iterable.empty()); live_range::MoveAwareChains chains(cfg); - live_range::Uses first_load_param_uses; + std::unordered_set first_load_param_uses; auto first_load_param = iterable.begin()->insn; first_load_param_uses = @@ -104,7 +104,6 @@ void make_static_and_relocate_method(DexMethod* method, DexType* to_type) { uses_this(method, true) ? mutators::KeepThis::Yes : mutators::KeepThis::No); } - change_visibility(method, to_type); relocate_method(method, to_type); } @@ -357,7 +356,7 @@ void KotlinObjectInliner::run_pass(DexStoresVector& stores, const auto scope = build_class_scope(stores); - InsertOnlyConcurrentMap map; + ConcurrentMap map; ConcurrentSet bad; std::unordered_map outer_cls_count; std::unordered_set do_not_inline_set; diff --git a/opt/kotlin-lambda/KotlinObjectInliner.h b/opt/kotlin-lambda/KotlinObjectInliner.h index 019da6a43c..4f89ce5436 100644 --- a/opt/kotlin-lambda/KotlinObjectInliner.h +++ b/opt/kotlin-lambda/KotlinObjectInliner.h @@ -40,7 +40,10 @@ class KotlinObjectInliner : public Pass { const override { using namespace redex_properties::interactions; using namespace redex_properties::names; - return {}; + return { + {HasSourceBlocks, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, + }; } void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; diff --git a/opt/kotlin-lambda/RewriteKotlinSingletonInstance.cpp b/opt/kotlin-lambda/RewriteKotlinSingletonInstance.cpp index 61eeb4c0a4..4b011540c6 100644 --- a/opt/kotlin-lambda/RewriteKotlinSingletonInstance.cpp +++ b/opt/kotlin-lambda/RewriteKotlinSingletonInstance.cpp @@ -12,6 +12,7 @@ #include "IRCode.h" #include "LocalPointersAnalysis.h" #include "PassManager.h" +#include "ScopedCFG.h" #include "SideEffectSummary.h" #include "SummarySerialization.h" @@ -21,9 +22,8 @@ bool check_inits_has_side_effects( init_classes_with_side_effects, IRCode* code, const std::unordered_set& safe_base_invoke) { - always_assert(code->editable_cfg_built()); - auto& cfg = code->cfg(); - auto iterable = cfg::InstructionIterable(cfg); + cfg::ScopedCFG cfg(code); + auto iterable = cfg::InstructionIterable(*cfg); side_effects::Summary summary(side_effects::EFF_NONE, {}); side_effects::InvokeToSummaryMap summary_map; @@ -38,10 +38,10 @@ bool check_inits_has_side_effects( } } - reaching_defs::MoveAwareFixpointIterator reaching_defs_iter(cfg); + reaching_defs::MoveAwareFixpointIterator reaching_defs_iter(*cfg); reaching_defs_iter.run({}); - local_pointers::FixpointIterator fp_iter(cfg); + local_pointers::FixpointIterator fp_iter(*cfg); fp_iter.run({}); auto side_effect_summary = @@ -50,7 +50,7 @@ bool check_inits_has_side_effects( /* analyze_external_reads */ true) .build(); - return !side_effect_summary.has_side_effects(); + return side_effect_summary.is_pure(); } bool init_for_type_has_side_effects( const init_classes::InitClassesWithSideEffects& diff --git a/opt/kotlin-lambda/RewriteKotlinSingletonInstance.h b/opt/kotlin-lambda/RewriteKotlinSingletonInstance.h index 12be678839..8ea1892007 100644 --- a/opt/kotlin-lambda/RewriteKotlinSingletonInstance.h +++ b/opt/kotlin-lambda/RewriteKotlinSingletonInstance.h @@ -44,9 +44,12 @@ class RewriteKotlinSingletonInstance : public Pass { using namespace redex_properties::interactions; using namespace redex_properties::names; return { + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } + bool is_cfg_legacy() override { return true; } void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; }; diff --git a/opt/layout-reachability/LayoutReachabilityPass.cpp b/opt/layout-reachability/LayoutReachabilityPass.cpp index eaab07f359..d38c020473 100644 --- a/opt/layout-reachability/LayoutReachabilityPass.cpp +++ b/opt/layout-reachability/LayoutReachabilityPass.cpp @@ -18,7 +18,7 @@ void LayoutReachabilityPass::run_pass(DexStoresVector& stores, TRACE(PGR, 1, "Recomputing layout classes"); std::string apk_dir; conf.get_json_config().get("apk_dir", "", apk_dir); - always_assert(!apk_dir.empty()); + always_assert(apk_dir.size()); auto scope = build_class_scope(stores); // Update the m_byresources rstate flags on classes/methods diff --git a/opt/layout-reachability/LayoutReachabilityPass.h b/opt/layout-reachability/LayoutReachabilityPass.h index 5929afc847..07d3774ea9 100644 --- a/opt/layout-reachability/LayoutReachabilityPass.h +++ b/opt/layout-reachability/LayoutReachabilityPass.h @@ -25,9 +25,13 @@ class LayoutReachabilityPass : Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } + bool is_cfg_legacy() override { return true; } + void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; }; diff --git a/opt/local-dce/LocalDcePass.h b/opt/local-dce/LocalDcePass.h index 62f336da89..255211fc84 100644 --- a/opt/local-dce/LocalDcePass.h +++ b/opt/local-dce/LocalDcePass.h @@ -20,14 +20,15 @@ class LocalDcePass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoInitClassInstructions, Preserves}, {NoUnreachableInstructions, Preserves}, {NoResolvablePureRefs, Preserves}, - {SpuriousGetClassCallsInterned, RequiresAndPreserves}, + {NoSpuriousGetClassCalls, Preserves}, {RenameClass, Preserves}, - {InitialRenameClass, Preserves}, }; } + bool is_cfg_legacy() override { return true; } void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; }; diff --git a/opt/make-public/MakePublicPass.h b/opt/make-public/MakePublicPass.h index 05c3fbd65d..344e0aa65d 100644 --- a/opt/make-public/MakePublicPass.h +++ b/opt/make-public/MakePublicPass.h @@ -24,6 +24,7 @@ class MakePublicPass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NeedsEverythingPublic, Destroys}, {NoInitClassInstructions, Preserves}, {NoResolvablePureRefs, Preserves}, @@ -32,5 +33,6 @@ class MakePublicPass : public Pass { }; } + bool is_cfg_legacy() override { return true; } void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; }; diff --git a/opt/merge_interface/MergeInterface.h b/opt/merge_interface/MergeInterface.h index 5a77fd4748..0f67df767c 100644 --- a/opt/merge_interface/MergeInterface.h +++ b/opt/merge_interface/MergeInterface.h @@ -27,7 +27,9 @@ class MergeInterfacePass : public Pass { const override { using namespace redex_properties::interactions; using namespace redex_properties::names; - return {}; + return { + {HasSourceBlocks, Preserves}, + }; } bool is_cfg_legacy() override { return true; } diff --git a/opt/methodinline/BridgeSynthInlinePass.cpp b/opt/methodinline/BridgeSynthInlinePass.cpp index a3ab9acd56..8fa74d13d2 100644 --- a/opt/methodinline/BridgeSynthInlinePass.cpp +++ b/opt/methodinline/BridgeSynthInlinePass.cpp @@ -15,7 +15,6 @@ void BridgeSynthInlinePass::run_pass(DexStoresVector& stores, inliner::run_inliner(stores, mgr, conf, - DEFAULT_COST_CONFIG, /* intra_dex */ false, /* inline_for_speed */ nullptr, /* inline_bridge_synth_only */ true); diff --git a/opt/methodinline/BridgeSynthInlinePass.h b/opt/methodinline/BridgeSynthInlinePass.h index 511b9bf388..4dc0db1b90 100644 --- a/opt/methodinline/BridgeSynthInlinePass.h +++ b/opt/methodinline/BridgeSynthInlinePass.h @@ -19,12 +19,12 @@ class BridgeSynthInlinePass : public Pass { using namespace redex_properties::interactions; using namespace redex_properties::names; return { + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, - // This may be too conservative as the inliner can be configured not to - // DCE in the shrinker. - {SpuriousGetClassCallsInterned, RequiresAndPreserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } + bool is_cfg_legacy() override { return true; } void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; }; diff --git a/opt/methodinline/IntraDexInlinePass.cpp b/opt/methodinline/IntraDexInlinePass.cpp index e072ea80a6..c34fbe4e1f 100644 --- a/opt/methodinline/IntraDexInlinePass.cpp +++ b/opt/methodinline/IntraDexInlinePass.cpp @@ -12,8 +12,7 @@ void IntraDexInlinePass::run_pass(DexStoresVector& stores, ConfigFiles& conf, PassManager& mgr) { - inliner::run_inliner( - stores, mgr, conf, DEFAULT_COST_CONFIG, /* intra_dex */ true); + inliner::run_inliner(stores, mgr, conf, /* intra_dex */ true); } static IntraDexInlinePass s_pass; diff --git a/opt/methodinline/IntraDexInlinePass.h b/opt/methodinline/IntraDexInlinePass.h index c8932a431f..b29168da9c 100644 --- a/opt/methodinline/IntraDexInlinePass.h +++ b/opt/methodinline/IntraDexInlinePass.h @@ -20,13 +20,13 @@ class IntraDexInlinePass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, - // This may be too conservative as the inliner can be configured not - // to DCE in the shrinker. - {SpuriousGetClassCallsInterned, RequiresAndPreserves}, - {InitialRenameClass, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } + bool is_cfg_legacy() override { return true; } + void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; }; diff --git a/opt/methodinline/LocalMethodInlinePass.cpp b/opt/methodinline/LocalMethodInlinePass.cpp index ca4fb4e65b..7901da0210 100644 --- a/opt/methodinline/LocalMethodInlinePass.cpp +++ b/opt/methodinline/LocalMethodInlinePass.cpp @@ -30,8 +30,7 @@ void LocalMethodInlinePass::run_pass(DexStoresVector& stores, ConfigFiles& conf, PassManager& mgr) { - inliner::run_inliner(stores, mgr, conf, DEFAULT_COST_CONFIG, - /* intra_dex */ false, + inliner::run_inliner(stores, mgr, conf, /* intra_dex */ false, /* inline_for_speed */ nullptr, /* inline_bridge_synth_only */ false, /* local_only */ true); diff --git a/opt/methodinline/LocalMethodInlinePass.h b/opt/methodinline/LocalMethodInlinePass.h index ce03652cd7..a26be3233e 100644 --- a/opt/methodinline/LocalMethodInlinePass.h +++ b/opt/methodinline/LocalMethodInlinePass.h @@ -19,12 +19,13 @@ class LocalMethodInlinePass : public Pass { using namespace redex_properties::interactions; using namespace redex_properties::names; return { + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, - // This may be too conservative as the inliner can be configured not to - // DCE in the shrinker. - {SpuriousGetClassCallsInterned, RequiresAndPreserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } + bool is_cfg_legacy() override { return true; } + void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; }; diff --git a/opt/methodinline/MethodInlinePass.cpp b/opt/methodinline/MethodInlinePass.cpp index 60cbf0b0d8..f3490ac401 100644 --- a/opt/methodinline/MethodInlinePass.cpp +++ b/opt/methodinline/MethodInlinePass.cpp @@ -7,62 +7,12 @@ #include "MethodInlinePass.h" -void MethodInlinePass::bind_config() { - size_t cost_invoke; - bind("cost_invoke", - (size_t)(DEFAULT_COST_CONFIG.cost_invoke * 100), - cost_invoke); - m_inliner_cost_config.cost_invoke = (float)cost_invoke / (float)100; - size_t cost_move_result; - bind("cost_move_result", - (size_t)(DEFAULT_COST_CONFIG.cost_move_result * 100), - cost_move_result); - m_inliner_cost_config.cost_move_result = (float)cost_move_result / (float)100; - bind("cost_method", - DEFAULT_COST_CONFIG.cost_method, - m_inliner_cost_config.cost_method); - bind("unused_args_discount", - DEFAULT_COST_CONFIG.unused_args_discount, - m_inliner_cost_config.unused_args_discount); - bind("reg_threshold_1", - DEFAULT_COST_CONFIG.reg_threshold_1, - m_inliner_cost_config.reg_threshold_1); - bind("reg_threshold_2", - DEFAULT_COST_CONFIG.reg_threshold_2, - m_inliner_cost_config.reg_threshold_2); - bind("op_init_class_cost", - DEFAULT_COST_CONFIG.op_init_class_cost, - m_inliner_cost_config.op_init_class_cost); - bind("op_injection_id_cost", - DEFAULT_COST_CONFIG.op_injection_id_cost, - m_inliner_cost_config.op_injection_id_cost); - bind("op_unreachable_cost", - DEFAULT_COST_CONFIG.op_unreachable_cost, - m_inliner_cost_config.op_unreachable_cost); - bind("op_move_exception_cost", - DEFAULT_COST_CONFIG.op_move_exception_cost, - m_inliner_cost_config.op_move_exception_cost); - bind("insn_cost_1", - DEFAULT_COST_CONFIG.insn_cost_1, - m_inliner_cost_config.insn_cost_1); - bind("insn_has_data_cost", - DEFAULT_COST_CONFIG.insn_has_data_cost, - m_inliner_cost_config.insn_has_data_cost); - bind("insn_has_lit_cost_1", - DEFAULT_COST_CONFIG.insn_has_lit_cost_1, - m_inliner_cost_config.insn_has_lit_cost_1); - bind("insn_has_lit_cost_2", - DEFAULT_COST_CONFIG.insn_has_lit_cost_2, - m_inliner_cost_config.insn_has_lit_cost_2); - bind("insn_has_lit_cost_3", - DEFAULT_COST_CONFIG.insn_has_lit_cost_3, - m_inliner_cost_config.insn_has_lit_cost_3); -} +#include "MethodInliner.h" void MethodInlinePass::run_pass(DexStoresVector& stores, ConfigFiles& conf, PassManager& mgr) { - inliner::run_inliner(stores, mgr, conf, m_inliner_cost_config); + inliner::run_inliner(stores, mgr, conf); } static MethodInlinePass s_pass; diff --git a/opt/methodinline/MethodInlinePass.h b/opt/methodinline/MethodInlinePass.h index c8fd1ba158..6ff3efa824 100644 --- a/opt/methodinline/MethodInlinePass.h +++ b/opt/methodinline/MethodInlinePass.h @@ -8,7 +8,6 @@ #pragma once #include "DexClass.h" -#include "MethodInliner.h" #include "Pass.h" class MethodInlinePass : public Pass { @@ -20,17 +19,13 @@ class MethodInlinePass : public Pass { using namespace redex_properties::interactions; using namespace redex_properties::names; return { + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, - // This may be too conservative as the inliner can be configured not to - // DCE in the shrinker. - {SpuriousGetClassCallsInterned, RequiresAndPreserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } - void bind_config() override; + bool is_cfg_legacy() override { return true; } void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; - - private: - InlinerCostConfig m_inliner_cost_config; }; diff --git a/opt/methodinline/PGIForest.h b/opt/methodinline/PGIForest.h index a463d99f73..cd338be74a 100644 --- a/opt/methodinline/PGIForest.h +++ b/opt/methodinline/PGIForest.h @@ -117,9 +117,9 @@ class MethodContextContext { // Somewhat expensive. loop_impl::LoopInfo info(*cfg); res.m_num_loops = info.num_loops(); - for (auto& loop : info) { + for (auto* loop : info) { res.m_deepest_loop = - std::max(res.m_deepest_loop, (uint32_t)loop.get_loop_depth()); + std::max(res.m_deepest_loop, (uint32_t)loop->get_loop_depth()); } return res; diff --git a/opt/methodinline/PerfMethodInlinePass.cpp b/opt/methodinline/PerfMethodInlinePass.cpp index 8eb6d2a177..c5c6a1aaa0 100644 --- a/opt/methodinline/PerfMethodInlinePass.cpp +++ b/opt/methodinline/PerfMethodInlinePass.cpp @@ -889,8 +889,6 @@ enum class IFSMode { } // namespace -PerfMethodInlinePass::PerfMethodInlinePass() : Pass("PerfMethodInlinePass") {} - PerfMethodInlinePass::~PerfMethodInlinePass() {} struct PerfMethodInlinePass::Config { @@ -1134,8 +1132,7 @@ void PerfMethodInlinePass::run_pass(DexStoresVector& stores, not_reached(); }()}; - inliner::run_inliner(stores, mgr, conf, DEFAULT_COST_CONFIG, - /* intra_dex */ true, + inliner::run_inliner(stores, mgr, conf, /* intra_dex */ true, /* inline_for_speed= */ ifs.get()); TRACE(METH_PROF, 1, "Accepted %zu out of %zu choices.", diff --git a/opt/methodinline/PerfMethodInlinePass.h b/opt/methodinline/PerfMethodInlinePass.h index 2bcbad1cac..e28399edfa 100644 --- a/opt/methodinline/PerfMethodInlinePass.h +++ b/opt/methodinline/PerfMethodInlinePass.h @@ -12,7 +12,7 @@ class PerfMethodInlinePass : public Pass { public: - PerfMethodInlinePass(); + PerfMethodInlinePass() : Pass("PerfMethodInlinePass") {} redex_properties::PropertyInteractions get_property_interactions() const override { @@ -22,10 +22,7 @@ class PerfMethodInlinePass : public Pass { {DexLimitsObeyed, Preserves}, {HasSourceBlocks, RequiresAndEstablishes}, {NoResolvablePureRefs, Preserves}, - // This may be too conservative as the inliner can be configured not to - // DCE in the shrinker. - {SpuriousGetClassCallsInterned, RequiresAndPreserves}, - {InitialRenameClass, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } @@ -33,9 +30,11 @@ class PerfMethodInlinePass : public Pass { void bind_config() override; + bool is_cfg_legacy() override { return true; } + void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; private: struct Config; - std::unique_ptr m_config{nullptr}; + std::unique_ptr m_config; }; diff --git a/opt/nullcheck_conversion/IntrinsifyNullChecksPass.cpp b/opt/nullcheck_conversion/IntrinsifyNullChecksPass.cpp index 52a556e642..350974cb7e 100644 --- a/opt/nullcheck_conversion/IntrinsifyNullChecksPass.cpp +++ b/opt/nullcheck_conversion/IntrinsifyNullChecksPass.cpp @@ -34,7 +34,6 @@ void IntrinsifyNullChecksPass::create_null_check_class( cls->rstate.set_generated(); cls->rstate.set_clinit_has_no_side_effects(); cls->rstate.set_name_used(); - cls->rstate.set_dont_rename(); // Crate method for null check. auto meth_name = DexString::make_string("null_check"); DexProto* proto = DexProto::make_proto( @@ -63,8 +62,6 @@ void IntrinsifyNullChecksPass::create_null_check_class( cfg.create_branch(entry_block, dasm(OPCODE_IF_EQZ, {0_v}), return_block, throw_block); cfg.recompute_registers_size(); - method->rstate.set_keepnames(keep_reason::KeepReasonType::UNKNOWN); - method->rstate.set_dont_inline(); cls->add_method(method); TRACE(NCI, 1, "the added method is %s\n", SHOW(method)); TRACE(NCI, 1, "the code is %s\n", SHOW(cfg)); diff --git a/opt/nullcheck_conversion/IntrinsifyNullChecksPass.h b/opt/nullcheck_conversion/IntrinsifyNullChecksPass.h index 935ec7d068..72b9aacc91 100644 --- a/opt/nullcheck_conversion/IntrinsifyNullChecksPass.h +++ b/opt/nullcheck_conversion/IntrinsifyNullChecksPass.h @@ -58,10 +58,8 @@ class IntrinsifyNullChecksPass : public Pass { using namespace redex_properties::interactions; using namespace redex_properties::names; return { - // This is so the checker verifies the efficacy of the pass. + {HasSourceBlocks, Preserves}, {NoSpuriousGetClassCalls, Establishes}, - // This is so the rest of the passes know that this ran. - {SpuriousGetClassCallsInterned, Establishes}, {UltralightCodePatterns, Preserves}, }; } diff --git a/opt/nullcheck_conversion/MaterializeNullChecksPass.h b/opt/nullcheck_conversion/MaterializeNullChecksPass.h index 9ee6cc06c4..f267a4d48d 100644 --- a/opt/nullcheck_conversion/MaterializeNullChecksPass.h +++ b/opt/nullcheck_conversion/MaterializeNullChecksPass.h @@ -43,12 +43,11 @@ class MaterializeNullChecksPass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, - {InitialRenameClass, Preserves}, + {HasSourceBlocks, Preserves}, {NoInitClassInstructions, Preserves}, {NoResolvablePureRefs, Preserves}, {NoUnreachableInstructions, Preserves}, {RenameClass, Preserves}, - {SpuriousGetClassCallsInterned, Requires}, }; } diff --git a/opt/obfuscate/Obfuscate.cpp b/opt/obfuscate/Obfuscate.cpp index d04a17d0f0..389900452a 100644 --- a/opt/obfuscate/Obfuscate.cpp +++ b/opt/obfuscate/Obfuscate.cpp @@ -79,21 +79,25 @@ template DexMember* find_renamable_ref( DexMemberRef* ref, - InsertOnlyConcurrentMap& ref_def_cache, + ConcurrentMap& ref_def_cache, DexElemManager& name_mapping) { TRACE(OBFUSCATE, 4, "Found a ref opcode"); - return *ref_def_cache - .get_or_create_and_assert_equal( - ref, [&](auto*) { return name_mapping.def_of_ref(ref); }) - .first; + DexMember* def = nullptr; + ref_def_cache.update(ref, [&](auto, auto& cache, bool exists) { + if (!exists) { + cache = name_mapping.def_of_ref(ref); + } + def = cache; + }); + return def; } void update_refs(Scope& scope, DexFieldManager& field_name_mapping, DexMethodManager& method_name_mapping, size_t* classes_made_public) { - InsertOnlyConcurrentMap f_ref_def_cache; - InsertOnlyConcurrentMap m_ref_def_cache; + ConcurrentMap f_ref_def_cache; + ConcurrentMap m_ref_def_cache; auto maybe_publicize_class = [&](DexMethod* referrer, DexClass* referree) { if (is_public(referree)) { diff --git a/opt/obfuscate/Obfuscate.h b/opt/obfuscate/Obfuscate.h index 80211115e6..8bf251785c 100644 --- a/opt/obfuscate/Obfuscate.h +++ b/opt/obfuscate/Obfuscate.h @@ -19,11 +19,13 @@ class ObfuscatePass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, - {InitialRenameClass, Preserves}, }; } + bool is_cfg_legacy() override { return true; } + void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; void bind_config() final { trait(Traits::Pass::unique, true); } diff --git a/opt/obfuscate/VirtualRenamer.cpp b/opt/obfuscate/VirtualRenamer.cpp index 17a30d9d10..20ea581225 100644 --- a/opt/obfuscate/VirtualRenamer.cpp +++ b/opt/obfuscate/VirtualRenamer.cpp @@ -107,7 +107,8 @@ DexMethod* find_method(const DexClass* cls, // keep a map from defs to all refs resolving to that def using RefsMap = - ConcurrentMap>; + std::unordered_map>; // Uncomment and use this as a prefix for virtual method // names for debugging @@ -481,6 +482,8 @@ int VirtualRenamer::rename_virtual_scopes(const DexType* type, int& seed) { * Collect all method refs to concrete methods (definitions). */ void collect_refs(Scope& scope, RefsMap& def_refs) { + ConcurrentMap> + concurrent_def_refs; walk::parallel::opcodes( scope, [](DexMethod*) { return true; }, [&](DexMethod*, IRInstruction* insn) { @@ -508,9 +511,10 @@ void collect_refs(Scope& scope, RefsMap& def_refs) { redex_assert(type_class(top->get_class()) != nullptr); if (type_class(top->get_class())->is_external()) return; // it's a top definition on an internal class, save it - def_refs.update(top, - [&](auto*, auto& set, bool) { set.insert(callee); }); + concurrent_def_refs.update( + top, [&](auto*, auto& set, bool) { set.insert(callee); }); }); + def_refs = concurrent_def_refs.move_to_container(); } } // namespace diff --git a/opt/obfuscate_resources/ObfuscateResourcesPass.cpp b/opt/obfuscate_resources/ObfuscateResourcesPass.cpp index 8eb1888777..ed05c248b4 100644 --- a/opt/obfuscate_resources/ObfuscateResourcesPass.cpp +++ b/opt/obfuscate_resources/ObfuscateResourcesPass.cpp @@ -7,7 +7,6 @@ #include "ObfuscateResourcesPass.h" -#include #include #include #include @@ -38,21 +37,19 @@ const std::string RESFILE_MAPPING = "resource-mapping.txt"; const std::string DOT_DELIM = "."; const std::string RES_START = std::string(RES_DIRECTORY) + "/"; const std::string SHORTEN_START = std::string(OBFUSCATED_RES_DIRECTORY) + "/"; -constexpr std::string_view PORT_CHAR = "abcdefghijklmnopqrstuvwxyz0123456789_-"; +const std::string PORT_CHAR = "abcdefghijklmnopqrstuvwxyz0123456789_-"; const std::string FONT_DIR = "/font/"; std::string get_short_name_from_index(size_t index) { always_assert(index >= 0); std::string to_return; auto PORT_CHAR_length = PORT_CHAR.length(); - to_return.reserve(1 + index / PORT_CHAR_length); while (index >= PORT_CHAR_length) { size_t i = index % PORT_CHAR_length; - to_return += PORT_CHAR[i]; + to_return = PORT_CHAR[i] + to_return; index = index / PORT_CHAR_length; } - to_return += PORT_CHAR[index]; - std::reverse(to_return.begin(), to_return.end()); + to_return = PORT_CHAR[index] + to_return; return to_return; } @@ -185,9 +182,7 @@ void collect_string_values_from_code( // Checking things like proto / type names is probably unnecessary. Just // look at instructions. if (code != nullptr) { - always_assert(code->editable_cfg_built()); - auto& cfg = code->cfg(); - cfg.gather_strings(strings); + code->gather_strings(strings); } } } @@ -314,14 +309,6 @@ void ObfuscateResourcesPass::run_pass(DexStoresVector& stores, m_keep_resource_name_prefixes, keep_resource_names_specific); mgr.incr_metric("num_anonymized_resource_names", changed); mgr.incr_metric("num_anonymized_resource_files", filepath_old_to_new.size()); - - // Obfuscation may create duplicate entry/value structures. Make sure to set - // relevant values in BundleConfig.pb to take advantage of that (if - // configured). - const auto& global_config = conf.get_global_config(); - auto global_resources_config = - global_config.get_config_by_name("resources"); - resources->finalize_bundle_config(*global_resources_config); } static ObfuscateResourcesPass s_pass; diff --git a/opt/obfuscate_resources/ObfuscateResourcesPass.h b/opt/obfuscate_resources/ObfuscateResourcesPass.h index badbba1a38..222376c8f8 100644 --- a/opt/obfuscate_resources/ObfuscateResourcesPass.h +++ b/opt/obfuscate_resources/ObfuscateResourcesPass.h @@ -36,6 +36,7 @@ class ObfuscateResourcesPass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoInitClassInstructions, Preserves}, {NoResolvablePureRefs, Preserves}, {NoUnreachableInstructions, Preserves}, @@ -87,6 +88,8 @@ class ObfuscateResourcesPass : public Pass { m_code_references_okay_to_obfuscate); } + bool is_cfg_legacy() override { return true; } + void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; private: diff --git a/opt/object-escape-analysis/ExpandableMethodParams.cpp b/opt/object-escape-analysis/ExpandableMethodParams.cpp index a502be46ea..6b57fdf32f 100644 --- a/opt/object-escape-analysis/ExpandableMethodParams.cpp +++ b/opt/object-escape-analysis/ExpandableMethodParams.cpp @@ -56,107 +56,107 @@ std::vector ExpandableMethodParams::get_expanded_args_vector( return args_vector; } -ExpandableMethodParams::MethodInfo ExpandableMethodParams::create_method_info( +// Get or create the method-info for a given type, method-name, rtype. +ExpandableMethodParams::MethodInfo* ExpandableMethodParams::get_method_info( const MethodKey& key) const { - MethodInfo res; - auto cls = type_class(key.type); - if (!cls) { - return res; + auto res = m_method_infos.get(key, nullptr); + if (res) { + return res.get(); } - std::set> args_vectors; - // First, for constructors, collect all of the (guaranteed to be distinct) - // args. - if (key.name->str() == "") { + res = std::make_shared(); + auto cls = type_class(key.type); + if (cls) { + std::set> args_vectors; + // First, for constructors, collect all of the (guaranteed to be distinct) + // args. + if (key.name->str() == "") { + for (auto* method : cls->get_all_methods()) { + if (method->get_name() != key.name || + method->get_proto()->get_rtype() != key.rtype) { + continue; + } + auto args = method->get_proto()->get_args(); + std::vector args_vector(args->begin(), args->end()); + auto inserted = args_vectors.insert(std::move(args_vector)).second; + always_assert(inserted); + } + } + // Second, for each matching method, and each (non-receiver) parameter + // that is only used in igets, compute the expanded constructor args and + // record them if they don't create a conflict. for (auto* method : cls->get_all_methods()) { if (method->get_name() != key.name || method->get_proto()->get_rtype() != key.rtype) { continue; } - auto args = method->get_proto()->get_args(); - std::vector args_vector(args->begin(), args->end()); - auto inserted = args_vectors.insert(std::move(args_vector)).second; - always_assert(inserted); - } - } - // Second, for each matching method, and each (non-receiver) parameter - // that is only used in igets, compute the expanded constructor args and - // record them if they don't create a conflict. - for (auto* method : cls->get_all_methods()) { - if (method->get_name() != key.name || - method->get_proto()->get_rtype() != key.rtype) { - continue; - } - auto code = method->get_code(); - if (!code || method->rstate.no_optimizations()) { - continue; - } - live_range::MoveAwareChains chains( - code->cfg(), - /* ignore_unreachable */ false, - [](auto* insn) { return opcode::is_a_load_param(insn->opcode()); }); - auto du_chains = chains.get_def_use_chains(); - param_index_t param_index{0}; - auto ii = code->cfg().get_param_instructions(); - auto begin = ii.begin(); - if (method::is_init(method)) { - begin++; - param_index++; - } - for (auto it = begin; it != ii.end(); it++, param_index++) { - auto insn = it->insn; - if (insn->opcode() != IOPCODE_LOAD_PARAM_OBJECT) { + auto code = method->get_code(); + if (!code || method->rstate.no_optimizations()) { continue; } - bool expandable{true}; - std::vector fields; - for (auto& use : du_chains[it->insn]) { - if (opcode::is_an_iget(use.insn->opcode())) { - auto* field = - resolve_field(use.insn->get_field(), FieldSearch::Instance); - if (field) { - fields.push_back(field); - continue; - } - } - expandable = false; - break; - } - if (!expandable) { - continue; - } - std::sort(fields.begin(), fields.end(), compare_dexfields); - // remove duplicates - fields.erase(std::unique(fields.begin(), fields.end()), fields.end()); - auto expanded_args_vector = - get_expanded_args_vector(method, param_index, fields); - // We need to check if we don't have too many args that won't fit - // into an invoke/range instruction. - uint32_t range_size = 0; + live_range::MoveAwareChains chains(code->cfg()); + auto du_chains = chains.get_def_use_chains(); + param_index_t param_index{0}; + auto ii = code->cfg().get_param_instructions(); + auto begin = ii.begin(); if (method::is_init(method)) { - range_size++; + begin++; + param_index++; } - for (auto arg_type : expanded_args_vector) { - range_size += type::is_wide_type(arg_type) ? 2 : 1; - } - if (range_size <= 0xff) { - auto inserted = - args_vectors.insert(std::move(expanded_args_vector)).second; - if (inserted) { - res[method].emplace(param_index, std::move(fields)); + for (auto it = begin; it != ii.end(); it++, param_index++) { + auto insn = it->insn; + if (insn->opcode() != IOPCODE_LOAD_PARAM_OBJECT) { + continue; + } + bool expandable{true}; + std::vector fields; + for (auto& use : du_chains[it->insn]) { + if (opcode::is_an_iget(use.insn->opcode())) { + auto* field = + resolve_field(use.insn->get_field(), FieldSearch::Instance); + if (field) { + fields.push_back(field); + continue; + } + } + expandable = false; + break; + } + if (!expandable) { + continue; + } + std::sort(fields.begin(), fields.end(), compare_dexfields); + // remove duplicates + fields.erase(std::unique(fields.begin(), fields.end()), fields.end()); + auto expanded_args_vector = + get_expanded_args_vector(method, param_index, fields); + // We need to check if we don't have too many args that won't fit + // into an invoke/range instruction. + uint32_t range_size = 0; + if (method::is_init(method)) { + range_size++; + } + for (auto arg_type : expanded_args_vector) { + range_size += type::is_wide_type(arg_type) ? 2 : 1; + } + if (range_size <= 0xff) { + auto inserted = + args_vectors.insert(std::move(expanded_args_vector)).second; + if (inserted) { + (*res)[method].emplace(param_index, std::move(fields)); + } } } } } - return res; -} - -// Get or create the method-info for a given type, method-name, rtype. -const ExpandableMethodParams::MethodInfo* -ExpandableMethodParams::get_method_info(const MethodKey& key) const { - return m_method_infos - .get_or_create_and_assert_equal( - key, [this](const auto& _) { return create_method_info(_); }) - .first; + m_method_infos.update(key, [&](const auto&, auto& value, bool exists) { + if (exists) { + // Oh well, we wasted some racing with another thread. + res = value; + return; + } + value = res; + }); + return res.get(); } DexMethod* ExpandableMethodParams::make_expanded_method_concrete( @@ -192,7 +192,7 @@ DexMethod* ExpandableMethodParams::make_expanded_method_concrete( std::vector new_load_param_insns; std::unordered_map field_regs; auto& fields = m_method_infos.at_unsafe(MethodKey::from_method(method)) - .at(method) + ->at(method) .at(param_index); for (auto field : fields) { auto reg = type::is_wide_type(field->get_type()) ? cfg.allocate_wide_temp() @@ -209,9 +209,7 @@ DexMethod* ExpandableMethodParams::make_expanded_method_concrete( // Replace all igets on the (newly created) object with moves from the new // field value load-params. No other (non-move) uses of the (newly created) // object can exist. - live_range::MoveAwareChains chains( - cfg, /* ignore_unreachable */ false, - [](auto* insn) { return opcode::is_a_load_param(insn->opcode()); }); + live_range::MoveAwareChains chains(cfg); auto du_chains = chains.get_def_use_chains(); std::unordered_set use_insns; for (auto& use : du_chains[load_param_it->insn]) { @@ -268,7 +266,7 @@ ExpandableMethodParams::ExpandableMethodParams(const Scope& scope) { DexMethodRef* ExpandableMethodParams::get_expanded_method_ref( DexMethod* method, param_index_t param_index, - std::vector const** fields) const { + std::vector** fields) const { auto method_info = get_method_info(MethodKey::from_method(method)); auto it = method_info->find(method); if (it == method_info->end()) { diff --git a/opt/object-escape-analysis/ExpandableMethodParams.h b/opt/object-escape-analysis/ExpandableMethodParams.h index c00f87642e..e17f77c869 100644 --- a/opt/object-escape-analysis/ExpandableMethodParams.h +++ b/opt/object-escape-analysis/ExpandableMethodParams.h @@ -56,11 +56,8 @@ class ExpandableMethodParams { param_index_t param_index, const std::vector& fields); - // create the method-info for a given type, method-name, rtype. - MethodInfo create_method_info(const MethodKey& key) const; - // Get or create the method-info for a given type, method-name, rtype. - const MethodInfo* get_method_info(const MethodKey& key) const; + MethodInfo* get_method_info(const MethodKey& key) const; // Given an earlier created expanded method ref, fill in the code. DexMethod* make_expanded_method_concrete(DexMethodRef* expanded_method_ref); @@ -74,14 +71,14 @@ class ExpandableMethodParams { DexMethodRef* get_expanded_method_ref( DexMethod* method, param_index_t param_index, - std::vector const** fields = nullptr) const; + std::vector** fields = nullptr) const; // Make sure that all newly used expanded ctors actually exist as concrete // methods. size_t flush(const Scope& scope); private: - mutable InsertOnlyConcurrentMap + mutable ConcurrentMap, MethodKeyHash> m_method_infos; // For each requested expanded method ref, we remember the // original method, and which parameter was expanded. diff --git a/opt/object-escape-analysis/ObjectEscapeAnalysis.cpp b/opt/object-escape-analysis/ObjectEscapeAnalysis.cpp index 4e68e92e92..c62b473044 100644 --- a/opt/object-escape-analysis/ObjectEscapeAnalysis.cpp +++ b/opt/object-escape-analysis/ObjectEscapeAnalysis.cpp @@ -55,22 +55,39 @@ #include "ObjectEscapeAnalysis.h" +#include #include +#include +#include +#include +#include +#include + #include "ApiLevelChecker.h" +#include "BaseIRAnalyzer.h" #include "CFGMutation.h" #include "ConfigFiles.h" +#include "ControlFlow.h" +#include "DexClass.h" #include "ExpandableMethodParams.h" +#include "HierarchyUtil.h" +#include "IRCode.h" #include "IRInstruction.h" +#include "IROpcode.h" #include "Inliner.h" -#include "ObjectEscapeAnalysisImpl.h" +#include "Lazy.h" +#include "LiveRange.h" +#include "MethodOverrideGraph.h" #include "PassManager.h" +#include "Resolver.h" +#include "Show.h" #include "StringBuilder.h" #include "Walkers.h" using namespace sparta; -using namespace object_escape_analysis_impl; namespace mog = method_override_graph; +namespace hier = hierarchy_util; namespace { @@ -101,34 +118,384 @@ constexpr int64_t COST_MOVE_RESULT = 30; // Overhead of a new-instance instruction, times 10. constexpr int64_t COST_NEW_INSTANCE = 20; +using Locations = std::vector>; + +// Collect all allocation and invoke instructions, as well as non-virtual +// invocation dependencies. +void analyze_scope( + const Scope& scope, + const std::unordered_set& non_overridden_virtuals, + std::unordered_map* new_instances, + std::unordered_map* invokes, + std::unordered_map>* + dependencies) { + Timer t("analyze_scope"); + ConcurrentMap concurrent_new_instances; + ConcurrentMap concurrent_invokes; + ConcurrentMap> + concurrent_dependencies; + walk::parallel::code(scope, [&](DexMethod* method, IRCode& code) { + code.build_cfg(/* editable */ true); + for (auto& mie : InstructionIterable(code.cfg())) { + auto insn = mie.insn; + if (insn->opcode() == OPCODE_NEW_INSTANCE) { + auto cls = type_class(insn->get_type()); + if (cls && !cls->is_external()) { + concurrent_new_instances.update( + insn->get_type(), + [&](auto*, auto& vec, bool) { vec.emplace_back(method, insn); }); + } + } else if (opcode::is_an_invoke(insn->opcode())) { + auto callee = + resolve_method(insn->get_method(), opcode_to_search(insn)); + if (callee && + (!callee->is_virtual() || non_overridden_virtuals.count(callee))) { + concurrent_invokes.update(callee, [&](auto*, auto& vec, bool) { + vec.emplace_back(method, insn); + }); + if (!method->is_virtual() || non_overridden_virtuals.count(method)) { + concurrent_dependencies.update( + callee, + [method](auto, auto& set, auto) { set.insert(method); }); + } + } + } + } + }); + *new_instances = concurrent_new_instances.move_to_container(); + *invokes = concurrent_invokes.move_to_container(); + *dependencies = concurrent_dependencies.move_to_container(); +} + +// A benign method invocation can be ignored during the escape analysis. +bool is_benign(const DexMethodRef* method_ref) { + static const std::unordered_set methods = { + // clang-format off + "Ljava/lang/Object;.:()V", + // clang-format on + }; + + return method_ref->is_def() && + methods.count( + method_ref->as_def()->get_deobfuscated_name_or_empty_copy()); +} + +constexpr const IRInstruction* NO_ALLOCATION = nullptr; + +using namespace ir_analyzer; + +// For each allocating instruction that escapes (not including returns), all +// uses by which it escapes. +using Escapes = std::unordered_map>; + +// For each object, we track which instruction might have allocated it: +// - new-instance, invoke-, and load-param-object instructions might represent +// allocation points +// - NO_ALLOCATION is a value for which the allocation instruction is not known, +// or it is not an object +using Domain = sparta::PatriciaTreeSetAbstractDomain; + +// For each register that holds a relevant value, keep track of it. +using Environment = sparta::PatriciaTreeMapAbstractEnvironment; + +struct MethodSummary { + // A parameter is "benign" if a provided argument does not escape + std::unordered_set benign_params; + // A method might contain a unique instruction which allocates an object that + // is eventually unconditionally returned. + const IRInstruction* allocation_insn{nullptr}; +}; + +using MethodSummaries = std::unordered_map; + +// The analyzer computes... +// - which instructions allocate (new-instance, invoke-) +// - which allocations escape (and how) +// - which allocations return +class Analyzer final : public BaseIRAnalyzer { + public: + explicit Analyzer(const MethodSummaries& method_summaries, + DexMethodRef* incomplete_marker_method, + cfg::ControlFlowGraph& cfg) + : BaseIRAnalyzer(cfg), + m_method_summaries(method_summaries), + m_incomplete_marker_method(incomplete_marker_method) { + MonotonicFixpointIterator::run(Environment::top()); + } + + static const IRInstruction* get_singleton_allocation(const Domain& domain) { + always_assert(domain.kind() == AbstractValueKind::Value); + auto& elements = domain.elements(); + if (elements.size() != 1) { + return nullptr; + } + return *elements.begin(); + } + + void analyze_instruction(const IRInstruction* insn, + Environment* current_state) const override { + + const auto escape = [&](src_index_t src_idx) { + auto reg = insn->src(src_idx); + const auto& domain = current_state->get(reg); + always_assert(domain.kind() == AbstractValueKind::Value); + for (auto allocation_insn : domain.elements()) { + if (allocation_insn != NO_ALLOCATION) { + m_escapes[allocation_insn].insert( + {const_cast(insn), src_idx}); + } + } + }; + + if (insn->opcode() == OPCODE_NEW_INSTANCE) { + auto type = insn->get_type(); + auto cls = type_class(type); + if (cls && !cls->is_external()) { + m_escapes[insn]; + current_state->set(RESULT_REGISTER, Domain(insn)); + return; + } + } else if (insn->opcode() == IOPCODE_LOAD_PARAM_OBJECT) { + m_escapes[insn]; + current_state->set(insn->dest(), Domain(insn)); + return; + } else if (insn->opcode() == OPCODE_RETURN_OBJECT) { + const auto& domain = current_state->get(insn->src(0)); + always_assert(domain.kind() == AbstractValueKind::Value); + m_returns.insert(domain.elements().begin(), domain.elements().end()); + return; + } else if (insn->opcode() == OPCODE_MOVE_RESULT_OBJECT || + insn->opcode() == IOPCODE_MOVE_RESULT_PSEUDO_OBJECT) { + const auto& domain = current_state->get(RESULT_REGISTER); + current_state->set(insn->dest(), domain); + return; + } else if (insn->opcode() == OPCODE_MOVE_OBJECT) { + const auto& domain = current_state->get(insn->src(0)); + current_state->set(insn->dest(), domain); + return; + } else if (insn->opcode() == OPCODE_INSTANCE_OF || + opcode::is_an_iget(insn->opcode())) { + if (get_singleton_allocation(current_state->get(insn->src(0)))) { + current_state->set(RESULT_REGISTER, Domain(NO_ALLOCATION)); + return; + } + } else if (opcode::is_a_monitor(insn->opcode()) || + insn->opcode() == OPCODE_IF_EQZ || + insn->opcode() == OPCODE_IF_NEZ) { + if (get_singleton_allocation(current_state->get(insn->src(0)))) { + return; + } + } else if (opcode::is_an_iput(insn->opcode())) { + if (get_singleton_allocation(current_state->get(insn->src(1)))) { + escape(0); + return; + } + } else if (opcode::is_an_invoke(insn->opcode())) { + if (is_benign(insn->get_method()) || is_incomplete_marker(insn)) { + current_state->set(RESULT_REGISTER, Domain(NO_ALLOCATION)); + return; + } + auto callee = resolve_method(insn->get_method(), opcode_to_search(insn)); + auto it = m_method_summaries.find(callee); + auto benign_params = + it == m_method_summaries.end() ? nullptr : &it->second.benign_params; + for (src_index_t i = 0; i < insn->srcs_size(); i++) { + if (!benign_params || !benign_params->count(i) || + !get_singleton_allocation(current_state->get(insn->src(i)))) { + escape(i); + } + } + + Domain domain(NO_ALLOCATION); + if (it != m_method_summaries.end() && it->second.allocation_insn) { + m_escapes[insn]; + domain = Domain(insn); + } + current_state->set(RESULT_REGISTER, domain); + return; + } + + for (src_index_t i = 0; i < insn->srcs_size(); i++) { + escape(i); + } + + if (insn->has_dest()) { + current_state->set(insn->dest(), Domain(NO_ALLOCATION)); + if (insn->dest_is_wide()) { + current_state->set(insn->dest() + 1, Domain::top()); + } + } else if (insn->has_move_result_any()) { + current_state->set(RESULT_REGISTER, Domain(NO_ALLOCATION)); + } + } + + const Escapes& get_escapes() { return m_escapes; } + + const std::unordered_set& get_returns() { + return m_returns; + } + + // Returns set of new-instance and invoke- allocating instructions that do not + // escape (or return). + std::unordered_set get_inlinables() { + std::unordered_set inlinables; + for (auto&& [insn, uses] : m_escapes) { + if (uses.empty() && insn->opcode() != IOPCODE_LOAD_PARAM_OBJECT && + !m_returns.count(insn)) { + inlinables.insert(const_cast(insn)); + } + } + return inlinables; + } + + private: + const MethodSummaries& m_method_summaries; + DexMethodRef* m_incomplete_marker_method; + mutable Escapes m_escapes; + mutable std::unordered_set m_returns; + + bool is_incomplete_marker(const IRInstruction* insn) const { + return insn->opcode() == OPCODE_INVOKE_STATIC && + insn->get_method() == m_incomplete_marker_method; + } +}; + +MethodSummaries compute_method_summaries( + PassManager& mgr, + const Scope& scope, + const std::unordered_map>& + dependencies, + const std::unordered_set& non_overridden_virtuals) { + Timer t("compute_method_summaries"); + + std::unordered_set impacted_methods; + walk::code(scope, [&](DexMethod* method, IRCode&) { + if (!method->is_virtual() || non_overridden_virtuals.count(method)) { + impacted_methods.insert(method); + } + }); + + MethodSummaries method_summaries; + size_t analysis_iterations = 0; + while (!impacted_methods.empty()) { + Timer t2("analysis iteration"); + analysis_iterations++; + TRACE(OEA, 2, "[object escape analysis] analysis_iteration %zu", + analysis_iterations); + ConcurrentMap recomputed_method_summaries; + workqueue_run( + [&](DexMethod* method) { + auto& cfg = method->get_code()->cfg(); + Analyzer analyzer(method_summaries, + /* incomplete_marker_method */ nullptr, cfg); + const auto& escapes = analyzer.get_escapes(); + const auto& returns = analyzer.get_returns(); + src_index_t src_index = 0; + for (auto& mie : InstructionIterable(cfg.get_param_instructions())) { + if (mie.insn->opcode() == IOPCODE_LOAD_PARAM_OBJECT && + escapes.at(mie.insn).empty() && !returns.count(mie.insn)) { + recomputed_method_summaries.update( + method, [src_index](DexMethod*, auto& ms, bool) { + ms.benign_params.insert(src_index); + }); + } + src_index++; + } + const IRInstruction* allocation_insn; + if (returns.size() == 1 && + (allocation_insn = *returns.begin()) != NO_ALLOCATION && + escapes.at(allocation_insn).empty() && + allocation_insn->opcode() != IOPCODE_LOAD_PARAM_OBJECT) { + recomputed_method_summaries.update( + method, [allocation_insn](DexMethod*, auto& ms, bool) { + ms.allocation_insn = allocation_insn; + }); + } + }, + impacted_methods); + + std::unordered_set changed_methods; + // (Recomputed) summaries can only grow; assert that, update summaries when + // necessary, and remember for which methods the summaries actually changed. + for (auto&& [method, recomputed_summary] : recomputed_method_summaries) { + auto& summary = method_summaries[method]; + for (auto src_index : summary.benign_params) { + always_assert(recomputed_summary.benign_params.count(src_index)); + } + if (recomputed_summary.benign_params.size() > + summary.benign_params.size()) { + summary.benign_params = std::move(recomputed_summary.benign_params); + changed_methods.insert(method); + } + if (recomputed_summary.allocation_insn) { + if (summary.allocation_insn) { + always_assert(summary.allocation_insn == + recomputed_summary.allocation_insn); + } else { + summary.allocation_insn = recomputed_summary.allocation_insn; + changed_methods.insert(method); + } + } else { + always_assert(summary.allocation_insn == nullptr); + } + } + impacted_methods.clear(); + for (auto method : changed_methods) { + auto it = dependencies.find(method); + if (it != dependencies.end()) { + impacted_methods.insert(it->second.begin(), it->second.end()); + } + } + } + mgr.incr_metric("analysis_iterations", analysis_iterations); + return method_summaries; +} + +// For an inlinable new-instance or invoke- instruction, determine first +// resolved callee (if any), and (eventually) allocated type +std::pair resolve_inlinable( + const MethodSummaries& method_summaries, const IRInstruction* insn) { + always_assert(insn->opcode() == OPCODE_NEW_INSTANCE || + opcode::is_an_invoke(insn->opcode())); + DexMethod* first_callee{nullptr}; + while (insn->opcode() != OPCODE_NEW_INSTANCE) { + always_assert(opcode::is_an_invoke(insn->opcode())); + auto callee = resolve_method(insn->get_method(), opcode_to_search(insn)); + if (!first_callee) { + first_callee = callee; + } + insn = method_summaries.at(callee).allocation_insn; + } + return std::make_pair(first_callee, insn->get_type()); +} + using InlineAnchorsOfType = std::unordered_map>; -ConcurrentMap compute_inline_anchors( - const Scope& scope, - const MethodSummaries& method_summaries, - const std::unordered_set& excluded_classes) { +std::unordered_map compute_inline_anchors( + const Scope& scope, const MethodSummaries& method_summaries) { Timer t("compute_inline_anchors"); - ConcurrentMap inline_anchors; + ConcurrentMap concurrent_inline_anchors; walk::parallel::code(scope, [&](DexMethod* method, IRCode& code) { - Analyzer analyzer(excluded_classes, method_summaries, - /* incomplete_marker_method */ nullptr, code.cfg()); + Analyzer analyzer(method_summaries, /* incomplete_marker_method */ nullptr, + code.cfg()); auto inlinables = analyzer.get_inlinables(); for (auto insn : inlinables) { auto [callee, type] = resolve_inlinable(method_summaries, insn); TRACE(OEA, 3, "[object escape analysis] inline anchor [%s] %s", SHOW(method), SHOW(insn)); - inline_anchors.update( + concurrent_inline_anchors.update( type, [&](auto*, auto& map, bool) { map[method].insert(insn); }); } }); - return inline_anchors; + return concurrent_inline_anchors.move_to_container(); } class InlinedCodeSizeEstimator { private: using DeltaKey = std::pair; LazyUnorderedMap m_inlined_code_sizes; - LazyUnorderedMap> m_uses; + LazyUnorderedMap m_du_chains; LazyUnorderedMap> m_deltas; public: @@ -148,22 +515,14 @@ class InlinedCodeSizeEstimator { } return code_size; }), - m_uses([&](DeltaKey key) { - auto allocation_insn = const_cast(key.second); - live_range::MoveAwareChains chains(key.first->get_code()->cfg(), - /* ignore_unreachable */ false, - [&](const IRInstruction* insn) { - return insn == allocation_insn; - }); - return chains.get_def_use_chains()[allocation_insn]; + m_du_chains([](DexMethod* method) { + live_range::MoveAwareChains chains(method->get_code()->cfg()); + return chains.get_def_use_chains(); }), m_deltas([&](DeltaKey key) { auto [method, allocation_insn] = key; - always_assert( - opcode::is_new_instance(allocation_insn->opcode()) || - opcode::is_an_invoke(allocation_insn->opcode()) || - opcode::is_load_param_object(allocation_insn->opcode())); int64_t delta = 0; + auto& du_chains = m_du_chains[method]; if (opcode::is_an_invoke(allocation_insn->opcode())) { auto callee = resolve_method(allocation_insn->get_method(), opcode_to_search(allocation_insn)); @@ -177,7 +536,8 @@ class InlinedCodeSizeEstimator { } else if (allocation_insn->opcode() == OPCODE_NEW_INSTANCE) { delta -= COST_NEW_INSTANCE; } - for (auto& use : m_uses[key]) { + for (auto& use : + du_chains[const_cast(allocation_insn)]) { if (opcode::is_an_invoke(use.insn->opcode())) { delta -= COST_INVOKE; auto callee = resolve_method(use.insn->get_method(), @@ -191,17 +551,14 @@ class InlinedCodeSizeEstimator { auto* load_param_insn = std::next(load_param_insns.begin(), use.src_index)->insn; always_assert(load_param_insn); - always_assert( - opcode::is_load_param_object(load_param_insn->opcode())); + always_assert(opcode::is_a_load_param(load_param_insn->opcode())); delta += 10 * (int64_t)m_inlined_code_sizes[callee] + get_delta(callee, load_param_insn); if (!callee->get_proto()->is_void()) { delta -= COST_MOVE_RESULT; } } else if (opcode::is_an_iget(use.insn->opcode()) || - opcode::is_an_iput(use.insn->opcode()) || - opcode::is_instance_of(use.insn->opcode()) || - opcode::is_a_monitor(use.insn->opcode())) { + opcode::is_an_iput(use.insn->opcode())) { delta -= 10 * (int64_t)use.insn->size(); } } @@ -231,10 +588,10 @@ using InlinableTypes = std::unordered_map; std::unordered_map compute_root_methods( PassManager& mgr, - const ConcurrentMap& new_instances, - const ConcurrentMap& invokes, + const std::unordered_map& new_instances, + const std::unordered_map& invokes, const MethodSummaries& method_summaries, - const ConcurrentMap& inline_anchors) { + const std::unordered_map& inline_anchors) { Timer t("compute_root_methods"); std::array candidate_types{ 0, 0, 0}; @@ -243,16 +600,11 @@ std::unordered_map compute_root_methods( std::mutex mutex; // protects candidate_types and root_methods std::atomic incomplete_estimated_delta_threshold_exceeded{0}; - auto concurrent_add_root_methods = [&](DexType* type, bool complete) { - const auto& inline_anchors_of_type = inline_anchors.at_unsafe(type); + const auto& inline_anchors_of_type = inline_anchors.at(type); InlinedCodeSizeEstimator inlined_code_size_estimator(method_summaries); std::vector methods; for (auto& [method, allocation_insns] : inline_anchors_of_type) { - if (method->rstate.no_optimizations()) { - always_assert(!complete); - continue; - } auto it2 = method_summaries.find(method); if (it2 != method_summaries.end() && it2->second.allocation_insn && resolve_inlinable(method_summaries, it2->second.allocation_insn) @@ -306,8 +658,8 @@ std::unordered_map compute_root_methods( } workqueue_run( [&](DexType* type) { - const auto& method_insn_pairs = new_instances.at_unsafe(type); - const auto& inline_anchors_of_type = inline_anchors.at_unsafe(type); + const auto& method_insn_pairs = new_instances.at(type); + const auto& inline_anchors_of_type = inline_anchors.at(type); std::function&)> is_anchored; @@ -365,9 +717,9 @@ std::unordered_map compute_root_methods( return root_methods; } -size_t shrink_root_methods( +void shrink_root_methods( MultiMethodInliner& inliner, - const ConcurrentMap>& + const std::unordered_map>& dependencies, const std::unordered_map& root_methods, MethodSummaries* method_summaries) { @@ -418,7 +770,6 @@ size_t shrink_root_methods( always_assert(allocation_insn != nullptr); allocation_insn = nullptr; } - return methods_that_lost_allocation_insns.size(); } size_t get_code_size(DexMethod* method) { @@ -427,9 +778,9 @@ size_t get_code_size(DexMethod* method) { struct Stats { std::atomic total_savings{0}; - size_t reduced_methods{0}; + std::atomic reduced_methods{0}; std::atomic reduced_methods_variants{0}; - size_t selected_reduced_methods{0}; + std::atomic selected_reduced_methods{0}; std::atomic invokes_not_inlinable_callee_unexpandable{0}; std::atomic invokes_not_inlinable_callee_inconcrete{0}; std::atomic invokes_not_inlinable_callee_too_many_params_to_expand{0}; @@ -443,9 +794,6 @@ struct Stats { std::atomic expanded_methods{0}; std::atomic calls_inlined{0}; std::atomic new_instances_eliminated{0}; - size_t inlined_methods_removed{0}; - size_t inlinable_methods_kept{0}; - size_t inlined_methods_mispredicted{0}; }; // Data structure to derive local or accumulated global net savings @@ -462,8 +810,7 @@ struct NetSavings { } // Estimate how many code units will be saved. - int get_value( - const InsertOnlyConcurrentSet* methods_kept = nullptr) const { + int get_value() const { int net_savings{local}; // A class will only eventually get deleted if all static methods are // inlined. @@ -479,8 +826,7 @@ struct NetSavings { } } for (auto method : methods) { - if (can_delete(method) && !method::is_argless_init(method) && - (!methods_kept || !methods_kept->count(method))) { + if (can_delete(method) && !method::is_argless_init(method)) { auto code_size = get_code_size(method); net_savings += COST_METHOD + code_size; if (is_static(method)) { @@ -570,7 +916,6 @@ class RootMethodReducer { DexMethodRef* m_incomplete_marker_method; MultiMethodInliner& m_inliner; const MethodSummaries& m_method_summaries; - const std::unordered_set& m_excluded_classes; Stats* m_stats; bool m_is_init_or_clinit; DexMethod* m_method; @@ -584,7 +929,6 @@ class RootMethodReducer { DexMethodRef* incomplete_marker_method, MultiMethodInliner& inliner, const MethodSummaries& method_summaries, - const std::unordered_set& excluded_classes, Stats* stats, bool is_init_or_clinit, DexMethod* method, @@ -594,7 +938,6 @@ class RootMethodReducer { m_incomplete_marker_method(incomplete_marker_method), m_inliner(inliner), m_method_summaries(method_summaries), - m_excluded_classes(excluded_classes), m_stats(stats), m_is_init_or_clinit(is_init_or_clinit), m_method(method), @@ -615,20 +958,9 @@ class RootMethodReducer { } auto* insn = find_incomplete_marker_methods(); - auto describe = [&]() { - std::ostringstream oss; - for (auto [type, kind] : m_types) { - oss << show(type) << ":" - << (kind == InlinableTypeKind::Incomplete ? "incomplete" : "") - << ", "; - } - return oss.str(); - }; - always_assert_log( - !insn, - "Incomplete marker {%s} present in {%s} after reduction of %s in\n%s", - SHOW(insn), SHOW(m_method), describe().c_str(), - SHOW(m_method->get_code()->cfg())); + always_assert_log(!insn, + "Incomplete marker {%s} present after reduction in\n%s", + SHOW(insn), SHOW(m_method->get_code()->cfg())); shrink(); return (ReducedMethod){ @@ -662,7 +994,7 @@ class RootMethodReducer { auto callee = resolve_method(insn->get_method(), opcode_to_search(insn)); always_assert(callee); always_assert(callee->is_concrete()); - std::vector const* fields; + std::vector* fields; auto expanded_method_ref = m_expandable_method_params.get_expanded_method_ref(callee, param_index, &fields); @@ -719,7 +1051,6 @@ class RootMethodReducer { auto param_index = it2->second; auto expanded_method_ref = expand_invoke(mutation, it, param_index); if (!expanded_method_ref) { - mutation.clear(); return false; } expanded_method_refs->insert(expanded_method_ref); @@ -733,15 +1064,11 @@ class RootMethodReducer { bool inline_anchors() { auto& cfg = m_method->get_code()->cfg(); for (int iteration = 0; true; iteration++) { - Analyzer analyzer(m_excluded_classes, m_method_summaries, - m_incomplete_marker_method, cfg); + Analyzer analyzer(m_method_summaries, m_incomplete_marker_method, cfg); std::unordered_set invokes_to_inline; auto inlinables = analyzer.get_inlinables(); Lazy du_chains([&]() { - live_range::MoveAwareChains chains( - cfg, /* ignore_unreachable */ false, [&](auto* insn) { - return inlinables.count(const_cast(insn)); - }); + live_range::MoveAwareChains chains(cfg); return chains.get_def_use_chains(); }); cfg::CFGMutation mutation(cfg); @@ -795,17 +1122,18 @@ class RootMethodReducer { } } - bool is_inlinable_new_instance(const IRInstruction* insn) const { + bool is_inlinable_new_instance(IRInstruction* insn) const { return insn->opcode() == OPCODE_NEW_INSTANCE && m_types.count(insn->get_type()); } - bool is_incomplete_marker(const IRInstruction* insn) const { + bool is_incomplete_marker(IRInstruction* insn) const { return insn->opcode() == OPCODE_INVOKE_STATIC && insn->get_method() == m_incomplete_marker_method; } - bool has_incomplete_marker(const live_range::Uses& uses) const { + bool has_incomplete_marker( + const std::unordered_set& uses) const { return std::any_of(uses.begin(), uses.end(), [&](auto& use) { return is_incomplete_marker(use.insn); }); @@ -821,14 +1149,11 @@ class RootMethodReducer { return nullptr; } - std::optional> + std::optional> find_inlinable_new_instance() const { auto& cfg = m_method->get_code()->cfg(); Lazy du_chains([&]() { - live_range::MoveAwareChains chains( - cfg, - /* ignore_unreachable */ false, - [&](auto* insn) { return is_inlinable_new_instance(insn); }); + live_range::MoveAwareChains chains(cfg); return chains.get_def_use_chains(); }); for (auto& mie : InstructionIterable(cfg)) { @@ -837,12 +1162,11 @@ class RootMethodReducer { continue; } auto type = insn->get_type(); - auto& p = *du_chains->find(insn); if (m_types.at(type) == InlinableTypeKind::Incomplete && - !has_incomplete_marker(p.second)) { + !has_incomplete_marker((*du_chains)[insn])) { continue; } - return std::make_optional(std::move(p)); + return std::make_optional(std::pair(std::move(*du_chains), insn)); } return std::nullopt; } @@ -878,10 +1202,7 @@ class RootMethodReducer { std::unordered_set invokes_to_inline; std::unordered_map invokes_to_expand; - live_range::MoveAwareChains chains( - cfg, - /* ignore_unreachable */ false, - [&](auto* insn) { return is_inlinable_new_instance(insn); }); + live_range::MoveAwareChains chains(cfg); auto du_chains = chains.get_def_use_chains(); std::unordered_map> @@ -957,8 +1278,8 @@ class RootMethodReducer { // Given a new-instance instruction whose (main) uses are as the receiver in // iget- and iput- instruction, transform all such field accesses into // accesses to registers, one per field. - bool stackify(IRInstruction* new_instance_insn, - const live_range::Uses& new_instance_insn_uses) { + bool stackify(live_range::DefUseChains& du_chains, + IRInstruction* new_instance_insn) { auto& cfg = m_method->get_code()->cfg(); std::unordered_map field_regs; std::vector ordered_fields; @@ -975,9 +1296,10 @@ class RootMethodReducer { return it->second; }; + auto& uses = du_chains[new_instance_insn]; std::unordered_set instructions_to_replace; bool identity_matters{false}; - for (auto& use : new_instance_insn_uses) { + for (auto& use : uses) { auto opcode = use.insn->opcode(); if (opcode::is_an_iput(opcode)) { always_assert(use.src_index == 1); @@ -1098,7 +1420,6 @@ compute_reduced_methods( ExpandableMethodParams& expandable_method_params, MultiMethodInliner& inliner, const MethodSummaries& method_summaries, - const std::unordered_set& excluded_classes, const std::unordered_map& root_methods, std::unordered_set* irreducible_types, Stats* stats, @@ -1167,15 +1488,14 @@ compute_reduced_methods( [&](const std::pair& p) { auto* method = p.first; const auto& types = p.second; - auto copy_name_str = method->get_name()->str() + "$oea$internal$" + - std::to_string(types.size()); + auto copy_name_str = + method->get_name()->str() + "$oea$" + std::to_string(types.size()); auto copy = DexMethod::make_method_from( method, method->get_class(), DexString::make_string(copy_name_str)); RootMethodReducer root_method_reducer{expandable_method_params, incomplete_marker_method, inliner, method_summaries, - excluded_classes, stats, method::is_init(method) || method::is_clinit(method), @@ -1226,13 +1546,11 @@ compute_reduced_methods( // Select all those reduced methods which will result in overall size savings, // either by looking at local net savings for just a single reduced method, or // by considering families of reduced methods that affect the same classes. -void select_reduced_methods( +std::unordered_map select_reduced_methods( const std::unordered_map>& reduced_methods, std::unordered_set* irreducible_types, - Stats* stats, - InsertOnlyConcurrentMap* - concurrent_selected_reduced_methods) { + Stats* stats) { Timer t("select_reduced_methods"); // First, we are going to identify all reduced methods which will result in @@ -1240,10 +1558,8 @@ void select_reduced_methods( // We'll also build up families of reduced methods for which we can later do a // global net savings analysis. - InsertOnlyConcurrentSet concurrent_irreducible_types; - InsertOnlyConcurrentSet concurrent_inlined_methods_removed; - InsertOnlyConcurrentSet concurrent_inlinable_methods_kept; - InsertOnlyConcurrentSet concurrent_kept_methods; + ConcurrentSet concurrent_irreducible_types; + ConcurrentMap concurrent_selected_reduced_methods; // A family of reduced methods for which we'll look at the combined global net // savings @@ -1261,44 +1577,31 @@ void select_reduced_methods( auto update_irreducible_types = [&](size_t i) { const auto& reduced_method = reduced_methods_variants.at(i); const auto& reduced_types = reduced_method.types; - const auto& inlined_methods = reduced_method.inlined_methods; - const auto& largest_reduced_method = reduced_methods_variants.at(0); - for (auto&& [type, kind] : largest_reduced_method.types) { + for (auto&& [type, kind] : reduced_methods_variants.at(0).types) { if (!reduced_types.count(type) && kind != InlinableTypeKind::Incomplete) { concurrent_irreducible_types.insert(type); } } - for (auto&& [inlined_method, _] : - largest_reduced_method.inlined_methods) { - if (!inlined_methods.count(inlined_method)) { - concurrent_inlinable_methods_kept.insert(inlined_method); - } - } }; // We'll try to find the maximal (involving most inlined non-incomplete // types) reduced method variant for which we can make the largest // non-negative local cost determination. - struct Candidate { - size_t index; - NetSavings net_savings; - int value; - }; - boost::optional best_candidate; + boost::optional> best_candidate; for (size_t i = 0; i < reduced_methods_variants.size(); i++) { // If we couldn't accommodate any types, we'll need to add them to the // irreducible types set. Except for the last variant with the least // inlined types, for which might below try to make a global cost // determination. const auto& reduced_method = reduced_methods_variants.at(i); - auto net_savings = reduced_method.get_net_savings(*irreducible_types); - auto value = net_savings.get_value(); - if (!best_candidate || value > best_candidate->value) { - best_candidate = (Candidate){i, std::move(net_savings), value}; + auto savings = + reduced_method.get_net_savings(*irreducible_types).get_value(); + if (!best_candidate || savings > best_candidate->second) { + best_candidate = std::make_pair(i, savings); } // If there are no incomplete types left, we can stop here const auto& reduced_types = reduced_method.types; - if (best_candidate && best_candidate->value >= 0 && + if (best_candidate && best_candidate->second >= 0 && !std::any_of(reduced_types.begin(), reduced_types.end(), [](auto& p) { return p.second == InlinableTypeKind::Incomplete; @@ -1306,14 +1609,11 @@ void select_reduced_methods( break; } } - if (best_candidate && best_candidate->value >= 0) { - stats->total_savings += best_candidate->value; - concurrent_selected_reduced_methods->emplace(method, - best_candidate->index); - update_irreducible_types(best_candidate->index); - for (auto* inlined_method : best_candidate->net_savings.methods) { - concurrent_inlined_methods_removed.insert(inlined_method); - } + if (best_candidate && best_candidate->second >= 0) { + auto [i, savings] = *best_candidate; + stats->total_savings += savings; + concurrent_selected_reduced_methods.emplace(method, i); + update_irreducible_types(i); return; } update_irreducible_types(reduced_methods_variants.size() - 1); @@ -1347,44 +1647,30 @@ void select_reduced_methods( for (auto [type, kind] : smallest_reduced_method.types) { concurrent_irreducible_types.insert(type); } - for (auto&& [inlined_method, _] : - smallest_reduced_method.inlined_methods) { - concurrent_inlinable_methods_kept.insert(inlined_method); - } }, reduced_methods); irreducible_types->insert(concurrent_irreducible_types.begin(), concurrent_irreducible_types.end()); - stats->inlinable_methods_kept = concurrent_inlinable_methods_kept.size(); // Second, perform global net savings analysis workqueue_run>( [&](const std::pair& p) { auto& [type, family] = p; if (irreducible_types->count(type) || - family.global_net_savings.get_value( - &concurrent_inlinable_methods_kept) < 0) { + family.global_net_savings.get_value() < 0) { stats->too_costly_globally += family.reduced_methods.size(); return; } - stats->total_savings += family.global_net_savings.get_value( - &concurrent_inlinable_methods_kept); + stats->total_savings += family.global_net_savings.get_value(); for (auto& [method, i] : family.reduced_methods) { - concurrent_selected_reduced_methods->emplace(method, i); - } - for (auto* inlined_method : family.global_net_savings.methods) { - concurrent_inlined_methods_removed.insert(inlined_method); + concurrent_selected_reduced_methods.emplace(method, i); } }, concurrent_families); - stats->inlined_methods_removed = concurrent_inlined_methods_removed.size(); - - for (auto* method : concurrent_inlined_methods_removed) { - if (concurrent_inlinable_methods_kept.count_unsafe(method)) { - stats->inlined_methods_mispredicted++; - } - } + return std::unordered_map( + concurrent_selected_reduced_methods.begin(), + concurrent_selected_reduced_methods.end()); } void reduce(DexStoresVector& stores, @@ -1394,7 +1680,6 @@ void reduce(DexStoresVector& stores, init_classes_with_side_effects, MultiMethodInliner& inliner, const MethodSummaries& method_summaries, - const std::unordered_set& excluded_classes, const std::unordered_map& root_methods, Stats* stats, size_t max_inline_size) { @@ -1405,14 +1690,13 @@ void reduce(DexStoresVector& stores, ExpandableMethodParams expandable_method_params(scope); std::unordered_set irreducible_types; auto reduced_methods = compute_reduced_methods( - expandable_method_params, inliner, method_summaries, excluded_classes, - root_methods, &irreducible_types, stats, max_inline_size); + expandable_method_params, inliner, method_summaries, root_methods, + &irreducible_types, stats, max_inline_size); stats->reduced_methods = reduced_methods.size(); // Second, we select reduced methods that will result in net savings - InsertOnlyConcurrentMap selected_reduced_methods; - select_reduced_methods(reduced_methods, &irreducible_types, stats, - &selected_reduced_methods); + auto selected_reduced_methods = + select_reduced_methods(reduced_methods, &irreducible_types, stats); stats->selected_reduced_methods = selected_reduced_methods.size(); // Finally, we are going to apply those selected methods, and clean up all @@ -1452,23 +1736,19 @@ void ObjectEscapeAnalysisPass::run_pass(DexStoresVector& stores, auto method_override_graph = mog::build_graph(scope); init_classes::InitClassesWithSideEffects init_classes_with_side_effects( scope, conf.create_init_class_insns(), method_override_graph.get()); + auto non_overridden_virtuals = + hier::find_non_overridden_virtuals(*method_override_graph); - auto excluded_classes = get_excluded_classes(*method_override_graph); - - ConcurrentMap new_instances; - ConcurrentMap invokes; - ConcurrentMap> dependencies; - analyze_scope(scope, *method_override_graph, &new_instances, &invokes, + std::unordered_map new_instances; + std::unordered_map invokes; + std::unordered_map> dependencies; + analyze_scope(scope, non_overridden_virtuals, &new_instances, &invokes, &dependencies); - size_t analysis_iterations; - auto method_summaries = - compute_method_summaries(scope, dependencies, *method_override_graph, - excluded_classes, &analysis_iterations); - mgr.incr_metric("analysis_iterations", analysis_iterations); + auto method_summaries = compute_method_summaries(mgr, scope, dependencies, + non_overridden_virtuals); - auto inline_anchors = - compute_inline_anchors(scope, method_summaries, excluded_classes); + auto inline_anchors = compute_inline_anchors(scope, method_summaries); auto root_methods = compute_root_methods(mgr, new_instances, invokes, method_summaries, inline_anchors); @@ -1482,7 +1762,6 @@ void ObjectEscapeAnalysisPass::run_pass(DexStoresVector& stores, inliner_config.shrinker.run_cse = true; inliner_config.shrinker.run_copy_prop = true; inliner_config.shrinker.run_local_dce = true; - inliner_config.shrinker.normalize_new_instances = false; inliner_config.shrinker.compute_pure_methods = false; int min_sdk = 0; MultiMethodInliner inliner( @@ -1490,14 +1769,16 @@ void ObjectEscapeAnalysisPass::run_pass(DexStoresVector& stores, std::ref(concurrent_method_resolver), inliner_config, min_sdk, MultiMethodInlinerMode::None); - auto lost_returns_through_shrinking = shrink_root_methods( - inliner, dependencies, root_methods, &method_summaries); + shrink_root_methods(inliner, dependencies, root_methods, &method_summaries); Stats stats; reduce(stores, scope, conf, init_classes_with_side_effects, inliner, - method_summaries, excluded_classes, root_methods, &stats, + method_summaries, root_methods, &stats, m_max_inline_size); + walk::parallel::code(scope, + [&](DexMethod*, IRCode& code) { code.clear_cfg(); }); + TRACE(OEA, 1, "[object escape analysis] total savings: %zu", (size_t)stats.total_savings); TRACE( @@ -1511,10 +1792,10 @@ void ObjectEscapeAnalysisPass::run_pass(DexStoresVector& stores, "inlinable after too many iterations, %zu stackify returned objects, " "%zu too costly with irreducible classes, %zu too costly with multiple " "conditional classes, %zu too costly globally; %zu expanded methods; %zu " - "calls inlined; %zu new-instances eliminated; %zu/%zu/%zu inlinable " - "methods removed/kept/mispredicted", - root_methods.size(), stats.reduced_methods, - (size_t)stats.reduced_methods_variants, stats.selected_reduced_methods, + "calls inlined; %zu new-instances eliminated", + root_methods.size(), (size_t)stats.reduced_methods, + (size_t)stats.reduced_methods_variants, + (size_t)stats.selected_reduced_methods, (size_t)stats.anchors_not_inlinable_inlining, (size_t)stats.invokes_not_inlinable_callee_unexpandable, (size_t)stats.invokes_not_inlinable_callee_too_many_params_to_expand, @@ -1525,16 +1806,15 @@ void ObjectEscapeAnalysisPass::run_pass(DexStoresVector& stores, (size_t)stats.too_costly_irreducible_classes, (size_t)stats.too_costly_multiple_conditional_classes, (size_t)stats.too_costly_globally, (size_t)stats.expanded_methods, - (size_t)stats.calls_inlined, (size_t)stats.new_instances_eliminated, - stats.inlined_methods_removed, stats.inlinable_methods_kept, - stats.inlined_methods_mispredicted); + (size_t)stats.calls_inlined, (size_t)stats.new_instances_eliminated); mgr.incr_metric("total_savings", stats.total_savings); mgr.incr_metric("root_methods", root_methods.size()); - mgr.incr_metric("reduced_methods", stats.reduced_methods); + mgr.incr_metric("reduced_methods", (size_t)stats.reduced_methods); mgr.incr_metric("reduced_methods_variants", (size_t)stats.reduced_methods_variants); - mgr.incr_metric("selected_reduced_methods", stats.selected_reduced_methods); + mgr.incr_metric("selected_reduced_methods", + (size_t)stats.selected_reduced_methods); mgr.incr_metric("root_method_anchors_not_inlinable_inlining", (size_t)stats.anchors_not_inlinable_inlining); mgr.incr_metric("root_method_invokes_not_inlinable_callee_unexpandable", @@ -1561,12 +1841,6 @@ void ObjectEscapeAnalysisPass::run_pass(DexStoresVector& stores, mgr.incr_metric("calls_inlined", (size_t)stats.calls_inlined); mgr.incr_metric("new_instances_eliminated", (size_t)stats.new_instances_eliminated); - mgr.incr_metric("inlined_methods_removed", stats.inlined_methods_removed); - mgr.incr_metric("inlinable_methods_kept", stats.inlinable_methods_kept); - mgr.incr_metric("inlined_methods_mispredicted", - stats.inlined_methods_mispredicted); - mgr.incr_metric("lost_returns_through_shrinking", - lost_returns_through_shrinking); } static ObjectEscapeAnalysisPass s_pass; diff --git a/opt/object-escape-analysis/ObjectEscapeAnalysis.h b/opt/object-escape-analysis/ObjectEscapeAnalysis.h index c636cad07a..eafc201798 100644 --- a/opt/object-escape-analysis/ObjectEscapeAnalysis.h +++ b/opt/object-escape-analysis/ObjectEscapeAnalysis.h @@ -18,13 +18,16 @@ class ObjectEscapeAnalysisPass : public Pass { using namespace redex_properties::interactions; using namespace redex_properties::names; return { + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, - {SpuriousGetClassCallsInterned, RequiresAndPreserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } void bind_config() override; + bool is_cfg_legacy() override { return true; } + void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; private: diff --git a/opt/object-escape-analysis/ObjectEscapeAnalysisImpl.cpp b/opt/object-escape-analysis/ObjectEscapeAnalysisImpl.cpp deleted file mode 100644 index 7764426834..0000000000 --- a/opt/object-escape-analysis/ObjectEscapeAnalysisImpl.cpp +++ /dev/null @@ -1,331 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * - * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. - */ - -#include "ObjectEscapeAnalysisImpl.h" - -#include "Walkers.h" - -using namespace sparta; -namespace mog = method_override_graph; - -namespace { -constexpr const IRInstruction* NO_ALLOCATION = nullptr; -} // namespace - -namespace object_escape_analysis_impl { - -std::unordered_set get_excluded_classes( - const mog::Graph& method_override_graph) { - std::unordered_set res; - for (auto* overriding_method : method_override_graph::get_overriding_methods( - method_override_graph, method::java_lang_Object_finalize())) { - auto* cls = type_class(overriding_method->get_class()); - if (cls && !cls->is_external()) { - res.insert(cls); - } - } - return res; -}; - -// Collect all allocation and invoke instructions, as well as non-virtual -// invocation dependencies. -void analyze_scope( - const Scope& scope, - const mog::Graph& method_override_graph, - ConcurrentMap* new_instances, - ConcurrentMap* invokes, - ConcurrentMap>* dependencies) { - Timer t("analyze_scope"); - walk::parallel::code(scope, [&](DexMethod* method, IRCode& code) { - always_assert(code.editable_cfg_built()); - LazyUnorderedMap is_not_overridden([&](auto* m) { - return !m->is_virtual() || - !mog::any_overriding_methods(method_override_graph, m); - }); - for (auto& mie : InstructionIterable(code.cfg())) { - auto insn = mie.insn; - if (insn->opcode() == OPCODE_NEW_INSTANCE) { - auto cls = type_class(insn->get_type()); - if (cls && !cls->is_external()) { - new_instances->update(insn->get_type(), [&](auto*, auto& vec, bool) { - vec.emplace_back(method, insn); - }); - } - } else if (opcode::is_an_invoke(insn->opcode())) { - auto callee = - resolve_method(insn->get_method(), opcode_to_search(insn)); - if (callee && callee->get_code() && !callee->is_external() && - is_not_overridden[callee]) { - invokes->update(callee, [&](auto*, auto& vec, bool) { - vec.emplace_back(method, insn); - }); - if (is_not_overridden[method]) { - dependencies->update(callee, [method](auto, auto& set, auto) { - set.insert(method); - }); - } - } - } - } - }); -} - -// A benign method invocation can be ignored during the escape analysis. -bool is_benign(const DexMethodRef* method_ref) { - static const std::unordered_set methods = { - // clang-format off - "Ljava/lang/Object;.:()V", - // clang-format on - }; - - return method_ref->is_def() && - methods.count( - method_ref->as_def()->get_deobfuscated_name_or_empty_copy()); -} - -Analyzer::Analyzer(const std::unordered_set& excluded_classes, - const MethodSummaries& method_summaries, - DexMethodRef* incomplete_marker_method, - cfg::ControlFlowGraph& cfg) - : BaseIRAnalyzer(cfg), - m_excluded_classes(excluded_classes), - m_method_summaries(method_summaries), - m_incomplete_marker_method(incomplete_marker_method) { - MonotonicFixpointIterator::run(Environment::top()); -} - -const IRInstruction* Analyzer::get_singleton_allocation(const Domain& domain) { - always_assert(domain.kind() == AbstractValueKind::Value); - auto& elements = domain.elements(); - if (elements.size() != 1) { - return nullptr; - } - return *elements.begin(); -} - -void Analyzer::analyze_instruction(const IRInstruction* insn, - Environment* current_state) const { - - const auto escape = [&](src_index_t src_idx) { - auto reg = insn->src(src_idx); - const auto& domain = current_state->get(reg); - always_assert(domain.kind() == AbstractValueKind::Value); - for (auto allocation_insn : domain.elements()) { - if (allocation_insn != NO_ALLOCATION) { - m_escapes[allocation_insn].insert( - {const_cast(insn), src_idx}); - } - } - }; - - if (insn->opcode() == OPCODE_NEW_INSTANCE) { - auto type = insn->get_type(); - auto cls = type_class(type); - if (cls && !cls->is_external() && !m_excluded_classes.count(cls)) { - m_escapes[insn]; - current_state->set(RESULT_REGISTER, Domain(insn)); - return; - } - } else if (insn->opcode() == IOPCODE_LOAD_PARAM_OBJECT) { - m_escapes[insn]; - current_state->set(insn->dest(), Domain(insn)); - return; - } else if (insn->opcode() == OPCODE_RETURN_OBJECT) { - const auto& domain = current_state->get(insn->src(0)); - always_assert(domain.kind() == AbstractValueKind::Value); - m_returns.insert(domain.elements().begin(), domain.elements().end()); - return; - } else if (insn->opcode() == OPCODE_MOVE_RESULT_OBJECT || - insn->opcode() == IOPCODE_MOVE_RESULT_PSEUDO_OBJECT) { - const auto& domain = current_state->get(RESULT_REGISTER); - current_state->set(insn->dest(), domain); - return; - } else if (insn->opcode() == OPCODE_MOVE_OBJECT) { - const auto& domain = current_state->get(insn->src(0)); - current_state->set(insn->dest(), domain); - return; - } else if (insn->opcode() == OPCODE_INSTANCE_OF || - opcode::is_an_iget(insn->opcode())) { - if (get_singleton_allocation(current_state->get(insn->src(0)))) { - current_state->set(RESULT_REGISTER, Domain(NO_ALLOCATION)); - return; - } - } else if (opcode::is_a_monitor(insn->opcode()) || - insn->opcode() == OPCODE_IF_EQZ || - insn->opcode() == OPCODE_IF_NEZ) { - if (get_singleton_allocation(current_state->get(insn->src(0)))) { - return; - } - } else if (opcode::is_an_iput(insn->opcode())) { - if (get_singleton_allocation(current_state->get(insn->src(1)))) { - escape(0); - return; - } - } else if (opcode::is_an_invoke(insn->opcode())) { - if (is_benign(insn->get_method()) || is_incomplete_marker(insn)) { - current_state->set(RESULT_REGISTER, Domain(NO_ALLOCATION)); - return; - } - auto callee = resolve_method(insn->get_method(), opcode_to_search(insn)); - auto it = m_method_summaries.find(callee); - auto benign_params = - it == m_method_summaries.end() ? nullptr : &it->second.benign_params; - for (src_index_t i = 0; i < insn->srcs_size(); i++) { - if (!benign_params || !benign_params->count(i) || - !get_singleton_allocation(current_state->get(insn->src(i)))) { - escape(i); - } - } - - Domain domain(NO_ALLOCATION); - if (it != m_method_summaries.end() && it->second.allocation_insn) { - m_escapes[insn]; - domain = Domain(insn); - } - current_state->set(RESULT_REGISTER, domain); - return; - } - - for (src_index_t i = 0; i < insn->srcs_size(); i++) { - escape(i); - } - - if (insn->has_dest()) { - current_state->set(insn->dest(), Domain(NO_ALLOCATION)); - if (insn->dest_is_wide()) { - current_state->set(insn->dest() + 1, Domain::top()); - } - } else if (insn->has_move_result_any()) { - current_state->set(RESULT_REGISTER, Domain(NO_ALLOCATION)); - } -} - -// Returns set of new-instance and invoke- allocating instructions that do not -// escape (or return). -std::unordered_set Analyzer::get_inlinables() { - std::unordered_set inlinables; - for (auto&& [insn, uses] : m_escapes) { - if (uses.empty() && insn->opcode() != IOPCODE_LOAD_PARAM_OBJECT && - !m_returns.count(insn)) { - inlinables.insert(const_cast(insn)); - } - } - return inlinables; -} - -MethodSummaries compute_method_summaries( - const Scope& scope, - const ConcurrentMap>& - dependencies, - const mog::Graph& method_override_graph, - const std::unordered_set& excluded_classes, - size_t* analysis_iterations) { - Timer t("compute_method_summaries"); - - std::unordered_set impacted_methods; - walk::code(scope, [&](DexMethod* method, IRCode&) { - if (!method->is_virtual() || - !mog::any_overriding_methods(method_override_graph, method)) { - impacted_methods.insert(method); - } - }); - - MethodSummaries method_summaries; - *analysis_iterations = 0; - while (!impacted_methods.empty()) { - Timer t2("analysis iteration"); - (*analysis_iterations)++; - TRACE(OEA, 2, "[object escape analysis] analysis_iteration %zu", - *analysis_iterations); - ConcurrentMap recomputed_method_summaries; - workqueue_run( - [&](DexMethod* method) { - auto& cfg = method->get_code()->cfg(); - Analyzer analyzer(excluded_classes, method_summaries, - /* incomplete_marker_method */ nullptr, cfg); - const auto& escapes = analyzer.get_escapes(); - const auto& returns = analyzer.get_returns(); - src_index_t src_index = 0; - for (auto& mie : InstructionIterable(cfg.get_param_instructions())) { - if (mie.insn->opcode() == IOPCODE_LOAD_PARAM_OBJECT && - escapes.at(mie.insn).empty() && !returns.count(mie.insn)) { - recomputed_method_summaries.update( - method, [src_index](DexMethod*, auto& ms, bool) { - ms.benign_params.insert(src_index); - }); - } - src_index++; - } - if (returns.size() == 1) { - const auto* allocation_insn = *returns.begin(); - if (allocation_insn != NO_ALLOCATION && - escapes.at(allocation_insn).empty() && - allocation_insn->opcode() != IOPCODE_LOAD_PARAM_OBJECT) { - recomputed_method_summaries.update( - method, [allocation_insn](DexMethod*, auto& ms, bool) { - ms.allocation_insn = allocation_insn; - }); - } - } - }, - impacted_methods); - - std::unordered_set changed_methods; - // (Recomputed) summaries can only grow; assert that, update summaries when - // necessary, and remember for which methods the summaries actually changed. - for (auto&& [method, recomputed_summary] : recomputed_method_summaries) { - auto& summary = method_summaries[method]; - for (auto src_index : summary.benign_params) { - always_assert(recomputed_summary.benign_params.count(src_index)); - } - if (recomputed_summary.benign_params.size() > - summary.benign_params.size()) { - summary.benign_params = std::move(recomputed_summary.benign_params); - changed_methods.insert(method); - } - if (recomputed_summary.allocation_insn) { - if (summary.allocation_insn) { - always_assert(summary.allocation_insn == - recomputed_summary.allocation_insn); - } else { - summary.allocation_insn = recomputed_summary.allocation_insn; - changed_methods.insert(method); - } - } else { - always_assert(summary.allocation_insn == nullptr); - } - } - impacted_methods.clear(); - for (auto method : changed_methods) { - auto it = dependencies.find(method); - if (it != dependencies.end()) { - impacted_methods.insert(it->second.begin(), it->second.end()); - } - } - } - return method_summaries; -} - -// For an inlinable new-instance or invoke- instruction, determine first -// resolved callee (if any), and (eventually) allocated type -std::pair resolve_inlinable( - const MethodSummaries& method_summaries, const IRInstruction* insn) { - always_assert(insn->opcode() == OPCODE_NEW_INSTANCE || - opcode::is_an_invoke(insn->opcode())); - DexMethod* first_callee{nullptr}; - while (insn->opcode() != OPCODE_NEW_INSTANCE) { - always_assert(opcode::is_an_invoke(insn->opcode())); - auto callee = resolve_method(insn->get_method(), opcode_to_search(insn)); - if (!first_callee) { - first_callee = callee; - } - insn = method_summaries.at(callee).allocation_insn; - } - return std::make_pair(first_callee, insn->get_type()); -} - -} // namespace object_escape_analysis_impl diff --git a/opt/object-escape-analysis/ObjectEscapeAnalysisImpl.h b/opt/object-escape-analysis/ObjectEscapeAnalysisImpl.h deleted file mode 100644 index 13ba75df87..0000000000 --- a/opt/object-escape-analysis/ObjectEscapeAnalysisImpl.h +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * - * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. - */ - -#pragma once - -#include - -#include -#include -#include -#include -#include - -#include "BaseIRAnalyzer.h" -#include "ControlFlow.h" -#include "DexClass.h" -#include "IRCode.h" -#include "IROpcode.h" -#include "Lazy.h" -#include "LiveRange.h" -#include "MethodOverrideGraph.h" -#include "Resolver.h" -#include "Show.h" - -namespace object_escape_analysis_impl { - -using Locations = std::vector>; - -std::unordered_set get_excluded_classes( - const method_override_graph::Graph& method_override_graph); - -// Collect all allocation and invoke instructions, as well as non-virtual -// invocation dependencies. -void analyze_scope( - const Scope& scope, - const method_override_graph::Graph& method_override_graph, - ConcurrentMap* new_instances, - ConcurrentMap* invokes, - ConcurrentMap>* dependencies); - -// A benign method invocation can be ignored during the escape analysis. -bool is_benign(const DexMethodRef* method_ref); - -// For each allocating instruction that escapes (not including returns), all -// uses by which it escapes. -using Escapes = std::unordered_map; - -// For each object, we track which instruction might have allocated it: -// - new-instance, invoke-, and load-param-object instructions might represent -// allocation points -// - NO_ALLOCATION is a value for which the allocation instruction is not known, -// or it is not an object -using Domain = sparta::PatriciaTreeSetAbstractDomain; - -// For each register that holds a relevant value, keep track of it. -using Environment = sparta::PatriciaTreeMapAbstractEnvironment; - -struct MethodSummary { - // A parameter is "benign" if a provided argument does not escape - std::unordered_set benign_params; - // A method might contain a unique instruction which allocates an object that - // is eventually unconditionally returned. - const IRInstruction* allocation_insn{nullptr}; -}; - -using MethodSummaries = std::unordered_map; - -// The analyzer computes... -// - which instructions allocate (new-instance, invoke-) -// - which allocations escape (and how) -// - which allocations return -class Analyzer final : public ir_analyzer::BaseIRAnalyzer { - public: - explicit Analyzer(const std::unordered_set& excluded_classes, - const MethodSummaries& method_summaries, - DexMethodRef* incomplete_marker_method, - cfg::ControlFlowGraph& cfg); - - static const IRInstruction* get_singleton_allocation(const Domain& domain); - - void analyze_instruction(const IRInstruction* insn, - Environment* current_state) const override; - - const Escapes& get_escapes() { return m_escapes; } - - const std::unordered_set& get_returns() { - return m_returns; - } - - // Returns set of new-instance and invoke- allocating instructions that do not - // escape (or return). - std::unordered_set get_inlinables(); - - private: - const std::unordered_set& m_excluded_classes; - const MethodSummaries& m_method_summaries; - DexMethodRef* m_incomplete_marker_method; - mutable Escapes m_escapes; - mutable std::unordered_set m_returns; - - bool is_incomplete_marker(const IRInstruction* insn) const { - return insn->opcode() == OPCODE_INVOKE_STATIC && - insn->get_method() == m_incomplete_marker_method; - } -}; - -MethodSummaries compute_method_summaries( - const Scope& scope, - const ConcurrentMap>& - dependencies, - const method_override_graph::Graph& method_override_graph, - const std::unordered_set& excluded_classes, - size_t* analysis_iterations); - -// For an inlinable new-instance or invoke- instruction, determine first -// resolved callee (if any), and (eventually) allocated type -std::pair resolve_inlinable( - const MethodSummaries& method_summaries, const IRInstruction* insn); - -} // namespace object_escape_analysis_impl diff --git a/opt/object-sensitive-dce/ObjectSensitiveDcePass.cpp b/opt/object-sensitive-dce/ObjectSensitiveDcePass.cpp index 8c7bfe393c..32fbd8027a 100644 --- a/opt/object-sensitive-dce/ObjectSensitiveDcePass.cpp +++ b/opt/object-sensitive-dce/ObjectSensitiveDcePass.cpp @@ -19,7 +19,6 @@ #include "InitClassesWithSideEffects.h" #include "LocalPointersAnalysis.h" #include "PassManager.h" -#include "Purity.h" #include "ScopedCFG.h" #include "SummarySerialization.h" #include "Walkers.h" @@ -47,47 +46,11 @@ namespace hier = hierarchy_util; namespace ptrs = local_pointers; namespace uv = used_vars; -namespace { - -class CallGraphStrategy final : public call_graph::MultipleCalleeStrategy { +class CallGraphStrategy final : public call_graph::BuildStrategy { public: - explicit CallGraphStrategy( - const method_override_graph::Graph& graph, - const Scope& scope, - const std::unordered_set& pure_methods, - const ptrs::SummaryMap& escape_summaries, - const side_effects::SummaryMap& effect_summaries, - uint32_t big_override_threshold) - : call_graph::MultipleCalleeStrategy( - graph, scope, big_override_threshold), - m_pure_methods(pure_methods), - m_escape_summaries(escape_summaries), - m_effect_summaries(effect_summaries) { - // XXX(jezng): We make every single method a root in order that all methods - // are seen as reachable. Unreachable methods will not have `get_callsites` - // run on them and will not have their outgoing edges added to the call - // graph, which means that the dead code removal will not optimize them - // fully. I'm not sure why these "unreachable" methods are not ultimately - // removed by RMU, but as it stands, properly optimizing them is a size win - // for us. - m_root_and_dynamic = MultipleCalleeStrategy::get_roots(); - walk::code(m_scope, [&](DexMethod* method, IRCode&) { - m_root_and_dynamic.roots.insert(method); - }); - } - - bool is_pure(IRInstruction* insn) const { - // This is what LocalDce does. - auto ref = insn->get_method(); - const auto meth = resolve_method(ref, opcode_to_search(insn)); - if (meth == nullptr) { - return false; - } - if (::assumenosideeffects(meth)) { - return true; - } - return m_pure_methods.count(ref); - } + explicit CallGraphStrategy(const Scope& scope) + : m_scope(scope), + m_non_overridden_virtuals(hier::find_non_overridden_virtuals(scope)) {} call_graph::CallSites get_callsites(const DexMethod* method) const override { call_graph::CallSites callsites; @@ -95,62 +58,15 @@ class CallGraphStrategy final : public call_graph::MultipleCalleeStrategy { if (code == nullptr) { return callsites; } - always_assert(code->editable_cfg_built()); - for (auto& mie : InstructionIterable(code->cfg())) { + auto& cfg = code->cfg(); + for (auto& mie : InstructionIterable(cfg)) { auto insn = mie.insn; - if (!opcode::is_an_invoke(insn->opcode())) { - continue; - } - auto callee = resolve_invoke_method(insn, method); - if (callee == nullptr) { - if (ptrs::is_array_clone(insn->get_method())) { - // We'll synthesize appropriate summaries for array clone methods on - // the fly. - callsites.emplace_back(/* callee */ nullptr, insn); - } - continue; - } - if (is_pure(insn)) { - // By including this in the call-graph with an empty callee, it will by - // default get trivial summaries, representing no interactions with - // objects, and no side effects. - callsites.emplace_back(/* callee */ nullptr, insn); - continue; - } - if (callee->is_external()) { - if ((opcode::is_invoke_super(insn->opcode()) || - !ptrs::may_be_overridden(callee)) && - has_summaries(callee)) { - callsites.emplace_back(callee, insn); - } - continue; - } - - if (is_definitely_virtual(callee) && - insn->opcode() != OPCODE_INVOKE_SUPER) { - if (m_root_and_dynamic.dynamic_methods.count(callee)) { - continue; - } - - // For true virtual callees, add the callee itself and all of its - // overrides if they are not in big virtuals. - if (m_big_virtuals.count_unsafe(callee)) { + if (opcode::is_an_invoke(insn->opcode())) { + auto callee = + resolve_method(insn->get_method(), opcode_to_search(insn), method); + if (callee == nullptr || may_be_overridden(callee)) { continue; } - const auto& overriding_methods = - get_ordered_overriding_methods_with_code_or_native(callee); - if (is_native(callee) || - std::any_of(overriding_methods.begin(), overriding_methods.end(), - [](auto* method) { return is_native(method); })) { - continue; - } - if (callee->get_code()) { - callsites.emplace_back(callee, insn); - } - for (auto overriding_method : overriding_methods) { - callsites.emplace_back(overriding_method, insn); - } - } else if (callee->is_concrete() && !is_native(callee)) { callsites.emplace_back(callee, insn); } } @@ -164,24 +80,41 @@ class CallGraphStrategy final : public call_graph::MultipleCalleeStrategy { // not sure why these "unreachable" methods are not ultimately removed by RMU, // but as it stands, properly optimizing them is a size win for us. call_graph::RootAndDynamic get_roots() const override { - return m_root_and_dynamic; + call_graph::RootAndDynamic root_and_dynamic; + auto& roots = root_and_dynamic.roots; + + walk::code(m_scope, [&](DexMethod* method, IRCode& code) { + roots.emplace_back(method); + }); + return root_and_dynamic; } private: - bool has_summaries(DexMethod* method) const { - if (m_escape_summaries.count(method) && m_effect_summaries.count(method)) { - return true; - } - return method == method::java_lang_Object_ctor(); + bool may_be_overridden(DexMethod* method) const { + return method->is_virtual() && m_non_overridden_virtuals.count(method) == 0; } - const std::unordered_set& m_pure_methods; - const ptrs::SummaryMap& m_escape_summaries; - const side_effects::SummaryMap& m_effect_summaries; - call_graph::RootAndDynamic m_root_and_dynamic; + const Scope& m_scope; + std::unordered_set m_non_overridden_virtuals; }; -} // namespace +static side_effects::InvokeToSummaryMap build_summary_map( + const side_effects::SummaryMap& effect_summaries, + const call_graph::Graph& call_graph, + DexMethod* method) { + side_effects::InvokeToSummaryMap invoke_to_summary_map; + if (call_graph.has_node(method)) { + const auto& callee_edges = call_graph.node(method)->callees(); + for (const auto& edge : callee_edges) { + auto* callee = edge->callee()->method(); + if (effect_summaries.count(callee) != 0) { + invoke_to_summary_map.emplace(edge->invoke_insn(), + effect_summaries.at(callee)); + } + } + } + return invoke_to_summary_map; +} void ObjectSensitiveDcePass::run_pass(DexStoresVector& stores, ConfigFiles& conf, @@ -192,16 +125,8 @@ void ObjectSensitiveDcePass::run_pass(DexStoresVector& stores, "init-class instructions."); auto scope = build_class_scope(stores); - auto method_override_graph = method_override_graph::build_graph(scope); init_classes::InitClassesWithSideEffects init_classes_with_side_effects( - scope, conf.create_init_class_insns(), method_override_graph.get()); - - auto pure_methods = get_pure_methods(); - auto configured_pure_methods = conf.get_pure_methods(); - pure_methods.insert(configured_pure_methods.begin(), - configured_pure_methods.end()); - auto immutable_getters = get_immutable_getters(scope); - pure_methods.insert(immutable_getters.begin(), immutable_getters.end()); + scope, conf.create_init_class_insns()); walk::parallel::code(scope, [&](const DexMethod* method, IRCode& code) { always_assert(code.editable_cfg_built()); @@ -209,6 +134,8 @@ void ObjectSensitiveDcePass::run_pass(DexStoresVector& stores, code.cfg().calculate_exit_block(); }); + auto call_graph = call_graph::Graph(CallGraphStrategy(scope)); + ptrs::SummaryMap escape_summaries; if (m_external_escape_summaries_file) { std::ifstream file_input(*m_external_escape_summaries_file); @@ -216,6 +143,11 @@ void ObjectSensitiveDcePass::run_pass(DexStoresVector& stores, } mgr.incr_metric("external_escape_summaries", escape_summaries.size()); + ptrs::SummaryCMap escape_summaries_cmap(escape_summaries.begin(), + escape_summaries.end()); + auto ptrs_fp_iter_map = + ptrs::analyze_scope(scope, call_graph, &escape_summaries_cmap); + side_effects::SummaryMap effect_summaries; if (m_external_side_effect_summaries_file) { std::ifstream file_input(*m_external_side_effect_summaries_file); @@ -223,38 +155,23 @@ void ObjectSensitiveDcePass::run_pass(DexStoresVector& stores, } mgr.incr_metric("external_side_effect_summaries", effect_summaries.size()); - auto call_graph = call_graph::Graph(CallGraphStrategy( - *method_override_graph, scope, pure_methods, escape_summaries, - effect_summaries, m_big_override_threshold)); - - auto ptrs_fp_iter_map = - ptrs::analyze_scope(scope, call_graph, &escape_summaries); - side_effects::analyze_scope(init_classes_with_side_effects, scope, call_graph, - ptrs_fp_iter_map, &effect_summaries); + *ptrs_fp_iter_map, &effect_summaries); std::atomic removed{0}; std::atomic init_class_instructions_added{0}; - std::mutex init_class_stats_mutex; init_classes::Stats init_class_stats; - std::mutex invokes_with_summaries_mutex; - std::unordered_map invokes_with_summaries{0}; - + std::mutex init_class_stats_mutex; walk::parallel::code(scope, [&](DexMethod* method, IRCode& code) { if (method->get_code() == nullptr || method->rstate.no_optimizations()) { return; } always_assert(code.editable_cfg_built()); auto& cfg = code.cfg(); - auto summary_map = build_summary_map(effect_summaries, call_graph, method); - std::unordered_map local_invokes_with_summaries; - for (auto&& [insn, summary] : summary_map) { - if (!summary.effects) { - local_invokes_with_summaries[insn->opcode()]++; - } - } - uv::FixpointIterator used_vars_fp_iter(*ptrs_fp_iter_map.at_unsafe(method), - std::move(summary_map), cfg, method); + uv::FixpointIterator used_vars_fp_iter( + *ptrs_fp_iter_map->find(method)->second, + build_summary_map(effect_summaries, call_graph, method), + cfg); used_vars_fp_iter.run(uv::UsedVarsSet()); cfg::CFGMutation mutator(cfg); @@ -295,11 +212,6 @@ void ObjectSensitiveDcePass::run_pass(DexStoresVector& stores, init_class_stats += init_class_pruner.get_stats(); } } - - std::lock_guard lock(invokes_with_summaries_mutex); - for (auto&& [opcode, count] : local_invokes_with_summaries) { - invokes_with_summaries[opcode] += count; - } }); mgr.set_metric("removed_instructions", removed); mgr.set_metric("init_class_instructions_added", @@ -308,29 +220,6 @@ void ObjectSensitiveDcePass::run_pass(DexStoresVector& stores, init_class_stats.init_class_instructions_removed); mgr.incr_metric("init_class_instructions_refined", init_class_stats.init_class_instructions_refined); - - size_t methods_with_summaries{0}; - size_t modified_params{0}; - for (auto&& [_, summary] : effect_summaries) { - if (!summary.effects) { - methods_with_summaries++; - } - modified_params += summary.modified_params.size(); - } - mgr.set_metric("methods_with_summaries", methods_with_summaries); - mgr.set_metric("modified_params", modified_params); - mgr.set_metric("invoke_direct_with_summaries", - invokes_with_summaries[OPCODE_INVOKE_DIRECT]); - mgr.set_metric("invoke_static_with_summaries", - invokes_with_summaries[OPCODE_INVOKE_STATIC]); - mgr.set_metric("invoke_interface_with_summaries", - invokes_with_summaries[OPCODE_INVOKE_INTERFACE]); - mgr.set_metric("invoke_virtual_with_summaries", - invokes_with_summaries[OPCODE_INVOKE_VIRTUAL]); - mgr.set_metric("invoke_super_with_summaries", - invokes_with_summaries[OPCODE_INVOKE_SUPER]); - TRACE(OSDCE, 1, "%zu methods with summaries, removed %zu instructions", - methods_with_summaries, (size_t)removed); } static ObjectSensitiveDcePass s_pass; diff --git a/opt/object-sensitive-dce/ObjectSensitiveDcePass.h b/opt/object-sensitive-dce/ObjectSensitiveDcePass.h index ea88870cfe..663be70e71 100644 --- a/opt/object-sensitive-dce/ObjectSensitiveDcePass.h +++ b/opt/object-sensitive-dce/ObjectSensitiveDcePass.h @@ -25,7 +25,9 @@ class ObjectSensitiveDcePass final : public Pass { using namespace redex_properties::interactions; using namespace redex_properties::names; return { + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } @@ -36,7 +38,6 @@ class ObjectSensitiveDcePass final : public Pass { bind("escape_summaries", {boost::none}, m_external_escape_summaries_file, "TODO: Document me!", Configurable::bindflags::optionals::skip_empty_string); - bind("big_override_threshold", UINT32_C(5), m_big_override_threshold); if (!m_external_escape_summaries_file || !m_external_side_effect_summaries_file) { @@ -51,5 +52,4 @@ class ObjectSensitiveDcePass final : public Pass { private: boost::optional m_external_side_effect_summaries_file; boost::optional m_external_escape_summaries_file; - uint32_t m_big_override_threshold; }; diff --git a/opt/object-sensitive-dce/SideEffectSummary.cpp b/opt/object-sensitive-dce/SideEffectSummary.cpp index 39ec8d0701..7be2552c25 100644 --- a/opt/object-sensitive-dce/SideEffectSummary.cpp +++ b/opt/object-sensitive-dce/SideEffectSummary.cpp @@ -20,8 +20,10 @@ namespace ptrs = local_pointers; namespace side_effects { -using SummaryConcurrentMap = - InsertOnlyConcurrentMap; +using PointersFixpointIteratorMap = + ConcurrentMap; + +using SummaryConcurrentMap = ConcurrentMap; SummaryBuilder::SummaryBuilder( const init_classes::InitClassesWithSideEffects& @@ -173,8 +175,13 @@ void SummaryBuilder::analyze_instruction_effects( classify_heap_write(env, insn->src(0), summary); break; } + case OPCODE_INVOKE_SUPER: - case OPCODE_INVOKE_INTERFACE: + case OPCODE_INVOKE_INTERFACE: { + TRACE(OSDCE, 3, "Unknown invoke: %s", SHOW(insn)); + summary->effects |= EFF_UNKNOWN_INVOKE; + break; + } case OPCODE_INVOKE_STATIC: case OPCODE_INVOKE_DIRECT: case OPCODE_INVOKE_VIRTUAL: { @@ -205,7 +212,7 @@ void SummaryBuilder::analyze_instruction_effects( void SummaryBuilder::classify_heap_write(const ptrs::Environment& env, reg_t modified_ptr_reg, Summary* summary) { - const auto& pointers = env.get_pointers(modified_ptr_reg); + auto pointers = env.get_pointers(modified_ptr_reg); if (!pointers.is_value()) { summary->effects |= EFF_WRITE_MAY_ESCAPE; return; @@ -222,55 +229,47 @@ void SummaryBuilder::classify_heap_write(const ptrs::Environment& env, } } -InvokeToSummaryMap build_summary_map(const SummaryMap& summary_map, - const call_graph::Graph& call_graph, - const DexMethod* method) { - InvokeToSummaryMap invoke_to_summary_map; +/* + * Analyze :method and insert its summary into :summary_cmap. Recursively + * analyze the callees if necessary. This method is thread-safe. + */ +void analyze_method_recursive(const init_classes::InitClassesWithSideEffects& + init_classes_with_side_effects, + const DexMethod* method, + const call_graph::Graph& call_graph, + const ptrs::FixpointIteratorMap& ptrs_fp_iter_map, + PatriciaTreeSet visiting, + SummaryConcurrentMap* summary_cmap) { + if (!method || summary_cmap->count(method) != 0 || + visiting.contains(method) || method->get_code() == nullptr) { + return; + } + visiting.insert(method); + + InvokeToSummaryMap invoke_to_summary_cmap; if (call_graph.has_node(method)) { const auto& callee_edges = call_graph.node(method)->callees(); for (const auto& edge : callee_edges) { - if (edge->callee() == call_graph.exit()) { - continue; - } - auto invoke_insn = edge->invoke_insn(); - auto& callee_summary = invoke_to_summary_map[invoke_insn]; auto* callee = edge->callee()->method(); - auto it = summary_map.find(callee); - if (it != summary_map.end()) { - callee_summary.join_with(it->second); - } else if (callee == nullptr && - ptrs::is_array_clone(invoke_insn->get_method())) { - // The array clone method doesn't have any effects, and doesn't modify - // any parameters; but may read heap locations (the elements of the - // array it clones). - callee_summary.join_with( - Summary(EFF_NONE, {}, /* may_read_external */ true)); + analyze_method_recursive(init_classes_with_side_effects, callee, + call_graph, ptrs_fp_iter_map, visiting, + summary_cmap); + if (summary_cmap->count(callee) != 0) { + invoke_to_summary_cmap.emplace(edge->invoke_insn(), + summary_cmap->at(callee)); } } } - return invoke_to_summary_map; -} -/* - * Analyze :method. - */ -Summary analyze_method(const init_classes::InitClassesWithSideEffects& - init_classes_with_side_effects, - const DexMethod* method, - const call_graph::Graph& call_graph, - const ptrs::FixpointIteratorMap& ptrs_fp_iter_map, - const SummaryMap& summary_map) { - auto invoke_to_summary_map = - build_summary_map(summary_map, call_graph, method); - - const auto& ptrs_fp_iter = *ptrs_fp_iter_map.at_unsafe(method); + const auto* ptrs_fp_iter = ptrs_fp_iter_map.find(method)->second; auto summary = - SummaryBuilder(init_classes_with_side_effects, invoke_to_summary_map, - ptrs_fp_iter, method->get_code()) + SummaryBuilder(init_classes_with_side_effects, invoke_to_summary_cmap, + *ptrs_fp_iter, method->get_code()) .build(); if (method->rstate.no_optimizations()) { summary.effects |= EFF_NO_OPTIMIZE; } + summary_cmap->emplace(method, summary); if (traceEnabled(OSDCE, 3)) { TRACE(OSDCE, 3, "%s %s unknown side effects (%zu)", SHOW(method), @@ -284,8 +283,6 @@ Summary analyze_method(const init_classes::InitClassesWithSideEffects& TRACE(OSDCE, 3, ""); } } - - return summary; } Summary analyze_code(const init_classes::InitClassesWithSideEffects& @@ -298,46 +295,34 @@ Summary analyze_code(const init_classes::InitClassesWithSideEffects& .build(); } -void analyze_scope(const init_classes::InitClassesWithSideEffects& - init_classes_with_side_effects, - const Scope& scope, - const call_graph::Graph& call_graph, - const ptrs::FixpointIteratorMap& ptrs_fp_iter_map, - SummaryMap* effect_summaries) { +void analyze_scope( + const init_classes::InitClassesWithSideEffects& + init_classes_with_side_effects, + const Scope& scope, + const call_graph::Graph& call_graph, + const ConcurrentMap& + ptrs_fp_iter_map, + SummaryMap* effect_summaries) { // This method is special: the bytecode verifier requires that this method // be called before a newly-allocated object gets used in any way. We can // model this by treating the method as modifying its `this` parameter -- // changing it from uninitialized to initialized. - (*effect_summaries)[method::java_lang_Object_ctor()] = Summary({0}); + (*effect_summaries)[DexMethod::get_method("Ljava/lang/Object;.:()V")] = + Summary({0}); - auto affected_methods = std::make_unique>(); - walk::parallel::code(scope, [&](const DexMethod* method, IRCode&) { - affected_methods->insert(method); + SummaryConcurrentMap summary_cmap; + for (auto& pair : *effect_summaries) { + summary_cmap.insert(pair); + } + + walk::parallel::code(scope, [&](const DexMethod* method, IRCode& code) { + PatriciaTreeSet visiting; + analyze_method_recursive(init_classes_with_side_effects, method, call_graph, + ptrs_fp_iter_map, visiting, &summary_cmap); }); - while (!affected_methods->empty()) { - SummaryConcurrentMap changed_effect_summaries; - auto next_affected_methods = - std::make_unique>(); - workqueue_run( - [&](const DexMethod* method) { - auto new_summary = - analyze_method(init_classes_with_side_effects, method, call_graph, - ptrs_fp_iter_map, *effect_summaries); - new_summary.normalize(); - auto it = effect_summaries->find(method); - if (it != effect_summaries->end() && it->second == new_summary) { - return; - } - changed_effect_summaries.emplace(method, std::move(new_summary)); - const auto& callers = call_graph.get_callers(method); - next_affected_methods->insert(callers.begin(), callers.end()); - }, - *affected_methods); - for (auto&& [method, summary] : changed_effect_summaries) { - (*effect_summaries)[method] = std::move(summary); - } - std::swap(next_affected_methods, affected_methods); + for (auto& pair : summary_cmap) { + effect_summaries->insert(pair); } } @@ -345,7 +330,6 @@ s_expr to_s_expr(const Summary& summary) { std::vector s_exprs; s_exprs.emplace_back(std::to_string(summary.effects)); std::vector mod_param_s_exprs; - mod_param_s_exprs.reserve(summary.modified_params.size()); for (auto idx : summary.modified_params) { mod_param_s_exprs.emplace_back(idx); } @@ -353,24 +337,6 @@ s_expr to_s_expr(const Summary& summary) { return s_expr(s_exprs); } -std::ostream& operator<<(std::ostream& o, const Summary& summary) { - o << "Effects: " << summary.effects << ", "; - o << "Modified parameters: "; - std::vector modified_params(summary.modified_params.begin(), - summary.modified_params.end()); - std::sort(modified_params.begin(), modified_params.end()); - bool first{true}; - for (auto p_idx : summary.modified_params) { - if (!first) { - o << ", "; - } - o << p_idx; - first = false; - } - o << "May-read-external: " << summary.may_read_external << ", "; - return o; -} - Summary Summary::from_s_expr(const s_expr& expr) { Summary summary; always_assert(expr.size() == 2); @@ -383,12 +349,4 @@ Summary Summary::from_s_expr(const s_expr& expr) { return summary; } -void Summary::join_with(const Summary& other) { - effects |= other.effects; - modified_params.insert(other.modified_params.begin(), - other.modified_params.end()); - if (other.may_read_external) { - may_read_external = true; - } -} } // namespace side_effects diff --git a/opt/object-sensitive-dce/SideEffectSummary.h b/opt/object-sensitive-dce/SideEffectSummary.h index 07239169ce..1dacbf1aea 100644 --- a/opt/object-sensitive-dce/SideEffectSummary.h +++ b/opt/object-sensitive-dce/SideEffectSummary.h @@ -31,8 +31,8 @@ * Now supposing that there are no other side effects in the method (such as * throwing an exception), we can use this classification as follows: * - * - Methods containing only #1 are always side-effect-free and can be elided - * if their return values are unused. + * - Methods containing only #1 are always pure and can be elided if their + * return values are unused. * - Methods containing only #1 and #2 can be elided if their arguments are * all non-escaping and unused, and if their return values are unused. */ @@ -53,7 +53,6 @@ enum Effects : size_t { // Marked by @DoNotOptimize EFF_NO_OPTIMIZE = 1 << 4, EFF_INIT_CLASS = 1 << 5, - EFF_NORMALIZED = 1 << 6, }; struct Summary { @@ -76,32 +75,19 @@ struct Summary { Summary(const std::initializer_list& modified_params) : modified_params(modified_params) {} - bool has_side_effects() const { - return effects != EFF_NONE || !modified_params.empty() || may_read_external; + bool is_pure() { + return effects == EFF_NONE && modified_params.empty() && !may_read_external; } - friend bool operator==(const Summary& a, const Summary& b) { return a.effects == b.effects && a.modified_params == b.modified_params && a.may_read_external == b.may_read_external; } - void normalize() { - if (effects != EFF_NONE) { - effects = EFF_NORMALIZED; - modified_params.clear(); - may_read_external = false; - } - } - - void join_with(const Summary& other); - static Summary from_s_expr(const sparta::s_expr&); }; sparta::s_expr to_s_expr(const Summary&); -std::ostream& operator<<(std::ostream& o, const Summary& summary); - using SummaryMap = std::unordered_map; using InvokeToSummaryMap = std::unordered_map; @@ -138,11 +124,6 @@ class SummaryBuilder final { reaching_defs::MoveAwareFixpointIterator* m_reaching_defs_fixpoint_iter; }; -// Builds a caller-specific summary from. -InvokeToSummaryMap build_summary_map(const SummaryMap& summary_map, - const call_graph::Graph& call_graph, - const DexMethod* method); - // For testing. Summary analyze_code(const init_classes::InitClassesWithSideEffects& init_classes_with_side_effects, @@ -157,7 +138,8 @@ void analyze_scope(const init_classes::InitClassesWithSideEffects& init_classes_with_side_effects, const Scope& scope, const call_graph::Graph&, - const local_pointers::FixpointIteratorMap&, + const ConcurrentMap&, SummaryMap* effect_summaries); } // namespace side_effects diff --git a/opt/object-sensitive-dce/UsedVarsAnalysis.cpp b/opt/object-sensitive-dce/UsedVarsAnalysis.cpp index 78b20ca187..edc224925b 100644 --- a/opt/object-sensitive-dce/UsedVarsAnalysis.cpp +++ b/opt/object-sensitive-dce/UsedVarsAnalysis.cpp @@ -45,10 +45,8 @@ using namespace ir_analyzer; FixpointIterator::FixpointIterator( const local_pointers::FixpointIterator& pointers_fp_iter, side_effects::InvokeToSummaryMap invoke_to_summary_map, - const cfg::ControlFlowGraph& cfg, - const DexMethod* method) + const cfg::ControlFlowGraph& cfg) : BaseBackwardsIRAnalyzer(cfg), - m_method(method), m_insn_env_map(gen_instruction_environment_map(cfg, pointers_fp_iter)), m_invoke_to_summary_map(std::move(invoke_to_summary_map)) {} @@ -80,7 +78,7 @@ void FixpointIterator::analyze_instruction(IRInstruction* insn, for (size_t i = 0; i < insn->srcs_size(); ++i) { auto reg = insn->src(i); used_vars->add(reg); - const auto& pointers = env.get_pointers(reg); + auto pointers = env.get_pointers(reg); if (!pointers.is_value()) { continue; } @@ -100,7 +98,7 @@ void FixpointIterator::analyze_instruction(IRInstruction* insn, bool FixpointIterator::is_used_or_escaping_write(const ptrs::Environment& env, const UsedVarsSet& used_vars, reg_t obj_reg) const { - const auto& pointers = env.get_pointers(obj_reg); + auto pointers = env.get_pointers(obj_reg); if (!pointers.is_value()) { return true; } @@ -127,6 +125,7 @@ bool FixpointIterator::is_required(const IRInstruction* insn, case OPCODE_RETURN_OBJECT: case OPCODE_MONITOR_ENTER: case OPCODE_MONITOR_EXIT: + case OPCODE_CHECK_CAST: case OPCODE_THROW: case OPCODE_GOTO: case OPCODE_SWITCH: @@ -174,22 +173,21 @@ bool FixpointIterator::is_required(const IRInstruction* insn, case OPCODE_SPUT_SHORT: { return true; } - case OPCODE_INVOKE_SUPER: - case OPCODE_INVOKE_INTERFACE: case OPCODE_INVOKE_DIRECT: case OPCODE_INVOKE_STATIC: case OPCODE_INVOKE_VIRTUAL: { - auto method = resolve_invoke_method(insn, m_method); + auto method = resolve_method(insn->get_method(), opcode_to_search(insn)); + if (method == nullptr) { + return true; + } const auto& env = m_insn_env_map.at(insn); - if (method != nullptr) { - if (method::is_init(method)) { - if (used_vars.contains(insn->src(0)) || - is_used_or_escaping_write(env, used_vars, insn->src(0))) { - return true; - } - } else if (assumenosideeffects(method)) { - return used_vars.contains(RESULT_REGISTER); + if (method::is_init(method)) { + if (used_vars.contains(insn->src(0)) || + is_used_or_escaping_write(env, used_vars, insn->src(0))) { + return true; } + } else if (assumenosideeffects(method)) { + return used_vars.contains(RESULT_REGISTER); } if (!m_invoke_to_summary_map.count(insn)) { return true; @@ -208,6 +206,10 @@ bool FixpointIterator::is_required(const IRInstruction* insn, return is_used_or_escaping_write(env, used_vars, insn->src(idx)); }); } + case OPCODE_INVOKE_SUPER: + case OPCODE_INVOKE_INTERFACE: { + return true; + } default: { if (insn->has_dest()) { return used_vars.contains(insn->dest()); diff --git a/opt/object-sensitive-dce/UsedVarsAnalysis.h b/opt/object-sensitive-dce/UsedVarsAnalysis.h index ef173d7227..042e06afcf 100644 --- a/opt/object-sensitive-dce/UsedVarsAnalysis.h +++ b/opt/object-sensitive-dce/UsedVarsAnalysis.h @@ -68,8 +68,7 @@ class FixpointIterator final public: FixpointIterator(const local_pointers::FixpointIterator& pointers_fp_iter, side_effects::InvokeToSummaryMap invoke_to_summary_map, - const cfg::ControlFlowGraph& cfg, - const DexMethod* method = nullptr); + const cfg::ControlFlowGraph& cfg); void analyze_instruction(IRInstruction* insn, UsedVarsSet* used_vars) const override; @@ -92,7 +91,6 @@ class FixpointIterator final } private: - const DexMethod* m_method; std::unordered_map m_insn_env_map; side_effects::InvokeToSummaryMap m_invoke_to_summary_map; diff --git a/opt/optimize_enums/EnumConfig.h b/opt/optimize_enums/EnumConfig.h index f2a9e30128..67e07ebd00 100644 --- a/opt/optimize_enums/EnumConfig.h +++ b/opt/optimize_enums/EnumConfig.h @@ -30,7 +30,7 @@ struct ParamSummary { void print(const DexMethodRef* method) const; }; -using SummaryMap = InsertOnlyConcurrentMap; +using SummaryMap = ConcurrentMap; struct Config { /** diff --git a/opt/optimize_enums/EnumTransformer.cpp b/opt/optimize_enums/EnumTransformer.cpp index 6070c4cb17..e11c422b71 100644 --- a/opt/optimize_enums/EnumTransformer.cpp +++ b/opt/optimize_enums/EnumTransformer.cpp @@ -84,8 +84,7 @@ struct EnumUtil { ConcurrentSet m_instance_methods; // Store methods for getting instance fields to be generated later. - InsertOnlyConcurrentMap - m_get_instance_field_methods; + ConcurrentMap m_get_instance_field_methods; DexMethodRef* m_values_method_ref = nullptr; @@ -138,14 +137,13 @@ struct EnumUtil { DexMethodRef* INTEGER_COMPARETO_METHOD = DexMethod::make_method( "Ljava/lang/Integer;.compareTo:(Ljava/lang/Integer;)I"); DexMethodRef* INTEGER_VALUEOF_METHOD = method::java_lang_Integer_valueOf(); + DexMethodRef* RTEXCEPTION_CTOR_METHOD = + method::java_lang_RuntimeException_init_String(); DexMethodRef* ILLEGAL_ARG_CONSTRUCT_METHOD = DexMethod::make_method( "Ljava/lang/IllegalArgumentException;.:(Ljava/lang/String;)V"); DexMethodRef* STRING_EQ_METHOD = DexMethod::make_method("Ljava/lang/String;.equals:(Ljava/lang/Object;)Z"); - InsertOnlyConcurrentMap - m_user_defined_tostring_method_cache; - explicit EnumUtil(const Config& config) : m_config(config) {} void create_util_class(DexStoresVector* stores, uint32_t fields_count) { @@ -295,19 +293,15 @@ struct EnumUtil { * Store the method ref at the same time. */ DexMethodRef* add_get_ifield_method(DexType* enum_type, DexFieldRef* ifield) { - return *m_get_instance_field_methods - .get_or_create_and_assert_equal( - ifield, - [&](auto*) { - auto proto = DexProto::make_proto( - ifield->get_type(), - DexTypeList::make_type_list({INTEGER_TYPE})); - auto method_name = DexString::make_string( - "redex$OE$get_" + ifield->str()); - return DexMethod::make_method(enum_type, method_name, - proto); - }) - .first; + if (m_get_instance_field_methods.count(ifield)) { + return m_get_instance_field_methods.at(ifield); + } + auto proto = DexProto::make_proto( + ifield->get_type(), DexTypeList::make_type_list({INTEGER_TYPE})); + auto method_name = DexString::make_string("redex$OE$get_" + ifield->str()); + auto method = DexMethod::make_method(enum_type, method_name, proto); + m_get_instance_field_methods.insert(std::make_pair(ifield, method)); + return method; } /** @@ -315,19 +309,18 @@ struct EnumUtil { * `Enum.toString()`. Return `nullptr` if `Enum.toString()` is not overridden. */ DexMethod* get_user_defined_tostring_method(DexClass* cls) { - return *m_user_defined_tostring_method_cache - .get_or_create_and_assert_equal( - cls, - [&](auto*) -> DexMethod* { - for (auto vmethod : cls->get_vmethods()) { - if (method::signatures_match(vmethod, - ENUM_TOSTRING_METHOD)) { - return vmethod; - } - } - return nullptr; - }) - .first; + static ConcurrentMap cache; + if (cache.count(cls)) { + return cache.at(cls); + } + for (auto vmethod : cls->get_vmethods()) { + if (method::signatures_match(vmethod, ENUM_TOSTRING_METHOD)) { + cache.insert(std::make_pair(cls, vmethod)); + return vmethod; + } + } + cache.insert(std::make_pair(cls, nullptr)); + return nullptr; } private: @@ -673,8 +666,8 @@ class CodeTransformer final { auto type = insn->get_type(); auto new_type = try_convert_to_int_type(type); if (new_type) { - const auto& possible_src_types = env.get(insn->src(0)); - if (!possible_src_types.empty()) { + auto possible_src_types = env.get(insn->src(0)); + if (possible_src_types.size() != 0) { DexType* candidate_type = extract_candidate_enum_type(possible_src_types); always_assert(candidate_type == type); @@ -971,28 +964,19 @@ class CodeTransformer final { // If this is toString() and there is no CandidateEnum.toString(), then we // call Enum.name() instead. - DexMethod* method = nullptr; if (method::signatures_match(method_ref, - m_enum_util->ENUM_TOSTRING_METHOD)) { - method = m_enum_util->get_user_defined_tostring_method( - type_class(candidate_type)); - if (method == nullptr) { - update_invoke_name(env, cfg, block, mie); - return; - } - } - if (method == nullptr) { - method = resolve_method(method_ref, opcode_to_search(insn)); + m_enum_util->ENUM_TOSTRING_METHOD) && + m_enum_util->get_user_defined_tostring_method( + type_class(candidate_type)) == nullptr) { + update_invoke_name(env, cfg, block, mie); + } else { + auto method = resolve_method(method_ref, opcode_to_search(insn)); + always_assert(method); + auto new_insn = (new IRInstruction(*insn)) + ->set_opcode(OPCODE_INVOKE_STATIC) + ->set_method(method); + m_replacements.push_back(InsnReplacement(cfg, block, mie, new_insn)); } - always_assert(method); - always_assert_log( - !method->is_external() || is_static(method), - "[%s] with candidate type %s resolved to external non-static method %s", - SHOW(insn), SHOW(candidate_type), SHOW(method)); - auto new_insn = (new IRInstruction(*insn)) - ->set_opcode(OPCODE_INVOKE_STATIC) - ->set_method(method); - m_replacements.push_back(InsnReplacement(cfg, block, mie, new_insn)); } /** @@ -1011,7 +995,7 @@ class CodeTransformer final { } else if (!m_enum_util->is_super_type_of_candidate_enum(target_type)) { return nullptr; } - const auto& type_set = reg_types.elements(); + auto type_set = reg_types.elements(); if (type_set.empty()) { // Register holds null value, we infer the type in instruction. return candidate_type; @@ -1552,7 +1536,7 @@ class EnumTransformer final { cfg.calculate_exit_block(); ptrs::FixpointIterator fp_iter(cfg); fp_iter.run(ptrs::Environment()); - used_vars::FixpointIterator uv_fpiter(fp_iter, summaries, cfg, ctor); + used_vars::FixpointIterator uv_fpiter(fp_iter, summaries, cfg); uv_fpiter.run(used_vars::UsedVarsSet()); auto dead_instructions = used_vars::get_dead_instructions(cfg, uv_fpiter); for (const auto& insn : dead_instructions) { diff --git a/opt/optimize_enums/OptimizeEnums.cpp b/opt/optimize_enums/OptimizeEnums.cpp index 1dd54ea570..cb52d65dd7 100644 --- a/opt/optimize_enums/OptimizeEnums.cpp +++ b/opt/optimize_enums/OptimizeEnums.cpp @@ -499,9 +499,8 @@ class OptimizeEnums { [](const auto& p) { return p.first; }); std::sort(unsafe_types.begin(), unsafe_types.end(), compare_dextypes); for (auto* t : unsafe_types) { - const auto& unsafe_enums_at_t = unsafe_enums.at_unsafe(t); - ofs << show(t) << ":" << unsafe_enums_at_t << "\n"; - for (auto u : unsafe_enums_at_t) { + ofs << show(t) << ":" << unsafe_enums.at(t) << "\n"; + for (auto u : unsafe_enums.at(t)) { ++unsafe_counts[u]; } } diff --git a/opt/optimize_enums/OptimizeEnums.h b/opt/optimize_enums/OptimizeEnums.h index 53bfc70ffa..b4e6f78596 100644 --- a/opt/optimize_enums/OptimizeEnums.h +++ b/opt/optimize_enums/OptimizeEnums.h @@ -21,7 +21,9 @@ class OptimizeEnumsPass : public Pass { using namespace redex_properties::interactions; using namespace redex_properties::names; return { + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } diff --git a/opt/optimize_resources/OptimizeResources.cpp b/opt/optimize_resources/OptimizeResources.cpp index 09eb10d30a..102407a485 100644 --- a/opt/optimize_resources/OptimizeResources.cpp +++ b/opt/optimize_resources/OptimizeResources.cpp @@ -83,12 +83,11 @@ void extract_resources_from_static_arrays( if (ir_code == nullptr) { continue; } - always_assert(ir_code->editable_cfg_built()); - auto& cfg = ir_code->cfg(); - live_range::MoveAwareChains move_aware_chains(cfg); + cfg::ScopedCFG cfg(ir_code); + live_range::MoveAwareChains move_aware_chains(*cfg); auto use_defs = move_aware_chains.get_use_def_chains(); auto def_uses = move_aware_chains.get_def_use_chains(); - for (const auto& mie : InstructionIterable(cfg)) { + for (const auto& mie : InstructionIterable(*cfg)) { auto insn = mie.insn; if (!opcode::is_an_sput(insn->opcode())) { continue; @@ -109,7 +108,7 @@ void extract_resources_from_static_arrays( always_assert_log(array_def->opcode() == OPCODE_NEW_ARRAY, "OptimizeResources does not support extracting " "resources from array created by %s\nin %s:\n%s", - SHOW(array_def), SHOW(clinit), SHOW(cfg)); + SHOW(array_def), SHOW(clinit), SHOW(*cfg)); // should be only one, but we can be conservative and consider all for (auto& use : uses) { switch (use.insn->opcode()) { @@ -192,9 +191,9 @@ void compute_transitive_closure( while (!potential_file_paths.empty()) { for (auto& str : potential_file_paths) { if (is_resource_xml(str)) { - auto r_str = std::string(zip_dir).append("/").append(str); + auto r_str = zip_dir + "/" + str; if (explored_xml_files->find(r_str) == explored_xml_files->end()) { - next_xml_files.emplace(std::move(r_str)); + next_xml_files.emplace(r_str); } } } @@ -390,12 +389,12 @@ void remap_resource_class_clinit( DexMethod* clinit) { const auto c_name = cls->get_name()->str(); IRCode* ir_code = clinit->get_code(); - always_assert(ir_code->editable_cfg_built()); + // Lookup from new-array instruction to an updated array size. std::unordered_map new_array_size_updates; IRInstruction* last_new_array = nullptr; - auto& cfg = ir_code->cfg(); - for (const MethodItemEntry& mie : cfg::InstructionIterable(cfg)) { + + for (const MethodItemEntry& mie : InstructionIterable(ir_code)) { IRInstruction* insn = mie.insn; if (insn->opcode() == OPCODE_CONST) { auto const_literal = insn->get_literal(); @@ -471,7 +470,7 @@ void remap_resource_class_clinit( if (new_array_size_updates.empty()) { return; } - auto ii = cfg::InstructionIterable(cfg); + auto ii = InstructionIterable(ir_code); for (auto it = ii.begin(); it != ii.end(); ++it) { IRInstruction* insn = it->insn; auto search = new_array_size_updates.find(insn); @@ -482,12 +481,12 @@ void remap_resource_class_clinit( // Make this additive to not impact other instructions. // Note that the naive numbering scheme here requires RegAlloc to be run // later (which should be the case for all apps). - auto new_reg = cfg.allocate_temp(); + auto new_reg = ir_code->allocate_temp(); auto const_insn = new IRInstruction(OPCODE_CONST); const_insn->set_literal(new_size); const_insn->set_dest(new_reg); insn->set_src(0, new_reg); - cfg.insert_before(it, const_insn); + ir_code->insert_before(it.unwrap(), const_insn); } } } @@ -631,7 +630,7 @@ void OptimizeResourcesPass::run_pass(DexStoresVector& stores, PassManager& mgr) { std::string zip_dir; conf.get_json_config().get("apk_dir", "", zip_dir); - always_assert(!zip_dir.empty()); + always_assert(zip_dir.size()); // 1. Get all known resource ID's from either resources.pb(AAB) or // resources.arsc(APK) file. diff --git a/opt/optimize_resources/OptimizeResources.h b/opt/optimize_resources/OptimizeResources.h index 74e65422bc..52a7086e7c 100644 --- a/opt/optimize_resources/OptimizeResources.h +++ b/opt/optimize_resources/OptimizeResources.h @@ -119,8 +119,9 @@ class OptimizeResourcesPass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, - {InitialRenameClass, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } @@ -135,6 +136,7 @@ class OptimizeResourcesPass : public Pass { } void eval_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; + bool is_cfg_legacy() override { return true; } void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; std::unique_ptr clone(const std::string& new_name) const override { return std::make_unique(new_name); diff --git a/opt/original_name/OriginalNamePass.cpp b/opt/original_name/OriginalNamePass.cpp index 0c427fb221..2640422795 100644 --- a/opt/original_name/OriginalNamePass.cpp +++ b/opt/original_name/OriginalNamePass.cpp @@ -108,7 +108,7 @@ void OriginalNamePass::run_pass(DexStoresVector& stores, auto simple_name = (lastDot != std::string::npos) ? external_name.substr(lastDot + 1) : external_name; - auto simple_name_s = DexString::make_string(simple_name); + auto simple_name_s = DexString::make_string(simple_name.c_str()); always_assert_log( DexField::get_field(cls_type, field_name, string_type) == nullptr, "field %s already exists!", diff --git a/opt/original_name/OriginalNamePass.h b/opt/original_name/OriginalNamePass.h index a65c900093..49032be018 100644 --- a/opt/original_name/OriginalNamePass.h +++ b/opt/original_name/OriginalNamePass.h @@ -20,9 +20,9 @@ class OriginalNamePass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, {RenameClass, Preserves}, - {InitialRenameClass, RequiresAndEstablishes}, }; } diff --git a/opt/outliner/InstructionSequenceOutliner.cpp b/opt/outliner/InstructionSequenceOutliner.cpp index ece6fa886f..57ae332eed 100644 --- a/opt/outliner/InstructionSequenceOutliner.cpp +++ b/opt/outliner/InstructionSequenceOutliner.cpp @@ -1208,9 +1208,9 @@ static void get_recurring_cores( const std::unordered_set& sufficiently_hot_methods, const RefChecker& ref_checker, CandidateInstructionCoresSet* recurring_cores, - InsertOnlyConcurrentMap* - block_deciders) { - AtomicMap + ConcurrentMap* block_deciders) { + ConcurrentMap concurrent_cores; walk::parallel::code( scope, [&config, &ref_checker, &sufficiently_warm_methods, @@ -1219,7 +1219,7 @@ static void get_recurring_cores( if (!can_outline_from_method(method)) { return; } - always_assert(code.editable_cfg_built()); + code.build_cfg(/* editable */ true); code.cfg().calculate_exit_block(); CanOutlineBlockDecider block_decider( config.profile_guidance, sufficiently_warm_methods.count(method), @@ -1248,7 +1248,10 @@ static void get_recurring_cores( } cores_builder.push_back(insn); if (cores_builder.has_value()) { - concurrent_cores.fetch_add(cores_builder.get_value(), 1); + concurrent_cores.update(cores_builder.get_value(), + [](const CandidateInstructionCores&, + size_t& occurrences, + bool /* exists */) { occurrences++; }); } } } @@ -1256,9 +1259,8 @@ static void get_recurring_cores( }); size_t singleton_cores{0}; for (auto& p : concurrent_cores) { - auto count = p.second.load(); - always_assert(count > 0); - if (count > 1) { + always_assert(p.second > 0); + if (p.second > 1) { recurring_cores->insert(p.first); } else { singleton_cores++; @@ -1285,12 +1287,9 @@ struct CandidateInfo { // store. Order vector is used for keeping the track of the order of each // candidate stored. struct ReusableOutlinedMethods { - struct OutlinedMethod { - const DexStore* store; - DexMethod* method; - std::set pattern_ids; - }; - std::unordered_map, CandidateHasher> + std::unordered_map>>, + CandidateHasher> map; std::vector order; }; @@ -1400,8 +1399,6 @@ std::unordered_set get_referenced_types(const Candidate& c) { // Finds an outlined method that either resides in an outlined helper class, or // a common base class, or is co-located with its references. static DexMethod* find_reusable_method( - const DexStore* store, - const DexStoreDependencies& store_dependencies, const Candidate& c, const CandidateInfo& ci, const ReusableOutlinedMethods& outlined_methods, @@ -1414,11 +1411,10 @@ static DexMethod* find_reusable_method( if (it == outlined_methods.map.end()) { return nullptr; } + auto& method_pattern_pairs = it->second; DexMethod* helper_class_method{nullptr}; - for (auto&& [other_store, method, _] : it->second) { - if (store != other_store && !store_dependencies.count(other_store)) { - continue; - } + for (const auto& vec_entry : method_pattern_pairs) { + auto method = vec_entry.first; auto cls = type_class(method->get_class()); if (cls->rstate.outlined()) { helper_class_method = method; @@ -1439,8 +1435,6 @@ static DexMethod* find_reusable_method( } static size_t get_savings(const Config& config, - const DexStore* store, - const DexStoreDependencies& store_dependencies, const Candidate& c, const CandidateInfo& ci, const ReusableOutlinedMethods& outlined_methods) { @@ -1449,7 +1443,7 @@ static size_t get_savings(const Config& config, COST_METHOD_METADATA + (c.res_type ? COST_INVOKE_WITH_RESULT : COST_INVOKE_WITHOUT_RESULT) * ci.count; - if (find_reusable_method(store, store_dependencies, c, ci, outlined_methods, + if (find_reusable_method(c, ci, outlined_methods, config.reuse_outlined_methods_across_dexes) == nullptr) { outlined_cost += COST_METHOD_BODY + c.size; @@ -1484,13 +1478,10 @@ struct CandidateWithInfo { static void get_beneficial_candidates( const Config& config, PassManager& mgr, - const DexStore* store, - const DexStoreDependencies& store_dependencies, - const Scope& dex, + const Scope& scope, const RefChecker& ref_checker, const CandidateInstructionCoresSet& recurring_cores, - const InsertOnlyConcurrentMap& - block_deciders, + const ConcurrentMap& block_deciders, const ReusableOutlinedMethods* outlined_methods, std::vector* candidates_with_infos, std::unordered_map>* @@ -1498,9 +1489,9 @@ static void get_beneficial_candidates( ConcurrentMap concurrent_candidates; FindCandidatesStats stats; - walk::parallel::code(dex, [&config, &ref_checker, &recurring_cores, - &concurrent_candidates, &block_deciders, - &stats](DexMethod* method, IRCode& code) { + walk::parallel::code(scope, [&config, &ref_checker, &recurring_cores, + &concurrent_candidates, &block_deciders, + &stats](DexMethod* method, IRCode& code) { if (!can_outline_from_method(method)) { return; } @@ -1525,8 +1516,7 @@ static void get_beneficial_candidates( candidates_by_methods; size_t beneficial_count{0}, maleficial_count{0}; for (auto& p : concurrent_candidates) { - if (get_savings(config, store, store_dependencies, p.first, p.second, - *outlined_methods) > 0) { + if (get_savings(config, p.first, p.second, *outlined_methods) > 0) { beneficial_count += p.second.count; for (auto& q : p.second.methods) { candidates_by_methods[q.first].insert(p.first); @@ -1738,7 +1728,6 @@ class OutlinedMethodCreator { auto manager = g_redex->get_position_pattern_switch_manager(); // Order methods to make sure we get deterministic pattern-ids. std::vector ordered_methods; - ordered_methods.reserve(ci.methods.size()); for (auto& p : ci.methods) { ordered_methods.push_back(p.first); } @@ -2325,8 +2314,6 @@ using NewlyOutlinedMethods = // Outlining all occurrences of a particular candidate. bool outline_candidate(const Config& config, - const DexStore* store, - const DexStoreDependencies& store_dependencies, const Candidate& c, const CandidateInfo& ci, ReusableOutlinedMethods* outlined_methods, @@ -2346,9 +2333,8 @@ bool outline_candidate(const Config& config, auto rtype = c.res_type ? c.res_type : type::_void(); type_refs_to_insert.insert(const_cast(rtype)); - DexMethod* outlined_method{ - find_reusable_method(store, store_dependencies, c, ci, *outlined_methods, - reuse_outlined_methods_across_dexes)}; + DexMethod* outlined_method{find_reusable_method( + c, ci, *outlined_methods, reuse_outlined_methods_across_dexes)}; if (outlined_method) { type_refs_to_insert.insert(outlined_method->get_class()); if (!dex_state->can_insert_type_refs(type_refs_to_insert)) { @@ -2356,14 +2342,16 @@ bool outline_candidate(const Config& config, } if (config.full_dbg_positions) { - auto& c_outlined_methods = outlined_methods->map.at(c); - auto it = - std::find_if(c_outlined_methods.begin(), c_outlined_methods.end(), - [outlined_method](const auto& p) { - return p.method == outlined_method; - }); - outlined_method_creator->add_outlined_dbg_position_patterns( - c, ci, &it->pattern_ids); + auto& pairs = outlined_methods->map.at(c); + auto it = std::find_if( + pairs.begin(), pairs.end(), + [&outlined_method]( + const std::pair>& pair) { + return pair.first == outlined_method; + }); + auto& pattern_ids = it->second; + outlined_method_creator->add_outlined_dbg_position_patterns(c, ci, + &pattern_ids); } (*num_reused_methods)++; @@ -2395,8 +2383,7 @@ bool outline_candidate(const Config& config, outlined_method = outlined_method_creator->create_outlined_method( c, ci, host_class, &position_pattern_ids); outlined_methods->order.push_back(c); - outlined_methods->map[c].push_back( - {store, outlined_method, position_pattern_ids}); + outlined_methods->map[c].push_back({outlined_method, position_pattern_ids}); auto& methods = (*newly_outlined_methods)[outlined_method]; for (auto& p : ci.methods) { methods.push_back(p.first); @@ -2439,8 +2426,6 @@ bool outline_candidate(const Config& config, static NewlyOutlinedMethods outline( const Config& config, PassManager& mgr, - const DexStore* store, - const DexStoreDependencies& store_dependencies, DexState& dex_state, int min_sdk, std::vector* candidates_with_infos, @@ -2461,12 +2446,11 @@ static NewlyOutlinedMethods outline( // impacted candidates, until there is no more beneficial candidate left. using Priority = uint64_t; MutablePriorityQueue pq; - auto get_priority = [&config, store, &store_dependencies, - &candidates_with_infos, + auto get_priority = [&config, &candidates_with_infos, outlined_methods](CandidateId id) { auto& cwi = candidates_with_infos->at(id); - auto savings = get_savings(config, store, store_dependencies, cwi.candidate, - cwi.info, *outlined_methods); + auto savings = + get_savings(config, cwi.candidate, cwi.info, *outlined_methods); auto size = cwi.candidate.size; auto count = cwi.info.count; Priority primary_priority = (savings + 1) * 100000 / (size * count); @@ -2502,8 +2486,8 @@ static NewlyOutlinedMethods outline( auto id = pq.front(); auto& cwi = candidates_with_infos->at(id); - auto savings = get_savings(config, store, store_dependencies, cwi.candidate, - cwi.info, *outlined_methods); + auto savings = + get_savings(config, cwi.candidate, cwi.info, *outlined_methods); always_assert(savings > 0); total_savings += savings; outlined_count += cwi.info.count; @@ -2513,10 +2497,10 @@ static NewlyOutlinedMethods outline( "[invoke sequence outliner] %4zx(%3zu) [%zu]: %zu byte savings", cwi.info.count, cwi.info.methods.size(), cwi.candidate.size, 2 * savings); - if (outline_candidate(config, store, store_dependencies, cwi.candidate, - cwi.info, outlined_methods, &newly_outlined_methods, - &dex_state, &host_class_selector, - &outlined_method_creator, num_reused_methods, + if (outline_candidate(config, cwi.candidate, cwi.info, outlined_methods, + &newly_outlined_methods, &dex_state, + &host_class_selector, &outlined_method_creator, + num_reused_methods, config.reuse_outlined_methods_across_dexes)) { dex_state.insert_method_ref(); } else { @@ -2553,9 +2537,8 @@ static NewlyOutlinedMethods outline( // Update priorities of affected candidates for (auto other_id : other_candidate_ids_with_changes) { auto& other_cwi = candidates_with_infos->at(other_id); - auto other_savings = - get_savings(config, store, store_dependencies, other_cwi.candidate, - other_cwi.info, *outlined_methods); + auto other_savings = get_savings(config, other_cwi.candidate, + other_cwi.info, *outlined_methods); if (other_savings == 0) { erase(other_id, other_cwi); } else { @@ -2738,7 +2721,6 @@ void reorder_with_method_profiles( "from %s to %s", SHOW(method->get_name()), SHOW(method->get_class()), SHOW(target_class)); - change_visibility(method, target_class->get_type()); relocate_method(method, target_class->get_type()); method->set_deobfuscated_name(show(method)); relocated_outlined_methods++; @@ -2755,12 +2737,15 @@ void reorder_with_method_profiles( // clear_cfgs //////////////////////////////////////////////////////////////////////////////// -static size_t count_methods(const Scope& scope) { +static size_t clear_cfgs(const Scope& scope) { std::atomic methods{0}; - walk::parallel::code(scope, [&methods](DexMethod* method, IRCode&) { + walk::parallel::code(scope, [&methods](DexMethod* method, IRCode& code) { if (!can_outline_from_method(method)) { return; } + + code.clear_cfg(); + methods++; }); @@ -2984,28 +2969,20 @@ class OutlinedMethodBodySetter { // set body for each method stored in ReusableOutlinedMethod void set_method_body(ReusableOutlinedMethods& outlined_methods) { for (auto& c : outlined_methods.order) { - auto it = outlined_methods.map.find(c); - always_assert(it != outlined_methods.map.end()); - auto& c_outlined_methods = it->second; - auto& front = c_outlined_methods.front(); - auto& outlined_method = front.method; - always_assert(!front.pattern_ids.empty()); + auto& method_pattern_pairs = outlined_methods.map[c]; + auto& outlined_method = method_pattern_pairs.front().first; + auto& position_pattern_ids = method_pattern_pairs.front().second; + always_assert(!position_pattern_ids.empty()); outlined_method->set_code( - get_outlined_code(outlined_method, c, front.pattern_ids)); + get_outlined_code(outlined_method, c, position_pattern_ids)); change_visibility(outlined_method->get_code(), outlined_method->get_class(), outlined_method); - outlined_method->get_code()->build_cfg(); TRACE(ISO, 5, "[invoke sequence outliner] set the body of %s as \n%s", SHOW(outlined_method), SHOW(outlined_method->get_code())); m_outlined_method_body_set++; // pop up the front pair to avoid duplicate - c_outlined_methods.pop_front(); - if (c_outlined_methods.empty()) { - outlined_methods.map.erase(c); - } + method_pattern_pairs.pop_front(); } - always_assert(outlined_methods.map.empty()); - outlined_methods.order.clear(); } }; @@ -3055,25 +3032,6 @@ size_t update_method_profiles( return count; } -std::vector get_stores(DexStoresVector& stores, - XStoreRefs& xstores) { - std::vector res; - res.reserve(stores.size()); - for (auto& store : stores) { - res.emplace_back(&store); - } - std::sort(res.begin(), res.end(), [&xstores](DexStore* s, DexStore* t) { - if (xstores.get_transitive_resolved_dependencies(t).count(s)) { - return true; - } - if (xstores.get_transitive_resolved_dependencies(s).count(t)) { - return false; - } - return s->get_name() < t->get_name(); - }); - return res; -} - } // namespace void InstructionSequenceOutliner::bind_config() { @@ -3202,12 +3160,11 @@ void InstructionSequenceOutliner::run_pass(DexStoresVector& stores, // keep track of the outlined methods and scope for reordering later OutlinedMethodsToReorder outlined_methods_to_reorder; size_t num_reused_methods{0}; + boost::optional last_store_idx; auto iteration = m_iteration++; bool is_primary_dex{true}; - for (auto* store : get_stores(stores, xstores)) { - auto store_dependencies = - xstores.get_transitive_resolved_dependencies(store); - for (auto& dex : store->get_dexen()) { + for (auto& store : stores) { + for (auto& dex : store.get_dexen()) { if (is_primary_dex) { is_primary_dex = false; if (!m_config.outline_from_primary_dex) { @@ -3224,35 +3181,46 @@ void InstructionSequenceOutliner::run_pass(DexStoresVector& stores, return xstores.get_store_idx( cls->get_type()) != store_idx; }) == dex.end()); + if (last_store_idx && + xstores.illegal_ref_between_stores(store_idx, *last_store_idx)) { + // TODO: Keep around all store dependencies and reuse when possible + // set method body before the storage is cleared. + outlined_method_body_setter.set_method_body(outlined_methods); + TRACE(ISO, 3, + "Clearing reusable outlined methods when transitioning from " + "store %zu to %zu", + *last_store_idx, store_idx); + outlined_methods.map.clear(); + outlined_methods.order.clear(); + } + last_store_idx = store_idx; RefChecker ref_checker{&xstores, store_idx, min_sdk_api}; CandidateInstructionCoresSet recurring_cores; - InsertOnlyConcurrentMap - block_deciders; + ConcurrentMap block_deciders; get_recurring_cores(m_config, mgr, dex, sufficiently_warm_methods, sufficiently_hot_methods, ref_checker, &recurring_cores, &block_deciders); std::vector candidates_with_infos; std::unordered_map> candidate_ids_by_methods; - get_beneficial_candidates(m_config, mgr, store, store_dependencies, dex, - ref_checker, recurring_cores, block_deciders, - &outlined_methods, &candidates_with_infos, - &candidate_ids_by_methods); + get_beneficial_candidates( + m_config, mgr, dex, ref_checker, recurring_cores, block_deciders, + &outlined_methods, &candidates_with_infos, &candidate_ids_by_methods); // TODO: Merge candidates that are equivalent except that one returns // something and the other doesn't. Affects around 1.5% of candidates. DexState dex_state(mgr, init_classes_with_side_effects, dex, dex_id++, reserved_trefs, reserved_mrefs); auto newly_outlined_methods = - outline(m_config, mgr, store, store_dependencies, dex_state, min_sdk, - &candidates_with_infos, &candidate_ids_by_methods, - &outlined_methods, iteration, &num_reused_methods); + outline(m_config, mgr, dex_state, min_sdk, &candidates_with_infos, + &candidate_ids_by_methods, &outlined_methods, iteration, + &num_reused_methods); outlined_methods_to_reorder.push_back({&dex, newly_outlined_methods}); auto affected_methods = count_affected_methods(newly_outlined_methods); - auto total_methods = count_methods(dex); + auto total_methods = clear_cfgs(dex); if (total_methods > 0) { mgr.incr_metric(std::string("percent_methods_affected_in_Dex") + - std::to_string(dex_id) + "_" + store->get_name(), + std::to_string(dex_id), affected_methods * 100 / total_methods); } } diff --git a/opt/outliner/InstructionSequenceOutliner.h b/opt/outliner/InstructionSequenceOutliner.h index 89ee5a5935..035caa54c5 100644 --- a/opt/outliner/InstructionSequenceOutliner.h +++ b/opt/outliner/InstructionSequenceOutliner.h @@ -42,13 +42,14 @@ class InstructionSequenceOutliner : public Pass { {DexLimitsObeyed, Preserves}, {HasSourceBlocks, RequiresAndEstablishes}, {NoResolvablePureRefs, Preserves}, - {SpuriousGetClassCallsInterned, RequiresAndEstablishes}, - {InitialRenameClass, Preserves}, + {NoSpuriousGetClassCalls, Establishes}, }; } void bind_config() override; + bool is_cfg_legacy() override { return true; } + void run_pass(DexStoresVector& stores, ConfigFiles& config, PassManager& mgr) override; diff --git a/opt/outliner/MethodClosures.cpp b/opt/outliner/MethodClosures.cpp index 2319bab450..5f91ce726c 100644 --- a/opt/outliner/MethodClosures.cpp +++ b/opt/outliner/MethodClosures.cpp @@ -167,7 +167,7 @@ std::shared_ptr discover_closures(DexMethod* method, any_throw = true; break; } - const auto& count = mca->get_exit_state_at(e->src()); + auto count = mca->get_exit_state_at(e->src()); if (count != sparta::ConstantAbstractDomain(0)) { any_non_zero_monitor_count = true; break; diff --git a/opt/outliner/MethodSplitter.cpp b/opt/outliner/MethodSplitter.cpp index fab3b51798..81cd63ec40 100644 --- a/opt/outliner/MethodSplitter.cpp +++ b/opt/outliner/MethodSplitter.cpp @@ -282,22 +282,21 @@ DexMethod* split_method(const SplittableClosure& splittable_closure, return split_method; } -ConcurrentSet split_splittable_closures( +std::unordered_set split_splittable_closures( const std::vector& dexen, int32_t min_sdk, const init_classes::InitClassesWithSideEffects& init_classes_with_side_effects, size_t reserved_trefs, size_t reserved_mrefs, - const ConcurrentMap>& + const std::unordered_map>& splittable_closures, const std::string& name_infix, - AtomicMap* uniquifiers, + ConcurrentMap* uniquifiers, Stats* stats, std::unordered_map>* dex_states, ConcurrentSet* concurrent_added_methods, - InsertOnlyConcurrentMap* - concurrent_new_hot_methods) { + ConcurrentMap* concurrent_new_hot_methods) { Timer t("split"); ConcurrentSet concurrent_affected_methods; auto process_dex = [&](DexClasses* dex) { @@ -342,7 +341,8 @@ ConcurrentSet split_splittable_closures( auto method = splittable_closure->method_closures->method; std::string id = method->get_class()->str() + "." + method->get_name()->str(); - size_t index = uniquifiers->fetch_add(id, 1); + size_t index{0}; + uniquifiers->update(id, [&index](auto, auto& v, bool) { index = v++; }); auto new_method = split_method(*splittable_closure, name_infix, index, dex_state.get()); if (!new_method) { @@ -365,8 +365,8 @@ ConcurrentSet split_splittable_closures( case HotSplitKind::Hot: { stats->hot_split_count++; if (concurrent_new_hot_methods) { - auto* ptr = concurrent_new_hot_methods->get(method); - auto* hot_root_method = ptr ? *ptr : method; + auto hot_root_method = + concurrent_new_hot_methods->get(method, method); concurrent_new_hot_methods->emplace(new_method, hot_root_method); } break; @@ -388,7 +388,7 @@ ConcurrentSet split_splittable_closures( affected_methods.end()); }; workqueue_run(process_dex, dexen); - return concurrent_affected_methods; + return concurrent_affected_methods.move_to_container(); } } // namespace @@ -404,23 +404,11 @@ void split_methods_in_stores( size_t reserved_trefs, Stats* stats, const std::string& name_infix, - InsertOnlyConcurrentMap* concurrent_new_hot_methods, - InsertOnlyConcurrentMap* - concurrent_splittable_no_optimizations_methods) { + ConcurrentMap* concurrent_new_hot_methods) { auto scope = build_class_scope(stores); init_classes::InitClassesWithSideEffects init_classes_with_side_effects( scope, create_init_class_insns); - ConcurrentSet methods; - std::vector dexen; - std::unordered_map> dex_states; - for (auto& store : stores) { - for (auto& dex : store.get_dexen()) { - dexen.push_back(&dex); - dex_states[&dex]; - } - } - auto is_excluded = [&](DexMethod* method) { auto name = method->get_deobfuscated_name_or_empty(); for (const auto& prefix : config.excluded_prefices) { @@ -430,21 +418,30 @@ void split_methods_in_stores( } return false; }; - walk::parallel::code(scope, [&](DexMethod* method, IRCode&) { - if (is_excluded(method)) { - stats->excluded_methods++; - return; + + std::unordered_set methods; + std::vector dexen; + std::unordered_map> dex_states; + for (auto& store : stores) { + for (auto& dex : store.get_dexen()) { + dexen.push_back(&dex); + dex_states[&dex]; + walk::code(dex, [&](DexMethod* method, IRCode&) { + if (is_excluded(method)) { + stats->excluded_methods++; + return; + } + methods.insert(method); + }); } - methods.insert(method); - }); + } size_t iteration{0}; - AtomicMap uniquifiers; + ConcurrentMap uniquifiers; while (!methods.empty() && iteration < config.max_iteration) { TRACE(MS, 2, "=== iteration[%zu]", iteration); Timer t("iteration " + std::to_string(iteration++)); - auto splittable_closures = select_splittable_closures( - methods, config, concurrent_splittable_no_optimizations_methods); + auto splittable_closures = select_splittable_closures(methods, config); ConcurrentSet concurrent_added_methods; methods = split_splittable_closures( dexen, min_sdk, init_classes_with_side_effects, reserved_trefs, diff --git a/opt/outliner/MethodSplitter.h b/opt/outliner/MethodSplitter.h index 2553e41a80..2f8cbfbb30 100644 --- a/opt/outliner/MethodSplitter.h +++ b/opt/outliner/MethodSplitter.h @@ -35,18 +35,15 @@ struct Stats { std::atomic excluded_methods{0}; }; -void split_methods_in_stores( - DexStoresVector& stores, - int32_t min_sdk, - const Config& config, - bool create_init_class_insns, - size_t reserved_mrefs, - size_t reserved_trefs, - Stats* stats, - const std::string& name_infix = "", - InsertOnlyConcurrentMap* - concurrent_new_hot_methods = nullptr, - InsertOnlyConcurrentMap* - concurrent_splittable_no_optimizations_methods = nullptr); +void split_methods_in_stores(DexStoresVector& stores, + int32_t min_sdk, + const Config& config, + bool create_init_class_insns, + size_t reserved_mrefs, + size_t reserved_trefs, + Stats* stats, + const std::string& name_infix = "", + ConcurrentMap* + concurrent_new_hot_methods = nullptr); } // namespace method_splitting_impl diff --git a/opt/outliner/MethodSplittingPass.cpp b/opt/outliner/MethodSplittingPass.cpp index bf4e6664c3..1cc96f201a 100644 --- a/opt/outliner/MethodSplittingPass.cpp +++ b/opt/outliner/MethodSplittingPass.cpp @@ -69,14 +69,11 @@ void MethodSplittingPass::run_pass(DexStoresVector& stores, auto name_infix = "$" + std::to_string(m_iteration) + "$"; Stats stats; - InsertOnlyConcurrentMap concurrent_new_hot_methods; - InsertOnlyConcurrentMap - concurrent_splittable_no_optimizations_methods; + ConcurrentMap concurrent_new_hot_methods; split_methods_in_stores(stores, mgr.get_redex_options().min_sdk, m_config, conf.create_init_class_insns(), reserved_mrefs, reserved_trefs, &stats, name_infix, - &concurrent_new_hot_methods, - &concurrent_splittable_no_optimizations_methods); + &concurrent_new_hot_methods); auto& method_profiles = conf.get_method_profiles(); size_t derived_method_profile_stats{0}; @@ -101,9 +98,6 @@ void MethodSplittingPass::run_pass(DexStoresVector& stores, mgr.set_metric("excluded_methods", (size_t)stats.excluded_methods); TRACE(MS, 1, "Split out %zu methods", stats.added_methods.size()); - for (auto [method, size] : concurrent_splittable_no_optimizations_methods) { - mgr.set_metric("no_optimizations_" + show_deobfuscated(method), size); - } m_iteration++; } diff --git a/opt/outliner/MethodSplittingPass.h b/opt/outliner/MethodSplittingPass.h index 0d3e51c0ce..1a143f285f 100644 --- a/opt/outliner/MethodSplittingPass.h +++ b/opt/outliner/MethodSplittingPass.h @@ -20,8 +20,9 @@ class MethodSplittingPass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, - {InitialRenameClass, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } diff --git a/opt/outliner/SplittableClosures.cpp b/opt/outliner/SplittableClosures.cpp index 0604899a29..67576d2344 100644 --- a/opt/outliner/SplittableClosures.cpp +++ b/opt/outliner/SplittableClosures.cpp @@ -457,11 +457,9 @@ std::vector to_splittable_closures( } // namespace namespace method_splitting_impl { -ConcurrentMap> -select_splittable_closures(const ConcurrentSet& methods, - const Config& config, - InsertOnlyConcurrentMap* - concurrent_splittable_no_optimizations_methods) { +std::unordered_map> +select_splittable_closures(const std::unordered_set& methods, + const Config& config) { Timer t("select_splittable_closures"); ConcurrentMap> concurrent_splittable_closures; @@ -488,13 +486,6 @@ select_splittable_closures(const ConcurrentSet& methods, if (scored_closures.empty()) { return; } - if (method->rstate.no_optimizations()) { - if (concurrent_splittable_no_optimizations_methods) { - concurrent_splittable_no_optimizations_methods->emplace( - method, mcs->original_size + adjustment); - } - return; - } auto splittable_closures = to_splittable_closures(config, mcs, std::move(scored_closures)); concurrent_splittable_closures.update( @@ -505,7 +496,7 @@ select_splittable_closures(const ConcurrentSet& methods, }); }; workqueue_run(concurrent_process_method, methods); - return concurrent_splittable_closures; + return concurrent_splittable_closures.move_to_container(); } } // namespace method_splitting_impl diff --git a/opt/outliner/SplittableClosures.h b/opt/outliner/SplittableClosures.h index e765cc2b0f..67c5173135 100644 --- a/opt/outliner/SplittableClosures.h +++ b/opt/outliner/SplittableClosures.h @@ -7,7 +7,6 @@ #pragma once -#include "ConcurrentContainers.h" #include "MethodClosures.h" namespace method_splitting_impl { @@ -46,10 +45,8 @@ struct SplittableClosure { }; // Selects splittable closures for a given set of methods. -ConcurrentMap> -select_splittable_closures(const ConcurrentSet& methods, - const Config& config, - InsertOnlyConcurrentMap* - concurrent_splittable_no_optimizations_methods); +std::unordered_map> +select_splittable_closures(const std::unordered_set& methods, + const Config& config); } // namespace method_splitting_impl diff --git a/opt/partial-application/PartialApplication.cpp b/opt/partial-application/PartialApplication.cpp index 299fa96ba6..63bf2b42cc 100644 --- a/opt/partial-application/PartialApplication.cpp +++ b/opt/partial-application/PartialApplication.cpp @@ -151,7 +151,7 @@ const api::AndroidSDK* get_min_sdk_api(ConfigFiles& conf, PassManager& mgr) { } } -using EnumUtilsCache = InsertOnlyConcurrentMap; +using EnumUtilsCache = ConcurrentMap; // Check if we have a boxed value for which there is a $EnumUtils field. DexField* try_get_enum_utils_f_field(EnumUtilsCache& cache, @@ -166,22 +166,19 @@ DexField* try_get_enum_utils_f_field(EnumUtilsCache& cache, object.attributes.front().value.get(); auto c = signed_value.get_constant(); always_assert(c); - return *cache - .get_or_create_and_assert_equal( - *c, - [&](int32_t key) -> DexField* { - auto cls = - type_class(DexType::make_type("Lredex/$EnumUtils;")); - if (!cls) { - return nullptr; - } - std::string field_name = "f" + std::to_string(key); - auto* field = cls->find_sfield(field_name.c_str(), - type::java_lang_Integer()); - always_assert(!field || is_static(field)); - return field; - }) - .first; + DexField* res; + cache.update(*c, [&res](int32_t key, DexField*& value, bool exists) { + if (!exists) { + auto cls = type_class(DexType::make_type("Lredex/$EnumUtils;")); + if (cls) { + std::string field_name = "f" + std::to_string(key); + value = cls->find_sfield(field_name.c_str(), type::java_lang_Integer()); + always_assert(!value || is_static(value)); + } + } + res = value; + }); + return res; } // Identify how many argument slots an invocation needs after expansion of wide @@ -249,10 +246,8 @@ ArgExclusivityVector get_arg_exclusivity(const UseDefChains& use_def_chains, return aev; } -using InsnsArgExclusivity = - InsertOnlyConcurrentMap; using CalleeCallerClasses = - ConcurrentMap>; + std::unordered_map>; // Gather all (caller, callee) pairs. Also compute arg exclusivity, which invoke // instructions we should exclude, and how many classes calls are distributed // over. @@ -262,17 +257,25 @@ void gather_caller_callees( const std::unordered_set& sufficiently_warm_methods, const std::unordered_set& sufficiently_hot_methods, const GetCalleeFunction& get_callee_fn, - ConcurrentMethodToMethodOccurrences* callee_caller, - ConcurrentMethodToMethodOccurrences* caller_callee, - InsnsArgExclusivity* arg_exclusivity, - ConcurrentSet* excluded_invoke_insns, + MethodToMethodOccurrences* callee_caller, + MethodToMethodOccurrences* caller_callee, + std::unordered_map* + arg_exclusivity, + std::unordered_set* excluded_invoke_insns, CalleeCallerClasses* callee_caller_classes) { Timer timer("gather_caller_callees"); + using ConcurrentMethodToMethodOccurrences = + ConcurrentMap>; + ConcurrentMethodToMethodOccurrences concurrent_callee_caller; + ConcurrentMethodToMethodOccurrences concurrent_caller_callee; + ConcurrentSet concurrent_excluded_invoke_insns; + ConcurrentMap + concurrent_arg_exclusivity; + ConcurrentMap> + concurrent_callee_caller_classes; + walk::parallel::code(scope, [&](DexMethod* caller, IRCode& code) { - if (caller->rstate.no_optimizations()) { - return; - } - always_assert(code.editable_cfg_built()); + code.build_cfg(true); CanOutlineBlockDecider block_decider( profile_guidance_config, sufficiently_warm_methods.count(caller), sufficiently_hot_methods.count(caller)); @@ -289,28 +292,28 @@ void gather_caller_callees( continue; } if (!can_outline) { - excluded_invoke_insns->insert(insn); + concurrent_excluded_invoke_insns.insert(insn); continue; } auto needs_range = analyze_args(callee).second; auto ae = get_arg_exclusivity(use_def_chains, def_use_chains, needs_range, insn); if (ae.empty()) { - excluded_invoke_insns->insert(insn); + concurrent_excluded_invoke_insns.insert(insn); continue; } - callee_caller->update( + concurrent_callee_caller.update( callee, [caller](const DexMethod*, std::unordered_map& v, bool) { ++v[caller]; }); - caller_callee->update( + concurrent_caller_callee.update( caller, [callee](const DexMethod*, std::unordered_map& v, bool) { ++v[callee]; }); - arg_exclusivity->emplace(insn, std::move(ae)); - callee_caller_classes->update( + concurrent_arg_exclusivity.emplace(insn, std::move(ae)); + concurrent_callee_caller_classes.update( callee, [caller](const DexMethod*, std::unordered_set& value, @@ -318,6 +321,12 @@ void gather_caller_callees( } } }); + + *callee_caller = concurrent_callee_caller.move_to_container(); + *caller_callee = concurrent_caller_callee.move_to_container(); + *excluded_invoke_insns = concurrent_excluded_invoke_insns.move_to_container(); + *arg_exclusivity = concurrent_arg_exclusivity.move_to_container(); + *callee_caller_classes = concurrent_callee_caller_classes.move_to_container(); } using InvokeCallSiteSummaries = @@ -384,7 +393,8 @@ class CalleeInvocationSelector { EnumUtilsCache& m_enum_utils_cache; CallSiteSummarizer& m_call_site_summarizer; const DexMethod* m_callee; - const InsnsArgExclusivity& m_arg_exclusivity; + const std::unordered_map& + m_arg_exclusivity; size_t m_callee_caller_classes; param_index_t m_src_regs; @@ -539,11 +549,13 @@ class CalleeInvocationSelector { MutablePriorityQueue m_pq; public: - CalleeInvocationSelector(EnumUtilsCache& enum_utils_cache, - CallSiteSummarizer& call_site_summarizer, - const DexMethod* callee, - const InsnsArgExclusivity& arg_exclusivity, - size_t callee_caller_classes) + CalleeInvocationSelector( + EnumUtilsCache& enum_utils_cache, + CallSiteSummarizer& call_site_summarizer, + const DexMethod* callee, + const std::unordered_map& + arg_exclusivity, + size_t callee_caller_classes) : m_enum_utils_cache(enum_utils_cache), m_call_site_summarizer(call_site_summarizer), m_callee(callee), @@ -583,7 +595,7 @@ class CalleeInvocationSelector { continue; } m_call_site_invoke_summaries.emplace_back(invoke_insn, css); - auto& aev = arg_exclusivity.at_unsafe(invoke_insn); + auto& aev = arg_exclusivity.at(invoke_insn); auto& aaem = m_aggregated_arg_exclusivity[css]; for (auto& p : aev) { auto& aae = aaem[p.first]; @@ -738,7 +750,7 @@ class CalleeInvocationSelector { // other invokes with the same css was beneficial on average. Check // (and filter out) if it's not actually beneficial for this particular // invoke. - auto& aev = m_arg_exclusivity.at_unsafe(invoke_insn); + auto& aev = m_arg_exclusivity.at(invoke_insn); const auto& bindings = reduced_css->arguments.bindings(); if (std::find_if(aev.begin(), aev.end(), [&bindings](auto& q) { return !bindings.at(q.first).is_top(); @@ -789,16 +801,16 @@ uint64_t get_stable_hash(const std::string& s) { return stable_hash; } -using PaMethodRefs = - InsertOnlyConcurrentMap>; +using PaMethodRefs = ConcurrentMap>; // Run the analysis over all callees. void select_invokes_and_callers( EnumUtilsCache& enum_utils_cache, CallSiteSummarizer& call_site_summarizer, - const ConcurrentMethodToMethodOccurrences& callee_caller, - const InsnsArgExclusivity& arg_exclusivity, + const MethodToMethodOccurrences& callee_caller, + const std::unordered_map& + arg_exclusivity, const CalleeCallerClasses& callee_caller_classes, size_t iteration, std::atomic* total_estimated_savings, @@ -820,9 +832,9 @@ void select_invokes_and_callers( workqueue_run( [&](const DexMethod* callee) { - CalleeInvocationSelector cis( - enum_utils_cache, call_site_summarizer, callee, arg_exclusivity, - callee_caller_classes.at_unsafe(callee).size()); + CalleeInvocationSelector cis(enum_utils_cache, call_site_summarizer, + callee, arg_exclusivity, + callee_caller_classes.at(callee).size()); cis.fill_pq(); cis.reduce_pq(); cis.select_invokes(total_estimated_savings, @@ -889,7 +901,7 @@ void select_invokes_and_callers( std::lock_guard lock_guard(mutex); selected_invokes->insert(callee_selected_invokes.begin(), callee_selected_invokes.end()); - for (auto& p : callee_caller.at_unsafe(callee)) { + for (auto& p : callee_caller.at(callee)) { selected_callers->insert(p.first); } } @@ -947,43 +959,42 @@ void rewrite_callers( }; walk::parallel::code(scope, [&](DexMethod* caller, IRCode& code) { - if (!selected_callers.count(caller)) { - return; - } - always_assert(!caller->rstate.no_optimizations()); - bool any_changes{false}; - auto& cfg = code.cfg(); - cfg::CFGMutation mutation(cfg); - auto ii = InstructionIterable(cfg); - size_t removed_srcs{0}; - std::unordered_set pas; - for (auto it = ii.begin(); it != ii.end(); it++) { - auto new_invoke_insn = - make_partial_application_invoke_insn(caller, it->insn); - if (!new_invoke_insn) { - continue; - } - removed_srcs += it->insn->srcs_size() - new_invoke_insn->srcs_size(); - std::vector new_insns{new_invoke_insn}; - auto move_result_it = cfg.move_result_of(it); - if (!move_result_it.is_end()) { - new_insns.push_back(new IRInstruction(*move_result_it->insn)); + if (selected_callers.count(caller)) { + bool any_changes{false}; + auto& cfg = code.cfg(); + cfg::CFGMutation mutation(cfg); + auto ii = InstructionIterable(cfg); + size_t removed_srcs{0}; + std::unordered_set pas; + for (auto it = ii.begin(); it != ii.end(); it++) { + auto new_invoke_insn = + make_partial_application_invoke_insn(caller, it->insn); + if (!new_invoke_insn) { + continue; + } + removed_srcs += it->insn->srcs_size() - new_invoke_insn->srcs_size(); + std::vector new_insns{new_invoke_insn}; + auto move_result_it = cfg.move_result_of(it); + if (!move_result_it.is_end()) { + new_insns.push_back(new IRInstruction(*move_result_it->insn)); + } + mutation.replace(it, new_insns); + auto pa = new_invoke_insn->get_method(); + if (pas.insert(pa).second) { + pa_callers->update( + pa, [caller](auto*, auto& vec, bool) { vec.push_back(caller); }); + } + any_changes = true; } - mutation.replace(it, new_insns); - auto pa = new_invoke_insn->get_method(); - if (pas.insert(pa).second) { - pa_callers->update( - pa, [caller](auto*, auto& vec, bool) { vec.push_back(caller); }); + mutation.flush(); + if (any_changes) { + TRACE(PA, 6, "[PartialApplication] Rewrote %s:\n%s", SHOW(caller), + SHOW(cfg)); + shrinker.shrink_method(caller); + (*removed_args) += removed_srcs; } - any_changes = true; - } - mutation.flush(); - if (any_changes) { - TRACE(PA, 6, "[PartialApplication] Rewrote %s:\n%s", SHOW(caller), - SHOW(cfg)); - shrinker.shrink_method(caller); - (*removed_args) += removed_srcs; } + code.clear_cfg(); }); } @@ -1099,11 +1110,9 @@ void create_partial_application_methods(EnumUtilsCache& enum_utils_cache, pa_method->set_virtual(true); } pa_method->set_deobfuscated_name(show_deobfuscated(pa_method)); - pa_method->get_code()->build_cfg(); cls->add_method(pa_method); TRACE(PA, 5, "[PartialApplication] Created %s binding %s:\n%s", - SHOW(pa_method), css->get_key().c_str(), - SHOW(pa_method->get_code()->cfg())); + SHOW(pa_method), css->get_key().c_str(), SHOW(pa_method->get_code())); } } @@ -1192,7 +1201,7 @@ void PartialApplicationPass::run_pass(DexStoresVector& stores, Shrinker shrinker(stores, scope, init_classes_with_side_effects, shrinker_config, min_sdk); - ConcurrentSet excluded_invoke_insns; + std::unordered_set excluded_invoke_insns; auto get_callee_fn = [&excluded_classes, &excluded_invoke_insns]( DexMethod* caller, IRInstruction* insn) -> DexMethod* { @@ -1225,9 +1234,10 @@ void PartialApplicationPass::run_pass(DexStoresVector& stores, return callee; }; - ConcurrentMethodToMethodOccurrences callee_caller; - ConcurrentMethodToMethodOccurrences caller_callee; - InsnsArgExclusivity arg_exclusivity; + MethodToMethodOccurrences callee_caller; + MethodToMethodOccurrences caller_callee; + std::unordered_map + arg_exclusivity; CalleeCallerClasses callee_caller_classes; gather_caller_callees( m_profile_guidance_config, scope, sufficiently_warm_methods, diff --git a/opt/partial-application/PartialApplication.h b/opt/partial-application/PartialApplication.h index 62ffda826b..4e47847b3d 100644 --- a/opt/partial-application/PartialApplication.h +++ b/opt/partial-application/PartialApplication.h @@ -19,13 +19,15 @@ class PartialApplicationPass : public Pass { using namespace redex_properties::interactions; using namespace redex_properties::names; return { + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, - {SpuriousGetClassCallsInterned, RequiresAndPreserves}, }; } void bind_config() override; + bool is_cfg_legacy() override { return true; } + void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; private: diff --git a/opt/peephole/Peephole.cpp b/opt/peephole/Peephole.cpp index 961a095fc9..fbf1574a5f 100644 --- a/opt/peephole/Peephole.cpp +++ b/opt/peephole/Peephole.cpp @@ -1733,7 +1733,7 @@ class alignas(CACHE_LINE_SIZE) PeepholeOptimizer { void peephole(DexMethod* method) { auto code = method->get_code(); - always_assert(code->editable_cfg_built()); + code->build_cfg(/* editable */ true); auto& cfg = code->cfg(); // do optimizations one at a time @@ -1797,6 +1797,8 @@ class alignas(CACHE_LINE_SIZE) PeepholeOptimizer { // Apply the mutator. mutator.flush(); } + + code->clear_cfg(); } void print_stats() { diff --git a/opt/peephole/Peephole.h b/opt/peephole/Peephole.h index 09659cdb60..659555ad7e 100644 --- a/opt/peephole/Peephole.h +++ b/opt/peephole/Peephole.h @@ -18,9 +18,13 @@ class PeepholePass : public Pass { const override { using namespace redex_properties::interactions; using namespace redex_properties::names; - return {}; + return { + {HasSourceBlocks, Preserves}, + }; } + bool is_cfg_legacy() override { return true; } + void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; void bind_config() override { diff --git a/opt/peephole/RedundantCheckCastRemover.cpp b/opt/peephole/RedundantCheckCastRemover.cpp index a63a1894be..e09a352ebe 100644 --- a/opt/peephole/RedundantCheckCastRemover.cpp +++ b/opt/peephole/RedundantCheckCastRemover.cpp @@ -39,9 +39,7 @@ void RedundantCheckCastRemover::run() { const std::vector& insns) { if (RedundantCheckCastRemover::can_remove_check_cast(insns)) { IRInstruction* check_cast = insns[2]; - auto& cfg = method->get_code()->cfg(); - auto pos_it = cfg.find_insn(check_cast); - cfg.remove_insn(pos_it); + method->get_code()->remove_opcode(check_cast); num_check_casts_removed++; TRACE(PEEPHOLE, 8, "redundant check cast in %s", SHOW(method)); diff --git a/opt/print-kotlin-stats/PrintKotlinStats.cpp b/opt/print-kotlin-stats/PrintKotlinStats.cpp index 49fb0ea0f4..22e6afc4f9 100644 --- a/opt/print-kotlin-stats/PrintKotlinStats.cpp +++ b/opt/print-kotlin-stats/PrintKotlinStats.cpp @@ -179,10 +179,9 @@ PrintKotlinStats::Stats PrintKotlinStats::handle_method(DexMethod* method) { } } - always_assert(method->get_code()->editable_cfg_built()); - auto& cfg = method->get_code()->cfg(); + auto code = method->get_code(); - for (const auto& it : cfg::InstructionIterable(cfg)) { + for (const auto& it : InstructionIterable(code)) { auto insn = it.insn; switch (insn->opcode()) { case OPCODE_INVOKE_STATIC: { diff --git a/opt/print-kotlin-stats/PrintKotlinStats.h b/opt/print-kotlin-stats/PrintKotlinStats.h index 5ea94a121b..abd012068f 100644 --- a/opt/print-kotlin-stats/PrintKotlinStats.h +++ b/opt/print-kotlin-stats/PrintKotlinStats.h @@ -63,13 +63,16 @@ class PrintKotlinStats : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, {UltralightCodePatterns, Preserves}, }; } void setup(); void eval_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; + bool is_cfg_legacy() override { return true; } void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; Stats handle_method(DexMethod* method); Stats handle_class(DexClass* cls); diff --git a/opt/print-members/PrintMembers.h b/opt/print-members/PrintMembers.h index d357bf0ea6..b53c93bf1c 100644 --- a/opt/print-members/PrintMembers.h +++ b/opt/print-members/PrintMembers.h @@ -19,7 +19,9 @@ class PrintMembersPass : public Pass { const override { using namespace redex_properties::interactions; using namespace redex_properties::names; - return {}; + return { + {HasSourceBlocks, Preserves}, + }; } void bind_config() override { diff --git a/opt/reachable-natives/ReachableNatives.h b/opt/reachable-natives/ReachableNatives.h index 79a75705f1..e2eaaa29f0 100644 --- a/opt/reachable-natives/ReachableNatives.h +++ b/opt/reachable-natives/ReachableNatives.h @@ -45,9 +45,9 @@ class ReachableNativesPass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, {UltralightCodePatterns, Preserves}, - {InitialRenameClass, Preserves}, }; } diff --git a/opt/rearrange-enum-clinit/RearrangeEnumClinit.cpp b/opt/rearrange-enum-clinit/RearrangeEnumClinit.cpp index b5a0f51b0c..afbd07b674 100644 --- a/opt/rearrange-enum-clinit/RearrangeEnumClinit.cpp +++ b/opt/rearrange-enum-clinit/RearrangeEnumClinit.cpp @@ -176,7 +176,7 @@ struct Rearranger { } reg_t move_new_array_to_front() { - redex_assert(array_new_array != CONSTP(b)->begin()); + redex_assert(array_new_array != b->begin()); auto size_def_it = use_def.find({array_new_array->insn, 0}); redex_assert(size_def_it != std::as_const(use_def).end()); @@ -221,7 +221,7 @@ struct Rearranger { IRInstruction* find_singleton_def(IRInstruction* use_insn, src_index_t src_index) { auto it = use_def.find(live_range::Use{use_insn, src_index}); - redex_assert(it != use_def.cend()); + redex_assert(it != use_def.end()); redex_assert(it->second.size() == 1); return *it->second.begin(); } @@ -229,7 +229,7 @@ struct Rearranger { std::pair find_move_point_new_instance( IRInstruction* object_insn) { auto it = def_use.find(object_insn); - redex_assert(it != def_use.cend()); + redex_assert(it != def_use.end()); std::optional which_reg{}; std::optional which_it{}; for (auto& obj_use : it->second) { @@ -292,7 +292,7 @@ struct Rearranger { // Find all the users of the array. This should be aput-object things. { auto new_array_uses_it = def_use.find(array_new_array->insn); - redex_assert(new_array_uses_it != def_use.cend()); + redex_assert(new_array_uses_it != def_use.end()); std::optional extra_reg{}; for (auto& use : new_array_uses_it->second) { // Skip the sput. @@ -378,7 +378,6 @@ void RearrangeEnumClinitPass::run_pass(DexStoresVector& stores, std::atomic cnt_below_threshold{0}; std::atomic cnt_failed{0}; std::atomic cnt_changed{0}; - std::atomic cnt_no_optimizations{0}; walk::parallel::classes(build_class_scope(stores), [&](DexClass* c) { if (c->is_external() || !is_enum(c)) { @@ -400,11 +399,6 @@ void RearrangeEnumClinitPass::run_pass(DexStoresVector& stores, return; } - if (m->rstate.no_optimizations()) { - cnt_no_optimizations.fetch_add(1, std::memory_order_relaxed); - return; - } - auto res = run(m, m->get_code()); switch (res) { @@ -429,7 +423,6 @@ void RearrangeEnumClinitPass::run_pass(DexStoresVector& stores, mgr.set_metric("failed", cnt_failed.load()); mgr.set_metric("no_clinit", cnt_no_clinit.load()); mgr.set_metric("below_threshold", cnt_below_threshold.load()); - mgr.set_metric("no_optimizations", cnt_no_optimizations.load()); mgr.set_metric("not_one_block", cnt_not_one_block.load()); mgr.set_metric("all_enum", cnt_all.load()); } diff --git a/opt/rebindrefs/ReBindRefs.cpp b/opt/rebindrefs/ReBindRefs.cpp index 98c98383a9..5df213d26b 100644 --- a/opt/rebindrefs/ReBindRefs.cpp +++ b/opt/rebindrefs/ReBindRefs.cpp @@ -105,9 +105,7 @@ struct Rebinder { } bool is_support_lib = api::is_support_lib_type(method->get_class()); RebinderRefs rebinder_refs; - always_assert(method->get_code()->editable_cfg_built()); - auto& cfg = method->get_code()->cfg(); - for (auto& mie : cfg::InstructionIterable(cfg)) { + for (auto& mie : InstructionIterable(method->get_code())) { auto insn = mie.insn; switch (insn->opcode()) { case OPCODE_INVOKE_VIRTUAL: { diff --git a/opt/rebindrefs/ReBindRefs.h b/opt/rebindrefs/ReBindRefs.h index 0b0c19d685..5676aff384 100644 --- a/opt/rebindrefs/ReBindRefs.h +++ b/opt/rebindrefs/ReBindRefs.h @@ -36,7 +36,9 @@ class ReBindRefsPass : public ExternalRefsManglingPass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } @@ -48,5 +50,6 @@ class ReBindRefsPass : public ExternalRefsManglingPass { ExternalRefsManglingPass::eval_pass(stores, conf, mgr); } + bool is_cfg_legacy() override { return true; } void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; }; diff --git a/opt/reduce-array-literals/ReduceArrayLiterals.cpp b/opt/reduce-array-literals/ReduceArrayLiterals.cpp index 003a7b7f5b..6e55dcf393 100644 --- a/opt/reduce-array-literals/ReduceArrayLiterals.cpp +++ b/opt/reduce-array-literals/ReduceArrayLiterals.cpp @@ -641,6 +641,15 @@ size_t ReduceArrayLiterals::patch_new_array_chunk( return chunk_size; } +class ReduceArrayLiteralsInterDexPlugin : public interdex::InterDexPassPlugin { + public: + ReserveRefsInfo reserve_refs() override { + return ReserveRefsInfo(/* frefs */ 0, + /* trefs */ 0, + /* mrefs */ 1); + } +}; + void ReduceArrayLiteralsPass::bind_config() { bind("debug", false, m_debug); // The default value 27 is somewhat arbitrary and could be tweaked. @@ -649,31 +658,22 @@ void ReduceArrayLiteralsPass::bind_config() { // runtime, while also being reasonably large so that this optimization still // results in a significant win in terms of instructions count. bind("max_filled_elements", 27, m_max_filled_elements); - after_configuration([this] { always_assert(m_max_filled_elements < 0xff); }); -} - -void ReduceArrayLiteralsPass::eval_pass(DexStoresVector&, - ConfigFiles&, - PassManager& mgr) { - if (m_eval++ == 0) { - m_reserved_refs_handle = mgr.reserve_refs(name(), - ReserveRefsInfo(/* frefs */ 0, - /* trefs */ 0, - /* mrefs */ 1)); - } + after_configuration([this] { + always_assert(m_max_filled_elements < 0xff); + interdex::InterDexRegistry* registry = + static_cast( + PluginRegistry::get().pass_registry(interdex::INTERDEX_PASS_NAME)); + std::function fn = + []() -> interdex::InterDexPassPlugin* { + return new ReduceArrayLiteralsInterDexPlugin(); + }; + registry->register_plugin("REDUCE_ARRAY_LITERALS_PLUGIN", std::move(fn)); + }); } void ReduceArrayLiteralsPass::run_pass(DexStoresVector& stores, ConfigFiles& /* conf */, PassManager& mgr) { - ++m_run; - // For the last invocation, release reserved refs. - if (m_eval == m_run) { - always_assert(m_reserved_refs_handle); - mgr.release_reserved_refs(*m_reserved_refs_handle); - m_reserved_refs_handle = std::nullopt; - } - int32_t min_sdk = mgr.get_redex_options().min_sdk; Architecture arch = mgr.get_redex_options().arch; TRACE(RAL, 1, "[RAL] min_sdk=%d, arch=%s", min_sdk, @@ -688,10 +688,12 @@ void ReduceArrayLiteralsPass::run_pass(DexStoresVector& stores, if (code == nullptr || m->rstate.no_optimizations()) { return ReduceArrayLiterals::Stats(); } - always_assert(code->editable_cfg_built()); + + code->build_cfg(/* editable */ true); ReduceArrayLiterals ral(code->cfg(), m_max_filled_elements, min_sdk, arch); ral.patch(); + code->clear_cfg(); return ral.get_stats(); }, m_debug ? 1 : redex_parallel::default_num_threads()); diff --git a/opt/reduce-array-literals/ReduceArrayLiterals.h b/opt/reduce-array-literals/ReduceArrayLiterals.h index aa4b198471..b05e3d18ff 100644 --- a/opt/reduce-array-literals/ReduceArrayLiterals.h +++ b/opt/reduce-array-literals/ReduceArrayLiterals.h @@ -11,7 +11,6 @@ #include "IRInstruction.h" // For reg_t. #include "Pass.h" -#include "PassManager.h" #include "RedexOptions.h" // For Architecture. class DexType; @@ -79,21 +78,17 @@ class ReduceArrayLiteralsPass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, - {InitialRenameClass, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } void bind_config() override; - - void eval_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; - + bool is_cfg_legacy() override { return true; } void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; private: size_t m_max_filled_elements; bool m_debug; - std::optional m_reserved_refs_handle; - size_t m_run{0}; // Which iteration of `run_pass`. - size_t m_eval{0}; // How many `eval_pass` iterations. }; diff --git a/opt/reduce-boolean-branches/ReduceBooleanBranchesPass.cpp b/opt/reduce-boolean-branches/ReduceBooleanBranchesPass.cpp index b620d4e408..aea3fa5179 100644 --- a/opt/reduce-boolean-branches/ReduceBooleanBranchesPass.cpp +++ b/opt/reduce-boolean-branches/ReduceBooleanBranchesPass.cpp @@ -39,10 +39,11 @@ void ReduceBooleanBranchesPass::run_pass(DexStoresVector& stores, scope, [&config = m_config, ©_prop_config, &pure_methods](DexMethod* method) { const auto code = method->get_code(); - if (!code || method->rstate.no_optimizations()) { + if (!code) { return reduce_boolean_branches_impl::Stats{}; } - always_assert(code->editable_cfg_built()); + + code->build_cfg(/* editable */ true); reduce_boolean_branches_impl::ReduceBooleanBranches rbb( config, is_static(method), method->get_proto()->get_args(), code); while (rbb.run()) { @@ -54,6 +55,7 @@ void ReduceBooleanBranchesPass::run_pass(DexStoresVector& stores, .dce(code); } + code->clear_cfg(); return rbb.get_stats(); }); diff --git a/opt/reduce-boolean-branches/ReduceBooleanBranchesPass.h b/opt/reduce-boolean-branches/ReduceBooleanBranchesPass.h index bc6733b54a..f4d13b3183 100644 --- a/opt/reduce-boolean-branches/ReduceBooleanBranchesPass.h +++ b/opt/reduce-boolean-branches/ReduceBooleanBranchesPass.h @@ -20,11 +20,12 @@ class ReduceBooleanBranchesPass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, - {InitialRenameClass, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } - + bool is_cfg_legacy() override { return true; } void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; private: diff --git a/opt/reduce-gotos/ReduceGotos.cpp b/opt/reduce-gotos/ReduceGotos.cpp index d85fba038b..963f47305f 100644 --- a/opt/reduce-gotos/ReduceGotos.cpp +++ b/opt/reduce-gotos/ReduceGotos.cpp @@ -480,11 +480,13 @@ void ReduceGotosPass::process_code_ifs(cfg::ControlFlowGraph& cfg, ReduceGotosPass::Stats ReduceGotosPass::process_code(IRCode* code) { Stats stats; - always_assert(code->editable_cfg_built()); + + code->build_cfg(/* editable = true*/); code->cfg().calculate_exit_block(); auto& cfg = code->cfg(); process_code_switches(cfg, stats); process_code_ifs(cfg, stats); + code->clear_cfg(); return stats; } @@ -496,7 +498,7 @@ void ReduceGotosPass::run_pass(DexStoresVector& stores, Stats stats = walk::parallel::methods(scope, [](DexMethod* method) { const auto code = method->get_code(); - if (!code || method->rstate.no_optimizations()) { + if (!code) { return Stats{}; } diff --git a/opt/reduce-gotos/ReduceGotos.h b/opt/reduce-gotos/ReduceGotos.h index dfbea19920..c573a969f4 100644 --- a/opt/reduce-gotos/ReduceGotos.h +++ b/opt/reduce-gotos/ReduceGotos.h @@ -40,12 +40,14 @@ class ReduceGotosPass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoInitClassInstructions, Preserves}, {NoResolvablePureRefs, Preserves}, {NoUnreachableInstructions, Preserves}, {RenameClass, Preserves}, }; } + bool is_cfg_legacy() override { return true; } void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; static Stats process_code(IRCode*); diff --git a/opt/regalloc-fast/FastRegAlloc.h b/opt/regalloc-fast/FastRegAlloc.h index 7aed5e63f9..577c78a2d6 100644 --- a/opt/regalloc-fast/FastRegAlloc.h +++ b/opt/regalloc-fast/FastRegAlloc.h @@ -24,12 +24,15 @@ class FastRegAllocPass : public Pass { const override { using namespace redex_properties::interactions; using namespace redex_properties::names; - return {}; + return { + {HasSourceBlocks, Preserves}, + }; } void eval_pass(DexStoresVector& stores, ConfigFiles& conf, PassManager& mgr) override; + bool is_cfg_legacy() override { return true; } void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; private: diff --git a/opt/regalloc/RegAlloc.h b/opt/regalloc/RegAlloc.h index 3629df57b2..97d6bfe5dc 100644 --- a/opt/regalloc/RegAlloc.h +++ b/opt/regalloc/RegAlloc.h @@ -24,11 +24,11 @@ class RegAllocPass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoInitClassInstructions, Preserves}, {NoResolvablePureRefs, Preserves}, {NoUnreachableInstructions, Preserves}, {RenameClass, Preserves}, - {MethodRegister, Establishes}, }; } diff --git a/opt/remove-apilevel-checks/RemoveApiLevelChecks.cpp b/opt/remove-apilevel-checks/RemoveApiLevelChecks.cpp index 2e827fcc32..0f1fd409d4 100644 --- a/opt/remove-apilevel-checks/RemoveApiLevelChecks.cpp +++ b/opt/remove-apilevel-checks/RemoveApiLevelChecks.cpp @@ -250,10 +250,10 @@ RemoveApiLevelChecksPass::ApiLevelStats RemoveApiLevelChecksPass::run( if (!code) { return ApiLevelStats{}; } - always_assert(code->editable_cfg_built()); - auto& cfg = code->cfg(); - auto sdk_int_sgets = find_sdk_int_sgets(cfg, sdk_int_field); + cfg::ScopedCFG cfg(code); + + auto sdk_int_sgets = find_sdk_int_sgets(*cfg, sdk_int_field); if (sdk_int_sgets.empty()) { return ApiLevelStats(); } @@ -261,7 +261,7 @@ RemoveApiLevelChecksPass::ApiLevelStats RemoveApiLevelChecksPass::run( ApiLevelStats ret; ret.num_field_gets = sdk_int_sgets.size(); ret.num_methods = 1; - ret.num_removed = analyze_and_rewrite(cfg, sdk_int_sgets, min_sdk); + ret.num_removed = analyze_and_rewrite(*cfg, sdk_int_sgets, min_sdk); return ret; } diff --git a/opt/remove-apilevel-checks/RemoveApiLevelChecks.h b/opt/remove-apilevel-checks/RemoveApiLevelChecks.h index 5c51aae28b..2d7f3fbd2f 100644 --- a/opt/remove-apilevel-checks/RemoveApiLevelChecks.h +++ b/opt/remove-apilevel-checks/RemoveApiLevelChecks.h @@ -22,9 +22,12 @@ class RemoveApiLevelChecksPass : public Pass { const override { using namespace redex_properties::interactions; using namespace redex_properties::names; - return {}; + return { + {HasSourceBlocks, Preserves}, + }; } + bool is_cfg_legacy() override { return true; } void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; private: diff --git a/opt/remove-builders/RemoveBuilders.h b/opt/remove-builders/RemoveBuilders.h index 4a5822b092..24e374cbeb 100644 --- a/opt/remove-builders/RemoveBuilders.h +++ b/opt/remove-builders/RemoveBuilders.h @@ -17,7 +17,9 @@ class RemoveBuildersPass : public Pass { const override { using namespace redex_properties::interactions; using namespace redex_properties::names; - return {}; + return { + {HasSourceBlocks, Preserves}, + }; } void bind_config() override { diff --git a/opt/remove-builders/RemoveBuildersHelper.cpp b/opt/remove-builders/RemoveBuildersHelper.cpp index 2d2c15c1d9..edc5dab803 100644 --- a/opt/remove-builders/RemoveBuildersHelper.cpp +++ b/opt/remove-builders/RemoveBuildersHelper.cpp @@ -1159,18 +1159,7 @@ bool BuilderTransform::inline_methods( } } - always_assert(!to_inline.count(method)); - always_assert(!method->get_code()->editable_cfg_built()); - method->get_code()->build_cfg(); - for (auto* m : to_inline) { - always_assert(!m->get_code()->editable_cfg_built()); - m->get_code()->build_cfg(); - } m_inliner->inline_callees(method, to_inline); - method->get_code()->clear_cfg(); - for (auto* m : to_inline) { - m->get_code()->clear_cfg(); - } // Check all possible methods were inlined. previous_to_inline = to_inline; diff --git a/opt/remove-interfaces/RemoveInterfacePass.cpp b/opt/remove-interfaces/RemoveInterfacePass.cpp index bfeace09bb..99791a11c1 100644 --- a/opt/remove-interfaces/RemoveInterfacePass.cpp +++ b/opt/remove-interfaces/RemoveInterfacePass.cpp @@ -28,11 +28,11 @@ namespace { // If an interface has more implementors, we do not remove it. constexpr size_t MAX_IMPLS_SIZE = 7; -std::vector get_args_for(DexProto* proto, MethodCreator& mc) { +std::vector get_args_for(DexProto* proto, MethodCreator* mc) { std::vector args; size_t args_size = proto->get_args()->size(); for (size_t arg_loc = 0; arg_loc < args_size; ++arg_loc) { - args.push_back(mc.get_local(arg_loc)); + args.push_back(mc->get_local(arg_loc)); } return args; @@ -44,8 +44,8 @@ std::unique_ptr get_anno_set(DexType* anno_type) { anno_type, DexAnnotationVisibility::DAV_BUILD)); return anno_set; } -DexMethod* materialized_dispatch(DexType* owner, MethodCreator&& mc) { - auto dispatch = mc.create(); +DexMethod* materialized_dispatch(DexType* owner, MethodCreator* mc) { + auto dispatch = mc->create(); TRACE(RM_INTF, 9, "Generated dispatch %s\n%s", @@ -94,16 +94,16 @@ DexMethod* generate_dispatch(const DexType* base_type, TRACE(RM_INTF, 9, "generating dispatch %s.%s for targets of size %zu", SHOW(dispatch_owner), dispatch_name->c_str(), targets.size()); auto anno_set = get_anno_set(dispatch_anno); - auto mc = MethodCreator(dispatch_owner, dispatch_name, new_proto, - ACC_STATIC | ACC_PUBLIC, std::move(anno_set), - keep_debug_info); + auto mc = new MethodCreator(dispatch_owner, dispatch_name, new_proto, + ACC_STATIC | ACC_PUBLIC, std::move(anno_set), + keep_debug_info); // Variable setup - auto self_loc = mc.get_local(0); - auto type_test_loc = mc.make_local(type::_boolean()); - auto ret_loc = new_proto->is_void() ? mc.get_local(0) // not used - : mc.make_local(new_proto->get_rtype()); + auto self_loc = mc->get_local(0); + auto type_test_loc = mc->make_local(type::_boolean()); + auto ret_loc = new_proto->is_void() ? mc->get_local(0) // not used + : mc->make_local(new_proto->get_rtype()); std::vector args = get_args_for(new_proto, mc); - auto mb = mc.get_main_block(); + auto mb = mc->get_main_block(); /** * In case all interface scopes can only be resolved to a single concrete @@ -121,7 +121,7 @@ DexMethod* generate_dispatch(const DexType* base_type, mb->move_result(ret_loc, new_proto->get_rtype()); } mb->ret(new_proto->get_rtype(), ret_loc); - return materialized_dispatch(dispatch_owner, std::move(mc)); + return materialized_dispatch(dispatch_owner, mc); } // Construct dispatchs for (size_t idx = 0; idx < targets.size(); idx++) { @@ -146,7 +146,7 @@ DexMethod* generate_dispatch(const DexType* base_type, curr_block->ret(new_proto->get_rtype(), ret_loc); } // Finalizing - return materialized_dispatch(dispatch_owner, std::move(mc)); + return materialized_dispatch(dispatch_owner, mc); } void update_interface_calls( @@ -281,7 +281,7 @@ size_t exclude_unremovables(const Scope& scope, const std::vector& excluded_interfaces, TypeSet& candidates) { size_t count = 0; - always_assert(!stores.empty()); + always_assert(stores.size()); XStoreRefs xstores(stores); // Excluded by config @@ -324,9 +324,7 @@ size_t exclude_unremovables(const Scope& scope, if (!code) { return current_excluded; } - always_assert(code->editable_cfg_built()); - auto& cfg = code->cfg(); - for (const auto& mie : cfg::InstructionIterable(cfg)) { + for (const auto& mie : InstructionIterable(code)) { auto insn = mie.insn; if (!insn->has_type() || insn->get_type() == nullptr) { continue; @@ -512,7 +510,6 @@ TypeSet RemoveInterfacePass::remove_leaf_interfaces( auto dispatch = generate_dispatch(replacement_type, dispatch_targets, meth, m_keep_debug_info, m_interface_dispatch_anno); - dispatch->get_code()->build_cfg(); local_dispatch_stats[dispatch_targets.size()]++; local_intf_meth_to_dispatch[meth] = dispatch; } diff --git a/opt/remove-interfaces/RemoveInterfacePass.h b/opt/remove-interfaces/RemoveInterfacePass.h index 26a7ae4739..a4339fb04c 100644 --- a/opt/remove-interfaces/RemoveInterfacePass.h +++ b/opt/remove-interfaces/RemoveInterfacePass.h @@ -41,11 +41,14 @@ class RemoveInterfacePass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } void bind_config() override; + bool is_cfg_legacy() override { return true; } void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; private: diff --git a/opt/remove-nullcheck-string-arg/RemoveNullcheckStringArg.cpp b/opt/remove-nullcheck-string-arg/RemoveNullcheckStringArg.cpp index 32fb5bfaec..b91f61c0d7 100644 --- a/opt/remove-nullcheck-string-arg/RemoveNullcheckStringArg.cpp +++ b/opt/remove-nullcheck-string-arg/RemoveNullcheckStringArg.cpp @@ -39,9 +39,10 @@ void RemoveNullcheckStringArg::run_pass(DexStoresVector& stores, new_methods.count(method)) { return Stats(); } - always_assert(code->editable_cfg_built()); + code->build_cfg(); auto local_stats = change_in_cfg(code->cfg(), transfer_map_param, transfer_map_expr, method->is_virtual()); + code->clear_cfg(); return local_stats; }); @@ -163,7 +164,6 @@ DexMethod* RemoveNullcheckStringArg::get_wrapper_method_with_msg( main_block->ret_void(); auto new_method = method_creator.create(); - new_method->get_code()->build_cfg(); TRACE(NULLCHECK, 5, "Created Method : %s", SHOW(new_method->get_code())); host_cls->add_method(new_method); return new_method; @@ -254,7 +254,6 @@ DexMethod* RemoveNullcheckStringArg::get_wrapper_method_with_int_index( main_block->ret_void(); auto new_method = method_creator.create(); - new_method->get_code()->build_cfg(); TRACE(NULLCHECK, 5, "Created Method : %s", SHOW(new_method->get_code())); host_cls->add_method(new_method); return new_method; @@ -280,9 +279,7 @@ RemoveNullcheckStringArg::Stats RemoveNullcheckStringArg::change_in_cfg( param_index.insert(std::make_pair(load_insn->dest(), arg_index++)); } - live_range::MoveAwareChains chains( - cfg, /* ignore_unreachable */ false, - [&](auto* insn) { return opcode::is_a_load_param(insn->opcode()); }); + live_range::MoveAwareChains chains(cfg); live_range::DefUseChains du_chains = chains.get_def_use_chains(); for (cfg::Block* block : cfg.blocks()) { @@ -356,7 +353,7 @@ RemoveNullcheckStringArg::Stats RemoveNullcheckStringArg::change_in_cfg( } else { // Handle null check for expr. Get the proper wrapper function to throw // more accurate exception message. - always_assert(!defs.empty()); + always_assert(defs.size() > 0); auto def = *defs.elements().begin(); NullErrSrc err_msg; if (defs.size() > 1) { diff --git a/opt/remove-nullcheck-string-arg/RemoveNullcheckStringArg.h b/opt/remove-nullcheck-string-arg/RemoveNullcheckStringArg.h index de609b458e..0a2835e04a 100644 --- a/opt/remove-nullcheck-string-arg/RemoveNullcheckStringArg.h +++ b/opt/remove-nullcheck-string-arg/RemoveNullcheckStringArg.h @@ -82,6 +82,8 @@ class RemoveNullcheckStringArg : public Pass { using namespace redex_properties::interactions; using namespace redex_properties::names; return { + {HasSourceBlocks, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, {UltralightCodePatterns, Preserves}, }; } @@ -89,6 +91,7 @@ class RemoveNullcheckStringArg : public Pass { bool setup(TransferMapForParam& transfer_map_param, TransferMapForExpr& transfer_map_expr, NewMethodSet& new_methods); + bool is_cfg_legacy() override { return true; } void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; RemoveNullcheckStringArg::Stats change_in_cfg( cfg::ControlFlowGraph& cfg, diff --git a/opt/remove-recursive-locks/RemoveRecursiveLocks.cpp b/opt/remove-recursive-locks/RemoveRecursiveLocks.cpp index aa6fa4cf84..33b0d8468d 100644 --- a/opt/remove-recursive-locks/RemoveRecursiveLocks.cpp +++ b/opt/remove-recursive-locks/RemoveRecursiveLocks.cpp @@ -24,6 +24,7 @@ #include "MethodProfiles.h" #include "PassManager.h" #include "ReachingDefinitions.h" +#include "ScopedCFG.h" #include "Show.h" #include "Trace.h" #include "Walkers.h" @@ -294,14 +295,14 @@ ComputeRDefsResult compute_rdefs(ControlFlowGraph& cfg) { std::unordered_map block_map; auto get_rdef = [&](IRInstruction* insn, reg_t reg) -> IRInstruction* { auto it = block_map.find(insn); - redex_assert(it != block_map.cend()); + redex_assert(it != block_map.end()); auto defs = get_defs(it->second, insn); return get_singleton(defs, reg); }; auto print_rdefs = [&](IRInstruction* insn, reg_t reg) -> std::string { auto it = block_map.find(insn); - redex_assert(it != block_map.cend()); + redex_assert(it != block_map.end()); auto defs = get_defs(it->second, insn); const auto& defs0 = defs.get(reg); if (defs0.is_top()) { @@ -590,7 +591,7 @@ size_t remove(ControlFlowGraph& cfg, AnalysisResult& analysis) { for (const auto& insn_it : ir_list::InstructionIterable{b}) { if (opcode::is_a_monitor(insn_it.insn->opcode())) { auto it = analysis.rdefs.find(insn_it.insn); - redex_assert(it != analysis.rdefs.cend()); + redex_assert(it != analysis.rdefs.end()); auto def = it->second; auto& bindings = state.bindings(); @@ -691,8 +692,8 @@ struct Stats { } }; -bool has_monitor_ops(cfg::ControlFlowGraph& cfg) { - for (const auto& mie : cfg::InstructionIterable(cfg)) { +bool has_monitor_ops(const IRCode* code) { + for (const auto& mie : ir_list::InstructionIterableImpl(code)) { if (opcode::is_a_monitor(mie.insn->opcode())) { return true; } @@ -702,14 +703,13 @@ bool has_monitor_ops(cfg::ControlFlowGraph& cfg) { Stats run_locks_removal(DexMethod* m, IRCode* code) { // 1) Check whether there are MONITOR_ENTER instructions. - always_assert(code->editable_cfg_built()); - auto& cfg = code->cfg(); - if (!has_monitor_ops(cfg)) { + if (!has_monitor_ops(code)) { return Stats{}; } + cfg::ScopedCFG cfg(code); Stats stats{}; - auto analysis = analyze(cfg); + auto analysis = analyze(*cfg); stats.methods_with_locks = analysis.method_with_locks ? 1 : 0; if (analysis.non_singleton_rdefs) { @@ -728,19 +728,19 @@ Stats run_locks_removal(DexMethod* m, IRCode* code) { stats.counts_per[analysis.max_same].insert(m); if (analysis.max_same > 1) { - size_t removed = remove(cfg, analysis); + size_t removed = remove(*cfg, analysis); redex_assert(removed > 0); - cfg.simplify(); // Remove dead blocks. + cfg->simplify(); // Remove dead blocks. // Run analysis again just to check. - auto analysis2 = analyze(cfg); - always_assert_log(!analysis2.non_singleton_rdefs, "%s", SHOW(cfg)); - always_assert_log(!analysis2.method_with_issues, "%s", SHOW(cfg)); - auto verify_res = verify(cfg, analysis, analysis2); + auto analysis2 = analyze(*cfg); + always_assert_log(!analysis2.non_singleton_rdefs, "%s", SHOW(*cfg)); + always_assert_log(!analysis2.method_with_issues, "%s", SHOW(*cfg)); + auto verify_res = verify(*cfg, analysis, analysis2); auto print_err = [&m, &verify_res, &analysis2, &cfg]() { std::ostringstream oss; oss << show(m) << ": " << *verify_res << std::endl; - print(oss, analysis2.rdefs, *analysis2.iter, cfg); + print(oss, analysis2.rdefs, *analysis2.iter, *cfg); return oss.str(); }; always_assert_log(!verify_res, "%s", print_err().c_str()); @@ -760,7 +760,7 @@ void run_impl(DexStoresVector& stores, Stats stats = walk::parallel::methods(scope, [](DexMethod* method) -> Stats { auto code = method->get_code(); - if (code != nullptr && !method->rstate.no_optimizations()) { + if (code != nullptr) { return run_locks_removal(method, code); } return Stats{}; diff --git a/opt/remove-recursive-locks/RemoveRecursiveLocks.h b/opt/remove-recursive-locks/RemoveRecursiveLocks.h index a8928c9972..ce6aa8f2cd 100644 --- a/opt/remove-recursive-locks/RemoveRecursiveLocks.h +++ b/opt/remove-recursive-locks/RemoveRecursiveLocks.h @@ -23,10 +23,12 @@ class RemoveRecursiveLocksPass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, - {InitialRenameClass, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } + bool is_cfg_legacy() override { return true; } void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; diff --git a/opt/remove-uninstantiables/RemoveUninstantiablesPass.cpp b/opt/remove-uninstantiables/RemoveUninstantiablesPass.cpp index 8db90a6e05..6ae738c7b3 100644 --- a/opt/remove-uninstantiables/RemoveUninstantiablesPass.cpp +++ b/opt/remove-uninstantiables/RemoveUninstantiablesPass.cpp @@ -11,6 +11,7 @@ #include "MethodFixup.h" #include "RemoveUninstantiablesImpl.h" +#include "ScopedCFG.h" #include "Walkers.h" namespace { @@ -57,7 +58,7 @@ class OverriddenVirtualScopesAnalysis { void compute_transitively_defined_virtual_scope( const std::unordered_map>& instantiable_children, - const InsertOnlyConcurrentMap& + const ConcurrentMap& defined_virtual_scopes, DexType* t) { auto it = m_transitively_defined_virtual_scopes.find(t); @@ -147,8 +148,7 @@ class OverriddenVirtualScopesAnalysis { scan_code(scope); - InsertOnlyConcurrentMap - defined_virtual_scopes; + ConcurrentMap defined_virtual_scopes; walk::parallel::classes(scope, [&](DexClass* cls) { VirtualScopeIdSet virtual_scopes; for (auto method : cls->get_vmethods()) { @@ -272,18 +272,18 @@ void RemoveUninstantiablesPass::run_pass(DexStoresVector& stores, if (method->rstate.no_optimizations() || code == nullptr) { return remove_uninstantiables_impl::Stats(); } - always_assert(code->editable_cfg_built()); + if (overridden_virtual_scopes_analysis.keep_code(method)) { - auto& cfg = code->cfg(); + cfg::ScopedCFG cfg(code); return remove_uninstantiables_impl::replace_uninstantiable_refs( - scoped_uninstantiable_types, cfg); + scoped_uninstantiable_types, *cfg); } uncallable_instance_methods.insert(method); return remove_uninstantiables_impl::Stats(); }); stats += remove_uninstantiables_impl::reduce_uncallable_instance_methods( - scope, uncallable_instance_methods, + scope, uncallable_instance_methods.move_to_container(), [&](const DexMethod* m) { return false; }); stats.report(mgr); diff --git a/opt/remove-uninstantiables/RemoveUninstantiablesPass.h b/opt/remove-uninstantiables/RemoveUninstantiablesPass.h index a3c9480ec3..4a03dbde1f 100644 --- a/opt/remove-uninstantiables/RemoveUninstantiablesPass.h +++ b/opt/remove-uninstantiables/RemoveUninstantiablesPass.h @@ -37,7 +37,9 @@ class RemoveUninstantiablesPass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } @@ -46,5 +48,7 @@ class RemoveUninstantiablesPass : public Pass { std::unordered_map>* instantiable_children = nullptr); + bool is_cfg_legacy() override { return true; } + void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; }; diff --git a/opt/remove-unreachable/RemoveUnreachable.cpp b/opt/remove-unreachable/RemoveUnreachable.cpp index 375ca457bc..6eadf19d45 100644 --- a/opt/remove-unreachable/RemoveUnreachable.cpp +++ b/opt/remove-unreachable/RemoveUnreachable.cpp @@ -14,8 +14,6 @@ #include "ConfigFiles.h" #include "DexUtil.h" #include "IOUtil.h" -#include "InitClassesWithSideEffects.h" -#include "LocalDce.h" #include "MethodOverrideGraph.h" #include "PassManager.h" #include "Show.h" @@ -178,18 +176,6 @@ void RemoveUnreachablePassBase::run_pass(DexStoresVector& stores, // Store names of removed classes and methods ConcurrentSet removed_symbols; - auto sweep_code = m_prune_uninstantiable_insns || m_throw_propagation; - auto scope = build_class_scope(stores); - always_assert(!pm.unreliable_virtual_scopes()); - auto method_override_graph = mog::build_graph(scope); - std::unique_ptr - init_classes_with_side_effects; - if (sweep_code && !pm.init_class_lowering_has_run()) { - init_classes_with_side_effects = - std::make_unique( - scope, conf.create_init_class_insns(), method_override_graph.get()); - } - root_metrics(stores, pm); bool emit_graph_this_run = @@ -207,8 +193,8 @@ void RemoveUnreachablePassBase::run_pass(DexStoresVector& stores, int num_ignore_check_strings = 0; reachability::ReachableAspects reachable_aspects; auto reachables = this->compute_reachable_objects( - scope, *method_override_graph, pm, &num_ignore_check_strings, - &reachable_aspects, emit_graph_this_run, m_relaxed_keep_class_members, + stores, pm, &num_ignore_check_strings, &reachable_aspects, + emit_graph_this_run, m_relaxed_keep_class_members, m_prune_unreferenced_interfaces, m_prune_uninstantiable_insns, m_prune_uncallable_instance_method_bodies, m_throw_propagation, m_remove_no_argument_constructors); @@ -233,34 +219,12 @@ void RemoveUnreachablePassBase::run_pass(DexStoresVector& stores, auto abstracted_classes = reachability::mark_classes_abstract( stores, *reachables, reachable_aspects); pm.incr_metric("abstracted_classes", abstracted_classes.size()); - if (sweep_code) { - remove_uninstantiables_impl::Stats remove_uninstantiables_stats; - std::atomic throws_inserted{0}; - InsertOnlyConcurrentSet affected_methods; - reachability::sweep_code(stores, m_prune_uncallable_instance_method_bodies, - m_prune_uncallable_virtual_methods, - reachable_aspects, &remove_uninstantiables_stats, - &throws_inserted, &affected_methods); - remove_uninstantiables_stats.report(pm); - pm.incr_metric("throws_inserted", (size_t)throws_inserted); - pm.incr_metric("methods_with_code_changes", affected_methods.size()); - std::unordered_set pure_methods; - LocalDce::Stats dce_stats; - std::mutex dce_stats_mutex; - workqueue_run( - [&](DexMethod* method) { - LocalDce dce(init_classes_with_side_effects.get(), pure_methods); - dce.dce(method->get_code()->cfg(), /* normalize_new_instances */ true, - method->get_class()); - auto local_stats = dce.get_stats(); - std::lock_guard lock(dce_stats_mutex); - dce_stats += local_stats; - }, - affected_methods); - pm.incr_metric("instructions_eliminated_localdce_dead", - dce_stats.dead_instruction_count); - pm.incr_metric("instructions_eliminated_localdce_unreachable", - dce_stats.unreachable_instruction_count); + if (m_prune_uninstantiable_insns || m_throw_propagation) { + auto [uninstantiables_stats, throws_inserted] = reachability::sweep_code( + stores, m_prune_uncallable_instance_method_bodies, + m_prune_uncallable_virtual_methods, reachable_aspects); + uninstantiables_stats.report(pm); + pm.incr_metric("throws_inserted", throws_inserted); } reachability::sweep(stores, *reachables, output_unreachable_symbols ? &removed_symbols : nullptr, @@ -299,7 +263,7 @@ void RemoveUnreachablePassBase::run_pass(DexStoresVector& stores, { std::ofstream os; open_or_die(conf.metafile("method-override-graph"), &os); - method_override_graph = mog::build_graph(build_class_scope(stores)); + auto method_override_graph = mog::build_graph(build_class_scope(stores)); method_override_graph->dump(os); } } @@ -332,8 +296,7 @@ void RemoveUnreachablePassBase::write_out_removed_symbols( std::unique_ptr RemoveUnreachablePass::compute_reachable_objects( - const Scope& scope, - const method_override_graph::Graph& method_override_graph, + const DexStoresVector& stores, PassManager& /* pm */, int* num_ignore_check_strings, reachability::ReachableAspects* reachable_aspects, @@ -345,11 +308,11 @@ RemoveUnreachablePass::compute_reachable_objects( bool cfg_gathering_check_returning, bool remove_no_argument_constructors) { return reachability::compute_reachable_objects( - scope, method_override_graph, m_ignore_sets, num_ignore_check_strings, - reachable_aspects, emit_graph_this_run, relaxed_keep_class_members, - relaxed_keep_interfaces, cfg_gathering_check_instantiable, - cfg_gathering_check_instance_callable, cfg_gathering_check_returning, - false, remove_no_argument_constructors); + stores, m_ignore_sets, num_ignore_check_strings, reachable_aspects, + emit_graph_this_run, relaxed_keep_class_members, relaxed_keep_interfaces, + cfg_gathering_check_instantiable, cfg_gathering_check_instance_callable, + cfg_gathering_check_returning, false, nullptr, + remove_no_argument_constructors); } static RemoveUnreachablePass s_pass; diff --git a/opt/remove-unreachable/RemoveUnreachable.h b/opt/remove-unreachable/RemoveUnreachable.h index 7f6bfd4de6..8b36816d4e 100644 --- a/opt/remove-unreachable/RemoveUnreachable.h +++ b/opt/remove-unreachable/RemoveUnreachable.h @@ -20,9 +20,10 @@ class RemoveUnreachablePassBase : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, {UltralightCodePatterns, Preserves}, - {InitialRenameClass, Preserves}, }; } @@ -64,19 +65,17 @@ class RemoveUnreachablePassBase : public Pass { void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; virtual std::unique_ptr - compute_reachable_objects( - const Scope& scope, - const method_override_graph::Graph& method_override_graph, - PassManager& pm, - int* num_ignore_check_strings, - reachability::ReachableAspects* reachable_aspects, - bool emit_graph_this_run, - bool relaxed_keep_class_members, - bool relaxed_keep_interfaces, - bool cfg_gathering_check_instantiable, - bool cfg_gathering_check_instance_callable, - bool cfg_gathering_check_returning, - bool remove_no_argument_constructors) = 0; + compute_reachable_objects(const DexStoresVector& stores, + PassManager& pm, + int* num_ignore_check_strings, + reachability::ReachableAspects* reachable_aspects, + bool emit_graph_this_run, + bool relaxed_keep_class_members, + bool relaxed_keep_interfaces, + bool cfg_gathering_check_instantiable, + bool cfg_gathering_check_instance_callable, + bool cfg_gathering_check_returning, + bool remove_no_argument_constructors) = 0; void write_out_removed_symbols( const std::string& filepath, @@ -103,8 +102,7 @@ class RemoveUnreachablePass : public RemoveUnreachablePassBase { : RemoveUnreachablePassBase("RemoveUnreachablePass") {} std::unique_ptr compute_reachable_objects( - const Scope& scope, - const method_override_graph::Graph& method_override_graph, + const DexStoresVector& stores, PassManager& pm, int* num_ignore_check_strings, reachability::ReachableAspects* reachable_aspects, diff --git a/opt/remove-unreachable/TypeAnalysisAwareRemoveUnreachable.cpp b/opt/remove-unreachable/TypeAnalysisAwareRemoveUnreachable.cpp index 3c0e32c17a..dda00f8e18 100644 --- a/opt/remove-unreachable/TypeAnalysisAwareRemoveUnreachable.cpp +++ b/opt/remove-unreachable/TypeAnalysisAwareRemoveUnreachable.cpp @@ -339,8 +339,7 @@ class TypeAnalysisAwareClosureMarkerWorker final }; std::unique_ptr compute_reachable_objects_with_type_anaysis( - const Scope& scope, - const method_override_graph::Graph& method_override_graph, + const DexStoresVector& stores, const IgnoreSets& ignore_sets, int* num_ignore_check_strings, ReachableAspects* reachable_aspects, @@ -356,16 +355,18 @@ std::unique_ptr compute_reachable_objects_with_type_anaysis( int* num_unreachable_invokes, int* num_null_invokes) { Timer t("Marking"); + auto scope = build_class_scope(stores); std::unordered_set scope_set(scope.begin(), scope.end()); walk::parallel::code(scope, [](DexMethod*, IRCode& code) { code.cfg().calculate_exit_block(); }); auto reachable_objects = std::make_unique(); ConditionallyMarked cond_marked; + auto method_override_graph = mog::build_graph(scope); ConcurrentSet root_set; bool remove_no_argument_constructors = false; - RootSetMarker root_set_marker(method_override_graph, record_reachability, + RootSetMarker root_set_marker(*method_override_graph, record_reachability, relaxed_keep_class_members, remove_no_argument_constructors, &cond_marked, reachable_objects.get(), &root_set); @@ -374,7 +375,7 @@ std::unique_ptr compute_reachable_objects_with_type_anaysis( size_t num_threads = redex_parallel::default_num_threads(); Stats stats; TypeAnalysisAwareClosureMarkerSharedState shared_state{ - {std::move(scope_set), &ignore_sets, &method_override_graph, + {std::move(scope_set), &ignore_sets, method_override_graph.get(), record_reachability, relaxed_keep_class_members, relaxed_keep_interfaces, cfg_gathering_check_instantiable, cfg_gathering_check_instance_callable, cfg_gathering_check_returning, &cond_marked, reachable_objects.get(), @@ -391,7 +392,7 @@ std::unique_ptr compute_reachable_objects_with_type_anaysis( }, root_set, num_threads, /* push_tasks_while_running*/ true); - compute_zombie_methods(method_override_graph, *reachable_objects, + compute_zombie_methods(*method_override_graph, *reachable_objects, *reachable_aspects); if (num_ignore_check_strings != nullptr) { @@ -416,8 +417,7 @@ std::unique_ptr compute_reachable_objects_with_type_anaysis( std::unique_ptr TypeAnalysisAwareRemoveUnreachablePass::compute_reachable_objects( - const Scope& scope, - const method_override_graph::Graph& method_override_graph, + const DexStoresVector& stores, PassManager& pm, int* num_ignore_check_strings, reachability::ReachableAspects* reachable_aspects, @@ -438,12 +438,11 @@ TypeAnalysisAwareRemoveUnreachablePass::compute_reachable_objects( int num_unreachable_invokes; int num_null_invokes; auto res = compute_reachable_objects_with_type_anaysis( - scope, method_override_graph, m_ignore_sets, num_ignore_check_strings, - reachable_aspects, emit_graph_this_run, relaxed_keep_class_members, - relaxed_keep_interfaces, cfg_gathering_check_instantiable, - cfg_gathering_check_instance_callable, cfg_gathering_check_returning, - gta.get(), remove_no_argument_constructors, &num_exact_resolved_callees, - &num_unreachable_invokes, &num_null_invokes); + stores, m_ignore_sets, num_ignore_check_strings, reachable_aspects, + emit_graph_this_run, relaxed_keep_class_members, relaxed_keep_interfaces, + cfg_gathering_check_instantiable, cfg_gathering_check_instance_callable, + cfg_gathering_check_returning, gta.get(), remove_no_argument_constructors, + &num_exact_resolved_callees, &num_unreachable_invokes, &num_null_invokes); pm.incr_metric("num_exact_resolved_callees", num_exact_resolved_callees); pm.incr_metric("num_unreachable_invokes", num_unreachable_invokes); pm.incr_metric("num_null_invokes", num_null_invokes); diff --git a/opt/remove-unreachable/TypeAnalysisAwareRemoveUnreachable.h b/opt/remove-unreachable/TypeAnalysisAwareRemoveUnreachable.h index c2a77e0140..9c4e59d807 100644 --- a/opt/remove-unreachable/TypeAnalysisAwareRemoveUnreachable.h +++ b/opt/remove-unreachable/TypeAnalysisAwareRemoveUnreachable.h @@ -23,8 +23,7 @@ class TypeAnalysisAwareRemoveUnreachablePass } std::unique_ptr compute_reachable_objects( - const Scope& scope, - const method_override_graph::Graph& method_override_graph, + const DexStoresVector& stores, PassManager& pm, int* num_ignore_check_strings, reachability::ReachableAspects* reachable_aspects, diff --git a/opt/remove-unused-args/RemoveUnusedArgs.cpp b/opt/remove-unused-args/RemoveUnusedArgs.cpp index f2ad8ca68d..ac5973d7f9 100644 --- a/opt/remove-unused-args/RemoveUnusedArgs.cpp +++ b/opt/remove-unused-args/RemoveUnusedArgs.cpp @@ -23,7 +23,6 @@ #include "OptData.h" #include "OptDataDefs.h" #include "PassManager.h" -#include "Purity.h" #include "Resolver.h" #include "Show.h" #include "Walkers.h" @@ -157,10 +156,12 @@ static bool any_external(const Collection& methods) { * that are externally defined or not-renamable. */ void RemoveArgs::compute_reordered_protos(const mog::Graph& override_graph) { - AtomicMap fixed_protos; + ConcurrentMap fixed_protos; ConcurrentSet defined_protos; auto record_fixed_proto = [&fixed_protos](DexProto* proto, size_t increment) { - fixed_protos.fetch_add(proto, increment); + fixed_protos.update(proto, + [increment](DexProto*, size_t& count, + bool /* exists */) { count += increment; }); }; walk::parallel::methods( m_scope, @@ -168,8 +169,7 @@ void RemoveArgs::compute_reordered_protos(const mog::Graph& override_graph) { &defined_protos](DexMethod* caller) { auto caller_proto = caller->get_proto(); defined_protos.insert(caller_proto); - if (!can_rename(caller) || is_native(caller) || - caller->rstate.no_optimizations()) { + if (!can_rename(caller) || is_native(caller)) { record_fixed_proto(caller_proto, 1); } else if (caller->is_virtual()) { auto is_interface_method = @@ -215,11 +215,8 @@ void RemoveArgs::compute_reordered_protos(const mog::Graph& override_graph) { } }); - std::vector> ordered_fixed_protos; - ordered_fixed_protos.reserve(fixed_protos.size()); - for (auto&& [proto, count] : fixed_protos) { - ordered_fixed_protos.emplace_back(proto, count.load()); - } + std::vector> ordered_fixed_protos( + fixed_protos.begin(), fixed_protos.end()); std::sort(ordered_fixed_protos.begin(), ordered_fixed_protos.end(), compare_weighted_dexprotos); std::unordered_map fixed_representatives; @@ -535,9 +532,9 @@ void run_cleanup(DexMethod* method, cfg::ControlFlowGraph& cfg, const init_classes::InitClassesWithSideEffects* init_classes_with_side_effects, - const std::unordered_set& pure_methods, std::mutex& mutex, LocalDce::Stats& stats) { + std::unordered_set pure_methods; auto local_dce = LocalDce(init_classes_with_side_effects, pure_methods); local_dce.dce(cfg, /* normalize_new_instances */ true, method->get_class()); const auto& local_stats = local_dce.get_stats(); @@ -565,7 +562,7 @@ RemoveArgs::MethodStats RemoveArgs::update_method_protos( DexProto* original_proto; DexProto* reordered_proto; }; - InsertOnlyConcurrentMap unordered_entries; + ConcurrentMap unordered_entries; walk::parallel::methods(m_scope, [&](DexMethod* method) { std::deque live_arg_idxs; std::vector dead_insns; @@ -590,9 +587,6 @@ RemoveArgs::MethodStats RemoveArgs::update_method_protos( if (has_any_annotation(method, no_devirtualize_annos)) { return; } - if (method->rstate.no_optimizations()) { - return; - } // Remember entry unordered_entries.emplace(method, @@ -669,7 +663,6 @@ RemoveArgs::MethodStats RemoveArgs::update_method_protos( run_cleanup(method, cfg, &m_init_classes_with_side_effects, - m_pure_methods, local_dce_stats_mutex, local_dce_stats); } @@ -748,7 +741,6 @@ std::pair RemoveArgs::update_callsites() { run_cleanup(method, cfg, &m_init_classes_with_side_effects, - m_pure_methods, local_dce_stats_mutex, local_dce_stats); } @@ -772,11 +764,10 @@ void RemoveUnusedArgsPass::run_pass(DexStoresVector& stores, size_t num_method_protos_reordered_count = 0; size_t num_iterations = 0; LocalDce::Stats local_dce_stats; - auto pure_methods = get_pure_methods(); while (true) { num_iterations++; RemoveArgs rm_args(scope, init_classes_with_side_effects, m_blocklist, - pure_methods, m_total_iterations++); + m_total_iterations++); auto pass_stats = rm_args.run(conf); if (pass_stats.methods_updated_count == 0) { break; diff --git a/opt/remove-unused-args/RemoveUnusedArgs.h b/opt/remove-unused-args/RemoveUnusedArgs.h index fa6220cccd..cb25c3bb1a 100644 --- a/opt/remove-unused-args/RemoveUnusedArgs.h +++ b/opt/remove-unused-args/RemoveUnusedArgs.h @@ -47,20 +47,18 @@ class RemoveArgs { const init_classes::InitClassesWithSideEffects& init_classes_with_side_effects, const std::vector& blocklist, - const std::unordered_set& pure_methods, size_t iteration = 0) : m_scope(scope), m_init_classes_with_side_effects(init_classes_with_side_effects), m_blocklist(blocklist), - m_iteration(iteration), - m_pure_methods(pure_methods) {} + m_iteration(iteration) {} RemoveArgs::PassStats run(ConfigFiles& conf); private: const Scope& m_scope; const init_classes::InitClassesWithSideEffects& m_init_classes_with_side_effects; - InsertOnlyConcurrentMap> m_live_arg_idxs_map; + ConcurrentMap> m_live_arg_idxs_map; // Data structure to remember running indices to make method names unique when // we reorder prototypes across virtual scopes, or do other general changes to // non-virtuals. @@ -74,7 +72,6 @@ class RemoveArgs { std::unordered_map m_reordered_protos; const std::vector& m_blocklist; size_t m_iteration; - const std::unordered_set& m_pure_methods; DexTypeList::ContainerType get_live_arg_type_list( DexMethod* method, const std::deque& live_arg_idxs); @@ -101,8 +98,9 @@ class RemoveUnusedArgsPass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, - {InitialRenameClass, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } diff --git a/opt/remove-unused-fields/RemoveUnusedFields.cpp b/opt/remove-unused-fields/RemoveUnusedFields.cpp index ac5c0b36e2..64af034033 100644 --- a/opt/remove-unused-fields/RemoveUnusedFields.cpp +++ b/opt/remove-unused-fields/RemoveUnusedFields.cpp @@ -118,15 +118,18 @@ class RemoveUnusedFields final { field_op_tracker::FieldStatsMap field_stats = field_op_tracker::analyze(m_scope); - std::unique_ptr field_writes; + // analyze_non_zero_writes and the later transform() need (editable) cfg + walk::parallel::code(m_scope, [&](const DexMethod*, IRCode& code) { + code.build_cfg(/* editable = true*/); + }); + + boost::optional field_writes; if (m_config.remove_zero_written_fields || m_config.remove_vestigial_objects_written_fields) { - field_writes = std::make_unique(); - field_op_tracker::analyze_writes( + field_writes = field_op_tracker::analyze_writes( m_scope, field_stats, m_config.remove_vestigial_objects_written_fields ? &m_type_lifetimes - : nullptr, - field_writes.get()); + : nullptr); } for (auto& pair : field_stats) { @@ -143,7 +146,7 @@ class RemoveUnusedFields final { if (m_config.remove_unread_fields && stats.reads == 0) { m_unread_fields.emplace(field); if (m_config.remove_vestigial_objects_written_fields && - !field_writes->non_vestigial_objects_written_fields.count_unsafe( + !field_writes->non_vestigial_objects_written_fields.count( field)) { m_vestigial_objects_written_fields.emplace(field); } @@ -151,7 +154,7 @@ class RemoveUnusedFields final { !has_non_zero_static_value(field)) { m_unwritten_fields.emplace(field); } else if (m_config.remove_zero_written_fields && - !field_writes->non_zero_written_fields.count_unsafe(field) && + !field_writes->non_zero_written_fields.count(field) && !has_non_zero_static_value(field)) { m_zero_written_fields.emplace(field); } @@ -170,10 +173,6 @@ class RemoveUnusedFields final { // Replace reads to unwritten fields with appropriate const-0 instructions, // and remove the writes to unread fields. walk::parallel::code(m_scope, [&](const DexMethod* method, IRCode& code) { - if (method->rstate.no_optimizations()) { - return; - } - always_assert(code.editable_cfg_built()); auto& cfg = code.cfg(); cfg::CFGMutation m(cfg); auto iterable = cfg::InstructionIterable(cfg); @@ -246,6 +245,7 @@ class RemoveUnusedFields final { if (any_changes) { m_shrinker.shrink_method(const_cast(method)); } + code.clear_cfg(); }); } diff --git a/opt/remove-unused-fields/RemoveUnusedFields.h b/opt/remove-unused-fields/RemoveUnusedFields.h index 426fcea2fa..e229cf3c9a 100644 --- a/opt/remove-unused-fields/RemoveUnusedFields.h +++ b/opt/remove-unused-fields/RemoveUnusedFields.h @@ -49,8 +49,9 @@ class PassImpl : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, - {SpuriousGetClassCallsInterned, RequiresAndPreserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } @@ -85,6 +86,8 @@ class PassImpl : public Pass { "Write all removed fields to " + std::string(REMOVED_FIELDS_FILENAME)); } + bool is_cfg_legacy() override { return true; } + void run_pass(DexStoresVector& stores, ConfigFiles& conf, PassManager& mgr) override; diff --git a/opt/remove_empty_classes/RemoveEmptyClasses.cpp b/opt/remove_empty_classes/RemoveEmptyClasses.cpp index 0a4326c300..ead6c00811 100644 --- a/opt/remove_empty_classes/RemoveEmptyClasses.cpp +++ b/opt/remove_empty_classes/RemoveEmptyClasses.cpp @@ -77,9 +77,7 @@ void process_code(ConcurrentSet* class_references, DexMethod* meth, IRCode& code) { // Types referenced in code. - always_assert(code.editable_cfg_built()); - auto& cfg = code.cfg(); - for (auto const& mie : cfg::InstructionIterable(cfg)) { + for (auto const& mie : InstructionIterable(meth->get_code())) { auto opcode = mie.insn; if (opcode->has_type()) { auto typ = type::get_element_type_if_array(opcode->get_type()); diff --git a/opt/remove_empty_classes/RemoveEmptyClasses.h b/opt/remove_empty_classes/RemoveEmptyClasses.h index 6185f0be30..08dc4d4325 100644 --- a/opt/remove_empty_classes/RemoveEmptyClasses.h +++ b/opt/remove_empty_classes/RemoveEmptyClasses.h @@ -17,8 +17,12 @@ class RemoveEmptyClassesPass : public Pass { const override { using namespace redex_properties::interactions; using namespace redex_properties::names; - return {}; + return { + {HasSourceBlocks, Preserves}, + }; } + bool is_cfg_legacy() override { return true; } + void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; }; diff --git a/opt/remove_redundant_check_casts/RemoveRedundantCheckCasts.cpp b/opt/remove_redundant_check_casts/RemoveRedundantCheckCasts.cpp index 2909e91511..07484e9330 100644 --- a/opt/remove_redundant_check_casts/RemoveRedundantCheckCasts.cpp +++ b/opt/remove_redundant_check_casts/RemoveRedundantCheckCasts.cpp @@ -24,11 +24,12 @@ impl::Stats remove_redundant_check_casts(const CheckCastConfig& config, } auto* code = method->get_code(); - always_assert(code->editable_cfg_built()); + code->build_cfg(/* editable */ true); impl::CheckCastAnalysis analysis(config, method, android_sdk); auto casts = analysis.collect_redundant_checks_replacement(); auto stats = impl::apply(method, casts); + code->clear_cfg(); return stats; } diff --git a/opt/remove_redundant_check_casts/RemoveRedundantCheckCasts.h b/opt/remove_redundant_check_casts/RemoveRedundantCheckCasts.h index 417d8b6eab..249625b985 100644 --- a/opt/remove_redundant_check_casts/RemoveRedundantCheckCasts.h +++ b/opt/remove_redundant_check_casts/RemoveRedundantCheckCasts.h @@ -22,13 +22,16 @@ class RemoveRedundantCheckCastsPass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, - {InitialRenameClass, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } void bind_config() override; + bool is_cfg_legacy() override { return true; } + void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; private: diff --git a/opt/renameclasses/InitialRenameClassesPass.h b/opt/renameclasses/InitialRenameClassesPass.h index b4289d385c..d411cac13c 100644 --- a/opt/renameclasses/InitialRenameClassesPass.h +++ b/opt/renameclasses/InitialRenameClassesPass.h @@ -29,7 +29,7 @@ class InitialRenameClassesPass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, - {InitialRenameClass, Establishes}, + {HasSourceBlocks, Preserves}, }; } diff --git a/opt/renameclasses/RenameClassesV2.h b/opt/renameclasses/RenameClassesV2.h index 5f71003504..7a5631c06f 100644 --- a/opt/renameclasses/RenameClassesV2.h +++ b/opt/renameclasses/RenameClassesV2.h @@ -47,8 +47,8 @@ class RenameClassesPassV2 : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, - {InitialRenameClass, Preserves}, {RenameClass, EstablishesAndRequiresFinally}, }; } diff --git a/opt/reorder-interfaces-decl/ReorderInterfacesDecl.h b/opt/reorder-interfaces-decl/ReorderInterfacesDecl.h index 4e6aafa6cc..436483893b 100644 --- a/opt/reorder-interfaces-decl/ReorderInterfacesDecl.h +++ b/opt/reorder-interfaces-decl/ReorderInterfacesDecl.h @@ -19,9 +19,12 @@ class ReorderInterfacesDeclPass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } + bool is_cfg_legacy() override { return true; } void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; }; diff --git a/opt/resolve-proguard-values/ResolveProguardAssumeValues.cpp b/opt/resolve-proguard-values/ResolveProguardAssumeValues.cpp index 9b5bd4a3df..6cb5aa5ef3 100644 --- a/opt/resolve-proguard-values/ResolveProguardAssumeValues.cpp +++ b/opt/resolve-proguard-values/ResolveProguardAssumeValues.cpp @@ -12,6 +12,7 @@ #include "ReachingDefinitions.h" #include "RedexContext.h" #include "Resolver.h" +#include "ScopedCFG.h" #include "Show.h" #include "Trace.h" #include "Walkers.h" @@ -27,9 +28,8 @@ void ResolveProguardAssumeValuesPass::run_pass(DexStoresVector& stores, if (!method || !method->get_code()) { return stat; } - always_assert(method->get_code()->editable_cfg_built()); - auto& cfg = method->get_code()->cfg(); - return process_for_code(cfg); + auto code = method->get_code(); + return process_for_code(code); }); stats.report(mgr); TRACE(PGR, @@ -43,12 +43,13 @@ void ResolveProguardAssumeValuesPass::run_pass(DexStoresVector& stores, } ResolveProguardAssumeValuesPass::Stats -ResolveProguardAssumeValuesPass::process_for_code(cfg::ControlFlowGraph& cfg) { +ResolveProguardAssumeValuesPass::process_for_code(IRCode* code) { Stats stat; - cfg::CFGMutation mutation(cfg); + cfg::ScopedCFG cfg(code); + cfg::CFGMutation mutation(*cfg); - auto ii = InstructionIterable(cfg); + auto ii = InstructionIterable(*cfg); for (auto insn_it = ii.begin(); insn_it != ii.end(); ++insn_it) { auto insn = insn_it->insn; switch (insn->opcode()) { @@ -60,7 +61,7 @@ ResolveProguardAssumeValuesPass::process_for_code(cfg::ControlFlowGraph& cfg) { break; } - auto move_result_it = cfg.move_result_of(insn_it); + auto move_result_it = cfg->move_result_of(insn_it); if (!move_result_it.is_end()) { auto move_insn = move_result_it->insn; int val = field_value->value.v; @@ -86,7 +87,7 @@ ResolveProguardAssumeValuesPass::process_for_code(cfg::ControlFlowGraph& cfg) { break; } - auto move_result_it = cfg.move_result_of(insn_it); + auto move_result_it = cfg->move_result_of(insn_it); if (!move_result_it.is_end()) { auto move_insn = move_result_it->insn; int val = (*return_value).value.v; diff --git a/opt/resolve-proguard-values/ResolveProguardAssumeValues.h b/opt/resolve-proguard-values/ResolveProguardAssumeValues.h index 73a636fba3..a5c7b076cc 100644 --- a/opt/resolve-proguard-values/ResolveProguardAssumeValues.h +++ b/opt/resolve-proguard-values/ResolveProguardAssumeValues.h @@ -6,7 +6,6 @@ */ #pragma once -#include "ControlFlow.h" #include "Pass.h" #include "PassManager.h" @@ -59,11 +58,14 @@ class ResolveProguardAssumeValuesPass : public Pass { const override { using namespace redex_properties::interactions; using namespace redex_properties::names; - return {}; + return { + {HasSourceBlocks, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, + }; } - static ResolveProguardAssumeValuesPass::Stats process_for_code( - cfg::ControlFlowGraph& cfg); + static ResolveProguardAssumeValuesPass::Stats process_for_code(IRCode* code); + bool is_cfg_legacy() override { return true; } void run_pass(DexStoresVector& stores, ConfigFiles&, PassManager& mgr) override; diff --git a/opt/resolve-refs/ExternalRefsManglingPass.h b/opt/resolve-refs/ExternalRefsManglingPass.h index a8fe1d602d..ba21762808 100644 --- a/opt/resolve-refs/ExternalRefsManglingPass.h +++ b/opt/resolve-refs/ExternalRefsManglingPass.h @@ -25,7 +25,9 @@ class ExternalRefsManglingPass : public Pass { const override { using namespace redex_properties::interactions; using namespace redex_properties::names; - return {}; + return { + {HasSourceBlocks, Preserves}, + }; } void bind_config() override { diff --git a/opt/resolve-refs/ResolveRefsPass.cpp b/opt/resolve-refs/ResolveRefsPass.cpp index b6be40c3d6..b57fb04a55 100644 --- a/opt/resolve-refs/ResolveRefsPass.cpp +++ b/opt/resolve-refs/ResolveRefsPass.cpp @@ -250,9 +250,17 @@ void ResolveRefsPass::resolve_method_refs(const DexMethod* caller, RefStats& stats) { always_assert(insn->has_method()); auto mref = insn->get_method(); - bool resolved_virtual_to_interface; - auto mdef = - resolve_invoke_method(insn, caller, &resolved_virtual_to_interface); + auto mdef = resolve_method(mref, opcode_to_search(insn), caller); + bool resolved_to_interface = false; + if (!mdef && opcode_to_search(insn) == MethodSearch::Virtual) { + mdef = resolve_method(mref, MethodSearch::InterfaceVirtual, caller); + if (mdef) { + TRACE(RESO, 4, "InterfaceVirtual resolve to %s in %s", SHOW(mdef), + SHOW(insn)); + const auto* cls = type_class(mdef->get_class()); + resolved_to_interface = cls && is_interface(cls); + } + } if (!mdef && is_array_clone(insn)) { auto* object_array_clone = method::java_lang_Objects_clone(); TRACE(RESO, 3, "Resolving %s\n\t=>%s", SHOW(mref), @@ -286,9 +294,7 @@ void ResolveRefsPass::resolve_method_refs(const DexMethod* caller, TRACE(RESO, 3, "Resolving %s\n\t=>%s", SHOW(mref), SHOW(mdef)); insn->set_method(mdef); stats.num_mref_simple_resolved++; - if (resolved_virtual_to_interface && cls && is_interface(cls)) { - TRACE(RESO, 4, "InterfaceVirtual resolve to %s in %s", SHOW(mdef), - SHOW(insn)); + if (resolved_to_interface && opcode::is_invoke_virtual(insn->opcode())) { insn->set_opcode(OPCODE_INVOKE_INTERFACE); stats.num_resolve_to_interface++; } diff --git a/opt/resolve-refs/ResolveRefsPass.h b/opt/resolve-refs/ResolveRefsPass.h index f79e1e3967..f8caead5da 100644 --- a/opt/resolve-refs/ResolveRefsPass.h +++ b/opt/resolve-refs/ResolveRefsPass.h @@ -41,7 +41,9 @@ class ResolveRefsPass : public ExternalRefsManglingPass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Establishes}, + {NoSpuriousGetClassCalls, Preserves}, }; } diff --git a/opt/resolve-refs/SpecializeRtype.cpp b/opt/resolve-refs/SpecializeRtype.cpp index 65119e6902..08663699c9 100644 --- a/opt/resolve-refs/SpecializeRtype.cpp +++ b/opt/resolve-refs/SpecializeRtype.cpp @@ -290,7 +290,7 @@ void RtypeSpecialization::specialize_non_true_virtuals( const method_override_graph::Graph& override_graph, DexMethod* meth, const DexType* better_rtype, - InsertOnlyConcurrentMap& virtual_roots, + ConcurrentMap& virtual_roots, RtypeStats& stats) const { const auto& overridings = method_override_graph::get_overriding_methods(override_graph, meth, true); @@ -303,7 +303,7 @@ void RtypeSpecialization::specialize_true_virtuals( const method_override_graph::Graph& override_graph, DexMethod* meth, const DexType* better_rtype, - InsertOnlyConcurrentMap& virtual_roots, + ConcurrentMap& virtual_roots, RtypeStats& stats) const { const auto& overridings = method_override_graph::get_overriding_methods(override_graph, meth, true); @@ -380,7 +380,7 @@ void RtypeSpecialization::specialize_true_virtuals( void RtypeSpecialization::specialize_rtypes(const Scope& scope) { Timer t("specialize_rtype"); const auto& override_graph = method_override_graph::build_graph(scope); - InsertOnlyConcurrentMap virtual_roots; + ConcurrentMap virtual_roots; // Preprocess the candidates to cut down the size of candidates. // The main logic is filtering out complex virtual scopes that we choose not diff --git a/opt/resolve-refs/SpecializeRtype.h b/opt/resolve-refs/SpecializeRtype.h index 708bd1709b..e096321b50 100644 --- a/opt/resolve-refs/SpecializeRtype.h +++ b/opt/resolve-refs/SpecializeRtype.h @@ -79,14 +79,14 @@ class RtypeSpecialization final { const method_override_graph::Graph& override_graph, DexMethod* meth, const DexType* better_rtype, - InsertOnlyConcurrentMap& virtual_roots, + ConcurrentMap& virtual_roots, RtypeStats& stats) const; void specialize_true_virtuals( const method_override_graph::Graph& override_graph, DexMethod* meth, const DexType* better_rtype, - InsertOnlyConcurrentMap& virtual_roots, + ConcurrentMap& virtual_roots, RtypeStats& stats) const; bool shares_identical_rtype_candidate(DexMethod* meth, const DexType* better_rtype) const; diff --git a/opt/result-propagation/ResultPropagation.cpp b/opt/result-propagation/ResultPropagation.cpp index 1015519511..c31f182480 100644 --- a/opt/result-propagation/ResultPropagation.cpp +++ b/opt/result-propagation/ResultPropagation.cpp @@ -18,6 +18,7 @@ #include "IRInstruction.h" #include "PassManager.h" #include "Resolver.h" +#include "ScopedCFG.h" #include "Show.h" #include "Trace.h" #include "Walkers.h" @@ -251,7 +252,12 @@ boost::optional ReturnParamResolver::get_return_param_index( return 0; } - auto callee = resolve_invoke_method(insn, resolved_refs); + auto callee = resolve_method(method, opcode_to_search(insn), resolved_refs); + if (callee == nullptr && opcode == OPCODE_INVOKE_VIRTUAL) { + // There are some invoke-virtual call on methods whose def are + // actually in interface. + callee = resolve_method(method, MethodSearch::InterfaceVirtual); + } if (callee == nullptr) { return boost::none; } @@ -425,11 +431,12 @@ boost::optional ReturnParamResolver::get_return_param_index( return return_param_index.get_constant(); } -void ResultPropagation::patch(PassManager& mgr, cfg::ControlFlowGraph& cfg) { +void ResultPropagation::patch(PassManager& mgr, IRCode* code) { // turn move-result-... into move instructions if the called method // is known to always return a particular parameter std::vector deletes; - auto ii = InstructionIterable(cfg); + cfg::ScopedCFG cfg(code); + auto ii = InstructionIterable(*cfg); for (auto it = ii.begin(); it != ii.end(); it++) { // do we have a sequence of invoke + move-result instruction? const auto insn = it->insn; @@ -440,7 +447,7 @@ void ResultPropagation::patch(PassManager& mgr, cfg::ControlFlowGraph& cfg) { continue; } - auto primary_it = cfg.primary_instruction_of_move_result(it); + auto primary_it = cfg->primary_instruction_of_move_result(it); if (primary_it.is_end()) { continue; } @@ -495,7 +502,7 @@ void ResultPropagation::patch(PassManager& mgr, cfg::ControlFlowGraph& cfg) { } } for (auto const& instr : deletes) { - cfg.remove_insn(instr); + cfg->remove_insn(instr); } } @@ -511,15 +518,13 @@ void ResultPropagationPass::run_pass(DexStoresVector& stores, const auto stats = walk::parallel::methods( scope, [&](DexMethod* m) { const auto code = m->get_code(); - if (code == nullptr || m->rstate.no_optimizations()) { + if (code == nullptr) { return ResultPropagation::Stats(); } ResultPropagation rp(methods_which_return_parameter, resolver, m_callee_blocklist); - always_assert(code->editable_cfg_built()); - auto& cfg = code->cfg(); - rp.patch(mgr, cfg); + rp.patch(mgr, code); return rp.get_stats(); }); mgr.incr_metric(METRIC_METHODS_WHICH_RETURN_PARAMETER, @@ -574,10 +579,9 @@ ResultPropagationPass::find_methods_which_return_parameter( return res; } - always_assert(code->editable_cfg_built()); - auto& cfg = code->cfg(); + cfg::ScopedCFG cfg(const_cast(code)); const auto return_param_index = resolver.get_return_param_index( - cfg, methods_which_return_parameter); + *cfg, methods_which_return_parameter); if (return_param_index) { res.insert({method, *return_param_index}); } diff --git a/opt/result-propagation/ResultPropagation.h b/opt/result-propagation/ResultPropagation.h index 145c6fe123..d08cf2f503 100644 --- a/opt/result-propagation/ResultPropagation.h +++ b/opt/result-propagation/ResultPropagation.h @@ -110,7 +110,7 @@ class ResultPropagation { /* * Patch code based on analysis results. */ - void patch(PassManager&, cfg::ControlFlowGraph&); + void patch(PassManager&, IRCode*); private: const std::unordered_map& @@ -139,7 +139,9 @@ class ResultPropagationPass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } diff --git a/opt/shorten-srcstrings/Shorten.h b/opt/shorten-srcstrings/Shorten.h index 17275e506d..2ffaf43c95 100644 --- a/opt/shorten-srcstrings/Shorten.h +++ b/opt/shorten-srcstrings/Shorten.h @@ -19,6 +19,7 @@ class ShortenSrcStringsPass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, {RenameClass, Preserves}, }; @@ -29,6 +30,8 @@ class ShortenSrcStringsPass : public Pass { trait(Traits::Pass::unique, true); } + bool is_cfg_legacy() override { return true; } + void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; private: diff --git a/opt/shrinker/ShrinkerPass.cpp b/opt/shrinker/ShrinkerPass.cpp index e35dd7c373..52562e35fd 100644 --- a/opt/shrinker/ShrinkerPass.cpp +++ b/opt/shrinker/ShrinkerPass.cpp @@ -79,10 +79,10 @@ void ShrinkerPass::run_pass(DexStoresVector& stores, shrinker.get_cse_stats().instructions_eliminated); mgr.incr_metric("instructions_eliminated_copy_prop", shrinker.get_copy_prop_stats().moves_eliminated); - mgr.incr_metric("instructions_eliminated_localdce_dead", - shrinker.get_local_dce_stats().dead_instruction_count); - mgr.incr_metric("instructions_eliminated_localdce_unreachable", - shrinker.get_local_dce_stats().unreachable_instruction_count); + mgr.incr_metric( + "instructions_eliminated_localdce", + shrinker.get_local_dce_stats().dead_instruction_count + + shrinker.get_local_dce_stats().unreachable_instruction_count); mgr.incr_metric("instructions_eliminated_dedup_blocks", shrinker.get_dedup_blocks_stats().insns_removed); mgr.incr_metric("blocks_eliminated_by_dedup_blocks", diff --git a/opt/shrinker/ShrinkerPass.h b/opt/shrinker/ShrinkerPass.h index 95d52a6773..c001457979 100644 --- a/opt/shrinker/ShrinkerPass.h +++ b/opt/shrinker/ShrinkerPass.h @@ -20,14 +20,14 @@ class ShrinkerPass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, - // This may be too conservative as the shrinker can be configured not to - // DCE. - {SpuriousGetClassCallsInterned, RequiresAndPreserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } void bind_config() override; + bool is_cfg_legacy() override { return true; } void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; private: diff --git a/opt/singleimpl/SingleImpl.h b/opt/singleimpl/SingleImpl.h index ab0762295f..f83e8b2de6 100644 --- a/opt/singleimpl/SingleImpl.h +++ b/opt/singleimpl/SingleImpl.h @@ -31,7 +31,9 @@ class SingleImplPass : public Pass { using namespace redex_properties::interactions; using namespace redex_properties::names; return { + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } diff --git a/opt/singleimpl/SingleImplAnalyze.cpp b/opt/singleimpl/SingleImplAnalyze.cpp index 49718b7bab..6355e106ff 100644 --- a/opt/singleimpl/SingleImplAnalyze.cpp +++ b/opt/singleimpl/SingleImplAnalyze.cpp @@ -51,6 +51,7 @@ struct AnalysisImpl : SingleImplAnalysis { void filter_list(const std::vector& list, bool keep_match); void filter_by_annotations(const std::vector& blocklist); + private: const Scope& scope; const ProguardMap& pg_map; XStoreRefs xstores; @@ -259,7 +260,7 @@ void AnalysisImpl::escape_with_clinit() { void AnalysisImpl::escape_with_sfields() { for (auto const& intf_it : single_impls) { auto intf_cls = type_class(intf_it.first); - redex_assert(CONSTP(intf_cls)->get_ifields().empty()); + redex_assert(intf_cls->get_ifields().empty()); always_assert(!intf_cls->is_external()); const auto& sfields = intf_cls->get_sfields(); if (sfields.empty()) continue; diff --git a/opt/singleimpl/SingleImplOptimize.cpp b/opt/singleimpl/SingleImplOptimize.cpp index a3a72160e3..31578ab60e 100644 --- a/opt/singleimpl/SingleImplOptimize.cpp +++ b/opt/singleimpl/SingleImplOptimize.cpp @@ -215,6 +215,7 @@ struct OptimizationImpl { const std::unordered_set& methods, std::vector* removed_instruction); + private: std::unique_ptr single_impls; // A map from interface method to implementing method. We maintain this global // map for rewriting method references in annotation. diff --git a/opt/split_huge_switches/SplitHugeSwitchPass.cpp b/opt/split_huge_switches/SplitHugeSwitchPass.cpp index dacb3f8ce1..b9e649710b 100644 --- a/opt/split_huge_switches/SplitHugeSwitchPass.cpp +++ b/opt/split_huge_switches/SplitHugeSwitchPass.cpp @@ -83,13 +83,11 @@ using Stats = SplitHugeSwitchPass::Stats; namespace { -bool has_switch(cfg::ControlFlowGraph& cfg) { - for (auto* block : cfg.blocks()) { - for (const auto& mie : InstructionIterable(block)) { - auto opcode = mie.insn->opcode(); - if (opcode::is_switch(opcode)) { - return true; - } +bool has_switch(IRCode* code) { + for (const auto& mie : InstructionIterable(*code)) { + auto opcode = mie.insn->opcode(); + if (opcode::is_switch(opcode)) { + return true; } } return false; @@ -103,7 +101,7 @@ cfg::InstructionIterator find_large_switch(cfg::ControlFlowGraph& cfg, continue; } auto block = it.block(); - redex_assert(it->insn == CONSTP(block)->get_last_insn()->insn); + redex_assert(it->insn == block->get_last_insn()->insn); if (block->succs().size() >= case_threshold) { break; } @@ -212,7 +210,7 @@ struct SwitchRange { SwitchRange get_switch_range(const cfg::ControlFlowGraph& cfg, cfg::Block* b, size_t split_into) { - redex_assert(CONSTP(b)->get_last_insn()->insn->opcode() == OPCODE_SWITCH); + redex_assert(b->get_last_insn()->insn->opcode() == OPCODE_SWITCH); std::vector cases; for (const auto* e : cfg.get_succ_edges_of_type(b, cfg::EDGE_BRANCH)) { cases.push_back(*e->case_key()); @@ -278,6 +276,7 @@ DexMethod* create_split(DexMethod* orig_method, cfg.simplify(); } + cloned_code->clear_cfg(); return create_dex_method(orig_method, std::move(cloned_code)); } @@ -531,15 +530,15 @@ AnalysisData analyze(DexMethod* m, data.under_code_units_threshold = false; // stats.large_methods_set.emplace(m); - cfg::ScopedCFG scoped_cfg(code); - - if (!has_switch(*scoped_cfg)) { + if (!has_switch(code)) { data.no_switch = true; return data; } data.no_switch = false; // stats.switch_methods_set.emplace(m); + cfg::ScopedCFG scoped_cfg(code); + auto switch_it = find_large_switch(*scoped_cfg, case_threshold); if (switch_it.is_end()) { data.no_large_switch = true; @@ -761,15 +760,13 @@ Stats run_split_dexes(DexStoresVector& stores, continue; } left -= required; - always_assert(data.m->get_code()); - always_assert(data.m->get_code()->editable_cfg_built()); - size_t orig_size = data.m->get_code()->cfg().estimate_code_units(); + size_t orig_size = data.m->get_code()->estimate_code_units(); auto new_methods = run_split(data, data.m, data.m->get_code(), case_threshold); - size_t new_size = data.m->get_code()->cfg().estimate_code_units(); + size_t new_size = data.m->get_code()->estimate_code_units(); for (DexMethod* m : new_methods) { type_class(m->get_class())->add_method(m); - new_size += m->get_code()->cfg().estimate_code_units(); + new_size += m->get_code()->estimate_code_units(); } std::lock_guard lock_guard(mutex); diff --git a/opt/split_huge_switches/SplitHugeSwitchPass.h b/opt/split_huge_switches/SplitHugeSwitchPass.h index ec4ecd1d6a..cf6efaadc9 100644 --- a/opt/split_huge_switches/SplitHugeSwitchPass.h +++ b/opt/split_huge_switches/SplitHugeSwitchPass.h @@ -49,14 +49,16 @@ class SplitHugeSwitchPass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, - {InitialRenameClass, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } SplitHugeSwitchPass() : Pass("SplitHugeSwitchPass") {} void bind_config() override; + bool is_cfg_legacy() override { return true; } void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; static Stats run(DexMethod* m, diff --git a/opt/split_resource_tables/SplitResourceTables.cpp b/opt/split_resource_tables/SplitResourceTables.cpp index 3b4ccf41b7..5b07e4f2b0 100644 --- a/opt/split_resource_tables/SplitResourceTables.cpp +++ b/opt/split_resource_tables/SplitResourceTables.cpp @@ -348,7 +348,7 @@ void SplitResourceTablesPass::run_pass(DexStoresVector& stores, PassManager& mgr) { std::string zip_dir; cfg.get_json_config().get("apk_dir", "", zip_dir); - always_assert(!zip_dir.empty()); + always_assert(zip_dir.size()); TRACE(SPLIT_RES, 2, "Begin SplitResourceTablesPass"); auto resources = create_resource_reader(zip_dir); @@ -457,7 +457,6 @@ void SplitResourceTablesPass::run_pass(DexStoresVector& stores, // Set up the new types that will actually be created by the next step. for (const auto& t : new_types) { std::vector config_ptrs; - config_ptrs.reserve(t.configs.size()); for (auto& config : t.configs) { config_ptrs.emplace_back(const_cast(&config)); } diff --git a/opt/split_resource_tables/SplitResourceTables.h b/opt/split_resource_tables/SplitResourceTables.h index 9f30ab7a26..626c04e800 100644 --- a/opt/split_resource_tables/SplitResourceTables.h +++ b/opt/split_resource_tables/SplitResourceTables.h @@ -89,7 +89,9 @@ class SplitResourceTablesPass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } @@ -104,6 +106,7 @@ class SplitResourceTablesPass : public Pass { bind("max_splits_per_type", 5, m_max_splits_per_type); } + bool is_cfg_legacy() override { return true; } void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; private: diff --git a/opt/staticrelo/StaticReloV2.cpp b/opt/staticrelo/StaticReloV2.cpp index 3803902bc4..12edc67731 100644 --- a/opt/staticrelo/StaticReloV2.cpp +++ b/opt/staticrelo/StaticReloV2.cpp @@ -102,12 +102,7 @@ void build_call_graph(const std::vector& candidate_classes, for (auto& meth_id : graph.method_id_map) { DexMethod* caller = meth_id.first; int caller_id = meth_id.second; - if (!caller->get_code()) { - continue; - } - always_assert(caller->get_code()->editable_cfg_built()); - for (const auto& mie : - cfg::InstructionIterable(caller->get_code()->cfg())) { + for (const auto& mie : InstructionIterable(caller->get_code())) { if (mie.insn->has_method()) { if (mie.insn->opcode() != OPCODE_INVOKE_STATIC) { continue; @@ -164,9 +159,7 @@ void color_from_a_class(StaticCallGraph& graph, DexClass* cls, int color) { if (code == nullptr) { return; } - always_assert(code->editable_cfg_built()); - auto& cfg = code->cfg(); - for (const auto& mie : cfg::InstructionIterable(cfg)) { + for (const auto& mie : InstructionIterable(code)) { if (mie.insn->has_method()) { if (mie.insn->opcode() != OPCODE_INVOKE_STATIC) { continue; @@ -208,7 +201,6 @@ int relocate_clusters(const StaticCallGraph& graph, const Scope& scope) { // caller int caller_id = *graph.callers[vertex.id].begin(); DexMethod* caller = graph.vertices[caller_id].method; - change_visibility(vertex.method, caller->get_class()); relocate_method(vertex.method, caller->get_class()); relocated_methods++; set_public(vertex.method); @@ -220,7 +212,6 @@ int relocate_clusters(const StaticCallGraph& graph, const Scope& scope) { // higher or equal to the api level of the method. if (to_class->rstate.get_api_level() >= api::LevelChecker::get_method_level(vertex.method)) { - change_visibility(vertex.method, to_class->get_type()); relocate_method(vertex.method, to_class->get_type()); relocated_methods++; } diff --git a/opt/staticrelo/StaticReloV2.h b/opt/staticrelo/StaticReloV2.h index e680c9ec40..44a581f9fd 100644 --- a/opt/staticrelo/StaticReloV2.h +++ b/opt/staticrelo/StaticReloV2.h @@ -20,12 +20,15 @@ class StaticReloPassV2 : public Pass { using namespace redex_properties::interactions; using namespace redex_properties::names; return { + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } static std::vector gen_candidates(const Scope&); static int run_relocation(const Scope&, std::vector&); + bool is_cfg_legacy() override { return true; } void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; }; } // namespace static_relo_v2 diff --git a/opt/string_concatenator/StringConcatenator.cpp b/opt/string_concatenator/StringConcatenator.cpp index 4179bcc736..b3ad32353f 100644 --- a/opt/string_concatenator/StringConcatenator.cpp +++ b/opt/string_concatenator/StringConcatenator.cpp @@ -383,15 +383,16 @@ void StringConcatenatorPass::run_pass(DexStoresVector& stores, if (code == nullptr) { return Stats{}; } - if (!method::is_clinit(m) || m->rstate.no_optimizations()) { + if (!method::is_clinit(m)) { // TODO maybe later? If we expand to non-clinit methods, `analyze()` // will have to consider StringBuilders passed in as arguments. return Stats{}; } - always_assert(code->editable_cfg_built()); + code->build_cfg(/* editable */ true); Stats stats = Concatenator{config}.run(&code->cfg(), m, &methods_to_remove); + code->clear_cfg(); return stats; }, diff --git a/opt/string_concatenator/StringConcatenator.h b/opt/string_concatenator/StringConcatenator.h index a550dca5b9..f05fa5d912 100644 --- a/opt/string_concatenator/StringConcatenator.h +++ b/opt/string_concatenator/StringConcatenator.h @@ -19,9 +19,12 @@ class StringConcatenatorPass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } + bool is_cfg_legacy() override { return true; } void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; }; diff --git a/opt/stringbuilder-outliner/StringBuilderOutliner.cpp b/opt/stringbuilder-outliner/StringBuilderOutliner.cpp index b16bf9baf6..413bbe08ac 100644 --- a/opt/stringbuilder-outliner/StringBuilderOutliner.cpp +++ b/opt/stringbuilder-outliner/StringBuilderOutliner.cpp @@ -191,7 +191,9 @@ void Outliner::gather_outline_candidate_typelists( for (const auto& p : tostring_instruction_to_state) { const auto& state = p.second; auto typelist = typelist_from_state(state); - m_outline_typelists.fetch_add(typelist, 1); + m_outline_typelists.update( + typelist, + [](const DexTypeList*, size_t& n, bool /* exists */) { ++n; }); } } @@ -234,7 +236,7 @@ void Outliner::create_outline_helpers(DexStoresVector* stores) { bool did_create_helper{false}; for (const auto& p : m_outline_typelists) { const auto* typelist = p.first; - auto count = p.second.load(); + auto count = p.second; if (count < m_config.min_outline_count || typelist->size() > m_config.max_outline_length) { @@ -454,18 +456,14 @@ void StringBuilderOutlinerPass::run_pass(DexStoresVector& stores, Outliner outliner(m_config); // 1) Determine which methods have candidates for outlining. walk::parallel::code(scope, [&](const DexMethod* method, IRCode& code) { - if (!method->rstate.no_optimizations()) { - outliner.analyze(code); - } + outliner.analyze(code); }); // 2) Determine which candidates occur frequently enough to be worth // outlining. Build the corresponding outline helper functions. outliner.create_outline_helpers(&stores); // 3) Actually do the outlining. walk::parallel::code(scope, [&](const DexMethod* method, IRCode& code) { - if (!method->rstate.no_optimizations()) { - outliner.transform(&code); - } + outliner.transform(&code); }); mgr.incr_metric("stringbuilders_removed", diff --git a/opt/stringbuilder-outliner/StringBuilderOutliner.h b/opt/stringbuilder-outliner/StringBuilderOutliner.h index 3b4ba0c057..9445f8b045 100644 --- a/opt/stringbuilder-outliner/StringBuilderOutliner.h +++ b/opt/stringbuilder-outliner/StringBuilderOutliner.h @@ -251,12 +251,12 @@ class Outliner { // Map typelists of potentially outlinable StringBuilder call sequence to // their number of occurrences. - AtomicMap m_outline_typelists; + ConcurrentMap m_outline_typelists; // Typelists of call sequences we have chosen to outline -> generated outline // helper method. std::unordered_map m_outline_helpers; - InsertOnlyConcurrentMap m_builder_state_maps; + ConcurrentMap m_builder_state_maps; }; class StringBuilderOutlinerPass : public Pass { @@ -268,7 +268,9 @@ class StringBuilderOutlinerPass : public Pass { using namespace redex_properties::interactions; using namespace redex_properties::names; return { + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } diff --git a/opt/strip-debug-info/StripDebugInfo.cpp b/opt/strip-debug-info/StripDebugInfo.cpp index 658fccfcb9..4c91823043 100644 --- a/opt/strip-debug-info/StripDebugInfo.cpp +++ b/opt/strip-debug-info/StripDebugInfo.cpp @@ -126,20 +126,16 @@ Stats StripDebugInfo::run(IRCode& code, bool should_drop_synth) { ++stats.num_matches; bool debug_info_empty = true; bool force_discard = m_config.drop_all_dbg_info || should_drop_synth; - always_assert(code.editable_cfg_built()); - auto& cfg = code.cfg(); - for (auto* b : cfg.blocks()) { - auto it = b->begin(); - while (it != b->end()) { - const auto& mie = *it; - if (should_remove(mie, stats) || (force_discard && is_debug_entry(mie))) { - // Even though force_discard will drop the debug item below, preventing - // any of the debug entries for :meth to be output, we still want to - // erase those entries here so that transformations like inlining won't - // move these entries into a method that does have a debug item. - it = b->remove_mie(it); - continue; - } + + for (auto it = code.begin(); it != code.end();) { + const auto& mie = *it; + if (should_remove(mie, stats) || (force_discard && is_debug_entry(mie))) { + // Even though force_discard will drop the debug item below, preventing + // any of the debug entries for :meth to be output, we still want to + // erase those entries here so that transformations like inlining won't + // move these entries into a method that does have a debug item. + it = code.erase(it); + } else { switch (mie.type) { case MFLOW_DEBUG: // Any debug information op other than an end sequence means diff --git a/opt/strip-debug-info/StripDebugInfo.h b/opt/strip-debug-info/StripDebugInfo.h index ced246e006..e7d988d052 100644 --- a/opt/strip-debug-info/StripDebugInfo.h +++ b/opt/strip-debug-info/StripDebugInfo.h @@ -21,6 +21,8 @@ class StripDebugInfoPass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, {UltralightCodePatterns, Preserves}, }; } @@ -39,6 +41,7 @@ class StripDebugInfoPass : public Pass { bind("drop_synth_conservative", false, m_config.drop_synth_conservative); } + bool is_cfg_legacy() override { return true; } void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; void set_drop_prologue_end(bool b) { m_config.drop_prologue_end = b; } diff --git a/opt/test_cfg/TestCFG.h b/opt/test_cfg/TestCFG.h index 0e86da4ba0..b83db643e0 100644 --- a/opt/test_cfg/TestCFG.h +++ b/opt/test_cfg/TestCFG.h @@ -17,9 +17,10 @@ class TestCFGPass : public Pass { const override { using namespace redex_properties::interactions; using namespace redex_properties::names; - return {}; + return { + {HasSourceBlocks, Preserves}, + }; } - // This pass is no longer in use. bool is_cfg_legacy() override { return true; } void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; }; diff --git a/opt/throw-propagation/ThrowPropagationPass.cpp b/opt/throw-propagation/ThrowPropagationPass.cpp index 7c6f66f7f0..db02a5e67b 100644 --- a/opt/throw-propagation/ThrowPropagationPass.cpp +++ b/opt/throw-propagation/ThrowPropagationPass.cpp @@ -65,6 +65,7 @@ bool exclude_method(DexMethod* method) { } // namespace void ThrowPropagationPass::bind_config() { + bind("debug", false, m_config.debug); bind("blocklist", {}, m_config.blocklist, @@ -93,7 +94,7 @@ bool ThrowPropagationPass::is_no_return_method(const Config& config, return !can_return; } -ConcurrentSet ThrowPropagationPass::get_no_return_methods( +std::unordered_set ThrowPropagationPass::get_no_return_methods( const Config& config, const Scope& scope) { ConcurrentSet concurrent_no_return_methods; walk::parallel::methods(scope, [&](DexMethod* method) { @@ -101,12 +102,12 @@ ConcurrentSet ThrowPropagationPass::get_no_return_methods( concurrent_no_return_methods.insert(method); } }); - return concurrent_no_return_methods; + return concurrent_no_return_methods.move_to_container(); } ThrowPropagationPass::Stats ThrowPropagationPass::run( const Config& config, - const ConcurrentSet& no_return_methods, + const std::unordered_set& no_return_methods, const method_override_graph::Graph& graph, IRCode* code, std::unordered_set* no_return_methods_checked) { @@ -136,7 +137,7 @@ ThrowPropagationPass::Stats ThrowPropagationPass::run( if (exclude_method(other_method)) { return false; } - if (!no_return_methods.count_unsafe(other_method)) { + if (!no_return_methods.count(other_method)) { return_methods.push_back(other_method); } return true; @@ -159,7 +160,7 @@ ThrowPropagationPass::Stats ThrowPropagationPass::run( return return_methods.empty(); }; - throw_propagation_impl::ThrowPropagator impl(cfg); + throw_propagation_impl::ThrowPropagator impl(cfg, config.debug); for (auto block : cfg.blocks()) { auto ii = InstructionIterable(block); for (auto it = ii.begin(); it != ii.end(); it++) { @@ -191,7 +192,7 @@ void ThrowPropagationPass::run_pass(DexStoresVector& stores, PassManager& mgr) { Scope scope = build_class_scope(stores); auto override_graph = method_override_graph::build_graph(scope); - ConcurrentSet no_return_methods; + std::unordered_set no_return_methods; { Timer t("get_no_return_methods"); no_return_methods = get_no_return_methods(m_config, scope); @@ -235,7 +236,7 @@ void ThrowPropagationPass::run_pass(DexStoresVector& stores, }); } if (local_stats.throws_inserted > 0) { - if (!no_return_methods.count_unsafe(method) && + if (!no_return_methods.count(method) && is_no_return_method(m_config, method)) { std::lock_guard lock_guard( new_no_return_methods_mutex); diff --git a/opt/throw-propagation/ThrowPropagationPass.h b/opt/throw-propagation/ThrowPropagationPass.h index 31c360ae31..e07aade2b7 100644 --- a/opt/throw-propagation/ThrowPropagationPass.h +++ b/opt/throw-propagation/ThrowPropagationPass.h @@ -14,6 +14,7 @@ class ThrowPropagationPass : public Pass { public: struct Config { + bool debug{false}; std::unordered_set blocklist; }; @@ -37,18 +38,20 @@ class ThrowPropagationPass : public Pass { using namespace redex_properties::interactions; using namespace redex_properties::names; return { + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } void bind_config() override; - static ConcurrentSet get_no_return_methods(const Config& config, - const Scope& scope); + static std::unordered_set get_no_return_methods( + const Config& config, const Scope& scope); static Stats run( const Config& config, - const ConcurrentSet& no_return_methods, + const std::unordered_set& no_return_methods, const method_override_graph::Graph& graph, IRCode* code, std::unordered_set* no_return_methods_checked = nullptr); diff --git a/opt/type-analysis/CallGraphFileGenerationPass.cpp b/opt/type-analysis/CallGraphFileGenerationPass.cpp index e87c9c7dac..380d662f3c 100644 --- a/opt/type-analysis/CallGraphFileGenerationPass.cpp +++ b/opt/type-analysis/CallGraphFileGenerationPass.cpp @@ -59,9 +59,9 @@ void gather_cg_information( always_assert(cg_stats.num_nodes == nodes->size()); } -void gather_method_positions(const Scope& scope, - InsertOnlyConcurrentMap* - method_to_first_position) { +void gather_method_positions( + const Scope& scope, + ConcurrentMap* method_to_first_position) { walk::parallel::code( scope, [&method_to_first_position](DexMethod* method, IRCode& code) { always_assert(code.editable_cfg_built()); @@ -98,7 +98,7 @@ void write_out_callgraph(const Scope& scope, std::unordered_map nodes_to_ids; std::unordered_map> nodes_to_succs; gather_cg_information(cg, &nodes, &nodes_to_ids, &nodes_to_succs); - InsertOnlyConcurrentMap method_to_first_position; + ConcurrentMap method_to_first_position; gather_method_positions(scope, &method_to_first_position); size_t bit_32_size = sizeof(uint32_t); always_assert(nodes.size() <= std::numeric_limits::max()); diff --git a/opt/type-analysis/GlobalTypeAnalysisPass.h b/opt/type-analysis/GlobalTypeAnalysisPass.h index ee750ff7bf..aa80cb89ac 100644 --- a/opt/type-analysis/GlobalTypeAnalysisPass.h +++ b/opt/type-analysis/GlobalTypeAnalysisPass.h @@ -55,7 +55,9 @@ class GlobalTypeAnalysisPass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } diff --git a/opt/type-analysis/TypeAnalysisCallGraphGenerationPass.cpp b/opt/type-analysis/TypeAnalysisCallGraphGenerationPass.cpp index 005ffec7ac..6e01928ee7 100644 --- a/opt/type-analysis/TypeAnalysisCallGraphGenerationPass.cpp +++ b/opt/type-analysis/TypeAnalysisCallGraphGenerationPass.cpp @@ -78,7 +78,7 @@ class TypeAnalysisBasedStrategy : public MultipleCalleeBaseStrategy { if (!opcode::is_an_invoke(insn->opcode())) { continue; } - auto* resolved_callee = resolve_invoke_method(insn, method); + auto* resolved_callee = this->resolve_callee(method, insn); if (resolved_callee == nullptr) { continue; } diff --git a/opt/type-analysis/TypeAnalysisCallGraphGenerationPass.h b/opt/type-analysis/TypeAnalysisCallGraphGenerationPass.h index 15b6d25a08..7db813f152 100644 --- a/opt/type-analysis/TypeAnalysisCallGraphGenerationPass.h +++ b/opt/type-analysis/TypeAnalysisCallGraphGenerationPass.h @@ -32,7 +32,8 @@ class TypeAnalysisCallGraphGenerationPass : public Pass { using namespace redex_properties::interactions; using namespace redex_properties::names; return { - + {HasSourceBlocks, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } diff --git a/opt/typedef-anno-checker/TypedefAnnoCheckerPass.cpp b/opt/typedef-anno-checker/TypedefAnnoCheckerPass.cpp index 0a2a1b4232..89d6f84feb 100644 --- a/opt/typedef-anno-checker/TypedefAnnoCheckerPass.cpp +++ b/opt/typedef-anno-checker/TypedefAnnoCheckerPass.cpp @@ -7,7 +7,10 @@ #include "TypedefAnnoCheckerPass.h" +#include + #include "AnnoUtils.h" +#include "IROpcode.h" #include "PassManager.h" #include "Resolver.h" #include "Show.h" @@ -19,123 +22,18 @@ constexpr const char* ACCESS_PREFIX = "access$"; constexpr const char* DEFAULT_SUFFIX = "$default"; namespace { - -bool is_int(const type_inference::TypeEnvironment& env, reg_t reg) { - return !env.get_int_type(reg).is_top() && !env.get_int_type(reg).is_bottom(); -} - -bool is_string(const type_inference::TypeEnvironment& env, reg_t reg) { - return env.get_dex_type(reg) && - *env.get_dex_type(reg) == type::java_lang_String(); -} - -bool is_not_str_nor_int(const type_inference::TypeEnvironment& env, reg_t reg) { - return !is_string(env, reg) && !is_int(env, reg); -} - -DexMethod* resolve_method(DexMethod* caller, IRInstruction* insn) { - auto def_method = - resolve_method(insn->get_method(), opcode_to_search(insn), caller); - if (def_method == nullptr && insn->opcode() == OPCODE_INVOKE_VIRTUAL) { - def_method = - resolve_method(insn->get_method(), MethodSearch::InterfaceVirtual); - } - return def_method; +bool is_int(const type_inference::TypeEnvironment* env, reg_t reg) { + return env->get_type(reg).element() == IRType::INT || + env->get_type(reg).element() == IRType::CONST || + env->get_type(reg).element() == IRType::ZERO; } -bool is_synthetic_accessor(DexMethod* m) { - return boost::starts_with(m->get_simple_deobfuscated_name(), ACCESS_PREFIX) || - boost::ends_with(m->get_simple_deobfuscated_name(), DEFAULT_SUFFIX); +bool is_string(const type_inference::TypeEnvironment* env, reg_t reg) { + return env->get_dex_type(reg) && + env->get_dex_type(reg).value() == type::java_lang_String(); } - -void collect_from_instruction( - TypeEnvironments& envs, - type_inference::TypeInference& inference, - DexMethod* caller, - IRInstruction* insn, - std::vector>& missing_param_annos) { - always_assert(opcode::is_an_invoke(insn->opcode())); - auto* def_method = resolve_method(caller, insn); - if (!def_method || !def_method->get_param_anno()) { - // callee cannot be resolved or has no param annotation. - return; - } - if (ACCESS_PREFIX + def_method->get_simple_deobfuscated_name() != - caller->get_simple_deobfuscated_name() && - def_method->get_simple_deobfuscated_name() + DEFAULT_SUFFIX != - caller->get_simple_deobfuscated_name()) { - // Not a matching synthetic accessor. - return; - } - - auto& env = envs.find(insn)->second; - for (auto const& param_anno : *def_method->get_param_anno()) { - auto annotation = - inference.get_typedef_annotation(param_anno.second->get_annotations()); - if (!annotation) { - continue; - } - int param_index = insn->opcode() == OPCODE_INVOKE_STATIC - ? param_anno.first - : param_anno.first + 1; - reg_t param_reg = insn->src(param_index); - auto anno_type = env.get_annotation(param_reg); - if (anno_type && anno_type == annotation) { - // Safe assignment. Nothing to do. - continue; - } - DexAnnotationSet& param_anno_set = *param_anno.second; - missing_param_annos.push_back({param_index, param_anno_set}); - TRACE(TAC, 2, "Missing param annotation %s in %s", SHOW(¶m_anno_set), - SHOW(caller)); - } -} - } // namespace -void SynthAccessorPatcher::run(const Scope& scope) { - walk::parallel::methods(scope, [this](DexMethod* m) { - if (is_synthetic(m) && is_synthetic_accessor(m)) { - collect_accessors(m); - } - }); -} - -void SynthAccessorPatcher::collect_accessors(DexMethod* m) { - IRCode* code = m->get_code(); - if (!code) { - return; - } - - always_assert_log(code->editable_cfg_built(), "%s has no cfg built", SHOW(m)); - auto& cfg = code->cfg(); - type_inference::TypeInference inference(cfg, false, m_typedef_annos); - inference.run(m); - - TypeEnvironments& envs = inference.get_type_environments(); - std::vector> missing_param_annos; - for (cfg::Block* b : cfg.blocks()) { - for (auto& mie : InstructionIterable(b)) { - auto* insn = mie.insn; - IROpcode opcode = insn->opcode(); - if (!opcode::is_an_invoke(opcode)) { - continue; - } - collect_from_instruction(envs, inference, m, insn, missing_param_annos); - } - } - - // Patch missing param annotations - for (auto& pair : missing_param_annos) { - int param_index = pair.first; - always_assert(is_static(m)); - m->attach_param_annotation_set( - param_index, std::make_unique(pair.second)); - TRACE(TAC, 2, "Add param annotation %s at %d to %s", SHOW(&pair.second), - param_index, SHOW(m)); - } -} - void TypedefAnnoChecker::run(DexMethod* m) { IRCode* code = m->get_code(); if (!code) { @@ -160,7 +58,6 @@ void TypedefAnnoChecker::run(DexMethod* m) { inference.get_typedef_annotation(return_annos->get_annotations()); } TypeEnvironments& envs = inference.get_type_environments(); - TRACE(TAC, 2, "Start checking %s\n%s", SHOW(m), SHOW(cfg)); for (cfg::Block* b : cfg.blocks()) { for (auto& mie : InstructionIterable(b)) { auto* insn = mie.insn; @@ -170,7 +67,7 @@ void TypedefAnnoChecker::run(DexMethod* m) { } } if (!m_good) { - TRACE(TAC, 2, "Done checking %s", SHOW(m)); + TRACE(TAC, 1, "%s\n%s", SHOW(m), SHOW(cfg)); } } @@ -192,12 +89,26 @@ void TypedefAnnoChecker::check_instruction( case OPCODE_INVOKE_DIRECT: case OPCODE_INVOKE_STATIC: case OPCODE_INVOKE_INTERFACE: { - auto* callee_def = resolve_method(m, insn); - if (!callee_def || !callee_def->get_param_anno()) { - // Callee does not expect any Typedef value. Nothing to do. + auto def_method = + resolve_method(insn->get_method(), opcode_to_search(insn), m); + if (def_method == nullptr && opcode == OPCODE_INVOKE_VIRTUAL) { + def_method = + resolve_method(insn->get_method(), MethodSearch::InterfaceVirtual); + } + if (!def_method) { return; } - for (auto const& param_anno : *callee_def->get_param_anno()) { + // some methods are called through their access or defaul + // counterpart, which do not retain the typedef annotation. + // In these cases, we will skip the checker + if (!def_method->get_param_anno() || + ACCESS_PREFIX + def_method->get_simple_deobfuscated_name() == + m->get_simple_deobfuscated_name() || + def_method->get_simple_deobfuscated_name() + DEFAULT_SUFFIX == + m->get_simple_deobfuscated_name()) { + return; + } + for (auto const& param_anno : *def_method->get_param_anno()) { auto annotation = inference->get_typedef_annotation( param_anno.second->get_annotations()); if (annotation == boost::none) { @@ -207,15 +118,14 @@ void TypedefAnnoChecker::check_instruction( ? param_anno.first : param_anno.first + 1; reg_t reg = insn->src(param_index); - auto anno_type = env.get_annotation(reg); - auto type = env.get_dex_type(reg); + auto env_anno = env.get_annotation(reg); // TypeInference inferred a different annotation - if (anno_type && anno_type != annotation) { + if (env_anno != boost::none && env_anno != annotation) { std::ostringstream out; - if (anno_type.value() == type::java_lang_Object()) { - out << "TypedefAnnoCheckerPass: while invoking " << show(callee_def) - << "\n in method " << show(m) << "\n parameter " + if (env_anno.value() == DexType::make_type("Ljava/lang/Object;")) { + out << "TypedefAnnoCheckerPass: while invoking " << SHOW(def_method) + << "\n in method " << SHOW(m) << "\n parameter " << param_anno.first << "should have the annotation " << annotation.value()->get_name()->c_str() << "\n but it instead contains an ambiguous annotation, " @@ -223,37 +133,39 @@ void TypedefAnnoChecker::check_instruction( "typedef annotation \n before the method invokation. The " "ambiguous annotation is unsafe, and typedef annotations " "should not be mixed.\n" - << " failed instruction: " << show(insn) << "\n\n"; + << " failed instruction: " << SHOW(insn) << "\n\n"; } else { - out << "TypedefAnnoCheckerPass: while invoking " << show(callee_def) - << "\n in method " << show(m) << "\n parameter " - << param_anno.first << " has the annotation " << show(anno_type) + out << "TypedefAnnoCheckerPass: while invoking " << SHOW(def_method) + << "\n in method " << SHOW(m) << "\n parameter " + << param_anno.first << " has the annotation " << SHOW(env_anno) << "\n but the method expects the annotation to be " << annotation.value()->get_name()->c_str() - << ".\n failed instruction: " << show(insn) << "\n\n"; + << ".\n failed instruction: " << SHOW(insn) << "\n\n"; } m_error += out.str(); m_good = false; - } else if (is_not_str_nor_int(env, reg)) { + } else if (!is_int(&env, reg) && !is_string(&env, reg) && + env.get_type(insn->src(0)) != + type_inference::TypeDomain(IRType::INT)) { std::ostringstream out; - out << "TypedefAnnoCheckerPass: the annotation " << show(annotation) + out << "TypedefAnnoCheckerPass: the annotation " << SHOW(annotation) << "\n annotates a parameter with an incompatible type " - << show(type) << "\n or a non-constant parameter in method " - << show(m) << "\n while trying to invoke the method " - << show(callee_def) << ".\n failed instruction: " << show(insn) - << "\n\n"; + << SHOW(env.get_type(reg)) + << "\n or a non-constant parameter in method " << SHOW(m) + << "\n while trying to invoke the method " << SHOW(def_method) + << ".\n failed instruction: " << SHOW(insn) << "\n\n"; m_error += out.str(); m_good = false; - } else if (!anno_type) { + } else if (env_anno == boost::none) { // TypeInference didn't infer anything bool good = check_typedef_value(m, annotation, ud_chains, insn, param_index, inference, envs); if (!good) { std::ostringstream out; - out << " Error invoking " << show(callee_def) << "\n"; + out << " Error invoking " << SHOW(def_method) << "\n"; out << " Incorrect parameter's index: " << param_index << "\n\n"; m_error += out.str(); - TRACE(TAC, 1, "invoke method: %s", SHOW(callee_def)); + TRACE(TAC, 1, "invoke method: %s", SHOW(def_method)); } } } @@ -268,28 +180,28 @@ void TypedefAnnoChecker::check_instruction( if (env_anno != boost::none && field_anno != boost::none && env_anno.value() != field_anno.value()) { std::ostringstream out; - out << "TypedefAnnoCheckerPass: The method " << show(m) + out << "TypedefAnnoCheckerPass: The method " << SHOW(m) << "\n assigned a field " << insn->get_field()->c_str() - << "\n with annotation " << show(field_anno) - << "\n to a value with annotation " << show(env_anno) - << ".\n failed instruction: " << show(insn) << "\n\n"; + << "\n with annotation " << SHOW(field_anno) + << "\n to a value with annotation " << SHOW(env_anno) + << ".\n failed instruction: " << SHOW(insn) << "\n\n"; m_error += out.str(); m_good = false; } break; } - // if there's an annotation that has a string typedef or an int typedef - // annotation in the method's signature, check that TypeInference - // inferred that annotation in the retured value + // if there's an annotation that has a string typedef or an int typedef + // annotation in the method's signature, check that TypeInference + // inferred that annotation in the retured value case OPCODE_RETURN: case OPCODE_RETURN_OBJECT: { if (return_annotation) { reg_t reg = insn->src(0); - auto anno_type = env.get_annotation(reg); - if (anno_type && anno_type != return_annotation) { + auto env_anno = env.get_annotation(reg); + if (env_anno != boost::none && env_anno != return_annotation) { std::ostringstream out; - if (anno_type.value() == type::java_lang_Object()) { - out << "TypedefAnnoCheckerPass: The method " << show(m) + if (env_anno.value() == DexType::make_type("Ljava/lang/Object;")) { + out << "TypedefAnnoCheckerPass: The method " << SHOW(m) << "\n has an annotation " << return_annotation.value()->get_name()->c_str() << "\n in its method signature, but the returned value has an " @@ -297,29 +209,31 @@ void TypedefAnnoChecker::check_instruction( "with another typedef annotation within the method. The " "ambiguous annotation is unsafe, \nand typedef annotations " "should not be mixed. \n" - << "failed instruction: " << show(insn) << "\n\n"; + << "failed instruction: " << SHOW(insn) << "\n\n"; } else { - out << "TypedefAnnoCheckerPass: The method " << show(m) + out << "TypedefAnnoCheckerPass: The method " << SHOW(m) << "\n has an annotation " << return_annotation.value()->get_name()->c_str() << "\n in its method signature, but the returned value " "contains the annotation \n" - << show(anno_type) << " instead.\n" - << " failed instruction: " << show(insn) << "\n\n"; + << SHOW(env_anno) << " instead.\n" + << " failed instruction: " << SHOW(insn) << "\n\n"; } m_error += out.str(); m_good = false; - } else if (is_not_str_nor_int(env, reg)) { + } else if (!is_int(&env, reg) && !is_string(&env, reg) && + env.get_type(insn->src(0)) != + type_inference::TypeDomain(IRType::INT)) { std::ostringstream out; out << "TypedefAnnoCheckerPass: the annotation " - << show(return_annotation) + << SHOW(return_annotation) << "\n annotates a value with an incompatible type or a " "non-constant value in method\n " - << show(m) << " .\n" - << " failed instruction: " << show(insn) << "\n\n"; + << SHOW(m) << " .\n" + << " failed instruction: " << SHOW(insn) << "\n\n"; m_error += out.str(); m_good = false; - } else if (!anno_type) { + } else if (env_anno == boost::none) { bool good = check_typedef_value(m, return_annotation, ud_chains, insn, 0, inference, envs); if (!good) { @@ -346,35 +260,34 @@ bool TypedefAnnoChecker::check_typedef_value( TypeEnvironments& envs) { auto anno_class = type_class(annotation.value()); - const auto* str_value_set = m_strdef_constants.get_unsafe(anno_class); - const auto* int_value_set = m_intdef_constants.get_unsafe(anno_class); + const auto str_value_set = m_strdef_constants.find(anno_class); + const auto int_value_set = m_intdef_constants.find(anno_class); - bool has_str_vals = str_value_set != nullptr && !str_value_set->empty(); - bool has_int_vals = int_value_set != nullptr && !int_value_set->empty(); - always_assert_log(has_int_vals ^ has_str_vals, - "%s has both str and int const values", SHOW(anno_class)); - if (!has_str_vals && !has_int_vals) { + if (str_value_set != m_strdef_constants.end() && str_value_set->second.empty()) { + TRACE(TAC, 1, "%s contains no annotation constants", SHOW(anno_class)); + return true; + } + if (int_value_set != m_intdef_constants.end() && int_value_set->second.empty()) { TRACE(TAC, 1, "%s contains no annotation constants", SHOW(anno_class)); return true; } live_range::Use use_of_id{insn, src}; auto udchains_it = ud_chains->find(use_of_id); - auto defs_set = udchains_it->second; - for (IRInstruction* def : defs_set) { + for (IRInstruction* def : udchains_it->second) { switch (def->opcode()) { case OPCODE_CONST_STRING: { auto const const_value = def->get_string(); - if (str_value_set->count(const_value) == 0) { + if (str_value_set->second.count(const_value) == 0) { std::ostringstream out; - out << "TypedefAnnoCheckerPass: in method " << show(m) - << "\n the string value " << show(const_value) + out << "TypedefAnnoCheckerPass: in method " << SHOW(m) + << "\n the string value " << SHOW(const_value) << " does not have the typedef annotation \n" - << show(annotation) + << SHOW(annotation) << " attached to it. \n Check that the value is annotated and " "exists in the typedef annotation class.\n" - << " failed instruction: " << show(def) << "\n"; + << " failed instruction: " << SHOW(def) << "\n"; m_good = false; m_error += out.str(); return false; @@ -383,20 +296,15 @@ bool TypedefAnnoChecker::check_typedef_value( } case OPCODE_CONST: { auto const const_value = def->get_literal(); - if (has_str_vals && const_value == 0) { - // Null assigned to a StringDef value. This is valid. We don't enforce - // nullness. - break; - } - if (int_value_set->count(const_value) == 0) { + if (int_value_set->second.count(const_value) == 0) { std::ostringstream out; - out << "TypedefAnnoCheckerPass: in method " << show(m) - << "\n the int value " << show(const_value) + out << "TypedefAnnoCheckerPass: in method " << SHOW(m) + << "\n the int value " << SHOW(const_value) << " does not have the typedef annotation \n" - << show(annotation) + << SHOW(annotation) << " attached to it. \n Check that the value is annotated and " "exists in its typedef annotation class.\n" - << " failed instruction: " << show(def) << "\n"; + << " failed instruction: " << SHOW(def) << "\n"; m_good = false; m_error += out.str(); return false; @@ -411,13 +319,13 @@ bool TypedefAnnoChecker::check_typedef_value( auto env = envs.find(def); if (env->second.get_int_type(def->dest()).element() == (IntType::BOOLEAN)) { - if (int_value_set->count(0) == 0 || int_value_set->count(1) == 0) { + if (int_value_set->second.count(0) == 0 || int_value_set->second.count(1) == 0) { std::ostringstream out; - out << "TypedefAnnoCheckerPass: the method" << show(m) - << "\n assigns a int with typedef annotation " << show(annotation) + out << "TypedefAnnoCheckerPass: the method" << SHOW(m) + << "\n assigns a int with typedef annotation " << SHOW(annotation) << "\n to either 0 or 1, which is invalid because the typedef " "annotation class does not contain both the values 0 and 1.\n" - << " failed instruction: " << show(def) << "\n"; + << " failed instruction: " << SHOW(def) << "\n"; m_good = false; return false; } @@ -426,12 +334,12 @@ bool TypedefAnnoChecker::check_typedef_value( auto anno = env->second.get_annotation(def->dest()); if (anno == boost::none || anno != annotation) { std::ostringstream out; - out << "TypedefAnnoCheckerPass: in method " << show(m) + out << "TypedefAnnoCheckerPass: in method " << SHOW(m) << "\n one of the parameters needs to have the typedef annotation " - << show(annotation) + << SHOW(annotation) << "\n attached to it. Check that the value is annotated and " "exists in the typedef annotation class.\n" - << " failed instruction: " << show(def) << "\n"; + << " failed instruction: " << SHOW(def) << "\n"; m_good = false; m_error += out.str(); return false; @@ -443,14 +351,19 @@ bool TypedefAnnoChecker::check_typedef_value( case OPCODE_INVOKE_DIRECT: case OPCODE_INVOKE_STATIC: case OPCODE_INVOKE_INTERFACE: { - auto def_method = resolve_method(m, def); + auto def_method = + resolve_method(def->get_method(), opcode_to_search(def), m); + if (def_method == nullptr && def->opcode() == OPCODE_INVOKE_VIRTUAL) { + def_method = + resolve_method(def->get_method(), MethodSearch::InterfaceVirtual); + } if (!def_method) { std::ostringstream out; - out << "TypedefAnnoCheckerPass: in the method " << show(m) - << "\n the source of the value with annotation " << show(annotation) + out << "TypedefAnnoCheckerPass: in the method " << SHOW(m) + << "\n the source of the value with annotation " << SHOW(annotation) << "\n is produced by invoking an unresolveable callee, so the " "value safety is not guaranteed.\n" - << " failed instruction: " << show(def) << "\n"; + << " failed instruction: " << SHOW(def) << "\n"; m_good = false; m_error += out.str(); return false; @@ -460,11 +373,11 @@ bool TypedefAnnoChecker::check_typedef_value( if (anno == boost::none || anno != annotation) { std::ostringstream out; out << "TypedefAnnoCheckerPass: the method " - << show(def->get_method()->as_def()) + << SHOW(def->get_method()->as_def()) << "\n needs to return a value with the anotation " - << show(annotation) + << SHOW(annotation) << "\n and include it in it's method signature.\n" - << " failed instruction: " << show(def) << "\n"; + << " failed instruction: " << SHOW(def) << "\n"; m_good = false; m_error += out.str(); return false; @@ -478,41 +391,26 @@ bool TypedefAnnoChecker::check_typedef_value( // of 1. We essentially end up with // mNotificationsSharedPrefsHelper.get().getAppBadgeEnabledStatus() ? 0 : // 1 which gets optimized to an XOR by the compiler - if (int_value_set->count(0) == 0 || int_value_set->count(1) == 0) { + if (int_value_set->second.count(0) == 0 || int_value_set->second.count(1) == 0) { std::ostringstream out; - out << "TypedefAnnoCheckerPass: the method" << show(m) - << "\n assigns a int with typedef annotation " << show(annotation) + out << "TypedefAnnoCheckerPass: the method" << SHOW(m) + << "\n assigns a int with typedef annotation " << SHOW(annotation) << "\n to either 0 or 1, which is invalid because the typedef " "annotation class does not contain both the values 0 and 1.\n" - << " failed instruction: " << show(def) << "\n"; + << " failed instruction: " << SHOW(def) << "\n"; m_good = false; return false; } break; } - case OPCODE_IGET: - case OPCODE_IGET_OBJECT: { - auto field_anno = - inference->get_typedef_anno_from_member(def->get_field()); - if (!field_anno || field_anno != annotation) { - std::ostringstream out; - out << "TypedefAnnoCheckerPass: in method " << show(m) - << "\n the field " << def->get_field()->str() - << "\n needs to have the annotation " << show(annotation) - << ".\n failed instruction: " << show(def) << "\n"; - m_error += out.str(); - m_good = false; - } - break; - } default: { std::ostringstream out; - out << "TypedefAnnoCheckerPass: the method " << show(m) + out << "TypedefAnnoCheckerPass: the method " << SHOW(m) << "\n does not guarantee value safety for the value with typedef " "annotation " - << show(annotation) + << SHOW(annotation) << " .\n Check that this value does not change within the method\n" - << " failed instruction: " << show(def) << "\n"; + << " failed instruction: " << SHOW(def) << "\n"; m_good = false; m_error += out.str(); return false; @@ -528,16 +426,13 @@ void TypedefAnnoCheckerPass::run_pass(DexStoresVector& stores, assert(m_config.int_typedef != nullptr); assert(m_config.str_typedef != nullptr); auto scope = build_class_scope(stores); - StrDefConstants strdef_constants; - IntDefConstants intdef_constants; + ConcurrentMap> + strdef_constants; + ConcurrentMap> intdef_constants; walk::parallel::classes(scope, [&](DexClass* cls) { gather_typedef_values(cls, strdef_constants, intdef_constants); }); - SynthAccessorPatcher patcher(m_config); - patcher.run(scope); - TRACE(TAC, 2, "Finish patching synth accessors"); - auto stats = walk::parallel::methods(scope, [&](DexMethod* m) { TypedefAnnoChecker checker = TypedefAnnoChecker(strdef_constants, intdef_constants, m_config); @@ -573,8 +468,10 @@ void TypedefAnnoCheckerPass::run_pass(DexStoresVector& stores, void TypedefAnnoCheckerPass::gather_typedef_values( const DexClass* cls, - StrDefConstants& strdef_constants, - IntDefConstants& intdef_constants) { + ConcurrentMap>& + strdef_constants, + ConcurrentMap>& + intdef_constants) { const std::vector& fields = cls->get_sfields(); if (get_annotation(cls, m_config.str_typedef)) { std::unordered_set str_values; @@ -583,13 +480,13 @@ void TypedefAnnoCheckerPass::gather_typedef_values( static_cast(field->get_static_value()) ->string()); } - strdef_constants.emplace(cls, std::move(str_values)); + strdef_constants.insert(std::pair(cls, std::move(str_values))); } else if (get_annotation(cls, m_config.int_typedef)) { std::unordered_set int_values; for (auto* field : fields) { int_values.emplace(field->get_static_value()->value()); } - intdef_constants.emplace(cls, std::move(int_values)); + intdef_constants.insert(std::pair(cls, std::move(int_values))); } } diff --git a/opt/typedef-anno-checker/TypedefAnnoCheckerPass.h b/opt/typedef-anno-checker/TypedefAnnoCheckerPass.h index ce4ebe9698..c2f67cf805 100644 --- a/opt/typedef-anno-checker/TypedefAnnoCheckerPass.h +++ b/opt/typedef-anno-checker/TypedefAnnoCheckerPass.h @@ -49,10 +49,9 @@ class TypedefAnnoCheckerPass : public Pass { private: void gather_typedef_values( const DexClass* cls, - InsertOnlyConcurrentMap>& + ConcurrentMap>& strdef_constants, - InsertOnlyConcurrentMap>& + ConcurrentMap>& intdef_constants); Config m_config; @@ -77,33 +76,14 @@ struct Stats { } }; -using StrDefConstants = - InsertOnlyConcurrentMap>; - -using IntDefConstants = - InsertOnlyConcurrentMap>; - -class SynthAccessorPatcher { - public: - explicit SynthAccessorPatcher(const TypedefAnnoCheckerPass::Config& config) { - m_typedef_annos.insert(config.int_typedef); - m_typedef_annos.insert(config.str_typedef); - } - - void run(const Scope& scope); - - private: - void collect_accessors(DexMethod* method); - - std::unordered_set m_typedef_annos; -}; - class TypedefAnnoChecker { public: - explicit TypedefAnnoChecker(const StrDefConstants& strdef_constants, - const IntDefConstants& intdef_constants, - const TypedefAnnoCheckerPass::Config& config) + explicit TypedefAnnoChecker( + ConcurrentMap>& + strdef_constants, + ConcurrentMap>& + intdef_constants, + const TypedefAnnoCheckerPass::Config& config) : m_config(config), m_strdef_constants(strdef_constants), m_intdef_constants(intdef_constants) {} @@ -135,6 +115,8 @@ class TypedefAnnoChecker { std::string m_error; TypedefAnnoCheckerPass::Config m_config; - const StrDefConstants& m_strdef_constants; - const IntDefConstants& m_intdef_constants; + const ConcurrentMap>& + m_strdef_constants; + const ConcurrentMap>& + m_intdef_constants; }; diff --git a/opt/unmark_proguard_keep/UnmarkProguardKeep.h b/opt/unmark_proguard_keep/UnmarkProguardKeep.h index 6e678b120d..f38e028edf 100644 --- a/opt/unmark_proguard_keep/UnmarkProguardKeep.h +++ b/opt/unmark_proguard_keep/UnmarkProguardKeep.h @@ -28,6 +28,7 @@ class UnmarkProguardKeepPass : Pass { bind("package_list", {}, m_package_list); } + bool is_cfg_legacy() override { return true; } void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; private: diff --git a/opt/unreachable/UnreachableLoweringPass.cpp b/opt/unreachable/UnreachableLoweringPass.cpp index f881ab713c..82d62a430c 100644 --- a/opt/unreachable/UnreachableLoweringPass.cpp +++ b/opt/unreachable/UnreachableLoweringPass.cpp @@ -7,14 +7,12 @@ #include "UnreachableLoweringPass.h" -#include "CFGMutation.h" #include "ControlFlow.h" #include "Lazy.h" #include "LiveRange.h" #include "Show.h" #include "Trace.h" #include "Walkers.h" -#include namespace { @@ -24,98 +22,11 @@ constexpr const char* METRIC_UNREACHABLE_METHODS = "unreachable_methods"; constexpr const char* METRIC_REACHABLE_METHODS_WITH_UNREACHABLE_INSTRUCTIONS = "reachable_methods_with_unreachable_instructions"; -constexpr const char* UNREACHABLE_EXCEPTION_CLASS_NAME = - "Lcom/redex/UnreachableException;"; -constexpr const char* UNREACHABLE_EXCEPTION_CREATE_AND_THROW_METHOD_NAME = - "createAndThrow"; - } // namespace -void UnreachableLoweringPass::eval_pass(DexStoresVector& stores, - ConfigFiles&, - PassManager& mgr) { - always_assert(!stores.empty()); - auto& root_store = stores.front(); - auto& primary_dex = root_store.get_dexen().at(0); - auto cls_name = DexString::make_string(UNREACHABLE_EXCEPTION_CLASS_NAME); - auto type = DexType::make_type(cls_name); - ClassCreator cls_creator(type); - cls_creator.set_access(ACC_PUBLIC | ACC_FINAL); - cls_creator.set_super(type::java_lang_RuntimeException()); - auto cls = cls_creator.create(); - cls->rstate.set_generated(); - cls->rstate.set_root(); - cls->set_perf_sensitive(PerfSensitiveGroup::UNREACHABLE); - primary_dex.push_back(cls); - - DexMethod* init_method; - - { - MethodCreator method_creator( - cls->get_type(), - DexString::make_string(""), - DexProto::make_proto(type::_void(), DexTypeList::make_type_list({})), - ACC_PUBLIC | ACC_CONSTRUCTOR); - auto this_arg = method_creator.get_local(0); - auto string_var = method_creator.make_local(type::java_lang_String()); - method_creator.make_local(type::java_lang_RuntimeException()); - auto main_block = method_creator.get_main_block(); - - main_block->load_const( - string_var, - DexString::make_string( - "Redex: Unreachable code. This should never get triggered.")); - main_block->invoke( - OPCODE_INVOKE_DIRECT, - DexMethod::make_method( - type::java_lang_RuntimeException(), - DexString::make_string(""), - DexProto::make_proto( - type::_void(), - DexTypeList::make_type_list({type::java_lang_String()}))), - {this_arg, string_var}); - main_block->ret_void(); - init_method = method_creator.create(); - cls->add_method(init_method); - init_method->get_code()->build_cfg(/* editable */ true); - init_method->rstate.set_generated(); - init_method->set_deobfuscated_name(show_deobfuscated(init_method)); - } - - { - MethodCreator method_creator( - cls->get_type(), - DexString::make_string( - UNREACHABLE_EXCEPTION_CREATE_AND_THROW_METHOD_NAME), - DexProto::make_proto(type, DexTypeList::make_type_list({})), - ACC_STATIC | ACC_PUBLIC); - auto var = method_creator.make_local(type); - auto main_block = method_creator.get_main_block(); - main_block->new_instance(type, var); - main_block->invoke(init_method, {var}); - main_block->throwex(var); - m_create_and_throw_method = method_creator.create(); - cls->add_method(m_create_and_throw_method); - m_create_and_throw_method->get_code()->build_cfg(/* editable */ true); - m_create_and_throw_method->rstate.set_generated(); - m_create_and_throw_method->rstate.set_root(); - m_create_and_throw_method->set_deobfuscated_name( - show_deobfuscated(m_create_and_throw_method)); - } - - m_reserved_refs_handle = mgr.reserve_refs(name(), - ReserveRefsInfo(/* frefs */ 0, - /* trefs */ 1, - /* mrefs */ 1)); -} - void UnreachableLoweringPass::run_pass(DexStoresVector& stores, ConfigFiles&, PassManager& mgr) { - always_assert(m_reserved_refs_handle); - mgr.release_reserved_refs(*m_reserved_refs_handle); - m_reserved_refs_handle = std::nullopt; - const auto scope = build_class_scope(stores); std::atomic unreachable_instructions{0}; std::atomic unreachable_methods{0}; @@ -128,20 +39,14 @@ void UnreachableLoweringPass::run_pass(DexStoresVector& stores, } size_t local_unreachable_instructions{0}; Lazy duchains([&cfg]() { - live_range::MoveAwareChains chains( - cfg, /* ignore_unreachable */ false, - [&](auto* insn) { return opcode::is_unreachable(insn->opcode()); }); + live_range::MoveAwareChains chains(cfg); return chains.get_def_use_chains(); }); - auto ii = InstructionIterable(cfg); - std::unique_ptr mutation; - for (auto it = ii.begin(); it != ii.end(); ++it) { - auto& mie = *it; + for (auto& mie : InstructionIterable(cfg)) { if (!opcode::is_unreachable(mie.insn->opcode())) { continue; } local_unreachable_instructions++; - // We want to enforce that the (dummy) value produced by the "unreachable" // instruction is only used by a "throw" instruction. // TODO: In practice, the InstrumentPass might also squeeze in an @@ -162,28 +67,14 @@ void UnreachableLoweringPass::run_pass(DexStoresVector& stores, // TODO: Consider other transformations, e.g. just return if there are no // monitor instructions, or embed a descriptive message. - if (!mutation) { - mutation = std::make_unique(cfg); - } - mutation->replace(it, - { - (new IRInstruction(OPCODE_INVOKE_STATIC)) - ->set_method(m_create_and_throw_method), - (new IRInstruction(OPCODE_MOVE_RESULT_OBJECT)) - ->set_dest(mie.insn->dest()), - }); + mie.insn->set_opcode(OPCODE_CONST); + mie.insn->set_literal(0); } - - if (mutation) { - always_assert(local_unreachable_instructions > 0); - mutation->flush(); - cfg.remove_unreachable_blocks(); + if (local_unreachable_instructions > 0) { unreachable_instructions += local_unreachable_instructions; if (!is_unreachable_method) { reachable_methods_with_unreachable_instructions++; } - } else { - always_assert(local_unreachable_instructions == 0); } }); mgr.incr_metric(METRIC_UNREACHABLE_INSTRUCTIONS, diff --git a/opt/unreachable/UnreachableLoweringPass.h b/opt/unreachable/UnreachableLoweringPass.h index 9e87aa4da4..ab1ede50a2 100644 --- a/opt/unreachable/UnreachableLoweringPass.h +++ b/opt/unreachable/UnreachableLoweringPass.h @@ -20,17 +20,12 @@ class UnreachableLoweringPass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoInitClassInstructions, Preserves}, {NoUnreachableInstructions, Establishes}, {RenameClass, Preserves}, }; } - void eval_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; - void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; - - private: - std::optional m_reserved_refs_handle; - DexMethod* m_create_and_throw_method{nullptr}; }; diff --git a/opt/unreferenced_interfaces/UnreferencedInterfaces.h b/opt/unreferenced_interfaces/UnreferencedInterfaces.h index d922eb03be..db2faf628a 100644 --- a/opt/unreferenced_interfaces/UnreferencedInterfaces.h +++ b/opt/unreferenced_interfaces/UnreferencedInterfaces.h @@ -25,10 +25,13 @@ class UnreferencedInterfacesPass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } + bool is_cfg_legacy() override { return true; } void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; struct Metric { diff --git a/opt/up-code-motion/UpCodeMotion.cpp b/opt/up-code-motion/UpCodeMotion.cpp index e4fe49c95c..2ccb3ff0ff 100644 --- a/opt/up-code-motion/UpCodeMotion.cpp +++ b/opt/up-code-motion/UpCodeMotion.cpp @@ -231,7 +231,8 @@ UpCodeMotionPass::Stats UpCodeMotionPass::process_code( IRCode* code, bool is_branch_hot_check) { Stats stats; - always_assert(code->editable_cfg_built()); + + code->build_cfg(/* editable = true*/); auto& cfg = code->cfg(); std::unique_ptr type_inference; std::unordered_set blocks_to_remove_set; @@ -361,6 +362,8 @@ UpCodeMotionPass::Stats UpCodeMotionPass::process_code( } cfg.remove_blocks(blocks_to_remove); + + code->clear_cfg(); return stats; } @@ -371,7 +374,7 @@ void UpCodeMotionPass::run_pass(DexStoresVector& stores, Stats stats = walk::parallel::methods(scope, [&](DexMethod* method) { const auto code = method->get_code(); - if (!code || method->rstate.no_optimizations()) { + if (!code) { return Stats{}; } diff --git a/opt/up-code-motion/UpCodeMotion.h b/opt/up-code-motion/UpCodeMotion.h index 6f59bc5b76..6d4eae613a 100644 --- a/opt/up-code-motion/UpCodeMotion.h +++ b/opt/up-code-motion/UpCodeMotion.h @@ -40,10 +40,11 @@ class UpCodeMotionPass : public Pass { {DexLimitsObeyed, Preserves}, {HasSourceBlocks, RequiresAndEstablishes}, {NoResolvablePureRefs, Preserves}, - {InitialRenameClass, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } + bool is_cfg_legacy() override { return true; } void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; void bind_config() override { bind("check_branch_hotness", diff --git a/opt/uses-names/UsesNames.h b/opt/uses-names/UsesNames.h index 0ee7b650f9..b12a154ef1 100644 --- a/opt/uses-names/UsesNames.h +++ b/opt/uses-names/UsesNames.h @@ -28,7 +28,9 @@ class ProcessUsesNamesAnnoPass : public Pass { using namespace redex_properties::interactions; using namespace redex_properties::names; return { + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } @@ -42,6 +44,7 @@ class ProcessUsesNamesAnnoPass : public Pass { m_uses_names_trans_annotation); } + bool is_cfg_legacy() override { return true; } void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; private: diff --git a/opt/verifier/Verifier.h b/opt/verifier/Verifier.h index adddf5728f..a5195647ae 100644 --- a/opt/verifier/Verifier.h +++ b/opt/verifier/Verifier.h @@ -19,9 +19,12 @@ class VerifierPass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, {UltralightCodePatterns, Preserves}, }; } + bool is_cfg_legacy() override { return true; } void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; }; diff --git a/opt/vertical_merging/VerticalMerging.cpp b/opt/vertical_merging/VerticalMerging.cpp index cbdaf2a961..0d3e2d54ff 100644 --- a/opt/vertical_merging/VerticalMerging.cpp +++ b/opt/vertical_merging/VerticalMerging.cpp @@ -664,7 +664,7 @@ void remove_merged(Scope& scope, const ClassMap& mergeable_to_merger) { void resolve_virtual_calls_to_merger(const Scope& scope, ClassMap& mergeable_to_merger) { ConcurrentSet excluded_mergeables; - InsertOnlyConcurrentMap resolved_virtual_calls; + ConcurrentMap resolved_virtual_calls; walk::parallel::code(scope, [&](DexMethod* /* method */, IRCode& code) { editable_cfg_adapter::iterate(&code, [&](MethodItemEntry& mie) { auto insn = mie.insn; @@ -686,7 +686,7 @@ void resolve_virtual_calls_to_merger(const Scope& scope, // merger class is not checked because the case is excluded earlier // in collect_can_merge. if (merger_method_ref && is_internal_def(merger_method_ref)) { - resolved_virtual_calls.emplace(insn, merger_method_ref); + resolved_virtual_calls.insert({insn, merger_method_ref}); } } else { // Merger is the superclass. if (resolve_virtual(find_merger->second, @@ -696,7 +696,7 @@ void resolve_virtual_calls_to_merger(const Scope& scope, DexMethod::make_method(find_merger->second->get_type(), mergeable_method_ref->get_name(), mergeable_method_ref->get_proto()); - resolved_virtual_calls.emplace(insn, merger_method_ref); + resolved_virtual_calls.insert({insn, merger_method_ref}); } else { // There is no instance of the mergeable class. So virtual calls // on the mergeable class should be invalid or unreachable. To diff --git a/opt/vertical_merging/VerticalMerging.h b/opt/vertical_merging/VerticalMerging.h index 215efa815a..f163f058ac 100644 --- a/opt/vertical_merging/VerticalMerging.h +++ b/opt/vertical_merging/VerticalMerging.h @@ -30,7 +30,9 @@ class VerticalMergingPass : public Pass { using namespace redex_properties::interactions; using namespace redex_properties::names; return { + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, {NeedsEverythingPublic, Establishes}, // TT150850158 }; } diff --git a/opt/virtual_merging/VirtualMerging.cpp b/opt/virtual_merging/VirtualMerging.cpp index 2c607f323d..114d2d100e 100644 --- a/opt/virtual_merging/VirtualMerging.cpp +++ b/opt/virtual_merging/VirtualMerging.cpp @@ -654,20 +654,19 @@ VirtualMerging::compute_mergeable_pairs_by_virtual_scopes( const method_profiles::MethodProfiles& profiles, Strategy strategy, VirtualMergingStats& stats) const { - InsertOnlyConcurrentMap local_stats; + ConcurrentMap local_stats; std::vector virtual_scopes; for (auto& p : m_mergeable_scope_methods) { virtual_scopes.push_back(p.first); } - InsertOnlyConcurrentMap< - const VirtualScope*, - std::vector>> + ConcurrentMap>> mergeable_pairs_by_virtual_scopes; SimpleOrderingProvider ordering_provider{profiles}; walk::parallel::virtual_scopes( virtual_scopes, [&](const virt_scope::VirtualScope* virtual_scope) { MergePairsBuilder mpb(virtual_scope, ordering_provider, m_perf_config); - auto res = mpb.build(m_mergeable_scope_methods.at_unsafe(virtual_scope), + auto res = mpb.build(m_mergeable_scope_methods.at(virtual_scope), m_xstores, m_xdexes, profiles, strategy); if (!res) { return; @@ -1375,8 +1374,6 @@ void VirtualMerging::merge_methods( // Part 5: Remove methods within classes. void VirtualMerging::remove_methods() { std::vector classes_with_virtual_methods_to_remove; - classes_with_virtual_methods_to_remove.reserve( - m_virtual_methods_to_remove.size()); for (auto& p : m_virtual_methods_to_remove) { classes_with_virtual_methods_to_remove.push_back(p.first); } diff --git a/opt/virtual_merging/VirtualMerging.h b/opt/virtual_merging/VirtualMerging.h index 898f2afc62..289cd88361 100644 --- a/opt/virtual_merging/VirtualMerging.h +++ b/opt/virtual_merging/VirtualMerging.h @@ -186,7 +186,7 @@ class VirtualMergingPass : public Pass { {DexLimitsObeyed, Preserves}, {HasSourceBlocks, RequiresAndEstablishes}, {NoResolvablePureRefs, Preserves}, - {InitialRenameClass, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } diff --git a/opt/virtual_scope/MethodDevirtualizationPass.h b/opt/virtual_scope/MethodDevirtualizationPass.h index 7d2b69bb3f..82e31c2201 100644 --- a/opt/virtual_scope/MethodDevirtualizationPass.h +++ b/opt/virtual_scope/MethodDevirtualizationPass.h @@ -19,7 +19,9 @@ class MethodDevirtualizationPass : public Pass { using namespace redex_properties::names; return { {DexLimitsObeyed, Preserves}, + {HasSourceBlocks, Preserves}, {NoResolvablePureRefs, Preserves}, + {NoSpuriousGetClassCalls, Preserves}, }; } @@ -39,6 +41,7 @@ class MethodDevirtualizationPass : public Pass { bind("ignore_keep", false, m_ignore_keep); } + bool is_cfg_legacy() override { return true; } void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; private: diff --git a/proto/config.proto b/proto/config.proto index 7a86c42597..05d9917b5f 100644 --- a/proto/config.proto +++ b/proto/config.proto @@ -37,9 +37,6 @@ message BundleConfig { ASSET_ONLY = 2; } BundleType type = 8; - - // Configuration for locales. - Locales locales = 9; } message Bundletool { @@ -111,7 +108,7 @@ message Optimizations { // This is for uncompressing native libraries on M+ devices (L+ devices on // instant apps). UncompressNativeLibraries uncompress_native_libraries = 2; - // This is for uncompressing dex files. + // This is for uncompressing dex files on P+ devices. UncompressDexFiles uncompress_dex_files = 3; // Configuration for the generation of standalone APKs. // If no StandaloneConfig is set, the configuration is inherited from @@ -126,123 +123,34 @@ message Optimizations { } message ResourceOptimizations { - enum SparseEncoding { - // Previously 'ENFORCED'. This option is deprecated because of issues found - // in Android O up to Android Sv2 and causes segfaults in - // Resources#getIdentifier. - reserved 1; - reserved "ENFORCED"; - - // Disables sparse encoding. - UNSPECIFIED = 0; - // Generates special APKs for Android SDK +32 with sparse resource tables. - // Devices with Android SDK below 32 will still receive APKs with regular - // resource tables. - VARIANT_FOR_SDK_32 = 2; - } - - // Pair of resource type and name, like 'layout/foo', 'string/bar'. - message ResourceTypeAndName { - string type = 1; - string name = 2; - } - - // Optimizations related to collapsed resource names. - message CollapsedResourceNames { - // Whether to collapse resource names. - // Resources with collapsed resource names are only accessible by their - // ids. These names are not stored inside 'resources.arsc'. - bool collapse_resource_names = 1; - - // Instructs to not collapse resource names for specific resources which - // makes certain resources to be accessible by their names. - // - // Applicable only if 'collapse_resource_names' is 'true'. - repeated ResourceTypeAndName no_collapse_resources = 2; - - // Instructs to not collapse all resources of certain types. - // - // Applicable only if 'collapse_resource_names' is 'true'. - repeated string no_collapse_resource_types = 4; - - // Whether to store only unique resource entries in 'resources.arsc'. - // - // For example if there are 3 'bool' resources defined with 'true' value - // with this flag only one 'true' entry is stored and all 3 resources - // are referencing this entry. By default all 3 entries are stored. - // - // This only works with resources where names are collapsed (either using - // 'collapse_resource_names' flag or manually) because resource name is a - // part of resource entry and if names are preserved - all entries are - // unique. - bool deduplicate_resource_entries = 3; - } - // Whether to use sparse encoding for resource tables. // Resources in sparse resource table are accessed using a binary search tree. // This decreases APK size at the cost of resource retrieval performance. SparseEncoding sparse_encoding = 1; - // Optimizations related to collapsed resource names. - CollapsedResourceNames collapsed_resource_names = 2; -} - -message UncompressNativeLibraries { - bool enabled = 1; - - enum PageAlignment { - PAGE_ALIGNMENT_UNSPECIFIED = 0; - PAGE_ALIGNMENT_4K = 1; - PAGE_ALIGNMENT_16K = 2; - PAGE_ALIGNMENT_64K = 3; + enum SparseEncoding { + // Disables sparse encoding. + UNSPECIFIED = 0; + // Forces sparse resourse tables in all produced variants. This mode is + // available only for applications with minSdk >= 26 because sparse + // resource table support was added in Android O. + ENFORCED = 1; } - - // This is an experimental setting. It's behavior might be changed or - // completely removed. - // - // Alignment used for uncompressed native libraries inside APKs generated - // by bundletool. - // - // PAGE_ALIGNMENT_4K by default. - PageAlignment alignment = 2; } -message UncompressDexFiles { - // A new variant with uncompressed dex will be generated. The sdk targeting - // of the variant is determined by 'uncompressed_dex_target_sdk'. - bool enabled = 1; +message UncompressNativeLibraries { bool enabled = 1; } - // If 'enabled' field is set, this will determine the sdk targeting of the - // generated variant. - UncompressedDexTargetSdk uncompressed_dex_target_sdk = 2; - - enum UncompressedDexTargetSdk { - // Q+ variant will be generated. - UNSPECIFIED = 0; - // S+ variant will be generated. - SDK_31 = 1; - } -} +message UncompressDexFiles { bool enabled = 1; } message StoreArchive { // Archive is an app state that allows an official app store to reclaim device // storage and disable app functionality temporarily until the user interacts // with the app again. Upon interaction the latest available version of the - // app will be restored while leaving user data unaffected. - // Enabled by default. + // app will be restored while leaving user data unaffected. Disabled by + // default. bool enabled = 1; } -message Locales { - // Instructs bundletool to generate locale config and inject it into - // AndroidManifest.xml. A locale is marked as supported by the application if - // there is at least one resource value in this locale. Be very careful with - // this setting because if some of your libraries expose resources in some - // locales which are not actually supported by your application it will mark - // this locale as supported. Disabled by default. - bool inject_locale_config = 1; -} - // Optimization configuration used to generate Split APKs. message SplitsConfig { repeated SplitDimension split_dimension = 1; } @@ -269,19 +177,6 @@ message StandaloneConfig { // application developers. NEVER_MERGE = 1; } - - // Defines how to deal with feature modules in standalone variants (minSdk < - // 21). - FeatureModulesMode feature_modules_mode = 4; - - enum FeatureModulesMode { - // Default mode which fuses feature modules with respect to its - // fusing attribute into base.apk. - FUSED_FEATURE_MODULES = 0; - // Advanced mode, which allows to generate a single separate apk per each - // feature module in variants with minSdk < 21. - SEPARATE_FEATURE_MODULES = 1; - } } message SplitDimension { @@ -292,7 +187,6 @@ message SplitDimension { LANGUAGE = 3; TEXTURE_COMPRESSION_FORMAT = 4; DEVICE_TIER = 6; - COUNTRY_SET = 7; } Value value = 1; @@ -334,15 +228,8 @@ message SuffixStripping { message ApexConfig { // Configuration for processing of APKs embedded in an APEX image. repeated ApexEmbeddedApkConfig apex_embedded_apk_config = 1; - - // Explicit list of supported ABIs. - // Default: See ApexBundleValidator.REQUIRED_ONE_OF_ABI_SETS - repeated SupportedAbiSet supported_abi_set = 2; } -// Represents a set of ABIs which must be supported by a single APEX image. -message SupportedAbiSet { repeated string abi = 1; } - message ApexEmbeddedApkConfig { // Android package name of the APK. string package_name = 1; diff --git a/pyredex/packer.py b/pyredex/packer.py index 3ccad899e8..cfccad5fa7 100644 --- a/pyredex/packer.py +++ b/pyredex/packer.py @@ -25,7 +25,6 @@ from contextlib import contextmanager from enum import Enum -from pyredex.utils import get_xz_path timer: typing.Callable[[], float] = timeit.default_timer @@ -200,13 +199,11 @@ def _compress_xz( from_file: str, to_file: str, compression_level: CompressionLevel ) -> None: comp = _get_xz_compress_level(compression_level) - xz = get_xz_path() - if xz is not None: - cmd = [xz, "-z", comp[0], "-T10"] - with open(from_file, "rb") as fin: - with open(to_file, "wb") as fout: - subprocess.check_call(cmd, stdin=fin, stdout=fout) - return + if shutil.which("xz"): + logging.debug("Using command line xz") + cmd = f'cat "{from_file}" | xz -z {comp[0]} -T10 - > "{to_file}"' + subprocess.check_call(cmd, shell=True) # noqa: P204 + return with lzma.open(filename=to_file, mode="wb", preset=comp[1]) as xz: with open(from_file, "rb") as f_in: diff --git a/pyredex/unpacker.py b/pyredex/unpacker.py index 8f32044b3b..b12825da50 100644 --- a/pyredex/unpacker.py +++ b/pyredex/unpacker.py @@ -30,7 +30,6 @@ from pyredex.utils import ( abs_glob, ensure_libs_dir, - get_xz_path, make_temp_dir, remove_signature_files, ) @@ -584,13 +583,11 @@ def _warn_xz() -> None: def unpack_xz(input: str, output: str) -> None: - xz = get_xz_path() - if xz is not None: - with open(input, "rb") as fin: - with open(output, "wb") as fout: - cmd = [xz, "-d", "--threads", "6"] - subprocess.check_call(cmd, stdin=fin, stdout=fout) - return + # See whether the `xz` binary exists. It may be faster because of multithreaded decoding. + if shutil.which("xz"): + cmd = 'cat "{}" | xz -d --threads 6 > "{}"'.format(input, output) + subprocess.check_call(cmd, shell=True) # noqa: P204 + return _warn_xz() @@ -612,8 +609,7 @@ def pack_xz( check: int = lzma.CHECK_CRC32, ) -> None: # See whether the `xz` binary exists. It may be faster because of multithreaded encoding. - xz = get_xz_path() - if xz is not None: + if shutil.which("xz"): check_map = { lzma.CHECK_CRC32: "crc32", lzma.CHECK_CRC64: "crc64", @@ -622,21 +618,14 @@ def pack_xz( None: None, } check_str = check_map[check] - with open(input, "rb") as fin: - with open(output, "wb") as fout: - cmd = [ - xz, - f"-z{compression_level}", - f"--threads={threads}", - "-c", - f"--check={check_str}" if check_str else "", - ] - subprocess.check_call( - cmd, - stdin=fin, - stdout=fout, - ) - return + + subprocess.check_call( # noqa(P204) + f"xz -z{compression_level} --threads={threads} -c" + + (f" --check={check_str}" if check_str else "") + + f" {input} > {output}", + shell=True, + ) + return _warn_xz() assert isinstance(compression_level, int) @@ -659,20 +648,10 @@ def pack_xz( def unpack_tar_xz(input: str, output_dir: str) -> None: # See whether the `xz` binary exists. It may be faster because of multithreaded decoding. - if shutil.which("tar"): - xz = get_xz_path() - - if xz is not None: - cmd = [ - "tar", - "xf", - input, - "-C", - output_dir, - f"--use-compress-program={xz}", - ] - subprocess.check_call(cmd) - return + if shutil.which("xz") and shutil.which("tar"): + cmd = f'XZ_OPT=-T6 tar xf "{input}" -C "{output_dir}"' + subprocess.check_call(cmd, shell=True) # noqa: P204 + return _warn_xz() diff --git a/pyredex/utils.py b/pyredex/utils.py index d94c28c993..e74687260d 100644 --- a/pyredex/utils.py +++ b/pyredex/utils.py @@ -517,23 +517,18 @@ def get_file_ext(file_name: str) -> str: def _verify_dex(dex_file: str, cmd: str) -> bool: - try: - logging.info("Verifying %s...", dex_file) - - res = subprocess.run( - f"{cmd} '{dex_file}'", shell=True, text=False, capture_output=True - ) - if res.returncode == 0: - return True + logging.debug("Verifying %s...", dex_file) - try: - stderr_str = res.stderr.decode("utf-8") - except BaseException: - stderr_str = "" + res = subprocess.run( + f"{cmd} '{dex_file}'", shell=True, text=False, capture_output=True + ) + if res.returncode == 0: + return True - except BaseException as e: - logging.exception("Error for dex file %s", dex_file) - stderr_str = f"{e}" + try: + stderr_str = res.stderr.decode("utf-8") + except BaseException: + stderr_str = "" logging.error("Failed verification for %s:\n%s", dex_file, stderr_str) return False @@ -562,26 +557,6 @@ def verify_dexes(dex_dir: str, cmd: str) -> None: logging.debug("Dex verification finished in {:.2f} seconds".format(end - start)) -try: - # This is an xz that is built from source and provided via BUCK. - from xz_for_python.xz import get_xz_bin_path -except ImportError: - get_xz_bin_path = None - - -def get_xz_path() -> typing.Optional[str]: - xz = None - if get_xz_bin_path is not None: - xz_bin_path = get_xz_bin_path() - assert xz_bin_path is not None - logging.debug("Using provided xz") - xz = xz_bin_path - elif shutil.which("xz"): - logging.debug("Using command line xz") - xz = "xz" - return xz - - _TIME_IT_DEPTH: int = 0 diff --git a/redex.py b/redex.py index 9c782bbd98..e623583c47 100755 --- a/redex.py +++ b/redex.py @@ -947,6 +947,7 @@ def _check_shrinker_heuristics(args: argparse.Namespace) -> None: # Nothing found, check whether we have files embedded logging.info("No shrinking heuristic found, searching for default.") try: + # pyre-ignore[21] from generated_shrinker_regalloc_heuristics import SHRINKER_HEURISTICS_FILE logging.info("Found embedded shrinker heuristics") @@ -971,6 +972,7 @@ def _check_android_sdk_api(args: argparse.Namespace) -> None: # Nothing found, check whether we have files embedded logging.info("No android_sdk_api_XX_file parameters found.") try: + # pyre-ignore[21] import generated_apilevels as ga levels = ga.get_api_levels() @@ -1331,7 +1333,6 @@ def get_compression_list() -> typing.List[CompressionEntry]: def finalize_redex(state: State) -> None: if state.args.verify_dexes: - # with BuckPartScope("Redex::VerifyDexes", "Verifying output dex files"): verify_dexes(state.dex_dir, state.args.verify_dexes) if state.dexen_initial_state is not None: diff --git a/service/class-merging/ClassAssemblingUtils.cpp b/service/class-merging/ClassAssemblingUtils.cpp index ce72487091..110329a23d 100644 --- a/service/class-merging/ClassAssemblingUtils.cpp +++ b/service/class-merging/ClassAssemblingUtils.cpp @@ -83,20 +83,20 @@ DexClass* create_class(const DexType* type, // Create ctor. auto super_ctors = type_class(super_type)->get_ctors(); for (auto super_ctor : super_ctors) { - auto mc = MethodCreator(t, - DexString::make_string(""), - super_ctor->get_proto(), - ACC_PUBLIC | ACC_CONSTRUCTOR); + auto mc = new MethodCreator(t, + DexString::make_string(""), + super_ctor->get_proto(), + ACC_PUBLIC | ACC_CONSTRUCTOR); // Call to super. std::vector args; size_t args_size = super_ctor->get_proto()->get_args()->size(); for (size_t arg_loc = 0; arg_loc < args_size + 1; ++arg_loc) { - args.push_back(mc.get_local(arg_loc)); + args.push_back(mc->get_local(arg_loc)); } - auto mb = mc.get_main_block(); + auto mb = mc->get_main_block(); mb->invoke(OPCODE_INVOKE_DIRECT, super_ctor, args); mb->ret_void(); - auto ctor = mc.create(); + auto ctor = mc->create(); ctor->get_code()->build_cfg(); TRACE(CLMG, 4, " default ctor created %s", SHOW(ctor)); cls->add_method(ctor); diff --git a/service/class-merging/ClassMerging.cpp b/service/class-merging/ClassMerging.cpp index ecdfecfdec..1f2701c9c0 100644 --- a/service/class-merging/ClassMerging.cpp +++ b/service/class-merging/ClassMerging.cpp @@ -135,12 +135,9 @@ ModelStats merge_model(const TypeSystem& type_system, auto model = Model::build_model(scope, stores, conf, spec, type_system, *refchecker); ModelStats stats = model.get_model_stats(); - bool update_method_profiles_stats; - conf.get_json_config().get( - "update_method_profiles_stats", false, update_method_profiles_stats); + ModelMerger mm; - auto merger_classes = - mm.merge_model(scope, stores, conf, model, update_method_profiles_stats); + auto merger_classes = mm.merge_model(scope, stores, conf, model); auto num_dedupped = method_dedup::dedup_constructors(merger_classes, scope); mm.increase_ctor_dedupped_stats(num_dedupped); stats += mm.get_model_stats(); diff --git a/service/class-merging/Model.cpp b/service/class-merging/Model.cpp index 9324b38a96..399ae28dc8 100644 --- a/service/class-merging/Model.cpp +++ b/service/class-merging/Model.cpp @@ -1296,7 +1296,6 @@ ModelStats& ModelStats::operator+=(const ModelStats& stats) { m_num_static_non_virt_dedupped += stats.m_num_static_non_virt_dedupped; m_num_vmethods_dedupped += stats.m_num_vmethods_dedupped; m_num_const_lifted_methods += stats.m_num_const_lifted_methods; - m_updated_profile_method += stats.m_updated_profile_method; return *this; } @@ -1333,7 +1332,6 @@ void ModelStats::update_redex_stats(const std::string& prefix, mgr.incr_metric(prefix + "_static_non_virt_dedupped", m_num_static_non_virt_dedupped); mgr.incr_metric(prefix + "_vmethods_dedupped", m_num_vmethods_dedupped); - mgr.incr_metric(prefix + "_updated_profile_method", m_updated_profile_method); mgr.set_metric(prefix + "_const_lifted_methods", m_num_const_lifted_methods); } diff --git a/service/class-merging/Model.h b/service/class-merging/Model.h index 3766cc402f..5c1b91947b 100644 --- a/service/class-merging/Model.h +++ b/service/class-merging/Model.h @@ -217,7 +217,6 @@ struct ModelStats { uint32_t m_num_static_non_virt_dedupped = 0; uint32_t m_num_vmethods_dedupped = 0; uint32_t m_num_const_lifted_methods = 0; - uint32_t m_updated_profile_method = 0; ModelStats& operator+=(const ModelStats& stats); @@ -249,7 +248,6 @@ class Model { const std::string& get_name() const { return m_spec.name; } std::vector get_roots() const { std::vector res; - res.reserve(m_roots.size()); for (const auto root_merger : m_roots) { res.push_back(root_merger->type); } diff --git a/service/class-merging/ModelMerger.cpp b/service/class-merging/ModelMerger.cpp index 80c8288987..0f6a3ba5a4 100644 --- a/service/class-merging/ModelMerger.cpp +++ b/service/class-merging/ModelMerger.cpp @@ -135,25 +135,27 @@ void update_code_type_refs( if (!type_reference::proto_has_reference_to(proto, mergeables)) { continue; } - bool resolved_virtual_to_interface; const auto meth_def = - resolve_invoke_method(insn, meth, &resolved_virtual_to_interface); + resolve_method(meth_ref, opcode_to_search(insn), meth); // This is a very tricky case where ResolveRefs cannot resolve a // MethodRef to MethodDef. It is a invoke-virtual with a MethodRef // referencing an interface method implmentation defined in a subclass // of the referenced type. To resolve the actual def we need to go // through another interface method search. Maybe we should fix it in // ResolveRefs. - always_assert_log(resolved_virtual_to_interface, - "Found mergeable referencing MethodRef %s\n", - SHOW(meth_ref)); - always_assert(insn->opcode() == OPCODE_INVOKE_VIRTUAL); - auto new_proto = - type_reference::get_new_proto(proto, mergeable_to_merger); - DexMethodSpec spec; - spec.proto = new_proto; - meth_ref->change(spec, true /*rename on collision*/); - continue; + if (meth_def == nullptr) { + auto intf_def = + resolve_method(meth_ref, MethodSearch::InterfaceVirtual); + always_assert(insn->opcode() == OPCODE_INVOKE_VIRTUAL && intf_def); + auto new_proto = + type_reference::get_new_proto(proto, mergeable_to_merger); + DexMethodSpec spec; + spec.proto = new_proto; + meth_ref->change(spec, true /*rename on collision*/); + continue; + } + not_reached_log("Found mergeable referencing MethodRef %s\n", + SHOW(meth_ref)); } //////////////////////////////////////// // Update simple type refs @@ -245,23 +247,23 @@ DexMethod* create_instanceof_method(const DexType* merger_type, DexTypeList::make_type_list({type::java_lang_Object(), type::_int()}); auto proto = DexProto::make_proto(type::_boolean(), arg_list); auto access = ACC_PUBLIC | ACC_STATIC; - auto mc = MethodCreator(const_cast(merger_type), - DexString::make_string(INSTANCE_OF_STUB_NAME), - proto, - access); - auto obj_loc = mc.get_local(0); - auto type_tag_loc = mc.get_local(1); + auto mc = new MethodCreator(const_cast(merger_type), + DexString::make_string(INSTANCE_OF_STUB_NAME), + proto, + access); + auto obj_loc = mc->get_local(0); + auto type_tag_loc = mc->get_local(1); // first type check result loc. - auto check_res_loc = mc.make_local(type::_boolean()); - auto mb = mc.get_main_block(); + auto check_res_loc = mc->make_local(type::_boolean()); + auto mb = mc->get_main_block(); mb->instance_of(obj_loc, check_res_loc, const_cast(merger_type)); // ret slot. - auto ret_loc = mc.make_local(type::_boolean()); + auto ret_loc = mc->make_local(type::_boolean()); // first check and branch off. Zero means fail. auto instance_of_block = mb->if_testz(OPCODE_IF_EQZ, check_res_loc); // Fall through. Check succeed. - auto itype_tag_loc = mc.make_local(type::_int()); + auto itype_tag_loc = mc->make_local(type::_int()); // CHECK_CAST obj to merger type. instance_of_block->check_cast(obj_loc, const_cast(merger_type)); instance_of_block->iget(type_tag_field, obj_loc, itype_tag_loc); @@ -275,7 +277,7 @@ DexMethod* create_instanceof_method(const DexType* merger_type, instance_of_block->load_const(ret_loc, 0); instance_of_block->ret(ret_loc); - return mc.create(); + return mc->create(); } void update_instance_of( @@ -522,12 +524,10 @@ void ModelMerger::update_stats(const std::string& model_name, m_stats += mm.get_stats(); } -std::vector ModelMerger::merge_model( - Scope& scope, - DexStoresVector& stores, - ConfigFiles& conf, - Model& model, - bool update_method_profiles_stats) { +std::vector ModelMerger::merge_model(Scope& scope, + DexStoresVector& stores, + ConfigFiles& conf, + Model& model) { Timer t("merge_model"); std::vector to_materialize; std::vector merger_classes; @@ -632,18 +632,13 @@ std::vector ModelMerger::merge_model( scope, to_materialize, mergeable_to_merger, m_merger_fields); // Merge methods - method_profiles::MethodProfiles& method_profile = conf.get_method_profiles(); - ModelMethodMerger mm( - scope, - to_materialize, - type_tag_fields, - &type_tags, - method_debug_map, - model.get_model_spec(), - model.get_model_spec().max_num_dispatch_target, - update_method_profiles_stats - ? boost::optional(&method_profile) - : boost::none); + ModelMethodMerger mm(scope, + to_materialize, + type_tag_fields, + &type_tags, + method_debug_map, + model.get_model_spec(), + model.get_model_spec().max_num_dispatch_target); auto mergeable_to_merger_ctor = mm.merge_methods(); update_stats(model.get_name(), to_materialize, mm); @@ -671,7 +666,7 @@ std::vector ModelMerger::merge_model( if (mergeable_to_merger.count(cls->get_type())) { cls->set_interfaces(no_interface); cls->set_super_class(type::java_lang_Object()); - redex_assert(CONSTP(cls)->get_vmethods().empty()); + redex_assert(cls->get_vmethods().empty()); if (!cls->get_clinit() && cls->get_sfields().empty()) { // Purge merged cls w/o static fields. return true; diff --git a/service/class-merging/ModelMerger.h b/service/class-merging/ModelMerger.h index 427f44c28e..c78f538f12 100644 --- a/service/class-merging/ModelMerger.h +++ b/service/class-merging/ModelMerger.h @@ -34,8 +34,7 @@ class ModelMerger { std::vector merge_model(Scope& scope, DexStoresVector& stores, ConfigFiles& conf, - Model& model, - bool update_method_profiles_stats); + Model& model); void increase_ctor_dedupped_stats(int64_t value) { m_stats.m_num_ctor_dedupped += value; diff --git a/service/class-merging/ModelMethodMerger.cpp b/service/class-merging/ModelMethodMerger.cpp index 0b1c1d838c..10b0aa6deb 100644 --- a/service/class-merging/ModelMethodMerger.cpp +++ b/service/class-merging/ModelMethodMerger.cpp @@ -56,7 +56,7 @@ void staticize_with_new_arg_head(DexMethod* meth, DexType* new_head) { mutators::make_static(meth, mutators::KeepThis::Yes); DexMethodSpec spec; auto args = meth->get_proto()->get_args(); - always_assert(!args->empty()); + always_assert(args->size()); auto new_type_list = args->replace_head(new_head); auto new_proto = DexProto::make_proto(meth->get_proto()->get_rtype(), new_type_list); @@ -158,7 +158,7 @@ static void find_common_ctor_invocations( return; } auto last_non_goto_insn = target->get_last_insn(); - assert_log(last_non_goto_insn != CONSTP(target)->end(), + assert_log(last_non_goto_insn != target->end(), "Should have at least one insn!"); if (!opcode::is_invoke_direct(last_non_goto_insn->insn->opcode())) { @@ -236,16 +236,14 @@ ModelMethodMerger::ModelMethodMerger( const TypeTags* type_tags, const std::unordered_map& method_debug_map, const ModelSpec& model_spec, - boost::optional max_num_dispatch_target, - boost::optional method_profiles) + boost::optional max_num_dispatch_target) : m_scope(scope), m_mergers(mergers), m_type_tag_fields(type_tag_fields), m_type_tags(type_tags), m_method_debug_map(method_debug_map), m_model_spec(model_spec), - m_max_num_dispatch_target(max_num_dispatch_target), - m_method_profiles(method_profiles) { + m_max_num_dispatch_target(max_num_dispatch_target) { for (const auto& mtf : type_tag_fields) { auto type_tag_field = mtf.second; if (model_spec.generate_type_tag()) { @@ -352,7 +350,7 @@ std::vector ModelMethodMerger::make_check_cast(DexType* type, dispatch::DispatchMethod ModelMethodMerger::create_dispatch_method( const dispatch::Spec& spec, const std::vector& targets) { - always_assert(!targets.empty()); + always_assert(targets.size()); TRACE(CLMG, 5, "creating dispatch %s.%s for targets of size %zu", @@ -370,12 +368,24 @@ dispatch::DispatchMethod ModelMethodMerger::create_dispatch_method( std::map ModelMethodMerger::get_dedupped_indices_map( const std::vector& targets) { - always_assert(!targets.empty()); + always_assert(targets.size()); std::map indices_to_callee; + + // TODO "structural_equals" feature of editable cfg hasn't been implenmented + // yet. Currently, we still need to use irlist::structural_equals. Therefore, + // we need to clear_cfg before finding equivalent methods. Once + // structural_equals of editable cfg is added, the following clear_cfg will be + // removed. + for (size_t i = 0; i < targets.size(); i++) { + targets[i]->get_code()->clear_cfg(); + } // Find equivalent methods. std::vector duplicates = method_dedup::group_identical_methods( targets, m_model_spec.dedup_fill_in_stack_trace); + for (size_t i = 0; i < targets.size(); i++) { + targets[i]->get_code()->build_cfg(); + } for (const auto& duplicate : duplicates) { SwitchIndices switch_indices; for (auto& meth : duplicate) { @@ -557,7 +567,7 @@ void ModelMethodMerger::merge_virtual_methods( DexClass* target_cls = type_class(target_type); for (auto& virt_meth : virt_methods) { auto& meth_lst = virt_meth.overrides; - always_assert(!meth_lst.empty()); + always_assert(meth_lst.size()); auto overridden_meth = virt_meth.base; auto front_meth = meth_lst.front(); auto access = front_meth->get_access(); @@ -591,11 +601,6 @@ void ModelMethodMerger::merge_virtual_methods( for (const auto& m : meth_lst) { old_to_new_callee[m] = dispatch.main_dispatch; } - if (m_method_profiles != boost::none) { - m_stats.m_updated_profile_method += - m_method_profiles.get()->substitute_stats(dispatch.main_dispatch, - meth_lst); - } // Populating method dedup map for (auto& type_to_sig : meth_signatures) { auto type = type_to_sig.first; @@ -696,10 +701,6 @@ void ModelMethodMerger::merge_ctors() { for (const auto& m : ctors) { old_to_new_callee[m] = dispatch; } - if (m_method_profiles != boost::none) { - m_stats.m_updated_profile_method += - m_method_profiles.get()->substitute_stats(dispatch, ctors); - } std::vector> not_inlined_ctors; type_class(target_type)->add_method(dispatch); // Inline entries @@ -788,9 +789,20 @@ void ModelMethodMerger::dedup_non_ctor_non_virt_methods() { auto new_to_old_optional = boost::optional>( new_to_old); + // TODO "structural_equals" feature of editable cfg hasn't been implenmented + // yet. Currently, we still need to use irlist::structural_equals. + // Therefore, we need to clear_cfg before finding equivalent methods. Once + // structural_equals of editable cfg is added, the following clear_cfg will + // be removed. + for (size_t i = 0; i < to_dedup.size(); i++) { + to_dedup[i]->get_code()->clear_cfg(); + } m_stats.m_num_static_non_virt_dedupped += method_dedup::dedup_methods( m_scope, to_dedup, m_model_spec.dedup_fill_in_stack_trace, replacements, new_to_old_optional); + for (size_t i = 0; i < replacements.size(); i++) { + replacements[i]->get_code()->build_cfg(); + } // Relocate the remainders. std::set to_relocate( replacements.begin(), replacements.end()); @@ -810,18 +822,12 @@ void ModelMethodMerger::dedup_non_ctor_non_virt_methods() { SHOW(merger_type)); TRACE(CLMG, 8, "dedup: moving static|non_virt method %s", SHOW(m)); - change_visibility(m, merger_type); relocate_method(m, merger_type); } // Update method dedup map for (auto& pair : new_to_old) { auto old_list = pair.second; - if (m_method_profiles != boost::none) { - std::vector old_list_vec{old_list.begin(), old_list.end()}; - m_stats.m_updated_profile_method += - m_method_profiles.get()->substitute_stats(pair.first, old_list_vec); - } for (auto old_meth : old_list) { auto type = old_meth->get_class(); if (m_mergeable_to_merger_ctor.count(type) == 0) { @@ -880,8 +886,6 @@ void ModelMethodMerger::merge_virt_itf_methods() { ? m_type_tag_fields.at(merger) : nullptr; std::vector virt_methods; - virt_methods.reserve(merger->vmethods.size() + - merger->intfs_methods.size()); for (auto& vm_lst : merger->vmethods) { virt_methods.emplace_back(vm_lst); @@ -916,7 +920,6 @@ void ModelMethodMerger::merge_virt_itf_methods() { for (const auto& pair : not_inlined_dispatch_entries) { auto merger_type = pair.first; auto not_inlined = pair.second; - change_visibility(not_inlined, merger_type); relocate_method(not_inlined, merger_type); } } diff --git a/service/class-merging/ModelMethodMerger.h b/service/class-merging/ModelMethodMerger.h index 2763d3d3c7..0b0c38dc3d 100644 --- a/service/class-merging/ModelMethodMerger.h +++ b/service/class-merging/ModelMethodMerger.h @@ -11,7 +11,6 @@ #include "DexClass.h" #include "MergerType.h" -#include "MethodProfiles.h" #include "Model.h" #include "TypeTags.h" @@ -57,8 +56,7 @@ class ModelMethodMerger { const TypeTags* type_tags, const std::unordered_map& method_debug_map, const ModelSpec& model_spec, - boost::optional max_num_dispatch_target, - boost::optional method_profiles); + boost::optional max_num_dispatch_target); TypeToMethod& merge_methods() { merge_ctors(); @@ -91,7 +89,6 @@ class ModelMethodMerger { // This member is only used for testing purpose. If its value is greator than // zero, the splitting decision will bypass the instruction count limit. boost::optional m_max_num_dispatch_target; - boost::optional m_method_profiles; // dmethods MergerToMethods m_merger_ctors; diff --git a/service/class-splitting/ClassSplitting.cpp b/service/class-splitting/ClassSplitting.cpp index dadfb01516..b0355ed876 100644 --- a/service/class-splitting/ClassSplitting.cpp +++ b/service/class-splitting/ClassSplitting.cpp @@ -367,7 +367,7 @@ DexClasses ClassSplitter::additional_classes(const DexClasses& classes) { ++m_stats.relocated_static_methods; } else if (!method->is_virtual()) { ++m_stats.relocated_non_static_direct_methods; - } else if (m_non_true_virtual_methods.count_unsafe(method)) { + } else if (m_non_true_virtual_methods.count(method)) { ++m_stats.relocated_non_true_virtual_methods; } else { ++m_stats.relocated_true_virtual_methods; @@ -524,8 +524,8 @@ void ClassSplitter::cleanup(const Scope& final_scope) { if (!is_static(method)) { mutators::make_static(method); } - change_visibility(method, target_cls->get_type()); relocate_method(method, target_cls->get_type()); + change_visibility(method); } TRACE(CS, 2, "[class splitting] Made {%zu} methods static.", methods_to_staticize.size()); @@ -698,8 +698,7 @@ bool ClassSplitter::can_relocate(bool cls_has_problematic_clinit, // fields, and carefully deal with super-init calls. return false; } - } else if (m_non_true_virtual_methods.count_unsafe( - const_cast(m))) { + } else if (m_non_true_virtual_methods.count(const_cast(m))) { if (!m_config.relocate_non_true_virtual_methods) { return false; } diff --git a/service/class-splitting/ClassSplitting.h b/service/class-splitting/ClassSplitting.h index eaac9da3d3..58792c2921 100644 --- a/service/class-splitting/ClassSplitting.h +++ b/service/class-splitting/ClassSplitting.h @@ -144,7 +144,7 @@ class ClassSplitter final { std::vector> m_methods_to_relocate; std::vector> m_methods_to_trampoline; ClassSplittingStats m_stats; - InsertOnlyConcurrentSet m_non_true_virtual_methods; + std::unordered_set m_non_true_virtual_methods; ClassSplittingConfig m_config; PassManager& m_mgr; const std::unordered_set& m_sufficiently_popular_methods; diff --git a/service/constant-propagation/ConstantPropagation.cpp b/service/constant-propagation/ConstantPropagation.cpp index 6b1047e6e6..c78a7e1e75 100644 --- a/service/constant-propagation/ConstantPropagation.cpp +++ b/service/constant-propagation/ConstantPropagation.cpp @@ -20,7 +20,7 @@ Transform::Stats ConstantPropagation::run( DexMethod* method, XStoreRefs* xstores, const Transform::RuntimeCache& runtime_cache) { - if (method->get_code() == nullptr || method->rstate.no_optimizations()) { + if (method->get_code() == nullptr) { return Transform::Stats(); } TRACE(CONSTP, 2, "Method: %s", SHOW(method)); diff --git a/service/constant-propagation/ConstantPropagationAnalysis.cpp b/service/constant-propagation/ConstantPropagationAnalysis.cpp index bcd88c38e2..07173b0c25 100644 --- a/service/constant-propagation/ConstantPropagationAnalysis.cpp +++ b/service/constant-propagation/ConstantPropagationAnalysis.cpp @@ -1015,52 +1015,31 @@ bool ImmutableAttributeAnalyzerState::is_jvm_cached_object( DexType* ImmutableAttributeAnalyzerState::initialized_type( const DexMethod* initialize_method) { - auto res = method::is_init(initialize_method) - ? initialize_method->get_class() - : initialize_method->get_proto()->get_rtype(); - always_assert(!type::is_primitive(res)); - always_assert(res != type::java_lang_Object()); - always_assert(!type::is_array(res)); - return res; + return method::is_init(initialize_method) + ? initialize_method->get_class() + : initialize_method->get_proto()->get_rtype(); } bool ImmutableAttributeAnalyzerState::may_be_initialized_type( DexType* type) const { - if (type == nullptr || type::is_array(type)) { - return false; - } - always_assert(!type::is_primitive(type)); - return *may_be_initialized_types - .get_or_create_and_assert_equal( - type, - [&](DexType*) { - return compute_may_be_initialized_type(type); - }) - .first; -} - -bool ImmutableAttributeAnalyzerState::compute_may_be_initialized_type( - DexType* type) const { - // Here we effectively check if check_cast(type, x) for any x in - // initialized_types. - always_assert(type != nullptr); - if (initialized_types.count_unsafe(type)) { - return true; - } - auto cls = type_class(type); - if (cls == nullptr) { - return false; - } - if (may_be_initialized_type(cls->get_super_class())) { - return true; - } - auto intfs = cls->get_interfaces(); - for (auto intf : *intfs) { - if (may_be_initialized_type(intf)) { - return true; + auto res = may_be_initialized_types.get(type, std::nullopt); + if (!res) { + res = false; + for (auto* initialized_type : initialized_types) { + if (type::check_cast(type, initialized_type)) { + res = true; + break; + } } + may_be_initialized_types.update(type, [&](auto*, auto& value, bool exists) { + if (exists) { + always_assert(value == res); + } else { + value = res; + } + }); } - return false; + return *res; } bool ImmutableAttributeAnalyzer::analyze_iget( diff --git a/service/constant-propagation/ConstantPropagationAnalysis.h b/service/constant-propagation/ConstantPropagationAnalysis.h index 67f19a369b..134ddb35a6 100644 --- a/service/constant-propagation/ConstantPropagationAnalysis.h +++ b/service/constant-propagation/ConstantPropagationAnalysis.h @@ -319,7 +319,7 @@ struct ImmutableAttributeAnalyzerState { ConcurrentSet attribute_fields; std::unordered_map cached_boxed_objects; ConcurrentSet initialized_types; - mutable InsertOnlyConcurrentMap may_be_initialized_types; + mutable ConcurrentMap> may_be_initialized_types; ImmutableAttributeAnalyzerState(); @@ -337,9 +337,6 @@ struct ImmutableAttributeAnalyzerState { static DexType* initialized_type(const DexMethod* initialize_method); bool may_be_initialized_type(DexType* type) const; - - private: - bool compute_may_be_initialized_type(DexType* type) const; }; class ImmutableAttributeAnalyzer final diff --git a/service/constant-propagation/ConstantPropagationTransform.cpp b/service/constant-propagation/ConstantPropagationTransform.cpp index d67e9676d7..5612b25ba9 100644 --- a/service/constant-propagation/ConstantPropagationTransform.cpp +++ b/service/constant-propagation/ConstantPropagationTransform.cpp @@ -965,10 +965,6 @@ bool Transform::has_problematic_return(cfg::ControlFlowGraph& cfg, DexType* declaring_type, DexProto* proto, const XStoreRefs* xstores) { - static AccumulatingTimer s_timer( - "constant_propagation::Transform::has_problematic_return"); - auto t = s_timer.scope(); - // Nothing to check without method information if (!declaring_type || !proto) { return false; @@ -983,25 +979,13 @@ bool Transform::has_problematic_return(cfg::ControlFlowGraph& cfg, // No return issues when there are no try/catch blocks auto blocks = cfg.blocks(); bool has_catch = - std::any_of(blocks.begin(), blocks.end(), - [](cfg::Block* block) { return block->is_catch(); }); + std::find_if(blocks.begin(), blocks.end(), [](cfg::Block* block) { + return block->is_catch(); + }) != blocks.end(); if (!has_catch) { return false; } - auto is_relevant_def = [](auto* insn) { - auto op = insn->opcode(); - return insn->has_type() || insn->has_method() || op == OPCODE_IGET_OBJECT || - op == OPCODE_SGET_OBJECT || op == OPCODE_AGET_OBJECT; - }; - auto ii = InstructionIterable(cfg); - auto has_relevant_def = std::any_of(ii.begin(), ii.end(), [&](auto& mie) { - return is_relevant_def(mie.insn); - }); - if (!has_relevant_def) { - return false; - } - // For all return instructions, check whether the reaching definitions are of // a type that's unavailable/external, or defined in a different store. auto declaring_class_idx = xstores->get_store_idx(declaring_type); @@ -1029,7 +1013,7 @@ bool Transform::has_problematic_return(cfg::ControlFlowGraph& cfg, t_idx, SHOW(insn)); return true; }; - reaching_defs::MoveAwareFixpointIterator fp_iter(cfg, is_relevant_def); + reaching_defs::MoveAwareFixpointIterator fp_iter(cfg); fp_iter.run({}); std::unique_ptr ti; for (cfg::Block* block : blocks) { @@ -1040,9 +1024,8 @@ bool Transform::has_problematic_return(cfg::ControlFlowGraph& cfg, for (auto& mie : InstructionIterable(block)) { IRInstruction* insn = mie.insn; if (opcode::is_a_return(insn->opcode())) { - const auto& defs = env.get(insn->src(0)); - always_assert(!defs.is_bottom()); - always_assert(!defs.is_top()); + auto defs = env.get(insn->src(0)); + always_assert(!defs.is_bottom() && !defs.is_top()); for (auto def : defs.elements()) { auto op = def->opcode(); if (def->has_type()) { @@ -1072,8 +1055,6 @@ bool Transform::has_problematic_return(cfg::ControlFlowGraph& cfg, type::get_array_component_type(*dex_type), def)) { return true; } - } else { - not_reached(); } } } diff --git a/service/constant-propagation/ConstantPropagationWholeProgramState.cpp b/service/constant-propagation/ConstantPropagationWholeProgramState.cpp index fb2d8c9395..e9b74ce3a5 100644 --- a/service/constant-propagation/ConstantPropagationWholeProgramState.cpp +++ b/service/constant-propagation/ConstantPropagationWholeProgramState.cpp @@ -138,7 +138,7 @@ namespace constant_propagation { WholeProgramState::WholeProgramState( const Scope& scope, const interprocedural::FixpointIterator& fp_iter, - const InsertOnlyConcurrentSet& non_true_virtuals, + const std::unordered_set& non_true_virtuals, const std::unordered_set& field_blocklist, const std::unordered_set& definitely_assigned_ifields, std::shared_ptr call_graph) @@ -358,7 +358,15 @@ bool WholeProgramAwareAnalyzer::analyze_invoke( return false; } if (whole_program_state->has_call_graph()) { - if (whole_program_state->invoke_is_dynamic(insn)) { + auto method = resolve_method(insn->get_method(), opcode_to_search(insn)); + if (method == nullptr && opcode_to_search(insn) == MethodSearch::Virtual) { + method = + resolve_method(insn->get_method(), MethodSearch::InterfaceVirtual); + } + if (method == nullptr) { + return false; + } + if (whole_program_state->method_is_dynamic(method)) { return false; } auto value = whole_program_state->get_return_value_from_cg(insn); diff --git a/service/constant-propagation/ConstantPropagationWholeProgramState.h b/service/constant-propagation/ConstantPropagationWholeProgramState.h index c40c9d7cdf..496f39d7d4 100644 --- a/service/constant-propagation/ConstantPropagationWholeProgramState.h +++ b/service/constant-propagation/ConstantPropagationWholeProgramState.h @@ -57,7 +57,7 @@ class WholeProgramState { WholeProgramState(const Scope&, const interprocedural::FixpointIterator&, - const InsertOnlyConcurrentSet&, + const std::unordered_set&, const std::unordered_set&, const std::unordered_set&, std::shared_ptr call_graph); @@ -122,8 +122,8 @@ class WholeProgramState { const call_graph::Graph* call_graph() const { return m_call_graph.get(); } - bool invoke_is_dynamic(const IRInstruction* insn) const { - return call_graph::invoke_is_dynamic(*m_call_graph, insn); + bool method_is_dynamic(const DexMethod* method) const { + return call_graph::method_is_dynamic(*m_call_graph, method); } private: @@ -187,8 +187,8 @@ class WholeProgramStateAccessor { bool has_call_graph() const { return m_wps.has_call_graph(); } - bool invoke_is_dynamic(const IRInstruction* insn) const { - return m_wps.invoke_is_dynamic(insn); + bool method_is_dynamic(const DexMethod* method) const { + return m_wps.method_is_dynamic(method); } ConstantValue get_field_value(const DexField* field) const { @@ -208,7 +208,6 @@ class WholeProgramStateAccessor { } for (const DexMethod* callee : callees) { if (!callee->get_code()) { - always_assert(is_abstract(callee) || is_native(callee)); return ConstantValue::top(); } } diff --git a/service/constant-propagation/ConstructorParams.cpp b/service/constant-propagation/ConstructorParams.cpp index 3a046fad1d..930e2c45bf 100644 --- a/service/constant-propagation/ConstructorParams.cpp +++ b/service/constant-propagation/ConstructorParams.cpp @@ -132,28 +132,24 @@ class InitFixpointIterator final env->set(RESULT_REGISTER, ParamIdxDomain::top()); return; } - bool first_initializer_obj_is_dest = false; - m_state.method_initializers.observe( - method, [&](auto*, const auto& initializers) { - for (auto& initializer : initializers) { - if (initializer->attr.is_method()) { - env->set( - initializer->attr.val.method, - env->get(insn->src(initializer->insn_src_id_of_attr))); - } else { // is_field - env->set( - initializer->attr.val.field, - env->get(insn->src(initializer->insn_src_id_of_attr))); - } - } - auto& first_initializer = *initializers.begin(); - if (first_initializer->obj_is_dest()) { - env->set(RESULT_REGISTER, obj_domain); - first_initializer_obj_is_dest = true; - } - }); - if (first_initializer_obj_is_dest) { - return; + std::unique_lock lock( + m_state.method_initializers.get_lock(method)); + auto it = m_state.method_initializers.find(method); + if (it != m_state.method_initializers.end()) { + for (auto& initializer : it->second) { + if (initializer->attr.is_method()) { + env->set(initializer->attr.val.method, + env->get(insn->src(initializer->insn_src_id_of_attr))); + } else { // is_field + env->set(initializer->attr.val.field, + env->get(insn->src(initializer->insn_src_id_of_attr))); + } + } + auto& first_initializer = *it->second.begin(); + if (first_initializer->obj_is_dest()) { + env->set(RESULT_REGISTER, obj_domain); + return; + } } } } diff --git a/service/constant-propagation/DefinitelyAssignedIFields.cpp b/service/constant-propagation/DefinitelyAssignedIFields.cpp index 0cff3d9520..455a1e34a6 100644 --- a/service/constant-propagation/DefinitelyAssignedIFields.cpp +++ b/service/constant-propagation/DefinitelyAssignedIFields.cpp @@ -293,33 +293,41 @@ namespace definitely_assigned_ifields { std::unordered_set get_definitely_assigned_ifields( const Scope& scope) { Timer t("get_definitely_assigned_ifields"); - InsertOnlyConcurrentMap analysis_results; + ConcurrentMap> analysis_results; std::function get_analysis_result; - get_analysis_result = [&](DexMethod* ctor) -> const AnalysisResult* { - return analysis_results - .get_or_create_and_assert_equal( - ctor, - [&](auto*) { - if (!ctor->is_external() && ctor->get_code()) { - auto& cfg = ctor->get_code()->cfg(); - Analyzer analyzer(cfg, ctor->get_class(), get_analysis_result); - const auto& env = analyzer.get_exit_state_at(cfg.exit_block()); - auto cls = type_class(ctor->get_class()); - return env.get_analysis_result(cls); - } - AnalysisResult res; - // Conservative assumption: All external ctors (without - // code) except Object:: may directly or - // indirectly read and write own fields. - if (ctor->get_class() != type::java_lang_Object()) { - // TODO: Consider using the SummaryGenerator to - // analyze AOSP classes to find other external - // constructors where this does not escape. - res.may_this_have_escaped = true; - } - return res; - }) - .first; + get_analysis_result = [&](DexMethod* ctor) { + auto res = analysis_results.get(ctor, nullptr); + if (res) { + return res.get(); + } + if (!ctor->is_external() && ctor->get_code()) { + auto& cfg = ctor->get_code()->cfg(); + Analyzer analyzer(cfg, ctor->get_class(), get_analysis_result); + const auto& env = analyzer.get_exit_state_at(cfg.exit_block()); + auto cls = type_class(ctor->get_class()); + res = std::make_shared(env.get_analysis_result(cls)); + } else { + res = std::make_shared(); + // Conservative assumption: All external ctors (without code) + // except Object:: may directly or indirectly read and write own + // fields. + if (ctor->get_class() != type::java_lang_Object()) { + // TODO: Consider using the SummaryGenerator to analyze AOSP classes to + // find other external constructors where this does not escape. + res->may_this_have_escaped = true; + } + } + analysis_results.update( + ctor, + [&](DexMethod*, std::shared_ptr& value, bool exists) { + if (exists) { + always_assert(*value == *res); + res = value; + } else { + value = res; + } + }); + return res.get(); }; ConcurrentSet res; walk::parallel::classes(scope, [&](DexClass* cls) { diff --git a/service/constant-propagation/DisjointUnionWithSignedConstantDomain.h b/service/constant-propagation/DisjointUnionWithSignedConstantDomain.h index 88da475fd0..869586c592 100644 --- a/service/constant-propagation/DisjointUnionWithSignedConstantDomain.h +++ b/service/constant-propagation/DisjointUnionWithSignedConstantDomain.h @@ -15,7 +15,6 @@ #include #include -#include "Debug.h" #include "ObjectWithImmutAttr.h" #include "SignedConstantDomain.h" #include "SingletonObject.h" @@ -228,6 +227,14 @@ void DisjointUnionWithSignedConstantDomain::join_with( if (other.is_bottom()) { return; } + // SingletonObjectDomain and ObjectWithImmutAttrDomain both represent object + // references and they have intersection. + // Handle their meet operator specially. + if ((this->is_singleton_object() && other.is_object_with_immutable_attr()) || + (other.is_singleton_object() && this->is_object_with_immutable_attr())) { + this->set_to_top(); + return; + } auto nez = (this->is_nez() || this->is_object()) && (other.is_nez() || other.is_object()); boost::apply_visitor(sparta::duad_impl::join_visitor(), this->m_variant, @@ -261,7 +268,7 @@ void DisjointUnionWithSignedConstantDomain::meet_with( // Handle their meet operator specially. if ((this->is_singleton_object() && other.is_object_with_immutable_attr()) || (other.is_singleton_object() && this->is_object_with_immutable_attr())) { - this->m_variant = SignedConstantDomain(sign_domain::Interval::NEZ); + this->set_to_top(); return; } diff --git a/service/constant-propagation/IPConstantPropagationAnalysis.cpp b/service/constant-propagation/IPConstantPropagationAnalysis.cpp index ee2b67ffa8..c7f10a0be3 100644 --- a/service/constant-propagation/IPConstantPropagationAnalysis.cpp +++ b/service/constant-propagation/IPConstantPropagationAnalysis.cpp @@ -127,7 +127,8 @@ void FixpointIterator::analyze_node(call_graph::NodeId const& node, } Domain FixpointIterator::analyze_edge( - const call_graph::EdgeId& edge, const Domain& exit_state_at_source) const { + const std::shared_ptr& edge, + const Domain& exit_state_at_source) const { Domain entry_state_at_dest; auto insn = edge->invoke_insn(); if (insn == nullptr) { @@ -219,9 +220,10 @@ FixpointIterator::find_matching_method_cache_entry( } // namespace interprocedural void set_encoded_values(const DexClass* cls, ConstantEnvironment* env) { - always_assert(!cls->is_external()); for (auto* sfield : cls->get_sfields()) { - always_assert(!sfield->is_external()); + if (sfield->is_external()) { + continue; + } auto value = sfield->get_static_value(); if (value == nullptr || value->evtype() == DEVT_NULL) { env->set(sfield, SignedConstantDomain(0)); @@ -244,15 +246,19 @@ void set_encoded_values(const DexClass* cls, ConstantEnvironment* env) { } /** - * This function is much simpler than set_ifield_values since there are no - * DexEncodedValues to handle. + * Bind all eligible fields to SignedConstantDomain(0) in :env, since all + * fields are initialized to zero by default at runtime. This function is + * much simpler than set_ifield_values since there are no DexEncodedValues + * to handle. */ void set_ifield_values(const DexClass* cls, const EligibleIfields& eligible_ifields, ConstantEnvironment* env) { always_assert(!cls->is_external()); + if (cls->get_ctors().size() > 1) { + return; + } for (auto* ifield : cls->get_ifields()) { - always_assert(!ifield->is_external()); if (!eligible_ifields.count(ifield)) { // If the field is not a eligible ifield, move on. continue; diff --git a/service/constant-propagation/IPConstantPropagationAnalysis.h b/service/constant-propagation/IPConstantPropagationAnalysis.h index d3efc7d943..11012e6bea 100644 --- a/service/constant-propagation/IPConstantPropagationAnalysis.h +++ b/service/constant-propagation/IPConstantPropagationAnalysis.h @@ -100,7 +100,7 @@ class FixpointIterator : public sparta::ParallelMonotonicFixpointIterator< void analyze_node(const call_graph::NodeId& node, Domain* current_state) const override; - Domain analyze_edge(const call_graph::EdgeId& edge, + Domain analyze_edge(const std::shared_ptr& edge, const Domain& exit_state_at_source) const override; std::unique_ptr get_intraprocedural_analysis( @@ -149,10 +149,6 @@ class FixpointIterator : public sparta::ParallelMonotonicFixpointIterator< */ void set_encoded_values(const DexClass* cls, ConstantEnvironment* env); -/** - * Bind all eligible fields to SignedConstantDomain(0) in :env, since all - * fields are initialized to zero by default at runtime. - */ void set_ifield_values(const DexClass* cls, const EligibleIfields& eligible_ifields, ConstantEnvironment* env); diff --git a/service/constant-propagation/NewObjectDomain.h b/service/constant-propagation/NewObjectDomain.h index 3d1313460d..0e4bb086d6 100644 --- a/service/constant-propagation/NewObjectDomain.h +++ b/service/constant-propagation/NewObjectDomain.h @@ -7,11 +7,12 @@ #pragma once +#include + #include #include #include "DexClass.h" -#include "IRInstruction.h" #include "Show.h" #include "SignedConstantDomain.h" diff --git a/service/constant-propagation/SignedConstantDomain.h b/service/constant-propagation/SignedConstantDomain.h index 50a653fbe1..fc2590889d 100644 --- a/service/constant-propagation/SignedConstantDomain.h +++ b/service/constant-propagation/SignedConstantDomain.h @@ -11,7 +11,6 @@ #include #include -#include "Debug.h" #include "SignDomain.h" using ConstantDomain = sparta::ConstantAbstractDomain; diff --git a/service/copy-propagation/AliasedRegisters.cpp b/service/copy-propagation/AliasedRegisters.cpp index 10168da6db..64143539d3 100644 --- a/service/copy-propagation/AliasedRegisters.cpp +++ b/service/copy-propagation/AliasedRegisters.cpp @@ -713,7 +713,7 @@ void AliasGraph::clear_vertex(vertex_t v) { auto v_in_it = m_vertices_ins.find(v); if (v_in_it != m_vertices_ins.end()) { auto& v_in = v_in_it->second; - always_assert(!v_in.empty()); + always_assert(v_in.size() >= 1); for (auto u : v_in) { always_assert(m_vertices_outs.at(u) == v); m_vertices_outs.erase(u); diff --git a/service/copy-propagation/CanonicalizeLocks.cpp b/service/copy-propagation/CanonicalizeLocks.cpp index c41d2efde7..a98a816400 100644 --- a/service/copy-propagation/CanonicalizeLocks.cpp +++ b/service/copy-propagation/CanonicalizeLocks.cpp @@ -63,7 +63,7 @@ boost::optional compute_rdefs(ControlFlowGraph& cfg) { std::unordered_map block_map; auto get_rdef = [&](IRInstruction* insn, reg_t reg) -> IRInstruction* { auto it = block_map.find(insn); - redex_assert(it != block_map.cend()); + redex_assert(it != block_map.end()); auto defs = get_defs(it->second, insn); return get_singleton(defs, reg); }; diff --git a/service/copy-propagation/CopyPropagation.cpp b/service/copy-propagation/CopyPropagation.cpp index 3650dec104..f11a8dc5a7 100644 --- a/service/copy-propagation/CopyPropagation.cpp +++ b/service/copy-propagation/CopyPropagation.cpp @@ -534,8 +534,7 @@ Stats CopyPropagation::run(const Scope& scope) { scope, [&](DexMethod* m) { IRCode* code = m->get_code(); - if (code == nullptr || - (m->rstate.no_optimizations() && !m_config.regalloc_has_run)) { + if (code == nullptr) { return Stats(); } @@ -562,10 +561,10 @@ Stats CopyPropagation::run(IRCode* code, DexTypeList* args, std::function method_describer) { Stats stats; - always_assert(code->editable_cfg_built()); - auto& cfg = code->cfg(); + cfg::ScopedCFG cfg(code); + if (m_config.canonicalize_locks && !m_config.regalloc_has_run) { - auto res = locks::run(cfg); + auto res = locks::run(*cfg); stats.lock_fixups = res.fixups; stats.non_singleton_lock_rdefs = res.non_singleton_rdefs ? 1 : 0; } @@ -576,7 +575,7 @@ Stats CopyPropagation::run(IRCode* code, // the registers. std::unordered_set range_set; if (m_config.regalloc_has_run) { - for (auto& mie : InstructionIterable(cfg)) { + for (auto& mie : InstructionIterable(*cfg)) { auto* insn = mie.insn; if (opcode::has_range_form(insn->opcode())) { insn->denormalize_registers(); @@ -588,15 +587,15 @@ Stats CopyPropagation::run(IRCode* code, } } - auto check_cast_throw_targets_regs = get_check_cast_throw_targets_regs(cfg); + auto check_cast_throw_targets_regs = get_check_cast_throw_targets_regs(*cfg); AliasFixpointIterator fixpoint( - cfg, is_static, declaring_type, rtype, args, std::move(method_describer), + *cfg, is_static, declaring_type, rtype, args, std::move(method_describer), m_config, range_set, stats, check_cast_throw_targets_regs); fixpoint.run(AliasDomain()); - cfg::CFGMutation mutation{cfg}; - for (auto block : cfg.blocks()) { + cfg::CFGMutation mutation{*cfg}; + for (auto block : cfg->blocks()) { AliasDomain domain = fixpoint.get_entry_state_at(block); domain.update( [&fixpoint, block, &mutation, &stats](AliasedRegisters& aliases) { diff --git a/service/cross-dex-ref-minimizer/CrossDexRefMinimizer.cpp b/service/cross-dex-ref-minimizer/CrossDexRefMinimizer.cpp index c5006f6bbd..0545dfce7e 100644 --- a/service/cross-dex-ref-minimizer/CrossDexRefMinimizer.cpp +++ b/service/cross-dex-ref-minimizer/CrossDexRefMinimizer.cpp @@ -136,7 +136,7 @@ void CrossDexRefMinimizer::reprioritize( } void CrossDexRefMinimizer::sample(DexClass* cls) { - const auto& cls_refs = m_cache->get(cls); + auto cls_refs = m_cache->get(cls); auto increment = [&ref_counts = m_ref_counts, &max_ref_count = m_max_ref_count](const void* ref) { size_t& count = ref_counts[ref]; @@ -144,25 +144,25 @@ void CrossDexRefMinimizer::sample(DexClass* cls) { max_ref_count = count; } }; - for (auto ref : cls_refs.method_refs) { + for (auto ref : cls_refs->method_refs) { increment(ref); } - for (auto ref : cls_refs.field_refs) { + for (auto ref : cls_refs->field_refs) { increment(ref); } - for (auto ref : cls_refs.types) { + for (auto ref : cls_refs->types) { increment(ref); } - for (auto ref : cls_refs.strings) { + for (auto ref : cls_refs->strings) { increment(ref); } if (m_json_classes) { Json::Value json_class; - json_class["method_refs"] = m_json_methods.get(cls_refs.method_refs); - json_class["field_refs"] = m_json_fields.get(cls_refs.field_refs); - json_class["types"] = m_json_types.get(cls_refs.types); - json_class["strings"] = m_json_strings.get(cls_refs.strings); + json_class["method_refs"] = m_json_methods.get(cls_refs->method_refs); + json_class["field_refs"] = m_json_fields.get(cls_refs->field_refs); + json_class["types"] = m_json_types.get(cls_refs->types); + json_class["strings"] = m_json_strings.get(cls_refs->strings); json_class["is_generated"] = cls->rstate.is_generated(); json_class["insert_index"] = -1; (*m_json_classes)[get_json_class_index(cls)] = json_class; @@ -181,11 +181,11 @@ void CrossDexRefMinimizer::insert(DexClass* cls) { // entries. // We don't bother with protos and type_lists, as they are directly related // to method refs (I tried, didn't help). - const auto& cls_refs = m_cache->get(cls); + auto cls_refs = m_cache->get(cls); auto& refs = *class_info.refs; - refs.reserve(cls_refs.method_refs.size() + cls_refs.field_refs.size() + - cls_refs.types.size() + cls_refs.strings.size()); + refs.reserve(cls_refs->method_refs.size() + cls_refs->field_refs.size() + + cls_refs->types.size() + cls_refs->strings.size()); uint64_t& refs_weight = class_info.refs_weight; uint64_t& seed_weight = class_info.seed_weight; @@ -215,22 +215,16 @@ void CrossDexRefMinimizer::insert(DexClass* cls) { // different values and observing the effect on APK size. // We discount references that occur in many classes. // TODO: Try some other variations. - for (auto mref : cls_refs.method_refs) { + for (auto mref : cls_refs->method_refs) { add_weight(mref, m_config.method_ref_weight, m_config.method_seed_weight); } - for (auto type : cls_refs.types) { + for (auto type : cls_refs->types) { add_weight(type, m_config.type_ref_weight, m_config.type_seed_weight); } - for (auto string : cls_refs.strings) { - if (string->length() > m_config.min_large_string_size) { - add_weight(string, m_config.small_string_ref_weight, - m_config.large_string_seed_weight); - } else { - add_weight(string, m_config.large_string_ref_weight, - m_config.small_string_seed_weight); - } + for (auto string : cls_refs->strings) { + add_weight(string, m_config.string_ref_weight, m_config.string_seed_weight); } - for (auto fref : cls_refs.field_refs) { + for (auto fref : cls_refs->field_refs) { add_weight(fref, m_config.field_ref_weight, m_config.field_seed_weight); } diff --git a/service/cross-dex-ref-minimizer/CrossDexRefMinimizer.h b/service/cross-dex-ref-minimizer/CrossDexRefMinimizer.h index 0e29abdeac..3d599ebddc 100644 --- a/service/cross-dex-ref-minimizer/CrossDexRefMinimizer.h +++ b/service/cross-dex-ref-minimizer/CrossDexRefMinimizer.h @@ -41,16 +41,13 @@ struct CrossDexRefMinimizerConfig { uint64_t method_ref_weight{100}; uint64_t field_ref_weight{90}; uint64_t type_ref_weight{100}; - uint64_t large_string_ref_weight{90}; - uint64_t small_string_ref_weight{90}; + uint64_t string_ref_weight{90}; uint64_t method_seed_weight{100}; uint64_t field_seed_weight{20}; uint64_t type_seed_weight{30}; - uint64_t large_string_seed_weight{20}; - uint64_t small_string_seed_weight{20}; + uint64_t string_seed_weight{20}; - uint32_t min_large_string_size{50}; bool emit_json{false}; }; diff --git a/service/cse/CommonSubexpressionElimination.cpp b/service/cse/CommonSubexpressionElimination.cpp index 5c7194a066..f5d5107a1d 100644 --- a/service/cse/CommonSubexpressionElimination.cpp +++ b/service/cse/CommonSubexpressionElimination.cpp @@ -1018,7 +1018,7 @@ SharedState::SharedState( } if (traceEnabled(CSE, 2)) { - m_barriers.reset(new AtomicMap()); + m_barriers.reset(new ConcurrentMap()); } const std::vector boxing_types = { @@ -1270,7 +1270,8 @@ CseUnorderedLocationSet SharedState::get_relevant_written_locations( void SharedState::log_barrier(const Barrier& barrier) { if (m_barriers) { - m_barriers->fetch_add(barrier, 1); + m_barriers->update( + barrier, [](const Barrier, size_t& v, bool /* exists */) { v++; }); } } @@ -1337,11 +1338,8 @@ void SharedState::cleanup() { return; } - std::vector> ordered_barriers; - ordered_barriers.reserve(m_barriers->size()); - for (auto&& [barrier, count] : *m_barriers) { - ordered_barriers.emplace_back(barrier, count.load()); - } + std::vector> ordered_barriers(m_barriers->begin(), + m_barriers->end()); std::sort( ordered_barriers.begin(), ordered_barriers.end(), [](const std::pair& a, diff --git a/service/cse/CommonSubexpressionElimination.h b/service/cse/CommonSubexpressionElimination.h index 906b53cc6e..6cf4308ea5 100644 --- a/service/cse/CommonSubexpressionElimination.h +++ b/service/cse/CommonSubexpressionElimination.h @@ -124,7 +124,7 @@ class SharedState { const std::unordered_set& m_finalish_field_names; const std::unordered_set& m_finalish_fields; std::unordered_set m_finalizable_fields; - std::unique_ptr> m_barriers; + std::unique_ptr> m_barriers; std::unordered_map m_method_written_locations; std::unordered_map diff --git a/service/dataflow/ConstantUses.cpp b/service/dataflow/ConstantUses.cpp index 10d58c1df5..7196911cc3 100644 --- a/service/dataflow/ConstantUses.cpp +++ b/service/dataflow/ConstantUses.cpp @@ -89,21 +89,13 @@ ConstantUses::ConstantUses(const cfg::ControlFlowGraph& cfg, DexTypeList* args, const std::function& method_describer, bool force_type_inference) - : m_rtype(rtype) { - static AccumulatingTimer s_timer("ConstantUses::ConstantUses"); - auto t = s_timer.scope(); - + : m_reaching_definitions(cfg), m_rtype(rtype) { always_assert(!force_type_inference || args); - auto is_const = [](auto* insn) { - return insn->opcode() == OPCODE_CONST || - insn->opcode() == OPCODE_CONST_WIDE; - }; - reaching_defs::MoveAwareFixpointIterator reaching_definitions(cfg, is_const); - reaching_definitions.run(reaching_defs::Environment()); + m_reaching_definitions.run(reaching_defs::Environment()); bool need_type_inference = false; for (cfg::Block* block : cfg.blocks()) { - auto env = reaching_definitions.get_entry_state_at(block); + auto env = m_reaching_definitions.get_entry_state_at(block); for (auto& mie : InstructionIterable(block)) { IRInstruction* insn = mie.insn; for (size_t src_index = 0; src_index < insn->srcs_size(); src_index++) { @@ -111,34 +103,36 @@ ConstantUses::ConstantUses(const cfg::ControlFlowGraph& cfg, const auto& defs = env.get(src); if (!defs.is_top() && !defs.is_bottom()) { for (auto def : defs.elements()) { - always_assert(is_const(def)); - m_constant_uses[def].emplace_back(insn, src_index); - // So there's an instruction that uses a const value. - // For some uses, get_type_demand(IRInstruction*, size_t) will - // need to know type inference information on operands. - // The following switch logic needs to be kept in sync with that - // actual usage of type inference information. - auto opcode = insn->opcode(); - switch (opcode) { - case OPCODE_APUT: - case OPCODE_APUT_WIDE: - if (src_index == 0) { + auto def_opcode = def->opcode(); + if (def_opcode == OPCODE_CONST || def_opcode == OPCODE_CONST_WIDE) { + m_constant_uses[def].emplace_back(insn, src_index); + // So there's an instruction that uses a const value. + // For some uses, get_type_demand(IRInstruction*, size_t) will + // need to know type inference information on operands. + // The following switch logic needs to be kept in sync with that + // actual usage of type inference information. + auto opcode = insn->opcode(); + switch (opcode) { + case OPCODE_APUT: + case OPCODE_APUT_WIDE: + if (src_index == 0) { + need_type_inference = true; + } + break; + case OPCODE_IF_EQ: + case OPCODE_IF_NE: + case OPCODE_IF_EQZ: + case OPCODE_IF_NEZ: need_type_inference = true; + break; + default: + break; } - break; - case OPCODE_IF_EQ: - case OPCODE_IF_NE: - case OPCODE_IF_EQZ: - case OPCODE_IF_NEZ: - need_type_inference = true; - break; - default: - break; } } } } - reaching_definitions.analyze_instruction(insn, &env); + m_reaching_definitions.analyze_instruction(insn, &env); } } diff --git a/service/dataflow/ConstantUses.h b/service/dataflow/ConstantUses.h index cadc8711ab..951dfd7e2f 100644 --- a/service/dataflow/ConstantUses.h +++ b/service/dataflow/ConstantUses.h @@ -62,6 +62,7 @@ class ConstantUses { TypeDemand get_type_demand(IRInstruction* insn, size_t src_index) const; mutable std::unique_ptr m_type_inference; + reaching_defs::MoveAwareFixpointIterator m_reaching_definitions; std::unordered_map>> m_constant_uses; diff --git a/service/dataflow/LiveRange.cpp b/service/dataflow/LiveRange.cpp index cc179f674f..7812cd5e0e 100644 --- a/service/dataflow/LiveRange.cpp +++ b/service/dataflow/LiveRange.cpp @@ -14,12 +14,9 @@ #include "IRCode.h" #include "ScopedCFG.h" #include "Show.h" -#include "Timer.h" namespace { -AccumulatingTimer s_timer("live_range"); - using namespace live_range; /* @@ -80,8 +77,6 @@ void replay_analysis_with_callback(const cfg::ControlFlowGraph& cfg, const Iter& iter, bool ignore_unreachable, Fn f) { - auto timer_scope = s_timer.scope(); - for (cfg::Block* block : cfg.blocks()) { auto defs_in = iter.get_entry_state_at(block); if (ignore_unreachable && defs_in.is_bottom()) { @@ -93,16 +88,9 @@ void replay_analysis_with_callback(const cfg::ControlFlowGraph& cfg, Use use{insn, i}; auto src = insn->src(i); const auto& defs = defs_in.get(src); - if (defs.is_top() || defs.empty()) { - if (iter.has_filter()) { - continue; - } - not_reached_log("Found use without def when processing [0x%p]%s", - &mie, SHOW(insn)); - } - always_assert_log(!defs.is_bottom(), - "Found unreachable use when processing [0x%p]%s", - &mie, SHOW(insn)); + always_assert_log( + !defs.is_top() && !defs.is_bottom() && defs.size() > 0, + "Found use without def when processing [0x%p]%s", &mie, SHOW(insn)); f(use, defs); } iter.analyze_instruction(insn, &defs_in); @@ -146,13 +134,8 @@ bool Use::operator==(const Use& that) const { return insn == that.insn && src_index == that.src_index; } -Chains::Chains(const cfg::ControlFlowGraph& cfg, - bool ignore_unreachable, - reaching_defs::Filter filter) - : m_cfg(cfg), - m_fp_iter(cfg, std::move(filter)), - m_ignore_unreachable(ignore_unreachable) { - auto timer_scope = s_timer.scope(); +Chains::Chains(const cfg::ControlFlowGraph& cfg, bool ignore_unreachable) + : m_cfg(cfg), m_fp_iter(cfg), m_ignore_unreachable(ignore_unreachable) { m_fp_iter.run(reaching_defs::Environment()); } @@ -165,12 +148,8 @@ DefUseChains Chains::get_def_use_chains() const { } MoveAwareChains::MoveAwareChains(const cfg::ControlFlowGraph& cfg, - bool ignore_unreachable, - reaching_defs::Filter filter) - : m_cfg(cfg), - m_fp_iter(cfg, std::move(filter)), - m_ignore_unreachable(ignore_unreachable) { - auto timer_scope = s_timer.scope(); + bool ignore_unreachable) + : m_cfg(cfg), m_fp_iter(cfg), m_ignore_unreachable(ignore_unreachable) { m_fp_iter.run(reaching_defs::Environment()); } diff --git a/service/dataflow/LiveRange.h b/service/dataflow/LiveRange.h index 6dbd2528a4..e538bee222 100644 --- a/service/dataflow/LiveRange.h +++ b/service/dataflow/LiveRange.h @@ -56,14 +56,12 @@ struct hash { namespace live_range { using UseDefChains = std::unordered_map>; -using Uses = std::unordered_set; using DefUseChains = std::unordered_map>; class Chains { public: explicit Chains(const cfg::ControlFlowGraph& cfg, - bool ignore_unreachable = false, - reaching_defs::Filter filter = nullptr); + bool ignore_unreachable = false); UseDefChains get_use_def_chains() const; DefUseChains get_def_use_chains() const; const reaching_defs::FixpointIterator& get_fp_iter() const { @@ -79,8 +77,7 @@ class Chains { class MoveAwareChains { public: explicit MoveAwareChains(const cfg::ControlFlowGraph& cfg, - bool ignore_unreachable = false, - reaching_defs::Filter filter = nullptr); + bool ignore_unreachable = false); UseDefChains get_use_def_chains() const; DefUseChains get_def_use_chains() const; const reaching_defs::MoveAwareFixpointIterator& get_fp_iter() const { diff --git a/service/dataflow/ReachingDefinitions.h b/service/dataflow/ReachingDefinitions.h index b33ebc9115..10521740b5 100644 --- a/service/dataflow/ReachingDefinitions.h +++ b/service/dataflow/ReachingDefinitions.h @@ -11,8 +11,6 @@ #include #include -#include - #include "BaseIRAnalyzer.h" #include "ControlFlow.h" #include "DexClass.h" @@ -24,43 +22,25 @@ using Domain = sparta::PatriciaTreeSetAbstractDomain; using Environment = sparta::PatriciaTreeMapAbstractEnvironment; -using Filter = std::function; - class FixpointIterator final : public ir_analyzer::BaseIRAnalyzer { - Filter m_filter; - public: - explicit FixpointIterator(const cfg::ControlFlowGraph& cfg, - Filter filter = nullptr) - : ir_analyzer::BaseIRAnalyzer(cfg), - m_filter(std::move(filter)) {} + explicit FixpointIterator(const cfg::ControlFlowGraph& cfg) + : ir_analyzer::BaseIRAnalyzer(cfg) {} void analyze_instruction(const IRInstruction* insn, Environment* current_state) const override { if (insn->has_dest()) { - current_state->set(insn->dest(), make_domain(insn)); - } - } - - bool has_filter() const { return m_filter != nullptr; } - - Domain make_domain(const IRInstruction* insn) const { - if (!m_filter || m_filter(insn)) { - return Domain(const_cast(insn)); + current_state->set(insn->dest(), + Domain(const_cast(insn))); } - return Domain(); } }; class MoveAwareFixpointIterator final : public ir_analyzer::BaseIRAnalyzer { - Filter m_filter; - public: - explicit MoveAwareFixpointIterator(const cfg::ControlFlowGraph& cfg, - Filter filter = nullptr) - : ir_analyzer::BaseIRAnalyzer(cfg), - m_filter(std::move(filter)) {} + explicit MoveAwareFixpointIterator(const cfg::ControlFlowGraph& cfg) + : ir_analyzer::BaseIRAnalyzer(cfg) {} void analyze_instruction(const IRInstruction* insn, Environment* current_state) const override { @@ -70,19 +50,12 @@ class MoveAwareFixpointIterator final current_state->set(insn->dest(), current_state->get(RESULT_REGISTER)); current_state->set(RESULT_REGISTER, Domain::top()); } else if (insn->has_move_result_any()) { - current_state->set(RESULT_REGISTER, make_domain(insn)); + current_state->set(RESULT_REGISTER, + Domain(const_cast(insn))); } else if (insn->has_dest()) { - current_state->set(insn->dest(), make_domain(insn)); - } - } - - bool has_filter() const { return m_filter != nullptr; } - - Domain make_domain(const IRInstruction* insn) const { - if (!m_filter || m_filter(insn)) { - return Domain(const_cast(insn)); + current_state->set(insn->dest(), + Domain(const_cast(insn))); } - return Domain(); } }; diff --git a/service/escape-analysis/BlamingAnalysis.h b/service/escape-analysis/BlamingAnalysis.h index 2370fa3f7c..8bf3df31d7 100644 --- a/service/escape-analysis/BlamingAnalysis.h +++ b/service/escape-analysis/BlamingAnalysis.h @@ -222,9 +222,7 @@ class BlameMap { const BlameStore::Value m_value; }; - // TODO: Tidy complains about an unnecessary copy when using a value type. - // This indicates that a move constructor may be missing for Domain. - explicit BlameMap(const BlameStore::Domain& domain) : m_domain(domain) {} + explicit BlameMap(BlameStore::Domain domain) : m_domain(std::move(domain)) {} size_t size() const { return m_domain.size(); } diff --git a/service/escape-analysis/LocalPointersAnalysis.cpp b/service/escape-analysis/LocalPointersAnalysis.cpp index 7a8c7acb9b..c613bb97a0 100644 --- a/service/escape-analysis/LocalPointersAnalysis.cpp +++ b/service/escape-analysis/LocalPointersAnalysis.cpp @@ -353,7 +353,7 @@ void FixpointIterator::analyze_instruction(const IRInstruction* insn, const auto& summary = m_invoke_to_summary_map.at(insn); analyze_invoke_with_summary(summary, insn, env); } else { - analyze_generic_invoke(insn, env); + default_instruction_handler(insn, env); } } else if (may_alloc(op)) { env->set_fresh_pointer(RESULT_REGISTER, insn); @@ -367,88 +367,80 @@ void FixpointIterator::analyze_instruction(const IRInstruction* insn, } } -std::pair, EscapeSummary> analyze_method( +void FixpointIteratorMapDeleter::operator()(FixpointIteratorMap* map) { + // Deletion is actually really expensive due to the reference counts of the + // shared_ptrs in the Patricia trees, so we do it in parallel. + auto wq = workqueue_foreach( + [](FixpointIterator* fp_iter) { delete fp_iter; }); + for (auto& pair : *map) { + wq.add_item(pair.second); + } + wq.run_all(); + delete map; +} + +static void analyze_method_recursive( const DexMethod* method, const call_graph::Graph& call_graph, - const SummaryMap& summary_map) { + sparta::PatriciaTreeSet visiting, + FixpointIteratorMap* fp_iter_map, + SummaryCMap* summary_map) { + if (!method || summary_map->count(method) != 0 || visiting.contains(method) || + method->get_code() == nullptr) { + return; + } + visiting.insert(method); + std::unordered_map invoke_to_summary_map; if (call_graph.has_node(method)) { const auto& callee_edges = call_graph.node(method)->callees(); for (const auto& edge : callee_edges) { - if (edge->callee() == call_graph.exit()) { - continue; - } - auto invoke_insn = edge->invoke_insn(); - auto& callee_summary = invoke_to_summary_map[invoke_insn]; auto* callee = edge->callee()->method(); - auto it = summary_map.find(callee); - if (it != summary_map.end()) { - callee_summary.join_with(it->second); - } else if (callee == nullptr && - is_array_clone(invoke_insn->get_method())) { - // The array clone method doesn't escape or return any parameters; it - // returns a new array. - callee_summary.join_with(EscapeSummary(ParamSet{FRESH_RETURN}, {})); + analyze_method_recursive(callee, call_graph, visiting, fp_iter_map, + summary_map); + if (summary_map->count(callee) != 0) { + invoke_to_summary_map.emplace(edge->invoke_insn(), + summary_map->at(callee)); } } } auto* code = method->get_code(); auto& cfg = code->cfg(); - auto fp_iter = - std::make_unique(cfg, - std::move(invoke_to_summary_map), - /* escape_check_cast */ false); + auto fp_iter = new FixpointIterator(cfg, std::move(invoke_to_summary_map)); fp_iter->run(Environment()); - auto summary = get_escape_summary(*fp_iter, *code); - return std::make_pair(std::move(fp_iter), std::move(summary)); + // The following updates form a critical section. + { + std::unique_lock lock(fp_iter_map->get_lock(method)); + fp_iter_map->update_unsafe(method, + [&](auto, FixpointIterator*& v, bool exists) { + redex_assert(!(exists ^ (v != nullptr))); + delete v; + v = fp_iter; + }); + summary_map->update(method, [&](auto, EscapeSummary& v, bool) { + v = get_escape_summary(*fp_iter, *code); + }); + } } -FixpointIteratorMap analyze_scope(const Scope& scope, - const call_graph::Graph& call_graph, - SummaryMap* summary_map_ptr) { - FixpointIteratorMap fp_iter_map; - SummaryMap summary_map; +FixpointIteratorMapPtr analyze_scope(const Scope& scope, + const call_graph::Graph& call_graph, + SummaryCMap* summary_map_ptr) { + FixpointIteratorMapPtr fp_iter_map(new FixpointIteratorMap()); + SummaryCMap summary_map; if (summary_map_ptr == nullptr) { summary_map_ptr = &summary_map; } - summary_map_ptr->emplace(method::java_lang_Object_ctor(), EscapeSummary{}); + summary_map_ptr->emplace( + DexMethod::get_method("Ljava/lang/Object;.:()V"), EscapeSummary{}); - auto affected_methods = std::make_unique>(); - walk::parallel::code(scope, [&](const DexMethod* method, IRCode&) { - affected_methods->insert(method); + walk::parallel::code(scope, [&](const DexMethod* method, IRCode& code) { + sparta::PatriciaTreeSet visiting; + analyze_method_recursive(method, call_graph, visiting, fp_iter_map.get(), + summary_map_ptr); }); - - while (!affected_methods->empty()) { - InsertOnlyConcurrentMap - changed_effect_summaries; - auto next_affected_methods = - std::make_unique>(); - workqueue_run( - [&](const DexMethod* method) { - auto p = analyze_method(method, call_graph, *summary_map_ptr); - auto& new_fp_iter = p.first; - auto& new_summary = p.second; - fp_iter_map.update(method, [&](auto*, auto& v, bool exists) { - redex_assert(!(exists ^ (v != nullptr))); - std::swap(new_fp_iter, v); - }); - auto it = summary_map_ptr->find(method); - if (it != summary_map_ptr->end() && it->second == new_summary) { - return; - } - changed_effect_summaries.emplace(method, std::move(new_summary)); - const auto& callers = call_graph.get_callers(method); - next_affected_methods->insert(callers.begin(), callers.end()); - }, - *affected_methods); - for (auto&& [method, summary] : changed_effect_summaries) { - (*summary_map_ptr)[method] = std::move(summary); - } - std::swap(next_affected_methods, affected_methods); - } - return fp_iter_map; } @@ -509,7 +501,6 @@ EscapeSummary get_escape_summary(const FixpointIterator& fp_iter, switch (returned_ptrs.kind()) { case sparta::AbstractValueKind::Value: { - summary.returned_parameters = ParamSet(); for (auto insn : returned_ptrs.elements()) { if (insn->opcode() == IOPCODE_LOAD_PARAM_OBJECT) { summary.returned_parameters.add(param_indexes.at(insn)); @@ -605,7 +596,6 @@ EscapeSummary EscapeSummary::from_s_expr(const sparta::s_expr& expr) { } } else { always_assert(returned_params_s_expr.is_list()); - summary.returned_parameters = ParamSet(); for (size_t i = 0; i < returned_params_s_expr.size(); ++i) { summary.returned_parameters.add(returned_params_s_expr[i].get_int32()); } @@ -613,20 +603,4 @@ EscapeSummary EscapeSummary::from_s_expr(const sparta::s_expr& expr) { return summary; } -void EscapeSummary::join_with(const EscapeSummary& other) { - escaping_parameters.insert(other.escaping_parameters.begin(), - other.escaping_parameters.end()); - returned_parameters.join_with(other.returned_parameters); -} - -bool may_be_overridden(const DexMethod* method) { - return method->is_virtual() && !is_final(method) && - !is_final(type_class(method->get_class())); -} - -bool is_array_clone(const DexMethodRef* mref) { - return type::is_array(mref->get_class()) && - mref->get_name()->str() == "clone"; -} - } // namespace local_pointers diff --git a/service/escape-analysis/LocalPointersAnalysis.h b/service/escape-analysis/LocalPointersAnalysis.h index 228e11b2f8..c7b640de27 100644 --- a/service/escape-analysis/LocalPointersAnalysis.h +++ b/service/escape-analysis/LocalPointersAnalysis.h @@ -60,7 +60,7 @@ class EnvironmentWithStore { virtual bool may_have_escaped(const IRInstruction* ptr) const = 0; - const PointerSet& get_pointers(reg_t reg) const { + PointerSet get_pointers(reg_t reg) const { return get_pointer_environment().get(reg); } @@ -119,7 +119,7 @@ class EnvironmentWithStoreImpl final void set_pointers(reg_t reg, PointerSet pset) override { Base::template apply<0>( - [&](PointerEnvironment* penv) { penv->set(reg, std::move(pset)); }); + [&](PointerEnvironment* penv) { penv->set(reg, pset); }); } void set_fresh_pointer(reg_t reg, const IRInstruction* pointer) override { @@ -141,7 +141,7 @@ class EnvironmentWithStoreImpl final template void update_store(reg_t reg, F updater) { - const auto& pointers = get_pointers(reg); + auto pointers = get_pointers(reg); if (!pointers.is_value()) { return; } @@ -196,23 +196,16 @@ struct EscapeSummary { // Note that if only some of the returned values are parameters, this will be // set to Top. A non-extremal value indicates that the return value must be // an element of the set. - ParamSet returned_parameters = ParamSet::bottom(); + ParamSet returned_parameters; EscapeSummary() = default; - // TODO: Tidy complains about an unnecessary copy when using a value type. - // This indicates that a move constructor may be missing for ParamSet. - EscapeSummary(const ParamSet& ps, std::initializer_list l) - : escaping_parameters(l), returned_parameters(ps) {} + EscapeSummary(std::initializer_list l) : escaping_parameters(l) {} - static EscapeSummary from_s_expr(const sparta::s_expr&); - - bool operator==(const EscapeSummary& other) const { - return escaping_parameters == other.escaping_parameters && - returned_parameters == other.returned_parameters; - } + EscapeSummary(ParamSet ps, std::initializer_list l) + : escaping_parameters(l), returned_parameters(std::move(ps)) {} - void join_with(const EscapeSummary& other); + static EscapeSummary from_s_expr(const sparta::s_expr&); }; std::ostream& operator<<(std::ostream& o, const EscapeSummary& summary); @@ -300,20 +293,29 @@ void default_instruction_handler(const IRInstruction* insn, EnvironmentWithStore* env); using FixpointIteratorMap = - ConcurrentMap>; + ConcurrentMap; + +struct FixpointIteratorMapDeleter final { + void operator()(FixpointIteratorMap*); +}; + +using FixpointIteratorMapPtr = + std::unique_ptr; using SummaryMap = std::unordered_map; +using SummaryCMap = ConcurrentMap; + /* * Analyze all methods in scope, making sure to analyze the callees before * their callers. * - * If a non-null SummaryMap pointer is passed in, it will get populated + * If a non-null SummaryCMap pointer is passed in, it will get populated * with the escape summaries of the methods in scope. */ -FixpointIteratorMap analyze_scope(const Scope&, - const call_graph::Graph&, - SummaryMap* = nullptr); +FixpointIteratorMapPtr analyze_scope(const Scope&, + const call_graph::Graph&, + SummaryCMap* = nullptr); /* * Join over all possible returned and thrown values. @@ -332,10 +334,4 @@ void collect_exiting_pointers(const FixpointIterator& fp_iter, EscapeSummary get_escape_summary(const FixpointIterator& fp_iter, const IRCode& code); -/* Whether a method is virtual but not final, or in a final class. */ -bool may_be_overridden(const DexMethod*); - -// Whether a given method ref is a method called "clone" defined on an array. -bool is_array_clone(const DexMethodRef*); - } // namespace local_pointers diff --git a/service/field-ops/FieldOpTracker.cpp b/service/field-ops/FieldOpTracker.cpp index 11061b9fd3..52ebc17cb7 100644 --- a/service/field-ops/FieldOpTracker.cpp +++ b/service/field-ops/FieldOpTracker.cpp @@ -39,52 +39,26 @@ struct Escapes { // Escape information for instructions define a value using InstructionEscapes = std::unordered_map; -bool operator==(const Escapes& a, const Escapes& b) { - return a.put_value_fields == b.put_value_fields && - a.invoked_ctors == b.invoked_ctors && a.other == b.other; -} - -bool operator==(const InstructionEscapes& a, const InstructionEscapes& b) { - if (a.size() != b.size()) { - return false; - } - for (auto&& [insn, escapes] : a) { - auto it = b.find(insn); - if (it == b.end()) { - return false; - } - if (!(it->second == escapes)) { - return false; - } - } - return true; -}; - class WritesAnalyzer { private: const field_op_tracker::TypeLifetimes* m_type_lifetimes; const field_op_tracker::FieldStatsMap& m_field_stats; - mutable InsertOnlyConcurrentMap + std::unordered_map m_method_insn_escapes; + // We track the set of actively analyzed methods to find recursive cases. + std::unordered_set m_active; - bool has_lifetime(const DexType* t) const { + bool has_lifetime(const DexType* t) { return (!m_type_lifetimes && type::is_object(t)) || m_type_lifetimes->has_lifetime(t); } - const InstructionEscapes& get_insn_escapes(const DexMethod* method) const { - return *m_method_insn_escapes - .get_or_create_and_assert_equal( - method, [&](auto*) { return compute_insn_escapes(method); }) - .first; - } - // Compute information about which values (represented by // instructions that create them) escape by being stored in fields, array // elements, passed as the first argument to a constructor, or escape // otherwise. We also record writing a non-zero value to a field / array // element with a (relevant) lifetime type as an "other" escape. - InstructionEscapes compute_insn_escapes(const DexMethod* method) const { + InstructionEscapes compute_insn_escapes(const DexMethod* method) { auto& cfg = method->get_code()->cfg(); Lazy type_inference([&cfg, method] { auto res = std::make_unique(cfg); @@ -244,8 +218,7 @@ class WritesAnalyzer { } // Whether a constructor can store any values with (relevant) lifetimes - bool may_capture(const sparta::PatriciaTreeSet& active, - const DexMethodRef* method) const { + bool may_capture(const DexMethodRef* method) { always_assert(method::is_init(method)); auto type = method->get_class(); if (type == type::java_lang_Object()) { @@ -258,22 +231,21 @@ class WritesAnalyzer { if (cls == nullptr || cls->is_external()) { return true; } - bool any_non_vestigial_objects_written_fields{false}; + std::unordered_set non_vestigial_objects_written_fields; std::unordered_set invoked_base_ctors; bool other_escapes{false}; - if (!get_writes(active, method->as_def(), + if (!get_writes(method->as_def(), /* non_zero_written_fields */ nullptr, - /* non_vestigial_objects_written_fields */ nullptr, - &any_non_vestigial_objects_written_fields, - &invoked_base_ctors, &other_escapes)) { + &non_vestigial_objects_written_fields, &invoked_base_ctors, + &other_escapes)) { // mutual recursion across constructor invocations, which can happen when // a constructor creates a new object of some other type return true; } - if (other_escapes || any_non_vestigial_objects_written_fields || + if (other_escapes || !non_vestigial_objects_written_fields.empty() || (std::find_if(invoked_base_ctors.begin(), invoked_base_ctors.end(), - [&](DexMethod* invoked_base_ctor) { - return may_capture(active, invoked_base_ctor); + [this](DexMethod* invoked_base_ctor) { + return may_capture(invoked_base_ctor); }) != invoked_base_ctors.end())) { return true; } @@ -282,9 +254,8 @@ class WritesAnalyzer { // Whether a newly created object may capture any values with (relevant) // lifetimes, or itself, as part of its creation - bool may_capture(const sparta::PatriciaTreeSet& active, - const IRInstruction* insn, - const std::unordered_set& invoked_ctors) const { + bool may_capture(const IRInstruction* insn, + const std::unordered_set& invoked_ctors) { switch (insn->opcode()) { case OPCODE_NEW_ARRAY: return false; @@ -296,7 +267,7 @@ class WritesAnalyzer { always_assert(!invoked_ctors.empty()); for (auto method : invoked_ctors) { always_assert(method->get_class() == insn->get_type()); - if (may_capture(active, method)) { + if (may_capture(method)) { return true; } } @@ -310,9 +281,16 @@ class WritesAnalyzer { explicit WritesAnalyzer(const Scope& scope, const field_op_tracker::FieldStatsMap& field_stats, const field_op_tracker::TypeLifetimes* type_lifetimes) - : m_type_lifetimes(type_lifetimes), m_field_stats(field_stats) {} + : m_type_lifetimes(type_lifetimes), m_field_stats(field_stats) { + walk::code(scope, [this](const DexMethod* method, const IRCode&) { + m_method_insn_escapes[method]; + }); + walk::parallel::code(scope, [&](const DexMethod* method, const IRCode&) { + m_method_insn_escapes.at(method) = compute_insn_escapes(method); + }); + } - bool any_read(const std::unordered_set& fields) const { + bool any_read(const std::unordered_set& fields) { for (auto field : fields) { if (m_field_stats.at(field).reads != 0) { return true; @@ -323,19 +301,15 @@ class WritesAnalyzer { // Result indicates whether we ran into a recursive case. bool get_writes( - const sparta::PatriciaTreeSet& old_active, const DexMethod* method, - ConcurrentSet* non_zero_written_fields, - ConcurrentSet* non_vestigial_objects_written_fields, - bool* any_non_vestigial_objects_written_fields, + std::unordered_set* non_zero_written_fields, + std::unordered_set* non_vestigial_objects_written_fields, std::unordered_set* invoked_base_ctors, - bool* other_escapes) const { - auto active = old_active; - active.insert(method); - if (active.reference_equals(old_active)) { + bool* other_escapes) { + if (!m_active.insert(method).second) { return false; } - auto& insn_escapes = get_insn_escapes(method); + auto& insn_escapes = m_method_insn_escapes.at(method); auto init_load_param_this = method::is_init(method) ? method->get_code()->cfg().get_param_instructions().front().insn @@ -353,19 +327,14 @@ class WritesAnalyzer { opcode::is_a_new(insn->opcode()) && !(any_read(escapes.put_value_fields) || escapes.other) && !(has_lifetime(insn->get_type()) && - may_capture(active, insn, escapes.invoked_ctors)); + may_capture(insn, escapes.invoked_ctors)); for (auto field : escapes.put_value_fields) { always_assert(field != nullptr); if (non_zero_written_fields) { non_zero_written_fields->insert(field); } if (!is_vestigial_object && has_lifetime(field->get_type())) { - if (non_vestigial_objects_written_fields) { - non_vestigial_objects_written_fields->insert(field); - } - if (any_non_vestigial_objects_written_fields) { - *any_non_vestigial_objects_written_fields = true; - } + non_vestigial_objects_written_fields->insert(field); } } if (!escapes.invoked_ctors.empty() && invoked_base_ctors && @@ -377,6 +346,7 @@ class WritesAnalyzer { *other_escapes = true; } } + m_active.erase(method); return true; } }; @@ -411,20 +381,22 @@ bool TypeLifetimes::has_lifetime(const DexType* t) const { return true; } -void analyze_writes(const Scope& scope, - const FieldStatsMap& field_stats, - const TypeLifetimes* type_lifetimes, - FieldWrites* res) { +FieldWrites analyze_writes(const Scope& scope, + const FieldStatsMap& field_stats, + const TypeLifetimes* type_lifetimes) { + std::unordered_set non_zero_written_fields; + std::unordered_set non_vestigial_objects_written_fields; WritesAnalyzer analyzer(scope, field_stats, type_lifetimes); - walk::parallel::code(scope, [&](const DexMethod* method, const IRCode&) { - auto success = analyzer.get_writes( - /*active*/ {}, method, &res->non_zero_written_fields, - &res->non_vestigial_objects_written_fields, - /* any_non_vestigial_objects_written_fields */ nullptr, - /* invoked_base_ctors */ nullptr, - /* other_escapes */ nullptr); + walk::code(scope, [&](const DexMethod* method, const IRCode&) { + auto success = analyzer.get_writes(method, + &non_zero_written_fields, + &non_vestigial_objects_written_fields, + /* invoked_base_ctors */ nullptr, + /* other_escapes */ nullptr); always_assert(success); }); + return FieldWrites{non_zero_written_fields, + non_vestigial_objects_written_fields}; }; FieldStatsMap analyze(const Scope& scope) { diff --git a/service/field-ops/FieldOpTracker.h b/service/field-ops/FieldOpTracker.h index 3b6c576c7a..d94671edca 100644 --- a/service/field-ops/FieldOpTracker.h +++ b/service/field-ops/FieldOpTracker.h @@ -38,6 +38,7 @@ class TypeLifetimes { private: std::unordered_set m_ignored_types; const DexType* m_java_lang_Enum; + mutable ConcurrentMap> m_cache; public: TypeLifetimes(); @@ -50,7 +51,7 @@ FieldStatsMap analyze(const Scope& scope); struct FieldWrites { // All fields to which some potentially non-zero value is written. - ConcurrentSet non_zero_written_fields; + std::unordered_set non_zero_written_fields; // All fields to which some non-vestigial object is written. // We say an object is "vestigial" when the only escaping reference to it is // stored in a particular field. In other words, the only way to retrieve and @@ -58,11 +59,10 @@ struct FieldWrites { // is unread, we can remove the iput/sput to it, as it is not possible that // the object's lifetime can be observed by a weak reference, at least after // the storing method returns. - ConcurrentSet non_vestigial_objects_written_fields; + std::unordered_set non_vestigial_objects_written_fields; }; -void analyze_writes(const Scope& scope, - const FieldStatsMap& field_stats, - const TypeLifetimes* type_lifetimes, - FieldWrites* res); +FieldWrites analyze_writes(const Scope& scope, + const FieldStatsMap& field_stats, + const TypeLifetimes* type_lifetimes = nullptr); } // namespace field_op_tracker diff --git a/service/init-classes/InitClassBackwardAnalysis.h b/service/init-classes/InitClassBackwardAnalysis.h index 8e1cebc71b..75cf0ce2f1 100644 --- a/service/init-classes/InitClassBackwardAnalysis.h +++ b/service/init-classes/InitClassBackwardAnalysis.h @@ -15,7 +15,6 @@ #include "BaseIRAnalyzer.h" #include "DexUtil.h" -#include "InitClassesWithSideEffects.h" namespace init_classes { diff --git a/service/kotlin-instance-rewrite/KotlinInstanceRewriter.cpp b/service/kotlin-instance-rewrite/KotlinInstanceRewriter.cpp index 33fd705467..9cf1b1665f 100644 --- a/service/kotlin-instance-rewrite/KotlinInstanceRewriter.cpp +++ b/service/kotlin-instance-rewrite/KotlinInstanceRewriter.cpp @@ -8,6 +8,7 @@ #include "KotlinInstanceRewriter.h" #include "CFGMutation.h" #include "PassManager.h" +#include "ScopedCFG.h" #include "Show.h" #include "Walkers.h" @@ -72,9 +73,8 @@ KotlinInstanceRewriter::Stats KotlinInstanceRewriter::remove_escaping_instance( return stats; } - always_assert(method->get_code()->editable_cfg_built()); - auto& cfg = method->get_code()->cfg(); - auto iterable = cfg::InstructionIterable(cfg); + cfg::ScopedCFG cfg(method->get_code()); + auto iterable = cfg::InstructionIterable(*cfg); for (auto it = iterable.begin(); it != iterable.end(); it++) { auto insn = it->insn; @@ -147,11 +147,11 @@ KotlinInstanceRewriter::Stats KotlinInstanceRewriter::transform( auto dmethods = cls->get_dmethods(); for (auto* meth : dmethods) { if (method::is_clinit(meth)) { - auto& cfg = meth->get_code()->cfg(); - cfg::CFGMutation m(cfg); + cfg::ScopedCFG cfg(meth->get_code()); + cfg::CFGMutation m(*cfg); TRACE(KOTLIN_INSTANCE, 5, "%s before\n%s", SHOW(cls), - SHOW(cfg)); - auto iterable = cfg::InstructionIterable(cfg); + SHOW(*cfg)); + auto iterable = cfg::InstructionIterable(*cfg); for (auto insn_it = iterable.begin(); insn_it != iterable.end(); insn_it++) { auto insn = insn_it->insn; @@ -165,7 +165,7 @@ KotlinInstanceRewriter::Stats KotlinInstanceRewriter::transform( } m.flush(); TRACE(KOTLIN_INSTANCE, 5, "%s after\n%s", SHOW(cls), - SHOW(cfg)); + SHOW(*cfg)); break; } } @@ -173,23 +173,23 @@ KotlinInstanceRewriter::Stats KotlinInstanceRewriter::transform( // Convert INSTANCE read to new instance creation for (auto& method_it : concurrent_instance_map.find(field)->second) { auto* meth = method_it.second; - auto& cfg = meth->get_code()->cfg(); - cfg::CFGMutation m(cfg); - TRACE(KOTLIN_INSTANCE, 5, "%s before\n%s", SHOW(meth), SHOW(cfg)); + cfg::ScopedCFG cfg(meth->get_code()); + cfg::CFGMutation m(*cfg); + TRACE(KOTLIN_INSTANCE, 5, "%s before\n%s", SHOW(meth), SHOW(*cfg)); DexMethodRef* init = DexMethod::get_method( cls->get_type(), DexString::make_string(""), DexProto::make_proto(type::_void(), DexTypeList::make_type_list({}))); always_assert(init); // Make this constructor publcic set_public(init->as_def()); - auto iterable = cfg::InstructionIterable(cfg); + auto iterable = cfg::InstructionIterable(*cfg); for (auto insn_it = iterable.begin(); insn_it != iterable.end(); insn_it++) { auto insn = insn_it->insn; if (!opcode::is_an_sget(insn->opcode()) || insn->get_field() != field) { continue; } - auto move_result_it = cfg.move_result_of(insn_it); + auto move_result_it = cfg->move_result_of(insn_it); IRInstruction* new_isn = new IRInstruction(OPCODE_NEW_INSTANCE); new_isn->set_type(cls->get_type()); IRInstruction* mov_result = @@ -203,7 +203,7 @@ KotlinInstanceRewriter::Stats KotlinInstanceRewriter::transform( stats.kotlin_new_inserted++; } m.flush(); - TRACE(KOTLIN_INSTANCE, 5, "%s after\n%s", SHOW(meth), SHOW(cfg)); + TRACE(KOTLIN_INSTANCE, 5, "%s after\n%s", SHOW(meth), SHOW(*cfg)); } cls->remove_field(resolve_field(field)); } diff --git a/service/local-dce/LocalDce.cpp b/service/local-dce/LocalDce.cpp index 3fc2130fda..78b898cbc4 100644 --- a/service/local-dce/LocalDce.cpp +++ b/service/local-dce/LocalDce.cpp @@ -183,17 +183,13 @@ void LocalDce::dce(cfg::ControlFlowGraph& cfg, const IRList::iterator& it = pair.second; auto insn = it->insn; auto cfg_it = b->to_cfg_instruction_iterator(it); - auto has_no_implementors = [&](auto* method) { - if (method == nullptr) { - return false; - } - return !has_implementor(m_method_override_graph, method); - }; + DexMethod* method; if (m_may_allocate_registers && m_method_override_graph && (insn->opcode() == OPCODE_INVOKE_VIRTUAL || insn->opcode() == OPCODE_INVOKE_INTERFACE) && - has_no_implementors( - resolve_method(insn->get_method(), opcode_to_search(insn)))) { + (method = resolve_method(insn->get_method(), opcode_to_search(insn))) != + nullptr && + !has_implementor(m_method_override_graph, method)) { TRACE(DCE, 2, "DEAD NPE: %s", SHOW(insn)); if (!npe_creator) { npe_creator = std::make_unique(&cfg); @@ -317,9 +313,6 @@ bool LocalDce::assumenosideeffects(DexMethodRef* ref, DexMethod* meth) { } void LocalDce::normalize_new_instances(cfg::ControlFlowGraph& cfg) { - static AccumulatingTimer s_timer("LocalDce::normalize_new_instances"); - auto t = s_timer.scope(); - // TODO: This normalization optimization doesn't really belong to local-dce, // but it combines nicely as local-dce will clean-up redundant new-instance // instructions and moves afterwards. @@ -338,8 +331,7 @@ void LocalDce::normalize_new_instances(cfg::ControlFlowGraph& cfg) { } cfg::CFGMutation mutation(cfg); - reaching_defs::MoveAwareFixpointIterator fp_iter( - cfg, [&](auto* insn) { return opcode::is_new_instance(insn->opcode()); }); + reaching_defs::MoveAwareFixpointIterator fp_iter(cfg); fp_iter.run({}); for (cfg::Block* block : cfg.blocks()) { auto env = fp_iter.get_entry_state_at(block); @@ -359,12 +351,18 @@ void LocalDce::normalize_new_instances(cfg::ControlFlowGraph& cfg) { const auto& defs = env.get(reg); always_assert(!defs.is_top()); always_assert(!defs.is_bottom()); - if (defs.empty()) { + IRInstruction* old_new_instance_insn{nullptr}; + for (auto def : defs.elements()) { + if (def->opcode() == OPCODE_NEW_INSTANCE) { + always_assert(old_new_instance_insn == nullptr); + old_new_instance_insn = def; + always_assert(def->get_type() == type); + } + } + if (old_new_instance_insn == nullptr) { // base constructor invocation continue; } - always_assert(defs.size() == 1); - IRInstruction* old_new_instance_insn = *defs.elements().begin(); if (last_insn != end && last_insn->insn->opcode() == IOPCODE_MOVE_RESULT_PSEUDO_OBJECT && last_insn->insn->dest() == reg) { diff --git a/service/loop-info/LoopInfo.cpp b/service/loop-info/LoopInfo.cpp index bc1162da77..227ea781ac 100644 --- a/service/loop-info/LoopInfo.cpp +++ b/service/loop-info/LoopInfo.cpp @@ -132,6 +132,12 @@ LoopInfo::LoopInfo(cfg::ControlFlowGraph& cfg) { }); } +LoopInfo::~LoopInfo() { + for (auto* loop : m_loops) { + delete loop; + } +} + template void LoopInfo::init(T& cfg, Fn preheader_fn) { sparta::WeakTopologicalOrdering wto( @@ -190,26 +196,27 @@ void LoopInfo::init(T& cfg, Fn preheader_fn) { auto loop_header = blocks_in_loop.front(); auto loop_preheader = preheader_fn(cfg, block_set, loop_header); - // we are traversing level_order backwards, so we insert in front to make it - // level_order; also note that insertions into deques does not invalidate - // references - auto& loop = - m_loops.emplace_front(blocks_in_loop, subloops, loop_preheader); + auto loop = new Loop(blocks_in_loop, subloops, loop_preheader); + + loop_heads.emplace(loop_header, loop); + m_loops.emplace_back(loop); for (auto block : blocks_in_loop) { - m_block_location.emplace(block, &loop); + m_block_location.emplace(block, loop); } - - loop_heads.emplace(loop_header, &loop); } + // we traversed level_order backwards, so reverse loops to make it level + // order + std::reverse(m_loops.begin(), m_loops.end()); + // update parent_loop - for (auto& loop : m_loops) { + for (auto loop : m_loops) { // since we are traversing in level order, if this is true then we are done - if (loop.get_parent_loop() != nullptr) { + if (loop->get_parent_loop() != nullptr) { break; } - loop.update_parent_loop_fields(); + loop->update_parent_loop_fields(); } } diff --git a/service/loop-info/LoopInfo.h b/service/loop-info/LoopInfo.h index 623eed0bc1..40dad46d33 100644 --- a/service/loop-info/LoopInfo.h +++ b/service/loop-info/LoopInfo.h @@ -128,10 +128,11 @@ class Loop { class LoopInfo { public: - using iterator = std::deque::iterator; - using reverse_iterator = std::deque::reverse_iterator; + using iterator = std::vector::iterator; + using reverse_iterator = std::vector::reverse_iterator; explicit LoopInfo(const cfg::ControlFlowGraph& cfg); explicit LoopInfo(cfg::ControlFlowGraph& cfg); + ~LoopInfo(); Loop* get_loop_for(cfg::Block* block); size_t num_loops(); iterator begin(); @@ -143,7 +144,7 @@ class LoopInfo { template void init(Cfg& cfg, Fn fn); - std::deque m_loops; + std::vector m_loops; std::unordered_map m_loop_depth; std::unordered_map m_block_location; }; diff --git a/service/method-dedup/ConstantLifting.cpp b/service/method-dedup/ConstantLifting.cpp index 07bc117c19..2909408b51 100644 --- a/service/method-dedup/ConstantLifting.cpp +++ b/service/method-dedup/ConstantLifting.cpp @@ -8,10 +8,8 @@ #include "ConstantLifting.h" #include "AnnoUtils.h" -#include "CFGMutation.h" #include "ConstantValue.h" #include "ControlFlow.h" -#include "DexAsm.h" #include "IRCode.h" #include "MethodReference.h" #include "Resolver.h" @@ -20,8 +18,6 @@ #include "TypeReference.h" #include "TypeTags.h" -using namespace dex_asm; - namespace { constexpr const char* METHOD_META = @@ -47,22 +43,6 @@ bool overlaps_with_an_existing_virtual_scope(DexType* type, return false; } -void patch_invoke(cfg::ControlFlowGraph& meth_cfg, - cfg::CFGMutation& mutation, - const cfg::InstructionIterator& cfg_it, - IRInstruction* invoke) { - mutation.insert_before(cfg_it, {invoke}); - - auto move_res_old = meth_cfg.move_result_of(cfg_it); - if (!move_res_old.is_end()) { - auto dest = move_res_old->insn->dest(); - auto move_res_new = dasm(move_res_old->insn->opcode(), {{VREG, dest}}); - mutation.insert_before(cfg_it, {move_res_new}); - } - - mutation.remove(cfg_it); -} - } // namespace const DexType* s_method_meta_anno; @@ -150,10 +130,11 @@ std::vector ConstantLifting::lift_constants_from( if (const_val.is_invalid()) { continue; } - auto opcode = const_val.is_int_value() ? IOPCODE_LOAD_PARAM - : IOPCODE_LOAD_PARAM_OBJECT; auto load_type_tag_param = - dasm(opcode, {{VREG, const_val.get_param_reg()}}); + const_val.is_int_value() + ? new IRInstruction(IOPCODE_LOAD_PARAM) + : new IRInstruction(IOPCODE_LOAD_PARAM_OBJECT); + load_type_tag_param->set_dest(const_val.get_param_reg()); if (last_loading != block->end()) { cfg.insert_after(block->to_cfg_instruction_iterator(last_loading), load_type_tag_param); @@ -170,9 +151,11 @@ std::vector ConstantLifting::lift_constants_from( auto const_val = load.first; auto insn_it = load.second.first; auto dest = load.second.second; - auto opcode = const_val.is_int_value() ? OPCODE_MOVE : OPCODE_MOVE_OBJECT; - auto move_const_arg = - dasm(opcode, {{VREG, dest}, {VREG, const_val.get_param_reg()}}); + auto move_const_arg = const_val.is_int_value() + ? new IRInstruction(OPCODE_MOVE) + : new IRInstruction(OPCODE_MOVE_OBJECT); + move_const_arg->set_dest(dest); + move_const_arg->set_src(0, const_val.get_param_reg()); cfg.insert_before(insn_it, move_const_arg); cfg.remove_insn(insn_it); } @@ -199,7 +182,6 @@ std::vector ConstantLifting::lift_constants_from( always_assert(callee != nullptr); auto const_vals = lifted_constants.at(callee); auto& meth_cfg = meth->get_code()->cfg(); - cfg::CFGMutation mutation(meth_cfg); auto cfg_it = meth_cfg.find_insn(insn); if (const_vals.needs_stub()) { // Insert const load @@ -210,8 +192,10 @@ std::vector ConstantLifting::lift_constants_from( auto stub = const_vals.create_stub_method(callee); stub->get_code()->build_cfg(); auto invoke = method_reference::make_invoke(stub, insn->opcode(), args); - patch_invoke(meth_cfg, mutation, cfg_it, invoke); - + IRInstruction* orig_invoke = cfg_it->insn; + cfg_it->insn = invoke; + // remove original call. + delete orig_invoke; stub_methods.push_back(stub); } else { // Make const load @@ -226,12 +210,14 @@ std::vector ConstantLifting::lift_constants_from( args.push_back(insn->src(i)); } args.insert(args.end(), const_regs.begin(), const_regs.end()); - mutation.insert_before(cfg_it, const_loads); + meth_cfg.insert_before(cfg_it, const_loads); + auto orig_invoke = cfg_it->insn; auto invoke = method_reference::make_invoke(callee, insn->opcode(), args); - patch_invoke(meth_cfg, mutation, cfg_it, invoke); + // replace to the call. + cfg_it->insn = invoke; + // remove original call. + delete orig_invoke; } - - mutation.flush(); TRACE(METH_DEDUP, 9, " patched call site in %s\n%s", SHOW(meth), SHOW(meth_cfg)); } diff --git a/service/method-dedup/ConstantValue.cpp b/service/method-dedup/ConstantValue.cpp index 36e4e32e1e..85c795afe2 100644 --- a/service/method-dedup/ConstantValue.cpp +++ b/service/method-dedup/ConstantValue.cpp @@ -10,13 +10,10 @@ #include #include "Creators.h" -#include "DexAsm.h" #include "Show.h" #include "Trace.h" #include "TypeReference.h" -using namespace dex_asm; - namespace { constexpr uint64_t MAX_NUM_CONST_VALUE = 10; @@ -24,10 +21,13 @@ constexpr uint64_t MAX_NUM_CONST_VALUE = 10; std::vector make_string_const(reg_t dest, const std::string& val) { std::vector res; - auto load = dasm(OPCODE_CONST_STRING, DexString::make_string(val)); - auto move_res = dasm(IOPCODE_MOVE_RESULT_PSEUDO_OBJECT, {{VREG, dest}}); + IRInstruction* load = new IRInstruction(OPCODE_CONST_STRING); + load->set_string(DexString::make_string(val)); + IRInstruction* move_result_pseudo = + new IRInstruction(IOPCODE_MOVE_RESULT_PSEUDO_OBJECT); + move_result_pseudo->set_dest(dest); res.push_back(load); - res.push_back(move_res); + res.push_back(move_result_pseudo); return res; } @@ -230,27 +230,28 @@ DexMethod* ConstantValues::create_stub_method(DexMethod* callee) { auto name = DexString::make_string(callee->get_name()->str() + "$stub"); name = DexMethod::get_unique_name(type, name, stub_proto); TRACE(METH_DEDUP, 9, "const value: stub name %s", name->c_str()); - auto mc = MethodCreator(type, - name, - stub_proto, - callee->get_access(), - /* anno=*/nullptr, - /* with_debug_item= */ false); - auto mb = mc.get_main_block(); + auto mc = new MethodCreator(type, + name, + stub_proto, + callee->get_access(), + nullptr, // anno + false // with_debug_item + ); + auto mb = mc->get_main_block(); // Setup args for calling the callee. size_t arg_loc = 0; std::vector args; if (!is_static(callee)) { - args.push_back(mc.get_local(arg_loc++)); + args.push_back(mc->get_local(arg_loc++)); } for (size_t i = 0; i < stub_arg_list->size(); ++i) { - args.push_back(mc.get_local(arg_loc++)); + args.push_back(mc->get_local(arg_loc++)); } for (auto& cval : m_const_vals) { if (cval.is_invalid()) { continue; } - auto loc = mc.make_local(cval.get_constant_type()); + auto loc = mc->make_local(cval.get_constant_type()); if (cval.is_int_value()) { mb->load_const(loc, static_cast(cval.get_int_value())); } else { @@ -264,12 +265,12 @@ DexMethod* ConstantValues::create_stub_method(DexMethod* callee) { if (ret_type == type::_void()) { mb->ret_void(); } else { - auto ret_loc = mc.make_local(ret_type); + auto ret_loc = mc->make_local(ret_type); mb->move_result(ret_loc, ret_type); mb->ret(ret_type, ret_loc); } - auto stub = mc.create(); + auto stub = mc->create(); // Propogate deobfuscated name const auto orig_name = callee->get_deobfuscated_name_or_empty(); auto pos = orig_name.find(':'); diff --git a/service/method-dedup/MethodDedup.cpp b/service/method-dedup/MethodDedup.cpp index 9276c40a80..1833986eb0 100644 --- a/service/method-dedup/MethodDedup.cpp +++ b/service/method-dedup/MethodDedup.cpp @@ -18,11 +18,11 @@ namespace { struct CodeAsKey { - const cfg::ControlFlowGraph& cfg; + const IRCode* code; const bool dedup_fill_in_stack_trace; - CodeAsKey(cfg::ControlFlowGraph& c, bool dedup_fill_in_stack_trace) - : cfg(c), dedup_fill_in_stack_trace(dedup_fill_in_stack_trace) {} + CodeAsKey(const IRCode* c, bool dedup_fill_in_stack_trace) + : code(c), dedup_fill_in_stack_trace(dedup_fill_in_stack_trace) {} static bool non_throw_instruction_equal(const IRInstruction& left, const IRInstruction& right) { @@ -33,16 +33,16 @@ struct CodeAsKey { bool operator==(const CodeAsKey& other) const { return dedup_fill_in_stack_trace - ? cfg.structural_equals(other.cfg) - : cfg.structural_equals(other.cfg, non_throw_instruction_equal); + ? code->structural_equals(*other.code) + : code->structural_equals(*other.code, + non_throw_instruction_equal); } }; struct CodeHasher { size_t operator()(const CodeAsKey& key) const { size_t result = 0; - for (auto& mie : cfg::InstructionIterable( - const_cast(key.cfg))) { + for (auto& mie : InstructionIterable(key.code)) { result ^= mie.insn->hash(); } return result; @@ -57,8 +57,7 @@ std::vector get_duplicate_methods_simple( DuplicateMethods duplicates; for (DexMethod* method : methods) { always_assert(method->get_code()); - always_assert(method->get_code()->editable_cfg_built()); - duplicates[CodeAsKey(method->get_code()->cfg(), dedup_fill_in_stack_trace)] + duplicates[CodeAsKey(method->get_code(), dedup_fill_in_stack_trace)] .emplace(method); } diff --git a/service/method-inliner/CallSiteSummaries.cpp b/service/method-inliner/CallSiteSummaries.cpp index 93f8f65d9a..5d4be881a1 100644 --- a/service/method-inliner/CallSiteSummaries.cpp +++ b/service/method-inliner/CallSiteSummaries.cpp @@ -158,8 +158,8 @@ namespace inliner { CallSiteSummarizer::CallSiteSummarizer( shrinker::Shrinker& shrinker, - const ConcurrentMethodToMethodOccurrences& callee_caller, - const ConcurrentMethodToMethodOccurrences& caller_callee, + const MethodToMethodOccurrences& callee_caller, + const MethodToMethodOccurrences& caller_callee, GetCalleeFunction get_callee_fn, HasCalleeOtherCallSitesPredicate has_callee_other_call_sites_fn, std::function* filter_fn, @@ -175,14 +175,34 @@ CallSiteSummarizer::CallSiteSummarizer( const CallSiteSummary* CallSiteSummarizer::internalize_call_site_summary( const CallSiteSummary& call_site_summary) { - return m_call_site_summaries - .get_or_emplace_and_assert_equal(call_site_summary.get_key(), - call_site_summary) - .first; + auto key = call_site_summary.get_key(); + const CallSiteSummary* res; + m_call_site_summaries.update( + key, [&](const std::string&, + std::unique_ptr& p, + bool exist) { + if (exist) { + always_assert_log(p->result_used == call_site_summary.result_used, + "same key %s for\n %d\nvs. %d", key.c_str(), + p->result_used, call_site_summary.result_used); + always_assert_log(p->arguments.equals(call_site_summary.arguments), + "same key %s for\n %s\nvs. %s", key.c_str(), + SHOW(p->arguments), + SHOW(call_site_summary.arguments)); + } else { + p = std::make_unique(call_site_summary); + } + res = p.get(); + }); + return res; } void CallSiteSummarizer::summarize() { Timer t("compute_call_site_summaries"); + struct CalleeInfo { + std::unordered_map occurrences; + std::vector invokes; + }; // We'll do a top-down traversal of all call-sites, in order to propagate // call-site information from outer call-sites to nested call-sites, improving @@ -194,17 +214,21 @@ void CallSiteSummarizer::summarize() { // virtual methods. PriorityThreadPoolDAGScheduler summaries_scheduler; + ConcurrentMap> + concurrent_callee_infos; + // Helper function to retrieve a list of callers of a callee such that all // possible call-sites to the callee are in the returned callers. auto get_dependencies = [&](DexMethod* callee) -> const std::unordered_map* { - const auto* ptr = m_callee_caller.get_unsafe(callee); - if (ptr == nullptr || m_has_callee_other_call_sites_fn(callee)) { + auto it = m_callee_caller.find(callee); + if (it == m_callee_caller.end() || + m_has_callee_other_call_sites_fn(callee)) { return nullptr; } // If we get here, then we know all possible call-sites to the callee, and // they reside in the known list of callers. - return ptr; + return &it->second; }; summaries_scheduler.set_executor([&](DexMethod* method) { @@ -214,18 +238,11 @@ void CallSiteSummarizer::summarize() { // constant arguments. arguments = CallSiteArguments::top(); } else { - CalleeInfo* ci = nullptr; - auto success = - m_callee_infos.observe(method, [&](auto*, const auto& val) { - ci = const_cast(&val); - }); - always_assert(success == !!ci); - if (ci == nullptr) { + auto ci = concurrent_callee_infos.get(method, nullptr); + if (!ci) { // All callers were unreachable arguments = CallSiteArguments::bottom(); } else { - // Release memory for indices - ci->indices.clear(); // The only way to call this method is by going through a set of known // call-sites. We join together all those incoming constant arguments. always_assert(!ci->occurrences.empty()); @@ -242,7 +259,7 @@ void CallSiteSummarizer::summarize() { m_stats->constant_invoke_callers_unreachable++; return; } - auto& callees = m_caller_callee.at_unsafe(method); + auto& callees = m_caller_callee.at(method); ConstantEnvironment initial_env = constant_propagation::interprocedural::env_with_params( is_static(method), method->get_code(), arguments); @@ -251,27 +268,23 @@ void CallSiteSummarizer::summarize() { auto insn = p.first; auto callee = m_get_callee_fn(method, insn); auto call_site_summary = p.second; - m_callee_infos.update( - callee, [&](const DexMethod*, CalleeInfo& ci, bool /* exists */) { - auto [it, emplaced] = - ci.indices.emplace(call_site_summary, ci.indices.size()); - if (emplaced) { - ci.occurrences.emplace_back(call_site_summary, 1); - } else { - ci.occurrences[it->second].second++; - } - ci.invokes.push_back(insn); - }); + concurrent_callee_infos.update(callee, + [&](const DexMethod*, + std::shared_ptr& ci, + bool /* exists */) { + if (!ci) { + ci = std::make_shared(); + } + ci->occurrences[call_site_summary]++; + ci->invokes.push_back(insn); + }); m_invoke_call_site_summaries.emplace(insn, call_site_summary); } m_stats->constant_invoke_callers_analyzed++; - if (res.dead_blocks > 0) { - m_stats->constant_invoke_callers_unreachable_blocks += res.dead_blocks; - } + m_stats->constant_invoke_callers_unreachable_blocks += res.dead_blocks; }); std::vector callers; - callers.reserve(m_caller_callee.size()); for (auto& p : m_caller_callee) { auto method = const_cast(p.first); callers.push_back(method); @@ -284,6 +297,19 @@ void CallSiteSummarizer::summarize() { } m_stats->constant_invoke_callers_critical_path_length = summaries_scheduler.run(callers.begin(), callers.end()); + + for (auto& p : concurrent_callee_infos) { + auto callee = p.first; + auto& v = m_callee_call_site_summary_occurrences[callee]; + auto& ci = p.second; + for (const auto& q : ci->occurrences) { + const auto call_site_summary = q.first; + const auto count = q.second; + v.emplace_back(call_site_summary, count); + } + auto& invokes = m_callee_call_site_invokes[callee]; + invokes.insert(invokes.end(), ci->invokes.begin(), ci->invokes.end()); + } } InvokeCallSiteSummariesAndDeadBlocks @@ -352,21 +378,22 @@ CallSiteSummarizer::get_invoke_call_site_summaries( const std::vector* CallSiteSummarizer::get_callee_call_site_summary_occurrences( const DexMethod* callee) const { - auto ptr = m_callee_infos.get_unsafe(callee); - return ptr == nullptr ? nullptr : &ptr->occurrences; + auto it = m_callee_call_site_summary_occurrences.find(callee); + return it == m_callee_call_site_summary_occurrences.end() ? nullptr + : &it->second; } const std::vector* CallSiteSummarizer::get_callee_call_site_invokes( const DexMethod* callee) const { - auto ptr = m_callee_infos.get_unsafe(callee); - return ptr == nullptr ? nullptr : &ptr->invokes; + auto it = m_callee_call_site_invokes.find(callee); + return it == m_callee_call_site_invokes.end() ? nullptr : &it->second; } const CallSiteSummary* CallSiteSummarizer::get_instruction_call_site_summary( const IRInstruction* invoke_insn) const { - auto ptr = m_invoke_call_site_summaries.get_unsafe(invoke_insn); - return ptr == nullptr ? nullptr : *ptr; + auto it = m_invoke_call_site_summaries.find(invoke_insn); + return it == m_invoke_call_site_summaries.end() ? nullptr : &*it->second; } } // namespace inliner diff --git a/service/method-inliner/CallSiteSummaries.h b/service/method-inliner/CallSiteSummaries.h index ba61a927d9..0f75256e23 100644 --- a/service/method-inliner/CallSiteSummaries.h +++ b/service/method-inliner/CallSiteSummaries.h @@ -26,10 +26,6 @@ struct CallSiteSummary { const ConstantValue& value); }; -inline bool operator==(const CallSiteSummary& a, const CallSiteSummary& b) { - return a.result_used == b.result_used && a.arguments.equals(b.arguments); -} - struct CalleeCallSiteSummary { const DexMethod* method; const CallSiteSummary* call_site_summary; @@ -54,9 +50,9 @@ struct InvokeCallSiteSummariesAndDeadBlocks { using CallSiteSummaryOccurrences = std::pair; -using ConcurrentMethodToMethodOccurrences = - ConcurrentMap>; - +using MethodToMethodOccurrences = + std::unordered_map>; namespace inliner { using GetCalleeFunction = std::function; @@ -71,35 +67,36 @@ struct CallSiteSummaryStats { class CallSiteSummarizer { shrinker::Shrinker& m_shrinker; - const ConcurrentMethodToMethodOccurrences& m_callee_caller; - const ConcurrentMethodToMethodOccurrences& m_caller_callee; + const MethodToMethodOccurrences& m_callee_caller; + const MethodToMethodOccurrences& m_caller_callee; GetCalleeFunction m_get_callee_fn; HasCalleeOtherCallSitesPredicate m_has_callee_other_call_sites_fn; std::function* m_filter_fn; CallSiteSummaryStats* m_stats; - struct CalleeInfo { - std::unordered_map indices; - std::vector occurrences; - std::vector invokes; - }; + /** + * For all (reachable) invoked methods, list of call-site summaries + */ + std::unordered_map> + m_callee_call_site_summary_occurrences; /** - * For all (reachable) invoked methods, call-site summaries and invoke - * instructions + * For all (reachable) invoked methods, list of vinoke instructions */ - ConcurrentMap m_callee_infos; + std::unordered_map> + m_callee_call_site_invokes; /** * For all (reachable) invoke instructions, constant arguments */ - InsertOnlyConcurrentMap + ConcurrentMap m_invoke_call_site_summaries; /** * Internalized call-site summaries. */ - InsertOnlyConcurrentMap m_call_site_summaries; + ConcurrentMap> + m_call_site_summaries; /** * For all (reachable) invoke instructions in a given method, collect @@ -114,8 +111,8 @@ class CallSiteSummarizer { public: CallSiteSummarizer( shrinker::Shrinker& shrinker, - const ConcurrentMethodToMethodOccurrences& callee_caller, - const ConcurrentMethodToMethodOccurrences& caller_callee, + const MethodToMethodOccurrences& callee_caller, + const MethodToMethodOccurrences& caller_callee, GetCalleeFunction get_callee_fn, HasCalleeOtherCallSitesPredicate has_callee_other_call_sites_fn, std::function* filter_fn, diff --git a/service/method-inliner/Inliner.cpp b/service/method-inliner/Inliner.cpp index 2705a4251b..0d2ff1547b 100644 --- a/service/method-inliner/Inliner.cpp +++ b/service/method-inliner/Inliner.cpp @@ -16,6 +16,7 @@ #include "ConstantPropagationWholeProgramState.h" #include "ConstructorAnalysis.h" #include "DexInstruction.h" +#include "EditableCfgAdapter.h" #include "GraphUtil.h" #include "InlineForSpeed.h" #include "InlinerConfig.h" @@ -39,6 +40,21 @@ using namespace opt_metadata; using namespace outliner; namespace { + +// The following costs are in terms of code-units (2 bytes). + +// Typical overhead of calling a method, without move-result overhead. +const float COST_INVOKE = 3.7f; + +// Typical overhead of having move-result instruction. +const float COST_MOVE_RESULT = 3.0f; + +// Overhead of having a method and its metadata. +const size_t COST_METHOD = 16; + +// Typical savings in caller when callee doesn't use any argument. +const float UNUSED_ARGS_DISCOUNT = 1.0f; + /* * This is the maximum size of method that Dex bytecode can encode. * The table of instructions is indexed by a 32 bit unsigned integer. @@ -56,18 +72,18 @@ std::unordered_set gather_resolved_init_class_types( init_classes_with_side_effects) { std::unordered_set refined_init_class_types; - always_assert(m->get_code()->editable_cfg_built()); - for (auto& mie : InstructionIterable(m->get_code()->cfg())) { + editable_cfg_adapter::iterate(m->get_code(), [&](const MethodItemEntry& mie) { auto insn = mie.insn; if (insn->opcode() != IOPCODE_INIT_CLASS) { - continue; + return editable_cfg_adapter::LOOP_CONTINUE; } auto* refined_type = init_classes_with_side_effects.refine(insn->get_type()); if (refined_type != nullptr) { refined_init_class_types.insert(const_cast(refined_type)); } - } + return editable_cfg_adapter::LOOP_CONTINUE; + }); return refined_init_class_types; } @@ -92,16 +108,14 @@ MultiMethodInliner::MultiMethodInliner( const api::AndroidSDK* min_sdk_api, bool cross_dex_penalty, const std::unordered_set& configured_finalish_field_names, - bool local_only, - InlinerCostConfig inliner_cost_config) + bool local_only) : m_concurrent_resolver(std::move(concurrent_resolve_fn)), m_scheduler( [this](DexMethod* method) { auto it = caller_callee.find(method); if (it != caller_callee.end()) { - always_assert(method->get_code()->editable_cfg_built()); + always_assert(!inline_inlinables_need_deconstruct(method)); std::unordered_set callees; - callees.reserve(it->second.size()); for (auto& p : it->second) { callees.insert(p.first); } @@ -129,8 +143,7 @@ MultiMethodInliner::MultiMethodInliner( min_sdk, configured_pure_methods, configured_finalish_field_names), - m_local_only(local_only), - m_inliner_cost_config(inliner_cost_config) { + m_local_only(local_only) { Timer t("MultiMethodInliner construction"); for (const auto& callee_callers : true_virtual_callers) { auto callee = callee_callers.first; @@ -151,6 +164,10 @@ MultiMethodInliner::MultiMethodInliner( // IntraDex, we properly exclude invocations where the caller is located in // another dex from the callee, and remember all such x-dex callees. ConcurrentSet concurrent_x_dex_callees; + ConcurrentMap> + concurrent_callee_caller; + ConcurrentMap> + concurrent_caller_callee; std::unique_ptr x_dex; if (mode == IntraDex) { x_dex = std::make_unique(stores); @@ -184,17 +201,21 @@ MultiMethodInliner::MultiMethodInliner( concurrent_x_dex_callees.insert(callee); return; } - callee_caller.update(callee, - [caller](const DexMethod*, - std::unordered_map& v, - bool) { ++v[caller]; }); - caller_callee.update(caller, - [callee](const DexMethod*, - std::unordered_map& v, - bool) { ++v[callee]; }); + concurrent_callee_caller.update( + callee, + [caller](const DexMethod*, + std::unordered_map& v, + bool) { ++v[caller]; }); + concurrent_caller_callee.update( + caller, + [callee](const DexMethod*, + std::unordered_map& v, + bool) { ++v[callee]; }); }); m_x_dex_callees.insert(concurrent_x_dex_callees.begin(), concurrent_x_dex_callees.end()); + callee_caller = concurrent_callee_caller.move_to_container(); + caller_callee = concurrent_caller_callee.move_to_container(); for (const auto& callee_callers : true_virtual_callers) { auto callee = callee_callers.first; for (const auto& caller_insns : callee_callers.second.caller_insns) { @@ -205,17 +226,8 @@ MultiMethodInliner::MultiMethodInliner( } auto count = caller_insns.second.size(); always_assert(count > 0); - callee_caller.update_unsafe(callee, - [&](const DexMethod*, - std::unordered_map& v, - bool) { v[caller] += count; }); - bool added_virtual_only = false; - caller_callee.update_unsafe( - caller, - [&](const DexMethod*, - std::unordered_map& v, - bool) { added_virtual_only = (v[callee] += count) == count; }); - if (added_virtual_only) { + callee_caller[callee][caller] += count; + if ((caller_callee[caller][callee] += count) == count) { // We added a new callee that is only valid via m_caller_virtual_callees m_caller_virtual_callees[caller].exclusive_callees.insert(callee); } @@ -223,7 +235,34 @@ MultiMethodInliner::MultiMethodInliner( } } -void MultiMethodInliner::inline_methods() { +void MultiMethodInliner::inline_methods(bool methods_need_deconstruct) { + std::unordered_set need_deconstruct; + if (methods_need_deconstruct) { + for (auto& p : caller_callee) { + need_deconstruct.insert(const_cast(p.first->get_code())); + } + for (auto& p : callee_caller) { + need_deconstruct.insert(const_cast(p.first->get_code())); + } + for (auto it = need_deconstruct.begin(); it != need_deconstruct.end();) { + if ((*it)->editable_cfg_built()) { + it = need_deconstruct.erase(it); + } else { + it++; + } + } + if (!need_deconstruct.empty()) { + workqueue_run( + [](IRCode* code) { code->build_cfg(/* editable */ true); }, + need_deconstruct); + } + } + + // Inlining and shrinking initiated from within this method will be done + // in parallel. + m_scheduler.get_thread_pool().set_num_threads( + m_config.debug ? 1 : redex_parallel::default_num_threads()); + // The order in which we inline is such that once a callee is considered to // be inlined, it's code will no longer change. So we can cache... // - its size @@ -232,15 +271,16 @@ void MultiMethodInliner::inline_methods() { // - whether all callers are in the same class, and are called from how many // classes m_callee_insn_sizes = - std::make_unique>(); - m_callee_type_refs = std::make_unique>>>(); + std::make_unique>(); + m_callee_type_refs = + std::make_unique>>>(); if (m_ref_checkers) { m_callee_code_refs = std::make_unique< - InsertOnlyConcurrentMap>>(); + ConcurrentMap>>(); } - m_callee_caller_refs = std::make_unique< - InsertOnlyConcurrentMap>(); + m_callee_caller_refs = + std::make_unique>(); // Instead of changing visibility as we inline, blocking other work on the // critical path, we do it all in parallel at the end. @@ -280,11 +320,6 @@ void MultiMethodInliner::inline_methods() { m_call_site_summarizer->summarize(); } - // Inlining and shrinking initiated from within this method will be done - // in parallel. - m_scheduler.get_thread_pool().set_num_threads( - m_config.debug ? 1 : redex_parallel::default_num_threads()); - // Second, compute caller priorities --- the callers get a priority assigned // that reflects how many other callers will be waiting for them. std::unordered_set methods_to_schedule; @@ -316,6 +351,11 @@ void MultiMethodInliner::inline_methods() { delayed_visibility_changes_apply(); delayed_invoke_direct_to_static(); info.waited_seconds = m_scheduler.get_thread_pool().get_waited_seconds(); + + if (!need_deconstruct.empty()) { + workqueue_run([](IRCode* code) { code->clear_cfg(); }, + need_deconstruct); + } } DexMethod* MultiMethodInliner::get_callee(DexMethod* caller, @@ -348,64 +388,61 @@ void MultiMethodInliner::inline_callees( { auto timer = m_inline_callees_timer.scope(); - always_assert(caller->get_code()->editable_cfg_built()); - auto ii = InstructionIterable(caller->get_code()->cfg()); // walk the caller opcodes collecting all candidates to inline // Build a callee to opcode map - for (auto it = ii.begin(); it != ii.end(); ++it) { - auto insn = it->insn; - auto callee = get_callee(caller, insn); - if (!callee || !callees.count(callee)) { - continue; - } - std::shared_ptr reduced_code; - bool no_return{false}; - size_t insn_size{0}; - if (filter_via_should_inline) { - auto timer2 = m_inline_callees_should_inline_timer.scope(); - // Cost model is based on fully inlining callee everywhere; let's - // see if we can get more detailed call-site specific information - if (should_inline_at_call_site(caller, insn, callee, &no_return, - &reduced_code, &insn_size)) { - always_assert(!no_return); - // Yes, we know might have dead_blocks and a refined insn_size - } else if (should_inline_always(callee)) { - // We'll fully inline the callee without any adjustments - no_return = false; - insn_size = get_callee_insn_size(callee); - } else if (no_return) { - always_assert(insn_size == 0); - always_assert(!reduced_code); - } else { - continue; - } - } else { - insn_size = get_callee_insn_size(callee); - } - always_assert(callee->is_concrete()); - if (m_analyze_and_prune_inits && method::is_init(callee) && !no_return) { - auto timer2 = m_inline_callees_init_timer.scope(); - if (!callee->get_code()->editable_cfg_built()) { - continue; - } - if (!can_inline_init(callee)) { - if (!method::is_init(caller) || - caller->get_class() != callee->get_class() || - !caller->get_code()->editable_cfg_built() || - !constructor_analysis::can_inline_inits_in_same_class( - caller, callee, insn)) { - continue; + editable_cfg_adapter::iterate_with_iterator( + caller->get_code(), [&](const IRList::iterator& it) { + auto insn = it->insn; + auto callee = get_callee(caller, insn); + if (!callee || !callees.count(callee)) { + return editable_cfg_adapter::LOOP_CONTINUE; + } + std::shared_ptr reduced_code; + bool no_return{false}; + size_t insn_size{0}; + if (filter_via_should_inline) { + auto timer2 = m_inline_callees_should_inline_timer.scope(); + // Cost model is based on fully inlining callee everywhere; let's + // see if we can get more detailed call-site specific information + if (should_inline_at_call_site(caller, insn, callee, &no_return, + &reduced_code, &insn_size)) { + always_assert(!no_return); + // Yes, we know might have dead_blocks and a refined insn_size + } else if (should_inline_always(callee)) { + // We'll fully inline the callee without any adjustments + no_return = false; + insn_size = get_callee_insn_size(callee); + } else if (no_return) { + always_assert(insn_size == 0); + always_assert(!reduced_code); + } else { + return editable_cfg_adapter::LOOP_CONTINUE; + } + } else { + insn_size = get_callee_insn_size(callee); + } + always_assert(callee->is_concrete()); + if (m_analyze_and_prune_inits && method::is_init(callee) && + !no_return) { + auto timer2 = m_inline_callees_init_timer.scope(); + if (!callee->get_code()->editable_cfg_built()) { + return editable_cfg_adapter::LOOP_CONTINUE; + } + if (!can_inline_init(callee)) { + if (!method::is_init(caller) || + caller->get_class() != callee->get_class() || + !caller->get_code()->editable_cfg_built() || + !constructor_analysis::can_inline_inits_in_same_class( + caller, callee, insn)) { + return editable_cfg_adapter::LOOP_CONTINUE; + } + } } - } - } - auto it2 = m_inlined_invokes_need_cast.find(insn); - auto needs_receiver_cast = - it2 == m_inlined_invokes_need_cast.end() ? nullptr : it2->second; - inlinables.push_back((Inlinable){callee, insn, no_return, - std::move(reduced_code), insn_size, - needs_receiver_cast}); - } + inlinables.push_back((Inlinable){callee, it, insn, no_return, + std::move(reduced_code), insn_size}); + return editable_cfg_adapter::LOOP_CONTINUE; + }); } if (!inlinables.empty()) { inline_inlinables(caller, inlinables); @@ -418,27 +455,30 @@ size_t MultiMethodInliner::inline_callees( std::vector* deleted_insns) { TraceContext context{caller}; std::vector inlinables; - always_assert(caller->get_code()->editable_cfg_built()); - for (auto& mie : InstructionIterable(caller->get_code()->cfg())) { - auto insn = mie.insn; - if (insns.count(insn)) { - auto callee = get_callee(caller, insn); - if (callee == nullptr) { - continue; - } - always_assert(callee->is_concrete()); - auto it2 = m_inlined_invokes_need_cast.find(insn); - auto needs_receiver_cast = - it2 == m_inlined_invokes_need_cast.end() ? nullptr : it2->second; - inlinables.push_back((Inlinable){callee, insn, false, nullptr, - get_callee_insn_size(callee), - needs_receiver_cast}); - } - } + editable_cfg_adapter::iterate_with_iterator( + caller->get_code(), [&](const IRList::iterator& it) { + auto insn = it->insn; + if (insns.count(insn)) { + auto callee = get_callee(caller, insn); + if (callee == nullptr) { + return editable_cfg_adapter::LOOP_CONTINUE; + } + always_assert(callee->is_concrete()); + inlinables.push_back((Inlinable){callee, it, insn, false, nullptr, + get_callee_insn_size(callee)}); + } + return editable_cfg_adapter::LOOP_CONTINUE; + }); return inline_inlinables(caller, inlinables, deleted_insns); } +bool MultiMethodInliner::inline_inlinables_need_deconstruct(DexMethod* method) { + // The mixed CFG, IRCode is used by Switch Inline (only?) where the caller is + // an IRCode and the callee is a CFG. + return !method->get_code()->editable_cfg_built(); +} + namespace { // Helper method, as computing inline for a trace could be too expensive. @@ -462,8 +502,8 @@ std::string create_inlining_trace_msg(const DexMethod* caller, loop_impl::LoopInfo info(code->cfg()); oss << "!" << info.num_loops(); size_t max_depth{0}; - for (auto& loop : info) { - max_depth = std::max(max_depth, (size_t)loop.get_loop_depth()); + for (auto* loop : info) { + max_depth = std::max(max_depth, (size_t)loop->get_loop_depth()); } oss << "!" << max_depth; if (insn != nullptr) { @@ -513,10 +553,24 @@ size_t MultiMethodInliner::inline_inlinables( std::vector* deleted_insns) { auto timer = m_inline_inlinables_timer.scope(); auto caller = caller_method->get_code(); - always_assert(caller->editable_cfg_built()); + std::unordered_set need_deconstruct; + if (inline_inlinables_need_deconstruct(caller_method)) { + need_deconstruct.reserve(1 + inlinables.size()); + need_deconstruct.insert(caller); + for (const auto& inlinable : inlinables) { + need_deconstruct.insert(inlinable.callee->get_code()); + } + for (auto code : need_deconstruct) { + always_assert(!code->editable_cfg_built()); + code->build_cfg(/* editable */ true); + // if (deleted_insns != nullptr) { + // code->cfg().set_removed_insn_ownership(false); + // } + } + } // attempt to inline all inlinable candidates - size_t estimated_caller_size = caller->cfg().estimate_code_units(); + size_t estimated_caller_size = caller->estimate_code_units(); // Prefer inlining smaller methods first, so that we are less likely to hit // overall size limit. @@ -591,8 +645,7 @@ size_t MultiMethodInliner::inline_inlinables( } if (inlinable.no_return) { - if (!m_config.throw_after_no_return || - caller_method->rstate.no_optimizations()) { + if (!m_config.throw_after_no_return) { continue; } // we are not actually inlining, but just cutting off control-flow @@ -675,7 +728,7 @@ size_t MultiMethodInliner::inline_inlinables( estimated_caller_size, inlinable.insn_size, &caller_too_large_); if (!not_inlinable && m_config.intermediate_shrinking && - m_shrinker.enabled() && !caller_method->rstate.no_optimizations()) { + m_shrinker.enabled()) { intermediate_shrinkings++; m_shrinker.shrink_method(caller_method); cfg_next_caller_reg = caller->cfg().get_registers_size(); @@ -707,13 +760,15 @@ size_t MultiMethodInliner::inline_inlinables( cfg_next_caller_reg = caller->cfg().get_registers_size(); } auto timer2 = m_inline_with_cfg_timer.scope(); + auto it = m_inlined_invokes_need_cast.find(callsite_insn); + auto needs_receiver_cast = + it == m_inlined_invokes_need_cast.end() ? nullptr : it->second; auto needs_init_class = get_needs_init_class(callee_method); const auto& reduced_code = inlinable.reduced_code; const auto* reduced_cfg = reduced_code ? &reduced_code->cfg() : nullptr; bool success = inliner::inline_with_cfg( - caller_method, callee_method, callsite_insn, - inlinable.needs_receiver_cast, needs_init_class, *cfg_next_caller_reg, - reduced_cfg); + caller_method, callee_method, callsite_insn, needs_receiver_cast, + needs_init_class, *cfg_next_caller_reg, reduced_cfg); if (!success) { calls_not_inlined++; continue; @@ -728,7 +783,7 @@ size_t MultiMethodInliner::inline_inlinables( } else { visibility_changes_for.insert(callee_method); } - if (needs_init_class) { + if (is_static(callee_method)) { init_classes++; } @@ -754,6 +809,10 @@ size_t MultiMethodInliner::inline_inlinables( m_inlined.insert(inlined_callees.begin(), inlined_callees.end()); } + for (IRCode* code : need_deconstruct) { + code->clear_cfg(nullptr, deleted_insns); + } + info.calls_inlined += inlined_callees.size(); if (calls_not_inlinable) { info.calls_not_inlinable += calls_not_inlinable; @@ -790,7 +849,7 @@ void MultiMethodInliner::postprocess_method(DexMethod* method) { m_shrinker.shrink_method(method); } - bool is_callee = !!callee_caller.count_unsafe(method); + bool is_callee = !!callee_caller.count(method); if (!is_callee) { // This method isn't the callee of another caller, so we can stop here. return; @@ -832,9 +891,9 @@ void MultiMethodInliner::compute_callee_costs(DexMethod* method) { } if (!keep_reduced_code) { CalleeCallSiteSummary key{method, call_site_summary}; - const auto* cached = m_call_site_inlined_costs.get(key); - if (cached) { - const_cast(cached)->reduced_code.reset(); + auto inlined_cost = m_call_site_inlined_costs.get(key, nullptr); + if (inlined_cost) { + inlined_cost->reduced_code.reset(); } } }); @@ -875,9 +934,6 @@ bool MultiMethodInliner::is_inlinable(const DexMethod* caller, if (caller_too_large_) { *caller_too_large_ = false; } - if (caller->rstate.no_optimizations()) { - return false; - } // don't inline cross store references if (cross_store_reference(caller, callee)) { if (insn) { @@ -1025,7 +1081,7 @@ bool MultiMethodInliner::should_inline_fast(const DexMethod* callee) { // non-root methods that are only ever called once should always be inlined, // as the method can be removed afterwards - const auto& callers = callee_caller.at_unsafe(callee); + const auto& callers = callee_caller.at(callee); if (callers.size() == 1 && callers.begin()->second == 1 && !root(callee) && !method::is_argless_init(callee) && !m_recursive_callees.count(callee) && !m_x_dex_callees.count(callee) && @@ -1045,26 +1101,29 @@ bool MultiMethodInliner::should_inline_always(const DexMethod* callee) { return true; } - return *m_should_inline - .get_or_create_and_assert_equal( - callee, - [&](const auto&) { - always_assert(!for_speed()); - always_assert(!callee->rstate.force_inline()); - if (too_many_callers(callee)) { - log_nopt(INL_TOO_MANY_CALLERS, callee); - return false; - } - return true; - }) - .first; + auto res = m_should_inline.get(callee, boost::none); + if (res) { + return *res; + } + + always_assert(!for_speed()); + always_assert(!callee->rstate.force_inline()); + if (too_many_callers(callee)) { + log_nopt(INL_TOO_MANY_CALLERS, callee); + res = false; + } else { + res = true; + } + m_should_inline.emplace(callee, res); + return *res; } size_t MultiMethodInliner::get_callee_insn_size(const DexMethod* callee) { if (m_callee_insn_sizes) { - auto* res = m_callee_insn_sizes->get(callee); - if (res) { - return *res; + const auto absent = std::numeric_limits::max(); + auto size = m_callee_insn_sizes->get(callee, absent); + if (size != absent) { + return size; } } @@ -1079,11 +1138,10 @@ size_t MultiMethodInliner::get_callee_insn_size(const DexMethod* callee) { /* * Estimate additional costs if an instruction takes many source registers. */ -static size_t get_inlined_regs_cost(size_t regs, - const InlinerCostConfig& cost_config) { +static size_t get_inlined_regs_cost(size_t regs) { size_t cost{0}; - if (regs > cost_config.reg_threshold_1) { - if (regs > cost_config.reg_threshold_1 + cost_config.reg_threshold_2) { + if (regs > 3) { + if (regs > 5) { // invoke with many args will likely need extra moves cost += regs; } else { @@ -1093,13 +1151,9 @@ static size_t get_inlined_regs_cost(size_t regs, return cost; } -static float get_invoke_cost(const InlinerCostConfig& cost_config, - const DexMethod* callee, - float result_used) { - float invoke_cost = - cost_config.cost_invoke + result_used * cost_config.cost_move_result; - invoke_cost += get_inlined_regs_cost(callee->get_proto()->get_args()->size(), - cost_config); +static float get_invoke_cost(const DexMethod* callee, float result_used) { + float invoke_cost = COST_INVOKE + result_used * COST_MOVE_RESULT; + invoke_cost += get_inlined_regs_cost(callee->get_proto()->get_args()->size()); return invoke_cost; } @@ -1112,41 +1166,39 @@ static float get_invoke_cost(const InlinerCostConfig& cost_config, * - Remove return opcodes, as they will disappear when gluing things * together. */ -static size_t get_inlined_cost(IRInstruction* insn, - const InlinerCostConfig& cost_config) { +static size_t get_inlined_cost(IRInstruction* insn) { auto op = insn->opcode(); size_t cost{0}; if (opcode::is_an_internal(op) || opcode::is_a_move(op) || opcode::is_a_return(op)) { if (op == IOPCODE_INIT_CLASS) { - cost += cost_config.op_init_class_cost; + cost += 2; } else if (op == IOPCODE_INJECTION_ID) { - cost += cost_config.op_injection_id_cost; + cost += 3; } else if (op == IOPCODE_UNREACHABLE) { - cost += cost_config.op_unreachable_cost; + cost += 1; } } else { cost++; auto regs = insn->srcs_size() + ((insn->has_dest() || insn->has_move_result_pseudo()) ? 1 : 0); - cost += get_inlined_regs_cost(regs, cost_config); + cost += get_inlined_regs_cost(regs); if (op == OPCODE_MOVE_EXCEPTION) { - cost += cost_config.op_move_exception_cost; // accounting for book-keeping - // overhead of throw-blocks + cost += 8; // accounting for book-keeping overhead of throw-blocks } else if (insn->has_method() || insn->has_field() || insn->has_type() || insn->has_string()) { - cost += cost_config.insn_cost_1; + cost++; } else if (insn->has_data()) { - cost += cost_config.insn_has_data_cost + insn->get_data()->size(); + cost += 4 + insn->get_data()->size(); } else if (insn->has_literal()) { auto lit = insn->get_literal(); if (lit < -2147483648 || lit > 2147483647) { - cost += cost_config.insn_has_lit_cost_1; + cost += 4; } else if (lit < -32768 || lit > 32767) { - cost += cost_config.insn_has_lit_cost_2; + cost += 2; } else if ((opcode::is_a_const(op) && (lit < -8 || lit > 7)) || (!opcode::is_a_const(op) && (lit < -128 || lit > 127))) { - cost += cost_config.insn_has_lit_cost_3; + cost++; } } } @@ -1161,8 +1213,7 @@ static size_t get_inlined_cost(IRInstruction* insn, */ static size_t get_inlined_cost(const std::vector& blocks, size_t index, - const std::vector& succs, - const InlinerCostConfig& cost_config) { + const std::vector& succs) { auto block = blocks.at(index); switch (block->branchingness()) { case opcode::Branchingness::BRANCH_GOTO: @@ -1282,36 +1333,45 @@ InlinedCost MultiMethodInliner::get_inlined_cost( }; size_t insn_size; float unused_args{0}; - always_assert(code->editable_cfg_built()); - reduced_code = apply_call_site_summary(is_static, declaring_type, proto, - code->cfg(), call_site_summary); - auto cfg = &(reduced_code ? &reduced_code->code() : code)->cfg(); - auto blocks = cfg->blocks(); - for (size_t i = 0; i < blocks.size(); ++i) { - auto block = blocks.at(i); - for (auto& mie : InstructionIterable(block)) { - auto insn = mie.insn; - cost += ::get_inlined_cost(insn, m_inliner_cost_config); - analyze_refs(insn); - } - cost += - ::get_inlined_cost(blocks, i, block->succs(), m_inliner_cost_config); - if (block->branchingness() == opcode::Branchingness::BRANCH_RETURN) { - returns++; + if (code->editable_cfg_built()) { + reduced_code = apply_call_site_summary(is_static, declaring_type, proto, + code->cfg(), call_site_summary); + auto cfg = &(reduced_code ? &reduced_code->code() : code)->cfg(); + auto blocks = cfg->blocks(); + for (size_t i = 0; i < blocks.size(); ++i) { + auto block = blocks.at(i); + for (auto& mie : InstructionIterable(block)) { + auto insn = mie.insn; + cost += ::get_inlined_cost(insn); + analyze_refs(insn); + } + cost += ::get_inlined_cost(blocks, i, block->succs()); + if (block->branchingness() == opcode::Branchingness::BRANCH_RETURN) { + returns++; + } } - } - live_range::MoveAwareChains chains( - *cfg, - /* ignore_unreachable */ !reduced_code, - [&](auto* insn) { return opcode::is_a_load_param(insn->opcode()); }); - auto def_use_chains = chains.get_def_use_chains(); - for (auto& mie : cfg->get_param_instructions()) { - auto uses = def_use_chains[mie.insn]; - if (uses.empty()) { - unused_args++; + live_range::MoveAwareChains chains(*cfg, + /* ignore_unreachable */ !reduced_code); + auto def_use_chains = chains.get_def_use_chains(); + for (auto& mie : cfg->get_param_instructions()) { + auto uses = def_use_chains[mie.insn]; + if (uses.empty()) { + unused_args++; + } } + insn_size = cfg->estimate_code_units(); + } else { + editable_cfg_adapter::iterate(code, [&](const MethodItemEntry& mie) { + auto insn = mie.insn; + cost += ::get_inlined_cost(insn); + if (opcode::is_a_return(insn->opcode())) { + returns++; + } + analyze_refs(insn); + return editable_cfg_adapter::LOOP_CONTINUE; + }); + insn_size = code->estimate_code_units(); } - insn_size = cfg->estimate_code_units(); if (returns > 1) { // if there's more than one return, gotos will get introduced to merge // control flow @@ -1334,43 +1394,55 @@ InlinedCost MultiMethodInliner::get_inlined_cost( const InlinedCost* MultiMethodInliner::get_fully_inlined_cost( const DexMethod* callee) { - return m_fully_inlined_costs - .get_or_create_and_assert_equal( - callee, - [&](const auto&) { - InlinedCost inlined_cost( - get_inlined_cost(is_static(callee), callee->get_class(), - callee->get_proto(), callee->get_code())); - TRACE(INLINE, 4, - "get_fully_inlined_cost(%s) = {%zu,%f,%f,%f,%s,%f,%d,%zu}", - SHOW(callee), inlined_cost.full_code, inlined_cost.code, - inlined_cost.method_refs, inlined_cost.other_refs, - inlined_cost.no_return ? "no_return" : "return", - inlined_cost.result_used, !!inlined_cost.reduced_code, - inlined_cost.insn_size); - return inlined_cost; - }) - .first; + auto inlined_cost = m_fully_inlined_costs.get(callee, nullptr); + if (inlined_cost) { + return inlined_cost.get(); + } + inlined_cost = std::make_shared( + get_inlined_cost(is_static(callee), callee->get_class(), + callee->get_proto(), callee->get_code())); + TRACE(INLINE, 4, "get_fully_inlined_cost(%s) = {%zu,%f,%f,%f,%s,%f,%d,%zu}", + SHOW(callee), inlined_cost->full_code, inlined_cost->code, + inlined_cost->method_refs, inlined_cost->other_refs, + inlined_cost->no_return ? "no_return" : "return", + inlined_cost->result_used, !!inlined_cost->reduced_code, + inlined_cost->insn_size); + m_fully_inlined_costs.update( + callee, + [&](const DexMethod*, std::shared_ptr& value, bool exists) { + if (exists) { + // We wasted some work, and some other thread beat us. Oh well... + always_assert(*value == *inlined_cost); + inlined_cost = value; + return; + } + value = inlined_cost; + }); + + return inlined_cost.get(); } const InlinedCost* MultiMethodInliner::get_call_site_inlined_cost( const IRInstruction* invoke_insn, const DexMethod* callee) { - return *m_invoke_call_site_inlined_costs - .get_or_create_and_assert_equal( - invoke_insn, - [&](const auto&) { - auto call_site_summary = - m_call_site_summarizer - ? m_call_site_summarizer - ->get_instruction_call_site_summary( - invoke_insn) - : nullptr; - return call_site_summary == nullptr - ? nullptr - : get_call_site_inlined_cost(call_site_summary, - callee); - }) - .first; + auto res = m_invoke_call_site_inlined_costs.get(invoke_insn, boost::none); + if (res) { + return *res; + } + + auto call_site_summary = + m_call_site_summarizer + ? m_call_site_summarizer->get_instruction_call_site_summary( + invoke_insn) + : nullptr; + if (call_site_summary == nullptr) { + res = nullptr; + } else { + res = get_call_site_inlined_cost(call_site_summary, callee); + } + + always_assert(res); + m_invoke_call_site_inlined_costs.emplace(invoke_insn, res); + return *res; } const InlinedCost* MultiMethodInliner::get_call_site_inlined_cost( @@ -1383,66 +1455,76 @@ const InlinedCost* MultiMethodInliner::get_call_site_inlined_cost( } CalleeCallSiteSummary key{callee, call_site_summary}; - return m_call_site_inlined_costs - .get_or_create_and_assert_equal( - key, - [&](const auto&) { - auto inlined_cost = get_inlined_cost( - is_static(callee), callee->get_class(), callee->get_proto(), - callee->get_code(), call_site_summary); - TRACE( - INLINE, 4, - "get_call_site_inlined_cost(%s) = {%zu,%f,%f,%f,%s,%f,%d,%zu}", - call_site_summary->get_key().c_str(), inlined_cost.full_code, - inlined_cost.code, inlined_cost.method_refs, - inlined_cost.other_refs, - inlined_cost.no_return ? "no_return" : "return", - inlined_cost.result_used, !!inlined_cost.reduced_code, - inlined_cost.insn_size); - if (inlined_cost.insn_size >= fully_inlined_cost->insn_size) { - inlined_cost.reduced_code.reset(); - } - return inlined_cost; - }) - .first; + auto inlined_cost = m_call_site_inlined_costs.get(key, nullptr); + if (inlined_cost) { + return inlined_cost.get(); + } + + inlined_cost = std::make_shared(get_inlined_cost( + is_static(callee), callee->get_class(), callee->get_proto(), + callee->get_code(), call_site_summary)); + TRACE(INLINE, 4, + "get_call_site_inlined_cost(%s) = {%zu,%f,%f,%f,%s,%f,%d,%zu}", + call_site_summary->get_key().c_str(), inlined_cost->full_code, + inlined_cost->code, inlined_cost->method_refs, inlined_cost->other_refs, + inlined_cost->no_return ? "no_return" : "return", + inlined_cost->result_used, !!inlined_cost->reduced_code, + inlined_cost->insn_size); + if (inlined_cost->insn_size >= fully_inlined_cost->insn_size) { + inlined_cost->reduced_code.reset(); + } + m_call_site_inlined_costs.update(key, + [&](const CalleeCallSiteSummary&, + std::shared_ptr& value, + bool exists) { + if (exists) { + // We wasted some work, and some other + // thread beat us. Oh well... + always_assert(*value == *inlined_cost); + inlined_cost = value; + return; + } + value = inlined_cost; + }); + + return inlined_cost.get(); } const InlinedCost* MultiMethodInliner::get_average_inlined_cost( const DexMethod* callee) { - const auto* cached = m_average_inlined_costs.get(callee); - if (cached) { - return cached; + auto inlined_cost = m_average_inlined_costs.get(callee, nullptr); + if (inlined_cost) { + return inlined_cost.get(); } + auto fully_inlined_cost = get_fully_inlined_cost(callee); + always_assert(fully_inlined_cost); + size_t callees_analyzed{0}; size_t callees_unused_results{0}; size_t callees_no_return{0}; - auto res = [&]() { - auto fully_inlined_cost = get_fully_inlined_cost(callee); - always_assert(fully_inlined_cost); - - if (fully_inlined_cost->full_code > - m_config.max_cost_for_constant_propagation) { - return *fully_inlined_cost; - } - const auto* callee_call_site_summary_occurrences = - m_call_site_summarizer - ? m_call_site_summarizer->get_callee_call_site_summary_occurrences( - callee) - : nullptr; - if (!callee_call_site_summary_occurrences) { - return *fully_inlined_cost; - } - InlinedCost inlined_cost((InlinedCost){fully_inlined_cost->full_code, - /* code */ 0.0f, - /* method_refs */ 0.0f, - /* other_refs */ 0.0f, - /* no_return */ true, - /* result_used */ 0.0f, - /* unused_args */ 0.0f, - /* reduced_cfg */ nullptr, - /* insn_size */ 0}); + const std::vector* + callee_call_site_summary_occurrences; + if (fully_inlined_cost->full_code > + m_config.max_cost_for_constant_propagation || + !(callee_call_site_summary_occurrences = + m_call_site_summarizer + ? m_call_site_summarizer + ->get_callee_call_site_summary_occurrences(callee) + : nullptr)) { + inlined_cost = std::make_shared(*fully_inlined_cost); + } else { + inlined_cost = std::make_shared( + (InlinedCost){fully_inlined_cost->full_code, + /* code */ 0.0f, + /* method_refs */ 0.0f, + /* other_refs */ 0.0f, + /* no_return */ true, + /* result_used */ 0.0f, + /* unused_args */ 0.0f, + /* reduced_cfg */ nullptr, + /* insn_size */ 0}); bool callee_has_result = !callee->get_proto()->is_void(); for (auto& p : *callee_call_site_summary_occurrences) { const auto call_site_summary = p.first; @@ -1453,56 +1535,75 @@ const InlinedCost* MultiMethodInliner::get_average_inlined_cost( if (callee_has_result && !call_site_summary->result_used) { callees_unused_results += count; } - inlined_cost.code += call_site_inlined_cost->code * count; - inlined_cost.method_refs += call_site_inlined_cost->method_refs * count; - inlined_cost.other_refs += call_site_inlined_cost->other_refs * count; - inlined_cost.result_used += call_site_inlined_cost->result_used * count; - inlined_cost.unused_args += call_site_inlined_cost->unused_args * count; + inlined_cost->code += call_site_inlined_cost->code * count; + inlined_cost->method_refs += call_site_inlined_cost->method_refs * count; + inlined_cost->other_refs += call_site_inlined_cost->other_refs * count; + inlined_cost->result_used += call_site_inlined_cost->result_used * count; + inlined_cost->unused_args += call_site_inlined_cost->unused_args * count; if (call_site_inlined_cost->no_return) { callees_no_return++; } else { - inlined_cost.no_return = false; + inlined_cost->no_return = false; } - if (call_site_inlined_cost->insn_size > inlined_cost.insn_size) { - inlined_cost.insn_size = call_site_inlined_cost->insn_size; + if (call_site_inlined_cost->insn_size > inlined_cost->insn_size) { + inlined_cost->insn_size = call_site_inlined_cost->insn_size; } callees_analyzed += count; }; always_assert(callees_analyzed > 0); // compute average costs - inlined_cost.code /= callees_analyzed; - inlined_cost.method_refs /= callees_analyzed; - inlined_cost.other_refs /= callees_analyzed; - inlined_cost.result_used /= callees_analyzed; - inlined_cost.unused_args /= callees_analyzed; - return inlined_cost; - }(); + inlined_cost->code /= callees_analyzed; + inlined_cost->method_refs /= callees_analyzed; + inlined_cost->other_refs /= callees_analyzed; + inlined_cost->result_used /= callees_analyzed; + inlined_cost->unused_args /= callees_analyzed; + } TRACE(INLINE, 4, "get_average_inlined_cost(%s) = {%zu,%f,%f,%f,%s,%f,%zu}", - SHOW(callee), res.full_code, res.code, res.method_refs, res.other_refs, - res.no_return ? "no_return" : "return", res.result_used, res.insn_size); - auto [ptr, emplaced] = - m_average_inlined_costs.get_or_emplace_and_assert_equal(callee, - std::move(res)); - if (emplaced && callees_analyzed >= 0) { - info.constant_invoke_callees_analyzed += callees_analyzed; - info.constant_invoke_callees_unused_results += callees_unused_results; - info.constant_invoke_callees_no_return += callees_no_return; - } - return ptr; + SHOW(callee), inlined_cost->full_code, inlined_cost->code, + inlined_cost->method_refs, inlined_cost->other_refs, + inlined_cost->no_return ? "no_return" : "return", + inlined_cost->result_used, inlined_cost->insn_size); + m_average_inlined_costs.update( + callee, + [&](const DexMethod*, std::shared_ptr& value, bool exists) { + if (exists) { + // We wasted some work, and some other thread beat us. Oh well... + always_assert(*value == *inlined_cost); + inlined_cost = value; + return; + } + value = inlined_cost; + if (callees_analyzed == 0) { + return; + } + info.constant_invoke_callees_analyzed += callees_analyzed; + info.constant_invoke_callees_unused_results += callees_unused_results; + info.constant_invoke_callees_no_return += callees_no_return; + }); + return inlined_cost.get(); } bool MultiMethodInliner::can_inline_init(const DexMethod* init_method) { - return *m_can_inline_init - .get_or_create_and_assert_equal( - init_method, - [&](const auto&) { - const auto* finalizable_fields = - m_shrinker.get_finalizable_fields(); - return constructor_analysis::can_inline_init( - init_method, finalizable_fields); - }) - .first; + auto opt_can_inline_init = m_can_inline_init.get(init_method, boost::none); + if (opt_can_inline_init) { + return *opt_can_inline_init; + } + + const auto* finalizable_fields = m_shrinker.get_finalizable_fields(); + bool res = + constructor_analysis::can_inline_init(init_method, finalizable_fields); + m_can_inline_init.update( + init_method, + [&](const DexMethod*, boost::optional& value, bool exists) { + if (exists) { + // We wasted some work, and some other thread beat us. Oh well... + always_assert(*value == res); + return; + } + value = res; + }); + return res; } bool MultiMethodInliner::too_many_callers(const DexMethod* callee) { @@ -1517,19 +1618,21 @@ bool MultiMethodInliner::too_many_callers(const DexMethod* callee) { can_delete_callee = false; } - const auto& callers = callee_caller.at_unsafe(callee); + const auto& callers = callee_caller.at(callee); // Can we inline the init-callee into all callers? // If not, then we can give up, as there's no point in making the case that // we can eliminate the callee method based on pervasive inlining. if (m_analyze_and_prune_inits && method::is_init(callee)) { - always_assert(callee->get_code()->editable_cfg_built()); + if (!callee->get_code()->editable_cfg_built()) { + return true; + } if (!can_inline_init(callee)) { for (auto& p : callers) { auto caller = p.first; - always_assert(caller->get_code()->editable_cfg_built()); if (!method::is_init(caller) || caller->get_class() != callee->get_class() || + !caller->get_code()->editable_cfg_built() || !constructor_analysis::can_inline_inits_in_same_class( caller, callee, /* callsite_insn */ nullptr)) { @@ -1567,8 +1670,7 @@ bool MultiMethodInliner::too_many_callers(const DexMethod* callee) { for (auto& p : callers) { caller_count += p.second; } - float invoke_cost = - get_invoke_cost(m_inliner_cost_config, callee, inlined_cost->result_used); + float invoke_cost = get_invoke_cost(callee, inlined_cost->result_used); TRACE(INLINE, 3, "[too_many_callers] %zu calls to %s; cost: inlined %f + %f, invoke %f", caller_count, SHOW(callee), inlined_cost->code, cross_dex_penalty, @@ -1580,14 +1682,13 @@ bool MultiMethodInliner::too_many_callers(const DexMethod* callee) { if (can_delete_callee) { // The cost of keeping a method amounts of somewhat fixed metadata overhead, // plus the method body, which we approximate with the inlined cost. - method_cost = m_inliner_cost_config.cost_method + inlined_cost->full_code; + method_cost = COST_METHOD + inlined_cost->full_code; } // If we inline invocations to this method everywhere, we could delete the // method. Is this worth it, given the number of callsites and costs // involved? - if ((inlined_cost->code - - inlined_cost->unused_args * m_inliner_cost_config.unused_args_discount) * + if ((inlined_cost->code - inlined_cost->unused_args * UNUSED_ARGS_DISCOUNT) * caller_count + classes * cross_dex_penalty > invoke_cost * caller_count + method_cost) { @@ -1648,11 +1749,8 @@ bool MultiMethodInliner::should_inline_at_call_site( } } - float invoke_cost = - get_invoke_cost(m_inliner_cost_config, callee, inlined_cost->result_used); - if (inlined_cost->code - - inlined_cost->unused_args * - m_inliner_cost_config.unused_args_discount + + float invoke_cost = get_invoke_cost(callee, inlined_cost->result_used); + if (inlined_cost->code - inlined_cost->unused_args * UNUSED_ARGS_DISCOUNT + cross_dex_penalty > invoke_cost) { if (no_return) { @@ -1797,7 +1895,7 @@ bool MultiMethodInliner::create_vmethod(IRInstruction* insn, // concrete ctors we can handle because they stay invoke_direct return false; } - if (!can_rename(method) || method->rstate.no_optimizations()) { + if (!can_rename(method)) { info.need_vmethod++; return true; } @@ -1918,9 +2016,9 @@ bool MultiMethodInliner::check_android_os_version(IRInstruction* insn) { std::shared_ptr MultiMethodInliner::get_callee_code_refs( const DexMethod* callee) { if (m_callee_code_refs) { - auto* res = m_callee_code_refs->get(callee); - if (res) { - return *res; + auto cached = m_callee_code_refs->get(callee, nullptr); + if (cached) { + return cached; } } @@ -1934,9 +2032,9 @@ std::shared_ptr MultiMethodInliner::get_callee_code_refs( std::shared_ptr> MultiMethodInliner::get_callee_type_refs( const DexMethod* callee) { if (m_callee_type_refs) { - auto* res = m_callee_type_refs->get(callee); - if (res) { - return *res; + auto cached = m_callee_type_refs->get(callee, nullptr); + if (cached) { + return cached; } } @@ -1983,13 +2081,14 @@ std::shared_ptr> MultiMethodInliner::get_callee_type_refs( CalleeCallerRefs MultiMethodInliner::get_callee_caller_refs( const DexMethod* callee) { if (m_callee_caller_refs) { - auto* res = m_callee_caller_refs->get(callee); - if (res) { - return *res; + CalleeCallerRefs absent = {false, std::numeric_limits::max()}; + auto cached = m_callee_caller_refs->get(callee, absent); + if (cached.classes != absent.classes) { + return cached; } } - const auto& callers = callee_caller.at_unsafe(callee); + const auto& callers = callee_caller.at(callee); std::unordered_set caller_classes; for (auto& p : callers) { auto caller = p.first; diff --git a/service/method-inliner/Inliner.h b/service/method-inliner/Inliner.h index e565045d29..392d2ca7b2 100644 --- a/service/method-inliner/Inliner.h +++ b/service/method-inliner/Inliner.h @@ -45,61 +45,6 @@ enum MultiMethodInlinerMode { IntraDex, }; -struct InlinerCostConfig { - // The following costs are in terms of code-units (2 bytes). - // Typical overhead of calling a method, without move-result overhead. - float cost_invoke; - - // Typical overhead of having move-result instruction. - float cost_move_result; - - // Overhead of having a method and its metadata. - size_t cost_method; - - // Typical savings in caller when callee doesn't use any argument. - float unused_args_discount; - - size_t reg_threshold_1; - - size_t reg_threshold_2; - - size_t op_init_class_cost; - - size_t op_injection_id_cost; - - size_t op_unreachable_cost; - - size_t op_move_exception_cost; - - size_t insn_cost_1; - - size_t insn_has_data_cost; - - size_t insn_has_lit_cost_1; - - size_t insn_has_lit_cost_2; - - size_t insn_has_lit_cost_3; -}; - -const struct InlinerCostConfig DEFAULT_COST_CONFIG = { - 3.7f, // cost_invoke - 3.0f, // cost_move_result - 16, // cost_method - 1.0f, // unused_args_discount - 3, // reg_threshold_1 - 2, // reg_threshold_2 - 2, // op_init_class_cost - 3, // op_injection_id_cost - 1, // op_unreachable_cost - 8, // op_move_exception_cost - 1, // insn_cost_1 - 4, // insn_has_data_cost - 4, // insn_has_lit_cost_1 - 2, // insn_has_lit_cost_2 - 1, // insn_has_lit_cost_3 -}; - // All call-sites of a callee. struct CallerInsns { // Invoke instructions per caller @@ -109,7 +54,6 @@ struct CallerInsns { std::unordered_map inlined_invokes_need_cast; // Whether there may be any other unknown call-sites. bool other_call_sites{false}; - bool other_call_sites_overriding_methods_added{false}; bool empty() const { return caller_insns.empty() && !other_call_sites; } }; @@ -127,6 +71,8 @@ class ReducedCode { struct Inlinable { DexMethod* callee; + // Only used when not using cfg; iterator to invoke instruction to callee + IRList::iterator iterator; // Invoke instruction to callee IRInstruction* insn; // Whether the invocation at a particular call-site is guaranteed to not @@ -138,9 +84,6 @@ struct Inlinable { std::shared_ptr reduced_code; // Estimated size of callee, possibly reduced by call-site specific knowledge size_t insn_size; - // Whether the callee is a virtual method different from the one referenced in - // the invoke instruction. - DexType* needs_receiver_cast; }; struct CalleeCallerRefs { @@ -172,7 +115,7 @@ struct InlinedCost { // Maximum or call-site specific estimated callee size after pruning size_t insn_size; - bool operator==(const InlinedCost& other) const { + bool operator==(const InlinedCost& other) { // TODO: Also check that reduced_cfg's are equivalent return full_code == other.full_code && code == other.code && method_refs == other.method_refs && other_refs == other.other_refs && @@ -215,15 +158,14 @@ class MultiMethodInliner { bool cross_dex_penalty = false, const std::unordered_set& configured_finalish_field_names = {}, - bool local_only = false, - InlinerCostConfig m_inliner_cost_config = DEFAULT_COST_CONFIG); + bool local_only = false); ~MultiMethodInliner() { delayed_invoke_direct_to_static(); } /** * attempt inlining for all candidates. */ - void inline_methods(); + void inline_methods(bool methods_need_deconstruct = true); /** * Return the set of unique inlined methods. @@ -510,6 +452,12 @@ class MultiMethodInliner { */ void shrink_method(DexMethod* method); + /** + * Whether inline_inlinables needs to deconstruct the caller's and callees' + * code. + */ + bool inline_inlinables_need_deconstruct(DexMethod* method); + // Checks that... // - there are no assignments to (non-inherited) instance fields before // a constructor call, and @@ -536,9 +484,9 @@ class MultiMethodInliner { // Maps from callee to callers and reverse map from caller to callees. // Those are used to perform bottom up inlining. // - ConcurrentMethodToMethodOccurrences callee_caller; + MethodToMethodOccurrences callee_caller; - ConcurrentMethodToMethodOccurrences caller_callee; + MethodToMethodOccurrences caller_callee; // Auxiliary data for a caller that contains true virtual callees struct CallerVirtualCallees { @@ -566,21 +514,22 @@ class MultiMethodInliner { // Cache of the inlined costs of fully inlining a calle without using any // summaries for pruning. - mutable InsertOnlyConcurrentMap + mutable ConcurrentMap> m_fully_inlined_costs; // Cache of the average inlined costs of each method. - mutable InsertOnlyConcurrentMap + mutable ConcurrentMap> m_average_inlined_costs; // Cache of the inlined costs of each call-site summary after pruning. - mutable InsertOnlyConcurrentMap> + mutable ConcurrentMap, + boost::hash> m_call_site_inlined_costs; // Cache of the inlined costs of each call-site after pruning. - mutable InsertOnlyConcurrentMap + mutable ConcurrentMap> m_invoke_call_site_inlined_costs; // Priority thread pool to handle parallel processing of methods, either @@ -601,29 +550,27 @@ class MultiMethodInliner { std::mutex m_visibility_changes_mutex; // Cache for should_inline function - InsertOnlyConcurrentMap m_should_inline; + ConcurrentMap> m_should_inline; // Optional cache for get_callee_insn_size function - std::unique_ptr> - m_callee_insn_sizes; + std::unique_ptr> m_callee_insn_sizes; // Optional cache for get_callee_type_refs function std::unique_ptr< - InsertOnlyConcurrentMap>>> + ConcurrentMap>>> m_callee_type_refs; // Optional cache for get_callee_code_refs function - std::unique_ptr< - InsertOnlyConcurrentMap>> + std::unique_ptr>> m_callee_code_refs; // Optional cache for get_callee_caller_res function - std::unique_ptr> + std::unique_ptr> m_callee_caller_refs; // Cache of whether a constructor can be unconditionally inlined. - mutable InsertOnlyConcurrentMap m_can_inline_init; + mutable ConcurrentMap> + m_can_inline_init; std::unique_ptr m_call_site_summarizer; @@ -701,8 +648,6 @@ class MultiMethodInliner { bool m_local_only; - InlinerCostConfig m_inliner_cost_config; - public: const InliningInfo& get_info() { return info; } diff --git a/service/method-inliner/MethodInliner.cpp b/service/method-inliner/MethodInliner.cpp index 9556149308..c4d8efed00 100644 --- a/service/method-inliner/MethodInliner.cpp +++ b/service/method-inliner/MethodInliner.cpp @@ -21,6 +21,7 @@ #include "DexUtil.h" #include "IRCode.h" #include "IRInstruction.h" +#include "Inliner.h" #include "LiveRange.h" #include "MethodOverrideGraph.h" #include "MethodProfiles.h" @@ -226,7 +227,7 @@ static void filter_candidates_local_only( */ std::unordered_set gather_non_virtual_methods( Scope& scope, - const InsertOnlyConcurrentSet* non_virtual, + const std::unordered_set* non_virtual, const std::unordered_set& no_devirtualize_anno) { // trace counter size_t all_methods = 0; @@ -333,14 +334,10 @@ SameImplementationMap get_same_implementation_map( SameImplementation same_implementation{nullptr, {}}; auto consider_method = [&](DexMethod* method) { always_assert(method->get_code()); - always_assert(method->get_code()->editable_cfg_built()); - if (same_implementation.representative != nullptr) { - always_assert(same_implementation.representative->get_code() - ->editable_cfg_built()); - if (!method->get_code()->cfg().structural_equals( - same_implementation.representative->get_code()->cfg())) { - return false; - } + if (same_implementation.representative != nullptr && + !method->get_code()->structural_equals( + *same_implementation.representative->get_code())) { + return false; } if (same_implementation.representative == nullptr || compare_dexmethods(method, same_implementation.representative)) { @@ -409,16 +406,15 @@ bool can_have_unknown_implementations(const mog::Graph& method_override_graph, return true; } // Also check that for all overridden methods. - return mog::any_overridden_methods( - method_override_graph, method, - [&](auto* overridden_method) { - if (is_interface(type_class(overridden_method->get_class())) && - (root(overridden_method) || !can_rename(overridden_method))) { - return true; - } - return false; - }, - /* include_interfaces */ true); + const auto& overridden_methods = mog::get_overridden_methods( + method_override_graph, method, /* include_interfaces */ true); + for (auto overridden_method : overridden_methods) { + if (is_interface(type_class(overridden_method->get_class())) && + (root(overridden_method) || !can_rename(overridden_method))) { + return true; + } + } + return false; }; /** @@ -431,13 +427,13 @@ bool can_have_unknown_implementations(const mog::Graph& method_override_graph, */ void gather_true_virtual_methods( const mog::Graph& method_override_graph, - const InsertOnlyConcurrentSet& non_virtual, + const std::unordered_set& non_virtual, const Scope& scope, const SameImplementationMap& same_implementation_map, CalleeCallerInsns* true_virtual_callers) { Timer t("gather_true_virtual_methods"); ConcurrentMap concurrent_true_virtual_callers; - InsertOnlyConcurrentMap + ConcurrentMap same_implementation_invokes; // Add mapping from callee to monomorphic callsites. auto add_monomorphic_call_site = [&](const DexMethod* caller, @@ -448,67 +444,22 @@ void gather_true_virtual_methods( m.caller_insns[caller].emplace(callsite); }); }; - auto add_other_call_site = - [&](const DexMethod* callee, - bool other_call_sites_overriding_methods_added = false) { - bool res; - concurrent_true_virtual_callers.update( - callee, [&](const DexMethod*, CallerInsns& m, bool) { - m.other_call_sites = true; - res = m.other_call_sites_overriding_methods_added; - if (other_call_sites_overriding_methods_added) { - m.other_call_sites_overriding_methods_added = true; - } - }); - return res; - }; + auto add_other_call_site = [&](const DexMethod* callee) { + concurrent_true_virtual_callers.update( + callee, [&](const DexMethod*, CallerInsns& m, bool) { + m.other_call_sites = true; + }); + }; auto add_candidate = [&](const DexMethod* callee) { concurrent_true_virtual_callers.emplace(callee, CallerInsns()); }; - struct Key { - DexMethod* callee; - DexType* static_base_type; - bool operator==(const Key& other) const { - return callee == other.callee && - static_base_type == other.static_base_type; - } - }; - struct Hash { - size_t operator()(const Key& key) const { - size_t hash = 0; - boost::hash_combine(hash, key.callee); - boost::hash_combine(hash, key.static_base_type); - return hash; - } - }; - InsertOnlyConcurrentMap, Hash> - concurrent_overriding_methods; - auto get_overriding_methods = - [&](DexMethod* callee, - DexType* static_base_type) -> const std::vector& { - return *concurrent_overriding_methods - .get_or_create_and_assert_equal( - Key{callee, static_base_type}, - [&](const Key&) { - auto overriding_methods = mog::get_overriding_methods( - method_override_graph, callee, - /* include_interfaces */ false, static_base_type); - std20::erase_if(overriding_methods, [&](auto* m) { - return !method::may_be_invoke_target(m); - }); - return overriding_methods; - }) - .first; - }; - walk::parallel::methods(scope, [&non_virtual, &method_override_graph, &add_monomorphic_call_site, &add_other_call_site, &add_candidate, &same_implementation_invokes, - &same_implementation_map, - &get_overriding_methods](DexMethod* method) { - if (method->is_virtual() && !non_virtual.count_unsafe(method)) { + &same_implementation_map](DexMethod* method) { + if (method->is_virtual() && !non_virtual.count(method)) { add_candidate(method); if (root(method)) { add_other_call_site(method); @@ -537,26 +488,32 @@ void gather_true_virtual_methods( continue; } auto insn_method = insn->get_method(); - auto callee = resolve_invoke_method(insn, method); + auto callee = + resolve_method(insn_method, opcode_to_search(insn), method); + if (callee == nullptr) { + // There are some invoke-virtual call on methods whose def are + // actually in interface. + callee = resolve_method(insn->get_method(), + MethodSearch::InterfaceVirtual); + } if (callee == nullptr) { continue; } - if (non_virtual.count_unsafe(callee) != 0) { + if (non_virtual.count(callee) != 0) { // Not true virtual, no need to continue; continue; } auto static_base_type = insn_method->get_class(); if (can_have_unknown_implementations(method_override_graph, callee)) { - bool consider_overriding_methods = - insn->opcode() != OPCODE_INVOKE_SUPER; - if (!add_other_call_site(callee, consider_overriding_methods) && - consider_overriding_methods) { - const auto& overriding_methods = - get_overriding_methods(callee, static_base_type); + add_other_call_site(callee); + if (insn->opcode() != OPCODE_INVOKE_SUPER) { + auto overriding_methods = mog::get_overriding_methods( + method_override_graph, callee, /* include_interfaces */ false, + static_base_type); for (auto overriding_method : overriding_methods) { - add_other_call_site( - overriding_method, - /* other_call_sites_overriding_methods_added */ true); + if (method::may_be_invoke_target(overriding_method)) { + add_other_call_site(overriding_method); + } } } continue; @@ -576,8 +533,12 @@ void gather_true_virtual_methods( same_implementation_invokes.emplace(insn, it->second.get()); continue; } - const auto& overriding_methods = - get_overriding_methods(callee, static_base_type); + auto overriding_methods = mog::get_overriding_methods( + method_override_graph, callee, /* include_interfaces */ false, + static_base_type); + std20::erase_if(overriding_methods, [&](auto* m) { + return !method::may_be_invoke_target(m); + }); if (overriding_methods.empty()) { // There is no override for this method add_monomorphic_call_site(method, insn, callee); @@ -587,14 +548,9 @@ void gather_true_virtual_methods( auto implementing_method = *overriding_methods.begin(); add_monomorphic_call_site(method, insn, implementing_method); } else { - if (!add_other_call_site( - callee, - /* other_call_sites_overriding_methods_added */ true)) { - for (auto overriding_method : overriding_methods) { - add_other_call_site( - overriding_method, - /* other_call_sites_overriding_methods_added */ true); - } + add_other_call_site(callee); + for (auto overriding_method : overriding_methods) { + add_other_call_site(overriding_method); } } } @@ -623,15 +579,11 @@ void gather_true_virtual_methods( } // Figure out if candidates use the receiver in a way that does require // a cast. - live_range::Uses first_load_param_uses; + std::unordered_set first_load_param_uses; { + live_range::MoveAwareChains chains(code->cfg()); auto ii = InstructionIterable(code->cfg().get_param_instructions()); auto first_load_param = ii.begin()->insn; - live_range::MoveAwareChains chains(code->cfg(), - /* ignore_unreachable */ false, - [first_load_param](auto* insn) { - return insn == first_load_param; - }); first_load_param_uses = std::move(chains.get_def_use_chains()[first_load_param]); } @@ -721,15 +673,13 @@ void gather_true_virtual_methods( } // namespace namespace inliner { -void run_inliner( - DexStoresVector& stores, - PassManager& mgr, - ConfigFiles& conf, - InlinerCostConfig inliner_cost_config /* DEFAULT_COST_CONFIG */, - bool intra_dex /* false */, - InlineForSpeed* inline_for_speed /* nullptr */, - bool inline_bridge_synth_only /* false */, - bool local_only /* false */) { +void run_inliner(DexStoresVector& stores, + PassManager& mgr, + ConfigFiles& conf, + bool intra_dex /* false */, + InlineForSpeed* inline_for_speed /* nullptr */, + bool inline_bridge_synth_only /* false */, + bool local_only /* false */) { always_assert_log( !mgr.init_class_lowering_has_run(), "Implementation limitation: The inliner could introduce new " @@ -786,10 +736,10 @@ void run_inliner( inliner_config.unique_inlined_registers = false; std::unique_ptr method_override_graph; - std::unique_ptr> non_virtual; + std::unique_ptr> non_virtual; if (inliner_config.virtual_inline) { method_override_graph = mog::build_graph(scope); - non_virtual = std::make_unique>( + non_virtual = std::make_unique>( mog::get_non_true_virtuals(*method_override_graph, scope)); } @@ -810,6 +760,10 @@ void run_inliner( get_same_implementation_map(scope, *method_override_graph)); } + walk::parallel::code(scope, [](DexMethod*, IRCode& code) { + code.build_cfg(/* editable */ true); + }); + if (inliner_config.virtual_inline && inliner_config.true_virtual_inline) { gather_true_virtual_methods(*method_override_graph, *non_virtual, scope, *same_implementation_map, @@ -843,9 +797,11 @@ void run_inliner( intra_dex ? IntraDex : InterDex, true_virtual_callers, inline_for_speed, analyze_and_prune_inits, conf.get_pure_methods(), min_sdk_api, cross_dex_penalty, - /* configured_finalish_field_names */ {}, local_only, - inliner_cost_config); - inliner.inline_methods(); + /* configured_finalish_field_names */ {}, local_only); + inliner.inline_methods(/* need_deconstruct */ false); + + walk::parallel::code(scope, + [](DexMethod*, IRCode& code) { code.clear_cfg(); }); // delete all methods that can be deleted auto inlined = inliner.get_inlined(); @@ -995,10 +951,10 @@ void run_inliner( shrinker.get_cse_stats().instructions_eliminated); mgr.incr_metric("instructions_eliminated_copy_prop", shrinker.get_copy_prop_stats().moves_eliminated); - mgr.incr_metric("instructions_eliminated_localdce_dead", - shrinker.get_local_dce_stats().dead_instruction_count); - mgr.incr_metric("instructions_eliminated_localdce_unreachable", - shrinker.get_local_dce_stats().unreachable_instruction_count); + mgr.incr_metric( + "instructions_eliminated_localdce", + shrinker.get_local_dce_stats().dead_instruction_count + + shrinker.get_local_dce_stats().unreachable_instruction_count); mgr.incr_metric("instructions_eliminated_unreachable", inliner.get_info().unreachable_insns); mgr.incr_metric("instructions_eliminated_dedup_blocks", diff --git a/service/method-inliner/MethodInliner.h b/service/method-inliner/MethodInliner.h index 17af6e5811..60d01f8b82 100644 --- a/service/method-inliner/MethodInliner.h +++ b/service/method-inliner/MethodInliner.h @@ -5,7 +5,6 @@ * LICENSE file in the root directory of this source tree. */ -#include "Inliner.h" #include "InlinerConfig.h" #include "PassManager.h" @@ -20,7 +19,6 @@ namespace inliner { void run_inliner(DexStoresVector& stores, PassManager& mgr, ConfigFiles& conf, - InlinerCostConfig inliner_cost_config = DEFAULT_COST_CONFIG, bool intra_dex = false, InlineForSpeed* inline_for_speed = nullptr, bool inline_bridge_synth_only = false, diff --git a/service/method-inliner/RecursionPruner.cpp b/service/method-inliner/RecursionPruner.cpp index 7a108b84b3..99f3a72168 100644 --- a/service/method-inliner/RecursionPruner.cpp +++ b/service/method-inliner/RecursionPruner.cpp @@ -10,8 +10,8 @@ namespace inliner { RecursionPruner::RecursionPruner( - ConcurrentMethodToMethodOccurrences& callee_caller, - ConcurrentMethodToMethodOccurrences& caller_callee, + MethodToMethodOccurrences& callee_caller, + MethodToMethodOccurrences& caller_callee, std::function exclude_fn) : m_callee_caller(callee_caller), m_caller_callee(caller_callee), @@ -83,7 +83,7 @@ size_t RecursionPruner::recurse( if (callees.empty()) { m_caller_callee.erase(caller); } - auto& callers = m_callee_caller.at_unsafe(callee); + auto& callers = m_callee_caller.at(callee); callers.erase(caller); if (callers.empty()) { m_callee_caller.erase(callee); diff --git a/service/method-inliner/RecursionPruner.h b/service/method-inliner/RecursionPruner.h index 09d062e182..d98874cadb 100644 --- a/service/method-inliner/RecursionPruner.h +++ b/service/method-inliner/RecursionPruner.h @@ -13,8 +13,8 @@ namespace inliner { class RecursionPruner { private: - ConcurrentMethodToMethodOccurrences& m_callee_caller; - ConcurrentMethodToMethodOccurrences& m_caller_callee; + MethodToMethodOccurrences& m_callee_caller; + MethodToMethodOccurrences& m_caller_callee; size_t m_recursive_call_sites{0}; size_t m_max_call_stack_depth{0}; std::unordered_set m_recursive_callees; @@ -22,8 +22,8 @@ class RecursionPruner { std::function m_exclude_fn; public: - RecursionPruner(ConcurrentMethodToMethodOccurrences& callee_caller, - ConcurrentMethodToMethodOccurrences& caller_callee, + RecursionPruner(MethodToMethodOccurrences& callee_caller, + MethodToMethodOccurrences& caller_callee, std::function exclude_fn); void run(); diff --git a/service/method-outliner/OutliningProfileGuidanceImpl.cpp b/service/method-outliner/OutliningProfileGuidanceImpl.cpp index 3791a977cc..a91ba890ca 100644 --- a/service/method-outliner/OutliningProfileGuidanceImpl.cpp +++ b/service/method-outliner/OutliningProfileGuidanceImpl.cpp @@ -12,6 +12,7 @@ #include "MethodOverrideGraph.h" #include "MethodProfiles.h" #include "PassManager.h" +#include "ScopedCFG.h" #include "Show.h" #include "SourceBlocks.h" #include "Walkers.h" @@ -181,7 +182,7 @@ std::vector get_possibly_warm_or_hot_methods( // next method on success/failure, as there's no point // checking the same fixed condition for other // interactions. - always_assert(code->editable_cfg_built()); + cfg::ScopedCFG scopedCFG(code); auto& cfg = code->cfg(); auto entry_block = cfg.entry_block(); auto entry_sb = source_blocks::get_first_source_block(entry_block); diff --git a/service/reference-update/TypeReference.h b/service/reference-update/TypeReference.h index 2294f3be39..ba0ff54f3d 100644 --- a/service/reference-update/TypeReference.h +++ b/service/reference-update/TypeReference.h @@ -65,7 +65,7 @@ class TypeRefUpdater final { */ bool mangling(DexMethodRef* method); - InsertOnlyConcurrentMap m_inits; + ConcurrentMap m_inits; const std::unordered_map& m_old_to_new; }; diff --git a/service/regalloc-fast/LiveInterval.cpp b/service/regalloc-fast/LiveInterval.cpp index 4da4c59c8f..9ec73820a5 100644 --- a/service/regalloc-fast/LiveInterval.cpp +++ b/service/regalloc-fast/LiveInterval.cpp @@ -164,7 +164,7 @@ VRegAliveRangeInBlock get_check_cast_throw_targets_live_range( const std::unordered_set& vregs) { VRegAliveRangeInBlock vreg_block_range; const LivenessDomain& live_in = fixpoint_iter.get_live_in_vars_at(block); - const auto& elements = live_in.elements(); + auto elements = live_in.elements(); for (auto vreg : vregs) { if (!elements.contains(vreg)) { LiveIntervalPoint first = LiveIntervalPoint::get_block_begin(block); diff --git a/service/regalloc/RegisterAllocation.cpp b/service/regalloc/RegisterAllocation.cpp index 4bfe155433..1c91c1b9bd 100644 --- a/service/regalloc/RegisterAllocation.cpp +++ b/service/regalloc/RegisterAllocation.cpp @@ -11,7 +11,6 @@ #include "CppUtil.h" #include "Debug.h" -#include "DebugUtils.h" #include "DexClass.h" #include "DexUtil.h" #include "GraphColoring.h" diff --git a/service/shrinker/Shrinker.cpp b/service/shrinker/Shrinker.cpp index a2b442a02a..52a849a653 100644 --- a/service/shrinker/Shrinker.cpp +++ b/service/shrinker/Shrinker.cpp @@ -56,6 +56,9 @@ Shrinker::ShrinkerForest load(const std::string& filename) { } bool should_shrink(IRCode* code, const Shrinker::ShrinkerForest& forest) { + if (!code->editable_cfg_built()) { + code->build_cfg(/* editable= */ true); + } const auto& cfg = code->cfg(); size_t regs = cfg.get_registers_size(); @@ -206,10 +209,12 @@ void Shrinker::shrink_code( DexType* declaring_type, DexProto* proto, const std::function& method_describer) { - always_assert(code->editable_cfg_built()); + bool editable_cfg_built = code->editable_cfg_built(); // force simplification/linearization of any existing editable cfg once, and // forget existing cfg for a clean start - code->cfg().recompute_registers_size(); + if (editable_cfg_built) { + code->cfg().recompute_registers_size(); + } code->clear_cfg(); constant_propagation::Transform::Stats const_prop_stats; @@ -218,9 +223,12 @@ void Shrinker::shrink_code( LocalDce::Stats local_dce_stats; dedup_blocks_impl::Stats dedup_blocks_stats; - code->build_cfg(); if (m_config.run_const_prop) { auto timer = m_const_prop_timer.scope(); + if (!code->editable_cfg_built()) { + code->build_cfg(/* editable */ true); + } + constant_propagation::Transform::Config config; config.pure_methods = &m_pure_methods; const_prop_stats = constant_propagation(is_static, declaring_type, proto, @@ -229,6 +237,10 @@ void Shrinker::shrink_code( if (m_config.run_cse) { auto timer = m_cse_timer.scope(); + if (!code->editable_cfg_built()) { + code->build_cfg(/* editable */ true); + } + cse_impl::CommonSubexpressionElimination cse( m_cse_shared_state.get(), code->cfg(), is_static, is_init_or_clinit, declaring_type, proto->get_args()); @@ -238,6 +250,10 @@ void Shrinker::shrink_code( if (m_config.run_copy_prop) { auto timer = m_copy_prop_timer.scope(); + if (!code->editable_cfg_built()) { + code->build_cfg(/* editable */ true); + } + copy_prop_stats = copy_propagation(code, is_static, declaring_type, proto->get_rtype(), proto->get_args(), method_describer); @@ -245,8 +261,12 @@ void Shrinker::shrink_code( if (m_config.run_local_dce) { auto timer = m_local_dce_timer.scope(); + if (!code->editable_cfg_built()) { + code->build_cfg(/* editable */ true); + } + local_dce_stats = - local_dce(code, m_config.normalize_new_instances, declaring_type); + local_dce(code, /* normalize_new_instances */ true, declaring_type); } using stats_t = std::tuple; @@ -254,6 +274,9 @@ void Shrinker::shrink_code( if (!traceEnabled(MMINL, mminl_level)) { return stats_t{}; } + if (!code->editable_cfg_built()) { + code->build_cfg(/* editable= */ true); + } const auto& cfg = code->cfg(); size_t regs_before = cfg.get_registers_size(); @@ -297,6 +320,10 @@ void Shrinker::shrink_code( if (m_config.run_fast_reg_alloc) { auto timer = m_reg_alloc_timer.scope(); + if (!code->editable_cfg_built()) { + code->build_cfg(/* editable= */ true); + } + auto allocator = fastregalloc::LinearScanAllocator(code, is_static, method_describer); allocator.allocate(); @@ -304,6 +331,10 @@ void Shrinker::shrink_code( if (m_config.run_dedup_blocks) { auto timer = m_dedup_blocks_timer.scope(); + if (!code->editable_cfg_built()) { + code->build_cfg(/* editable */ true); + } + dedup_blocks_impl::Config config; dedup_blocks_impl::DedupBlocks dedup_blocks( &config, code, is_static, declaring_type, proto->get_args()); @@ -322,6 +353,12 @@ void Shrinker::shrink_code( std::get<2>(data_after_dedup), std::get<3>(data_after_dedup)); } + if (editable_cfg_built && !code->editable_cfg_built()) { + code->build_cfg(/* editable */ true); + } else if (!editable_cfg_built && code->editable_cfg_built()) { + code->clear_cfg(); + } + std::lock_guard guard(m_stats_mutex); m_const_prop_stats += const_prop_stats; m_cse_stats += cse_stats; diff --git a/service/switch-dispatch/SwitchDispatch.cpp b/service/switch-dispatch/SwitchDispatch.cpp index 378019d3e0..5a6a078ce7 100644 --- a/service/switch-dispatch/SwitchDispatch.cpp +++ b/service/switch-dispatch/SwitchDispatch.cpp @@ -245,7 +245,7 @@ size_t estimate_num_switch_dispatch_needed( DexMethod* create_simple_switch_dispatch( const dispatch::Spec& spec, const std::map& indices_to_callee) { - always_assert(!indices_to_callee.empty()); + always_assert(indices_to_callee.size()); TRACE(SDIS, 5, "creating leaf switch dispatch %s.%s for targets of size %zu", @@ -689,11 +689,9 @@ bool may_be_dispatch(const DexMethod* method) { if (name.find(DISPATCH_PREFIX) != 0) { return false; } - auto code = const_cast(method)->get_code(); - always_assert(code->editable_cfg_built()); - auto& cfg = code->cfg(); + auto code = method->get_code(); uint32_t branches = 0; - for (auto& mie : cfg::InstructionIterable(cfg)) { + for (auto& mie : InstructionIterable(code)) { auto op = mie.insn->opcode(); if (opcode::is_switch(op)) { return true; diff --git a/service/switch-partitioning/SwitchEquivFinder.cpp b/service/switch-partitioning/SwitchEquivFinder.cpp index 68801ed93d..f91603f300 100644 --- a/service/switch-partitioning/SwitchEquivFinder.cpp +++ b/service/switch-partitioning/SwitchEquivFinder.cpp @@ -594,8 +594,8 @@ void SwitchEquivFinder::find_case_keys(const std::vector& leaves) { // Get the inferred value of m_switching_reg at the end of `edge_to_leaf` // but before the beginning of the leaf block because we would lose the // information by merging all the incoming edges. - const auto& exit_env = fixpoint.get_exit_state_at(edge_to_leaf->src()); - auto env = fixpoint.analyze_edge(edge_to_leaf, exit_env); + auto env = fixpoint.get_exit_state_at(edge_to_leaf->src()); + env = fixpoint.analyze_edge(edge_to_leaf, env); const auto& val = env.get(m_switching_reg); return ConstantValue::apply_visitor(key_creating_visitor(), val); }; diff --git a/service/switch-partitioning/SwitchEquivFinder.h b/service/switch-partitioning/SwitchEquivFinder.h index afe416403b..7e8d78b121 100644 --- a/service/switch-partitioning/SwitchEquivFinder.h +++ b/service/switch-partitioning/SwitchEquivFinder.h @@ -206,6 +206,5 @@ class SwitchEquivEditor { // turned into a duplicate of its successor (attaching its successor's // successors onto itself, etc). This is meant to ensure ExtraLoads state is // accurate. - static size_t normalize_sled_blocks( - cfg::ControlFlowGraph* cfg, const uint32_t leaf_duplication_threshold); + static size_t normalize_sled_blocks(cfg::ControlFlowGraph* cfg, const uint32_t leaf_duplication_threshold); }; diff --git a/service/switch-partitioning/SwitchEquivPrerequisites.h b/service/switch-partitioning/SwitchEquivPrerequisites.h index 2c58e42452..e47b562950 100644 --- a/service/switch-partitioning/SwitchEquivPrerequisites.h +++ b/service/switch-partitioning/SwitchEquivPrerequisites.h @@ -30,13 +30,8 @@ inline bool gather_linear_prologue_blocks( return false; } } - std::unordered_set visited; for (cfg::Block* b = cfg->entry_block(); b != nullptr; b = b->goes_to_only_edge()) { - if (!visited.insert(b).second) { - // non-terminating loop - return false; - } prologue_blocks->push_back(b); } if (prologue_blocks->empty()) { diff --git a/service/switch-partitioning/SwitchMethodPartitioning.cpp b/service/switch-partitioning/SwitchMethodPartitioning.cpp new file mode 100644 index 0000000000..22b506abd3 --- /dev/null +++ b/service/switch-partitioning/SwitchMethodPartitioning.cpp @@ -0,0 +1,115 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "SwitchMethodPartitioning.h" + +#include +#include + +#include "ConstantEnvironment.h" +#include "ConstantPropagationAnalysis.h" +#include "ControlFlow.h" +#include "SwitchEquivFinder.h" +#include "SwitchEquivPrerequisites.h" + +namespace cp = constant_propagation; + +namespace { +// See comments in SwitchEquivFinder.h for an explanation. +constexpr size_t DEFAULT_LEAF_DUP_THRESHOLD = 50; + +// Check whether, possibly at the end of a chain of gotos, the block will +// unconditionally throw. +bool throws(cfg::Block* block) { + std::unordered_set visited{block}; + for (; block->goes_to_only_edge(); block = block->goes_to_only_edge()) { + if (!visited.insert(block->goes_to_only_edge()).second) { + // non-terminating loop + return false; + } + } + auto last_insn_it = block->get_last_insn(); + return last_insn_it != block->end() && + last_insn_it->insn->opcode() == OPCODE_THROW; +} +} // namespace + +std::unique_ptr SwitchMethodPartitioning::create( + IRCode* code, bool verify_default_case_throws) { + cfg::ScopedCFG cfg(code); + // Check for a throw only method up front. SwitchEquivFinder will not + // represent this out of the box, so convert directly to + // SwitchMethodPartitioning representation. + if (throws(cfg->entry_block())) { + TRACE(SW, 3, "Special case: method always throws"); + std::vector entry_blocks; + entry_blocks.emplace_back(cfg->entry_block()); + return std::unique_ptr( + new SwitchMethodPartitioning(std::move(cfg), std::move(entry_blocks), + {})); + } + + // Note that a single-case switch can be compiled as either a switch opcode or + // a series of if-* opcodes. We can use constant propagation to handle these + // cases uniformly: to determine the case key, we use the inferred value of + // the operand to the branching opcode in the successor blocks. + std::vector prologue_blocks; + if (!gather_linear_prologue_blocks(cfg.get(), &prologue_blocks)) { + TRACE(SW, 3, "Prologue blocks do not have expected branching"); + return nullptr; + } + + // Ensure that cfg forms that are not simplified (due to existence of source + // blocks) can get handled gracefully. Use the same leaf duplication strategy + // as the finder would. + auto blocks_changed = SwitchEquivEditor::normalize_sled_blocks( + cfg.get(), DEFAULT_LEAF_DUP_THRESHOLD); + if (blocks_changed > 0 && traceEnabled(SW, 2)) { + TRACE(SW, 2, "Replaced %zu block(s) to normalize; %s", blocks_changed, + SHOW(*cfg)); + } + + auto fixpoint = std::make_shared( + *cfg, SwitchEquivFinder::Analyzer()); + fixpoint->run(ConstantEnvironment()); + reg_t determining_reg; + if (!find_determining_reg(*fixpoint, prologue_blocks.back(), + &determining_reg)) { + TRACE(SW, 3, "Unknown const for branching"); + return nullptr; + } + auto last_prologue_block = prologue_blocks.back(); + auto last_prologue_insn = last_prologue_block->get_last_insn(); + auto root_branch = cfg->find_insn(last_prologue_insn->insn); + auto finder = std::make_unique( + cfg.get(), root_branch, determining_reg, DEFAULT_LEAF_DUP_THRESHOLD, + fixpoint, SwitchEquivFinder::EXECUTION_ORDER); + if (!finder->success() || + !finder->are_keys_uniform(SwitchEquivFinder::KeyKind::INT)) { + TRACE(SW, 3, "Cannot represent method as switch equivalent"); + return nullptr; + } + + if (verify_default_case_throws) { + always_assert_log(finder->default_case() != boost::none, + "Method does not have default case"); + auto default_block = *finder->default_case(); + always_assert_log(throws(default_block), "Default case B%zu should throw", + default_block->id()); + } + + // Method is supported, munge into simpler format expected by callers. + std::map key_to_block; + for (auto&& [key, block] : finder->key_to_case()) { + if (!SwitchEquivFinder::is_default_case(key)) { + auto i = boost::get(key); + key_to_block.emplace(i, block); + } + } + return std::unique_ptr(new SwitchMethodPartitioning( + std::move(cfg), std::move(prologue_blocks), std::move(key_to_block))); +} diff --git a/service/switch-partitioning/SwitchMethodPartitioning.h b/service/switch-partitioning/SwitchMethodPartitioning.h new file mode 100644 index 0000000000..c517e4c5a0 --- /dev/null +++ b/service/switch-partitioning/SwitchMethodPartitioning.h @@ -0,0 +1,72 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include +#include +#include + +#include "ConstantPropagationAnalysis.h" +#include "DexClass.h" +#include "IRCode.h" +#include "ScopedCFG.h" + +/* + * This is designed to work on methods with a very specific control-flow graph + * -- methods whose sources contain a single switch statement (or if-else tree) + * and no other control-flow structures (like catch blocks). We expect the CFG + * to be of the following form: + * + * [Prologue block(s)] ____ + * _/ | \_ \______ + * / | ... \ \ + * [case 0] [case 1] ... [case N] [default case (may throw)] + * \_ | ... _/ _______/ + * \ | / __/ + * [Exit block(s)] + * + * We partition the method into these prologue blocks and case blocks. The + * default case and the exit blocks, if any, are omitted. The current usages + * of SMP have no need for the default case, and they can find the exit blocks + * easily enough by following the successor edges from the case blocks. + * + * It's also possible that there are no exit blocks, rather each case has a + * return opcode. + * + * SwitchMethodPartitioning is slightly a misnomer. It was originally designed + * for methods that had a single switch statement, but was later extended to + * support methods that use an if-else tree to choose a case block (instead of + * a switch). These methods may have been switch-only in source code, but have + * been compiled into if-else trees (usually by d8). + */ +class SwitchMethodPartitioning final { + public: + static std::unique_ptr create( + IRCode* code, bool verify_default_case_throws = true); + + const std::vector& get_prologue_blocks() const { + return m_prologue_blocks; + } + + const std::map& get_key_to_block() const { + return m_key_to_block; + } + + private: + SwitchMethodPartitioning( + cfg::ScopedCFG cfg, + std::vector prologue_blocks, + std::map key_to_block) + : m_prologue_blocks(std::move(prologue_blocks)), + m_key_to_block(std::move(key_to_block)), + m_cfg(std::move(cfg)) {} + + std::vector m_prologue_blocks; + std::map m_key_to_block; + cfg::ScopedCFG m_cfg; +}; diff --git a/service/type-analysis/GlobalTypeAnalyzer.cpp b/service/type-analysis/GlobalTypeAnalyzer.cpp index 41182e69c8..599f4243af 100644 --- a/service/type-analysis/GlobalTypeAnalyzer.cpp +++ b/service/type-analysis/GlobalTypeAnalyzer.cpp @@ -85,7 +85,7 @@ void scan_any_init_reachables( SHOW(method)); continue; } - auto callees = resolve_callees_in_graph(cg, insn); + auto callees = resolve_callees_in_graph(cg, method, insn); for (const DexMethod* callee : callees) { scan_any_init_reachables(cg, method_override_graph, callee, false, reachables); @@ -179,7 +179,7 @@ void GlobalTypeAnalyzer::analyze_node( } ArgumentTypePartition GlobalTypeAnalyzer::analyze_edge( - const call_graph::EdgeId& edge, + const std::shared_ptr& edge, const ArgumentTypePartition& exit_state_at_source) const { ArgumentTypePartition entry_state_at_dest; auto insn = edge->invoke_insn(); @@ -227,7 +227,8 @@ bool GlobalTypeAnalyzer::is_reachable(const DexMethod* method) const { } using CombinedAnalyzer = - InstructionAnalyzerCombiner; @@ -253,8 +254,10 @@ std::unique_ptr GlobalTypeAnalyzer::analyze_method( } auto env = env_with_params(&code, args); - DexType* ctor_type{nullptr}; - if (method::is_init(method)) { + DexType *clinit_type{nullptr}, *ctor_type{nullptr}; + if (method::is_clinit(method)) { + clinit_type = method->get_class(); + } else if (method::is_init(method)) { ctor_type = method->get_class(); } TRACE(TYPE, 5, "%s", SHOW(code.cfg())); @@ -263,7 +266,8 @@ std::unique_ptr GlobalTypeAnalyzer::analyze_method( ? std::make_unique( code.cfg(), CombinedReplayAnalyzer(&wps, nullptr)) : std::make_unique( - code.cfg(), CombinedAnalyzer(&wps, ctor_type, nullptr)); + code.cfg(), + CombinedAnalyzer(clinit_type, &wps, ctor_type, nullptr)); local_ta->run(env); return local_ta; @@ -389,7 +393,7 @@ void GlobalTypeAnalysis::find_any_init_reachables( SHOW(method)); continue; } - auto callees = resolve_callees_in_graph(*cg, insn); + auto callees = resolve_callees_in_graph(*cg, method, insn); for (const DexMethod* callee : callees) { bool trace_callbacks_in_callee_cls = is_leaking_this_in_ctor(method, callee); @@ -457,26 +461,21 @@ std::unique_ptr GlobalTypeAnalysis::analyze( {CURRENT_PARTITION_LABEL, ArgumentTypeEnvironment()}}); auto non_true_virtuals = mog::get_non_true_virtuals(*method_override_graph, scope); - EligibleIfields eligible_ifields; - if (m_only_aggregate_safely_inferrable_fields) { - eligible_ifields = - constant_propagation::gather_safely_inferable_ifield_candidates(scope, - {}); - } + EligibleIfields eligible_ifields = + constant_propagation::gather_safely_inferable_ifield_candidates(scope, + {}); size_t iteration_cnt = 0; for (size_t i = 0; i < m_max_global_analysis_iteration; ++i) { // Build an approximation of all the field values and method return values. TRACE(TYPE, 2, "[global] Collecting WholeProgramState"); - auto wps = - m_use_multiple_callee_callgraph - ? std::make_unique( - scope, *gta, non_true_virtuals, m_any_init_reachables, - eligible_ifields, m_only_aggregate_safely_inferrable_fields, - cg) - : std::make_unique( - scope, *gta, non_true_virtuals, m_any_init_reachables, - eligible_ifields, m_only_aggregate_safely_inferrable_fields); + auto wps = m_use_multiple_callee_callgraph + ? std::make_unique( + scope, *gta, non_true_virtuals, m_any_init_reachables, + eligible_ifields, cg) + : std::make_unique( + scope, *gta, non_true_virtuals, m_any_init_reachables, + eligible_ifields); trace_whole_program_state(*wps); trace_stats(*wps); trace_whole_program_state_diff(gta->get_whole_program_state(), *wps); diff --git a/service/type-analysis/GlobalTypeAnalyzer.h b/service/type-analysis/GlobalTypeAnalyzer.h index f0b2e66cfb..8f18584ae9 100644 --- a/service/type-analysis/GlobalTypeAnalyzer.h +++ b/service/type-analysis/GlobalTypeAnalyzer.h @@ -79,7 +79,7 @@ class GlobalTypeAnalyzer : public sparta::ParallelMonotonicFixpointIterator< ArgumentTypePartition* current_partition) const override; ArgumentTypePartition analyze_edge( - const call_graph::EdgeId& edge, + const std::shared_ptr& edge, const ArgumentTypePartition& exit_state_at_source) const override; /* @@ -125,14 +125,10 @@ class GlobalTypeAnalyzer : public sparta::ParallelMonotonicFixpointIterator< class GlobalTypeAnalysis { public: - explicit GlobalTypeAnalysis( - size_t max_global_analysis_iteration = 10, - bool use_multiple_callee_callgraph = false, - bool only_aggregate_safely_inferrable_fields = true) + explicit GlobalTypeAnalysis(size_t max_global_analysis_iteration = 10, + bool use_multiple_callee_callgraph = false) : m_max_global_analysis_iteration(max_global_analysis_iteration), - m_use_multiple_callee_callgraph(use_multiple_callee_callgraph), - m_only_aggregate_safely_inferrable_fields( - only_aggregate_safely_inferrable_fields) {} + m_use_multiple_callee_callgraph(use_multiple_callee_callgraph) {} void run(Scope& scope) { analyze(scope); } @@ -141,7 +137,6 @@ class GlobalTypeAnalysis { private: size_t m_max_global_analysis_iteration; bool m_use_multiple_callee_callgraph; - bool m_only_aggregate_safely_inferrable_fields; // Methods reachable from clinit that read static fields and reachable from // ctors that read instance fields. ConcurrentSet m_any_init_reachables; diff --git a/service/type-analysis/LocalTypeAnalyzer.cpp b/service/type-analysis/LocalTypeAnalyzer.cpp index 71189f22f4..a38b9f9f24 100644 --- a/service/type-analysis/LocalTypeAnalyzer.cpp +++ b/service/type-analysis/LocalTypeAnalyzer.cpp @@ -51,16 +51,6 @@ bool RegisterTypeAnalyzer::analyze_default(const IRInstruction* insn, return true; } -bool RegisterTypeAnalyzer::analyze_check_cast(const IRInstruction* insn, - DexTypeEnvironment* env) { - if (type::is_array(insn->get_type())) { - env->set(RESULT_REGISTER, DexTypeDomain::top()); - } else { - env->set(RESULT_REGISTER, env->get(insn->src(0))); - } - return true; -} - bool RegisterTypeAnalyzer::analyze_const(const IRInstruction* insn, DexTypeEnvironment* env) { if (insn->opcode() != OPCODE_CONST) { @@ -69,22 +59,20 @@ bool RegisterTypeAnalyzer::analyze_const(const IRInstruction* insn, if (insn->get_literal() == 0) { env->set(insn->dest(), DexTypeDomain::null()); } else { - env->set(insn->dest(), DexTypeDomain::top()); + env->set(insn->dest(), DexTypeDomain(insn->get_literal())); } return true; } bool RegisterTypeAnalyzer::analyze_const_string(const IRInstruction*, DexTypeEnvironment* env) { - env->set(RESULT_REGISTER, - DexTypeDomain::create_not_null(type::java_lang_String())); + env->set(RESULT_REGISTER, DexTypeDomain(type::java_lang_String())); return true; } bool RegisterTypeAnalyzer::analyze_const_class(const IRInstruction*, DexTypeEnvironment* env) { - env->set(RESULT_REGISTER, - DexTypeDomain::create_not_null(type::java_lang_Class())); + env->set(RESULT_REGISTER, DexTypeDomain(type::java_lang_Class())); return true; } @@ -98,8 +86,207 @@ bool RegisterTypeAnalyzer::analyze_aget(const IRInstruction* insn, always_assert_log(type::is_array(*array_type), "Wrong array type %s in %s", SHOW(*array_type), SHOW(insn)); + auto idx_opt = env->get(insn->src(1)).get_constant(); + auto nullness = env->get(insn->src(0)).get_array_element_nullness(idx_opt); const auto ctype = type::get_array_component_type(*array_type); - env->set(RESULT_REGISTER, DexTypeDomain::create_nullable(ctype)); + auto cls = type_class(ctype); + bool is_type_exact = cls && !cls->is_external() && is_final(cls); + // is_type_exact is to decide whether to populate the + // small-set-dex-type-domain, which should only hold exact (non-interface) + // class (and possibly java.lang.Throwable, but we ignore that here). + env->set(RESULT_REGISTER, + DexTypeDomain(ctype, nullness.element(), is_type_exact)); + return true; +} + +/* + * Only populating array nullness since we don't model array element types. + */ +bool RegisterTypeAnalyzer::analyze_aput(const IRInstruction* insn, + DexTypeEnvironment* env) { + if (insn->opcode() != OPCODE_APUT_OBJECT) { + return false; + } + boost::optional idx_opt = env->get(insn->src(2)).get_constant(); + auto nullness = env->get(insn->src(0)).get_nullness(); + env->mutate_reg_environment([&](RegTypeEnvironment* env) { + auto array_reg = insn->src(1); + env->update(array_reg, [&](const DexTypeDomain& domain) { + auto copy = domain; + copy.set_array_element_nullness(idx_opt, nullness); + return copy; + }); + }); + return true; +} + +bool RegisterTypeAnalyzer::analyze_array_length(const IRInstruction* insn, + DexTypeEnvironment* env) { + auto array_nullness = env->get(insn->src(0)).get_array_nullness(); + if (array_nullness.is_top()) { + env->set(RESULT_REGISTER, DexTypeDomain::top()); + return true; + } + if (auto array_length = array_nullness.get_length()) { + env->set(RESULT_REGISTER, DexTypeDomain(*array_length)); + } else { + env->set(RESULT_REGISTER, DexTypeDomain::top()); + } + return true; +} + +bool RegisterTypeAnalyzer::analyze_binop_lit(const IRInstruction* insn, + DexTypeEnvironment* env) { + auto op = insn->opcode(); + int32_t lit = insn->get_literal(); + auto int_val = env->get(insn->src(0)).get_constant(); + boost::optional result = boost::none; + + if (!int_val) { + return analyze_default(insn, env); + } + + bool use_result_reg = false; + switch (op) { + case OPCODE_ADD_INT_LIT: { + result = (*int_val) + lit; + break; + } + case OPCODE_RSUB_INT_LIT: { + result = lit - (*int_val); + break; + } + case OPCODE_MUL_INT_LIT: { + result = (*int_val) * lit; + break; + } + case OPCODE_DIV_INT_LIT: { + if (lit != 0) { + result = (*int_val) / lit; + } + use_result_reg = true; + break; + } + case OPCODE_REM_INT_LIT: { + if (lit != 0) { + result = (*int_val) % lit; + } + use_result_reg = true; + break; + } + case OPCODE_AND_INT_LIT: { + result = (*int_val) & lit; + break; + } + case OPCODE_OR_INT_LIT: { + result = (*int_val) | lit; + break; + } + case OPCODE_XOR_INT_LIT: { + result = (*int_val) ^ lit; + break; + } + // as in https://source.android.com/devices/tech/dalvik/dalvik-bytecode + // the following operations have the second operand masked. + case OPCODE_SHL_INT_LIT: { + uint32_t ucst = *int_val; + uint32_t uresult = ucst << (lit & 0x1f); + result = (int32_t)uresult; + break; + } + case OPCODE_SHR_INT_LIT: { + result = (*int_val) >> (lit & 0x1f); + break; + } + case OPCODE_USHR_INT_LIT: { + uint32_t ucst = *int_val; + // defined in dalvik spec + result = ucst >> (lit & 0x1f); + break; + } + default: + break; + } + auto res_dom = DexTypeDomain::top(); + if (result != boost::none) { + int32_t result32 = (int32_t)(*result & 0xFFFFFFFF); + res_dom = DexTypeDomain(result32); + } + env->set(use_result_reg ? RESULT_REGISTER : insn->dest(), res_dom); + return true; +} + +bool RegisterTypeAnalyzer::analyze_binop(const IRInstruction* insn, + DexTypeEnvironment* env) { + auto op = insn->opcode(); + auto int_left = env->get(insn->src(0)).get_constant(); + auto int_right = env->get(insn->src(1)).get_constant(); + if (!int_left || !int_right) { + return analyze_default(insn, env); + } + + boost::optional result = boost::none; + bool use_result_reg = false; + switch (op) { + case OPCODE_ADD_INT: + case OPCODE_ADD_LONG: { + result = (*int_left) + (*int_right); + break; + } + case OPCODE_SUB_INT: + case OPCODE_SUB_LONG: { + result = (*int_left) - (*int_right); + break; + } + case OPCODE_MUL_INT: + case OPCODE_MUL_LONG: { + result = (*int_left) * (*int_right); + break; + } + case OPCODE_DIV_INT: + case OPCODE_DIV_LONG: { + if ((*int_right) != 0) { + result = (*int_left) / (*int_right); + } + use_result_reg = true; + break; + } + case OPCODE_REM_INT: + case OPCODE_REM_LONG: { + if ((*int_right) != 0) { + result = (*int_left) % (*int_right); + } + use_result_reg = true; + break; + } + case OPCODE_AND_INT: + case OPCODE_AND_LONG: { + result = (*int_left) & (*int_right); + break; + } + case OPCODE_OR_INT: + case OPCODE_OR_LONG: { + result = (*int_left) | (*int_right); + break; + } + case OPCODE_XOR_INT: + case OPCODE_XOR_LONG: { + result = (*int_left) ^ (*int_right); + break; + } + default: + return analyze_default(insn, env); + } + auto res_dom = DexTypeDomain::top(); + if (result != boost::none) { + if (opcode::is_binop64(op)) { + res_dom = DexTypeDomain(*result); + } else { + int32_t result32 = (int32_t)(*result & 0xFFFFFFFF); + res_dom = DexTypeDomain(result32); + } + } + env->set(use_result_reg ? RESULT_REGISTER : insn->dest(), res_dom); return true; } @@ -119,27 +306,26 @@ bool RegisterTypeAnalyzer::analyze_move_exception(const IRInstruction* insn, DexTypeEnvironment* env) { // We don't know where to grab the type of the just-caught exception. // Simply set to j.l.Throwable here. - env->set(insn->dest(), - DexTypeDomain::create_nullable(type::java_lang_Throwable())); + env->set(insn->dest(), DexTypeDomain(type::java_lang_Throwable())); return true; } bool RegisterTypeAnalyzer::analyze_new_instance(const IRInstruction* insn, DexTypeEnvironment* env) { - env->set(RESULT_REGISTER, DexTypeDomain::create_not_null(insn->get_type())); + env->set(RESULT_REGISTER, DexTypeDomain(insn->get_type())); return true; } bool RegisterTypeAnalyzer::analyze_new_array(const IRInstruction* insn, DexTypeEnvironment* env) { // Skip array element nullness domains. - env->set(RESULT_REGISTER, DexTypeDomain::create_not_null(insn->get_type())); + env->set(RESULT_REGISTER, DexTypeDomain(insn->get_type())); return true; } bool RegisterTypeAnalyzer::analyze_filled_new_array(const IRInstruction* insn, DexTypeEnvironment* env) { - env->set(RESULT_REGISTER, DexTypeDomain::create_not_null(insn->get_type())); + env->set(RESULT_REGISTER, DexTypeDomain(insn->get_type())); return true; } @@ -152,6 +338,36 @@ bool RegisterTypeAnalyzer::analyze_invoke(const IRInstruction* insn, // Note we don't need to take care of the RESULT_REGISTER update from this // point. The remaining cases are already taken care by the // WholeProgramAwareAnalyzer::analyze_invoke. + // + // When passed through a call, we need to reset the elements of an + // ArrayNullnessDomain. The domain passed to the callee is a copy and can be + // written over there. That means that the local ArrayNullnessDomain stored in + // the caller environment might be out of date. + // + // E.g., a newly allocated array in a caller environment has its elements + // initially as UNINITIALIED. The array elements can be updated by a callee + // which has access to the array. At that point, the updated element is no + // longer UNINITIALIED. However, the change is not propagated to the caller + // environment. Reference: T107422148, T123970364 + for (auto src : insn->srcs()) { + auto type_domain = env->get(src); + auto array_nullness = type_domain.get_array_nullness(); + auto dex_type = type_domain.get_dex_type(); + + if (!array_nullness.is_top() && array_nullness.get_length() && + *array_nullness.get_length() > 0 && dex_type) { + env->mutate_reg_environment([&](RegTypeEnvironment* env) { + env->transform([&](const DexTypeDomain& domain) { + auto dex_type_local = domain.get_dex_type(); + if (dex_type_local && *dex_type == *dex_type_local) { + return DexTypeDomain(*dex_type_local, + domain.get_nullness().element()); + } + return domain; + }); + }); + } + } return false; } @@ -187,6 +403,18 @@ bool field_put_helper(const DexType* class_under_init, } // namespace +bool ClinitFieldAnalyzer::analyze_sget(const DexType* class_under_init, + const IRInstruction* insn, + DexTypeEnvironment* env) { + return field_get_helper(class_under_init, insn, env); +} + +bool ClinitFieldAnalyzer::analyze_sput(const DexType* class_under_init, + const IRInstruction* insn, + DexTypeEnvironment* env) { + return field_put_helper(class_under_init, insn, env); +} + bool CtorFieldAnalyzer::analyze_default(const DexType* class_under_init, const IRInstruction* insn, DexTypeEnvironment* env) { diff --git a/service/type-analysis/LocalTypeAnalyzer.h b/service/type-analysis/LocalTypeAnalyzer.h index c6ff88074b..ed3e4fcce1 100644 --- a/service/type-analysis/LocalTypeAnalyzer.h +++ b/service/type-analysis/LocalTypeAnalyzer.h @@ -40,9 +40,6 @@ class RegisterTypeAnalyzer final static bool analyze_default(const IRInstruction* insn, DexTypeEnvironment* env); - static bool analyze_check_cast(const IRInstruction* insn, - DexTypeEnvironment* env); - static bool analyze_const(const IRInstruction* insn, DexTypeEnvironment* env); static bool analyze_const_string(const IRInstruction*, @@ -53,6 +50,16 @@ class RegisterTypeAnalyzer final static bool analyze_aget(const IRInstruction* insn, DexTypeEnvironment* env); + static bool analyze_aput(const IRInstruction* insn, DexTypeEnvironment* env); + + static bool analyze_array_length(const IRInstruction* insn, + DexTypeEnvironment* env); + + static bool analyze_binop_lit(const IRInstruction* insn, + DexTypeEnvironment* env); + + static bool analyze_binop(const IRInstruction* insn, DexTypeEnvironment* env); + static bool analyze_move(const IRInstruction* insn, DexTypeEnvironment* env); static bool analyze_move_result(const IRInstruction* insn, @@ -74,6 +81,39 @@ class RegisterTypeAnalyzer final DexTypeEnvironment* env); }; +/* + * Unlike in other methods where we always propagate field type info from + * the WholeProgramState, in s, we directly propagate static field type + * info through the local FieldTypeEnvironment. This is similar to what we do + * for constant values in IPCP. + * + * The reason is that the is the 1st method of the class being executed + * after class loading. Therefore, the field 'writes' in the happens + * before other 'writes' to the same field. In other words, the field type state + * of s are self-contained. Note that we are limiting ourselves to + * static fields belonging to the same class here. + * + * We don't throw away our results if there're invoke-statics in the . + * Since the field 'write' in the invoke-static callee will be aggregated in the + * final type mapping in WholeProgramState. Before that happens, we do not + * propagate incomplete field type info to other methods. As stated above, we + * only propagate field type info from WholeProgramState computed in the + * previous global iteration. + */ +class ClinitFieldAnalyzer final + : public InstructionAnalyzerBase { + public: + static bool analyze_sget(const DexType* class_under_init, + const IRInstruction* insn, + DexTypeEnvironment* env); + + static bool analyze_sput(const DexType* class_under_init, + const IRInstruction* insn, + DexTypeEnvironment* env); +}; + /* * Similarly CtorFieldAnalyzer populates local FieldTypeEnvironment when * analyzing a ctor. We only do so for instance fields that belong to the class diff --git a/service/type-analysis/TypeAnalysisRuntimeAssert.cpp b/service/type-analysis/TypeAnalysisRuntimeAssert.cpp index cb2d7314dd..665ebea524 100644 --- a/service/type-analysis/TypeAnalysisRuntimeAssert.cpp +++ b/service/type-analysis/TypeAnalysisRuntimeAssert.cpp @@ -291,10 +291,15 @@ bool RuntimeAssertTransform::insert_return_value_assert( DexMethod* callee = nullptr; DexTypeDomain domain = DexTypeDomain::top(); if (wps.has_call_graph()) { - if (wps.invoke_is_dynamic(insn)) { + callee = resolve_method(insn->get_method(), opcode_to_search(insn)); + if (callee == nullptr && opcode_to_search(insn) == MethodSearch::Virtual) { + callee = + resolve_method(insn->get_method(), MethodSearch::InterfaceVirtual); + } + if (callee == nullptr || wps.method_is_dynamic(callee)) { + domain = DexTypeDomain::top(); return false; } - callee = resolve_invoke_method(insn); domain = wps.get_return_type_from_cg(insn); } else { callee = resolve_method(insn->get_method(), opcode_to_search(insn)); diff --git a/service/type-analysis/WholeProgramState.cpp b/service/type-analysis/WholeProgramState.cpp index 9303cb1efa..b9fec0ac4a 100644 --- a/service/type-analysis/WholeProgramState.cpp +++ b/service/type-analysis/WholeProgramState.cpp @@ -66,11 +66,10 @@ void set_encoded_values(const DexClass* cls, DexTypeEnvironment* env) { env->set(sfield, DexTypeDomain::null()); } else if (sfield->get_type() == type::java_lang_String() && value->evtype() == DEVT_STRING) { - env->set(sfield, - DexTypeDomain::create_not_null(type::java_lang_String())); + env->set(sfield, DexTypeDomain(type::java_lang_String())); } else if (sfield->get_type() == type::java_lang_Class() && value->evtype() == DEVT_TYPE) { - env->set(sfield, DexTypeDomain::create_not_null(type::java_lang_Class())); + env->set(sfield, DexTypeDomain(type::java_lang_Class())); } else { env->set(sfield, DexTypeDomain::top()); } @@ -109,15 +108,12 @@ void set_sfields_in_partition(const DexClass* cls, * initialized in any ctor, it is nullalbe. That's why we need to join the type * mapping across all ctors. */ -void set_ifields_in_partition( - const DexClass* cls, - const DexTypeEnvironment& env, - const EligibleIfields& eligible_ifields, - const bool only_aggregate_safely_inferrable_fields, - DexTypeFieldPartition* field_partition) { +void set_ifields_in_partition(const DexClass* cls, + const DexTypeEnvironment& env, + const EligibleIfields& eligible_ifields, + DexTypeFieldPartition* field_partition) { for (auto& field : cls->get_ifields()) { - if (!is_reference(field) || (only_aggregate_safely_inferrable_fields && - eligible_ifields.count(field) == 0)) { + if (!is_reference(field) || eligible_ifields.count(field) == 0) { continue; } auto domain = env.get(field); @@ -157,13 +153,10 @@ namespace type_analyzer { WholeProgramState::WholeProgramState( const Scope& scope, const global::GlobalTypeAnalyzer& gta, - const InsertOnlyConcurrentSet& non_true_virtuals, + const std::unordered_set& non_true_virtuals, const ConcurrentSet& any_init_reachables, - const EligibleIfields& eligible_ifields, - const bool only_aggregate_safely_inferrable_fields) - : m_any_init_reachables(&any_init_reachables), - m_only_aggregate_safely_inferrable_fields( - only_aggregate_safely_inferrable_fields) { + const EligibleIfields& eligible_ifields) + : m_any_init_reachables(any_init_reachables) { // Exclude fields we cannot correctly analyze. walk::fields(scope, [&](DexField* field) { if (!type::is_object(field->get_type())) { @@ -199,17 +192,15 @@ WholeProgramState::WholeProgramState( WholeProgramState::WholeProgramState( const Scope& scope, const global::GlobalTypeAnalyzer& gta, - const InsertOnlyConcurrentSet& non_true_virtuals, + const std::unordered_set& non_true_virtuals, const ConcurrentSet& any_init_reachables, const EligibleIfields& eligible_ifields, - const bool only_aggregate_safely_inferrable_fields, std::shared_ptr call_graph) : WholeProgramState(scope, gta, non_true_virtuals, any_init_reachables, - eligible_ifields, - only_aggregate_safely_inferrable_fields) { + eligible_ifields) { m_call_graph = std::move(call_graph); } @@ -220,8 +211,9 @@ std::string WholeProgramState::show_method(const DexMethod* m) { void WholeProgramState::setup_known_method_returns() { for (auto& p : STATIC_METHOD_TO_TYPE_MAP) { auto method = DexMethod::make_method(p.first); - auto type = DexTypeDomain::create_not_null( - DexType::make_type(DexString::make_string(p.second))); + auto type = + DexTypeDomain(DexType::make_type(DexString::make_string(p.second)), + NOT_NULL, /* is_dex_type_exact */ true); m_known_method_returns.insert(std::make_pair(method, type)); } } @@ -280,7 +272,6 @@ void WholeProgramState::analyze_clinits_and_ctors( auto lta = gta.get_internal_local_analysis(ctor); const auto& env = lta->get_exit_state_at(cfg.exit_block()); set_ifields_in_partition(cls, env, eligible_ifields, - m_only_aggregate_safely_inferrable_fields, &cls_field_partition); } @@ -341,7 +332,6 @@ void WholeProgramState::collect_field_types( return; } if (opcode::is_an_iput(insn->opcode()) && - m_only_aggregate_safely_inferrable_fields && eligible_ifields.count(field) == 0) { // Skip writes to non-eligible instance fields. return; @@ -418,7 +408,8 @@ std::string WholeProgramState::print_field_partition_diff( return ss.str(); } const auto& this_field_bindings = m_field_partition.bindings(); - const auto& other_field_bindings = other.m_field_partition.bindings(); + const auto& other_field_bindings = + other.m_field_partition.bindings(); for (auto& pair : this_field_bindings) { auto field = pair.first; if (!other_field_bindings.count(field)) { @@ -454,7 +445,8 @@ std::string WholeProgramState::print_method_partition_diff( return ss.str(); } const auto& this_method_bindings = m_method_partition.bindings(); - const auto& other_method_bindings = other.m_method_partition.bindings(); + const auto& other_method_bindings = + other.m_method_partition.bindings(); for (auto& pair : this_method_bindings) { auto method = pair.first; if (!other_method_bindings.count(method)) { @@ -507,7 +499,12 @@ bool WholeProgramAwareAnalyzer::analyze_invoke( } if (whole_program_state->has_call_graph()) { - if (whole_program_state->invoke_is_dynamic(insn)) { + auto method = resolve_method(insn->get_method(), opcode_to_search(insn)); + if (method == nullptr && opcode_to_search(insn) == MethodSearch::Virtual) { + method = + resolve_method(insn->get_method(), MethodSearch::InterfaceVirtual); + } + if (method == nullptr || whole_program_state->method_is_dynamic(method)) { env->set(RESULT_REGISTER, DexTypeDomain::top()); return false; } diff --git a/service/type-analysis/WholeProgramState.h b/service/type-analysis/WholeProgramState.h index a66b070cca..2f163e8af3 100644 --- a/service/type-analysis/WholeProgramState.h +++ b/service/type-analysis/WholeProgramState.h @@ -42,20 +42,17 @@ class WholeProgramState { // By default, the field and method partitions are initialized to Bottom. WholeProgramState() = default; - WholeProgramState( - const Scope&, - const global::GlobalTypeAnalyzer&, - const InsertOnlyConcurrentSet& non_true_virtuals, - const ConcurrentSet& any_init_reachables, - const EligibleIfields& eligible_ifields, - const bool only_aggregate_safely_inferrable_fields); + WholeProgramState(const Scope&, + const global::GlobalTypeAnalyzer&, + const std::unordered_set& non_true_virtuals, + const ConcurrentSet& any_init_reachables, + const EligibleIfields& eligible_ifields); WholeProgramState(const Scope&, const global::GlobalTypeAnalyzer&, - const InsertOnlyConcurrentSet&, + const std::unordered_set&, const ConcurrentSet&, const EligibleIfields&, - const bool, std::shared_ptr call_graph); void set_to_top() { @@ -129,7 +126,7 @@ class WholeProgramState { } bool is_any_init_reachable(const DexMethod* method) const { - return m_any_init_reachables && m_any_init_reachables->count(method); + return m_any_init_reachables.count(method); } /* @@ -152,10 +149,6 @@ class WholeProgramState { } DexTypeDomain ret = DexTypeDomain::bottom(); for (const DexMethod* callee : callees) { - if (!callee->get_code()) { - always_assert(is_abstract(callee) || is_native(callee)); - return DexTypeDomain::top(); - } auto val = m_method_partition.get(callee); ret.join_with(val); } @@ -165,8 +158,8 @@ class WholeProgramState { return ret; } - bool invoke_is_dynamic(const IRInstruction* insn) const { - return call_graph::invoke_is_dynamic(*m_call_graph, insn); + bool method_is_dynamic(const DexMethod* method) const { + return call_graph::method_is_dynamic(*m_call_graph, method); } // For debugging @@ -228,12 +221,11 @@ class WholeProgramState { std::unordered_set m_known_methods; // Methods reachable from clinit that read static fields and reachable from // ctors that raed instance fields. - const ConcurrentSet* m_any_init_reachables{nullptr}; + ConcurrentSet m_any_init_reachables; DexTypeFieldPartition m_field_partition; DexTypeMethodPartition m_method_partition; std::unordered_map m_known_method_returns; - bool m_only_aggregate_safely_inferrable_fields = false; }; class WholeProgramAwareAnalyzer final diff --git a/setup_oss_toolchain.sh b/setup_oss_toolchain.sh index e1beed5a3f..68aeb6d1f3 100755 --- a/setup_oss_toolchain.sh +++ b/setup_oss_toolchain.sh @@ -15,14 +15,8 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" # Temporary directory for toolchain sources. Build artifacts will be # installed to /usr/local. -echo "toolchain tmp = $TOOLCHAIN_TMP" -if [ -z "$TOOLCHAIN_TMP" ] ; then - TOOLCHAIN_TMP=$(mktemp -d 2>/dev/null) - trap 'rm -r $TOOLCHAIN_TMP' EXIT -else - echo "Using toolchain tmp $TOOLCHAIN_TMP" - mkdir -p "$TOOLCHAIN_TMP" -fi +TMP=$(mktemp -d 2>/dev/null) +trap 'rm -r $TMP' EXIT if [ "$1" = "32" ] ; then BITNESS="32" @@ -50,21 +44,26 @@ BOOST_DEB_UBUNTU_PKGS="libboost-filesystem-dev$BITNESS_SUFFIX PROTOBUF_DEB_UBUNTU_PKGS="libprotobuf-dev$BITNESS_SUFFIX protobuf-compiler" +function install_python36_from_source { + pushd "$TMP" + wget https://www.python.org/ftp/python/3.6.10/Python-3.6.10.tgz + tar -xvf Python-3.6.10.tgz + pushd Python-3.6.10 + + # Always compile Python as host-preferred. + ./configure + make V=0 && make install V=0 +} + function install_boost_from_source { - pushd "$TOOLCHAIN_TMP" + pushd "$TMP" "$ROOT"/get_boost.sh } function install_protobuf3_from_source { - pushd "$TOOLCHAIN_TMP" - mkdir -p dl_cache/protobuf - if [ ! -f dl_cache/protobuf/protobuf-cpp-3.17.3.tar.gz ] ; then - wget https://github.com/protocolbuffers/protobuf/releases/download/v3.17.3/protobuf-cpp-3.17.3.tar.gz -O dl_cache/protobuf/protobuf-cpp-3.17.3.tar.gz - fi - - mkdir -p toolchain_install/protobuf - pushd toolchain_install/protobuf - tar -xf ../../dl_cache/protobuf/protobuf-cpp-3.17.3.tar.gz --no-same-owner + pushd "$TMP" + wget https://github.com/protocolbuffers/protobuf/releases/download/v3.17.3/protobuf-cpp-3.17.3.tar.gz + tar -xvf protobuf-cpp-3.17.3.tar.gz --no-same-owner pushd protobuf-3.17.3 ./configure $BITNESS_CONFIGURE @@ -88,8 +87,8 @@ function install_from_apt { make wget zlib1g-dev$BITNESS_SUFFIX $BITNESS_PKGS $*" - apt-get update -q - apt-get install -q --no-install-recommends -y ${PKGS} + apt-get update + apt-get install --no-install-recommends -y ${PKGS} } function handle_debian { diff --git a/test/common/RedexTest.h b/test/common/RedexTest.h index 7844d0fc05..3745ba5bb0 100644 --- a/test/common/RedexTest.h +++ b/test/common/RedexTest.h @@ -27,17 +27,9 @@ #include "RedexTestUtils.h" struct RedexTest : public testing::Test { - public: RedexTest() { g_redex = new RedexContext(); } ~RedexTest() { delete g_redex; } - - std::string android_sdk_jar_path() { - const char* android_sdk = std::getenv("sdk_path"); - std::string android_target(std::getenv("android_target")); - return std::string(android_sdk) + "/platforms/" + android_target + - "/android.jar"; - } }; struct RedexIntegrationTest : public RedexTest { diff --git a/test/common/TypeAnalysisTestBase.h b/test/common/TypeAnalysisTestBase.h index 5e7e9743ba..0d1d20480b 100644 --- a/test/common/TypeAnalysisTestBase.h +++ b/test/common/TypeAnalysisTestBase.h @@ -47,19 +47,18 @@ class TypeAnalysisTestBase : public RedexIntegrationTest { DexTypeDomain get_type_domain(const std::string& type_name) { std::string full_name = "Lcom/facebook/redextest/" + type_name + ";"; - return DexTypeDomain::create_not_null( - DexType::make_type(DexString::make_string(full_name))); + return DexTypeDomain(DexType::make_type(DexString::make_string(full_name))); } - DexTypeDomain get_type_domain_simple(const std::string& type_name, - bool is_not_null = false) { - if (is_not_null) { - return DexTypeDomain::create_not_null( - DexType::make_type(DexString::make_string(type_name))); - } + DexTypeDomain get_type_domain_simple(const std::string& type_name) { + return DexTypeDomain(DexType::make_type(DexString::make_string(type_name))); + } - return DexTypeDomain::create_nullable( - DexType::make_type(DexString::make_string(type_name))); + DexTypeDomain get_type_domain_simple(const std::string& type_name, + const Nullness nullness, + bool is_dex_type_exact) { + return DexTypeDomain(DexType::make_type(DexString::make_string(type_name)), + nullness, is_dex_type_exact); } DexType* get_type_simple(const std::string& type_name) { diff --git a/test/equivalence/TestGenerator.cpp b/test/equivalence/TestGenerator.cpp index 0dc21c3a57..f1e848a3ed 100644 --- a/test/equivalence/TestGenerator.cpp +++ b/test/equivalence/TestGenerator.cpp @@ -70,7 +70,7 @@ int main(int argc, char* argv[]) { "Lcom/facebook/redex/equivalence/EquivalenceMain;"); }); - always_assert(runner_cls != classes.end()); + redex_assert(runner_cls != classes.end()); EquivalenceTest::generate_all(*runner_cls); diff --git a/test/instr/ConstClassBranches.java b/test/instr/ConstClassBranches.java index 467f10a9aa..d41cba3067 100644 --- a/test/instr/ConstClassBranches.java +++ b/test/instr/ConstClassBranches.java @@ -76,40 +76,4 @@ public static final Integer get(Class clazz) { } } } - - public static class Complicated { - Complicated() {} - - public static final Integer get(Class clazz, String s) { - if ("a".equals(s)) { - if (clazz == java.util.Map.class) { - // See if this is the same block of code as the below case. - return floop(1000); - } else if (clazz == java.util.List.class) { - return floop(1); - } else if (clazz == java.util.Set.class) { - return floop(2); - } else if (clazz == java.util.Deque.class) { - return floop(3); - } else if (clazz == java.util.Iterator.class) { - return floop(4); - } else { - return clazz == java.util.Collection.class ? floop(5) : null; - } - } - if (clazz == java.util.Date.class) { - return floop(1000); - } else if (clazz == java.util.List.class) { - return floop(1001); - } else if (clazz == java.util.Set.class) { - return floop(1002); - } else if (clazz == java.util.Deque.class) { - return floop(1003); - } else if (clazz == java.util.Iterator.class) { - return floop(1004); - } else { - return clazz == java.util.Collection.class ? floop(1005) : null; - } - } - } } diff --git a/test/instr/ConstClassBranchesTest.java b/test/instr/ConstClassBranchesTest.java index cbaa6dd4c1..868fb409dc 100644 --- a/test/instr/ConstClassBranchesTest.java +++ b/test/instr/ConstClassBranchesTest.java @@ -49,26 +49,4 @@ public void testDuplicateLookup() { assertThat(ConstClassBranches.Duplicates.get(java.lang.String.class)).isNull(); assertThat(ConstClassBranches.Duplicates.get(null)).isNull(); } - - @Test - public void testMultipleTransformsInMethod() { - String s = "a"; - assertThat(ConstClassBranches.Complicated.get(java.util.Map.class, s)).isEqualTo(1000); - assertThat(ConstClassBranches.Complicated.get(java.util.List.class, s)).isEqualTo(1); - assertThat(ConstClassBranches.Complicated.get(java.util.Set.class, s)).isEqualTo(2); - assertThat(ConstClassBranches.Complicated.get(java.util.Deque.class, s)).isEqualTo(3); - assertThat(ConstClassBranches.Complicated.get(java.util.Iterator.class, s)).isEqualTo(4); - assertThat(ConstClassBranches.Complicated.get(java.util.Collection.class, s)).isEqualTo(5); - assertThat(ConstClassBranches.Complicated.get(java.lang.String.class, s)).isNull(); - assertThat(ConstClassBranches.Complicated.get(null, s)).isNull(); - s = "b"; - assertThat(ConstClassBranches.Complicated.get(java.util.Date.class, s)).isEqualTo(1000); - assertThat(ConstClassBranches.Complicated.get(java.util.List.class, s)).isEqualTo(1001); - assertThat(ConstClassBranches.Complicated.get(java.util.Set.class, s)).isEqualTo(1002); - assertThat(ConstClassBranches.Complicated.get(java.util.Deque.class, s)).isEqualTo(1003); - assertThat(ConstClassBranches.Complicated.get(java.util.Iterator.class, s)).isEqualTo(1004); - assertThat(ConstClassBranches.Complicated.get(java.util.Collection.class, s)).isEqualTo(1005); - assertThat(ConstClassBranches.Complicated.get(java.lang.String.class, s)).isNull(); - assertThat(ConstClassBranches.Complicated.get(null, s)).isNull(); - } } diff --git a/test/instr/ConstClassBranchesTestVerify.cpp b/test/instr/ConstClassBranchesTestVerify.cpp index 63b4de24e8..70ba4c6b43 100644 --- a/test/instr/ConstClassBranchesTestVerify.cpp +++ b/test/instr/ConstClassBranchesTestVerify.cpp @@ -49,14 +49,6 @@ TEST_F(PreVerify, VerifyBaseState) { ASSERT_NE(method_dup, nullptr); EXPECT_EQ(count_switches(method_dup), 0) << "Method does not match expected input state"; - - auto cls_multi = find_class_named( - classes, "Lcom/facebook/redex/ConstClassBranches$Complicated;"); - ASSERT_NE(cls_multi, nullptr); - auto method_multi = find_dmethod_named(*cls_multi, "get"); - ASSERT_NE(method_multi, nullptr); - EXPECT_EQ(count_switches(method_multi), 0) - << "Method does not match expected input state"; } TEST_F(PostVerify, VerifyTransformedA) { @@ -86,13 +78,3 @@ TEST_F(PostVerify, VerifyTransformedDuplicate) { EXPECT_EQ(count_switches(method_dup), 1) << "Duplicates.get should be transformed"; } - -TEST_F(PostVerify, VerifyTransformedMulti) { - auto cls_multi = find_class_named( - classes, "Lcom/facebook/redex/ConstClassBranches$Complicated;"); - ASSERT_NE(cls_multi, nullptr); - auto method_multi = find_dmethod_named(*cls_multi, "get"); - ASSERT_NE(method_multi, nullptr); - EXPECT_EQ(count_switches(method_multi), 2) - << "Complicated.get should have two transforms applied"; -} diff --git a/test/instr/EnumTransformTest.java b/test/instr/EnumTransformTest.java index 21df30f2e6..ae5c4641e9 100644 --- a/test/instr/EnumTransformTest.java +++ b/test/instr/EnumTransformTest.java @@ -380,12 +380,6 @@ public void test_string_valueof() { } } - @Test - public void toString_regression() { - Enum e = SCORE.ONE; - assertThat(e.toString()).isEqualTo("UNO"); - } - // NullPointerException. @Test(expected = NullPointerException.class) public void test_npe() { diff --git a/test/instr/KtnonclTestVerify.cpp b/test/instr/KtnonclTestVerify.cpp index 4750e530c5..50863c2012 100644 --- a/test/instr/KtnonclTestVerify.cpp +++ b/test/instr/KtnonclTestVerify.cpp @@ -49,22 +49,20 @@ TEST_F(PostVerify, KotlinGeneratedClass) { // After opt, there is no invoke-interface, which is replaced with // invoke-virtual in doCalc. - // This is not optimized anymore!!! auto* intf_cls = find_class_named(classes, fn2); - ASSERT_NE(nullptr, find_invoke(meth_doCalc, DOPCODE_INVOKE_INTERFACE, + ASSERT_EQ(nullptr, find_invoke(meth_doCalc, DOPCODE_INVOKE_INTERFACE, "invoke", intf_cls->get_type())); auto* impl_cls = find_class_named(classes, foo); - ASSERT_EQ(nullptr, find_invoke(meth_doCalc, DOPCODE_INVOKE_VIRTUAL, "invoke", + ASSERT_NE(nullptr, find_invoke(meth_doCalc, DOPCODE_INVOKE_VIRTUAL, "invoke", impl_cls->get_type())); auto* meth_doCalc1 = find_vmethod_named(*cls, "doCalc1"); EXPECT_NE(nullptr, meth_doCalc1); // After opt, there is no invoke-interface, which is replaced with // invoke-virtual in doCalc1. - // This is not optimized anymore!!! - ASSERT_NE(nullptr, find_invoke(meth_doCalc1, DOPCODE_INVOKE_INTERFACE, + ASSERT_EQ(nullptr, find_invoke(meth_doCalc1, DOPCODE_INVOKE_INTERFACE, "invoke", intf_cls->get_type())); impl_cls = find_class_named(classes, foo1); - ASSERT_EQ(nullptr, find_invoke(meth_doCalc1, DOPCODE_INVOKE_VIRTUAL, "invoke", + ASSERT_NE(nullptr, find_invoke(meth_doCalc1, DOPCODE_INVOKE_VIRTUAL, "invoke", impl_cls->get_type())); } diff --git a/test/instr/constclassbranches.config b/test/instr/constclassbranches.config index c5792b83d0..9bdea60bf5 100644 --- a/test/instr/constclassbranches.config +++ b/test/instr/constclassbranches.config @@ -1,7 +1,7 @@ { "TransformConstClassBranchesPass": { "consider_external_classes": true, - "min_cases": 3, + "min_cases": 2, "string_tree_lookup_method": "Lcom/facebook/common/dextricks/StringTreeSet;.searchMapStringify:(Ljava/lang/Object;Ljava/lang/String;I)I" }, "redex" : { diff --git a/test/instr/obfuscateresources.config b/test/instr/obfuscateresources.config index 14b27eea02..e5bd656d93 100644 --- a/test/instr/obfuscateresources.config +++ b/test/instr/obfuscateresources.config @@ -30,11 +30,6 @@ ], "obfuscate_xml_attributes": false }, - "resources": { - "canonical_entry_types": [ - "id" - ] - }, "compute_xml_reachability": true, "finalize_resource_table": true } diff --git a/test/integ/CallGraphTest.cpp b/test/integ/CallGraphTest.cpp index ba3e4e505e..145de00d6b 100644 --- a/test/integ/CallGraphTest.cpp +++ b/test/integ/CallGraphTest.cpp @@ -39,24 +39,21 @@ struct CallGraphTest : public RedexIntegrationTest { DexMethod* pure_ref_intf_return; DexMethod* pure_ref_3_return; DexMethod* pure_ref_3_init; - DexMethod* more_than_5_class_extends_1_init; - DexMethod* more_than_5_class_extends_1_return_super_num; - DexMethod* more_than_5_class_return_num; Scope scope; std::unique_ptr method_override_graph; - std::unique_ptr complete_graph; - std::unique_ptr multiple_graph; + boost::optional complete_graph; + boost::optional multiple_graph; public: CallGraphTest() {} void SetUp() override { scope = build_class_scope(stores); method_override_graph = method_override_graph::build_graph(scope); - complete_graph = std::make_unique( - call_graph::complete_call_graph(*method_override_graph, scope)); - multiple_graph = std::make_unique( - call_graph::multiple_callee_graph(*method_override_graph, scope, 5)); + complete_graph = + call_graph::complete_call_graph(*method_override_graph, scope); + multiple_graph = + call_graph::multiple_callee_graph(*method_override_graph, scope, 5); clinit = DexMethod::get_method( "Lcom/facebook/redextest/CallGraphTest;.:()V") ->as_def(); @@ -143,22 +140,6 @@ struct CallGraphTest : public RedexIntegrationTest { pure_ref_3_init = DexMethod::get_method( "Lcom/facebook/redextest/PureRefImpl3;.:()V") ->as_def(); - - more_than_5_class_extends_1_init = DexMethod::get_method( - "Lcom/facebook/redextest/" - "MoreThan5ClassExtends1;.:()V") - ->as_def(); - - more_than_5_class_extends_1_return_super_num = - DexMethod::get_method( - "Lcom/facebook/redextest/" - "MoreThan5ClassExtends1;.returnSuperNum:()I") - ->as_def(); - - more_than_5_class_return_num = - DexMethod::get_method( - "Lcom/facebook/redextest/MoreThan5Class;.returnNum:()I") - ->as_def(); } std::vector get_callees(const call_graph::Graph& graph, @@ -188,8 +169,8 @@ TEST_F(CallGraphTest, test_resolve_static_callees) { } } ASSERT_NE(invoke_insn, nullptr); - auto callees = - call_graph::resolve_callees_in_graph(*complete_graph, invoke_insn); + auto callees = call_graph::resolve_callees_in_graph( + *complete_graph, clinit, invoke_insn); EXPECT_THAT(callees, ::testing::UnorderedElementsAre(base_foo)); } @@ -202,8 +183,8 @@ TEST_F(CallGraphTest, test_resolve_virtual_callees) { } } ASSERT_NE(invoke_insn, nullptr); - auto callees = - call_graph::resolve_callees_in_graph(*complete_graph, invoke_insn); + auto callees = call_graph::resolve_callees_in_graph( + *complete_graph, calls_returns_int, invoke_insn); EXPECT_THAT(callees, ::testing::UnorderedElementsAre(base_returns_int, extended_returns_int, @@ -230,20 +211,15 @@ TEST_F(CallGraphTest, test_multiple_callee_graph_entry) { TEST_F(CallGraphTest, test_multiple_callee_graph_clinit) { auto clinit_callees = get_callees(*multiple_graph, clinit); EXPECT_THAT(clinit_callees, - ::testing::UnorderedElementsAre( - calls_returns_int, - base_foo, - extended_init, - less_impl3_init, - more_impl1_init, - more_impl1_init, - more_impl1_return, - less_impl1_return, - less_impl2_return, - less_impl3_return, - less_impl4_return, - more_than_5_class_extends_1_init, - more_than_5_class_extends_1_return_super_num)); + ::testing::UnorderedElementsAre(calls_returns_int, + base_foo, + extended_init, + less_impl3_init, + more_impl1_init, + less_impl1_return, + less_impl2_return, + less_impl3_return, + less_impl4_return)); } TEST_F(CallGraphTest, test_multiple_callee_graph_return4) { @@ -268,10 +244,3 @@ TEST_F(CallGraphTest, test_multiple_callee_graph_extended_returns_int) { EXPECT_THAT(extendedextended_returns_int_callees, ::testing::UnorderedElementsAre(extended_returns_int)); } - -TEST_F(CallGraphTest, test_multiple_callee_graph_invoke_super) { - auto callees = get_callees(*multiple_graph, - more_than_5_class_extends_1_return_super_num); - EXPECT_THAT(callees, - ::testing::UnorderedElementsAre(more_than_5_class_return_num)); -} diff --git a/test/integ/CallGraphTest.java b/test/integ/CallGraphTest.java index 95b424c253..28cb3627a4 100644 --- a/test/integ/CallGraphTest.java +++ b/test/integ/CallGraphTest.java @@ -15,12 +15,8 @@ public class CallGraphTest { Extended.foo(); MoreThan5 moreThan5 = new MoreThan5Impl1(); int get1 = moreThan5.returnNum(); - MoreThan5Impl1 moreThan5Impl1 = new MoreThan5Impl1(); - int get2 = moreThan5Impl1.returnNum(); LessThan5 lessThan5 = new LessThan5Impl3(); int get3 = lessThan5.returnNum(); - MoreThan5ClassExtends1 moreThan5ClassExtends1 = new MoreThan5ClassExtends1(); - int get4 = moreThan5ClassExtends1.returnSuperNum(); } static int callsReturnsInt(Base b) { @@ -113,49 +109,3 @@ abstract class PureRefImpl2 extends PureRefImpl1 {} class PureRefImpl3 extends PureRefImpl2 { public int returnNum() { return 5; } } - -class MoreThan5Class { - public int returnNum() { - return 0; - } -} - -class MoreThan5ClassExtends1 extends MoreThan5Class { - public int returnNum() { - return 1; - } - - public int returnSuperNum() { - return super.returnNum(); - } -} - -class MoreThan5ClassExtends2 extends MoreThan5Class { - public int returnNum() { - return 2; - } -} - -class MoreThan5ClassExtends3 extends MoreThan5Class { - public int returnNum() { - return 3; - } -} - -class MoreThan5ClassExtends4 extends MoreThan5Class { - public int returnNum() { - return 4; - } -} - -class MoreThan5ClassExtends5 extends MoreThan5Class { - public int returnNum() { - return 5; - } -} - -class MoreThan5ClassExtends6 extends MoreThan5Class { - public int returnNum() { - return 6; - } -} diff --git a/test/integ/Dex038Test.cpp b/test/integ/Dex038Test.cpp index 837c57e319..c5e607e566 100644 --- a/test/integ/Dex038Test.cpp +++ b/test/integ/Dex038Test.cpp @@ -353,21 +353,20 @@ TEST(Dex038Test, ReadWriteDex038) { std::string output_dex = tmpdir.path + "/output.dex"; auto gtypes = std::make_shared(&classes); - write_classes_to_dex( - output_dex, - &classes, - std::move(gtypes), - nullptr, - 0, - nullptr, - 0, - dummy_cfg, - pos_mapper.get(), - DebugInfoKind::NoCustomSymbolication, - &method_to_id, - &code_debug_lines, - nullptr, - "dex\n038\0"); // NOLINT(bugprone-string-literal-with-embedded-nul) + write_classes_to_dex(output_dex, + &classes, + std::move(gtypes), + nullptr, + 0, + nullptr, + 0, + dummy_cfg, + pos_mapper.get(), + DebugInfoKind::NoCustomSymbolication, + &method_to_id, + &code_debug_lines, + nullptr, + "dex\n038\0"); delete g_redex; g_redex = new RedexContext(); diff --git a/test/integ/FlowSensitiveReachabilityTest.cpp b/test/integ/FlowSensitiveReachabilityTest.cpp index 3b2b80f10d..5630f02880 100644 --- a/test/integ/FlowSensitiveReachabilityTest.cpp +++ b/test/integ/FlowSensitiveReachabilityTest.cpp @@ -31,11 +31,9 @@ TEST_F(FlowSensitiveReachabilityTest, reachability::ReachableAspects reachable_aspects; auto scope = build_class_scope(stores); walk::parallel::code(scope, [&](auto*, auto& code) { code.build_cfg(); }); - auto method_override_graph = method_override_graph::build_graph(scope); auto reachable_objects = reachability::compute_reachable_objects( - scope, *method_override_graph, ig_sets, &num_ignore_check_strings, - &reachable_aspects, false, + stores, ig_sets, &num_ignore_check_strings, &reachable_aspects, false, /* relaxed_keep_class_members */ true, /* relaxed_keep_interfaces */ false, /* cfg_gathering_check_instantiable */ true); @@ -83,17 +81,13 @@ TEST_F(FlowSensitiveReachabilityTest, EXPECT_TRUE(is_uninstantiable_dependency("LDataHolder;")); // Code sweeping - remove_uninstantiables_impl::Stats remove_uninstantiables_stats; - std::atomic throws_inserted{0}; - InsertOnlyConcurrentSet affected_methods; - reachability::sweep_code( + auto [uninstantiables_stats, throws_inserted] = reachability::sweep_code( stores, /* prune_uncallable_instance_method_bodies */ false, - /* skip_uncallable_virtual_methods */ false, reachable_aspects, - &remove_uninstantiables_stats, &throws_inserted, &affected_methods); - EXPECT_EQ(remove_uninstantiables_stats.field_accesses_on_uninstantiable, 3); - EXPECT_EQ(remove_uninstantiables_stats.invokes, 7); - EXPECT_EQ(remove_uninstantiables_stats.check_casts, 1); - EXPECT_EQ(remove_uninstantiables_stats.instance_ofs, 1); + /* skip_uncallable_virtual_methods */ false, reachable_aspects); + EXPECT_EQ(uninstantiables_stats.field_accesses_on_uninstantiable, 3); + EXPECT_EQ(uninstantiables_stats.invokes, 7); + EXPECT_EQ(uninstantiables_stats.check_casts, 1); + EXPECT_EQ(uninstantiables_stats.instance_ofs, 1); walk::parallel::code(scope, [&](auto*, auto& code) { code.clear_cfg(); }); } @@ -114,11 +108,9 @@ TEST_F(FlowSensitiveReachabilityTest, cfg_gathering_check_instance_callable) { reachability::ReachableAspects reachable_aspects; auto scope = build_class_scope(stores); walk::parallel::code(scope, [&](auto*, auto& code) { code.build_cfg(); }); - auto method_override_graph = method_override_graph::build_graph(scope); auto reachable_objects = reachability::compute_reachable_objects( - scope, *method_override_graph, ig_sets, &num_ignore_check_strings, - &reachable_aspects, false, + stores, ig_sets, &num_ignore_check_strings, &reachable_aspects, false, /* relaxed_keep_class_members */ true, /* relaxed_keep_interfaces */ false, /* cfg_gathering_check_instantiable */ true, @@ -167,18 +159,14 @@ TEST_F(FlowSensitiveReachabilityTest, cfg_gathering_check_instance_callable) { EXPECT_TRUE(is_uninstantiable_dependency("LDataHolder;")); // Code sweeping - remove_uninstantiables_impl::Stats remove_uninstantiables_stats; - std::atomic throws_inserted{0}; - InsertOnlyConcurrentSet affected_methods; - reachability::sweep_code( + auto [uninstantiables_stats, throws_inserted] = reachability::sweep_code( stores, /* prune_uncallable_instance_method_bodies */ true, - /* skip_uncallable_virtual_methods */ false, reachable_aspects, - &remove_uninstantiables_stats, &throws_inserted, &affected_methods); - EXPECT_EQ(remove_uninstantiables_stats.field_accesses_on_uninstantiable, 1); - EXPECT_EQ(remove_uninstantiables_stats.invokes, 5); - EXPECT_EQ(remove_uninstantiables_stats.check_casts, 1); - EXPECT_EQ(remove_uninstantiables_stats.instance_ofs, 1); - EXPECT_EQ(remove_uninstantiables_stats.throw_null_methods, 12); + /* skip_uncallable_virtual_methods */ false, reachable_aspects); + EXPECT_EQ(uninstantiables_stats.field_accesses_on_uninstantiable, 1); + EXPECT_EQ(uninstantiables_stats.invokes, 5); + EXPECT_EQ(uninstantiables_stats.check_casts, 1); + EXPECT_EQ(uninstantiables_stats.instance_ofs, 1); + EXPECT_EQ(uninstantiables_stats.throw_null_methods, 12); walk::parallel::code(scope, [&](auto*, auto& code) { code.clear_cfg(); }); } @@ -199,11 +187,9 @@ TEST_F(FlowSensitiveReachabilityTest, sweep_uncallable_virtual_methods) { reachability::ReachableAspects reachable_aspects; auto scope = build_class_scope(stores); walk::parallel::code(scope, [&](auto*, auto& code) { code.build_cfg(); }); - auto method_override_graph = method_override_graph::build_graph(scope); auto reachable_objects = reachability::compute_reachable_objects( - scope, *method_override_graph, ig_sets, &num_ignore_check_strings, - &reachable_aspects, false, + stores, ig_sets, &num_ignore_check_strings, &reachable_aspects, false, /* relaxed_keep_class_members */ true, /* relaxed_keep_interfaces */ false, /* cfg_gathering_check_instantiable */ true, @@ -252,28 +238,24 @@ TEST_F(FlowSensitiveReachabilityTest, sweep_uncallable_virtual_methods) { EXPECT_TRUE(is_uninstantiable_dependency("LDataHolder;")); // Code sweeping - remove_uninstantiables_impl::Stats remove_uninstantiables_stats; - std::atomic throws_inserted{0}; - InsertOnlyConcurrentSet affected_methods; - reachability::sweep_code( + auto [uninstantiables_stats, throws_inserted] = reachability::sweep_code( stores, /* prune_uncallable_instance_method_bodies */ true, - /* skip_uncallable_virtual_methods */ true, reachable_aspects, - &remove_uninstantiables_stats, &throws_inserted, &affected_methods); - EXPECT_EQ(remove_uninstantiables_stats.field_accesses_on_uninstantiable, 1); - EXPECT_EQ(remove_uninstantiables_stats.invokes, 5); - EXPECT_EQ(remove_uninstantiables_stats.check_casts, 1); - EXPECT_EQ(remove_uninstantiables_stats.instance_ofs, 1); - EXPECT_EQ(remove_uninstantiables_stats.throw_null_methods, 7); + /* skip_uncallable_virtual_methods */ true, reachable_aspects); + EXPECT_EQ(uninstantiables_stats.field_accesses_on_uninstantiable, 1); + EXPECT_EQ(uninstantiables_stats.invokes, 5); + EXPECT_EQ(uninstantiables_stats.check_casts, 1); + EXPECT_EQ(uninstantiables_stats.instance_ofs, 1); + EXPECT_EQ(uninstantiables_stats.throw_null_methods, 7); auto abstracted_classes = reachability::mark_classes_abstract( stores, *reachable_objects, reachable_aspects); EXPECT_EQ(abstracted_classes.size(), 5); reachability::sweep(stores, *reachable_objects); - remove_uninstantiables_stats = + uninstantiables_stats = reachability::sweep_uncallable_virtual_methods(stores, reachable_aspects); - EXPECT_EQ(remove_uninstantiables_stats.abstracted_vmethods, 1); - EXPECT_EQ(remove_uninstantiables_stats.abstracted_classes, 0); - EXPECT_EQ(remove_uninstantiables_stats.removed_vmethods, 1); + EXPECT_EQ(uninstantiables_stats.abstracted_vmethods, 1); + EXPECT_EQ(uninstantiables_stats.abstracted_classes, 0); + EXPECT_EQ(uninstantiables_stats.removed_vmethods, 1); walk::parallel::code(scope, [&](auto*, auto& code) { code.clear_cfg(); }); } @@ -294,11 +276,9 @@ TEST_F(FlowSensitiveReachabilityTest, abstract_overrides_non_abstract) { reachability::ReachableAspects reachable_aspects; auto scope = build_class_scope(stores); walk::parallel::code(scope, [&](auto*, auto& code) { code.build_cfg(); }); - auto method_override_graph = method_override_graph::build_graph(scope); auto reachable_objects = reachability::compute_reachable_objects( - scope, *method_override_graph, ig_sets, &num_ignore_check_strings, - &reachable_aspects, false, + stores, ig_sets, &num_ignore_check_strings, &reachable_aspects, false, /* relaxed_keep_class_members */ true, /* relaxed_keep_interfaces */ false, /* cfg_gathering_check_instantiable */ true); @@ -341,11 +321,9 @@ TEST_F(FlowSensitiveReachabilityTest, throw_propagation) { reachability::ReachableAspects reachable_aspects; auto scope = build_class_scope(stores); walk::parallel::code(scope, [&](auto*, auto& code) { code.build_cfg(); }); - auto method_override_graph = method_override_graph::build_graph(scope); auto reachable_objects = reachability::compute_reachable_objects( - scope, *method_override_graph, ig_sets, &num_ignore_check_strings, - &reachable_aspects, false, + stores, ig_sets, &num_ignore_check_strings, &reachable_aspects, false, /* relaxed_keep_class_members */ true, /* relaxed_keep_interfaces */ true, /* cfg_gathering_check_instantiable */ true, @@ -362,13 +340,9 @@ TEST_F(FlowSensitiveReachabilityTest, throw_propagation) { ASSERT_FALSE(reachable_objects->marked_unsafe(dead_cls)); // Code sweeping - remove_uninstantiables_impl::Stats remove_uninstantiables_stats; - std::atomic throws_inserted{0}; - InsertOnlyConcurrentSet affected_methods; reachability::sweep_code( stores, /* prune_uncallable_instance_method_bodies */ true, - /* skip_uncallable_virtual_methods */ true, reachable_aspects, - &remove_uninstantiables_stats, &throws_inserted, &affected_methods); + /* skip_uncallable_virtual_methods */ true, reachable_aspects); walk::parallel::code(scope, [&](auto*, auto& code) { code.clear_cfg(); }); diff --git a/test/integ/GlobalTypeAnalysisTest.cpp b/test/integ/GlobalTypeAnalysisTest.cpp index f8645ef6c2..4498fc19b4 100644 --- a/test/integ/GlobalTypeAnalysisTest.cpp +++ b/test/integ/GlobalTypeAnalysisTest.cpp @@ -59,28 +59,28 @@ TEST_F(GlobalTypeAnalysisTest, ConstsAndAGETTest) { auto meth_pass_string = get_method("TestB;.passString", "Ljava/lang/String;", "Ljava/lang/String;"); - EXPECT_EQ( - wps.get_return_type(meth_pass_string), - get_type_domain_simple("Ljava/lang/String;", /* is_not_null */ true)); + EXPECT_EQ(wps.get_return_type(meth_pass_string), + get_type_domain_simple("Ljava/lang/String;")); auto meth_pass_class = get_method("TestB;.passClass", "Ljava/lang/Class;", "Ljava/lang/Class;"); - EXPECT_EQ( - wps.get_return_type(meth_pass_class), - get_type_domain_simple("Ljava/lang/Class;", /* is_not_null */ true)); + EXPECT_EQ(wps.get_return_type(meth_pass_class), + get_type_domain_simple("Ljava/lang/Class;")); auto meth_array_comp = get_method("TestB;.getStringArrayComponent", "[Ljava/lang/String;", "Ljava/lang/String;"); EXPECT_EQ(wps.get_return_type(meth_array_comp), - get_type_domain_simple("Ljava/lang/String;")); + get_type_domain_simple("Ljava/lang/String;", Nullness::NN_TOP, + /* is_dex_type_exact */ false)); auto meth_nested_array_comp = get_method("TestB;.getNestedStringArrayComponent", "[[Ljava/lang/String;", "[Ljava/lang/String;"); EXPECT_EQ(wps.get_return_type(meth_nested_array_comp), - get_type_domain_simple("[Ljava/lang/String;")); + get_type_domain_simple("[Ljava/lang/String;", Nullness::NN_TOP, + /* is_dex_type_exact */ false)); } TEST_F(GlobalTypeAnalysisTest, NullableFieldTypeTest) { @@ -157,7 +157,9 @@ TEST_F(GlobalTypeAnalysisTest, ConstNullnessDomainTest) { auto lta = gta->get_replayable_local_analysis(meth_foo); auto code = meth_foo->get_code(); auto foo_exit_env = lta->get_exit_state_at(code->cfg().exit_block()); - EXPECT_TRUE(foo_exit_env.get_reg_environment().get(0).is_top()); + EXPECT_FALSE(foo_exit_env.get_reg_environment().get(0).is_top()); + EXPECT_EQ(*foo_exit_env.get_reg_environment().get(0).get_constant(), 1); + EXPECT_TRUE(foo_exit_env.get_reg_environment().get(0).is_not_null()); } TEST_F(GlobalTypeAnalysisTest, ArrayConstNullnessDomainTest) { @@ -190,8 +192,11 @@ TEST_F(GlobalTypeAnalysisTest, ClinitFieldAnalyzerTest) { auto field_sbase = get_field("TestH;.BASE:Lcom/facebook/redextest/TestH$Base;"); auto ftype = wps.get_field_type(field_sbase); - EXPECT_TRUE(ftype.is_top()); + EXPECT_FALSE(ftype.is_top()); EXPECT_TRUE(ftype.is_nullable()); + EXPECT_EQ(ftype.get_single_domain(), + SingletonDexTypeDomain(get_type("TestH$Base"))); + EXPECT_EQ(ftype.get_set_domain(), get_small_set_domain({"TestH$Base"})); auto field_mbase = get_field("TestH;.mBase:Lcom/facebook/redextest/TestH$Base;"); @@ -214,8 +219,11 @@ TEST_F(GlobalTypeAnalysisTest, ClinitFieldAnalyzerTest) { auto meth_baz = get_method("TestH;.baz", "", "Lcom/facebook/redextest/TestH$Base;"); rtype = wps.get_return_type(meth_baz); - EXPECT_TRUE(rtype.is_top()); + EXPECT_FALSE(rtype.is_top()); EXPECT_TRUE(rtype.is_nullable()); + EXPECT_EQ(rtype.get_single_domain(), + SingletonDexTypeDomain(get_type("TestH$Base"))); + EXPECT_EQ(rtype.get_set_domain(), get_small_set_domain({"TestH$Base"})); } TEST_F(GlobalTypeAnalysisTest, IFieldsNullnessTest) { @@ -262,6 +270,7 @@ TEST_F(GlobalTypeAnalysisTest, PrimitiveArrayTest) { EXPECT_TRUE(rtype.is_not_null()); EXPECT_EQ(rtype.get_single_domain(), SingletonDexTypeDomain(get_type_simple("[B"))); + EXPECT_TRUE(rtype.get_array_nullness().is_top()); } TEST_F(GlobalTypeAnalysisTest, InstanceSensitiveCtorTest) { @@ -306,6 +315,7 @@ TEST_F(GlobalTypeAnalysisTest, ArrayNullnessEscapeTest) { EXPECT_EQ(rtype.get_single_domain(), SingletonDexTypeDomain( get_type_simple("Lcom/facebook/redextest/TestM$A;"))); + EXPECT_TRUE(rtype.get_array_nullness().is_top()); } TEST_F(GlobalTypeAnalysisTest, ArrayNullnessEscape2Test) { @@ -325,6 +335,7 @@ TEST_F(GlobalTypeAnalysisTest, ArrayNullnessEscape2Test) { EXPECT_EQ(rtype.get_single_domain(), SingletonDexTypeDomain( get_type_simple("Lcom/facebook/redextest/TestN$A;"))); + EXPECT_TRUE(rtype.get_array_nullness().is_top()); auto dance2 = get_method("TestN;.danceWithArray2", "", "Lcom/facebook/redextest/TestN$A;"); @@ -335,6 +346,7 @@ TEST_F(GlobalTypeAnalysisTest, ArrayNullnessEscape2Test) { EXPECT_EQ(rtype.get_single_domain(), SingletonDexTypeDomain( get_type_simple("Lcom/facebook/redextest/TestN$A;"))); + EXPECT_TRUE(rtype.get_array_nullness().is_top()); } TEST_F(GlobalTypeAnalysisTest, MultipleCalleeTest) { diff --git a/test/integ/IODI.cpp b/test/integ/IODI.cpp index 4ae277b191..959102293b 100644 --- a/test/integ/IODI.cpp +++ b/test/integ/IODI.cpp @@ -94,14 +94,12 @@ class IODITest : public ::testing::Test { &method_to_id, &code_debug_lines); output.prepare( - // NOLINTNEXTLINE(bugprone-string-literal-with-embedded-nul) SortMode::DEFAULT, {SortMode::DEFAULT}, dummy_cfg, "dex\n035\0"); if (mids) { for (auto& iter : method_to_id) { DexMethod* method = iter.first; - redex_assert(CONSTP(method)->get_dex_code() != nullptr); - redex_assert(CONSTP(method)->get_dex_code()->get_debug_item() != - nullptr); + redex_assert(method->get_dex_code() != nullptr); + redex_assert(method->get_dex_code()->get_debug_item() != nullptr); mids->emplace(show(method), iter.second); } } @@ -584,8 +582,8 @@ class IODIEncodingTest : public IODITest { if (layered) { auto layer = get_iodi_layer(*debug_item); if (layer) { - pretty_name = IODIMetadata::get_layered_name(pretty_name, *layer, - pretty_name); + pretty_name = IODIMetadata::get_layered_name( + pretty_name, *layer, pretty_name); } } if (is_plain) { diff --git a/test/integ/MaxDepthAnalysisTest.cpp b/test/integ/MaxDepthAnalysisTest.cpp index cad0fdcb54..ad7c3f60e6 100644 --- a/test/integ/MaxDepthAnalysisTest.cpp +++ b/test/integ/MaxDepthAnalysisTest.cpp @@ -24,7 +24,9 @@ class AnalysisConsumerPass : public Pass { const override { using namespace redex_properties::interactions; using namespace redex_properties::names; - return {}; + return { + {HasSourceBlocks, Preserves}, + }; } void set_analysis_usage(AnalysisUsage& au) const override { diff --git a/test/integ/ObjectEscapeAnalysisTest.config b/test/integ/ObjectEscapeAnalysisTest.config index 5e52f53d92..34efd26b7d 100644 --- a/test/integ/ObjectEscapeAnalysisTest.config +++ b/test/integ/ObjectEscapeAnalysisTest.config @@ -1,7 +1,6 @@ { "redex": { "passes": [ - "BranchPrefixHoistingPass", "ObjectEscapeAnalysisPass" ] }, diff --git a/test/integ/ObjectEscapeAnalysisTest.cpp b/test/integ/ObjectEscapeAnalysisTest.cpp index cbf280d657..23eaf9dfd0 100644 --- a/test/integ/ObjectEscapeAnalysisTest.cpp +++ b/test/integ/ObjectEscapeAnalysisTest.cpp @@ -14,7 +14,6 @@ #include #include -#include "BranchPrefixHoisting.h" #include "ControlFlow.h" #include "DexAsm.h" #include "DexClass.h" @@ -54,7 +53,6 @@ class ObjectEscapeAnalysisTest : public RedexIntegrationTest { config_file >> cfg; std::vector passes = { - new BranchPrefixHoistingPass(), new ObjectEscapeAnalysisPass(), }; @@ -292,9 +290,9 @@ TEST_F(ObjectEscapeAnalysisTest, reduceTo42WithExpandedCtor) { "redextest/ObjectEscapeAnalysisTest$N;"); auto expected = assembler::ircode_from_string(R"( ( + (const v3 42) (new-instance "Lcom/facebook/redextest/ObjectEscapeAnalysisTest$N;") (move-result-pseudo-object v0) - (const v3 42) (invoke-direct (v0 v3) "Lcom/facebook/redextest/ObjectEscapeAnalysisTest$N;.:(I)V") (return-object v0) ) @@ -374,9 +372,9 @@ TEST_F(ObjectEscapeAnalysisTest, reduceTo42IncompleteInlinableTypeB) { "ObjectEscapeAnalysisTest;.reduceTo42IncompleteInlinableTypeB:()I"); auto expected = assembler::ircode_from_string(R"( ( + (const v2 16) (new-instance "Lcom/facebook/redextest/ObjectEscapeAnalysisTest$O;") (move-result-pseudo-object v1) - (const v2 16) (invoke-direct (v1 v2) "Lcom/facebook/redextest/ObjectEscapeAnalysisTest$O;.:(I)V") (sput-object v1 "Lcom/facebook/redextest/ObjectEscapeAnalysisTest$O;.instance:Lcom/facebook/redextest/ObjectEscapeAnalysisTest$O;") (const v1 42) @@ -386,53 +384,6 @@ TEST_F(ObjectEscapeAnalysisTest, reduceTo42IncompleteInlinableTypeB) { ASSERT_EQ(actual.str(), assembler::to_s_expr(expected.get()).str()); } -TEST_F(ObjectEscapeAnalysisTest, reduceIncompleteInlinableType) { - run(); - - auto actual = get_s_expr( - "Lcom/facebook/redextest/" - "ObjectEscapeAnalysisTest;.reduceIncompleteInlinableType:(Z)I"); - // We reduce away the creation of the "O" type, but not yet the D type, as - // that's a second-order problem... - auto expected = assembler::ircode_from_string(R"( - ( - (load-param v5) - (const v8 42) - (invoke-static (v8) "Lcom/facebook/redextest/ObjectEscapeAnalysisTest$D;.allocator:(I)Lcom/facebook/redextest/ObjectEscapeAnalysisTest$D;") - (move-result-object v6) - (invoke-virtual (v6) "Lcom/facebook/redextest/ObjectEscapeAnalysisTest$D;.getX:()I") - (if-eqz v5 :L1) - (const v20 42) - (:L0) - (return v20) - (:L1) - (const v20 23) - (goto :L0) - ) -)"); - ASSERT_EQ(actual.str(), assembler::to_s_expr(expected.get()).str()); -} - -TEST_F(ObjectEscapeAnalysisTest, doNotReduceTo42Finalize) { - run(); - - auto actual = get_s_expr( - "Lcom/facebook/redextest/" - "ObjectEscapeAnalysisTest;.doNotReduceTo42Finalize:()I"); - auto expected = assembler::ircode_from_string(R"( - ( - (new-instance "Lcom/facebook/redextest/ObjectEscapeAnalysisTest$P;") - (move-result-pseudo-object v0) - (const v1 42) - (invoke-direct (v0 v1) "Lcom/facebook/redextest/ObjectEscapeAnalysisTest$P;.:(I)V") - (invoke-virtual (v0) "Lcom/facebook/redextest/ObjectEscapeAnalysisTest$P;.getX:()I") - (move-result v1) - (return v1) - ) -)"); - ASSERT_EQ(actual.str(), assembler::to_s_expr(expected.get()).str()); -} - TEST_F(ObjectEscapeAnalysisTest, nothingToReduce) { run(); diff --git a/test/integ/ObjectEscapeAnalysisTest.java b/test/integ/ObjectEscapeAnalysisTest.java index a2e9c88d7a..98d098fe65 100644 --- a/test/integ/ObjectEscapeAnalysisTest.java +++ b/test/integ/ObjectEscapeAnalysisTest.java @@ -323,10 +323,6 @@ public O(int x) { public int getX() { return this.x; } - - public int getX4(int dummy_a, int dummy_b, int dummy_c, int dummy_d) { - return this.x; - } } public static int reduceTo42IncompleteInlinableType() { @@ -342,38 +338,6 @@ public static int reduceTo42IncompleteInlinableTypeB() { return o.getX(); } - public static O helper(boolean b) { - // This completely reducible class D makes the helper into a root. - D d = D.allocator(42); - d.getX(); - return b ? new O(42) : new O(23); - } - - public static int reduceIncompleteInlinableType(boolean b) { - O o = helper(b); - return o.getX4(1, 2, 3, 4); - } - - public static class P { - int x; - - public P(int x) { - this.x = x; - } - - public int getX() { - return this.x; - } - - protected void finalize() {} - } - - public static int doNotReduceTo42Finalize() { - // This object creation can NOT be reduced - P p = new P(42); - return p.getX(); - } - static class Q { public Q() {} diff --git a/test/integ/ObjectSensitiveDceTest.cpp b/test/integ/ObjectSensitiveDceTest.cpp index a6f99d899b..752d7764e4 100644 --- a/test/integ/ObjectSensitiveDceTest.cpp +++ b/test/integ/ObjectSensitiveDceTest.cpp @@ -60,149 +60,6 @@ TEST_F(ObjectSensitiveDceTest, basic) { ASSERT_EQ(it->insn->opcode(), OPCODE_RETURN_VOID); } -TEST_F(ObjectSensitiveDceTest, invoke_super) { - auto method_ref = DexMethod::get_method( - "Lcom/facebook/redextest/ObjectSensitiveDceTest;.invoke_super:()V"); - EXPECT_NE(method_ref, nullptr); - auto method = method_ref->as_def(); - EXPECT_NE(method, nullptr); - - std::vector passes = { - new ObjectSensitiveDcePass(), - }; - - run_passes(passes); - - auto ii = InstructionIterable(method->get_code()); - auto it = ii.begin(); - ASSERT_TRUE(it != ii.end()); - ASSERT_EQ(it->insn->opcode(), OPCODE_RETURN_VOID); -} - -TEST_F(ObjectSensitiveDceTest, invoke_virtual_with_overrides) { - auto method_ref = DexMethod::get_method( - "Lcom/facebook/redextest/" - "ObjectSensitiveDceTest;.invoke_virtual_with_overrides:()V"); - EXPECT_NE(method_ref, nullptr); - auto method = method_ref->as_def(); - EXPECT_NE(method, nullptr); - - std::vector passes = { - new ObjectSensitiveDcePass(), - }; - - run_passes(passes); - - auto ii = InstructionIterable(method->get_code()); - auto it = ii.begin(); - ASSERT_TRUE(it != ii.end()); - ASSERT_EQ(it->insn->opcode(), OPCODE_RETURN_VOID); -} - -TEST_F(ObjectSensitiveDceTest, invoke_virtual_with_overrides_with_side_effect) { - auto method_ref = DexMethod::get_method( - "Lcom/facebook/redextest/" - "ObjectSensitiveDceTest;.invoke_virtual_with_overrides_with_side_effect:(" - ")V"); - EXPECT_NE(method_ref, nullptr); - auto method = method_ref->as_def(); - EXPECT_NE(method, nullptr); - - std::vector passes = { - new ObjectSensitiveDcePass(), - }; - - run_passes(passes); - - // Nothing could get optimized away, because the invoke-virtual to bar - // has an override with side-effects, so it couldn't get removed, and thus - // the object creation itself is required. - auto code = method->get_code(); - ASSERT_EQ(method::count_opcode_of_types(code, {OPCODE_NEW_INSTANCE}), 1); - ASSERT_EQ(method::count_opcode_of_types(code, {OPCODE_INVOKE_DIRECT}), 1); - ASSERT_EQ(method::count_opcode_of_types(code, {OPCODE_INVOKE_VIRTUAL}), 1); -} - -TEST_F(ObjectSensitiveDceTest, invoke_virtual_with_too_many_overrides) { - auto method_ref = DexMethod::get_method( - "Lcom/facebook/redextest/" - "ObjectSensitiveDceTest;.invoke_virtual_with_too_many_overrides:()V"); - EXPECT_NE(method_ref, nullptr); - auto method = method_ref->as_def(); - EXPECT_NE(method, nullptr); - - std::vector passes = { - new ObjectSensitiveDcePass(), - }; - - run_passes(passes); - - // Nothing could get optimized away, because the invoke-virtual to bar - // has an override with side-effects, so it couldn't get removed, and thus - // the object creation itself is required. - auto code = method->get_code(); - ASSERT_EQ(method::count_opcode_of_types(code, {OPCODE_NEW_INSTANCE}), 1); - ASSERT_EQ(method::count_opcode_of_types(code, {OPCODE_INVOKE_DIRECT}), 1); - ASSERT_EQ(method::count_opcode_of_types(code, {OPCODE_INVOKE_VIRTUAL}), 1); -} - -TEST_F(ObjectSensitiveDceTest, non_termination) { - auto method_ref = DexMethod::get_method( - "Lcom/facebook/redextest/ObjectSensitiveDceTest;.non_termination:()V"); - EXPECT_NE(method_ref, nullptr); - auto method = method_ref->as_def(); - EXPECT_NE(method, nullptr); - - std::vector passes = { - new ObjectSensitiveDcePass(), - }; - - run_passes(passes); - - auto ii = InstructionIterable(method->get_code()); - auto it = ii.begin(); - ASSERT_TRUE(it != ii.end()); - ASSERT_EQ(it->insn->opcode(), OPCODE_RETURN_VOID); -} - -TEST_F(ObjectSensitiveDceTest, recursive) { - auto method_ref = DexMethod::get_method( - "Lcom/facebook/redextest/ObjectSensitiveDceTest;.recursive:()V"); - EXPECT_NE(method_ref, nullptr); - auto method = method_ref->as_def(); - EXPECT_NE(method, nullptr); - - std::vector passes = { - new ObjectSensitiveDcePass(), - }; - - run_passes(passes); - - auto ii = InstructionIterable(method->get_code()); - auto it = ii.begin(); - ASSERT_TRUE(it != ii.end()); - ASSERT_EQ(it->insn->opcode(), OPCODE_RETURN_VOID); -} - -TEST_F(ObjectSensitiveDceTest, mutually_recursive) { - auto method_ref = DexMethod::get_method( - "Lcom/facebook/redextest/ObjectSensitiveDceTest;.mutually_recursive:()V"); - EXPECT_NE(method_ref, nullptr); - auto method = method_ref->as_def(); - EXPECT_NE(method, nullptr); - - std::vector passes = { - new ObjectSensitiveDcePass(), - }; - - run_passes(passes); - - auto ii = InstructionIterable(method->get_code()); - auto it = ii.begin(); - ASSERT_TRUE(it != ii.end()); - ASSERT_EQ(it->insn->opcode(), OPCODE_RETURN_VOID); -} - TEST_F(ObjectSensitiveDceTest, clinit_with_side_effects) { auto method_ref = DexMethod::get_method( "Lcom/facebook/redextest/" @@ -250,43 +107,3 @@ TEST_F(ObjectSensitiveDceTest, method_needing_init_class) { ASSERT_EQ(method::count_opcode_of_types(code, {OPCODE_INVOKE_DIRECT}), 1); ASSERT_EQ(method::count_opcode_of_types(code, {OPCODE_INVOKE_VIRTUAL}), 1); } - -TEST_F(ObjectSensitiveDceTest, pure_method) { - auto method_ref = DexMethod::get_method( - "Lcom/facebook/redextest/" - "ObjectSensitiveDceTest;.pure_method:()V"); - EXPECT_NE(method_ref, nullptr); - auto method = method_ref->as_def(); - EXPECT_NE(method, nullptr); - - std::vector passes = { - new ObjectSensitiveDcePass(), - }; - - run_passes(passes); - - auto ii = InstructionIterable(method->get_code()); - auto it = ii.begin(); - ASSERT_TRUE(it != ii.end()); - ASSERT_EQ(it->insn->opcode(), OPCODE_RETURN_VOID); -} - -TEST_F(ObjectSensitiveDceTest, array_clone) { - auto method_ref = DexMethod::get_method( - "Lcom/facebook/redextest/" - "ObjectSensitiveDceTest;.array_clone:()V"); - EXPECT_NE(method_ref, nullptr); - auto method = method_ref->as_def(); - EXPECT_NE(method, nullptr); - - std::vector passes = { - new ObjectSensitiveDcePass(), - }; - - run_passes(passes); - - auto ii = InstructionIterable(method->get_code()); - auto it = ii.begin(); - ASSERT_TRUE(it != ii.end()); - ASSERT_EQ(it->insn->opcode(), OPCODE_RETURN_VOID); -} diff --git a/test/integ/ObjectSensitiveDceTest.java b/test/integ/ObjectSensitiveDceTest.java index 23da281bbb..c620d0d15d 100644 --- a/test/integ/ObjectSensitiveDceTest.java +++ b/test/integ/ObjectSensitiveDceTest.java @@ -13,41 +13,6 @@ public static void basic() { useless.foo(); } - public static void recursive() { - Useless useless = new Useless(); - useless.recursive_foo(100); - } - - public static void mutually_recursive() { - Useless useless = new Useless(); - useless.mutually_recursive_foo(100); - } - - public static void invoke_super() { - UselessDerived useless = new UselessDerived(); - useless.foo(); - } - - public static void invoke_virtual_with_overrides() { - UselessBase useless = new UselessDerived(); - useless.bar(); - } - - public static void invoke_virtual_with_overrides_with_side_effect() { - UselessBase useless = new UselessDerived(); - useless.bar_where_override_has_side_effect(); - } - - public static void invoke_virtual_with_too_many_overrides() { - UselessBase useless = new UselessDerived(); - useless.bar_with_invoke_virtual_with_too_many_overrides(); - } - - public static void non_termination() { - Useless useless = new Useless(); - useless.non_terminating_foo(); - } - public static void clinit_with_side_effects() { UselessWithClInitWithSideEffects useless = new UselessWithClInitWithSideEffects(); useless.foo(); @@ -57,17 +22,6 @@ public static void method_needing_init_class() { UselessWithMethodNeedingInitClass useless = new UselessWithMethodNeedingInitClass(); useless.foo(); } - - public static void pure_method() { - Useless useless = new Useless(); - useless.getClass(); - } - - public static void array_clone() { - int[] a = new int[1]; - a = a.clone(); - a.clone(); - } } class Useless { @@ -76,95 +30,6 @@ public Useless() {} public void foo() { F = 42; } - public void non_terminating_foo() { - while (true) {} - } - public void recursive_foo(int x) { - if (x<=0) { - F = 42; - } else { - recursive_foo(x-1); - } - } - public void mutually_recursive_foo(int x) { - if (x<=0) { - F = 42; - } else { - mutually_recursive_bar(x-1); - } - } - public void mutually_recursive_bar(int x) { - if (x<=0) { - F = 42; - } else { - mutually_recursive_foo(x-1); - } - } -} - -class UselessBase { - int F; - public UselessBase() {} - public void foo() { - F = 42; - } - public void bar() { - F = 42; - } - public void bar_where_override_has_side_effect() { - F = 42; - } - public void bar_with_invoke_virtual_with_too_many_overrides() { - F = 42; - } -} - -class UselessDerived extends UselessBase { - public UselessDerived() {} - public void foo() { - super.foo(); - } - public void bar() { - F = 42; - } - public void bar_where_override_has_side_effect() { - try { - System.loadLibrary("boo"); // side effect - } catch (Throwable t) { } - } - public void bar_with_invoke_virtual_with_too_many_overrides() { - F = 42; - } -} - -class UselessDerived2 extends UselessBase { - public void bar_with_invoke_virtual_with_too_many_overrides() { - F = 42; - } -} - -class UselessDerived3 extends UselessBase { - public void bar_with_invoke_virtual_with_too_many_overrides() { - F = 42; - } -} - -class UselessDerived4 extends UselessBase { - public void bar_with_invoke_virtual_with_too_many_overrides() { - F = 42; - } -} - -class UselessDerived5 extends UselessBase { - public void bar_with_invoke_virtual_with_too_many_overrides() { - F = 42; - } -} - -class UselessDerived6 extends UselessBase { - public void bar_with_invoke_virtual_with_too_many_overrides() { - F = 42; - } } class UselessWithClInitWithSideEffects { diff --git a/test/integ/OptimizeResourcesTest.cpp b/test/integ/OptimizeResourcesTest.cpp index 9e8a2cdcb1..1ae2f5adbf 100644 --- a/test/integ/OptimizeResourcesTest.cpp +++ b/test/integ/OptimizeResourcesTest.cpp @@ -24,8 +24,7 @@ uint32_t find_const_value(IRCode* code, IRInstruction* use, uint16_t reg) { // Janky scan that certainly won't work with control flow. Shouldn't matter // for just the autogenerated method. int64_t literal = -1; - auto& cfg = code->cfg(); - for (const auto& mie : cfg::InstructionIterable(cfg)) { + for (const auto& mie : InstructionIterable(code)) { auto insn = mie.insn; if (insn == use) { break; @@ -37,9 +36,8 @@ uint32_t find_const_value(IRCode* code, IRInstruction* use, uint16_t reg) { return literal; } -void dump_code_verbose(IRCode* code) { - auto& cfg = code->cfg(); - for (const auto& mie : cfg::InstructionIterable(cfg)) { +void dump_code_verbose(const IRCode* code) { + for (const auto& mie : InstructionIterable(code)) { auto insn = mie.insn; std::cout << SHOW(mie) << std::endl; if (insn->opcode() == OPCODE_FILL_ARRAY_DATA) { @@ -90,7 +88,6 @@ TEST_F(OptimizeResourcesTest, remapResourceClassArrays) { std::cout << "BASELINE R :" << std::endl; auto clinit = base_r_class->get_clinit(); auto code = clinit->get_code(); - code->build_cfg(); dump_code_verbose(code); // A typical styleable inner class, which has different conventions and is @@ -100,7 +97,6 @@ TEST_F(OptimizeResourcesTest, remapResourceClassArrays) { std::cout << std::endl << "BASELINE R$styleable :" << std::endl; auto styleable_clinit = styleable_class->get_clinit(); auto styleable_code = styleable_clinit->get_code(); - styleable_code->build_cfg(); dump_code_verbose(styleable_code); std::map old_to_remapped_ids; @@ -126,8 +122,7 @@ TEST_F(OptimizeResourcesTest, remapResourceClassArrays) { std::vector expected_sizes = {4, 2, 1}; size_t count = 0; - auto& cfg = code->cfg(); - for (const auto& mie : cfg::InstructionIterable(cfg)) { + for (const auto& mie : InstructionIterable(code)) { auto insn = mie.insn; if (insn->opcode() == OPCODE_NEW_ARRAY) { auto size_reg = insn->src(0); @@ -139,8 +134,7 @@ TEST_F(OptimizeResourcesTest, remapResourceClassArrays) { std::cout << std::endl << "MODIFIED R$styleable :" << std::endl; dump_code_verbose(styleable_code); // Despite deleting one item, size should still be 2 - auto& styleable_cfg = styleable_code->cfg(); - for (const auto& mie : cfg::InstructionIterable(styleable_cfg)) { + for (const auto& mie : InstructionIterable(styleable_code)) { auto insn = mie.insn; if (insn->opcode() == OPCODE_NEW_ARRAY) { auto size_reg = insn->src(0); diff --git a/test/integ/PointsToSemanticsTest.cpp b/test/integ/PointsToSemanticsTest.cpp index a52780aea9..bd45eec280 100644 --- a/test/integ/PointsToSemanticsTest.cpp +++ b/test/integ/PointsToSemanticsTest.cpp @@ -304,7 +304,20 @@ void patch_filled_new_array_test(Scope& scope) { class PointsToSemanticsTest : public RedexIntegrationTest {}; TEST_F(PointsToSemanticsTest, semanticActionGeneration) { - std::string sdk_jar = android_sdk_jar_path(); + const char* android_env_sdk = std::getenv("ANDROID_SDK"); + const char* android_config_sdk = std::getenv("sdk_path"); + + const char* android_sdk = (strncmp(android_config_sdk, "None", 4) != 0) + ? android_config_sdk + : android_env_sdk; + + ASSERT_NE(nullptr, android_sdk); + const char* android_target = std::getenv("android_target"); + ASSERT_NE(nullptr, android_target); + std::string android_version(android_target); + ASSERT_NE("NotFound", android_version); + std::string sdk_jar = std::string(android_sdk) + "/platforms/" + + android_version + "/android.jar"; ASSERT_TRUE(load_jar_file(DexLocation::make_location("", sdk_jar))); DexStoreClassesIterator it(stores); diff --git a/test/integ/ReachabilityTest.cpp b/test/integ/ReachabilityTest.cpp index 97ac3ca6ac..949f78f272 100644 --- a/test/integ/ReachabilityTest.cpp +++ b/test/integ/ReachabilityTest.cpp @@ -40,11 +40,8 @@ TEST_F(ReachabilityTest, ReachabilityFromProguardTest) { reachability::ReachableAspects reachable_aspects; auto scope = build_class_scope(stores); walk::parallel::code(scope, [&](auto*, auto& code) { code.build_cfg(); }); - auto method_override_graph = method_override_graph::build_graph(scope); - auto reachable_objects = reachability::compute_reachable_objects( - scope, *method_override_graph, ig_sets, &num_ignore_check_strings, - &reachable_aspects); + stores, ig_sets, &num_ignore_check_strings, &reachable_aspects); walk::parallel::code(scope, [&](auto*, auto& code) { code.clear_cfg(); }); reachability::mark_classes_abstract(stores, *reachable_objects, @@ -85,17 +82,14 @@ TEST_F(ReachabilityTest, ReachabilityMarkAllTest) { reachability::ReachableAspects reachable_aspects; auto scope = build_class_scope(stores); walk::parallel::code(scope, [&](auto*, auto& code) { code.build_cfg(); }); - auto method_override_graph = method_override_graph::build_graph(scope); - auto reachable_objects = reachability::compute_reachable_objects( - scope, *method_override_graph, ig_sets, &num_ignore_check_strings, - &reachable_aspects, + stores, ig_sets, &num_ignore_check_strings, &reachable_aspects, /* record_reachability */ false, /* relaxed_keep_class_members */ false, /* relaxed_keep_interfaces */ false, /* cfg_gathering_check_instantiable */ false, /* cfg_gathering_check_instance_callable */ false, /* cfg_gathering_check_returning */ false, - /* should_mark_all_as_seed */ true); + /* should_mark_all_as_seed */ true, nullptr); walk::parallel::code(scope, [&](auto*, auto& code) { code.clear_cfg(); }); reachability::mark_classes_abstract(stores, *reachable_objects, @@ -127,11 +121,8 @@ TEST_F(ReachabilityTest, NotDireclyInstantiatedClassesBecomeAbstract) { reachability::ReachableAspects reachable_aspects; auto scope = build_class_scope(stores); walk::parallel::code(scope, [&](auto*, auto& code) { code.build_cfg(); }); - auto method_override_graph = method_override_graph::build_graph(scope); - auto reachable_objects = reachability::compute_reachable_objects( - scope, *method_override_graph, ig_sets, &num_ignore_check_strings, - &reachable_aspects); + stores, ig_sets, &num_ignore_check_strings, &reachable_aspects); walk::parallel::code(scope, [&](auto*, auto& code) { code.clear_cfg(); }); auto abstracted_classes = reachability::mark_classes_abstract( stores, *reachable_objects, reachable_aspects); @@ -182,11 +173,8 @@ TEST_F(ReachabilityTest, SharpeningCreatesMoreZombies) { reachability::ReachableAspects reachable_aspects; auto scope = build_class_scope(stores); walk::parallel::code(scope, [&](auto*, auto& code) { code.build_cfg(); }); - auto method_override_graph = method_override_graph::build_graph(scope); - auto reachable_objects = reachability::compute_reachable_objects( - scope, *method_override_graph, ig_sets, &num_ignore_check_strings, - &reachable_aspects); + stores, ig_sets, &num_ignore_check_strings, &reachable_aspects); walk::parallel::code(scope, [&](auto*, auto& code) { code.clear_cfg(); }); auto is_callable_instance_method = [&](const auto& s) { diff --git a/test/integ/ResolveRefsTest.cpp b/test/integ/ResolveRefsTest.cpp index 39a36a9f60..f99cfc9325 100644 --- a/test/integ/ResolveRefsTest.cpp +++ b/test/integ/ResolveRefsTest.cpp @@ -6,14 +6,9 @@ */ #include -#include -#include "JarLoader.h" -#include "RedexOptions.h" #include "RedexTest.h" #include "ResolveRefsPass.h" -#include "ScopedCFG.h" -#include "Show.h" class ResolveRefsTest : public RedexIntegrationTest { @@ -115,77 +110,3 @@ TEST_F(ResolveRefsTest, test_rtype_not_specialized_with_cross_dexstore_refs) { ASSERT_TRUE(rtype_after); EXPECT_EQ(rtype_after, m_base_cls->get_type()); } - -TEST_F(ResolveRefsTest, test_invoke_virtual_specialization_to_interface) { - // Ensure that invoke-virtual on Object.toString() results in a correct opcode - // when an interface also defines a toString() method. This test relies on jdk - // classes (as it mimics a real world scenario), so manually suck them in to - // make the code resolvable. - auto& root_store = stores.at(0); - std::string sdk_jar = android_sdk_jar_path(); - ASSERT_TRUE(load_jar_file(DexLocation::make_location("", sdk_jar))); - - ClassCreator foo_creator(DexType::make_type("LFoo;")); - foo_creator.set_super(type::java_lang_Object()); - - auto method = - DexMethod::make_method("LFoo;.bar:()V") - ->make_concrete(ACC_STATIC | ACC_PUBLIC, false /* is_virtual */); - auto code_str = R"( - ( - (sget-object "Landroid/os/Build;.BRAND:Ljava/lang/String;") - (move-result-pseudo-object v0) - (const v1 0) - (const v2 1) - (invoke-interface (v0 v1 v2) "Ljava/lang/CharSequence;.subSequence:(II)Ljava/lang/CharSequence;") - (move-result-object v3) - (invoke-virtual (v3) "Ljava/lang/Object;.toString:()Ljava/lang/String;") - (move-result-object v4) - (invoke-static (v4 v4) "Landroid/util/Log;.v:(Ljava/lang/String;Ljava/lang/String;)I") - (return-void) - ) - )"; - method->set_code(assembler::ircode_from_string(code_str)); - foo_creator.add_method(method); - auto cls = foo_creator.create(); - method->get_code()->build_cfg(); - - root_store.add_classes(std::vector{cls}); - - std::vector passes = { - new ResolveRefsPass(), - }; - - RedexOptions options{}; - // A sensible lower bound for most of our apps. Need to kick on the resolving - // of external refs for the above ir code to be relevant. - options.min_sdk = 21; - - Json::Value root; - auto val_cstr = std::getenv("api"); - redex_assert(val_cstr != nullptr); - root["android_sdk_api_21_file"] = val_cstr; - - run_passes(passes, nullptr, root, options); - - cfg::ScopedCFG cfg(method->get_code()); - IRInstruction* invoke_to_string{nullptr}; - for (auto it = cfg::ConstInstructionIterator(*cfg, true); !it.is_end(); - ++it) { - if (it->insn->has_method()) { - auto method_name = it->insn->get_method()->get_name(); - if (method_name->str_copy() == "toString") { - invoke_to_string = it->insn; - } - } - } - - EXPECT_NE(invoke_to_string, nullptr) - << "Relevant instruction to assert was not found!"; - EXPECT_EQ(invoke_to_string->opcode(), OPCODE_INVOKE_VIRTUAL) - << "Incorrect invoke type!"; - EXPECT_EQ(invoke_to_string->get_method()->get_class(), - type::java_lang_Object()) - << "Should not rebind toString! Got " - << show(invoke_to_string->get_method()->get_class()); -} diff --git a/test/integ/SourceBlocksTest.cpp b/test/integ/SourceBlocksTest.cpp index e015aec70b..9c0d5b6353 100644 --- a/test/integ/SourceBlocksTest.cpp +++ b/test/integ/SourceBlocksTest.cpp @@ -29,7 +29,7 @@ class SourceBlocksTest : public RedexIntegrationTest { SourceBlocksTest() { // The loading code in integ-test does not insert deobfuscated names. walk::methods(*classes, [](auto* m) { - always_assert(m->get_deobfuscated_name_or_null() == nullptr); + redex_assert(m->get_deobfuscated_name_or_null() == nullptr); m->set_deobfuscated_name(show(m)); }); } @@ -195,7 +195,6 @@ TEST_F(SourceBlocksTest, source_blocks) { { inliner::InlinerConfig conf{}; auto scope = build_class_scope(stores); - walk::parallel::code(scope, [&](auto*, IRCode& code) { code.build_cfg(); }); conf.populate(scope); init_classes::InitClassesWithSideEffects init_classes_with_side_effects( scope, /* create_init_class_insns */ false); @@ -216,8 +215,6 @@ TEST_F(SourceBlocksTest, source_blocks) { std::ref(concurrent_method_resolver), conf, min_sdk, MultiMethodInlinerMode::IntraDex); inliner.inline_methods(); - walk::parallel::code(scope, [&](auto*, IRCode& code) { code.clear_cfg(); }); - ASSERT_EQ(inliner.get_inlined().size(), 1u); auto bar_ref = DexMethod::get_method( @@ -432,7 +429,6 @@ TEST_F(SourceBlocksTest, scaling) { { inliner::InlinerConfig conf{}; auto scope = build_class_scope(stores); - walk::parallel::code(scope, [&](auto*, IRCode& code) { code.build_cfg(); }); conf.populate(scope); init_classes::InitClassesWithSideEffects init_classes_with_side_effects( scope, /* create_init_class_insns */ false); @@ -448,8 +444,6 @@ TEST_F(SourceBlocksTest, scaling) { std::ref(concurrent_method_resolver), conf, min_sdk, MultiMethodInlinerMode::IntraDex); inliner.inline_methods(); - walk::parallel::code(scope, [&](auto*, IRCode& code) { code.clear_cfg(); }); - ASSERT_EQ(inliner.get_inlined().size(), 1u); ASSERT_EQ(inliner.get_info().calls_inlined, 4u); diff --git a/test/integ/TypeAnalysisRemoveRedundantCmp.cpp b/test/integ/TypeAnalysisRemoveRedundantCmp.cpp index 6048b356ca..c0a1078501 100644 --- a/test/integ/TypeAnalysisRemoveRedundantCmp.cpp +++ b/test/integ/TypeAnalysisRemoveRedundantCmp.cpp @@ -50,11 +50,6 @@ TEST_F(TypeAnalysisTransformTest, MethodHasNoEqDefined) { ->as_def(); auto codey = y_method->get_code(); ASSERT_NE(nullptr, codey); - auto jj = InstructionIterable(y_method->get_code()); - auto end2 = jj.end(); - for (auto it = jj.begin(); it != end2; ++it) { - auto insn = it->insn; - } } } // namespace diff --git a/test/integ/TypeInferenceTest.cpp b/test/integ/TypeInferenceTest.cpp index e3b8b7544a..e5061f4754 100644 --- a/test/integ/TypeInferenceTest.cpp +++ b/test/integ/TypeInferenceTest.cpp @@ -256,9 +256,14 @@ TEST_F(TypeInferenceTest, test_join_with_interface) { } auto ret_type = exit_env.get_type_domain(insn->src(0)); EXPECT_TRUE(ret_type.get_dex_type()); + const auto& type_set = ret_type.get_type_set(); EXPECT_EQ(*ret_type.get_dex_type(), DexType::get_type("Lcom/facebook/redextest/I;")); - EXPECT_TRUE(ret_type.get_set_domain().is_top()); + EXPECT_EQ(type_set.size(), 2); + EXPECT_TRUE( + type_set.contains(DexType::get_type("Lcom/facebook/redextest/I;"))); + EXPECT_TRUE( + type_set.contains(DexType::get_type("Lcom/facebook/redextest/C;"))); } } diff --git a/test/integ/TypedefAnnoCheckerTest.cpp b/test/integ/TypedefAnnoCheckerTest.cpp index 0dd8a0ebf2..ce32a18e9c 100644 --- a/test/integ/TypedefAnnoCheckerTest.cpp +++ b/test/integ/TypedefAnnoCheckerTest.cpp @@ -22,51 +22,41 @@ struct TypedefAnnoCheckerTest : public RedexIntegrationTest { return m_config; } - void build_cfg(Scope& scope) { - for (auto* cls : scope) { - always_assert(cls); - for (auto* m : cls->get_dmethods()) { - if (m && m->get_code()) { - m->get_code()->build_cfg(); - } - } - for (auto* m : cls->get_vmethods()) { - if (m && m->get_code()) { - m->get_code()->build_cfg(); - } - } - } - } - - void gather_typedef_values(TypedefAnnoCheckerPass pass, - DexClass* cls, - StrDefConstants& strdef_constants, - IntDefConstants& intdef_constants) { + void gather_typedef_values( + TypedefAnnoCheckerPass pass, + DexClass* cls, + ConcurrentMap>& + strdef_constants, + ConcurrentMap>& + intdef_constants) { pass.gather_typedef_values(cls, strdef_constants, intdef_constants); } }; TEST_F(TypedefAnnoCheckerTest, TestValidIntAnnoReturn) { auto scope = build_class_scope(stores); - build_cfg(scope); - auto* method = DexMethod::get_method( - "Lcom/facebook/redextest/" - "TypedefAnnoCheckerTest;.testValidIntAnnoReturn:(I)I") - ->as_def(); - - StrDefConstants strdef_constants; - IntDefConstants intdef_constants; + auto method = DexMethod::get_method( + "Lcom/facebook/redextest/" + "TypedefAnnoCheckerTest;.testValidIntAnnoReturn:(I)I") + ->as_def(); + + IRCode* code = method->get_code(); + code->build_cfg(); + ConcurrentMap> + strdef_constants; + ConcurrentMap> intdef_constants; TypedefAnnoChecker checker = TypedefAnnoChecker(strdef_constants, intdef_constants, get_config()); checker.run(method); + code->clear_cfg(); + EXPECT_TRUE(checker.complete()); } TEST_F(TypedefAnnoCheckerTest, TestValidStrAnnoReturn) { auto scope = build_class_scope(stores); - build_cfg(scope); - auto* method = + auto method = DexMethod::get_method( "Lcom/facebook/redextest/" "TypedefAnnoCheckerTest;.testValidStrAnnoReturn:(Ljava/lang/" @@ -74,36 +64,44 @@ TEST_F(TypedefAnnoCheckerTest, TestValidStrAnnoReturn) { "String;") ->as_def(); - StrDefConstants strdef_constants; - IntDefConstants intdef_constants; + IRCode* code = method->get_code(); + code->build_cfg(); + ConcurrentMap> + strdef_constants; + ConcurrentMap> intdef_constants; TypedefAnnoChecker checker = TypedefAnnoChecker(strdef_constants, intdef_constants, get_config()); checker.run(method); + code->clear_cfg(); + EXPECT_TRUE(checker.complete()); } TEST_F(TypedefAnnoCheckerTest, TestIntAnnoInvokeStatic) { auto scope = build_class_scope(stores); - build_cfg(scope); - auto* method = DexMethod::get_method( - "Lcom/facebook/redextest/" - "TypedefAnnoCheckerTest;.testIntAnnoInvokeStatic:(I)I") - ->as_def(); - - StrDefConstants strdef_constants; - IntDefConstants intdef_constants; + auto method = DexMethod::get_method( + "Lcom/facebook/redextest/" + "TypedefAnnoCheckerTest;.testIntAnnoInvokeStatic:(I)I") + ->as_def(); + + IRCode* code = method->get_code(); + code->build_cfg(); + ConcurrentMap> + strdef_constants; + ConcurrentMap> intdef_constants; TypedefAnnoChecker checker = TypedefAnnoChecker(strdef_constants, intdef_constants, get_config()); checker.run(method); + code->clear_cfg(); + EXPECT_TRUE(checker.complete()); } TEST_F(TypedefAnnoCheckerTest, TestStringAnnoInvokeStatic) { auto scope = build_class_scope(stores); - build_cfg(scope); - auto* method = + auto method = DexMethod::get_method( "Lcom/facebook/redextest/" "TypedefAnnoCheckerTest;.testStringAnnoInvokeStatic:(Ljava/lang/" @@ -111,19 +109,23 @@ TEST_F(TypedefAnnoCheckerTest, TestStringAnnoInvokeStatic) { "String;") ->as_def(); - StrDefConstants strdef_constants; - IntDefConstants intdef_constants; + IRCode* code = method->get_code(); + code->build_cfg(); + ConcurrentMap> + strdef_constants; + ConcurrentMap> intdef_constants; TypedefAnnoChecker checker = TypedefAnnoChecker(strdef_constants, intdef_constants, get_config()); checker.run(method); + code->clear_cfg(); + EXPECT_TRUE(checker.complete()); } TEST_F(TypedefAnnoCheckerTest, TestWrongAnnotationReturned) { auto scope = build_class_scope(stores); - build_cfg(scope); - auto* method = + auto method = DexMethod::get_method( "Lcom/facebook/redextest/" "TypedefAnnoCheckerTest;.testWrongAnnotationReturned:(Ljava/lang/" @@ -131,12 +133,17 @@ TEST_F(TypedefAnnoCheckerTest, TestWrongAnnotationReturned) { "String;") ->as_def(); - StrDefConstants strdef_constants; - IntDefConstants intdef_constants; + IRCode* code = method->get_code(); + code->build_cfg(); + ConcurrentMap> + strdef_constants; + ConcurrentMap> intdef_constants; TypedefAnnoChecker checker = TypedefAnnoChecker(strdef_constants, intdef_constants, get_config()); checker.run(method); + code->clear_cfg(); + EXPECT_FALSE(checker.complete()); EXPECT_EQ( checker.error(), @@ -149,18 +156,22 @@ TEST_F(TypedefAnnoCheckerTest, TestWrongAnnotationReturned) { TEST_F(TypedefAnnoCheckerTest, TestWrongAnnoInvokeStatic) { auto scope = build_class_scope(stores); - build_cfg(scope); - auto* method = DexMethod::get_method( - "Lcom/facebook/redextest/" - "TypedefAnnoCheckerTest;.testWrongAnnoInvokeStatic:(I)I") - ->as_def(); - - StrDefConstants strdef_constants; - IntDefConstants intdef_constants; + auto method = DexMethod::get_method( + "Lcom/facebook/redextest/" + "TypedefAnnoCheckerTest;.testWrongAnnoInvokeStatic:(I)I") + ->as_def(); + + IRCode* code = method->get_code(); + code->build_cfg(); + ConcurrentMap> + strdef_constants; + ConcurrentMap> intdef_constants; TypedefAnnoChecker checker = TypedefAnnoChecker(strdef_constants, intdef_constants, get_config()); checker.run(method); + code->clear_cfg(); + EXPECT_FALSE(checker.complete()); EXPECT_EQ( checker.error(), @@ -173,35 +184,43 @@ TEST_F(TypedefAnnoCheckerTest, TestWrongAnnoInvokeStatic) { TEST_F(TypedefAnnoCheckerTest, TestIntField) { auto scope = build_class_scope(stores); - build_cfg(scope); - auto* method = DexMethod::get_method( - "Lcom/facebook/redextest/" - "TypedefAnnoCheckerTest;.testIntField:(I)V") - ->as_def(); - - StrDefConstants strdef_constants; - IntDefConstants intdef_constants; + auto method = DexMethod::get_method( + "Lcom/facebook/redextest/" + "TypedefAnnoCheckerTest;.testIntField:(I)V") + ->as_def(); + + IRCode* code = method->get_code(); + code->build_cfg(); + ConcurrentMap> + strdef_constants; + ConcurrentMap> intdef_constants; TypedefAnnoChecker checker = TypedefAnnoChecker(strdef_constants, intdef_constants, get_config()); checker.run(method); + code->clear_cfg(); + EXPECT_TRUE(checker.complete()); } TEST_F(TypedefAnnoCheckerTest, TestWrongIntField) { auto scope = build_class_scope(stores); - build_cfg(scope); - auto* method = DexMethod::get_method( - "Lcom/facebook/redextest/" - "TypedefAnnoCheckerTest;.testWrongIntField:(I)V") - ->as_def(); - - StrDefConstants strdef_constants; - IntDefConstants intdef_constants; + auto method = DexMethod::get_method( + "Lcom/facebook/redextest/" + "TypedefAnnoCheckerTest;.testWrongIntField:(I)V") + ->as_def(); + + IRCode* code = method->get_code(); + code->build_cfg(); + ConcurrentMap> + strdef_constants; + ConcurrentMap> intdef_constants; TypedefAnnoChecker checker = TypedefAnnoChecker(strdef_constants, intdef_constants, get_config()); checker.run(method); + code->clear_cfg(); + EXPECT_FALSE(checker.complete()); EXPECT_EQ( checker.error(), @@ -214,61 +233,79 @@ TEST_F(TypedefAnnoCheckerTest, TestWrongIntField) { TEST_F(TypedefAnnoCheckerTest, TestStringField) { auto scope = build_class_scope(stores); - build_cfg(scope); - auto* method = DexMethod::get_method( - "Lcom/facebook/redextest/" - "TypedefAnnoCheckerTest;.testStringField:(Ljava/lang/" - "String;)V") - ->as_def(); - - StrDefConstants strdef_constants; - IntDefConstants intdef_constants; + auto method = DexMethod::get_method( + "Lcom/facebook/redextest/" + "TypedefAnnoCheckerTest;.testStringField:(Ljava/lang/" + "String;)V") + ->as_def(); + + IRCode* code = method->get_code(); + code->build_cfg(); + ConcurrentMap> + strdef_constants; + ConcurrentMap> intdef_constants; TypedefAnnoChecker checker = TypedefAnnoChecker(strdef_constants, intdef_constants, get_config()); checker.run(method); + code->clear_cfg(); + EXPECT_TRUE(checker.complete()); } TEST_F(TypedefAnnoCheckerTest, TestConstReturn) { auto scope = build_class_scope(stores); - build_cfg(scope); - auto* method = DexMethod::get_method( - "Lcom/facebook/redextest/" - "TypedefAnnoCheckerTest;.testConstReturn:()I") - ->as_def(); - - StrDefConstants strdef_constants; - IntDefConstants intdef_constants; + auto method = DexMethod::get_method( + "Lcom/facebook/redextest/" + "TypedefAnnoCheckerTest;.testConstReturn:()I") + ->as_def(); + + IRCode* code = method->get_code(); + code->build_cfg(); + auto& cfg = code->cfg(); + cfg.calculate_exit_block(); + + ConcurrentMap> + strdef_constants; + ConcurrentMap> intdef_constants; TypedefAnnoCheckerPass pass = TypedefAnnoCheckerPass(get_config()); - for (auto* cls : scope) { + for (auto cls : scope) { gather_typedef_values(pass, cls, strdef_constants, intdef_constants); } TypedefAnnoChecker checker = TypedefAnnoChecker(strdef_constants, intdef_constants, get_config()); checker.run(method); + code->clear_cfg(); + EXPECT_TRUE(checker.complete()); } TEST_F(TypedefAnnoCheckerTest, TestInvalidConstReturn) { auto scope = build_class_scope(stores); - build_cfg(scope); - auto* method = DexMethod::get_method( - "Lcom/facebook/redextest/" - "TypedefAnnoCheckerTest;.testInvalidConstReturn:()I") - ->as_def(); - - StrDefConstants strdef_constants; - IntDefConstants intdef_constants; + auto method = DexMethod::get_method( + "Lcom/facebook/redextest/" + "TypedefAnnoCheckerTest;.testInvalidConstReturn:()I") + ->as_def(); + + IRCode* code = method->get_code(); + code->build_cfg(); + auto& cfg = code->cfg(); + cfg.calculate_exit_block(); + + ConcurrentMap> + strdef_constants; + ConcurrentMap> intdef_constants; TypedefAnnoCheckerPass pass = TypedefAnnoCheckerPass(get_config()); - for (auto* cls : scope) { + for (auto cls : scope) { gather_typedef_values(pass, cls, strdef_constants, intdef_constants); } TypedefAnnoChecker checker = TypedefAnnoChecker(strdef_constants, intdef_constants, get_config()); checker.run(method); + code->clear_cfg(); + EXPECT_FALSE(checker.complete()); EXPECT_EQ( checker.error(), @@ -282,22 +319,29 @@ TEST_F(TypedefAnnoCheckerTest, TestInvalidConstReturn) { TEST_F(TypedefAnnoCheckerTest, TestInvalidConstReturn2) { auto scope = build_class_scope(stores); - build_cfg(scope); - auto* method = DexMethod::get_method( - "Lcom/facebook/redextest/" - "TypedefAnnoCheckerTest;.testInvalidConstReturn2:()I") - ->as_def(); - - StrDefConstants strdef_constants; - IntDefConstants intdef_constants; + auto method = DexMethod::get_method( + "Lcom/facebook/redextest/" + "TypedefAnnoCheckerTest;.testInvalidConstReturn2:()I") + ->as_def(); + + IRCode* code = method->get_code(); + code->build_cfg(); + auto& cfg = code->cfg(); + cfg.calculate_exit_block(); + + ConcurrentMap> + strdef_constants; + ConcurrentMap> intdef_constants; TypedefAnnoCheckerPass pass = TypedefAnnoCheckerPass(get_config()); - for (auto* cls : scope) { + for (auto cls : scope) { gather_typedef_values(pass, cls, strdef_constants, intdef_constants); } TypedefAnnoChecker checker = TypedefAnnoChecker(strdef_constants, intdef_constants, get_config()); checker.run(method); + code->clear_cfg(); + EXPECT_FALSE(checker.complete()); EXPECT_EQ( checker.error(), @@ -311,24 +355,31 @@ TEST_F(TypedefAnnoCheckerTest, TestInvalidConstReturn2) { TEST_F(TypedefAnnoCheckerTest, TestInvalidConstStrReturn) { auto scope = build_class_scope(stores); - build_cfg(scope); - auto* method = + auto method = DexMethod::get_method( "Lcom/facebook/redextest/" "TypedefAnnoCheckerTest;.testInvalidConstStrReturn:()Ljava/lang/" "String;") ->as_def(); - StrDefConstants strdef_constants; - IntDefConstants intdef_constants; + IRCode* code = method->get_code(); + code->build_cfg(); + auto& cfg = code->cfg(); + cfg.calculate_exit_block(); + + ConcurrentMap> + strdef_constants; + ConcurrentMap> intdef_constants; TypedefAnnoCheckerPass pass = TypedefAnnoCheckerPass(get_config()); - for (auto* cls : scope) { + for (auto cls : scope) { gather_typedef_values(pass, cls, strdef_constants, intdef_constants); } TypedefAnnoChecker checker = TypedefAnnoChecker(strdef_constants, intdef_constants, get_config()); checker.run(method); + code->clear_cfg(); + EXPECT_FALSE(checker.complete()); EXPECT_EQ( checker.error(), @@ -342,22 +393,29 @@ TEST_F(TypedefAnnoCheckerTest, TestInvalidConstStrReturn) { TEST_F(TypedefAnnoCheckerTest, TestInvalidConstInvokeStatic) { auto scope = build_class_scope(stores); - build_cfg(scope); - auto* method = DexMethod::get_method( - "Lcom/facebook/redextest/" - "TypedefAnnoCheckerTest;.testInvalidConstInvokeStatic:()I") - ->as_def(); - - StrDefConstants strdef_constants; - IntDefConstants intdef_constants; + auto method = DexMethod::get_method( + "Lcom/facebook/redextest/" + "TypedefAnnoCheckerTest;.testInvalidConstInvokeStatic:()I") + ->as_def(); + + IRCode* code = method->get_code(); + code->build_cfg(); + auto& cfg = code->cfg(); + cfg.calculate_exit_block(); + + ConcurrentMap> + strdef_constants; + ConcurrentMap> intdef_constants; TypedefAnnoCheckerPass pass = TypedefAnnoCheckerPass(get_config()); - for (auto* cls : scope) { + for (auto cls : scope) { gather_typedef_values(pass, cls, strdef_constants, intdef_constants); } TypedefAnnoChecker checker = TypedefAnnoChecker(strdef_constants, intdef_constants, get_config()); checker.run(method); + code->clear_cfg(); + EXPECT_FALSE(checker.complete()); EXPECT_EQ( checker.error(), @@ -372,23 +430,29 @@ TEST_F(TypedefAnnoCheckerTest, TestInvalidConstInvokeStatic) { TEST_F(TypedefAnnoCheckerTest, TestInvalidConstInvokeStatic2) { auto scope = build_class_scope(stores); - build_cfg(scope); - auto* method = - DexMethod::get_method( - "Lcom/facebook/redextest/" - "TypedefAnnoCheckerTest;.testInvalidConstInvokeStatic2:()I") - ->as_def(); + auto method = DexMethod::get_method( + "Lcom/facebook/redextest/" + "TypedefAnnoCheckerTest;.testInvalidConstInvokeStatic2:()I") + ->as_def(); + + IRCode* code = method->get_code(); + code->build_cfg(); + auto& cfg = code->cfg(); + cfg.calculate_exit_block(); - StrDefConstants strdef_constants; - IntDefConstants intdef_constants; + ConcurrentMap> + strdef_constants; + ConcurrentMap> intdef_constants; TypedefAnnoCheckerPass pass = TypedefAnnoCheckerPass(get_config()); - for (auto* cls : scope) { + for (auto cls : scope) { gather_typedef_values(pass, cls, strdef_constants, intdef_constants); } TypedefAnnoChecker checker = TypedefAnnoChecker(strdef_constants, intdef_constants, get_config()); checker.run(method); + code->clear_cfg(); + EXPECT_FALSE(checker.complete()); EXPECT_EQ( checker.error(), @@ -403,29 +467,33 @@ TEST_F(TypedefAnnoCheckerTest, TestInvalidConstInvokeStatic2) { TEST_F(TypedefAnnoCheckerTest, TestMultipleBlocksInt) { auto scope = build_class_scope(stores); - build_cfg(scope); - auto* method = DexMethod::get_method( - "Lcom/facebook/redextest/" - "TypedefAnnoCheckerTest;.testMultipleBlocksInt:(I)I") - ->as_def(); - - StrDefConstants strdef_constants; - IntDefConstants intdef_constants; + auto method = DexMethod::get_method( + "Lcom/facebook/redextest/" + "TypedefAnnoCheckerTest;.testMultipleBlocksInt:(I)I") + ->as_def(); + + IRCode* code = method->get_code(); + code->build_cfg(); + + ConcurrentMap> + strdef_constants; + ConcurrentMap> intdef_constants; TypedefAnnoCheckerPass pass = TypedefAnnoCheckerPass(get_config()); - for (auto* cls : scope) { + for (auto cls : scope) { gather_typedef_values(pass, cls, strdef_constants, intdef_constants); } TypedefAnnoChecker checker = TypedefAnnoChecker(strdef_constants, intdef_constants, get_config()); checker.run(method); + code->clear_cfg(); + EXPECT_TRUE(checker.complete()); } TEST_F(TypedefAnnoCheckerTest, TestMultipleBlocksString) { auto scope = build_class_scope(stores); - build_cfg(scope); - auto* method = + auto method = DexMethod::get_method( "Lcom/facebook/redextest/" "TypedefAnnoCheckerTest;.testMultipleBlocksString:(Ljava/lang/" @@ -433,40 +501,50 @@ TEST_F(TypedefAnnoCheckerTest, TestMultipleBlocksString) { "String;") ->as_def(); - StrDefConstants strdef_constants; - IntDefConstants intdef_constants; + IRCode* code = method->get_code(); + code->build_cfg(); + + ConcurrentMap> + strdef_constants; + ConcurrentMap> intdef_constants; TypedefAnnoCheckerPass pass = TypedefAnnoCheckerPass(get_config()); - for (auto* cls : scope) { + for (auto cls : scope) { gather_typedef_values(pass, cls, strdef_constants, intdef_constants); } TypedefAnnoChecker checker = TypedefAnnoChecker(strdef_constants, intdef_constants, get_config()); checker.run(method); + code->clear_cfg(); + EXPECT_TRUE(checker.complete()); } TEST_F(TypedefAnnoCheckerTest, TestInvalidMultipleBlocksString) { auto scope = build_class_scope(stores); - build_cfg(scope); - auto* method = + auto method = DexMethod::get_method( "Lcom/facebook/redextest/" "TypedefAnnoCheckerTest;.testInvalidMultipleBlocksString:(Ljava/lang/" "String;)Ljava/lang/" "String;") ->as_def(); + IRCode* code = method->get_code(); + code->build_cfg(); - StrDefConstants strdef_constants; - IntDefConstants intdef_constants; + ConcurrentMap> + strdef_constants; + ConcurrentMap> intdef_constants; TypedefAnnoCheckerPass pass = TypedefAnnoCheckerPass(get_config()); - for (auto* cls : scope) { + for (auto cls : scope) { gather_typedef_values(pass, cls, strdef_constants, intdef_constants); } TypedefAnnoChecker checker = TypedefAnnoChecker(strdef_constants, intdef_constants, get_config()); checker.run(method); + code->clear_cfg(); + EXPECT_FALSE(checker.complete()); EXPECT_EQ( checker.error(), @@ -479,22 +557,26 @@ TEST_F(TypedefAnnoCheckerTest, TestInvalidMultipleBlocksString) { TEST_F(TypedefAnnoCheckerTest, TestNonConstInt) { auto scope = build_class_scope(stores); - build_cfg(scope); - auto* method = DexMethod::get_method( - "Lcom/facebook/redextest/" - "TypedefAnnoCheckerTest;.testNonConstInt:(I)I") - ->as_def(); - - StrDefConstants strdef_constants; - IntDefConstants intdef_constants; + auto method = DexMethod::get_method( + "Lcom/facebook/redextest/" + "TypedefAnnoCheckerTest;.testNonConstInt:(I)I") + ->as_def(); + IRCode* code = method->get_code(); + code->build_cfg(); + + ConcurrentMap> + strdef_constants; + ConcurrentMap> intdef_constants; TypedefAnnoCheckerPass pass = TypedefAnnoCheckerPass(get_config()); - for (auto* cls : scope) { + for (auto cls : scope) { gather_typedef_values(pass, cls, strdef_constants, intdef_constants); } TypedefAnnoChecker checker = TypedefAnnoChecker(strdef_constants, intdef_constants, get_config()); checker.run(method); + code->clear_cfg(); + EXPECT_FALSE(checker.complete()); EXPECT_EQ( checker.error(), @@ -507,18 +589,22 @@ TEST_F(TypedefAnnoCheckerTest, TestNonConstInt) { TEST_F(TypedefAnnoCheckerTest, TestInvalidType) { auto scope = build_class_scope(stores); - build_cfg(scope); - auto* method = DexMethod::get_method( - "Lcom/facebook/redextest/" - "TypedefAnnoCheckerTest;.testInvalidType:(Lcom/facebook/" - "redextest/I;)Lcom/facebook/redextest/I;") - ->as_def(); - - StrDefConstants strdef_constants; - IntDefConstants intdef_constants; + auto method = DexMethod::get_method( + "Lcom/facebook/redextest/" + "TypedefAnnoCheckerTest;.testInvalidType:(Lcom/facebook/" + "redextest/I;)Lcom/facebook/redextest/I;") + ->as_def(); + IRCode* code = method->get_code(); + code->build_cfg(); + + ConcurrentMap> + strdef_constants; + ConcurrentMap> intdef_constants; TypedefAnnoChecker checker = TypedefAnnoChecker(strdef_constants, intdef_constants, get_config()); checker.run(method); + code->clear_cfg(); + EXPECT_FALSE(checker.complete()); EXPECT_EQ(checker.error(), "TypedefAnnoCheckerPass: the annotation Linteg/TestIntDef;\n\ @@ -529,24 +615,29 @@ TEST_F(TypedefAnnoCheckerTest, TestInvalidType) { TEST_F(TypedefAnnoCheckerTest, TestJoiningTwoAnnotations) { auto scope = build_class_scope(stores); - build_cfg(scope); - auto* method = + auto method = DexMethod::get_method( "Lcom/facebook/redextest/" "TypedefAnnoCheckerTest;.testJoiningTwoAnnotations:(Ljava/lang/" "String;Ljava/lang/String;)Ljava/lang/String;") ->as_def(); - StrDefConstants strdef_constants; - IntDefConstants intdef_constants; + IRCode* code = method->get_code(); + code->build_cfg(); + + ConcurrentMap> + strdef_constants; + ConcurrentMap> intdef_constants; TypedefAnnoCheckerPass pass = TypedefAnnoCheckerPass(get_config()); - for (auto* cls : scope) { + for (auto cls : scope) { gather_typedef_values(pass, cls, strdef_constants, intdef_constants); } TypedefAnnoChecker checker = TypedefAnnoChecker(strdef_constants, intdef_constants, get_config()); checker.run(method); + code->clear_cfg(); + EXPECT_FALSE(checker.complete()); EXPECT_EQ( checker.error(), @@ -559,142 +650,172 @@ TEST_F(TypedefAnnoCheckerTest, TestJoiningTwoAnnotations) { TEST_F(TypedefAnnoCheckerTest, TestJoiningTwoAnnotations2) { auto scope = build_class_scope(stores); - build_cfg(scope); - auto* method = + auto method = DexMethod::get_method( "Lcom/facebook/redextest/" "TypedefAnnoCheckerTest;.testJoiningTwoAnnotations2:(Ljava/lang/" "String;Ljava/lang/String;)Ljava/lang/String;") ->as_def(); - StrDefConstants strdef_constants; - IntDefConstants intdef_constants; + IRCode* code = method->get_code(); + code->build_cfg(); + + ConcurrentMap> + strdef_constants; + ConcurrentMap> intdef_constants; TypedefAnnoCheckerPass pass = TypedefAnnoCheckerPass(get_config()); - for (auto* cls : scope) { + for (auto cls : scope) { gather_typedef_values(pass, cls, strdef_constants, intdef_constants); } TypedefAnnoChecker checker = TypedefAnnoChecker(strdef_constants, intdef_constants, get_config()); checker.run(method); + code->clear_cfg(); + EXPECT_TRUE(checker.complete()); } TEST_F(TypedefAnnoCheckerTest, TestReassigningInt) { auto scope = build_class_scope(stores); - build_cfg(scope); - auto* method = DexMethod::get_method( - "Lcom/facebook/redextest/" - "TypedefAnnoCheckerTest;.testReassigningInt:(II)I") - ->as_def(); - - StrDefConstants strdef_constants; - IntDefConstants intdef_constants; + auto method = DexMethod::get_method( + "Lcom/facebook/redextest/" + "TypedefAnnoCheckerTest;.testReassigningInt:(II)I") + ->as_def(); + + IRCode* code = method->get_code(); + code->build_cfg(); + + ConcurrentMap> + strdef_constants; + ConcurrentMap> intdef_constants; TypedefAnnoCheckerPass pass = TypedefAnnoCheckerPass(get_config()); - for (auto* cls : scope) { + for (auto cls : scope) { gather_typedef_values(pass, cls, strdef_constants, intdef_constants); } TypedefAnnoChecker checker = TypedefAnnoChecker(strdef_constants, intdef_constants, get_config()); checker.run(method); + code->clear_cfg(); + EXPECT_TRUE(checker.complete()); } TEST_F(TypedefAnnoCheckerTest, TestIfElse) { auto scope = build_class_scope(stores); - build_cfg(scope); - auto* method = DexMethod::get_method( - "Lcom/facebook/redextest/" - "TypedefAnnoCheckerTest;.testIfElse:()I") - ->as_def(); - - StrDefConstants strdef_constants; - IntDefConstants intdef_constants; + auto method = DexMethod::get_method( + "Lcom/facebook/redextest/" + "TypedefAnnoCheckerTest;.testIfElse:()I") + ->as_def(); + + IRCode* code = method->get_code(); + code->build_cfg(); + + ConcurrentMap> + strdef_constants; + ConcurrentMap> intdef_constants; TypedefAnnoCheckerPass pass = TypedefAnnoCheckerPass(get_config()); - for (auto* cls : scope) { + for (auto cls : scope) { gather_typedef_values(pass, cls, strdef_constants, intdef_constants); } TypedefAnnoChecker checker = TypedefAnnoChecker(strdef_constants, intdef_constants, get_config()); checker.run(method); + code->clear_cfg(); + EXPECT_TRUE(checker.complete()); } TEST_F(TypedefAnnoCheckerTest, TestIfElseParam) { auto scope = build_class_scope(stores); - build_cfg(scope); - auto* method = DexMethod::get_method( - "Lcom/facebook/redextest/" - "TypedefAnnoCheckerTest;.testIfElseParam:(Z)I") - ->as_def(); - - StrDefConstants strdef_constants; - IntDefConstants intdef_constants; + auto method = DexMethod::get_method( + "Lcom/facebook/redextest/" + "TypedefAnnoCheckerTest;.testIfElseParam:(Z)I") + ->as_def(); + + IRCode* code = method->get_code(); + code->build_cfg(); + + ConcurrentMap> + strdef_constants; + ConcurrentMap> intdef_constants; TypedefAnnoCheckerPass pass = TypedefAnnoCheckerPass(get_config()); - for (auto* cls : scope) { + for (auto cls : scope) { gather_typedef_values(pass, cls, strdef_constants, intdef_constants); } TypedefAnnoChecker checker = TypedefAnnoChecker(strdef_constants, intdef_constants, get_config()); checker.run(method); + code->clear_cfg(); + EXPECT_TRUE(checker.complete()); } TEST_F(TypedefAnnoCheckerTest, TestIfElseString) { auto scope = build_class_scope(stores); - build_cfg(scope); - auto* method = DexMethod::get_method( - "Lcom/facebook/redextest/" - "TypedefAnnoCheckerTest;.testIfElseString:(Z)Ljava/lang/" - "String;") - ->as_def(); - - StrDefConstants strdef_constants; - IntDefConstants intdef_constants; + auto method = DexMethod::get_method( + "Lcom/facebook/redextest/" + "TypedefAnnoCheckerTest;.testIfElseString:(Z)Ljava/lang/" + "String;") + ->as_def(); + + IRCode* code = method->get_code(); + code->build_cfg(); + + ConcurrentMap> + strdef_constants; + ConcurrentMap> intdef_constants; TypedefAnnoCheckerPass pass = TypedefAnnoCheckerPass(get_config()); - for (auto* cls : scope) { + for (auto cls : scope) { gather_typedef_values(pass, cls, strdef_constants, intdef_constants); } TypedefAnnoChecker checker = TypedefAnnoChecker(strdef_constants, intdef_constants, get_config()); checker.run(method); + code->clear_cfg(); + EXPECT_TRUE(checker.complete()); } TEST_F(TypedefAnnoCheckerTest, TestXORIfElse) { auto scope = build_class_scope(stores); - build_cfg(scope); - auto* method = DexMethod::get_method( - "Lcom/facebook/redextest/" - "TypedefAnnoCheckerTest;.testXORIfElse:(Z)I") - ->as_def(); - - StrDefConstants strdef_constants; - IntDefConstants intdef_constants; + auto method = DexMethod::get_method( + "Lcom/facebook/redextest/" + "TypedefAnnoCheckerTest;.testXORIfElse:(Z)I") + ->as_def(); + + IRCode* code = method->get_code(); + code->build_cfg(); + + ConcurrentMap> + strdef_constants; + ConcurrentMap> intdef_constants; TypedefAnnoCheckerPass pass = TypedefAnnoCheckerPass(get_config()); - for (auto* cls : scope) { + for (auto cls : scope) { gather_typedef_values(pass, cls, strdef_constants, intdef_constants); } TypedefAnnoChecker checker = TypedefAnnoChecker(strdef_constants, intdef_constants, get_config()); checker.run(method); + code->clear_cfg(); + EXPECT_TRUE(checker.complete()); } TEST_F(TypedefAnnoCheckerTest, TestXORIfElseZero) { auto scope = build_class_scope(stores); - build_cfg(scope); - auto* method = DexMethod::get_method( - "Lcom/facebook/redextest/" - "TypedefAnnoCheckerTest;.testXORIfElseZero:()I") - ->as_def(); + auto method = DexMethod::get_method( + "Lcom/facebook/redextest/" + "TypedefAnnoCheckerTest;.testXORIfElseZero:()I") + ->as_def(); IRCode* code = method->get_code(); + code->build_cfg(); auto& cfg = code->cfg(); type_inference::TypeInference inference(cfg); @@ -711,250 +832,18 @@ TEST_F(TypedefAnnoCheckerTest, TestXORIfElseZero) { inference.analyze_instruction(insn, &env); } - StrDefConstants strdef_constants; - IntDefConstants intdef_constants; + ConcurrentMap> + strdef_constants; + ConcurrentMap> intdef_constants; TypedefAnnoCheckerPass pass = TypedefAnnoCheckerPass(get_config()); - for (auto* cls : scope) { + for (auto cls : scope) { gather_typedef_values(pass, cls, strdef_constants, intdef_constants); } TypedefAnnoChecker checker = TypedefAnnoChecker(strdef_constants, intdef_constants, get_config()); checker.run(method); - EXPECT_TRUE(checker.complete()); -} - -TEST_F(TypedefAnnoCheckerTest, testSynthAccessor) { - auto scope = build_class_scope(stores); - build_cfg(scope); - auto* accessor = DexMethod::get_method( - "Lcom/facebook/redextest/" - "TypedefAnnoCheckerKtTest;.access$takesStrConst:(Lcom/" - "facebook/redextest/TypedefAnnoCheckerKtTest;Ljava/lang/" - "String;)Ljava/lang/String;") - ->as_def(); - EXPECT_TRUE(accessor != nullptr); - - StrDefConstants strdef_constants; - IntDefConstants intdef_constants; - TypedefAnnoCheckerPass pass = TypedefAnnoCheckerPass(get_config()); - for (auto* cls : scope) { - gather_typedef_values(pass, cls, strdef_constants, intdef_constants); - } - - auto config = get_config(); - TypedefAnnoChecker checker = - TypedefAnnoChecker(strdef_constants, intdef_constants, config); - checker.run(accessor); - - // Without patching the accessor, the checker will fail. - EXPECT_FALSE(checker.complete()); - EXPECT_EQ( - checker.error(), - "TypedefAnnoCheckerPass: in method Lcom/facebook/redextest/TypedefAnnoCheckerKtTest;.access$takesStrConst:(Lcom/facebook/redextest/TypedefAnnoCheckerKtTest;Ljava/lang/String;)Ljava/lang/String;\n\ - one of the parameters needs to have the typedef annotation Linteg/TestStringDef;\n\ - attached to it. Check that the value is annotated and exists in the typedef annotation class.\n\ - failed instruction: IOPCODE_LOAD_PARAM_OBJECT v2\n\ - Error invoking Lcom/facebook/redextest/TypedefAnnoCheckerKtTest;.takesStrConst:(Ljava/lang/String;)Ljava/lang/String;\n\ - Incorrect parameter's index: 1\n\n"); + code->clear_cfg(); - SynthAccessorPatcher patcher(config); - patcher.run(scope); - - TypedefAnnoChecker checker2 = - TypedefAnnoChecker(strdef_constants, intdef_constants, config); - checker2.run(accessor); - // After patching the accessor, the checker should succeed. - EXPECT_TRUE(checker2.complete()); - - auto* accessor_caller = - DexMethod::get_method( - "Lcom/facebook/redextest/" - "TypedefAnnoCheckerKtTest$testSynthAccessor$lmd$1;" - ".invoke:()Ljava/lang/String;") - ->as_def(); - EXPECT_TRUE(accessor_caller != nullptr); - - TypedefAnnoChecker checker3 = - TypedefAnnoChecker(strdef_constants, intdef_constants, config); - checker3.run(accessor_caller); - // The caller of the accessor has the actual violation. - EXPECT_FALSE(checker3.complete()); - EXPECT_EQ( - checker3.error(), - "TypedefAnnoCheckerPass: in method Lcom/facebook/redextest/TypedefAnnoCheckerKtTest$testSynthAccessor$lmd$1;.invoke:()Ljava/lang/String;\n\ - the string value liu does not have the typedef annotation \n\ - Linteg/TestStringDef; attached to it. \n\ - Check that the value is annotated and exists in the typedef annotation class.\n\ - failed instruction: CONST_STRING \"liu\"\n\ - Error invoking Lcom/facebook/redextest/TypedefAnnoCheckerKtTest;.access$takesStrConst:(Lcom/facebook/redextest/TypedefAnnoCheckerKtTest;Ljava/lang/String;)Ljava/lang/String;\n\ - Incorrect parameter's index: 1\n\n"); -} - -TEST_F(TypedefAnnoCheckerTest, testDefaultArg) { - // Dex code example: https://fburl.com/dexbolt/o35r4sgv - auto scope = build_class_scope(stores); - build_cfg(scope); - auto* wrong_default_arg = - DexMethod::get_method( - "Lcom/facebook/redextest/" - "TypedefAnnoCheckerKtTest;.wrongDefaultArg$default:(Lcom/" - "facebook/redextest/TypedefAnnoCheckerKtTest;Ljava/lang/" - "String;ILjava/lang/Object;)Ljava/lang/String;") - ->as_def(); - EXPECT_TRUE(wrong_default_arg != nullptr); - - auto* wrong_default_caller = - DexMethod::get_method( - "Lcom/facebook/redextest/" - "TypedefAnnoCheckerKtTest;.wrongDefaultCaller:(Ljava/lang/" - "String;)V") - ->as_def(); - EXPECT_TRUE(wrong_default_caller != nullptr); - - auto* right_default_arg = - DexMethod::get_method( - "Lcom/facebook/redextest/" - "TypedefAnnoCheckerKtTest;.rightDefaultArg$default:(Lcom/" - "facebook/redextest/TypedefAnnoCheckerKtTest;Ljava/lang/" - "String;ILjava/lang/Object;)Ljava/lang/String;") - ->as_def(); - EXPECT_TRUE(right_default_arg != nullptr); - - auto* right_default_caller = - DexMethod::get_method( - "Lcom/facebook/redextest/" - "TypedefAnnoCheckerKtTest;.rightDefaultCaller:(Ljava/lang/" - "String;)V") - ->as_def(); - EXPECT_TRUE(right_default_caller != nullptr); - - StrDefConstants strdef_constants; - IntDefConstants intdef_constants; - TypedefAnnoCheckerPass pass = TypedefAnnoCheckerPass(get_config()); - for (auto* cls : scope) { - gather_typedef_values(pass, cls, strdef_constants, intdef_constants); - } - - auto config = get_config(); - TypedefAnnoChecker checker = - TypedefAnnoChecker(strdef_constants, intdef_constants, config); - checker.run(wrong_default_arg); - // Without patching the accessor, the checker will fail. - // The default arg is not safe value. The param is not annotated. - // We don't check the detailed error msg, since multiple errors are possible. - EXPECT_FALSE(checker.complete()); - - TypedefAnnoChecker checker1 = - TypedefAnnoChecker(strdef_constants, intdef_constants, config); - checker1.run(wrong_default_caller); - EXPECT_TRUE(checker1.complete()); - - TypedefAnnoChecker checker2 = - TypedefAnnoChecker(strdef_constants, intdef_constants, config); - checker2.run(right_default_arg); - // Without patching the accessor, the checker will fail. - // The default arg is a safe value, but the param is not annotated. - EXPECT_FALSE(checker2.complete()); - EXPECT_EQ( - checker2.error(), - "TypedefAnnoCheckerPass: in method Lcom/facebook/redextest/TypedefAnnoCheckerKtTest;.rightDefaultArg$default:(Lcom/facebook/redextest/TypedefAnnoCheckerKtTest;Ljava/lang/String;ILjava/lang/Object;)Ljava/lang/String;\n\ - one of the parameters needs to have the typedef annotation Linteg/TestStringDef;\n\ - attached to it. Check that the value is annotated and exists in the typedef annotation class.\n\ - failed instruction: IOPCODE_LOAD_PARAM_OBJECT v1\n\ - Error invoking Lcom/facebook/redextest/TypedefAnnoCheckerKtTest;.rightDefaultArg:(Ljava/lang/String;)Ljava/lang/String;\n\ - Incorrect parameter's index: 1\n\n"); - - TypedefAnnoChecker checker3 = - TypedefAnnoChecker(strdef_constants, intdef_constants, config); - checker3.run(right_default_caller); - EXPECT_TRUE(checker3.complete()); - - /////////////////////////////////////////////////// - // Patch the default synth stub param !!! - /////////////////////////////////////////////////// - SynthAccessorPatcher patcher(config); - patcher.run(scope); - - TypedefAnnoChecker checker4 = - TypedefAnnoChecker(strdef_constants, intdef_constants, config); - checker4.run(wrong_default_arg); - // After patching the accessor, the param annotation is patched. But the wrong - // constant error remains. - EXPECT_FALSE(checker4.complete()); - EXPECT_EQ( - checker4.error(), - "TypedefAnnoCheckerPass: in method Lcom/facebook/redextest/TypedefAnnoCheckerKtTest;.wrongDefaultArg$default:(Lcom/facebook/redextest/TypedefAnnoCheckerKtTest;Ljava/lang/String;ILjava/lang/Object;)Ljava/lang/String;\n\ - the string value default does not have the typedef annotation \n\ - Linteg/TestStringDef; attached to it. \n\ - Check that the value is annotated and exists in the typedef annotation class.\n\ - failed instruction: CONST_STRING \"default\"\n\ - Error invoking Lcom/facebook/redextest/TypedefAnnoCheckerKtTest;.wrongDefaultArg:(Ljava/lang/String;)Ljava/lang/String;\n\ - Incorrect parameter's index: 1\n\n"); - - TypedefAnnoChecker checker5 = - TypedefAnnoChecker(strdef_constants, intdef_constants, config); - checker5.run(wrong_default_caller); - EXPECT_TRUE(checker5.complete()); - - TypedefAnnoChecker checker6 = - TypedefAnnoChecker(strdef_constants, intdef_constants, config); - checker6.run(right_default_arg); - // After patching the accessor, the param annotation is patched. The default - // arg is correct. - EXPECT_TRUE(checker6.complete()); - - TypedefAnnoChecker checker7 = - TypedefAnnoChecker(strdef_constants, intdef_constants, config); - checker7.run(right_default_caller); - EXPECT_TRUE(checker7.complete()); -} - -TEST_F(TypedefAnnoCheckerTest, testAssignNullToString) { - auto scope = build_class_scope(stores); - build_cfg(scope); - auto* method = - DexMethod::get_method( - "Lcom/facebook/redextest/" - "TypedefAnnoCheckerTest;.testAssignNullToString:()Ljava/lang/String;") - ->as_def(); - - StrDefConstants strdef_constants; - IntDefConstants intdef_constants; - TypedefAnnoCheckerPass pass = TypedefAnnoCheckerPass(get_config()); - for (auto* cls : scope) { - gather_typedef_values(pass, cls, strdef_constants, intdef_constants); - } - - TypedefAnnoChecker checker = - TypedefAnnoChecker(strdef_constants, intdef_constants, get_config()); - checker.run(method); EXPECT_TRUE(checker.complete()); } - -TEST_F(TypedefAnnoCheckerTest, TestNoAnnoField) { - auto scope = build_class_scope(stores); - build_cfg(scope); - auto* method = DexMethod::get_method( - "Lcom/facebook/redextest/" - "TypedefAnnoCheckerTest;.testNoAnnoField:()I") - ->as_def(); - - StrDefConstants strdef_constants; - IntDefConstants intdef_constants; - TypedefAnnoCheckerPass pass = TypedefAnnoCheckerPass(get_config()); - for (auto* cls : scope) { - gather_typedef_values(pass, cls, strdef_constants, intdef_constants); - } - - TypedefAnnoChecker checker = - TypedefAnnoChecker(strdef_constants, intdef_constants, get_config()); - checker.run(method); - EXPECT_FALSE(checker.complete()); - EXPECT_EQ( - checker.error(), - "TypedefAnnoCheckerPass: in method Lcom/facebook/redextest/TypedefAnnoCheckerTest;.testNoAnnoField:()I\n\ - the field no_anno_field\n\ - needs to have the annotation Linteg/TestIntDef;.\n\ - failed instruction: IGET v1, Lcom/facebook/redextest/TypedefAnnoCheckerTest;.no_anno_field:I\n"); -} diff --git a/test/integ/TypedefAnnoCheckerTest.java b/test/integ/TypedefAnnoCheckerTest.java index 1963c5cb2f..28ecf6257f 100644 --- a/test/integ/TypedefAnnoCheckerTest.java +++ b/test/integ/TypedefAnnoCheckerTest.java @@ -21,7 +21,6 @@ public class TypedefAnnoCheckerTest { @TestIntDef int int_field; @TestStringDef int wrong_anno_field; @TestStringDef String str_field; - int no_anno_field = 6; void testIntField(@TestIntDef int val) { int_field = val; @@ -35,10 +34,6 @@ void testStringField(@TestStringDef String val) { str_field = val; } - @TestIntDef int testNoAnnoField() { - return testValidIntAnnoReturn(no_anno_field); - } - static @NotSafeAnno @TestIntDef int testValidIntAnnoReturn(@NotSafeAnno @TestIntDef int val) { return val; } @@ -186,8 +181,4 @@ void testStringField(@TestStringDef String val) { int res = flag ? TestIntDef.ZERO : TestIntDef.ONE; return res; } - - static @TestStringDef String testAssignNullToString() { - return null; - } } diff --git a/test/integ/TypedefAnnoCheckerTest.kt b/test/integ/TypedefAnnoCheckerTest.kt deleted file mode 100644 index 6f353531a5..0000000000 --- a/test/integ/TypedefAnnoCheckerTest.kt +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * - * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. - */ - -package com.facebook.redextest - -import integ.TestStringDef - -public class TypedefAnnoCheckerKtTest { - - fun testSynthAccessor() { - val lmd: () -> String = { takesStrConst("liu") } - lmd() - } - - private fun takesStrConst(@TestStringDef str: String): String { - return str - } - - fun wrongDefaultCaller(@TestStringDef arg: String) { - wrongDefaultArg(arg) - wrongDefaultArg() - } - - private fun wrongDefaultArg(@TestStringDef str: String = "default"): String { - return str - } - - fun rightDefaultCaller(@TestStringDef arg: String) { - rightDefaultArg(arg) - rightDefaultArg() - } - - private fun rightDefaultArg(@TestStringDef str: String = "one"): String { - return str - } -} diff --git a/test/integ/TypedefAnnosTest.cpp b/test/integ/TypedefAnnosTest.cpp index 53f6b82702..bff59210af 100644 --- a/test/integ/TypedefAnnosTest.cpp +++ b/test/integ/TypedefAnnosTest.cpp @@ -181,9 +181,9 @@ TEST_F(TypedefAnnosTest, test_int_field) { DexType::get_type("Linteg/TestIntDef;")); auto domain = env.get_type_domain(insn->src(1)); auto dex_type = DexType::make_type("I"); - EXPECT_TRUE(domain.is_nullable()); + EXPECT_EQ(domain.get<0>(), ConstNullnessDomain(Nullness::NOT_NULL)); EXPECT_EQ(domain.get<1>(), SingletonDexTypeDomain(dex_type)); - EXPECT_TRUE(domain.get<2>().is_top()); + EXPECT_EQ(domain.get<2>(), SmallSetDexTypeDomain(dex_type)); } } } @@ -210,9 +210,9 @@ TEST_F(TypedefAnnosTest, test_str_field) { DexType::make_type("Lcom/facebook/redextest/TypedefAnnosTest;"); EXPECT_EQ(*(env.get_annotation(insn->src(1))), DexType::get_type("Linteg/TestStringDef;")); - EXPECT_TRUE(domain.is_nullable()); + EXPECT_EQ(domain.get<0>(), ConstNullnessDomain(Nullness::NOT_NULL)); EXPECT_EQ(domain.get<1>(), SingletonDexTypeDomain(dex_type)); - EXPECT_TRUE(domain.get<2>().is_top()); + EXPECT_EQ(domain.get<2>(), SmallSetDexTypeDomain(dex_type)); } } } diff --git a/test/integ/VirtualTargetsReachabilityTest.cpp b/test/integ/VirtualTargetsReachabilityTest.cpp index abdc828e3d..1c83bf0e7c 100644 --- a/test/integ/VirtualTargetsReachabilityTest.cpp +++ b/test/integ/VirtualTargetsReachabilityTest.cpp @@ -32,11 +32,9 @@ TEST_F(VirtualTargetsReachabilityTest, invoke_super_subtlety) { reachability::ReachableAspects reachable_aspects; auto scope = build_class_scope(stores); walk::parallel::code(scope, [&](auto*, auto& code) { code.build_cfg(); }); - auto method_override_graph = method_override_graph::build_graph(scope); auto reachable_objects = reachability::compute_reachable_objects( - scope, *method_override_graph, ig_sets, &num_ignore_check_strings, - &reachable_aspects, false, + stores, ig_sets, &num_ignore_check_strings, &reachable_aspects, false, /* relaxed_keep_class_members */ true, /* relaxed_keep_interfaces */ false, /* cfg_gathering_check_instantiable */ true); diff --git a/test/integ/arsc_tool_test.sh b/test/integ/arsc_tool_test.sh deleted file mode 100755 index be3febf7bd..0000000000 --- a/test/integ/arsc_tool_test.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -set -e - -# Expects the executable and an .apk file, which will have its .arsc file -# extracted and inspected. This test is supposed to be a very minimal "does the -# tool compile and spew something" check, as a signal for automated deployments. -# The APIs are unit tested separately, with specially crafted input data. -if [ "$#" -le 1 ]; then - echo "arsc_attribution binary, target apk required" - exit 1 -fi -ATTRIBUTION_BIN=$1 -APK=$2 - -TMP_DIR=$(mktemp -d) -trap 'rm -r $TMP_DIR' EXIT - -unzip "$APK" resources.arsc -d "$TMP_DIR" -ARSC_FILE="$TMP_DIR/resources.arsc" -CSV_FILE="$TMP_DIR/out.csv" - -echo "Running binary" -pushd "$TMP_DIR" -"$ATTRIBUTION_BIN" --file "$ARSC_FILE" > out.csv -popd - -# Just check that some resource ID was spit out. -grep -q "^0x7f010000.*default$" "$CSV_FILE" diff --git a/test/pg-config-e2e/ProguardTest.cpp b/test/pg-config-e2e/ProguardTest.cpp index 44ef17e521..773e381fb7 100644 --- a/test/pg-config-e2e/ProguardTest.cpp +++ b/test/pg-config-e2e/ProguardTest.cpp @@ -137,7 +137,20 @@ TEST_F(ProguardTest, assortment) { keep_rules::proguard_parser::parse_file(configuraiton_file, &pg_config); EXPECT_TRUE(pg_config.ok); - std::string sdk_jar = android_sdk_jar_path(); + auto android_env_sdk = std::getenv("ANDROID_SDK"); + auto android_config_sdk = std::getenv("sdk_path"); + + auto android_sdk = (strncmp(android_config_sdk, "None", 4) != 0) + ? android_config_sdk + : android_env_sdk; + + ASSERT_NE(nullptr, android_sdk); + auto android_target = std::getenv("android_target"); + ASSERT_NE(nullptr, android_target); + std::string android_version(android_target); + ASSERT_NE("NotFound", android_version); + std::string sdk_jar = std::string(android_sdk) + "/platforms/" + + android_version + "/android.jar"; Scope external_classes; EXPECT_TRUE(load_jar_file(DexLocation::make_location("", sdk_jar), &external_classes)); diff --git a/test/unit/AnalysisUsageTest.cpp b/test/unit/AnalysisUsageTest.cpp index b66cd74dd5..9f95a9ed0f 100644 --- a/test/unit/AnalysisUsageTest.cpp +++ b/test/unit/AnalysisUsageTest.cpp @@ -41,7 +41,9 @@ class MyAnalysisPass : public Pass { const override { using namespace redex_properties::interactions; using namespace redex_properties::names; - return {}; + return { + {HasSourceBlocks, Preserves}, + }; } void run_pass(DexStoresVector& /* stores */, @@ -70,7 +72,9 @@ class MyAnalysisPass2 : public Pass { const override { using namespace redex_properties::interactions; using namespace redex_properties::names; - return {}; + return { + {HasSourceBlocks, Preserves}, + }; } void run_pass(DexStoresVector& /* stores */, @@ -98,7 +102,9 @@ class ConsumeAnalysisAndInvalidatePass : public Pass { const override { using namespace redex_properties::interactions; using namespace redex_properties::names; - return {}; + return { + {HasSourceBlocks, Preserves}, + }; } void set_analysis_usage(AnalysisUsage& au) const override { @@ -123,7 +129,9 @@ class ConsumeAnalysisAndPreservePass : public Pass { const override { using namespace redex_properties::interactions; using namespace redex_properties::names; - return {}; + return { + {HasSourceBlocks, Preserves}, + }; } void set_analysis_usage(AnalysisUsage& au) const override { @@ -150,7 +158,9 @@ class ConsumeAnalysisAndPreserveOnePass : public Pass { const override { using namespace redex_properties::interactions; using namespace redex_properties::names; - return {}; + return { + {HasSourceBlocks, Preserves}, + }; } void set_analysis_usage(AnalysisUsage& au) const override { @@ -176,7 +186,9 @@ class ConsumeAnalysis2Pass : public Pass { const override { using namespace redex_properties::interactions; using namespace redex_properties::names; - return {}; + return { + {HasSourceBlocks, Preserves}, + }; } void set_analysis_usage(AnalysisUsage& au) const override { diff --git a/test/unit/ArscAttributionTest.cpp b/test/unit/ArscAttributionTest.cpp deleted file mode 100644 index 20ed9217e2..0000000000 --- a/test/unit/ArscAttributionTest.cpp +++ /dev/null @@ -1,331 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * - * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. - */ - -#include -#include - -#include "ArscStats.h" -#include "Debug.h" -#include "RedexTestUtils.h" -#include "Util.h" -#include "androidfw/ResourceTypes.h" -#include "arsc/TestStructures.h" -#include "utils/Errors.h" -#include "utils/Serialize.h" - -namespace { -constexpr uint32_t UTF8_POOL = android::ResStringPool_header::UTF8_FLAG; -constexpr uint32_t UTF16_POOL = 0; -// The following accounts for null zero, utf-8 size (one byte) and utf-16 -// size (one byte) for strings with small lengths. -constexpr size_t SIZE_AND_NULL_ZERO = 3; -constexpr size_t OFFSET_SIZE = sizeof(uint32_t); -attribution::ResourceNames no_names; - -// Makes a string pool and calls the API method to count padding bytes. -size_t count_padding(const std::vector& items, - uint32_t pool_flags) { - arsc::ResStringPoolBuilder builder(pool_flags); - for (const auto& s : items) { - builder.add_string(s); - } - android::Vector pool_data; - builder.serialize(&pool_data); - auto pool_header = (android::ResStringPool_header*)pool_data.array(); - android::ResStringPool pool; - pool.setTo(pool_header, pool_data.size(), true); - return attribution::count_padding(pool_header, pool); -} -} // namespace - -// clang-format off -// This test case builds up an .arsc file that should be counted like this: -// -// ID | Type | Name | Private Size | Shared Size | Proportional Size | Config Count | Configs -// 0x7f010000 | dimen | yolo | 148 | 0 | 247.33 | 2 | default land -// 0x7f010001 | dimen | second | 37 | 0 | 136.33 | 1 | default -// 0x7f010002 | dimen | third | 36 | 0 | 135.33 | 1 | default -// 0x7f020000 | style | fourth | 179 | 0 | 239 | 1 | xxhdpi -// 0x7f030000 | xml | fifth | 52 | 0 | 184.50 | 1 | default -// 0x7f030001 | xml | sixth | 53 | 0 | 185.50 | 1 | default -// -// clang-format on -TEST(Arsc, BuildFileForAttribution) { - auto global_strings_builder = - std::make_shared(UTF8_POOL); - global_strings_builder->add_string("res/a.xml"); - global_strings_builder->add_string("res/bb.xml"); - - auto key_strings_builder = - std::make_shared(UTF8_POOL); - std::vector entry_names{"first", "second", "third", - "fourth", "fifth", "sixth"}; - for (const auto& s : entry_names) { - key_strings_builder->add_string(s); - } - - auto type_strings_builder = - std::make_shared(UTF16_POOL); - std::vector type_names{"dimen", "style", "xml"}; - for (const auto& s : type_names) { - type_strings_builder->add_string(s); - } - - auto package_builder = - std::make_shared(&foo_package); - package_builder->set_key_strings(key_strings_builder); - package_builder->set_type_strings(type_strings_builder); - - auto table_builder = std::make_shared(); - table_builder->set_global_strings(global_strings_builder); - table_builder->add_package(package_builder); - - // dimen - std::vector dimen_configs = {&default_config, - &land_config}; - // First res ID has entries in two different configs (this flag denotes that). - // Subsequent two entries only have default config entries (hence zero). - std::vector dimen_flags = { - android::ResTable_config::CONFIG_ORIENTATION, 0, 0}; - auto dimen_type_definer = std::make_shared( - foo_package.id, 1, dimen_configs, dimen_flags); - package_builder->add_type(dimen_type_definer); - - dimen_type_definer->add(&default_config, &e0); - dimen_type_definer->add(&land_config, &e0_land); - dimen_type_definer->add(&default_config, &e1); - dimen_type_definer->add_empty(&land_config); - dimen_type_definer->add(&default_config, &e2); - dimen_type_definer->add_empty(&land_config); - - // style - std::vector style_configs = {&xxhdpi_config}; - std::vector style_flags = { - android::ResTable_config::CONFIG_DENSITY}; - auto style_type_definer = std::make_shared( - foo_package.id, 2, style_configs, style_flags); - package_builder->add_type(style_type_definer); - - style.item0.name.ident = 0x01010098; // android:textColor - style.item0.value.dataType = android::Res_value::TYPE_INT_COLOR_RGB8; - style.item0.value.data = 0xFF0000FF; - - style.item1.name.ident = 0x010100d4; // android:background - style.item1.value.dataType = android::Res_value::TYPE_INT_COLOR_RGB8; - style.item1.value.data = 0xFF00FF00; - - style_type_definer->add(&xxhdpi_config, &style); - - // xml - std::vector xml_configs = {&default_config}; - std::vector xml_flags = {0, 0}; - auto xml_type_definer = std::make_shared( - foo_package.id, 3, xml_configs, xml_flags); - package_builder->add_type(xml_type_definer); - EntryAndValue x0(4, android::Res_value::TYPE_STRING, 0); - EntryAndValue x1(5, android::Res_value::TYPE_STRING, 1); - xml_type_definer->add(&default_config, &x0); - xml_type_definer->add(&default_config, &x1); - - android::Vector table_data; - table_builder->serialize(&table_data); - - // Make a fake rename map. - attribution::ResourceNames names{{0x7f010000, "yolo"}}; - attribution::ArscStats stats(table_data.array(), table_data.size(), names); - auto results = stats.compute(); - EXPECT_EQ(results.size(), entry_names.size()); - std::vector expected_private_sizes{148, 37, 36, 179, 52, 53}; - // For ease of comparison, these are the floor of expected values. - std::vector expected_proportional_sizes{247, 136, 135, 239, 184, 185}; - size_t idx = 0; - for (const auto& result : results) { - if (idx == 0) { - // Make sure the given rename map takes priority. - EXPECT_STREQ(result.name.c_str(), "yolo") - << "Incorrect name for 0x" << std::hex << result.id; - EXPECT_EQ(result.configs.size(), 2); - } else { - EXPECT_EQ(result.name, entry_names.at(idx)) - << "Incorrect name for 0x" << std::hex << result.id; - EXPECT_EQ(result.configs.size(), 1); - } - EXPECT_EQ(result.sizes.private_size, expected_private_sizes.at(idx)) - << "Incorrect size for 0x" << std::hex << result.id; - EXPECT_EQ(result.sizes.shared_size, 0); - EXPECT_EQ((size_t)std::floor(result.sizes.proportional_size), - expected_proportional_sizes.at(idx)) - << "Incorrect proportional size for 0x" << std::hex << result.id; - idx++; - } -} - -TEST(Arsc, StringSpanAttribution) { - // Make a string pool with two strings, first being a styled string with 2 - // html style tags and the second being a regular string. In human readable - // form it looks like this: - // clang-format off - // - // Package Groups (1) - // Package Group 0 id=0x7f packageCount=1 name=foo - // Package 0 id=0x7f name=foo - // type 0 configCount=1 entryCount=2 - // spec resource 0x7f010000 foo:string/first: flags=0x00000000 - // spec resource 0x7f010001 foo:string/second: flags=0x00000000 - // config (default): - // resource 0x7f010000 foo:string/first: t=0x03 d=0x00000000 (s=0x0008 r=0x00) - // (string8) "I like a fine glass of H20 in the morning." - // resource 0x7f010001 foo:string/second: t=0x03 d=0x00000001 (s=0x0008 r=0x00) - // (string8) "regular string" - // clang-format on - auto global_strings_builder = - std::make_shared(UTF8_POOL); - android::ResStringPool_span em{.name = {2}, .firstChar = 9, .lastChar = 12}; - android::ResStringPool_span sub{.name = {3}, .firstChar = 24, .lastChar = 24}; - std::string styled_string("I like a fine glass of H20 in the morning."); - std::string regular_string("regular string"); - global_strings_builder->add_style(styled_string, {&em, &sub}); - global_strings_builder->add_string(regular_string); - global_strings_builder->add_string("em"); - global_strings_builder->add_string("sub"); - - // Check some things regarding the pool itself - android::Vector pool_data; - global_strings_builder->serialize(&pool_data); - auto pool_header = (android::ResStringPool_header*)pool_data.array(); - android::ResStringPool pool; - pool.setTo(pool_header, pool_data.size(), true); - EXPECT_EQ(attribution::count_padding(pool_header, pool), 3); - - // This API call is just for the bytes of styled_string itself. - EXPECT_EQ(attribution::compute_string_character_size(pool, 0), - styled_string.size() + SIZE_AND_NULL_ZERO); - // Entire size to represent styled_string which includes an offset for it, - // offsets for the 2 html tag names, as well as the size of the span - // information for where the tags should be plus another offset to where the - // span information starts. - auto styled_string_data_size = - styled_string.size() + strlen("em") + strlen("sub") + - 3 * SIZE_AND_NULL_ZERO + 3 * OFFSET_SIZE + - 2 * sizeof(android::ResStringPool_span) + - sizeof(android::ResStringPool_span::END) + OFFSET_SIZE; - EXPECT_EQ(attribution::compute_string_size(pool, 0), styled_string_data_size); - - // Just the bytes of regular_string - EXPECT_EQ(attribution::compute_string_character_size(pool, 1), - regular_string.size() + SIZE_AND_NULL_ZERO); - // Entire size to represent regular_string which includes an offset. - EXPECT_EQ(attribution::compute_string_size(pool, 1), - regular_string.size() + SIZE_AND_NULL_ZERO + OFFSET_SIZE); - - // Continue on to build a full .arsc file and get the stats. - auto key_strings_builder = - std::make_shared(UTF8_POOL); - key_strings_builder->add_string("first"); - key_strings_builder->add_string("second"); - - auto type_strings_builder = - std::make_shared(UTF16_POOL); - type_strings_builder->add_string("string"); - - auto package_builder = - std::make_shared(&foo_package); - package_builder->set_key_strings(key_strings_builder); - package_builder->set_type_strings(type_strings_builder); - - auto table_builder = std::make_shared(); - table_builder->set_global_strings(global_strings_builder); - table_builder->add_package(package_builder); - - // string type - std::vector string_configs = {&default_config}; - std::vector string_flags = {0, 0}; - auto string_type_definer = std::make_shared( - foo_package.id, 1, string_configs, string_flags); - package_builder->add_type(string_type_definer); - EntryAndValue s0(0, android::Res_value::TYPE_STRING, 0); - EntryAndValue s1(1, android::Res_value::TYPE_STRING, 1); - string_type_definer->add(&default_config, &s0); - string_type_definer->add(&default_config, &s1); - - android::Vector table_data; - table_builder->serialize(&table_data); - - attribution::ArscStats stats(table_data.array(), table_data.size(), no_names); - auto results = stats.compute(); - EXPECT_EQ(results.size(), 2); - - const auto& result = results.at(0); - auto size_of_key_string = OFFSET_SIZE + strlen("first") + SIZE_AND_NULL_ZERO; - EXPECT_EQ(result.sizes.private_size, - styled_string_data_size + size_of_key_string + - OFFSET_SIZE /* typeSpec flag */ + - OFFSET_SIZE /* type offset */ + sizeof(android::Res_value) + - sizeof(android::ResTable_entry)); -} - -TEST(Arsc, CountPadding) { - std::vector odd = {"array"}; - std::vector even = {"string"}; - EXPECT_EQ(count_padding(odd, UTF16_POOL), 2); - EXPECT_EQ(count_padding(even, UTF16_POOL), 0); -} - -TEST(Arsc, DuplicateDataAttribution) { - auto global_strings_builder = - std::make_shared(UTF8_POOL); - auto key_strings_builder = - std::make_shared(UTF8_POOL); - key_strings_builder->add_string("(name removed)"); - auto type_strings_builder = - std::make_shared(UTF16_POOL); - type_strings_builder->add_string("dimen"); - - auto package_builder = - std::make_shared(&foo_package); - package_builder->set_key_strings(key_strings_builder); - package_builder->set_type_strings(type_strings_builder); - - auto table_builder = std::make_shared(); - table_builder->set_global_strings(global_strings_builder); - table_builder->add_package(package_builder); - - // dimen - std::vector dimen_configs{&default_config}; - std::vector dimen_flags{0, 0}; - auto dimen_type_definer = std::make_shared( - foo_package.id, - 1, - dimen_configs, - dimen_flags, - true /* enable_canonical_entries */, - true /* enable_sparse_encoding */); - package_builder->add_type(dimen_type_definer); - - EntryAndValue duplicate(0, android::Res_value::TYPE_DIMENSION, 9999); - dimen_type_definer->add(&default_config, &duplicate); - dimen_type_definer->add(&default_config, &duplicate); - - android::Vector table_data; - table_builder->serialize(&table_data); - - attribution::ArscStats stats(table_data.array(), table_data.size(), no_names); - auto results = stats.compute(); - EXPECT_EQ(results.size(), 2); - - auto expected_shared_size = - sizeof(android::Res_value) + sizeof(android::ResTable_entry) + - strlen("(name removed)") + SIZE_AND_NULL_ZERO + OFFSET_SIZE; - auto& first_result = results.at(0); - EXPECT_EQ(first_result.sizes.shared_size, expected_shared_size); - EXPECT_EQ(first_result.sizes.private_size, 2 * OFFSET_SIZE); - - // They are sharing same data and string name. - auto& second_result = results.at(1); - EXPECT_EQ(second_result.sizes.shared_size, expected_shared_size); - EXPECT_EQ(second_result.sizes.private_size, 2 * OFFSET_SIZE); -} diff --git a/test/unit/AtomicMapTest.cpp b/test/unit/AtomicMapTest.cpp deleted file mode 100644 index b0131ebb85..0000000000 --- a/test/unit/AtomicMapTest.cpp +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * - * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. - */ - -#include "ConcurrentContainers.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -class AtomicMapTest : public ::testing::Test {}; - -TEST_F(AtomicMapTest, concurrentFetchAdd) { - const size_t N_THREADS = 1000; - const size_t N = 100000; - AtomicMap map; - std::vector threads; - for (size_t t = 0; t < N_THREADS; ++t) { - threads.emplace_back([&]() { - for (size_t i = 0; i < N; ++i) { - map.fetch_add(i, 1); - } - }); - } - for (auto& thread : threads) { - thread.join(); - } - EXPECT_EQ(N, map.size()); - for (size_t i = 0; i < N; ++i) { - auto value = map.load(i); - EXPECT_EQ(value, N_THREADS); - } -} - -TEST_F(AtomicMapTest, concurrentStore) { - const size_t N_THREADS = 1000; - const size_t N = 100000; - AtomicMap map; - std::vector threads; - for (size_t t = 0; t < N_THREADS; ++t) { - threads.emplace_back([&]() { - for (size_t i = 0; i < N; ++i) { - map.store(i, i); - } - }); - } - for (auto& thread : threads) { - thread.join(); - } - EXPECT_EQ(N, map.size()); - for (size_t i = 0; i < N; ++i) { - auto value = map.load(i); - EXPECT_EQ(value, i); - } -} - -TEST_F(AtomicMapTest, exchange) { - const size_t N = 100000; - AtomicMap map; - for (size_t i = 0; i < N; ++i) { - map.store(i, i); - } - for (size_t i = 0; i < N; ++i) { - auto old = map.exchange(i, N); - EXPECT_EQ(old, i); - } - for (size_t i = 0; i < N; ++i) { - auto current = map.load(i); - EXPECT_EQ(current, N); - } -} - -TEST_F(AtomicMapTest, concurrentCompareExchange) { - const size_t N_THREADS = 1000; - const size_t N = 100000; - AtomicMap map; - std::vector threads; - for (size_t t = 0; t < N_THREADS; ++t) { - threads.emplace_back([&]() { - for (size_t i = 0; i < N; ++i) { - size_t expected = map.load(i); - while (!map.compare_exchange(i, expected, expected + 1)) { - } - } - }); - } - for (auto& thread : threads) { - thread.join(); - } - EXPECT_EQ(N, map.size()); - for (size_t i = 0; i < N; ++i) { - auto value = map.load(i); - EXPECT_EQ(value, N_THREADS); - } -} diff --git a/test/unit/CheckBreadCrumbsTest.cpp b/test/unit/CheckBreadCrumbsTest.cpp index a4da304595..240d768a17 100644 --- a/test/unit/CheckBreadCrumbsTest.cpp +++ b/test/unit/CheckBreadCrumbsTest.cpp @@ -14,7 +14,6 @@ #include "DexUtil.h" #include "IRAssembler.h" #include "RedexTest.h" -#include "Walkers.h" // Create the following hierarchy // @@ -224,17 +223,11 @@ TEST_F(CheckBreadcrumbsTest, AccessValidityTest) { /* verify_proto_cross_dex= */ false, /* enforce_allowed_violations_file= */ false); std::vector method_list = call_a_fields_and_methods_methods(); - method_list[0]->get_code()->build_cfg(); EXPECT_EQ(bc.has_illegal_access(method_list[0]), false); - method_list[1]->get_code()->build_cfg(); EXPECT_EQ(bc.has_illegal_access(method_list[1]), false); - method_list[2]->get_code()->build_cfg(); EXPECT_EQ(bc.has_illegal_access(method_list[2]), true); - method_list[3]->get_code()->build_cfg(); EXPECT_EQ(bc.has_illegal_access(method_list[3]), false); - method_list[4]->get_code()->build_cfg(); EXPECT_EQ(bc.has_illegal_access(method_list[4]), false); - method_list[5]->get_code()->build_cfg(); EXPECT_EQ(bc.has_illegal_access(method_list[5]), true); std::ostringstream expected; expected << "Bad methods in class LB;\n" @@ -313,7 +306,6 @@ TEST_F(CheckBreadcrumbsTest, CrossStoreValidityTest) { stores.emplace_back(std::move(store_s_B_C)); auto scope = build_class_scope(stores); - Breadcrumbs bc_shared(scope, "", stores, diff --git a/test/unit/CheckCastAnalysisTest.cpp b/test/unit/CheckCastAnalysisTest.cpp index 328de929cc..8528f9263e 100644 --- a/test/unit/CheckCastAnalysisTest.cpp +++ b/test/unit/CheckCastAnalysisTest.cpp @@ -47,7 +47,6 @@ struct CheckCastAnalysisTest : public RedexTest { } }; - TEST_F(CheckCastAnalysisTest, simple_string) { auto method = assembler::method_from_string(R"( (method (public) "LFoo;.bar:()Ljava/lang/String;" diff --git a/test/unit/ConcurrentContainersTest.cpp b/test/unit/ConcurrentContainersTest.cpp index c72b3bee09..1410c81699 100644 --- a/test/unit/ConcurrentContainersTest.cpp +++ b/test/unit/ConcurrentContainersTest.cpp @@ -278,9 +278,7 @@ TEST_F(ConcurrentContainersTest, concurrentMapTest) { run_on_subset_samples([&map](const std::vector& sample) { for (size_t i = 0; i < sample.size(); ++i) { - auto* p = map.get_and_erase(std::to_string(sample[i])); - EXPECT_TRUE(p); - EXPECT_EQ(*p, sample[i] + 1); + map.erase(std::to_string(sample[i])); } }); @@ -313,116 +311,3 @@ TEST_F(ConcurrentContainersTest, concurrentMapTest) { map.clear(); EXPECT_EQ(0, map.size()); } - -TEST_F(ConcurrentContainersTest, insertOnlyConcurrentMapTest) { - InsertOnlyConcurrentMap map; - - InsertOnlyConcurrentMap ptrs; - run_on_samples([&map, &ptrs](const std::vector& sample) { - for (size_t i = 0; i < sample.size(); ++i) { - std::string s = std::to_string(sample[i]); - if (i % 3 == 0) { - map.insert({s, sample[i]}); - ptrs.emplace(s, map.get(s)); - } else if (i % 3 == 1) { - auto [ptr, created] = map.get_or_create_and_assert_equal( - s, [](const auto& t) { return (uint32_t)atoi(t.c_str()); }); - EXPECT_TRUE(created); - ptrs.emplace(s, ptr); - } else { - auto [ptr, emplaced] = - map.get_or_emplace_and_assert_equal(s, sample[i]); - EXPECT_TRUE(emplaced); - ptrs.emplace(s, ptr); - } - EXPECT_EQ(1, map.count(s)); - } - }); - - run_on_samples([&map, &ptrs](const std::vector& sample) { - for (size_t i = 0; i < sample.size(); ++i) { - std::string s = std::to_string(sample[i]); - auto [ptr1, emplaced] = map.get_or_emplace_and_assert_equal(s, sample[i]); - EXPECT_FALSE(emplaced); - auto [ptr2, created] = map.get_or_create_and_assert_equal( - s, [](const auto& t) -> uint32_t { not_reached(); }); - EXPECT_FALSE(created); - EXPECT_EQ(ptrs.at(s), ptr1); - EXPECT_EQ(ptr1, ptr2); - } - }); - EXPECT_EQ(m_data_set.size(), map.size()); - for (uint32_t x : m_data) { - std::string s = std::to_string(x); - EXPECT_EQ(1, map.count(s)); - auto it = map.find(s); - EXPECT_NE(map.end(), it); - EXPECT_EQ(s, it->first); - EXPECT_EQ(x, it->second); - auto p = ptrs.at(s); - EXPECT_EQ(p, map.get(s)); - EXPECT_EQ(p, map.get_unsafe(s)); - } - - EXPECT_EQ(m_data_set.size(), map.size()); -} - -TEST_F(ConcurrentContainersTest, move) { - ConcurrentMap map1; - map1.emplace(nullptr, nullptr); - EXPECT_EQ(1, map1.size()); - auto map2 = std::move(map1); - EXPECT_EQ(1, map2.size()); - map1 = std::move(map2); - EXPECT_EQ(1, map1.size()); -} - -TEST_F(ConcurrentContainersTest, copy) { - ConcurrentMap map1; - map1.emplace(nullptr, nullptr); - EXPECT_EQ(1, map1.size()); - auto map2 = map1; - EXPECT_EQ(1, map1.size()); - EXPECT_EQ(1, map2.size()); -} - -TEST_F(ConcurrentContainersTest, insert_or_assign) { - ConcurrentMap> map; - run_on_samples([&map](const std::vector& sample) { - for (auto x : sample) { - map.insert_or_assign(std::make_pair(x, std::make_unique(x))); - } - }); - EXPECT_EQ(m_data_set.size(), map.size()); - for (uint32_t x : m_data) { - EXPECT_TRUE(map.count(x)); - auto& p = map.at_unsafe(x); - EXPECT_TRUE(p); - EXPECT_EQ(x, *p); - } - - run_on_samples([&map](const std::vector& sample) { - for (auto x : sample) { - map.insert_or_assign( - std::make_pair(x, std::make_unique(x + 1))); - } - }); - EXPECT_EQ(m_data_set.size(), map.size()); - for (uint32_t x : m_data) { - EXPECT_TRUE(map.count(x)); - auto& p = map.at_unsafe(x); - EXPECT_TRUE(p); - EXPECT_EQ(x + 1, *p); - } -} - -TEST_F(ConcurrentContainersTest, atThrows) { - ConcurrentMap empty; - bool threw = false; - try { - empty.at(nullptr); - } catch (const std::out_of_range&) { - threw = true; - } - EXPECT_TRUE(threw); -} diff --git a/test/unit/ConcurrentHashtableTest.cpp b/test/unit/ConcurrentHashtableTest.cpp deleted file mode 100644 index bdf37b1bf1..0000000000 --- a/test/unit/ConcurrentHashtableTest.cpp +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * - * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. - */ - -#include "ConcurrentContainers.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -using namespace cc_impl; - -class ConcurrentHashtableTest : public ::testing::Test {}; - -TEST_F(ConcurrentHashtableTest, sequentialInsertGet) { - const size_t N = 10000; - ConcurrentHashtable, - std::equal_to> - set; - for (size_t i = 0; i < N; ++i) { - auto insertion_result = set.try_insert(i); - EXPECT_TRUE(insertion_result.success); - EXPECT_NE(nullptr, insertion_result.stored_value_ptr); - EXPECT_EQ(i, *insertion_result.stored_value_ptr); - } - EXPECT_EQ(N, set.size()); - for (size_t i = 0; i < N; ++i) { - auto insertion_result = set.try_insert(i); - EXPECT_FALSE(insertion_result.success); - EXPECT_NE(nullptr, insertion_result.stored_value_ptr); - EXPECT_EQ(i, *insertion_result.stored_value_ptr); - } - EXPECT_EQ(N, set.size()); - for (size_t i = 0; i < N; ++i) { - auto ptr = set.get(i); - EXPECT_NE(nullptr, ptr); - EXPECT_EQ(i, *ptr); - } - EXPECT_EQ(nullptr, set.get(N)); -} - -TEST_F(ConcurrentHashtableTest, sequentialInsertEraseGet) { - const size_t N = 10000; - ConcurrentHashtable, - std::equal_to> - set; - for (size_t i = 0; i < N; ++i) { - auto insertion_result = set.try_insert(i); - EXPECT_TRUE(insertion_result.success); - EXPECT_NE(nullptr, insertion_result.stored_value_ptr); - EXPECT_EQ(i, *insertion_result.stored_value_ptr); - } - EXPECT_EQ(N, set.size()); - for (size_t i = 0; i < N; ++i) { - auto erased = set.erase(i); - EXPECT_TRUE(erased); - } - EXPECT_TRUE(set.empty()); - EXPECT_EQ(nullptr, set.get(0)); -} - -TEST_F(ConcurrentHashtableTest, concurrentInsertGet) { - const size_t N_THREADS = 1000; - const size_t N = 100000; - ConcurrentHashtable, - std::equal_to> - set; - std::vector threads; - for (size_t t = 0; t < N_THREADS; ++t) { - threads.emplace_back([&]() { - for (size_t i = 0; i < N; ++i) { - auto insertion_result = set.try_insert(i); - EXPECT_NE(nullptr, insertion_result.stored_value_ptr); - EXPECT_EQ(i, *insertion_result.stored_value_ptr); - } - }); - } - for (auto& thread : threads) { - thread.join(); - } - EXPECT_EQ(N, set.size()); - for (size_t i = 0; i < N; ++i) { - auto ptr = set.get(i); - EXPECT_NE(nullptr, ptr); - EXPECT_EQ(i, *ptr); - } - EXPECT_EQ(nullptr, set.get(N)); -} - -TEST_F(ConcurrentHashtableTest, primeProgression) { - size_t i = 5; - i = cc_impl::get_prime_number_greater_or_equal_to(i * 2); - ASSERT_EQ(i, 13); - i = cc_impl::get_prime_number_greater_or_equal_to(i * 2); - ASSERT_EQ(i, 29); - i = cc_impl::get_prime_number_greater_or_equal_to(i * 2); - ASSERT_EQ(i, 61); - i = cc_impl::get_prime_number_greater_or_equal_to(i * 2); - ASSERT_EQ(i, 113); - i = cc_impl::get_prime_number_greater_or_equal_to(i * 2); - ASSERT_EQ(i, 251); - i = cc_impl::get_prime_number_greater_or_equal_to(i * 2); - ASSERT_EQ(i, 509); - i = cc_impl::get_prime_number_greater_or_equal_to(i * 2); - ASSERT_EQ(i, 1021); - i = cc_impl::get_prime_number_greater_or_equal_to(i * 2); - ASSERT_EQ(i, 2039); - i = cc_impl::get_prime_number_greater_or_equal_to(i * 2); - ASSERT_EQ(i, 4093); - i = cc_impl::get_prime_number_greater_or_equal_to(i * 2); - ASSERT_EQ(i, 8179); - i = cc_impl::get_prime_number_greater_or_equal_to(i * 2); - ASSERT_EQ(i, 16381); - i = cc_impl::get_prime_number_greater_or_equal_to(i * 2); - ASSERT_EQ(i, 32749); - - i = 1073741789; - i = cc_impl::get_prime_number_greater_or_equal_to(i * 2); - ASSERT_EQ(i, 2147483647); - i = cc_impl::get_prime_number_greater_or_equal_to(i * 2); - ASSERT_EQ(i, 4294967295); - i = cc_impl::get_prime_number_greater_or_equal_to(i * 2); - ASSERT_EQ(i, 8589934591); -} diff --git a/test/unit/ControlFlowTest.cpp b/test/unit/ControlFlowTest.cpp index c85ec2d550..fa7f232971 100644 --- a/test/unit/ControlFlowTest.cpp +++ b/test/unit/ControlFlowTest.cpp @@ -1042,7 +1042,6 @@ TEST_F(ControlFlowTest, deep_copy1) { cfg::ControlFlowGraph copy; orig.deep_copy(©); - EXPECT_TRUE(orig.structural_equals(copy)); IRList* orig_list = orig.linearize(); IRList* copy_list = copy.linearize(); @@ -1072,7 +1071,6 @@ TEST_F(ControlFlowTest, deep_copy2) { cfg::ControlFlowGraph copy; orig.deep_copy(©); - EXPECT_TRUE(orig.structural_equals(copy)); IRList* orig_list = orig.linearize(); IRList* copy_list = copy.linearize(); @@ -1111,7 +1109,6 @@ TEST_F(ControlFlowTest, deep_copy3) { cfg::ControlFlowGraph copy; orig.deep_copy(©); - EXPECT_TRUE(orig.structural_equals(copy)); IRList* orig_list = orig.linearize(); IRList* copy_list = copy.linearize(); diff --git a/test/unit/CopyPropagationTest.cpp b/test/unit/CopyPropagationTest.cpp index ce3a906ff8..a943641cbe 100644 --- a/test/unit/CopyPropagationTest.cpp +++ b/test/unit/CopyPropagationTest.cpp @@ -25,7 +25,7 @@ TEST_F(CopyPropagationTest, simple) { ) )"); code->set_registers_size(3); - code->build_cfg(); + copy_propagation_impl::Config config; CopyPropagation(config).run(code.get()); @@ -39,7 +39,7 @@ TEST_F(CopyPropagationTest, simple) { (return v0) ) )"); - code->clear_cfg(); + EXPECT_CODE_EQ(code.get(), expected_code.get()); } @@ -60,7 +60,7 @@ TEST_F(CopyPropagationTest, deleteRepeatedMove) { ) )"); code->set_registers_size(2); - code->build_cfg(); + copy_propagation_impl::Config config; CopyPropagation(config).run(code.get()); @@ -73,7 +73,7 @@ TEST_F(CopyPropagationTest, deleteRepeatedMove) { (return v0) ) )"); - code->clear_cfg(); + EXPECT_CODE_EQ(code.get(), expected_code.get()); } @@ -92,7 +92,7 @@ TEST_F(CopyPropagationTest, noRemapRange) { ) )"); code->set_registers_size(7); - code->build_cfg(); + copy_propagation_impl::Config config; config.regalloc_has_run = true; CopyPropagation(config).run(code.get()); @@ -105,7 +105,7 @@ TEST_F(CopyPropagationTest, noRemapRange) { (return v0) ) )"); - code->clear_cfg(); + EXPECT_CODE_EQ(code.get(), expected_code.get()); } @@ -118,7 +118,7 @@ TEST_F(CopyPropagationTest, deleteSelfMove) { ) )"); code->set_registers_size(2); - code->build_cfg(); + copy_propagation_impl::Config config; CopyPropagation(config).run(code.get()); @@ -128,7 +128,7 @@ TEST_F(CopyPropagationTest, deleteSelfMove) { (return-void) ) )"); - code->clear_cfg(); + EXPECT_CODE_EQ(code.get(), expected_code.get()); } @@ -143,7 +143,7 @@ TEST_F(CopyPropagationTest, representative) { ) )"); code->set_registers_size(2); - code->build_cfg(); + copy_propagation_impl::Config config; CopyPropagation(config).run(code.get()); @@ -156,7 +156,7 @@ TEST_F(CopyPropagationTest, representative) { (return-void) ) )"); - code->clear_cfg(); + EXPECT_CODE_EQ(code.get(), expected_code.get()); } @@ -172,7 +172,7 @@ TEST_F(CopyPropagationTest, verifyEnabled) { ) )"); code->set_registers_size(2); - code->build_cfg(); + copy_propagation_impl::Config config; CopyPropagation(config).run(code.get()); @@ -185,7 +185,7 @@ TEST_F(CopyPropagationTest, verifyEnabled) { (return-void) ) )"); - code->clear_cfg(); + EXPECT_CODE_EQ(code.get(), expected_code.get()); } @@ -201,7 +201,7 @@ TEST_F(CopyPropagationTest, consts_safe_by_constant_uses) { ) )"); code->set_registers_size(2); - code->build_cfg(); + copy_propagation_impl::Config config; CopyPropagation(config).run(code.get()); @@ -213,7 +213,7 @@ TEST_F(CopyPropagationTest, consts_safe_by_constant_uses) { (return-void) ) )"); - code->clear_cfg(); + EXPECT_CODE_EQ(code.get(), expected_code.get()); } @@ -235,7 +235,7 @@ TEST_F(CopyPropagationTest, non_zero_constant_cannot_have_object_type_demand) { )"); auto code = method->get_code(); code->set_registers_size(2); - code->build_cfg(); + copy_propagation_impl::Config config; CopyPropagation(config).run(code, method); @@ -248,7 +248,7 @@ TEST_F(CopyPropagationTest, non_zero_constant_cannot_have_object_type_demand) { (return-void) ) )"); - code->clear_cfg(); + EXPECT_CODE_EQ(code, expected_code.get()); } @@ -281,7 +281,7 @@ TEST_F(CopyPropagationTest, if_eqz_can_create_demand) { )"); auto code = method->get_code(); code->set_registers_size(2); - code->build_cfg(); + copy_propagation_impl::Config config; CopyPropagation(config).run(code, method); @@ -306,7 +306,7 @@ TEST_F(CopyPropagationTest, if_eqz_can_create_demand) { (return-void) ) )"); - code->clear_cfg(); + EXPECT_CODE_EQ(code, expected_code.get()); } @@ -331,7 +331,7 @@ TEST_F(CopyPropagationTest, consts_safe_by_constant_uses_aput) { )"); auto code = method->get_code(); code->set_registers_size(3); - code->build_cfg(); + copy_propagation_impl::Config config; CopyPropagation(config).run(code, method); @@ -346,7 +346,7 @@ TEST_F(CopyPropagationTest, consts_safe_by_constant_uses_aput) { (return-void) ) )"); - code->clear_cfg(); + EXPECT_CODE_EQ(code, expected_code.get()); } @@ -370,7 +370,7 @@ TEST_F(CopyPropagationTest, consts_unsafe_by_constant_uses_aput) { )"); auto code = method->get_code(); code->set_registers_size(3); - code->build_cfg(); + copy_propagation_impl::Config config; CopyPropagation(config).run(code, method); @@ -387,7 +387,7 @@ TEST_F(CopyPropagationTest, consts_unsafe_by_constant_uses_aput) { (return-void) ) )"); - code->clear_cfg(); + EXPECT_CODE_EQ(code, expected_code.get()); } @@ -403,7 +403,7 @@ TEST_F(CopyPropagationTest, wide_consts_safe_by_constant_uses) { ) )"); code->set_registers_size(4); - code->build_cfg(); + copy_propagation_impl::Config config; CopyPropagation(config).run(code.get()); @@ -415,7 +415,7 @@ TEST_F(CopyPropagationTest, wide_consts_safe_by_constant_uses) { (return-void) ) )"); - code->clear_cfg(); + EXPECT_CODE_EQ(code.get(), expected_code.get()); } @@ -445,7 +445,7 @@ TEST_F(CopyPropagationTest, if_constraints_with_constant_uses) { )"); auto code = method->get_code(); code->set_registers_size(4); - code->build_cfg(); + copy_propagation_impl::Config config; CopyPropagation(config).run(code, method); @@ -465,7 +465,7 @@ TEST_F(CopyPropagationTest, if_constraints_with_constant_uses) { (return-object v1) ) )"); - code->clear_cfg(); + EXPECT_CODE_EQ(code, expected_code.get()); } @@ -480,7 +480,7 @@ TEST_F(CopyPropagationTest, cliqueAliasing) { ) )"); code->set_registers_size(4); - code->build_cfg(); + copy_propagation_impl::Config config; config.replace_with_representative = false; CopyPropagation(config).run(code.get()); @@ -493,7 +493,7 @@ TEST_F(CopyPropagationTest, cliqueAliasing) { (return-void) ) )"); - code->clear_cfg(); + EXPECT_CODE_EQ(code.get(), expected_code.get()); } @@ -513,7 +513,7 @@ TEST_F(CopyPropagationTest, loopNoChange) { ) )"); code->set_registers_size(2); - code->build_cfg(); + copy_propagation_impl::Config config; CopyPropagation(config).run(code.get()); @@ -531,7 +531,7 @@ TEST_F(CopyPropagationTest, loopNoChange) { (return-void) ) )"); - code->clear_cfg(); + EXPECT_CODE_EQ(code.get(), expected_code.get()); } @@ -554,12 +554,12 @@ TEST_F(CopyPropagationTest, branchNoChange) { auto code = assembler::ircode_from_string(no_change); code->set_registers_size(4); - code->build_cfg(); + copy_propagation_impl::Config config; CopyPropagation(config).run(code.get()); auto expected_code = assembler::ircode_from_string(no_change); - code->clear_cfg(); + EXPECT_CODE_EQ(code.get(), expected_code.get()); } @@ -580,7 +580,7 @@ TEST_F(CopyPropagationTest, intersect1) { ) )"); code->set_registers_size(4); - code->build_cfg(); + copy_propagation_impl::Config config; CopyPropagation(config).run(code.get()); @@ -598,7 +598,7 @@ TEST_F(CopyPropagationTest, intersect1) { (goto :end) ) )"); - code->clear_cfg(); + EXPECT_CODE_EQ(code.get(), expected_code.get()); } @@ -621,13 +621,13 @@ TEST_F(CopyPropagationTest, intersect2) { )"; auto code = assembler::ircode_from_string(no_change); code->set_registers_size(5); - code->build_cfg(); + copy_propagation_impl::Config config; config.replace_with_representative = false; CopyPropagation(config).run(code.get()); auto expected_code = assembler::ircode_from_string(no_change); - code->clear_cfg(); + EXPECT_CODE_EQ(code.get(), expected_code.get()); } @@ -640,7 +640,7 @@ TEST_F(CopyPropagationTest, wide) { ) )"); code->set_registers_size(4); - code->build_cfg(); + copy_propagation_impl::Config config; config.wide_registers = true; CopyPropagation(config).run(code.get()); @@ -651,7 +651,7 @@ TEST_F(CopyPropagationTest, wide) { (return-void) ) )"); - code->clear_cfg(); + EXPECT_CODE_EQ(code.get(), expected_code.get()); } @@ -665,7 +665,7 @@ TEST_F(CopyPropagationTest, wideClobber) { ) )"); code->set_registers_size(5); - code->build_cfg(); + copy_propagation_impl::Config config; config.wide_registers = false; CopyPropagation(config).run(code.get()); @@ -678,7 +678,7 @@ TEST_F(CopyPropagationTest, wideClobber) { (return-void) ) )"); - code->clear_cfg(); + EXPECT_CODE_EQ(code.get(), expected_code.get()); } @@ -692,7 +692,7 @@ TEST_F(CopyPropagationTest, wideClobberWideTrue) { ) )"); code->set_registers_size(5); - code->build_cfg(); + copy_propagation_impl::Config config; config.wide_registers = true; CopyPropagation(config).run(code.get()); @@ -705,7 +705,7 @@ TEST_F(CopyPropagationTest, wideClobberWideTrue) { (return-void) ) )"); - code->clear_cfg(); + EXPECT_CODE_EQ(code.get(), expected_code.get()); } @@ -718,7 +718,7 @@ TEST_F(CopyPropagationTest, wideClobberWideOddUp) { ) )"); code->set_registers_size(5); - code->build_cfg(); + copy_propagation_impl::Config config; config.wide_registers = true; CopyPropagation(config).run(code.get()); @@ -730,7 +730,7 @@ TEST_F(CopyPropagationTest, wideClobberWideOddUp) { (return-void) ) )"); - code->clear_cfg(); + EXPECT_CODE_EQ(code.get(), expected_code.get()); } @@ -743,7 +743,7 @@ TEST_F(CopyPropagationTest, wideClobberWideOddDown) { ) )"); code->set_registers_size(5); - code->build_cfg(); + copy_propagation_impl::Config config; config.wide_registers = true; CopyPropagation(config).run(code.get()); @@ -755,7 +755,7 @@ TEST_F(CopyPropagationTest, wideClobberWideOddDown) { (return-void) ) )"); - code->clear_cfg(); + EXPECT_CODE_EQ(code.get(), expected_code.get()); } @@ -770,7 +770,7 @@ TEST_F(CopyPropagationTest, repWide) { ) )"); code->set_registers_size(5); - code->build_cfg(); + copy_propagation_impl::Config config; config.wide_registers = true; config.replace_with_representative = true; @@ -786,7 +786,7 @@ TEST_F(CopyPropagationTest, repWide) { (return-void) ) )"); - code->clear_cfg(); + EXPECT_CODE_EQ(code.get(), expected_code.get()); } @@ -810,13 +810,13 @@ TEST_F(CopyPropagationTest, whichRep) { )"; auto code = assembler::ircode_from_string(no_change); code->set_registers_size(4); - code->build_cfg(); + copy_propagation_impl::Config config; config.replace_with_representative = true; CopyPropagation(config).run(code.get()); auto expected_code = assembler::ircode_from_string(no_change); - code->clear_cfg(); + EXPECT_CODE_EQ(code.get(), expected_code.get()); } @@ -838,11 +838,11 @@ TEST_F(CopyPropagationTest, whichRep2) { )"; auto code = assembler::ircode_from_string(no_change); code->set_registers_size(4); - code->build_cfg(); + copy_propagation_impl::Config config; config.replace_with_representative = true; CopyPropagation(config).run(code.get()); - code->clear_cfg(); + auto expected_code = assembler::ircode_from_string(no_change); } @@ -864,7 +864,7 @@ TEST_F(CopyPropagationTest, whichRepPreserve) { ) )"); code->set_registers_size(4); - code->build_cfg(); + copy_propagation_impl::Config config; config.replace_with_representative = true; CopyPropagation(config).run(code.get()); @@ -884,7 +884,7 @@ TEST_F(CopyPropagationTest, whichRepPreserve) { (goto :end) ) )"); - code->clear_cfg(); + EXPECT_CODE_EQ(code.get(), expected_code.get()); } @@ -899,7 +899,7 @@ TEST_F(CopyPropagationTest, wideInvokeSources) { )"; auto code = assembler::ircode_from_string(no_change); code->set_registers_size(16); - code->build_cfg(); + copy_propagation_impl::Config config; config.replace_with_representative = true; config.wide_registers = true; @@ -907,7 +907,7 @@ TEST_F(CopyPropagationTest, wideInvokeSources) { CopyPropagation(config).run(code.get()); auto expected_code = assembler::ircode_from_string(no_change); - code->clear_cfg(); + EXPECT_CODE_EQ(code.get(), expected_code.get()); } @@ -925,7 +925,7 @@ TEST_F(CopyPropagationTest, use_does_not_kill_type_demands) { )"); auto code = method->get_code(); code->set_registers_size(2); - code->build_cfg(); + copy_propagation_impl::Config config; CopyPropagation(config).run(code, method); @@ -937,7 +937,7 @@ TEST_F(CopyPropagationTest, use_does_not_kill_type_demands) { (return-object v0) ) )"); - code->clear_cfg(); + EXPECT_CODE_EQ(code, expected_code.get()); } @@ -955,7 +955,7 @@ TEST_F(CopyPropagationTest, instance_of_kills_type_demands) { )"); auto code = method->get_code(); code->set_registers_size(2); - code->build_cfg(); + copy_propagation_impl::Config config; CopyPropagation(config).run(code, method); @@ -968,7 +968,7 @@ TEST_F(CopyPropagationTest, instance_of_kills_type_demands) { (return-object v0) ) )"); - code->clear_cfg(); + EXPECT_CODE_EQ(code, expected_code.get()); } @@ -988,7 +988,7 @@ TEST_F(CopyPropagationTest, ResueConst) { auto code = method->get_code(); code->set_registers_size(4); - code->build_cfg(); + copy_propagation_impl::Config config; config.regalloc_has_run = false; CopyPropagation(config).run(code, method); @@ -1003,7 +1003,6 @@ TEST_F(CopyPropagationTest, ResueConst) { (invoke-static (v1 v2 v2 v2 v2 v2) "LFoo;.bar:(IIIIII)V") ) )"); - code->clear_cfg(); EXPECT_CODE_EQ(code, expected_code.get()); } @@ -1026,7 +1025,7 @@ TEST_F(CopyPropagationTest, lock_canonicalization_none) { )"); auto code = method->get_code(); code->set_registers_size(4); - code->build_cfg(); + copy_propagation_impl::Config config; CopyPropagation(config).run(code, method); @@ -1044,7 +1043,6 @@ TEST_F(CopyPropagationTest, lock_canonicalization_none) { (monitor-exit v3) ) )"); - code->clear_cfg(); EXPECT_CODE_EQ(code, expected_code.get()); } @@ -1065,7 +1063,7 @@ TEST_F(CopyPropagationTest, lock_canonicalization) { )"); auto code = method->get_code(); code->set_registers_size(2); - code->build_cfg(); + copy_propagation_impl::Config config; CopyPropagation(config).run(code, method); @@ -1081,7 +1079,6 @@ TEST_F(CopyPropagationTest, lock_canonicalization) { (monitor-exit v2) ) )"); - code->clear_cfg(); EXPECT_CODE_EQ(code, expected_code.get()); } @@ -1115,13 +1112,12 @@ TEST_F(CopyPropagationTest, check_cast_throw_targets_regs) { auto code = method->get_code(); code->set_registers_size(3); - code->build_cfg(); + copy_propagation_impl::Config config; config.replace_with_representative = true; config.eliminate_const_literals_with_same_type_demands = true; CopyPropagation(config).run(code, method); - code->clear_cfg(); // Expect that `v2` is not replaced in `add-int`. auto expected_code = assembler::ircode_from_string(code_str); EXPECT_CODE_EQ(code, expected_code.get()); diff --git a/test/unit/CreatorsTest.cpp b/test/unit/CreatorsTest.cpp index 696d33e838..de719b3186 100644 --- a/test/unit/CreatorsTest.cpp +++ b/test/unit/CreatorsTest.cpp @@ -120,7 +120,7 @@ TEST_F(CreatorsTest, MakeSwitchMultiIndices) { TEST_F(CreatorsTest, ClassCreator) { std::string foo("Lfoo;"); - ClassCreator cc(DexType::make_type(foo)); + ClassCreator cc(DexType::make_type(foo.c_str())); cc.set_super(type::java_lang_Object()); auto cls = cc.create(); std::string bar("Lbar;"); diff --git a/test/unit/DedupBlocksTest.cpp b/test/unit/DedupBlocksTest.cpp index a7ed1e1fb9..1ed7a8b001 100644 --- a/test/unit/DedupBlocksTest.cpp +++ b/test/unit/DedupBlocksTest.cpp @@ -1180,16 +1180,17 @@ TEST_F(DedupBlocksTest, androidIsAfraidOfArraysOfInterfaces) { ClassCreator i_creator(DexType::make_type("LI;")); i_creator.set_access(ACC_INTERFACE); i_creator.set_super(type::java_lang_Object()); - auto i_cls = i_creator.create(); + i_creator.create(); ClassCreator a_creator(DexType::make_type("LA;")); - a_creator.add_interface(i_cls->get_type()); + a_creator.add_interface(i_creator.get_class()->get_type()); a_creator.set_super(type::java_lang_Object()); a_creator.create(); ClassCreator b_creator(DexType::make_type("LB;")); - b_creator.add_interface(i_cls->get_type()); + b_creator.add_interface(i_creator.get_class()->get_type()); b_creator.set_super(type::java_lang_Object()); + b_creator.create(); auto str = R"( ( @@ -1237,10 +1238,10 @@ TEST_F(DedupBlocksTest, trivialJoinOfArrayOfClassesIsFine) { ClassCreator i_creator(DexType::make_type("LI;")); i_creator.set_access(ACC_INTERFACE); i_creator.set_super(type::java_lang_Object()); - auto i_cls = i_creator.create(); + i_creator.create(); ClassCreator a_creator(DexType::make_type("LA;")); - a_creator.add_interface(i_cls->get_type()); + a_creator.add_interface(i_creator.get_class()->get_type()); a_creator.set_super(type::java_lang_Object()); a_creator.create(); diff --git a/test/unit/EvaluateTypeChecksTest.cpp b/test/unit/EvaluateTypeChecksTest.cpp index 9e1e7f3adc..f4af022da0 100644 --- a/test/unit/EvaluateTypeChecksTest.cpp +++ b/test/unit/EvaluateTypeChecksTest.cpp @@ -22,7 +22,7 @@ class EvaluateTypeChecksTest : public RedexTest { public: void SetUp() override { auto simple_class = [](const std::string& name, const DexType* super_type) { - ClassCreator cc(DexType::make_type(name)); + ClassCreator cc(DexType::make_type(name.c_str())); cc.set_super(const_cast(super_type)); return cc.create()->get_type(); }; @@ -73,9 +73,9 @@ class EvaluateTypeChecksTest : public RedexTest { auto method_str = std::string("(") + method_line + " " + in + " )"; auto method = assembler::class_with_method(type, method_str); - method->get_code()->build_cfg(); + check_casts::EvaluateTypeChecksPass::optimize(method, shrinker); - method->get_code()->clear_cfg(); + auto expected_str = regularize(out); auto actual_str = assembler::to_string(method->get_code()); if (expected_str == actual_str) { diff --git a/test/unit/HierarchyUtilTest.cpp b/test/unit/HierarchyUtilTest.cpp index 7677dbab1c..01f25a203e 100644 --- a/test/unit/HierarchyUtilTest.cpp +++ b/test/unit/HierarchyUtilTest.cpp @@ -56,16 +56,8 @@ TEST_F(RedexTest, findNonOverriddenVirtuals) { ext_cc.create(); - NonOverriddenVirtuals non_overridden_virtuals({cls}); - std::unordered_set set; - g_redex->walk_type_class([&](const DexType*, const DexClass* cls) { - for (auto* method : cls->get_vmethods()) { - if (non_overridden_virtuals.count(method)) { - set.insert(method); - } - } - }); - EXPECT_THAT(set, + const auto& non_overridden = find_non_overridden_virtuals({cls}); + EXPECT_THAT(non_overridden, ::testing::UnorderedElementsAre(final_method, nonfinal_method, ext_final_method)); } diff --git a/test/unit/IRTypeCheckerTest.cpp b/test/unit/IRTypeCheckerTest.cpp index c591b7913f..c03a5d4827 100644 --- a/test/unit/IRTypeCheckerTest.cpp +++ b/test/unit/IRTypeCheckerTest.cpp @@ -2828,46 +2828,3 @@ TEST_F(IRTypeCheckerTest, synchronizedThrowOutsideCatchAllInTry) { checker.run(); EXPECT_TRUE(checker.fail()); } - -TEST_F(IRTypeCheckerTest, invokeVirtualOnInterfaceMethod) { - const auto interface_type = DexType::make_type("LI;"); - ClassCreator interface_type_creator(interface_type); - interface_type_creator.set_super(type::java_lang_Object()); - interface_type_creator.set_access(ACC_INTERFACE); - auto foo_method = - DexMethod::make_method("LI;.foo:()V")->make_concrete(ACC_PUBLIC, true); - interface_type_creator.add_method(foo_method); - interface_type_creator.create(); - - { - auto method = DexMethod::make_method("LFoo;.bar:(LI;)V;") - ->make_concrete(ACC_PUBLIC, /* is_virtual */ false); - method->set_code(assembler::ircode_from_string(R"( - ( - (load-param-object v0) - (load-param-object v1) - (invoke-virtual (v1) "LI;.foo:()V") - (return-void) - ) - )")); - IRTypeChecker checker(method); - checker.run(); - EXPECT_TRUE(checker.fail()); - } - - { - auto method = DexMethod::make_method("LFoo;.bar:(LI;)V;") - ->make_concrete(ACC_PUBLIC, /* is_virtual */ false); - method->set_code(assembler::ircode_from_string(R"( - ( - (load-param-object v0) - (load-param-object v1) - (invoke-interface (v1) "LI;.foo:()V") - (return-void) - ) - )")); - IRTypeChecker checker(method); - checker.run(); - EXPECT_FALSE(checker.fail()); - } -} diff --git a/test/unit/InitClassLoweringPassTest.cpp b/test/unit/InitClassLoweringPassTest.cpp index ef206ff2db..313aba2bc7 100644 --- a/test/unit/InitClassLoweringPassTest.cpp +++ b/test/unit/InitClassLoweringPassTest.cpp @@ -51,7 +51,7 @@ class InitClassLoweringPassTest : public RedexTest { add_sfield(c_type, type::_double()); std::string class_name = "LTest;"; - ClassCreator creator(DexType::make_type(class_name)); + ClassCreator creator(DexType::make_type(class_name.c_str())); creator.set_super(type::java_lang_Object()); auto signature = class_name + ".foo:()V"; auto method = DexMethod::make_method(signature)->make_concrete( diff --git a/test/unit/LocalPointersTest.cpp b/test/unit/LocalPointersTest.cpp index a5dcdf678a..c880327639 100644 --- a/test/unit/LocalPointersTest.cpp +++ b/test/unit/LocalPointersTest.cpp @@ -80,7 +80,7 @@ TEST_F(LocalPointersTest, simple) { ptrs::FixpointIterator fp_iter(cfg, invoke_to_summary_map); fp_iter.run(ptrs::Environment()); - const auto& exit_env = fp_iter.get_exit_state_at(cfg.exit_block()); + auto exit_env = fp_iter.get_exit_state_at(cfg.exit_block()); EXPECT_EQ(exit_env.get_pointers(0).size(), 2); EXPECT_THAT(exit_env.get_pointers(0).elements(), UnorderedElementsAre( @@ -88,7 +88,7 @@ TEST_F(LocalPointersTest, simple) { .set_type(DexType::get_type("LFoo;"))))), Pointee(Eq(*( IRInstruction(IOPCODE_LOAD_PARAM_OBJECT).set_dest(0)))))); - const auto& pointers = exit_env.get_pointers(0); + auto pointers = exit_env.get_pointers(0); for (auto insn : pointers.elements()) { if (insn->opcode() == IOPCODE_LOAD_PARAM_OBJECT || insn->opcode() == OPCODE_NEW_INSTANCE) { @@ -121,8 +121,8 @@ TEST_F(LocalPointersTest, aliasEscape) { ptrs::FixpointIterator fp_iter(cfg, invoke_to_summary_map); fp_iter.run(ptrs::Environment()); - const auto& exit_env = fp_iter.get_exit_state_at(cfg.exit_block()); - const auto& returned_ptrs = exit_env.get_pointers(0); + auto exit_env = fp_iter.get_exit_state_at(cfg.exit_block()); + auto returned_ptrs = exit_env.get_pointers(0); EXPECT_EQ(returned_ptrs.size(), 2); EXPECT_THAT( returned_ptrs.elements(), @@ -155,10 +155,10 @@ TEST_F(LocalPointersTest, filledNewArrayEscape) { ptrs::FixpointIterator fp_iter(cfg, invoke_to_summary_map); fp_iter.run(ptrs::Environment()); - const auto& exit_env = fp_iter.get_exit_state_at(cfg.exit_block()); - const auto& foo_ptr_set = exit_env.get_pointers(0); + auto exit_env = fp_iter.get_exit_state_at(cfg.exit_block()); + auto foo_ptr_set = exit_env.get_pointers(0); EXPECT_EQ(foo_ptr_set.size(), 1); - const auto& foo_ptr = *foo_ptr_set.elements().begin(); + auto foo_ptr = *foo_ptr_set.elements().begin(); EXPECT_EQ(*foo_ptr, *(IRInstruction(OPCODE_NEW_INSTANCE) .set_type(DexType::get_type("LFoo;")))); @@ -326,7 +326,7 @@ TEST_F(LocalPointersTest, returnFreshValue) { ptrs::FixpointIterator fp_iter(cfg, invoke_to_summary_map); fp_iter.run(ptrs::Environment()); - const auto& exit_env = fp_iter.get_exit_state_at(cfg.exit_block()); + auto exit_env = fp_iter.get_exit_state_at(cfg.exit_block()); EXPECT_THAT(exit_env.get_pointers(0).elements(), UnorderedElementsAre(Pointee(Eq(*invoke_insn)))); EXPECT_FALSE(exit_env.may_have_escaped(invoke_insn)); @@ -392,7 +392,7 @@ TEST_F(LocalPointersTest, returnEscapedValue) { ptrs::FixpointIterator fp_iter(cfg, invoke_to_summary_map); fp_iter.run(ptrs::Environment()); - const auto& exit_env = fp_iter.get_exit_state_at(cfg.exit_block()); + auto exit_env = fp_iter.get_exit_state_at(cfg.exit_block()); EXPECT_THAT(exit_env.get_pointers(0).elements(), UnorderedElementsAre(Pointee(Eq(*invoke_insn)))); EXPECT_TRUE(exit_env.may_have_escaped(invoke_insn)); diff --git a/test/unit/Makefile.am b/test/unit/Makefile.am index c3bf93cce2..9e1ec82762 100644 --- a/test/unit/Makefile.am +++ b/test/unit/Makefile.am @@ -17,7 +17,6 @@ check_PROGRAMS = \ analysis_usage_test \ array_propagation_test \ assert_test \ - atomic_map_test \ blaming_escape_test \ boxed_boolean_propagation_test \ branch_prefix_hoisting_test \ @@ -28,7 +27,6 @@ check_PROGRAMS = \ check_breadcrumbs_test \ check_cast_analysis_test \ concurrent_containers_test \ - concurrent_hashtable_test \ configurable_test \ constructor_analysis_test \ control_flow_test \ @@ -133,6 +131,7 @@ check_PROGRAMS = \ strip_debug_info_test \ switch_dispatch_test \ switch_equiv_test \ + switch_partitioning_test \ timer_test \ trace_multithreading_test \ true_virtuals_test \ @@ -184,10 +183,6 @@ class_checker_test_SOURCES = ClassCheckerTest.cpp ScopeHelper.cpp concurrent_containers_test_SOURCES = ConcurrentContainersTest.cpp -concurrent_hashtable_test_SOURCES = ConcurrentHashtableTest.cpp - -atomic_map_test_SOURCES = AtomicMapTest.cpp - configurable_test_SOURCES = ConfigurableTest.cpp configurable_test_LDADD = $(COMMON_MOCK_TEST_LIBS) @@ -432,6 +427,8 @@ switch_dispatch_test_SOURCES = SwitchDispatchTest.cpp switch_equiv_test_SOURCES = SwitchEquivFinderTest.cpp switch_equiv_test_LDADD = $(COMMON_MOCK_TEST_LIBS) +switch_partitioning_test_SOURCES = SwitchPartitioningTest.cpp + # throw_propagation_test_SOURCES = ThrowPropagationTest.cpp # throw_propagation_test_LDADD = $(COMMON_MOCK_TEST_LIBS) @@ -493,8 +490,6 @@ TESTS = \ check_cast_analysis_test \ class_checker_test \ concurrent_containers_test \ - concurrent_hashtable_test \ - atomic_map_test \ configurable_test \ constructor_analysis_test \ control_flow_test \ @@ -592,6 +587,7 @@ TESTS = \ strip_debug_info_test \ switch_dispatch_test \ switch_equiv_test \ + switch_partitioning_test \ timer_test \ trace_multithreading_test \ true_virtuals_test \ diff --git a/test/unit/MethodInlineTest.cpp b/test/unit/MethodInlineTest.cpp index 4079d86ea2..7ab761165d 100644 --- a/test/unit/MethodInlineTest.cpp +++ b/test/unit/MethodInlineTest.cpp @@ -18,7 +18,6 @@ #include "LegacyInliner.h" #include "RedexTest.h" #include "VirtualScope.h" -#include "Walkers.h" struct MethodInlineTest : public RedexTest { MethodInlineTest() { @@ -465,7 +464,6 @@ TEST_F(MethodInlineTest, test_intra_dex_inlining) { expected_inlined.insert(bar_m2); } auto scope = build_class_scope(stores); - walk::parallel::code(scope, [&](auto*, IRCode& code) { code.build_cfg(); }); api::LevelChecker::init(0, scope); inliner::InlinerConfig inliner_config; @@ -526,7 +524,6 @@ TEST_F(MethodInlineTest, test_intra_dex_inlining_new_references) { // bring a new reference from baz_m1. } auto scope = build_class_scope(stores); - walk::parallel::code(scope, [&](auto*, IRCode& code) { code.build_cfg(); }); api::LevelChecker::init(0, scope); inliner::InlinerConfig inliner_config; @@ -612,7 +609,6 @@ TEST_F(MethodInlineTest, test_intra_dex_inlining_init_class) { // Expect bar_m1 not to be inlined, as it has an init-class instruction. } auto scope = build_class_scope(stores); - walk::parallel::code(scope, [&](auto*, IRCode& code) { code.build_cfg(); }); api::LevelChecker::init(0, scope); inliner::InlinerConfig inliner_config; @@ -662,7 +658,6 @@ TEST_F(MethodInlineTest, size_limit) { make_a_method_calls_others(bar_cls, "bar_main", {bar_m1}); } auto scope = build_class_scope(stores); - walk::parallel::code(scope, [&](auto*, IRCode& code) { code.build_cfg(); }); api::LevelChecker::init(0, scope); inliner::InlinerConfig inliner_config; @@ -702,7 +697,6 @@ TEST_F(MethodInlineTest, minimal_self_loop_regression) { expected_inlined.insert(foo_m1); } auto scope = build_class_scope(stores); - walk::parallel::code(scope, [&](auto*, IRCode& code) { code.build_cfg(); }); api::LevelChecker::init(0, scope); inliner::InlinerConfig inliner_config; inliner_config.populate(scope); @@ -749,7 +743,6 @@ TEST_F(MethodInlineTest, non_unique_inlined_registers) { expected_inlined.insert(foo_m2); } auto scope = build_class_scope(stores); - walk::parallel::code(scope, [&](auto*, IRCode& code) { code.build_cfg(); }); api::LevelChecker::init(0, scope); inliner::InlinerConfig inliner_config; inliner_config.populate(scope); @@ -768,7 +761,6 @@ TEST_F(MethodInlineTest, non_unique_inlined_registers) { EXPECT_EQ(inlined.count(method), 1); } - walk::parallel::code(scope, [&](auto*, IRCode& code) { code.clear_cfg(); }); // Note: the position is an artifact and may get cleaned up. const auto& expected_str = R"( ( diff --git a/test/unit/MethodSplittingTest.cpp b/test/unit/MethodSplittingTest.cpp index a9285cd8e8..c1d2e0d7f2 100644 --- a/test/unit/MethodSplittingTest.cpp +++ b/test/unit/MethodSplittingTest.cpp @@ -29,7 +29,7 @@ class MethodSplitterTest : public RedexTest { // Create a totally new class. size_t c = s_counter.fetch_add(1); std::string name = std::string("LFoo") + std::to_string(c) + ";"; - ClassCreator cc{DexType::make_type(name)}; + ClassCreator cc{DexType::make_type(name.c_str())}; cc.set_super(type::java_lang_Object()); auto m = diff --git a/test/unit/ObjectInlinerTest.cpp b/test/unit/ObjectInlinerTest.cpp index 4bc188a5bf..3bb2468b58 100644 --- a/test/unit/ObjectInlinerTest.cpp +++ b/test/unit/ObjectInlinerTest.cpp @@ -62,8 +62,8 @@ void test_object_inliner( const std::vector& srcs, const std::string& expected_str, boost::optional callee_ctor_str = boost::none) { - DexType* callee_type = DexType::make_type(callee_class); - DexType* caller_type = DexType::make_type(caller_class); + DexType* callee_type = DexType::make_type(callee_class.c_str()); + DexType* caller_type = DexType::make_type(caller_class.c_str()); std::vector field_refs = {}; for (const auto& field_data : fields) { diff --git a/test/unit/PeepholeTest.cpp b/test/unit/PeepholeTest.cpp index 30168a31be..30bbe7339e 100644 --- a/test/unit/PeepholeTest.cpp +++ b/test/unit/PeepholeTest.cpp @@ -24,7 +24,7 @@ class PeepholeTest : public RedexTest { public: ::sparta::s_expr run_peephole_pass(const std::string& code) { auto class_name = next_class(); - ClassCreator creator(DexType::make_type(class_name)); + ClassCreator creator(DexType::make_type(class_name.c_str())); creator.set_super(type::java_lang_Object()); auto signature = class_name + ".foo:()V"; auto method = DexMethod::make_method(signature)->make_concrete( diff --git a/test/unit/PrintKotlinStatsTest.cpp b/test/unit/PrintKotlinStatsTest.cpp index ef54053ef0..1108607c30 100644 --- a/test/unit/PrintKotlinStatsTest.cpp +++ b/test/unit/PrintKotlinStatsTest.cpp @@ -89,10 +89,7 @@ TEST_F(PrintKotlinStatsTest, SimpleArgumentPassingTest) { pass.setup(); PrintKotlinStats::Stats stats = walk::parallel::methods( - scope, [&](DexMethod* meth) { - meth->get_code()->build_cfg(); - return pass.handle_method(meth); - }); + scope, [&](DexMethod* meth) { return pass.handle_method(meth); }); ASSERT_EQ(stats.kotlin_null_check_insns, 2); } diff --git a/test/unit/ReduceGotosTest.cpp b/test/unit/ReduceGotosTest.cpp index c0ea72c88f..a286d0c7c9 100644 --- a/test/unit/ReduceGotosTest.cpp +++ b/test/unit/ReduceGotosTest.cpp @@ -28,7 +28,7 @@ void test(const std::string& code_str, auto code = assembler::ircode_from_string(code_str); auto expected = assembler::ircode_from_string(expected_str); - code->build_cfg(); + ReduceGotosPass::Stats stats = ReduceGotosPass::process_code(code.get()); EXPECT_EQ(expected_replaced_gotos_with_returns, stats.replaced_gotos_with_returns); @@ -42,7 +42,7 @@ void test(const std::string& code_str, EXPECT_EQ(expected_removed_switch_cases, stats.removed_switch_cases); EXPECT_EQ(expected_replaced_trivial_switches, stats.replaced_trivial_switches); - code->clear_cfg(); + EXPECT_EQ(assembler::to_s_expr(code.get()), assembler::to_s_expr(expected.get())) << " " << assembler::to_s_expr(code.get()).str() << "\n---\n" diff --git a/test/unit/RemoveApiLevelChecksTest.cpp b/test/unit/RemoveApiLevelChecksTest.cpp index 1e09ce6b5f..51833506e5 100644 --- a/test/unit/RemoveApiLevelChecksTest.cpp +++ b/test/unit/RemoveApiLevelChecksTest.cpp @@ -20,9 +20,7 @@ class RemoveApiLevelChecksTest : public RedexTest { public: size_t run(IRCode* code, int32_t min_sdk) { const auto* sdk_int_field = RemoveApiLevelChecksPass::get_sdk_int_field(); - code->build_cfg(); auto res = RemoveApiLevelChecksPass::run(code, min_sdk, sdk_int_field); - code->clear_cfg(); return res.num_removed; } diff --git a/test/unit/RemoveRecursiveLocksTest.cpp b/test/unit/RemoveRecursiveLocksTest.cpp index d25c2019a4..7a6f28a4d0 100644 --- a/test/unit/RemoveRecursiveLocksTest.cpp +++ b/test/unit/RemoveRecursiveLocksTest.cpp @@ -34,7 +34,6 @@ TEST_F(RemoveRecursiveLocksTest, no_single_blocks) { (monitor-exit v1) ) ))"); - method->get_code()->build_cfg(); auto res = RemoveRecursiveLocksPass::run(method, method->get_code()); ASSERT_FALSE(res); } @@ -82,10 +81,10 @@ TEST_F(RemoveRecursiveLocksTest, recursion) { ) ))"); auto code = method->get_code(); - code->build_cfg(); + auto res = RemoveRecursiveLocksPass::run(method, code); ASSERT_TRUE(res); - code->clear_cfg(); + auto expected_code = assembler::ircode_from_string(R"( ( (load-param-object v0) @@ -178,10 +177,10 @@ TEST_F(RemoveRecursiveLocksTest, recursion_nested) { ) ))"); auto code = method->get_code(); - code->build_cfg(); + auto res = RemoveRecursiveLocksPass::run(method, code); ASSERT_TRUE(res); - code->clear_cfg(); + auto expected_code = assembler::ircode_from_string(R"( ( (load-param-object v1) diff --git a/test/unit/RemoveUnusedArgsTest.cpp b/test/unit/RemoveUnusedArgsTest.cpp index 99c315c1d6..666512534a 100644 --- a/test/unit/RemoveUnusedArgsTest.cpp +++ b/test/unit/RemoveUnusedArgsTest.cpp @@ -23,7 +23,6 @@ struct RemoveUnusedArgsTest : public RedexTest { remove_unused_args::RemoveArgs* m_remove_args; std::vector m_blocklist; - std::unordered_set m_pure_methods; RemoveUnusedArgsTest() { Scope dummy_scope; @@ -35,8 +34,7 @@ struct RemoveUnusedArgsTest : public RedexTest { auto dummy_cls = create_internal_class(dummy_t, obj_t, {}); dummy_scope.push_back(dummy_cls); m_remove_args = new remove_unused_args::RemoveArgs( - dummy_scope, dummy_init_classes_with_side_effects, m_blocklist, - m_pure_methods); + dummy_scope, dummy_init_classes_with_side_effects, m_blocklist); } ~RemoveUnusedArgsTest() {} diff --git a/test/unit/ResolveProguardAssumeValuesTest.cpp b/test/unit/ResolveProguardAssumeValuesTest.cpp index af9bd5b058..6bdd0cf2cb 100644 --- a/test/unit/ResolveProguardAssumeValuesTest.cpp +++ b/test/unit/ResolveProguardAssumeValuesTest.cpp @@ -44,9 +44,10 @@ void test(const std::string& code_str, const std::string& expected_str) { auto code = assembler::ircode_from_string(code_str); auto expected = assembler::ircode_from_string(expected_str); + + ResolveProguardAssumeValuesPass::process_for_code(code.get()); auto code_ptr = code.get(); code_ptr->build_cfg(); - ResolveProguardAssumeValuesPass::process_for_code(code_ptr->cfg()); auto& cfg = code_ptr->cfg(); std::cerr << "after:" << std::endl << SHOW(cfg); diff --git a/test/unit/ResourceSerializationTest.cpp b/test/unit/ResourceSerializationTest.cpp index 9ee86e39f9..287c468998 100644 --- a/test/unit/ResourceSerializationTest.cpp +++ b/test/unit/ResourceSerializationTest.cpp @@ -21,7 +21,6 @@ #include "SanitizersConfig.h" #include "Util.h" #include "androidfw/ResourceTypes.h" -#include "arsc/TestStructures.h" #include "utils/Errors.h" #include "utils/Serialize.h" #include "utils/Visitor.h" @@ -388,8 +387,8 @@ ParsedAaptOutput aapt_dump_and_parse(const std::string& arsc_path, uint32_t data = std::stoul(what[5], nullptr, 16); output.id_fully_qualified_names.emplace(id, fully_qualified); output.fully_qualified_name_to_id.emplace(fully_qualified, id); - android::Res_value value{sizeof(android::Res_value), 0, (uint8_t)type, - data}; + android::Res_value value{ + sizeof(android::Res_value), 0, (uint8_t)type, data}; SimpleEntry entry{id, fully_qualified, value}; simple_values.emplace(id, std::move(entry)); } else if (boost::regex_search(line, what, bag_exp)) { @@ -570,6 +569,11 @@ TEST(ResTable, AppendNewType) { } } + // Create a default looking ResTable_config + android::ResTable_config default_config; + memset(&default_config, 0, sizeof(android::ResTable_config)); + default_config.size = sizeof(android::ResTable_config); + // Write a new .arsc file { ResourcesArscFile arsc_file(dest_file_path); @@ -814,7 +818,12 @@ TEST(ResTableParse, TestUnknownPackageChunks) { } TEST(Configs, TestConfigEquivalence) { + android::ResTable_config default_config{}; + default_config.size = sizeof(android::ResTable_config); EXPECT_TRUE(arsc::are_configs_equivalent(&default_config, &default_config)); + android::ResTable_config land_config{}; + land_config.size = sizeof(android::ResTable_config); + land_config.orientation = android::ResTable_config::ORIENTATION_LAND; EXPECT_FALSE(arsc::are_configs_equivalent(&default_config, &land_config)); // Configs of different sizes (simulate some of our snapshots of older files) { @@ -856,6 +865,70 @@ TEST(ResTable, TestBuilderRoundTrip) { } namespace { +PACKED(struct EntryAndValue { + android::ResTable_entry entry{}; + android::Res_value value{}; + EntryAndValue(uint32_t key_string_idx, uint8_t data_type, uint32_t data) { + entry.size = sizeof(android::ResTable_entry); + entry.key.index = key_string_idx; + value.size = sizeof(android::Res_value); + value.dataType = data_type; + value.data = data; + } +}); + +// For testing simplicity, a map that has two items in it. +PACKED(struct MapEntryAndValues { + android::ResTable_map_entry entry{}; + android::ResTable_map item0{}; + android::ResTable_map item1{}; + MapEntryAndValues(uint32_t key_string_idx, uint32_t parent_ident) { + entry.size = sizeof(android::ResTable_map_entry); + entry.count = 2; + entry.flags = android::ResTable_entry::FLAG_COMPLEX; + entry.key.index = key_string_idx; + entry.parent.ident = parent_ident; + item0.value.size = sizeof(android::Res_value); + item1.value.size = sizeof(android::Res_value); + } +}); +} // namespace + +TEST(ResTable, ComputeSizes) { + EntryAndValue simple(0, android::Res_value::TYPE_DIMENSION, 1000); + EXPECT_EQ(arsc::compute_entry_value_length(&simple.entry), + sizeof(EntryAndValue)); + MapEntryAndValues complex(1, 0); + EXPECT_EQ(arsc::compute_entry_value_length(&complex.entry), + sizeof(MapEntryAndValues)); +} + +namespace { +// Data for a simple arsc file that many tests can get written against. +EntryAndValue e0(0, android::Res_value::TYPE_DIMENSION, 1000); +EntryAndValue e0_land(0, android::Res_value::TYPE_DIMENSION, 1001); +EntryAndValue e1(1, android::Res_value::TYPE_DIMENSION, 2000); +EntryAndValue e2(2, android::Res_value::TYPE_REFERENCE, 0x7f010001); +EntryAndValue id_0(0, android::Res_value::TYPE_INT_BOOLEAN, 0); +EntryAndValue id_1(1, android::Res_value::TYPE_INT_BOOLEAN, 0); +EntryAndValue id_2(2, android::Res_value::TYPE_INT_BOOLEAN, 0); +MapEntryAndValues style(3, 0); + +// The package that all tests to follow will be in +android::ResTable_package package_header{.id = 0x7f, + .name = {'f', 'o', 'o', '\0'}}; +// Create a default ResTable_config +android::ResTable_config default_config = { + .size = sizeof(android::ResTable_config)}; +// Create a landscape config +android::ResTable_config land_config = { + .size = sizeof(android::ResTable_config), + .orientation = android::ResTable_config::ORIENTATION_LAND}; +// And a xxhdpi config +android::ResTable_config xxhdpi_config = { + .size = sizeof(android::ResTable_config), + .density = android::ResTable_config::DENSITY_XXHIGH}; + void build_arsc_file_and_validate( const std::function& callback) { @@ -880,7 +953,7 @@ void build_arsc_file_and_validate( } auto package_builder = - std::make_shared(&foo_package); + std::make_shared(&package_header); package_builder->set_key_strings(key_strings_builder); package_builder->set_type_strings(type_strings_builder); @@ -896,7 +969,7 @@ void build_arsc_file_and_validate( std::vector dimen_flags = { android::ResTable_config::CONFIG_ORIENTATION, 0, 0}; auto dimen_type_definer = std::make_shared( - foo_package.id, 1, dimen_configs, dimen_flags); + package_header.id, 1, dimen_configs, dimen_flags); package_builder->add_type(dimen_type_definer); dimen_type_definer->add(&default_config, &e0); @@ -911,7 +984,7 @@ void build_arsc_file_and_validate( std::vector style_flags = { android::ResTable_config::CONFIG_DENSITY}; auto style_type_definer = std::make_shared( - foo_package.id, 2, style_configs, style_flags); + package_header.id, 2, style_configs, style_flags); package_builder->add_type(style_type_definer); style.item0.name.ident = 0x01010098; // android:textColor @@ -1041,15 +1114,6 @@ TEST(ResTable, BuildNewTable) { }); } -TEST(ResTable, ComputeSizes) { - EntryAndValue simple(0, android::Res_value::TYPE_DIMENSION, 1000); - EXPECT_EQ(arsc::compute_entry_value_length(&simple.entry), - sizeof(EntryAndValue)); - MapEntryAndValues complex(1, 0); - EXPECT_EQ(arsc::compute_entry_value_length(&complex.entry), - sizeof(MapEntryAndValues)); -} - TEST(ResTable, DeleteAllEntriesInType) { build_arsc_file_and_validate( [&](const std::string& /* unused */, const std::string& arsc_path) { @@ -1142,7 +1206,7 @@ TEST(ResTable, SerializeTypeWithAllEmpty) { type_strings_builder->add_string(type_name.c_str(), type_name.size()); auto package_builder = - std::make_shared(&foo_package); + std::make_shared(&package_header); package_builder->set_key_strings(key_strings_builder); package_builder->set_type_strings(type_strings_builder); @@ -1154,7 +1218,7 @@ TEST(ResTable, SerializeTypeWithAllEmpty) { std::vector dimen_configs = {&default_config}; std::vector dimen_flags = {0, 0, 0}; auto dimen_type_definer = std::make_shared( - foo_package.id, 1, dimen_configs, dimen_flags); + package_header.id, 1, dimen_configs, dimen_flags); package_builder->add_type(dimen_type_definer); dimen_type_definer->add_empty(&default_config); dimen_type_definer->add_empty(&default_config); @@ -1204,7 +1268,7 @@ void build_table_with_ids(const std::string& dest_file_path, type_strings_builder->add_string(type_name.c_str(), type_name.size()); auto package_builder = - std::make_shared(&foo_package); + std::make_shared(&package_header); package_builder->set_key_strings(key_strings_builder); package_builder->set_type_strings(type_strings_builder); @@ -1216,7 +1280,7 @@ void build_table_with_ids(const std::string& dest_file_path, std::vector id_configs = {&default_config}; std::vector flags = {0, 0, 0, 0, 0, 0}; auto type_definer = std::make_shared( - foo_package.id, 1, id_configs, flags, canonical_entries); + package_header.id, 1, id_configs, flags, canonical_entries); package_builder->add_type(type_definer); type_definer->add(&default_config, &id_0); // When canonical_entries is true, following three items will generate three @@ -1248,7 +1312,7 @@ TEST(ResTable, ValueEquality) { type_strings_builder->add_string("dimen"); auto package_builder = - std::make_shared(&foo_package); + std::make_shared(&package_header); package_builder->set_key_strings(key_strings_builder); package_builder->set_type_strings(type_strings_builder); @@ -1260,7 +1324,7 @@ TEST(ResTable, ValueEquality) { &default_config, &land_config, &xxhdpi_config}; std::vector dimen_flags = {0, 0, 0, 0, 0}; auto type_definer = std::make_shared( - foo_package.id, 1, dimen_configs, dimen_flags); + package_header.id, 1, dimen_configs, dimen_flags); package_builder->add_type(type_definer); EntryAndValue a(0, android::Res_value::TYPE_INT_COLOR_RGB8, 123456); @@ -1361,7 +1425,7 @@ TEST(ResTable, CanonicalEntryData) { apk::TableParser parsed_table; parsed_table.visit((void*)no_canon_file.const_data(), no_canon_file.size()); auto package_builder = - std::make_shared(&foo_package); + std::make_shared(&package_header); package_builder->set_key_strings( parsed_table.m_package_key_string_headers.begin()->second); package_builder->set_type_strings( @@ -1369,7 +1433,7 @@ TEST(ResTable, CanonicalEntryData) { auto& id_type = parsed_table.m_package_types.begin()->second.at(0); auto type_projector = std::make_shared( - foo_package.id, id_type.spec, id_type.configs, true); + package_header.id, id_type.spec, id_type.configs, true); package_builder->add_type(type_projector); auto table_builder = std::make_shared(); @@ -1405,7 +1469,7 @@ TEST(ResTable, GetStringsByName) { type_strings_builder->add_string("string"); auto package_builder = - std::make_shared(&foo_package); + std::make_shared(&package_header); package_builder->set_key_strings(key_strings_builder); package_builder->set_type_strings(type_strings_builder); @@ -1420,7 +1484,7 @@ TEST(ResTable, GetStringsByName) { android::ResTable_config::CONFIG_ORIENTATION, android::ResTable_config::CONFIG_ORIENTATION}; auto type_definer = std::make_shared( - foo_package.id, 1, string_configs, string_flags); + package_header.id, 1, string_configs, string_flags); package_builder->add_type(type_definer); EntryAndValue first(0, android::Res_value::TYPE_STRING, 0); @@ -1489,7 +1553,7 @@ TEST(ResTable, BuildDumpAndParseSparseType) { type_strings_builder->add_string("dimen"); auto package_builder = - std::make_shared(&foo_package); + std::make_shared(&package_header); package_builder->set_key_strings(key_strings_builder); package_builder->set_type_strings(type_strings_builder); @@ -1505,7 +1569,7 @@ TEST(ResTable, BuildDumpAndParseSparseType) { 0, 0, 0, 0}; auto type_definer = std::make_shared( - foo_package.id, 1, dimen_configs, flags, true, true); + package_header.id, 1, dimen_configs, flags, true, true); package_builder->add_type(type_definer); // Add all 8 values diff --git a/test/unit/ScopeHelper.cpp b/test/unit/ScopeHelper.cpp index 42bb61fef4..d17c7dc36c 100644 --- a/test/unit/ScopeHelper.cpp +++ b/test/unit/ScopeHelper.cpp @@ -83,12 +83,16 @@ DexClass* create_java_lang_object() { method->set_external(); obj_cls->add_method(method); - // protected java.lang.Object.finalize()V method = static_cast( - DexMethod::make_method(obj_t, finalize, void_void)); - method->set_access(ACC_PROTECTED); - method->set_virtual(true); - method->set_external(); + DexMethod::get_method(obj_t, finalize, void_void)); + if (method == nullptr) { + // protected java.lang.Object.finalize()V + method = static_cast( + DexMethod::make_method(obj_t, finalize, void_void)); + method->set_access(ACC_PROTECTED); + method->set_virtual(true); + method->set_external(); + } obj_cls->add_method(method); method = static_cast( diff --git a/test/unit/SimpleGraph.h b/test/unit/SimpleGraph.h index 117a2e1ab2..e9a3ae6e4e 100644 --- a/test/unit/SimpleGraph.h +++ b/test/unit/SimpleGraph.h @@ -7,8 +7,6 @@ #pragma once -#include - #include #include diff --git a/test/unit/SourceBlocksTest.cpp b/test/unit/SourceBlocksTest.cpp index 2f40b64415..4c00aad98a 100644 --- a/test/unit/SourceBlocksTest.cpp +++ b/test/unit/SourceBlocksTest.cpp @@ -36,7 +36,7 @@ class SourceBlocksTest : public RedexTest { // Create a totally new class. size_t c = s_counter.fetch_add(1); std::string name = class_name + std::to_string(c) + ";"; - ClassCreator cc{DexType::make_type(name)}; + ClassCreator cc{DexType::make_type(name.c_str())}; cc.set_super(type::java_lang_Object()); // Empty code isn't really legal. But it does not matter for us. diff --git a/test/unit/SplitHugeSwitchTest.cpp b/test/unit/SplitHugeSwitchTest.cpp index fbcb2382b3..d90c613576 100644 --- a/test/unit/SplitHugeSwitchTest.cpp +++ b/test/unit/SplitHugeSwitchTest.cpp @@ -33,7 +33,7 @@ class SplitHugeSwitchTest : public RedexTest { // Create a totally new class. size_t c = s_counter.fetch_add(1); std::string name = std::string("LFoo") + std::to_string(c) + ";"; - ClassCreator cc{DexType::make_type(name)}; + ClassCreator cc{DexType::make_type(name.c_str())}; cc.set_super(type::java_lang_Object()); auto m = diff --git a/test/unit/StaticReloV2Test.cpp b/test/unit/StaticReloV2Test.cpp index 412bdf8578..ee7d7785e3 100644 --- a/test/unit/StaticReloV2Test.cpp +++ b/test/unit/StaticReloV2Test.cpp @@ -10,11 +10,10 @@ #include "ApiLevelChecker.h" #include "Creators.h" -#include "DexAsm.h" #include "IRAssembler.h" #include "RedexTest.h" + #include "StaticReloV2.h" -#include "Walkers.h" namespace static_relo_v2 { @@ -40,8 +39,6 @@ struct StaticReloV2Test : public RedexTest { cls->get_type(), DexString::make_string(method_name), m_proto) ->make_concrete(access, false); method->set_code(std::make_unique(method, 1)); - method->get_code()->push_back(dex_asm::dasm(OPCODE_RETURN_VOID)); - method->get_code()->build_cfg(); cls->add_method(method); return method; } @@ -49,9 +46,7 @@ struct StaticReloV2Test : public RedexTest { void call(DexMethod* caller, DexMethod* callee) { IRInstruction* inst = new IRInstruction(OPCODE_INVOKE_STATIC); inst->set_method(callee); - auto& cfg = caller->get_code()->cfg(); - auto ii = cfg::InstructionIterable(cfg); - cfg.insert_before(ii.begin(), inst); + caller->get_code()->push_back(inst); } }; diff --git a/test/unit/StringBuilderOutlinerTest.cpp b/test/unit/StringBuilderOutlinerTest.cpp index aa710f3cfc..0bbc2a9198 100644 --- a/test/unit/StringBuilderOutlinerTest.cpp +++ b/test/unit/StringBuilderOutlinerTest.cpp @@ -22,7 +22,10 @@ namespace uv = used_vars; class StringBuilderOutlinerTest : public RedexTest { public: void SetUp() override { - std::string sdk_jar = android_sdk_jar_path(); + const char* android_sdk = std::getenv("sdk_path"); + std::string android_target(std::getenv("android_target")); + std::string sdk_jar = std::string(android_sdk) + "/platforms/" + + android_target + "/android.jar"; // StringBuilderOutliner requires bunch of java.lang.* classes to be // defined. Loading the SDK JAR here ensures that. ASSERT_TRUE(load_jar_file(DexLocation::make_location("", sdk_jar))); diff --git a/test/unit/StripDebugInfoTest.cpp b/test/unit/StripDebugInfoTest.cpp index 0a251e5a6e..537394d163 100644 --- a/test/unit/StripDebugInfoTest.cpp +++ b/test/unit/StripDebugInfoTest.cpp @@ -24,9 +24,7 @@ void test(const StripDebugInfoPass::Config& config, const char* o) { auto code = assembler::ircode_from_string(i); code->set_registers_size(3); - code->build_cfg(); StripDebugInfo(config).run(*code); - code->clear_cfg(); auto expected_code = assembler::ircode_from_string(o); EXPECT_EQ(assembler::to_s_expr(code.get()), assembler::to_s_expr(expected_code.get())); diff --git a/test/unit/SwitchPartitioningTest.cpp b/test/unit/SwitchPartitioningTest.cpp new file mode 100644 index 0000000000..096c869d0e --- /dev/null +++ b/test/unit/SwitchPartitioningTest.cpp @@ -0,0 +1,65 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "IRAssembler.h" +#include "RedexTest.h" +#include "Show.h" +#include "SwitchMethodPartitioning.h" + +class SwitchPartitioningTest : public RedexTest {}; + +TEST_F(SwitchPartitioningTest, if_chains) { + auto code1 = assembler::ircode_from_string(R"( + ( + (load-param v1) + (const v0 1) + (if-eq v1 v0 :if-1) + (const v0 2) + (if-eq v1 v0 :if-2) + (const v1 48) + (return v1) + (:if-2) + (const v1 47) + (return v1) + (:if-1) + (const v1 46) + (return v1) + ) + )"); + auto smp1 = SwitchMethodPartitioning::create(code1.get(), + /* verify_default_case */ false); + ASSERT_TRUE(smp1); + const auto& key_to_block1 = smp1->get_key_to_block(); + EXPECT_EQ(key_to_block1.size(), 2); + + auto code2 = assembler::ircode_from_string(R"( + ( + (load-param v0) + (switch v0 (:case-1 :case-2)) + (const v1 48) + (return v1) + (:case-1 1) + (const v1 46) + (return v1) + (:case-2 2) + (const v1 47) + (return v1) + ) + )"); + auto smp2 = SwitchMethodPartitioning::create(code2.get(), + /* verify_default_case */ false); + ASSERT_TRUE(smp2); + const auto& key_to_block2 = smp2->get_key_to_block(); + EXPECT_EQ(key_to_block2.size(), 2); + for (size_t key = 1; key <= 2; key++) { + auto* block1 = key_to_block1.at(key); + auto* block2 = key_to_block2.at(key); + EXPECT_TRUE(block1->structural_equals(block2)) << key << " : \n" + << show(block1) << "v.s.\n" + << show(block2); + } +} diff --git a/test/unit/ThrowPropagationTest.cpp b/test/unit/ThrowPropagationTest.cpp index f61681ca12..a8c4d41b58 100644 --- a/test/unit/ThrowPropagationTest.cpp +++ b/test/unit/ThrowPropagationTest.cpp @@ -97,7 +97,11 @@ TEST_F(ThrowPropagationTest, cannot_return_simple) { auto expected_str = R"( ( (invoke-static () "LFoo;.bar:()V") - (const v0 0) + (new-instance "Ljava/lang/RuntimeException;") + (move-result-pseudo-object v0) + (const-string "Redex: Unreachable code after no-return invoke") + (move-result-pseudo-object v1) + (invoke-direct (v0 v1) "Ljava/lang/RuntimeException;.:(Ljava/lang/String;)V") (throw v0) ) )"; @@ -129,7 +133,11 @@ TEST_F(ThrowPropagationTest, cannot_return_remove_move_result) { auto expected_str = R"( ( (invoke-static () "LFoo;.bar:()I") - (const v2 0) + (new-instance "Ljava/lang/RuntimeException;") + (move-result-pseudo-object v2) + (const-string "Redex: Unreachable code after no-return invoke") + (move-result-pseudo-object v3) + (invoke-direct (v2 v3) "Ljava/lang/RuntimeException;.:(Ljava/lang/String;)V") (throw v2) ) )"; diff --git a/test/unit/TrueVirtualsTest.cpp b/test/unit/TrueVirtualsTest.cpp index c56aaf50a2..a7e618b0c1 100644 --- a/test/unit/TrueVirtualsTest.cpp +++ b/test/unit/TrueVirtualsTest.cpp @@ -409,9 +409,8 @@ std::vector create_scope_10() { // Utilities for tests // -template std::unordered_set get_method_names( - const MethodCollection& methods) { + const std::unordered_set& methods) { std::unordered_set result; for (auto* method : methods) { result.emplace(show(method)); diff --git a/test/unit/UnreachableLoweringPassTest.cpp b/test/unit/UnreachableLoweringPassTest.cpp index 70869ae8cb..0de7f1703d 100644 --- a/test/unit/UnreachableLoweringPassTest.cpp +++ b/test/unit/UnreachableLoweringPassTest.cpp @@ -29,7 +29,7 @@ class UnreachableLoweringPassTest : public RedexTest { virt_scope::get_vmethods(type::java_lang_Object()); std::string class_name = "LTest;"; - ClassCreator creator(DexType::make_type(class_name)); + ClassCreator creator(DexType::make_type(class_name.c_str())); creator.set_super(type::java_lang_Object()); auto signature = class_name + ".foo:()V"; auto method = DexMethod::make_method(signature)->make_concrete( @@ -77,8 +77,7 @@ TEST_F(UnreachableLoweringPassTest, simple) { )"; auto expected_code = R"( ( - (invoke-static () "Lcom/redex/UnreachableException;.createAndThrow:()Lcom/redex/UnreachableException;") - (move-result-object v0) + (const v0 0) (throw v0) ) )"; @@ -95,8 +94,7 @@ TEST_F(UnreachableLoweringPassTest, move_objects_are_tolerated) { )"; auto expected_code = R"( ( - (invoke-static () "Lcom/redex/UnreachableException;.createAndThrow:()Lcom/redex/UnreachableException;") - (move-result-object v0) + (const v0 0) (move-object v1 v0) (throw v1) ) @@ -115,8 +113,7 @@ TEST_F(UnreachableLoweringPassTest, invokes_are_tolerated) { )"; auto expected_code = R"( ( - (invoke-static () "Lcom/redex/UnreachableException;.createAndThrow:()Lcom/redex/UnreachableException;") - (move-result-object v0) + (const v0 0) (move-object v1 v0) (invoke-static () "Lcom/facebook/redex/dynamicanalysis/DynamicAnalysis;.onMethodExit:()V") (throw v1) diff --git a/test/unit/UpCodeMotionTest.cpp b/test/unit/UpCodeMotionTest.cpp index a19d0c4de9..79b8f4e22e 100644 --- a/test/unit/UpCodeMotionTest.cpp +++ b/test/unit/UpCodeMotionTest.cpp @@ -28,7 +28,6 @@ void test(const std::string& code_str, bool is_static = true; DexTypeList* args = DexTypeList::make_type_list({}); DexType* declaring_type = nullptr; - code->build_cfg(); UpCodeMotionPass::Stats stats = UpCodeMotionPass::process_code( is_static, declaring_type, args, code.get(), branch_hotness_check); EXPECT_EQ(expected_instructions_moved, stats.instructions_moved); @@ -37,7 +36,6 @@ void test(const std::string& code_str, stats.inverted_conditional_branches); EXPECT_EQ(expected_clobbered_registers, stats.clobbered_registers); - code->clear_cfg(); printf("%s\n", assembler::to_string(code.get()).c_str()); EXPECT_CODE_EQ(code.get(), expected.get()); }; diff --git a/test/unit/VirtualMergingTest.cpp b/test/unit/VirtualMergingTest.cpp index 1f16961967..04fecd6dc1 100644 --- a/test/unit/VirtualMergingTest.cpp +++ b/test/unit/VirtualMergingTest.cpp @@ -131,7 +131,6 @@ class VirtualMergingTest : public RedexTest { return *fail; } - // NOLINTNEXTLINE(google-explicit-constructor) operator bool() const { return fail.operator bool(); } ::testing::AssertionResult result() { diff --git a/test/unit/arsc/TestStructures.cpp b/test/unit/arsc/TestStructures.cpp deleted file mode 100644 index 9f6a9ee32a..0000000000 --- a/test/unit/arsc/TestStructures.cpp +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * - * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. - */ - -#include "TestStructures.h" - -#include "androidfw/ResourceTypes.h" - -// Sample data for building arsc test cases -EntryAndValue e0(0, android::Res_value::TYPE_DIMENSION, 1000); -EntryAndValue e0_land(0, android::Res_value::TYPE_DIMENSION, 1001); -EntryAndValue e1(1, android::Res_value::TYPE_DIMENSION, 2000); -EntryAndValue e2(2, android::Res_value::TYPE_REFERENCE, 0x7f010001); -EntryAndValue id_0(0, android::Res_value::TYPE_INT_BOOLEAN, 0); -EntryAndValue id_1(1, android::Res_value::TYPE_INT_BOOLEAN, 0); -EntryAndValue id_2(2, android::Res_value::TYPE_INT_BOOLEAN, 0); -MapEntryAndValues style(3, 0); - -// The package that many unit tests will be in. -android::ResTable_package foo_package{.id = 0x7f, - .name = {'f', 'o', 'o', '\0'}}; -// Create a default ResTable_config -android::ResTable_config default_config = { - .size = sizeof(android::ResTable_config)}; -// Create a landscape config -android::ResTable_config land_config = { - .size = sizeof(android::ResTable_config), - .orientation = android::ResTable_config::ORIENTATION_LAND}; -// And a xxhdpi config -android::ResTable_config xxhdpi_config = { - .size = sizeof(android::ResTable_config), - .density = android::ResTable_config::DENSITY_XXHIGH}; diff --git a/test/unit/arsc/TestStructures.h b/test/unit/arsc/TestStructures.h deleted file mode 100644 index 7476221179..0000000000 --- a/test/unit/arsc/TestStructures.h +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * - * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. - */ - -#pragma once - -#include "Util.h" -#include "androidfw/ResourceTypes.h" - -// Data that is used to write many test cases against. Meant to be included from -// individual test cpp files that want to code against it. - -PACKED(struct EntryAndValue { - android::ResTable_entry entry{}; - android::Res_value value{}; - EntryAndValue(uint32_t key_string_idx, uint8_t data_type, uint32_t data) { - entry.size = sizeof(android::ResTable_entry); - entry.key.index = key_string_idx; - value.size = sizeof(android::Res_value); - value.dataType = data_type; - value.data = data; - } -}); - -// For testing simplicity, a map that has two items in it. -PACKED(struct MapEntryAndValues { - android::ResTable_map_entry entry{}; - android::ResTable_map item0{}; - android::ResTable_map item1{}; - MapEntryAndValues(uint32_t key_string_idx, uint32_t parent_ident) { - entry.size = sizeof(android::ResTable_map_entry); - entry.count = 2; - entry.flags = android::ResTable_entry::FLAG_COMPLEX; - entry.key.index = key_string_idx; - entry.parent.ident = parent_ident; - item0.value.size = sizeof(android::Res_value); - item1.value.size = sizeof(android::Res_value); - } -}); - -// Sample data for building arsc test cases -extern EntryAndValue e0; -extern EntryAndValue e0_land; -extern EntryAndValue e1; -extern EntryAndValue e2; -extern EntryAndValue id_0; -extern EntryAndValue id_1; -extern EntryAndValue id_2; -extern MapEntryAndValues style; - -// A package called "foo" -extern android::ResTable_package foo_package; -// Create a default ResTable_config -extern android::ResTable_config default_config; -// Create a landscape config -extern android::ResTable_config land_config; -// And a xxhdpi config -extern android::ResTable_config xxhdpi_config; diff --git a/test/unit/constant-propagation/ConstantValueTest.cpp b/test/unit/constant-propagation/ConstantValueTest.cpp index 3c7d1e5d42..52f3ffbd5b 100644 --- a/test/unit/constant-propagation/ConstantValueTest.cpp +++ b/test/unit/constant-propagation/ConstantValueTest.cpp @@ -79,8 +79,8 @@ TEST_F(ConstantValueTest, meet) { EXPECT_EQ(meet(ConstantValue::top(), owia), owia); EXPECT_EQ(meet(owia, ConstantValue::top()), owia); - EXPECT_EQ(meet(sod, owia), nez); - EXPECT_EQ(meet(owia, sod), nez); + EXPECT_EQ(meet(sod, owia), ConstantValue::top()); + EXPECT_EQ(meet(owia, sod), ConstantValue::top()); EXPECT_EQ(meet(sd_a, sd_b), ConstantValue::bottom()); EXPECT_EQ(meet(sd_b, sd_a), ConstantValue::bottom()); @@ -103,8 +103,8 @@ TEST_F(ConstantValueTest, join) { EXPECT_EQ(join(ConstantValue::top(), owia), ConstantValue::top()); EXPECT_EQ(join(owia, ConstantValue::top()), ConstantValue::top()); - EXPECT_EQ(join(sod, owia), nez); - EXPECT_EQ(join(owia, sod), nez); + EXPECT_EQ(join(sod, owia), ConstantValue::top()); + EXPECT_EQ(join(owia, sod), ConstantValue::top()); EXPECT_EQ(join(sd_a, sd_b), nez); EXPECT_EQ(join(sd_b, sd_a), nez); diff --git a/test/unit/constant-propagation/IPConstantPropagationTest.cpp b/test/unit/constant-propagation/IPConstantPropagationTest.cpp index e5c3ba3fe2..037ab01888 100644 --- a/test/unit/constant-propagation/IPConstantPropagationTest.cpp +++ b/test/unit/constant-propagation/IPConstantPropagationTest.cpp @@ -14,7 +14,6 @@ #include "ConstantPropagationRuntimeAssert.h" #include "Creators.h" #include "Debug.h" -#include "DebugUtils.h" #include "DexUtil.h" #include "IPConstantPropagationAnalysis.h" #include "IRAssembler.h" diff --git a/test/unit/object-sensitive-dce/UsedVarsTest.cpp b/test/unit/object-sensitive-dce/UsedVarsTest.cpp index 7ccfe44b12..7a9adb9d5f 100644 --- a/test/unit/object-sensitive-dce/UsedVarsTest.cpp +++ b/test/unit/object-sensitive-dce/UsedVarsTest.cpp @@ -50,7 +50,7 @@ void optimize(const uv::FixpointIterator& fp_iter, cfg::ControlFlowGraph& cfg) { // will call resolve_method() during its analysis. resolve_method() needs the // method to reside in a class hierarchy in order to work correctly. DexClass* create_simple_class(const std::string& name) { - ClassCreator cc(DexType::make_type(name)); + ClassCreator cc(DexType::make_type(name.c_str())); cc.set_super(type::java_lang_Object()); auto* ctor = DexMethod::make_method(name + ".:()V") ->make_concrete(ACC_PUBLIC, /* is_virtual */ false); @@ -66,8 +66,6 @@ TEST_F(UsedVarsTest, simple) { (new-instance "LFoo;") (move-result-pseudo-object v0) (invoke-direct (v0) "LFoo;.:()V") - (check-cast v0 "LFoo;") - (move-result-pseudo-object v0) (const v1 0) (iput v1 v0 "LFoo;.bar:I") (return-void) diff --git a/test/unit/type-analysis/DexTypeEnvironmentTest.cpp b/test/unit/type-analysis/DexTypeEnvironmentTest.cpp index ccca4128de..8b5a97447b 100644 --- a/test/unit/type-analysis/DexTypeEnvironmentTest.cpp +++ b/test/unit/type-analysis/DexTypeEnvironmentTest.cpp @@ -328,20 +328,20 @@ TEST_F(DexTypeEnvironmentTest, RegisterEnvTest) { auto type = env.get(v0); EXPECT_TRUE(type.is_top()); - env.set(v0, DexTypeDomain::create_not_null(m_type_a)); - EXPECT_EQ(env.get(v0), DexTypeDomain::create_not_null(m_type_a)); + env.set(v0, DexTypeDomain(m_type_a)); + EXPECT_EQ(env.get(v0), DexTypeDomain(m_type_a)); reg_t v1 = 1; - env.set(v1, DexTypeDomain::create_not_null(m_type_a1)); - EXPECT_EQ(env.get(v1), DexTypeDomain::create_not_null(m_type_a1)); + env.set(v1, DexTypeDomain(m_type_a1)); + EXPECT_EQ(env.get(v1), DexTypeDomain(m_type_a1)); - auto a_join_a1 = DexTypeDomain::create_not_null(m_type_a); + auto a_join_a1 = DexTypeDomain(m_type_a); a_join_a1.join_with(env.get(v1)); EXPECT_EQ(a_join_a1.get_single_domain(), SingletonDexTypeDomain(m_type_a)); EXPECT_EQ(a_join_a1.get_annotation_domain(), TypedefAnnotationDomain::top()); EXPECT_EQ(a_join_a1.get_type_set(), get_type_set({m_type_a, m_type_a1})); - auto a1_join_a = DexTypeDomain::create_not_null(m_type_a1); + auto a1_join_a = DexTypeDomain(m_type_a1); a1_join_a.join_with(env.get(v0)); EXPECT_EQ(a1_join_a.get_single_domain(), SingletonDexTypeDomain(m_type_a)); EXPECT_EQ(a1_join_a.get_annotation_domain(), TypedefAnnotationDomain::top()); @@ -354,28 +354,28 @@ TEST_F(DexTypeEnvironmentTest, AnnotationRegisterEnvTest) { auto type = env.get(v0); EXPECT_TRUE(type.is_top()); - env.set(v0, DexTypeDomain::create_nullable(m_type_a, m_anno_d1)); - EXPECT_EQ(env.get(v0), DexTypeDomain::create_nullable(m_type_a, m_anno_d1)); + env.set(v0, DexTypeDomain(m_type_a, m_anno_d1)); + EXPECT_EQ(env.get(v0), DexTypeDomain(m_type_a, m_anno_d1)); reg_t v1 = 1; - env.set(v1, DexTypeDomain::create_nullable(m_type_a1, m_anno_d2)); - EXPECT_EQ(env.get(v1), DexTypeDomain::create_nullable(m_type_a1, m_anno_d2)); + env.set(v1, DexTypeDomain(m_type_a1, m_anno_d2)); + EXPECT_EQ(env.get(v1), DexTypeDomain(m_type_a1, m_anno_d2)); - auto a_join_a1 = DexTypeDomain::create_nullable(m_type_a, m_anno_d1); + auto a_join_a1 = DexTypeDomain(m_type_a, m_anno_d1); a_join_a1.join_with(env.get(v1)); EXPECT_EQ(a_join_a1.get_single_domain(), SingletonDexTypeDomain(m_type_a)); EXPECT_EQ(a_join_a1.get_annotation_domain(), TypedefAnnotationDomain(type::java_lang_Object())); - EXPECT_TRUE(a_join_a1.get_set_domain().is_top()); + EXPECT_EQ(a_join_a1.get_type_set(), get_type_set({m_type_a, m_type_a1})); - auto a1_join_a = DexTypeDomain::create_nullable(m_type_a1, m_anno_d1); + auto a1_join_a = DexTypeDomain(m_type_a1, m_anno_d1); a1_join_a.join_with(env.get(v0)); EXPECT_EQ(a1_join_a.get_single_domain(), SingletonDexTypeDomain(m_type_a)); EXPECT_EQ(a1_join_a.get_annotation_domain(), TypedefAnnotationDomain(m_type_d1)); - EXPECT_TRUE(a1_join_a.get_set_domain().is_top()); + EXPECT_EQ(a1_join_a.get_type_set(), get_type_set({m_type_a, m_type_a1})); } TEST_F(DexTypeEnvironmentTest, FieldEnvTest) { @@ -384,29 +384,29 @@ TEST_F(DexTypeEnvironmentTest, FieldEnvTest) { auto type = env.get(f1); EXPECT_TRUE(type.is_top()); - env.set(f1, DexTypeDomain::create_not_null(m_type_a1)); - EXPECT_EQ(env.get(f1), DexTypeDomain::create_not_null(m_type_a1)); + env.set(f1, DexTypeDomain(m_type_a1)); + EXPECT_EQ(env.get(f1), DexTypeDomain(m_type_a1)); DexField* f2 = (DexField*)2; EXPECT_TRUE(env.get(f2).is_top()); - env.set(f2, DexTypeDomain::create_not_null(m_type_a)); - EXPECT_EQ(env.get(f2), DexTypeDomain::create_not_null(m_type_a)); + env.set(f2, DexTypeDomain(m_type_a)); + EXPECT_EQ(env.get(f2), DexTypeDomain(m_type_a)); auto a_join_a1 = env.get(f2); a_join_a1.join_with(env.get(f1)); EXPECT_EQ(a_join_a1.get_single_domain(), SingletonDexTypeDomain(m_type_a)); EXPECT_EQ(a_join_a1.get_annotation_domain(), TypedefAnnotationDomain::top()); EXPECT_EQ(a_join_a1.get_type_set(), get_type_set({m_type_a, m_type_a1})); - EXPECT_EQ(env.get(f1), DexTypeDomain::create_not_null(m_type_a1)); - EXPECT_EQ(env.get(f2), DexTypeDomain::create_not_null(m_type_a)); + EXPECT_EQ(env.get(f1), DexTypeDomain(m_type_a1)); + EXPECT_EQ(env.get(f2), DexTypeDomain(m_type_a)); auto a1_join_a = env.get(f1); a1_join_a.join_with(env.get(f2)); EXPECT_EQ(a1_join_a.get_single_domain(), SingletonDexTypeDomain(m_type_a)); EXPECT_EQ(a1_join_a.get_annotation_domain(), TypedefAnnotationDomain::top()); EXPECT_EQ(a1_join_a.get_type_set(), get_type_set({m_type_a, m_type_a1})); - EXPECT_EQ(env.get(f1), DexTypeDomain::create_not_null(m_type_a1)); - EXPECT_EQ(env.get(f2), DexTypeDomain::create_not_null(m_type_a)); + EXPECT_EQ(env.get(f1), DexTypeDomain(m_type_a1)); + EXPECT_EQ(env.get(f2), DexTypeDomain(m_type_a)); } TEST_F(DexTypeEnvironmentTest, ThisPointerEnvTest) { @@ -425,26 +425,26 @@ TEST_F(DexTypeEnvironmentTest, ThisPointerEnvTest) { } TEST_F(DexTypeEnvironmentTest, JoinWithTest) { - auto domain_a1 = DexTypeDomain::create_not_null(m_type_a1); - auto domain_a2 = DexTypeDomain::create_not_null(m_type_a2); + auto domain_a1 = DexTypeDomain(m_type_a1); + auto domain_a2 = DexTypeDomain(m_type_a2); domain_a1.join_with(domain_a2); EXPECT_EQ(domain_a1.get_single_domain(), SingletonDexTypeDomain(m_type_a)); EXPECT_EQ(domain_a1.get_type_set(), get_type_set({m_type_a1, m_type_a2})); - domain_a1 = DexTypeDomain::create_not_null(m_type_a1); - auto domain_a21 = DexTypeDomain::create_not_null(m_type_a21); + domain_a1 = DexTypeDomain(m_type_a1); + auto domain_a21 = DexTypeDomain(m_type_a21); domain_a1.join_with(domain_a21); EXPECT_EQ(domain_a1.get_single_domain(), SingletonDexTypeDomain(m_type_a)); EXPECT_EQ(domain_a1.get_type_set(), get_type_set({m_type_a1, m_type_a21})); - domain_a1 = DexTypeDomain::create_not_null(m_type_a1); - auto domain_a211 = DexTypeDomain::create_not_null(m_type_a211); + domain_a1 = DexTypeDomain(m_type_a1); + auto domain_a211 = DexTypeDomain(m_type_a211); domain_a1.join_with(domain_a211); EXPECT_EQ(domain_a1.get_single_domain(), SingletonDexTypeDomain(m_type_a)); EXPECT_EQ(domain_a1.get_type_set(), get_type_set({m_type_a1, m_type_a211})); - auto domain_a = DexTypeDomain::create_not_null(m_type_a); - domain_a211 = DexTypeDomain::create_not_null(m_type_a211); + auto domain_a = DexTypeDomain(m_type_a); + domain_a211 = DexTypeDomain(m_type_a211); domain_a.join_with(domain_a211); EXPECT_EQ(domain_a.get_single_domain(), SingletonDexTypeDomain(m_type_a)); EXPECT_EQ(domain_a.get_type_set(), get_type_set({m_type_a, m_type_a211})); @@ -455,36 +455,36 @@ TEST_F(DexTypeEnvironmentTest, JoinWithTest) { EXPECT_TRUE(top1.is_top()); EXPECT_TRUE(top2.is_top()); - domain_a = DexTypeDomain::create_not_null(m_type_a); - auto domain_b = DexTypeDomain::create_not_null(m_type_b); + domain_a = DexTypeDomain(m_type_a); + auto domain_b = DexTypeDomain(m_type_b); domain_a.join_with(domain_b); EXPECT_EQ(domain_a.get_single_domain(), SingletonDexTypeDomain(type::java_lang_Object())); EXPECT_EQ(domain_a.get_type_set(), get_type_set({m_type_a, m_type_b})); - domain_a1 = DexTypeDomain::create_not_null(m_type_a1); - domain_b = DexTypeDomain::create_not_null(m_type_b); + domain_a1 = DexTypeDomain(m_type_a1); + domain_b = DexTypeDomain(m_type_b); domain_a1.join_with(domain_b); EXPECT_EQ(domain_a1.get_single_domain(), SingletonDexTypeDomain(type::java_lang_Object())); EXPECT_EQ(domain_a1.get_type_set(), get_type_set({m_type_a1, m_type_b})); - domain_a21 = DexTypeDomain::create_not_null(m_type_a21); - domain_b = DexTypeDomain::create_not_null(m_type_b); + domain_a21 = DexTypeDomain(m_type_a21); + domain_b = DexTypeDomain(m_type_b); domain_a21.join_with(domain_b); EXPECT_EQ(domain_a21.get_single_domain(), SingletonDexTypeDomain(type::java_lang_Object())); EXPECT_EQ(domain_a21.get_type_set(), get_type_set({m_type_a21, m_type_b})); - domain_a211 = DexTypeDomain::create_not_null(m_type_a211); - domain_b = DexTypeDomain::create_not_null(m_type_b); + domain_a211 = DexTypeDomain(m_type_a211); + domain_b = DexTypeDomain(m_type_b); domain_a211.join_with(domain_b); EXPECT_EQ(domain_a211.get_single_domain(), SingletonDexTypeDomain(type::java_lang_Object())); EXPECT_EQ(domain_a211.get_type_set(), get_type_set({m_type_a211, m_type_b})); - domain_a1 = DexTypeDomain::create_not_null(m_type_a1); - auto domain_b1 = DexTypeDomain::create_not_null(m_type_b1); + domain_a1 = DexTypeDomain(m_type_a1); + auto domain_b1 = DexTypeDomain(m_type_b1); domain_a1.join_with(domain_b1); EXPECT_EQ(domain_a1.get_single_domain(), SingletonDexTypeDomain(type::java_lang_Object())); @@ -492,7 +492,7 @@ TEST_F(DexTypeEnvironmentTest, JoinWithTest) { EXPECT_FALSE(domain_a1.get_single_domain().is_top()); EXPECT_FALSE(domain_b1.get_single_domain().is_top()); - domain_a1 = DexTypeDomain::create_not_null(m_type_a1); + domain_a1 = DexTypeDomain(m_type_a1); domain_b1.join_with(domain_a1); EXPECT_EQ(domain_b1.get_single_domain(), SingletonDexTypeDomain(type::java_lang_Object())); @@ -502,28 +502,28 @@ TEST_F(DexTypeEnvironmentTest, JoinWithTest) { } TEST_F(DexTypeEnvironmentTest, AnnotationJoinWithTest) { - auto domain_a1 = DexTypeDomain::create_nullable(m_type_a1, m_anno_d1); - auto domain_a2 = DexTypeDomain::create_nullable(m_type_a2, m_anno_d2); + auto domain_a1 = DexTypeDomain(m_type_a1, m_anno_d1); + auto domain_a2 = DexTypeDomain(m_type_a2, m_anno_d2); domain_a1.join_with(domain_a2); EXPECT_EQ(domain_a1.get_annotation_domain(), TypedefAnnotationDomain(type::java_lang_Object())); - domain_a1 = DexTypeDomain::create_nullable(m_type_a1, m_anno_d3); - auto domain_a21 = DexTypeDomain::create_nullable(m_type_a21, nullptr); + domain_a1 = DexTypeDomain(m_type_a1, m_anno_d3); + auto domain_a21 = DexTypeDomain(m_type_a21, nullptr); domain_a1.join_with(domain_a21); EXPECT_EQ(domain_a1.get_annotation_domain(), TypedefAnnotationDomain::top()); - EXPECT_TRUE(domain_a1.get_set_domain().is_top()); + EXPECT_EQ(domain_a1.get_type_set(), get_type_set({m_type_a1, m_type_a21})); - domain_a1 = DexTypeDomain::create_nullable(m_type_a1, nullptr); - auto domain_a211 = DexTypeDomain::create_nullable(m_type_a211, m_anno_d3); + domain_a1 = DexTypeDomain(m_type_a1, nullptr); + auto domain_a211 = DexTypeDomain(m_type_a211, m_anno_d3); domain_a1.join_with(domain_a211); EXPECT_EQ(domain_a1.get_annotation_domain(), TypedefAnnotationDomain::top()); - EXPECT_TRUE(domain_a1.get_set_domain().is_top()); + EXPECT_EQ(domain_a1.get_type_set(), get_type_set({m_type_a1, m_type_a211})); - auto domain_a = DexTypeDomain::create_nullable(m_type_a, m_anno_d4); - domain_a211 = DexTypeDomain::create_nullable(m_type_a211, m_anno_d4); + auto domain_a = DexTypeDomain(m_type_a, m_anno_d4); + domain_a211 = DexTypeDomain(m_type_a211, m_anno_d4); domain_a.join_with(domain_a211); EXPECT_EQ(domain_a.get_annotation_domain(), TypedefAnnotationDomain(m_type_d4)); @@ -534,13 +534,13 @@ TEST_F(DexTypeEnvironmentTest, AnnotationJoinWithTest) { EXPECT_TRUE(top1.is_top()); EXPECT_TRUE(top2.is_top()); - domain_a = DexTypeDomain::create_nullable(m_type_a); - auto domain_b = DexTypeDomain::create_nullable(m_type_b); + domain_a = DexTypeDomain(m_type_a); + auto domain_b = DexTypeDomain(m_type_b); domain_a.join_with(domain_b); EXPECT_EQ(domain_a.get_annotation_domain(), TypedefAnnotationDomain::top()); - domain_a1 = DexTypeDomain::create_nullable(m_type_a1, nullptr); - domain_b = DexTypeDomain::create_nullable(m_type_b, nullptr); + domain_a1 = DexTypeDomain(m_type_a1, nullptr); + domain_b = DexTypeDomain(m_type_b, nullptr); domain_a1.join_with(domain_b); EXPECT_EQ(domain_a1.get_annotation_domain(), TypedefAnnotationDomain::top()); } @@ -779,29 +779,27 @@ TEST_F(DexTypeEnvironmentTest, NullableDexTypeDomainTest) { EXPECT_TRUE(null1.get_single_domain().is_none()); EXPECT_TRUE(null1.get_annotation_domain().is_none()); - auto type_a = DexTypeDomain::create_nullable(m_type_a, m_anno_d1); + auto type_a = DexTypeDomain(m_type_a, m_anno_d1); null1.join_with(type_a); EXPECT_FALSE(null1.is_null()); EXPECT_FALSE(null1.is_not_null()); EXPECT_TRUE(null1.is_nullable()); - // Both Nullalbe - EXPECT_EQ(null1, DexTypeDomain::create_nullable(m_type_a, m_anno_d1)); + EXPECT_NE(null1, DexTypeDomain(m_type_a, m_anno_d1)); EXPECT_EQ(*null1.get_dex_type(), m_type_a); EXPECT_EQ(*null1.get_annotation_type(), m_type_d1); - EXPECT_EQ(type_a, DexTypeDomain::create_nullable(m_type_a, m_anno_d1)); + EXPECT_EQ(type_a, DexTypeDomain(m_type_a, m_anno_d1)); EXPECT_FALSE(null1.get_single_domain().is_none()); EXPECT_FALSE(type_a.get_single_domain().is_none()); EXPECT_FALSE(null1.get_annotation_domain().is_none()); EXPECT_FALSE(type_a.get_annotation_domain().is_none()); - type_a = DexTypeDomain::create_nullable(m_type_a, m_anno_d1); + type_a = DexTypeDomain(m_type_a, m_anno_d1); null1 = DexTypeDomain::null(); type_a.join_with(null1); EXPECT_FALSE(type_a.is_null()); EXPECT_FALSE(type_a.is_not_null()); EXPECT_TRUE(type_a.is_nullable()); - // Both Nullalbe - EXPECT_EQ(type_a, DexTypeDomain::create_nullable(m_type_a, m_anno_d1)); + EXPECT_NE(type_a, DexTypeDomain(m_type_a, m_anno_d1)); EXPECT_EQ(*type_a.get_dex_type(), m_type_a); EXPECT_EQ(*type_a.get_annotation_type(), m_type_d1); EXPECT_EQ(null1, DexTypeDomain::null()); @@ -1014,10 +1012,11 @@ TEST_F(DexTypeEnvironmentTest, SmallSetDexTypeDomainMixedHierarchyTest) { } TEST_F(DexTypeEnvironmentTest, DexTypeDomainReduceProductTest) { - auto domain = DexTypeDomain::create_not_null(type::java_lang_Object()); + auto domain = DexTypeDomain(type::java_lang_Object(), + new DexAnnoType(type::java_lang_Object())); - domain.join_with(DexTypeDomain::create_not_null( - type::make_array_type(type::java_lang_String()))); + domain.join_with( + DexTypeDomain(type::make_array_type(type::java_lang_String()))); EXPECT_TRUE(domain.get_single_domain().is_top()); EXPECT_TRUE(domain.get_annotation_domain().is_top()); EXPECT_FALSE(domain.get_set_domain().is_top()); @@ -1025,29 +1024,124 @@ TEST_F(DexTypeEnvironmentTest, DexTypeDomainReduceProductTest) { get_type_set({type::java_lang_Object(), type::make_array_type(type::java_lang_String())})); - auto domain_c1 = DexTypeDomain::create_nullable(m_type_c1, m_anno_d1); - domain_c1.join_with(DexTypeDomain::create_nullable(m_type_c2, m_anno_d2)); - domain_c1.join_with(DexTypeDomain::create_nullable(m_type_c3, m_anno_d3)); - domain_c1.join_with(DexTypeDomain::create_nullable(m_type_c4, m_anno_d4)); - domain_c1.join_with(DexTypeDomain::create_nullable(m_type_c5, m_anno_d5)); + auto domain_c1 = DexTypeDomain(m_type_c1, m_anno_d1); + domain_c1.join_with(DexTypeDomain(m_type_c2, m_anno_d2)); + domain_c1.join_with(DexTypeDomain(m_type_c3, m_anno_d3)); + domain_c1.join_with(DexTypeDomain(m_type_c4, m_anno_d4)); + domain_c1.join_with(DexTypeDomain(m_type_c5, m_anno_d5)); EXPECT_FALSE(domain_c1.get_single_domain().is_top()); EXPECT_FALSE(domain_c1.get_annotation_domain().is_top()); EXPECT_TRUE(domain_c1.get_set_domain().is_top()); - domain_c1 = DexTypeDomain::create_nullable(m_type_c1, m_anno_d1); - auto domain_c2 = DexTypeDomain::create_nullable(m_type_c2, m_anno_d2); - domain_c2.join_with(DexTypeDomain::create_nullable(m_type_c3, m_anno_d3)); - domain_c2.join_with(DexTypeDomain::create_nullable(m_type_c4, m_anno_d4)); - domain_c2.join_with(DexTypeDomain::create_nullable(m_type_c5, m_anno_d5)); + domain_c1 = DexTypeDomain(m_type_c1, m_anno_d1); + auto domain_c2 = DexTypeDomain(m_type_c2, m_anno_d2); + domain_c2.join_with(DexTypeDomain(m_type_c3, m_anno_d3)); + domain_c2.join_with(DexTypeDomain(m_type_c4, m_anno_d4)); + domain_c2.join_with(DexTypeDomain(m_type_c5, m_anno_d5)); EXPECT_FALSE(domain_c2.get_single_domain().is_top()); EXPECT_FALSE(domain_c2.get_annotation_domain().is_top()); - EXPECT_TRUE(domain_c2.get_set_domain().is_top()); + EXPECT_FALSE(domain_c2.get_set_domain().is_top()); domain_c1.join_with(domain_c2); EXPECT_FALSE(domain_c1.get_single_domain().is_top()); EXPECT_FALSE(domain_c1.get_annotation_domain().is_top()); EXPECT_TRUE(domain_c1.get_set_domain().is_top()); } +TEST_F(DexTypeEnvironmentTest, ConstNullnessDomainTest) { + auto c1 = DexTypeDomain(1); + EXPECT_FALSE(c1.is_top()); + EXPECT_EQ(*c1.get_constant(), 1); + + auto nl = DexTypeDomain::null(); + EXPECT_FALSE(nl.is_top()); + EXPECT_TRUE(nl.is_null()); + + c1.join_with(nl); + EXPECT_TRUE(c1.is_top()); + EXPECT_TRUE(c1.get<0>().get().const_domain().is_top()); + EXPECT_TRUE(c1.get_nullness().is_top()); + EXPECT_TRUE(c1.is_nullable()); +} + +TEST_F(DexTypeEnvironmentTest, ArrayNullnessDomainTest) { + auto a1 = ArrayNullnessDomain(1); + EXPECT_FALSE(a1.is_top()); + EXPECT_FALSE(a1.is_bottom()); + EXPECT_FALSE(a1.get_nullness().is_top()); + EXPECT_EQ(a1.get_nullness(), NullnessDomain(NOT_NULL)); + EXPECT_EQ(*a1.get_length(), 1); + EXPECT_EQ(a1.get_element(0), NullnessDomain(UNINITIALIZED)); + EXPECT_TRUE(a1.get_element(1).is_top()); + + auto a2 = ArrayNullnessDomain(2); + EXPECT_FALSE(a2.is_top()); + EXPECT_FALSE(a2.is_bottom()); + EXPECT_EQ(a2.get_nullness(), NullnessDomain(NOT_NULL)); + EXPECT_EQ(*a2.get_length(), 2); + EXPECT_EQ(a2.get_element(0), NullnessDomain(UNINITIALIZED)); + EXPECT_EQ(a2.get_element(1), NullnessDomain(UNINITIALIZED)); + + a1.join_with(a2); + EXPECT_FALSE(a1.is_top()); + EXPECT_FALSE(a1.is_bottom()); + EXPECT_EQ(a1.get_nullness(), NullnessDomain(NOT_NULL)); + EXPECT_TRUE(a1.get<1>().is_top()); + EXPECT_TRUE(a1.get_element(0).is_top()); + EXPECT_TRUE(a1.get_element(1).is_top()); +} + +TEST_F(DexTypeEnvironmentTest, ArrayConstNullnessDomain) { + auto array1 = DexTypeDomain(m_string_array, 1); + EXPECT_EQ(array1.get_array_nullness().get_nullness(), + NullnessDomain(NOT_NULL)); + EXPECT_EQ(*array1.get_array_nullness().get_length(), 1); + EXPECT_EQ(array1.get_array_element_nullness(0), + NullnessDomain(UNINITIALIZED)); + + array1.set_array_element_nullness(0, NullnessDomain(NOT_NULL)); + EXPECT_EQ(array1.get_array_element_nullness(0), NullnessDomain(NOT_NULL)); + EXPECT_TRUE(array1.get_array_element_nullness(1).is_top()); + + array1.set_array_element_nullness(1, NullnessDomain(NOT_NULL)); + EXPECT_TRUE(array1.get_array_element_nullness(1).is_top()); + + EXPECT_TRUE(array1.get_array_element_nullness(-1).is_top()); + array1.set_array_element_nullness(-1, NullnessDomain(NOT_NULL)); + EXPECT_TRUE(array1.get_array_nullness().get_elements().is_top()); + + auto array2 = DexTypeDomain(m_string_array, 2); + EXPECT_EQ(array2.get_array_nullness().get_nullness(), + NullnessDomain(NOT_NULL)); + EXPECT_EQ(*array2.get_array_nullness().get_length(), 2); + EXPECT_EQ(array2.get_array_element_nullness(0), + NullnessDomain(UNINITIALIZED)); + EXPECT_EQ(array2.get_array_element_nullness(1), + NullnessDomain(UNINITIALIZED)); + array2.set_array_element_nullness(0, NullnessDomain(NOT_NULL)); + array2.set_array_element_nullness(1, NullnessDomain(NOT_NULL)); + EXPECT_EQ(array2.get_array_element_nullness(0), NullnessDomain(NOT_NULL)); + EXPECT_EQ(array2.get_array_element_nullness(1), NullnessDomain(NOT_NULL)); + + array1.join_with(array2); + EXPECT_EQ(array2.get_array_nullness().get_nullness(), + NullnessDomain(NOT_NULL)); + EXPECT_TRUE(array1.get_array_nullness().get<1>().is_top()); + EXPECT_TRUE(array1.get_array_nullness().get_elements().is_top()); + + // DisjoinUnion crossing access + EXPECT_FALSE(DexTypeDomain::top().get_constant()); + EXPECT_EQ(*DexTypeDomain::null().get_constant(), 0); + EXPECT_FALSE(DexTypeDomain(m_string_array, 3).get_constant()); + EXPECT_FALSE(DexTypeDomain(m_string_array).get_constant()); + + EXPECT_TRUE(DexTypeDomain::top().get_array_nullness().is_top()); + EXPECT_TRUE(DexTypeDomain::null().get_array_nullness().is_top()); + EXPECT_FALSE(DexTypeDomain(m_string_array, 3).get_array_nullness().is_top()); + EXPECT_FALSE( + DexTypeDomain(m_string_array, 3).get_array_nullness().is_bottom()); + EXPECT_TRUE(DexTypeDomain(m_string_array).get_array_nullness().is_top()); +} + TEST_F(DexTypeEnvironmentTest, BaseClassInterfaceJoinTest) { auto abs_me = SingletonDexTypeDomain(m_abs_map_entry); auto intf = SingletonDexTypeDomain(m_map_entry); @@ -1077,11 +1171,11 @@ TEST_F(DexTypeEnvironmentTest, BaseClassInterfaceJoinTest) { } TEST_F(DexTypeEnvironmentTest, TypedefAnnotationDomain) { - auto d1 = DexTypeDomain::create_for_anno(m_anno_d1); + auto d1 = DexTypeDomain(m_anno_d1); EXPECT_FALSE(d1.is_top()); EXPECT_EQ(*d1.get_annotation_type(), m_type_d1); - auto d2 = DexTypeDomain::create_for_anno(m_anno_d2); + auto d2 = DexTypeDomain(m_anno_d2); d1.join_with(d2); EXPECT_EQ(d1.get_annotation_domain(), TypedefAnnotationDomain(type::java_lang_Object())); diff --git a/test/unit/type-analysis/GlobalTypeAnalysisTest.cpp b/test/unit/type-analysis/GlobalTypeAnalysisTest.cpp index 06ba2ee6c8..a9ff328420 100644 --- a/test/unit/type-analysis/GlobalTypeAnalysisTest.cpp +++ b/test/unit/type-analysis/GlobalTypeAnalysisTest.cpp @@ -39,8 +39,7 @@ struct GlobalTypeAnalysisTest : public RedexTest { void prepare_scope(Scope& scope) { scope.push_back(m_cls_o); } DexTypeDomain get_type_domain(const std::string& type_name) { - return DexTypeDomain::create_not_null( - DexType::make_type(DexString::make_string(type_name))); + return DexTypeDomain(DexType::make_type(DexString::make_string(type_name))); } SingletonDexTypeDomain get_singleton_type_domain( @@ -358,12 +357,15 @@ TEST_F(GlobalTypeAnalysisTest, ClinitSimpleTest) { GlobalTypeAnalysis analysis; auto gta = analysis.analyze(scope); auto wps = gta->get_whole_program_state(); - EXPECT_TRUE(wps.get_field_type(field_1).is_top()); - EXPECT_TRUE(wps.get_return_type(meth_bar).is_top()); + EXPECT_EQ(wps.get_field_type(field_1), + get_type_domain("LO;").join(DexTypeDomain::null())); + EXPECT_EQ(wps.get_return_type(meth_bar), + get_type_domain("LO;").join(DexTypeDomain::null())); auto lta = gta->get_replayable_local_analysis(meth_foo); auto code = meth_foo->get_code(); auto foo_exit_env = lta->get_exit_state_at(code->cfg().exit_block()); - EXPECT_TRUE(foo_exit_env.get_reg_environment().get(1).is_top()); + EXPECT_EQ(foo_exit_env.get_reg_environment().get(1), + get_type_domain("LO;").join(DexTypeDomain::null())); } TEST_F(GlobalTypeAnalysisTest, StaticFieldWithEncodedValueTest) { @@ -474,17 +476,15 @@ TEST_F(GlobalTypeAnalysisTest, StaticFieldWithEncodedValueTest) { EXPECT_EQ(wps.get_field_type(field_1), DexTypeDomain::null()); EXPECT_EQ(wps.get_return_type(meth_bar), DexTypeDomain::null()); - EXPECT_EQ(wps.get_field_type(field_2), - DexTypeDomain::create_not_null(type::java_lang_String()) - .join(DexTypeDomain::null())); - EXPECT_EQ(wps.get_return_type(meth_baz), - DexTypeDomain::create_not_null(type::java_lang_String()) - .join(DexTypeDomain::null())); + EXPECT_EQ( + wps.get_field_type(field_2), + DexTypeDomain(type::java_lang_String()).join(DexTypeDomain::null())); + EXPECT_EQ( + wps.get_return_type(meth_baz), + DexTypeDomain(type::java_lang_String()).join(DexTypeDomain::null())); EXPECT_EQ(wps.get_field_type(field_3), - DexTypeDomain::create_not_null(type::java_lang_Class()) - .join(DexTypeDomain::null())); + DexTypeDomain(type::java_lang_Class()).join(DexTypeDomain::null())); EXPECT_EQ(wps.get_return_type(meth_buk), - DexTypeDomain::create_not_null(type::java_lang_Class()) - .join(DexTypeDomain::null())); + DexTypeDomain(type::java_lang_Class()).join(DexTypeDomain::null())); } diff --git a/tools/arsc/ArscStats.cpp b/tools/arsc/ArscStats.cpp deleted file mode 100644 index d2d9816c3f..0000000000 --- a/tools/arsc/ArscStats.cpp +++ /dev/null @@ -1,546 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * - * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. - */ - -#include "ArscStats.h" - -#include -#include -#include -#include - -#include "ApkResources.h" -#include "Debug.h" -#include "RedexResources.h" -#include "Trace.h" -#include "androidfw/ResourceTypes.h" -#include "utils/ByteOrder.h" -#include "utils/Errors.h" -#include "utils/Log.h" -#include "utils/Serialize.h" -#include "utils/Unicode.h" -#include "utils/Vector.h" -#include "utils/Visitor.h" - -using namespace attribution; - -namespace { -constexpr size_t OFFSET_SIZE = sizeof(uint32_t); - -// Will be iterated over for output, other collections can be unordered. -using ResourceSizes = std::map; -using ResourceConfigs = std::unordered_map>; -using TypeNames = std::unordered_map; -using StringUsages = std::vector>; - -void add_size(const char* audit_msg, - uint32_t id, - size_t amount, - size_t usage_count, - ResourceSizes* resource_sizes) { - always_assert(usage_count > 0); - auto& size_struct = resource_sizes->at(id); - if (usage_count == 1) { - TRACE(ARSC, 2, "%s: 0x%x adding private size %zu", audit_msg, id, amount); - size_struct.private_size += amount; - } - auto size = (double)amount / usage_count; - TRACE(ARSC, 2, "%s: 0x%x adding proportional size (%zu / %zu) = %f", - audit_msg, id, amount, usage_count, size); - size_struct.proportional_size += size; -} - -void add_shared_size(const char* audit_msg, - uint32_t id, - size_t amount, - ResourceSizes* resource_sizes) { - auto& size_struct = resource_sizes->at(id); - TRACE(ARSC, 2, "%s: 0x%x adding shared size %zu", audit_msg, id, amount); - size_struct.shared_size += amount; -} - -void populate_string_usages(apk::TableEntryParser& parser, - StringUsages* global_usages, - StringUsages* key_usages, - StringUsages* type_usages) { - auto handle_value = [&](const uint32_t& id, android::Res_value* value_ptr) { - if (value_ptr->dataType == android::Res_value::TYPE_STRING) { - auto string_idx = dtohl(value_ptr->data); - global_usages->at(string_idx).emplace(id); - } - }; - for (const auto& id_to_entries : parser.m_res_id_to_entries) { - auto id = id_to_entries.first; - for (const auto& pair : id_to_entries.second) { - if (arsc::is_empty(pair.second)) { - continue; - } - auto entry = (android::ResTable_entry*)pair.second.getKey(); - auto value = arsc::get_value_data(pair.second); - - auto key_idx = dtohl(entry->key.index); - key_usages->at(key_idx).emplace(id); - auto flags = dtohs(entry->flags); - - uint8_t type_id = (id & TYPE_MASK_BIT) >> TYPE_INDEX_BIT_SHIFT; - type_usages->at(type_id - 1).emplace(id); - - if ((flags & android::ResTable_entry::FLAG_COMPLEX) != 0) { - auto complex_entry = (android::ResTable_map_entry*)entry; - auto count = dtohl(complex_entry->count); - auto complex_item = (android::ResTable_map*)value.getKey(); - for (size_t i = 0; i < count; i++, complex_item++) { - handle_value(id, &complex_item->value); - } - } else { - auto value_ptr = (android::Res_value*)value.getKey(); - handle_value(id, value_ptr); - } - } - } -} - -template -size_t length_units(size_t length) { - // see aosp - // https://cs.android.com/android/platform/superproject/main/+/main:frameworks/base/tools/aapt2/StringPool.cpp;l=356;drc=1dbbd3bd6ed466d9c3f284caad7adb8ed0f827d3 - static_assert(std::is_integral::value); - constexpr size_t MASK = 1 << ((sizeof(T) * 8) - 1); - constexpr size_t MAX_SIZE = MASK - 1; - return length > MAX_SIZE ? 2 : 1; -} - -// Actual implementation of the string counting, which allows for -// differentiating whether or not we are currently computing a styled string -// (which is not expected to have a span index that is a style string). -size_t compute_string_size_impl(const android::ResStringPool& pool, - uint32_t idx, - bool allow_styles) { - always_assert_log(idx < pool.size(), - "idx out of range, got %u for a pool of size %zu", - idx, - pool.size()); - size_t result = OFFSET_SIZE + compute_string_character_size(pool, idx); - if (idx < pool.styleCount()) { - always_assert_log(allow_styles, - "Got style index %u while computing size of style", idx); - // for the span start - result += OFFSET_SIZE; - auto span_ptr = pool.styleAt(idx); - std::vector vec; - arsc::collect_spans((android::ResStringPool_span*)span_ptr, &vec); - result += vec.size() * sizeof(android::ResStringPool_span); - for (const auto span : vec) { - result += compute_string_size_impl(pool, dtohl(span->name.index), false); - } - result += sizeof(android::ResStringPool_span::END); - } - return result; -} - -// Return the size of the string pool data structure header, padding, and END -// section, plus the string size for any unused string entries. -size_t compute_overhead(const android::ResStringPool_header* header, - const android::ResStringPool& pool, - const StringUsages& usages) { - size_t padding = count_padding(header, pool); - TRACE(ARSC, 1, "pool padding: %zu bytes", padding); - size_t overhead = dtohs(header->header.headerSize) + padding; - if (pool.styleCount() > 0) { - overhead += 2 * sizeof(android::ResStringPool_span::END); - } - for (uint32_t idx = 0; idx < pool.size(); idx++) { - if (usages.at(idx).empty()) { - overhead += compute_string_size(pool, idx); - } - if (traceEnabled(ARSC, 3)) { - auto str = arsc::get_string_from_pool(pool, idx); - auto len = compute_string_size(pool, idx); - TRACE_NO_LINE(ARSC, 3, "%u: \"%s\", length = %zu bytes. ", idx, - str.c_str(), len); - auto set = usages.at(idx); - if (set.empty()) { - TRACE(ARSC, 3, "No uses."); - } else { - bool first = true; - TRACE_NO_LINE(ARSC, 3, "Used by { "); - for (const auto& id : set) { - if (!first) { - TRACE_NO_LINE(ARSC, 3, ", "); - } - TRACE_NO_LINE(ARSC, 3, "0x%x", id); - first = false; - } - TRACE(ARSC, 3, " }"); - } - } - } - return overhead; -} - -void initialize_resource_sizes(const apk::TableEntryParser& parser, - ResourceSizes* resource_sizes) { - for (const auto& entry : parser.m_res_id_to_entries) { - resource_sizes->emplace(entry.first, ResourceSize{}); - } -} - -// Attributes the size of string data and offsets to resource ids. Caller -// chooses whether a string value used by many ids should be considered as -// shared data or not. -void tally_string_sizes(const char* audit_msg, - const android::ResStringPool& pool, - const StringUsages& usages, - bool count_as_sharable, - size_t overhead, - ResourceSizes* resource_sizes) { - auto entry_audit_message = std::string(audit_msg) + " pool entry"; - auto overhead_audit_message = std::string(audit_msg) + " pool overhead"; - - std::set all_ids; - for (uint32_t idx = 0; idx < pool.size(); idx++) { - auto set = usages.at(idx); - if (!set.empty()) { - auto amount = compute_string_size(pool, idx); - for (const auto id : set) { - auto usage_count = set.size(); - add_size(entry_audit_message.c_str(), id, amount, usage_count, - resource_sizes); - if (usage_count > 1 && count_as_sharable) { - add_shared_size("string pool", id, amount, resource_sizes); - } - all_ids.emplace(id); - } - } - } - for (const auto id : all_ids) { - add_size(overhead_audit_message.c_str(), id, overhead, all_ids.size(), - resource_sizes); - } -} - -// Attributes the size of string data and offsets to resource ids. Any string -// value that has many ids pointed to them will get counted as shared data. -void tally_string_sizes(const char* audit_msg, - const android::ResStringPool& pool, - const StringUsages& usages, - size_t overhead, - ResourceSizes* resource_sizes) { - tally_string_sizes(audit_msg, pool, usages, true, overhead, resource_sizes); -} - -// Attributes the typeSpec structure and zero to many type structures to the -// resource ids which are responsible for them. This is the step at which the -// table's chunk size and package header will be distributed to all non-empty -// resource ids. -std::set tally_type_and_entries( - const android::ResTable_package* package, - const android::ResTable_typeSpec* type_spec, - const std::vector& types, - apk::TableEntryParser& parser, - android::ResStringPool& key_strings, - ResourceSizes* resource_sizes, - ResourceConfigs* resource_configs, - ResourceNames* resource_names) { - // Reverse map of actual data to the potentially many entries that it may - // represent. This is to take into consideration the "canonical_entries" Redex - // config item and make sure to represent this as shared size in the many ids - // which can be represented with a single part of the arsc file. - std::map> data_to_ids; - std::set non_empty_res_ids; - std::unordered_map> - type_to_non_empty_ids; - - auto package_id = dtohl(package->id); - auto type_id = type_spec->id; - auto entry_count = dtohl(type_spec->entryCount); - always_assert_log(entry_count <= std::numeric_limits::max(), - "entry count %u too large for type", - entry_count); - - uint32_t upper = - (PACKAGE_MASK_BIT & (package_id << PACKAGE_INDEX_BIT_SHIFT)) | - (TYPE_MASK_BIT & (type_id << TYPE_INDEX_BIT_SHIFT)); - // Note: this vector could be empty. - for (const auto& type : types) { - for (uint16_t i = 0; i < entry_count; i++) { - uint32_t res_id = upper | i; - if (resource_configs->count(res_id) == 0) { - std::vector empty_vec; - resource_configs->emplace(res_id, empty_vec); - } - auto ev = parser.get_entry_for_config(res_id, &type->config); - if (!arsc::is_empty(ev)) { - non_empty_res_ids.emplace(res_id); - type_to_non_empty_ids[type].emplace(res_id); - auto entry = (android::ResTable_entry*)ev.getKey(); - // Store name of entry and name of its configs. - if (resource_names->count(res_id) == 0) { - auto entry_name = - arsc::get_string_from_pool(key_strings, dtohl(entry->key.index)); - resource_names->emplace(res_id, - entry_name.empty() ? "unknown" : entry_name); - } - auto config_name = std::string(type->config.toString().string()); - if (config_name.empty()) { - resource_configs->at(res_id).emplace_back("default"); - } else { - resource_configs->at(res_id).emplace_back(config_name); - } - // Keep track of if we've seen a redundant pointer before - data_to_ids[entry].emplace(res_id); - } - } - } - - // typeSpec overhead will be the size of the header itself, plus 4 bytes for - // every completely dead entry - size_t spec_overhead = dtohs(type_spec->header.headerSize) + - (entry_count - non_empty_res_ids.size()) * OFFSET_SIZE; - for (const auto res_id : non_empty_res_ids) { - add_size("ResTable_typeSpec flag", res_id, OFFSET_SIZE, 1, resource_sizes); - add_size("ResTable_typeSpec overhead", - res_id, - spec_overhead, - non_empty_res_ids.size(), - resource_sizes); - } - - // Last step, re-iterate over the resource ids in each type, and compute - // overhead of the type - for (const auto& type : types) { - auto& this_non_empty_set = type_to_non_empty_ids.at(type); - size_t type_overhead = dtohs(type->header.headerSize); - if ((type->flags & android::ResTable_type::FLAG_SPARSE) == 0) { - type_overhead += (entry_count - this_non_empty_set.size()) * OFFSET_SIZE; - } - for (uint16_t i = 0; i < entry_count; i++) { - uint32_t res_id = upper | i; - auto ev = parser.get_entry_for_config(res_id, &type->config); - if (!arsc::is_empty(ev)) { - add_size("ResTable_type offset", res_id, OFFSET_SIZE, 1, - resource_sizes); - add_size("ResTable_type overhead", res_id, type_overhead, - this_non_empty_set.size(), resource_sizes); - auto entry = (android::ResTable_entry*)ev.getKey(); - auto entry_value_size = ev.getValue(); - - auto& shared_set = data_to_ids.at(entry); - always_assert_log(!shared_set.empty(), - "Inconsistent entry pointers for res id 0x%x", - res_id); - add_size("ResTable_type entry and value", res_id, entry_value_size, - shared_set.size(), resource_sizes); - if (shared_set.size() != 1) { - add_shared_size("ResTable_type entry and value", res_id, - entry_value_size, resource_sizes); - } - } - } - } - return non_empty_res_ids; -} - -// Flattens data structures into an easily consumable form for outputting to a -// table / csv / whatever. -std::vector flatten(const ResourceSizes& resource_sizes, - const ResourceConfigs& resource_configs, - const ResourceNames& resource_names, - const TypeNames& type_names) { - std::vector results; - for (const auto& pair : resource_sizes) { - auto res_id = pair.first; - uint8_t type_id = (res_id >> TYPE_INDEX_BIT_SHIFT) & 0xFF; - const auto& type_name = type_names.at(type_id); - std::string resource_name; - auto search = resource_names.find(res_id); - if (search != resource_names.end()) { - resource_name = search->second; - } - Result r{res_id, type_name, resource_name, pair.second, - resource_configs.at(res_id)}; - results.emplace_back(r); - } - return results; -} -} // namespace - -namespace attribution { -// Returns the number of bytes used to encode the string's length, the string's -// characters and the null zero. -size_t compute_string_character_size(const android::ResStringPool& pool, - uint32_t idx) { - size_t len; - if (pool.isUTF8()) { - auto ptr = pool.string8At(idx, &len); - if (ptr != nullptr) { - // UTF-8 length of this string will be either 1 or two bytes preceeding - // the string. - auto utf8_units = length_units(len); - // UTF-16 length is also stored, same way as above (one or two bytes) - // preceeding the encoded UTF-8 length. - auto utf16_length = utf8_to_utf16_length((const uint8_t*)ptr, len); - auto utf16_units = length_units(utf16_length); - return utf16_units + utf8_units + len + 1; - } - } else { - auto ptr = pool.stringAt(idx, &len); - if (ptr != nullptr) { - // length, char data, plus null zero. - return (length_units(len) + len + 1) * sizeof(uint16_t); - } - } - TRACE(ARSC, 1, "BAD STRING INDEX %u", idx); - return 0; -} - -// Character data that makes up the string pool needs to be 4 byte aligned. This -// counts how many bytes of padding were added. -size_t count_padding(const android::ResStringPool_header* header, - const android::ResStringPool& pool) { - auto strings_start = dtohl(header->stringsStart); - if (pool.size() == 0 || strings_start == 0) { - return 0; - } - auto style_start = dtohl(header->stylesStart); - auto strings_end = - style_start == 0 ? dtohl(header->header.size) : style_start; - auto total_characters_size = strings_end - strings_start; - always_assert_log(total_characters_size >= 0, "Invalid string pool header"); - - size_t current_characters_size = 0; - for (uint32_t idx = 0; idx < pool.size(); idx++) { - current_characters_size += compute_string_character_size(pool, idx); - } - always_assert_log(total_characters_size >= current_characters_size, - "Miscount of character data"); - return total_characters_size - current_characters_size; -} - -// Return the number of bytes needed to encode the offset to string data, the -// number of bytes needed to encode the string's length, the character data, the -// null zero, and optionally how much data is needed to encode the spans and -// their character data. -size_t compute_string_size(const android::ResStringPool& pool, uint32_t idx) { - return compute_string_size_impl(pool, idx, true); -} - -std::vector ArscStats::compute() { - apk::TableEntryParser parser; - auto chunk_header = (android::ResChunk_header*)m_data; - auto success = parser.visit(chunk_header, m_file_len); - always_assert_log(success, "Could not parse arsc file!"); - // Maybe some day lift the following restriction, but we have no test data to - // exercise >1 package so assert for now. - always_assert_log(parser.m_packages.size() == 1, "Expected only 1 package."); - auto package_header = *parser.m_packages.begin(); - - // Step 1: parse the string pools and build up a vector of idx -> vector of - // resource ids that use it. - android::ResStringPool global_strings; - auto global_strings_header = parser.m_global_pool_header; - auto global_strings_size = dtohl(global_strings_header->header.size); - { - auto status = - global_strings.setTo(global_strings_header, global_strings_size, true); - always_assert_log(status == android::NO_ERROR, - "Could not parse global strings"); - } - - android::ResStringPool key_strings; - auto key_strings_header = parser.m_package_key_string_headers.begin()->second; - auto key_strings_size = dtohl(key_strings_header->header.size); - { - auto status = key_strings.setTo(key_strings_header, key_strings_size, true); - always_assert_log(status == android::NO_ERROR, - "Could not parse key strings"); - } - - android::ResStringPool type_strings; - auto type_strings_header = - parser.m_package_type_string_headers.begin()->second; - auto type_strings_size = dtohl(type_strings_header->header.size); - { - auto status = - type_strings.setTo(type_strings_header, type_strings_size, true); - always_assert_log(status == android::NO_ERROR, - "Could not parse type strings"); - } - - std::set empty_set; - StringUsages global_usages(global_strings.size(), empty_set); - StringUsages key_usages(key_strings.size(), empty_set); - StringUsages type_usages(type_strings.size(), empty_set); - populate_string_usages(parser, &global_usages, &key_usages, &type_usages); - - TRACE(ARSC, 1, "Global strings size: %u", global_strings_size); - auto global_overhead = - compute_overhead(global_strings_header, global_strings, global_usages); - TRACE(ARSC, 1, "Global strings overhead: %zu\n******************************", - global_overhead); - - TRACE(ARSC, 1, "Key strings size: %u", key_strings_size); - auto key_overhead = - compute_overhead(key_strings_header, key_strings, key_usages); - TRACE(ARSC, 1, "Key strings overhead: %zu\n******************************", - key_overhead); - - TRACE(ARSC, 1, "Type strings size: %u", type_strings_size); - auto type_strings_overhead = - compute_overhead(type_strings_header, type_strings, type_usages); - TRACE(ARSC, 1, "Type strings overhead: %zu\n******************************", - type_strings_overhead); - - // All the various maps to hold output data. - ResourceSizes resource_sizes; - ResourceConfigs resource_configs; - // Copy this to a new map, as the resid to name map may not be given. Any id - // not present in the map will be outputted as it appears in the arsc file. - ResourceNames resid_to_name(m_given_resid_to_name); - TypeNames type_names; - - initialize_resource_sizes(parser, &resource_sizes); - tally_string_sizes("global", global_strings, global_usages, global_overhead, - &resource_sizes); - tally_string_sizes("key", key_strings, key_usages, key_overhead, - &resource_sizes); - tally_string_sizes("type", type_strings, type_usages, - false /* don't count as sharable */, type_strings_overhead, - &resource_sizes); - - always_assert_log(type_strings.size() <= std::numeric_limits::max(), - "type strings too large"); - for (uint8_t t = 0; t < type_strings.size(); t++) { - auto type_name = arsc::get_string_from_pool(type_strings, t); - type_names.emplace(t + 1, type_name); - } - - // Add up sizes for every typeSpec and its type(s). - std::set all_non_empty_res_ids; - for (auto& type_info : parser.m_package_types.at(package_header)) { - // NOTE: we need to gather globally, the non-empty resource ids so we can - // distribute the table_overhead figure above. - auto non_empty_res_ids = tally_type_and_entries(package_header, - type_info.spec, - type_info.configs, - parser, - key_strings, - &resource_sizes, - &resource_configs, - &resid_to_name); - all_non_empty_res_ids.insert(non_empty_res_ids.begin(), - non_empty_res_ids.end()); - } - auto table_overhead = dtohs(chunk_header->headerSize) + - dtohs(package_header->header.headerSize); - for (const auto& res_id : all_non_empty_res_ids) { - add_size("table, package headers", res_id, table_overhead, - all_non_empty_res_ids.size(), &resource_sizes); - } - - return flatten(resource_sizes, resource_configs, resid_to_name, type_names); -} -} // namespace attribution diff --git a/tools/arsc/ArscStats.h b/tools/arsc/ArscStats.h deleted file mode 100644 index 6ecc48c9c3..0000000000 --- a/tools/arsc/ArscStats.h +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * - * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. - */ - -#pragma once - -#include -#include -#include -#include - -#include "androidfw/ResourceTypes.h" - -namespace attribution { -// -// Helper functions that are exposed to be easily testable. -// - -// Returns the number of bytes used to pad character data to 4 byte alignment. -size_t count_padding(const android::ResStringPool_header* header, - const android::ResStringPool& pool); -// Returns the number of bytes used to encode the length of the string, -// the string characters, and the null zero. -size_t compute_string_character_size(const android::ResStringPool& pool, - uint32_t idx); -// Returns the number of bytes used to represent a string in the pool as used by -// a resource reference, which will count the bytes for the offset and chase -// down spans and count that up too. -size_t compute_string_size(const android::ResStringPool& pool, uint32_t idx); - -// -// API for callers follows. -// - -struct ResourceSize { - // Number of bytes that exist in the arsc file only because of this single - // resource id. - size_t private_size = 0; - // Number of bytes that represent this resource (its name/value) and some - // other resource(s). Deduplication is and name obfuscation is what - // contributes to this. - size_t shared_size = 0; - // The amount of space in the file divided by the number of other resource ids - // that are responsible for the bytes. - double proportional_size = 0; -}; - -// Represents all computed data, for formatting/presenting in another format by -// caller. -struct Result { - uint32_t id; - std::string type; - std::string name; - ResourceSize sizes; - std::vector configs; -}; - -using ResourceNames = std::unordered_map; - -class ArscStats { - public: - ArscStats(const void* arsc_data, - size_t file_len, - const ResourceNames& resid_to_name) - : m_data(arsc_data), - m_file_len(file_len), - m_given_resid_to_name(resid_to_name) {} - - std::vector compute(); - - private: - const void* m_data; - size_t m_file_len; - const ResourceNames& m_given_resid_to_name; -}; - -} // namespace attribution diff --git a/tools/arsc/arsc_attribution.cpp b/tools/arsc/arsc_attribution.cpp deleted file mode 100644 index 23d6baca42..0000000000 --- a/tools/arsc/arsc_attribution.cpp +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * - * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "ArscStats.h" -#include "Debug.h" - -namespace { -inline std::string csv_escape(std::string value) { - always_assert_log(value.find('\n') == std::string::npos, - "not supporting new lines"); - if (value.find(',') != std::string::npos || - value.find('\"') != std::string::npos) { - boost::replace_all(value, "\"", "\"\""); - return "\"" + value + "\""; - } - return value; -} - -void print_csv(const std::vector& results, - bool hide_uninteresting) { - std::cout << "ID,Type,Name,Private Size,Shared Size,Proportional Size,Config " - "Count,Configs" - << std::endl; - for (const auto& result : results) { - if (hide_uninteresting && result.sizes.proportional_size == 0) { - continue; - } - auto joined_configs = boost::algorithm::join(result.configs, " "); - std::cout << "0x" << std::hex << result.id << std::dec << "," - << csv_escape(result.type) << "," << csv_escape(result.name) - << "," << result.sizes.private_size << "," - << result.sizes.shared_size << "," - << result.sizes.proportional_size << "," << result.configs.size() - << "," << csv_escape(joined_configs) << std::endl; - } -} - -bool read_rename_map(const std::string& resid_to_name_path, - attribution::ResourceNames* out) { - std::ifstream deob_file(resid_to_name_path, std::ifstream::binary); - Json::Reader reader; - Json::Value root; - if (!reader.parse(deob_file, root)) { - std::cerr << reader.getFormatedErrorMessages() << std::endl; - return false; - } - for (Json::ValueIterator it = root.begin(); it != root.end(); ++it) { - std::string hex_str(it.key().asCString()); - uint32_t id = std::stoul(hex_str, nullptr, 16); - std::string name(it->asCString()); - out->emplace(id, name); - } - return true; -} - -void do_attribution(const void* data, - size_t len, - bool hide_uninteresting, - const attribution::ResourceNames& given_resid_to_name) { - attribution::ArscStats stats(data, len, given_resid_to_name); - print_csv(stats.compute(), hide_uninteresting); -} - -void run(int argc, char** argv) { - namespace po = boost::program_options; - po::options_description desc("Allowed options"); - desc.add_options()("help,h", "print this help message"); - desc.add_options()( - "file", po::value(), "required path to arsc file"); - desc.add_options()("resid", - po::value(), - "optional path to resource id to name json file"); - desc.add_options()("hide-uninteresting", - po::bool_switch(), - "suppress resource ids that are empty"); - - po::variables_map vm; - try { - po::store(po::parse_command_line(argc, argv, desc), vm); - po::notify(vm); - } catch (std::exception& e) { - std::cerr << e.what() << std::endl << std::endl; - exit(EXIT_FAILURE); - } - - if (vm.count("help")) { - std::cout << desc << "\n"; - exit(EXIT_SUCCESS); - } - if (!vm.count("file")) { - std::cerr << desc << "\n"; - exit(EXIT_FAILURE); - } - - attribution::ResourceNames resid_to_name; - if (vm.count("resid")) { - auto resid_to_name_path = vm["resid"].as(); - if (!read_rename_map(resid_to_name_path, &resid_to_name)) { - std::cerr << "Failed to parse resid to name json file: " - << resid_to_name_path << std::endl; - exit(EXIT_FAILURE); - } - } - auto arsc_path = vm["file"].as(); - auto map = std::make_unique(); - auto mode = (std::ios_base::openmode)(std::ios_base::in); - map->open(arsc_path, mode); - if (!map->is_open()) { - std::cerr << "Could not map " << arsc_path << std::endl; - exit(EXIT_FAILURE); - } - do_attribution(map->const_data(), - map->size(), - vm["hide-uninteresting"].as(), - resid_to_name); -} -} // namespace - -// This tool accepts an .arsc file and spits out a csv of useful stats on how -// much space is being taken up by what resource id. It aims to be similar in -// concept to https://github.com/google/android-arscblamer but operate -// differently to: -// 1) handle arsc files that have been obfuscated, apply a deobfuscation map. -// 2) be able to traverse files that have been mangled substantially by -// deduplication, canonical offsets, etc. -// 3) handle shared data in a more sensible way (it seems more intuitive to -// count type string data as overhead, not shared data). -int main(int argc, char** argv) { - try { - run(argc, argv); - } catch (std::exception& e) { - std::cerr << e.what() << std::endl << std::endl; - exit(EXIT_FAILURE); - } - return 0; -} diff --git a/tools/python/symbolicator/symbolicator.py b/tools/python/symbolicator/symbolicator.py index 6b616738e0..711467d74e 100755 --- a/tools/python/symbolicator/symbolicator.py +++ b/tools/python/symbolicator/symbolicator.py @@ -14,12 +14,12 @@ import signal import sys -from debug_line_map import DebugLineMap -from dexdump import DexdumpSymbolicator -from iodi import IODIMetadata -from line_unmap import PositionMap -from logcat import LogcatSymbolicator -from symbol_files import SymbolFiles +from debug_line_map import DebugLineMap # pyre-fixme[21] +from dexdump import DexdumpSymbolicator # pyre-fixme[21] +from iodi import IODIMetadata # pyre-fixme[21] +from line_unmap import PositionMap # pyre-fixme[21] +from logcat import LogcatSymbolicator # pyre-fixme[21] +from symbol_files import SymbolFiles # pyre-fixme[21] # A simple symbolicator for line-based input, diff --git a/tools/redex-all/main.cpp b/tools/redex-all/main.cpp index f2a9e77489..e5a8723e48 100644 --- a/tools/redex-all/main.cpp +++ b/tools/redex-all/main.cpp @@ -42,7 +42,6 @@ #include "ConfigFiles.h" #include "ControlFlow.h" // To set DEBUG. #include "Debug.h" -#include "DebugUtils.h" #include "DexClass.h" #include "DexHasher.h" #include "DexLoader.h" @@ -56,7 +55,6 @@ #include "JemallocUtil.h" #include "KeepReason.h" #include "Macros.h" -#include "MallocDebug.h" #include "MonitorCount.h" #include "NoOptimizationsMatcher.h" #include "OptData.h" @@ -68,7 +66,6 @@ #include "ProguardPrintConfiguration.h" // New ProGuard configuration #include "ReachableClasses.h" #include "RedexContext.h" -#include "RedexProperties.h" #include "RedexPropertiesManager.h" #include "RedexPropertyCheckerRegistry.h" #include "RedexResources.h" @@ -76,7 +73,6 @@ #include "SanitizersConfig.h" #include "ScopedMemStats.h" #include "Show.h" -#include "ThreadPool.h" #include "Timer.h" #include "ToolsCommon.h" #include "Walkers.h" @@ -243,7 +239,6 @@ Json::Value reflect_config(const Configurable::Reflection& cr) { } void add_pass_properties_reflection(Json::Value& value, Pass* pass) { - using namespace redex_properties; auto interactions = pass->get_property_interactions(); if (interactions.empty()) { return; @@ -256,16 +251,16 @@ void add_pass_properties_reflection(Json::Value& value, Pass* pass) { for (const auto& [property, inter] : interactions) { if (inter.establishes) { - establishes.append(get_name(property)); + establishes.append(property); } if (inter.requires_) { - requires_.append(get_name(property)); + requires_.append(property); } if (inter.preserves) { - preserves.append(get_name(property)); + preserves.append(property); } if (inter.requires_finally) { - requires_finally.append(get_name(property)); + requires_finally.append(property); } } @@ -283,23 +278,23 @@ Json::Value reflect_property_definitions() { properties["properties"] = []() { Json::Value prop_map; - auto all = redex_properties::get_all_properties(); + auto all = redex_properties::Manager::get_all_properties(); for (auto& prop : all) { Json::Value prop_value; - prop_value["negative"] = redex_properties::is_negative(prop); - prop_map[redex_properties::get_name(prop)] = std::move(prop_value); + prop_value["negative"] = redex_properties::Manager::is_negative(prop); + prop_map[prop] = std::move(prop_value); } return prop_map; }(); auto create_sorted = [](const auto& input) { - std::vector tmp; + std::vector tmp; std::copy(input.begin(), input.end(), std::back_inserter(tmp)); std::sort(tmp.begin(), tmp.end()); Json::Value holder = Json::arrayValue; for (auto& prop : tmp) { - holder.append(redex_properties::get_name(prop)); + holder.append(prop); } return holder; }; @@ -660,7 +655,6 @@ Json::Value get_stats(const dex_stats_t& stats) { val["num_annotations"] = stats.num_annotations; val["num_bytes"] = stats.num_bytes; val["num_instructions"] = stats.num_instructions; - val["num_tries"] = stats.num_tries; val["num_unique_types"] = stats.num_unique_types; val["num_unique_protos"] = stats.num_unique_protos; @@ -862,12 +856,6 @@ Json::Value get_times(double cpu_time_s) { cpu_element["cpu_time"] = std::round(cpu_time_s * 10) / 10.0; list.append(cpu_element); } - { - Json::Value thread_pool_element; - thread_pool_element["thread_pool_size"] = - redex_thread_pool::ThreadPool::get_instance()->size() * 1.0; - list.append(thread_pool_element); - } return list; } @@ -1521,25 +1509,17 @@ void redex_backend(ConfigFiles& conf, auto method_move_map = conf.metafile(json_config.get("method_move_map", std::string())); if (needs_addresses) { - Timer t2{"Writing debug line mapping"}; write_debug_line_mapping(debug_line_map_filename, method_to_id, code_debug_lines, stores, needs_debug_line_mapping); } if (is_iodi(dik)) { - Timer t2{"Writing IODI metadata"}; iodi_metadata.write(iodi_metadata_filename, method_to_id); } - { - Timer t2{"Writing position map"}; - pos_mapper->write_map(); - } - { - Timer t2{"Collecting output stats"}; - stats["output_stats"] = - get_output_stats(output_totals, output_dexes_stats, manager, - instruction_lowering_stats, pos_mapper.get()); - } + pos_mapper->write_map(); + stats["output_stats"] = + get_output_stats(output_totals, output_dexes_stats, manager, + instruction_lowering_stats, pos_mapper.get()); print_warning_summary(); if (dex_output_config.write_class_sizes) { @@ -1664,7 +1644,6 @@ void copy_proguard_stats(Json::Value& stats) { } int check_pass_properties(const Arguments& args) { - using namespace redex_properties; // Cannot parse GlobalConfig nor passes, as they may require binding // to dex elements. So this looks more complicated than necessary. @@ -1683,12 +1662,13 @@ int check_pass_properties(const Arguments& args) { } auto const& all_passes = PassRegistry::get().get_passes(); - auto props_manager = - Manager(conf, PropertyCheckerRegistry::get().get_checkers()); + auto props_manager = redex_properties::Manager( + conf, redex_properties::PropertyCheckerRegistry::get().get_checkers()); auto active_passes = PassManager::compute_activated_passes(all_passes, conf, &pmc); - std::vector> pass_interactions; + std::vector> + pass_interactions; for (const auto& [pass, _] : active_passes.activated_passes) { auto m = pass->get_property_interactions(); for (auto it = m.begin(); it != m.end();) { @@ -1701,12 +1681,13 @@ int check_pass_properties(const Arguments& args) { always_assert_log(property_interaction.is_valid(), "%s has an invalid property interaction for %s", - pass->name().c_str(), get_name(name)); + pass->name().c_str(), name.c_str()); ++it; } pass_interactions.emplace_back(pass->name(), std::move(m)); } - auto failure = Manager::verify_pass_interactions(pass_interactions, conf); + auto failure = redex_properties::Manager::verify_pass_interactions( + pass_interactions, conf); if (failure) { std::cerr << "Illegal pass order:\n" << *failure << std::endl; return 1; @@ -1743,8 +1724,6 @@ int main(int argc, char* argv[]) { auto maybe_global_profile = ScopedCommandProfiling::maybe_from_env("GLOBAL_", "global"); - redex_thread_pool::ThreadPool::create(); - ConcurrentContainerConcurrentDestructionScope concurrent_container_destruction_scope; @@ -1892,9 +1871,5 @@ int main(int argc, char* argv[]) { pretty_bytes(vm_stats.vm_hwm).c_str()); } - redex_thread_pool::ThreadPool::destroy(); - - malloc_debug::set_shutdown(); - return 0; } diff --git a/util/MallocDebug.cpp b/util/MallocDebug.cpp index 9868ab3233..18674a11fe 100644 --- a/util/MallocDebug.cpp +++ b/util/MallocDebug.cpp @@ -95,18 +95,15 @@ static auto libc_posix_memalign = #ifdef __linux__ extern "C" { -extern void* __libc_malloc(size_t size); // NOLINT(bugprone-reserved-identifier) -// NOLINTNEXTLINE(bugprone-reserved-identifier) +extern void* __libc_malloc(size_t size); extern void* __libc_calloc(size_t nelem, size_t elsize); -// NOLINTNEXTLINE(bugprone-reserved-identifier) extern void* __libc_memalign(size_t alignment, size_t size); // This isn't found? // extern int __posix_memalign(void** out, size_t alignment, size_t size); } -static auto libc_malloc = __libc_malloc; // NOLINT(bugprone-reserved-identifier) -static auto libc_calloc = __libc_calloc; // NOLINT(bugprone-reserved-identifier) -// NOLINTNEXTLINE(bugprone-reserved-identifier) +static auto libc_malloc = __libc_malloc; +static auto libc_calloc = __libc_calloc; static auto libc_memalign = __libc_memalign; // static auto libc_posix_memalign = __posix_memalign; #endif @@ -288,44 +285,22 @@ class MallocDebug { }; thread_local MallocDebug malloc_debug; -bool shutdown{false}; } // namespace extern "C" { -void* malloc(size_t sz) { - if (shutdown) { - return libc_malloc(sz); - } - return malloc_debug.malloc(sz); -} +void* malloc(size_t sz) { return malloc_debug.malloc(sz); } void* calloc(size_t nelem, size_t elsize) { - if (shutdown) { - return libc_calloc(nelem, elsize); - } return malloc_debug.calloc(nelem, elsize); } void* memalign(size_t alignment, size_t bytes) { - if (shutdown) { - return libc_memalign(alignment, bytes); - } return malloc_debug.memalign(alignment, bytes); } int posix_memalign(void** out, size_t alignment, size_t size) { - if (shutdown) { - *out = memalign(alignment, size); - return 0; - } return malloc_debug.posix_memalign(out, alignment, size); } -} // extern "C" - -namespace malloc_debug { - -void set_shutdown() { shutdown = true; } - -} // namespace malloc_debug +} diff --git a/util/MallocDebug.h b/util/MallocDebug.h deleted file mode 100644 index b5920e4e04..0000000000 --- a/util/MallocDebug.h +++ /dev/null @@ -1,15 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * - * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. - */ - -#pragma once - -namespace malloc_debug { - -// NOLINTNEXTLINE(misc-definitions-in-headers) -void __attribute__((weak)) set_shutdown() {} - -} // namespace malloc_debug diff --git a/util/Sha1.cpp b/util/Sha1.cpp index 19eb98e402..3007e401ce 100644 --- a/util/Sha1.cpp +++ b/util/Sha1.cpp @@ -232,8 +232,8 @@ void sha1_update(Sha1Context* context, index = (unsigned int)((context->count[0] >> 3) & 0x3F); /* Update number of bits */ - context->count[0] += ((unsigned int)inputLen << 3); - if (context->count[0] < ((unsigned int)inputLen << 3)) { + if ((context->count[0] += ((unsigned int)inputLen << 3)) < + ((unsigned int)inputLen << 3)) { context->count[1]++; } context->count[1] += ((unsigned int)inputLen >> 29);