From 6d40ac01f72543e78f102cc53b27f734f36c3c10 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 18 Oct 2022 22:18:53 +0200 Subject: [PATCH 01/30] Profiling, first steps Prepare for profiling with NDK's `simpleperf`: * Build shared libraries with full debug info, using DWARF5 (for better compression) and debug info optimized for lldb * Adapt `simpleperf` C++ API to Xamarin.Android's needs and style (the original source uses streams which we can't support and also... doesn't build out of the box) --- src/monodroid/CMakeLists.txt | 24 +- src/monodroid/jni/monodroid-glue.cc | 3 + src/monodroid/jni/simpleperf.cc | 546 ++++++++++++++++++++++++++++ src/monodroid/jni/simpleperf.hh | 246 +++++++++++++ 4 files changed, 816 insertions(+), 3 deletions(-) create mode 100644 src/monodroid/jni/simpleperf.cc create mode 100644 src/monodroid/jni/simpleperf.hh diff --git a/src/monodroid/CMakeLists.txt b/src/monodroid/CMakeLists.txt index 7f45848792b..db522376056 100644 --- a/src/monodroid/CMakeLists.txt +++ b/src/monodroid/CMakeLists.txt @@ -37,15 +37,15 @@ endif() option(ENABLE_CLANG_ASAN "Enable the clang AddressSanitizer support" OFF) option(ENABLE_CLANG_UBSAN "Enable the clang UndefinedBehaviorSanitizer support" OFF) +option(ENABLE_NET "Enable compilation for .NET 6+" OFF) +option(ENABLE_TIMING "Build with timing support" OFF) -if(ENABLE_CLANG_ASAN OR ENABLE_CLANG_UBSAN) +if(ENABLE_CLANG_ASAN OR ENABLE_CLANG_UBSAN OR (ANDROID AND ENABLE_NET)) set(STRIP_DEBUG_DEFAULT OFF) else() set(STRIP_DEBUG_DEFAULT ON) endif() -option(ENABLE_NET "Enable compilation for .NET 6+" OFF) -option(ENABLE_TIMING "Build with timing support" OFF) option(STRIP_DEBUG "Strip debugging information when linking" ${STRIP_DEBUG_DEFAULT}) option(DISABLE_DEBUG "Disable the built-in debugging code" OFF) option(USE_CCACHE "Use ccache, if found, to speed up recompilation" ${CCACHE_OPTION_DEFAULT}) @@ -365,6 +365,18 @@ if(ANDROID) ) endif() + # Always build with the most extensive debug info possible, we can strip it out later on and + # it may be really useful to have it even in release builds, should the user choose so. + list(APPEND LOCAL_COMMON_COMPILER_ARGS + -glldb # NDK uses lldb, makes sense to optimize for it + -gdwarf-5 # DWARF-4 is the current default, but DWARF-5 produces smaller debug sections (among other changes: https://dwarfstd.org/Dwarf5Std.php) + ) + + list(APPEND LOCAL_COMMON_LINKER_ARGS + -glldb + -gdwarf-5 + ) + unset(SANITIZER_FLAGS) if (ENABLE_CLANG_ASAN) set(SANITIZER_FLAGS -fsanitize=address) @@ -512,6 +524,12 @@ if(ANDROID) ) endif() + if(ENABLE_NET) + list(APPEND XAMARIN_MONODROID_SOURCES + ${SOURCES_DIR}/simpleperf.cc + ) + endif() + if(NOT USES_LIBSTDCPP) list(APPEND XAMARIN_MONODROID_SOURCES ${BIONIC_SOURCES_DIR}/cxa_guard.cc diff --git a/src/monodroid/jni/monodroid-glue.cc b/src/monodroid/jni/monodroid-glue.cc index 5a8a3500951..571f18e9e13 100644 --- a/src/monodroid/jni/monodroid-glue.cc +++ b/src/monodroid/jni/monodroid-glue.cc @@ -2169,6 +2169,9 @@ MonodroidRuntime::Java_mono_android_Runtime_initInternal (JNIEnv *env, jclass kl jobject loader, jobjectArray assembliesJava, jint apiLevel, jboolean isEmulator, jboolean haveSplitApks) { + log_warn (LOG_DEFAULT, "Testing symbols"); + abort (); + char *mono_log_mask_raw = nullptr; char *mono_log_level_raw = nullptr; diff --git a/src/monodroid/jni/simpleperf.cc b/src/monodroid/jni/simpleperf.cc new file mode 100644 index 00000000000..8704ebd0ab8 --- /dev/null +++ b/src/monodroid/jni/simpleperf.cc @@ -0,0 +1,546 @@ +// +// SPDX-License-Identifier: MIT +// SPDX-License-Identifier: Apache-2.0 +// +// Support for managing simpleperf session from within the runtime +// +// Heavily based on https://android.googlesource.com/platform/system/extras/+/refs/tags/android-13.0.0_r11/simpleperf/app_api/cpp/simpleperf.cpp +// +// Because the original code is licensed under the `Apache-2.0` license, this file is dual-licensed under the `MIT` and +// `Apache-2.0` licenses +// +// We can't use the original source because of the C++ stdlib features it uses (I/O streams which we can't use because +// we don't reference libc++) +// +// The API is very similar to the original, with occasional stylistic changes and some behavioral changes (for instance, +// we do not abort the process if tracing fails - instead we log errors and continue running) +// +#include +#include +#include +#include +#include +#include +#include + +#include "android-system.hh" +#include "logger.hh" +#include "simpleperf.hh" +#include "strings.hh" + +using namespace xamarin::android::internal; + +std::string +RecordOptions::get_default_output_filename () noexcept +{ + time_t t = time (nullptr); + + struct tm tm; + if (localtime_r (&t, &tm) != &tm) { + return "perf.data"; + } + + char* buf = nullptr; + + // TODO: don't use asprintf + asprintf (&buf, "perf-%02d-%02d-%02d-%02d-%02d.data", tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); + + std::string result = buf; + free (buf); + + return result; +} + +std::vector +RecordOptions::to_record_args () noexcept +{ + std::vector args; + if (output_filename.empty ()) { + output_filename = get_default_output_filename (); + } + + args.insert (args.end (), {"-o", output_filename}); + args.insert (args.end (), {"-e", event}); + args.insert (args.end (), {"-f", std::to_string(freq)}); + + if (duration_in_seconds != 0.0) { + args.insert (args.end (), {"--duration", std::to_string (duration_in_seconds)}); + } + + if (threads.empty ()) { + args.insert (args.end (), {"-p", std::to_string (getpid ())}); + } else { + std::string threads_arg; + + for (auto const& thread_id : threads) { + if (!threads_arg.empty ()) { + threads_arg.append (","); + } + + threads_arg.append (std::to_string (thread_id)); + } + + args.insert(args.end (), {"-t", threads_arg}); + } + + if (dwarf_callgraph) { + args.push_back ("-g"); + } else if (fp_callgraph) { + args.insert (args.end (), {"--call-graph", "fp"}); + } + + if (trace_offcpu) { + args.push_back ("--trace-offcpu"); + } + + return args; +} + +ProfileSession::ProfileSession () noexcept +{ + std::string input_file {"/proc/self/cmdline"}; + FILE* fp = fopen (input_file.c_str (), "r"); + if (fp == nullptr) { + log_error (LOG_DEFAULT, "simpleperf: failed to open %s: %s", input_file.c_str (), strerror (errno)); + return; + } + + std::string s = read_file (fp, input_file); + for (size_t i = 0; i < s.size (); i++) { + if (s[i] == '\0') { + s = s.substr (0, i); + break; + } + } + + std::string app_data_dir = "/data/data/" + s; + uid_t uid = getuid (); + if (uid >= AID_USER_OFFSET) { + int user_id = uid / AID_USER_OFFSET; + app_data_dir = "/data/user/" + std::to_string (user_id) + "/" + s; + } + + session_valid = true; +} + +std::string +ProfileSession::read_file (FILE* fp, std::string const& path) noexcept +{ + std::string s; + if (fp == nullptr) { + return s; + } + + constexpr size_t BUF_SIZE = 200; + std::array buf; + + while (true) { + size_t n = fread (buf.data (), 1, buf.size (), fp); + if (n < buf.size ()) { + if (ferror (fp)) { + log_warn (LOG_DEFAULT, "simpleperf: an error occurred while reading input file %s: %s", path.c_str (), strerror (errno)); + } + + break; + } + + s.insert (s.end (), buf.data (), buf.data () + n); + } + + fclose (fp); + return s; +} + +bool +ProfileSession::session_is_valid () const noexcept +{ + if (session_valid) { + return true; + } + + log_warn (LOG_DEFAULT, "simpleperf: profiling session object hasn't been initialized properly, profiling will NOT produce any results"); + return false; +} + +std::string +ProfileSession::find_simpleperf_in_temp_dir () const noexcept +{ + const std::string path = "/data/local/tmp/simpleperf"; + if (!is_executable_file (path)) { + return ""; + } + // Copy it to app_dir to execute it. + const std::string to_path = app_data_dir_ + "/simpleperf"; + if (!run_cmd ({"/system/bin/cp", path.c_str(), to_path.c_str()}, nullptr)) { + return ""; + } + + // For apps with target sdk >= 29, executing app data file isn't allowed. + // For android R, app context isn't allowed to use perf_event_open. + // So test executing downloaded simpleperf. + std::string s; + if (!run_cmd ({to_path.c_str(), "list", "sw"}, &s)) { + return ""; + } + + if (s.find ("cpu-clock") == std::string::npos) { + return ""; + } + + return to_path; +} + +bool +ProfileSession::run_cmd (std::vector args, std::string* standard_output) noexcept +{ + std::array stdout_fd; + if (pipe (stdout_fd.data ()) != 0) { + return false; + } + + args.push_back (nullptr); + + // Fork handlers (like gsl_library_close) may hang in a multi-thread environment. + // So we use vfork instead of fork to avoid calling them. + int pid = vfork (); + if (pid == -1) { + log_warn (LOG_DEFAULT, "simpleperf: `vfork` failed: %s", strerror (errno)); + return false; + } + + if (pid == 0) { + // child process + close (stdout_fd[0]); + dup2 (stdout_fd[1], 1); + close (stdout_fd[1]); + + execvp (const_cast(args[0]), const_cast(args.data ())); + + log_error (LOG_DEFAULT, "simpleperf: failed to run %s: %s", args[0], strerror (errno)); + _exit (1); + } + + // parent process + close (stdout_fd[1]); + + int status; + pid_t result = TEMP_FAILURE_RETRY (waitpid (pid, &status, 0)); + if (result == -1) { + log_error (LOG_DEFAULT, "simpleperf: failed to call waitpid: %s", strerror (errno)); + } + + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + return false; + } + + if (standard_output == nullptr) { + close (stdout_fd[0]); + } else { + *standard_output = read_file (fdopen (stdout_fd[0], "r"), "pipe"); + } + + return true; +} + +bool +ProfileSession::is_executable_file (const std::string& path) noexcept +{ + struct stat st; + + if (stat (path.c_str (), &st) != 0) { + return false; + } + + return S_ISREG(st.st_mode) && ((st.st_mode & S_IXUSR) == S_IXUSR); +} + +std::string +ProfileSession::find_simpleperf () const noexcept +{ + // 1. Try /data/local/tmp/simpleperf first. Probably it's newer than /system/bin/simpleperf. + std::string simpleperf_path = find_simpleperf_in_temp_dir (); + if (!simpleperf_path.empty()) { + return simpleperf_path; + } + + // 2. Try /system/bin/simpleperf, which is available on Android >= Q. + simpleperf_path = "/system/bin/simpleperf"; + if (is_executable_file (simpleperf_path)) { + return simpleperf_path; + } + + log_error (LOG_DEFAULT, "simpleperf: can't find simpleperf on device. Please run api_profiler.py."); + return ""; +} + +bool +ProfileSession::check_if_perf_enabled () noexcept +{ + dynamic_local_string prop; + + if (AndroidSystem::monodroid_get_system_property ("persist.simpleperf.profile_app_uid", prop) <= 0) { + return false; + } + + if (prop.get () == std::to_string (getuid ())) { + prop.clear (); + + AndroidSystem::monodroid_get_system_property ("persist.simpleperf.profile_app_expiration_time", prop); + if (!prop.empty ()) { + errno = 0; + long expiration_time = strtol (prop.get (), nullptr, 10); + if (errno == 0 && expiration_time > time (nullptr)) { + return true; + } + } + } + + if (AndroidSystem::monodroid_get_system_property ("security.perf_harden", prop) <= 0 || prop.empty ()) { + return true; + } + + if (prop.get ()[0] == '1') { + log_error (LOG_DEFAULT, "simpleperf: recording app isn't enabled on the device. Please run api_profiler.py."); + return false; + } + + return true; +} + +bool +ProfileSession::create_simpleperf_data_dir () const noexcept +{ + struct stat st; + if (stat (simpleperf_data_dir_.c_str (), &st) == 0 && S_ISDIR (st.st_mode)) { + return true; + } + + if (mkdir (simpleperf_data_dir_.c_str (), 0700) == -1) { + log_error (LOG_DEFAULT, "simpleperf: failed to create simpleperf data dir %s: %s", simpleperf_data_dir_.c_str(), strerror (errno)); + return false; + } + + return true; +} + +bool +ProfileSession::create_simpleperf_process (std::string const& simpleperf_path, std::vector const& record_args) noexcept +{ + // 1. Create control/reply pips. + std::array control_fd { -1, -1 }; + std::array reply_fd { -1, -1 }; + + if (pipe (control_fd.data ()) != 0 || pipe (reply_fd.data ()) != 0) { + log_error (LOG_DEFAULT, "simpleperf: failed to call pipe: %s", strerror (errno)); + return false; + } + + // 2. Prepare simpleperf arguments. + std::vector args; + args.emplace_back (simpleperf_path); + args.emplace_back ("record"); + args.emplace_back ("--log-to-android-buffer"); + args.insert (args.end (), {"--log", "debug"}); + args.emplace_back ("--stdio-controls-profiling"); + args.emplace_back ("--in-app"); + args.insert (args.end (), {"--tracepoint-events", "/data/local/tmp/tracepoint_events"}); + args.insert (args.end (), record_args.begin (), record_args.end ()); + + char* argv[args.size () + 1]; + for (size_t i = 0; i < args.size (); ++i) { + argv[i] = &args[i][0]; + } + argv[args.size ()] = nullptr; + + // 3. Start simpleperf process. + // Fork handlers (like gsl_library_close) may hang in a multi-thread environment. + // So we use vfork instead of fork to avoid calling them. + int pid = vfork (); + if (pid == -1) { + auto close_fds = [](std::array const& fds) { + for (auto fd : fds) { + close (fd); + } + }; + + log_error (LOG_DEFAULT, "simpleperf: failed to fork: %s", strerror (errno)); + close_fds (control_fd); + close_fds (reply_fd); + + return false; + } + + if (pid == 0) { + // child process + close (control_fd[1]); + dup2 (control_fd[0], 0); // simpleperf read control cmd from fd 0. + close (control_fd[0]); + close (reply_fd[0]); + dup2 (reply_fd[1], 1); // simpleperf writes reply to fd 1. + close (reply_fd[0]); + chdir (simpleperf_data_dir_.c_str()); + execvp (argv[0], argv); + + log_fatal (LOG_DEFAULT, "simpleperf: failed to call exec: %s", strerror (errno)); + } + + // parent process + close (control_fd[0]); + control_fd_ = control_fd[1]; + close (reply_fd[1]); + reply_fd_ = reply_fd[0]; + simpleperf_pid_ = pid; + + // 4. Wait until simpleperf starts recording. + std::string start_flag = read_reply (); + if (start_flag != "started") { + log_error (LOG_DEFAULT, "simpleperf: failed to receive simpleperf start flag"); + return false; + } + + return true; +} + +void +ProfileSession::start_recording (std::vector const& record_args) noexcept +{ + if (!session_is_valid () || !check_if_perf_enabled ()) { + return; + } + + std::lock_guard guard {lock_}; + if (state_ != State::NOT_YET_STARTED) { + log_error (LOG_DEFAULT, "simpleperf: start_recording: session in wrong state %d", state_); + } + + for (auto const& arg : record_args) { + if (arg == "--trace-offcpu") { + trace_offcpu_ = true; + } + } + + std::string simpleperf_path = find_simpleperf (); + + if (!create_simpleperf_data_dir ()) { + return; + } + + if (!create_simpleperf_process (simpleperf_path, record_args)) { + return; + } + + state_ = State::STARTED; +} + +void +ProfileSession::pause_recording () noexcept +{ + if (!session_is_valid ()) { + return; + } + + std::lock_guard guard(lock_); + if (state_ != State::STARTED) { + log_error (LOG_DEFAULT, "simpleperf: pause_recording: session in wrong state %d", state_); + return; + } + + if (trace_offcpu_) { + log_warn (LOG_DEFAULT, "simpleperf: --trace-offcpu doesn't work well with pause/resume recording"); + } + + if (!send_cmd ("pause")) { + return; + } + + state_ = State::PAUSED; +} + +void +ProfileSession::resume_recording () noexcept +{ + if (!session_is_valid ()) { + return; + } + + std::lock_guard guard {lock_}; + + if (state_ != State::PAUSED) { + log_error (LOG_DEFAULT, "simpleperf: resume_recording: session in wrong state %d", state_); + } + + if (!send_cmd ("resume")) { + return; + } + + state_ = State::STARTED; +} + +void +ProfileSession::stop_recording () noexcept +{ + if (!session_is_valid ()) { + return; + } + + std::lock_guard guard {lock_}; + + if (state_ != State::STARTED && state_ != State::PAUSED) { + log_error (LOG_DEFAULT, "simpleperf: stop_recording: session in wrong state %d", state_); + return; + } + + // Send SIGINT to simpleperf to stop recording. + if (kill (simpleperf_pid_, SIGINT) == -1) { + log_error (LOG_DEFAULT, "simpleperf: failed to stop simpleperf: %s", strerror (errno)); + return; + } + + int status; + pid_t result = TEMP_FAILURE_RETRY(waitpid(simpleperf_pid_, &status, 0)); + if (result == -1) { + log_error (LOG_DEFAULT, "simpleperf: failed to call waitpid: %s", strerror (errno)); + return; + } + + if (!WIFEXITED (status) || WEXITSTATUS (status) != 0) { + log_error (LOG_DEFAULT, "simpleperf: simpleperf exited with error, status = 0x%x", status); + return; + } + + state_ = State::STOPPED; +} + +std::string +ProfileSession::read_reply () noexcept +{ + std::string s; + while (true) { + char c; + ssize_t result = TEMP_FAILURE_RETRY (read (reply_fd_, &c, 1)); + if (result <= 0 || c == '\n') { + break; + } + s.push_back(c); + } + + return s; +} + +bool +ProfileSession::send_cmd (std::string const& cmd) noexcept +{ + std::string data = cmd + "\n"; + + if (TEMP_FAILURE_RETRY (write (control_fd_, &data[0], data.size())) != static_cast(data.size ())) { + log_error (LOG_DEFAULT, "simpleperf: failed to send cmd to simpleperf: %s", strerror (errno)); + return false; + } + + if (read_reply () != "ok") { + log_error (LOG_DEFAULT, "simpleperf: failed to run cmd in simpleperf: %s", cmd.c_str ()); + return false; + } + + return true; +} diff --git a/src/monodroid/jni/simpleperf.hh b/src/monodroid/jni/simpleperf.hh new file mode 100644 index 00000000000..2f737d73b4c --- /dev/null +++ b/src/monodroid/jni/simpleperf.hh @@ -0,0 +1,246 @@ +// +// SPDX-License-Identifier: MIT +// SPDX-License-Identifier: Apache-2.0 +// +// Support for managing simpleperf session from within the runtime +// +// Heavily based on https://android.googlesource.com/platform/system/extras/+/refs/tags/android-13.0.0_r11/simpleperf/app_api/cpp/simpleperf.cpp +// +// Because the original code is licensed under the `Apache-2.0` license, this file is dual-licensed under the `MIT` and +// `Apache-2.0` licenses +// +// We can't use the original source because of the C++ stdlib features it uses (I/O streams which we can't use because +// we don't reference libc++) +// +// The API is very similar to the original, with occasional stylistic changes and some behavioral changes (for instance, +// we do not abort the process if tracing fails - instead we log errors and continue running). Portions are +// reimplemented in a way that better fits Xamarin.Android's purposes. We also don't split the classes into interface +// and implementation bits - there's no need since we use this API entirely internally. +// +// Stylistic changes include indentation and renaming of all methods to use lower-case words separated by underscores instead of +// camel case (because that's the style Xamarin.Android sources use). Names are otherwise the same as in the original +// code, for easier porting of potential changes. +// +#if !defined (__SIMPLEPERF_HH) +#define __SIMPLEPERF_HH + +#include +#include + +#include "cppcompat.hh" + +namespace xamarin::android::internal +{ + enum class RecordCmd + { + CMD_PAUSE_RECORDING = 1, + CMD_RESUME_RECORDING, + }; + + /** + * RecordOptions sets record options used by ProfileSession. The options are + * converted to a string list in toRecordArgs(), which is then passed to + * `simpleperf record` cmd. Run `simpleperf record -h` or + * `run_simpleperf_on_device.py record -h` for help messages. + * + * Example: + * RecordOptions options; + * options.set_duration (3).record_dwarf_call_graph ().set_output_filename ("perf.data"); + * ProfileSession session; + * session.start_recording (options); + */ + class RecordOptions final + { + public: + /** + * Set output filename. Default is perf-----.data. + * The file will be generated under simpleperf_data/. + */ + RecordOptions& set_output_filename (std::string const& filename) noexcept + { + output_filename = filename; + return *this; + } + + /** + * Set event to record. Default is cpu-cycles. See `simpleperf list` for all available events. + */ + RecordOptions& set_event (std::string const& wanted_event) noexcept + { + event = wanted_event; + return *this; + } + + /** + * Set how many samples to generate each second running. Default is 4000. + */ + RecordOptions& set_sample_frequency (size_t wanted_freq) noexcept + { + freq = wanted_freq; + return *this; + } + + /** + * Set record duration. The record stops after `durationInSecond` seconds. By default, + * record stops only when stopRecording() is called. + */ + RecordOptions& set_duration (double wanted_duration_in_seconds) noexcept + { + duration_in_seconds = wanted_duration_in_seconds; + return *this; + } + + /** + * Record some threads in the app process. By default, record all threads in the process. + */ + RecordOptions& set_sample_threads (std::vector const& wanted_threads) noexcept + { + threads = wanted_threads; + return *this; + } + + /** + * Record dwarf based call graph. It is needed to get Java callstacks. + */ + RecordOptions& record_dwarf_call_graph () noexcept + { + dwarf_callgraph = true; + fp_callgraph = false; + return *this; + } + + /** + * Record frame pointer based call graph. It is suitable to get C++ callstacks on 64bit devices. + */ + RecordOptions& record_frame_pointer_call_graph () noexcept + { + fp_callgraph = true; + dwarf_callgraph = false; + return *this; + } + + /** + * Trace context switch info to show where threads spend time off cpu. + */ + RecordOptions& trace_off_cpu () noexcept + { + trace_offcpu = true; + return *this; + } + + /** + * Translate record options into arguments for `simpleperf record` cmd. + */ + std::vector to_record_args () noexcept; + + private: + static std::string get_default_output_filename () noexcept; + + private: + std::string output_filename; + std::string event = "cpu-cycles"; + size_t freq = 4000; + double duration_in_seconds = 0.0; + std::vector threads; + bool dwarf_callgraph = false; + bool fp_callgraph = false; + bool trace_offcpu = false; + }; + + + enum class State + { + NOT_YET_STARTED, + STARTED, + PAUSED, + STOPPED, + }; + + /** + * ProfileSession uses `simpleperf record` cmd to generate a recording file. + * It allows users to start recording with some options, pause/resume recording + * to only profile interested code, and stop recording. + * + * Example: + * RecordOptions options; + * options.set_dwarf_call_graph (); + * ProfileSession session; + * session.start_recording (options); + * sleep(1); + * session.pause_recording (); + * sleep(1); + * session.resume_recording (); + * sleep(1); + * session.stop_recording (); + * + * It logs when error happens, does not abort the process. To read error messages of simpleperf record + * process, filter logcat with `simpleperf`. + */ + class ProfileSession final + { + private: + static constexpr uid_t AID_USER_OFFSET = 100000; + + public: + ProfileSession () noexcept; + + /** + * Start recording. + * @param options RecordOptions + */ + void start_recording (RecordOptions& options) noexcept + { + start_recording (options.to_record_args ()); + } + + /** + * Start recording. + * @param args arguments for `simpleperf record` cmd. + */ + void start_recording (std::vector const& record_args) noexcept; + + /** + * Pause recording. No samples are generated in paused state. + */ + void pause_recording () noexcept; + + /** + * Resume a paused session. + */ + void resume_recording () noexcept; + + /** + * Stop recording and generate a recording file under appDataDir/simpleperf_data/. + */ + void stop_recording () noexcept; + + private: + bool session_is_valid () const noexcept; + + std::string find_simpleperf_in_temp_dir () const noexcept; + std::string find_simpleperf () const noexcept; + bool create_simpleperf_data_dir () const noexcept; + bool create_simpleperf_process (std::string const& simpleperf_path, std::vector const& record_args) noexcept; + std::string read_reply () noexcept; + bool send_cmd (std::string const& cmd) noexcept; + + static std::string read_file (FILE* fp, std::string const& path) noexcept; + static bool is_executable_file (const std::string& path) noexcept; + static bool run_cmd (std::vector args, std::string* standard_output) noexcept; + static bool check_if_perf_enabled () noexcept; + + private: + // Clunky, but we want error in initialization to be non-fatal to the app + bool session_valid = false; + + const std::string app_data_dir_; + const std::string simpleperf_data_dir_; + std::mutex lock_; // Protect all members below. + State state_ = State::NOT_YET_STARTED; + pid_t simpleperf_pid_ = -1; + int control_fd_ = -1; + int reply_fd_ = -1; + bool trace_offcpu_ = false; + }; +} +#endif // ndef __SIMPLEPERF_HH From a7e9dbb7b702fcc2cb0716f1d68d7c3fd428d20d Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 3 Nov 2022 22:39:23 +0100 Subject: [PATCH 02/30] [WIP, PROF] MSBuild tasks to drive the profiling session --- .../Tasks/ProfileNativeCode.cs | 25 ++ .../Tasks/SetupNativeCodeProfiling.cs | 74 ++++ .../Utilities/AdbRunner.cs | 112 ++++++ .../Utilities/AndroidDeviceInfo.cs | 124 ++++++ .../Utilities/ProcessRunner.cs | 361 ++++++++++++++++++ .../Utilities/ProcessStandardStreamWrapper.cs | 82 ++++ .../Utilities/ToolRunner.cs | 86 +++++ .../Xamarin.Android.Application.targets | 41 ++ src/monodroid/jni/monodroid-glue.cc | 3 - src/monodroid/jni/simpleperf.cc | 7 +- 10 files changed, 909 insertions(+), 6 deletions(-) create mode 100644 src/Xamarin.Android.Build.Tasks/Tasks/ProfileNativeCode.cs create mode 100644 src/Xamarin.Android.Build.Tasks/Tasks/SetupNativeCodeProfiling.cs create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/AndroidDeviceInfo.cs create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/ProcessRunner.cs create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/ProcessStandardStreamWrapper.cs create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/ToolRunner.cs diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/ProfileNativeCode.cs b/src/Xamarin.Android.Build.Tasks/Tasks/ProfileNativeCode.cs new file mode 100644 index 00000000000..a53181cdca7 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Tasks/ProfileNativeCode.cs @@ -0,0 +1,25 @@ +using System; +using System.IO; + +using Microsoft.Android.Build.Tasks; +using Microsoft.Build.Framework; +using Xamarin.Android.Tools; + +namespace Xamarin.Android.Tasks +{ + public class ProfileNativeCode : AndroidAsyncTask + { + public override string TaskPrefix => "PNC"; + + public string DeviceSdkVersion { get; set; } + public bool DeviceIsEmulator { get; set; } + public string[] DeviceSupportedAbis { get; set; } + public string DevicePrimaryABI { get; set; } + public string SimplePerfDirectory { get; set; } + public string NdkPythonDirectory { get; set; } + + public async override System.Threading.Tasks.Task RunTaskAsync () + { + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/SetupNativeCodeProfiling.cs b/src/Xamarin.Android.Build.Tasks/Tasks/SetupNativeCodeProfiling.cs new file mode 100644 index 00000000000..bc7e7d4aa51 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Tasks/SetupNativeCodeProfiling.cs @@ -0,0 +1,74 @@ +using System; +using System.IO; + +using Microsoft.Android.Build.Tasks; +using Microsoft.Build.Framework; +using Xamarin.Android.Tools; + +namespace Xamarin.Android.Tasks +{ + public class SetupNativeCodeProfiling : AndroidAsyncTask + { + public override string TaskPrefix => "SNCP"; + + [Required] + public string AdbPath { get; set; } + + [Required] + public string AndroidNdkPath { get; set; } + + public string TargetDeviceName { get; set; } + + [Output] + public string DeviceSdkVersion { get; set; } + + [Output] + public bool DeviceIsEmulator { get; set; } + + [Output] + public string[] DeviceSupportedAbis { get; set; } + + [Output] + public string DevicePrimaryABI { get; set; } + + [Output] + public string SimplePerfDirectory { get; set; } + + [Output] + public string NdkPythonDirectory { get; set; } + + public async override System.Threading.Tasks.Task RunTaskAsync () + { + var adi = new AndroidDeviceInfo (Log, AdbPath, TargetDeviceName); + await adi.Detect (); + + DeviceSdkVersion = adi.DeviceSdkVersion ?? String.Empty; + DeviceIsEmulator = adi.DeviceIsEmulator; + DeviceSupportedAbis = adi.DeviceSupportedAbis ?? new string[] {}; + DevicePrimaryABI = adi.DevicePrimaryABI ?? String.Empty; + + string simplePerfPath = Path.Combine (AndroidNdkPath, "simpleperf"); + if (Directory.Exists (simplePerfPath)) { + SimplePerfDirectory = simplePerfPath; + } else { + Log.LogError ($"Simpleperf directory '{simplePerfPath}' not found"); + } + + string os; + if (OS.IsWindows) { + os = "windows"; + } else if (OS.IsMac) { + os = "darwin"; + } else { + os = "linux"; + } + + string ndkPythonPath = Path.Combine (AndroidNdkPath, "toolchains", "llvm", "prebuilt", $"{os}-x86_64", "python3"); + if (Directory.Exists (ndkPythonPath)) { + NdkPythonDirectory = ndkPythonPath; + } else { + Log.LogWarning ($"NDK Python 3 directory '{ndkPythonPath}' does not exist"); + } + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs new file mode 100644 index 00000000000..cf5116e0b9c --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs @@ -0,0 +1,112 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading.Tasks; + +using Microsoft.Build.Utilities; + +namespace Xamarin.Android.Tasks +{ + class AdbRunner : ToolRunner + { + class AdbOutputSink : ToolOutputSink + { + public Action? LineCallback { get; set; } + + public AdbOutputSink (TaskLoggingHelper logger) + : base (logger) + {} + + public override void WriteLine (string? value) + { + base.WriteLine (value); + LineCallback?.Invoke (value ?? String.Empty); + } + } + + string[]? initialParams; + + public AdbRunner (TaskLoggingHelper logger, string adbPath, string? deviceSerial = null) + : base (logger, adbPath) + { + if (!String.IsNullOrEmpty (deviceSerial)) { + initialParams = new string[] { "-s", deviceSerial }; + } + } + + public async Task<(bool success, string output)> GetPropertyValue (string propertyName) + { + var runner = CreateAdbRunner (); + return await GetPropertyValue (runner, propertyName); + } + + async Task<(bool success, string output)> GetPropertyValue (ProcessRunner runner, string propertyName) + { + runner.ClearArguments (); + runner.ClearOutputSinks (); + runner.AddArgument ("shell"); + runner.AddArgument ("getprop"); + runner.AddArgument (propertyName); + + return await CaptureAdbOutput (runner); + } + + async Task<(bool success, string output)> CaptureAdbOutput (ProcessRunner runner, bool firstLineOnly = false) + { + string? outputLine = null; + List? lines = null; + + using (var outputSink = (AdbOutputSink)SetupOutputSink (runner, ignoreStderr: true)) { + outputSink.LineCallback = (string line) => { + if (firstLineOnly) { + if (outputLine != null) { + return; + } + outputLine = line.Trim (); + return; + } + + if (lines == null) { + lines = new List (); + } + lines.Add (line.Trim ()); + }; + + if (!await RunAdb (runner, setupOutputSink: false)) { + return (false, String.Empty); + } + } + + if (firstLineOnly) { + return (true, outputLine ?? String.Empty); + } + + return (true, lines != null ? String.Join (Environment.NewLine, lines) : String.Empty); + } + + async Task RunAdb (ProcessRunner runner, bool setupOutputSink = true, bool ignoreStderr = true) + { + return await RunTool ( + () => { + TextWriter? sink = null; + if (setupOutputSink) { + sink = SetupOutputSink (runner, ignoreStderr: ignoreStderr); + } + + try { + return runner.Run (); + } finally { + sink?.Dispose (); + } + } + ); + } + + ProcessRunner CreateAdbRunner () => CreateProcessRunner (initialParams); + + protected override TextWriter CreateLogSink (TaskLoggingHelper logger) + { + return new AdbOutputSink (logger); + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AndroidDeviceInfo.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AndroidDeviceInfo.cs new file mode 100644 index 00000000000..3feb6c8417d --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AndroidDeviceInfo.cs @@ -0,0 +1,124 @@ +using System; +using System.Threading.Tasks; + +using Microsoft.Build.Utilities; + +using TPLTask = System.Threading.Tasks.Task; + +namespace Xamarin.Android.Tasks +{ + class AndroidDeviceInfo + { + string? targetDevice; + TaskLoggingHelper log; + string adbPath; + + public string? DeviceSdkVersion { get; private set; } + public bool DeviceIsEmulator { get; private set; } + public string[]? DeviceSupportedAbis { get; private set; } + public string? DevicePrimaryABI { get; private set; } + + public AndroidDeviceInfo (TaskLoggingHelper logger, string adbPath, string? targetDevice = null) + { + this.targetDevice = targetDevice; + this.log = logger; + this.adbPath = adbPath; + } + + public async TPLTask Detect () + { + var adb = new AdbRunner (log, adbPath, targetDevice); + + DeviceSdkVersion = await GetProperty (adb, "ro.build.version.sdk", "Android SDK version"); + DevicePrimaryABI = await GetProperty (adb, "ro.product.cpu.abi", "primary ABI"); + + string abis = await GetProperty (adb, "ro.product.cpu.abilist", "ABI list"); + DeviceSupportedAbis = abis?.Split (','); + + string? fingerprint = await GetProperty (adb, "ro.build.fingerprint", "fingerprint"); + if (CheckProperty (fingerprint, (string v) => v.StartsWith ("generic", StringComparison.Ordinal))) { + DeviceIsEmulator = true; + return; + } + + string? model = await GetProperty (adb, "ro.product.model", "product model"); + if (!String.IsNullOrEmpty (model)) { + if (Contains (model, "google_sdk") || + Contains (model, "droid4x", StringComparison.OrdinalIgnoreCase) || + Contains (model, "Emulator") || + Contains (model, "Android SDK built for x86", StringComparison.OrdinalIgnoreCase) + ) { + DeviceIsEmulator = true; + return; + } + } + + string? manufacturer = await GetProperty (adb, "ro.product.manufacturer", "product manufacturer"); + if (CheckProperty (manufacturer, (string v) => Contains (v, "Genymotion", StringComparison.OrdinalIgnoreCase))) { + DeviceIsEmulator = true; + return; + } + + string? hardware = await GetProperty (adb, "ro.hardware", "hardware model"); + if (!String.IsNullOrEmpty (hardware)) { + if (Contains (hardware, "goldfish") || + Contains (hardware, "ranchu") || + Contains (hardware, "vbox86") + ) { + DeviceIsEmulator = true; + return; + } + } + + string? product = await GetProperty (adb, "ro.product.name", "product name"); + if (!String.IsNullOrEmpty (product)) { + if (Contains (product, "sdk_google") || + Contains (product, "google_sdk") || + Contains (product, "sdk") || + Contains (product, "sdk_x86") || + Contains (product, "sdk_gphone64_arm64") || + Contains (product, "vbox86p") || + Contains (product, "emulator") || + Contains (product, "simulator") + ) { + DeviceIsEmulator = true; + return; + } + } + + bool Contains (string s, string sub, StringComparison comparison = StringComparison.Ordinal) + { +#if NETCOREAPP + return s.Contains (sub, comparison); +#else + return s.IndexOf (sub, comparison) >= 0; +#endif + } + + bool CheckProperty (string? value, Func checker) + { + if (String.IsNullOrEmpty (value)) { + return false; + } + + return checker (value); + } + } + + async Task GetProperty (AdbRunner adb, string propertyName, string errorWhat) + { + (bool success, string propertyValue) = await adb.GetPropertyValue (propertyName); + if (!success) { + log.LogWarning ($"Failed to get {errorWhat} from device"); + return default; + } + + return propertyValue; + } + + bool IsEmulator (string? model) + { + return false; + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ProcessRunner.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ProcessRunner.cs new file mode 100644 index 00000000000..67499e27d9e --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ProcessRunner.cs @@ -0,0 +1,361 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Text; +using System.Threading; + +using Microsoft.Android.Build.Tasks; +using Microsoft.Build.Utilities; + +namespace Xamarin.Android.Tasks +{ + class ProcessRunner + { + public const string StdoutSeverityName = "stdout | "; + public const string StderrSeverityName = "stderr | "; + + public enum ErrorReasonCode + { + NotExecutedYet, + NoError, + CommandNotFound, + ExecutionTimedOut, + ExitCodeNotZero, + }; + + static readonly TimeSpan DefaultProcessTimeout = TimeSpan.FromMinutes (5); + static readonly TimeSpan DefaultOutputTimeout = TimeSpan.FromSeconds (10); + + sealed class WriterGuard + { + public readonly object WriteLock = new object (); + public readonly TextWriter Writer; + + public WriterGuard (TextWriter writer) + { + Writer = writer; + } + } + + string command; + List? arguments; + List? stderrSinks; + List? stdoutSinks; + Dictionary? guardCache; + bool defaultStdoutEchoWrapperAdded; + ProcessStandardStreamWrapper? defaultStderrEchoWrapper; + TaskLoggingHelper log; + + public string Command => command; + + public string Arguments { + get { + if (arguments == null) + return String.Empty; + return String.Join (" ", arguments); + } + } + + public string FullCommandLine { + get { + string args = Arguments; + if (String.IsNullOrEmpty (args)) + return command; + return $"{command} {args}"; + } + } + + public Dictionary Environment { get; } = new Dictionary (StringComparer.Ordinal); + public int ExitCode { get; private set; } = -1; + public ErrorReasonCode ErrorReason { get; private set; } = ErrorReasonCode.NotExecutedYet; + public bool EchoCmdAndArguments { get; set; } = true; + public bool EchoStandardOutput { get; set; } + public ProcessStandardStreamWrapper.LogLevel EchoStandardOutputLevel { get; set; } = ProcessStandardStreamWrapper.LogLevel.Message; + public bool EchoStandardError { get; set; } + public ProcessStandardStreamWrapper.LogLevel EchoStandardErrorLevel { get; set; } = ProcessStandardStreamWrapper.LogLevel.Error; + public ProcessStandardStreamWrapper? StandardOutputEchoWrapper { get; set; } + public ProcessStandardStreamWrapper? StandardErrorEchoWrapper { get; set; } + public Encoding StandardOutputEncoding { get; set; } = Encoding.Default; + public Encoding StandardErrorEncoding { get; set; } = Encoding.Default; + public TimeSpan StandardOutputTimeout { get; set; } = DefaultOutputTimeout; + public TimeSpan StandardErrorTimeout { get; set; } = DefaultOutputTimeout; + public TimeSpan ProcessTimeout { get; set; } = DefaultProcessTimeout; + public string? WorkingDirectory { get; set; } + public Action? StartInfoCallback { get; set; } + + public ProcessRunner (TaskLoggingHelper logger, string command, params string?[] arguments) + : this (logger, command, false, arguments) + {} + + public ProcessRunner (TaskLoggingHelper logger, string command, bool ignoreEmptyArguments, params string?[] arguments) + { + if (String.IsNullOrEmpty (command)) { + throw new ArgumentException ("must not be null or empty", nameof (command)); + } + + log = logger; + this.command = command; + AddArgumentsInternal (ignoreEmptyArguments, arguments); + } + + public ProcessRunner ClearArguments () + { + arguments?.Clear (); + return this; + } + + public ProcessRunner ClearOutputSinks () + { + stderrSinks?.Clear (); + stdoutSinks?.Clear (); + return this; + } + + public ProcessRunner AddArguments (params string?[] arguments) + { + return AddArguments (true, arguments); + } + + public ProcessRunner AddArguments (bool ignoreEmptyArguments, params string?[] arguments) + { + AddArgumentsInternal (ignoreEmptyArguments, arguments); + return this; + } + + void AddArgumentsInternal (bool ignoreEmptyArguments, params string?[] arguments) + { + if (arguments == null) { + return; + } + + for (int i = 0; i < arguments.Length; i++) { + string? argument = arguments [i]?.Trim (); + if (String.IsNullOrEmpty (argument)) { + if (ignoreEmptyArguments) { + continue; + } + throw new InvalidOperationException ($"Argument {i} is null or empty"); + } + + AddQuotedArgument (argument!); + } + } + + public ProcessRunner AddArgument (string argument) + { + if (String.IsNullOrEmpty (argument)) { + throw new ArgumentException ("must not be null or empty", nameof (argument)); + } + + AddToList (argument, ref arguments); + return this; + } + + public ProcessRunner AddQuotedArgument (string argument) + { + if (String.IsNullOrEmpty (argument)) { + throw new ArgumentException ("must not be null or empty", nameof (argument)); + } + + return AddArgument (QuoteArgument (argument)); + } + + public static string QuoteArgument (string argument) + { + if (String.IsNullOrEmpty (argument)) { + return String.Empty; + } + + if (argument.IndexOf ('"') >= 0) { + argument = argument.Replace ("\"", "\\\""); + } + + return $"\"{argument}\""; + } + + public ProcessRunner AddStandardErrorSink (TextWriter writer) + { + if (writer == null) { + throw new ArgumentNullException (nameof (writer)); + } + + AddToList (GetGuard (writer), ref stderrSinks); + return this; + } + + public ProcessRunner AddStandardOutputSink (TextWriter writer) + { + if (writer == null) { + throw new ArgumentNullException (nameof (writer)); + } + + AddToList (GetGuard (writer), ref stdoutSinks); + return this; + } + + WriterGuard GetGuard (TextWriter writer) + { + if (guardCache == null) + guardCache = new Dictionary (); + + if (guardCache.TryGetValue (writer, out WriterGuard? ret) && ret != null) + return ret; + + ret = new WriterGuard (writer); + guardCache.Add (writer, ret); + return ret; + } + + public bool Run () + { + if (EchoStandardOutput) { + if (StandardOutputEchoWrapper != null) { + AddStandardOutputSink (StandardOutputEchoWrapper); + } else if (!defaultStdoutEchoWrapperAdded) { + AddStandardOutputSink (new ProcessStandardStreamWrapper (log) { LoggingLevel = EchoStandardOutputLevel, LogPrefix = StdoutSeverityName }); + defaultStdoutEchoWrapperAdded = true; + } + } + + if (EchoStandardError) { + if (StandardErrorEchoWrapper != null) { + AddStandardErrorSink (StandardErrorEchoWrapper); + } else if (defaultStderrEchoWrapper == null) { + defaultStderrEchoWrapper = new ProcessStandardStreamWrapper (log) { LoggingLevel = EchoStandardErrorLevel, LogPrefix = StderrSeverityName }; + AddStandardErrorSink (defaultStderrEchoWrapper); + } + } + + ManualResetEventSlim? stdout_done = null; + ManualResetEventSlim? stderr_done = null; + + if (stderrSinks != null && stderrSinks.Count > 0) { + stderr_done = new ManualResetEventSlim (false); + } + + if (stdoutSinks != null && stdoutSinks.Count > 0) { + stdout_done = new ManualResetEventSlim (false); + } + + var psi = new ProcessStartInfo (command) { + UseShellExecute = false, + CreateNoWindow = true, + WindowStyle = ProcessWindowStyle.Hidden, + RedirectStandardError = stderr_done != null, + RedirectStandardOutput = stdout_done != null, + }; + + if (arguments != null) { + psi.Arguments = String.Join (" ", arguments); + } + + if (!String.IsNullOrEmpty (WorkingDirectory)) { + psi.WorkingDirectory = WorkingDirectory; + } + + if (psi.RedirectStandardError) { + StandardErrorEncoding = StandardErrorEncoding; + } + + if (psi.RedirectStandardOutput) { + StandardOutputEncoding = StandardOutputEncoding; + } + + if (StartInfoCallback != null) { + StartInfoCallback (psi); + } + + var process = new Process { + StartInfo = psi + }; + + if (EchoCmdAndArguments) { + log.LogDebugMessage ($"Running: {FullCommandLine}"); + } + + try { + process.Start (); + } catch (System.ComponentModel.Win32Exception ex) { + log.LogError ($"Process failed to start: {ex.Message}"); + log.LogDebugMessage (ex.ToString ()); + + ErrorReason = ErrorReasonCode.CommandNotFound; + return false; + } + + if (psi.RedirectStandardError) { + process.ErrorDataReceived += (object sender, DataReceivedEventArgs e) => { + if (e.Data != null) { + WriteOutput (e.Data, stderrSinks!); + } else { + stderr_done!.Set (); + } + }; + process.BeginErrorReadLine (); + } + + if (psi.RedirectStandardOutput) { + process.OutputDataReceived += (object sender, DataReceivedEventArgs e) => { + if (e.Data != null) { + WriteOutput (e.Data, stdoutSinks!); + } else { + stdout_done!.Set (); + } + }; + process.BeginOutputReadLine (); + } + + bool exited = process.WaitForExit ((int)ProcessTimeout.TotalMilliseconds); + if (!exited) { + log.LogError ($"Process '{FullCommandLine}' timed out after {ProcessTimeout}"); + ErrorReason = ErrorReasonCode.ExecutionTimedOut; + process.Kill (); + } + + // See: https://docs.microsoft.com/en-us/dotnet/api/system.diagnostics.process.waitforexit?view=netframework-4.7.2#System_Diagnostics_Process_WaitForExit) + if (psi.RedirectStandardError || psi.RedirectStandardOutput) { + process.WaitForExit (); + } + + if (stderr_done != null) { + stderr_done.Wait (StandardErrorTimeout); + } + + if (stdout_done != null) { + stdout_done.Wait (StandardOutputTimeout); + } + + ExitCode = process.ExitCode; + if (ExitCode != 0 && ErrorReason == ErrorReasonCode.NotExecutedYet) { + ErrorReason = ErrorReasonCode.ExitCodeNotZero; + return false; + } + + if (exited) { + ErrorReason = ErrorReasonCode.NoError; + } + + return exited; + } + + void WriteOutput (string data, List sinks) + { + foreach (WriterGuard wg in sinks) { + if (wg == null || wg.Writer == null) + continue; + + lock (wg.WriteLock) { + wg.Writer.WriteLine (data); + } + } + } + + void AddToList (T item, ref List? list) + { + if (list == null) + list = new List (); + list.Add (item); + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ProcessStandardStreamWrapper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ProcessStandardStreamWrapper.cs new file mode 100644 index 00000000000..613a7ab2ecc --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ProcessStandardStreamWrapper.cs @@ -0,0 +1,82 @@ +using System; +using System.IO; +using System.Text; + +using Microsoft.Android.Build.Tasks; +using Microsoft.Build.Utilities; + +namespace Xamarin.Android.Tasks +{ + class ProcessStandardStreamWrapper : TextWriter + { + public enum LogLevel + { + Error, + Warning, + Info, + Message, + Debug, + } + + TaskLoggingHelper log; + + public LogLevel LoggingLevel { get; set; } = LogLevel.Debug; + public string? LogPrefix { get; set; } + + public override Encoding Encoding => Encoding.Default; + + public ProcessStandardStreamWrapper (TaskLoggingHelper logger) + { + log = logger; + } + + public override void WriteLine (string? value) + { + DoWrite (value); + } + + protected virtual string? PreprocessMessage (string? message, out bool ignoreLine) + { + ignoreLine = false; + return message; + } + + void DoWrite (string? message) + { + bool ignoreLine; + + message = PreprocessMessage (message, out ignoreLine) ?? String.Empty; + if (ignoreLine) { + return; + } + + if (!String.IsNullOrEmpty (LogPrefix)) { + message = $"{LogPrefix}{message}"; + } + + switch (LoggingLevel) { + case LogLevel.Error: + log.LogError (message); + break; + + case LogLevel.Warning: + log.LogWarning (message); + break; + + case LogLevel.Info: + case LogLevel.Message: + log.LogMessage (message); + break; + + case LogLevel.Debug: + log.LogDebugMessage (message); + break; + + default: + log.LogWarning ($"Unsupported log level {LoggingLevel}"); + log.LogMessage (message); + break; + } + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ToolRunner.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ToolRunner.cs new file mode 100644 index 00000000000..4fe86dbe01d --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ToolRunner.cs @@ -0,0 +1,86 @@ +using System; +using System.IO; +using System.Text; +using System.Threading.Tasks; + +using Microsoft.Build.Utilities; + +using TPLTask = System.Threading.Tasks.Task; + +namespace Xamarin.Android.Tasks +{ + abstract class ToolRunner + { + protected abstract class ToolOutputSink : TextWriter + { + TaskLoggingHelper log; + + public override Encoding Encoding => Encoding.Default; + + protected ToolOutputSink (TaskLoggingHelper logger) + { + log = logger; + } + + public override void WriteLine (string? value) + { + log.LogMessage (value ?? String.Empty); + } + } + + static readonly TimeSpan DefaultProcessTimeout = TimeSpan.FromMinutes (15); + + protected TaskLoggingHelper Logger { get; } + + public string ToolPath { get; } + public bool EchoCmdAndArguments { get; set; } = true; + public bool EchoStandardError { get; set; } = true; + public bool EchoStandardOutput { get; set; } + public virtual TimeSpan ProcessTimeout { get; set; } = DefaultProcessTimeout; + + protected ToolRunner (TaskLoggingHelper logger, string toolPath) + { + if (String.IsNullOrEmpty (toolPath)) { + throw new ArgumentException ("must not be null or empty", nameof (toolPath)); + } + + Logger = logger; + ToolPath = toolPath; + } + + protected virtual ProcessRunner CreateProcessRunner (params string[] initialParams) + { + var runner = new ProcessRunner (Logger, ToolPath) { + ProcessTimeout = ProcessTimeout, + EchoCmdAndArguments = EchoCmdAndArguments, + EchoStandardError = EchoStandardError, + EchoStandardOutput = EchoStandardOutput, + }; + + runner.AddArguments (initialParams); + return runner; + } + + protected virtual async Task RunTool (Func runner) + { + return await TPLTask.Run (runner); + } + + protected TextWriter SetupOutputSink (ProcessRunner runner, bool ignoreStderr = false) + { + TextWriter ret = CreateLogSink (Logger); + + if (!ignoreStderr) { + runner.AddStandardErrorSink (ret); + } + runner.AddStandardOutputSink (ret); + + return ret; + } + + protected virtual TextWriter CreateLogSink (TaskLoggingHelper logger) + { + throw new NotSupportedException ("Child class must implement this method if it uses output sinks"); + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Application.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Application.targets index 06735a9e2b7..4db3f1edb8f 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Application.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Application.targets @@ -12,6 +12,8 @@ Copyright (C) 2019 Microsoft Corporation. All rights reserved. --> + + + + + <_AndroidEnableNativeCodeProfiling>False + + _ResolveMonoAndroidSdks; + _SetupNativeCodeProfiling; + Build; + Install; + + + + + + + + + + + + + + + + + + + + + diff --git a/src/monodroid/jni/monodroid-glue.cc b/src/monodroid/jni/monodroid-glue.cc index 571f18e9e13..5a8a3500951 100644 --- a/src/monodroid/jni/monodroid-glue.cc +++ b/src/monodroid/jni/monodroid-glue.cc @@ -2169,9 +2169,6 @@ MonodroidRuntime::Java_mono_android_Runtime_initInternal (JNIEnv *env, jclass kl jobject loader, jobjectArray assembliesJava, jint apiLevel, jboolean isEmulator, jboolean haveSplitApks) { - log_warn (LOG_DEFAULT, "Testing symbols"); - abort (); - char *mono_log_mask_raw = nullptr; char *mono_log_level_raw = nullptr; diff --git a/src/monodroid/jni/simpleperf.cc b/src/monodroid/jni/simpleperf.cc index 8704ebd0ab8..b840dcacd3c 100644 --- a/src/monodroid/jni/simpleperf.cc +++ b/src/monodroid/jni/simpleperf.cc @@ -24,6 +24,7 @@ #include #include "android-system.hh" +#include "globals.hh" #include "logger.hh" #include "simpleperf.hh" #include "strings.hh" @@ -278,14 +279,14 @@ ProfileSession::check_if_perf_enabled () noexcept { dynamic_local_string prop; - if (AndroidSystem::monodroid_get_system_property ("persist.simpleperf.profile_app_uid", prop) <= 0) { + if (androidSystem.monodroid_get_system_property ("persist.simpleperf.profile_app_uid", prop) <= 0) { return false; } if (prop.get () == std::to_string (getuid ())) { prop.clear (); - AndroidSystem::monodroid_get_system_property ("persist.simpleperf.profile_app_expiration_time", prop); + androidSystem.monodroid_get_system_property ("persist.simpleperf.profile_app_expiration_time", prop); if (!prop.empty ()) { errno = 0; long expiration_time = strtol (prop.get (), nullptr, 10); @@ -295,7 +296,7 @@ ProfileSession::check_if_perf_enabled () noexcept } } - if (AndroidSystem::monodroid_get_system_property ("security.perf_harden", prop) <= 0 || prop.empty ()) { + if (androidSystem.monodroid_get_system_property ("security.perf_harden", prop) <= 0 || prop.empty ()) { return true; } From f155b087fee8f8988ee428b0a9e7d6d5f3a51613 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Fri, 4 Nov 2022 23:12:05 +0100 Subject: [PATCH 03/30] [WIP] Add wrap.sh, start a python runner, support newer Android versions --- ...rosoft.Android.Sdk.NativeProfiling.targets | 47 ++++++++++ .../Tasks/BuildApk.cs | 31 ++++++- .../Tasks/GenerateJavaStubs.cs | 18 +++- .../Tasks/ProfileNativeCode.cs | 86 +++++++++++++++++++ .../Utilities/ManifestDocument.cs | 11 +++ .../Utilities/PythonRunner.cs | 35 ++++++++ .../Xamarin.Android.Application.targets | 41 --------- .../Xamarin.Android.Common.targets | 13 ++- 8 files changed, 234 insertions(+), 48 deletions(-) create mode 100644 src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeProfiling.targets create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/PythonRunner.cs diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeProfiling.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeProfiling.targets new file mode 100644 index 00000000000..b254ee3b3ab --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeProfiling.targets @@ -0,0 +1,47 @@ + + + + + + + <_AndroidEnableNativeCodeProfiling>False + + _ResolveMonoAndroidSdks; + _SetupNativeCodeProfiling; + Build; + Install; + + + + + + <_AndroidEnableNativeCodeProfiling>True + + + apk + + + + + + + + + + + + + + + + diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs index 28cca622f9b..e6b3d1c5b32 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs @@ -95,6 +95,10 @@ public class BuildApk : AndroidTask public bool UseAssemblyStore { get; set; } + public bool NativeCodeProfilingEnabled { get; set; } + + public string DeviceSdkVersion { get; set; } + [Required] public string ProjectFullPath { get; set; } @@ -191,6 +195,7 @@ void ExecuteWithAbi (string [] supportedAbis, string apkInputPath, string apkOut } AddRuntimeLibraries (apk, supportedAbis); + AddProfilingScripts (apk, supportedAbis); apk.Flush(); AddNativeLibraries (files, supportedAbis); AddAdditionalNativeLibraries (files, supportedAbis); @@ -616,9 +621,11 @@ CompressionMethod GetCompressionMethod (string fileName) return CompressionMethod.Default; } + string GetArchiveAbiLibPath (string abi, string fileName) => $"lib/{abi}/{fileName}"; + void AddNativeLibraryToArchive (ZipArchiveEx apk, string abi, string filesystemPath, string inArchiveFileName) { - string archivePath = $"lib/{abi}/{inArchiveFileName}"; + string archivePath = GetArchiveAbiLibPath (abi, inArchiveFileName); existingEntries.Remove (archivePath); CompressionMethod compressionMethod = GetCompressionMethod (archivePath); if (apk.SkipExistingFile (filesystemPath, archivePath, compressionMethod)) { @@ -629,6 +636,26 @@ void AddNativeLibraryToArchive (ZipArchiveEx apk, string abi, string filesystemP apk.Archive.AddEntry (archivePath, File.OpenRead (filesystemPath), compressionMethod); } + void AddProfilingScripts (ZipArchiveEx apk, string [] supportedAbis) + { + if (!NativeCodeProfilingEnabled || !Int32.TryParse (DeviceSdkVersion, out int sdkVersion) || sdkVersion < 26) { + // wrap.sh is available on Android O (API 26) or newer + return; + } + + string wrapScript = "#!/system/bin/sh\n$@\n"; + byte[] wrapScriptBytes = new UTF8Encoding (false).GetBytes (wrapScript); + EntryPermissions wrapScriptPermissions = + EntryPermissions.WorldRead | EntryPermissions.WorldExecute | + EntryPermissions.GroupRead | EntryPermissions.GroupExecute | + EntryPermissions.OwnerRead | EntryPermissions.OwnerWrite | EntryPermissions.OwnerExecute; + + foreach (var abi in supportedAbis) { + string path = GetArchiveAbiLibPath (abi, "wrap.sh"); + apk.Archive.AddEntry (wrapScriptBytes, path, wrapScriptPermissions, CompressionMethod.Default); + } + } + void AddRuntimeLibraries (ZipArchiveEx apk, string [] supportedAbis) { foreach (var abi in supportedAbis) { @@ -802,7 +829,7 @@ private void AddAdditionalNativeLibraries (ArchiveFileList files, string [] supp void AddNativeLibrary (ArchiveFileList files, string path, string abi, string archiveFileName) { string fileName = string.IsNullOrEmpty (archiveFileName) ? Path.GetFileName (path) : archiveFileName; - var item = (filePath: path, archivePath: $"lib/{abi}/{fileName}"); + var item = (filePath: path, archivePath: GetArchiveAbiLibPath (abi, fileName)); if (files.Any (x => x.archivePath == item.archivePath)) { Log.LogCodedWarning ("XA4301", path, 0, Properties.Resources.XA4301, item.archivePath); return; diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs index 4053a0f9b1e..a94cfb313f6 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs @@ -92,6 +92,9 @@ public class GenerateJavaStubs : AndroidTask public ITaskItem[] Environments { get; set; } + public bool NativeCodeProfilingEnabled { get; set; } + public string DeviceSdkVersion { get; set; } + [Output] public string [] GeneratedBinaryTypeMaps { get; set; } @@ -355,11 +358,22 @@ void Run (DirectoryAssemblyResolver res, bool useMarshalMethods) } manifest.Assemblies.AddRange (userAssemblies.Values); - if (!String.IsNullOrWhiteSpace (CheckedBuild)) { + bool checkedBuild = !String.IsNullOrWhiteSpace (CheckedBuild); + if (NativeCodeProfilingEnabled || checkedBuild) { // We don't validate CheckedBuild value here, this will be done in BuildApk. We just know that if it's // on then we need android:debuggable=true and android:extractNativeLibs=true + // + // For profiling we only need android:debuggable=true + // manifest.ForceDebuggable = true; - manifest.ForceExtractNativeLibs = true; + if (checkedBuild) { + manifest.ForceExtractNativeLibs = true; + } + + // is supported on Android Q (API 29) or newer + if (NativeCodeProfilingEnabled && Int32.TryParse (DeviceSdkVersion, out int sdkVersion) && sdkVersion >= 29) { + manifest.Profileable = true; + } } var additionalProviders = manifest.Merge (Log, cache, allJavaTypes, ApplicationJavaClass, EmbedAssemblies, BundledWearApplicationName, MergedManifestDocuments); diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/ProfileNativeCode.cs b/src/Xamarin.Android.Build.Tasks/Tasks/ProfileNativeCode.cs index a53181cdca7..eb3d9142f85 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/ProfileNativeCode.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/ProfileNativeCode.cs @@ -9,17 +9,103 @@ namespace Xamarin.Android.Tasks { public class ProfileNativeCode : AndroidAsyncTask { + const string PythonName = "python3"; + public override string TaskPrefix => "PNC"; + string[]? pathExt; + public string DeviceSdkVersion { get; set; } public bool DeviceIsEmulator { get; set; } public string[] DeviceSupportedAbis { get; set; } public string DevicePrimaryABI { get; set; } + + [Required] public string SimplePerfDirectory { get; set; } public string NdkPythonDirectory { get; set; } public async override System.Threading.Tasks.Task RunTaskAsync () { + string? pythonPath = null; + + if (!String.IsNullOrEmpty (NdkPythonDirectory)) { + pythonPath = Path.Combine (NdkPythonDirectory, "bin", PythonName); + if (!File.Exists (pythonPath)) { + pythonPath = null; + } + } + + if (String.IsNullOrEmpty (pythonPath)) { + Log.LogWarning ($"NDK {PythonName} not found, will attempt to find one in a system location"); + pythonPath = FindPython (); + if (String.IsNullOrEmpty (pythonPath)) { + Log.LogWarning ($"System {PythonName} not found, will attempt to use executable name without path"); + pythonPath = PythonName; + } + } + + string? appProfilerScript = Path.Combine (SimplePerfDirectory, "app_profiler.py"); + if (!File.Exists (appProfilerScript)) { + Log.LogError ($"Profiling script {appProfilerScript} not found"); + return; + } + + // TODO: prepare a directory with unstripped native libraries (for use with the profiler's -lib argument) + Console.WriteLine ($"python3 path: {pythonPath}"); + Console.WriteLine ($"profiler script path: {appProfilerScript}"); + + var python = new PythonRunner (Log, pythonPath); + + // TODO: params + bool success = await python.RunScript (appProfilerScript); + } + + string? FindPython () + { + // TODO: might be a good idea to try to look for `python` and check its version, if python3 isn't found + if (OS.IsWindows) { + string? envvar = Environment.GetEnvironmentVariable ("PATHEXT"); + if (String.IsNullOrEmpty (envvar)) { + pathExt = new string[] { ".exe", ".bat", ".cmd" }; + } else { + pathExt = envvar.Split (Path.PathSeparator); + } + } + + string? pathVar = Environment.GetEnvironmentVariable ("PATH")?.Trim (); + if (String.IsNullOrEmpty (pathVar)) { + return null; + } + + foreach (string dir in pathVar.Split (Path.PathSeparator)) { + string? exe = GetExecutablePath (dir, PythonName); + if (!String.IsNullOrEmpty (exe)) { + return exe; + } + } + + return null; + } + + string? GetExecutablePath (string dir, string baseExeName) + { + string exePath = Path.Combine (dir, baseExeName); + if (!OS.IsWindows) { + if (File.Exists (exePath)) { + return exePath; + } + + return null; + } + + foreach (string ext in pathExt) { + string exe = $"{exePath}{ext}"; + if (File.Exists (exe)) { + return exe; + } + } + + return null; } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ManifestDocument.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ManifestDocument.cs index 9dd676dfc92..1fac33b9c40 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ManifestDocument.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ManifestDocument.cs @@ -95,6 +95,11 @@ internal class ManifestDocument public bool ForceDebuggable { get; set; } public string VersionName { get; set; } + /// + /// Available on API 29 or newer + /// + public bool Profileable { get; set; } + string versionCode; /// @@ -419,6 +424,12 @@ public IList Merge (TaskLoggingHelper log, TypeDefinitionCache cache, Li debuggable.Value = "true"; } + if (Profileable) { + var profileable = new XElement ("profileable"); + profileable.Add (new XAttribute (androidNs + "shell", "true")); + app.Add (profileable); + } + if (Debug || NeedsInternet) AddInternetPermissionForDebugger (); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/PythonRunner.cs b/src/Xamarin.Android.Build.Tasks/Utilities/PythonRunner.cs new file mode 100644 index 00000000000..dc60b76a5c5 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/PythonRunner.cs @@ -0,0 +1,35 @@ +using System.Threading.Tasks; + +using Microsoft.Build.Utilities; + +namespace Xamarin.Android.Tasks +{ + class PythonRunner : ToolRunner + { + public PythonRunner (TaskLoggingHelper logger, string pythonPath) + : base (logger, pythonPath) + {} + + public async Task RunScript (string scriptPath, params string[] arguments) + { + ProcessRunner runner = CreateProcessRunner (); + + if (arguments != null && arguments.Length > 0) { + foreach (string arg in arguments) { + runner.AddArgument (arg); + } + } + + return await RunPython (runner); + } + + async Task RunPython (ProcessRunner runner) + { + return await RunTool ( + () => { + return runner.Run (); + } + ); + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Application.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Application.targets index 4db3f1edb8f..06735a9e2b7 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Application.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Application.targets @@ -12,8 +12,6 @@ Copyright (C) 2019 Microsoft Corporation. All rights reserved. --> - - - - - <_AndroidEnableNativeCodeProfiling>False - - _ResolveMonoAndroidSdks; - _SetupNativeCodeProfiling; - Build; - Install; - - - - - - - - - - - - - - - - - - - - - diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index dc0f72bf029..812ec5b5f79 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -366,6 +366,7 @@ Copyright (C) 2011-2012 Xamarin. All rights reserved. Imports ******************************************* --> + @@ -1526,7 +1527,9 @@ because xbuild doesn't support framework reference assemblies. LinkingEnabled="$(_LinkingEnabled)" HaveMultipleRIDs="$(_HaveMultipleRIDs)" IntermediateOutputDirectory="$(IntermediateOutputPath)" - Environments="@(AndroidEnvironment);@(LibraryEnvironments)"> + Environments="@(AndroidEnvironment);@(LibraryEnvironments)" + NativeCodeProfilingEnabled="$(_AndroidEnableNativeCodeProfiling)" + DeviceSdkVersion="$(_AndroidDeviceSdkVersion)"> @@ -2116,7 +2119,9 @@ because xbuild doesn't support framework reference assemblies. CheckedBuild="$(_AndroidCheckedBuild)" RuntimeConfigBinFilePath="$(_BinaryRuntimeConfigPath)" ExcludeFiles="@(AndroidPackagingOptionsExclude)" - UseAssemblyStore="$(AndroidUseAssemblyStore)"> + UseAssemblyStore="$(AndroidUseAssemblyStore)" + NativeCodeProfilingEnabled="$(_AndroidEnableNativeCodeProfiling)" + DeviceSdkVersion="$(_AndroidDeviceSdkVersion)"> + UseAssemblyStore="$(AndroidUseAssemblyStore)" + NativeCodeProfilingEnabled="$(_AndroidEnableNativeCodeProfiling)" + DeviceSdkVersion="$(_AndroidDeviceSdkVersion)"> Date: Wed, 9 Nov 2022 22:31:54 +0100 Subject: [PATCH 04/30] [WIP] Beginnings of debugging support --- .../Tasks/SetupNativeCodeProfiling.cs | 12 +- .../Utilities/AdbRunner.cs | 35 ++- .../Utilities/NativeDebugger.cs | 249 ++++++++++++++++++ .../Utilities/NdkHelper.cs | 45 ++++ 4 files changed, 324 insertions(+), 17 deletions(-) create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/NativeDebugger.cs create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/NdkHelper.cs diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/SetupNativeCodeProfiling.cs b/src/Xamarin.Android.Build.Tasks/Tasks/SetupNativeCodeProfiling.cs index bc7e7d4aa51..da01314b91d 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/SetupNativeCodeProfiling.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/SetupNativeCodeProfiling.cs @@ -3,7 +3,6 @@ using Microsoft.Android.Build.Tasks; using Microsoft.Build.Framework; -using Xamarin.Android.Tools; namespace Xamarin.Android.Tasks { @@ -54,16 +53,7 @@ public async override System.Threading.Tasks.Task RunTaskAsync () Log.LogError ($"Simpleperf directory '{simplePerfPath}' not found"); } - string os; - if (OS.IsWindows) { - os = "windows"; - } else if (OS.IsMac) { - os = "darwin"; - } else { - os = "linux"; - } - - string ndkPythonPath = Path.Combine (AndroidNdkPath, "toolchains", "llvm", "prebuilt", $"{os}-x86_64", "python3"); + string ndkPythonPath = Path.Combine (AndroidNdkPath, "toolchains", "llvm", "prebuilt", NdkHelper.ToolchainHostName, "python3"); if (Directory.Exists (ndkPythonPath)) { NdkPythonDirectory = ndkPythonPath; } else { diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs index cf5116e0b9c..a7f54ad8b3e 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs @@ -26,6 +26,8 @@ public override void WriteLine (string? value) string[]? initialParams; + public int ExitCode { get; private set; } + public AdbRunner (TaskLoggingHelper logger, string adbPath, string? deviceSerial = null) : base (logger, adbPath) { @@ -34,6 +36,26 @@ public AdbRunner (TaskLoggingHelper logger, string adbPath, string? deviceSerial } } + public async Task<(bool success, string output)> GetAppDataDirectory (string packageName) + { + return await Shell ("run-as", packageName, "/system/bin/sh", "-c", "pwd", "2>/dev/null"); + } + + public async Task<(bool success, string output)> Shell (string command, params string[] args) + { + var runner = CreateAdbRunner (); + + runner.AddArgument ("shell"); + runner.AddArgument (command); + if (args != null && args.Length > 0) { + foreach (string arg in args) { + runner.AddArgument (arg); + } + } + + return await CaptureAdbOutput (runner); + } + public async Task<(bool success, string output)> GetPropertyValue (string propertyName) { var runner = CreateAdbRunner (); @@ -44,11 +66,7 @@ public AdbRunner (TaskLoggingHelper logger, string adbPath, string? deviceSerial { runner.ClearArguments (); runner.ClearOutputSinks (); - runner.AddArgument ("shell"); - runner.AddArgument ("getprop"); - runner.AddArgument (propertyName); - - return await CaptureAdbOutput (runner); + return await Shell ("getprop", propertyName); } async Task<(bool success, string output)> CaptureAdbOutput (ProcessRunner runner, bool firstLineOnly = false) @@ -94,7 +112,12 @@ async Task RunAdb (ProcessRunner runner, bool setupOutputSink = true, bool } try { - return runner.Run (); + bool ret = runner.Run (); + ExitCode = runner.ExitCode; + return ret; + } catch { + ExitCode = -0xDEAD; + throw; } finally { sink?.Dispose (); } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeDebugger.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeDebugger.cs new file mode 100644 index 00000000000..c91afe444e9 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeDebugger.cs @@ -0,0 +1,249 @@ +using System; +using System.Collections.Generic; +using System.IO; + +using Microsoft.Build.Utilities; + +namespace Xamarin.Android.Tasks +{ + /// + /// Interface to lldb, the NDK native code debugger. + /// + class NativeDebugger + { + sealed class Context + { + public AdbRunner adb; + public int apiLevel; + public string abi; + public string arch; + public string appDataDir; + } + + // We want the shell/batch scripts first, since they set up Python environment for the debugger + static readonly string[] lldbNames = { + "lldb.sh", + "lldb.cmd", + "lldb", + "lldb.exe", + }; + + static readonly string[] abiProperties = { + // new properties + "ro.product.cpu.abilist", + + // old properties + "ro.product.cpu.abi", + "ro.product.cpu.abi2", + }; + + TaskLoggingHelper log; + string packageName; + string lldbPath; + string adbPath; + Dictionary hostLldbServerPaths; + string[] supportedAbis; + + public string? AdbDeviceTarget { get; set; } + + public NativeDebugger (TaskLoggingHelper logger, string adbPath, string ndkRootPath, string packageName, string[] supportedAbis) + { + this.log = logger; + this.packageName = packageName; + this.adbPath = adbPath; + this.supportedAbis = supportedAbis; + + FindTools (ndkRootPath, supportedAbis); + } + + /// + /// Detect PID of the running application and attach the debugger to it + /// + public void Attach () + { + Context context = Init (); + } + + /// + /// Launch the application under control of the debugger. If is provided, + /// it will be launched instead of the default launcher activity. + /// + public void Launch (string? activityName) + { + Context context = Init (); + } + + Context Init () + { + var context = new Context { + adb = new AdbRunner (log, adbPath) + }; + + (bool success, string output) = context.adb.GetPropertyValue ("ro.build.version.sdk").Result; + if (!success || String.IsNullOrEmpty (output) || !Int32.TryParse (output, out int apiLevel)) { + throw new InvalidOperationException ("Unable to determine connected device's API level"); + } + context.apiLevel = apiLevel; + + // Warn on old Pixel C firmware (b/29381985). Newer devices may have Yama + // enabled but still work with ndk-gdb (b/19277529). + (success, output) = context.adb.Shell ("cat", "/proc/sys/kernel/yama/ptrace_scope", "2>/dev/null").Result; + if (success && + YamaOK (output.Trim ()) && + PropertyHasValue (context.adb.GetPropertyValue ("ro.build.product").Result, "dragon") && + PropertyHasValue (context.adb.GetPropertyValue ("ro.product.name").Result, "ryu") + ) { + LogLine ("WARNING: The device uses Yama ptrace_scope to restrict debugging. ndk-gdb will"); + LogLine (" likely be unable to attach to a process. With root access, the restriction"); + LogLine (" can be lifted by writing 0 to /proc/sys/kernel/yama/ptrace_scope. Consider"); + LogLine (" upgrading your Pixel C to MXC89L or newer, where Yama is disabled."); + LogLine (); + } + + DetermineABI (context); + context.arch = context.abi switch { + "armeabi" => "arm", + "armeabi-v7a" => "arm", + "arm64-v8a" => "arm64", + _ => context.abi, + }; + + DetermineAppDataDirectory (context); + LogLine ($"Application data directory: {context.appDataDir}"); + + return context; + + bool YamaOK (string output) + { + return !String.IsNullOrEmpty (output) && String.Compare ("0", output, StringComparison.Ordinal) != 0; + } + + bool PropertyHasValue ((bool haveProperty, string value) result, string expected) + { + return + result.haveProperty && + !String.IsNullOrEmpty (result.value) && + String.Compare (result.value, expected, StringComparison.Ordinal) == 0; + } + } + + void DetermineAppDataDirectory (Context context) + { + (bool success, string output) = context.adb.GetAppDataDirectory (packageName).Result; + if (!success) { + throw new InvalidOperationException ($"Unable to determine data directory for package '{packageName}'"); + } + + context.appDataDir = output.Trim (); + + // Applications with minSdkVersion >= 24 will have their data directories + // created with rwx------ permissions, preventing adbd from forwarding to + // the gdbserver socket. To be safe, if we're on a device >= 24, always + // chmod the directory. + if (context.apiLevel >= 24) { + (success, output) = context.adb.Shell ("/system/bin/chmod", "a+x", context.appDataDir).Result; + if (!success) { + throw new InvalidOperationException ("Failed to make application data directory world executable"); + } + } + } + + void DetermineABI (Context context) + { + string[]? deviceABIs = null; + + foreach (string prop in abiProperties) { + (bool success, string value) = context.adb.GetPropertyValue (prop).Result; + if (!success) { + continue; + } + + deviceABIs = value.Split (','); + } + + if (deviceABIs == null || deviceABIs.Length == 0) { + throw new InvalidOperationException ("Unable to determine device ABI"); + } + + LogLine ($"Application ABIs: {String.Join ("", "", supportedAbis)}"); + LogLine ($"Device ABIs: {String.Join ("", "", deviceABIs)}"); + foreach (string deviceABI in deviceABIs) { + foreach (string appABI in supportedAbis) { + if (String.Compare (appABI, deviceABI, StringComparison.OrdinalIgnoreCase) == 0) { + context.abi = deviceABI; + return; + } + } + } + + throw new InvalidOperationException ($"Application cannot run on the selected device: no matching ABI found"); + } + + string GetLlvmVersion (string toolchainDir) + { + string path = Path.Combine (toolchainDir, "AndroidVersion.txt"); + if (!File.Exists (path)) { + throw new InvalidOperationException ($"LLVM version file not found at '{path}'"); + } + + string[] lines = File.ReadAllLines (path); + string? line = lines.Length >= 1 ? lines[0].Trim () : null; + if (String.IsNullOrEmpty (line)) { + throw new InvalidOperationException ($"Unable to read LLVM version from '{path}'"); + } + + return line; + } + + void FindTools (string ndkRootPath, string[] supportedAbis) + { + string toolchainDir = Path.Combine (ndkRootPath, NdkHelper.RelativeToolchainDir); + string toolchainBinDir = Path.Combine (toolchainDir, "bin"); + string? path = null; + + foreach (string lldb in lldbNames) { + path = Path.Combine (toolchainBinDir, lldb); + if (File.Exists (path)) { + break; + } + } + + if (String.IsNullOrEmpty (path)) { + throw new InvalidOperationException ($"Unable to locate lldb executable in '{toolchainBinDir}'"); + } + lldbPath = path; + + hostLldbServerPaths = new Dictionary (StringComparer.OrdinalIgnoreCase); + string llvmVersion = GetLlvmVersion (toolchainDir); + foreach (string abi in supportedAbis) { + string llvmAbi = NdkHelper.TranslateAbiToLLVM (abi); + path = Path.Combine (ndkRootPath, NdkHelper.RelativeToolchainDir, "lib64", "clang", llvmVersion, "lib", "linux", abi, "lldb-server"); + if (!File.Exists (path)) { + throw new InvalidOperationException ($"LLVM lldb server component for ABI '{abi}' not found at '{path}'"); + } + + hostLldbServerPaths.Add (abi, path); + } + + if (hostLldbServerPaths.Count == 0) { + throw new InvalidOperationException ("Unable to find any lldb-server executables, debugging not possible"); + } + } + + void LogLine (string? message = null, bool isError = false) + { + Log (message, isError); + Log (Environment.NewLine, isError); + } + + void Log (string? message = null, bool isError = false) + { + TextWriter writer = isError ? Console.Error : Console.Out; + message = message ?? String.Empty; + writer.Write (message); + if (!String.IsNullOrEmpty (message)) { + log.LogError (message); + } + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NdkHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NdkHelper.cs new file mode 100644 index 00000000000..ac514d05a3e --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NdkHelper.cs @@ -0,0 +1,45 @@ +using System; +using System.IO; + +using Xamarin.Android.Tools; + +namespace Xamarin.Android.Tasks +{ + static class NdkHelper + { + static readonly string toolchainHostName; + static readonly string relativeToolchainDir; + + public static string ToolchainHostName => toolchainHostName; + public static string RelativeToolchainDir => relativeToolchainDir; + + static NdkHelper () + { + string os; + if (OS.IsWindows) { + os = "windows"; + } else if (OS.IsMac) { + os = "darwin"; + } else { + os = "linux"; + } + + // We care only about the latest NDK versions, they have only x86_64 versions. We'll need to revisit the code once + // native macOS/arm64 toolchain is out. + toolchainHostName = $"{os}-x86_64"; + + relativeToolchainDir = Path.Combine ("toolchains", "llvm", "prebuilt", toolchainHostName); + } + + public static string TranslateAbiToLLVM (string xaAbi) + { + return xaAbi switch { + "armeabi-v7a" => "arm", + "arm64-v8a" => "aarch64", + "x86" => "i386", + "x86_64" => "x86_64", + _ => throw new InvalidOperationException ($"Unknown ABI '{xaAbi}'") + }; + } + } +} From 7760be1c3960146ae5f21f9d803604f02ba8a275 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 10 Nov 2022 22:46:26 +0100 Subject: [PATCH 05/30] [WIP] Prepare device and host for debugging --- ...d.Sdk.NativeProfilingAndDebugging.targets} | 45 +++- .../Tasks/DebugNativeCode.cs | 80 +++++++ .../Tasks/GenerateJavaStubs.cs | 6 +- .../Tasks/LinkApplicationSharedLibraries.cs | 4 +- .../Utilities/AdbRunner.cs | 52 ++++- .../Utilities/NativeDebugger.cs | 200 ++++++++++++++++-- .../Utilities/NdkHelper.cs | 19 +- .../Xamarin.Android.Common.targets | 11 +- 8 files changed, 381 insertions(+), 36 deletions(-) rename src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/{Microsoft.Android.Sdk.NativeProfiling.targets => Microsoft.Android.Sdk.NativeProfilingAndDebugging.targets} (56%) create mode 100644 src/Xamarin.Android.Build.Tasks/Tasks/DebugNativeCode.cs diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeProfiling.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeProfilingAndDebugging.targets similarity index 56% rename from src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeProfiling.targets rename to src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeProfilingAndDebugging.targets index b254ee3b3ab..71c41d8d788 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeProfiling.targets +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeProfilingAndDebugging.targets @@ -1,21 +1,34 @@ - + + + - + <_AndroidEnableNativeCodeProfiling>False + <_AndroidEnableNativeDebugging>False + _ResolveMonoAndroidSdks; _SetupNativeCodeProfiling; Build; Install; + + + _ResolveMonoAndroidSdks; + _SetupNativeCodeDebugging; + Build; + Install; + <_AndroidEnableNativeCodeProfiling>True + <_AndroidAotStripLibraries>False + <_AndroidStripNativeLibraries>False apk @@ -44,4 +57,32 @@ SimplePerfDirectory="$(_AndroidSimplePerfDirectory)" NdkPythonDirectory="$(_AndroidNdkPythonDirectory)" /> + + + + <_AndroidEnableNativeDebugging>True + <_AndroidAotStripLibraries>False + <_AndroidStripNativeLibraries>False + + + + + + + + + + diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/DebugNativeCode.cs b/src/Xamarin.Android.Build.Tasks/Tasks/DebugNativeCode.cs new file mode 100644 index 00000000000..b30083e1236 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Tasks/DebugNativeCode.cs @@ -0,0 +1,80 @@ +using System; +using System.Collections.Generic; + +using Microsoft.Android.Build.Tasks; +using Microsoft.Build.Framework; + +namespace Xamarin.Android.Tasks +{ + public class DebugNativeCode : AndroidAsyncTask + { + public override string TaskPrefix => "DNC"; + + [Required] + public string AndroidNdkPath { get; set; } + + [Required] + public string PackageName { get; set; } + + [Required] + public string[] SupportedAbis { get; set; } + + [Required] + public string AdbPath { get; set; } + + [Required] + public string MainActivityName { get; set; } + + [Required] + public string IntermediateOutputDir { get; set; } + + [Required] + public ITaskItem[] NativeLibraries { get; set; } + + public string ActivityName { get; set; } + public string TargetDeviceName { get; set; } + + public async override System.Threading.Tasks.Task RunTaskAsync () + { + var nativeLibs = new Dictionary> (StringComparer.OrdinalIgnoreCase); + foreach (ITaskItem item in NativeLibraries) { + string? abi = null; + string? rid = item.GetMetadata ("RuntimeIdentifier"); + + if (!String.IsNullOrEmpty (rid)) { + abi = NdkHelper.RIDToABI (rid); + } + + if (String.IsNullOrEmpty (abi)) { + abi = item.GetMetadata ("abi"); + } + + if (String.IsNullOrEmpty (abi)) { + Log.LogDebugMessage ($"Ignoring native library {item.ItemSpec} because it doesn't specify its ABI"); + continue; + } + + if (!nativeLibs.TryGetValue (abi, out List abiLibs)) { + abiLibs = new List (); + nativeLibs.Add (abi, abiLibs); + } + abiLibs.Add (item.ItemSpec); + } + + var debugger = new NativeDebugger ( + Log, + AdbPath, + AndroidNdkPath, + IntermediateOutputDir, + PackageName, + SupportedAbis + ) { + AdbDeviceTarget = TargetDeviceName, + NativeLibrariesPerABI = nativeLibs, + }; + + string activity = String.IsNullOrEmpty (ActivityName) ? MainActivityName : ActivityName; + debugger.Launch (activity); + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs index a94cfb313f6..a1a43449888 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs @@ -95,6 +95,8 @@ public class GenerateJavaStubs : AndroidTask public bool NativeCodeProfilingEnabled { get; set; } public string DeviceSdkVersion { get; set; } + public bool NativeDebuggingEnabled { get; set; } + [Output] public string [] GeneratedBinaryTypeMaps { get; set; } @@ -359,14 +361,14 @@ void Run (DirectoryAssemblyResolver res, bool useMarshalMethods) manifest.Assemblies.AddRange (userAssemblies.Values); bool checkedBuild = !String.IsNullOrWhiteSpace (CheckedBuild); - if (NativeCodeProfilingEnabled || checkedBuild) { + if (NativeCodeProfilingEnabled || NativeDebuggingEnabled || checkedBuild) { // We don't validate CheckedBuild value here, this will be done in BuildApk. We just know that if it's // on then we need android:debuggable=true and android:extractNativeLibs=true // // For profiling we only need android:debuggable=true // manifest.ForceDebuggable = true; - if (checkedBuild) { + if (checkedBuild || NativeDebuggingEnabled) { manifest.ForceExtractNativeLibs = true; } diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/LinkApplicationSharedLibraries.cs b/src/Xamarin.Android.Build.Tasks/Tasks/LinkApplicationSharedLibraries.cs index 0810ba6f0de..d52af1a73a2 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/LinkApplicationSharedLibraries.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/LinkApplicationSharedLibraries.cs @@ -37,7 +37,7 @@ sealed class InputFiles public ITaskItem[] ApplicationSharedLibraries { get; set; } [Required] - public bool DebugBuild { get; set; } + public bool KeepDebugInfo { get; set; } [Required] public string AndroidBinUtilsDirectory { get; set; } @@ -124,7 +124,7 @@ IEnumerable GetLinkerConfigs () "--warn-shared-textrel " + "--fatal-warnings"; - string stripSymbolsArg = DebugBuild ? String.Empty : " -s"; + string stripSymbolsArg = KeepDebugInfo ? String.Empty : " -s"; string ld = Path.Combine (AndroidBinUtilsDirectory, MonoAndroidHelper.GetExecutablePath (AndroidBinUtilsDirectory, "ld")); var targetLinkerArgs = new List (); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs index a7f54ad8b3e..e979b597fd5 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs @@ -36,21 +36,63 @@ public AdbRunner (TaskLoggingHelper logger, string adbPath, string? deviceSerial } } + public async Task Pull (string remotePath, string localPath) + { + var runner = CreateAdbRunner (); + runner.AddArgument ("pull"); + runner.AddArgument (remotePath); + runner.AddArgument (localPath); + + return await RunAdb (runner); + } + + public async Task Push (string localPath, string remotePath) + { + var runner = CreateAdbRunner (); + runner.AddArgument ("push"); + runner.AddArgument (localPath); + runner.AddArgument (remotePath); + + return await RunAdb (runner); + } + + public async Task<(bool success, string output)> RunAs (string packageName, string command, params string[] args) + { + var shellArgs = new List { + packageName, + command, + }; + + if (args != null && args.Length > 0) { + shellArgs.AddRange (args); + } + + return await Shell ("run-as", (IEnumerable)shellArgs); + } + public async Task<(bool success, string output)> GetAppDataDirectory (string packageName) { - return await Shell ("run-as", packageName, "/system/bin/sh", "-c", "pwd", "2>/dev/null"); + return await RunAs (packageName, "/system/bin/sh", "-c", "pwd", "2>/dev/null"); + } + + public async Task<(bool success, string output)> Shell (string command, List args) + { + return await Shell (command, (IEnumerable)args); } public async Task<(bool success, string output)> Shell (string command, params string[] args) + { + return await Shell (command, (IEnumerable)args); + } + + async Task<(bool success, string output)> Shell (string command, IEnumerable args) { var runner = CreateAdbRunner (); runner.AddArgument ("shell"); runner.AddArgument (command); - if (args != null && args.Length > 0) { - foreach (string arg in args) { - runner.AddArgument (arg); - } + foreach (string arg in args) { + runner.AddArgument (arg); } return await CaptureAdbOutput (runner); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeDebugger.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeDebugger.cs index c91afe444e9..9d1d1c28425 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/NativeDebugger.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeDebugger.cs @@ -3,6 +3,7 @@ using System.IO; using Microsoft.Build.Utilities; +using Xamarin.Android.Tools; namespace Xamarin.Android.Tasks { @@ -17,7 +18,12 @@ sealed class Context public int apiLevel; public string abi; public string arch; + public bool appIs64Bit; public string appDataDir; + public string debugServerPath; + public string outputDir; + public string appLibrariesDir; + public List? nativeLibraries; } // We want the shell/batch scripts first, since they set up Python environment for the debugger @@ -37,21 +43,30 @@ sealed class Context "ro.product.cpu.abi2", }; + static readonly string[] deviceLibraries = { + "libc.so", + "libm.so", + "libdl.so", + }; + TaskLoggingHelper log; string packageName; string lldbPath; string adbPath; + string outputDir; Dictionary hostLldbServerPaths; string[] supportedAbis; public string? AdbDeviceTarget { get; set; } + public IDictionary>? NativeLibrariesPerABI { get; set; } - public NativeDebugger (TaskLoggingHelper logger, string adbPath, string ndkRootPath, string packageName, string[] supportedAbis) + public NativeDebugger (TaskLoggingHelper logger, string adbPath, string ndkRootPath, string outputDirRoot, string packageName, string[] supportedAbis) { this.log = logger; this.packageName = packageName; this.adbPath = adbPath; this.supportedAbis = supportedAbis; + outputDir = Path.Combine (outputDirRoot, "native-debug"); FindTools (ndkRootPath, supportedAbis); } @@ -65,16 +80,21 @@ public void Attach () } /// - /// Launch the application under control of the debugger. If is provided, - /// it will be launched instead of the default launcher activity. + /// Launch the application under control of the debugger. /// - public void Launch (string? activityName) + public void Launch (string activityName) { + if (String.IsNullOrEmpty (activityName)) { + throw new ArgumentException ("must not be null or empty", nameof (activityName)); + } + Context context = Init (); } Context Init () { + LogLine (); + var context = new Context { adb = new AdbRunner (log, adbPath) }; @@ -100,16 +120,10 @@ Context Init () LogLine (); } - DetermineABI (context); - context.arch = context.abi switch { - "armeabi" => "arm", - "armeabi-v7a" => "arm", - "arm64-v8a" => "arm64", - _ => context.abi, - }; - + DetermineArchitectureAndABI (context); DetermineAppDataDirectory (context); - LogLine ($"Application data directory: {context.appDataDir}"); + PushDebugServer (context); + CopyLibraries (context); return context; @@ -127,6 +141,127 @@ bool PropertyHasValue ((bool haveProperty, string value) result, string expected } } + void CopyLibraries (Context context) + { + LogLine ("Populating local native library cache"); + context.appLibrariesDir = Path.Combine (context.outputDir, "app", "lib"); + if (!Directory.Exists (context.appLibrariesDir)) { + Directory.CreateDirectory (context.appLibrariesDir); + } + + if (context.nativeLibraries != null) { + LogLine (" Copying application native libraries"); + foreach (string library in context.nativeLibraries) { + LogLine ($" {library}"); + + string fileName = Path.GetFileName (library); + if (fileName.StartsWith ("libmono-android.")) { + fileName = "libmonodroid.so"; + } + File.Copy (library, Path.Combine (context.appLibrariesDir, fileName), true); + } + } + + var requiredFiles = new List (); + var libraries = new List (); + string libraryPath; + + if (context.appIs64Bit) { + libraryPath = "/system/lib64"; + requiredFiles.Add ("/system/bin/app_process64"); + requiredFiles.Add ("/system/bin/linker64"); + } else { + libraryPath = "/system/lib"; + requiredFiles.Add ("/system/bin/linker"); + } + + foreach (string lib in deviceLibraries) { + requiredFiles.Add ($"{libraryPath}/{lib}"); + } + + LogLine (" Copying binaries from device"); + bool isWindows = OS.IsWindows; + foreach (string file in requiredFiles) { + string filePath = ToLocalPathFormat (file); + string localPath = $"{context.outputDir}{filePath}"; + string localDir = Path.GetDirectoryName (localPath); + + if (!Directory.Exists (localDir)) { + Directory.CreateDirectory (localDir); + } + + Log ($" From '{file}' to '{localPath}' "); + if (!context.adb.Pull (file, localPath).Result) { + LogLine ("[FAILED]"); + } else { + LogLine ("[SUCCESS]"); + } + } + + if (context.appIs64Bit) { + return; + } + + // /system/bin/app_process is 32-bit on 32-bit devices, but a symlink to + // # app_process64 on 64-bit. If we need the 32-bit version, try to pull + // # app_process32, and if that fails, pull app_process. + string destination = $"{context.outputDir}{ToLocalPathFormat ("/system/bin/app_process")}"; + string? source = "/system/bin/app_process32"; + + if (!context.adb.Pull (source, destination).Result) { + source = "/system/bin/app_process"; + if (!context.adb.Pull (source, destination).Result) { + source = null; + } + } + + if (String.IsNullOrEmpty (source)) { + LogLine (" Failed to copy 32-bit app_process"); + } else { + Log ($" From '{source}' to '{destination}' "); + } + + string ToLocalPathFormat (string path) => isWindows ? path.Replace ("/", "\\") : path; + } + + void PushDebugServer (Context context) + { + if (!hostLldbServerPaths.TryGetValue (context.abi, out string debugServerPath)) { + throw new InvalidOperationException ($"Debug server for abi '{context.abi}' not found."); + } + + string serverName = $"{context.arch}-{Path.GetFileName (debugServerPath)}"; + string deviceServerPath = Path.Combine (context.appDataDir, serverName); + + // Always push the server binary, as we don't know what version might already be there + LogLine ($"Uploading {debugServerPath} to device"); + + // First upload to temporary path, as it's writable for everyone + string remotePath = $"/data/local/tmp/{serverName}"; + if (!context.adb.Push (debugServerPath, remotePath).Result) { + throw new InvalidOperationException ($"Failed to upload debug server {debugServerPath} to device path {remotePath}"); + } + + // Next, copy it to the app dir, with run-as + (bool success, string output) = context.adb.Shell ( + "cat", remotePath, "|", + "run-as", packageName, + "sh", "-c", $"'cat > {deviceServerPath}'" + ).Result; + + if (!success) { + throw new InvalidOperationException ($"Failed to copy debug server on device, from {remotePath} to {deviceServerPath}"); + } + + (success, output) = context.adb.RunAs (packageName, "chmod", "700", deviceServerPath).Result; + if (!success) { + throw new InvalidOperationException ($"Failed to make debug server executable on device, at {deviceServerPath}"); + } + + context.debugServerPath = deviceServerPath; + LogLine ($"Debug server path on device: {context.debugServerPath}"); + } + void DetermineAppDataDirectory (Context context) { (bool success, string output) = context.adb.GetAppDataDirectory (packageName).Result; @@ -135,20 +270,21 @@ void DetermineAppDataDirectory (Context context) } context.appDataDir = output.Trim (); + LogLine ($"Application data directory on device: {context.appDataDir}"); // Applications with minSdkVersion >= 24 will have their data directories // created with rwx------ permissions, preventing adbd from forwarding to // the gdbserver socket. To be safe, if we're on a device >= 24, always // chmod the directory. if (context.apiLevel >= 24) { - (success, output) = context.adb.Shell ("/system/bin/chmod", "a+x", context.appDataDir).Result; + (success, output) = context.adb.RunAs (packageName, "/system/bin/chmod", "a+x", context.appDataDir).Result; if (!success) { throw new InvalidOperationException ("Failed to make application data directory world executable"); } } } - void DetermineABI (Context context) + void DetermineArchitectureAndABI (Context context) { string[]? deviceABIs = null; @@ -159,24 +295,46 @@ void DetermineABI (Context context) } deviceABIs = value.Split (','); + break; } if (deviceABIs == null || deviceABIs.Length == 0) { throw new InvalidOperationException ("Unable to determine device ABI"); } - LogLine ($"Application ABIs: {String.Join ("", "", supportedAbis)}"); - LogLine ($"Device ABIs: {String.Join ("", "", deviceABIs)}"); + LogABIs ("Application", supportedAbis); + LogABIs (" Device", deviceABIs); + foreach (string deviceABI in deviceABIs) { foreach (string appABI in supportedAbis) { if (String.Compare (appABI, deviceABI, StringComparison.OrdinalIgnoreCase) == 0) { context.abi = deviceABI; + context.arch = context.abi switch { + "armeabi" => "arm", + "armeabi-v7a" => "arm", + "arm64-v8a" => "arm64", + _ => context.abi, + }; + + LogLine ($" Selected ABI: {context.abi} (architecture: {context.arch})"); + + context.appIs64Bit = context.abi.IndexOf ("64", StringComparison.Ordinal) >= 0; + context.outputDir = Path.Combine (outputDir, context.abi); + if (NativeLibrariesPerABI != null && NativeLibrariesPerABI.TryGetValue (context.abi, out List abiLibraries)) { + context.nativeLibraries = abiLibraries; + } return; } } } throw new InvalidOperationException ($"Application cannot run on the selected device: no matching ABI found"); + + void LogABIs (string which, string[] abis) + { + string list = String.Join (", ", abis); + LogLine ($"{which} ABIs: {list}"); + } } string GetLlvmVersion (string toolchainDir) @@ -217,7 +375,7 @@ void FindTools (string ndkRootPath, string[] supportedAbis) string llvmVersion = GetLlvmVersion (toolchainDir); foreach (string abi in supportedAbis) { string llvmAbi = NdkHelper.TranslateAbiToLLVM (abi); - path = Path.Combine (ndkRootPath, NdkHelper.RelativeToolchainDir, "lib64", "clang", llvmVersion, "lib", "linux", abi, "lldb-server"); + path = Path.Combine (ndkRootPath, NdkHelper.RelativeToolchainDir, "lib64", "clang", llvmVersion, "lib", "linux", llvmAbi, "lldb-server"); if (!File.Exists (path)) { throw new InvalidOperationException ($"LLVM lldb server component for ABI '{abi}' not found at '{path}'"); } @@ -242,7 +400,11 @@ void Log (string? message = null, bool isError = false) message = message ?? String.Empty; writer.Write (message); if (!String.IsNullOrEmpty (message)) { - log.LogError (message); + if (isError) { + log.LogError (message); + } else { + log.LogMessage (message); + } } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NdkHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NdkHelper.cs index ac514d05a3e..c8bda862396 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/NdkHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NdkHelper.cs @@ -35,10 +35,21 @@ public static string TranslateAbiToLLVM (string xaAbi) { return xaAbi switch { "armeabi-v7a" => "arm", - "arm64-v8a" => "aarch64", - "x86" => "i386", - "x86_64" => "x86_64", - _ => throw new InvalidOperationException ($"Unknown ABI '{xaAbi}'") + "arm64-v8a" => "aarch64", + "x86" => "i386", + "x86_64" => "x86_64", + _ => throw new InvalidOperationException ($"Unknown ABI '{xaAbi}'"), + }; + } + + public static string RIDToABI (string rid) + { + return rid switch { + "android-arm" => "armeabi-v7a", + "android-arm64" => "arm64-v8a", + "android-x86" => "x86", + "android-x64" => "x86_64", + _ => throw new InvalidOperationException ($"Unknown RID '{rid}'") }; } } diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index 812ec5b5f79..d680f5d352c 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -329,6 +329,7 @@ Copyright (C) 2011-2012 Xamarin. All rights reserved. <_AndroidAotStripLibraries Condition=" '$(_AndroidAotStripLibraries)' == '' And '$(AndroidIncludeDebugSymbols)' != 'true' ">True + <_AndroidStripNativeLibraries Condition=" '$(_AndroidStripNativeLibraries)' == '' And '$(AndroidIncludeDebugSymbols)' != 'true' ">True false true True @@ -366,7 +367,7 @@ Copyright (C) 2011-2012 Xamarin. All rights reserved. Imports ******************************************* --> - + @@ -1528,6 +1529,7 @@ because xbuild doesn't support framework reference assemblies. HaveMultipleRIDs="$(_HaveMultipleRIDs)" IntermediateOutputDirectory="$(IntermediateOutputPath)" Environments="@(AndroidEnvironment);@(LibraryEnvironments)" + NativeDebuggingEnabled="$(_AndroidEnableNativeDebugging)" NativeCodeProfilingEnabled="$(_AndroidEnableNativeCodeProfiling)" DeviceSdkVersion="$(_AndroidDeviceSdkVersion)"> @@ -2036,10 +2038,15 @@ because xbuild doesn't support framework reference assemblies. DependsOnTargets="_CompileNativeAssemblySources;_PrepareApplicationSharedLibraryItems" Inputs="@(_NativeAssemblyTarget)" Outputs="@(_ApplicationSharedLibrary)"> + + <_KeepDebugInfo Condition=" '$(_AndroidStripNativeLibraries)' != 'true' ">True + <_KeepDebugInfo Condition=" '$(_AndroidStripNativeLibraries)' == 'true' ">False + + From 69185ac4aa5c334e81d8d5b1e80715dce42c36e8 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Mon, 14 Nov 2022 22:27:21 +0100 Subject: [PATCH 06/30] [WIP] Library symbolication + some progress towards running lldb --- .../Utilities/DotnetSymbolRunner.cs | 41 ++ .../Utilities/ELFHelper.cs | 113 +++++- .../Utilities/LldbRunner.cs | 30 ++ .../Utilities/NativeDebugger.cs | 359 +++++++++++++++--- 4 files changed, 471 insertions(+), 72 deletions(-) create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/DotnetSymbolRunner.cs create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/LldbRunner.cs diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/DotnetSymbolRunner.cs b/src/Xamarin.Android.Build.Tasks/Utilities/DotnetSymbolRunner.cs new file mode 100644 index 00000000000..9b8fed3d8aa --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/DotnetSymbolRunner.cs @@ -0,0 +1,41 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading.Tasks; + +using Microsoft.Build.Utilities; + +namespace Xamarin.Android.Tasks +{ + class DotnetSymbolRunner : ToolRunner + { + public DotnetSymbolRunner (TaskLoggingHelper logger, string dotnetSymbolPath) + : base (logger, dotnetSymbolPath) + {} + + public async Task Fetch (string nativeLibraryPath, bool enableDiagnostics = false) + { + var runner = CreateProcessRunner (); + runner.AddArgument ("--symbols"); + runner.AddArgument ("--timeout").AddArgument ("1"); + runner.AddArgument ("--overwrite"); + + if (enableDiagnostics) { + runner.AddArgument ("--diagnostics"); + } + + runner.AddArgument (nativeLibraryPath); + + return await Run (runner); + } + + async Task Run (ProcessRunner runner) + { + return await RunTool ( + () => { + return runner.Run (); + } + ); + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ELFHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ELFHelper.cs index 66cc957cab7..85bf7a52927 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ELFHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ELFHelper.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Concurrent; using System.IO; +using System.Text; using ELFSharp; using ELFSharp.ELF; @@ -15,23 +16,17 @@ namespace Xamarin.Android.Tasks { static class ELFHelper { - public static bool IsEmptyAOTLibrary (TaskLoggingHelper log, string path) + public static bool IsAOTLibrary (TaskLoggingHelper log, string path) { - if (String.IsNullOrEmpty (path) || !File.Exists (path)) { - return false; - } - - try { - return IsEmptyAOTLibrary (log, path, ELFReader.Load (path)); - } catch (Exception ex) { - log.LogWarning ($"Attempt to check whether '{path}' is a valid ELF file failed with exception, ignoring AOT check for the file."); - log.LogWarningFromException (ex, showStackTrace: true); + IELF? elf = ReadElfFile (log, path, "Unable to check if file is an AOT shared library."); + if (elf == null) { return false; } + return IsAOTLibrary (elf); } - static bool IsEmptyAOTLibrary (TaskLoggingHelper log, string path, IELF elf) + static bool IsAOTLibrary (IELF elf) { ISymbolTable? symtab = GetSymbolTable (elf, ".dynsym"); if (symtab == null) { @@ -39,20 +34,95 @@ static bool IsEmptyAOTLibrary (TaskLoggingHelper log, string path, IELF elf) return false; } - bool mono_aot_file_info_found = false; foreach (var entry in symtab.Entries) { if (String.Compare ("mono_aot_file_info", entry.Name, StringComparison.Ordinal) == 0 && entry.Type == ELFSymbolType.Object) { - mono_aot_file_info_found = true; + return true; + } + } + + return false; + } + + public static bool HasDebugSymbols (TaskLoggingHelper log, string path) + { + return HasDebugSymbols (log, path, out bool _); + } + + public static bool HasDebugSymbols (TaskLoggingHelper log, string path, out bool usesDebugLink) + { + usesDebugLink = false; + IELF? elf = ReadElfFile (log, path, "Skipping debug symbols presence check."); + if (elf == null) { + return false; + } + + if (HasDebugSymbols (elf)) { + return true; + } + + ISection? gnuDebugLink = GetSection (elf, ".gnu_debuglink"); + if (gnuDebugLink == null) { + return false; + } + usesDebugLink = true; + + byte[] contents = gnuDebugLink.GetContents (); + if (contents == null || contents.Length == 0) { + return false; + } + + // .gnu_debuglink section format: https://sourceware.org/gdb/current/onlinedocs/gdb/Separate-Debug-Files.html#index-_002egnu_005fdebuglink-sections + int nameEnd = -1; + for (int i = 0; i < contents.Length; i++) { + if (contents[i] == 0) { + nameEnd = i; break; } } - if (!mono_aot_file_info_found) { - // Not a MonoVM AOT assembly + if (nameEnd < 2) { + // Name is terminated with a 0 byte, so we need at least 2 bytes + return false; + } + + string debugInfoFileName = Encoding.UTF8.GetString (contents, 0, nameEnd); + if (String.IsNullOrEmpty (debugInfoFileName)) { return false; } - symtab = GetSymbolTable (elf, ".symtab"); + string debugFilePath = Path.Combine (Path.GetDirectoryName (path), debugInfoFileName); + return File.Exists (debugFilePath); + } + + static bool HasDebugSymbols (IELF elf) + { + return GetSymbolTable (elf, ".symtab") != null; + } + + public static bool IsEmptyAOTLibrary (TaskLoggingHelper log, string path) + { + if (String.IsNullOrEmpty (path) || !File.Exists (path)) { + return false; + } + + try { + return IsEmptyAOTLibrary (log, path, ELFReader.Load (path)); + } catch (Exception ex) { + log.LogWarning ($"Attempt to check whether '{path}' is a valid ELF file failed with exception, ignoring AOT check for the file."); + log.LogWarningFromException (ex, showStackTrace: true); + return false; + } + + } + + static bool IsEmptyAOTLibrary (TaskLoggingHelper log, string path, IELF elf) + { + if (!IsAOTLibrary (elf)) { + // Not a MonoVM AOT shared library + return false; + } + + ISymbolTable? symtab = GetSymbolTable (elf, ".symtab"); if (symtab == null) { // The DSO is stripped, we can't tell if there are any functions defined (.text will be present anyway) // We perhaps **can** take a look at the .text section size, but it's not a solid check... @@ -121,5 +191,16 @@ bool IsNonEmptyCodeSymbol (SymbolEntry? symbolEntry) where T : struct return section; } + + static IELF? ReadElfFile (TaskLoggingHelper log, string path, string customErrorMessage) + { + try { + return ELFReader.Load (path); + } catch (Exception ex) { + log.LogWarning ($"{path} may not be a valid ELF binary. ${customErrorMessage}"); + log.LogWarningFromException (ex, showStackTrace: false); + return null; + } + } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LldbRunner.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LldbRunner.cs new file mode 100644 index 00000000000..7cf3303306b --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LldbRunner.cs @@ -0,0 +1,30 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading.Tasks; + +using Microsoft.Build.Utilities; + +namespace Xamarin.Android.Tasks +{ + // NOTE: export TERMINFO=path/to/terminfo on Unix, because lldb bundled with the NDK is unable to find it on its own. + // It searches for the database at "/buildbot/src/android/llvm-toolchain/out/lib/libncurses-linux-install/share/terminfo" + class LldbRunner : ToolRunner + { + bool needPythonEnvvars; + + public LldbRunner (TaskLoggingHelper logger, string lldbPath) + : base (logger, lldbPath) + { + // If we're invoking the executable directly, we need to set up Python environment variables or lldb won't run + string ext = Path.GetExtension (lldbPath); + if (String.Compare (".sh", ext, StringComparison.OrdinalIgnoreCase) == 0 || + String.Compare (".cmd", ext, StringComparison.OrdinalIgnoreCase) == 0 || + String.Compare (".bat", ext, StringComparison.OrdinalIgnoreCase) == 0) { + needPythonEnvvars = false; + } else { + needPythonEnvvars = true; + } + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeDebugger.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeDebugger.cs index 9d1d1c28425..3565e7084c5 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/NativeDebugger.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeDebugger.cs @@ -2,9 +2,12 @@ using System.Collections.Generic; using System.IO; +using Microsoft.Android.Build.Tasks; using Microsoft.Build.Utilities; using Xamarin.Android.Tools; +using TPL = System.Threading.Tasks; + namespace Xamarin.Android.Tasks { /// @@ -12,6 +15,23 @@ namespace Xamarin.Android.Tasks /// class NativeDebugger { + const ConsoleColor ErrorColor = ConsoleColor.Red; + const ConsoleColor DebugColor = ConsoleColor.DarkGray; + const ConsoleColor InfoColor = ConsoleColor.Green; + const ConsoleColor MessageColor = ConsoleColor.Gray; + const ConsoleColor WarningColor = ConsoleColor.Yellow; + const ConsoleColor StatusLabel = ConsoleColor.Cyan; + const ConsoleColor StatusText = ConsoleColor.White; + + enum LogLevel + { + Error, + Warning, + Info, + Message, + Debug + } + sealed class Context { public AdbRunner adb; @@ -23,14 +43,15 @@ sealed class Context public string debugServerPath; public string outputDir; public string appLibrariesDir; + public uint applicationPID; public List? nativeLibraries; } // We want the shell/batch scripts first, since they set up Python environment for the debugger static readonly string[] lldbNames = { "lldb.sh", - "lldb.cmd", "lldb", + "lldb.cmd", "lldb.exe", }; @@ -49,6 +70,14 @@ sealed class Context "libdl.so", }; + static HashSet xaLibraries = new HashSet (StringComparer.Ordinal) { + "libmonodroid.so", + "libxamarin-app.so", + "libxamarin-debug-app-helper.so", + }; + + static readonly object consoleLock = new object (); + TaskLoggingHelper log; string packageName; string lldbPath; @@ -68,30 +97,72 @@ public NativeDebugger (TaskLoggingHelper logger, string adbPath, string ndkRootP this.supportedAbis = supportedAbis; outputDir = Path.Combine (outputDirRoot, "native-debug"); - FindTools (ndkRootPath, supportedAbis); + if (!FindTools (ndkRootPath, supportedAbis)) { + throw new InvalidOperationException ("Failed to find all the required tools and utilities"); + } } /// /// Detect PID of the running application and attach the debugger to it /// - public void Attach () + public bool Attach () { - Context context = Init (); + Context? context = Init (); + + return context != null; } /// /// Launch the application under control of the debugger. /// - public void Launch (string activityName) + public bool Launch (string activityName) { if (String.IsNullOrEmpty (activityName)) { throw new ArgumentException ("must not be null or empty", nameof (activityName)); } - Context context = Init (); + Context? context = Init (); + if (context == null) { + return false; + } + + // Start the app, tell it to wait for debugger to attach and to kill any running instance + // We tell `am` to wait ('-W') for the app to start, so that `pidof` then can find the process + string launchName = $"{packageName}/{activityName}"; + LogLine (); + LogStatusLine ("Launching activity", launchName); + Log ("Waiting for the activity to start..."); + (bool success, string output) = context.adb.Shell ("am", "start", "-D", "-S", "-W", launchName).Result; + if (!success) { + LogErrorLine ("Failed to launch the activity"); + return false; + } + + (success, output) = context.adb.Shell ("pidof", packageName).Result; + if (!success) { + LogErrorLine ("Failed to obtain PID of the running application"); + LogErrorLine (output); + return false; + } + + output = output.Trim (); + if (!UInt32.TryParse (output, out context.applicationPID)) { + LogErrorLine ($"Unable to parse string '{output}' as the package's PID"); + return false; + } + + LogStatusLine ("Application PID", output); + + TPL.Task debugServerTask = StartDebugServer (context); + return true; + } + + TPL.Task StartDebugServer (Context context) + { + return null; } - Context Init () + Context? Init () { LogLine (); @@ -101,7 +172,8 @@ Context Init () (bool success, string output) = context.adb.GetPropertyValue ("ro.build.version.sdk").Result; if (!success || String.IsNullOrEmpty (output) || !Int32.TryParse (output, out int apiLevel)) { - throw new InvalidOperationException ("Unable to determine connected device's API level"); + LogErrorLine ("Unable to determine connected device's API level"); + return null; } context.apiLevel = apiLevel; @@ -110,19 +182,28 @@ Context Init () (success, output) = context.adb.Shell ("cat", "/proc/sys/kernel/yama/ptrace_scope", "2>/dev/null").Result; if (success && YamaOK (output.Trim ()) && - PropertyHasValue (context.adb.GetPropertyValue ("ro.build.product").Result, "dragon") && - PropertyHasValue (context.adb.GetPropertyValue ("ro.product.name").Result, "ryu") + PropertyIsEqualTo (context.adb.GetPropertyValue ("ro.build.product").Result, "dragon") && + PropertyIsEqualTo (context.adb.GetPropertyValue ("ro.product.name").Result, "ryu") ) { - LogLine ("WARNING: The device uses Yama ptrace_scope to restrict debugging. ndk-gdb will"); - LogLine (" likely be unable to attach to a process. With root access, the restriction"); - LogLine (" can be lifted by writing 0 to /proc/sys/kernel/yama/ptrace_scope. Consider"); - LogLine (" upgrading your Pixel C to MXC89L or newer, where Yama is disabled."); + LogWarningLine ("WARNING: The device uses Yama ptrace_scope to restrict debugging. ndk-gdb will"); + LogWarningLine (" likely be unable to attach to a process. With root access, the restriction"); + LogWarningLine (" can be lifted by writing 0 to /proc/sys/kernel/yama/ptrace_scope. Consider"); + LogWarningLine (" upgrading your Pixel C to MXC89L or newer, where Yama is disabled."); LogLine (); } - DetermineArchitectureAndABI (context); - DetermineAppDataDirectory (context); - PushDebugServer (context); + if (!DetermineArchitectureAndABI (context)) { + return null; + } + + if (!DetermineAppDataDirectory (context)) { + return null; + } + + if (!PushDebugServer (context)) { + return null; + } + CopyLibraries (context); return context; @@ -132,7 +213,7 @@ bool YamaOK (string output) return !String.IsNullOrEmpty (output) && String.Compare ("0", output, StringComparison.Ordinal) != 0; } - bool PropertyHasValue ((bool haveProperty, string value) result, string expected) + bool PropertyIsEqualTo ((bool haveProperty, string value) result, string expected) { return result.haveProperty && @@ -141,16 +222,61 @@ bool PropertyHasValue ((bool haveProperty, string value) result, string expected } } + bool EnsureSharedLibraryHasSymboles (string libraryPath, DotnetSymbolRunner? dotnetSymbol) + { + bool tryToFetchSymbols = false; + bool hasSymbols = ELFHelper.HasDebugSymbols (log, libraryPath, out bool usesDebugLink); + string libName = Path.GetFileName (libraryPath); + + if (!xaLibraries.Contains (libName)) { + if (ELFHelper.IsAOTLibrary (log, libraryPath)) { + return true; // We don't are about symbols, AOT libraries are only data + } + + // It might be a framework shared library, we'll try to fetch symbols if necessary and possible + tryToFetchSymbols = !hasSymbols && usesDebugLink; + } + + if (tryToFetchSymbols && dotnetSymbol != null) { + LogInfoLine ($" Attempting to download debug symbols from symbol server"); + if (!dotnetSymbol.Fetch (libraryPath).Result) { + LogWarningLine ($" Warning: failed to download debug symbols for {libraryPath}"); + } else { + LogLine (); + } + } + + hasSymbols = ELFHelper.HasDebugSymbols (log, libraryPath); + return hasSymbols; + } + + DotnetSymbolRunner? GetDotnetSymbolRunner () + { + string dotnetSymbolPath = Path.Combine (Environment.GetFolderPath (Environment.SpecialFolder.UserProfile), ".dotnet", "tools", "dotnet-symbol"); + if (OS.IsWindows) { + dotnetSymbolPath = $"{dotnetSymbolPath}.exe"; + } + + if (!File.Exists (dotnetSymbolPath)) { + return null; + } + + return new DotnetSymbolRunner (log, dotnetSymbolPath); + } + void CopyLibraries (Context context) { - LogLine ("Populating local native library cache"); + LogInfoLine ("Populating local native library cache"); context.appLibrariesDir = Path.Combine (context.outputDir, "app", "lib"); if (!Directory.Exists (context.appLibrariesDir)) { Directory.CreateDirectory (context.appLibrariesDir); } if (context.nativeLibraries != null) { - LogLine (" Copying application native libraries"); + LogInfoLine (" Copying application native libraries"); + bool haveLibsWithoutSymbols = false; + DotnetSymbolRunner? dotnetSymbol = GetDotnetSymbolRunner (); + foreach (string library in context.nativeLibraries) { LogLine ($" {library}"); @@ -158,7 +284,19 @@ void CopyLibraries (Context context) if (fileName.StartsWith ("libmono-android.")) { fileName = "libmonodroid.so"; } - File.Copy (library, Path.Combine (context.appLibrariesDir, fileName), true); + string destPath = Path.Combine (context.appLibrariesDir, fileName); + File.Copy (library, destPath, true); + + if (!EnsureSharedLibraryHasSymboles (destPath, dotnetSymbol)) { + haveLibsWithoutSymbols = true; + } + } + + if (haveLibsWithoutSymbols) { + LogWarningLine ($"One or more native libraries have no debug symbols."); + if (dotnetSymbol == null) { + LogWarningLine ($"The dotnet-symbol tool was not found. It can be installed using: dotnet tool install -g dotnet-symbol"); + } } } @@ -179,7 +317,8 @@ void CopyLibraries (Context context) requiredFiles.Add ($"{libraryPath}/{lib}"); } - LogLine (" Copying binaries from device"); + LogLine (); + LogInfoLine (" Copying binaries from device"); bool isWindows = OS.IsWindows; foreach (string file in requiredFiles) { string filePath = ToLocalPathFormat (file); @@ -216,30 +355,32 @@ void CopyLibraries (Context context) } if (String.IsNullOrEmpty (source)) { - LogLine (" Failed to copy 32-bit app_process"); + LogWarningLine (" Failed to copy 32-bit app_process"); } else { - Log ($" From '{source}' to '{destination}' "); + LogLine ($" From '{source}' to '{destination}' "); } string ToLocalPathFormat (string path) => isWindows ? path.Replace ("/", "\\") : path; } - void PushDebugServer (Context context) + bool PushDebugServer (Context context) { if (!hostLldbServerPaths.TryGetValue (context.abi, out string debugServerPath)) { - throw new InvalidOperationException ($"Debug server for abi '{context.abi}' not found."); + LogErrorLine ($"Debug server for abi '{context.abi}' not found."); + return false; } string serverName = $"{context.arch}-{Path.GetFileName (debugServerPath)}"; string deviceServerPath = Path.Combine (context.appDataDir, serverName); // Always push the server binary, as we don't know what version might already be there - LogLine ($"Uploading {debugServerPath} to device"); + LogDebugLine ($"Uploading {debugServerPath} to device"); // First upload to temporary path, as it's writable for everyone string remotePath = $"/data/local/tmp/{serverName}"; if (!context.adb.Push (debugServerPath, remotePath).Result) { - throw new InvalidOperationException ($"Failed to upload debug server {debugServerPath} to device path {remotePath}"); + LogErrorLine ($"Failed to upload debug server {debugServerPath} to device path {remotePath}"); + return false; } // Next, copy it to the app dir, with run-as @@ -250,27 +391,34 @@ void PushDebugServer (Context context) ).Result; if (!success) { - throw new InvalidOperationException ($"Failed to copy debug server on device, from {remotePath} to {deviceServerPath}"); + LogErrorLine ($"Failed to copy debug server on device, from {remotePath} to {deviceServerPath}"); + return false; } (success, output) = context.adb.RunAs (packageName, "chmod", "700", deviceServerPath).Result; if (!success) { - throw new InvalidOperationException ($"Failed to make debug server executable on device, at {deviceServerPath}"); + LogErrorLine ($"Failed to make debug server executable on device, at {deviceServerPath}"); + return false; } context.debugServerPath = deviceServerPath; - LogLine ($"Debug server path on device: {context.debugServerPath}"); + LogStatusLine ("Debug server path on device", context.debugServerPath); + LogLine (); + + return true; } - void DetermineAppDataDirectory (Context context) + bool DetermineAppDataDirectory (Context context) { (bool success, string output) = context.adb.GetAppDataDirectory (packageName).Result; if (!success) { - throw new InvalidOperationException ($"Unable to determine data directory for package '{packageName}'"); + LogErrorLine ($"Unable to determine data directory for package '{packageName}'"); + return false; } context.appDataDir = output.Trim (); - LogLine ($"Application data directory on device: {context.appDataDir}"); + LogStatusLine ($"Application data directory on device", context.appDataDir); + LogLine (); // Applications with minSdkVersion >= 24 will have their data directories // created with rwx------ permissions, preventing adbd from forwarding to @@ -279,12 +427,15 @@ void DetermineAppDataDirectory (Context context) if (context.apiLevel >= 24) { (success, output) = context.adb.RunAs (packageName, "/system/bin/chmod", "a+x", context.appDataDir).Result; if (!success) { - throw new InvalidOperationException ("Failed to make application data directory world executable"); + LogErrorLine ("Failed to make application data directory world executable"); + return false; } } + + return true; } - void DetermineArchitectureAndABI (Context context) + bool DetermineArchitectureAndABI (Context context) { string[]? deviceABIs = null; @@ -299,7 +450,8 @@ void DetermineArchitectureAndABI (Context context) } if (deviceABIs == null || deviceABIs.Length == 0) { - throw new InvalidOperationException ("Unable to determine device ABI"); + LogErrorLine ("Unable to determine device ABI"); + return false; } LogABIs ("Application", supportedAbis); @@ -316,24 +468,24 @@ void DetermineArchitectureAndABI (Context context) _ => context.abi, }; - LogLine ($" Selected ABI: {context.abi} (architecture: {context.arch})"); + LogStatusLine ($" Selected ABI", $"{context.abi} (architecture: {context.arch})"); context.appIs64Bit = context.abi.IndexOf ("64", StringComparison.Ordinal) >= 0; context.outputDir = Path.Combine (outputDir, context.abi); if (NativeLibrariesPerABI != null && NativeLibrariesPerABI.TryGetValue (context.abi, out List abiLibraries)) { context.nativeLibraries = abiLibraries; } - return; + return true; } } } - throw new InvalidOperationException ($"Application cannot run on the selected device: no matching ABI found"); + LogErrorLine ($"Application cannot run on the selected device: no matching ABI found"); + return false; void LogABIs (string which, string[] abis) { - string list = String.Join (", ", abis); - LogLine ($"{which} ABIs: {list}"); + LogStatusLine ($"{which} ABIs", String.Join (", ", abis)); } } @@ -353,7 +505,7 @@ string GetLlvmVersion (string toolchainDir) return line; } - void FindTools (string ndkRootPath, string[] supportedAbis) + bool FindTools (string ndkRootPath, string[] supportedAbis) { string toolchainDir = Path.Combine (ndkRootPath, NdkHelper.RelativeToolchainDir); string toolchainBinDir = Path.Combine (toolchainDir, "bin"); @@ -367,7 +519,8 @@ void FindTools (string ndkRootPath, string[] supportedAbis) } if (String.IsNullOrEmpty (path)) { - throw new InvalidOperationException ($"Unable to locate lldb executable in '{toolchainBinDir}'"); + LogErrorLine ($"Unable to locate lldb executable in '{toolchainBinDir}'"); + return false; } lldbPath = path; @@ -377,35 +530,129 @@ void FindTools (string ndkRootPath, string[] supportedAbis) string llvmAbi = NdkHelper.TranslateAbiToLLVM (abi); path = Path.Combine (ndkRootPath, NdkHelper.RelativeToolchainDir, "lib64", "clang", llvmVersion, "lib", "linux", llvmAbi, "lldb-server"); if (!File.Exists (path)) { - throw new InvalidOperationException ($"LLVM lldb server component for ABI '{abi}' not found at '{path}'"); + LogErrorLine ($"LLVM lldb server component for ABI '{abi}' not found at '{path}'"); + return false; } hostLldbServerPaths.Add (abi, path); } if (hostLldbServerPaths.Count == 0) { - throw new InvalidOperationException ("Unable to find any lldb-server executables, debugging not possible"); + LogErrorLine ("Unable to find any lldb-server executables, debugging not possible"); + return false; } + + return true; + } + + void Log (string? message) + { + Log (LogLevel.Message, message); + } + + void LogLine (string? message = null) + { + Log ($"{message}{Environment.NewLine}"); + } + + void LogWarning (string? message) + { + Log (LogLevel.Warning, message); + } + + void LogWarningLine (string? message) + { + LogWarning ($"{message}{Environment.NewLine}"); + } + + void LogError (string? message) + { + Log (LogLevel.Error, message); } - void LogLine (string? message = null, bool isError = false) + void LogErrorLine (string? message) { - Log (message, isError); - Log (Environment.NewLine, isError); + LogError ($"{message}{Environment.NewLine}"); } - void Log (string? message = null, bool isError = false) + void LogInfo (string? message) { - TextWriter writer = isError ? Console.Error : Console.Out; + Log (LogLevel.Info, message); + } + + void LogInfoLine (string? message) + { + LogInfo ($"{message}{Environment.NewLine}"); + } + + void LogDebug (string? message) + { + Log (LogLevel.Debug, message); + } + + void LogDebugLine (string? message) + { + LogDebug ($"{message}{Environment.NewLine}"); + } + + void LogStatusLine (string label, string text) + { + Log (LogLevel.Info, $"{label}: ", StatusLabel); + Log (LogLevel.Info, $"{text}{Environment.NewLine}", StatusText); + } + + void Log (LogLevel level, string? message) + { + Log (level, message, ForegroundColor (level)); + } + + void Log (LogLevel level, string? message, ConsoleColor color) + { + TextWriter writer = level == LogLevel.Error ? Console.Error : Console.Out; message = message ?? String.Empty; - writer.Write (message); + + ConsoleColor fg = ConsoleColor.Gray; + try { + lock (consoleLock) { + fg = Console.ForegroundColor; + Console.ForegroundColor = color; + } + + writer.Write (message); + } finally { + Console.ForegroundColor = fg; + } + if (!String.IsNullOrEmpty (message)) { - if (isError) { - log.LogError (message); - } else { - log.LogMessage (message); + switch (level) { + case LogLevel.Error: + log.LogError (message); + break; + + case LogLevel.Warning: + log.LogWarning (message); + break; + + default: + case LogLevel.Message: + case LogLevel.Info: + log.LogMessage (message); + break; + + case LogLevel.Debug: + log.LogDebugMessage (message); + break; } } } + + ConsoleColor ForegroundColor (LogLevel level) => level switch { + LogLevel.Error => ErrorColor, + LogLevel.Warning => WarningColor, + LogLevel.Info => InfoColor, + LogLevel.Debug => DebugColor, + LogLevel.Message => MessageColor, + _ => MessageColor, + }; } } From c7ed034897c5e177e167d5ae23ee4527e7ce7ab3 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Mon, 14 Nov 2022 22:52:58 +0100 Subject: [PATCH 07/30] [WIP] Debug server runner progress --- .../Utilities/AdbRunner.cs | 10 ++++++ .../Utilities/NativeDebugger.cs | 32 ++++++++++++++++--- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs index e979b597fd5..d240c227301 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs @@ -36,6 +36,16 @@ public AdbRunner (TaskLoggingHelper logger, string adbPath, string? deviceSerial } } + public async Task<(bool success, string output)> Forward (string local, string remote) + { + var runner = CreateAdbRunner (); + runner.AddArgument ("forward"); + runner.AddArgument (local); + runner.AddArgument (remote); + + return await CaptureAdbOutput (runner); + } + public async Task Pull (string remotePath, string localPath) { var runner = CreateAdbRunner (); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeDebugger.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeDebugger.cs index 3565e7084c5..11ff55ebe76 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/NativeDebugger.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeDebugger.cs @@ -23,6 +23,9 @@ class NativeDebugger const ConsoleColor StatusLabel = ConsoleColor.Cyan; const ConsoleColor StatusText = ConsoleColor.White; + + const int DefaultHostDebugPort = 5039; + enum LogLevel { Error, @@ -41,6 +44,7 @@ sealed class Context public bool appIs64Bit; public string appDataDir; public string debugServerPath; + public string debugSocketPath; public string outputDir; public string appLibrariesDir; public uint applicationPID; @@ -87,6 +91,7 @@ sealed class Context string[] supportedAbis; public string? AdbDeviceTarget { get; set; } + public int HostDebugPort { get; set; } = -1; public IDictionary>? NativeLibrariesPerABI { get; set; } public NativeDebugger (TaskLoggingHelper logger, string adbPath, string ndkRootPath, string outputDirRoot, string packageName, string[] supportedAbis) @@ -153,21 +158,39 @@ public bool Launch (string activityName) LogStatusLine ("Application PID", output); - TPL.Task debugServerTask = StartDebugServer (context); + (AdbRunner? debugServerRunner, TPL.Task<(bool success, string output)>? debugServerTask) = StartDebugServer (context); + if (debugServerRunner == null || debugServerTask == null) { + return false; + } + return true; } - TPL.Task StartDebugServer (Context context) + (AdbRunner? runner, TPL.Task<(bool success, string output)>? task) StartDebugServer (Context context) { - return null; + var runner = CreateAdbRunner (); + TPL.Task<(bool success, string output)> task = runner.RunAs (packageName, context.debugServerPath, "gdbserver", $"unix://{context.debugSocketPath}"); + + int port = HostDebugPort <= 0 ? DefaultHostDebugPort : HostDebugPort; + (bool success, string output) = context.adb.Forward ($"tcp:{port}", $"localfilesystem:{context.debugSocketPath}").Result; + + if (!success) { + LogErrorLine ("Failed to forward remote device socket to a local port"); + // TODO: kill the process and wait for the task to complete + return (null, null); + } + + return (runner, task); } + AdbRunner CreateAdbRunner () => new AdbRunner (log, adbPath, AdbDeviceTarget); + Context? Init () { LogLine (); var context = new Context { - adb = new AdbRunner (log, adbPath) + adb = CreateAdbRunner () }; (bool success, string output) = context.adb.GetPropertyValue ("ro.build.version.sdk").Result; @@ -205,6 +228,7 @@ TPL.Task StartDebugServer (Context context) } CopyLibraries (context); + context.debugSocketPath = $"{context.appDataDir}/debug_socket"; return context; From 99a07cd35415908fcc0e21e8971b529a3b2eaddf Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 15 Nov 2022 14:30:41 +0100 Subject: [PATCH 08/30] [WIP] Well... it won't work this way Unfortunately, neither connecting to the remote server nor interacting with local lldb is possible. Even the native `ndk-gdb` script won't work with the modern Android apps, and faithfully reproducing what it does also led to: (lldb) gdb-remote 5039 error: Failed to connect port Next step: examine Android Studio sources and do what they do, since that works fine. Luckily, most of the "framework" code written in the previous commit can be reused. The goal now is to generate a shell script (both on Unix and Windows) to actually run lldb, and use msbuild task(s) to prepare the application, pull binaries, make sure they have symbols etc - most of this code is already done and won't have to change much. --- .../Utilities/LldbRunner.cs | 43 ++++- .../Utilities/NativeDebugger.cs | 162 +++++++++++++++--- .../Utilities/ProcessRunner.cs | 9 +- 3 files changed, 182 insertions(+), 32 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LldbRunner.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LldbRunner.cs index 7cf3303306b..285c1901092 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LldbRunner.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LldbRunner.cs @@ -4,16 +4,16 @@ using System.Threading.Tasks; using Microsoft.Build.Utilities; +using Xamarin.Android.Tools; namespace Xamarin.Android.Tasks { - // NOTE: export TERMINFO=path/to/terminfo on Unix, because lldb bundled with the NDK is unable to find it on its own. - // It searches for the database at "/buildbot/src/android/llvm-toolchain/out/lib/libncurses-linux-install/share/terminfo" class LldbRunner : ToolRunner { bool needPythonEnvvars; + string scriptPath; - public LldbRunner (TaskLoggingHelper logger, string lldbPath) + public LldbRunner (TaskLoggingHelper logger, string lldbPath, string lldbScriptPath) : base (logger, lldbPath) { // If we're invoking the executable directly, we need to set up Python environment variables or lldb won't run @@ -25,6 +25,43 @@ public LldbRunner (TaskLoggingHelper logger, string lldbPath) } else { needPythonEnvvars = true; } + + scriptPath = lldbScriptPath; + + EchoStandardError = false; + EchoStandardOutput = false; + ProcessTimeout = TimeSpan.MaxValue; + } + + public bool Run () + { + var runner = CreateProcessRunner ("--source", scriptPath); + if (!OS.IsWindows) { + // lldb bundled with the NDK is unable to find it on its own. + // It searches for the database at "/buildbot/src/android/llvm-toolchain/out/lib/libncurses-linux-install/share/terminfo" + runner.Environment.Add ("TERMINFO", "/usr/share/terminfo"); + } + + if (needPythonEnvvars) { + // We assume our LLDB path is within the NDK root + string pythonDir = Path.GetFullPath (Path.Combine (Path.GetDirectoryName (ToolPath), "..", "python3")); + runner.Environment.Add ("PYTHONHOME", pythonDir); + + if (!OS.IsWindows) { + string envvarName = OS.IsMac ? "DYLD_LIBRARY_PATH" : "LD_LIBRARY_PATH"; + string oldLibraryPath = Environment.GetEnvironmentVariable (envvarName) ?? String.Empty; + string pythonLibDir = Path.Combine (pythonDir, "lib"); + runner.Environment.Add (envvarName, $"{pythonLibDir}:${oldLibraryPath}"); + } + } + + try { + return runner.Run (); + } catch (Exception ex) { + Logger.LogWarning ("LLDB failed with exception"); + Logger.LogWarningFromException (ex); + return false; + } } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeDebugger.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeDebugger.cs index 11ff55ebe76..c688b010b85 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/NativeDebugger.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeDebugger.cs @@ -1,6 +1,8 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; +using System.Text; using Microsoft.Android.Build.Tasks; using Microsoft.Build.Utilities; @@ -48,7 +50,11 @@ sealed class Context public string outputDir; public string appLibrariesDir; public uint applicationPID; + public string lldbScriptPath; + public string zygotePath; + public int debugServerPort; public List? nativeLibraries; + public List deviceBinaryDirs; } // We want the shell/batch scripts first, since they set up Python environment for the debugger @@ -92,6 +98,9 @@ sealed class Context public string? AdbDeviceTarget { get; set; } public int HostDebugPort { get; set; } = -1; + public bool UseLldbGUI { get; set; } = true; + public string? CustomLldbCommandsFilePath { get; set; } + public IDictionary>? NativeLibrariesPerABI { get; set; } public NativeDebugger (TaskLoggingHelper logger, string adbPath, string ndkRootPath, string outputDirRoot, string packageName, string[] supportedAbis) @@ -136,43 +145,103 @@ public bool Launch (string activityName) string launchName = $"{packageName}/{activityName}"; LogLine (); LogStatusLine ("Launching activity", launchName); - Log ("Waiting for the activity to start..."); - (bool success, string output) = context.adb.Shell ("am", "start", "-D", "-S", "-W", launchName).Result; + LogLine ("Waiting for the activity to start..."); + (bool success, string output) = context.adb.Shell ("am", "start", "-S", "-W", launchName).Result; if (!success) { LogErrorLine ("Failed to launch the activity"); return false; } - (success, output) = context.adb.Shell ("pidof", packageName).Result; - if (!success) { + long appPID = GetDeviceProcessID (context, packageName); + if (appPID <= 0) { LogErrorLine ("Failed to obtain PID of the running application"); LogErrorLine (output); return false; } + context.applicationPID = (uint)appPID; - output = output.Trim (); - if (!UInt32.TryParse (output, out context.applicationPID)) { - LogErrorLine ($"Unable to parse string '{output}' as the package's PID"); - return false; - } - - LogStatusLine ("Application PID", output); + LogStatusLine ("Application PID", $"{context.applicationPID}"); (AdbRunner? debugServerRunner, TPL.Task<(bool success, string output)>? debugServerTask) = StartDebugServer (context); if (debugServerRunner == null || debugServerTask == null) { return false; } + GenerateLldbScript (context); + + LogDebugLine ($"Starting LLDB: {lldbPath}"); + var lldb = new LldbRunner (log, lldbPath, context.lldbScriptPath); + if (lldb.Run ()) { + LogWarning ("LLDB failed?"); + } + + KillDebugServer (context); + LogDebugLine ("Waiting on the debug server process to quit"); + (success, output) = debugServerTask.Result; + return true; } + void GenerateLldbScript (Context context) + { + context.lldbScriptPath = Path.Combine (context.outputDir, $"{context.arch}-lldb-script.txt"); + + using (var f = File.OpenWrite (context.lldbScriptPath)) { + using (var sw = new StreamWriter (f, new UTF8Encoding (false))) { + string systemPaths = String.Join (" ", context.deviceBinaryDirs.Select (d => $"'{Path.GetFullPath(d)}'" )); + sw.WriteLine ($"settings append target.exec-search-paths '{Path.GetFullPath (context.appLibrariesDir)}' {systemPaths}"); + sw.WriteLine ($"target create '{Path.GetFullPath (context.zygotePath)}'"); + sw.WriteLine ($"target modules search-paths add / '{Path.GetFullPath (outputDir)}/'"); + sw.WriteLine ($"gdb-remote {context.debugServerPort}"); + + if (UseLldbGUI) { + sw.WriteLine ($"gui"); + } + + if (!String.IsNullOrEmpty (CustomLldbCommandsFilePath)) { + sw.WriteLine (); + sw.Write (File.ReadAllText (CustomLldbCommandsFilePath)); + sw.WriteLine (); + } + + sw.Flush (); + } + } + } + + bool KillDebugServer (Context context) + { + long serverPID = GetDeviceProcessID (context, context.debugServerPath, quiet: true); + if (serverPID <= 0) { + return true; + } + + LogDebugLine ("Killing previous instance of the debug server"); + (bool success, string _) = context.adb.RunAs (packageName, "kill", "-9", $"{serverPID}").Result; + return success; + } + (AdbRunner? runner, TPL.Task<(bool success, string output)>? task) StartDebugServer (Context context) { + LogDebugLine ($"Starting debug server on device: {context.debugServerPath}"); + + (bool success, string output) = context.adb.RunAs (packageName, "rm", "-f", context.debugSocketPath).Result; + if (!success) { + LogWarningLine ($"Failed to remove debug socket on device, {context.debugSocketPath}"); + LogWarningLine (output); + } + + if (!KillDebugServer (context)) { + LogWarningLine ("Failed to kill previous instance of the debug server"); + } + var runner = CreateAdbRunner (); + runner.ProcessTimeout = TimeSpan.MaxValue; + TPL.Task<(bool success, string output)> task = runner.RunAs (packageName, context.debugServerPath, "gdbserver", $"unix://{context.debugSocketPath}"); - int port = HostDebugPort <= 0 ? DefaultHostDebugPort : HostDebugPort; - (bool success, string output) = context.adb.Forward ($"tcp:{port}", $"localfilesystem:{context.debugSocketPath}").Result; + context.debugServerPort = HostDebugPort <= 0 ? DefaultHostDebugPort : HostDebugPort; + (success, output) = context.adb.Forward ($"tcp:{context.debugServerPort}", $"localfilesystem:{context.debugSocketPath}").Result; if (!success) { LogErrorLine ("Failed to forward remote device socket to a local port"); @@ -183,6 +252,28 @@ public bool Launch (string activityName) return (runner, task); } + long GetDeviceProcessID (Context context, string processName, bool quiet = false) + { + (bool success, string output) = context.adb.Shell ("pidof", processName).Result; + if (!success) { + if (!quiet) { + LogErrorLine ($"Failed to obtain PID of process '{processName}'"); + LogErrorLine (output); + } + return -1; + } + + output = output.Trim (); + if (!UInt32.TryParse (output, out uint pid)) { + if (!quiet) { + LogErrorLine ($"Unable to parse string '{output}' as the package's PID"); + } + return -1; + } + + return pid; + } + AdbRunner CreateAdbRunner () => new AdbRunner (log, adbPath, AdbDeviceTarget); Context? Init () @@ -227,7 +318,10 @@ public bool Launch (string activityName) return null; } - CopyLibraries (context); + if (!CopyLibraries (context)) { + return null; + } + context.debugSocketPath = $"{context.appDataDir}/debug_socket"; return context; @@ -288,7 +382,7 @@ bool EnsureSharedLibraryHasSymboles (string libraryPath, DotnetSymbolRunner? dot return new DotnetSymbolRunner (log, dotnetSymbolPath); } - void CopyLibraries (Context context) + bool CopyLibraries (Context context) { LogInfoLine ("Populating local native library cache"); context.appLibrariesDir = Path.Combine (context.outputDir, "app", "lib"); @@ -330,7 +424,10 @@ void CopyLibraries (Context context) if (context.appIs64Bit) { libraryPath = "/system/lib64"; - requiredFiles.Add ("/system/bin/app_process64"); + + string zygotePath = "/system/bin/app_process64"; + requiredFiles.Add (zygotePath); + context.zygotePath = $"{context.outputDir}{ToLocalPathFormat (zygotePath)}"; requiredFiles.Add ("/system/bin/linker64"); } else { libraryPath = "/system/lib"; @@ -343,7 +440,8 @@ void CopyLibraries (Context context) LogLine (); LogInfoLine (" Copying binaries from device"); - bool isWindows = OS.IsWindows; + var dirs = new HashSet (StringComparer.Ordinal); + foreach (string file in requiredFiles) { string filePath = ToLocalPathFormat (file); string localPath = $"{context.outputDir}{filePath}"; @@ -351,6 +449,9 @@ void CopyLibraries (Context context) if (!Directory.Exists (localDir)) { Directory.CreateDirectory (localDir); + if (!dirs.Contains (localDir)) { + dirs.Add (localDir); + } } Log ($" From '{file}' to '{localPath}' "); @@ -361,8 +462,9 @@ void CopyLibraries (Context context) } } + context.deviceBinaryDirs = new List (dirs); if (context.appIs64Bit) { - return; + return true; } // /system/bin/app_process is 32-bit on 32-bit devices, but a symlink to @@ -379,12 +481,15 @@ void CopyLibraries (Context context) } if (String.IsNullOrEmpty (source)) { - LogWarningLine (" Failed to copy 32-bit app_process"); - } else { - LogLine ($" From '{source}' to '{destination}' "); + LogErrorLine ("Failed to copy 32-bit app_process"); + return false; } + LogLine ($" From '{source}' to '{destination}' "); + context.zygotePath = destination; - string ToLocalPathFormat (string path) => isWindows ? path.Replace ("/", "\\") : path; + return true; + + string ToLocalPathFormat (string path) => OS.IsWindows ? path.Replace ("/", "\\") : path; } bool PushDebugServer (Context context) @@ -395,7 +500,9 @@ bool PushDebugServer (Context context) } string serverName = $"{context.arch}-{Path.GetFileName (debugServerPath)}"; - string deviceServerPath = Path.Combine (context.appDataDir, serverName); + context.debugServerPath = Path.Combine (context.appDataDir, serverName); + + KillDebugServer (context); // Always push the server binary, as we don't know what version might already be there LogDebugLine ($"Uploading {debugServerPath} to device"); @@ -411,21 +518,20 @@ bool PushDebugServer (Context context) (bool success, string output) = context.adb.Shell ( "cat", remotePath, "|", "run-as", packageName, - "sh", "-c", $"'cat > {deviceServerPath}'" + "sh", "-c", $"'cat > {context.debugServerPath}'" ).Result; if (!success) { - LogErrorLine ($"Failed to copy debug server on device, from {remotePath} to {deviceServerPath}"); + LogErrorLine ($"Failed to copy debug server on device, from {remotePath} to {context.debugServerPath}"); return false; } - (success, output) = context.adb.RunAs (packageName, "chmod", "700", deviceServerPath).Result; + (success, output) = context.adb.RunAs (packageName, "chmod", "700", context.debugServerPath).Result; if (!success) { - LogErrorLine ($"Failed to make debug server executable on device, at {deviceServerPath}"); + LogErrorLine ($"Failed to make debug server executable on device, at {context.debugServerPath}"); return false; } - context.debugServerPath = deviceServerPath; LogStatusLine ("Debug server path on device", context.debugServerPath); LogLine (); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ProcessRunner.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ProcessRunner.cs index 67499e27d9e..28a2962218d 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ProcessRunner.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ProcessRunner.cs @@ -246,6 +246,12 @@ public bool Run () RedirectStandardOutput = stdout_done != null, }; + if (Environment.Count > 0) { + foreach (var kvp in Environment) { + psi.Environment.Add (kvp.Key, kvp.Value); + } + } + if (arguments != null) { psi.Arguments = String.Join (" ", arguments); } @@ -306,7 +312,8 @@ public bool Run () process.BeginOutputReadLine (); } - bool exited = process.WaitForExit ((int)ProcessTimeout.TotalMilliseconds); + int timeout = ProcessTimeout == TimeSpan.MaxValue ? -1 : (int)ProcessTimeout.TotalMilliseconds; + bool exited = process.WaitForExit (timeout); if (!exited) { log.LogError ($"Process '{FullCommandLine}' timed out after {ProcessTimeout}"); ErrorReason = ErrorReasonCode.ExecutionTimedOut; From a362b9e6bfb3da83b4c3671d5ec2cf8a7396d175 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 16 Nov 2022 23:19:23 +0100 Subject: [PATCH 09/30] [WIP] Working on new approach to running the debug server --- .../Resources/xa_start_lldb_server.sh | 48 +++++++ .../Tasks/CreateTypeManagerJava.cs | 19 +-- .../Utilities/AdbRunner.cs | 5 + .../Utilities/MonoAndroidHelper.cs | 14 ++ .../Utilities/NativeDebugger.cs | 136 ++++++++++++------ .../Xamarin.Android.Build.Tasks.csproj | 3 + 6 files changed, 160 insertions(+), 65 deletions(-) create mode 100755 src/Xamarin.Android.Build.Tasks/Resources/xa_start_lldb_server.sh diff --git a/src/Xamarin.Android.Build.Tasks/Resources/xa_start_lldb_server.sh b/src/Xamarin.Android.Build.Tasks/Resources/xa_start_lldb_server.sh new file mode 100755 index 00000000000..246c89b53c7 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Resources/xa_start_lldb_server.sh @@ -0,0 +1,48 @@ +#!/system/bin/sh + +# +# A modified version of https://android.googlesource.com/platform/prebuilts/tools/+/e55133323a87d21dc5b73bc5539cc8522dc83f69/common/lldb/android/start_lldb_server.sh +# + +# This script launches lldb-server on Android device from application subfolder - /data/data/$packageId/lldb/bin. +# Native run configuration is expected to push this script along with lldb-server to the device prior to its execution. +# Following command arguments are expected to be passed - lldb package directory and lldb-server listen port. + +set -x + +umask 0002 + +LLDB_DIR="$1" +LISTENER_SCHEME="$2" +DOMAINSOCKET_DIR="$3" +PLATFORM_SOCKET="$4" +LOG_CHANNELS="$5" +LLDB_ARCH="$6" + +BIN_DIR="$LLDB_DIR/bin" +LOG_DIR="$LLDB_DIR/log" +TMP_DIR="$LLDB_DIR/tmp" +PLATFORM_LOG_FILE="$LOG_DIR/xa-${LLDB_ARCH}-platform.log" + +export LLDB_DEBUGSERVER_LOG_FILE="$LOG_DIR/xa-${LLDB_ARCH}-lldb-server.log" +export LLDB_SERVER_LOG_CHANNELS="$LOG_CHANNELS" +export LLDB_DEBUGSERVER_DOMAINSOCKET_DIR="$DOMAINSOCKET_DIR" + +# This directory already exists. Make sure it has the right permissions. +chmod 0775 "$LLDB_DIR" + +rm -r "$TMP_DIR" +mkdir "$TMP_DIR" +export TMPDIR="$TMP_DIR" + +rm -r "$LOG_DIR" +mkdir "$LOG_DIR" + +# LLDB would create these files with more restrictive permissions than our umask above. Make sure +# it doesn't get a chance. +# "touch" does not exist on pre API-16 devices. This is a poor man's replacement +cat < /dev/null >"$LLDB_DEBUGSERVER_LOG_FILE" 2> "$PLATFORM_LOG_FILE" + +cd "$TMP_DIR" # change cwd + +exec "$BIN_DIR"/xa-${LLDB_ARCH}-lldb-server platform --server --listen "$LISTENER_SCHEME://$DOMAINSOCKET_DIR/$PLATFORM_SOCKET" --log-file "$PLATFORM_LOG_FILE" --log-channels "$LOG_CHANNELS" < /dev/null > "$LOG_DIR"/xa-${LLDB_ARCH}-platform-stdout.log 2>&1 diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/CreateTypeManagerJava.cs b/src/Xamarin.Android.Build.Tasks/Tasks/CreateTypeManagerJava.cs index 37872f31927..6b7fa4ac594 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/CreateTypeManagerJava.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/CreateTypeManagerJava.cs @@ -1,5 +1,4 @@ using System.IO; -using System.Reflection; using System.Text; using System; @@ -18,11 +17,9 @@ public class CreateTypeManagerJava : AndroidTask [Required] public string OutputFilePath { get; set; } - static readonly Assembly ExecutingAssembly = Assembly.GetExecutingAssembly (); - public override bool RunTask () { - string? content = ReadResource (ResourceName); + string? content = MonoAndroidHelper.ReadManifestResource (Log, ResourceName); if (String.IsNullOrEmpty (content)) { return false; @@ -61,19 +58,5 @@ public override bool RunTask () return !Log.HasLoggedErrors; } - - string? ReadResource (string resourceName) - { - using (var from = ExecutingAssembly.GetManifestResourceStream (resourceName)) { - if (from == null) { - Log.LogCodedError ("XA0116", Properties.Resources.XA0116, resourceName); - return null; - } - - using (var sr = new StreamReader (from)) { - return sr.ReadToEnd (); - } - } - } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs index d240c227301..0aebdf227bf 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs @@ -66,6 +66,11 @@ public async Task Push (string localPath, string remotePath) return await RunAdb (runner); } + public async Task<(bool success, string output)> CreateDirectoryAs (string packageName, string directoryPath) + { + return await RunAs (packageName, "mkdir", "-p", directoryPath); + } + public async Task<(bool success, string output)> RunAs (string packageName, string command, params string[] args) { var shellArgs = new List { diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs index 73485dffb91..72042c7f004 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs @@ -496,5 +496,19 @@ public static string GetRelativePathForAndroidAsset (string assetsDirectory, ITa path = head.Length == path.Length ? path : path.Substring ((head.Length == 0 ? 0 : head.Length + 1) + assetsDirectory.Length).TrimStart (DirectorySeparators); return path; } + + public static string? ReadManifestResource (TaskLoggingHelper log, string resourceName) + { + using (var from = System.Reflection.Assembly.GetExecutingAssembly ().GetManifestResourceStream (resourceName)) { + if (from == null) { + log.LogCodedError ("XA0116", Properties.Resources.XA0116, resourceName); + return null; + } + + using (var sr = new StreamReader (from)) { + return sr.ReadToEnd (); + } + } + } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeDebugger.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeDebugger.cs index c688b010b85..1a74bcdcff9 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/NativeDebugger.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeDebugger.cs @@ -12,6 +12,8 @@ namespace Xamarin.Android.Tasks { + // Starting LLDB server: /data/data/net.twistedcode.myapplication/lldb/bin/start_lldb_server.sh /data/data/net.twistedcode.myapplication/lldb unix-abstract /net.twistedcode.myapplication-0 platform-1668626161269.sock "lldb process:gdb-remote packets" + /// /// Interface to lldb, the NDK native code debugger. /// @@ -25,8 +27,7 @@ class NativeDebugger const ConsoleColor StatusLabel = ConsoleColor.Cyan; const ConsoleColor StatusText = ConsoleColor.White; - - const int DefaultHostDebugPort = 5039; + const string ServerLauncherScriptName = "xa_start_lldb_server.sh"; enum LogLevel { @@ -46,13 +47,19 @@ sealed class Context public bool appIs64Bit; public string appDataDir; public string debugServerPath; + public string debugServerScriptPath; public string debugSocketPath; public string outputDir; public string appLibrariesDir; + public string appLldbBaseDir; + public string appLldbBinDir; + public string appLldbLogDir; public uint applicationPID; public string lldbScriptPath; public string zygotePath; public int debugServerPort; + public string domainSocketDir; + public string platformSocketName; public List? nativeLibraries; public List deviceBinaryDirs; } @@ -87,6 +94,7 @@ sealed class Context }; static readonly object consoleLock = new object (); + static readonly UTF8Encoding UTF8NoBOM = new UTF8Encoding (false); TaskLoggingHelper log; string packageName; @@ -97,7 +105,6 @@ sealed class Context string[] supportedAbis; public string? AdbDeviceTarget { get; set; } - public int HostDebugPort { get; set; } = -1; public bool UseLldbGUI { get; set; } = true; public string? CustomLldbCommandsFilePath { get; set; } @@ -162,22 +169,22 @@ public bool Launch (string activityName) LogStatusLine ("Application PID", $"{context.applicationPID}"); - (AdbRunner? debugServerRunner, TPL.Task<(bool success, string output)>? debugServerTask) = StartDebugServer (context); - if (debugServerRunner == null || debugServerTask == null) { - return false; - } + // (AdbRunner? debugServerRunner, TPL.Task<(bool success, string output)>? debugServerTask) = StartDebugServer (context); + // if (debugServerRunner == null || debugServerTask == null) { + // return false; + // } - GenerateLldbScript (context); + // GenerateLldbScript (context); - LogDebugLine ($"Starting LLDB: {lldbPath}"); - var lldb = new LldbRunner (log, lldbPath, context.lldbScriptPath); - if (lldb.Run ()) { - LogWarning ("LLDB failed?"); - } + // LogDebugLine ($"Starting LLDB: {lldbPath}"); + // var lldb = new LldbRunner (log, lldbPath, context.lldbScriptPath); + // if (lldb.Run ()) { + // LogWarning ("LLDB failed?"); + // } - KillDebugServer (context); - LogDebugLine ("Waiting on the debug server process to quit"); - (success, output) = debugServerTask.Result; + // KillDebugServer (context); + // LogDebugLine ("Waiting on the debug server process to quit"); + // (success, output) = debugServerTask.Result; return true; } @@ -187,7 +194,7 @@ void GenerateLldbScript (Context context) context.lldbScriptPath = Path.Combine (context.outputDir, $"{context.arch}-lldb-script.txt"); using (var f = File.OpenWrite (context.lldbScriptPath)) { - using (var sw = new StreamWriter (f, new UTF8Encoding (false))) { + using (var sw = new StreamWriter (f, UTF8NoBOM)) { string systemPaths = String.Join (" ", context.deviceBinaryDirs.Select (d => $"'{Path.GetFullPath(d)}'" )); sw.WriteLine ($"settings append target.exec-search-paths '{Path.GetFullPath (context.appLibrariesDir)}' {systemPaths}"); sw.WriteLine ($"target create '{Path.GetFullPath (context.zygotePath)}'"); @@ -223,31 +230,30 @@ bool KillDebugServer (Context context) (AdbRunner? runner, TPL.Task<(bool success, string output)>? task) StartDebugServer (Context context) { - LogDebugLine ($"Starting debug server on device: {context.debugServerPath}"); - - (bool success, string output) = context.adb.RunAs (packageName, "rm", "-f", context.debugSocketPath).Result; - if (!success) { - LogWarningLine ($"Failed to remove debug socket on device, {context.debugSocketPath}"); - LogWarningLine (output); - } + LogDebugLine ($"Starting debug server on device: {context.debugServerScriptPath}"); if (!KillDebugServer (context)) { LogWarningLine ("Failed to kill previous instance of the debug server"); } - var runner = CreateAdbRunner (); - runner.ProcessTimeout = TimeSpan.MaxValue; + context.domainSocketDir = $"xa-{packageName}-0"; - TPL.Task<(bool success, string output)> task = runner.RunAs (packageName, context.debugServerPath, "gdbserver", $"unix://{context.debugSocketPath}"); + var rnd = new Random (); + context.platformSocketName = $"xa-platform-{rnd.Next ()}.sock"; - context.debugServerPort = HostDebugPort <= 0 ? DefaultHostDebugPort : HostDebugPort; - (success, output) = context.adb.Forward ($"tcp:{context.debugServerPort}", $"localfilesystem:{context.debugSocketPath}").Result; + var runner = CreateAdbRunner (); + runner.ProcessTimeout = TimeSpan.MaxValue; - if (!success) { - LogErrorLine ("Failed to forward remote device socket to a local port"); - // TODO: kill the process and wait for the task to complete - return (null, null); - } + TPL.Task<(bool success, string output)> task = runner.RunAs ( + packageName, + context.debugServerScriptPath, + context.appLldbBaseDir, // LLDB directory + "unix-abstract", // Listener socket scheme (unix-abstract: virtual, not on the filesystem) + context.domainSocketDir, // Directory where listener socket will be created + context.platformSocketName, // name of the socket to create + "'lldb process:gdb-remote packets'", // LLDB log channels + context.arch // LLDB architecture + ); return (runner, task); } @@ -499,18 +505,53 @@ bool PushDebugServer (Context context) return false; } - string serverName = $"{context.arch}-{Path.GetFileName (debugServerPath)}"; - context.debugServerPath = Path.Combine (context.appDataDir, serverName); + debugServerPath = "/tmp/lldb-server"; + if (!context.adb.CreateDirectoryAs (packageName, context.appLldbBinDir).Result.success) { + LogErrorLine ($"Failed to create debug server destination directory on device, {context.appLldbBinDir}"); + return false; + } + + string serverName = $"xa-{context.arch}-{Path.GetFileName (debugServerPath)}"; + context.debugServerPath = $"{context.appLldbBinDir}/{serverName}"; KillDebugServer (context); // Always push the server binary, as we don't know what version might already be there - LogDebugLine ($"Uploading {debugServerPath} to device"); + if (!PushServerExecutable (context, debugServerPath, context.debugServerPath)) { + return false; + } + LogStatusLine ("Debug server path on device", context.debugServerPath); + + string? launcherScript = MonoAndroidHelper.ReadManifestResource (log, ServerLauncherScriptName); + if (String.IsNullOrEmpty (launcherScript)) { + return false; + } + + string launcherScriptPath = Path.Combine (context.outputDir, ServerLauncherScriptName); + Directory.CreateDirectory (Path.GetDirectoryName (launcherScriptPath)); + File.WriteAllText (launcherScriptPath, launcherScript, UTF8NoBOM); + + context.debugServerScriptPath = $"{context.appLldbBinDir}/{Path.GetFileName (launcherScriptPath)}"; + if (!PushServerExecutable (context, launcherScriptPath, context.debugServerScriptPath)) { + return false; + } + LogStatusLine ("Debug server launcher script path on device", context.debugServerScriptPath); + LogLine (); + + return true; + } + + bool PushServerExecutable (Context context, string hostSource, string deviceDestination) + { + string executableName = Path.GetFileName (deviceDestination); + + // Always push the executable, as we don't know what version might already be there + LogDebugLine ($"Uploading {hostSource} to device"); // First upload to temporary path, as it's writable for everyone - string remotePath = $"/data/local/tmp/{serverName}"; - if (!context.adb.Push (debugServerPath, remotePath).Result) { - LogErrorLine ($"Failed to upload debug server {debugServerPath} to device path {remotePath}"); + string remotePath = $"/data/local/tmp/{executableName}"; + if (!context.adb.Push (hostSource, remotePath).Result) { + LogErrorLine ($"Failed to upload debug server {hostSource} to device path {remotePath}"); return false; } @@ -518,23 +559,20 @@ bool PushDebugServer (Context context) (bool success, string output) = context.adb.Shell ( "cat", remotePath, "|", "run-as", packageName, - "sh", "-c", $"'cat > {context.debugServerPath}'" + "sh", "-c", $"'cat > {deviceDestination}'" ).Result; if (!success) { - LogErrorLine ($"Failed to copy debug server on device, from {remotePath} to {context.debugServerPath}"); + LogErrorLine ($"Failed to copy debug executable to device, from {hostSource} to {deviceDestination}"); return false; } - (success, output) = context.adb.RunAs (packageName, "chmod", "700", context.debugServerPath).Result; + (success, output) = context.adb.RunAs (packageName, "chmod", "700", deviceDestination).Result; if (!success) { - LogErrorLine ($"Failed to make debug server executable on device, at {context.debugServerPath}"); + LogErrorLine ($"Failed to make debug server executable on device, at {deviceDestination}"); return false; } - LogStatusLine ("Debug server path on device", context.debugServerPath); - LogLine (); - return true; } @@ -550,6 +588,10 @@ bool DetermineAppDataDirectory (Context context) LogStatusLine ($"Application data directory on device", context.appDataDir); LogLine (); + context.appLldbBaseDir = $"{context.appDataDir}/lldb"; + context.appLldbBinDir = $"{context.appLldbBaseDir}/bin"; + context.appLldbLogDir = $"{context.appLldbBaseDir}/log"; + // Applications with minSdkVersion >= 24 will have their data directories // created with rwx------ permissions, preventing adbd from forwarding to // the gdbserver socket. To be safe, if we're on a device >= 24, always diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.csproj b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.csproj index ae4a7e2eb2e..099f290ebe3 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.csproj +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.csproj @@ -407,6 +407,9 @@ JavaInteropTypeManager.java + + xa_start_lldb_server.sh + From 6923ffa58e85b836f5834caa842e74ea89ae9a36 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 17 Nov 2022 22:41:45 +0100 Subject: [PATCH 10/30] [WIP] 4h spent looking for a missing /, doh But it finally works! --- .../Resources/xa_start_lldb_server.sh | 45 ++++++++----------- .../Utilities/NativeDebugger.cs | 43 +++++++++++------- 2 files changed, 47 insertions(+), 41 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Resources/xa_start_lldb_server.sh b/src/Xamarin.Android.Build.Tasks/Resources/xa_start_lldb_server.sh index 246c89b53c7..14f030aaa1f 100755 --- a/src/Xamarin.Android.Build.Tasks/Resources/xa_start_lldb_server.sh +++ b/src/Xamarin.Android.Build.Tasks/Resources/xa_start_lldb_server.sh @@ -1,48 +1,41 @@ #!/system/bin/sh -# -# A modified version of https://android.googlesource.com/platform/prebuilts/tools/+/e55133323a87d21dc5b73bc5539cc8522dc83f69/common/lldb/android/start_lldb_server.sh -# - # This script launches lldb-server on Android device from application subfolder - /data/data/$packageId/lldb/bin. # Native run configuration is expected to push this script along with lldb-server to the device prior to its execution. # Following command arguments are expected to be passed - lldb package directory and lldb-server listen port. - set -x - umask 0002 -LLDB_DIR="$1" -LISTENER_SCHEME="$2" -DOMAINSOCKET_DIR="$3" -PLATFORM_SOCKET="$4" -LOG_CHANNELS="$5" -LLDB_ARCH="$6" +LLDB_DIR=$1 +LISTENER_SCHEME=$2 +DOMAINSOCKET_DIR=$3 +PLATFORM_SOCKET=$4 +LOG_CHANNELS=$5 -BIN_DIR="$LLDB_DIR/bin" -LOG_DIR="$LLDB_DIR/log" -TMP_DIR="$LLDB_DIR/tmp" -PLATFORM_LOG_FILE="$LOG_DIR/xa-${LLDB_ARCH}-platform.log" +BIN_DIR=$LLDB_DIR/bin +LOG_DIR=$LLDB_DIR/log +TMP_DIR=$LLDB_DIR/tmp +PLATFORM_LOG_FILE=$LOG_DIR/platform.log -export LLDB_DEBUGSERVER_LOG_FILE="$LOG_DIR/xa-${LLDB_ARCH}-lldb-server.log" +export LLDB_DEBUGSERVER_LOG_FILE=$LOG_DIR/gdb-server.log export LLDB_SERVER_LOG_CHANNELS="$LOG_CHANNELS" -export LLDB_DEBUGSERVER_DOMAINSOCKET_DIR="$DOMAINSOCKET_DIR" +export LLDB_DEBUGSERVER_DOMAINSOCKET_DIR=$DOMAINSOCKET_DIR # This directory already exists. Make sure it has the right permissions. chmod 0775 "$LLDB_DIR" -rm -r "$TMP_DIR" -mkdir "$TMP_DIR" -export TMPDIR="$TMP_DIR" +rm -r $TMP_DIR +mkdir $TMP_DIR +export TMPDIR=$TMP_DIR -rm -r "$LOG_DIR" -mkdir "$LOG_DIR" +rm -r $LOG_DIR +mkdir $LOG_DIR # LLDB would create these files with more restrictive permissions than our umask above. Make sure # it doesn't get a chance. # "touch" does not exist on pre API-16 devices. This is a poor man's replacement -cat < /dev/null >"$LLDB_DEBUGSERVER_LOG_FILE" 2> "$PLATFORM_LOG_FILE" +cat "$LLDB_DEBUGSERVER_LOG_FILE" 2>"$PLATFORM_LOG_FILE" -cd "$TMP_DIR" # change cwd +cd $TMP_DIR # change cwd -exec "$BIN_DIR"/xa-${LLDB_ARCH}-lldb-server platform --server --listen "$LISTENER_SCHEME://$DOMAINSOCKET_DIR/$PLATFORM_SOCKET" --log-file "$PLATFORM_LOG_FILE" --log-channels "$LOG_CHANNELS" < /dev/null > "$LOG_DIR"/xa-${LLDB_ARCH}-platform-stdout.log 2>&1 +$BIN_DIR/lldb-server platform --server --listen $LISTENER_SCHEME://$DOMAINSOCKET_DIR/$PLATFORM_SOCKET --log-file "$PLATFORM_LOG_FILE" --log-channels "$LOG_CHANNELS" $LOG_DIR/platform-stdout.log 2>&1 diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeDebugger.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeDebugger.cs index 1a74bcdcff9..76bb6a3431e 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/NativeDebugger.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeDebugger.cs @@ -169,10 +169,7 @@ public bool Launch (string activityName) LogStatusLine ("Application PID", $"{context.applicationPID}"); - // (AdbRunner? debugServerRunner, TPL.Task<(bool success, string output)>? debugServerTask) = StartDebugServer (context); - // if (debugServerRunner == null || debugServerTask == null) { - // return false; - // } + StartDebugServer (context); // GenerateLldbScript (context); @@ -236,26 +233,42 @@ bool KillDebugServer (Context context) LogWarningLine ("Failed to kill previous instance of the debug server"); } - context.domainSocketDir = $"xa-{packageName}-0"; + context.domainSocketDir = $"/xa-{packageName}-0"; var rnd = new Random (); context.platformSocketName = $"xa-platform-{rnd.Next ()}.sock"; - var runner = CreateAdbRunner (); - runner.ProcessTimeout = TimeSpan.MaxValue; - - TPL.Task<(bool success, string output)> task = runner.RunAs ( + var args = new List { + "shell", + "run-as", packageName, context.debugServerScriptPath, context.appLldbBaseDir, // LLDB directory "unix-abstract", // Listener socket scheme (unix-abstract: virtual, not on the filesystem) context.domainSocketDir, // Directory where listener socket will be created context.platformSocketName, // name of the socket to create - "'lldb process:gdb-remote packets'", // LLDB log channels - context.arch // LLDB architecture - ); + "'\"lldb process:gdb-remote packets\"'", // LLDB log channels + context.arch + }; + + string command = String.Join (" ", args); + LogDebugLine ($"Launch command: adb {command}"); + + var runner = CreateAdbRunner (); + runner.ProcessTimeout = TimeSpan.MaxValue; + + // TPL.Task<(bool success, string output)> task = runner.RunAs ( + // packageName, + // context.debugServerScriptPath, + // context.appLldbBaseDir, // LLDB directory + // "unix-abstract", // Listener socket scheme (unix-abstract: virtual, not on the filesystem) + // context.domainSocketDir, // Directory where listener socket will be created + // context.platformSocketName, // name of the socket to create + // "'\"lldb process:gdb-remote packets\"'", // LLDB log channels + // context.arch // LLDB architecture + // ); - return (runner, task); + return (runner, null); } long GetDeviceProcessID (Context context, string processName, bool quiet = false) @@ -505,13 +518,13 @@ bool PushDebugServer (Context context) return false; } - debugServerPath = "/tmp/lldb-server"; if (!context.adb.CreateDirectoryAs (packageName, context.appLldbBinDir).Result.success) { LogErrorLine ($"Failed to create debug server destination directory on device, {context.appLldbBinDir}"); return false; } - string serverName = $"xa-{context.arch}-{Path.GetFileName (debugServerPath)}"; + //string serverName = $"xa-{context.arch}-{Path.GetFileName (debugServerPath)}"; + string serverName = Path.GetFileName (debugServerPath); context.debugServerPath = $"{context.appLldbBinDir}/{serverName}"; KillDebugServer (context); From 02f72bf7fbf50577ba662676bfe7aa8e28551665 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Mon, 21 Nov 2022 23:19:52 +0100 Subject: [PATCH 11/30] [WIP] Different direction --- .../Resources/lldb-debug-session.sh | 159 ++++++++++++++++ .../Utilities/AdbRunner.cs | 12 +- .../Utilities/ProcessRunner.cs | 20 +- .../Utilities/ProcessStandardStreamWrapper.cs | 10 +- .../Utilities/ToolRunner.cs | 19 +- .../Xamarin.Android.Build.Tasks.csproj | 3 + .../Debug.Session.Prep/AndroidDevice.cs | 171 ++++++++++++++++++ .../Debug.Session.Prep/AndroidNdk.cs | 12 ++ tools/debug-session-prep/Main.cs | 86 +++++++++ .../XamarinLoggingHelper.cs | 152 ++++++++++++++++ .../debug-session-prep.csproj | 22 +++ 11 files changed, 645 insertions(+), 21 deletions(-) create mode 100755 src/Xamarin.Android.Build.Tasks/Resources/lldb-debug-session.sh create mode 100644 tools/debug-session-prep/Debug.Session.Prep/AndroidDevice.cs create mode 100644 tools/debug-session-prep/Debug.Session.Prep/AndroidNdk.cs create mode 100644 tools/debug-session-prep/Main.cs create mode 100644 tools/debug-session-prep/Xamarin.Android.Utilities/XamarinLoggingHelper.cs create mode 100644 tools/debug-session-prep/debug-session-prep.csproj diff --git a/src/Xamarin.Android.Build.Tasks/Resources/lldb-debug-session.sh b/src/Xamarin.Android.Build.Tasks/Resources/lldb-debug-session.sh new file mode 100755 index 00000000000..b880d883e3d --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Resources/lldb-debug-session.sh @@ -0,0 +1,159 @@ +#!/bin/bash + +# Passed on command line +ADB_DEVICE="" +ARCH="" + +# Values set by the preparation task +SESSION_LOG_DIR="." +SESSION_STDERR_LOG_FILE="" + +# Detected during run +DEVICE_API_LEVEL="" +DEVICE_ABI="" +DEVICE_ARCH="" + +# Constants +ABI_PROPERTIES=( + # new properties + "ro.product.cpu.abilist" + + # old properties + "ro.product.cpu.abi" + "ro.product.cpu.abi2" +) + +function die() +{ + echo "$@" >&2 + exit 1 +} + +function die_with_log() +{ + local log_file="${1}" + + shift + + echo "$@" >&2 + if [ -f "${log_file}" ]; then + echo >&2 + cat "${log_file}" >&2 + echo >&2 + fi + + exit 1 +} + +function run_adb_nocheck() +{ + local args="" + + if [ -n "${ADB_DEVICE}" ]; then + args="-s ${ADB_DEVICE}" + fi + + COMMAND_OUTPUT="$(adb ${args} "$@")" +} + +function run_adb() +{ + local command_stderr="${SESSION_LOG_DIR}/adb-cmd-stderr.log" + + run_adb_nocheck "$@" 2> "${command_stderr}" + if [ $? -ne 0 ]; then + cat "${command_stderr}" >> "${SESSION_STDERR_LOG_FILE}" + die_with_log "${command_stderr}" "ADB command failed: " adb "${args}" "$@" + fi +} + +function adb_shell() +{ + run_adb shell "$@" +} + +function adb_shell_nocheck() +{ + run_adb_nocheck shell "$@" +} + +function adb_get_property() +{ + adb_shell getprop "$@" +} + +function get_api_level() +{ + adb_get_property ro.build.version.sdk + if [ $? -ne 0 ]; then + die "Unable to determine API level of the connected device" + fi + DEVICE_API_LEVEL="${COMMAND_OUTPUT}" +} + +function property_is_equal_to() +{ + local prop_name="${1}" + local expected_value="${2}" + + local prop_value + adb_get_property "${prop_name}" + prop_value=${COMMAND_OUTPUT} + + if [ -z "${prop_value}" -o "${prop_value}" != "${expected_value}" ]; then + false + return + fi + + true +} + +function warn_old_pixel_c() +{ + adb_shell_nocheck cat /proc/sys/kernel/yama/ptrace_scope "2> /dev/null" + if [ $? -ne 0 ]; then + true + return + fi + + local yama=${COMMAND_OUTPUT} + if [ -z "${yama}" -o "${yama}" == "0" ]; then + true + return + fi + + local prop_value + adb_get_property ro.build.product + prop_value=${COMMAND_OUTPUT} + + if ! property_is_equal_to "ro.build.product" "dragon"; then + true + return + fi + + if ! property_is_equal_to "ro.product.name" "ryu"; then + true + return + fi + + cat <&2 + +WARNING: The device uses Yama ptrace_scope to restrict debugging. ndk-gdb will + likely be unable to attach to a process. With root access, the restriction + can be lifted by writing 0 to /proc/sys/kernel/yama/ptrace_scope. Consider + upgrading your Pixel C to MXC89L or newer, where Yama is disabled. + +EOF +} + +if [ ! -d "${SESSION_LOG_DIR}" ]; then + install -d -m 755 "${SESSION_LOG_DIR}" +fi + +SESSION_STDERR_LOG_FILE="${SESSION_LOG_DIR}/adb-stderr.log" +rm -f "${SESSION_STDERR_LOG_FILE}" + +warn_old_pixel_c +get_api_level + +echo API: ${DEVICE_API_LEVEL} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs index 0aebdf227bf..36d28056e09 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs @@ -3,7 +3,11 @@ using System.IO; using System.Threading.Tasks; -using Microsoft.Build.Utilities; +#if NO_MSBUILD +using LoggerType = Xamarin.Android.Utilities.XamarinLoggingHelper; +#else // def NO_MSBUILD +using LoggerType = Microsoft.Build.Utilities.TaskLoggingHelper; +#endif // ndef NO_MSBUILD namespace Xamarin.Android.Tasks { @@ -13,7 +17,7 @@ class AdbOutputSink : ToolOutputSink { public Action? LineCallback { get; set; } - public AdbOutputSink (TaskLoggingHelper logger) + public AdbOutputSink (LoggerType logger) : base (logger) {} @@ -28,7 +32,7 @@ public override void WriteLine (string? value) public int ExitCode { get; private set; } - public AdbRunner (TaskLoggingHelper logger, string adbPath, string? deviceSerial = null) + public AdbRunner (LoggerType logger, string adbPath, string? deviceSerial = null) : base (logger, adbPath) { if (!String.IsNullOrEmpty (deviceSerial)) { @@ -184,7 +188,7 @@ async Task RunAdb (ProcessRunner runner, bool setupOutputSink = true, bool ProcessRunner CreateAdbRunner () => CreateProcessRunner (initialParams); - protected override TextWriter CreateLogSink (TaskLoggingHelper logger) + protected override TextWriter CreateLogSink (LoggerType logger) { return new AdbOutputSink (logger); } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ProcessRunner.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ProcessRunner.cs index 28a2962218d..f7e45f8b822 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ProcessRunner.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ProcessRunner.cs @@ -5,9 +5,15 @@ using System.Text; using System.Threading; +#if NO_MSBUILD +using LoggerType = Xamarin.Android.Utilities.XamarinLoggingHelper; +#else using Microsoft.Android.Build.Tasks; using Microsoft.Build.Utilities; +using LoggerType = Microsoft.Build.Utilities.TaskLoggingHelper; +#endif + namespace Xamarin.Android.Tasks { class ProcessRunner @@ -45,7 +51,7 @@ public WriterGuard (TextWriter writer) Dictionary? guardCache; bool defaultStdoutEchoWrapperAdded; ProcessStandardStreamWrapper? defaultStderrEchoWrapper; - TaskLoggingHelper log; + LoggerType log; public string Command => command; @@ -84,11 +90,11 @@ public string FullCommandLine { public string? WorkingDirectory { get; set; } public Action? StartInfoCallback { get; set; } - public ProcessRunner (TaskLoggingHelper logger, string command, params string?[] arguments) + public ProcessRunner (LoggerType logger, string command, params string?[] arguments) : this (logger, command, false, arguments) {} - public ProcessRunner (TaskLoggingHelper logger, string command, bool ignoreEmptyArguments, params string?[] arguments) + public ProcessRunner (LoggerType logger, string command, bool ignoreEmptyArguments, params string?[] arguments) { if (String.IsNullOrEmpty (command)) { throw new ArgumentException ("must not be null or empty", nameof (command)); @@ -112,20 +118,20 @@ public ProcessRunner ClearOutputSinks () return this; } - public ProcessRunner AddArguments (params string?[] arguments) + public ProcessRunner AddArguments (params string?[]? arguments) { return AddArguments (true, arguments); } - public ProcessRunner AddArguments (bool ignoreEmptyArguments, params string?[] arguments) + public ProcessRunner AddArguments (bool ignoreEmptyArguments, params string?[]? arguments) { AddArgumentsInternal (ignoreEmptyArguments, arguments); return this; } - void AddArgumentsInternal (bool ignoreEmptyArguments, params string?[] arguments) + void AddArgumentsInternal (bool ignoreEmptyArguments, params string?[]? arguments) { - if (arguments == null) { + if (arguments == null || arguments.Length == 0) { return; } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ProcessStandardStreamWrapper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ProcessStandardStreamWrapper.cs index 613a7ab2ecc..907052dc473 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ProcessStandardStreamWrapper.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ProcessStandardStreamWrapper.cs @@ -2,9 +2,15 @@ using System.IO; using System.Text; +#if NO_MSBUILD +using LoggerType = Xamarin.Android.Utilities.XamarinLoggingHelper; +#else using Microsoft.Android.Build.Tasks; using Microsoft.Build.Utilities; +using LoggerType = Microsoft.Build.Utilities.TaskLoggingHelper; +#endif + namespace Xamarin.Android.Tasks { class ProcessStandardStreamWrapper : TextWriter @@ -18,14 +24,14 @@ public enum LogLevel Debug, } - TaskLoggingHelper log; + LoggerType log; public LogLevel LoggingLevel { get; set; } = LogLevel.Debug; public string? LogPrefix { get; set; } public override Encoding Encoding => Encoding.Default; - public ProcessStandardStreamWrapper (TaskLoggingHelper logger) + public ProcessStandardStreamWrapper (LoggerType logger) { log = logger; } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ToolRunner.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ToolRunner.cs index 4fe86dbe01d..86e9b40d71f 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ToolRunner.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ToolRunner.cs @@ -3,7 +3,11 @@ using System.Text; using System.Threading.Tasks; -using Microsoft.Build.Utilities; +#if NO_MSBUILD +using LoggerType = Xamarin.Android.Utilities.XamarinLoggingHelper; +#else // def NO_MSBUILD +using LoggerType = Microsoft.Build.Utilities.TaskLoggingHelper; +#endif // ndef NO_MSBUILD using TPLTask = System.Threading.Tasks.Task; @@ -13,11 +17,11 @@ abstract class ToolRunner { protected abstract class ToolOutputSink : TextWriter { - TaskLoggingHelper log; + LoggerType log; public override Encoding Encoding => Encoding.Default; - protected ToolOutputSink (TaskLoggingHelper logger) + protected ToolOutputSink (LoggerType logger) { log = logger; } @@ -30,15 +34,14 @@ public override void WriteLine (string? value) static readonly TimeSpan DefaultProcessTimeout = TimeSpan.FromMinutes (15); - protected TaskLoggingHelper Logger { get; } - + protected LoggerType Logger { get; } public string ToolPath { get; } public bool EchoCmdAndArguments { get; set; } = true; public bool EchoStandardError { get; set; } = true; public bool EchoStandardOutput { get; set; } public virtual TimeSpan ProcessTimeout { get; set; } = DefaultProcessTimeout; - protected ToolRunner (TaskLoggingHelper logger, string toolPath) + protected ToolRunner (LoggerType logger, string toolPath) { if (String.IsNullOrEmpty (toolPath)) { throw new ArgumentException ("must not be null or empty", nameof (toolPath)); @@ -48,7 +51,7 @@ protected ToolRunner (TaskLoggingHelper logger, string toolPath) ToolPath = toolPath; } - protected virtual ProcessRunner CreateProcessRunner (params string[] initialParams) + protected virtual ProcessRunner CreateProcessRunner (params string?[]? initialParams) { var runner = new ProcessRunner (Logger, ToolPath) { ProcessTimeout = ProcessTimeout, @@ -78,7 +81,7 @@ protected TextWriter SetupOutputSink (ProcessRunner runner, bool ignoreStderr = return ret; } - protected virtual TextWriter CreateLogSink (TaskLoggingHelper logger) + protected virtual TextWriter CreateLogSink (LoggerType logger) { throw new NotSupportedException ("Child class must implement this method if it uses output sinks"); } diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.csproj b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.csproj index a3116f3fdc4..9fb6982f52a 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.csproj +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.csproj @@ -410,6 +410,9 @@ xa_start_lldb_server.sh + + lldb-debug-session.sh + diff --git a/tools/debug-session-prep/Debug.Session.Prep/AndroidDevice.cs b/tools/debug-session-prep/Debug.Session.Prep/AndroidDevice.cs new file mode 100644 index 00000000000..efc81b8ba76 --- /dev/null +++ b/tools/debug-session-prep/Debug.Session.Prep/AndroidDevice.cs @@ -0,0 +1,171 @@ +using System; + +using Xamarin.Android.Tasks; +using Xamarin.Android.Utilities; + +namespace Xamarin.Debug.Session.Prep; + +class AndroidDevice +{ + static readonly string[] abiProperties = { + // new properties + "ro.product.cpu.abilist", + + // old properties + "ro.product.cpu.abi", + "ro.product.cpu.abi2", + }; + + string packageName; + string adbPath; + string[] supportedAbis; + int apiLevel = -1; + string? appDataDir; + string? appLldbBaseDir; + string? appLldbBinDir; + string? appLldbLogDir; + string? abi; + string? arch; + bool appIs64Bit; + + XamarinLoggingHelper log; + AdbRunner adb; + + public AndroidDevice (XamarinLoggingHelper log, string adbPath, string packageName, string[] supportedAbis, string? adbTargetDevice = null) + { + this.adbPath = adbPath; + this.log = log; + this.packageName = packageName; + this.supportedAbis = supportedAbis; + + adb = new AdbRunner (log, adbPath, adbTargetDevice); + } + + public bool GatherInfo () + { + (bool success, string output) = adb.GetPropertyValue ("ro.build.version.sdk").Result; + if (!success || String.IsNullOrEmpty (output) || !Int32.TryParse (output, out apiLevel)) { + log.ErrorLine ("Unable to determine connected device's API level"); + return false; + } + + // Warn on old Pixel C firmware (b/29381985). Newer devices may have Yama + // enabled but still work with ndk-gdb (b/19277529). + (success, output) = adb.Shell ("cat", "/proc/sys/kernel/yama/ptrace_scope", "2>/dev/null").Result; + if (success && + YamaOK (output.Trim ()) && + PropertyIsEqualTo (adb.GetPropertyValue ("ro.build.product").Result, "dragon") && + PropertyIsEqualTo (adb.GetPropertyValue ("ro.product.name").Result, "ryu") + ) { + log.WarningLine ("WARNING: The device uses Yama ptrace_scope to restrict debugging. ndk-gdb will"); + log.WarningLine (" likely be unable to attach to a process. With root access, the restriction"); + log.WarningLine (" can be lifted by writing 0 to /proc/sys/kernel/yama/ptrace_scope. Consider"); + log.WarningLine (" upgrading your Pixel C to MXC89L or newer, where Yama is disabled."); + log.WarningLine (); + } + + if (!DetermineArchitectureAndABI ()) { + return false; + } + + if (!DetermineAppDataDirectory ()) { + return false; + } + + return true; + + bool YamaOK (string output) + { + return !String.IsNullOrEmpty (output) && String.Compare ("0", output, StringComparison.Ordinal) != 0; + } + + bool PropertyIsEqualTo ((bool haveProperty, string value) result, string expected) + { + return + result.haveProperty && + !String.IsNullOrEmpty (result.value) && + String.Compare (result.value, expected, StringComparison.Ordinal) == 0; + } + } + + bool DetermineAppDataDirectory () + { + (bool success, string output) = adb.GetAppDataDirectory (packageName).Result; + if (!success) { + log.ErrorLine ($"Unable to determine data directory for package '{packageName}'"); + return false; + } + + appDataDir = output.Trim (); + log.StatusLine ($"Application data directory on device", appDataDir); + log.MessageLine (); + + appLldbBaseDir = $"{appDataDir}/lldb"; + appLldbBinDir = $"{appLldbBaseDir}/bin"; + appLldbLogDir = $"{appLldbBaseDir}/log"; + + // Applications with minSdkVersion >= 24 will have their data directories + // created with rwx------ permissions, preventing adbd from forwarding to + // the gdbserver socket. To be safe, if we're on a device >= 24, always + // chmod the directory. + if (apiLevel >= 24) { + (success, output) = adb.RunAs (packageName, "/system/bin/chmod", "a+x", appDataDir).Result; + if (!success) { + log.ErrorLine ("Failed to make application data directory world executable"); + return false; + } + } + + return true; + } + + bool DetermineArchitectureAndABI () + { + string[]? deviceABIs = null; + + foreach (string prop in abiProperties) { + (bool success, string value) = adb.GetPropertyValue (prop).Result; + if (!success) { + continue; + } + + deviceABIs = value.Split (','); + break; + } + + if (deviceABIs == null || deviceABIs.Length == 0) { + log.ErrorLine ("Unable to determine device ABI"); + return false; + } + + LogABIs ("Application", supportedAbis); + LogABIs (" Device", deviceABIs); + + foreach (string deviceABI in deviceABIs) { + foreach (string appABI in supportedAbis) { + if (String.Compare (appABI, deviceABI, StringComparison.OrdinalIgnoreCase) == 0) { + abi = deviceABI; + arch = abi switch { + "armeabi" => "arm", + "armeabi-v7a" => "arm", + "arm64-v8a" => "arm64", + _ => abi, + }; + + log.StatusLine ($" Selected ABI", $"{abi} (architecture: {arch})"); + + appIs64Bit = abi.IndexOf ("64", StringComparison.Ordinal) >= 0; + return true; + } + } + } + + log.ErrorLine ($"Application cannot run on the selected device: no matching ABI found"); + return false; + + void LogABIs (string which, string[] abis) + { + log.StatusLine ($"{which} ABIs", String.Join (", ", abis)); + } + } +} diff --git a/tools/debug-session-prep/Debug.Session.Prep/AndroidNdk.cs b/tools/debug-session-prep/Debug.Session.Prep/AndroidNdk.cs new file mode 100644 index 00000000000..2ce8156c6a0 --- /dev/null +++ b/tools/debug-session-prep/Debug.Session.Prep/AndroidNdk.cs @@ -0,0 +1,12 @@ +namespace Xamarin.Debug.Session.Prep; + +class AndroidNdk +{ + // We want the shell/batch scripts first, since they set up Python environment for the debugger + static readonly string[] lldbNames = { + "lldb.sh", + "lldb", + "lldb.cmd", + "lldb.exe", + }; +} diff --git a/tools/debug-session-prep/Main.cs b/tools/debug-session-prep/Main.cs new file mode 100644 index 00000000000..6e7d023cd02 --- /dev/null +++ b/tools/debug-session-prep/Main.cs @@ -0,0 +1,86 @@ +using System; +using System.Collections.Generic; + +using Mono.Options; +using Xamarin.Android.Utilities; + +namespace Xamarin.Debug.Session.Prep; + +class App +{ + sealed class ParsedOptions + { + public string? AdbPath = "adb"; + public string? PackageName; + public string[]? SupportedABIs; + public string? TargetDevice; + public bool ShowHelp; + public bool Verbose; + } + + static int Main (string[] args) + { + bool haveOptionErrors = false; + var log = new XamarinLoggingHelper (); + var parsedOptions = new ParsedOptions (); + + var opts = new OptionSet { + "Usage: debug-session-prep [REQUIRED_OPTIONS] [OPTIONS]", + "", + "REQUIRED_OPTIONS are:", + { "p|package-name=", "name of the application package", v => parsedOptions.PackageName = EnsureNonEmptyString (log, "p|package-name", v, ref haveOptionErrors) }, + { "s|supported-abis=", "comma-separated list of ABIs the application supports", v => parsedOptions.SupportedABIs = EnsureSupportedABIs (log, "s|supported-abis", v, ref haveOptionErrors) }, + "", + "OPTIONS are:", + { "a|adb=", "{PATH} to adb to use for this session", v => parsedOptions.AdbPath = EnsureNonEmptyString (log, "a|adb", v, ref haveOptionErrors) }, + { "d|device=", "ID of {DEVICE} to target for this session", v => parsedOptions.TargetDevice = EnsureNonEmptyString (log, "d|device", v, ref haveOptionErrors) }, + "", + { "v|verbose", "Show debug messages", v => parsedOptions.Verbose = true }, + { "h|help|?", "Show this help screen", v => parsedOptions.ShowHelp = true }, + }; + + List rest = opts.Parse (args); + + if (parsedOptions.ShowHelp) { + opts.WriteOptionDescriptions (Console.Out); + return 0; + } + + if (haveOptionErrors) { + return 1; + } + + return 0; + } + + static string[]? EnsureSupportedABIs (XamarinLoggingHelper log, string paramName, string? value, ref bool haveOptionErrors) + { + string? abis = EnsureNonEmptyString (log, paramName, value, ref haveOptionErrors); + if (abis == null) { + return null; + } + + var list = new List (); + foreach (string s in abis.Split (',')) { + string? abi = s?.Trim (); + if (String.IsNullOrEmpty (abi)) { + continue; + } + + list.Add (abi); + } + + return list.ToArray (); + } + + static string? EnsureNonEmptyString (XamarinLoggingHelper log, string paramName, string? value, ref bool haveOptionErrors) + { + if (String.IsNullOrEmpty (value)) { + haveOptionErrors = true; + log.ErrorLine ($"Parameter '{paramName}' requires a non-empty string as its value"); + return null; + } + + return value; + } +} diff --git a/tools/debug-session-prep/Xamarin.Android.Utilities/XamarinLoggingHelper.cs b/tools/debug-session-prep/Xamarin.Android.Utilities/XamarinLoggingHelper.cs new file mode 100644 index 00000000000..383272cc139 --- /dev/null +++ b/tools/debug-session-prep/Xamarin.Android.Utilities/XamarinLoggingHelper.cs @@ -0,0 +1,152 @@ +using System; +using System.IO; + +namespace Xamarin.Android.Utilities; + +enum LogLevel +{ + Error, + Warning, + Info, + Message, + Debug +} + +class XamarinLoggingHelper +{ + static readonly object consoleLock = new object (); + + public const ConsoleColor ErrorColor = ConsoleColor.Red; + public const ConsoleColor DebugColor = ConsoleColor.DarkGray; + public const ConsoleColor InfoColor = ConsoleColor.Green; + public const ConsoleColor MessageColor = ConsoleColor.Gray; + public const ConsoleColor WarningColor = ConsoleColor.Yellow; + public const ConsoleColor StatusLabel = ConsoleColor.Cyan; + public const ConsoleColor StatusText = ConsoleColor.White; + + public void Message (string? message) + { + Log (LogLevel.Message, message); + } + + public void MessageLine (string? message = null) + { + Message ($"{message ?? String.Empty}{Environment.NewLine}"); + } + + public void Warning (string? message) + { + Log (LogLevel.Warning, message); + } + + public void WarningLine (string? message = null) + { + Warning ($"{message ?? String.Empty}{Environment.NewLine}"); + } + + public void Error (string? message) + { + Log (LogLevel.Error, message); + } + + public void ErrorLine (string? message = null) + { + Error ($"{message ?? String.Empty}{Environment.NewLine}"); + } + + public void Info (string? message) + { + Log (LogLevel.Info, message); + } + + public void InfoLine (string? message = null) + { + Info ($"{message ?? String.Empty}{Environment.NewLine}"); + } + + public void Debug (string? message) + { + Log (LogLevel.Debug, message); + } + + public void DebugLine (string? message = null) + { + Debug ($"{message ?? String.Empty}{Environment.NewLine}"); + } + + public void StatusLine (string label, string text) + { + Log (LogLevel.Info, $"{label}: ", StatusLabel); + Log (LogLevel.Info, $"{text}{Environment.NewLine}", StatusText); + } + + public void Log (LogLevel level, string? message) + { + Log (level, message, ForegroundColor (level)); + } + + public void Log (LogLevel level, string? message, ConsoleColor color) + { + TextWriter writer = level == LogLevel.Error ? Console.Error : Console.Out; + message = message ?? String.Empty; + + ConsoleColor fg = ConsoleColor.Gray; + try { + lock (consoleLock) { + fg = Console.ForegroundColor; + Console.ForegroundColor = color; + } + + writer.Write (message); + } finally { + Console.ForegroundColor = fg; + } + } + + ConsoleColor ForegroundColor (LogLevel level) => level switch { + LogLevel.Error => ErrorColor, + LogLevel.Warning => WarningColor, + LogLevel.Info => InfoColor, + LogLevel.Debug => DebugColor, + LogLevel.Message => MessageColor, + _ => MessageColor, + }; + +#region MSBuild compatibility methods + public void LogDebugMessage (string message, params object[] messageArgs) + { + if (messageArgs == null || messageArgs.Length == 0) { + DebugLine (message); + } else { + DebugLine (String.Format (message, messageArgs)); + } + } + + public void LogError (string message, params object[] messageArgs) + { + if (messageArgs == null || messageArgs.Length == 0) { + ErrorLine (message); + } else { + ErrorLine (String.Format (message, messageArgs)); + } + } + + public void LogMessage (string message, params object[] messageArgs) + { + if (messageArgs == null || messageArgs.Length == 0) { + MessageLine (message); + } else { + MessageLine (String.Format (message, messageArgs)); + } + } + + public void LogWarning (string message, params object[] messageArgs) + { + if (messageArgs == null || messageArgs.Length == 0) { + WarningLine (message); + } else { + WarningLine (String.Format (message, messageArgs)); + } + } +#endregion +} diff --git a/tools/debug-session-prep/debug-session-prep.csproj b/tools/debug-session-prep/debug-session-prep.csproj new file mode 100644 index 00000000000..851db034bce --- /dev/null +++ b/tools/debug-session-prep/debug-session-prep.csproj @@ -0,0 +1,22 @@ + + + Exe + net7.0 + debug_session_prep + enable + NO_MSBUILD + + + + + + + + + + + + + + + From 583cc394d993067b1241761bfb338442f3aa796c Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 22 Nov 2022 23:29:06 +0100 Subject: [PATCH 12/30] [WIP] Code migration to standalone app --- .../Utilities/AdbRunner.cs | 4 +- .../Utilities/ToolRunner.cs | 14 ++++++- .../Debug.Session.Prep/AndroidDevice.cs | 41 +++++++++++++------ tools/debug-session-prep/Main.cs | 34 ++++++++++++--- .../XamarinLoggingHelper.cs | 10 +++++ 5 files changed, 83 insertions(+), 20 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs index 36d28056e09..deecbc03546 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs @@ -19,7 +19,9 @@ class AdbOutputSink : ToolOutputSink public AdbOutputSink (LoggerType logger) : base (logger) - {} + { + LogLinePrefix = "adb"; + } public override void WriteLine (string? value) { diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ToolRunner.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ToolRunner.cs index 86e9b40d71f..e1f2de0dd12 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ToolRunner.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ToolRunner.cs @@ -6,6 +6,8 @@ #if NO_MSBUILD using LoggerType = Xamarin.Android.Utilities.XamarinLoggingHelper; #else // def NO_MSBUILD +using Microsoft.Android.Build.Tasks; + using LoggerType = Microsoft.Build.Utilities.TaskLoggingHelper; #endif // ndef NO_MSBUILD @@ -17,6 +19,8 @@ abstract class ToolRunner { protected abstract class ToolOutputSink : TextWriter { + protected string LogLinePrefix { get; set; } = String.Empty; + LoggerType log; public override Encoding Encoding => Encoding.Default; @@ -28,7 +32,15 @@ protected ToolOutputSink (LoggerType logger) public override void WriteLine (string? value) { - log.LogMessage (value ?? String.Empty); + string message; + + if (!String.IsNullOrEmpty (LogLinePrefix)) { + message = $"{LogLinePrefix}> {value ?? String.Empty}"; + } else { + message = value ?? String.Empty; + } + + log.LogDebugMessage (message); } } diff --git a/tools/debug-session-prep/Debug.Session.Prep/AndroidDevice.cs b/tools/debug-session-prep/Debug.Session.Prep/AndroidDevice.cs index efc81b8ba76..d2b8e41b8c6 100644 --- a/tools/debug-session-prep/Debug.Session.Prep/AndroidDevice.cs +++ b/tools/debug-session-prep/Debug.Session.Prep/AndroidDevice.cs @@ -16,6 +16,11 @@ class AndroidDevice "ro.product.cpu.abi2", }; + static readonly string[] serialNumberProperties = { + "ro.serialno", + "ro.boot.serialno", + }; + string packageName; string adbPath; string[] supportedAbis; @@ -26,6 +31,7 @@ class AndroidDevice string? appLldbLogDir; string? abi; string? arch; + string? serialNumber; bool appIs64Bit; XamarinLoggingHelper log; @@ -72,6 +78,13 @@ public bool GatherInfo () return false; } + serialNumber = GetFirstFoundPropertyValue (serialNumberProperties); + if (String.IsNullOrEmpty (serialNumber)) { + log.WarningLine ("Unable to determine device serial number"); + } else { + log.StatusLine ($"Device serial number", serialNumber); + } + return true; bool YamaOK (string output) @@ -98,7 +111,6 @@ bool DetermineAppDataDirectory () appDataDir = output.Trim (); log.StatusLine ($"Application data directory on device", appDataDir); - log.MessageLine (); appLldbBaseDir = $"{appDataDir}/lldb"; appLldbBinDir = $"{appLldbBaseDir}/bin"; @@ -121,17 +133,8 @@ bool DetermineAppDataDirectory () bool DetermineArchitectureAndABI () { - string[]? deviceABIs = null; - - foreach (string prop in abiProperties) { - (bool success, string value) = adb.GetPropertyValue (prop).Result; - if (!success) { - continue; - } - - deviceABIs = value.Split (','); - break; - } + string? propValue = GetFirstFoundPropertyValue (abiProperties); + string[]? deviceABIs = propValue?.Split (','); if (deviceABIs == null || deviceABIs.Length == 0) { log.ErrorLine ("Unable to determine device ABI"); @@ -168,4 +171,18 @@ void LogABIs (string which, string[] abis) log.StatusLine ($"{which} ABIs", String.Join (", ", abis)); } } + + string? GetFirstFoundPropertyValue (string[] propertyNames) + { + foreach (string prop in propertyNames) { + (bool success, string value) = adb.GetPropertyValue (prop).Result; + if (!success) { + continue; + } + + return value; + } + + return null; + } } diff --git a/tools/debug-session-prep/Main.cs b/tools/debug-session-prep/Main.cs index 6e7d023cd02..ce0ddd9b58c 100644 --- a/tools/debug-session-prep/Main.cs +++ b/tools/debug-session-prep/Main.cs @@ -8,14 +8,16 @@ namespace Xamarin.Debug.Session.Prep; class App { + const string DefaultAdbPath = "adb"; + sealed class ParsedOptions { - public string? AdbPath = "adb"; + public string? AdbPath; public string? PackageName; public string[]? SupportedABIs; public string? TargetDevice; public bool ShowHelp; - public bool Verbose; + public bool Verbose = true; // TODO: remove the default once development is done } static int Main (string[] args) @@ -28,12 +30,12 @@ static int Main (string[] args) "Usage: debug-session-prep [REQUIRED_OPTIONS] [OPTIONS]", "", "REQUIRED_OPTIONS are:", - { "p|package-name=", "name of the application package", v => parsedOptions.PackageName = EnsureNonEmptyString (log, "p|package-name", v, ref haveOptionErrors) }, - { "s|supported-abis=", "comma-separated list of ABIs the application supports", v => parsedOptions.SupportedABIs = EnsureSupportedABIs (log, "s|supported-abis", v, ref haveOptionErrors) }, + { "p|package-name=", "name of the application package", v => parsedOptions.PackageName = EnsureNonEmptyString (log, "-p|--package-name", v, ref haveOptionErrors) }, + { "s|supported-abis=", "comma-separated list of ABIs the application supports", v => parsedOptions.SupportedABIs = EnsureSupportedABIs (log, "-s|--supported-abis", v, ref haveOptionErrors) }, "", "OPTIONS are:", - { "a|adb=", "{PATH} to adb to use for this session", v => parsedOptions.AdbPath = EnsureNonEmptyString (log, "a|adb", v, ref haveOptionErrors) }, - { "d|device=", "ID of {DEVICE} to target for this session", v => parsedOptions.TargetDevice = EnsureNonEmptyString (log, "d|device", v, ref haveOptionErrors) }, + { "a|adb=", "{PATH} to adb to use for this session", v => parsedOptions.AdbPath = EnsureNonEmptyString (log, "-a|--adb", v, ref haveOptionErrors) }, + { "d|device=", "ID of {DEVICE} to target for this session", v => parsedOptions.TargetDevice = EnsureNonEmptyString (log, "-d|--device", v, ref haveOptionErrors) }, "", { "v|verbose", "Show debug messages", v => parsedOptions.Verbose = true }, { "h|help|?", "Show this help screen", v => parsedOptions.ShowHelp = true }, @@ -50,6 +52,26 @@ static int Main (string[] args) return 1; } + bool missingRequiredOptions = false; + if (parsedOptions.SupportedABIs == null || parsedOptions.SupportedABIs.Length == 0) { + log.ErrorLine ("The '-s|--supported-abis' option must be used to provide a non-empty list of Android ABIs supported by the application"); + missingRequiredOptions = true; + } + + if (String.IsNullOrEmpty (parsedOptions.PackageName)) { + log.ErrorLine ("The '-p|--package-name' option must be used to provide non-empty application package name"); + missingRequiredOptions = true; + } + + if (missingRequiredOptions) { + return 1; + } + + var device = new AndroidDevice (log, parsedOptions.AdbPath ?? DefaultAdbPath, parsedOptions.PackageName!, parsedOptions.SupportedABIs!, parsedOptions.TargetDevice); + if (!device.GatherInfo ()) { + return 1; + } + return 0; } diff --git a/tools/debug-session-prep/Xamarin.Android.Utilities/XamarinLoggingHelper.cs b/tools/debug-session-prep/Xamarin.Android.Utilities/XamarinLoggingHelper.cs index 383272cc139..af31a9e824b 100644 --- a/tools/debug-session-prep/Xamarin.Android.Utilities/XamarinLoggingHelper.cs +++ b/tools/debug-session-prep/Xamarin.Android.Utilities/XamarinLoggingHelper.cs @@ -24,6 +24,8 @@ class XamarinLoggingHelper public const ConsoleColor StatusLabel = ConsoleColor.Cyan; public const ConsoleColor StatusText = ConsoleColor.White; + public bool Verbose { get; set; } + public void Message (string? message) { Log (LogLevel.Message, message); @@ -82,11 +84,19 @@ public void StatusLine (string label, string text) public void Log (LogLevel level, string? message) { + if (!Verbose && level == LogLevel.Debug) { + return; + } + Log (level, message, ForegroundColor (level)); } public void Log (LogLevel level, string? message, ConsoleColor color) { + if (!Verbose && level == LogLevel.Debug) { + return; + } + TextWriter writer = level == LogLevel.Error ? Console.Error : Console.Out; message = message ?? String.Empty; From b5346a8ac4a7cf43c78234e855bd9252b3ceedab Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 23 Nov 2022 22:43:39 +0100 Subject: [PATCH 13/30] [WIP] Further progress --- .../Utilities/AdbRunner.cs | 2 +- .../Utilities/NdkHelper.cs | 10 +- .../Xamarin.Android.Build.Tasks.csproj | 6 - .../Debug.Session.Prep/AndroidDevice.cs | 175 +++++++++++++++++- .../Debug.Session.Prep/AndroidNdk.cs | 87 +++++++++ .../Debug.Session.Prep/Utilities.cs | 23 +++ tools/debug-session-prep/Main.cs | 67 ++++++- .../Resources/lldb-debug-session.sh | 0 .../Resources/xa_start_lldb_server.sh | 0 .../debug-session-prep.csproj | 14 +- 10 files changed, 366 insertions(+), 18 deletions(-) create mode 100644 tools/debug-session-prep/Debug.Session.Prep/Utilities.cs rename {src/Xamarin.Android.Build.Tasks => tools/debug-session-prep}/Resources/lldb-debug-session.sh (100%) rename {src/Xamarin.Android.Build.Tasks => tools/debug-session-prep}/Resources/xa_start_lldb_server.sh (100%) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs index deecbc03546..b9b422a37c5 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs @@ -93,7 +93,7 @@ public async Task Push (string localPath, string remotePath) public async Task<(bool success, string output)> GetAppDataDirectory (string packageName) { - return await RunAs (packageName, "/system/bin/sh", "-c", "pwd", "2>/dev/null"); + return await RunAs (packageName, "/system/bin/sh", "-c", "pwd"); } public async Task<(bool success, string output)> Shell (string command, List args) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NdkHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NdkHelper.cs index c8bda862396..f5062abeea5 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/NdkHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NdkHelper.cs @@ -1,7 +1,7 @@ using System; using System.IO; -using Xamarin.Android.Tools; +using System.Runtime.InteropServices; namespace Xamarin.Android.Tasks { @@ -16,12 +16,14 @@ static class NdkHelper static NdkHelper () { string os; - if (OS.IsWindows) { + if (RuntimeInformation.IsOSPlatform (OSPlatform.Windows)) { os = "windows"; - } else if (OS.IsMac) { + } else if (RuntimeInformation.IsOSPlatform (OSPlatform.OSX)) { os = "darwin"; - } else { + } else if (RuntimeInformation.IsOSPlatform (OSPlatform.Linux)) { os = "linux"; + } else { + throw new InvalidOperationException ($"Unsupported OS {RuntimeInformation.OSDescription}"); } // We care only about the latest NDK versions, they have only x86_64 versions. We'll need to revisit the code once diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.csproj b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.csproj index 9fb6982f52a..ee9e29a5c39 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.csproj +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.csproj @@ -407,12 +407,6 @@ JavaInteropTypeManager.java - - xa_start_lldb_server.sh - - - lldb-debug-session.sh - diff --git a/tools/debug-session-prep/Debug.Session.Prep/AndroidDevice.cs b/tools/debug-session-prep/Debug.Session.Prep/AndroidDevice.cs index d2b8e41b8c6..92bd62d450d 100644 --- a/tools/debug-session-prep/Debug.Session.Prep/AndroidDevice.cs +++ b/tools/debug-session-prep/Debug.Session.Prep/AndroidDevice.cs @@ -1,4 +1,6 @@ using System; +using System.IO; +using System.Text; using Xamarin.Android.Tasks; using Xamarin.Android.Utilities; @@ -7,6 +9,8 @@ namespace Xamarin.Debug.Session.Prep; class AndroidDevice { + const string ServerLauncherScriptName = "xa_start_lldb_server.sh"; + static readonly string[] abiProperties = { // new properties "ro.product.cpu.abilist", @@ -21,6 +25,8 @@ class AndroidDevice "ro.boot.serialno", }; + static readonly UTF8Encoding UTF8NoBOM = new UTF8Encoding (false); + string packageName; string adbPath; string[] supportedAbis; @@ -33,20 +39,29 @@ class AndroidDevice string? arch; string? serialNumber; bool appIs64Bit; + string? deviceLdd; + string? deviceDebugServerPath; + string? deviceDebugServerScriptPath; + string outputDir; XamarinLoggingHelper log; AdbRunner adb; + AndroidNdk ndk; - public AndroidDevice (XamarinLoggingHelper log, string adbPath, string packageName, string[] supportedAbis, string? adbTargetDevice = null) + public AndroidDevice (XamarinLoggingHelper log, AndroidNdk ndk, string outputDir, string adbPath, string packageName, string[] supportedAbis, string? adbTargetDevice = null) { this.adbPath = adbPath; this.log = log; this.packageName = packageName; this.supportedAbis = supportedAbis; + this.ndk = ndk; + this.outputDir = outputDir; adb = new AdbRunner (log, adbPath, adbTargetDevice); } + // TODO: implement manual error checking on API 21, since `adb` won't ever return any error code other than 0 - we need to look at the output of any command to determine + // whether or not it was successful. Ugh. public bool GatherInfo () { (bool success, string output) = adb.GetPropertyValue ("ro.build.version.sdk").Result; @@ -85,6 +100,14 @@ public bool GatherInfo () log.StatusLine ($"Device serial number", serialNumber); } + if (!DetectTools ()) { + return false; + } + + if (!PushDebugServer ()) { + return false; + } + return true; bool YamaOK (string output) @@ -101,10 +124,143 @@ bool PropertyIsEqualTo ((bool haveProperty, string value) result, string expecte } } + bool PushDebugServer () + { + string? debugServerPath = ndk.GetDebugServerPath (abi!); + if (String.IsNullOrEmpty (debugServerPath)) { + return false; + } + + if (!adb.CreateDirectoryAs (packageName, appLldbBinDir!).Result.success) { + log.ErrorLine ($"Failed to create debug server destination directory on device, {appLldbBinDir}"); + return false; + } + + //string serverName = $"xa-{context.arch}-{Path.GetFileName (debugServerPath)}"; + string serverName = Path.GetFileName (debugServerPath); + deviceDebugServerPath = $"{appLldbBinDir}/{serverName}"; + + KillDebugServer (deviceDebugServerPath); + + // Always push the server binary, as we don't know what version might already be there + if (!PushServerExecutable (debugServerPath, deviceDebugServerPath)) { + return false; + } + log.StatusLine ("Debug server path on device", deviceDebugServerPath); + + string? launcherScript = Utilities.ReadManifestResource (log, ServerLauncherScriptName); + if (String.IsNullOrEmpty (launcherScript)) { + return false; + } + + string launcherScriptPath = Path.Combine (outputDir, ServerLauncherScriptName); + Directory.CreateDirectory (Path.GetDirectoryName (launcherScriptPath)!); + File.WriteAllText (launcherScriptPath, launcherScript, UTF8NoBOM); + + deviceDebugServerScriptPath = $"{appLldbBinDir}/{Path.GetFileName (launcherScriptPath)}"; + if (!PushServerExecutable (launcherScriptPath, deviceDebugServerScriptPath)) { + return false; + } + log.StatusLine ("Debug server launcher script path on device", deviceDebugServerScriptPath); + log.MessageLine (); + + return true; + } + + bool PushServerExecutable (string hostSource, string deviceDestination) + { + string executableName = Path.GetFileName (deviceDestination); + + // Always push the executable, as we don't know what version might already be there + log.DebugLine ($"Uploading {hostSource} to device"); + + // First upload to temporary path, as it's writable for everyone + string remotePath = $"/data/local/tmp/{executableName}"; + if (!adb.Push (hostSource, remotePath).Result) { + log.ErrorLine ($"Failed to upload debug server {hostSource} to device path {remotePath}"); + return false; + } + + // Next, copy it to the app dir, with run-as + (bool success, string output) = adb.Shell ( + "cat", remotePath, "|", + "run-as", packageName, + "sh", "-c", $"'cat > {deviceDestination}'" + ).Result; + + if (!success) { + log.ErrorLine ($"Failed to copy debug executable to device, from {hostSource} to {deviceDestination}"); + return false; + } + + (success, output) = adb.RunAs (packageName, "chmod", "700", deviceDestination).Result; + if (!success) { + log.ErrorLine ($"Failed to make debug server executable on device, at {deviceDestination}"); + return false; + } + + return true; + } + + bool KillDebugServer (string debugServerPath) + { + long serverPID = GetDeviceProcessID (debugServerPath, quiet: true); + if (serverPID <= 0) { + return true; + } + + log.DebugLine ("Killing previous instance of the debug server"); + (bool success, string _) = adb.RunAs (packageName, "kill", "-9", $"{serverPID}").Result; + return success; + } + + long GetDeviceProcessID (string processName, bool quiet = false) + { + (bool success, string output) = adb.Shell ("pidof", processName).Result; + if (!success) { + if (!quiet) { + log.ErrorLine ($"Failed to obtain PID of process '{processName}'"); + log.ErrorLine (output); + } + return -1; + } + + output = output.Trim (); + if (!UInt32.TryParse (output, out uint pid)) { + if (!quiet) { + log.ErrorLine ($"Unable to parse string '{output}' as the package's PID"); + } + return -1; + } + + return pid; + } + + bool DetectTools () + { + // Not all versions of Android have the `which` utility, all of them have `whence` + // Also, API 21 adbd will not return an error code to us... But since we know that 21 + // doesn't have LDD, we'll cheat + deviceLdd = null; + if (apiLevel > 21) { + (bool success, string output) = adb.Shell ("whence", "ldd").Result; + if (success) { + log.DebugLine ($"Found `ldd` on device at '{output}'"); + deviceLdd = output; + } + } + + if (String.IsNullOrEmpty (deviceLdd)) { + log.DebugLine ("`ldd` not found on device"); + } + + return true; + } + bool DetermineAppDataDirectory () { (bool success, string output) = adb.GetAppDataDirectory (packageName).Result; - if (!success) { + if (!AppDataDirFound (success, output)) { log.ErrorLine ($"Unable to determine data directory for package '{packageName}'"); return false; } @@ -129,6 +285,21 @@ bool DetermineAppDataDirectory () } return true; + + bool AppDataDirFound (bool success, string output) + { + if (apiLevel > 21) { + return success; + } + + if (output.IndexOf ("run-as: Package", StringComparison.OrdinalIgnoreCase) >= 0 && + output.IndexOf ("is unknown", StringComparison.OrdinalIgnoreCase) >= 0) + { + return false; + } + + return true; + } } bool DetermineArchitectureAndABI () diff --git a/tools/debug-session-prep/Debug.Session.Prep/AndroidNdk.cs b/tools/debug-session-prep/Debug.Session.Prep/AndroidNdk.cs index 2ce8156c6a0..58df546dde1 100644 --- a/tools/debug-session-prep/Debug.Session.Prep/AndroidNdk.cs +++ b/tools/debug-session-prep/Debug.Session.Prep/AndroidNdk.cs @@ -1,3 +1,10 @@ +using System; +using System.Collections.Generic; +using System.IO; + +using Xamarin.Android.Utilities; +using Xamarin.Android.Tasks; + namespace Xamarin.Debug.Session.Prep; class AndroidNdk @@ -9,4 +16,84 @@ class AndroidNdk "lldb.cmd", "lldb.exe", }; + + Dictionary hostLldbServerPaths; + XamarinLoggingHelper log; + string? lldbPath; + + public AndroidNdk (XamarinLoggingHelper log, string ndkRootPath, string[] supportedAbis) + { + this.log = log; + hostLldbServerPaths = new Dictionary (StringComparer.Ordinal); + + if (!FindTools (ndkRootPath, supportedAbis)) { + throw new InvalidOperationException ("Failed to find all the required NDK tools"); + } + } + + public string? GetDebugServerPath (string abi) + { + if (!hostLldbServerPaths.TryGetValue (abi, out string? debugServerPath) || String.IsNullOrEmpty (debugServerPath)) { + log.ErrorLine ($"Debug server for abi '{abi}' not found."); + return null; + } + + return debugServerPath; + } + + bool FindTools (string ndkRootPath, string[] supportedAbis) + { + string toolchainDir = Path.Combine (ndkRootPath, NdkHelper.RelativeToolchainDir); + string toolchainBinDir = Path.Combine (toolchainDir, "bin"); + string? path = null; + + foreach (string lldb in lldbNames) { + path = Path.Combine (toolchainBinDir, lldb); + if (File.Exists (path)) { + break; + } + } + + if (String.IsNullOrEmpty (path)) { + log.ErrorLine ($"Unable to locate lldb executable in '{toolchainBinDir}'"); + return false; + } + lldbPath = path; + + hostLldbServerPaths = new Dictionary (StringComparer.OrdinalIgnoreCase); + string llvmVersion = GetLlvmVersion (toolchainDir); + foreach (string abi in supportedAbis) { + string llvmAbi = NdkHelper.TranslateAbiToLLVM (abi); + path = Path.Combine (ndkRootPath, NdkHelper.RelativeToolchainDir, "lib64", "clang", llvmVersion, "lib", "linux", llvmAbi, "lldb-server"); + if (!File.Exists (path)) { + log.ErrorLine ($"LLVM lldb server component for ABI '{abi}' not found at '{path}'"); + return false; + } + + hostLldbServerPaths.Add (abi, path); + } + + if (hostLldbServerPaths.Count == 0) { + log.ErrorLine ("Unable to find any lldb-server executables, debugging not possible"); + return false; + } + + return true; + } + + string GetLlvmVersion (string toolchainDir) + { + string path = Path.Combine (toolchainDir, "AndroidVersion.txt"); + if (!File.Exists (path)) { + throw new InvalidOperationException ($"LLVM version file not found at '{path}'"); + } + + string[] lines = File.ReadAllLines (path); + string? line = lines.Length >= 1 ? lines[0].Trim () : null; + if (String.IsNullOrEmpty (line)) { + throw new InvalidOperationException ($"Unable to read LLVM version from '{path}'"); + } + + return line; + } } diff --git a/tools/debug-session-prep/Debug.Session.Prep/Utilities.cs b/tools/debug-session-prep/Debug.Session.Prep/Utilities.cs new file mode 100644 index 00000000000..f76bd1e9e70 --- /dev/null +++ b/tools/debug-session-prep/Debug.Session.Prep/Utilities.cs @@ -0,0 +1,23 @@ +using System.IO; +using System.Reflection; + +using Xamarin.Android.Utilities; + +namespace Xamarin.Debug.Session.Prep; + +static class Utilities +{ + public static string? ReadManifestResource (XamarinLoggingHelper log, string resourceName) + { + using (var from = Assembly.GetExecutingAssembly ().GetManifestResourceStream (resourceName)) { + if (from == null) { + log.ErrorLine ($"Manifest resource '{resourceName}' cannot be loaded"); + return null; + } + + using (var sr = new StreamReader (from)) { + return sr.ReadToEnd (); + } + } + } +} diff --git a/tools/debug-session-prep/Main.cs b/tools/debug-session-prep/Main.cs index ce0ddd9b58c..cf8d9f780db 100644 --- a/tools/debug-session-prep/Main.cs +++ b/tools/debug-session-prep/Main.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Linq; using Mono.Options; using Xamarin.Android.Utilities; @@ -10,6 +11,17 @@ class App { const string DefaultAdbPath = "adb"; + static readonly Dictionary SupportedAbiMap = new Dictionary (StringComparer.OrdinalIgnoreCase) { + {"arm32", "armeabi-v7a"}, + {"arm64", "arm64-v8a"}, + {"arm64-v8a", "arm64-v8a"}, + {"armeabi", "armeabi-v7a"}, + {"armeabi-v7a", "armeabi-v7a"}, + {"x86", "x86"}, + {"x86_64", "x86_64"}, + {"x64", "x86_64"} + }; + sealed class ParsedOptions { public string? AdbPath; @@ -18,13 +30,18 @@ sealed class ParsedOptions public string? TargetDevice; public bool ShowHelp; public bool Verbose = true; // TODO: remove the default once development is done + public string? AppNativeLibrariesDir; + public string? NdkDirPath; + public string? OutputDirPath; } static int Main (string[] args) { bool haveOptionErrors = false; - var log = new XamarinLoggingHelper (); var parsedOptions = new ParsedOptions (); + var log = new XamarinLoggingHelper { + Verbose = parsedOptions.Verbose, + }; var opts = new OptionSet { "Usage: debug-session-prep [REQUIRED_OPTIONS] [OPTIONS]", @@ -32,6 +49,9 @@ static int Main (string[] args) "REQUIRED_OPTIONS are:", { "p|package-name=", "name of the application package", v => parsedOptions.PackageName = EnsureNonEmptyString (log, "-p|--package-name", v, ref haveOptionErrors) }, { "s|supported-abis=", "comma-separated list of ABIs the application supports", v => parsedOptions.SupportedABIs = EnsureSupportedABIs (log, "-s|--supported-abis", v, ref haveOptionErrors) }, + { "l|lib-dir=", "{PATH} to the directory where application native libraries were copied", v => parsedOptions.AppNativeLibrariesDir = v }, + { "n|ndk-dir=", "{PATH} to to the Android NDK root directory", v => parsedOptions.NdkDirPath = v }, + { "o|output-dir=", "{PATH} to directory which will contain various generated files (logs, scripts etc)", v => parsedOptions.OutputDirPath = v }, "", "OPTIONS are:", { "a|adb=", "{PATH} to adb to use for this session", v => parsedOptions.AdbPath = EnsureNonEmptyString (log, "-a|--adb", v, ref haveOptionErrors) }, @@ -39,9 +59,13 @@ static int Main (string[] args) "", { "v|verbose", "Show debug messages", v => parsedOptions.Verbose = true }, { "h|help|?", "Show this help screen", v => parsedOptions.ShowHelp = true }, + "", + $"Supported ABI names are: {GetSupportedAbiNames ()}", + "", }; List rest = opts.Parse (args); + log.Verbose = parsedOptions.Verbose; if (parsedOptions.ShowHelp) { opts.WriteOptionDescriptions (Console.Out); @@ -63,11 +87,36 @@ static int Main (string[] args) missingRequiredOptions = true; } + if (String.IsNullOrEmpty (parsedOptions.NdkDirPath)) { + log.ErrorLine ("The '-n|--ndk-dir' option must be used to specify the directory where Android NDK is installed"); + missingRequiredOptions = true; + } + + if (String.IsNullOrEmpty (parsedOptions.OutputDirPath)) { + log.ErrorLine ("The '-o|--output-dir' option must be used to specify the directory where generated files will be placed"); + missingRequiredOptions = true; + } + + if (String.IsNullOrEmpty (parsedOptions.AppNativeLibrariesDir)) { + log.ErrorLine ("The '-l|--lib-dir' option must be used to specify the directory where application shared libraries were copied"); + // missingRequiredOptions = true; + } + if (missingRequiredOptions) { return 1; } - var device = new AndroidDevice (log, parsedOptions.AdbPath ?? DefaultAdbPath, parsedOptions.PackageName!, parsedOptions.SupportedABIs!, parsedOptions.TargetDevice); + var ndk = new AndroidNdk (log, parsedOptions.NdkDirPath!, parsedOptions.SupportedABIs!); + var device = new AndroidDevice ( + log, + ndk, + parsedOptions.OutputDirPath!, + parsedOptions.AdbPath ?? DefaultAdbPath, + parsedOptions.PackageName!, + parsedOptions.SupportedABIs!, + parsedOptions.TargetDevice + ); + if (!device.GatherInfo ()) { return 1; } @@ -82,6 +131,7 @@ static int Main (string[] args) return null; } + bool haveInvalidAbis = false; var list = new List (); foreach (string s in abis.Split (',')) { string? abi = s?.Trim (); @@ -89,7 +139,16 @@ static int Main (string[] args) continue; } - list.Add (abi); + if (!SupportedAbiMap.TryGetValue (abi, out string? mappedAbi) || String.IsNullOrEmpty (mappedAbi)) { + log.ErrorLine ($"Unsupported ABI: {abi}"); + haveInvalidAbis = true; + } + + list.Add (mappedAbi!); + } + + if (haveInvalidAbis) { + return null; } return list.ToArray (); @@ -105,4 +164,6 @@ static int Main (string[] args) return value; } + + static string GetSupportedAbiNames () => String.Join (", ", SupportedAbiMap.Keys); } diff --git a/src/Xamarin.Android.Build.Tasks/Resources/lldb-debug-session.sh b/tools/debug-session-prep/Resources/lldb-debug-session.sh similarity index 100% rename from src/Xamarin.Android.Build.Tasks/Resources/lldb-debug-session.sh rename to tools/debug-session-prep/Resources/lldb-debug-session.sh diff --git a/src/Xamarin.Android.Build.Tasks/Resources/xa_start_lldb_server.sh b/tools/debug-session-prep/Resources/xa_start_lldb_server.sh similarity index 100% rename from src/Xamarin.Android.Build.Tasks/Resources/xa_start_lldb_server.sh rename to tools/debug-session-prep/Resources/xa_start_lldb_server.sh diff --git a/tools/debug-session-prep/debug-session-prep.csproj b/tools/debug-session-prep/debug-session-prep.csproj index 851db034bce..cf002f923f9 100644 --- a/tools/debug-session-prep/debug-session-prep.csproj +++ b/tools/debug-session-prep/debug-session-prep.csproj @@ -10,10 +10,20 @@ - + + - + + + + + + xa_start_lldb_server.sh + + + lldb-debug-session.sh + From 0dbaca4d9b8878773506b0d6bec3f0918e0c42cc Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 24 Nov 2022 23:29:48 +0100 Subject: [PATCH 14/30] [WIP] Compatibility between Android version is a mess... Whole day spent researching and still not done... :) --- .../Debug.Session.Prep/AndroidDevice.cs | 31 ++++++ .../DeviceLibrariesCopier.cs | 68 ++++++++++++++ .../LddDeviceLibrariesCopier.cs | 18 ++++ .../NoLddDeviceLibrariesCopier.cs | 94 +++++++++++++++++++ .../Debug.Session.Prep/Utilities.cs | 31 ++++++ tools/debug-session-prep/Main.cs | 6 +- 6 files changed, 247 insertions(+), 1 deletion(-) create mode 100644 tools/debug-session-prep/Debug.Session.Prep/DeviceLibrariesCopier.cs create mode 100644 tools/debug-session-prep/Debug.Session.Prep/LddDeviceLibrariesCopier.cs create mode 100644 tools/debug-session-prep/Debug.Session.Prep/NoLddDeviceLibrariesCopier.cs diff --git a/tools/debug-session-prep/Debug.Session.Prep/AndroidDevice.cs b/tools/debug-session-prep/Debug.Session.Prep/AndroidDevice.cs index 92bd62d450d..b2c49c21f8b 100644 --- a/tools/debug-session-prep/Debug.Session.Prep/AndroidDevice.cs +++ b/tools/debug-session-prep/Debug.Session.Prep/AndroidDevice.cs @@ -48,6 +48,8 @@ class AndroidDevice AdbRunner adb; AndroidNdk ndk; + public int ApiLevel => apiLevel; + public AndroidDevice (XamarinLoggingHelper log, AndroidNdk ndk, string outputDir, string adbPath, string packageName, string[] supportedAbis, string? adbTargetDevice = null) { this.adbPath = adbPath; @@ -108,6 +110,10 @@ public bool GatherInfo () return false; } + if (!PullLibraries ()) { + return false; + } + return true; bool YamaOK (string output) @@ -124,6 +130,31 @@ bool PropertyIsEqualTo ((bool haveProperty, string value) result, string expecte } } + bool PullLibraries () + { + DeviceLibraryCopier copier; + + if (String.IsNullOrEmpty (deviceLdd)) { + copier = new NoLddDeviceLibraryCopier ( + log, + adb, + appIs64Bit, + outputDir, + apiLevel + ); + } else { + copier = new LddDeviceLibraryCopier ( + log, + adb, + appIs64Bit, + outputDir, + apiLevel + ); + } + + return copier.Copy (); + } + bool PushDebugServer () { string? debugServerPath = ndk.GetDebugServerPath (abi!); diff --git a/tools/debug-session-prep/Debug.Session.Prep/DeviceLibrariesCopier.cs b/tools/debug-session-prep/Debug.Session.Prep/DeviceLibrariesCopier.cs new file mode 100644 index 00000000000..7a39616f89a --- /dev/null +++ b/tools/debug-session-prep/Debug.Session.Prep/DeviceLibrariesCopier.cs @@ -0,0 +1,68 @@ +using System; + +using Xamarin.Android.Utilities; +using Xamarin.Android.Tasks; + +namespace Xamarin.Debug.Session.Prep; + +abstract class DeviceLibraryCopier +{ + protected XamarinLoggingHelper Log { get; } + protected bool AppIs64Bit { get; } + protected string LocalDestinationDir { get; } + protected AdbRunner Adb { get; } + protected int DeviceApiLevel { get; } + + protected DeviceLibraryCopier (XamarinLoggingHelper log, AdbRunner adb, bool appIs64Bit, string localDestinationDir, int deviceApiLevel) + { + Log = log; + Adb = adb; + AppIs64Bit = appIs64Bit; + LocalDestinationDir = localDestinationDir; + DeviceApiLevel = deviceApiLevel; + } + + protected string? FetchZygote () + { + string zygotePath; + string destination; + + if (AppIs64Bit) { + zygotePath = "/system/bin/app_process64"; + destination = $"{LocalDestinationDir}{ToLocalPathFormat (zygotePath)}"; + + Utilities.MakeFileDirectory (destination); + if (!Adb.Pull (zygotePath, destination).Result) { + Log.ErrorLine ("Failed to copy 64-bit app_process64"); + return null; + } + } else { + // /system/bin/app_process is 32-bit on 32-bit devices, but a symlink to + // app_process64 on 64-bit. If we need the 32-bit version, try to pull + // app_process32, and if that fails, pull app_process. + destination = $"{LocalDestinationDir}{ToLocalPathFormat ("/system/bin/app_process")}"; + string? source = "/system/bin/app_process32"; + + Utilities.MakeFileDirectory (destination); + if (!Adb.Pull (source, destination).Result) { + source = "/system/bin/app_process"; + if (!Adb.Pull (source, destination).Result) { + source = null; + } + } + + if (String.IsNullOrEmpty (source)) { + Log.ErrorLine ("Failed to copy 32-bit app_process"); + return null; + } + + zygotePath = destination; + } + + return zygotePath; + } + + protected string ToLocalPathFormat (string path) => Utilities.IsWindows ? path.Replace ("/", "\\") : path; + + public abstract bool Copy (); +} diff --git a/tools/debug-session-prep/Debug.Session.Prep/LddDeviceLibrariesCopier.cs b/tools/debug-session-prep/Debug.Session.Prep/LddDeviceLibrariesCopier.cs new file mode 100644 index 00000000000..07b26d37cc4 --- /dev/null +++ b/tools/debug-session-prep/Debug.Session.Prep/LddDeviceLibrariesCopier.cs @@ -0,0 +1,18 @@ +using System; + +using Xamarin.Android.Utilities; +using Xamarin.Android.Tasks; + +namespace Xamarin.Debug.Session.Prep; + +class LddDeviceLibraryCopier : DeviceLibraryCopier +{ + public LddDeviceLibraryCopier (XamarinLoggingHelper log, AdbRunner adb, bool appIs64Bit, string localDestinationDir, int deviceApiLevel) + : base (log, adb, appIs64Bit, localDestinationDir, deviceApiLevel) + {} + + public override bool Copy () + { + throw new NotImplementedException(); + } +} diff --git a/tools/debug-session-prep/Debug.Session.Prep/NoLddDeviceLibrariesCopier.cs b/tools/debug-session-prep/Debug.Session.Prep/NoLddDeviceLibrariesCopier.cs new file mode 100644 index 00000000000..929c191c956 --- /dev/null +++ b/tools/debug-session-prep/Debug.Session.Prep/NoLddDeviceLibrariesCopier.cs @@ -0,0 +1,94 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; + +using Xamarin.Android.Utilities; +using Xamarin.Android.Tasks; + +namespace Xamarin.Debug.Session.Prep; + +class NoLddDeviceLibraryCopier : DeviceLibraryCopier +{ + const string LdConfigPath = "/system/etc/ld.config.txt"; + + // To make things interesting, it turns out that API29 devices have **both** API 28 and 29 (they report 28) and for that reason they have TWO config files for ld... + const string LdConfigPath28 = "/etc/ld.config.28.txt"; + const string LdConfigPath29 = "/etc/ld.config.29.txt"; + + // TODO: We probably need a "provider" for the list of paths, since on ARM devices, /system/lib{64} directories contain x86/x64 binaries, and the ARM binaries are found in + // /system/lib{64]/arm{64} (but not on all devices, of course... e.g. Pixel 6 Pro doesn't have these) + // + // List of directory paths to use when the device has neither ldd nor /system/etc/ld.config.txt + static readonly string[] FallbackLibraryDirectories = { + "/system/@LIB@", + "/system/@LIB@/drm", + "/system/@LIB@/egl", + "/system/@LIB@/hw", + "/system/@LIB@/soundfx", + "/system/@LIB@/ssl", + "/system/@LIB@/ssl/engines", + + // /system/vendor is a symlink to /vendor on some Android versions, we'll skip the latter then + "/system/vendor/@LIB@", + "/system/vendor/@LIB@/egl", + "/system/vendor/@LIB@/mediadrm", + }; + + + public NoLddDeviceLibraryCopier (XamarinLoggingHelper log, AdbRunner adb, bool appIs64Bit, string localDestinationDir, int deviceApiLevel) + : base (log, adb, appIs64Bit, localDestinationDir, deviceApiLevel) + {} + + public override bool Copy () + { + string? zygotePath = FetchZygote (); + if (String.IsNullOrEmpty (zygotePath)) { + Log.ErrorLine ("Unable to determine path of the zygote process on device"); + return false; + } + + throw new NotImplementedException(); + } + + List GetLibraryDirectories () + { + if (DeviceApiLevel == 21) { + // API21 devices (at least emulators) don't return adb error codes, so to avoid awkward error message parsing, we're going to just skip detection since we + // know what API21 has and doesn't have + return GetFallbackDirs (); + } + + string localLdConfigPath = Path.Combine (LocalDestinationDir, ToLocalPathFormat (LdConfigPath)); + Utilities.MakeFileDirectory (localLdConfigPath); + + string deviceLdConfigPath; + + if (DeviceApiLevel == 28) { + deviceLdConfigPath = LdConfigPath28; + } else if (DeviceApiLevel == 29) { + deviceLdConfigPath = LdConfigPath29; + } else { + deviceLdConfigPath = LdConfigPath; + } + + if (!Adb.Pull (deviceLdConfigPath, localLdConfigPath).Result) { + Log.DebugLine ($"Device doesn't have {LdConfigPath}"); + return GetFallbackDirs (); + } + + var ret = new List (); + + // TODO: parse ldconfig + // TODO: must check if device has APEX mountpoints nad include dirs from them as well + return ret; + + List GetFallbackDirs () + { + string lib = AppIs64Bit ? "lib64" : "lib"; + + Log.DebugLine ("Using fallback library directories for this device"); + return FallbackLibraryDirectories.Select (l => l.Replace ("@LIB@", lib)).ToList (); + } + } +} diff --git a/tools/debug-session-prep/Debug.Session.Prep/Utilities.cs b/tools/debug-session-prep/Debug.Session.Prep/Utilities.cs index f76bd1e9e70..50e4bef43ff 100644 --- a/tools/debug-session-prep/Debug.Session.Prep/Utilities.cs +++ b/tools/debug-session-prep/Debug.Session.Prep/Utilities.cs @@ -1,5 +1,7 @@ +using System; using System.IO; using System.Reflection; +using System.Runtime.InteropServices; using Xamarin.Android.Utilities; @@ -7,6 +9,35 @@ namespace Xamarin.Debug.Session.Prep; static class Utilities { + public static bool IsMacOS { get; private set; } + public static bool IsLinux { get; private set; } + public static bool IsWindows { get; private set; } + + static Utilities () + { + if (RuntimeInformation.IsOSPlatform (OSPlatform.Windows)) { + IsWindows = true; + } else if (RuntimeInformation.IsOSPlatform (OSPlatform.OSX)) { + IsMacOS = true; + } else if (RuntimeInformation.IsOSPlatform (OSPlatform.Linux)) { + IsLinux = true; + } + } + + public static void MakeFileDirectory (string filePath) + { + if (String.IsNullOrEmpty (filePath)) { + return; + } + + string? dirName = Path.GetDirectoryName (filePath); + if (String.IsNullOrEmpty (dirName)) { + return; + } + + Directory.CreateDirectory (dirName); + } + public static string? ReadManifestResource (XamarinLoggingHelper log, string resourceName) { using (var from = Assembly.GetExecutingAssembly ().GetManifestResourceStream (resourceName)) { diff --git a/tools/debug-session-prep/Main.cs b/tools/debug-session-prep/Main.cs index cf8d9f780db..85dcd39911c 100644 --- a/tools/debug-session-prep/Main.cs +++ b/tools/debug-session-prep/Main.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.Linq; using Mono.Options; using Xamarin.Android.Utilities; @@ -121,6 +120,11 @@ static int Main (string[] args) return 1; } + if (device.ApiLevel < 21) { + log.ErrorLine ($"Only Android API level 21 and newer are supported"); + return 1; + } + return 0; } From a06e281c42e8f954a4cac84f363450ff522f19b2 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Fri, 25 Nov 2022 21:46:15 +0100 Subject: [PATCH 15/30] [WIP] ld config and directory listing parsing for devices without ldd --- .../Utilities/AdbRunner.cs | 6 +- .../Debug.Session.Prep/LdConfigParser.cs | 167 ++++++++++++++++++ .../NoLddDeviceLibrariesCopier.cs | 107 +++++++++-- .../Debug.Session.Prep/Utilities.cs | 9 + 4 files changed, 275 insertions(+), 14 deletions(-) create mode 100644 tools/debug-session-prep/Debug.Session.Prep/LdConfigParser.cs diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs index b9b422a37c5..eba40c9d113 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs @@ -154,7 +154,7 @@ public async Task Push (string localPath, string remotePath) }; if (!await RunAdb (runner, setupOutputSink: false)) { - return (false, String.Empty); + return (false, FormatOutputWithLines (lines)); } } @@ -162,7 +162,9 @@ public async Task Push (string localPath, string remotePath) return (true, outputLine ?? String.Empty); } - return (true, lines != null ? String.Join (Environment.NewLine, lines) : String.Empty); + return (true, FormatOutputWithLines (lines)); + + string FormatOutputWithLines (List? lines) => lines != null ? String.Join (Environment.NewLine, lines) : String.Empty; } async Task RunAdb (ProcessRunner runner, bool setupOutputSink = true, bool ignoreStderr = true) diff --git a/tools/debug-session-prep/Debug.Session.Prep/LdConfigParser.cs b/tools/debug-session-prep/Debug.Session.Prep/LdConfigParser.cs new file mode 100644 index 00000000000..71b33086520 --- /dev/null +++ b/tools/debug-session-prep/Debug.Session.Prep/LdConfigParser.cs @@ -0,0 +1,167 @@ +using System; +using System.Collections.Generic; +using System.IO; + +using Xamarin.Android.Utilities; + +namespace Xamarin.Debug.Session.Prep; + +class LdConfigParser +{ + XamarinLoggingHelper log; + + public LdConfigParser (XamarinLoggingHelper log) + { + this.log = log; + } + + // Format: https://android.googlesource.com/platform/bionic/+/master/linker/ld.config.format.md + // + public (List searchPaths, HashSet permittedPaths) Parse (string localLdConfigPath, string deviceBinDirectory, string libDirName) + { + var searchPaths = new List (); + var permittedPaths = new HashSet (); + bool foundSomeSection = false; + bool insideMatchingSection = false; + string normalizedDeviceBinDirectory = Utilities.NormalizeDirectoryPath (deviceBinDirectory); + string? sectionName = null; + + log.DebugLine ($"Parsing LD config file '{localLdConfigPath}'"); + int lineCounter = 0; + var namespaces = new List { + "default" + }; + + foreach (string l in File.ReadLines (localLdConfigPath)) { + lineCounter++; + string line = l.Trim (); + if (line.Length == 0 || line.StartsWith ('#')) { + continue; + } + + // The `dir.*` entries are before any section, don't waste time looking for them if we've parsed a section already + if (!foundSomeSection && sectionName == null) { + sectionName = GetMatchingDirMapping (normalizedDeviceBinDirectory, line); + if (sectionName != null) { + log.DebugLine ($"Found section name on line {lineCounter}: '{sectionName}'"); + continue; + } + } + + if (line[0] == '[') { + foundSomeSection = true; + insideMatchingSection = String.Compare (line, $"[{sectionName}]", StringComparison.Ordinal) == 0; + if (insideMatchingSection) { + log.DebugLine ($"Found section '{sectionName}' start on line {lineCounter}"); + } + } + + if (!insideMatchingSection) { + continue; + } + + if (line.StartsWith ("additional.namespaces", StringComparison.Ordinal) && GetVariableAssignmentParts (line, out string? name, out string? value)) { + foreach (string v in value!.Split (',')) { + string nsName = v.Trim (); + if (nsName.Length == 0) { + continue; + } + + log.DebugLine ($"Adding additional namespace '{nsName}'"); + namespaces.Add (nsName); + } + continue; + } + + MaybeAddLibraryPath (searchPaths, permittedPaths, namespaces, line, libDirName); + } + + return (searchPaths, permittedPaths); + + } + + void MaybeAddLibraryPath (List searchPaths, HashSet permittedPaths, List knownNamespaces, string configLine, string libDirName) + { + if (!configLine.StartsWith ("namespace.", StringComparison.Ordinal)) { + return; + } + + // not interested in ASAN libraries + if (configLine.IndexOf (".asan.", StringComparison.Ordinal) > 0) { + return; + } + + foreach (string ns in knownNamespaces) { + if (!GetVariableAssignmentParts (configLine, out string? name, out string? value)) { + continue; + } + + string varName = $"namespace.{ns}.search.paths"; + if (String.Compare (varName, name, StringComparison.Ordinal) == 0) { + AddPath (searchPaths, "search", value!); + continue; + } + + varName = $"namespace.{ns}.permitted.paths"; + if (String.Compare (varName, name, StringComparison.Ordinal) == 0) { + AddPath (permittedPaths, "permitted", value!, checkIfAlreadyAdded: true); + } + } + + void AddPath (ICollection list, string which, string value, bool checkIfAlreadyAdded = false) + { + string path = Utilities.NormalizeDirectoryPath (value.Replace ("${LIB}", libDirName)); + + if (checkIfAlreadyAdded && list.Contains (path)) { + return; + } + + log.DebugLine ($"Adding library {which} path: {path}"); + list.Add (path); + } + } + + string? GetMatchingDirMapping (string deviceBinDirectory, string configLine) + { + const string LinePrefix = "dir."; + + string line = configLine.Trim (); + if (line.Length == 0 || !line.StartsWith (LinePrefix, StringComparison.Ordinal)) { + return null; + } + + if (!GetVariableAssignmentParts (line, out string? name, out string? value)) { + return null; + } + + string dirPath = Utilities.NormalizeDirectoryPath (value!); + if (String.Compare (dirPath, deviceBinDirectory, StringComparison.Ordinal) != 0) { + return null; + } + + string ns = name!.Substring (LinePrefix.Length).Trim (); + if (String.IsNullOrEmpty (ns)) { + return null; + } + + return ns; + } + + bool GetVariableAssignmentParts (string line, out string? name, out string? value) + { + name = value = null; + + string[] parts = line.Split ("+=", 2); + if (parts.Length != 2) { + parts = line.Split ('=', 2); + if (parts.Length != 2) { + return false; + } + } + + name = parts[0].Trim (); + value = parts[1].Trim (); + + return true; + } +} diff --git a/tools/debug-session-prep/Debug.Session.Prep/NoLddDeviceLibrariesCopier.cs b/tools/debug-session-prep/Debug.Session.Prep/NoLddDeviceLibrariesCopier.cs index 929c191c956..5f82b58613e 100644 --- a/tools/debug-session-prep/Debug.Session.Prep/NoLddDeviceLibrariesCopier.cs +++ b/tools/debug-session-prep/Debug.Session.Prep/NoLddDeviceLibrariesCopier.cs @@ -35,7 +35,6 @@ class NoLddDeviceLibraryCopier : DeviceLibraryCopier "/system/vendor/@LIB@/mediadrm", }; - public NoLddDeviceLibraryCopier (XamarinLoggingHelper log, AdbRunner adb, bool appIs64Bit, string localDestinationDir, int deviceApiLevel) : base (log, adb, appIs64Bit, localDestinationDir, deviceApiLevel) {} @@ -48,18 +47,97 @@ public override bool Copy () return false; } - throw new NotImplementedException(); + (List searchPaths, HashSet permittedPaths) = GetLibraryPaths (); + + // Collect file listings for all the search directories + var sharedLibraries = new List (); + foreach (string path in searchPaths) { + AddSharedLibraries (sharedLibraries, path, permittedPaths); + } + + return true; } - List GetLibraryDirectories () + void AddSharedLibraries (List sharedLibraries, string deviceDirPath, HashSet permittedPaths) { + (bool success, string output) = Adb.Shell ("ls", "-l", deviceDirPath).Result; + if (!success) { + // We can't rely on `success` because `ls -l` will return an error code if the directory exists but has any entries access to whose is not permitted + if (output.IndexOf ("No such file or directory") >= 0) { + Log.DebugLine ($"Shared libraries directory {deviceDirPath} not found on device"); + return; + } + } + + Log.DebugLine ($"Adding shared libraries from {deviceDirPath}"); + foreach (string l in output.Split ('\n')) { + string line = l.Trim (); + if (line.Length == 0) { + continue; + } + + // `ls -l` output has 8 columns for filesystem entries + string[] parts = line.Split (' ', 8, StringSplitOptions.RemoveEmptyEntries); + if (parts.Length != 8) { + continue; + } + + string permissions = parts[0].Trim (); + string name = parts[7].Trim (); + + // First column, permissions: `drwxr-xr-x`, `-rw-r--r--` etc + if (permissions[0] == 'd') { + // Directory + string nestedDirPath = $"{deviceDirPath}{name}/"; + if (permittedPaths.Count > 0 && !permittedPaths.Contains (nestedDirPath)) { + Log.DebugLine ($"Directory '{nestedDirPath}' is not in the list of permitted directories, ignoring"); + continue; + } + + AddSharedLibraries (sharedLibraries, nestedDirPath, permittedPaths); + continue; + } + + // Ignore entries that aren't regular .so files or symlinks + if ((permissions[0] != '-' && permissions[0] != 'l') || !name.EndsWith (".so", StringComparison.Ordinal)) { + continue; + } + + string libPath; + if (permissions[0] == 'l') { + // Let's hope there are no libraries with -> in their name :P (if there are, we should use `readlink`) + const string SymlinkArrow = "->"; + + // Symlink, we'll add the target library instead + int idx = name.IndexOf (SymlinkArrow); + if (idx > 0) { + libPath = name.Substring (idx + SymlinkArrow.Length).Trim (); + } else { + Log.WarningLine ($"'ls -l' output line contains a symbolic link, but I can't determine the target:"); + Log.WarningLine ($" '{line}'"); + Log.WarningLine ("Ignoring this entry"); + continue; + } + } else { + libPath = $"{deviceDirPath}{name}"; + } + + Log.DebugLine ($" {libPath}"); + sharedLibraries.Add (libPath); + } + } + + (List searchPaths, HashSet permittedPaths) GetLibraryPaths () + { + string lib = AppIs64Bit ? "lib64" : "lib"; + if (DeviceApiLevel == 21) { // API21 devices (at least emulators) don't return adb error codes, so to avoid awkward error message parsing, we're going to just skip detection since we // know what API21 has and doesn't have - return GetFallbackDirs (); + return (GetFallbackDirs (), new HashSet ()); } - string localLdConfigPath = Path.Combine (LocalDestinationDir, ToLocalPathFormat (LdConfigPath)); + string localLdConfigPath = $"{LocalDestinationDir}{ToLocalPathFormat (LdConfigPath)}"; Utilities.MakeFileDirectory (localLdConfigPath); string deviceLdConfigPath; @@ -74,19 +152,24 @@ List GetLibraryDirectories () if (!Adb.Pull (deviceLdConfigPath, localLdConfigPath).Result) { Log.DebugLine ($"Device doesn't have {LdConfigPath}"); - return GetFallbackDirs (); + return (GetFallbackDirs (), new HashSet ()); + } else { + Log.DebugLine ($"Downloaded {deviceLdConfigPath} to {localLdConfigPath}"); } - var ret = new List (); + var parser = new LdConfigParser (Log); - // TODO: parse ldconfig - // TODO: must check if device has APEX mountpoints nad include dirs from them as well - return ret; + // The app executables (app_process and app_process32) are both in /system/bin, so we can limit our + // library search paths to this location. + (List searchPaths, HashSet permittedPaths) = parser.Parse (localLdConfigPath, "/system/bin", lib); + if (searchPaths.Count == 0) { + searchPaths = GetFallbackDirs (); + } + + return (searchPaths, permittedPaths); List GetFallbackDirs () { - string lib = AppIs64Bit ? "lib64" : "lib"; - Log.DebugLine ("Using fallback library directories for this device"); return FallbackLibraryDirectories.Select (l => l.Replace ("@LIB@", lib)).ToList (); } diff --git a/tools/debug-session-prep/Debug.Session.Prep/Utilities.cs b/tools/debug-session-prep/Debug.Session.Prep/Utilities.cs index 50e4bef43ff..544bf64c74e 100644 --- a/tools/debug-session-prep/Debug.Session.Prep/Utilities.cs +++ b/tools/debug-session-prep/Debug.Session.Prep/Utilities.cs @@ -51,4 +51,13 @@ public static void MakeFileDirectory (string filePath) } } } + + public static string NormalizeDirectoryPath (string dirPath) + { + if (dirPath.EndsWith ('/')) { + return dirPath; + } + + return $"{dirPath}/"; + } } From b095fdf41e9b99b81ee93607b6c2c4a974881954 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Mon, 28 Nov 2022 22:00:05 +0100 Subject: [PATCH 16/30] [WIP] Fetching libraries from devices without ldd --- .../Utilities/AdbRunner.cs | 130 ++++++++++---- .../Utilities/ToolRunner.cs | 15 +- .../Debug.Session.Prep/AndroidDevice.cs | 6 +- .../DeviceLibrariesCopier.cs | 12 +- .../LddDeviceLibrariesCopier.cs | 4 +- .../Debug.Session.Prep/LldbModuleCache.cs | 170 ++++++++++++++++++ .../NoLddDeviceLibrariesCopier.cs | 31 +++- .../NoLddLldbModuleCache.cs | 41 +++++ .../Debug.Session.Prep/Utilities.cs | 12 ++ .../XamarinLoggingHelper.cs | 16 +- .../debug-session-prep.csproj | 1 + 11 files changed, 366 insertions(+), 72 deletions(-) create mode 100644 tools/debug-session-prep/Debug.Session.Prep/LldbModuleCache.cs create mode 100644 tools/debug-session-prep/Debug.Session.Prep/NoLddLldbModuleCache.cs diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs index eba40c9d113..631fa863cf0 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs @@ -13,20 +13,53 @@ namespace Xamarin.Android.Tasks { class AdbRunner : ToolRunner { + public delegate bool OutputLineFilter (bool isStdErr, string line); + class AdbOutputSink : ToolOutputSink { - public Action? LineCallback { get; set; } + bool isStdError; + OutputLineFilter? lineFilter; + + public Action? LineCallback { get; set; } - public AdbOutputSink (LoggerType logger) + public AdbOutputSink (LoggerType logger, bool isStdError, OutputLineFilter? lineFilter) : base (logger) { + this.isStdError = isStdError; + this.lineFilter = lineFilter; + LogLinePrefix = "adb"; } public override void WriteLine (string? value) { + if (lineFilter != null && lineFilter (isStdError, value ?? String.Empty)) { + return; + } + base.WriteLine (value); - LineCallback?.Invoke (value ?? String.Empty); + LineCallback?.Invoke (isStdError, value ?? String.Empty); + } + } + + class AdbStdErrorWrapper : ProcessStandardStreamWrapper + { + OutputLineFilter? lineFilter; + + public AdbStdErrorWrapper (LoggerType logger, OutputLineFilter? lineFilter) + : base (logger) + { + this.lineFilter = lineFilter; + } + + protected override string? PreprocessMessage (string? message, out bool ignoreLine) + { + if (lineFilter == null) { + return base.PreprocessMessage (message, out ignoreLine); + } + + ignoreLine = lineFilter (isStdErr: true, line: message ?? String.Empty); + return message; } } @@ -88,7 +121,7 @@ public async Task Push (string localPath, string remotePath) shellArgs.AddRange (args); } - return await Shell ("run-as", (IEnumerable)shellArgs); + return await Shell ("run-as", (IEnumerable)shellArgs, lineFilter: null); } public async Task<(bool success, string output)> GetAppDataDirectory (string packageName) @@ -96,27 +129,34 @@ public async Task Push (string localPath, string remotePath) return await RunAs (packageName, "/system/bin/sh", "-c", "pwd"); } - public async Task<(bool success, string output)> Shell (string command, List args) + public async Task<(bool success, string output)> Shell (string command, List args, OutputLineFilter? lineFilter = null) { - return await Shell (command, (IEnumerable)args); + return await Shell (command, (IEnumerable)args, lineFilter: null); } public async Task<(bool success, string output)> Shell (string command, params string[] args) { - return await Shell (command, (IEnumerable)args); + return await Shell (command, (IEnumerable)args, lineFilter: null); + } + + public async Task<(bool success, string output)> Shell (OutputLineFilter lineFilter, string command, params string[] args) + { + return await Shell (command, (IEnumerable)args, lineFilter); } - async Task<(bool success, string output)> Shell (string command, IEnumerable args) + async Task<(bool success, string output)> Shell (string command, IEnumerable? args, OutputLineFilter? lineFilter) { var runner = CreateAdbRunner (); runner.AddArgument ("shell"); runner.AddArgument (command); - foreach (string arg in args) { - runner.AddArgument (arg); + if (args != null) { + foreach (string arg in args) { + runner.AddArgument (arg); + } } - return await CaptureAdbOutput (runner); + return await CaptureAdbOutput (runner, lineFilter); } public async Task<(bool success, string output)> GetPropertyValue (string propertyName) @@ -132,30 +172,40 @@ public async Task Push (string localPath, string remotePath) return await Shell ("getprop", propertyName); } - async Task<(bool success, string output)> CaptureAdbOutput (ProcessRunner runner, bool firstLineOnly = false) + async Task<(bool success, string output)> CaptureAdbOutput (ProcessRunner runner, OutputLineFilter? lineFilter = null, bool firstLineOnly = false) { string? outputLine = null; List? lines = null; - using (var outputSink = (AdbOutputSink)SetupOutputSink (runner, ignoreStderr: true)) { - outputSink.LineCallback = (string line) => { - if (firstLineOnly) { - if (outputLine != null) { - return; - } - outputLine = line.Trim (); - return; - } - - if (lines == null) { - lines = new List (); - } - lines.Add (line.Trim ()); - }; - - if (!await RunAdb (runner, setupOutputSink: false)) { - return (false, FormatOutputWithLines (lines)); - } + using AdbOutputSink? stderrSink = lineFilter != null ? new AdbOutputSink (Logger, isStdError: true, lineFilter: lineFilter) : null; + using var stdoutSink = new AdbOutputSink (Logger, isStdError: false, lineFilter: lineFilter); + + SetupOutputSinks (runner, stdoutSink, stderrSink, ignoreStderr: stderrSink == null); + stdoutSink.LineCallback = (bool isStdErr, string line) => { + if (firstLineOnly) { + if (outputLine != null) { + return; + } + outputLine = line.Trim (); + return; + } + + if (lines == null) { + lines = new List (); + } + lines.Add (line.Trim ()); + }; + + ProcessStandardStreamWrapper? origStderrWrapper = runner.StandardErrorEchoWrapper; + using AdbStdErrorWrapper? stderrWrapper = lineFilter != null ? new AdbStdErrorWrapper (Logger, lineFilter) : null; + + try { + runner.StandardErrorEchoWrapper = stderrWrapper; + if (!await RunAdb (runner, setupOutputSink: false)) { + return (false, FormatOutputWithLines (lines)); + } + } finally { + runner.StandardErrorEchoWrapper = origStderrWrapper; } if (firstLineOnly) { @@ -171,9 +221,15 @@ async Task RunAdb (ProcessRunner runner, bool setupOutputSink = true, bool { return await RunTool ( () => { - TextWriter? sink = null; + TextWriter? stdoutSink = null; + TextWriter? stderrSink = null; if (setupOutputSink) { - sink = SetupOutputSink (runner, ignoreStderr: ignoreStderr); + stdoutSink = new AdbOutputSink (Logger, isStdError: false, lineFilter: null); + if (!ignoreStderr) { + stderrSink = new AdbOutputSink (Logger, isStdError: true, lineFilter: null); + } + + SetupOutputSinks (runner, stdoutSink, stderrSink, ignoreStderr); } try { @@ -184,17 +240,13 @@ async Task RunAdb (ProcessRunner runner, bool setupOutputSink = true, bool ExitCode = -0xDEAD; throw; } finally { - sink?.Dispose (); + stdoutSink?.Dispose (); + stderrSink?.Dispose (); } } ); } ProcessRunner CreateAdbRunner () => CreateProcessRunner (initialParams); - - protected override TextWriter CreateLogSink (LoggerType logger) - { - return new AdbOutputSink (logger); - } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ToolRunner.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ToolRunner.cs index e1f2de0dd12..876a1fb33d2 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ToolRunner.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ToolRunner.cs @@ -81,21 +81,12 @@ protected virtual async Task RunTool (Func runner) return await TPLTask.Run (runner); } - protected TextWriter SetupOutputSink (ProcessRunner runner, bool ignoreStderr = false) + protected void SetupOutputSinks (ProcessRunner runner, TextWriter stdoutSink, TextWriter? stderrSink = null, bool ignoreStderr = false) { - TextWriter ret = CreateLogSink (Logger); - if (!ignoreStderr) { - runner.AddStandardErrorSink (ret); + runner.AddStandardErrorSink (stderrSink ?? stdoutSink); } - runner.AddStandardOutputSink (ret); - - return ret; + runner.AddStandardOutputSink (stdoutSink); } - - protected virtual TextWriter CreateLogSink (LoggerType logger) - { - throw new NotSupportedException ("Child class must implement this method if it uses output sinks"); - } } } diff --git a/tools/debug-session-prep/Debug.Session.Prep/AndroidDevice.cs b/tools/debug-session-prep/Debug.Session.Prep/AndroidDevice.cs index b2c49c21f8b..dae40a7795b 100644 --- a/tools/debug-session-prep/Debug.Session.Prep/AndroidDevice.cs +++ b/tools/debug-session-prep/Debug.Session.Prep/AndroidDevice.cs @@ -49,6 +49,8 @@ class AndroidDevice AndroidNdk ndk; public int ApiLevel => apiLevel; + public string SerialNumber => serialNumber ?? String.Empty; + public AdbRunner AdbRunner => adb; public AndroidDevice (XamarinLoggingHelper log, AndroidNdk ndk, string outputDir, string adbPath, string packageName, string[] supportedAbis, string? adbTargetDevice = null) { @@ -140,7 +142,7 @@ bool PullLibraries () adb, appIs64Bit, outputDir, - apiLevel + this ); } else { copier = new LddDeviceLibraryCopier ( @@ -148,7 +150,7 @@ bool PullLibraries () adb, appIs64Bit, outputDir, - apiLevel + this ); } diff --git a/tools/debug-session-prep/Debug.Session.Prep/DeviceLibrariesCopier.cs b/tools/debug-session-prep/Debug.Session.Prep/DeviceLibrariesCopier.cs index 7a39616f89a..baeb4037bec 100644 --- a/tools/debug-session-prep/Debug.Session.Prep/DeviceLibrariesCopier.cs +++ b/tools/debug-session-prep/Debug.Session.Prep/DeviceLibrariesCopier.cs @@ -11,15 +11,15 @@ abstract class DeviceLibraryCopier protected bool AppIs64Bit { get; } protected string LocalDestinationDir { get; } protected AdbRunner Adb { get; } - protected int DeviceApiLevel { get; } + protected AndroidDevice Device { get; } - protected DeviceLibraryCopier (XamarinLoggingHelper log, AdbRunner adb, bool appIs64Bit, string localDestinationDir, int deviceApiLevel) + protected DeviceLibraryCopier (XamarinLoggingHelper log, AdbRunner adb, bool appIs64Bit, string localDestinationDir, AndroidDevice device) { Log = log; Adb = adb; AppIs64Bit = appIs64Bit; LocalDestinationDir = localDestinationDir; - DeviceApiLevel = deviceApiLevel; + Device = device; } protected string? FetchZygote () @@ -29,7 +29,7 @@ protected DeviceLibraryCopier (XamarinLoggingHelper log, AdbRunner adb, bool app if (AppIs64Bit) { zygotePath = "/system/bin/app_process64"; - destination = $"{LocalDestinationDir}{ToLocalPathFormat (zygotePath)}"; + destination = Utilities.MakeLocalPath (LocalDestinationDir, zygotePath); Utilities.MakeFileDirectory (destination); if (!Adb.Pull (zygotePath, destination).Result) { @@ -40,7 +40,7 @@ protected DeviceLibraryCopier (XamarinLoggingHelper log, AdbRunner adb, bool app // /system/bin/app_process is 32-bit on 32-bit devices, but a symlink to // app_process64 on 64-bit. If we need the 32-bit version, try to pull // app_process32, and if that fails, pull app_process. - destination = $"{LocalDestinationDir}{ToLocalPathFormat ("/system/bin/app_process")}"; + destination = Utilities.MakeLocalPath (LocalDestinationDir, "/system/bin/app_process"); string? source = "/system/bin/app_process32"; Utilities.MakeFileDirectory (destination); @@ -62,7 +62,5 @@ protected DeviceLibraryCopier (XamarinLoggingHelper log, AdbRunner adb, bool app return zygotePath; } - protected string ToLocalPathFormat (string path) => Utilities.IsWindows ? path.Replace ("/", "\\") : path; - public abstract bool Copy (); } diff --git a/tools/debug-session-prep/Debug.Session.Prep/LddDeviceLibrariesCopier.cs b/tools/debug-session-prep/Debug.Session.Prep/LddDeviceLibrariesCopier.cs index 07b26d37cc4..1ccd6240bb6 100644 --- a/tools/debug-session-prep/Debug.Session.Prep/LddDeviceLibrariesCopier.cs +++ b/tools/debug-session-prep/Debug.Session.Prep/LddDeviceLibrariesCopier.cs @@ -7,8 +7,8 @@ namespace Xamarin.Debug.Session.Prep; class LddDeviceLibraryCopier : DeviceLibraryCopier { - public LddDeviceLibraryCopier (XamarinLoggingHelper log, AdbRunner adb, bool appIs64Bit, string localDestinationDir, int deviceApiLevel) - : base (log, adb, appIs64Bit, localDestinationDir, deviceApiLevel) + public LddDeviceLibraryCopier (XamarinLoggingHelper log, AdbRunner adb, bool appIs64Bit, string localDestinationDir, AndroidDevice device) + : base (log, adb, appIs64Bit, localDestinationDir, device) {} public override bool Copy () diff --git a/tools/debug-session-prep/Debug.Session.Prep/LldbModuleCache.cs b/tools/debug-session-prep/Debug.Session.Prep/LldbModuleCache.cs new file mode 100644 index 00000000000..89a9b4cfcf3 --- /dev/null +++ b/tools/debug-session-prep/Debug.Session.Prep/LldbModuleCache.cs @@ -0,0 +1,170 @@ +using System; +using System.Collections.Generic; +using System.IO; + +using ELFSharp.ELF; +using ELFSharp.ELF.Sections; +using Xamarin.Android.Utilities; + +namespace Xamarin.Debug.Session.Prep; + +abstract class LldbModuleCache +{ + protected AndroidDevice Device { get; } + protected string CacheDirPath { get; } + protected XamarinLoggingHelper Log { get; } + + protected LldbModuleCache (XamarinLoggingHelper log, AndroidDevice device) + { + Device = device; + Log = log; + + CacheDirPath = Path.Combine ( + Environment.GetFolderPath (Environment.SpecialFolder.UserProfile), + ".lldb", + "module_cache", + "remote-android", // "platform" used by LLDB in our case + device.SerialNumber + ); + } + + public void Populate (string zygotePath) + { + string? localPath = FetchFileFromDevice (zygotePath); + if (localPath == null) { + // TODO: should we perhaps fetch a set of "basic" libraries here, as a fallback? + Log.WarningLine ($"Unable to fetch Android application launcher binary ('{zygotePath}') from device. No cache of shared modules will be generated"); + return; + } + + var alreadyDownloaded = new HashSet (StringComparer.Ordinal); + using IELF? elf = ReadElfFile (localPath); + + FetchDependencies (elf, alreadyDownloaded, localPath); + } + + void FetchDependencies (IELF? elf, HashSet alreadyDownloaded, string localPath) + { + if (elf == null) { + Log.DebugLine ($"Failed to open '{localPath}' as an ELF file. Ignoring."); + return; + } + + var dynstr = GetSection (elf, ".dynstr") as IStringTable; + if (dynstr == null) { + Log.DebugLine ($"ELF binary {localPath} has no .dynstr section, unable to read referenced shared library names"); + return; + } + + var needed = new HashSet (StringComparer.Ordinal); + foreach (IDynamicSection section in elf.GetSections ()) { + foreach (IDynamicEntry entry in section.Entries) { + if (entry.Tag != DynamicTag.Needed) { + continue; + } + + AddNeeded (dynstr, entry); + } + } + + Log.DebugLine ($"Binary {localPath} references the following libraries:"); + foreach (string lib in needed) { + Log.Debug ($" {lib}"); + if (alreadyDownloaded.Contains (lib)) { + Log.DebugLine (" [already downloaded]"); + continue; + } + + string? deviceLibraryPath = GetSharedLibraryPath (lib); + if (String.IsNullOrEmpty (deviceLibraryPath)) { + Log.DebugLine (" [device path unknown]"); + Log.WarningLine ($"Referenced libary '{lib}' not found on device"); + continue; + } + + Log.DebugLine (" [downloading]"); + Log.Status ("Downloading", deviceLibraryPath); + string? localLibraryPath = FetchFileFromDevice (deviceLibraryPath); + if (String.IsNullOrEmpty (localLibraryPath)) { + Log.Log (LogLevel.Info, " [FAILED]", XamarinLoggingHelper.ErrorColor); + continue; + } + Log.LogLine (LogLevel.Info, " [SUCCESS]", XamarinLoggingHelper.InfoColor); + + alreadyDownloaded.Add (lib); + using IELF? libElf = ReadElfFile (localLibraryPath); + FetchDependencies (libElf, alreadyDownloaded, localLibraryPath); + } + + void AddNeeded (IStringTable stringTable, IDynamicEntry entry) + { + ulong index; + if (entry is DynamicEntry entry64) { + index = entry64.Value; + } else if (entry is DynamicEntry entry32) { + index = (ulong)entry32.Value; + } else { + Log.WarningLine ($"DynamicEntry neither 32 nor 64 bit? Weird"); + return; + } + + string name = stringTable[(long)index]; + if (needed.Contains (name)) { + return; + } + + needed.Add (name); + } + } + + string? FetchFileFromDevice (string deviceFilePath) + { + string localFilePath = Utilities.MakeLocalPath (CacheDirPath, deviceFilePath); + string localTempFilePath = $"{localFilePath}.tmp"; + + Directory.CreateDirectory (Path.GetDirectoryName (localFilePath)!); + + if (!Device.AdbRunner.Pull (deviceFilePath, localTempFilePath).Result) { + Log.ErrorLine ($"Failed to download {deviceFilePath} from the attached device"); + return null; + } + + File.Move (localTempFilePath, localFilePath, true); + return localFilePath; + } + + protected string GetUnixFileName (string path) + { + int idx = path.LastIndexOf ('/'); + if (idx >= 0 && idx != path.Length - 1) { + return path.Substring (idx + 1); + } + + return path; + } + + protected abstract string? GetSharedLibraryPath (string libraryName); + + IELF? ReadElfFile (string path) + { + try { + if (ELFReader.TryLoad (path, out IELF ret)) { + return ret; + } + } catch (Exception ex) { + Log.WarningLine ($"{path} may not be a valid ELF binary."); + Log.WarningLine (ex.ToString ()); + } + + return null; + } + + ISection? GetSection (IELF elf, string sectionName) + { + if (!elf.TryGetSection (sectionName, out ISection section)) { + return null; + } + + return section; + } +} diff --git a/tools/debug-session-prep/Debug.Session.Prep/NoLddDeviceLibrariesCopier.cs b/tools/debug-session-prep/Debug.Session.Prep/NoLddDeviceLibrariesCopier.cs index 5f82b58613e..6f06da3ba2a 100644 --- a/tools/debug-session-prep/Debug.Session.Prep/NoLddDeviceLibrariesCopier.cs +++ b/tools/debug-session-prep/Debug.Session.Prep/NoLddDeviceLibrariesCopier.cs @@ -1,8 +1,9 @@ using System; using System.Collections.Generic; -using System.IO; using System.Linq; +using ELFSharp.ELF; + using Xamarin.Android.Utilities; using Xamarin.Android.Tasks; @@ -35,8 +36,8 @@ class NoLddDeviceLibraryCopier : DeviceLibraryCopier "/system/vendor/@LIB@/mediadrm", }; - public NoLddDeviceLibraryCopier (XamarinLoggingHelper log, AdbRunner adb, bool appIs64Bit, string localDestinationDir, int deviceApiLevel) - : base (log, adb, appIs64Bit, localDestinationDir, deviceApiLevel) + public NoLddDeviceLibraryCopier (XamarinLoggingHelper log, AdbRunner adb, bool appIs64Bit, string localDestinationDir, AndroidDevice device) + : base (log, adb, appIs64Bit, localDestinationDir, device) {} public override bool Copy () @@ -55,12 +56,26 @@ public override bool Copy () AddSharedLibraries (sharedLibraries, path, permittedPaths); } + var moduleCache = new NoLddLldbModuleCache (Log, Device, sharedLibraries); + moduleCache.Populate (zygotePath); + return true; } void AddSharedLibraries (List sharedLibraries, string deviceDirPath, HashSet permittedPaths) { - (bool success, string output) = Adb.Shell ("ls", "-l", deviceDirPath).Result; + AdbRunner.OutputLineFilter filterOutErrors = (bool isStdError, string line) => { + if (!isStdError) { + return false; // don't suppress any lines on stdout + } + + // Ignore these, since we don't really care and there's no point in spamming the output with red + return + line.IndexOf ("Permission denied", StringComparison.OrdinalIgnoreCase) >= 0 || + line.IndexOf ("No such file or directory", StringComparison.OrdinalIgnoreCase) >= 0; + }; + + (bool success, string output) = Adb.Shell (filterOutErrors, "ls", "-l", deviceDirPath).Result; if (!success) { // We can't rely on `success` because `ls -l` will return an error code if the directory exists but has any entries access to whose is not permitted if (output.IndexOf ("No such file or directory") >= 0) { @@ -131,20 +146,20 @@ void AddSharedLibraries (List sharedLibraries, string deviceDirPath, Has { string lib = AppIs64Bit ? "lib64" : "lib"; - if (DeviceApiLevel == 21) { + if (Device.ApiLevel == 21) { // API21 devices (at least emulators) don't return adb error codes, so to avoid awkward error message parsing, we're going to just skip detection since we // know what API21 has and doesn't have return (GetFallbackDirs (), new HashSet ()); } - string localLdConfigPath = $"{LocalDestinationDir}{ToLocalPathFormat (LdConfigPath)}"; + string localLdConfigPath = Utilities.MakeLocalPath (LocalDestinationDir, LdConfigPath); Utilities.MakeFileDirectory (localLdConfigPath); string deviceLdConfigPath; - if (DeviceApiLevel == 28) { + if (Device.ApiLevel == 28) { deviceLdConfigPath = LdConfigPath28; - } else if (DeviceApiLevel == 29) { + } else if (Device.ApiLevel == 29) { deviceLdConfigPath = LdConfigPath29; } else { deviceLdConfigPath = LdConfigPath; diff --git a/tools/debug-session-prep/Debug.Session.Prep/NoLddLldbModuleCache.cs b/tools/debug-session-prep/Debug.Session.Prep/NoLddLldbModuleCache.cs new file mode 100644 index 00000000000..ca22fdf332d --- /dev/null +++ b/tools/debug-session-prep/Debug.Session.Prep/NoLddLldbModuleCache.cs @@ -0,0 +1,41 @@ +using System; +using System.Collections.Generic; + +using Xamarin.Android.Utilities; + +namespace Xamarin.Debug.Session.Prep; + +class NoLddLldbModuleCache : LldbModuleCache +{ + List deviceSharedLibraries; + Dictionary libraryCache; + + public NoLddLldbModuleCache (XamarinLoggingHelper log, AndroidDevice device, List deviceSharedLibraries) + : base (log, device) + { + this.deviceSharedLibraries = deviceSharedLibraries; + libraryCache = new Dictionary (StringComparer.Ordinal); + } + + protected override string? GetSharedLibraryPath (string libraryName) + { + if (libraryCache.TryGetValue (libraryName, out string? libraryPath)) { + return libraryPath; + } + + // List is sorted on the order of directories as specified by ld.config.txt, file entries aren't + // sorted inside. + foreach (string libPath in deviceSharedLibraries) { + string fileName = GetUnixFileName (libPath); + + if (String.Compare (libraryName, fileName, StringComparison.Ordinal) == 0) { + libraryCache.Add (libraryName, libPath); + return libPath; + } + } + + // Cache misses, too, the list isn't going to change + libraryCache.Add (libraryName, null); + return null; + } +} diff --git a/tools/debug-session-prep/Debug.Session.Prep/Utilities.cs b/tools/debug-session-prep/Debug.Session.Prep/Utilities.cs index 544bf64c74e..24bf8ee984d 100644 --- a/tools/debug-session-prep/Debug.Session.Prep/Utilities.cs +++ b/tools/debug-session-prep/Debug.Session.Prep/Utilities.cs @@ -60,4 +60,16 @@ public static string NormalizeDirectoryPath (string dirPath) return $"{dirPath}/"; } + + public static string ToLocalPathFormat (string path) => IsWindows ? path.Replace ("/", "\\") : path; + + public static string MakeLocalPath (string localDirectory, string remotePath) + { + string remotePathLocalFormat = ToLocalPathFormat (remotePath); + if (remotePath[0] == '/') { + return $"{localDirectory}{remotePathLocalFormat}"; + } + + return Path.Combine (localDirectory, remotePathLocalFormat); + } } diff --git a/tools/debug-session-prep/Xamarin.Android.Utilities/XamarinLoggingHelper.cs b/tools/debug-session-prep/Xamarin.Android.Utilities/XamarinLoggingHelper.cs index af31a9e824b..4167c8852b6 100644 --- a/tools/debug-session-prep/Xamarin.Android.Utilities/XamarinLoggingHelper.cs +++ b/tools/debug-session-prep/Xamarin.Android.Utilities/XamarinLoggingHelper.cs @@ -76,10 +76,16 @@ public void DebugLine (string? message = null) Debug ($"{message ?? String.Empty}{Environment.NewLine}"); } - public void StatusLine (string label, string text) + public void Status (string label, string text) { Log (LogLevel.Info, $"{label}: ", StatusLabel); - Log (LogLevel.Info, $"{text}{Environment.NewLine}", StatusText); + Log (LogLevel.Info, $"{text}", StatusText); + } + + public void StatusLine (string label, string text) + { + Status (label, text); + Log (LogLevel.Info, Environment.NewLine); } public void Log (LogLevel level, string? message) @@ -91,6 +97,12 @@ public void Log (LogLevel level, string? message) Log (level, message, ForegroundColor (level)); } + public void LogLine (LogLevel level, string? message, ConsoleColor color) + { + Log (level, message, color); + Log (level, Environment.NewLine, color); + } + public void Log (LogLevel level, string? message, ConsoleColor color) { if (!Verbose && level == LogLevel.Debug) { diff --git a/tools/debug-session-prep/debug-session-prep.csproj b/tools/debug-session-prep/debug-session-prep.csproj index cf002f923f9..f030e3a184e 100644 --- a/tools/debug-session-prep/debug-session-prep.csproj +++ b/tools/debug-session-prep/debug-session-prep.csproj @@ -28,5 +28,6 @@ + From 2361e126b1722308f45d7397126cc32e4d01ad2f Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 30 Nov 2022 22:45:26 +0100 Subject: [PATCH 17/30] [WIP] Prep app mostly done, onto the script phase --- .../Resources/debug-app.ps1 | 9 ++ .../Resources/debug-app.sh | 54 +++++++++ .../Utilities/AdbRunner.cs | 9 +- .../Xamarin.Android.Build.Tasks.csproj | 6 + .../Debug.Session.Prep/AndroidDevice.cs | 93 ++++++++++------ .../Debug.Session.Prep/AndroidNdk.cs | 2 + .../NoLddDeviceLibrariesCopier.cs | 2 - .../Debug.Session.Prep/Utilities.cs | 3 + tools/debug-session-prep/Main.cs | 104 +++++++++++++++++- .../debug-session-prep.csproj | 6 + 10 files changed, 250 insertions(+), 38 deletions(-) create mode 100644 src/Xamarin.Android.Build.Tasks/Resources/debug-app.ps1 create mode 100755 src/Xamarin.Android.Build.Tasks/Resources/debug-app.sh diff --git a/src/Xamarin.Android.Build.Tasks/Resources/debug-app.ps1 b/src/Xamarin.Android.Build.Tasks/Resources/debug-app.ps1 new file mode 100644 index 00000000000..b12418a1581 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Resources/debug-app.ps1 @@ -0,0 +1,9 @@ +# +# How to read vars from a file +# +# $vars = Get-Content -Raw ./settings.txt | ConvertFrom-StringData +# +# Access to them: +# +# $vars.one +# write-host "$($vars.one)" diff --git a/src/Xamarin.Android.Build.Tasks/Resources/debug-app.sh b/src/Xamarin.Android.Build.Tasks/Resources/debug-app.sh new file mode 100755 index 00000000000..aac26d4ab5e --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Resources/debug-app.sh @@ -0,0 +1,54 @@ +#!env bash +PACKAGE_NAME="@PACKAGE_NAME@" +OUTPUT_DIR="@OUTPUT_DIR@" +SUPPORTED_ABIS_ARRAY=(@SUPPORTED_ABIS@) +APP_LIBS_DIR="@APP_LIBS_DIR@" +NDK_DIR="@NDK_DIR@" +CONFIG_SCRIPT_NAME="@CONFIG_SCRIPT_NAME@" +LLDB_SCRIPT_NAME="@LLDB_SCRIPT_NAME@" +ADB_PATH="@ADB_PATH@" +DEBUG_SESSION_PREP_PATH="@DEBUG_SESSION_PREP_PATH@" + +function die() +{ + echo "$@" + exit 1 +} + +function die_if_failed() +{ + if [ $? -ne 0 ]; then + die "$@" + fi +} + +#TODO: APP_LIBS_DIR needs to be appended the abi-specific subdir +#TODO: make NDK_DIR overridable via a parameter +#TOOD: add a parameter to specify the Android device to target +#TODO: add a parameter to specify the arch to use, verify against both SUPPORTED_ABIS_ARRAY and the device ABIs + +SUPPORTED_ABIS_ARG="" +for sa in "${SUPPORTED_ABIS_ARRAY[@]}"; do + if [ -z "${SUPPORTED_ABIS_ARG}" ]; then + SUPPORTED_ABIS_ARG="${sa}" + else + SUPPORTED_ABIS_ARG="${SUPPORTED_ABIS_ARG},${sa}" + fi +done + +"${DEBUG_SESSION_PREP_PATH} -s "${SUPPORTED_ABIS_ARG}" \ + -p "${PACKAGE_NAME}" \ + -n "${NDK_DIR}" \ + -o "${OUTPUT_DIR}" \ + -l "${APP_LIBS_DIR}" \ + -c "${CONFIG_SCRIPT_NAME}" \ + -g "${LLDB_SCRIPT_NAME}" + +die_if_failed Debug preparation app failed + +CONFIG_SCRIPT_PATH="${OUTPUT_DIR}/${CONFIG_SCRIPT_NAME}" +if [ ! -f "${CONFIG_SCRIPT_PATH}" ]; then + die Config script ${CONFIG_SCRIPT_PATH} not found +fi + +source "${CONFIG_SCRIPT_PATH}" diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs index 631fa863cf0..6aaa1e86d73 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs @@ -247,6 +247,13 @@ async Task RunAdb (ProcessRunner runner, bool setupOutputSink = true, bool ); } - ProcessRunner CreateAdbRunner () => CreateProcessRunner (initialParams); + ProcessRunner CreateAdbRunner () + { + ProcessRunner ret = CreateProcessRunner (initialParams); + + // Let's make sure all the messages we get are in English, since we need to parse some of them to detect problems + ret.Environment["LANG"] = "C"; + return ret; + } } } diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.csproj b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.csproj index ee9e29a5c39..c0d0d38db97 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.csproj +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.csproj @@ -407,6 +407,12 @@ JavaInteropTypeManager.java + + debug-app.sh + + + debug-app.ps1 + diff --git a/tools/debug-session-prep/Debug.Session.Prep/AndroidDevice.cs b/tools/debug-session-prep/Debug.Session.Prep/AndroidDevice.cs index dae40a7795b..95bb7ff61cb 100644 --- a/tools/debug-session-prep/Debug.Session.Prep/AndroidDevice.cs +++ b/tools/debug-session-prep/Debug.Session.Prep/AndroidDevice.cs @@ -1,6 +1,6 @@ using System; +using System.Collections.Generic; using System.IO; -using System.Text; using Xamarin.Android.Tasks; using Xamarin.Android.Utilities; @@ -25,8 +25,6 @@ class AndroidDevice "ro.boot.serialno", }; - static readonly UTF8Encoding UTF8NoBOM = new UTF8Encoding (false); - string packageName; string adbPath; string[] supportedAbis; @@ -35,8 +33,10 @@ class AndroidDevice string? appLldbBaseDir; string? appLldbBinDir; string? appLldbLogDir; - string? abi; - string? arch; + string? mainAbi; + string? mainArch; + string[]? availableAbis; + string[]? availableArches; string? serialNumber; bool appIs64Bit; string? deviceLdd; @@ -49,6 +49,10 @@ class AndroidDevice AndroidNdk ndk; public int ApiLevel => apiLevel; + public string[] AvailableAbis => availableAbis ?? new string[] {}; + public string[] AvailableArches => availableArches ?? new string[] {}; + public string MainArch => mainArch ?? String.Empty; + public string MainAbi => mainAbi ?? String.Empty; public string SerialNumber => serialNumber ?? String.Empty; public AdbRunner AdbRunner => adb; @@ -104,18 +108,6 @@ public bool GatherInfo () log.StatusLine ($"Device serial number", serialNumber); } - if (!DetectTools ()) { - return false; - } - - if (!PushDebugServer ()) { - return false; - } - - if (!PullLibraries ()) { - return false; - } - return true; bool YamaOK (string output) @@ -132,6 +124,23 @@ bool PropertyIsEqualTo ((bool haveProperty, string value) result, string expecte } } + public bool Prepare () + { + if (!DetectTools ()) { + return false; + } + + if (!PushDebugServer ()) { + return false; + } + + if (!PullLibraries ()) { + return false; + } + + return true; + } + bool PullLibraries () { DeviceLibraryCopier copier; @@ -159,7 +168,7 @@ bool PullLibraries () bool PushDebugServer () { - string? debugServerPath = ndk.GetDebugServerPath (abi!); + string? debugServerPath = ndk.GetDebugServerPath (mainAbi!); if (String.IsNullOrEmpty (debugServerPath)) { return false; } @@ -188,7 +197,7 @@ bool PushDebugServer () string launcherScriptPath = Path.Combine (outputDir, ServerLauncherScriptName); Directory.CreateDirectory (Path.GetDirectoryName (launcherScriptPath)!); - File.WriteAllText (launcherScriptPath, launcherScript, UTF8NoBOM); + File.WriteAllText (launcherScriptPath, launcherScript, Utilities.UTF8NoBOM); deviceDebugServerScriptPath = $"{appLldbBinDir}/{Path.GetFileName (launcherScriptPath)}"; if (!PushServerExecutable (launcherScriptPath, deviceDebugServerScriptPath)) { @@ -348,32 +357,50 @@ bool DetermineArchitectureAndABI () LogABIs ("Application", supportedAbis); LogABIs (" Device", deviceABIs); + bool gotValidAbi = false; + var possibleAbis = new List (); + var possibleArches = new List (); + foreach (string deviceABI in deviceABIs) { foreach (string appABI in supportedAbis) { if (String.Compare (appABI, deviceABI, StringComparison.OrdinalIgnoreCase) == 0) { - abi = deviceABI; - arch = abi switch { - "armeabi" => "arm", - "armeabi-v7a" => "arm", - "arm64-v8a" => "arm64", - _ => abi, - }; - - log.StatusLine ($" Selected ABI", $"{abi} (architecture: {arch})"); - - appIs64Bit = abi.IndexOf ("64", StringComparison.Ordinal) >= 0; - return true; + string arch = AbiToArch (deviceABI); + + if (!gotValidAbi) { + mainAbi = deviceABI; + mainArch = arch; + + log.StatusLine ($" Selected ABI", $"{mainAbi} (architecture: {mainArch})"); + + appIs64Bit = mainAbi.IndexOf ("64", StringComparison.Ordinal) >= 0; + gotValidAbi = true; + } + + possibleAbis.Add (deviceABI); + possibleArches.Add (arch); } } } - log.ErrorLine ($"Application cannot run on the selected device: no matching ABI found"); - return false; + if (!gotValidAbi) { + log.ErrorLine ($"Application cannot run on the selected device: no matching ABI found"); + } + + availableAbis = possibleAbis.ToArray (); + availableArches = possibleArches.ToArray (); + return gotValidAbi; void LogABIs (string which, string[] abis) { log.StatusLine ($"{which} ABIs", String.Join (", ", abis)); } + + string AbiToArch (string abi) => abi switch { + "armeabi" => "arm", + "armeabi-v7a" => "arm", + "arm64-v8a" => "arm64", + _ => abi, + }; } string? GetFirstFoundPropertyValue (string[] propertyNames) diff --git a/tools/debug-session-prep/Debug.Session.Prep/AndroidNdk.cs b/tools/debug-session-prep/Debug.Session.Prep/AndroidNdk.cs index 58df546dde1..48ec5dca23c 100644 --- a/tools/debug-session-prep/Debug.Session.Prep/AndroidNdk.cs +++ b/tools/debug-session-prep/Debug.Session.Prep/AndroidNdk.cs @@ -21,6 +21,8 @@ class AndroidNdk XamarinLoggingHelper log; string? lldbPath; + public string LldbPath => lldbPath ?? String.Empty; + public AndroidNdk (XamarinLoggingHelper log, string ndkRootPath, string[] supportedAbis) { this.log = log; diff --git a/tools/debug-session-prep/Debug.Session.Prep/NoLddDeviceLibrariesCopier.cs b/tools/debug-session-prep/Debug.Session.Prep/NoLddDeviceLibrariesCopier.cs index 6f06da3ba2a..9193bf77adb 100644 --- a/tools/debug-session-prep/Debug.Session.Prep/NoLddDeviceLibrariesCopier.cs +++ b/tools/debug-session-prep/Debug.Session.Prep/NoLddDeviceLibrariesCopier.cs @@ -2,8 +2,6 @@ using System.Collections.Generic; using System.Linq; -using ELFSharp.ELF; - using Xamarin.Android.Utilities; using Xamarin.Android.Tasks; diff --git a/tools/debug-session-prep/Debug.Session.Prep/Utilities.cs b/tools/debug-session-prep/Debug.Session.Prep/Utilities.cs index 24bf8ee984d..5ff6f08075c 100644 --- a/tools/debug-session-prep/Debug.Session.Prep/Utilities.cs +++ b/tools/debug-session-prep/Debug.Session.Prep/Utilities.cs @@ -2,6 +2,7 @@ using System.IO; using System.Reflection; using System.Runtime.InteropServices; +using System.Text; using Xamarin.Android.Utilities; @@ -9,6 +10,8 @@ namespace Xamarin.Debug.Session.Prep; static class Utilities { + public static readonly UTF8Encoding UTF8NoBOM = new UTF8Encoding (false); + public static bool IsMacOS { get; private set; } public static bool IsLinux { get; private set; } public static bool IsWindows { get; private set; } diff --git a/tools/debug-session-prep/Main.cs b/tools/debug-session-prep/Main.cs index 85dcd39911c..11e17001626 100644 --- a/tools/debug-session-prep/Main.cs +++ b/tools/debug-session-prep/Main.cs @@ -1,5 +1,7 @@ using System; using System.Collections.Generic; +using System.IO; +using System.Text; using Mono.Options; using Xamarin.Android.Utilities; @@ -32,6 +34,8 @@ sealed class ParsedOptions public string? AppNativeLibrariesDir; public string? NdkDirPath; public string? OutputDirPath; + public string? ConfigScriptName; + public string? LldbScriptName; } static int Main (string[] args) @@ -48,9 +52,11 @@ static int Main (string[] args) "REQUIRED_OPTIONS are:", { "p|package-name=", "name of the application package", v => parsedOptions.PackageName = EnsureNonEmptyString (log, "-p|--package-name", v, ref haveOptionErrors) }, { "s|supported-abis=", "comma-separated list of ABIs the application supports", v => parsedOptions.SupportedABIs = EnsureSupportedABIs (log, "-s|--supported-abis", v, ref haveOptionErrors) }, - { "l|lib-dir=", "{PATH} to the directory where application native libraries were copied", v => parsedOptions.AppNativeLibrariesDir = v }, + { "l|lib-dir=", "{PATH} to the directory where application native libraries were copied (relative to output directory, below)", v => parsedOptions.AppNativeLibrariesDir = v }, { "n|ndk-dir=", "{PATH} to to the Android NDK root directory", v => parsedOptions.NdkDirPath = v }, { "o|output-dir=", "{PATH} to directory which will contain various generated files (logs, scripts etc)", v => parsedOptions.OutputDirPath = v }, + { "c|config-script=", "{NAME} of the launcher configuration script which will be created in the output directory", v => parsedOptions.ConfigScriptName = v }, + { "g|lldb-script=", "{NAME} of the LLDB script which will be created in the output directory", v => parsedOptions.LldbScriptName = v }, "", "OPTIONS are:", { "a|adb=", "{PATH} to adb to use for this session", v => parsedOptions.AdbPath = EnsureNonEmptyString (log, "-a|--adb", v, ref haveOptionErrors) }, @@ -98,7 +104,17 @@ static int Main (string[] args) if (String.IsNullOrEmpty (parsedOptions.AppNativeLibrariesDir)) { log.ErrorLine ("The '-l|--lib-dir' option must be used to specify the directory where application shared libraries were copied"); - // missingRequiredOptions = true; + missingRequiredOptions = true; + } + + if (String.IsNullOrEmpty (parsedOptions.ConfigScriptName)) { + log.ErrorLine ("The '-c|--config-script' option must be used to specify name of the launcher configuration script"); + missingRequiredOptions = true; + } + + if (String.IsNullOrEmpty (parsedOptions.LldbScriptName)) { + log.ErrorLine ("The '-g|--lldb-script' option must be used to specify name of the LLDB script"); + missingRequiredOptions = true; } if (missingRequiredOptions) { @@ -125,9 +141,93 @@ static int Main (string[] args) return 1; } + if (!device.Prepare ()) { + log.ErrorLine ("Failed to prepare for debugging session"); + return 1; + } + + string socketScheme = "unix-abstract"; + string socketDir = $"/xa-{parsedOptions.PackageName}"; + + var rnd = new Random (); + string socketName = $"xa-platform-{rnd.NextInt64 ()}.sock"; + + WriteConfigScript (parsedOptions, device, ndk, socketScheme, socketDir, socketName); + WriteLldbScript (parsedOptions, socketScheme, socketDir, socketName); + return 0; } + static FileStream OpenScriptStream (string path) + { + return File.Open (path, FileMode.Create, FileAccess.Write, FileShare.Read); + } + + static StreamWriter OpenScriptWriter (FileStream fs) + { + return new StreamWriter (fs, Utilities.UTF8NoBOM); + } + + static void WriteLldbScript (ParsedOptions parsedOptions, string socketScheme, string socketDir, string socketName) + { + string outputFile = Path.Combine (parsedOptions.OutputDirPath!, parsedOptions.LldbScriptName!); + string fullLibsDir = Path.GetFullPath (Path.Combine (parsedOptions.OutputDirPath!, parsedOptions.AppNativeLibrariesDir!)); + using FileStream fs = OpenScriptStream (outputFile); + using StreamWriter sw = OpenScriptWriter (fs); + + // TODO: add support for appending user commands + sw.WriteLine ($"settings append target.exec-search-paths \"{fullLibsDir}\""); + sw.WriteLine ("platform remote-android"); + sw.WriteLine ($"platform connect {socketScheme}-connect:///{socketDir}/{socketName}"); + sw.WriteLine ("gui"); // TODO: make it optional + sw.Flush (); + } + + static void WriteConfigScript (ParsedOptions parsedOptions, AndroidDevice device, AndroidNdk ndk, string socketScheme, string socketDir, string socketName) + { + bool powershell = Utilities.IsWindows; + string outputFile = Path.Combine (parsedOptions.OutputDirPath!, parsedOptions.ConfigScriptName!); + using FileStream fs = OpenScriptStream (outputFile); + using StreamWriter sw = OpenScriptWriter (fs); + + sw.WriteLine ($"DEVICE_SERIAL=\"{device.SerialNumber}\""); + sw.WriteLine ($"DEVICE_API_LEVEL={device.ApiLevel}"); + sw.WriteLine ($"DEVICE_MAIN_ABI={device.MainAbi}"); + sw.WriteLine ($"DEVICE_MAIN_ARCH={device.MainArch}"); + sw.WriteLine ($"DEVICE_AVAILABLE_ABIS={FormatArray (device.AvailableAbis)}"); + sw.WriteLine ($"DEVICE_AVAILABLE_ARCHES={FormatArray (device.AvailableArches)}"); + sw.WriteLine ($"SOCKET_SCHEME={socketScheme}"); + sw.WriteLine ($"SOCKET_DIR={socketDir}"); + sw.WriteLine ($"SOCKET_NAME={socketName}"); + sw.WriteLine ($"LLDB_PATH=\"{ndk.LldbPath}\""); + sw.Flush (); + + string FormatArray (string[] values) + { + var sb = new StringBuilder (); + if (powershell) { + sb.Append ('@'); + } + sb.Append ('('); + + bool first = true; + foreach (string v in values) { + if (first) { + first = false; + } else { + sb.Append (powershell ? ", " : " "); + } + sb.Append ('"'); + sb.Append (v); + sb.Append ('"'); + } + + sb.Append (')'); + + return sb.ToString (); + } + } + static string[]? EnsureSupportedABIs (XamarinLoggingHelper log, string paramName, string? value, ref bool haveOptionErrors) { string? abis = EnsureNonEmptyString (log, paramName, value, ref haveOptionErrors); diff --git a/tools/debug-session-prep/debug-session-prep.csproj b/tools/debug-session-prep/debug-session-prep.csproj index f030e3a184e..4d72bd741de 100644 --- a/tools/debug-session-prep/debug-session-prep.csproj +++ b/tools/debug-session-prep/debug-session-prep.csproj @@ -24,6 +24,12 @@ lldb-debug-session.sh + + debug-app.sh + + + debug-app.ps1 + From 033954a7d22d9c668f052c1a637eba898f2a2b26 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 1 Dec 2022 22:29:57 +0100 Subject: [PATCH 18/30] [WIP] Install the console app in tools; script progress --- Xamarin.Android.sln | 7 + .../create-packs/Microsoft.Android.Sdk.proj | 4 + .../Resources/debug-app.sh | 35 ++-- .../Tasks/DebugNativeCode.cs | 17 +- .../Utilities/NativeDebugPrep.cs | 183 ++++++++++++++++++ .../Utilities/NativeDebugger.cs | 8 +- .../NoLddDeviceLibrariesCopier.cs | 4 +- .../debug-session-prep.csproj | 12 +- 8 files changed, 231 insertions(+), 39 deletions(-) create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/NativeDebugPrep.cs diff --git a/Xamarin.Android.sln b/Xamarin.Android.sln index baba28b1734..ad7fa0c0178 100644 --- a/Xamarin.Android.sln +++ b/Xamarin.Android.sln @@ -156,6 +156,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Java.Runtime.Environment", EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "create-android-api", "build-tools\create-android-api\create-android-api.csproj", "{BA4D889D-066B-4C2C-A973-09E319CBC396}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "debug-session-prep", "tools\debug-session-prep\debug-session-prep.csproj", "{087C42C4-6B45-4020-AB39-52515265082E}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|AnyCPU = Debug|AnyCPU @@ -416,6 +418,10 @@ Global {D8E14B43-E929-4C18-9FA6-2C3DC47EFC17}.Debug|AnyCPU.Build.0 = Debug|Any CPU {D8E14B43-E929-4C18-9FA6-2C3DC47EFC17}.Release|AnyCPU.ActiveCfg = Release|Any CPU {D8E14B43-E929-4C18-9FA6-2C3DC47EFC17}.Release|AnyCPU.Build.0 = Release|Any CPU + {087C42C4-6B45-4020-AB39-52515265082E}.Debug|AnyCPU.ActiveCfg = Debug|Any CPU + {087C42C4-6B45-4020-AB39-52515265082E}.Debug|AnyCPU.Build.0 = Debug|Any CPU + {087C42C4-6B45-4020-AB39-52515265082E}.Release|AnyCPU.ActiveCfg = Release|Any CPU + {087C42C4-6B45-4020-AB39-52515265082E}.Release|AnyCPU.Build.0 = Release|Any CPU {C0E44558-FEE3-4DD3-986A-3F46DD1BF41B}.Debug|AnyCPU.ActiveCfg = Debug|Any CPU {C0E44558-FEE3-4DD3-986A-3F46DD1BF41B}.Debug|AnyCPU.Build.0 = Debug|Any CPU {C0E44558-FEE3-4DD3-986A-3F46DD1BF41B}.Release|AnyCPU.ActiveCfg = Release|Any CPU @@ -493,6 +499,7 @@ Global {DA50FC92-7FE7-48B5-BDB6-CDA57B37BB51} = {864062D3-A415-4A6F-9324-5820237BA058} {4EFCED6E-9A6B-453A-94E4-CE4B736EC684} = {864062D3-A415-4A6F-9324-5820237BA058} {D8E14B43-E929-4C18-9FA6-2C3DC47EFC17} = {864062D3-A415-4A6F-9324-5820237BA058} + {087C42C4-6B45-4020-AB39-52515265082E} = {864062D3-A415-4A6F-9324-5820237BA058} {C0E44558-FEE3-4DD3-986A-3F46DD1BF41B} = {04E3E11E-B47D-4599-8AFC-50515A95E715} {BA4D889D-066B-4C2C-A973-09E319CBC396} = {E351F97D-EA4F-4E7F-AAA0-8EBB1F2A4A62} EndGlobalSection diff --git a/build-tools/create-packs/Microsoft.Android.Sdk.proj b/build-tools/create-packs/Microsoft.Android.Sdk.proj index 7deb4a8975a..957a493d033 100644 --- a/build-tools/create-packs/Microsoft.Android.Sdk.proj +++ b/build-tools/create-packs/Microsoft.Android.Sdk.proj @@ -51,6 +51,10 @@ core workload SDK packs imported by WorkloadManifest.targets. <_PackageFiles Include="$(MicrosoftAndroidSdkOutDir)class-parse.dll" PackagePath="tools" /> <_PackageFiles Include="$(MicrosoftAndroidSdkOutDir)class-parse.pdb" PackagePath="tools" /> <_PackageFiles Include="$(MicrosoftAndroidSdkOutDir)class-parse.runtimeconfig.json" PackagePath="tools" /> + <_PackageFiles Include="$(MicrosoftAndroidSdkOutDir)debug-session-prep.deps.json" PackagePath="tools" /> + <_PackageFiles Include="$(MicrosoftAndroidSdkOutDir)debug-session-prep.dll" PackagePath="tools" /> + <_PackageFiles Include="$(MicrosoftAndroidSdkOutDir)debug-session-prep.pdb" PackagePath="tools" /> + <_PackageFiles Include="$(MicrosoftAndroidSdkOutDir)debug-session-prep.runtimeconfig.json" PackagePath="tools" /> <_PackageFiles Include="$(MicrosoftAndroidSdkOutDir)generator.dll" PackagePath="tools" /> <_PackageFiles Include="$(MicrosoftAndroidSdkOutDir)generator.pdb" PackagePath="tools" /> <_PackageFiles Include="$(MicrosoftAndroidSdkOutDir)generator.runtimeconfig.json" PackagePath="tools" /> diff --git a/src/Xamarin.Android.Build.Tasks/Resources/debug-app.sh b/src/Xamarin.Android.Build.Tasks/Resources/debug-app.sh index aac26d4ab5e..a04acb94c7e 100755 --- a/src/Xamarin.Android.Build.Tasks/Resources/debug-app.sh +++ b/src/Xamarin.Android.Build.Tasks/Resources/debug-app.sh @@ -1,13 +1,16 @@ -#!env bash -PACKAGE_NAME="@PACKAGE_NAME@" -OUTPUT_DIR="@OUTPUT_DIR@" -SUPPORTED_ABIS_ARRAY=(@SUPPORTED_ABIS@) +#!/bin/bash +MY_DIR="$(cd $(dirname $0);pwd)" + +ADB_PATH="@ADB_PATH@" APP_LIBS_DIR="@APP_LIBS_DIR@" -NDK_DIR="@NDK_DIR@" CONFIG_SCRIPT_NAME="@CONFIG_SCRIPT_NAME@" -LLDB_SCRIPT_NAME="@LLDB_SCRIPT_NAME@" -ADB_PATH="@ADB_PATH@" DEBUG_SESSION_PREP_PATH="@DEBUG_SESSION_PREP_PATH@" +DEFAULT_ACTIVITY_NAME="@ACTIVITY_NAME@" +LLDB_SCRIPT_NAME="@LLDB_SCRIPT_NAME@" +NDK_DIR="@NDK_DIR@" +OUTPUT_DIR="@OUTPUT_DIR@" +PACKAGE_NAME="@PACKAGE_NAME@" +SUPPORTED_ABIS_ARRAY=(@SUPPORTED_ABIS@) function die() { @@ -26,6 +29,7 @@ function die_if_failed() #TODO: make NDK_DIR overridable via a parameter #TOOD: add a parameter to specify the Android device to target #TODO: add a parameter to specify the arch to use, verify against both SUPPORTED_ABIS_ARRAY and the device ABIs +#TODO: detect whether we have dotnet in $PATH and whether it's a compatible version SUPPORTED_ABIS_ARG="" for sa in "${SUPPORTED_ABIS_ARRAY[@]}"; do @@ -36,19 +40,20 @@ for sa in "${SUPPORTED_ABIS_ARRAY[@]}"; do fi done -"${DEBUG_SESSION_PREP_PATH} -s "${SUPPORTED_ABIS_ARG}" \ - -p "${PACKAGE_NAME}" \ - -n "${NDK_DIR}" \ - -o "${OUTPUT_DIR}" \ - -l "${APP_LIBS_DIR}" \ - -c "${CONFIG_SCRIPT_NAME}" \ - -g "${LLDB_SCRIPT_NAME}" +dotnet "${DEBUG_SESSION_PREP_PATH}" \ + -s "${SUPPORTED_ABIS_ARG}" \ + -p "${PACKAGE_NAME}" \ + -n "${NDK_DIR}" \ + -o "${OUTPUT_DIR}" \ + -l "${APP_LIBS_DIR}" \ + -c "${CONFIG_SCRIPT_NAME}" \ + -g "${LLDB_SCRIPT_NAME}" die_if_failed Debug preparation app failed CONFIG_SCRIPT_PATH="${OUTPUT_DIR}/${CONFIG_SCRIPT_NAME}" if [ ! -f "${CONFIG_SCRIPT_PATH}" ]; then - die Config script ${CONFIG_SCRIPT_PATH} not found + die Config script ${CONFIG_SCRIPT_PATH} not found fi source "${CONFIG_SCRIPT_PATH}" diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/DebugNativeCode.cs b/src/Xamarin.Android.Build.Tasks/Tasks/DebugNativeCode.cs index b30083e1236..bc8bdb1c89d 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/DebugNativeCode.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/DebugNativeCode.cs @@ -61,20 +61,17 @@ public async override System.Threading.Tasks.Task RunTaskAsync () abiLibs.Add (item.ItemSpec); } - var debugger = new NativeDebugger ( - Log, + var prep = new NativeDebugPrep (Log); + prep.Prepare ( AdbPath, AndroidNdkPath, + String.IsNullOrEmpty (ActivityName) ? MainActivityName : ActivityName, IntermediateOutputDir, PackageName, - SupportedAbis - ) { - AdbDeviceTarget = TargetDeviceName, - NativeLibrariesPerABI = nativeLibs, - }; - - string activity = String.IsNullOrEmpty (ActivityName) ? MainActivityName : ActivityName; - debugger.Launch (activity); + SupportedAbis, + nativeLibs, + TargetDeviceName + ); } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeDebugPrep.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeDebugPrep.cs new file mode 100644 index 00000000000..22cf8aa9967 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeDebugPrep.cs @@ -0,0 +1,183 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Reflection; +using System.Runtime.InteropServices; +using System.Text; + +using Microsoft.Android.Build.Tasks; +using Microsoft.Build.Utilities; +using Xamarin.Android.Tools; + +namespace Xamarin.Android.Tasks +{ + class NativeDebugPrep + { + const string ConfigScriptName = "debug-app-config"; + const string LldbScriptName = "lldb.x"; + + static HashSet xaLibraries = new HashSet (StringComparer.Ordinal) { + "libmonodroid.so", + "libxamarin-app.so", + "libxamarin-debug-app-helper.so", + }; + + TaskLoggingHelper log; + + public NativeDebugPrep (TaskLoggingHelper logger) + { + log = logger; + } + + [DllImport ("c", SetLastError=true, EntryPoint="chmod")] + private static extern int chmod (string path, uint mode); + + public void Prepare (string adbPath, string ndkRootPath, string activityName, + string outputDirRoot, string packageName, string[] supportedAbis, + Dictionary> nativeLibsPerAbi, string? targetDevice) + { + bool isWindows = OS.IsWindows; + string scriptResourceName = isWindows ? "debug-app.ps1" : "debug-app.sh"; + string? script = MonoAndroidHelper.ReadManifestResource (log, scriptResourceName); + + if (String.IsNullOrEmpty (script)) { + log.LogError ($"Failed to read script resource '{scriptResourceName}'"); + return; + } + + string? appLibrariesRoot = CopyLibraries (outputDirRoot, nativeLibsPerAbi); + string scriptConfigExt = isWindows ? ".ps1" : ".sh"; + string scriptOutput = Path.Combine (outputDirRoot, scriptResourceName); + + var sb = new StringBuilder (script); + + // TODO: perhaps use relative paths for APP_LIBS_DIR and OUTPUT_DIR? + sb.Replace ("@ACTIVITY_NAME@", activityName); + sb.Replace ("@ADB_PATH@", adbPath); + sb.Replace ("@APP_LIBS_DIR@", appLibrariesRoot ?? String.Empty); + sb.Replace ("@CONFIG_SCRIPT_NAME@", $"{ConfigScriptName}{scriptConfigExt}"); + sb.Replace ("@DEBUG_SESSION_PREP_PATH@", Path.Combine (Path.GetDirectoryName (typeof(NativeDebugPrep).Assembly.Location), "debug-session-prep.dll")); + sb.Replace ("@LLDB_SCRIPT_NAME@", LldbScriptName); + sb.Replace ("@NDK_DIR@", Path.GetFullPath (ndkRootPath)); + sb.Replace ("@OUTPUT_DIR@", Path.GetFullPath (outputDirRoot)); + sb.Replace ("@PACKAGE_NAME@", packageName); + + var abis = new StringBuilder (); + bool first = true; + foreach (string abi in supportedAbis) { + if (first) { + first = false; + } else { + abis.Append (isWindows ? ", " : " "); + } + abis.Append ($"\"{abi}\""); + } + sb.Replace ("@SUPPORTED_ABIS@", abis.ToString ()); + + Directory.CreateDirectory (Path.GetDirectoryName (scriptOutput)); + + using var fs = File.Open (scriptOutput, FileMode.Create, FileAccess.Write, FileShare.Read); + using var sw = new StreamWriter (fs, Files.UTF8withoutBOM); + + sw.Write (sb.ToString ()); + sw.Flush (); + sw.Close (); + fs.Close (); + + // 493 decimal is 0755 octal - makes the file rwx for the owner and rx for everybody else + if (!isWindows && chmod (scriptOutput, 493) != 0) { + log.LogWarning ($"Failed to make {scriptOutput} executable"); + } + + // TODO: color? + Console.WriteLine (); + Console.WriteLine ("You can start the debugging session by running the following command now:"); + Console.WriteLine ($" {scriptOutput}"); + Console.WriteLine (); + } + + string? CopyLibraries (string outputDirRoot, Dictionary> nativeLibsPerAbi) + { + if (nativeLibsPerAbi.Count == 0) { + return null; + } + + string appLibsRoot = Path.Combine (outputDirRoot, "app", "lib"); + log.LogDebugMessage ($"Copying application native libararies to {appLibsRoot}"); + + DotnetSymbolRunner? dotnetSymbol = GetDotnetSymbolRunner (); + bool haveLibsWithoutSymbols = false; + foreach (var kvp in nativeLibsPerAbi) { + string abi = kvp.Key; + List libs = kvp.Value; + + string abiDir = Path.Combine (appLibsRoot, abi); + foreach (string library in libs) { + log.LogDebugMessage ($" [{abi}] {library}"); + + string fileName = Path.GetFileName (library); + if (fileName.StartsWith ("libmono-android.")) { + fileName = "libmonodroid.so"; + } + + string destPath = Path.Combine (appLibsRoot, abi, fileName); + Directory.CreateDirectory (Path.GetDirectoryName (destPath)); + File.Copy (library, destPath, true); + + if (!EnsureSharedLibraryHasSymboles (destPath, dotnetSymbol)) { + haveLibsWithoutSymbols = true; + } + } + } + + if (haveLibsWithoutSymbols) { + log.LogWarning ($"One or more native libraries have no debug symbols."); + if (dotnetSymbol == null) { + log.LogWarning ($"The dotnet-symbol tool was not found. It can be installed using: dotnet tool install -g dotnet-symbol"); + } + } + + return Path.GetFullPath (appLibsRoot); + } + + bool EnsureSharedLibraryHasSymboles (string libraryPath, DotnetSymbolRunner? dotnetSymbol) + { + bool tryToFetchSymbols = false; + bool hasSymbols = ELFHelper.HasDebugSymbols (log, libraryPath, out bool usesDebugLink); + string libName = Path.GetFileName (libraryPath); + + if (!xaLibraries.Contains (libName)) { + if (ELFHelper.IsAOTLibrary (log, libraryPath)) { + return true; // We don't care about symbols, AOT libraries are only data + } + + // It might be a framework shared library, we'll try to fetch symbols if necessary and possible + tryToFetchSymbols = !hasSymbols && usesDebugLink; + } + + if (tryToFetchSymbols && dotnetSymbol != null) { + log.LogMessage ($"Attempting to download debug symbols from symbol server"); + if (!dotnetSymbol.Fetch (libraryPath).Result) { + log.LogWarning ($"Failed to download debug symbols for {libraryPath}"); + } + } + + hasSymbols = ELFHelper.HasDebugSymbols (log, libraryPath); + return hasSymbols; + } + + DotnetSymbolRunner? GetDotnetSymbolRunner () + { + string dotnetSymbolPath = Path.Combine (Environment.GetFolderPath (Environment.SpecialFolder.UserProfile), ".dotnet", "tools", "dotnet-symbol"); + if (OS.IsWindows) { + dotnetSymbolPath = $"{dotnetSymbolPath}.exe"; + } + + if (!File.Exists (dotnetSymbolPath)) { + return null; + } + + return new DotnetSymbolRunner (log, dotnetSymbolPath); + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeDebugger.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeDebugger.cs index 76bb6a3431e..8c63b12d763 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/NativeDebugger.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeDebugger.cs @@ -20,10 +20,10 @@ namespace Xamarin.Android.Tasks class NativeDebugger { const ConsoleColor ErrorColor = ConsoleColor.Red; - const ConsoleColor DebugColor = ConsoleColor.DarkGray; - const ConsoleColor InfoColor = ConsoleColor.Green; - const ConsoleColor MessageColor = ConsoleColor.Gray; - const ConsoleColor WarningColor = ConsoleColor.Yellow; + const ConsoleColor DebugColor = ConsoleColor.DarkGray; + const ConsoleColor InfoColor = ConsoleColor.Green; + const ConsoleColor MessageColor = ConsoleColor.Gray; + const ConsoleColor WarningColor = ConsoleColor.Yellow; const ConsoleColor StatusLabel = ConsoleColor.Cyan; const ConsoleColor StatusText = ConsoleColor.White; diff --git a/tools/debug-session-prep/Debug.Session.Prep/NoLddDeviceLibrariesCopier.cs b/tools/debug-session-prep/Debug.Session.Prep/NoLddDeviceLibrariesCopier.cs index 9193bf77adb..8ff3a07c922 100644 --- a/tools/debug-session-prep/Debug.Session.Prep/NoLddDeviceLibrariesCopier.cs +++ b/tools/debug-session-prep/Debug.Session.Prep/NoLddDeviceLibrariesCopier.cs @@ -76,7 +76,7 @@ void AddSharedLibraries (List sharedLibraries, string deviceDirPath, Has (bool success, string output) = Adb.Shell (filterOutErrors, "ls", "-l", deviceDirPath).Result; if (!success) { // We can't rely on `success` because `ls -l` will return an error code if the directory exists but has any entries access to whose is not permitted - if (output.IndexOf ("No such file or directory") >= 0) { + if (output.IndexOf ("No such file or directory", StringComparison.OrdinalIgnoreCase) >= 0) { Log.DebugLine ($"Shared libraries directory {deviceDirPath} not found on device"); return; } @@ -122,7 +122,7 @@ void AddSharedLibraries (List sharedLibraries, string deviceDirPath, Has const string SymlinkArrow = "->"; // Symlink, we'll add the target library instead - int idx = name.IndexOf (SymlinkArrow); + int idx = name.IndexOf (SymlinkArrow, StringComparison.Ordinal); if (idx > 0) { libPath = name.Substring (idx + SymlinkArrow.Length).Trim (); } else { diff --git a/tools/debug-session-prep/debug-session-prep.csproj b/tools/debug-session-prep/debug-session-prep.csproj index 4d72bd741de..5bca33d124f 100644 --- a/tools/debug-session-prep/debug-session-prep.csproj +++ b/tools/debug-session-prep/debug-session-prep.csproj @@ -1,14 +1,16 @@ + + Exe + False + $(MicrosoftAndroidSdkOutDir) net7.0 debug_session_prep enable NO_MSBUILD - - @@ -24,12 +26,6 @@ lldb-debug-session.sh - - debug-app.sh - - - debug-app.ps1 - From 5a013dc263c27fe0efe859bb97a38aaf3cdfc368 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Fri, 2 Dec 2022 23:20:31 +0100 Subject: [PATCH 19/30] [WIP] Change of direction New direction - going to implement everything as a dotnet tool, with support for debugging an .apk directly (as well as for building the apk, if necessary and possible) --- Xamarin.Android.sln | 9 +- .../Resources/debug-app.sh | 97 +++++++++- .../Utilities/NativeDebugPrep.cs | 2 +- .../Debug.Session.Prep/AndroidDevice.cs | 32 +++- .../DeviceLibrariesCopier.cs | 2 +- .../LddDeviceLibrariesCopier.cs | 2 +- .../NoLddDeviceLibrariesCopier.cs | 4 +- tools/debug-session-prep/Main.cs | 41 +++-- .../Resources/xa_start_lldb_server.sh | 2 +- tools/xadebug/LICENSE | 24 +++ tools/xadebug/Main.cs | 106 +++++++++++ tools/xadebug/README.md | 1 + .../Xamarin.Android.Debug/AXMLParser.cs | 9 + .../Xamarin.Android.Debug/Utilities.cs | 78 ++++++++ .../XamarinLoggingHelper.cs | 174 ++++++++++++++++++ tools/xadebug/xadebug.csproj | 43 +++++ 16 files changed, 597 insertions(+), 29 deletions(-) create mode 100644 tools/xadebug/LICENSE create mode 100644 tools/xadebug/Main.cs create mode 100644 tools/xadebug/README.md create mode 100644 tools/xadebug/Xamarin.Android.Debug/AXMLParser.cs create mode 100644 tools/xadebug/Xamarin.Android.Debug/Utilities.cs create mode 100644 tools/xadebug/Xamarin.Android.Utilities/XamarinLoggingHelper.cs create mode 100644 tools/xadebug/xadebug.csproj diff --git a/Xamarin.Android.sln b/Xamarin.Android.sln index ad7fa0c0178..3163b084448 100644 --- a/Xamarin.Android.sln +++ b/Xamarin.Android.sln @@ -158,6 +158,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "create-android-api", "build EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "debug-session-prep", "tools\debug-session-prep\debug-session-prep.csproj", "{087C42C4-6B45-4020-AB39-52515265082E}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "xadebug", "tools\xadebug\xadebug.csproj", "{F4D105FA-D394-437A-B3BA-0EC56C6734C3}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|AnyCPU = Debug|AnyCPU @@ -300,12 +302,10 @@ Global {B8105878-D423-4159-A3E7-028298281EC6}.Debug|AnyCPU.Build.0 = Debug|Any CPU {B8105878-D423-4159-A3E7-028298281EC6}.Release|AnyCPU.ActiveCfg = Release|Any CPU {B8105878-D423-4159-A3E7-028298281EC6}.Release|AnyCPU.Build.0 = Release|Any CPU - {43564FB3-0F79-4FF4-A2B0-B1637072FF01}.Debug|AnyCPU.ActiveCfg = Debug|Any CPU {43564FB3-0F79-4FF4-A2B0-B1637072FF01}.Debug|AnyCPU.Build.0 = Debug|Any CPU {43564FB3-0F79-4FF4-A2B0-B1637072FF01}.Release|AnyCPU.ActiveCfg = Release|Any CPU {43564FB3-0F79-4FF4-A2B0-B1637072FF01}.Release|AnyCPU.Build.0 = Release|Any CPU - {3DE17662-DCD6-4F49-AF06-D39AACC8649A}.Debug|AnyCPU.ActiveCfg = Debug|Any CPU {3DE17662-DCD6-4F49-AF06-D39AACC8649A}.Debug|AnyCPU.Build.0 = Debug|Any CPU {3DE17662-DCD6-4F49-AF06-D39AACC8649A}.Release|AnyCPU.ActiveCfg = Release|Any CPU @@ -430,6 +430,10 @@ Global {BA4D889D-066B-4C2C-A973-09E319CBC396}.Debug|AnyCPU.Build.0 = Debug|Any CPU {BA4D889D-066B-4C2C-A973-09E319CBC396}.Release|AnyCPU.ActiveCfg = Release|Any CPU {BA4D889D-066B-4C2C-A973-09E319CBC396}.Release|AnyCPU.Build.0 = Release|Any CPU + {F4D105FA-D394-437A-B3BA-0EC56C6734C3}.Debug|AnyCPU.ActiveCfg = Debug|Any CPU + {F4D105FA-D394-437A-B3BA-0EC56C6734C3}.Debug|AnyCPU.Build.0 = Debug|Any CPU + {F4D105FA-D394-437A-B3BA-0EC56C6734C3}.Release|AnyCPU.ActiveCfg = Release|Any CPU + {F4D105FA-D394-437A-B3BA-0EC56C6734C3}.Release|AnyCPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -502,6 +506,7 @@ Global {087C42C4-6B45-4020-AB39-52515265082E} = {864062D3-A415-4A6F-9324-5820237BA058} {C0E44558-FEE3-4DD3-986A-3F46DD1BF41B} = {04E3E11E-B47D-4599-8AFC-50515A95E715} {BA4D889D-066B-4C2C-A973-09E319CBC396} = {E351F97D-EA4F-4E7F-AAA0-8EBB1F2A4A62} + {F4D105FA-D394-437A-B3BA-0EC56C6734C3} = {864062D3-A415-4A6F-9324-5820237BA058} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {53A1F287-EFB2-4D97-A4BB-4A5E145613F6} diff --git a/src/Xamarin.Android.Build.Tasks/Resources/debug-app.sh b/src/Xamarin.Android.Build.Tasks/Resources/debug-app.sh index a04acb94c7e..fcc0b4e1ea2 100755 --- a/src/Xamarin.Android.Build.Tasks/Resources/debug-app.sh +++ b/src/Xamarin.Android.Build.Tasks/Resources/debug-app.sh @@ -11,6 +11,7 @@ NDK_DIR="@NDK_DIR@" OUTPUT_DIR="@OUTPUT_DIR@" PACKAGE_NAME="@PACKAGE_NAME@" SUPPORTED_ABIS_ARRAY=(@SUPPORTED_ABIS@) +ADB_DEVICE="" function die() { @@ -25,11 +26,59 @@ function die_if_failed() fi } +function run_adb_no_echo() +{ + local args="" + + if [ -n "${ADB_DEVICE}" ]; then + args="-s ${ADB_DEVICE}" + fi + + adb ${args} "$@" +} + +function run_adb() +{ + echo Running: adb ${args} "$@" + run_adb_no_echo "$@" +} + +function run_adb_echo_info() +{ + local log_file="$1" + shift + + echo Running: adb ${args} "$@" + echo Logging to: "${log_file}" + echo +} + +function run_adb_with_log() +{ + local log_file="$1" + shift + + run_adb_echo_info "${log_file}" "$@" + run_adb_no_echo "$@" > "${log_file}" 2>&1 +} + +function run_adb_with_log_bg() +{ + local log_file="$1" + shift + + run_adb_echo_info "${log_file}" "$@" + run_adb_no_echo "$@" > "${log_file}" 2>&1 & +} + #TODO: APP_LIBS_DIR needs to be appended the abi-specific subdir #TODO: make NDK_DIR overridable via a parameter #TOOD: add a parameter to specify the Android device to target -#TODO: add a parameter to specify the arch to use, verify against both SUPPORTED_ABIS_ARRAY and the device ABIs #TODO: detect whether we have dotnet in $PATH and whether it's a compatible version +#TODO: add an option to make XA wait for debugger to connect +#TODO: add a parameter to specify activity to start + +ACTIVITY_NAME="${DEFAULT_ACTIVITY_NAME}" SUPPORTED_ABIS_ARG="" for sa in "${SUPPORTED_ABIS_ARRAY[@]}"; do @@ -57,3 +106,49 @@ if [ ! -f "${CONFIG_SCRIPT_PATH}" ]; then fi source "${CONFIG_SCRIPT_PATH}" + +# Determine cross section of supported and device ABIs +ALLOWED_ABIS=() + +for dabi in "${DEVICE_AVAILABLE_ABIS[@]}"; do + for sabi in "${SUPPORTED_ABIS_ARRAY[@]}"; do + if [ "${dabi}" == "${sabi}" ]; then + ALLOWED_ABIS+="${dabi}" + fi + done +done + +if [ ${#ALLOWED_ABIS[@]} -le 0 ]; then + die Application does not support any ABIs available on device +fi + +ADB_DEBUG_SERVER_LOG="${OUTPUT_DIR}/lldb-debug-server.log" +echo Starting debug server on device +echo stdout and stderr will be redirected to: ${ADB_DEBUG_SERVER_LOG} + +LLDB_SERVER_PLATFORM_LOG="${DEVICE_LLDB_DIR}/log/platform.log" +LLDB_SERVER_STDOUT_LOG="${DEVICE_LLDB_DIR}/log/platform-stdout.log" +LLDB_SERVER_GDB_LOG="${DEVICE_LLDB_DIR}/log/gdb-server.log" + +run_adb shell run-as ${PACKAGE_NAME} kill -9 "\"\`pidof ${DEVICE_DEBUG_SERVER_LAUNCHER}\`\"" +run_adb_with_log_bg "${ADB_DEBUG_SERVER_LOG}" shell run-as ${PACKAGE_NAME} ${DEVICE_DEBUG_SERVER_LAUNCHER} ${DEVICE_LLDB_DIR} ${SOCKET_SCHEME} ${SOCKET_DIR} ${SOCKET_NAME} "\"lldb process:gdb-remote packets\"" + +LAUNCH_SPEC=${PACKAGE_NAME}/${ACTIVITY_NAME} +run_adb shell am start -S -W ${LAUNCH_SPEC} +die_if_failed Failed to start ${LAUNCH_SPEC} + +APP_PID=$(run_adb_no_echo shell pidof ${PACKAGE_NAME}) +die_if_failed Failed to get ${PACKAGE_NAME} PID on device + +LLDB_SCRIPT_PATH="${OUTPUT_DIR}/${LLDB_SCRIPT_NAME}" +echo App PID: ${APP_PID} +echo "attach ${APP_PID}" >> "${LLDB_SCRIPT_PATH}" + +#TODO: start the app if not running +#TODO: pass app pid to lldb, with -p or --attach-pid +export TERMINFO=/usr/share/terminfo +"${LLDB_PATH}" --source "${LLDB_SCRIPT_PATH}" + +run_adb_with_log "${OUTPUT_DIR}/lldb-platform.log" shell run-as ${PACKAGE_NAME} cat ${LLDB_SERVER_PLATFORM_LOG} +run_adb_with_log "${OUTPUT_DIR}/lldb-platform-stdout.log" shell run-as ${PACKAGE_NAME} cat ${LLDB_SERVER_STDOUT_LOG} +run_adb_with_log "${OUTPUT_DIR}/lldb-gdb-server.log" shell run-as ${PACKAGE_NAME} cat ${LLDB_SERVER_GDB_LOG} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeDebugPrep.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeDebugPrep.cs index 22cf8aa9967..76d94a1a3eb 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/NativeDebugPrep.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeDebugPrep.cs @@ -102,7 +102,7 @@ public void Prepare (string adbPath, string ndkRootPath, string activityName, return null; } - string appLibsRoot = Path.Combine (outputDirRoot, "app", "lib"); + string appLibsRoot = Path.Combine (outputDirRoot, "lldb", "lib"); log.LogDebugMessage ($"Copying application native libararies to {appLibsRoot}"); DotnetSymbolRunner? dotnetSymbol = GetDotnetSymbolRunner (); diff --git a/tools/debug-session-prep/Debug.Session.Prep/AndroidDevice.cs b/tools/debug-session-prep/Debug.Session.Prep/AndroidDevice.cs index 95bb7ff61cb..c91199f532a 100644 --- a/tools/debug-session-prep/Debug.Session.Prep/AndroidDevice.cs +++ b/tools/debug-session-prep/Debug.Session.Prep/AndroidDevice.cs @@ -33,6 +33,7 @@ class AndroidDevice string? appLldbBaseDir; string? appLldbBinDir; string? appLldbLogDir; + string? appLldbTmpDir; string? mainAbi; string? mainArch; string[]? availableAbis; @@ -54,6 +55,8 @@ class AndroidDevice public string MainArch => mainArch ?? String.Empty; public string MainAbi => mainAbi ?? String.Empty; public string SerialNumber => serialNumber ?? String.Empty; + public string DebugServerLauncherScriptPath => deviceDebugServerScriptPath; + public string LldbBaseDir => appLldbBaseDir; public AdbRunner AdbRunner => adb; public AndroidDevice (XamarinLoggingHelper log, AndroidNdk ndk, string outputDir, string adbPath, string packageName, string[] supportedAbis, string? adbTargetDevice = null) @@ -124,8 +127,9 @@ bool PropertyIsEqualTo ((bool haveProperty, string value) result, string expecte } } - public bool Prepare () + public bool Prepare (out string? mainProcessPath) { + mainProcessPath = null; if (!DetectTools ()) { return false; } @@ -134,15 +138,16 @@ public bool Prepare () return false; } - if (!PullLibraries ()) { + if (!PullLibraries (out mainProcessPath)) { return false; } return true; } - bool PullLibraries () + bool PullLibraries (out string? mainProcessPath) { + mainProcessPath = null; DeviceLibraryCopier copier; if (String.IsNullOrEmpty (deviceLdd)) { @@ -163,7 +168,7 @@ bool PullLibraries () ); } - return copier.Copy (); + return copier.Copy (out mainProcessPath); } bool PushDebugServer () @@ -173,8 +178,7 @@ bool PushDebugServer () return false; } - if (!adb.CreateDirectoryAs (packageName, appLldbBinDir!).Result.success) { - log.ErrorLine ($"Failed to create debug server destination directory on device, {appLldbBinDir}"); + if (!CreateLldbDir (appLldbBinDir!) || !CreateLldbDir (appLldbLogDir) || !CreateLldbDir (appLldbTmpDir)) { return false; } @@ -207,6 +211,16 @@ bool PushDebugServer () log.MessageLine (); return true; + + bool CreateLldbDir (string dir) + { + if (!adb.CreateDirectoryAs (packageName, dir).Result.success) { + log.ErrorLine ($"Failed to create debug server destination directory on device, {dir}"); + return false; + } + + return true; + } } bool PushServerExecutable (string hostSource, string deviceDestination) @@ -244,9 +258,10 @@ bool PushServerExecutable (string hostSource, string deviceDestination) return true; } + // TODO: handle multiple pids bool KillDebugServer (string debugServerPath) { - long serverPID = GetDeviceProcessID (debugServerPath, quiet: true); + long serverPID = GetDeviceProcessID (debugServerPath, quiet: false); if (serverPID <= 0) { return true; } @@ -258,7 +273,7 @@ bool KillDebugServer (string debugServerPath) long GetDeviceProcessID (string processName, bool quiet = false) { - (bool success, string output) = adb.Shell ("pidof", processName).Result; + (bool success, string output) = adb.Shell ("pidof", Path.GetFileName (processName)).Result; if (!success) { if (!quiet) { log.ErrorLine ($"Failed to obtain PID of process '{processName}'"); @@ -313,6 +328,7 @@ bool DetermineAppDataDirectory () appLldbBaseDir = $"{appDataDir}/lldb"; appLldbBinDir = $"{appLldbBaseDir}/bin"; appLldbLogDir = $"{appLldbBaseDir}/log"; + appLldbTmpDir = $"{appLldbBaseDir}/tmp"; // Applications with minSdkVersion >= 24 will have their data directories // created with rwx------ permissions, preventing adbd from forwarding to diff --git a/tools/debug-session-prep/Debug.Session.Prep/DeviceLibrariesCopier.cs b/tools/debug-session-prep/Debug.Session.Prep/DeviceLibrariesCopier.cs index baeb4037bec..96a793a08aa 100644 --- a/tools/debug-session-prep/Debug.Session.Prep/DeviceLibrariesCopier.cs +++ b/tools/debug-session-prep/Debug.Session.Prep/DeviceLibrariesCopier.cs @@ -62,5 +62,5 @@ protected DeviceLibraryCopier (XamarinLoggingHelper log, AdbRunner adb, bool app return zygotePath; } - public abstract bool Copy (); + public abstract bool Copy (out string? zygotePath); } diff --git a/tools/debug-session-prep/Debug.Session.Prep/LddDeviceLibrariesCopier.cs b/tools/debug-session-prep/Debug.Session.Prep/LddDeviceLibrariesCopier.cs index 1ccd6240bb6..ea7216399cc 100644 --- a/tools/debug-session-prep/Debug.Session.Prep/LddDeviceLibrariesCopier.cs +++ b/tools/debug-session-prep/Debug.Session.Prep/LddDeviceLibrariesCopier.cs @@ -11,7 +11,7 @@ public LddDeviceLibraryCopier (XamarinLoggingHelper log, AdbRunner adb, bool app : base (log, adb, appIs64Bit, localDestinationDir, device) {} - public override bool Copy () + public override bool Copy (out string? zygotePath) { throw new NotImplementedException(); } diff --git a/tools/debug-session-prep/Debug.Session.Prep/NoLddDeviceLibrariesCopier.cs b/tools/debug-session-prep/Debug.Session.Prep/NoLddDeviceLibrariesCopier.cs index 8ff3a07c922..a33f14917a2 100644 --- a/tools/debug-session-prep/Debug.Session.Prep/NoLddDeviceLibrariesCopier.cs +++ b/tools/debug-session-prep/Debug.Session.Prep/NoLddDeviceLibrariesCopier.cs @@ -38,9 +38,9 @@ public NoLddDeviceLibraryCopier (XamarinLoggingHelper log, AdbRunner adb, bool a : base (log, adb, appIs64Bit, localDestinationDir, device) {} - public override bool Copy () + public override bool Copy (out string? zygotePath) { - string? zygotePath = FetchZygote (); + zygotePath = FetchZygote (); if (String.IsNullOrEmpty (zygotePath)) { Log.ErrorLine ("Unable to determine path of the zygote process on device"); return false; diff --git a/tools/debug-session-prep/Main.cs b/tools/debug-session-prep/Main.cs index 11e17001626..e3f5e5714c9 100644 --- a/tools/debug-session-prep/Main.cs +++ b/tools/debug-session-prep/Main.cs @@ -141,7 +141,7 @@ static int Main (string[] args) return 1; } - if (!device.Prepare ()) { + if (!device.Prepare (out string? mainProcessPath) || String.IsNullOrEmpty (mainProcessPath)) { log.ErrorLine ("Failed to prepare for debugging session"); return 1; } @@ -153,7 +153,7 @@ static int Main (string[] args) string socketName = $"xa-platform-{rnd.NextInt64 ()}.sock"; WriteConfigScript (parsedOptions, device, ndk, socketScheme, socketDir, socketName); - WriteLldbScript (parsedOptions, socketScheme, socketDir, socketName); + WriteLldbScript (parsedOptions, device, socketScheme, socketDir, socketName, mainProcessPath); return 0; } @@ -168,7 +168,7 @@ static StreamWriter OpenScriptWriter (FileStream fs) return new StreamWriter (fs, Utilities.UTF8NoBOM); } - static void WriteLldbScript (ParsedOptions parsedOptions, string socketScheme, string socketDir, string socketName) + static void WriteLldbScript (ParsedOptions parsedOptions, AndroidDevice device, string socketScheme, string socketDir, string socketName, string mainProcessPath) { string outputFile = Path.Combine (parsedOptions.OutputDirPath!, parsedOptions.LldbScriptName!); string fullLibsDir = Path.GetFullPath (Path.Combine (parsedOptions.OutputDirPath!, parsedOptions.AppNativeLibrariesDir!)); @@ -176,10 +176,24 @@ static void WriteLldbScript (ParsedOptions parsedOptions, string socketScheme, s using StreamWriter sw = OpenScriptWriter (fs); // TODO: add support for appending user commands - sw.WriteLine ($"settings append target.exec-search-paths \"{fullLibsDir}\""); - sw.WriteLine ("platform remote-android"); - sw.WriteLine ($"platform connect {socketScheme}-connect:///{socketDir}/{socketName}"); - sw.WriteLine ("gui"); // TODO: make it optional + var searchPathsList = new List { + $"\"{Path.Combine (fullLibsDir, device.MainAbi)}\"" + }; + + foreach (string abi in device.AvailableAbis) { + if (String.Compare (abi, device.MainAbi, StringComparison.Ordinal) == 0) { + continue; + } + + searchPathsList.Add ($"\"{Path.Combine (fullLibsDir, abi)}\""); + } + + string searchPaths = String.Join (" ", searchPathsList); + sw.WriteLine ($"settings append target.exec-search-paths {searchPaths}"); + sw.WriteLine ("platform select remote-android"); + sw.WriteLine ($"platform connect {socketScheme}-connect://{socketDir}/{socketName}"); + sw.WriteLine ($"file \"{mainProcessPath}\""); + sw.Flush (); } @@ -190,16 +204,19 @@ static void WriteConfigScript (ParsedOptions parsedOptions, AndroidDevice device using FileStream fs = OpenScriptStream (outputFile); using StreamWriter sw = OpenScriptWriter (fs); - sw.WriteLine ($"DEVICE_SERIAL=\"{device.SerialNumber}\""); sw.WriteLine ($"DEVICE_API_LEVEL={device.ApiLevel}"); - sw.WriteLine ($"DEVICE_MAIN_ABI={device.MainAbi}"); - sw.WriteLine ($"DEVICE_MAIN_ARCH={device.MainArch}"); sw.WriteLine ($"DEVICE_AVAILABLE_ABIS={FormatArray (device.AvailableAbis)}"); sw.WriteLine ($"DEVICE_AVAILABLE_ARCHES={FormatArray (device.AvailableArches)}"); - sw.WriteLine ($"SOCKET_SCHEME={socketScheme}"); + sw.WriteLine ($"DEVICE_DEBUG_SERVER_LAUNCHER=\"{device.DebugServerLauncherScriptPath}\""); + sw.WriteLine ($"DEVICE_LLDB_DIR=\"{device.LldbBaseDir}\""); + sw.WriteLine ($"DEVICE_MAIN_ABI={device.MainAbi}"); + sw.WriteLine ($"DEVICE_MAIN_ARCH={device.MainArch}"); + sw.WriteLine ($"DEVICE_SERIAL=\"{device.SerialNumber}\""); + sw.WriteLine ($"LLDB_PATH=\"{ndk.LldbPath}\""); sw.WriteLine ($"SOCKET_DIR={socketDir}"); sw.WriteLine ($"SOCKET_NAME={socketName}"); - sw.WriteLine ($"LLDB_PATH=\"{ndk.LldbPath}\""); + sw.WriteLine ($"SOCKET_SCHEME={socketScheme}"); + sw.Flush (); string FormatArray (string[] values) diff --git a/tools/debug-session-prep/Resources/xa_start_lldb_server.sh b/tools/debug-session-prep/Resources/xa_start_lldb_server.sh index 14f030aaa1f..8fa6cdb3141 100755 --- a/tools/debug-session-prep/Resources/xa_start_lldb_server.sh +++ b/tools/debug-session-prep/Resources/xa_start_lldb_server.sh @@ -10,7 +10,7 @@ LLDB_DIR=$1 LISTENER_SCHEME=$2 DOMAINSOCKET_DIR=$3 PLATFORM_SOCKET=$4 -LOG_CHANNELS=$5 +LOG_CHANNELS="$5" BIN_DIR=$LLDB_DIR/bin LOG_DIR=$LLDB_DIR/log diff --git a/tools/xadebug/LICENSE b/tools/xadebug/LICENSE new file mode 100644 index 00000000000..cc166e065e9 --- /dev/null +++ b/tools/xadebug/LICENSE @@ -0,0 +1,24 @@ +The MIT License (MIT) + +Copyright (c) .NET Foundation Contributors + +All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/tools/xadebug/Main.cs b/tools/xadebug/Main.cs new file mode 100644 index 00000000000..fb968704235 --- /dev/null +++ b/tools/xadebug/Main.cs @@ -0,0 +1,106 @@ +using System; +using System.Collections.Generic; +using System.IO; + +using Mono.Options; +using Xamarin.Android.Utilities; +using Xamarin.Tools.Zip; + +namespace Xamarin.Android.Debug; + +class XADebug +{ + sealed class ParsedOptions + { + public bool ShowHelp; + public bool Verbose = true; // TODO: remove the default once development is done + public string Configuration = "Debug"; + public string? PackageName; + } + + static XamarinLoggingHelper log = new XamarinLoggingHelper (); + + static int Main (string[] args) + { + bool haveOptionErrors = false; + var parsedOptions = new ParsedOptions (); + log.Verbose = parsedOptions.Verbose; + + var opts = new OptionSet { + "Usage: dotnet xadebug [OPTIONS] ", + "", + { "p|package-name=", "name of the application package", v => parsedOptions.PackageName = EnsureNonEmptyString (log, "-p|--package-name", v, ref haveOptionErrors) }, + { "c|configuration=", "{CONFIGURATION} in which to build the application. Ignored when running in APK-only mode", v => parsedOptions.Configuration = v }, + "", + { "v|verbose", "Show debug messages", v => parsedOptions.Verbose = true }, + { "h|help|?", "Show this help screen", v => parsedOptions.ShowHelp = true }, + }; + + List rest = opts.Parse (args); + log.Verbose = parsedOptions.Verbose; + + if (parsedOptions.ShowHelp || rest.Count == 0) { + int ret = 0; + if (rest.Count == 0) { + log.ErrorLine ("Path to application APK or directory with a C# project must be specified"); + log.ErrorLine (); + ret = 1; + } + + opts.WriteOptionDescriptions (Console.Out); + return ret; + } + + if (haveOptionErrors) { + return 1; + } + + string? apkFilePath = null; + ZipArchive? apk = null; + + if (Directory.Exists (rest[0])) { + // TODO: build app in this directory and set apkFilePath appropriately + throw new NotImplementedException ("Building the application is not implemented yet"); + } else if (File.Exists (rest[0])) { + if (!IsAndroidPackageFile (rest[0], out apk)) { + log.ErrorLine ($"File '{rest[0]}' is not an Android APK package"); + log.ErrorLine (); + } else { + apkFilePath = rest[0]; + } + } else { + log.ErrorLine ($"Neither directory nor file '{rest[0]}' exist"); + log.ErrorLine (); + } + + if (String.IsNullOrEmpty (apkFilePath)) { + return 1; + } + + return 0; + } + + static string? EnsureNonEmptyString (XamarinLoggingHelper log, string paramName, string? value, ref bool haveOptionErrors) + { + if (String.IsNullOrEmpty (value)) { + haveOptionErrors = true; + log.ErrorLine ($"Parameter '{paramName}' requires a non-empty string as its value"); + return null; + } + + return value; + } + + static bool IsAndroidPackageFile (string filePath, out ZipArchive? apk) + { + try { + apk = ZipArchive.Open (filePath, FileMode.Open); + } catch (ZipIOException ex) { + log.DebugLine ($"Failed to open '{filePath}' as ZIP archive: {ex.Message}"); + apk = null; + return false; + } + + return apk.ContainsEntry ("AndroidManifest.xml"); + } +} diff --git a/tools/xadebug/README.md b/tools/xadebug/README.md new file mode 100644 index 00000000000..30404ce4c54 --- /dev/null +++ b/tools/xadebug/README.md @@ -0,0 +1 @@ +TODO \ No newline at end of file diff --git a/tools/xadebug/Xamarin.Android.Debug/AXMLParser.cs b/tools/xadebug/Xamarin.Android.Debug/AXMLParser.cs new file mode 100644 index 00000000000..47de43507eb --- /dev/null +++ b/tools/xadebug/Xamarin.Android.Debug/AXMLParser.cs @@ -0,0 +1,9 @@ +using System.IO; + +namespace Xamarin.Android.Debug; + +class AXMLParser +{ + public AXMLParser (Stream data) + {} +} diff --git a/tools/xadebug/Xamarin.Android.Debug/Utilities.cs b/tools/xadebug/Xamarin.Android.Debug/Utilities.cs new file mode 100644 index 00000000000..95d70314f3d --- /dev/null +++ b/tools/xadebug/Xamarin.Android.Debug/Utilities.cs @@ -0,0 +1,78 @@ +using System; +using System.IO; +using System.Reflection; +using System.Runtime.InteropServices; +using System.Text; + +using Xamarin.Android.Utilities; + +namespace Xamarin.Android.Debug; + +static class Utilities +{ + public static readonly UTF8Encoding UTF8NoBOM = new UTF8Encoding (false); + + public static bool IsMacOS { get; private set; } + public static bool IsLinux { get; private set; } + public static bool IsWindows { get; private set; } + + static Utilities () + { + if (RuntimeInformation.IsOSPlatform (OSPlatform.Windows)) { + IsWindows = true; + } else if (RuntimeInformation.IsOSPlatform (OSPlatform.OSX)) { + IsMacOS = true; + } else if (RuntimeInformation.IsOSPlatform (OSPlatform.Linux)) { + IsLinux = true; + } + } + + public static void MakeFileDirectory (string filePath) + { + if (String.IsNullOrEmpty (filePath)) { + return; + } + + string? dirName = Path.GetDirectoryName (filePath); + if (String.IsNullOrEmpty (dirName)) { + return; + } + + Directory.CreateDirectory (dirName); + } + + public static string? ReadManifestResource (XamarinLoggingHelper log, string resourceName) + { + using (var from = Assembly.GetExecutingAssembly ().GetManifestResourceStream (resourceName)) { + if (from == null) { + log.ErrorLine ($"Manifest resource '{resourceName}' cannot be loaded"); + return null; + } + + using (var sr = new StreamReader (from)) { + return sr.ReadToEnd (); + } + } + } + + public static string NormalizeDirectoryPath (string dirPath) + { + if (dirPath.EndsWith ('/')) { + return dirPath; + } + + return $"{dirPath}/"; + } + + public static string ToLocalPathFormat (string path) => IsWindows ? path.Replace ("/", "\\") : path; + + public static string MakeLocalPath (string localDirectory, string remotePath) + { + string remotePathLocalFormat = ToLocalPathFormat (remotePath); + if (remotePath[0] == '/') { + return $"{localDirectory}{remotePathLocalFormat}"; + } + + return Path.Combine (localDirectory, remotePathLocalFormat); + } +} diff --git a/tools/xadebug/Xamarin.Android.Utilities/XamarinLoggingHelper.cs b/tools/xadebug/Xamarin.Android.Utilities/XamarinLoggingHelper.cs new file mode 100644 index 00000000000..4167c8852b6 --- /dev/null +++ b/tools/xadebug/Xamarin.Android.Utilities/XamarinLoggingHelper.cs @@ -0,0 +1,174 @@ +using System; +using System.IO; + +namespace Xamarin.Android.Utilities; + +enum LogLevel +{ + Error, + Warning, + Info, + Message, + Debug +} + +class XamarinLoggingHelper +{ + static readonly object consoleLock = new object (); + + public const ConsoleColor ErrorColor = ConsoleColor.Red; + public const ConsoleColor DebugColor = ConsoleColor.DarkGray; + public const ConsoleColor InfoColor = ConsoleColor.Green; + public const ConsoleColor MessageColor = ConsoleColor.Gray; + public const ConsoleColor WarningColor = ConsoleColor.Yellow; + public const ConsoleColor StatusLabel = ConsoleColor.Cyan; + public const ConsoleColor StatusText = ConsoleColor.White; + + public bool Verbose { get; set; } + + public void Message (string? message) + { + Log (LogLevel.Message, message); + } + + public void MessageLine (string? message = null) + { + Message ($"{message ?? String.Empty}{Environment.NewLine}"); + } + + public void Warning (string? message) + { + Log (LogLevel.Warning, message); + } + + public void WarningLine (string? message = null) + { + Warning ($"{message ?? String.Empty}{Environment.NewLine}"); + } + + public void Error (string? message) + { + Log (LogLevel.Error, message); + } + + public void ErrorLine (string? message = null) + { + Error ($"{message ?? String.Empty}{Environment.NewLine}"); + } + + public void Info (string? message) + { + Log (LogLevel.Info, message); + } + + public void InfoLine (string? message = null) + { + Info ($"{message ?? String.Empty}{Environment.NewLine}"); + } + + public void Debug (string? message) + { + Log (LogLevel.Debug, message); + } + + public void DebugLine (string? message = null) + { + Debug ($"{message ?? String.Empty}{Environment.NewLine}"); + } + + public void Status (string label, string text) + { + Log (LogLevel.Info, $"{label}: ", StatusLabel); + Log (LogLevel.Info, $"{text}", StatusText); + } + + public void StatusLine (string label, string text) + { + Status (label, text); + Log (LogLevel.Info, Environment.NewLine); + } + + public void Log (LogLevel level, string? message) + { + if (!Verbose && level == LogLevel.Debug) { + return; + } + + Log (level, message, ForegroundColor (level)); + } + + public void LogLine (LogLevel level, string? message, ConsoleColor color) + { + Log (level, message, color); + Log (level, Environment.NewLine, color); + } + + public void Log (LogLevel level, string? message, ConsoleColor color) + { + if (!Verbose && level == LogLevel.Debug) { + return; + } + + TextWriter writer = level == LogLevel.Error ? Console.Error : Console.Out; + message = message ?? String.Empty; + + ConsoleColor fg = ConsoleColor.Gray; + try { + lock (consoleLock) { + fg = Console.ForegroundColor; + Console.ForegroundColor = color; + } + + writer.Write (message); + } finally { + Console.ForegroundColor = fg; + } + } + + ConsoleColor ForegroundColor (LogLevel level) => level switch { + LogLevel.Error => ErrorColor, + LogLevel.Warning => WarningColor, + LogLevel.Info => InfoColor, + LogLevel.Debug => DebugColor, + LogLevel.Message => MessageColor, + _ => MessageColor, + }; + +#region MSBuild compatibility methods + public void LogDebugMessage (string message, params object[] messageArgs) + { + if (messageArgs == null || messageArgs.Length == 0) { + DebugLine (message); + } else { + DebugLine (String.Format (message, messageArgs)); + } + } + + public void LogError (string message, params object[] messageArgs) + { + if (messageArgs == null || messageArgs.Length == 0) { + ErrorLine (message); + } else { + ErrorLine (String.Format (message, messageArgs)); + } + } + + public void LogMessage (string message, params object[] messageArgs) + { + if (messageArgs == null || messageArgs.Length == 0) { + MessageLine (message); + } else { + MessageLine (String.Format (message, messageArgs)); + } + } + + public void LogWarning (string message, params object[] messageArgs) + { + if (messageArgs == null || messageArgs.Length == 0) { + WarningLine (message); + } else { + WarningLine (String.Format (message, messageArgs)); + } + } +#endregion +} diff --git a/tools/xadebug/xadebug.csproj b/tools/xadebug/xadebug.csproj new file mode 100644 index 00000000000..591de7bea9b --- /dev/null +++ b/tools/xadebug/xadebug.csproj @@ -0,0 +1,43 @@ + + + + + Exe + net7.0 + False + enable + NO_MSBUILD + + Marek Habersack + Microsoft Corporation + 2022 © Microsoft Corporation + true + xadebug + nupkg + A tool to debug native code of Xamarin.Android applications + README.md + LICENSE + https://github.com/xamarin/xamarin-android + Major + + + + + + + + + + + + + + + + + + + + + + From 88c4e408d8f163514419e390d8de4eb48257fa0a Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Mon, 5 Dec 2022 23:07:34 +0100 Subject: [PATCH 20/30] [WIP] AXML parser --- tools/xadebug/Main.cs | 55 +++- .../Xamarin.Android.Debug/AXMLParser.cs | 241 +++++++++++++++++- .../Xamarin.Android.Debug/ApplicationInfo.cs | 11 + 3 files changed, 294 insertions(+), 13 deletions(-) create mode 100644 tools/xadebug/Xamarin.Android.Debug/ApplicationInfo.cs diff --git a/tools/xadebug/Main.cs b/tools/xadebug/Main.cs index fb968704235..cf25a24e74f 100644 --- a/tools/xadebug/Main.cs +++ b/tools/xadebug/Main.cs @@ -18,6 +18,8 @@ sealed class ParsedOptions public string? PackageName; } + const string AndroidManifestZipPath = "AndroidManifest.xml"; + static XamarinLoggingHelper log = new XamarinLoggingHelper (); static int Main (string[] args) @@ -55,21 +57,24 @@ static int Main (string[] args) return 1; } + string aPath = rest[0]; string? apkFilePath = null; ZipArchive? apk = null; - if (Directory.Exists (rest[0])) { - // TODO: build app in this directory and set apkFilePath appropriately - throw new NotImplementedException ("Building the application is not implemented yet"); - } else if (File.Exists (rest[0])) { - if (!IsAndroidPackageFile (rest[0], out apk)) { - log.ErrorLine ($"File '{rest[0]}' is not an Android APK package"); - log.ErrorLine (); + if (Directory.Exists (aPath)) { + apkFilePath = BuildApp (aPath); + } else if (File.Exists (aPath)) { + if (String.Compare (".csproj", Path.GetExtension (aPath), StringComparison.OrdinalIgnoreCase) == 0) { + // Let's see if we can trust the file name... + apkFilePath = BuildApp (aPath); + } else if (IsAndroidPackageFile (aPath, out apk)) { + apkFilePath = aPath; } else { - apkFilePath = rest[0]; + log.ErrorLine ($"File '{aPath}' is not an Android APK package"); + log.ErrorLine (); } } else { - log.ErrorLine ($"Neither directory nor file '{rest[0]}' exist"); + log.ErrorLine ($"Neither directory nor file '{aPath}' exist"); log.ErrorLine (); } @@ -77,9 +82,30 @@ static int Main (string[] args) return 1; } + if (apk == null) { + apk = OpenApk (apkFilePath); + } + + // Extract app information fromn the embedded manifest + ApplicationInfo appInfo = ReadManifest (apk); + return 0; } + static ApplicationInfo ReadManifest (ZipArchive apk) + { + ZipEntry entry = apk.ReadEntry (AndroidManifestZipPath); + + using var manifest = new MemoryStream (); + entry.Extract (manifest); + manifest.Seek (0, SeekOrigin.Begin); + + var axml = new AXMLParser (manifest, log); + string packageName = String.Empty; + + return new ApplicationInfo (packageName); + } + static string? EnsureNonEmptyString (XamarinLoggingHelper log, string paramName, string? value, ref bool haveOptionErrors) { if (String.IsNullOrEmpty (value)) { @@ -91,16 +117,23 @@ static int Main (string[] args) return value; } + static ZipArchive OpenApk (string filePath) => ZipArchive.Open (filePath, FileMode.Open); + static bool IsAndroidPackageFile (string filePath, out ZipArchive? apk) { try { - apk = ZipArchive.Open (filePath, FileMode.Open); + apk = OpenApk (filePath); } catch (ZipIOException ex) { log.DebugLine ($"Failed to open '{filePath}' as ZIP archive: {ex.Message}"); apk = null; return false; } - return apk.ContainsEntry ("AndroidManifest.xml"); + return apk.ContainsEntry (AndroidManifestZipPath); + } + + static string BuildApp (string projectPath) + { + throw new NotImplementedException ("Building the application is not implemented yet"); } } diff --git a/tools/xadebug/Xamarin.Android.Debug/AXMLParser.cs b/tools/xadebug/Xamarin.Android.Debug/AXMLParser.cs index 47de43507eb..e30dc00389e 100644 --- a/tools/xadebug/Xamarin.Android.Debug/AXMLParser.cs +++ b/tools/xadebug/Xamarin.Android.Debug/AXMLParser.cs @@ -1,9 +1,246 @@ +using System; +using System.Collections.Generic; using System.IO; +using System.Text; + +using Xamarin.Android.Utilities; namespace Xamarin.Android.Debug; +enum ChunkType : ushort +{ + Null = 0x0000, + StringPool = 0x0001, + Table = 0x0002, + Xml = 0x0003, + + XmlFirstChunk = 0x0100, + XmlStartNamespace = 0x0100, + XmlEndNamespace = 0x0101, + XmlStartElement = 0x0102, + XmlEndElement = 0x0103, + XmlCData = 0x0104, + XmlLastChunk = 0x017f, + XmlResourceMap = 0x0180, + + TablePackage = 0x0200, + TableType = 0x0201, + TableTypeSpec = 0x0202, + TableLibrary = 0x0203, +} + +// +// Based on https://github.com/androguard/androguard/tree/832104db3eb5dc3cc66b30883fa8ce8712dfa200/androguard/core/axml +// class AXMLParser { - public AXMLParser (Stream data) - {} + enum ParsingState + { + StartDocument = 0, + EndDocument = 1, + StartTag = 2, + EndTag = 3, + Text = 4, + } + + // Position of fields inside an attribute + const int ATTRIBUTE_IX_NAMESPACE_URI = 0; + const int ATTRIBUTE_IX_NAME = 1; + const int ATTRIBUTE_IX_VALUE_STRING = 2; + const int ATTRIBUTE_IX_VALUE_TYPE = 3; + const int ATTRIBUTE_IX_VALUE_DATA = 4; + const int ATTRIBUTE_LENGHT = 5; + + const long MinimumDataSize = 8; + const long MaximumDataSize = (long)UInt32.MaxValue; + + readonly XamarinLoggingHelper log; + + bool axmlTampered; + Stream data; + long dataSize; + ARSCHeader axmlHeader; + uint fileSize; + StringBlock stringPool; + + public AXMLParser (Stream data, XamarinLoggingHelper logger) + { + log = logger; + + this.data = data; + dataSize = data.Length; + + // Minimum is a single ARSCHeader, which would be a strange edge case... + if (dataSize < MinimumDataSize) { + throw new InvalidDataException ($"Input data size too small for it to be valid AXML content ({dataSize} < {MinimumDataSize})"); + } + + // This would be even stranger, if an AXML file is larger than 4GB... + // But this is not possible as the maximum chunk size is a unsigned 4 byte int. + if (dataSize > MaximumDataSize) { + throw new InvalidDataException ($"Input data size too large for it to be a valid AXML content ({dataSize} > {MaximumDataSize})"); + } + + try { + axmlHeader = new ARSCHeader (data); + } catch (Exception) { + log.ErrorLine ("Error parsing the first data header"); + throw; + } + + if (axmlHeader.HeaderSize != 8) { + throw new InvalidDataException ($"This does not look like AXML data. header size does not equal 8. header size = {axmlHeader.Size}"); + } + + fileSize = axmlHeader.Size; + if (fileSize > dataSize) { + throw new InvalidDataException ($"This does not look like AXML data. Declared data size does not match real size: {fileSize} vs {dataSize}"); + } + + if (fileSize < dataSize) { + axmlTampered = true; + log.WarningLine ($"Declared data size ({fileSize}) is smaller than total data size ({dataSize}). Was something appended to the file? Trying to parse it anyways."); + } + + if (axmlHeader.Type != ChunkType.Xml) { + axmlTampered = true; + log.WarningLine ($"AXML file has an unusual resource type, trying to parse it anyways. Resource Type: 0x{(ushort)axmlHeader.Type:04x}"); + } + + ARSCHeader stringPoolHeader = new ARSCHeader (data, ChunkType.StringPool); + if (stringPoolHeader.HeaderSize != 28) { + throw new InvalidDataException ($"This does not look like an AXML file. String chunk header size does not equal 28. Header size = {stringPoolHeader.Size}"); + } + + stringPool = new StringBlock (logger, data, stringPoolHeader); + } +} + +class StringBlock +{ + const uint FlagSorted = 1 << 0; + const uint FlagUTF8 = 1 << 0; + + XamarinLoggingHelper log; + ARSCHeader header; + uint stringCount; + uint styleCount; + uint stringsOffset; + uint stylesOffset; + uint flags; + bool isUTF8; + List stringOffsets; + byte[] chars; + + public StringBlock (XamarinLoggingHelper logger, Stream data, ARSCHeader stringPoolHeader) + { + log = logger; + header = stringPoolHeader; + + using var reader = new BinaryReader (data, Encoding.UTF8, leaveOpen: true); + + stringCount = reader.ReadUInt32 (); + styleCount = reader.ReadUInt32 (); + + flags = reader.ReadUInt32 (); + isUTF8 = (flags & FlagUTF8) == FlagUTF8; + + stringsOffset = reader.ReadUInt32 (); + stylesOffset = reader.ReadUInt32 (); + + if (styleCount == 0 && stylesOffset > 0) { + log.InfoLine ("Styles Offset given, but styleCount is zero. This is not a problem but could indicate packers."); + } + + stringOffsets = new List (); + + for (uint i = 0; i < stringCount; i++) { + stringOffsets.Add (reader.ReadUInt32 ()); + } + + // We're not interested in styles, skip over their offsets + for (uint i = 0; i < styleCount; i++) { + reader.ReadUInt32 (); + } + + bool haveStyles = stylesOffset != 0 && styleCount != 0; + uint size = header.Size - stringsOffset; + if (haveStyles) { + size = stylesOffset - stringsOffset; + } + + if (size % 4 != 0) { + log.WarningLine ("Size of strings is not aligned on four bytes."); + } + + chars = new byte[size]; + reader.Read (chars, 0, (int)size); + + if (haveStyles) { + size = header.Size - stylesOffset; + + if (size % 4 != 0) { + log.WarningLine ("Size of styles is not aligned on four bytes."); + } + + // Not interested in them, skip + for (uint i = 0; i < size / 4; i++) { + reader.ReadUInt32 (); + } + } + } +} + +class ARSCHeader +{ + // This is the minimal size such a header must have. There might be other header data too! + const long MinimumSize = 2 + 2 + 4; + + long start; + uint size; + ushort type; + ushort headerSize; + + public ChunkType Type => (ChunkType)type; + public ushort HeaderSize => headerSize; + public uint Size => size; + public long End => start + (long)size; + + public ARSCHeader (Stream data, ChunkType? expectedType = null) + { + start = data.Position; + if (data.Length < start + MinimumSize) { + throw new InvalidDataException ($"Input data not large enough. Offset: {start}"); + } + + // Data in AXML is little-endian, which is fortuitous as that's the only format BinaryReader understands. + using BinaryReader reader = new BinaryReader (data, Encoding.UTF8, leaveOpen: true); + + // ushort: type + // ushort: header_size + // uint: size + type = reader.ReadUInt16 (); + headerSize = reader.ReadUInt16 (); + size = reader.ReadUInt32 (); + + if (expectedType != null && type != (ushort)expectedType) { + throw new InvalidOperationException ($"Header type is not equal to the expected type ({expectedType}): got 0x{type:x}, expected 0x{(ushort)expectedType:x}"); + } + + if (!Enum.IsDefined (typeof(ChunkType), type)) { + throw new InvalidOperationException ($"Internal error: unsupported chunk type 0x{type:x}"); + } + + if (headerSize < MinimumSize) { + throw new InvalidDataException ($"Declared header size is smaller than required size of {MinimumSize}. Offset: {start}"); + } + + if (size < MinimumSize) { + throw new InvalidDataException ($"Declared chunk size is smaller than required size of {MinimumSize}. Offset: {start}"); + } + + if (size < headerSize) { + throw new InvalidDataException ($"Declared chunk size ({size}) is smaller than header size ({headerSize})! Offset: {start}"); + } + } } diff --git a/tools/xadebug/Xamarin.Android.Debug/ApplicationInfo.cs b/tools/xadebug/Xamarin.Android.Debug/ApplicationInfo.cs new file mode 100644 index 00000000000..2a04ecb018f --- /dev/null +++ b/tools/xadebug/Xamarin.Android.Debug/ApplicationInfo.cs @@ -0,0 +1,11 @@ +namespace Xamarin.Android.Debug; + +class ApplicationInfo +{ + public string PackageName { get; } + + public ApplicationInfo (string packageName) + { + PackageName = packageName; + } +} From c7a0478b2bc4d4a8d7673484cddd3cd7a9104ab1 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 6 Dec 2022 22:56:36 +0100 Subject: [PATCH 21/30] [WIP] AXML parsing continued --- tools/xadebug/Main.cs | 13 +- .../Xamarin.Android.Debug/AXMLParser.cs | 208 +++++++++++++++++- 2 files changed, 209 insertions(+), 12 deletions(-) diff --git a/tools/xadebug/Main.cs b/tools/xadebug/Main.cs index cf25a24e74f..104b39a124b 100644 --- a/tools/xadebug/Main.cs +++ b/tools/xadebug/Main.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Xml; using Mono.Options; using Xamarin.Android.Utilities; @@ -96,11 +97,15 @@ static ApplicationInfo ReadManifest (ZipArchive apk) { ZipEntry entry = apk.ReadEntry (AndroidManifestZipPath); - using var manifest = new MemoryStream (); - entry.Extract (manifest); - manifest.Seek (0, SeekOrigin.Begin); + using var manifestData = new MemoryStream (); + entry.Extract (manifestData); + manifestData.Seek (0, SeekOrigin.Begin); + + // TODO: make provisions for plain XML AndroidManifest.xml, perhaps? Although not sure if it's really necesary these days anymore as the APKs should all have the + // binary version of the manifest. + var axml = new AXMLParser (manifestData, log); + XmlDocument? manifest = axml.Parse (); - var axml = new AXMLParser (manifest, log); string packageName = String.Empty; return new ApplicationInfo (packageName); diff --git a/tools/xadebug/Xamarin.Android.Debug/AXMLParser.cs b/tools/xadebug/Xamarin.Android.Debug/AXMLParser.cs index e30dc00389e..2eccaefc53d 100644 --- a/tools/xadebug/Xamarin.Android.Debug/AXMLParser.cs +++ b/tools/xadebug/Xamarin.Android.Debug/AXMLParser.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO; using System.Text; +using System.Xml; using Xamarin.Android.Utilities; @@ -62,6 +63,10 @@ enum ParsingState ARSCHeader axmlHeader; uint fileSize; StringBlock stringPool; + bool valid = true; + long initialPosition; + + public bool IsValid => valid; public AXMLParser (Stream data, XamarinLoggingHelper logger) { @@ -113,6 +118,93 @@ public AXMLParser (Stream data, XamarinLoggingHelper logger) } stringPool = new StringBlock (logger, data, stringPoolHeader); + initialPosition = data.Position; + } + + public XmlDocument? Parse () + { + // Reset position in case we're called more than once, for whatever reason + data.Seek (initialPosition, SeekOrigin.Begin); + valid = true; + + XmlDocument ret = new XmlDocument (); + XmlDeclaration declaration = ret.CreateXmlDeclaration ("1.0", stringPool.IsUTF8 ? "UTF-8" : "UTF-16", null); + ret.InsertBefore (declaration, ret.DocumentElement); + + using var reader = new BinaryReader (data, Encoding.UTF8, leaveOpen: true); + ARSCHeader? header; + while (data.Position < dataSize) { + header = new ARSCHeader (data); + + // Special chunk: Resource Map. This chunk might follow the string pool. + if (header.Type == ChunkType.XmlResourceMap) { + if (!SkipOverResourceMap (header, reader)) { + valid = false; + break; + } + continue; + } + + // XML chunks + + // Skip over unknown types + if (!Enum.IsDefined (typeof(ChunkType), header.TypeRaw)) { + log.WarningLine ($"Unknown chunk type 0x{header.TypeRaw:x} at offset {data.Position}. Skipping over {header.Size} bytes"); + data.Seek (header.Size, SeekOrigin.Current); + continue; + } + + // Check that we read a correct header + if (header.HeaderSize != 16) { + log.WarningLine ($"XML chunk header size is not 16. Chunk type {header.Type} (0x{header.TypeRaw:x}), chunk size {header.Size}"); + data.Seek (header.Size, SeekOrigin.Current); + continue; + } + + // Line Number of the source file, only used as meta information + uint lineNumber = reader.ReadUInt32 (); + + // Comment_Index (usually 0xffffffff) + uint commentIndex = reader.ReadUInt32 (); + + if (commentIndex != 0xffffffff && (header.Type == ChunkType.XmlStartNamespace || header.Type == ChunkType.XmlEndNamespace)) { + log.WarningLine ($"Unhandled Comment at namespace chunk: {commentIndex}"); + } + + uint prefixIndex; + uint uriIndex; + + if (header.Type == ChunkType.XmlStartNamespace) { + prefixIndex = reader.ReadUInt32 (); + uriIndex = reader.ReadUInt32 (); + + string? prefix = stringPool.GetString (prefixIndex); + string? uri = stringPool.GetString (uriIndex); + + continue; + } + } + + return ret; + } + + bool SkipOverResourceMap (ARSCHeader header, BinaryReader reader) + { + log.DebugLine ("AXML contains a resource map"); + + // Check size: < 8 bytes mean that the chunk is not complete + // Should be aligned to 4 bytes. + if (header.Size < 8 || (header.Size % 4) != 0) { + log.ErrorLine ("Invalid chunk size in chunk XML_RESOURCE_MAP"); + return false; + } + + // Since our main interest is in reading AndroidManifest.xml, we're going to skip over the table + for (int i = 0; i < (header.Size - header.HeaderSize) / 4; i++) { + reader.ReadUInt32 (); + } + + return true; } } @@ -124,13 +216,15 @@ class StringBlock XamarinLoggingHelper log; ARSCHeader header; uint stringCount; - uint styleCount; uint stringsOffset; - uint stylesOffset; uint flags; bool isUTF8; List stringOffsets; byte[] chars; + Dictionary stringCache; + + public uint StringCount => stringCount; + public bool IsUTF8 => isUTF8; public StringBlock (XamarinLoggingHelper logger, Stream data, ARSCHeader stringPoolHeader) { @@ -140,13 +234,13 @@ public StringBlock (XamarinLoggingHelper logger, Stream data, ARSCHeader stringP using var reader = new BinaryReader (data, Encoding.UTF8, leaveOpen: true); stringCount = reader.ReadUInt32 (); - styleCount = reader.ReadUInt32 (); + uint styleCount = reader.ReadUInt32 (); flags = reader.ReadUInt32 (); isUTF8 = (flags & FlagUTF8) == FlagUTF8; stringsOffset = reader.ReadUInt32 (); - stylesOffset = reader.ReadUInt32 (); + uint stylesOffset = reader.ReadUInt32 (); if (styleCount == 0 && stylesOffset > 0) { log.InfoLine ("Styles Offset given, but styleCount is zero. This is not a problem but could indicate packers."); @@ -188,6 +282,102 @@ public StringBlock (XamarinLoggingHelper logger, Stream data, ARSCHeader stringP reader.ReadUInt32 (); } } + + stringCache = new Dictionary (); + } + + public string? GetString (uint idx) + { + if (stringCache.TryGetValue (idx, out string? ret)) { + return ret; + } + + if (idx < 0 || idx > stringOffsets.Count || stringOffsets.Count == 0) { + return null; + } + + uint offset = stringOffsets[(int)idx]; + if (isUTF8) { + ret = DecodeUTF8 (offset); + } else { + ret = DecodeUTF16 (offset); + } + stringCache[idx] = ret; + + return ret; + } + + string DecodeUTF8 (uint offset) + { + // UTF-8 Strings contain two lengths, as they might differ: + // 1) the string length in characters + (uint length, uint nbytes) = DecodeLength (offset, sizeOfChar: 1); + offset += nbytes; + + // 2) the number of bytes the encoded string occupies + (uint encodedBytes, nbytes) = DecodeLength (offset, sizeOfChar: 1); + offset += nbytes; + + if (chars[offset + encodedBytes] != 0) { + throw new InvalidDataException ($"UTF-8 string is not NUL-terminated. Offset: offset"); + } + + return Encoding.UTF8.GetString (chars, (int)offset, (int)encodedBytes); + } + + string DecodeUTF16 (uint offset) + { + (uint length, uint nbytes) = DecodeLength (offset, sizeOfChar: 2); + offset += nbytes; + + uint encodedBytes = length * 2; + if (chars[offset + encodedBytes] != 0 && chars[offset + encodedBytes + 1] != 0) { + throw new InvalidDataException ($"UTF-16 string is not NUL-terminated. Offset: offset"); + } + + return Encoding.Unicode.GetString (chars, (int)offset, (int)encodedBytes); + } + + (uint length, uint nbytes) DecodeLength (uint offset, uint sizeOfChar) + { + uint sizeOfTwoChars = sizeOfChar << 1; + uint highBit = 0x80u << (8 * ((int)sizeOfChar - 1)); + uint length1, length2; + + // Length is tored as 1 or 2 characters of `sizeofChar` size + if (sizeOfChar == 1) { + // UTF-8 encoding, each character is a byte + length1 = chars[offset]; + length2 = chars[offset + 1]; + } else { + // UTF-16 encoding, each character is a short + length1 = (uint)((chars[offset]) | (chars[offset + 1] << 8)); + length2 = (uint)((chars[offset + 2]) | (chars[offset + 3] << 8)); + } + + uint length; + uint nbytes; + if ((length1 & highBit) != 0) { + length = ((length1 & ~highBit) << (8 * (int)sizeOfChar)) | length2; + nbytes = sizeOfTwoChars; + } else { + length = length1; + nbytes = sizeOfChar; + } + + // 8 bit strings: maximum of 0x7FFF bytes, http://androidxref.com/9.0.0_r3/xref/frameworks/base/libs/androidfw/ResourceTypes.cpp#692 + // 16 bit strings: maximum of 0x7FFFFFF bytes, http://androidxref.com/9.0.0_r3/xref/frameworks/base/libs/androidfw/ResourceTypes.cpp#670 + if (sizeOfChar == 1) { + if (length > 0x7fff) { + throw new InvalidDataException ("UTF-8 string is too long. Offset: {offset}"); + } + } else { + if (length > 0x7fffffff) { + throw new InvalidDataException ("UTF-16 string is too long. Offset: {offset}"); + } + } + + return (length, nbytes); } } @@ -200,8 +390,10 @@ class ARSCHeader uint size; ushort type; ushort headerSize; + bool unknownType; - public ChunkType Type => (ChunkType)type; + public ChunkType Type => unknownType ? ChunkType.Null : (ChunkType)type; + public ushort TypeRaw => type; public ushort HeaderSize => headerSize; public uint Size => size; public long End => start + (long)size; @@ -221,15 +413,15 @@ public ARSCHeader (Stream data, ChunkType? expectedType = null) // uint: size type = reader.ReadUInt16 (); headerSize = reader.ReadUInt16 (); + + // Total size of the chunk, including the header size = reader.ReadUInt32 (); if (expectedType != null && type != (ushort)expectedType) { throw new InvalidOperationException ($"Header type is not equal to the expected type ({expectedType}): got 0x{type:x}, expected 0x{(ushort)expectedType:x}"); } - if (!Enum.IsDefined (typeof(ChunkType), type)) { - throw new InvalidOperationException ($"Internal error: unsupported chunk type 0x{type:x}"); - } + unknownType = !Enum.IsDefined (typeof(ChunkType), type); if (headerSize < MinimumSize) { throw new InvalidDataException ($"Declared header size is smaller than required size of {MinimumSize}. Offset: {start}"); From dfcddc3a941c8e25a88096d8ccec0ed741d6ce2f Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 7 Dec 2022 22:22:29 +0100 Subject: [PATCH 22/30] [WIP] AXML parser works Implemented enough of it for our needs. --- tools/xadebug/Main.cs | 113 ++++++- .../Xamarin.Android.Debug/AXMLParser.cs | 281 +++++++++++++++++- .../Xamarin.Android.Debug/ApplicationInfo.cs | 12 +- 3 files changed, 387 insertions(+), 19 deletions(-) diff --git a/tools/xadebug/Main.cs b/tools/xadebug/Main.cs index 104b39a124b..41641f57ec4 100644 --- a/tools/xadebug/Main.cs +++ b/tools/xadebug/Main.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Text; using System.Xml; using Mono.Options; @@ -17,9 +18,11 @@ sealed class ParsedOptions public bool Verbose = true; // TODO: remove the default once development is done public string Configuration = "Debug"; public string? PackageName; + public string? Activity; } const string AndroidManifestZipPath = "AndroidManifest.xml"; + const string DefaultMinSdkVersion = "21"; static XamarinLoggingHelper log = new XamarinLoggingHelper (); @@ -34,6 +37,7 @@ static int Main (string[] args) "", { "p|package-name=", "name of the application package", v => parsedOptions.PackageName = EnsureNonEmptyString (log, "-p|--package-name", v, ref haveOptionErrors) }, { "c|configuration=", "{CONFIGURATION} in which to build the application. Ignored when running in APK-only mode", v => parsedOptions.Configuration = v }, + { "a|activity=", "Name of the {ACTIVITY} to start the application. The default is determined from AndroidManifest.xml", v => parsedOptions.Activity = v }, "", { "v|verbose", "Show debug messages", v => parsedOptions.Verbose = true }, { "h|help|?", "Show this help screen", v => parsedOptions.ShowHelp = true }, @@ -88,12 +92,15 @@ static int Main (string[] args) } // Extract app information fromn the embedded manifest - ApplicationInfo appInfo = ReadManifest (apk); + ApplicationInfo? appInfo = ReadManifest (apk, parsedOptions); + if (appInfo == null) { + return 1; + } return 0; } - static ApplicationInfo ReadManifest (ZipArchive apk) + static ApplicationInfo? ReadManifest (ZipArchive apk, ParsedOptions parsedOptions) { ZipEntry entry = apk.ReadEntry (AndroidManifestZipPath); @@ -105,10 +112,108 @@ static ApplicationInfo ReadManifest (ZipArchive apk) // binary version of the manifest. var axml = new AXMLParser (manifestData, log); XmlDocument? manifest = axml.Parse (); + if (manifest == null) { + log.ErrorLine ("Unable to parse Android manifest from the apk"); + return null; + } + + var writerSettings = new XmlWriterSettings { + Encoding = new UTF8Encoding (false), + Indent = true, + IndentChars = "\t", + NewLineOnAttributes = false, + OmitXmlDeclaration = false, + WriteEndDocumentOnClose = true, + }; + + var manifestXml = new StringBuilder (); + using var writer = XmlWriter.Create (manifestXml, writerSettings); + manifest.WriteTo (writer); + writer.Flush (); + log.DebugLine ("Android manifest from the apk: START"); + log.DebugLine (manifestXml.ToString ()); + log.DebugLine ("Android manifest from the apk: END"); + + string? packageName = null; + XmlNode? node; + + node = manifest.SelectSingleNode ("//manifest"); + if (node == null) { + log.ErrorLine ("Unable to find root element 'manifest' of AndroidManifest.xml"); + return null; + } + + var nsManager = new XmlNamespaceManager (manifest.NameTable); + if (node.Attributes != null) { + const string nsPrefix = "xmlns:"; + + foreach (XmlAttribute attr in node.Attributes) { + if (!attr.Name.StartsWith (nsPrefix, StringComparison.Ordinal)) { + continue; + } + + nsManager.AddNamespace (attr.Name.Substring (nsPrefix.Length), attr.Value); + } + } + + if (String.IsNullOrEmpty (parsedOptions.PackageName)) { + packageName = GetAttributeValue (node, "package"); + } else { + packageName = parsedOptions.PackageName; + } + + if (String.IsNullOrEmpty (packageName)) { + log.ErrorLine ("Unable to determine the package name"); + return null; + } + + node = manifest.SelectSingleNode ("//manifest/uses-sdk"); + string? minSdkVersion = GetAttributeValue (node, "android:minSdkVersion"); + if (String.IsNullOrEmpty (minSdkVersion)) { + log.WarningLine ($"Android manifest doesn't specify the minimum SDK version supported by the application, assuming the default of {DefaultMinSdkVersion}"); + minSdkVersion = DefaultMinSdkVersion; + } - string packageName = String.Empty; + ApplicationInfo? ret; + try { + ret = new ApplicationInfo (packageName, minSdkVersion); + } catch (Exception ex) { + log.ErrorLine ($"Exception {ex.GetType ()} thrown while constructing application info: {ex.Message}"); + return null; + } + + if (String.IsNullOrEmpty (parsedOptions.Activity)) { + node = manifest.SelectSingleNode ("//manifest/application"); + string? debuggable = GetAttributeValue (node, "android:debuggable"); + if (!String.IsNullOrEmpty (debuggable)) { + ret.Debuggable = String.Compare ("true", debuggable, StringComparison.OrdinalIgnoreCase) == 0; + } + + node = manifest.SelectSingleNode ("//manifest/application/activity[./intent-filter/action[@android:name='android.intent.action.MAIN']]", nsManager); + if (node != null) { + ret.Activity = GetAttributeValue (node, "android:name"); + log.DebugLine ($"Detected main activity: {ret.Activity}"); + } + } else { + ret.Activity = parsedOptions.Activity; + } + + return ret; + } + + static string? GetAttributeValue (XmlNode? node, string prefixedAttributeName) + { + if (node?.Attributes == null) { + return null; + } + + foreach (XmlAttribute attr in node.Attributes) { + if (String.Compare (prefixedAttributeName, attr.Name, StringComparison.Ordinal) == 0) { + return attr.Value; + } + } - return new ApplicationInfo (packageName); + return null; } static string? EnsureNonEmptyString (XamarinLoggingHelper log, string paramName, string? value, ref bool haveOptionErrors) diff --git a/tools/xadebug/Xamarin.Android.Debug/AXMLParser.cs b/tools/xadebug/Xamarin.Android.Debug/AXMLParser.cs index 2eccaefc53d..71322671721 100644 --- a/tools/xadebug/Xamarin.Android.Debug/AXMLParser.cs +++ b/tools/xadebug/Xamarin.Android.Debug/AXMLParser.cs @@ -30,20 +30,62 @@ enum ChunkType : ushort TableLibrary = 0x0203, } +enum AttributeType : uint +{ + // The 'data' field is either 0 or 1, specifying this resource is either undefined or empty, respectively. + Null = 0x00, + + // The 'data' field holds a ResTable_ref, a reference to another resource + Reference = 0x01, + + // The 'data' field holds an attribute resource identifier. + Attribute = 0x02, + + // The 'data' field holds an index into the containing resource table's global value string pool. + String = 0x03, + + // The 'data' field holds a single-precision floating point number. + Float = 0x04, + + // The 'data' holds a complex number encoding a dimension value such as "100in". + Dimension = 0x05, + + // The 'data' holds a complex number encoding a fraction of a container. + Fraction = 0x06, + + // The 'data' holds a dynamic ResTable_ref, which needs to be resolved before it can be used like a Reference + DynamicReference = 0x07, + + // The 'data' holds an attribute resource identifier, which needs to be resolved before it can be used like a Attribute. + DynamicAttribute = 0x08, + + // The 'data' is a raw integer value of the form n..n. + IntDec = 0x10, + + // The 'data' is a raw integer value of the form 0xn..n. + IntHex = 0x11, + + // The 'data' is either 0 or 1, for input "false" or "true" respectively. + IntBoolean = 0x12, + + // The 'data' is a raw integer value of the form #aarrggbb. + IntColorARGB8 = 0x1c, + + // The 'data' is a raw integer value of the form #rrggbb. + IntColorRGB8 = 0x1d, + + // The 'data' is a raw integer value of the form #argb. + IntColorARGB4 = 0x1e, + + // The 'data' is a raw integer value of the form #rgb. + IntColorRGB4 = 0x1f, +} + // // Based on https://github.com/androguard/androguard/tree/832104db3eb5dc3cc66b30883fa8ce8712dfa200/androguard/core/axml // class AXMLParser { - enum ParsingState - { - StartDocument = 0, - EndDocument = 1, - StartTag = 2, - EndTag = 3, - Text = 4, - } - // Position of fields inside an attribute const int ATTRIBUTE_IX_NAMESPACE_URI = 0; const int ATTRIBUTE_IX_NAME = 1; @@ -55,6 +97,29 @@ enum ParsingState const long MinimumDataSize = 8; const long MaximumDataSize = (long)UInt32.MaxValue; + const uint ComplexUnitMask = 0x0f; + + static readonly float[] RadixMultipliers = { + 0.00390625f, + 3.051758E-005f, + 1.192093E-007f, + 4.656613E-010f, + }; + + static readonly string[] DimensionUnits = { + "px", + "dip", + "sp", + "pt", + "in", + "mm", + }; + + static readonly string[] FractionUnits = { + "%", + "%p", + }; + readonly XamarinLoggingHelper log; bool axmlTampered; @@ -133,6 +198,13 @@ public AXMLParser (Stream data, XamarinLoggingHelper logger) using var reader = new BinaryReader (data, Encoding.UTF8, leaveOpen: true); ARSCHeader? header; + string? nsPrefix = null; + string? nsUri = null; + uint prefixIndex = 0; + uint uriIndex = 0; + var nsUriToPrefix = new Dictionary (StringComparer.Ordinal); + XmlNode? currentNode = ret.DocumentElement; + while (data.Position < dataSize) { header = new ARSCHeader (data); @@ -171,23 +243,204 @@ public AXMLParser (Stream data, XamarinLoggingHelper logger) log.WarningLine ($"Unhandled Comment at namespace chunk: {commentIndex}"); } - uint prefixIndex; - uint uriIndex; - if (header.Type == ChunkType.XmlStartNamespace) { prefixIndex = reader.ReadUInt32 (); uriIndex = reader.ReadUInt32 (); - string? prefix = stringPool.GetString (prefixIndex); - string? uri = stringPool.GetString (uriIndex); + nsPrefix = stringPool.GetString (prefixIndex); + nsUri = stringPool.GetString (uriIndex); + + if (!String.IsNullOrEmpty (nsUri)) { + nsUriToPrefix[nsUri] = nsPrefix ?? String.Empty; + } + + log.DebugLine ($"Start of Namespace mapping: prefix {prefixIndex}: '{nsPrefix}' --> uri {uriIndex}: '{nsUri}'"); + + if (String.IsNullOrEmpty (nsUri)) { + log.WarningLine ($"Namespace prefix '{nsPrefix}' resolves to empty URI."); + } + + continue; + } + + if (header.Type == ChunkType.XmlEndNamespace) { + // Namespace handling is **really** simplified, since we expect to deal only with AndroidManifest.xml which should have just one namespace. + // There should be no problems with that. Famous last words. + uint endPrefixIndex = reader.ReadUInt32 (); + uint endUriIndex = reader.ReadUInt32 (); + + log.DebugLine ($"End of Namespace mapping: prefix {endPrefixIndex}, uri {endUriIndex}"); + if (endPrefixIndex != prefixIndex) { + log.WarningLine ($"Prefix index of Namespace end doesn't match the last Namespace prefix index: {prefixIndex} != {endPrefixIndex}"); + } + + if (endUriIndex != uriIndex) { + log.WarningLine ($"URI index of Namespace end doesn't match the last Namespace URI index: {uriIndex} != {endUriIndex}"); + } + + string? endUri = stringPool.GetString (endUriIndex); + if (!String.IsNullOrEmpty (endUri) && nsUriToPrefix.ContainsKey (endUri)) { + nsUriToPrefix.Remove (endUri); + } + + nsPrefix = null; + nsUri = null; + prefixIndex = 0; + uriIndex = 0; + + continue; + } + + uint tagNsUriIndex; + uint tagNameIndex; + string? tagName; + string? tagNs; // TODO: implement + + if (header.Type == ChunkType.XmlStartElement) { + // The TAG consists of some fields: + // * (chunk_size, line_number, comment_index - we read before) + // * namespace_uri + // * name + // * flags + // * attribute_count + // * class_attribute + // After that, there are two lists of attributes, 20 bytes each + tagNsUriIndex = reader.ReadUInt32 (); + tagNameIndex = reader.ReadUInt32 (); + uint tagFlags = reader.ReadUInt32 (); + uint attributeCount = reader.ReadUInt32 () & 0xffff; + uint classAttribute = reader.ReadUInt32 (); + + // Tag name is, of course, required but instead of throwing an exception should we find none, we use a fake name in hope that we can still salvage + // the document. + tagName = stringPool.GetString (tagNameIndex) ?? "unnamedTag"; + log.DebugLine ($"Start of tag '{tagName}', NS URI index {tagNsUriIndex}"); + log.DebugLine ($"Reading tag attributes ({attributeCount}):"); + + string? tagNsUri = tagNsUriIndex != 0xffffffff ? stringPool.GetString (tagNsUriIndex) : null; + string? tagNsPrefix; + + if (String.IsNullOrEmpty (tagNsUri) || !nsUriToPrefix.TryGetValue (tagNsUri, out tagNsPrefix)) { + tagNsPrefix = null; + } + XmlElement element = ret.CreateElement (tagNsPrefix, tagName, tagNsUri); + if (currentNode == null) { + ret.AppendChild (element); + if (!String.IsNullOrEmpty (nsPrefix) && !String.IsNullOrEmpty (nsUri)) { + ret.DocumentElement!.SetAttribute ($"xmlns:{nsPrefix}", nsUri); + } + } else { + currentNode.AppendChild (element); + } + currentNode = element; + + for (uint i = 0; i < attributeCount; i++) { + uint attrNsIdx = reader.ReadUInt32 (); // string index + uint attrNameIdx = reader.ReadUInt32 (); // string index + uint attrValue = reader.ReadUInt32 (); + uint attrType = reader.ReadUInt32 () >> 24; + uint attrData = reader.ReadUInt32 (); + + string? attrNs = attrNsIdx != 0xffffffff ? stringPool.GetString (attrNsIdx) : String.Empty; + string? attrName = stringPool.GetString (attrNameIdx); + + if (String.IsNullOrEmpty (attrName)) { + log.WarningLine ($"Attribute without name, ignoring. Offset: {data.Position}"); + continue; + } + + log.DebugLine ($" '{attrName}': ns == '{attrNs}'; value == 0x{attrValue:x}; type == 0x{attrType:x}; data == 0x{attrData:x}"); + XmlAttribute attr; + + if (!String.IsNullOrEmpty (attrNs)) { + attr = ret.CreateAttribute (nsUriToPrefix[attrNs], attrName, attrNs); + } else { + attr = ret.CreateAttribute (attrName!); + } + attr.Value = GetAttributeValue (attrValue, attrType, attrData); + element.SetAttributeNode (attr); + } continue; } + + if (header.Type == ChunkType.XmlEndElement) { + tagNsUriIndex = reader.ReadUInt32 (); + tagNameIndex = reader.ReadUInt32 (); + + tagName = stringPool.GetString (tagNameIndex); + log.DebugLine ($"End of tag '{tagName}', NS URI index {tagNsUriIndex}"); + currentNode = currentNode.ParentNode!; + continue; + } + + // TODO: add support for CDATA } return ret; } + string GetAttributeValue (uint attrValue, uint attrType, uint attrData) + { + if (!Enum.IsDefined (typeof(AttributeType), attrType)) { + log.WarningLine ($"Unknown attribute type value 0x{attrType:x}, returning empty attribute value (data == 0x{attrData:x}). Offset: {data.Position}"); + return String.Empty; + } + + switch ((AttributeType)attrType) { + case AttributeType.Null: + return attrData == 0 ? "?NULL?" : String.Empty; + + case AttributeType.Reference: + return $"@{MaybePrefix()}{attrData:x08}"; + + case AttributeType.Attribute: + return $"?{MaybePrefix()}{attrData:x08}"; + + case AttributeType.String: + return stringPool.GetString (attrData) ?? String.Empty; + + case AttributeType.Float: + return $"{(float)attrData}"; + + case AttributeType.Dimension: + return $"{ComplexToFloat(attrData)}{DimensionUnits[attrData & ComplexUnitMask]}"; + + case AttributeType.Fraction: + return $"{ComplexToFloat(attrData) * 100.0f}{FractionUnits[attrData & ComplexUnitMask]}"; + + case AttributeType.IntDec: + return attrData.ToString (); + + case AttributeType.IntHex: + return $"0x{attrData:X08}"; + + case AttributeType.IntBoolean: + return attrData == 0 ? "false" : "true"; + + case AttributeType.IntColorARGB8: + case AttributeType.IntColorRGB8: + case AttributeType.IntColorARGB4: + case AttributeType.IntColorRGB4: + return $"#{attrData:X08}"; + } + + return String.Empty; + + string MaybePrefix () + { + if (attrData >> 24 == 1) { + return "android:"; + } + return String.Empty; + } + + float ComplexToFloat (uint value) + { + return (float)(value & 0xffffff00) * RadixMultipliers[(value >> 4) & 3]; + } + } + bool SkipOverResourceMap (ARSCHeader header, BinaryReader reader) { log.DebugLine ("AXML contains a resource map"); diff --git a/tools/xadebug/Xamarin.Android.Debug/ApplicationInfo.cs b/tools/xadebug/Xamarin.Android.Debug/ApplicationInfo.cs index 2a04ecb018f..1cfbe3e00e1 100644 --- a/tools/xadebug/Xamarin.Android.Debug/ApplicationInfo.cs +++ b/tools/xadebug/Xamarin.Android.Debug/ApplicationInfo.cs @@ -1,11 +1,21 @@ +using System; + namespace Xamarin.Android.Debug; class ApplicationInfo { public string PackageName { get; } + public uint MinSdkVersion { get; } + public bool Debuggable { get; set; } + public string? Activity { get; set; } - public ApplicationInfo (string packageName) + public ApplicationInfo (string packageName, string minSdkVersion) { PackageName = packageName; + + if (!UInt32.TryParse (minSdkVersion, out uint ver)) { + throw new ArgumentException ($"Unable to parse minimum SDK version from '{minSdkVersion}'", nameof (minSdkVersion)); + } + MinSdkVersion = ver; } } From 82d90f4ae7f3d27cbf19810a00523a4a01fd6814 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 8 Dec 2022 23:36:08 +0100 Subject: [PATCH 23/30] [WIP] App building and APK discovery --- tools/xadebug/Main.cs | 158 +++++++++++++++++- .../Xamarin.Android.Utilities/DotNetRunner.cs | 107 ++++++++++++ tools/xadebug/xadebug.csproj | 1 + 3 files changed, 259 insertions(+), 7 deletions(-) create mode 100644 tools/xadebug/Xamarin.Android.Utilities/DotNetRunner.cs diff --git a/tools/xadebug/Main.cs b/tools/xadebug/Main.cs index 41641f57ec4..244228dd515 100644 --- a/tools/xadebug/Main.cs +++ b/tools/xadebug/Main.cs @@ -4,6 +4,7 @@ using System.Text; using System.Xml; +using Microsoft.Build.Logging.StructuredLogger; using Mono.Options; using Xamarin.Android.Utilities; using Xamarin.Tools.Zip; @@ -19,6 +20,8 @@ sealed class ParsedOptions public string Configuration = "Debug"; public string? PackageName; public string? Activity; + public string DotNetCommand = "dotnet"; + public string WorkDirectory = "xadebug-data"; } const string AndroidManifestZipPath = "AndroidManifest.xml"; @@ -36,8 +39,10 @@ static int Main (string[] args) "Usage: dotnet xadebug [OPTIONS] ", "", { "p|package-name=", "name of the application package", v => parsedOptions.PackageName = EnsureNonEmptyString (log, "-p|--package-name", v, ref haveOptionErrors) }, - { "c|configuration=", "{CONFIGURATION} in which to build the application. Ignored when running in APK-only mode", v => parsedOptions.Configuration = v }, - { "a|activity=", "Name of the {ACTIVITY} to start the application. The default is determined from AndroidManifest.xml", v => parsedOptions.Activity = v }, + { "c|configuration=", $"{{CONFIGURATION}} in which to build the application. Ignored when running in APK-only mode. Default: {parsedOptions.Configuration}", v => parsedOptions.Configuration = v }, + { "a|activity=", "Name of the {ACTIVITY} to start the application. Default: determined from AndroidManifest.xml inside the APK", v => parsedOptions.Activity = v }, + { "d|dotnet=", $"Name of the dotnet {{COMMAND}} to use when building a project. Defaults to {parsedOptions.DotNetCommand}", v => parsedOptions.DotNetCommand = v }, + { "w|work-dir=", $"{{DIRECTORY}} in which xadebug will store build and debug logs, as well as shared libraries with symbols. Default: {parsedOptions.WorkDirectory}", v => parsedOptions.WorkDirectory = v }, "", { "v|verbose", "Show debug messages", v => parsedOptions.Verbose = true }, { "h|help|?", "Show this help screen", v => parsedOptions.ShowHelp = true }, @@ -48,7 +53,7 @@ static int Main (string[] args) if (parsedOptions.ShowHelp || rest.Count == 0) { int ret = 0; - if (rest.Count == 0) { + if (!parsedOptions.ShowHelp) { log.ErrorLine ("Path to application APK or directory with a C# project must be specified"); log.ErrorLine (); ret = 1; @@ -58,6 +63,11 @@ static int Main (string[] args) return ret; } + if (String.IsNullOrEmpty (parsedOptions.DotNetCommand)) { + log.ErrorLine ("Empty string passed in the `-d|--dotnet` parameter. It must be a non-empty string."); + haveOptionErrors = true; + } + if (haveOptionErrors) { return 1; } @@ -67,11 +77,11 @@ static int Main (string[] args) ZipArchive? apk = null; if (Directory.Exists (aPath)) { - apkFilePath = BuildApp (aPath); + apkFilePath = BuildApp (aPath, parsedOptions); } else if (File.Exists (aPath)) { if (String.Compare (".csproj", Path.GetExtension (aPath), StringComparison.OrdinalIgnoreCase) == 0) { // Let's see if we can trust the file name... - apkFilePath = BuildApp (aPath); + apkFilePath = BuildApp (aPath, parsedOptions); } else if (IsAndroidPackageFile (aPath, out apk)) { apkFilePath = aPath; } else { @@ -97,6 +107,15 @@ static int Main (string[] args) return 1; } + if (!appInfo.Debuggable) { + log.ErrorLine ($"Application {apkFilePath} is not debuggable."); + log.MessageLine (); + log.MessageLine ("Please rebuild the aplication either in `Debug` configuration or with appropriate properties set in `Release` configuration:"); + log.MessageLine ("TODO: fill in instructions"); + log.MessageLine (); + return 1; + } + return 0; } @@ -242,8 +261,133 @@ static bool IsAndroidPackageFile (string filePath, out ZipArchive? apk) return apk.ContainsEntry (AndroidManifestZipPath); } - static string BuildApp (string projectPath) + static string? BuildApp (string projectPath, ParsedOptions parsedOptions) { - throw new NotImplementedException ("Building the application is not implemented yet"); + var dotnet = new DotNetRunner (log, parsedOptions.DotNetCommand, parsedOptions.WorkDirectory); + string? logPath = dotnet.Build ( + projectPath, + parsedOptions.Configuration, + "-p:AndroidCreatePackagePerAbi=False", + "-p:AndroidPackageFormat=apk", + "-p:_AndroidAotStripLibraries=False", + "-p:_AndroidEnableNativeDebugging=True", + "-p:_AndroidStripNativeLibraries=False" + ).Result; + string? apkPath = FindApkPathFromLog (logPath); + + if (String.IsNullOrEmpty (apkPath)) { + apkPath = TryToGuessApkPath (projectPath, parsedOptions); + } + + if (String.IsNullOrEmpty (apkPath)) { + log.ErrorLine ("Unable to determine path to the application APK file after build."); + log.MessageLine (); + log.MessageLine ("Please run `xadebug` again, passing it path to the produced APK file"); + log.MessageLine (); + return null; + } + + return apkPath; + } + + static string? TryToGuessApkPath (string projectPath, ParsedOptions parsedOptions) + { + string projectDir; + + if (File.Exists (projectPath)) { + projectDir = Path.GetDirectoryName (projectPath) ?? "."; + } else { + projectDir = projectPath; + } + + log.DebugLine ("Trying to find application APK in {projectDir}"); + + string binDir = Path.Combine (projectDir, "bin", parsedOptions.Configuration); + if (!Directory.Exists (binDir)) { + log.WarningLine ($"Bin output directory '{binDir}' does not exist. Unable to determine path to the produced APK"); + return null; + } + + const string ApkSuffix = "-Signed.apk"; + string apkName; + bool apkNameIsGlob; + + if (!String.IsNullOrEmpty (parsedOptions.PackageName)) { + apkName = $"{parsedOptions.PackageName}{ApkSuffix}"; + apkNameIsGlob = false; + } else { + apkName = $"*{ApkSuffix}"; + apkNameIsGlob = true; + } + + log.StatusLine ("Looking for APK with name", apkName); + + if (!apkNameIsGlob) { + string apkPath = Path.Combine (binDir, apkName); + LogPotentialPath (apkPath); + + if (File.Exists (apkPath)) { + return LogFoundAndReturn (apkPath); + } + } + + // Find subdirectories named netX.Y-android and the apk files inside them + var apkFiles = new List (); + foreach (string dir in Directory.EnumerateDirectories (binDir, "net*-android")) { + if (apkNameIsGlob) { + foreach (string file in Directory.EnumerateFiles (dir, apkName)) { + // We know it exists, but the method also logs paths + AddApkIfExists (file, apkFiles); + } + } else { + AddApkIfExists (Path.Combine (dir, apkName), apkFiles); + } + } + + if (apkFiles.Count == 0) { + return null; + } + + string selectedApkPath; + if (apkFiles.Count > 1) { + // TODO: ask the user to select one + throw new NotImplementedException ("Support for multiple APK files not implemented yet"); + } else { + selectedApkPath = apkFiles[0]; + } + + return LogFoundAndReturn (selectedApkPath); + + void AddApkIfExists (string apkPath, List apkFiles) + { + LogPotentialPath (apkPath); + if (File.Exists (apkPath)) { + log.DebugLine ($"Found APK: {apkPath}"); + apkFiles.Add (apkPath); + } + } + + void LogPotentialPath (string path) + { + log.DebugLine ($"Trying path: {path}"); + } + + string LogFoundAndReturn (string path) + { + log.DebugLine ($"Returning APK path: {path}"); + return path; + } + } + + static string? FindApkPathFromLog (string? logPath) + { + if (String.IsNullOrEmpty (logPath)) { + return null; + } + + Build build = BinaryLog.ReadBuild (logPath); + // TODO: figure out how to find the `_Sign` target's Outputs... + + return null; } } diff --git a/tools/xadebug/Xamarin.Android.Utilities/DotNetRunner.cs b/tools/xadebug/Xamarin.Android.Utilities/DotNetRunner.cs new file mode 100644 index 00000000000..145492a4615 --- /dev/null +++ b/tools/xadebug/Xamarin.Android.Utilities/DotNetRunner.cs @@ -0,0 +1,107 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading.Tasks; + +using Xamarin.Android.Tasks; + +namespace Xamarin.Android.Utilities; + +class DotNetRunner : ToolRunner +{ + class StreamOutputSink : ToolOutputSink + { + readonly StreamWriter output; + + public StreamOutputSink (XamarinLoggingHelper logger, StreamWriter output) + : base (logger) + { + this.output = output; + } + + public override void WriteLine (string? value) + { + output.WriteLine (value ?? String.Empty); + } + } + + readonly string workDirectory; + + public DotNetRunner (XamarinLoggingHelper logger, string toolPath, string workDirectory) + : base (logger, toolPath) + { + this.workDirectory = workDirectory; + } + + /// + /// Build project at (either path to a project directory or a project file) and return full + /// path to binary build log, if the build succeded, or null otherwise. + /// + public async Task Build (string projectPath, string configuration, params string[] extraArgs) + { + // TODO: use CRC64 instead of GetHashCode(), as the latter is not deterministic in .NET6+ + string projectWorkDir = Path.Combine (workDirectory, projectPath.GetHashCode ().ToString ("x")); + + Directory.CreateDirectory (projectWorkDir); + string binlogPath = Path.Combine (projectWorkDir, "build.binlog"); + + var runner = CreateProcessRunner ("build"); + runner. + AddArgument ("-c").AddArgument (configuration). + AddArgument ($"-bl:\"{binlogPath}\""). + AddQuotedArgument (projectPath); + + if (!await RunDotNet (runner)) { + return null; + } + + //return await ConvertBinlogToText (binlogPath); + return binlogPath; + } + + public async Task ConvertBinlogToText (string binlogPath) + { + if (!File.Exists (binlogPath)) { + Logger.ErrorLine ($"Binlog '{binlogPath}' does not exist, cannot convert to text"); + return null; + } + + bool stdoutEchoOrig = EchoStandardOutput; + ProcessRunner runner; + + try { + EchoStandardOutput = false; + runner = CreateProcessRunner ("msbuild"); + } finally { + EchoStandardOutput = stdoutEchoOrig; + } + + runner. + AddArgument ("-v:diag"). + AddQuotedArgument (binlogPath); + + string logOutput = Path.ChangeExtension (binlogPath, ".txt"); + using var fs = File.Open (logOutput, FileMode.Create); + using var sw = new StreamWriter (fs, Xamarin.Android.Debug.Utilities.UTF8NoBOM); + using var sink = new StreamOutputSink (Logger, sw); + + try { + if (!await RunDotNet (runner, sink)) { + return null; + } + } finally { + sw.Flush (); + } + + return logOutput; + } + + async Task RunDotNet (ProcessRunner runner, ToolOutputSink? outputSink = null) + { + if (outputSink != null) { + SetupOutputSinks (runner, stdoutSink: outputSink, stderrSink: null, ignoreStderr: true); + } + + return await RunTool (() => runner.Run ()); + } +} diff --git a/tools/xadebug/xadebug.csproj b/tools/xadebug/xadebug.csproj index 591de7bea9b..3784c8cc11a 100644 --- a/tools/xadebug/xadebug.csproj +++ b/tools/xadebug/xadebug.csproj @@ -34,6 +34,7 @@ + From a62ef71dad4262ad3cb50ef33ab9d35ebafc97ba Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Fri, 9 Dec 2022 22:23:32 +0100 Subject: [PATCH 24/30] [WIP] Debug session, beginnings --- tools/xadebug/Main.cs | 135 ++++-- .../Xamarin.Android.Debug/AXMLParser.cs | 7 +- .../Xamarin.Android.Debug/AndroidDevice.cs | 439 ++++++++++++++++++ .../Xamarin.Android.Debug/AndroidNdk.cs | 101 ++++ .../Xamarin.Android.Debug/DebugSession.cs | 88 ++++ .../DeviceLibrariesCopier.cs | 66 +++ .../Xamarin.Android.Debug/LdConfigParser.cs | 167 +++++++ .../LddDeviceLibrariesCopier.cs | 18 + .../Xamarin.Android.Debug/LldbModuleCache.cs | 170 +++++++ .../NoLddDeviceLibrariesCopier.cs | 190 ++++++++ .../NoLddLldbModuleCache.cs | 41 ++ .../Xamarin.Android.Debug/Utilities.cs | 20 + .../Xamarin.Android.Utilities/DotNetRunner.cs | 17 +- tools/xadebug/xadebug.csproj | 12 +- 14 files changed, 1422 insertions(+), 49 deletions(-) create mode 100644 tools/xadebug/Xamarin.Android.Debug/AndroidDevice.cs create mode 100644 tools/xadebug/Xamarin.Android.Debug/AndroidNdk.cs create mode 100644 tools/xadebug/Xamarin.Android.Debug/DebugSession.cs create mode 100644 tools/xadebug/Xamarin.Android.Debug/DeviceLibrariesCopier.cs create mode 100644 tools/xadebug/Xamarin.Android.Debug/LdConfigParser.cs create mode 100644 tools/xadebug/Xamarin.Android.Debug/LddDeviceLibrariesCopier.cs create mode 100644 tools/xadebug/Xamarin.Android.Debug/LldbModuleCache.cs create mode 100644 tools/xadebug/Xamarin.Android.Debug/NoLddDeviceLibrariesCopier.cs create mode 100644 tools/xadebug/Xamarin.Android.Debug/NoLddLldbModuleCache.cs diff --git a/tools/xadebug/Main.cs b/tools/xadebug/Main.cs index 244228dd515..287eb4613c9 100644 --- a/tools/xadebug/Main.cs +++ b/tools/xadebug/Main.cs @@ -11,22 +11,30 @@ namespace Xamarin.Android.Debug; -class XADebug +sealed class ParsedOptions { - sealed class ParsedOptions - { - public bool ShowHelp; - public bool Verbose = true; // TODO: remove the default once development is done - public string Configuration = "Debug"; - public string? PackageName; - public string? Activity; - public string DotNetCommand = "dotnet"; - public string WorkDirectory = "xadebug-data"; - } + public bool ShowHelp; + public bool Verbose = true; // TODO: remove the default once development is done + public string Configuration = "Debug"; + public string? PackageName; + public string? Activity; + public string DotNetCommand = "dotnet"; + public string WorkDirectory = "xadebug-data"; + public string AdbPath = "adb"; + public string? TargetDevice; + public string? NdkDirPath; +} +class XADebug +{ const string AndroidManifestZipPath = "AndroidManifest.xml"; const string DefaultMinSdkVersion = "21"; + static readonly string[] NdkEnvvars = { + "ANDROID_NDK_PATH", + "ANDROID_NDK_ROOT", + }; + static XamarinLoggingHelper log = new XamarinLoggingHelper (); static int Main (string[] args) @@ -44,6 +52,10 @@ static int Main (string[] args) { "d|dotnet=", $"Name of the dotnet {{COMMAND}} to use when building a project. Defaults to {parsedOptions.DotNetCommand}", v => parsedOptions.DotNetCommand = v }, { "w|work-dir=", $"{{DIRECTORY}} in which xadebug will store build and debug logs, as well as shared libraries with symbols. Default: {parsedOptions.WorkDirectory}", v => parsedOptions.WorkDirectory = v }, "", + { "s|adb=", "{PATH} to adb to use for this session", v => parsedOptions.AdbPath = EnsureNonEmptyString (log, "-s|--adb", v, ref haveOptionErrors) }, + { "e|device=", "ID of {DEVICE} to target for this session", v => parsedOptions.TargetDevice = EnsureNonEmptyString (log, "-e|--device", v, ref haveOptionErrors) }, + { "n|ndk-dir=", "{PATH} to to the Android NDK root directory", v => parsedOptions.NdkDirPath = v }, + "", { "v|verbose", "Show debug messages", v => parsedOptions.Verbose = true }, { "h|help|?", "Show this help screen", v => parsedOptions.ShowHelp = true }, }; @@ -68,20 +80,48 @@ static int Main (string[] args) haveOptionErrors = true; } + if (String.IsNullOrEmpty (parsedOptions.NdkDirPath)) { + string? ndk = null; + + foreach (string envvar in NdkEnvvars) { + log.DebugLine ($"Trying to read NDK path environment variable '{envvar}'"); + ndk = Environment.GetEnvironmentVariable (envvar); + if (!String.IsNullOrEmpty (ndk)) { + log.DebugLine ($"Potential NDK location: {ndk}"); + break; + } + } + + if (String.IsNullOrEmpty (ndk)) { + log.ErrorLine ("Unable to locate Android NDK from environment variables"); + log.MessageLine ("Please provide path to the NDK using the '-n|--ndk' argument"); + haveOptionErrors = true; + } else { + parsedOptions.NdkDirPath = ndk; + } + } + + if (!Directory.Exists (parsedOptions.NdkDirPath)) { + log.ErrorLine ($"NDK directory '{parsedOptions.NdkDirPath}' does not exist"); + return 1; + } + if (haveOptionErrors) { return 1; } + log.StatusLine ("Using NDK", parsedOptions.NdkDirPath); + string aPath = rest[0]; string? apkFilePath = null; ZipArchive? apk = null; if (Directory.Exists (aPath)) { - apkFilePath = BuildApp (aPath, parsedOptions); + apkFilePath = BuildApp (aPath, parsedOptions, projectPathIsDirectory: true); } else if (File.Exists (aPath)) { if (String.Compare (".csproj", Path.GetExtension (aPath), StringComparison.OrdinalIgnoreCase) == 0) { // Let's see if we can trust the file name... - apkFilePath = BuildApp (aPath, parsedOptions); + apkFilePath = BuildApp (aPath, parsedOptions, projectPathIsDirectory: false); } else if (IsAndroidPackageFile (aPath, out apk)) { apkFilePath = aPath; } else { @@ -97,6 +137,8 @@ static int Main (string[] args) return 1; } + log.StatusLine ("Input APK", apkFilePath); + if (apk == null) { apk = OpenApk (apkFilePath); } @@ -116,7 +158,8 @@ static int Main (string[] args) return 1; } - return 0; + var debugSession = new DebugSession (log, apkFilePath, apk, parsedOptions); + return debugSession.Prepare () ? 0 : 1; } static ApplicationInfo? ReadManifest (ZipArchive apk, ParsedOptions parsedOptions) @@ -235,12 +278,12 @@ static int Main (string[] args) return null; } - static string? EnsureNonEmptyString (XamarinLoggingHelper log, string paramName, string? value, ref bool haveOptionErrors) + static string EnsureNonEmptyString (XamarinLoggingHelper log, string paramName, string? value, ref bool haveOptionErrors) { if (String.IsNullOrEmpty (value)) { haveOptionErrors = true; log.ErrorLine ($"Parameter '{paramName}' requires a non-empty string as its value"); - return null; + return String.Empty; } return value; @@ -261,7 +304,7 @@ static bool IsAndroidPackageFile (string filePath, out ZipArchive? apk) return apk.ContainsEntry (AndroidManifestZipPath); } - static string? BuildApp (string projectPath, ParsedOptions parsedOptions) + static string? BuildApp (string projectPath, ParsedOptions parsedOptions, bool projectPathIsDirectory) { var dotnet = new DotNetRunner (log, parsedOptions.DotNetCommand, parsedOptions.WorkDirectory); string? logPath = dotnet.Build ( @@ -273,10 +316,17 @@ static bool IsAndroidPackageFile (string filePath, out ZipArchive? apk) "-p:_AndroidEnableNativeDebugging=True", "-p:_AndroidStripNativeLibraries=False" ).Result; - string? apkPath = FindApkPathFromLog (logPath); + + if (String.IsNullOrEmpty (logPath)) { + return null; + } + + string projectDir = projectPathIsDirectory ? projectPath : Path.GetDirectoryName (projectPath) ?? "."; + string? apkPath = FindApkPathFromLog (projectDir, logPath); if (String.IsNullOrEmpty (apkPath)) { - apkPath = TryToGuessApkPath (projectPath, parsedOptions); + log.DebugLine ("Could not get APK path from build log, trying to guess"); + apkPath = TryToGuessApkPath (projectDir, parsedOptions); } if (String.IsNullOrEmpty (apkPath)) { @@ -287,19 +337,16 @@ static bool IsAndroidPackageFile (string filePath, out ZipArchive? apk) return null; } + if (!File.Exists (apkPath)) { + log.ErrorLine ($"APK file '{apkPath}' not found after build"); + return null; + }; + return apkPath; } - static string? TryToGuessApkPath (string projectPath, ParsedOptions parsedOptions) + static string? TryToGuessApkPath (string projectDir, ParsedOptions parsedOptions) { - string projectDir; - - if (File.Exists (projectPath)) { - projectDir = Path.GetDirectoryName (projectPath) ?? "."; - } else { - projectDir = projectPath; - } - log.DebugLine ("Trying to find application APK in {projectDir}"); string binDir = Path.Combine (projectDir, "bin", parsedOptions.Configuration); @@ -327,7 +374,7 @@ static bool IsAndroidPackageFile (string filePath, out ZipArchive? apk) LogPotentialPath (apkPath); if (File.Exists (apkPath)) { - return LogFoundAndReturn (apkPath); + return LogFoundApkPathAndReturn (apkPath); } } @@ -356,7 +403,7 @@ static bool IsAndroidPackageFile (string filePath, out ZipArchive? apk) selectedApkPath = apkFiles[0]; } - return LogFoundAndReturn (selectedApkPath); + return LogFoundApkPathAndReturn (selectedApkPath); void AddApkIfExists (string apkPath, List apkFiles) { @@ -371,23 +418,35 @@ void LogPotentialPath (string path) { log.DebugLine ($"Trying path: {path}"); } - - string LogFoundAndReturn (string path) - { - log.DebugLine ($"Returning APK path: {path}"); - return path; - } } - static string? FindApkPathFromLog (string? logPath) + static string? FindApkPathFromLog (string projectDir, string? logPath) { if (String.IsNullOrEmpty (logPath)) { return null; } + log.DebugLine ($"Trying to find APK file path in the build log ('{logPath}')"); + Build build = BinaryLog.ReadBuild (logPath); - // TODO: figure out how to find the `_Sign` target's Outputs... + foreach (Property prop in build.FindChildrenRecursive ()) { + if (String.Compare ("ApkFileSigned", prop.Name, StringComparison.Ordinal) != 0) { + continue; + } + + if (Path.IsPathRooted (prop.Value)) { + return LogFoundApkPathAndReturn (prop.Value); + } + + return LogFoundApkPathAndReturn (Path.Combine (projectDir, prop.Value)); + } return null; } + + static string LogFoundApkPathAndReturn (string path) + { + log.DebugLine ($"Returning APK path: {path}"); + return path; + } } diff --git a/tools/xadebug/Xamarin.Android.Debug/AXMLParser.cs b/tools/xadebug/Xamarin.Android.Debug/AXMLParser.cs index 71322671721..90b6f088e0f 100644 --- a/tools/xadebug/Xamarin.Android.Debug/AXMLParser.cs +++ b/tools/xadebug/Xamarin.Android.Debug/AXMLParser.cs @@ -122,7 +122,6 @@ class AXMLParser readonly XamarinLoggingHelper log; - bool axmlTampered; Stream data; long dataSize; ARSCHeader axmlHeader; @@ -168,12 +167,10 @@ public AXMLParser (Stream data, XamarinLoggingHelper logger) } if (fileSize < dataSize) { - axmlTampered = true; log.WarningLine ($"Declared data size ({fileSize}) is smaller than total data size ({dataSize}). Was something appended to the file? Trying to parse it anyways."); } if (axmlHeader.Type != ChunkType.Xml) { - axmlTampered = true; log.WarningLine ($"AXML file has an unusual resource type, trying to parse it anyways. Resource Type: 0x{(ushort)axmlHeader.Type:04x}"); } @@ -294,7 +291,7 @@ public AXMLParser (Stream data, XamarinLoggingHelper logger) uint tagNsUriIndex; uint tagNameIndex; string? tagName; - string? tagNs; // TODO: implement +// string? tagNs; // TODO: implement if (header.Type == ChunkType.XmlStartElement) { // The TAG consists of some fields: @@ -370,7 +367,7 @@ public AXMLParser (Stream data, XamarinLoggingHelper logger) tagName = stringPool.GetString (tagNameIndex); log.DebugLine ($"End of tag '{tagName}', NS URI index {tagNsUriIndex}"); - currentNode = currentNode.ParentNode!; + currentNode = currentNode?.ParentNode!; continue; } diff --git a/tools/xadebug/Xamarin.Android.Debug/AndroidDevice.cs b/tools/xadebug/Xamarin.Android.Debug/AndroidDevice.cs new file mode 100644 index 00000000000..4e8db407821 --- /dev/null +++ b/tools/xadebug/Xamarin.Android.Debug/AndroidDevice.cs @@ -0,0 +1,439 @@ +using System; +using System.Collections.Generic; +using System.IO; + +using Xamarin.Android.Tasks; +using Xamarin.Android.Utilities; + +namespace Xamarin.Android.Debug; + +class AndroidDevice +{ + const string ServerLauncherScriptName = "xa_start_lldb_server.sh"; + + static readonly string[] abiProperties = { + // new properties + "ro.product.cpu.abilist", + + // old properties + "ro.product.cpu.abi", + "ro.product.cpu.abi2", + }; + + static readonly string[] serialNumberProperties = { + "ro.serialno", + "ro.boot.serialno", + }; + + string packageName; + string adbPath; + List supportedAbis; + int apiLevel = -1; + string? appDataDir; + string? appLldbBaseDir; + string? appLldbBinDir; + string? appLldbLogDir; + string? appLldbTmpDir; + string? mainAbi; + string? mainArch; + string[]? availableAbis; + string[]? availableArches; + string? serialNumber; + bool appIs64Bit; + string? deviceLdd; + string? deviceDebugServerPath; + string? deviceDebugServerScriptPath; + string outputDir; + + XamarinLoggingHelper log; + AdbRunner adb; + AndroidNdk ndk; + + public int ApiLevel => apiLevel; + public string[] AvailableAbis => availableAbis ?? new string[] {}; + public string[] AvailableArches => availableArches ?? new string[] {}; + public string MainArch => mainArch ?? String.Empty; + public string MainAbi => mainAbi ?? String.Empty; + public string SerialNumber => serialNumber ?? String.Empty; + public string DebugServerLauncherScriptPath => deviceDebugServerScriptPath ?? String.Empty; + public string LldbBaseDir => appLldbBaseDir ?? String.Empty; + public AdbRunner AdbRunner => adb; + + public AndroidDevice (XamarinLoggingHelper log, AndroidNdk ndk, string outputDir, string adbPath, string packageName, List supportedAbis, string? adbTargetDevice = null) + { + this.adbPath = adbPath; + this.log = log; + this.packageName = packageName; + this.supportedAbis = supportedAbis; + this.ndk = ndk; + this.outputDir = outputDir; + + adb = new AdbRunner (log, adbPath, adbTargetDevice); + } + + // TODO: implement manual error checking on API 21, since `adb` won't ever return any error code other than 0 - we need to look at the output of any command to determine + // whether or not it was successful. Ugh. + public bool GatherInfo () + { + (bool success, string output) = adb.GetPropertyValue ("ro.build.version.sdk").Result; + if (!success || String.IsNullOrEmpty (output) || !Int32.TryParse (output, out apiLevel)) { + log.ErrorLine ("Unable to determine connected device's API level"); + return false; + } + + // Warn on old Pixel C firmware (b/29381985). Newer devices may have Yama + // enabled but still work with ndk-gdb (b/19277529). + (success, output) = adb.Shell ("cat", "/proc/sys/kernel/yama/ptrace_scope", "2>/dev/null").Result; + if (success && + YamaOK (output.Trim ()) && + PropertyIsEqualTo (adb.GetPropertyValue ("ro.build.product").Result, "dragon") && + PropertyIsEqualTo (adb.GetPropertyValue ("ro.product.name").Result, "ryu") + ) { + log.WarningLine ("WARNING: The device uses Yama ptrace_scope to restrict debugging. ndk-gdb will"); + log.WarningLine (" likely be unable to attach to a process. With root access, the restriction"); + log.WarningLine (" can be lifted by writing 0 to /proc/sys/kernel/yama/ptrace_scope. Consider"); + log.WarningLine (" upgrading your Pixel C to MXC89L or newer, where Yama is disabled."); + log.WarningLine (); + } + + if (!DetermineArchitectureAndABI ()) { + return false; + } + + if (!DetermineAppDataDirectory ()) { + return false; + } + + serialNumber = GetFirstFoundPropertyValue (serialNumberProperties); + if (String.IsNullOrEmpty (serialNumber)) { + log.WarningLine ("Unable to determine device serial number"); + } else { + log.StatusLine ($"Device serial number", serialNumber); + } + + return true; + + bool YamaOK (string output) + { + return !String.IsNullOrEmpty (output) && String.Compare ("0", output, StringComparison.Ordinal) != 0; + } + + bool PropertyIsEqualTo ((bool haveProperty, string value) result, string expected) + { + return + result.haveProperty && + !String.IsNullOrEmpty (result.value) && + String.Compare (result.value, expected, StringComparison.Ordinal) == 0; + } + } + + public bool Prepare (out string? mainProcessPath) + { + mainProcessPath = null; + if (!DetectTools ()) { + return false; + } + + if (!PushDebugServer ()) { + return false; + } + + if (!PullLibraries (out mainProcessPath)) { + return false; + } + + return true; + } + + bool PullLibraries (out string? mainProcessPath) + { + mainProcessPath = null; + DeviceLibraryCopier copier; + + if (String.IsNullOrEmpty (deviceLdd)) { + copier = new NoLddDeviceLibraryCopier ( + log, + adb, + appIs64Bit, + outputDir, + this + ); + } else { + copier = new LddDeviceLibraryCopier ( + log, + adb, + appIs64Bit, + outputDir, + this + ); + } + + return copier.Copy (out mainProcessPath); + } + + bool PushDebugServer () + { + string? debugServerPath = ndk.GetDebugServerPath (mainAbi!); + if (String.IsNullOrEmpty (debugServerPath)) { + return false; + } + + if (!CreateLldbDir (appLldbBinDir!) || !CreateLldbDir (appLldbLogDir) || !CreateLldbDir (appLldbTmpDir)) { + return false; + } + + //string serverName = $"xa-{context.arch}-{Path.GetFileName (debugServerPath)}"; + string serverName = Path.GetFileName (debugServerPath); + deviceDebugServerPath = $"{appLldbBinDir}/{serverName}"; + + KillDebugServer (deviceDebugServerPath); + + // Always push the server binary, as we don't know what version might already be there + if (!PushServerExecutable (debugServerPath, deviceDebugServerPath)) { + return false; + } + log.StatusLine ("Debug server path on device", deviceDebugServerPath); + + string? launcherScript = Utilities.ReadManifestResource (log, ServerLauncherScriptName); + if (String.IsNullOrEmpty (launcherScript)) { + return false; + } + + string launcherScriptPath = Path.Combine (outputDir, ServerLauncherScriptName); + Directory.CreateDirectory (Path.GetDirectoryName (launcherScriptPath)!); + File.WriteAllText (launcherScriptPath, launcherScript, Utilities.UTF8NoBOM); + + deviceDebugServerScriptPath = $"{appLldbBinDir}/{Path.GetFileName (launcherScriptPath)}"; + if (!PushServerExecutable (launcherScriptPath, deviceDebugServerScriptPath)) { + return false; + } + log.StatusLine ("Debug server launcher script path on device", deviceDebugServerScriptPath); + log.MessageLine (); + + return true; + + bool CreateLldbDir (string? dir) + { + if (String.IsNullOrEmpty (dir)) { + return false; + } + + if (!adb.CreateDirectoryAs (packageName, dir).Result.success) { + log.ErrorLine ($"Failed to create debug server destination directory on device, {dir}"); + return false; + } + + return true; + } + } + + bool PushServerExecutable (string hostSource, string deviceDestination) + { + string executableName = Path.GetFileName (deviceDestination); + + // Always push the executable, as we don't know what version might already be there + log.DebugLine ($"Uploading {hostSource} to device"); + + // First upload to temporary path, as it's writable for everyone + string remotePath = $"/data/local/tmp/{executableName}"; + if (!adb.Push (hostSource, remotePath).Result) { + log.ErrorLine ($"Failed to upload debug server {hostSource} to device path {remotePath}"); + return false; + } + + // Next, copy it to the app dir, with run-as + (bool success, string output) = adb.Shell ( + "cat", remotePath, "|", + "run-as", packageName, + "sh", "-c", $"'cat > {deviceDestination}'" + ).Result; + + if (!success) { + log.ErrorLine ($"Failed to copy debug executable to device, from {hostSource} to {deviceDestination}"); + return false; + } + + (success, output) = adb.RunAs (packageName, "chmod", "700", deviceDestination).Result; + if (!success) { + log.ErrorLine ($"Failed to make debug server executable on device, at {deviceDestination}"); + return false; + } + + return true; + } + + // TODO: handle multiple pids + bool KillDebugServer (string debugServerPath) + { + long serverPID = GetDeviceProcessID (debugServerPath, quiet: false); + if (serverPID <= 0) { + return true; + } + + log.DebugLine ("Killing previous instance of the debug server"); + (bool success, string _) = adb.RunAs (packageName, "kill", "-9", $"{serverPID}").Result; + return success; + } + + long GetDeviceProcessID (string processName, bool quiet = false) + { + (bool success, string output) = adb.Shell ("pidof", Path.GetFileName (processName)).Result; + if (!success) { + if (!quiet) { + log.ErrorLine ($"Failed to obtain PID of process '{processName}'"); + log.ErrorLine (output); + } + return -1; + } + + output = output.Trim (); + if (!UInt32.TryParse (output, out uint pid)) { + if (!quiet) { + log.ErrorLine ($"Unable to parse string '{output}' as the package's PID"); + } + return -1; + } + + return pid; + } + + bool DetectTools () + { + // Not all versions of Android have the `which` utility, all of them have `whence` + // Also, API 21 adbd will not return an error code to us... But since we know that 21 + // doesn't have LDD, we'll cheat + deviceLdd = null; + if (apiLevel > 21) { + (bool success, string output) = adb.Shell ("whence", "ldd").Result; + if (success) { + log.DebugLine ($"Found `ldd` on device at '{output}'"); + deviceLdd = output; + } + } + + if (String.IsNullOrEmpty (deviceLdd)) { + log.DebugLine ("`ldd` not found on device"); + } + + return true; + } + + bool DetermineAppDataDirectory () + { + (bool success, string output) = adb.GetAppDataDirectory (packageName).Result; + if (!AppDataDirFound (success, output)) { + log.ErrorLine ($"Unable to determine data directory for package '{packageName}'"); + return false; + } + + appDataDir = output.Trim (); + log.StatusLine ($"Application data directory on device", appDataDir); + + appLldbBaseDir = $"{appDataDir}/lldb"; + appLldbBinDir = $"{appLldbBaseDir}/bin"; + appLldbLogDir = $"{appLldbBaseDir}/log"; + appLldbTmpDir = $"{appLldbBaseDir}/tmp"; + + // Applications with minSdkVersion >= 24 will have their data directories + // created with rwx------ permissions, preventing adbd from forwarding to + // the gdbserver socket. To be safe, if we're on a device >= 24, always + // chmod the directory. + if (apiLevel >= 24) { + (success, output) = adb.RunAs (packageName, "/system/bin/chmod", "a+x", appDataDir).Result; + if (!success) { + log.ErrorLine ("Failed to make application data directory world executable"); + return false; + } + } + + return true; + + bool AppDataDirFound (bool success, string output) + { + if (apiLevel > 21) { + return success; + } + + if (output.IndexOf ("run-as: Package", StringComparison.OrdinalIgnoreCase) >= 0 && + output.IndexOf ("is unknown", StringComparison.OrdinalIgnoreCase) >= 0) + { + return false; + } + + return true; + } + } + + bool DetermineArchitectureAndABI () + { + string? propValue = GetFirstFoundPropertyValue (abiProperties); + string[]? deviceABIs = propValue?.Split (','); + + if (deviceABIs == null || deviceABIs.Length == 0) { + log.ErrorLine ("Unable to determine device ABI"); + return false; + } + + LogABIs ("Application", supportedAbis); + LogABIs (" Device", deviceABIs); + + bool gotValidAbi = false; + var possibleAbis = new List (); + var possibleArches = new List (); + + foreach (string deviceABI in deviceABIs) { + foreach (string appABI in supportedAbis) { + if (String.Compare (appABI, deviceABI, StringComparison.OrdinalIgnoreCase) == 0) { + string arch = AbiToArch (deviceABI); + + if (!gotValidAbi) { + mainAbi = deviceABI; + mainArch = arch; + + log.StatusLine ($" Selected ABI", $"{mainAbi} (architecture: {mainArch})"); + + appIs64Bit = mainAbi.IndexOf ("64", StringComparison.Ordinal) >= 0; + gotValidAbi = true; + } + + possibleAbis.Add (deviceABI); + possibleArches.Add (arch); + } + } + } + + if (!gotValidAbi) { + log.ErrorLine ($"Application cannot run on the selected device: no matching ABI found"); + } + + availableAbis = possibleAbis.ToArray (); + availableArches = possibleArches.ToArray (); + return gotValidAbi; + + void LogABIs (string which, IEnumerable abis) + { + log.StatusLine ($"{which} ABIs", String.Join (", ", abis)); + } + + string AbiToArch (string abi) => abi switch { + "armeabi" => "arm", + "armeabi-v7a" => "arm", + "arm64-v8a" => "arm64", + _ => abi, + }; + } + + string? GetFirstFoundPropertyValue (string[] propertyNames) + { + foreach (string prop in propertyNames) { + (bool success, string value) = adb.GetPropertyValue (prop).Result; + if (!success) { + continue; + } + + return value; + } + + return null; + } +} diff --git a/tools/xadebug/Xamarin.Android.Debug/AndroidNdk.cs b/tools/xadebug/Xamarin.Android.Debug/AndroidNdk.cs new file mode 100644 index 00000000000..b75c188d388 --- /dev/null +++ b/tools/xadebug/Xamarin.Android.Debug/AndroidNdk.cs @@ -0,0 +1,101 @@ +using System; +using System.Collections.Generic; +using System.IO; + +using Xamarin.Android.Utilities; +using Xamarin.Android.Tasks; + +namespace Xamarin.Android.Debug; + +class AndroidNdk +{ + // We want the shell/batch scripts first, since they set up Python environment for the debugger + static readonly string[] lldbNames = { + "lldb.sh", + "lldb", + "lldb.cmd", + "lldb.exe", + }; + + Dictionary hostLldbServerPaths; + XamarinLoggingHelper log; + string? lldbPath; + + public string LldbPath => lldbPath ?? String.Empty; + + public AndroidNdk (XamarinLoggingHelper log, string ndkRootPath, List supportedAbis) + { + this.log = log; + hostLldbServerPaths = new Dictionary (StringComparer.Ordinal); + + if (!FindTools (ndkRootPath, supportedAbis)) { + throw new InvalidOperationException ("Failed to find all the required NDK tools"); + } + } + + public string? GetDebugServerPath (string abi) + { + if (!hostLldbServerPaths.TryGetValue (abi, out string? debugServerPath) || String.IsNullOrEmpty (debugServerPath)) { + log.ErrorLine ($"Debug server for abi '{abi}' not found."); + return null; + } + + return debugServerPath; + } + + bool FindTools (string ndkRootPath, List supportedAbis) + { + string toolchainDir = Path.Combine (ndkRootPath, NdkHelper.RelativeToolchainDir); + string toolchainBinDir = Path.Combine (toolchainDir, "bin"); + string? path = null; + + foreach (string lldb in lldbNames) { + path = Path.Combine (toolchainBinDir, lldb); + if (File.Exists (path)) { + break; + } + } + + if (String.IsNullOrEmpty (path)) { + log.ErrorLine ($"Unable to locate lldb executable in '{toolchainBinDir}'"); + return false; + } + lldbPath = path; + + hostLldbServerPaths = new Dictionary (StringComparer.OrdinalIgnoreCase); + string llvmVersion = GetLlvmVersion (toolchainDir); + foreach (string abi in supportedAbis) { + string llvmAbi = NdkHelper.TranslateAbiToLLVM (abi); + path = Path.Combine (ndkRootPath, NdkHelper.RelativeToolchainDir, "lib64", "clang", llvmVersion, "lib", "linux", llvmAbi, "lldb-server"); + if (!File.Exists (path)) { + log.ErrorLine ($"LLVM lldb server component for ABI '{abi}' not found at '{path}'"); + return false; + } + + hostLldbServerPaths.Add (abi, path); + } + + if (hostLldbServerPaths.Count == 0) { + log.ErrorLine ("Unable to find any lldb-server executables, debugging not possible"); + return false; + } + + return true; + } + + string GetLlvmVersion (string toolchainDir) + { + string path = Path.Combine (toolchainDir, "AndroidVersion.txt"); + if (!File.Exists (path)) { + throw new InvalidOperationException ($"LLVM version file not found at '{path}'"); + } + + string[] lines = File.ReadAllLines (path); + string? line = lines.Length >= 1 ? lines[0].Trim () : null; + if (String.IsNullOrEmpty (line)) { + throw new InvalidOperationException ($"Unable to read LLVM version from '{path}'"); + } + + return line; + } +} diff --git a/tools/xadebug/Xamarin.Android.Debug/DebugSession.cs b/tools/xadebug/Xamarin.Android.Debug/DebugSession.cs new file mode 100644 index 00000000000..b860ea0e8fe --- /dev/null +++ b/tools/xadebug/Xamarin.Android.Debug/DebugSession.cs @@ -0,0 +1,88 @@ +using System; +using System.Collections.Generic; +using System.IO; + +using Xamarin.Android.Utilities; +using Xamarin.Tools.Zip; + +namespace Xamarin.Android.Debug; + +class DebugSession +{ + static readonly Dictionary KnownAbiDirs = new Dictionary (StringComparer.Ordinal) { + { "lib/arm64-v8a", "arm64-v8a" }, + { "lib/armeabi-v7a", "armeabi-v7a" }, + { "lib/x86_64", "x86_64" }, + { "lib/x86", "x86" }, + }; + + readonly string apkPath; + readonly ParsedOptions parsedOptions; + readonly XamarinLoggingHelper log; + readonly ZipArchive apk; + readonly string workDirectory; + + public DebugSession (XamarinLoggingHelper logger, string apkPath, ZipArchive apk, ParsedOptions parsedOptions) + { + log = logger; + this.apkPath = apkPath; + this.parsedOptions = parsedOptions; + this.apk = apk; + workDirectory = Path.Combine (parsedOptions.WorkDirectory, Utilities.StringHash (apkPath)); + } + + public bool Prepare () + { + List supportedAbis = DetectSupportedAbis (); + if (supportedAbis.Count == 0) { + log.ErrorLine ("Unable to detect ABIs supported by the application"); + return false; + } + + var ndk = new AndroidNdk (log, parsedOptions.NdkDirPath!, supportedAbis); + var device = new AndroidDevice ( + log, + ndk, + workDirectory, + parsedOptions.AdbPath, + parsedOptions.PackageName!, + supportedAbis, + parsedOptions.TargetDevice + ); + + if (!device.GatherInfo ()) { + return false; + } + + return false; + } + + List DetectSupportedAbis () + { + var ret = new List (); + + log.DebugLine ($"Detecting ABIs supported by '{apkPath}'"); + HashSet seenAbis = new HashSet (StringComparer.Ordinal); + foreach (ZipEntry entry in apk) { + if (seenAbis.Count == KnownAbiDirs.Count) { + break; + } + + // There might not be separate entries for lib/{ARCH} directories, so we look for the first file + // inside one of them to determine if an ABI is supported + string entryDir = Path.GetDirectoryName (entry.FullName) ?? String.Empty; + if (!KnownAbiDirs.TryGetValue (entryDir, out string? abi) || seenAbis.Contains (abi)) { + continue; + } + + seenAbis.Add (abi); + ret.Add (abi); + } + + if (ret.Count > 0) { + log.StatusLine ("Supported ABIs", String.Join (", ", ret)); + } + + return ret; + } +} diff --git a/tools/xadebug/Xamarin.Android.Debug/DeviceLibrariesCopier.cs b/tools/xadebug/Xamarin.Android.Debug/DeviceLibrariesCopier.cs new file mode 100644 index 00000000000..4799e276bd6 --- /dev/null +++ b/tools/xadebug/Xamarin.Android.Debug/DeviceLibrariesCopier.cs @@ -0,0 +1,66 @@ +using System; + +using Xamarin.Android.Utilities; +using Xamarin.Android.Tasks; + +namespace Xamarin.Android.Debug; + +abstract class DeviceLibraryCopier +{ + protected XamarinLoggingHelper Log { get; } + protected bool AppIs64Bit { get; } + protected string LocalDestinationDir { get; } + protected AdbRunner Adb { get; } + protected AndroidDevice Device { get; } + + protected DeviceLibraryCopier (XamarinLoggingHelper log, AdbRunner adb, bool appIs64Bit, string localDestinationDir, AndroidDevice device) + { + Log = log; + Adb = adb; + AppIs64Bit = appIs64Bit; + LocalDestinationDir = localDestinationDir; + Device = device; + } + + protected string? FetchZygote () + { + string zygotePath; + string destination; + + if (AppIs64Bit) { + zygotePath = "/system/bin/app_process64"; + destination = Utilities.MakeLocalPath (LocalDestinationDir, zygotePath); + + Utilities.MakeFileDirectory (destination); + if (!Adb.Pull (zygotePath, destination).Result) { + Log.ErrorLine ("Failed to copy 64-bit app_process64"); + return null; + } + } else { + // /system/bin/app_process is 32-bit on 32-bit devices, but a symlink to + // app_process64 on 64-bit. If we need the 32-bit version, try to pull + // app_process32, and if that fails, pull app_process. + destination = Utilities.MakeLocalPath (LocalDestinationDir, "/system/bin/app_process"); + string? source = "/system/bin/app_process32"; + + Utilities.MakeFileDirectory (destination); + if (!Adb.Pull (source, destination).Result) { + source = "/system/bin/app_process"; + if (!Adb.Pull (source, destination).Result) { + source = null; + } + } + + if (String.IsNullOrEmpty (source)) { + Log.ErrorLine ("Failed to copy 32-bit app_process"); + return null; + } + + zygotePath = destination; + } + + return zygotePath; + } + + public abstract bool Copy (out string? zygotePath); +} diff --git a/tools/xadebug/Xamarin.Android.Debug/LdConfigParser.cs b/tools/xadebug/Xamarin.Android.Debug/LdConfigParser.cs new file mode 100644 index 00000000000..983ae60e5d4 --- /dev/null +++ b/tools/xadebug/Xamarin.Android.Debug/LdConfigParser.cs @@ -0,0 +1,167 @@ +using System; +using System.Collections.Generic; +using System.IO; + +using Xamarin.Android.Utilities; + +namespace Xamarin.Android.Debug; + +class LdConfigParser +{ + XamarinLoggingHelper log; + + public LdConfigParser (XamarinLoggingHelper log) + { + this.log = log; + } + + // Format: https://android.googlesource.com/platform/bionic/+/master/linker/ld.config.format.md + // + public (List searchPaths, HashSet permittedPaths) Parse (string localLdConfigPath, string deviceBinDirectory, string libDirName) + { + var searchPaths = new List (); + var permittedPaths = new HashSet (); + bool foundSomeSection = false; + bool insideMatchingSection = false; + string normalizedDeviceBinDirectory = Utilities.NormalizeDirectoryPath (deviceBinDirectory); + string? sectionName = null; + + log.DebugLine ($"Parsing LD config file '{localLdConfigPath}'"); + int lineCounter = 0; + var namespaces = new List { + "default" + }; + + foreach (string l in File.ReadLines (localLdConfigPath)) { + lineCounter++; + string line = l.Trim (); + if (line.Length == 0 || line.StartsWith ('#')) { + continue; + } + + // The `dir.*` entries are before any section, don't waste time looking for them if we've parsed a section already + if (!foundSomeSection && sectionName == null) { + sectionName = GetMatchingDirMapping (normalizedDeviceBinDirectory, line); + if (sectionName != null) { + log.DebugLine ($"Found section name on line {lineCounter}: '{sectionName}'"); + continue; + } + } + + if (line[0] == '[') { + foundSomeSection = true; + insideMatchingSection = String.Compare (line, $"[{sectionName}]", StringComparison.Ordinal) == 0; + if (insideMatchingSection) { + log.DebugLine ($"Found section '{sectionName}' start on line {lineCounter}"); + } + } + + if (!insideMatchingSection) { + continue; + } + + if (line.StartsWith ("additional.namespaces", StringComparison.Ordinal) && GetVariableAssignmentParts (line, out string? name, out string? value)) { + foreach (string v in value!.Split (',')) { + string nsName = v.Trim (); + if (nsName.Length == 0) { + continue; + } + + log.DebugLine ($"Adding additional namespace '{nsName}'"); + namespaces.Add (nsName); + } + continue; + } + + MaybeAddLibraryPath (searchPaths, permittedPaths, namespaces, line, libDirName); + } + + return (searchPaths, permittedPaths); + + } + + void MaybeAddLibraryPath (List searchPaths, HashSet permittedPaths, List knownNamespaces, string configLine, string libDirName) + { + if (!configLine.StartsWith ("namespace.", StringComparison.Ordinal)) { + return; + } + + // not interested in ASAN libraries + if (configLine.IndexOf (".asan.", StringComparison.Ordinal) > 0) { + return; + } + + foreach (string ns in knownNamespaces) { + if (!GetVariableAssignmentParts (configLine, out string? name, out string? value)) { + continue; + } + + string varName = $"namespace.{ns}.search.paths"; + if (String.Compare (varName, name, StringComparison.Ordinal) == 0) { + AddPath (searchPaths, "search", value!); + continue; + } + + varName = $"namespace.{ns}.permitted.paths"; + if (String.Compare (varName, name, StringComparison.Ordinal) == 0) { + AddPath (permittedPaths, "permitted", value!, checkIfAlreadyAdded: true); + } + } + + void AddPath (ICollection list, string which, string value, bool checkIfAlreadyAdded = false) + { + string path = Utilities.NormalizeDirectoryPath (value.Replace ("${LIB}", libDirName)); + + if (checkIfAlreadyAdded && list.Contains (path)) { + return; + } + + log.DebugLine ($"Adding library {which} path: {path}"); + list.Add (path); + } + } + + string? GetMatchingDirMapping (string deviceBinDirectory, string configLine) + { + const string LinePrefix = "dir."; + + string line = configLine.Trim (); + if (line.Length == 0 || !line.StartsWith (LinePrefix, StringComparison.Ordinal)) { + return null; + } + + if (!GetVariableAssignmentParts (line, out string? name, out string? value)) { + return null; + } + + string dirPath = Utilities.NormalizeDirectoryPath (value!); + if (String.Compare (dirPath, deviceBinDirectory, StringComparison.Ordinal) != 0) { + return null; + } + + string ns = name!.Substring (LinePrefix.Length).Trim (); + if (String.IsNullOrEmpty (ns)) { + return null; + } + + return ns; + } + + bool GetVariableAssignmentParts (string line, out string? name, out string? value) + { + name = value = null; + + string[] parts = line.Split ("+=", 2); + if (parts.Length != 2) { + parts = line.Split ('=', 2); + if (parts.Length != 2) { + return false; + } + } + + name = parts[0].Trim (); + value = parts[1].Trim (); + + return true; + } +} diff --git a/tools/xadebug/Xamarin.Android.Debug/LddDeviceLibrariesCopier.cs b/tools/xadebug/Xamarin.Android.Debug/LddDeviceLibrariesCopier.cs new file mode 100644 index 00000000000..8f49c0b3542 --- /dev/null +++ b/tools/xadebug/Xamarin.Android.Debug/LddDeviceLibrariesCopier.cs @@ -0,0 +1,18 @@ +using System; + +using Xamarin.Android.Utilities; +using Xamarin.Android.Tasks; + +namespace Xamarin.Android.Debug; + +class LddDeviceLibraryCopier : DeviceLibraryCopier +{ + public LddDeviceLibraryCopier (XamarinLoggingHelper log, AdbRunner adb, bool appIs64Bit, string localDestinationDir, AndroidDevice device) + : base (log, adb, appIs64Bit, localDestinationDir, device) + {} + + public override bool Copy (out string? zygotePath) + { + throw new NotImplementedException(); + } +} diff --git a/tools/xadebug/Xamarin.Android.Debug/LldbModuleCache.cs b/tools/xadebug/Xamarin.Android.Debug/LldbModuleCache.cs new file mode 100644 index 00000000000..d31ed16a7ac --- /dev/null +++ b/tools/xadebug/Xamarin.Android.Debug/LldbModuleCache.cs @@ -0,0 +1,170 @@ +using System; +using System.Collections.Generic; +using System.IO; + +using ELFSharp.ELF; +using ELFSharp.ELF.Sections; +using Xamarin.Android.Utilities; + +namespace Xamarin.Android.Debug; + +abstract class LldbModuleCache +{ + protected AndroidDevice Device { get; } + protected string CacheDirPath { get; } + protected XamarinLoggingHelper Log { get; } + + protected LldbModuleCache (XamarinLoggingHelper log, AndroidDevice device) + { + Device = device; + Log = log; + + CacheDirPath = Path.Combine ( + Environment.GetFolderPath (Environment.SpecialFolder.UserProfile), + ".lldb", + "module_cache", + "remote-android", // "platform" used by LLDB in our case + device.SerialNumber + ); + } + + public void Populate (string zygotePath) + { + string? localPath = FetchFileFromDevice (zygotePath); + if (localPath == null) { + // TODO: should we perhaps fetch a set of "basic" libraries here, as a fallback? + Log.WarningLine ($"Unable to fetch Android application launcher binary ('{zygotePath}') from device. No cache of shared modules will be generated"); + return; + } + + var alreadyDownloaded = new HashSet (StringComparer.Ordinal); + using IELF? elf = ReadElfFile (localPath); + + FetchDependencies (elf, alreadyDownloaded, localPath); + } + + void FetchDependencies (IELF? elf, HashSet alreadyDownloaded, string localPath) + { + if (elf == null) { + Log.DebugLine ($"Failed to open '{localPath}' as an ELF file. Ignoring."); + return; + } + + var dynstr = GetSection (elf, ".dynstr") as IStringTable; + if (dynstr == null) { + Log.DebugLine ($"ELF binary {localPath} has no .dynstr section, unable to read referenced shared library names"); + return; + } + + var needed = new HashSet (StringComparer.Ordinal); + foreach (IDynamicSection section in elf.GetSections ()) { + foreach (IDynamicEntry entry in section.Entries) { + if (entry.Tag != DynamicTag.Needed) { + continue; + } + + AddNeeded (dynstr, entry); + } + } + + Log.DebugLine ($"Binary {localPath} references the following libraries:"); + foreach (string lib in needed) { + Log.Debug ($" {lib}"); + if (alreadyDownloaded.Contains (lib)) { + Log.DebugLine (" [already downloaded]"); + continue; + } + + string? deviceLibraryPath = GetSharedLibraryPath (lib); + if (String.IsNullOrEmpty (deviceLibraryPath)) { + Log.DebugLine (" [device path unknown]"); + Log.WarningLine ($"Referenced libary '{lib}' not found on device"); + continue; + } + + Log.DebugLine (" [downloading]"); + Log.Status ("Downloading", deviceLibraryPath); + string? localLibraryPath = FetchFileFromDevice (deviceLibraryPath); + if (String.IsNullOrEmpty (localLibraryPath)) { + Log.Log (LogLevel.Info, " [FAILED]", XamarinLoggingHelper.ErrorColor); + continue; + } + Log.LogLine (LogLevel.Info, " [SUCCESS]", XamarinLoggingHelper.InfoColor); + + alreadyDownloaded.Add (lib); + using IELF? libElf = ReadElfFile (localLibraryPath); + FetchDependencies (libElf, alreadyDownloaded, localLibraryPath); + } + + void AddNeeded (IStringTable stringTable, IDynamicEntry entry) + { + ulong index; + if (entry is DynamicEntry entry64) { + index = entry64.Value; + } else if (entry is DynamicEntry entry32) { + index = (ulong)entry32.Value; + } else { + Log.WarningLine ($"DynamicEntry neither 32 nor 64 bit? Weird"); + return; + } + + string name = stringTable[(long)index]; + if (needed.Contains (name)) { + return; + } + + needed.Add (name); + } + } + + string? FetchFileFromDevice (string deviceFilePath) + { + string localFilePath = Utilities.MakeLocalPath (CacheDirPath, deviceFilePath); + string localTempFilePath = $"{localFilePath}.tmp"; + + Directory.CreateDirectory (Path.GetDirectoryName (localFilePath)!); + + if (!Device.AdbRunner.Pull (deviceFilePath, localTempFilePath).Result) { + Log.ErrorLine ($"Failed to download {deviceFilePath} from the attached device"); + return null; + } + + File.Move (localTempFilePath, localFilePath, true); + return localFilePath; + } + + protected string GetUnixFileName (string path) + { + int idx = path.LastIndexOf ('/'); + if (idx >= 0 && idx != path.Length - 1) { + return path.Substring (idx + 1); + } + + return path; + } + + protected abstract string? GetSharedLibraryPath (string libraryName); + + IELF? ReadElfFile (string path) + { + try { + if (ELFReader.TryLoad (path, out IELF ret)) { + return ret; + } + } catch (Exception ex) { + Log.WarningLine ($"{path} may not be a valid ELF binary."); + Log.WarningLine (ex.ToString ()); + } + + return null; + } + + ISection? GetSection (IELF elf, string sectionName) + { + if (!elf.TryGetSection (sectionName, out ISection section)) { + return null; + } + + return section; + } +} diff --git a/tools/xadebug/Xamarin.Android.Debug/NoLddDeviceLibrariesCopier.cs b/tools/xadebug/Xamarin.Android.Debug/NoLddDeviceLibrariesCopier.cs new file mode 100644 index 00000000000..a066f29c608 --- /dev/null +++ b/tools/xadebug/Xamarin.Android.Debug/NoLddDeviceLibrariesCopier.cs @@ -0,0 +1,190 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +using Xamarin.Android.Utilities; +using Xamarin.Android.Tasks; + +namespace Xamarin.Android.Debug; + +class NoLddDeviceLibraryCopier : DeviceLibraryCopier +{ + const string LdConfigPath = "/system/etc/ld.config.txt"; + + // To make things interesting, it turns out that API29 devices have **both** API 28 and 29 (they report 28) and for that reason they have TWO config files for ld... + const string LdConfigPath28 = "/etc/ld.config.28.txt"; + const string LdConfigPath29 = "/etc/ld.config.29.txt"; + + // TODO: We probably need a "provider" for the list of paths, since on ARM devices, /system/lib{64} directories contain x86/x64 binaries, and the ARM binaries are found in + // /system/lib{64]/arm{64} (but not on all devices, of course... e.g. Pixel 6 Pro doesn't have these) + // + // List of directory paths to use when the device has neither ldd nor /system/etc/ld.config.txt + static readonly string[] FallbackLibraryDirectories = { + "/system/@LIB@", + "/system/@LIB@/drm", + "/system/@LIB@/egl", + "/system/@LIB@/hw", + "/system/@LIB@/soundfx", + "/system/@LIB@/ssl", + "/system/@LIB@/ssl/engines", + + // /system/vendor is a symlink to /vendor on some Android versions, we'll skip the latter then + "/system/vendor/@LIB@", + "/system/vendor/@LIB@/egl", + "/system/vendor/@LIB@/mediadrm", + }; + + public NoLddDeviceLibraryCopier (XamarinLoggingHelper log, AdbRunner adb, bool appIs64Bit, string localDestinationDir, AndroidDevice device) + : base (log, adb, appIs64Bit, localDestinationDir, device) + {} + + public override bool Copy (out string? zygotePath) + { + zygotePath = FetchZygote (); + if (String.IsNullOrEmpty (zygotePath)) { + Log.ErrorLine ("Unable to determine path of the zygote process on device"); + return false; + } + + (List searchPaths, HashSet permittedPaths) = GetLibraryPaths (); + + // Collect file listings for all the search directories + var sharedLibraries = new List (); + foreach (string path in searchPaths) { + AddSharedLibraries (sharedLibraries, path, permittedPaths); + } + + var moduleCache = new NoLddLldbModuleCache (Log, Device, sharedLibraries); + moduleCache.Populate (zygotePath); + + return true; + } + + void AddSharedLibraries (List sharedLibraries, string deviceDirPath, HashSet permittedPaths) + { + AdbRunner.OutputLineFilter filterOutErrors = (bool isStdError, string line) => { + if (!isStdError) { + return false; // don't suppress any lines on stdout + } + + // Ignore these, since we don't really care and there's no point in spamming the output with red + return + line.IndexOf ("Permission denied", StringComparison.OrdinalIgnoreCase) >= 0 || + line.IndexOf ("No such file or directory", StringComparison.OrdinalIgnoreCase) >= 0; + }; + + (bool success, string output) = Adb.Shell (filterOutErrors, "ls", "-l", deviceDirPath).Result; + if (!success) { + // We can't rely on `success` because `ls -l` will return an error code if the directory exists but has any entries access to whose is not permitted + if (output.IndexOf ("No such file or directory", StringComparison.OrdinalIgnoreCase) >= 0) { + Log.DebugLine ($"Shared libraries directory {deviceDirPath} not found on device"); + return; + } + } + + Log.DebugLine ($"Adding shared libraries from {deviceDirPath}"); + foreach (string l in output.Split ('\n')) { + string line = l.Trim (); + if (line.Length == 0) { + continue; + } + + // `ls -l` output has 8 columns for filesystem entries + string[] parts = line.Split (' ', 8, StringSplitOptions.RemoveEmptyEntries); + if (parts.Length != 8) { + continue; + } + + string permissions = parts[0].Trim (); + string name = parts[7].Trim (); + + // First column, permissions: `drwxr-xr-x`, `-rw-r--r--` etc + if (permissions[0] == 'd') { + // Directory + string nestedDirPath = $"{deviceDirPath}{name}/"; + if (permittedPaths.Count > 0 && !permittedPaths.Contains (nestedDirPath)) { + Log.DebugLine ($"Directory '{nestedDirPath}' is not in the list of permitted directories, ignoring"); + continue; + } + + AddSharedLibraries (sharedLibraries, nestedDirPath, permittedPaths); + continue; + } + + // Ignore entries that aren't regular .so files or symlinks + if ((permissions[0] != '-' && permissions[0] != 'l') || !name.EndsWith (".so", StringComparison.Ordinal)) { + continue; + } + + string libPath; + if (permissions[0] == 'l') { + // Let's hope there are no libraries with -> in their name :P (if there are, we should use `readlink`) + const string SymlinkArrow = "->"; + + // Symlink, we'll add the target library instead + int idx = name.IndexOf (SymlinkArrow, StringComparison.Ordinal); + if (idx > 0) { + libPath = name.Substring (idx + SymlinkArrow.Length).Trim (); + } else { + Log.WarningLine ($"'ls -l' output line contains a symbolic link, but I can't determine the target:"); + Log.WarningLine ($" '{line}'"); + Log.WarningLine ("Ignoring this entry"); + continue; + } + } else { + libPath = $"{deviceDirPath}{name}"; + } + + Log.DebugLine ($" {libPath}"); + sharedLibraries.Add (libPath); + } + } + + (List searchPaths, HashSet permittedPaths) GetLibraryPaths () + { + string lib = AppIs64Bit ? "lib64" : "lib"; + + if (Device.ApiLevel == 21) { + // API21 devices (at least emulators) don't return adb error codes, so to avoid awkward error message parsing, we're going to just skip detection since we + // know what API21 has and doesn't have + return (GetFallbackDirs (), new HashSet ()); + } + + string localLdConfigPath = Utilities.MakeLocalPath (LocalDestinationDir, LdConfigPath); + Utilities.MakeFileDirectory (localLdConfigPath); + + string deviceLdConfigPath; + + if (Device.ApiLevel == 28) { + deviceLdConfigPath = LdConfigPath28; + } else if (Device.ApiLevel == 29) { + deviceLdConfigPath = LdConfigPath29; + } else { + deviceLdConfigPath = LdConfigPath; + } + + if (!Adb.Pull (deviceLdConfigPath, localLdConfigPath).Result) { + Log.DebugLine ($"Device doesn't have {LdConfigPath}"); + return (GetFallbackDirs (), new HashSet ()); + } else { + Log.DebugLine ($"Downloaded {deviceLdConfigPath} to {localLdConfigPath}"); + } + + var parser = new LdConfigParser (Log); + + // The app executables (app_process and app_process32) are both in /system/bin, so we can limit our + // library search paths to this location. + (List searchPaths, HashSet permittedPaths) = parser.Parse (localLdConfigPath, "/system/bin", lib); + if (searchPaths.Count == 0) { + searchPaths = GetFallbackDirs (); + } + + return (searchPaths, permittedPaths); + + List GetFallbackDirs () + { + Log.DebugLine ("Using fallback library directories for this device"); + return FallbackLibraryDirectories.Select (l => l.Replace ("@LIB@", lib)).ToList (); + } + } +} diff --git a/tools/xadebug/Xamarin.Android.Debug/NoLddLldbModuleCache.cs b/tools/xadebug/Xamarin.Android.Debug/NoLddLldbModuleCache.cs new file mode 100644 index 00000000000..c45042446dc --- /dev/null +++ b/tools/xadebug/Xamarin.Android.Debug/NoLddLldbModuleCache.cs @@ -0,0 +1,41 @@ +using System; +using System.Collections.Generic; + +using Xamarin.Android.Utilities; + +namespace Xamarin.Android.Debug; + +class NoLddLldbModuleCache : LldbModuleCache +{ + List deviceSharedLibraries; + Dictionary libraryCache; + + public NoLddLldbModuleCache (XamarinLoggingHelper log, AndroidDevice device, List deviceSharedLibraries) + : base (log, device) + { + this.deviceSharedLibraries = deviceSharedLibraries; + libraryCache = new Dictionary (StringComparer.Ordinal); + } + + protected override string? GetSharedLibraryPath (string libraryName) + { + if (libraryCache.TryGetValue (libraryName, out string? libraryPath)) { + return libraryPath; + } + + // List is sorted on the order of directories as specified by ld.config.txt, file entries aren't + // sorted inside. + foreach (string libPath in deviceSharedLibraries) { + string fileName = GetUnixFileName (libPath); + + if (String.Compare (libraryName, fileName, StringComparison.Ordinal) == 0) { + libraryCache.Add (libraryName, libPath); + return libPath; + } + } + + // Cache misses, too, the list isn't going to change + libraryCache.Add (libraryName, null); + return null; + } +} diff --git a/tools/xadebug/Xamarin.Android.Debug/Utilities.cs b/tools/xadebug/Xamarin.Android.Debug/Utilities.cs index 95d70314f3d..4b4cfba1bb3 100644 --- a/tools/xadebug/Xamarin.Android.Debug/Utilities.cs +++ b/tools/xadebug/Xamarin.Android.Debug/Utilities.cs @@ -4,6 +4,7 @@ using System.Runtime.InteropServices; using System.Text; +using Java.Interop.Tools.JavaCallableWrappers; using Xamarin.Android.Utilities; namespace Xamarin.Android.Debug; @@ -75,4 +76,23 @@ public static string MakeLocalPath (string localDirectory, string remotePath) return Path.Combine (localDirectory, remotePathLocalFormat); } + + public static string StringHash (string input, Encoding? encoding = null) + { + if (encoding == null) { + encoding = UTF8NoBOM; + } + + byte[] hash = Crc64Helper.Compute (encoding.GetBytes (input)); + if (hash.Length == 0) { + return input.GetHashCode ().ToString ("x"); + } + + var sb = new StringBuilder (); + foreach (byte b in hash) { + sb.Append (b.ToString ("x02")); + } + + return sb.ToString (); + } } diff --git a/tools/xadebug/Xamarin.Android.Utilities/DotNetRunner.cs b/tools/xadebug/Xamarin.Android.Utilities/DotNetRunner.cs index 145492a4615..82b13cff55b 100644 --- a/tools/xadebug/Xamarin.Android.Utilities/DotNetRunner.cs +++ b/tools/xadebug/Xamarin.Android.Utilities/DotNetRunner.cs @@ -7,6 +7,8 @@ namespace Xamarin.Android.Utilities; +using Utils = Xamarin.Android.Debug.Utilities; + class DotNetRunner : ToolRunner { class StreamOutputSink : ToolOutputSink @@ -39,8 +41,7 @@ public DotNetRunner (XamarinLoggingHelper logger, string toolPath, string workDi /// public async Task Build (string projectPath, string configuration, params string[] extraArgs) { - // TODO: use CRC64 instead of GetHashCode(), as the latter is not deterministic in .NET6+ - string projectWorkDir = Path.Combine (workDirectory, projectPath.GetHashCode ().ToString ("x")); + string projectWorkDir = Path.Combine (workDirectory, Utils.StringHash (projectPath)); Directory.CreateDirectory (projectWorkDir); string binlogPath = Path.Combine (projectWorkDir, "build.binlog"); @@ -48,8 +49,14 @@ public DotNetRunner (XamarinLoggingHelper logger, string toolPath, string workDi var runner = CreateProcessRunner ("build"); runner. AddArgument ("-c").AddArgument (configuration). - AddArgument ($"-bl:\"{binlogPath}\""). - AddQuotedArgument (projectPath); + AddArgument ($"-bl:\"{binlogPath}\""); + + if (extraArgs != null && extraArgs.Length > 0) { + foreach (string arg in extraArgs) { + runner.AddArgument (arg); + } + } + runner.AddQuotedArgument (projectPath); if (!await RunDotNet (runner)) { return null; @@ -82,7 +89,7 @@ public DotNetRunner (XamarinLoggingHelper logger, string toolPath, string workDi string logOutput = Path.ChangeExtension (binlogPath, ".txt"); using var fs = File.Open (logOutput, FileMode.Create); - using var sw = new StreamWriter (fs, Xamarin.Android.Debug.Utilities.UTF8NoBOM); + using var sw = new StreamWriter (fs, Utils.UTF8NoBOM); using var sink = new StreamOutputSink (Logger, sw); try { diff --git a/tools/xadebug/xadebug.csproj b/tools/xadebug/xadebug.csproj index 3784c8cc11a..038def1bd44 100644 --- a/tools/xadebug/xadebug.csproj +++ b/tools/xadebug/xadebug.csproj @@ -7,6 +7,7 @@ False enable NO_MSBUILD + true Marek Habersack Microsoft Corporation @@ -19,7 +20,6 @@ LICENSE https://github.com/xamarin/xamarin-android Major - @@ -28,6 +28,16 @@ + + + Crc64.cs + + + Crc64Helper.cs + + + Crc64.Table.cs + From 8b6b0b3c2f86de758a468a6e99934e2f3cf3d804 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 3 Jan 2023 15:44:33 +0100 Subject: [PATCH 25/30] [WIP] Update MSBuild.StructuredLogger version --- tools/xadebug/Xamarin.Android.Debug/DebugSession.cs | 2 +- tools/xadebug/xadebug.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/xadebug/Xamarin.Android.Debug/DebugSession.cs b/tools/xadebug/Xamarin.Android.Debug/DebugSession.cs index b860ea0e8fe..954979867db 100644 --- a/tools/xadebug/Xamarin.Android.Debug/DebugSession.cs +++ b/tools/xadebug/Xamarin.Android.Debug/DebugSession.cs @@ -29,7 +29,7 @@ public DebugSession (XamarinLoggingHelper logger, string apkPath, ZipArchive apk this.parsedOptions = parsedOptions; this.apk = apk; workDirectory = Path.Combine (parsedOptions.WorkDirectory, Utilities.StringHash (apkPath)); - } + } public bool Prepare () { diff --git a/tools/xadebug/xadebug.csproj b/tools/xadebug/xadebug.csproj index 038def1bd44..e256350004e 100644 --- a/tools/xadebug/xadebug.csproj +++ b/tools/xadebug/xadebug.csproj @@ -44,7 +44,7 @@ - + From 230c13275100430f15563054da14d4c436d48549 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 4 Jan 2023 21:59:13 +0100 Subject: [PATCH 26/30] [WIP] Getting back in the saddle --- .../Utilities/AdbRunner.cs | 8 ++ tools/xadebug/Main.cs | 10 +- .../xadebug/Resources/xa_start_lldb_server.sh | 41 ++++++ .../Xamarin.Android.Debug/AndroidDevice.cs | 24 +--- .../Xamarin.Android.Debug/DebugSession.cs | 105 +++++++++++++- .../DeviceLibrariesCopier.cs | 53 ++++--- .../LddDeviceLibrariesCopier.cs | 64 ++++++++- .../LddLldbModuleCache.cs | 24 ++++ .../Xamarin.Android.Debug/LldbModuleCache.cs | 136 ++++++------------ .../NoLddDeviceLibrariesCopier.cs | 1 - .../NoLddLldbModuleCache.cs | 95 +++++++++--- .../Xamarin.Android.Debug/Utilities.cs | 20 +++ tools/xadebug/xadebug.csproj | 6 + 13 files changed, 424 insertions(+), 163 deletions(-) create mode 100755 tools/xadebug/Resources/xa_start_lldb_server.sh create mode 100644 tools/xadebug/Xamarin.Android.Debug/LddLldbModuleCache.cs diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs index 6aaa1e86d73..e8659bd4845 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs @@ -112,6 +112,10 @@ public async Task Push (string localPath, string remotePath) public async Task<(bool success, string output)> RunAs (string packageName, string command, params string[] args) { + if (String.IsNullOrEmpty (packageName)) { + throw new ArgumentException ("must not be null or empty", nameof (packageName)); + } + var shellArgs = new List { packageName, command, @@ -146,6 +150,10 @@ public async Task Push (string localPath, string remotePath) async Task<(bool success, string output)> Shell (string command, IEnumerable? args, OutputLineFilter? lineFilter) { + if (String.IsNullOrEmpty (command)) { + throw new ArgumentException ("must not be null or empty", nameof (command)); + } + var runner = CreateAdbRunner (); runner.AddArgument ("shell"); diff --git a/tools/xadebug/Main.cs b/tools/xadebug/Main.cs index 287eb4613c9..33f35c3eab4 100644 --- a/tools/xadebug/Main.cs +++ b/tools/xadebug/Main.cs @@ -43,7 +43,7 @@ static int Main (string[] args) var parsedOptions = new ParsedOptions (); log.Verbose = parsedOptions.Verbose; - var opts = new OptionSet { + var opts = new OptionSet { "Usage: dotnet xadebug [OPTIONS] ", "", { "p|package-name=", "name of the application package", v => parsedOptions.PackageName = EnsureNonEmptyString (log, "-p|--package-name", v, ref haveOptionErrors) }, @@ -158,8 +158,12 @@ static int Main (string[] args) return 1; } - var debugSession = new DebugSession (log, apkFilePath, apk, parsedOptions); - return debugSession.Prepare () ? 0 : 1; + var debugSession = new DebugSession (log, appInfo, apkFilePath, apk, parsedOptions); + if (!debugSession.Prepare ()) { + return 1; + } + + return 0; } static ApplicationInfo? ReadManifest (ZipArchive apk, ParsedOptions parsedOptions) diff --git a/tools/xadebug/Resources/xa_start_lldb_server.sh b/tools/xadebug/Resources/xa_start_lldb_server.sh new file mode 100755 index 00000000000..8fa6cdb3141 --- /dev/null +++ b/tools/xadebug/Resources/xa_start_lldb_server.sh @@ -0,0 +1,41 @@ +#!/system/bin/sh + +# This script launches lldb-server on Android device from application subfolder - /data/data/$packageId/lldb/bin. +# Native run configuration is expected to push this script along with lldb-server to the device prior to its execution. +# Following command arguments are expected to be passed - lldb package directory and lldb-server listen port. +set -x +umask 0002 + +LLDB_DIR=$1 +LISTENER_SCHEME=$2 +DOMAINSOCKET_DIR=$3 +PLATFORM_SOCKET=$4 +LOG_CHANNELS="$5" + +BIN_DIR=$LLDB_DIR/bin +LOG_DIR=$LLDB_DIR/log +TMP_DIR=$LLDB_DIR/tmp +PLATFORM_LOG_FILE=$LOG_DIR/platform.log + +export LLDB_DEBUGSERVER_LOG_FILE=$LOG_DIR/gdb-server.log +export LLDB_SERVER_LOG_CHANNELS="$LOG_CHANNELS" +export LLDB_DEBUGSERVER_DOMAINSOCKET_DIR=$DOMAINSOCKET_DIR + +# This directory already exists. Make sure it has the right permissions. +chmod 0775 "$LLDB_DIR" + +rm -r $TMP_DIR +mkdir $TMP_DIR +export TMPDIR=$TMP_DIR + +rm -r $LOG_DIR +mkdir $LOG_DIR + +# LLDB would create these files with more restrictive permissions than our umask above. Make sure +# it doesn't get a chance. +# "touch" does not exist on pre API-16 devices. This is a poor man's replacement +cat "$LLDB_DEBUGSERVER_LOG_FILE" 2>"$PLATFORM_LOG_FILE" + +cd $TMP_DIR # change cwd + +$BIN_DIR/lldb-server platform --server --listen $LISTENER_SCHEME://$DOMAINSOCKET_DIR/$PLATFORM_SOCKET --log-file "$PLATFORM_LOG_FILE" --log-channels "$LOG_CHANNELS" $LOG_DIR/platform-stdout.log 2>&1 diff --git a/tools/xadebug/Xamarin.Android.Debug/AndroidDevice.cs b/tools/xadebug/Xamarin.Android.Debug/AndroidDevice.cs index 4e8db407821..162b4293caf 100644 --- a/tools/xadebug/Xamarin.Android.Debug/AndroidDevice.cs +++ b/tools/xadebug/Xamarin.Android.Debug/AndroidDevice.cs @@ -36,6 +36,7 @@ class AndroidDevice string? appLldbTmpDir; string? mainAbi; string? mainArch; + string[]? deviceABIs; string[]? availableAbis; string[]? availableArches; string? serialNumber; @@ -52,11 +53,15 @@ class AndroidDevice public int ApiLevel => apiLevel; public string[] AvailableAbis => availableAbis ?? new string[] {}; public string[] AvailableArches => availableArches ?? new string[] {}; + public string[] DeviceAbis => deviceABIs ?? new string[] {}; public string MainArch => mainArch ?? String.Empty; public string MainAbi => mainAbi ?? String.Empty; public string SerialNumber => serialNumber ?? String.Empty; public string DebugServerLauncherScriptPath => deviceDebugServerScriptPath ?? String.Empty; + public string DebugServerPath => deviceDebugServerPath ?? String.Empty; public string LldbBaseDir => appLldbBaseDir ?? String.Empty; + public string AppDataDir => appDataDir ?? String.Empty; + public string? DeviceLddPath => deviceLdd; public AdbRunner AdbRunner => adb; public AndroidDevice (XamarinLoggingHelper log, AndroidNdk ndk, string outputDir, string adbPath, string packageName, List supportedAbis, string? adbTargetDevice = null) @@ -107,8 +112,6 @@ public bool GatherInfo () serialNumber = GetFirstFoundPropertyValue (serialNumberProperties); if (String.IsNullOrEmpty (serialNumber)) { log.WarningLine ("Unable to determine device serial number"); - } else { - log.StatusLine ($"Device serial number", serialNumber); } return true; @@ -192,7 +195,6 @@ bool PushDebugServer () if (!PushServerExecutable (debugServerPath, deviceDebugServerPath)) { return false; } - log.StatusLine ("Debug server path on device", deviceDebugServerPath); string? launcherScript = Utilities.ReadManifestResource (log, ServerLauncherScriptName); if (String.IsNullOrEmpty (launcherScript)) { @@ -207,8 +209,6 @@ bool PushDebugServer () if (!PushServerExecutable (launcherScriptPath, deviceDebugServerScriptPath)) { return false; } - log.StatusLine ("Debug server launcher script path on device", deviceDebugServerScriptPath); - log.MessageLine (); return true; @@ -327,7 +327,6 @@ bool DetermineAppDataDirectory () } appDataDir = output.Trim (); - log.StatusLine ($"Application data directory on device", appDataDir); appLldbBaseDir = $"{appDataDir}/lldb"; appLldbBinDir = $"{appLldbBaseDir}/bin"; @@ -367,16 +366,13 @@ bool AppDataDirFound (bool success, string output) bool DetermineArchitectureAndABI () { string? propValue = GetFirstFoundPropertyValue (abiProperties); - string[]? deviceABIs = propValue?.Split (','); + deviceABIs = propValue?.Split (','); if (deviceABIs == null || deviceABIs.Length == 0) { log.ErrorLine ("Unable to determine device ABI"); return false; } - LogABIs ("Application", supportedAbis); - LogABIs (" Device", deviceABIs); - bool gotValidAbi = false; var possibleAbis = new List (); var possibleArches = new List (); @@ -389,9 +385,6 @@ bool DetermineArchitectureAndABI () if (!gotValidAbi) { mainAbi = deviceABI; mainArch = arch; - - log.StatusLine ($" Selected ABI", $"{mainAbi} (architecture: {mainArch})"); - appIs64Bit = mainAbi.IndexOf ("64", StringComparison.Ordinal) >= 0; gotValidAbi = true; } @@ -410,11 +403,6 @@ bool DetermineArchitectureAndABI () availableArches = possibleArches.ToArray (); return gotValidAbi; - void LogABIs (string which, IEnumerable abis) - { - log.StatusLine ($"{which} ABIs", String.Join (", ", abis)); - } - string AbiToArch (string abi) => abi switch { "armeabi" => "arm", "armeabi-v7a" => "arm", diff --git a/tools/xadebug/Xamarin.Android.Debug/DebugSession.cs b/tools/xadebug/Xamarin.Android.Debug/DebugSession.cs index 954979867db..7e89729cf0a 100644 --- a/tools/xadebug/Xamarin.Android.Debug/DebugSession.cs +++ b/tools/xadebug/Xamarin.Android.Debug/DebugSession.cs @@ -21,13 +21,15 @@ class DebugSession readonly XamarinLoggingHelper log; readonly ZipArchive apk; readonly string workDirectory; + readonly ApplicationInfo appInfo; - public DebugSession (XamarinLoggingHelper logger, string apkPath, ZipArchive apk, ParsedOptions parsedOptions) + public DebugSession (XamarinLoggingHelper logger, ApplicationInfo appInfo, string apkPath, ZipArchive apk, ParsedOptions parsedOptions) { log = logger; this.apkPath = apkPath; this.parsedOptions = parsedOptions; this.apk = apk; + this.appInfo = appInfo; workDirectory = Path.Combine (parsedOptions.WorkDirectory, Utilities.StringHash (apkPath)); } @@ -45,7 +47,7 @@ public bool Prepare () ndk, workDirectory, parsedOptions.AdbPath, - parsedOptions.PackageName!, + appInfo.PackageName, supportedAbis, parsedOptions.TargetDevice ); @@ -54,7 +56,102 @@ public bool Prepare () return false; } + string? mainProcessPath; + if (!device.Prepare (out mainProcessPath) || String.IsNullOrEmpty (mainProcessPath)) { + return false; + } + + string appLibsDirectory = Path.Combine (workDirectory, "lib", device.MainAbi); + CopyAppLibs (device, appLibsDirectory); + + log.MessageLine (); + + if (supportedAbis.Count > 0) { + log.StatusLine ("All supported ABIs", String.Join (", ", supportedAbis)); + } + + LogABIs ("Application", supportedAbis); + LogABIs (" Device", device.DeviceAbis); + log.StatusLine (" Selected ABI", $"{device.MainAbi} (architecture: {device.MainArch})"); + log.MessageLine (); + log.StatusLine ("Application data directory on device", device.AppDataDir); + log.StatusLine ("Device serial number", device.SerialNumber); + log.StatusLine ("Debug server path on device", device.DebugServerPath); + log.StatusLine ("Debug server launcher script path on device", device.DebugServerLauncherScriptPath); + log.MessageLine (); + + // TODO: install the apk + // TODO: start the app + // TODO: start the app so that it waits for the debugger (until monodroid_gdb_wait is set) + + string socketScheme = "unix-abstract"; + string socketDir = $"/xa-{appInfo.PackageName}"; + + var rnd = new Random (); + string socketName = $"xa-platform-{rnd.NextInt64 ()}.sock"; + string lldbScriptPath = WriteLldbScript (appLibsDirectory, device, socketScheme, socketDir, socketName, mainProcessPath); + return false; + + void LogABIs (string which, IEnumerable abis) + { + log.StatusLine ($"{which} ABIs", String.Join (", ", abis)); + } + } + + string WriteLldbScript (string appLibsDir, AndroidDevice device, string socketScheme, string socketDir, string socketName, string mainProcessPath) + { + string outputFile = Path.Combine (workDirectory, "lldb.x"); + string fullLibsDir = Path.GetFullPath (appLibsDir); + using FileStream fs = File.Open (outputFile, FileMode.Create, FileAccess.Write, FileShare.Read); + using StreamWriter sw = new StreamWriter (fs, Utilities.UTF8NoBOM); + + // TODO: add support for appending user commands + var searchPathsList = new List { + $"\"{fullLibsDir}\"" + }; + + string searchPaths = String.Join (" ", searchPathsList); + sw.WriteLine ($"settings append target.exec-search-paths {searchPaths}"); + sw.WriteLine ("platform select remote-android"); + sw.WriteLine ($"platform connect {socketScheme}-connect://{socketDir}/{socketName}"); + sw.WriteLine ($"file \"{mainProcessPath}\""); + + log.DebugLine ($"Writing LLDB startup script: {outputFile}"); + sw.Flush (); + + return outputFile; + } + + void CopyAppLibs (AndroidDevice device, string libDir) + { + log.DebugLine ($"Copying application shared libraries to '{libDir}'"); + Directory.CreateDirectory (libDir); + + string entryDir = $"lib/{device.MainAbi}/"; + log.DebugLine ($"Looking for shared libraries inside APK, stored in the {entryDir} directory"); + + foreach (ZipEntry entry in apk) { + if (entry.IsDirectory) { + continue; + } + + string dirName = Utilities.GetZipEntryDirName (entry.FullName); + if (dirName.Length == 0 || String.Compare (entryDir, dirName, StringComparison.Ordinal) != 0) { + continue; + } + + string destPath = Path.Combine (libDir, Utilities.GetZipEntryFileName (entry.FullName)); + log.DebugLine ($"Copying app library '{entry.FullName}' to '{destPath}'"); + + using var libraryData = File.Open (destPath, FileMode.Create); + entry.Extract (libraryData); + libraryData.Seek (0, SeekOrigin.Begin); + libraryData.Flush (); + libraryData.Close (); + + // TODO: fetch symbols for libs which don't start with `libaot*` and aren't known Xamarin.Android libraries + } } List DetectSupportedAbis () @@ -79,10 +176,6 @@ List DetectSupportedAbis () ret.Add (abi); } - if (ret.Count > 0) { - log.StatusLine ("Supported ABIs", String.Join (", ", ret)); - } - return ret; } } diff --git a/tools/xadebug/Xamarin.Android.Debug/DeviceLibrariesCopier.cs b/tools/xadebug/Xamarin.Android.Debug/DeviceLibrariesCopier.cs index 4799e276bd6..50ec4bb65ac 100644 --- a/tools/xadebug/Xamarin.Android.Debug/DeviceLibrariesCopier.cs +++ b/tools/xadebug/Xamarin.Android.Debug/DeviceLibrariesCopier.cs @@ -33,33 +33,46 @@ protected DeviceLibraryCopier (XamarinLoggingHelper log, AdbRunner adb, bool app Utilities.MakeFileDirectory (destination); if (!Adb.Pull (zygotePath, destination).Result) { - Log.ErrorLine ("Failed to copy 64-bit app_process64"); - return null; + return ReportFailureAndReturn (); } } else { // /system/bin/app_process is 32-bit on 32-bit devices, but a symlink to - // app_process64 on 64-bit. If we need the 32-bit version, try to pull - // app_process32, and if that fails, pull app_process. + // app_process64 on 64-bit. If we need the 32-bit version, try to pull + // app_process32, and if that fails, pull app_process. destination = Utilities.MakeLocalPath (LocalDestinationDir, "/system/bin/app_process"); - string? source = "/system/bin/app_process32"; - - Utilities.MakeFileDirectory (destination); - if (!Adb.Pull (source, destination).Result) { - source = "/system/bin/app_process"; - if (!Adb.Pull (source, destination).Result) { - source = null; - } - } - - if (String.IsNullOrEmpty (source)) { - Log.ErrorLine ("Failed to copy 32-bit app_process"); - return null; - } - - zygotePath = destination; + string? source = "/system/bin/app_process32"; + + Utilities.MakeFileDirectory (destination); + if (!Adb.Pull (source, destination).Result) { + source = "/system/bin/app_process"; + if (!Adb.Pull (source, destination).Result) { + source = null; + } + } + + if (String.IsNullOrEmpty (source)) { + return ReportFailureAndReturn (); + } + + zygotePath = destination; } + Log.DebugLine ($"Zygote path: {zygotePath}"); return zygotePath; + + string? ReportFailureAndReturn () + { + const string appProcess32 = "app_process"; + const string appProcess64 = appProcess32 + "64"; + + string bitness = AppIs64Bit ? "64" : "32"; + string process = AppIs64Bit ? appProcess64 : appProcess32; + + Log.ErrorLine ($"Failed to copy {bitness}-bit {process}"); + Log.ErrorLine ("Unable to determine path of the zygote process on device"); + + return null; + } } public abstract bool Copy (out string? zygotePath); diff --git a/tools/xadebug/Xamarin.Android.Debug/LddDeviceLibrariesCopier.cs b/tools/xadebug/Xamarin.Android.Debug/LddDeviceLibrariesCopier.cs index 8f49c0b3542..2a2fc1348a7 100644 --- a/tools/xadebug/Xamarin.Android.Debug/LddDeviceLibrariesCopier.cs +++ b/tools/xadebug/Xamarin.Android.Debug/LddDeviceLibrariesCopier.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using Xamarin.Android.Utilities; using Xamarin.Android.Tasks; @@ -13,6 +14,67 @@ public LddDeviceLibraryCopier (XamarinLoggingHelper log, AdbRunner adb, bool app public override bool Copy (out string? zygotePath) { - throw new NotImplementedException(); + string lddPath = Device.DeviceLddPath ?? throw new InvalidOperationException ("On-device `ldd` binary is required"); + + zygotePath = FetchZygote (); + if (String.IsNullOrEmpty (zygotePath)) { + return false; + } + + (bool success, string zygoteLibs) = Adb.Shell (lddPath, zygotePath).Result; + if (!success) { + Log.ErrorLine ($"On-device ldd ({lddPath}) failed to return list of dependencies for Android application process {zygotePath}"); + return false; + } + + Log.DebugLine ("Zygote libs:"); + Log.DebugLine (zygoteLibs); + + (List libraryPaths, List libraryNames) = LddOutputToLibraryList (zygoteLibs); + if (libraryPaths.Count == 0 || libraryNames.Count == 0) { + Log.WarningLine ($"ldd didn't report any shared libraries on-device application process '{zygotePath}' depends on"); + return true; // Not an error, per se + } + + var moduleCache = new LddLldbModuleCache (Log, Device, libraryPaths, libraryNames); + moduleCache.Populate (zygotePath); + + return true; + } + + (List libraryPaths, List libraryNames) LddOutputToLibraryList (string output) + { + var libraryPaths = new List (); + var libraryNames = new List (); + if (String.IsNullOrEmpty (output)) { + return (libraryPaths, libraryNames); + } + + // Overall line format is: LIBRARY_NAME => LIBRARY_PATH (HEX_ADDRESS) + // Lines are split on space, in assumption (and hope) that Android will not use filenames with spaces in them. This way we don't have to worry about the `=>` + // separator (which can, in theory, be changed to something else on some version of Android) + foreach (string l in output.Split ('\n')) { + string line = l.Trim (); + if (line.Length == 0) { + continue; + } + + string[] parts = line.Split (' '); + if (parts.Length != 4) { + Log.WarningLine ($"ldd line has unsupported format, ignoring: '{line}'"); + continue; + } + + string path = parts[2]; + if (String.Compare (path, "[vdso]", StringComparison.OrdinalIgnoreCase) == 0) { + // virtual library, doesn't exist on disk + continue; + } + + libraryPaths.Add (path); + libraryNames.Add (parts[0]); + } + + return (libraryPaths, libraryNames); } } diff --git a/tools/xadebug/Xamarin.Android.Debug/LddLldbModuleCache.cs b/tools/xadebug/Xamarin.Android.Debug/LddLldbModuleCache.cs new file mode 100644 index 00000000000..edf833cf536 --- /dev/null +++ b/tools/xadebug/Xamarin.Android.Debug/LddLldbModuleCache.cs @@ -0,0 +1,24 @@ +using System.Collections.Generic; + +using Xamarin.Android.Utilities; + +namespace Xamarin.Android.Debug; + +class LddLldbModuleCache : LldbModuleCache +{ + List dependencyLibraryNames; + + public LddLldbModuleCache (XamarinLoggingHelper log, AndroidDevice device, List deviceSharedLibraryPaths, List dependencyLibraryNames) + : base (log, device, deviceSharedLibraryPaths) + { + this.dependencyLibraryNames = dependencyLibraryNames; + } + + protected override void FetchDependencies (HashSet alreadyDownloaded, string localPath) + { + Log.DebugLine ($"Binary {localPath} references the following libraries:"); + foreach (string lib in dependencyLibraryNames) { + FetchLibrary (lib, alreadyDownloaded); + } + } +} diff --git a/tools/xadebug/Xamarin.Android.Debug/LldbModuleCache.cs b/tools/xadebug/Xamarin.Android.Debug/LldbModuleCache.cs index d31ed16a7ac..1d58b0d5c6d 100644 --- a/tools/xadebug/Xamarin.Android.Debug/LldbModuleCache.cs +++ b/tools/xadebug/Xamarin.Android.Debug/LldbModuleCache.cs @@ -2,19 +2,20 @@ using System.Collections.Generic; using System.IO; -using ELFSharp.ELF; -using ELFSharp.ELF.Sections; using Xamarin.Android.Utilities; namespace Xamarin.Android.Debug; abstract class LldbModuleCache { + List deviceSharedLibraries; + Dictionary libraryCache; + protected AndroidDevice Device { get; } protected string CacheDirPath { get; } protected XamarinLoggingHelper Log { get; } - protected LldbModuleCache (XamarinLoggingHelper log, AndroidDevice device) + protected LldbModuleCache (XamarinLoggingHelper log, AndroidDevice device, List deviceSharedLibraries) { Device = device; Log = log; @@ -26,6 +27,9 @@ protected LldbModuleCache (XamarinLoggingHelper log, AndroidDevice device) "remote-android", // "platform" used by LLDB in our case device.SerialNumber ); + + this.deviceSharedLibraries = deviceSharedLibraries; + libraryCache = new Dictionary (StringComparer.Ordinal); } public void Populate (string zygotePath) @@ -38,86 +42,41 @@ public void Populate (string zygotePath) } var alreadyDownloaded = new HashSet (StringComparer.Ordinal); - using IELF? elf = ReadElfFile (localPath); - - FetchDependencies (elf, alreadyDownloaded, localPath); + FetchDependencies (alreadyDownloaded, localPath); } - void FetchDependencies (IELF? elf, HashSet alreadyDownloaded, string localPath) - { - if (elf == null) { - Log.DebugLine ($"Failed to open '{localPath}' as an ELF file. Ignoring."); - return; - } + protected abstract void FetchDependencies (HashSet alreadyDownloaded, string localPath); - var dynstr = GetSection (elf, ".dynstr") as IStringTable; - if (dynstr == null) { - Log.DebugLine ($"ELF binary {localPath} has no .dynstr section, unable to read referenced shared library names"); - return; + protected string? FetchLibrary (string lib, HashSet alreadyDownloaded) + { + Log.Debug ($" {lib}"); + if (alreadyDownloaded.Contains (lib)) { + Log.DebugLine (" [already downloaded]"); + return null; } - var needed = new HashSet (StringComparer.Ordinal); - foreach (IDynamicSection section in elf.GetSections ()) { - foreach (IDynamicEntry entry in section.Entries) { - if (entry.Tag != DynamicTag.Needed) { - continue; - } - - AddNeeded (dynstr, entry); - } + string? deviceLibraryPath = GetSharedLibraryPath (lib); + if (String.IsNullOrEmpty (deviceLibraryPath)) { + Log.DebugLine (" [device path unknown]"); + Log.WarningLine ($"Referenced libary '{lib}' not found on device"); + return null; } - Log.DebugLine ($"Binary {localPath} references the following libraries:"); - foreach (string lib in needed) { - Log.Debug ($" {lib}"); - if (alreadyDownloaded.Contains (lib)) { - Log.DebugLine (" [already downloaded]"); - continue; - } - - string? deviceLibraryPath = GetSharedLibraryPath (lib); - if (String.IsNullOrEmpty (deviceLibraryPath)) { - Log.DebugLine (" [device path unknown]"); - Log.WarningLine ($"Referenced libary '{lib}' not found on device"); - continue; - } - - Log.DebugLine (" [downloading]"); - Log.Status ("Downloading", deviceLibraryPath); - string? localLibraryPath = FetchFileFromDevice (deviceLibraryPath); - if (String.IsNullOrEmpty (localLibraryPath)) { - Log.Log (LogLevel.Info, " [FAILED]", XamarinLoggingHelper.ErrorColor); - continue; - } - Log.LogLine (LogLevel.Info, " [SUCCESS]", XamarinLoggingHelper.InfoColor); - - alreadyDownloaded.Add (lib); - using IELF? libElf = ReadElfFile (localLibraryPath); - FetchDependencies (libElf, alreadyDownloaded, localLibraryPath); + Log.DebugLine (" [downloading]"); + Log.Status ("Downloading", deviceLibraryPath); + string? localLibraryPath = FetchFileFromDevice (deviceLibraryPath); + if (String.IsNullOrEmpty (localLibraryPath)) { + Log.Log (LogLevel.Info, " [FAILED]", XamarinLoggingHelper.ErrorColor); + return null; } + Log.LogLine (LogLevel.Info, " [SUCCESS]", XamarinLoggingHelper.InfoColor); - void AddNeeded (IStringTable stringTable, IDynamicEntry entry) - { - ulong index; - if (entry is DynamicEntry entry64) { - index = entry64.Value; - } else if (entry is DynamicEntry entry32) { - index = (ulong)entry32.Value; - } else { - Log.WarningLine ($"DynamicEntry neither 32 nor 64 bit? Weird"); - return; - } - - string name = stringTable[(long)index]; - if (needed.Contains (name)) { - return; - } + alreadyDownloaded.Add (lib); - needed.Add (name); - } + return localLibraryPath; } - string? FetchFileFromDevice (string deviceFilePath) + protected string? FetchFileFromDevice (string deviceFilePath) { string localFilePath = Utilities.MakeLocalPath (CacheDirPath, deviceFilePath); string localTempFilePath = $"{localFilePath}.tmp"; @@ -133,38 +92,23 @@ void AddNeeded (IStringTable stringTable, IDynamicEntry entry) return localFilePath; } - protected string GetUnixFileName (string path) + string? GetSharedLibraryPath (string libraryName) { - int idx = path.LastIndexOf ('/'); - if (idx >= 0 && idx != path.Length - 1) { - return path.Substring (idx + 1); + if (libraryCache.TryGetValue (libraryName, out string? libraryPath)) { + return libraryPath; } - return path; - } - - protected abstract string? GetSharedLibraryPath (string libraryName); + foreach (string libPath in deviceSharedLibraries) { + string fileName = Utilities.GetZipEntryFileName (libPath); - IELF? ReadElfFile (string path) - { - try { - if (ELFReader.TryLoad (path, out IELF ret)) { - return ret; + if (String.Compare (libraryName, fileName, StringComparison.Ordinal) == 0) { + libraryCache.Add (libraryName, libPath); + return libPath; } - } catch (Exception ex) { - Log.WarningLine ($"{path} may not be a valid ELF binary."); - Log.WarningLine (ex.ToString ()); } + // Cache misses, too, the list isn't going to change + libraryCache.Add (libraryName, null); return null; } - - ISection? GetSection (IELF elf, string sectionName) - { - if (!elf.TryGetSection (sectionName, out ISection section)) { - return null; - } - - return section; - } } diff --git a/tools/xadebug/Xamarin.Android.Debug/NoLddDeviceLibrariesCopier.cs b/tools/xadebug/Xamarin.Android.Debug/NoLddDeviceLibrariesCopier.cs index a066f29c608..31f50651b7d 100644 --- a/tools/xadebug/Xamarin.Android.Debug/NoLddDeviceLibrariesCopier.cs +++ b/tools/xadebug/Xamarin.Android.Debug/NoLddDeviceLibrariesCopier.cs @@ -42,7 +42,6 @@ public override bool Copy (out string? zygotePath) { zygotePath = FetchZygote (); if (String.IsNullOrEmpty (zygotePath)) { - Log.ErrorLine ("Unable to determine path of the zygote process on device"); return false; } diff --git a/tools/xadebug/Xamarin.Android.Debug/NoLddLldbModuleCache.cs b/tools/xadebug/Xamarin.Android.Debug/NoLddLldbModuleCache.cs index c45042446dc..4cd03dcdac7 100644 --- a/tools/xadebug/Xamarin.Android.Debug/NoLddLldbModuleCache.cs +++ b/tools/xadebug/Xamarin.Android.Debug/NoLddLldbModuleCache.cs @@ -1,41 +1,100 @@ using System; using System.Collections.Generic; +using ELFSharp.ELF; +using ELFSharp.ELF.Sections; using Xamarin.Android.Utilities; namespace Xamarin.Android.Debug; class NoLddLldbModuleCache : LldbModuleCache { - List deviceSharedLibraries; - Dictionary libraryCache; - public NoLddLldbModuleCache (XamarinLoggingHelper log, AndroidDevice device, List deviceSharedLibraries) - : base (log, device) + : base (log, device, deviceSharedLibraries) + {} + + protected override void FetchDependencies (HashSet alreadyDownloaded, string localPath) { - this.deviceSharedLibraries = deviceSharedLibraries; - libraryCache = new Dictionary (StringComparer.Ordinal); + using IELF? elf = ReadElfFile (localPath); + FetchDependencies (elf, alreadyDownloaded, localPath); } - protected override string? GetSharedLibraryPath (string libraryName) + void FetchDependencies (IELF? elf, HashSet alreadyDownloaded, string localPath) { - if (libraryCache.TryGetValue (libraryName, out string? libraryPath)) { - return libraryPath; + if (elf == null) { + Log.DebugLine ($"Failed to open '{localPath}' as an ELF file. Ignoring."); + return; + } + + var dynstr = GetSection (elf, ".dynstr") as IStringTable; + if (dynstr == null) { + Log.DebugLine ($"ELF binary {localPath} has no .dynstr section, unable to read referenced shared library names"); + return; + } + + var needed = new HashSet (StringComparer.Ordinal); + foreach (IDynamicSection section in elf.GetSections ()) { + foreach (IDynamicEntry entry in section.Entries) { + if (entry.Tag != DynamicTag.Needed) { + continue; + } + + AddNeeded (dynstr, entry); + } + } + + Log.DebugLine ($"Binary {localPath} references the following libraries:"); + foreach (string lib in needed) { + string? localLibraryPath = FetchLibrary (lib, alreadyDownloaded); + if (String.IsNullOrEmpty (localLibraryPath)) { + continue; + } + + using IELF? libElf = ReadElfFile (localLibraryPath); + FetchDependencies (libElf, alreadyDownloaded, localLibraryPath); } - // List is sorted on the order of directories as specified by ld.config.txt, file entries aren't - // sorted inside. - foreach (string libPath in deviceSharedLibraries) { - string fileName = GetUnixFileName (libPath); + void AddNeeded (IStringTable stringTable, IDynamicEntry entry) + { + ulong index; + if (entry is DynamicEntry entry64) { + index = entry64.Value; + } else if (entry is DynamicEntry entry32) { + index = (ulong)entry32.Value; + } else { + Log.WarningLine ($"DynamicEntry neither 32 nor 64 bit? Weird"); + return; + } + + string name = stringTable[(long)index]; + if (needed.Contains (name)) { + return; + } + + needed.Add (name); + } + } - if (String.Compare (libraryName, fileName, StringComparison.Ordinal) == 0) { - libraryCache.Add (libraryName, libPath); - return libPath; + IELF? ReadElfFile (string path) + { + try { + if (ELFReader.TryLoad (path, out IELF ret)) { + return ret; } + } catch (Exception ex) { + Log.WarningLine ($"{path} may not be a valid ELF binary."); + Log.WarningLine (ex.ToString ()); } - // Cache misses, too, the list isn't going to change - libraryCache.Add (libraryName, null); return null; } + + ISection? GetSection (IELF elf, string sectionName) + { + if (!elf.TryGetSection (sectionName, out ISection section)) { + return null; + } + + return section; + } } diff --git a/tools/xadebug/Xamarin.Android.Debug/Utilities.cs b/tools/xadebug/Xamarin.Android.Debug/Utilities.cs index 4b4cfba1bb3..acda6cd6ff2 100644 --- a/tools/xadebug/Xamarin.Android.Debug/Utilities.cs +++ b/tools/xadebug/Xamarin.Android.Debug/Utilities.cs @@ -95,4 +95,24 @@ public static string StringHash (string input, Encoding? encoding = null) return sb.ToString (); } + + public static string GetZipEntryFileName (string zipEntryName) + { + int idx = zipEntryName.LastIndexOf ('/'); + if (idx >= 0 && idx != zipEntryName.Length - 1) { + return zipEntryName.Substring (idx + 1); + } + + return zipEntryName; + } + + public static string GetZipEntryDirName (string zipEntryName) + { + int idx = zipEntryName.LastIndexOf ('/'); + if (idx < 0) { + return String.Empty; + } + + return zipEntryName.Substring (0, idx + 1); + } } diff --git a/tools/xadebug/xadebug.csproj b/tools/xadebug/xadebug.csproj index e256350004e..c2784f5a620 100644 --- a/tools/xadebug/xadebug.csproj +++ b/tools/xadebug/xadebug.csproj @@ -40,6 +40,12 @@ + + + xa_start_lldb_server.sh + + + From e146d06c35d7e611e91b4798bd2b0f92c051365e Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 17 Jan 2023 21:06:05 +0100 Subject: [PATCH 27/30] A handful of changes, lots of testing on various versions of Android --- .../Utilities/AdbRunner.cs | 22 +++++ tools/xadebug/Main.cs | 37 ++++++-- .../xadebug/Resources/xa_start_lldb_server.sh | 34 ++++---- .../Xamarin.Android.Debug/DebugSession.cs | 85 ++++++++++++++++--- .../XamarinLoggingHelper.cs | 32 +++++++ 5 files changed, 172 insertions(+), 38 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs index e8659bd4845..02e7cbd3da2 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AdbRunner.cs @@ -105,6 +105,28 @@ public async Task Push (string localPath, string remotePath) return await RunAdb (runner); } + public async Task Install (string apkPath, bool apkIsDebuggable = false, bool replaceExisting = true, bool noStreaming = true) + { + var runner = CreateAdbRunner (); + runner.AddArgument ("install"); + + if (replaceExisting) { + runner.AddArgument ("-r"); + } + + if (apkIsDebuggable) { + runner.AddArgument ("-d"); // Allow version code downgrade + } + + if (noStreaming) { + runner.AddArgument ("--no-streaming"); + } + + runner.AddQuotedArgument (apkPath); + + return await RunAdb (runner); + } + public async Task<(bool success, string output)> CreateDirectoryAs (string packageName, string directoryPath) { return await RunAs (packageName, "mkdir", "-p", directoryPath); diff --git a/tools/xadebug/Main.cs b/tools/xadebug/Main.cs index 33f35c3eab4..c3f8a53b954 100644 --- a/tools/xadebug/Main.cs +++ b/tools/xadebug/Main.cs @@ -14,7 +14,7 @@ namespace Xamarin.Android.Debug; sealed class ParsedOptions { public bool ShowHelp; - public bool Verbose = true; // TODO: remove the default once development is done + public bool Verbose; public string Configuration = "Debug"; public string? PackageName; public string? Activity; @@ -63,6 +63,10 @@ static int Main (string[] args) List rest = opts.Parse (args); log.Verbose = parsedOptions.Verbose; + DateTime now = DateTime.Now; + log.LogFilePath = Path.Combine (Path.GetFullPath (parsedOptions.WorkDirectory), $"session-{now.Year}-{now.Month:00}-{now.Day:00}-{now.Hour:00}:{now.Minute:00}:{now.Second:00}.log"); + log.StatusLine ("Session log file", log.LogFilePath); + if (parsedOptions.ShowHelp || rest.Count == 0) { int ret = 0; if (!parsedOptions.ShowHelp) { @@ -114,14 +118,15 @@ static int Main (string[] args) string aPath = rest[0]; string? apkFilePath = null; + string? buildLogPath = null; ZipArchive? apk = null; if (Directory.Exists (aPath)) { - apkFilePath = BuildApp (aPath, parsedOptions, projectPathIsDirectory: true); + (apkFilePath, buildLogPath) = BuildApp (aPath, parsedOptions, projectPathIsDirectory: true); } else if (File.Exists (aPath)) { if (String.Compare (".csproj", Path.GetExtension (aPath), StringComparison.OrdinalIgnoreCase) == 0) { // Let's see if we can trust the file name... - apkFilePath = BuildApp (aPath, parsedOptions, projectPathIsDirectory: false); + (apkFilePath, buildLogPath) = BuildApp (aPath, parsedOptions, projectPathIsDirectory: false); } else if (IsAndroidPackageFile (aPath, out apk)) { apkFilePath = aPath; } else { @@ -133,6 +138,10 @@ static int Main (string[] args) log.ErrorLine (); } + if (!String.IsNullOrEmpty (buildLogPath)) { + log.StatusLine ("Build log", buildLogPath); + } + if (String.IsNullOrEmpty (apkFilePath)) { return 1; } @@ -163,6 +172,10 @@ static int Main (string[] args) return 1; } + if (!debugSession.Run ()) { + return 1; + } + return 0; } @@ -308,8 +321,10 @@ static bool IsAndroidPackageFile (string filePath, out ZipArchive? apk) return apk.ContainsEntry (AndroidManifestZipPath); } - static string? BuildApp (string projectPath, ParsedOptions parsedOptions, bool projectPathIsDirectory) + static (string? apkPath, string? buildLogPath) BuildApp (string projectPath, ParsedOptions parsedOptions, bool projectPathIsDirectory) { + log.MessageLine (); + var dotnet = new DotNetRunner (log, parsedOptions.DotNetCommand, parsedOptions.WorkDirectory); string? logPath = dotnet.Build ( projectPath, @@ -322,7 +337,7 @@ static bool IsAndroidPackageFile (string filePath, out ZipArchive? apk) ).Result; if (String.IsNullOrEmpty (logPath)) { - return null; + return FinishAndReturn (null, null); } string projectDir = projectPathIsDirectory ? projectPath : Path.GetDirectoryName (projectPath) ?? "."; @@ -338,15 +353,21 @@ static bool IsAndroidPackageFile (string filePath, out ZipArchive? apk) log.MessageLine (); log.MessageLine ("Please run `xadebug` again, passing it path to the produced APK file"); log.MessageLine (); - return null; + return FinishAndReturn (null, logPath); } if (!File.Exists (apkPath)) { log.ErrorLine ($"APK file '{apkPath}' not found after build"); - return null; + return FinishAndReturn (null, logPath); }; - return apkPath; + return FinishAndReturn (apkPath, logPath); + + (string? apkPath, string? buildLogPath) FinishAndReturn (string? apkPath, string? buildLogPath) + { + log.MessageLine (); + return (apkPath, buildLogPath); + } } static string? TryToGuessApkPath (string projectDir, ParsedOptions parsedOptions) diff --git a/tools/xadebug/Resources/xa_start_lldb_server.sh b/tools/xadebug/Resources/xa_start_lldb_server.sh index 8fa6cdb3141..d024cced9c7 100755 --- a/tools/xadebug/Resources/xa_start_lldb_server.sh +++ b/tools/xadebug/Resources/xa_start_lldb_server.sh @@ -6,36 +6,36 @@ set -x umask 0002 -LLDB_DIR=$1 -LISTENER_SCHEME=$2 -DOMAINSOCKET_DIR=$3 -PLATFORM_SOCKET=$4 +LLDB_DIR="$1" +LISTENER_SCHEME="$2" +DOMAINSOCKET_DIR="$3" +PLATFORM_SOCKET="$4" LOG_CHANNELS="$5" -BIN_DIR=$LLDB_DIR/bin -LOG_DIR=$LLDB_DIR/log -TMP_DIR=$LLDB_DIR/tmp -PLATFORM_LOG_FILE=$LOG_DIR/platform.log +BIN_DIR="$LLDB_DIR/bin" +LOG_DIR="$LLDB_DIR/log" +TMP_DIR="$LLDB_DIR/tmp" +PLATFORM_LOG_FILE="$LOG_DIR/platform.log" -export LLDB_DEBUGSERVER_LOG_FILE=$LOG_DIR/gdb-server.log +export LLDB_DEBUGSERVER_LOG_FILE="$LOG_DIR/gdb-server.log" export LLDB_SERVER_LOG_CHANNELS="$LOG_CHANNELS" -export LLDB_DEBUGSERVER_DOMAINSOCKET_DIR=$DOMAINSOCKET_DIR +export LLDB_DEBUGSERVER_DOMAINSOCKET_DIR="$DOMAINSOCKET_DIR" # This directory already exists. Make sure it has the right permissions. chmod 0775 "$LLDB_DIR" -rm -r $TMP_DIR -mkdir $TMP_DIR -export TMPDIR=$TMP_DIR +rm -r "$TMP_DIR" +mkdir "$TMP_DIR" +export TMPDIR="$TMP_DIR" -rm -r $LOG_DIR -mkdir $LOG_DIR +rm -r "$LOG_DIR" +mkdir "$LOG_DIR" # LLDB would create these files with more restrictive permissions than our umask above. Make sure # it doesn't get a chance. # "touch" does not exist on pre API-16 devices. This is a poor man's replacement -cat "$LLDB_DEBUGSERVER_LOG_FILE" 2>"$PLATFORM_LOG_FILE" +cat "$LLDB_DEBUGSERVER_LOG_FILE" 2> "$PLATFORM_LOG_FILE" cd $TMP_DIR # change cwd -$BIN_DIR/lldb-server platform --server --listen $LISTENER_SCHEME://$DOMAINSOCKET_DIR/$PLATFORM_SOCKET --log-file "$PLATFORM_LOG_FILE" --log-channels "$LOG_CHANNELS" $LOG_DIR/platform-stdout.log 2>&1 +"$BIN_DIR/lldb-server" platform --server --listen "$LISTENER_SCHEME://$DOMAINSOCKET_DIR/$PLATFORM_SOCKET" --log-file "$PLATFORM_LOG_FILE" --log-channels "$LOG_CHANNELS" "$LOG_DIR/platform-stdout.log" 2>&1 diff --git a/tools/xadebug/Xamarin.Android.Debug/DebugSession.cs b/tools/xadebug/Xamarin.Android.Debug/DebugSession.cs index 7e89729cf0a..5d5832a048a 100644 --- a/tools/xadebug/Xamarin.Android.Debug/DebugSession.cs +++ b/tools/xadebug/Xamarin.Android.Debug/DebugSession.cs @@ -23,6 +23,13 @@ class DebugSession readonly string workDirectory; readonly ApplicationInfo appInfo; + const string socketScheme = "unix-abstract"; + string? socketDir = null; + string? socketName = null; + string? lldbScriptPath = null; + AndroidDevice? device = null; + AndroidNdk? ndk = null; + public DebugSession (XamarinLoggingHelper logger, ApplicationInfo appInfo, string apkPath, ZipArchive apk, ParsedOptions parsedOptions) { log = logger; @@ -31,7 +38,7 @@ public DebugSession (XamarinLoggingHelper logger, ApplicationInfo appInfo, strin this.apk = apk; this.appInfo = appInfo; workDirectory = Path.Combine (parsedOptions.WorkDirectory, Utilities.StringHash (apkPath)); - } + } public bool Prepare () { @@ -41,8 +48,8 @@ public bool Prepare () return false; } - var ndk = new AndroidNdk (log, parsedOptions.NdkDirPath!, supportedAbis); - var device = new AndroidDevice ( + ndk = new AndroidNdk (log, parsedOptions.NdkDirPath!, supportedAbis); + device = new AndroidDevice ( log, ndk, workDirectory, @@ -52,6 +59,12 @@ public bool Prepare () parsedOptions.TargetDevice ); + // Install first, since there might already be a version of this application on device that is not debuggable + if (!device.AdbRunner.Install (apkPath, apkIsDebuggable: true).Result) { + log.ErrorLine ($"Failed to install package '{apkPath}' to device"); + return false; + } + if (!device.GatherInfo ()) { return false; } @@ -70,6 +83,12 @@ public bool Prepare () log.StatusLine ("All supported ABIs", String.Join (", ", supportedAbis)); } + socketDir = $"/xa-{appInfo.PackageName}"; + + var rnd = new Random (); + socketName = $"xa-platform-{rnd.NextInt64 ()}.sock"; + lldbScriptPath = WriteLldbScript (appLibsDirectory, device, socketScheme, socketDir, socketName, mainProcessPath); + LogABIs ("Application", supportedAbis); LogABIs (" Device", device.DeviceAbis); log.StatusLine (" Selected ABI", $"{device.MainAbi} (architecture: {device.MainArch})"); @@ -80,22 +99,62 @@ public bool Prepare () log.StatusLine ("Debug server launcher script path on device", device.DebugServerLauncherScriptPath); log.MessageLine (); - // TODO: install the apk + return true; + + void LogABIs (string which, IEnumerable abis) + { + log.StatusLine ($"{which} ABIs", String.Join (", ", abis)); + } + } + + public bool Run () + { + if (!EnsureValidState (nameof (socketDir), socketDir) || + !EnsureValidState (nameof (socketName), socketName) || + !EnsureValidState (nameof (lldbScriptPath), socketName) || + !EnsureValidState (nameof (device), device) || + !EnsureValidState (nameof (ndk), ndk) + ) { + return false; + } + + log.InfoLine ("Starting lldb server on device"); + + // TODO: either start the server in the background of the remote shell or keep this `RunAs` in the background + (bool success, string output) = device!.AdbRunner.RunAs ( + appInfo.PackageName, + device.DebugServerLauncherScriptPath, + device.LldbBaseDir, + socketScheme, + socketDir!, + socketName!, + "\\\"lldb process:gdb-remote packets\\\"").Result; + + if (!success) { + return false; + } + // TODO: start the app // TODO: start the app so that it waits for the debugger (until monodroid_gdb_wait is set) - string socketScheme = "unix-abstract"; - string socketDir = $"/xa-{appInfo.PackageName}"; + return true; - var rnd = new Random (); - string socketName = $"xa-platform-{rnd.NextInt64 ()}.sock"; - string lldbScriptPath = WriteLldbScript (appLibsDirectory, device, socketScheme, socketDir, socketName, mainProcessPath); + bool EnsureValidState (string name, T? variable) + { + bool valid; - return false; + if (typeof(T) == typeof(string)) { + valid = !String.IsNullOrEmpty ((string?)(object?)variable); + } else { + valid = variable != null; + } - void LogABIs (string which, IEnumerable abis) - { - log.StatusLine ($"{which} ABIs", String.Join (", ", abis)); + if (valid) { + return true; + } + + log.ErrorLine ($"Debug session hasn't been initialized properly. Required variable '{name}' not set"); + return false; } } diff --git a/tools/xadebug/Xamarin.Android.Utilities/XamarinLoggingHelper.cs b/tools/xadebug/Xamarin.Android.Utilities/XamarinLoggingHelper.cs index 4167c8852b6..c32d031555a 100644 --- a/tools/xadebug/Xamarin.Android.Utilities/XamarinLoggingHelper.cs +++ b/tools/xadebug/Xamarin.Android.Utilities/XamarinLoggingHelper.cs @@ -15,6 +15,8 @@ enum LogLevel class XamarinLoggingHelper { static readonly object consoleLock = new object (); + string? logFilePath = null; + string? logFileDir = null; public const ConsoleColor ErrorColor = ConsoleColor.Red; public const ConsoleColor DebugColor = ConsoleColor.DarkGray; @@ -25,6 +27,20 @@ class XamarinLoggingHelper public const ConsoleColor StatusText = ConsoleColor.White; public bool Verbose { get; set; } + public string? LogFilePath { + get => logFilePath; + set { + if (!String.IsNullOrEmpty (value)) { + string? dir = Path.GetDirectoryName (value); + if (!String.IsNullOrEmpty (dir)) { + Directory.CreateDirectory (dir); + } + } + + logFilePath = value; + logFileDir = Path.GetDirectoryName (value); + } + } public void Message (string? message) { @@ -91,6 +107,7 @@ public void StatusLine (string label, string text) public void Log (LogLevel level, string? message) { if (!Verbose && level == LogLevel.Debug) { + LogToFile (message); return; } @@ -105,6 +122,8 @@ public void LogLine (LogLevel level, string? message, ConsoleColor color) public void Log (LogLevel level, string? message, ConsoleColor color) { + LogToFile (message); + if (!Verbose && level == LogLevel.Debug) { return; } @@ -125,6 +144,19 @@ public void Log (LogLevel level, string? message, ConsoleColor color) } } + void LogToFile (string? message) + { + if (String.IsNullOrEmpty (LogFilePath)) { + return; + } + + if (!String.IsNullOrEmpty (logFileDir) && !Directory.Exists (logFileDir)) { + Directory.CreateDirectory (logFileDir); + } + + File.AppendAllText (LogFilePath, message); + } + ConsoleColor ForegroundColor (LogLevel level) => level switch { LogLevel.Error => ErrorColor, LogLevel.Warning => WarningColor, From 371d3bb321c6766698c9c8ae4588e4a8e77e8b13 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 18 Jan 2023 22:42:42 +0100 Subject: [PATCH 28/30] Support for background processes + TPL, attempt #3 BackgroundProcessManager will probably go away, too clunky. ProcessRunner2 should provide a much better way to do it. --- .../BackgroundProcessManager.cs | 68 +++++ .../Xamarin.Android.Utilities/ILogger.cs | 22 ++ .../IProcessOutputLogger.cs | 7 + .../ProcessRunner.cs | 269 ++++++++++++++++++ .../ProcessStatus.cs | 18 ++ .../XamarinLoggingHelper.cs | 2 +- 6 files changed, 385 insertions(+), 1 deletion(-) create mode 100644 tools/xadebug/Xamarin.Android.Utilities/BackgroundProcessManager.cs create mode 100644 tools/xadebug/Xamarin.Android.Utilities/ILogger.cs create mode 100644 tools/xadebug/Xamarin.Android.Utilities/IProcessOutputLogger.cs create mode 100644 tools/xadebug/Xamarin.Android.Utilities/ProcessRunner.cs create mode 100644 tools/xadebug/Xamarin.Android.Utilities/ProcessStatus.cs diff --git a/tools/xadebug/Xamarin.Android.Utilities/BackgroundProcessManager.cs b/tools/xadebug/Xamarin.Android.Utilities/BackgroundProcessManager.cs new file mode 100644 index 00000000000..a8b07892184 --- /dev/null +++ b/tools/xadebug/Xamarin.Android.Utilities/BackgroundProcessManager.cs @@ -0,0 +1,68 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; + +using Xamarin.Android.Tasks; + +namespace Xamarin.Android.Utilities; + +class BackgroundProcessManager : IDisposable +{ + readonly object runnersLock = new object (); + readonly List runners; + + bool disposed; + + public BackgroundProcessManager () + { + runners = new List (); + Console.CancelKeyPress += ConsoleCanceled; + } + + // TODO: override finalizer only if 'Dispose(bool disposing)' has code to free unmanaged resources + ~BackgroundProcessManager () + { + Dispose (disposing: false); + } + + protected virtual void Dispose (bool disposing) + { + if (!disposed) { + if (disposing) { + // TODO: dispose managed state (managed objects) + } + + FinishAllTasks (); + disposed = true; + } + } + + public void Dispose () + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + Dispose (disposing: true); + GC.SuppressFinalize (this); + } + + public void Add (ToolRunner runner) + { + // Task continuation = task.ContinueWith (TaskFailed, TaskContinuationOptions.OnlyOnFaulted); + // tasks.Add (task); + // tasks.Add (continuation); + lock (runnersLock) { + runners.Add (runner); + } + } + + void TaskFailed (Task task) + { + } + + void FinishAllTasks () + { + } + + void ConsoleCanceled (object? sender, ConsoleCancelEventArgs args) + { + } +} diff --git a/tools/xadebug/Xamarin.Android.Utilities/ILogger.cs b/tools/xadebug/Xamarin.Android.Utilities/ILogger.cs new file mode 100644 index 00000000000..cd7838781a8 --- /dev/null +++ b/tools/xadebug/Xamarin.Android.Utilities/ILogger.cs @@ -0,0 +1,22 @@ +using System; + +namespace Xamarin.Android.Utilities; + +interface ILogger +{ + void Message (string? message); + void MessageLine (string? message = null); + void Warning (string? message); + void WarningLine (string? message = null); + void Error (string? message); + void ErrorLine (string? message = null); + void Info (string? message); + void InfoLine (string? message = null); + void Debug (string? message); + void DebugLine (string? message = null); + void Status (string label, string text); + void StatusLine (string label, string text); + void Log (LogLevel level, string? message); + void LogLine (LogLevel level, string? message, ConsoleColor color); + void Log (LogLevel level, string? message, ConsoleColor color); +} diff --git a/tools/xadebug/Xamarin.Android.Utilities/IProcessOutputLogger.cs b/tools/xadebug/Xamarin.Android.Utilities/IProcessOutputLogger.cs new file mode 100644 index 00000000000..e90db418684 --- /dev/null +++ b/tools/xadebug/Xamarin.Android.Utilities/IProcessOutputLogger.cs @@ -0,0 +1,7 @@ +namespace Xamarin.Android.Utilities; + +interface IProcessOutputLogger +{ + void WriteStdout (string text); + void WriteStderr (string text); +} diff --git a/tools/xadebug/Xamarin.Android.Utilities/ProcessRunner.cs b/tools/xadebug/Xamarin.Android.Utilities/ProcessRunner.cs new file mode 100644 index 00000000000..1fd733f984e --- /dev/null +++ b/tools/xadebug/Xamarin.Android.Utilities/ProcessRunner.cs @@ -0,0 +1,269 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace Xamarin.Android.Utilities; + +class ProcessRunner2 : IDisposable +{ + static readonly TimeSpan DefaultProcessTimeout = TimeSpan.FromMinutes (5); + static readonly TimeSpan DefaultOutputTimeout = TimeSpan.FromSeconds (10); + + readonly object runLock = new object (); + readonly IProcessOutputLogger? outputLogger; + readonly ILogger? logger; + readonly string command; + + bool disposed; + bool running; + List? arguments; + + public bool CreateWindow { get; set; } + public Dictionary Environment { get; } = new Dictionary (StringComparer.Ordinal); + public string? FullCommandLine { get; private set; } + public bool LogRunInfo { get; set; } = true; + public bool LogStderr { get; set; } + public bool LogStdout { get; set; } + public bool MakeProcessGroupLeader { get; set; } + public TimeSpan ProcessTimeout { get; set; } = DefaultProcessTimeout; + public Encoding StandardOutputEncoding { get; set; } = Encoding.Default; + public Encoding StandardErrorEncoding { get; set; } = Encoding.Default; + public TimeSpan StandardOutputTimeout { get; set; } = DefaultOutputTimeout; + public TimeSpan StandardErrorTimeout { get; set; } = DefaultOutputTimeout; + public Action? CustomizeStartInfo { get; set; } + public bool UseShell { get; set; } + public ProcessWindowStyle WindowStyle { get; set; } = ProcessWindowStyle.Hidden; + public string? WorkingDirectory { get; set; } + + public ProcessRunner2 (string command, IProcessOutputLogger? outputLogger = null, ILogger? logger = null) + { + if (String.IsNullOrEmpty (command)) { + throw new ArgumentException ("must not be null or empty", nameof (command)); + } + + this.command = command; + this.outputLogger = outputLogger; + this.logger = logger; + } + + ~ProcessRunner2 () + { + Dispose (disposing: false); + } + + public void Kill (bool gracefully = true) + {} + + public void AddArgument (string arg) + { + if (arguments == null) { + arguments = new List (); + } + + arguments.Add (arg); + } + + public void AddQuotedArgument (string arg) + { + AddArgument ($"\"{arg}\""); + } + + /// + /// Run process synchronously on the calling thread + /// + public ProcessStatus Run () + { + try { + return DoRun (PrepareForRun ()); + } finally { + MarkNotRunning (); + } + } + + ProcessStatus DoRun (ProcessStartInfo psi) + { + ManualResetEventSlim? stdout_done = null; + ManualResetEventSlim? stderr_done = null; + + if (LogStderr) { + stderr_done = new ManualResetEventSlim (false); + } + + if (LogStdout) { + stdout_done = new ManualResetEventSlim (false); + } + + if (LogRunInfo) { + logger?.DebugLine ($"Running: {FullCommandLine}"); + } + + var process = new Process { + StartInfo = psi + }; + + try { + process.Start (); + } catch (System.ComponentModel.Win32Exception ex) { + if (logger != null) { + logger.ErrorLine ($"Process failed to start: {ex.Message}"); + logger.DebugLine (ex.ToString ()); + } + + return new ProcessStatus (); + } + + if (psi.RedirectStandardError) { + process.ErrorDataReceived += (object sender, DataReceivedEventArgs e) => { + if (e.Data != null) { + outputLogger!.WriteStderr (e.Data ?? String.Empty); + } else { + stderr_done!.Set (); + } + }; + process.BeginErrorReadLine (); + } + + if (psi.RedirectStandardOutput) { + process.OutputDataReceived += (object sender, DataReceivedEventArgs e) => { + if (e.Data != null) { + outputLogger!.WriteStdout (e.Data ?? String.Empty); + } else { + stdout_done!.Set (); + } + }; + process.BeginOutputReadLine (); + } + + int timeout = ProcessTimeout == TimeSpan.MaxValue ? -1 : (int)ProcessTimeout.TotalMilliseconds; + bool exited = process.WaitForExit (timeout); + if (!exited) { + logger?.ErrorLine ($"Process '{FullCommandLine}' timed out after {ProcessTimeout}"); + process.Kill (); + } + + // See: https://docs.microsoft.com/en-us/dotnet/api/system.diagnostics.process.waitforexit?view=netframework-4.7.2#System_Diagnostics_Process_WaitForExit) + if (psi.RedirectStandardError || psi.RedirectStandardOutput) { + process.WaitForExit (); + } + + if (stderr_done != null) { + stderr_done.Wait (StandardErrorTimeout); + } + + if (stdout_done != null) { + stdout_done.Wait (StandardOutputTimeout); + } + + return new ProcessStatus (process.ExitCode, exited, process.ExitCode == 0); + } + + /// + /// Run process in a separate thread. The caller is responsible for awaiting on the returned Task + /// + public Task RunAsync () + { + return Task.Run (() => Run ()); + } + + /// + /// Run process in background, calling the on completion. This is meant to be used for processes which are to run under control of our + /// process but without us actively monitoring them or awaiting their completion. + /// + public void RunInBackground (Action completionHandler) + { + ProcessStartInfo psi = PrepareForRun (); + } + + protected virtual void Dispose (bool disposing) + { + if (disposed) { + return; + } + + if (disposing) { + // TODO: dispose managed state (managed objects) + } + + // TODO: free unmanaged resources (unmanaged objects) and override finalizer + // TODO: set large fields to null + disposed = true; + } + + public void Dispose () + { + Dispose (disposing: true); + GC.SuppressFinalize (this); + } + + ProcessStartInfo PrepareForRun () + { + MarkRunning (); + + var psi = new ProcessStartInfo (command) { + CreateNoWindow = !CreateWindow, + RedirectStandardError = LogStderr, + RedirectStandardOutput = LogStdout, + UseShellExecute = UseShell, + WindowStyle = WindowStyle, + }; + + if (arguments != null && arguments.Count > 0) { + psi.Arguments = String.Join (" ", arguments); + } + + if (Environment.Count > 0) { + foreach (var kvp in Environment) { + psi.Environment.Add (kvp.Key, kvp.Value); + } + } + + if (!String.IsNullOrEmpty (WorkingDirectory)) { + psi.WorkingDirectory = WorkingDirectory; + } + + if (psi.RedirectStandardError) { + StandardErrorEncoding = StandardErrorEncoding; + } + + if (psi.RedirectStandardOutput) { + StandardOutputEncoding = StandardOutputEncoding; + } + + if (CustomizeStartInfo != null) { + CustomizeStartInfo (psi); + } + + EnsureValidConfig (psi); + + FullCommandLine = $"{psi.FileName} {psi.Arguments}"; + return psi; + } + + void MarkRunning () + { + lock (runLock) { + if (running) { + throw new InvalidOperationException ("Process already running"); + } + + running = true; + } + } + + void MarkNotRunning () + { + lock (runLock) { + running = false; + } + } + + void EnsureValidConfig (ProcessStartInfo psi) + { + if ((psi.RedirectStandardOutput || psi.RedirectStandardError) && outputLogger == null) { + throw new InvalidOperationException ("Process output logger must be set in order to capture standard output streams"); + } + } +} diff --git a/tools/xadebug/Xamarin.Android.Utilities/ProcessStatus.cs b/tools/xadebug/Xamarin.Android.Utilities/ProcessStatus.cs new file mode 100644 index 00000000000..8d6e352bf3d --- /dev/null +++ b/tools/xadebug/Xamarin.Android.Utilities/ProcessStatus.cs @@ -0,0 +1,18 @@ +namespace Xamarin.Android.Utilities; + +class ProcessStatus +{ + public int ExitCode { get; } = -1; + public bool Exited { get; } = false; + public bool Success { get; } = false; + + public ProcessStatus () + {} + + public ProcessStatus (int exitCode, bool exited, bool success) + { + ExitCode = exitCode; + Exited = exited; + Success = success; + } +} diff --git a/tools/xadebug/Xamarin.Android.Utilities/XamarinLoggingHelper.cs b/tools/xadebug/Xamarin.Android.Utilities/XamarinLoggingHelper.cs index c32d031555a..4d5227d71ad 100644 --- a/tools/xadebug/Xamarin.Android.Utilities/XamarinLoggingHelper.cs +++ b/tools/xadebug/Xamarin.Android.Utilities/XamarinLoggingHelper.cs @@ -12,7 +12,7 @@ enum LogLevel Debug } -class XamarinLoggingHelper +class XamarinLoggingHelper : ILogger { static readonly object consoleLock = new object (); string? logFilePath = null; From 61ce7241dd0860e372a8dcfddacd18121187da22 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 19 Jan 2023 22:31:02 +0100 Subject: [PATCH 29/30] New tool runner, based on the new process runner AdbRunner implemented with the new tool runner --- .../Xamarin.Android.Debug/AndroidDevice.cs | 8 +- .../Xamarin.Android.Debug/DebugSession.cs | 3 +- .../DeviceLibrariesCopier.cs | 4 +- .../LddDeviceLibrariesCopier.cs | 3 +- .../NoLddDeviceLibrariesCopier.cs | 4 +- .../Xamarin.Android.Utilities/AdbRunner.cs | 230 ++++++++++++++++++ .../IProcessOutputLogger.cs | 8 +- .../ProcessRunner.cs | 152 ++++++++---- .../ProcessStatus.cs | 14 +- .../Xamarin.Android.Utilities/ToolRunner.cs | 126 ++++++++++ .../XamarinLoggingHelper.cs | 32 ++- 11 files changed, 514 insertions(+), 70 deletions(-) create mode 100644 tools/xadebug/Xamarin.Android.Utilities/AdbRunner.cs create mode 100644 tools/xadebug/Xamarin.Android.Utilities/ToolRunner.cs diff --git a/tools/xadebug/Xamarin.Android.Debug/AndroidDevice.cs b/tools/xadebug/Xamarin.Android.Debug/AndroidDevice.cs index 162b4293caf..3a74e0a5602 100644 --- a/tools/xadebug/Xamarin.Android.Debug/AndroidDevice.cs +++ b/tools/xadebug/Xamarin.Android.Debug/AndroidDevice.cs @@ -47,7 +47,7 @@ class AndroidDevice string outputDir; XamarinLoggingHelper log; - AdbRunner adb; + AdbRunner2 adb; AndroidNdk ndk; public int ApiLevel => apiLevel; @@ -62,9 +62,9 @@ class AndroidDevice public string LldbBaseDir => appLldbBaseDir ?? String.Empty; public string AppDataDir => appDataDir ?? String.Empty; public string? DeviceLddPath => deviceLdd; - public AdbRunner AdbRunner => adb; + public AdbRunner2 AdbRunner => adb; - public AndroidDevice (XamarinLoggingHelper log, AndroidNdk ndk, string outputDir, string adbPath, string packageName, List supportedAbis, string? adbTargetDevice = null) + public AndroidDevice (XamarinLoggingHelper log, IProcessOutputLogger processLogger, AndroidNdk ndk, string outputDir, string adbPath, string packageName, List supportedAbis, string? adbTargetDevice = null) { this.adbPath = adbPath; this.log = log; @@ -73,7 +73,7 @@ public AndroidDevice (XamarinLoggingHelper log, AndroidNdk ndk, string outputDir this.ndk = ndk; this.outputDir = outputDir; - adb = new AdbRunner (log, adbPath, adbTargetDevice); + adb = new AdbRunner2 (log, processLogger, adbPath, adbTargetDevice); } // TODO: implement manual error checking on API 21, since `adb` won't ever return any error code other than 0 - we need to look at the output of any command to determine diff --git a/tools/xadebug/Xamarin.Android.Debug/DebugSession.cs b/tools/xadebug/Xamarin.Android.Debug/DebugSession.cs index 5d5832a048a..d817d5ac851 100644 --- a/tools/xadebug/Xamarin.Android.Debug/DebugSession.cs +++ b/tools/xadebug/Xamarin.Android.Debug/DebugSession.cs @@ -50,7 +50,8 @@ public bool Prepare () ndk = new AndroidNdk (log, parsedOptions.NdkDirPath!, supportedAbis); device = new AndroidDevice ( - log, + log, // general logger + log, // process output logger ndk, workDirectory, parsedOptions.AdbPath, diff --git a/tools/xadebug/Xamarin.Android.Debug/DeviceLibrariesCopier.cs b/tools/xadebug/Xamarin.Android.Debug/DeviceLibrariesCopier.cs index 50ec4bb65ac..73982bfa919 100644 --- a/tools/xadebug/Xamarin.Android.Debug/DeviceLibrariesCopier.cs +++ b/tools/xadebug/Xamarin.Android.Debug/DeviceLibrariesCopier.cs @@ -10,10 +10,10 @@ abstract class DeviceLibraryCopier protected XamarinLoggingHelper Log { get; } protected bool AppIs64Bit { get; } protected string LocalDestinationDir { get; } - protected AdbRunner Adb { get; } + protected AdbRunner2 Adb { get; } protected AndroidDevice Device { get; } - protected DeviceLibraryCopier (XamarinLoggingHelper log, AdbRunner adb, bool appIs64Bit, string localDestinationDir, AndroidDevice device) + protected DeviceLibraryCopier (XamarinLoggingHelper log, AdbRunner2 adb, bool appIs64Bit, string localDestinationDir, AndroidDevice device) { Log = log; Adb = adb; diff --git a/tools/xadebug/Xamarin.Android.Debug/LddDeviceLibrariesCopier.cs b/tools/xadebug/Xamarin.Android.Debug/LddDeviceLibrariesCopier.cs index 2a2fc1348a7..166b611ecd1 100644 --- a/tools/xadebug/Xamarin.Android.Debug/LddDeviceLibrariesCopier.cs +++ b/tools/xadebug/Xamarin.Android.Debug/LddDeviceLibrariesCopier.cs @@ -2,13 +2,12 @@ using System.Collections.Generic; using Xamarin.Android.Utilities; -using Xamarin.Android.Tasks; namespace Xamarin.Android.Debug; class LddDeviceLibraryCopier : DeviceLibraryCopier { - public LddDeviceLibraryCopier (XamarinLoggingHelper log, AdbRunner adb, bool appIs64Bit, string localDestinationDir, AndroidDevice device) + public LddDeviceLibraryCopier (XamarinLoggingHelper log, AdbRunner2 adb, bool appIs64Bit, string localDestinationDir, AndroidDevice device) : base (log, adb, appIs64Bit, localDestinationDir, device) {} diff --git a/tools/xadebug/Xamarin.Android.Debug/NoLddDeviceLibrariesCopier.cs b/tools/xadebug/Xamarin.Android.Debug/NoLddDeviceLibrariesCopier.cs index 31f50651b7d..646891d819a 100644 --- a/tools/xadebug/Xamarin.Android.Debug/NoLddDeviceLibrariesCopier.cs +++ b/tools/xadebug/Xamarin.Android.Debug/NoLddDeviceLibrariesCopier.cs @@ -34,7 +34,7 @@ class NoLddDeviceLibraryCopier : DeviceLibraryCopier "/system/vendor/@LIB@/mediadrm", }; - public NoLddDeviceLibraryCopier (XamarinLoggingHelper log, AdbRunner adb, bool appIs64Bit, string localDestinationDir, AndroidDevice device) + public NoLddDeviceLibraryCopier (XamarinLoggingHelper log, AdbRunner2 adb, bool appIs64Bit, string localDestinationDir, AndroidDevice device) : base (log, adb, appIs64Bit, localDestinationDir, device) {} @@ -61,7 +61,7 @@ public override bool Copy (out string? zygotePath) void AddSharedLibraries (List sharedLibraries, string deviceDirPath, HashSet permittedPaths) { - AdbRunner.OutputLineFilter filterOutErrors = (bool isStdError, string line) => { + AdbRunner2.OutputLineFilter filterOutErrors = (bool isStdError, string line) => { if (!isStdError) { return false; // don't suppress any lines on stdout } diff --git a/tools/xadebug/Xamarin.Android.Utilities/AdbRunner.cs b/tools/xadebug/Xamarin.Android.Utilities/AdbRunner.cs new file mode 100644 index 00000000000..c1c142c5736 --- /dev/null +++ b/tools/xadebug/Xamarin.Android.Utilities/AdbRunner.cs @@ -0,0 +1,230 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace Xamarin.Android.Utilities; + +class AdbRunner2 : ToolRunner2 +{ + public delegate bool OutputLineFilter (bool isStdErr, string line); + + sealed class CaptureOutputState + { + public OutputLineFilter? LineFilter; + public CaptureProcessOutputLogger? Logger; + } + + sealed class CaptureProcessOutputLogger : IProcessOutputLogger + { + IProcessOutputLogger? wrappedLogger; + OutputLineFilter? lineFilter; + List lines; + string? stderrPrefix; + string? stdoutPrefix; + + public List Lines => lines; + + public IProcessOutputLogger? WrappedLogger => wrappedLogger; + + public string? StdoutPrefix { + get => stdoutPrefix ?? wrappedLogger?.StdoutPrefix ?? String.Empty; + set => stdoutPrefix = value; + } + + public string? StderrPrefix { + get => stderrPrefix ?? wrappedLogger?.StderrPrefix ?? String.Empty; + set => stderrPrefix = value; + } + + public CaptureProcessOutputLogger (IProcessOutputLogger? wrappedLogger, OutputLineFilter? lineFilter = null) + { + this.wrappedLogger = wrappedLogger; + this.lineFilter = lineFilter; + + lines = new List (); + } + + public void WriteStderr (string text, bool writeLine = true) + { + if (LineFiltered (text, isStdError: true)) { + return; + } + + wrappedLogger?.WriteStderr (text, writeLine); + } + + public void WriteStdout (string text, bool writeLine = true) + { + if (LineFiltered (text, isStdError: false)) { + return; + } + + lines.Add (text); + } + + bool LineFiltered (string text, bool isStdError) + { + if (lineFilter == null) { + return false; + } + + return lineFilter (isStdError, text); + } + } + + string[]? initialParams; + + public AdbRunner2 (ILogger logger, IProcessOutputLogger processOutputLogger, string adbPath, string? deviceSerial = null) + : base (adbPath, logger, processOutputLogger) + { + if (!String.IsNullOrEmpty (deviceSerial)) { + initialParams = new string[] { "-s", deviceSerial }; + } + } + + public async Task Pull (string remotePath, string localPath) + { + var runner = CreateAdbRunner (); + runner.AddArgument ("pull"); + runner.AddArgument (remotePath); + runner.AddArgument (localPath); + + return await RunAdbAsync (runner); + } + + public async Task Push (string localPath, string remotePath) + { + var runner = CreateAdbRunner (); + runner.AddArgument ("push"); + runner.AddArgument (localPath); + runner.AddArgument (remotePath); + + return await RunAdbAsync (runner); + } + + public async Task Install (string apkPath, bool apkIsDebuggable = false, bool replaceExisting = true, bool noStreaming = true) + { + var runner = CreateAdbRunner (); + runner.AddArgument ("install"); + + if (replaceExisting) { + runner.AddArgument ("-r"); + } + + if (apkIsDebuggable) { + runner.AddArgument ("-d"); // Allow version code downgrade + } + + if (noStreaming) { + runner.AddArgument ("--no-streaming"); + } + + runner.AddQuotedArgument (apkPath); + + return await RunAdbAsync (runner); + } + + public async Task<(bool success, string output)> GetAppDataDirectory (string packageName) + { + return await RunAs (packageName, "/system/bin/sh", "-c", "pwd"); + } + + + public async Task<(bool success, string output)> CreateDirectoryAs (string packageName, string directoryPath) + { + return await RunAs (packageName, "mkdir", "-p", directoryPath); + } + + public async Task<(bool success, string output)> GetPropertyValue (string propertyName) + { + var runner = CreateAdbRunner (); + return await Shell ("getprop", propertyName); + } + + public async Task<(bool success, string output)> RunAs (string packageName, string command, params string[] args) + { + if (String.IsNullOrEmpty (packageName)) { + throw new ArgumentException ("must not be null or empty", nameof (packageName)); + } + + var shellArgs = new List { + packageName, + command, + }; + + if (args != null && args.Length > 0) { + shellArgs.AddRange (args); + } + + return await Shell ("run-as", (IEnumerable)shellArgs, lineFilter: null); + } + + public async Task<(bool success, string output)> Shell (string command, List args, OutputLineFilter? lineFilter = null) + { + return await Shell (command, (IEnumerable)args, lineFilter); + } + + public async Task<(bool success, string output)> Shell (string command, params string[] args) + { + return await Shell (command, (IEnumerable)args, lineFilter: null); + } + + public async Task<(bool success, string output)> Shell (OutputLineFilter lineFilter, string command, params string[] args) + { + return await Shell (command, (IEnumerable)args, lineFilter); + } + + async Task<(bool success, string output)> Shell (string command, IEnumerable? args, OutputLineFilter? lineFilter) + { + if (String.IsNullOrEmpty (command)) { + throw new ArgumentException ("must not be null or empty", nameof (command)); + } + + var captureState = new CaptureOutputState { + LineFilter = lineFilter, + }; + + var runner = CreateAdbRunner (captureState); + + runner.AddArgument ("shell"); + runner.AddArgument (command); + runner.AddArguments (args); + + return await CaptureAdbOutput (runner, captureState); + } + + async Task RunAdbAsync (ProcessRunner2 runner) + { + ProcessStatus status = await runner.RunAsync (); + return status.Success; + } + + async Task<(bool success, string output)> CaptureAdbOutput (ProcessRunner2 runner, CaptureOutputState captureState) + { + ProcessStatus status = await runner.RunAsync (); + + string output = captureState.Logger != null ? String.Join (Environment.NewLine, captureState.Logger.Lines) : String.Empty; + return (status.Success, output); + } + + ProcessRunner2 CreateAdbRunner (CaptureOutputState? state = null) => InitProcessRunner (state, initialParams); + + protected override ProcessRunner2 CreateProcessRunner (IProcessOutputLogger consoleProcessLogger, object? state, params string?[]? initialParams) + { + IProcessOutputLogger outputLogger; + + if (state is CaptureOutputState captureState) { + captureState.Logger = new CaptureProcessOutputLogger (consoleProcessLogger, captureState.LineFilter); + outputLogger = captureState.Logger; + } else { + outputLogger = consoleProcessLogger; + } + + outputLogger.StderrPrefix = "adb> "; + ProcessRunner2 ret = base.CreateProcessRunner (outputLogger, initialParams); + + // Let's make sure all the messages we get are in English, since we need to parse some of them to detect problems + ret.Environment["LANG"] = "C"; + return ret; + } +} diff --git a/tools/xadebug/Xamarin.Android.Utilities/IProcessOutputLogger.cs b/tools/xadebug/Xamarin.Android.Utilities/IProcessOutputLogger.cs index e90db418684..3ffb3eda893 100644 --- a/tools/xadebug/Xamarin.Android.Utilities/IProcessOutputLogger.cs +++ b/tools/xadebug/Xamarin.Android.Utilities/IProcessOutputLogger.cs @@ -2,6 +2,10 @@ namespace Xamarin.Android.Utilities; interface IProcessOutputLogger { - void WriteStdout (string text); - void WriteStderr (string text); + IProcessOutputLogger? WrappedLogger { get; } + string? StdoutPrefix { get; set; } + string? StderrPrefix { get; set; } + + void WriteStdout (string text, bool writeLine = true); + void WriteStderr (string text, bool writeLine = true); } diff --git a/tools/xadebug/Xamarin.Android.Utilities/ProcessRunner.cs b/tools/xadebug/Xamarin.Android.Utilities/ProcessRunner.cs index 1fd733f984e..512e9683a26 100644 --- a/tools/xadebug/Xamarin.Android.Utilities/ProcessRunner.cs +++ b/tools/xadebug/Xamarin.Android.Utilities/ProcessRunner.cs @@ -20,10 +20,12 @@ class ProcessRunner2 : IDisposable bool disposed; bool running; List? arguments; + Task? backgroundProcess; public bool CreateWindow { get; set; } public Dictionary Environment { get; } = new Dictionary (StringComparer.Ordinal); public string? FullCommandLine { get; private set; } + public bool LeaveRunning { get; set; } public bool LogRunInfo { get; set; } = true; public bool LogStderr { get; set; } public bool LogStdout { get; set; } @@ -57,6 +59,21 @@ public ProcessRunner2 (string command, IProcessOutputLogger? outputLogger = null public void Kill (bool gracefully = true) {} + public void AddArguments (IEnumerable? args) + { + if (args == null) { + return; + } + + foreach (string? a in args) { + if (String.IsNullOrEmpty (a)) { + continue; + } + + AddArgument (a); + } + } + public void AddArgument (string arg) { if (arguments == null) { @@ -76,22 +93,66 @@ public void AddQuotedArgument (string arg) /// public ProcessStatus Run () { + return Run (ProcessTimeout); + } + + public ProcessStatus Run (TimeSpan processTimeout) + { + MarkRunning (); + try { - return DoRun (PrepareForRun ()); + return DoRun (processTimeout); } finally { MarkNotRunning (); } } - ProcessStatus DoRun (ProcessStartInfo psi) + ProcessStatus DoRun (TimeSpan processTimeout) { - ManualResetEventSlim? stdout_done = null; - ManualResetEventSlim? stderr_done = null; + var psi = new ProcessStartInfo (command) { + CreateNoWindow = !CreateWindow, + RedirectStandardError = LogStderr, + RedirectStandardOutput = LogStdout, + UseShellExecute = UseShell, + WindowStyle = WindowStyle, + }; + + if (arguments != null && arguments.Count > 0) { + psi.Arguments = String.Join (" ", arguments); + } + if (Environment.Count > 0) { + foreach (var kvp in Environment) { + psi.Environment.Add (kvp.Key, kvp.Value); + } + } + + if (!String.IsNullOrEmpty (WorkingDirectory)) { + psi.WorkingDirectory = WorkingDirectory; + } + + if (psi.RedirectStandardError) { + StandardErrorEncoding = StandardErrorEncoding; + } + + if (psi.RedirectStandardOutput) { + StandardOutputEncoding = StandardOutputEncoding; + } + + if (CustomizeStartInfo != null) { + CustomizeStartInfo (psi); + } + + EnsureValidConfig (psi); + + FullCommandLine = $"{psi.FileName} {psi.Arguments}"; + + ManualResetEventSlim? stderr_done = null; if (LogStderr) { stderr_done = new ManualResetEventSlim (false); } + ManualResetEventSlim? stdout_done = null; if (LogStdout) { stdout_done = new ManualResetEventSlim (false); } @@ -137,7 +198,7 @@ ProcessStatus DoRun (ProcessStartInfo psi) process.BeginOutputReadLine (); } - int timeout = ProcessTimeout == TimeSpan.MaxValue ? -1 : (int)ProcessTimeout.TotalMilliseconds; + int timeout = processTimeout == TimeSpan.MaxValue ? -1 : (int)processTimeout.TotalMilliseconds; bool exited = process.WaitForExit (timeout); if (!exited) { logger?.ErrorLine ($"Process '{FullCommandLine}' timed out after {ProcessTimeout}"); @@ -170,11 +231,42 @@ public Task RunAsync () /// /// Run process in background, calling the on completion. This is meant to be used for processes which are to run under control of our - /// process but without us actively monitoring them or awaiting their completion. + /// process but without us actively monitoring them or awaiting their completion. By default the process will run without a timeout (the + /// property is ignored). Timeout can be changed by setting the parameter to anything other than TimeSpan.MaxValue /// - public void RunInBackground (Action completionHandler) + public void RunInBackground (Action completionHandler, TimeSpan? processTimeout = null) { - ProcessStartInfo psi = PrepareForRun (); + backgroundProcess = new Task ( + () => Run (processTimeout ?? TimeSpan.MaxValue), + TaskCreationOptions.LongRunning + ).ContinueWith ( + (Task task) => { + ProcessStatus status; + if (task.IsFaulted) { + status = new ProcessStatus (task.Exception!); + } else { + status = new ProcessStatus (); + } + completionHandler (this, status); + return status; + }, TaskContinuationOptions.OnlyOnFaulted + ).ContinueWith ( + (Task task) => { + completionHandler (this, task.Result); + return task.Result; + }, + TaskContinuationOptions.OnlyOnRanToCompletion + ).ContinueWith ( + (Task task) => { + var status = new ProcessStatus (); + completionHandler (this, status); + return status; + }, + TaskContinuationOptions.OnlyOnCanceled + ); + + backgroundProcess.ConfigureAwait (false); + backgroundProcess.Start (); } protected virtual void Dispose (bool disposing) @@ -198,50 +290,6 @@ public void Dispose () GC.SuppressFinalize (this); } - ProcessStartInfo PrepareForRun () - { - MarkRunning (); - - var psi = new ProcessStartInfo (command) { - CreateNoWindow = !CreateWindow, - RedirectStandardError = LogStderr, - RedirectStandardOutput = LogStdout, - UseShellExecute = UseShell, - WindowStyle = WindowStyle, - }; - - if (arguments != null && arguments.Count > 0) { - psi.Arguments = String.Join (" ", arguments); - } - - if (Environment.Count > 0) { - foreach (var kvp in Environment) { - psi.Environment.Add (kvp.Key, kvp.Value); - } - } - - if (!String.IsNullOrEmpty (WorkingDirectory)) { - psi.WorkingDirectory = WorkingDirectory; - } - - if (psi.RedirectStandardError) { - StandardErrorEncoding = StandardErrorEncoding; - } - - if (psi.RedirectStandardOutput) { - StandardOutputEncoding = StandardOutputEncoding; - } - - if (CustomizeStartInfo != null) { - CustomizeStartInfo (psi); - } - - EnsureValidConfig (psi); - - FullCommandLine = $"{psi.FileName} {psi.Arguments}"; - return psi; - } - void MarkRunning () { lock (runLock) { diff --git a/tools/xadebug/Xamarin.Android.Utilities/ProcessStatus.cs b/tools/xadebug/Xamarin.Android.Utilities/ProcessStatus.cs index 8d6e352bf3d..9ef291305a9 100644 --- a/tools/xadebug/Xamarin.Android.Utilities/ProcessStatus.cs +++ b/tools/xadebug/Xamarin.Android.Utilities/ProcessStatus.cs @@ -1,14 +1,22 @@ +using System; + namespace Xamarin.Android.Utilities; class ProcessStatus { - public int ExitCode { get; } = -1; - public bool Exited { get; } = false; - public bool Success { get; } = false; + public int ExitCode { get; } = -1; + public bool Exited { get; } = false; + public bool Success { get; } = false; + public Exception? Exception { get; } public ProcessStatus () {} + public ProcessStatus (Exception ex) + { + Exception = ex; + } + public ProcessStatus (int exitCode, bool exited, bool success) { ExitCode = exitCode; diff --git a/tools/xadebug/Xamarin.Android.Utilities/ToolRunner.cs b/tools/xadebug/Xamarin.Android.Utilities/ToolRunner.cs new file mode 100644 index 00000000000..90561688101 --- /dev/null +++ b/tools/xadebug/Xamarin.Android.Utilities/ToolRunner.cs @@ -0,0 +1,126 @@ +using System; + +namespace Xamarin.Android.Utilities; + +abstract class ToolRunner2 : IDisposable +{ + sealed class ConsoleProcessLogger : IProcessOutputLogger + { + bool echoStdout; + bool echoStderr; + IProcessOutputLogger wrappedLogger; + ILogger logger; + + public IProcessOutputLogger? WrappedLogger => wrappedLogger; + public string? StdoutPrefix { get; set; } + public string? StderrPrefix { get; set; } = "stderr> "; + + public ConsoleProcessLogger (ILogger logger, IProcessOutputLogger wrappedLogger, bool echoStdout, bool echoStderr) + { + this.logger = logger; + this.wrappedLogger = wrappedLogger; + this.echoStdout = echoStdout; + this.echoStderr = echoStderr; + } + + public void WriteStderr (string text, bool writeNewline) + { + if (echoStderr) { + string message = $"{GetPrefix (StderrPrefix)}{text}"; + if (writeNewline) { + logger.ErrorLine (message); + } else { + logger.Error (message); + } + } + + wrappedLogger.WriteStderr (text, writeNewline); + } + + public void WriteStdout (string text, bool writeNewline) + { + if (echoStdout) { + string message = $"{GetPrefix (StdoutPrefix)}{text}"; + if (writeNewline) { + logger.MessageLine (message); + } else { + logger.Message (message); + } + } + + wrappedLogger.WriteStdout (text, writeNewline); + } + + string GetPrefix (string? prefix) => prefix ?? String.Empty; + } + + static readonly TimeSpan DefaultProcessTimeout = TimeSpan.FromMinutes (15); + + bool disposed; + ILogger logger; + IProcessOutputLogger processOutputLogger; + + protected ILogger Log => logger; + protected IProcessOutputLogger OutputLogger => processOutputLogger; + + public string ToolPath { get; } + public bool EchoCmdAndArguments { get; set; } = true; + public bool EchoStandardError { get; set; } = true; + public bool EchoStandardOutput { get; set; } + public TimeSpan ProcessTimeout { get; set; } = DefaultProcessTimeout; + + protected ToolRunner2 (string toolPath, ILogger logger, IProcessOutputLogger processOutputLogger) + { + if (String.IsNullOrEmpty (toolPath)) { + throw new ArgumentException ("must not be null or empty", nameof (toolPath)); + } + + this.logger = logger; + this.processOutputLogger = processOutputLogger; + + ToolPath = toolPath; + } + + ~ToolRunner2 () + { + Dispose (disposing: false); + } + + protected ProcessRunner2 InitProcessRunner (object? state, params string?[]? initialParams) + { + var consoleLogger = new ConsoleProcessLogger (logger, processOutputLogger, EchoStandardOutput, EchoStandardError); + return CreateProcessRunner (consoleLogger, state, initialParams); + } + + protected virtual ProcessRunner2 CreateProcessRunner (IProcessOutputLogger consoleProcessLogger, object? state, params string?[]? initialParams) + { + var runner = new ProcessRunner2 (ToolPath, consoleProcessLogger, logger) { + ProcessTimeout = ProcessTimeout, + LogRunInfo = EchoCmdAndArguments, + LogStdout = true, + LogStderr = true, + }; + + runner.AddArguments (initialParams); + return runner; + } + + protected virtual void Dispose (bool disposing) + { + if (disposed) { + return; + } + + if (disposing) { + // TODO: dispose managed state (managed objects) + } + + disposed = true; + } + + public void Dispose () + { + Dispose (disposing: true); + GC.SuppressFinalize (this); + } +} diff --git a/tools/xadebug/Xamarin.Android.Utilities/XamarinLoggingHelper.cs b/tools/xadebug/Xamarin.Android.Utilities/XamarinLoggingHelper.cs index 4d5227d71ad..a17aa6d8018 100644 --- a/tools/xadebug/Xamarin.Android.Utilities/XamarinLoggingHelper.cs +++ b/tools/xadebug/Xamarin.Android.Utilities/XamarinLoggingHelper.cs @@ -12,7 +12,7 @@ enum LogLevel Debug } -class XamarinLoggingHelper : ILogger +class XamarinLoggingHelper : ILogger, IProcessOutputLogger { static readonly object consoleLock = new object (); string? logFilePath = null; @@ -42,6 +42,22 @@ public string? LogFilePath { } } + public IProcessOutputLogger? WrappedLogger => null; + + public string? StdoutPrefix { + get => String.Empty; + set { + // no op + } + } + + public string? StderrPrefix { + get => "stderr> "; + set { + // no op + } + } + public void Message (string? message) { Log (LogLevel.Message, message); @@ -202,5 +218,17 @@ public void LogWarning (string message, params object[] messageArgs) WarningLine (String.Format (message, messageArgs)); } } -#endregion + + public void WriteStdout (string text, bool writeLine = true) + { + LogToFile ($"{StdoutPrefix}{text}{GetNewline (writeLine)}"); + } + + public void WriteStderr (string text, bool writeLine = true) + { + LogToFile ($"{StderrPrefix}{text}{GetNewline (writeLine)}"); + } + + string GetNewline (bool yes) => yes ? Environment.NewLine : String.Empty; + #endregion } From e97c68052bfc02b929994bfb5e9a1580c481ae4c Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 6 Apr 2023 20:39:46 +0200 Subject: [PATCH 30/30] Some little stuff --- .../Xamarin.Android.Debug/AndroidDevice.cs | 9 ++--- .../Xamarin.Android.Utilities/AdbRunner.cs | 39 ++++++++++++++++++- .../BackgroundProcessManager.cs | 11 +----- .../ProcessRunner.cs | 5 ++- 4 files changed, 46 insertions(+), 18 deletions(-) diff --git a/tools/xadebug/Xamarin.Android.Debug/AndroidDevice.cs b/tools/xadebug/Xamarin.Android.Debug/AndroidDevice.cs index 3a74e0a5602..ad444409cc6 100644 --- a/tools/xadebug/Xamarin.Android.Debug/AndroidDevice.cs +++ b/tools/xadebug/Xamarin.Android.Debug/AndroidDevice.cs @@ -265,7 +265,7 @@ bool PushServerExecutable (string hostSource, string deviceDestination) // TODO: handle multiple pids bool KillDebugServer (string debugServerPath) { - long serverPID = GetDeviceProcessID (debugServerPath, quiet: false); + long serverPID = GetDeviceProcessID (debugServerPath, quiet: true); if (serverPID <= 0) { return true; } @@ -279,10 +279,9 @@ long GetDeviceProcessID (string processName, bool quiet = false) { (bool success, string output) = adb.Shell ("pidof", Path.GetFileName (processName)).Result; if (!success) { - if (!quiet) { - log.ErrorLine ($"Failed to obtain PID of process '{processName}'"); - log.ErrorLine (output); - } + Action logger = quiet ? log.DebugLine : log.ErrorLine; + logger ($"Failed to obtain PID of process '{processName}'"); + logger (output); return -1; } diff --git a/tools/xadebug/Xamarin.Android.Utilities/AdbRunner.cs b/tools/xadebug/Xamarin.Android.Utilities/AdbRunner.cs index c1c142c5736..87d8a9aa57b 100644 --- a/tools/xadebug/Xamarin.Android.Utilities/AdbRunner.cs +++ b/tools/xadebug/Xamarin.Android.Utilities/AdbRunner.cs @@ -129,7 +129,6 @@ public async Task Install (string apkPath, bool apkIsDebuggable = false, b return await RunAs (packageName, "/system/bin/sh", "-c", "pwd"); } - public async Task<(bool success, string output)> CreateDirectoryAs (string packageName, string directoryPath) { return await RunAs (packageName, "mkdir", "-p", directoryPath); @@ -142,6 +141,37 @@ public async Task Install (string apkPath, bool apkIsDebuggable = false, b } public async Task<(bool success, string output)> RunAs (string packageName, string command, params string[] args) + { + return await Shell ( + "run-as", + RunAsPrepareArgs (packageName, command, args), + lineFilter: null + ); + } + + public void RunAsInBackground (BackgroundProcessManager processManager, string packageName, string command, params string[] args) + { + RunAsInBackground (processManager, null, null, packageName, command, args); + } + + public void RunAsInBackground (BackgroundProcessManager processManager, ProcessRunner2.BackgroundActionCompletionHandler? completionHandler, string packageName, string command, params string[] args) + { + RunAsInBackground (processManager, completionHandler, null, packageName, command, args); + } + + public void RunAsInBackground (BackgroundProcessManager processManager, ProcessRunner2.BackgroundActionCompletionHandler? completionHandler, TimeSpan? processTimeout, string packageName, string command, params string[] args) + { + ShellInBackground ( + processManager, + completionHandler, + processTimeout, + "run-as", + RunAsPrepareArgs (packageName, command, args), + lineFilter: null + ); + } + + IEnumerable RunAsPrepareArgs (string packageName, string command, params string[] args) { if (String.IsNullOrEmpty (packageName)) { throw new ArgumentException ("must not be null or empty", nameof (packageName)); @@ -156,7 +186,7 @@ public async Task Install (string apkPath, bool apkIsDebuggable = false, b shellArgs.AddRange (args); } - return await Shell ("run-as", (IEnumerable)shellArgs, lineFilter: null); + return shellArgs; } public async Task<(bool success, string output)> Shell (string command, List args, OutputLineFilter? lineFilter = null) @@ -193,6 +223,11 @@ public async Task Install (string apkPath, bool apkIsDebuggable = false, b return await CaptureAdbOutput (runner, captureState); } + void ShellInBackground (BackgroundProcessManager processManager, ProcessRunner2.BackgroundActionCompletionHandler? completionHandler, TimeSpan? processTimeout, + string command, IEnumerable? args, OutputLineFilter? lineFilter) + { + } + async Task RunAdbAsync (ProcessRunner2 runner) { ProcessStatus status = await runner.RunAsync (); diff --git a/tools/xadebug/Xamarin.Android.Utilities/BackgroundProcessManager.cs b/tools/xadebug/Xamarin.Android.Utilities/BackgroundProcessManager.cs index a8b07892184..3cc81af3f5b 100644 --- a/tools/xadebug/Xamarin.Android.Utilities/BackgroundProcessManager.cs +++ b/tools/xadebug/Xamarin.Android.Utilities/BackgroundProcessManager.cs @@ -44,15 +44,8 @@ public void Dispose () GC.SuppressFinalize (this); } - public void Add (ToolRunner runner) - { - // Task continuation = task.ContinueWith (TaskFailed, TaskContinuationOptions.OnlyOnFaulted); - // tasks.Add (task); - // tasks.Add (continuation); - lock (runnersLock) { - runners.Add (runner); - } - } + public void RunInBackground (ProcessRunner2 runner, ProcessRunner2.BackgroundActionCompletionHandler? completionHandler) + {} void TaskFailed (Task task) { diff --git a/tools/xadebug/Xamarin.Android.Utilities/ProcessRunner.cs b/tools/xadebug/Xamarin.Android.Utilities/ProcessRunner.cs index 512e9683a26..486a8a7a1f4 100644 --- a/tools/xadebug/Xamarin.Android.Utilities/ProcessRunner.cs +++ b/tools/xadebug/Xamarin.Android.Utilities/ProcessRunner.cs @@ -9,6 +9,8 @@ namespace Xamarin.Android.Utilities; class ProcessRunner2 : IDisposable { + public delegate void BackgroundActionCompletionHandler (ProcessRunner2 runner, ProcessStatus status); + static readonly TimeSpan DefaultProcessTimeout = TimeSpan.FromMinutes (5); static readonly TimeSpan DefaultOutputTimeout = TimeSpan.FromSeconds (10); @@ -234,7 +236,7 @@ public Task RunAsync () /// process but without us actively monitoring them or awaiting their completion. By default the process will run without a timeout (the /// property is ignored). Timeout can be changed by setting the parameter to anything other than TimeSpan.MaxValue /// - public void RunInBackground (Action completionHandler, TimeSpan? processTimeout = null) + public void RunInBackground (BackgroundActionCompletionHandler completionHandler, TimeSpan? processTimeout = null) { backgroundProcess = new Task ( () => Run (processTimeout ?? TimeSpan.MaxValue), @@ -265,7 +267,6 @@ public void RunInBackground (Action completionHan TaskContinuationOptions.OnlyOnCanceled ); - backgroundProcess.ConfigureAwait (false); backgroundProcess.Start (); }