From 971f86758a0d45f9abda2b63fd95fdf7da1a1924 Mon Sep 17 00:00:00 2001 From: TheAssassin Date: Thu, 6 Jul 2023 14:10:23 +0200 Subject: [PATCH 1/3] Download suitable runtime from GitHub automatically We no longer ship any runtimes with the AppImage, but just download the latest continuous release. TODO: download latest *released* runtime, and allow users to specify the release they want to download a runtime for. (We need to parse the GitHub API to find the latest release; the latter variant just skips that step and replaces one component in the URL.) --- CMakeLists.txt | 1 + ci/build-in-docker.sh | 10 +- src/CMakeLists.txt | 2 + src/appimagetool.c | 75 ++++++++------- src/appimagetool_fetch_runtime.c | 155 +++++++++++++++++++++++++++++++ src/appimagetool_fetch_runtime.h | 7 ++ 6 files changed, 216 insertions(+), 34 deletions(-) create mode 100644 src/appimagetool_fetch_runtime.c create mode 100644 src/appimagetool_fetch_runtime.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 1a7c2b7..611eaf2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -78,6 +78,7 @@ pkg_check_modules(libgcrypt REQUIRED libgcrypt IMPORTED_TARGET) # TODO: in the long term, we plan to get rid of GLib/GIO by moving to C++ pkg_check_modules(libglib REQUIRED glib-2.0 IMPORTED_TARGET) pkg_check_modules(libgio REQUIRED gio-2.0 IMPORTED_TARGET) +pkg_check_modules(libcurl REQUIRED libcurl IMPORTED_TARGET) # Alpine Linux does not ship an argp.h as part of the standard compiler toolchain unset(ARGP_H_FOUND CACHE) diff --git a/ci/build-in-docker.sh b/ci/build-in-docker.sh index 949b3ef..5ee5ee9 100755 --- a/ci/build-in-docker.sh +++ b/ci/build-in-docker.sh @@ -65,7 +65,15 @@ set -euxo pipefail apk add bash git gcc g++ cmake make file desktop-file-utils wget \ gpgme-dev libgcrypt-dev libgcrypt-static argp-standalone zstd-dev zstd-static util-linux-static \ - glib-static libassuan-static zlib-static libgpg-error-static + glib-static libassuan-static zlib-static libgpg-error-static \ + curl-dev curl-static nghttp2-static libidn2-static openssl-libs-static brotli-static c-ares-static libunistring-static + +# libcurl's pkg-config scripts are broken. everywhere, everytime. +# these additional flags have been collected from all the .pc files whose libs are mentioned as -l in Libs.private +# first, let's make sure there is no Requires.private section +grep -qv Requires.private /usr/lib/pkgconfig/libcurl.pc +# now, let's add one +echo "Requires.private: libcares libnghttp2 libidn2 libssl openssl libcrypto libbrotlicommon zlib" | tee -a /usr/lib/pkgconfig/libcurl.pc # in a Docker container, we can safely disable this check git config --global --add safe.directory '*' diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c75c31f..e9365a7 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,6 +1,7 @@ add_executable(appimagetool appimagetool.c appimagetool_sign.c + appimagetool_fetch_runtime.c hexlify.c elf.c digest.c @@ -15,6 +16,7 @@ target_link_libraries(appimagetool PkgConfig::libgio PkgConfig::libgcrypt PkgConfig::libgpgme + PkgConfig::libcurl ) target_compile_definitions(appimagetool diff --git a/src/appimagetool.c b/src/appimagetool.c index 5605a31..06d6675 100644 --- a/src/appimagetool.c +++ b/src/appimagetool.c @@ -55,14 +55,15 @@ #include "util.h" +#include "appimagetool_fetch_runtime.h" #include "appimagetool_sign.h" -enum fARCH { - fARCH_i386, +typedef enum { + fARCH_i686, fARCH_x86_64, - fARCH_arm, + fARCH_armhf, fARCH_aarch64 -}; +} fARCH; static gchar const APPIMAGEIGNORE[] = ".appimageignore"; static char _exclude_file_desc[256]; @@ -286,42 +287,55 @@ int count_archs(bool* archs) { return countArchs; } +gchar* archToName(fARCH arch) { + switch (arch) { + case fARCH_aarch64: + return "aarch64"; + case fARCH_armhf: + return "armhf"; + case fARCH_i686: + return "i686"; + case fARCH_x86_64: + return "x86_64"; + } +} + gchar* getArchName(bool* archs) { - if (archs[fARCH_i386]) - return "i386"; + if (archs[fARCH_i686]) + return archToName(fARCH_i686); else if (archs[fARCH_x86_64]) - return "x86_64"; - else if (archs[fARCH_arm]) - return "armhf"; + return archToName(fARCH_x86_64); + else if (archs[fARCH_armhf]) + return archToName(fARCH_armhf); else if (archs[fARCH_aarch64]) - return "aarch64"; + return archToName(fARCH_aarch64); else return "all"; } void extract_arch_from_e_machine_field(int16_t e_machine, const gchar* sourcename, bool* archs) { if (e_machine == 3) { - archs[fARCH_i386] = 1; + archs[fARCH_i686] = 1; if(verbose) - fprintf(stderr, "%s used for determining architecture i386\n", sourcename); + fprintf(stderr, "%s used for determining architecture %s\n", sourcename, archToName(fARCH_i686)); } if (e_machine == 62) { archs[fARCH_x86_64] = 1; if(verbose) - fprintf(stderr, "%s used for determining architecture x86_64\n", sourcename); + fprintf(stderr, "%s used for determining architecture %s\n", sourcename, archToName(fARCH_x86_64)); } if (e_machine == 40) { - archs[fARCH_arm] = 1; + archs[fARCH_armhf] = 1; if(verbose) - fprintf(stderr, "%s used for determining architecture armhf\n", sourcename); + fprintf(stderr, "%s used for determining architecture %s\n", sourcename, archToName(fARCH_armhf)); } if (e_machine == 183) { archs[fARCH_aarch64] = 1; if(verbose) - fprintf(stderr, "%s used for determining architecture aarch64\n", sourcename); + fprintf(stderr, "%s used for determining architecture %s\n", sourcename, archToName(fARCH_aarch64)); } } @@ -340,7 +354,7 @@ void extract_arch_from_text(gchar *archname, const gchar* sourcename, bool* arch || g_ascii_strncasecmp("intel_80586", archname, 20) == 0 || g_ascii_strncasecmp("intel_80686", archname, 20) == 0 ) { - archs[fARCH_i386] = 1; + archs[fARCH_i686] = 1; if (verbose) fprintf(stderr, "%s used for determining architecture i386\n", sourcename); } else if (g_ascii_strncasecmp("x86_64", archname, 20) == 0) { @@ -348,7 +362,7 @@ void extract_arch_from_text(gchar *archname, const gchar* sourcename, bool* arch if (verbose) fprintf(stderr, "%s used for determining architecture x86_64\n", sourcename); } else if (g_ascii_strncasecmp("arm", archname, 20) == 0) { - archs[fARCH_arm] = 1; + archs[fARCH_armhf] = 1; if (verbose) fprintf(stderr, "%s used for determining architecture ARM\n", sourcename); } else if (g_ascii_strncasecmp("arm_aarch64", archname, 20) == 0) { @@ -429,7 +443,7 @@ gchar* get_desktop_entry(GKeyFile *kf, char *key) { return value; } -bool readFile(char* filename, int* size, char** buffer) { +bool readFile(char* filename, size_t* size, char** buffer) { FILE* f = fopen(filename, "rb"); if (f==NULL) { *buffer = 0; @@ -858,22 +872,17 @@ main (int argc, char *argv[]) * should hopefully change that. */ fprintf (stderr, "Generating squashfs...\n"); - int size = 0; + size_t size = 0; char* data = NULL; - bool using_external_data = false; + // TODO: just write to the output file directly, we don't really need a memory buffer if (runtime_file != NULL) { - if (!readFile(runtime_file, &size, &data)) + if (!readFile(runtime_file, &size, &data)) { die("Unable to load provided runtime file"); - using_external_data = true; + } } else { -#ifdef HAVE_BINARY_RUNTIME - /* runtime is embedded into this executable - * http://stupefydeveloper.blogspot.de/2008/08/cc-embed-binary-data-into-elf.html */ - size = runtime_len; - data = runtime; -#else - die("No runtime file was provided"); -#endif + if (!fetch_runtime(arch, &size, &data, verbose)) { + die("Failed to download runtime file"); + } } if (verbose) printf("Size of the embedded runtime: %d bytes\n", size); @@ -891,8 +900,8 @@ main (int argc, char *argv[]) fseek(fpdst, 0, SEEK_SET); fwrite(data, size, 1, fpdst); fclose(fpdst); - if (using_external_data) - free(data); + // TODO: avoid memory buffer (see above) + free(data); fprintf (stderr, "Marking the AppImage as executable...\n"); if (chmod (destination, 0755) < 0) { diff --git a/src/appimagetool_fetch_runtime.c b/src/appimagetool_fetch_runtime.c new file mode 100644 index 0000000..7c16a13 --- /dev/null +++ b/src/appimagetool_fetch_runtime.c @@ -0,0 +1,155 @@ +// need to define this to enable asprintf +#define _GNU_SOURCE +#include +#include + +#include +#include +#include +#include + +#include "appimagetool_fetch_runtime.h" + +bool fetch_runtime(char* arch, size_t* size, char** buffer, bool verbose) { + // not the cleanest approach to globally init curl here, but this method shouldn't be called more than once anyway + curl_global_init(CURL_GLOBAL_ALL); + + char* url = NULL; + int url_size = asprintf(&url, "https://github.com/AppImage/type2-runtime/releases/download/continuous/runtime-%s", arch); + if (url_size <= 0) { + fprintf(stderr, "Failed to generate runtime URL\n"); + curl_global_cleanup(); + return false; + } + + fprintf(stderr, "Downloading runtime file from %s\n", url); + + char curl_error_buf[CURL_ERROR_SIZE]; + CURL* handle = NULL; + char* effective_url; + int success = -1L; + + curl_off_t content_length = -1; + + // first, we perform a HEAD request to determine the required buffer size to write the file to + // of course, this assumes that a) GitHub sends a Content-Length header and b) that it is correct and will be in + // the GET request, too + // we store the URL we are redirected to (which probably lies on some AWS and is unique to that file) for use in + // the GET request, which should ensure we really download the file whose size we check now + handle = curl_easy_init(); + + if (handle == NULL) { + fprintf(stderr, "Failed to initialize libcurl\n"); + curl_global_cleanup(); + return false; + } + + curl_easy_setopt(handle, CURLOPT_URL, url); + curl_easy_setopt(handle, CURLOPT_FOLLOWLOCATION, 1L); + // should be plenty for GitHub + curl_easy_setopt(handle, CURLOPT_MAXREDIRS, 12L); + curl_easy_setopt(handle, CURLOPT_NOBODY, 1L); + curl_easy_setopt(handle, CURLOPT_ERRORBUFFER, curl_error_buf); + if (verbose) { + curl_easy_setopt(handle, CURLOPT_VERBOSE, 1L); + } + + success = curl_easy_perform(handle); + + if (success == CURLE_OK) { + if (curl_easy_getinfo(handle, CURLINFO_EFFECTIVE_URL, &effective_url) == CURLE_OK) { + if (strcmp(url, effective_url) != 0) { + fprintf(stderr, "Redirected to %s\n", effective_url); + } + } else { + fprintf(stderr, "Error: failed to determine effective URL\n"); + // we recycle the cleanup call below and check whether effective_url was set to anything meaningful below + } + + if (curl_easy_getinfo(handle, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, &content_length) == CURLE_OK) { + fprintf(stderr, "Downloading runtime binary of size %" CURL_FORMAT_CURL_OFF_T "\n", content_length); + } else { + fprintf(stderr, "Error: no Content-Length header sent by GitHub\n"); + // we recycle the cleanup call below and check whether content_length was set to anything meaningful below + } + } else { + fprintf(stderr, "HEAD request to %s failed: %s\n", url, curl_error_buf); + } + + curl_easy_cleanup(handle); + handle = NULL; + + if (success != CURLE_OK || effective_url == NULL || content_length <= 0) { + curl_global_cleanup(); + return false; + } + + // now that we know the required buffer size, we allocate a suitable in-memory buffer and perform the GET request + // we allocate our own so that we don't have to use fread(...) to get the data + char raw_buffer[content_length]; + FILE* file_buffer = fmemopen(raw_buffer, sizeof(raw_buffer), "w+b"); + setbuf(file_buffer, NULL); + + if (file_buffer == NULL) { + fprintf(stderr, "fmemopen failed: %s\n", strerror(errno)); + curl_global_cleanup(); + return false; + } + + handle = curl_easy_init(); + + if (handle == NULL) { + fprintf(stderr, "Failed to initialize libcurl\n"); + curl_global_cleanup(); + return false; + } + + int blub; + + // note: we should not need any more redirects + blub = curl_easy_setopt(handle, CURLOPT_URL, effective_url); + blub = curl_easy_setopt(handle, CURLOPT_WRITEDATA, (void*) file_buffer); + blub = curl_easy_setopt(handle, CURLOPT_ERRORBUFFER, curl_error_buf); + if (verbose) { + curl_easy_setopt(handle, CURLOPT_VERBOSE, 1L); + } + + success = curl_easy_perform(handle); + + int get_content_length; + + if (success != CURLE_OK) { + fprintf(stderr, "GET request to %s failed: %s\n", effective_url, curl_error_buf); + curl_easy_cleanup(handle); + curl_global_cleanup(); + return false; + } else { + if (curl_easy_getinfo(handle, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, &get_content_length) != CURLE_OK) { + fprintf(stderr, "Error: no Content-Length header sent by GitHub\n"); + // we recycle the cleanup call below and check whether content_length was set to anything meaningful below + } + } + + curl_easy_cleanup(handle); + handle = NULL; + + // done with libcurl + curl_global_cleanup(); + + if (get_content_length != content_length) { + fprintf(stderr, "Downloading runtime binary of size %" CURL_FORMAT_CURL_OFF_T "\n", content_length); + } + + *size = content_length; + + *buffer = (void*) calloc(content_length + 1, 1); + + if (*buffer == NULL) { + fprintf(stderr, "Failed to allocate buffer\n"); + return false; + } + + memcpy(*buffer, raw_buffer, sizeof(raw_buffer)); + + return true; +} diff --git a/src/appimagetool_fetch_runtime.h b/src/appimagetool_fetch_runtime.h new file mode 100644 index 0000000..4110f9b --- /dev/null +++ b/src/appimagetool_fetch_runtime.h @@ -0,0 +1,7 @@ +#pragma once + +/** + * Download runtime from GitHub into a buffer. + * This function allocates a buffer of the right size internally, which needs to be cleaned up by the caller. + */ +bool fetch_runtime(char* arch, size_t* size, char** buffer, bool verbose); From 10dec875bebba3fd65a34be08079dd20200f3bc9 Mon Sep 17 00:00:00 2001 From: TheAssassin Date: Thu, 6 Jul 2023 22:49:04 +0200 Subject: [PATCH 2/3] fixup! Download suitable runtime from GitHub automatically --- src/appimagetool_fetch_runtime.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/appimagetool_fetch_runtime.c b/src/appimagetool_fetch_runtime.c index 7c16a13..092b9d3 100644 --- a/src/appimagetool_fetch_runtime.c +++ b/src/appimagetool_fetch_runtime.c @@ -104,12 +104,10 @@ bool fetch_runtime(char* arch, size_t* size, char** buffer, bool verbose) { return false; } - int blub; - // note: we should not need any more redirects - blub = curl_easy_setopt(handle, CURLOPT_URL, effective_url); - blub = curl_easy_setopt(handle, CURLOPT_WRITEDATA, (void*) file_buffer); - blub = curl_easy_setopt(handle, CURLOPT_ERRORBUFFER, curl_error_buf); + curl_easy_setopt(handle, CURLOPT_URL, effective_url); + curl_easy_setopt(handle, CURLOPT_WRITEDATA, (void*) file_buffer); + curl_easy_setopt(handle, CURLOPT_ERRORBUFFER, curl_error_buf); if (verbose) { curl_easy_setopt(handle, CURLOPT_VERBOSE, 1L); } From 554110ec8cfb4bb9162177c49f2947bfa0f73b30 Mon Sep 17 00:00:00 2001 From: TheAssassin Date: Thu, 6 Jul 2023 23:02:05 +0200 Subject: [PATCH 3/3] fixup! fixup! Download suitable runtime from GitHub automatically --- src/appimagetool_fetch_runtime.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/appimagetool_fetch_runtime.c b/src/appimagetool_fetch_runtime.c index 092b9d3..ad51f83 100644 --- a/src/appimagetool_fetch_runtime.c +++ b/src/appimagetool_fetch_runtime.c @@ -14,9 +14,10 @@ bool fetch_runtime(char* arch, size_t* size, char** buffer, bool verbose) { // not the cleanest approach to globally init curl here, but this method shouldn't be called more than once anyway curl_global_init(CURL_GLOBAL_ALL); - char* url = NULL; - int url_size = asprintf(&url, "https://github.com/AppImage/type2-runtime/releases/download/continuous/runtime-%s", arch); - if (url_size <= 0) { + // should be plenty big for the URL + char url[1024]; + int url_size = snprintf(url, sizeof(url), "https://github.com/AppImage/type2-runtime/releases/download/continuous/runtime-%s", arch); + if (url_size <= 0 || url_size >= sizeof(url)) { fprintf(stderr, "Failed to generate runtime URL\n"); curl_global_cleanup(); return false; @@ -140,14 +141,14 @@ bool fetch_runtime(char* arch, size_t* size, char** buffer, bool verbose) { *size = content_length; - *buffer = (void*) calloc(content_length + 1, 1); + *buffer = (char*) calloc(content_length + 1, 1); if (*buffer == NULL) { fprintf(stderr, "Failed to allocate buffer\n"); return false; } - memcpy(*buffer, raw_buffer, sizeof(raw_buffer)); + memcpy((void* ) *buffer, raw_buffer, sizeof(raw_buffer)); return true; }