Download suitable runtime from GitHub automatically

CMakeLists.txt

@@ -78,6 +78,7 @@ pkg_check_modules(libgcrypt REQUIRED libgcrypt IMPORTED_TARGET)
 # TODO: in the long term, we plan to get rid of GLib/GIO by moving to C++
 pkg_check_modules(libglib REQUIRED glib-2.0 IMPORTED_TARGET)
 pkg_check_modules(libgio REQUIRED gio-2.0 IMPORTED_TARGET)
+pkg_check_modules(libcurl REQUIRED libcurl IMPORTED_TARGET)
 
 # Alpine Linux does not ship an argp.h as part of the standard compiler toolchain
 unset(ARGP_H_FOUND CACHE)

ci/build-in-docker.sh

@@ -65,7 +65,15 @@ set -euxo pipefail
 
 apk add bash git gcc g++ cmake make file desktop-file-utils wget \
     gpgme-dev libgcrypt-dev libgcrypt-static argp-standalone zstd-dev zstd-static util-linux-static \
-    glib-static libassuan-static zlib-static libgpg-error-static
+    glib-static libassuan-static zlib-static libgpg-error-static \
+    curl-dev curl-static nghttp2-static libidn2-static openssl-libs-static brotli-static c-ares-static libunistring-static
+
+# libcurl's pkg-config scripts are broken. everywhere, everytime.
+# these additional flags have been collected from all the .pc files whose libs are mentioned as -l<lib> in Libs.private
+# first, let's make sure there is no Requires.private section
+grep -qv Requires.private /usr/lib/pkgconfig/libcurl.pc
+# now, let's add one
+echo "Requires.private: libcares libnghttp2 libidn2 libssl openssl libcrypto libbrotlicommon zlib" | tee -a /usr/lib/pkgconfig/libcurl.pc
 
 # in a Docker container, we can safely disable this check
 git config --global --add safe.directory '*'

src/CMakeLists.txt

@@ -1,6 +1,7 @@
 add_executable(appimagetool
     appimagetool.c
     appimagetool_sign.c
+    appimagetool_fetch_runtime.c
     hexlify.c
     elf.c
     digest.c
@@ -15,6 +16,7 @@ target_link_libraries(appimagetool
     PkgConfig::libgio
     PkgConfig::libgcrypt
     PkgConfig::libgpgme
+    PkgConfig::libcurl
 )
 
 target_compile_definitions(appimagetool

src/appimagetool.c

@@ -55,14 +55,15 @@
 
 #include "util.h"
 
+#include "appimagetool_fetch_runtime.h"
 #include "appimagetool_sign.h"
 
-enum fARCH { 
-    fARCH_i386,
+typedef enum {
+    fARCH_i686,
     fARCH_x86_64,
-    fARCH_arm,
+    fARCH_armhf,
     fARCH_aarch64
-};
+} fARCH;
 
 static gchar const APPIMAGEIGNORE[] = ".appimageignore";
 static char _exclude_file_desc[256];
@@ -286,42 +287,55 @@ int count_archs(bool* archs) {
     return countArchs;
 }
 
+gchar* archToName(fARCH arch) {
+    switch (arch) {
+        case fARCH_aarch64:
+            return "aarch64";
+        case fARCH_armhf:
+            return "armhf";
+        case fARCH_i686:
+            return "i686";
+        case fARCH_x86_64:
+            return "x86_64";
+    }
+}
+
 gchar* getArchName(bool* archs) {
-    if (archs[fARCH_i386])
-        return "i386";
+    if (archs[fARCH_i686])
+        return archToName(fARCH_i686);
     else if (archs[fARCH_x86_64])
-        return "x86_64";
-    else if (archs[fARCH_arm])
-        return "armhf";
+        return archToName(fARCH_x86_64);
+    else if (archs[fARCH_armhf])
+        return archToName(fARCH_armhf);
     else if (archs[fARCH_aarch64])
-        return "aarch64";
+        return archToName(fARCH_aarch64);
     else
         return "all";
 }
 
 void extract_arch_from_e_machine_field(int16_t e_machine, const gchar* sourcename, bool* archs) {
     if (e_machine == 3) {
-        archs[fARCH_i386] = 1;
+        archs[fARCH_i686] = 1;
         if(verbose)
-            fprintf(stderr, "%s used for determining architecture i386\n", sourcename);
+            fprintf(stderr, "%s used for determining architecture %s\n", sourcename, archToName(fARCH_i686));
     }
 
     if (e_machine == 62) {
         archs[fARCH_x86_64] = 1;
         if(verbose)
-            fprintf(stderr, "%s used for determining architecture x86_64\n", sourcename);
+            fprintf(stderr, "%s used for determining architecture %s\n", sourcename, archToName(fARCH_x86_64));
     }
 
     if (e_machine == 40) {
-        archs[fARCH_arm] = 1;
+        archs[fARCH_armhf] = 1;
         if(verbose)
-            fprintf(stderr, "%s used for determining architecture armhf\n", sourcename);
+            fprintf(stderr, "%s used for determining architecture %s\n", sourcename, archToName(fARCH_armhf));
     }
 
     if (e_machine == 183) {
         archs[fARCH_aarch64] = 1;
         if(verbose)
-            fprintf(stderr, "%s used for determining architecture aarch64\n", sourcename);
+            fprintf(stderr, "%s used for determining architecture %s\n", sourcename, archToName(fARCH_aarch64));
     }
 }
 
@@ -340,15 +354,15 @@ void extract_arch_from_text(gchar *archname, const gchar* sourcename, bool* arch
                     || g_ascii_strncasecmp("intel_80586", archname, 20) == 0
                     || g_ascii_strncasecmp("intel_80686", archname, 20) == 0
                     ) {
-                archs[fARCH_i386] = 1;
+                archs[fARCH_i686] = 1;
                 if (verbose)
                     fprintf(stderr, "%s used for determining architecture i386\n", sourcename);
             } else if (g_ascii_strncasecmp("x86_64", archname, 20) == 0) {
                 archs[fARCH_x86_64] = 1;
                 if (verbose)
                     fprintf(stderr, "%s used for determining architecture x86_64\n", sourcename);
             } else if (g_ascii_strncasecmp("arm", archname, 20) == 0) {
-                archs[fARCH_arm] = 1;
+                archs[fARCH_armhf] = 1;
                 if (verbose)
                     fprintf(stderr, "%s used for determining architecture ARM\n", sourcename);
             } else if (g_ascii_strncasecmp("arm_aarch64", archname, 20) == 0) {
@@ -429,7 +443,7 @@ gchar* get_desktop_entry(GKeyFile *kf, char *key) {
     return value;
 }
 
-bool readFile(char* filename, int* size, char** buffer) {
+bool readFile(char* filename, size_t* size, char** buffer) {
     FILE* f = fopen(filename, "rb");
     if (f==NULL) {
         *buffer = 0;
@@ -858,22 +872,17 @@ main (int argc, char *argv[])
         * should hopefully change that. */
 
         fprintf (stderr, "Generating squashfs...\n");
-        int size = 0;
+        size_t size = 0;
         char* data = NULL;
-        bool using_external_data = false;
+        // TODO: just write to the output file directly, we don't really need a memory buffer
         if (runtime_file != NULL) {
-            if (!readFile(runtime_file, &size, &data))
+            if (!readFile(runtime_file, &size, &data)) {
                 die("Unable to load provided runtime file");
-            using_external_data = true;
+            }
         } else {
-#ifdef HAVE_BINARY_RUNTIME
-            /* runtime is embedded into this executable
-            * http://stupefydeveloper.blogspot.de/2008/08/cc-embed-binary-data-into-elf.html */
-            size = runtime_len;
-            data = runtime;
-#else
-            die("No runtime file was provided");
-#endif
+            if (!fetch_runtime(arch, &size, &data, verbose)) {
+                die("Failed to download runtime file");
+            }
         }
         if (verbose)
             printf("Size of the embedded runtime: %d bytes\n", size);
@@ -891,8 +900,8 @@ main (int argc, char *argv[])
         fseek(fpdst, 0, SEEK_SET);
         fwrite(data, size, 1, fpdst);
         fclose(fpdst);
-        if (using_external_data)
-            free(data);
+        // TODO: avoid memory buffer (see above)
+        free(data);
 
         fprintf (stderr, "Marking the AppImage as executable...\n");
         if (chmod (destination, 0755) < 0) {

src/appimagetool_fetch_runtime.c

@@ -0,0 +1,154 @@
+// need to define this to enable asprintf
+#define _GNU_SOURCE
+#include <stdbool.h>
+#include <stdio.h>
+
+#include <curl/curl.h>
+#include <malloc.h>
+#include <string.h>
+#include <errno.h>
+
+#include "appimagetool_fetch_runtime.h"
+
+bool fetch_runtime(char* arch, size_t* size, char** buffer, bool verbose) {
+    // not the cleanest approach to globally init curl here, but this method shouldn't be called more than once anyway
+    curl_global_init(CURL_GLOBAL_ALL);
+
+    // should be plenty big for the URL
+    char url[1024];
+    int url_size = snprintf(url, sizeof(url), "https://github.com/AppImage/type2-runtime/releases/download/continuous/runtime-%s", arch);
+    if (url_size <= 0 || url_size >= sizeof(url)) {
+        fprintf(stderr, "Failed to generate runtime URL\n");
+        curl_global_cleanup();
+        return false;
+    }
+
+    fprintf(stderr, "Downloading runtime file from %s\n", url);
+
+    char curl_error_buf[CURL_ERROR_SIZE];
+    CURL* handle = NULL;
+    char* effective_url;
+    int success = -1L;
+
+    curl_off_t content_length = -1;
+
+    // first, we perform a HEAD request to determine the required buffer size to write the file to
+    // of course, this assumes that a) GitHub sends a Content-Length header and b) that it is correct and will be in
+    // the GET request, too
+    // we store the URL we are redirected to (which probably lies on some AWS and is unique to that file) for use in
+    // the GET request, which should ensure we really download the file whose size we check now
+    handle = curl_easy_init();
+
+    if (handle == NULL) {
+        fprintf(stderr, "Failed to initialize libcurl\n");
+        curl_global_cleanup();
+        return false;
+    }
+
+    curl_easy_setopt(handle, CURLOPT_URL, url);
+    curl_easy_setopt(handle, CURLOPT_FOLLOWLOCATION, 1L);
+    // should be plenty for GitHub
+    curl_easy_setopt(handle, CURLOPT_MAXREDIRS, 12L);
+    curl_easy_setopt(handle, CURLOPT_NOBODY, 1L);
+    curl_easy_setopt(handle, CURLOPT_ERRORBUFFER, curl_error_buf);
+    if (verbose) {
+        curl_easy_setopt(handle, CURLOPT_VERBOSE, 1L);
+    }
+
+    success = curl_easy_perform(handle);
+
+    if (success == CURLE_OK) {
+        if (curl_easy_getinfo(handle, CURLINFO_EFFECTIVE_URL, &effective_url) == CURLE_OK) {
+            if (strcmp(url, effective_url) != 0) {
+                fprintf(stderr, "Redirected to %s\n", effective_url);
+            }
+        } else {
+            fprintf(stderr, "Error: failed to determine effective URL\n");
+            // we recycle the cleanup call below and check whether effective_url was set to anything meaningful below
+        }
+
+        if (curl_easy_getinfo(handle, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, &content_length) == CURLE_OK) {
+            fprintf(stderr, "Downloading runtime binary of size %" CURL_FORMAT_CURL_OFF_T "\n", content_length);
+        } else {
+            fprintf(stderr, "Error: no Content-Length header sent by GitHub\n");
+            // we recycle the cleanup call below and check whether content_length was set to anything meaningful below
+        }
+    } else {
+        fprintf(stderr, "HEAD request to %s failed: %s\n", url, curl_error_buf);
+    }
+
+    curl_easy_cleanup(handle);
+    handle = NULL;
+
+    if (success != CURLE_OK || effective_url == NULL || content_length <= 0) {
+        curl_global_cleanup();
+        return false;
+    }
+
+    // now that we know the required buffer size, we allocate a suitable in-memory buffer and perform the GET request
+    // we allocate our own so that we don't have to use fread(...) to get the data
+    char raw_buffer[content_length];
+    FILE* file_buffer = fmemopen(raw_buffer, sizeof(raw_buffer), "w+b");
+    setbuf(file_buffer, NULL);
+
+    if (file_buffer == NULL) {
+        fprintf(stderr, "fmemopen failed: %s\n", strerror(errno));
+        curl_global_cleanup();
+        return false;
+    }
+
+    handle = curl_easy_init();
+
+    if (handle == NULL) {
+        fprintf(stderr, "Failed to initialize libcurl\n");
+        curl_global_cleanup();
+        return false;
+    }
+
+    // note: we should not need any more redirects
+    curl_easy_setopt(handle, CURLOPT_URL, effective_url);
+    curl_easy_setopt(handle, CURLOPT_WRITEDATA, (void*) file_buffer);
+    curl_easy_setopt(handle, CURLOPT_ERRORBUFFER, curl_error_buf);
+    if (verbose) {
+        curl_easy_setopt(handle, CURLOPT_VERBOSE, 1L);
+    }
+
+    success = curl_easy_perform(handle);
+
+    int get_content_length;
+
+    if (success != CURLE_OK) {
+        fprintf(stderr, "GET request to %s failed: %s\n", effective_url, curl_error_buf);
+        curl_easy_cleanup(handle);
+        curl_global_cleanup();
+        return false;
+    } else {
+        if (curl_easy_getinfo(handle, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, &get_content_length) != CURLE_OK) {
+            fprintf(stderr, "Error: no Content-Length header sent by GitHub\n");
+            // we recycle the cleanup call below and check whether content_length was set to anything meaningful below
+        }
+    }
+
+    curl_easy_cleanup(handle);
+    handle = NULL;
+
+    // done with libcurl
+    curl_global_cleanup();
+
+    if (get_content_length != content_length) {
+        fprintf(stderr, "Downloading runtime binary of size %" CURL_FORMAT_CURL_OFF_T "\n", content_length);
+    }
+
+    *size = content_length;
+
+    *buffer = (char*) calloc(content_length + 1, 1);
+
+    if (*buffer == NULL) {
+        fprintf(stderr, "Failed to allocate buffer\n");
+        return false;
+    }
+
+    memcpy((void* ) *buffer, raw_buffer, sizeof(raw_buffer));
+
+    return true;
+}

src/appimagetool_fetch_runtime.h

@@ -0,0 +1,7 @@
+#pragma once
+
+/**
+ * Download runtime from GitHub into a buffer.
+ * This function allocates a buffer of the right size internally, which needs to be cleaned up by the caller.
+ */
+bool fetch_runtime(char* arch, size_t* size, char** buffer, bool verbose);