From 1b513e0ed6198b3dba6fd6e5565bfb03778483b5 Mon Sep 17 00:00:00 2001 From: John Parent Date: Sat, 19 Oct 2024 01:06:36 -0400 Subject: [PATCH] Gpt4all Custom Updater This PR establishes the initial infrastructure required to migrate gpt4all away from reliance on the IFW framwork by implementing a custom updater. This custom updater (currently headless only) can perform the same operations as the existing updater with the option for more functionality to be added and most importantly, is installer agnostic, meaning gpt4all can start to leverage platform specific installers. Implements both offline and online installation and update mechanisms. Initial implementation of: https://github.com/nomic-ai/gpt4all/issues/2878 --- gpt4all-updater/CMakeLists.txt | 73 ++++++ gpt4all-updater/README | 30 +++ gpt4all-updater/include/Command.h | 18 ++ gpt4all-updater/include/CommandFactory.h | 12 + gpt4all-updater/include/CommandLine.h | 30 +++ gpt4all-updater/include/Downgrade.h | 17 ++ gpt4all-updater/include/Download.h | 64 ++++++ gpt4all-updater/include/Embeded.h | 35 +++ gpt4all-updater/include/Manifest.h | 46 ++++ gpt4all-updater/include/Modify.h | 15 ++ gpt4all-updater/include/Package.h | 29 +++ gpt4all-updater/include/Resource.h | 18 ++ gpt4all-updater/include/State.h | 35 +++ gpt4all-updater/include/Uninstall.h | 16 ++ gpt4all-updater/include/Update.h | 15 ++ gpt4all-updater/include/utils.h | 5 + gpt4all-updater/src/Command.cxx | 10 + gpt4all-updater/src/CommandFactory.cxx | 31 +++ gpt4all-updater/src/CommandLine.cxx | 54 +++++ gpt4all-updater/src/Downgrade.cxx | 9 + gpt4all-updater/src/Download.cxx.in | 216 ++++++++++++++++++ gpt4all-updater/src/Embeded.cxx | 32 +++ gpt4all-updater/src/Manifest.cxx | 81 +++++++ gpt4all-updater/src/Modify.cxx | 9 + gpt4all-updater/src/Package.cxx | 29 +++ gpt4all-updater/src/Resource.cxx | 38 +++ gpt4all-updater/src/State.cxx | 73 ++++++ gpt4all-updater/src/Uninstall.cxx | 11 + gpt4all-updater/src/Update.cxx | 11 + gpt4all-updater/src/asm/bake_installer.S.in | 8 + gpt4all-updater/src/main.cxx | 19 ++ gpt4all-updater/src/resources/installer.rc.in | 1 + gpt4all-updater/src/utils.cxx | 12 + gpt4all-updater/tmp/manifest.xml | 16 ++ 34 files changed, 1118 insertions(+) create mode 100644 gpt4all-updater/CMakeLists.txt create mode 100644 gpt4all-updater/README create mode 100644 gpt4all-updater/include/Command.h create mode 100644 gpt4all-updater/include/CommandFactory.h create mode 100644 gpt4all-updater/include/CommandLine.h create mode 100644 gpt4all-updater/include/Downgrade.h create mode 100644 gpt4all-updater/include/Download.h create mode 100644 gpt4all-updater/include/Embeded.h create mode 100644 gpt4all-updater/include/Manifest.h create mode 100644 gpt4all-updater/include/Modify.h create mode 100644 gpt4all-updater/include/Package.h create mode 100644 gpt4all-updater/include/Resource.h create mode 100644 gpt4all-updater/include/State.h create mode 100644 gpt4all-updater/include/Uninstall.h create mode 100644 gpt4all-updater/include/Update.h create mode 100644 gpt4all-updater/include/utils.h create mode 100644 gpt4all-updater/src/Command.cxx create mode 100644 gpt4all-updater/src/CommandFactory.cxx create mode 100644 gpt4all-updater/src/CommandLine.cxx create mode 100644 gpt4all-updater/src/Downgrade.cxx create mode 100644 gpt4all-updater/src/Download.cxx.in create mode 100644 gpt4all-updater/src/Embeded.cxx create mode 100644 gpt4all-updater/src/Manifest.cxx create mode 100644 gpt4all-updater/src/Modify.cxx create mode 100644 gpt4all-updater/src/Package.cxx create mode 100644 gpt4all-updater/src/Resource.cxx create mode 100644 gpt4all-updater/src/State.cxx create mode 100644 gpt4all-updater/src/Uninstall.cxx create mode 100644 gpt4all-updater/src/Update.cxx create mode 100644 gpt4all-updater/src/asm/bake_installer.S.in create mode 100644 gpt4all-updater/src/main.cxx create mode 100644 gpt4all-updater/src/resources/installer.rc.in create mode 100644 gpt4all-updater/src/utils.cxx create mode 100644 gpt4all-updater/tmp/manifest.xml diff --git a/gpt4all-updater/CMakeLists.txt b/gpt4all-updater/CMakeLists.txt new file mode 100644 index 000000000000..1b734f6f87d3 --- /dev/null +++ b/gpt4all-updater/CMakeLists.txt @@ -0,0 +1,73 @@ +cmake_minimum_required(VERSION 3.16) + +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +set(APP_VERSION_MAJOR 0) +set(APP_VERSION_MINOR 0) +set(APP_VERSION_PATCH 0) + +set(APP_VERSION ${APP_VERSION_MAJOR}.${APP_VERSION_MINOR}.${APP_VERSION_PATCH}) + +project(Gpt4AllAutoUpdater VERSION ${APP_VERSION} LANGUAGES CXX) + +option(BUILD_OFFLINE_UPDATER "Build an offline updater" OFF) + +if(BUILD_OFFLINE_UPDATER AND NOT GPT4ALL_INSTALLER_PATH) + message(FATAL_ERROR "The path to GPT4ALL's installer is required to construct this updater. +Please provide it on the command line using the argument -DGPT4ALL_INSTALLER_PATH=") +endif() + +if(NOT BUILD_OFFLINE_UPDATER AND NOT GPT4ALL_MANIFEST_ENDPOINT) + message(FATAL_ERROR "The manifest endpoint was not provided, the online installer will be unable to detect updates") +endif() + +if(APPLE) + option(BUILD_UNIVERSAL "Build universal binary on MacOS" OFF) + if(BUILD_UNIVERSAL) + set(CMAKE_OSX_ARCHITECTURES "arm64;x86_64" CACHE STRING "" FORCE) + else() + set(CMAKE_OSX_ARCHITECTURES "${CMAKE_HOST_SYSTEM_PROCESSOR}" CACHE STRING "" FORCE) + endif() +endif() + +if(APPLE AND BUILD_OFFLINE_UPDATER) + enable_language(ASM) + configure_file(src/asm/bake_installer.S.in ${CMAKE_BINARY_DIR}/bake_installer.S @ONLY) + set(ASSEMBLER_SOURCES ${CMAKE_BINARY_DIR}/bake_installer.S src/Embeded.cxx) +elseif(WIN32 AND BUILD_OFFLINE_UPDATER) + configure_file(src/resources/installer.rc.in ${CMAKE_BINARY_DIR}/resource.rc @ONLY) + set(RC_FILES ${CMAKE_BINARY_DIR}/resource.rc src/Resource.cxx) +endif() + +if(NOT BUILD_OFFLINE_UPDATER) + configure_file(src/Download.cxx.in ${CMAKE_BINARY_DIR}/Download.cxx @ONLY) +endif() + +find_package(Qt6 REQUIRED COMPONENTS Core Network) + +set(auto_updater_sources + src/Command.cxx + src/CommandFactory.cxx + src/CommandLine.cxx + src/Downgrade.cxx + ${CMAKE_BINARY_DIR}/Download.cxx + src/Manifest.cxx + src/Modify.cxx + src/Package.cxx + src/State.cxx + src/Uninstall.cxx + src/Update.cxx + src/utils.cxx + src/main.cxx + ${ASSEMBLER_SOURCES} + ${RC_FILES} +) + +add_executable(autoupdater ${auto_updater_sources}) +target_link_libraries(autoupdater PRIVATE Qt6::Core Qt6::Network) +target_include_directories(autoupdater PRIVATE include) + +if(BUILD_OFFLINE_UPDATER) + target_compile_definitions(autoupdater PRIVATE OFFLINE) +endif() diff --git a/gpt4all-updater/README b/gpt4all-updater/README new file mode 100644 index 000000000000..dd845cf08ebb --- /dev/null +++ b/gpt4all-updater/README @@ -0,0 +1,30 @@ +# Gpt4All Updater + +## Testing + +Testing this updater requires a bit of manual setup and walking through the +integration with gpt4all's installer, as this process has yet to be fully automated. + +There are two modes, offline and online, each is tested differently: + +### Online + +The online updater workflow takes a url endpoint (for early testing, this will be a file url) that points to a manifest file, a sample of which has been provided under the `tmp` directory relative to this README. The manifest file includes, amount other things specified in the updater RFC, another file URL to a gpt4all installer. + +The first thing testing this will require is a manual update of that file url in the mainfest xml file to point to the DMG on MacOS or exe on Windows, of an existing online Gpt4All installer, as well as a corresponding sha256 sum of the given installer. The manifest is filled with other stub info for testing, you're welcome to leave it as is or fill it out correctly for each testing iteration. + +That is all that is required for configuration. Now simply build this project via CMake, using standard CMake build practices. CMake will build the online installer by default, so no extra arguments are required. + +One argument is required for the online installer, the url where the updater should expect the mainfest file to be. This can be any url accessible to your system, but for simplicity and testing reasons, its usually best to use a file url. +This is provided via the cmake command line argument `-DGPT4ALL_MANIFEST_ENDPOINT=`. + +Now configure and build the updater with CMake. + +To test the installer, query the command line interface using the `--help` argument to determine which actions are available. Then select a given action, and provide the required arguments (there shouldn't be any at the moment), and let the updater drive. The updater will determine the operation requested, fetch the appropriate installer, and drive said installer with the appropriate arguments to effect the requested operation. If the operation is a modification or uninstall, the updater will not fetch a new installer, and instead will execute the older installer, as a new installer is not required. + +### Offline + +The offline updater is somewhat simpler. To instruct CMake to build the offline updater, specify the `-DBUILD_OFFLINE_UPDATER=ON` argument to CMake on the command line. One additional argument is required to properly configure CMake for the offline updater project, `-DGPT4ALL_INSTALLER_PATH` which should be set to the path to the DMG file containing an offline version of the GPT4All installer. + +Now, after building, simply run the updater. It should remove your current installation of Gpt4All (but not the config or models), and then run the offline installer in install mode. Once that process is complete, you should have an upgraded Gpt4All available on your system. + diff --git a/gpt4all-updater/include/Command.h b/gpt4all-updater/include/Command.h new file mode 100644 index 000000000000..60b8ce96f036 --- /dev/null +++ b/gpt4all-updater/include/Command.h @@ -0,0 +1,18 @@ +#pragma once +#include +#include +#include +#include + +namespace gpt4all { +namespace command{ + +class Command +{ +public: + virtual bool execute(); + QStringList arguments; + QFile* installer; +}; +} +} \ No newline at end of file diff --git a/gpt4all-updater/include/CommandFactory.h b/gpt4all-updater/include/CommandFactory.h new file mode 100644 index 000000000000..c97bdf711d12 --- /dev/null +++ b/gpt4all-updater/include/CommandFactory.h @@ -0,0 +1,12 @@ +#pragma once + +#include "Command.h" +#include "CommandLine.h" + + + +class CommandFactory : public QObject +{ +public: + std::shared_ptr GetCommand(gpt4all::command::CommandType type); +}; \ No newline at end of file diff --git a/gpt4all-updater/include/CommandLine.h b/gpt4all-updater/include/CommandLine.h new file mode 100644 index 000000000000..9f8c471ae0e8 --- /dev/null +++ b/gpt4all-updater/include/CommandLine.h @@ -0,0 +1,30 @@ +#pragma once + +#include + + +namespace gpt4all { +namespace command { + +enum CommandType { + UPDATE, + MODIFY, + DOWNGRADE, + UNINSTALL +}; + +class CommandLine : public QObject +{ +public: + CommandLine(); + ~CommandLine(); + void parse(QCoreApplication &app); + CommandType command(); +private: + CommandType type; + QCommandLineParser * parser; +}; + +} +} + diff --git a/gpt4all-updater/include/Downgrade.h b/gpt4all-updater/include/Downgrade.h new file mode 100644 index 000000000000..f87b6a525b2b --- /dev/null +++ b/gpt4all-updater/include/Downgrade.h @@ -0,0 +1,17 @@ +#pragma once + +#include "Command.h" + + +namespace gpt4all { +namespace downgrade { + +class Downgrade : public gpt4all::command::Command +{ +public: + Downgrade(QFile &installer); +}; + + +} +} \ No newline at end of file diff --git a/gpt4all-updater/include/Download.h b/gpt4all-updater/include/Download.h new file mode 100644 index 000000000000..d1d361caa1ba --- /dev/null +++ b/gpt4all-updater/include/Download.h @@ -0,0 +1,64 @@ +#pragma once + +#include "Manifest.h" +#include "State.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace gpt4all { +namespace download { + +class HashFile : public QObject +{ +public: + HashFile() : QObject(nullptr) {} + void hashAndInstall(const QString &expected, QCryptographicHash::Algorithm algo, QFile *temp, const QString &file, QNetworkReply *reply); +}; + +class Download : public QObject +{ +public: + static Download *instance(); + Q_INVOKABLE void driveFetchAndInstall(); + Q_INVOKABLE void downloadManifest(); + Q_INVOKABLE void downloadInstaller(); + Q_INVOKABLE void cancelDownload(); + Q_INVOKABLE void installInstaller(QString &expected, QFile *temp, QNetworkReply *installerResponse); + +private Q_SLOTS: + void handleSslErrors(QNetworkReply *reply, const QList &errors); + void handleErrorOccurred(QNetworkReply::NetworkError code); + void handleInstallerDownloadFinished(); + void handleReadyRead(); + +private: + explicit Download(); + ~Download() noexcept {}; + QNetworkReply * downloadInMemory(QUrl &url); + QNetworkReply * downloadLargeFile(QUrl &url); + QIODevice * handleManifestRequestResponse(QNetworkReply * reply); + gpt4all::manifest::ManifestFile *manifest; + QNetworkAccessManager m_networkManager; + QMap download_tracking; + HashFile *saver; + QDateTime m_startTime; + QString platform_ext; + friend class Downloader; +}; + +} +} diff --git a/gpt4all-updater/include/Embeded.h b/gpt4all-updater/include/Embeded.h new file mode 100644 index 000000000000..a6f4dc1008ae --- /dev/null +++ b/gpt4all-updater/include/Embeded.h @@ -0,0 +1,35 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include + +// Symbols indicating beginning and end of embedded installer +extern "C" { + extern char data_start_gpt4all_installer, data_stop_gpt4all_installer; + extern int gpt4all_installer_size; +} + +namespace gpt4all { +namespace embedded { + +class EmbeddedInstaller : public QObject +{ +public: + EmbeddedInstaller(); + void extractAndDecode(); +private: + char * start; + char * end; + int size; + QByteArray data; +}; + + +} +} + diff --git a/gpt4all-updater/include/Manifest.h b/gpt4all-updater/include/Manifest.h new file mode 100644 index 000000000000..3bb41d5d10ea --- /dev/null +++ b/gpt4all-updater/include/Manifest.h @@ -0,0 +1,46 @@ +#pragma once + +#include "Package.h" + +#include +#include +#include +#include +#include +#include + +namespace gpt4all{ +namespace manifest { + +enum releaseType{ + RELEASE, + DEBUG +}; + +class ManifestFile { +public: + static ManifestFile * parseManifest(QIODevice *manifestInput); + QUrl & getInstallerEndpoint(); + QString & getExpectedHash(); +private: + ManifestFile() {} + QString name; + QVersionNumber release_ver; + QString notes; + QStringList authors; + QDate release_date; + releaseType config; + QVersionNumber last_supported_version; + QString entity; + QStringList component_list; + Package pkg; + void parsePkgDescription(); + void parsePkgManifest(); + QXmlStreamReader xml; +}; + + + + +} +} \ No newline at end of file diff --git a/gpt4all-updater/include/Modify.h b/gpt4all-updater/include/Modify.h new file mode 100644 index 000000000000..13de51c72d40 --- /dev/null +++ b/gpt4all-updater/include/Modify.h @@ -0,0 +1,15 @@ +#pragma once + +#include "Command.h" + + +namespace gpt4all { +namespace modify { + +class Modify : public gpt4all::command::Command +{ +public: + Modify(QFile &installer); +}; +} +} \ No newline at end of file diff --git a/gpt4all-updater/include/Package.h b/gpt4all-updater/include/Package.h new file mode 100644 index 000000000000..3d9bf89bad74 --- /dev/null +++ b/gpt4all-updater/include/Package.h @@ -0,0 +1,29 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include +#include + +namespace gpt4all { +namespace manifest { + +class Package { +public: + Package() {} + static Package parsePackage(QXmlStreamReader &xml); + + QString checksum_sha256; + bool is_signed; + QUrl installer_endpoint; + QUrl sbom_manifest; + QStringList installer_args; +}; + +} + +} \ No newline at end of file diff --git a/gpt4all-updater/include/Resource.h b/gpt4all-updater/include/Resource.h new file mode 100644 index 000000000000..b971192484b4 --- /dev/null +++ b/gpt4all-updater/include/Resource.h @@ -0,0 +1,18 @@ +#pragma once + +#include +#include + +#include + +namespace gpt4all { +namespace resource { + + +class WinInstallerResources : public QObject +{ +public: + static int extractAndInstall(); +} +} +} \ No newline at end of file diff --git a/gpt4all-updater/include/State.h b/gpt4all-updater/include/State.h new file mode 100644 index 000000000000..5f17a5a41c52 --- /dev/null +++ b/gpt4all-updater/include/State.h @@ -0,0 +1,35 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace gpt4all { +namespace state { + +class Gpt4AllState : public QObject { +public: + virtual ~Gpt4AllState() = default; + static Gpt4AllState & getInstance(); + QVersionNumber &getCurrentGpt4AllVersion(); + QFile &getCurrentGpt4AllConfig(); + QFile &getCurrentGpt4AllInstallRoot(); + bool checkForExistingInstall(); + QFile &getInstaller(); + void setInstaller(QFile &installer); + void removeCurrentInstallation(); + void driveOffline(); + +private: + explicit Gpt4AllState(QObject *parent = nullptr) {} + QFile *installer; + QFile currentGpt4AllConfig; + QDir currentGpt4AllInstallPath; + QVersionNumber currentGpt4AllVersion; +}; +} +} + diff --git a/gpt4all-updater/include/Uninstall.h b/gpt4all-updater/include/Uninstall.h new file mode 100644 index 000000000000..5e22b7e7b7e3 --- /dev/null +++ b/gpt4all-updater/include/Uninstall.h @@ -0,0 +1,16 @@ +#pragma once + +#include "Command.h" + + +namespace gpt4all { +namespace uninstall { + +class Uninstall : public gpt4all::command::Command +{ +public: + Uninstall(QFile &uninstaller); +}; + +} +} \ No newline at end of file diff --git a/gpt4all-updater/include/Update.h b/gpt4all-updater/include/Update.h new file mode 100644 index 000000000000..c372a21e2cc4 --- /dev/null +++ b/gpt4all-updater/include/Update.h @@ -0,0 +1,15 @@ +#pragma once + +#include "Command.h" + +namespace gpt4all { +namespace updater { + +class Update : public gpt4all::command::Command +{ +public: + Update(QFile &installer); +}; + +} +} diff --git a/gpt4all-updater/include/utils.h b/gpt4all-updater/include/utils.h new file mode 100644 index 000000000000..5e2f1c6767bf --- /dev/null +++ b/gpt4all-updater/include/utils.h @@ -0,0 +1,5 @@ +#pragma once + +#include + +constexpr uint32_t hash(const char* data) noexcept; diff --git a/gpt4all-updater/src/Command.cxx b/gpt4all-updater/src/Command.cxx new file mode 100644 index 000000000000..2dfd96c18066 --- /dev/null +++ b/gpt4all-updater/src/Command.cxx @@ -0,0 +1,10 @@ +#include "Command.h" + +using namespace gpt4all::command; + +bool Command::execute() { + if (!this->installer->exists()) + qDebug() << "Couldn't find the installer, there may have been an issue extracting or downloading the installer"; + return false; + return QProcess::startDetached(this->installer->fileName(), this->arguments); +} diff --git a/gpt4all-updater/src/CommandFactory.cxx b/gpt4all-updater/src/CommandFactory.cxx new file mode 100644 index 000000000000..70aa9140db95 --- /dev/null +++ b/gpt4all-updater/src/CommandFactory.cxx @@ -0,0 +1,31 @@ +#include "CommandFactory.h" +#include "Downgrade.h" +#include "Modify.h" +#include "Uninstall.h" +#include "Update.h" +#include "State.h" +#include "Download.h" + +using namespace gpt4all::command; + +std::shared_ptr CommandFactory::GetCommand(CommandType type) +{ + std::shared_ptr active_command; +#ifdef OFFLINE + gpt4all::stage::Gpt4AllState::getInstance().driveOffline(); +#else + gpt4all::download::Download::instance()->driveFetchAndInstall(); +#endif + QFile &installer = gpt4all::state::Gpt4AllState::getInstance().getInstaller(); + switch(type){ + case CommandType::UPDATE: + active_command = std::make_shared(installer); + case CommandType::DOWNGRADE: + active_command = std::make_shared(installer); + case CommandType::MODIFY: + active_command = std::make_shared(installer); + case CommandType::UNINSTALL: + active_command = std::make_shared(installer); + } + return active_command; +} diff --git a/gpt4all-updater/src/CommandLine.cxx b/gpt4all-updater/src/CommandLine.cxx new file mode 100644 index 000000000000..27bdf2c8c4c3 --- /dev/null +++ b/gpt4all-updater/src/CommandLine.cxx @@ -0,0 +1,54 @@ +#include "CommandLine.h" + +#include "utils.h" + +gpt4all::command::CommandLine::CommandLine() +{ + this->parser = new QCommandLineParser(); + this->parser->addHelpOption(); + this->parser->addVersionOption(); + this->parser->addPositionalArgument("command", QCoreApplication::translate("main", "Gpt4All installation modification command, must be one of: 'update', 'downgrade', 'uninstall', 'modify'")); +} + +gpt4all::command::CommandLine::~CommandLine() +{ + free(this->parser); +} + +void gpt4all::command::CommandLine::parse(QCoreApplication &app) +{ + this->parser->process(app); +} + +constexpr uint32_t hash(const char* data) noexcept{ + uint32_t hash = 5381; + + for(const char *c = data; *c; ++c) + hash = ((hash << 5) + hash) + (unsigned char) *c; + + return hash; +} + +gpt4all::command::CommandType gpt4all::command::CommandLine::command() +{ + const QStringList args(this->parser->positionalArguments()); + if (args.empty()){ + // raise error + } + gpt4all::command::CommandType ret; + QString command_arg = args.first(); + switch(hash(command_arg.toStdString().c_str())) { + case hash("update"): + ret = gpt4all::command::CommandType::UPDATE; + case hash("modify"): + ret = gpt4all::command::CommandType::MODIFY; + case hash("downgrade"): + ret = gpt4all::command::CommandType::DOWNGRADE; + case hash("uninstall"): + ret = gpt4all::command::CommandType::UNINSTALL; + default: + // throw error, although the commnd line parser should have + // caught this + } + return ret; +} diff --git a/gpt4all-updater/src/Downgrade.cxx b/gpt4all-updater/src/Downgrade.cxx new file mode 100644 index 000000000000..5cf6f4a84bac --- /dev/null +++ b/gpt4all-updater/src/Downgrade.cxx @@ -0,0 +1,9 @@ +#include "Downgrade.h" + +using namespace gpt4all::downgrade; + +Downgrade::Downgrade(QFile &installer) +{ + this->installer = &installer; + this->arguments = {"-c", "rm"}; +} diff --git a/gpt4all-updater/src/Download.cxx.in b/gpt4all-updater/src/Download.cxx.in new file mode 100644 index 000000000000..be3df921f2c2 --- /dev/null +++ b/gpt4all-updater/src/Download.cxx.in @@ -0,0 +1,216 @@ +#include "Download.h" + +#include +#include +#include +#include +#include + +using namespace gpt4all::download; +using namespace Qt::Literals::StringLiterals; + +Download * Download::instance() +{ + return new Download(); +} + +Download::Download() + : QObject(nullptr), + saver(new HashFile) +{ +#if defined(Q_OS_WINDOWS) + platform_ext = ".exe"; +#else + platform_ext = ""; +#endif + connect(&m_networkManager, &QNetworkAccessManager::sslErrors, this, &Download::handleSslErrors); +} + +void Download::driveFetchAndInstall() +{ + this->downloadManifest(); + this->downloadInstaller(); +} + +void Download::downloadInstaller() +{ + this->downloadLargeFile(this->manifest->getInstallerEndpoint()); +} + +void Download::downloadManifest() +{ + QUrl manifestEndpoint("@GPT4ALL_MANIFEST_ENDPOINT@"); + QNetworkReply *manifestResponse = this->downloadInMemory(manifestEndpoint); + this->manifest = gpt4all::manifest::ManifestFile::parseManifest(manifestResponse); +} + +void Download::installInstaller(QString &expected, QFile *temp, QNetworkReply *installerResponse) +{ + const QString saveFilePath = QCoreApplication::applicationDirPath() + "gpt4all-installer" + platform_ext; + this->saver->hashAndInstall(expected, + QCryptographicHash::Sha256, + temp, saveFilePath, installerResponse); + QFile saveFile(saveFilePath); + gpt4all::state::Gpt4AllState::getInstance().setInstaller(saveFile); +} + +QNetworkReply * Download::downloadInMemory(QUrl &url) +{ + QNetworkRequest request(url); + QSslConfiguration conf = request.sslConfiguration(); + conf.setPeerVerifyMode(QSslSocket::VerifyNone); + request.setSslConfiguration(conf); + QNetworkReply *reply = m_networkManager.get(request); + if (!reply) { + return nullptr; + } + if (reply->error() != QNetworkReply::NoError) { + qWarning() << "Error: network error occured while downloading the release manifest:" << reply->errorString(); + reply->deleteLater(); + } + return reply; +} + +QNetworkReply * Download::downloadLargeFile(QUrl &url) +{ + QFile *tempFile = new QFile("gpt4all-installer" + platform_ext + "-partial"); + bool success = tempFile->open(QIODevice::WriteOnly | QIODevice::Append); + qWarning() << "Opening temp file for writing:" << tempFile->fileName(); + if (!success) { + const QString error = u"ERROR: Could not open temp file: %1 %2"_s.arg(tempFile->fileName()); + qWarning() << error; + return nullptr; + } + tempFile->flush(); + size_t incomplete_size = tempFile->size(); + if (incomplete_size > 0) { + bool success = tempFile->seek(incomplete_size); + if (!success) { + incomplete_size = 0; + success = tempFile->seek(incomplete_size); + Q_ASSERT(success); + } + } + QNetworkRequest request(url); + request.setAttribute(QNetworkRequest::User, "gpt4all-installer" + platform_ext); + request.setRawHeader("range", u"bytes=%1-"_s.arg(tempFile->pos()).toUtf8()); + QSslConfiguration conf = request.sslConfiguration(); + conf.setPeerVerifyMode(QSslSocket::VerifyNone); + request.setSslConfiguration(conf); + QNetworkReply *installerDownResponse = m_networkManager.get(request); + connect(installerDownResponse, &QNetworkReply::errorOccurred, this, &Download::handleErrorOccurred); + connect(installerDownResponse, &QNetworkReply::finished, this, &Download::handleInstallerDownloadFinished); + connect(installerDownResponse, &QNetworkReply::readyRead, this, &Download::handleReadyRead); + download_tracking.insert(installerDownResponse, tempFile); +} + +void Download::handleSslErrors(QNetworkReply *reply, const QList &errors) +{ + QUrl url = reply->request().url(); + for (const auto &e : errors) + qWarning() << "ERROR: Received ssl error:" << e.errorString() << "for" << url; +} + +void Download::handleErrorOccurred(QNetworkReply::NetworkError code) +{ + QNetworkReply *installerResponse = qobject_cast(sender()); + if (!installerResponse) + return; + if (code == QNetworkReply::OperationCanceledError) + return; + + QString installerfile = installerResponse->request().attribute(QNetworkRequest::User).toString(); + + const QString error = u"ERROR: Network error occurred attempting to download %1 code: %2 errorString %3"_s.arg(installerfile).arg(code).arg(installerResponse->errorString()); + qWarning() << error; + disconnect(installerResponse, &QNetworkReply::finished, this, &Download::handleInstallerDownloadFinished); + installerResponse->abort(); + installerResponse->deleteLater(); + QFile * tempfile = download_tracking.value(installerResponse); + tempfile->deleteLater(); + download_tracking.remove(installerResponse); +} + +void Download::handleReadyRead() +{ + QNetworkReply *installerResponse = qobject_cast(sender()); + if (!installerResponse) + return; + QFile * tempfile = download_tracking.value(installerResponse); + while(!installerResponse->atEnd()) { + tempfile->write(installerResponse->read(8192)); + } + tempfile->flush(); +} + +void Download::handleInstallerDownloadFinished() +{ + QNetworkReply *installerResponse = qobject_cast(sender()); + if (!installerResponse) + return; + + QString installerName = installerResponse->request().attribute(QNetworkRequest::User).toString(); + QFile *tempfile = download_tracking.value(installerResponse); + download_tracking.remove(installerResponse); + + if (installerResponse->error()) { + const QString errorString + = u"ERROR: Downloading failed with code %1 \"%2\""_s.arg(installerResponse->error()).arg(installerResponse->errorString()); + qWarning() << errorString; + installerResponse->deleteLater(); + tempfile->deleteLater(); + return; + } + tempfile->close(); + QString expected = this->manifest->getExpectedHash(); + this->installInstaller(expected, tempfile, installerResponse); +} + +void HashFile::hashAndInstall(const QString &expected, QCryptographicHash::Algorithm algo, QFile * temp, const QString &file, QNetworkReply *response) +{ + Q_ASSERT(!temp->isOpen()); + QString installerFile = response->request().attribute(QNetworkRequest::User).toString(); + if (!temp->open(QIODevice::ReadOnly)) { + const QString error = u"ERROR: Could not open temp file for hashing: %1 %2"_s.arg(temp->fileName(), installerFile); + qWarning() << error; + return; + } + + QCryptographicHash hash(algo); + while(!temp->atEnd()) + hash.addData(temp->read(8192)); + if (hash.result().toHex() != expected.toLatin1()) { + temp->close(); + const QString error = u"ERROR: Download error hash did not match: %1 != %2 for %3"_s.arg(hash.result().toHex(), expected.toLatin1(), installerFile); + qWarning() << error; + temp->remove(); + return; + } + + temp->close(); + if (temp->rename(file)) { + return; + } + + if (!temp->open(QIODevice::ReadOnly)) { + const QString error = u"ERROR: Could not open temp file at finish: %1 %2"_s.arg(temp->fileName(), file); + qWarning() << error; + return; + } + + QFile savefile(file); + if (savefile.open(QIODevice::WriteOnly)) { + QByteArray buffer; + while (!temp->atEnd()) { + buffer = temp->read(8192); + savefile.write(buffer); + } + savefile.close(); + temp->close(); + } else { + QFile::FileError error = savefile.error(); + const QString errorString = u"ERROR: Could not save installer to: %1 failed with code %1"_s.arg(file).arg(error); + qWarning() << errorString; + temp->close(); + } +} diff --git a/gpt4all-updater/src/Embeded.cxx b/gpt4all-updater/src/Embeded.cxx new file mode 100644 index 000000000000..05c67a8108b6 --- /dev/null +++ b/gpt4all-updater/src/Embeded.cxx @@ -0,0 +1,32 @@ +#include "Embeded.h" + +#include + +using namespace gpt4all::embedded; + +EmbeddedInstaller::EmbeddedInstaller() + : start(&data_start_gpt4all_installer), + end(&data_stop_gpt4all_installer), + size(gpt4all_installer_size) {} + +EmbeddedInstaller::extractAndDecode() +{ + QByteArray installerDat64(this->start, this->size); + QByteArray installerDat = QByteArray::fromBase64(installerDat64); + this->data(installerDat); +} + +EmbeddedInstaller::installInstaller(QFile *installerFilepath) +{ + bool success = installerPath->open(QIODevice::WriteOnly | QIODevice::Append); + qWarning() << "Opening installer file for writing:" << installerPath->fileName(); + if (!success) { + const QString error + = u"ERROR: Could not open temp file: %1 %2"_s.arg(installerPath->fileName()); + qWarning() << error; + return nullptr; + } + installerFilepath->write(this->data, this->size); + installerFilePath.close(); + gpt4all::state::Gpt4AllState::getInstance().setInstaller(installerPath); +} diff --git a/gpt4all-updater/src/Manifest.cxx b/gpt4all-updater/src/Manifest.cxx new file mode 100644 index 000000000000..9b2666c3c7a1 --- /dev/null +++ b/gpt4all-updater/src/Manifest.cxx @@ -0,0 +1,81 @@ +#include "Manifest.h" + +using namespace gpt4all::manifest; +using namespace Qt::Literals::StringLiterals; + +ManifestFile *ManifestFile::parseManifest(QIODevice *manifestInput) +{ + ManifestFile * manifest = new ManifestFile(); + manifest->xml.setDevice(manifestInput); + if(manifest->xml.readNextStartElement()) { + if (manifest->xml.name() == "gpt4all-updater"_L1 && manifest->xml.attributes().value("version"_L1) == "1.0"_L1){ + manifest->parsePkgDescription(); + manifest->parsePkgManifest(); + } + } + return manifest; +} + +void ManifestFile::parsePkgDescription() +{ + Q_ASSERT(xml.isStartElement() && xml.name() == "pkg-description"_L1); + + while(xml.readNextStartElement()){ + if(xml.name() == "name"_L1){ + this->name = xml.readElementText(); + } + else if(xml.name() == "version"_L1){ + qsizetype suffix; + this->release_ver = QVersionNumber::fromString(xml.readElementText(), &suffix); + } + else if(xml.name() == "notes"_L1) { + this->notes = xml.readElementText(); + } + else if(xml.name() == "authors"_L1) { + this->authors = xml.readElementText().split(","); + } + else if(xml.name() == "date"_L1) { + this->release_date = QDate::fromString(xml.readElementText(), Qt::ISODate); + } + else if(xml.name() == "release-type"_L1) { + if ( xml.readElementText() == "release"_L1 ) { + this->config = releaseType::RELEASE; + } + else { + this->config = releaseType::DEBUG; + } + } + else if(xml.name() == "last-compatible-version"_L1) { + qsizetype suffix; + this->last_supported_version = QVersionNumber::fromString(xml.readElementText(), &suffix); + } + else if(xml.name() == "entity"_L1) { + this->entity = xml.readElementText(); + } + else if(xml.name() == "component-list"_L1) { + this->component_list = xml.readElementText().split(","); + } + else if(xml.name() == "pkg-manifest"_L1) { + this->parsePkgManifest(); + } + else { + // Should probably throw here, but I'd rather implement schema validation + xml.skipCurrentElement(); + } + } +} + +void ManifestFile::parsePkgManifest() +{ + this->pkg = Package::parsePackage(xml); +} + +QUrl & ManifestFile::getInstallerEndpoint() +{ + return this->pkg.installer_endpoint; +} + +QString & ManifestFile::getExpectedHash() +{ + return this->pkg.checksum_sha256; +} diff --git a/gpt4all-updater/src/Modify.cxx b/gpt4all-updater/src/Modify.cxx new file mode 100644 index 000000000000..c3f33c7e4119 --- /dev/null +++ b/gpt4all-updater/src/Modify.cxx @@ -0,0 +1,9 @@ +#include "Modify.h" + +using namespace gpt4all::modify; + +Modify::Modify(QFile &installer) +{ + this->installer = &installer; + this->arguments = {"-c", "ch"}; +} \ No newline at end of file diff --git a/gpt4all-updater/src/Package.cxx b/gpt4all-updater/src/Package.cxx new file mode 100644 index 000000000000..3d5e08b9762a --- /dev/null +++ b/gpt4all-updater/src/Package.cxx @@ -0,0 +1,29 @@ +#include "Package.h" + + +using namespace Qt::Literals::StringLiterals; +using namespace gpt4all::manifest; + + +Package Package::parsePackage(QXmlStreamReader &xml) +{ + Q_ASSERT(xml.isStartElement() && xml.name() == "pkg-manifest"_L1); + Package p; + while(xml.readNextStartElement()) { + if(xml.name() == "installer_uri"_L1) { + p.installer_endpoint = QUrl(xml.readElementText()); + } + else if(xml.name() == "sha256"_L1) { + p.checksum_sha256 = xml.readElementText(); + } + else if(xml.name() == "args"_L1) { + p.installer_args = xml.readElementText().split(" "); + } + else if(xml.name() == "signed"_L1) { + p.is_signed = xml.readElementText() == "true"; + } + else { + xml.skipCurrentElement(); + } + } +} diff --git a/gpt4all-updater/src/Resource.cxx b/gpt4all-updater/src/Resource.cxx new file mode 100644 index 000000000000..a0732cc4d42f --- /dev/null +++ b/gpt4all-updater/src/Resource.cxx @@ -0,0 +1,38 @@ +#include "Resource.h" + + + +using namespace gpt4all::resource + + +int WinInstallerResources::extractAndInstall(QFile *installerPath) +{ + HMODULE hModule = NULL; + HRSRC resourceInfo = FindResourceA(hModule, "gpt4allInstaller", "CUSTOMDATA"); + if (!resourceInfo) + { + fprintf(stderr, "Could not find resource\n"); + return -1; + } + + HGLOBAL resource = LoadResource(hModule, resourceInfo); + if (!resource) + { + qWarning() << "Could not load resource"; + return -1; + } + + DWORD resourceSize = SizeofResource(hModule, resourceInfo); + bool success = installerPath->open(QIODevice::WriteOnly | QIODevice::Append); + qWarning() << "Opening installer file for writing:" << installerPath->fileName(); + if (!success) { + const QString error + = u"ERROR: Could not open temp file: %1 %2"_s.arg(installerPath->fileName()); + qWarning() << error; + return nullptr; + } + const char* installerdat = (const char*)resource; + installerPath.write(installerdat); + installerPath.close() + gpt4all::state::Gpt4AllState::getInstance().setInstaller(installerPath); +} \ No newline at end of file diff --git a/gpt4all-updater/src/State.cxx b/gpt4all-updater/src/State.cxx new file mode 100644 index 000000000000..0cbe7b6500c0 --- /dev/null +++ b/gpt4all-updater/src/State.cxx @@ -0,0 +1,73 @@ +#include "State.h" + +#if defined(Q_OS_WINDOWS) + #include "Resource.h" +#elif defined(Q_OS_DARWIN) + #include "Embeded.h" +#endif + + +using namespace gpt4all::state; + +Gpt4AllState & Gpt4AllState::getInstance() +{ + static Gpt4AllState instance; + return instance; +} + +void Gpt4AllState::driveOffline() +{ + if(this->checkForExistingInstall()) + this->removeCurrentInstallation(); +#if defined(Q_OS_WINDOWS) + gpt4all::resource::WinInstallerResources::extractAndInstall(); +#elif defined(Q_OS_DARWIN) + gpt4all::embedded::EmbeddedInstaller embed; + embed.extractAndDecode(); +#endif +} + +QVersionNumber &Gpt4AllState::getCurrentGpt4AllVersion() +{ + return this->currentGpt4AllVersion; +} + +QFile &Gpt4AllState::getCurrentGpt4AllConfig() +{ + return this->currentGpt4AllConfig; +} + +QDir &Gpt4AllState::getCurrentGpt4AllInstallRoot() +{ + return this->currentGpt4AllInstallPath; +} + +QFile & Gpt4AllState::getInstaller() +{ + return *this->installer; +} + +void Gpt4AllState::setInstaller(QFile &installer_) +{ + this->installer = &installer_; +} + +void Gpt4AllState::removeCurrentInstallation() { + if (!this->currentGpt4AllInstallPath.exists()) + return + this->currentGpt4AllInstallPath.removeRecursively(); +} + +bool Gpt4AllState::checkForExistingInstall() +{ +#if defined(Q_OS_DARWIN) + QStringList potentialInstallPaths = {"/Applications/gpt4all"}; +#elif defined(Q_OS_WINDOWS) + QStringList potentialInstallPaths = {"C:\\Program Files\\gpt4all"}; +#endif + foreach( const QString &str, potentialInstallPaths) { + QDir potentialPath(str); + if(potentialPath.exists()) + this->currentGpt4AllInstallPath = potentialPath; + } +} diff --git a/gpt4all-updater/src/Uninstall.cxx b/gpt4all-updater/src/Uninstall.cxx new file mode 100644 index 000000000000..d7e3b61087e4 --- /dev/null +++ b/gpt4all-updater/src/Uninstall.cxx @@ -0,0 +1,11 @@ +#include "Uninstall.h" + + +using namespace gpt4all::uninstall; + + +Uninstall::Uninstall(QFile &uninstaller) +{ + this->installer = &uninstaller; + this->arguments = {"-c", "pr"}; +} \ No newline at end of file diff --git a/gpt4all-updater/src/Update.cxx b/gpt4all-updater/src/Update.cxx new file mode 100644 index 000000000000..1467975eb714 --- /dev/null +++ b/gpt4all-updater/src/Update.cxx @@ -0,0 +1,11 @@ +#include "Update.h" + + +using namespace gpt4all::updater; + + +Update::Update(QFile &installer) +{ + this->installer = &installer; + this->arguments = {"-c", "install"}; +} \ No newline at end of file diff --git a/gpt4all-updater/src/asm/bake_installer.S.in b/gpt4all-updater/src/asm/bake_installer.S.in new file mode 100644 index 000000000000..77bf21312561 --- /dev/null +++ b/gpt4all-updater/src/asm/bake_installer.S.in @@ -0,0 +1,8 @@ + .global _data_start_gpt4all_installer + .global _data_stop_gpt4all_installer +_data_start_gpt4all_installer: + .incbin "@GPT4ALL_INSTALLER_PATH@" +_data_stop_gpt4all_installer: + .global _gpt4all_installer_size +_gpt4all_installer_size: + .int _data_stop_gpt4all_installer - _data_start_gpt4all_installer diff --git a/gpt4all-updater/src/main.cxx b/gpt4all-updater/src/main.cxx new file mode 100644 index 000000000000..d96d97126964 --- /dev/null +++ b/gpt4all-updater/src/main.cxx @@ -0,0 +1,19 @@ +#include "CommandLine.h" +#include "CommandFactory.h" + + +int main(int argc, char ** argv) +{ + QCoreApplication app(argc, argv); + QCoreApplication::setOrganizationName("nomic.ai"); + QCoreApplication::setOrganizationDomain("gpt4all.io"); + QCoreApplication::setApplicationName("gpt4all-auto-updater"); + QCoreApplication::setApplicationVersion("0.0.1"); + gpt4all::command::CommandLine cli; + cli.parse(app); + CommandFactory cmd; + gpt4all::command::CommandType command_type(cli.command()); + std::shared_ptr installer_command = cmd.GetCommand(command_type); + installer_command->execute(); + return 0; +} \ No newline at end of file diff --git a/gpt4all-updater/src/resources/installer.rc.in b/gpt4all-updater/src/resources/installer.rc.in new file mode 100644 index 000000000000..f27cce014032 --- /dev/null +++ b/gpt4all-updater/src/resources/installer.rc.in @@ -0,0 +1 @@ +gpt4allInstaller CUSTOMDATA @GPT4ALL_INSTALLER_PATH@ diff --git a/gpt4all-updater/src/utils.cxx b/gpt4all-updater/src/utils.cxx new file mode 100644 index 000000000000..13c47ce0719e --- /dev/null +++ b/gpt4all-updater/src/utils.cxx @@ -0,0 +1,12 @@ +#include "utils.h" + + +constexpr uint32_t hash(const char* data) noexcept +{ + uint32_t hash = 5381; + + for(const char *c = data; *c; ++c) + hash = ((hash << 5) + hash) + (unsigned char) *c; + + return hash; +} \ No newline at end of file diff --git a/gpt4all-updater/tmp/manifest.xml b/gpt4all-updater/tmp/manifest.xml new file mode 100644 index 000000000000..feb47b6e26f7 --- /dev/null +++ b/gpt4all-updater/tmp/manifest.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + \ No newline at end of file