feat: SQLite+Compressed save format (Phase 2 of v2 save format)

.github/workflows/android.yml

@@ -42,7 +42,7 @@ jobs:
       - name: Setup build and dependencies
         run: |
           sudo apt-get update
-          sudo apt-get install gettext
+          sudo apt-get install gettext libsqlite3-dev zlib1g-dev
 
       - name: Build
         working-directory: ./android

.github/workflows/clang-tidy.yml

@@ -50,7 +50,8 @@ jobs:
           sudo apt-get install \
             cmake gettext ninja-build mold ccache jq \
             clang-18 libclang-18-dev llvm-18 llvm-18-dev clang-tidy-18 \
-            libsdl2-dev libsdl2-ttf-dev libsdl2-image-dev libsdl2-mixer-dev libpulse-dev libflac-dev
+            libsdl2-dev libsdl2-ttf-dev libsdl2-image-dev libsdl2-mixer-dev libpulse-dev libflac-dev \
+            sqlite3 libsqlite3-dev zlib1g-dev
 
       - name: ensure clang 18 is installed
         run: |

.github/workflows/manual-release.yml

@@ -228,11 +228,11 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install libncursesw5-dev libsdl2-dev libsdl2-ttf-dev libsdl2-image-dev \
-            libsdl2-mixer-dev libpulse-dev ccache gettext parallel
+            libsdl2-mixer-dev libpulse-dev ccache gettext parallel libsqlite3-dev zlib1g-dev
       - name: Install dependencies (mac)
         if: runner.os == 'macOS'
         run: |
-          HOMEBREW_NO_AUTO_UPDATE=yes HOMEBREW_NO_INSTALL_CLEANUP=yes brew install sdl2 sdl2_image sdl2_ttf sdl2_mixer gettext ccache parallel llvm astyle
+          HOMEBREW_NO_AUTO_UPDATE=yes HOMEBREW_NO_INSTALL_CLEANUP=yes brew install sdl2 sdl2_image sdl2_ttf sdl2_mixer gettext ccache parallel llvm astyle sqlite3 zlib
           python3 -m venv ./venv
           source ./venv/bin/activate
           pip3 install mac_alias==2.2.0 dmgbuild==1.6.1 biplist polib luaparser

.github/workflows/matrix.yml

@@ -156,7 +156,7 @@ jobs:
         if: ${{ env.SKIP == 'false' && runner.os == 'Linux' }}
         run: |
           sudo apt-get update
-          sudo apt-get install libncursesw5-dev ccache gettext parallel
+          sudo apt-get install libncursesw5-dev ccache gettext parallel libsqlite3-dev zlib1g-dev
           sudo locale-gen en_US.UTF-8 fr_FR.UTF-8 ru_RU.UTF-8
 
       - name: install SDL2 dependencies (ubuntu)
@@ -189,7 +189,7 @@ jobs:
           # Ensure that it is actually needed version
           ${{ matrix.compiler }} --version | grep -q -E "${{ matrix.grep_clang_version_rxp }}"
 
-          HOMEBREW_NO_AUTO_UPDATE=yes HOMEBREW_NO_INSTALL_CLEANUP=yes brew install gettext ccache parallel
+          HOMEBREW_NO_AUTO_UPDATE=yes HOMEBREW_NO_INSTALL_CLEANUP=yes brew install gettext ccache parallel sqlite3 zlib
 
       # problem matchers are only ran on GCC to reduce error spam
       - name: add problem matcher

.github/workflows/msys2-cmake.yml

@@ -55,6 +55,8 @@ jobs:
           mingw-w64-x86_64-SDL2_ttf
           mingw-w64-x86_64-toolchain
           mingw-w64-x86_64-cmake
+          mingw-w64-x86_64-sqlite3
+          mingw-w64-x86_64-zstd
 
     - name: Create build directory
       run: mkdir build

.github/workflows/osx.yml

@@ -80,7 +80,7 @@ jobs:
 
       - name: Install build dependencies
         run: |
-          HOMEBREW_NO_AUTO_UPDATE=yes HOMEBREW_NO_INSTALL_CLEANUP=yes brew install sdl2 sdl2_image sdl2_ttf sdl2_mixer gettext ccache parallel llvm astyle
+          HOMEBREW_NO_AUTO_UPDATE=yes HOMEBREW_NO_INSTALL_CLEANUP=yes brew install sdl2 sdl2_image sdl2_ttf sdl2_mixer gettext ccache parallel llvm astyle sqlite3 zlib
           python3 -m venv ./venv
           source ./venv/bin/activate
           pip3 install mac_alias==2.2.0 dmgbuild==1.6.1 biplist polib luaparser

.github/workflows/release.yml

@@ -262,7 +262,7 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install libncursesw5-dev libsdl2-dev libsdl2-ttf-dev libsdl2-image-dev \
-            libsdl2-mixer-dev libpulse-dev ccache gettext parallel
+            libsdl2-mixer-dev libpulse-dev ccache gettext parallel libsqlite3-dev zlib1g-dev
 
       - name: Install runtime dependencies (mac)
         if: runner.os == 'macOS'
@@ -275,7 +275,7 @@ jobs:
       - name: Install build dependencies (mac)
         if: runner.os == 'macOS'
         run: |
-          HOMEBREW_NO_AUTO_UPDATE=yes HOMEBREW_NO_INSTALL_CLEANUP=yes brew install gettext ccache parallel llvm astyle
+          HOMEBREW_NO_AUTO_UPDATE=yes HOMEBREW_NO_INSTALL_CLEANUP=yes brew install gettext ccache parallel llvm astyle sqlite3 zlib
           python3 -m venv ./venv
           source ./venv/bin/activate
           pip3 install mac_alias==2.2.0 dmgbuild==1.6.1 biplist polib luaparser

CMakeLists.txt

@@ -113,6 +113,8 @@ include(CheckCXXCompilerFlag)
 #SET(CMAKE_SHARED_LIBRARY_CXX_FLAGS "${CMAKE_SHARED_LIBRARY_CXX_FLAGS} -m32")
 
 find_package(PkgConfig)
+find_package(SQLite3)
+find_package(ZLIB)
 if (NOT DYNAMIC_LINKING)
     if(NOT MSVC)
         set(CMAKE_FIND_LIBRARY_SUFFIXES ".a;.dll.a")

Makefile

@@ -607,6 +607,11 @@ endif
 
 PKG_CONFIG = $(CROSS)pkg-config
 
+CXXFLAGS += $(shell $(PKG_CONFIG) --cflags sqlite3)
+LDFLAGS += $(shell $(PKG_CONFIG) --libs sqlite3)
+CXXFLAGS += $(shell $(PKG_CONFIG) --cflags zlib)
+LDFLAGS += $(shell $(PKG_CONFIG) --libs zlib)
+
 ifeq ($(SOUND), 1)
   ifneq ($(TILES),1)
     $(error "SOUND=1 only works with TILES=1")

android/.gitignore

@@ -11,6 +11,8 @@
 /app/jni/SDL2_image
 /app/jni/SDL2_mixer
 /app/jni/SDL2_ttf
+/app/jni/sqlite3/*.c
+/app/jni/sqlite3/*.h
 /app/jni/libintl-lite
 /app/jni/libhidapi
 /app/jni/lua

android/app/build.gradle

@@ -142,6 +142,29 @@ task unzipDeps(type: Copy) {
     into outputDir
 }
 
+task fetchSqlite {
+    doLast {
+        def url = new URL('https://www.sqlite.org/2025/sqlite-amalgamation-3480000.zip')
+        def file = new File('/tmp/sqlite-amalgamation-3480000.zip')
+        file.withOutputStream { out ->
+            url.withInputStream { from -> out << from }
+        }
+    }
+}
+
+task unzipSqlite(type: Copy) {
+    dependsOn fetchSqlite
+    from(zipTree(file('/tmp/sqlite-amalgamation-3480000.zip'))) {
+        includeEmptyDirs = false
+        eachFile { fileCopyDetails ->
+            if (fileCopyDetails.path.startsWith('sqlite-amalgamation-3480000/')) {
+                fileCopyDetails.path = fileCopyDetails.path - 'sqlite-amalgamation-3480000/'
+            }
+        }
+    }
+    into 'jni/sqlite3'
+}
+
 task makeLocalization(type: Exec) {
     if (localize) {
         println("Building with localization'")
@@ -168,6 +191,7 @@ task makeLocalization(type: Exec) {
 makeLocalization
 unzipDeps.dependsOn makeLocalization
 preBuild.dependsOn unzipDeps
+preBuild.dependsOn unzipSqlite
 
 android {
     namespace "com.cleverraven.cataclysmdda"

android/app/jni/sqlite3/Android.mk

@@ -0,0 +1,49 @@
+# Reference: https://developer.android.com/ndk/guides/android_mk.html
+
+# Based on https://sqlite.org/android/file?name=sqlite3/src/main/jni/sqlite/Android.mk&ci=tip
+
+LOCAL_PATH:= $(call my-dir)
+include $(CLEAR_VARS)
+
+# If using SEE, uncomment the following:
+# LOCAL_CFLAGS += -DSQLITE_HAS_CODEC
+
+#Define HAVE_USLEEP, otherwise ALL sleep() calls take at least 1000ms
+LOCAL_CFLAGS += -DHAVE_USLEEP=1
+
+# Enable SQLite extensions.
+LOCAL_CFLAGS += -DSQLITE_ENABLE_FTS5 
+LOCAL_CFLAGS += -DSQLITE_ENABLE_RTREE
+LOCAL_CFLAGS += -DSQLITE_ENABLE_FTS3
+LOCAL_CFLAGS += -DSQLITE_ENABLE_BATCH_ATOMIC_WRITE
+
+# This is important - it causes SQLite to use memory for temp files. Since 
+# Android has no globally writable temp directory, if this is not defined the
+# application throws an exception when it tries to create a temp file.
+#
+LOCAL_CFLAGS += -DSQLITE_TEMP_STORE=3
+
+LOCAL_CFLAGS += -DHAVE_CONFIG_H -DKHTML_NO_EXCEPTIONS -DGKWQ_NO_JAVA
+LOCAL_CFLAGS += -DNO_SUPPORT_JS_BINDING -DQT_NO_WHEELEVENT -DKHTML_NO_XBL
+LOCAL_CFLAGS += -U__APPLE__
+LOCAL_CFLAGS += -DHAVE_STRCHRNUL=0
+LOCAL_CFLAGS += -DSQLITE_USE_URI=1
+LOCAL_CFLAGS += -Wno-unused-parameter -Wno-int-to-pointer-cast
+LOCAL_CFLAGS += -Wno-uninitialized -Wno-parentheses
+LOCAL_CPPFLAGS += -Wno-conversion-null
+
+ifeq ($(TARGET_ARCH), arm)
+	LOCAL_CFLAGS += -DPACKED="__attribute__ ((packed))"
+else
+	LOCAL_CFLAGS += -DPACKED=""
+endif
+
+LOCAL_SRC_FILES := sqlite3.c
+
+LOCAL_C_INCLUDES += $(LOCAL_PATH)
+LOCAL_EXPORT_C_INCLUDES := $(LOCAL_PATH)
+
+LOCAL_MODULE := libsqlite3
+LOCAL_LDLIBS += -ldl -llog 
+
+include $(BUILD_SHARED_LIBRARY)

data/raw/keybindings/keybindings.json

@@ -744,6 +744,13 @@
     "name": "Pick random world name",
     "bindings": [ { "input_method": "keyboard", "key": "*" } ]
   },
+  {
+    "type": "keybinding",
+    "id": "TOGGLE_V2_SAVE_FORMAT",
+    "category": "WORLDGEN_CONFIRM_DIALOG",
+    "name": "Toggle new world save format",
+    "bindings": [ { "input_method": "keyboard", "key": "=" } ]
+  },
   {
     "type": "keybinding",
     "id": "QUIT",

doc/src/content/docs/en/dev/guides/building/atomic_cmake.md

@@ -59,7 +59,8 @@ installation script. For Bazzite, that script looks like:
 ```sh
 $ sudo dnf install git cmake clang ninja-build mold ccache \
   SDL2-devel SDL2_image-devel SDL2_ttf-devel SDL2_mixer-devel \
-  freetype glibc bzip2 zlib-ng libvorbis ncurses gettext flac-devel
+  freetype glibc bzip2 zlib-ng libvorbis ncurses gettext flac-devel \
+  sqlite-devel zlib-devel
 ```
 
 After this, your container is set up for all your building needs in the future! Now, onto the steps

doc/src/content/docs/en/dev/guides/building/cmake.md

@@ -21,6 +21,7 @@ CataclysmBN:
   - `glibc`
   - `zlib`
   - `bzip2`
+  - `sqlite3`
 - Curses
   - `ncurses`
 - Tiles
@@ -50,15 +51,17 @@ Obtain packages specified above with your system package manager.
 ```sh
 $ sudo apt install git cmake ninja-build mold clang ccache \ 
   libsdl2-dev libsdl2-image-dev libsdl2-ttf-dev libsdl2-mixer-dev \ 
-  freetype glibc bzip2 zlib libvorbis ncurses gettext libflac++-dev
+  freetype glibc bzip2 zlib libvorbis ncurses gettext libflac++-dev \
+  libsqlite3-dev zlib1g-dev
 ```
 
 - For Fedora-based distros:
 
 ```sh
 $ sudo dnf install git cmake ninja-build mold clang ccache \
   SDL2-devel SDL2_image-devel SDL2_ttf-devel SDL2_mixer-devel \
-  freetype glibc bzip2 zlib-ng libvorbis ncurses gettext flac-devel
+  freetype glibc bzip2 zlib-ng libvorbis ncurses gettext flac-devel \
+  sqlite-devel zlib-devel
 ```
 
 ### Windows Environment (MSYS2)

doc/src/content/docs/en/dev/guides/building/makefile.md

@@ -58,7 +58,7 @@ your distro packages libraries and their development files separately (e.g. Debi
 
 Rough list based on building on Arch:
 
-- General: `gcc-libs`, `glibc`, `zlib`, `bzip2`
+- General: `gcc-libs`, `glibc`, `zlib`, `bzip2`, `sqlite3`
 - Optional: `intltool`
 - Curses: `ncurses`
 - Tiles: `sdl2`, `sdl2_image`, `sdl2_ttf`, `sdl2_mixer`, `freetype2`
@@ -137,7 +137,7 @@ Dependencies:
 Install:
 
 ```sh
-sudo apt-get install libncurses5-dev libncursesw5-dev build-essential astyle
+sudo apt-get install libncurses5-dev libncursesw5-dev build-essential astyle libsqlite3-dev zlib1g-dev
 ```
 
 ### Building

msvc-full-features/vcpkg.json

@@ -5,7 +5,8 @@
     "sdl2",
     { "name": "sdl2-image", "features": [ "libjpeg-turbo" ] },
     { "name": "sdl2-mixer", "features": [ "libflac", "mpg123", "libmodplug" ] },
-    "sdl2-ttf"
+    "sdl2-ttf",
+    "sqlite3", "zlib"
   ],
   "builtin-baseline": "b322364f06308bdd24823f9d8f03fe0cc86fd46f",
   "overrides": [

scripts/affected_files.ts

@@ -25,7 +25,7 @@ const getDiffs = async (pr: number) => {
     octokit.rest.pulls.listFiles,
     { owner: "cataclysmbnteam", repo: "Cataclysm-BN", pull_number: pr },
   )
-  const pathsFilter = new RegExp(`(^${paths.join("|")})/`)
+  const pathsFilter = new RegExp(`(^${paths.join("|")})/.*\\.(cpp|h|hpp)`)
 
   return res
     .filter((x) => ACMRT.has(x.status))
@@ -79,7 +79,7 @@ const main = new Command()
       getDiffs(pr).then((xs) => partition(xs, (x) => x.endsWith(".h"))),
     ])
     // console.log(deps)
-    // console.log({ src, headers })
+    console.log({ src, headers })
 
     const affected = new Set(src.concat(headers.flatMap((x) => Array.from(deps.get(x) ?? []))))
     console.log(affected)

src/Android.mk

@@ -17,9 +17,9 @@ LOCAL_SRC_FILES := $(FILE_LIST:$(LOCAL_PATH)/%=%)
 FILE_LIST_LUA := $(wildcard $(LOCAL_PATH)/lua/*.c)
 LOCAL_SRC_FILES += $(FILE_LIST_LUA:$(LOCAL_PATH)/%=%)
 
-LOCAL_SHARED_LIBRARIES := libhidapi SDL2 SDL2_mixer SDL2_image SDL2_ttf mpg123
+LOCAL_SHARED_LIBRARIES := libhidapi SDL2 SDL2_mixer SDL2_image SDL2_ttf mpg123 libsqlite3
 
-LOCAL_LDLIBS := -lGLESv1_CM -lGLESv2 -llog
+LOCAL_LDLIBS := -lGLESv1_CM -lGLESv2 -ldl -llog -lz
 
 LOCAL_CFLAGS += -DTILES=1 -DSDL_SOUND=1 -DLUA=1 -DBACKTRACE=1 -Wextra -Wall -fsigned-char -ffast-math
 

src/CMakeLists.txt

@@ -61,6 +61,8 @@ if (TILES)
 
     add_dependencies(cataclysm-bn-tiles-common get_version)
 
+    target_link_libraries(cataclysm-bn-tiles-common PUBLIC ZLIB::ZLIB)
+    target_link_libraries(cataclysm-bn-tiles-common PUBLIC SQLite::SQLite3)
     target_link_libraries(cataclysm-bn-tiles PRIVATE cataclysm-bn-tiles-common)
     target_compile_definitions(cataclysm-bn-tiles-common PUBLIC TILES )
 
@@ -176,9 +178,12 @@ if (CURSES)
     endif ()
 
     add_dependencies(cataclysm-bn-common get_version)
+
     target_link_libraries(cataclysm-bn PRIVATE cataclysm-bn-common)
 
     target_include_directories(cataclysm-bn-common PUBLIC ${CURSES_INCLUDE_DIR})
+    target_link_libraries(cataclysm-bn-common PUBLIC ZLIB::ZLIB)
+    target_link_libraries(cataclysm-bn-common PUBLIC SQLite::SQLite3)
     target_link_libraries(cataclysm-bn-common PUBLIC ${CURSES_LIBRARIES})
 
     if (CMAKE_USE_PTHREADS_INIT)

src/compress.cpp

@@ -0,0 +1,53 @@
+#include "compress.h"
+
+#include <zlib.h>
+#include <vector>
+#include <string>
+#include <stdexcept>
+#include <cstddef>
+
+void zlib_compress( const std::string &input, std::vector<std::byte> &output )
+{
+    uLongf compressedSize = compressBound( input.size() );
+    output.resize( compressedSize );
+
+    int result = compress2(
+                     reinterpret_cast<Bytef *>( output.data() ),
+                     &compressedSize,
+                     reinterpret_cast<const Bytef *>( input.data() ),
+                     input.size(),
+                     Z_BEST_SPEED
+                 );
+
+    if( result != Z_OK ) {
+        throw std::runtime_error( "Zlib compression error" );
+    }
+
+    output.resize( compressedSize );
+}
+
+void zlib_decompress( const void *compressed_data, int compressed_size, std::string &output )
+{
+    // We need to guess at the decompressed size - we expect things to compress fairly well.
+    uLongf decompressedSize = static_cast<uLongf>( compressed_size ) * 8;
+    output.resize( decompressedSize );
+
+    int result;
+    do {
+        result = uncompress(
+                     reinterpret_cast<Bytef *>( output.data() ),
+                     &decompressedSize,
+                     reinterpret_cast<const Bytef *>( compressed_data ),
+                     compressed_size
+                 );
+
+        if( result == Z_BUF_ERROR ) {
+            decompressedSize *= 2; // Double the buffer size and retry
+            output.resize( decompressedSize );
+        } else if( result != Z_OK ) {
+            throw std::runtime_error( "Zlib decompression failed" );
+        }
+    } while( result == Z_BUF_ERROR );
+
+    output.resize( decompressedSize );
+}