Support compressed ELF sections

CMakeLists.txt

@@ -145,6 +145,8 @@ target_sources(
     src/unwind/unwind_with_nothing.cpp
     src/unwind/unwind_with_unwind.cpp
     src/unwind/unwind_with_winapi.cpp
+    src/utils/decompress/decompress_zlib.cpp
+    src/utils/decompress/decompress_zstd.cpp
     src/utils/io/file.cpp
     src/utils/io/memory_file_view.cpp
     src/utils/error.cpp

src/binary/elf.cpp

@@ -1,5 +1,8 @@
 #include "binary/elf.hpp"
+#include <memory>
 
+#include "utils/decompress/decompress_zlib.h"
+#include "utils/decompress/decompress_zstd.h"
 #include "utils/error.hpp"
 #include "utils/io/base_file.hpp"
 #include "utils/io/memory_file_view.hpp"
@@ -293,6 +296,7 @@ namespace detail {
             section_info info;
             info.sh_name = byteswap_if_needed(section_header.sh_name);
             info.sh_type = byteswap_if_needed(section_header.sh_type);
+            info.sh_flags = byteswap_if_needed(section_header.sh_flags);
             info.sh_addr = byteswap_if_needed(section_header.sh_addr);
             info.sh_offset = byteswap_if_needed(section_header.sh_offset);
             info.sh_size = byteswap_if_needed(section_header.sh_size);
@@ -304,6 +308,59 @@ namespace detail {
         return sections;
     }
 
+    template<typename T>
+    Result<std::vector<T>, internal_error> elf::read_compressed_section(const section_info& section) {
+        if(is_64) {
+            return this->read_compressed_section_impl<T, 64>(section);
+        } else {
+            return this->read_compressed_section_impl<T, 32>(section);
+        }
+    }
+
+    template<typename T, std::size_t Bits>
+    Result<std::vector<T>, internal_error> elf::read_compressed_section_impl(const section_info& section) {
+        static_assert(Bits == 32 || Bits == 64, "Unexpected Bits argument");
+        using CHeader = typename std::conditional<Bits == 32, Elf32_Chdr, Elf64_Chdr>::type;
+        if((section.sh_flags & SHF_COMPRESSED) == 0) {
+            return internal_error("requested section is not compressed");
+        }
+        if(section.sh_entsize > 0 && section.sh_entsize != sizeof(T)) {
+            return internal_error("compressed section entsize mismatch: {} != {}", section.sh_entsize, sizeof(T));
+        }
+        auto loaded_sh = file->read<CHeader>(static_cast<off_t>(section.sh_offset));
+        if(loaded_sh.is_error()) {
+            return std::move(loaded_sh).unwrap_error();
+        }
+        const CHeader& comp_header = loaded_sh.unwrap_value();
+        auto decompression_type = byteswap_if_needed(comp_header.ch_type);
+        decltype(&decompress_zlib) decompress_function = nullptr;
+        switch(decompression_type) {
+            case ELFCOMPRESS_ZLIB:
+                decompress_function = decompress_zlib;
+                break;
+            case ELFCOMPRESS_ZSTD:
+                decompress_function = decompress_zstd;
+                break;
+            default:
+                return internal_error("unsupported compression type {} in {}", decompression_type, file->path());
+        }
+        auto decompressed_size = byteswap_if_needed(comp_header.ch_size);
+        if(decompressed_size % sizeof(T) != 0) {
+            return internal_error("decompressed size not a multiple of entry size");
+        }
+        std::vector<T> buffer(decompressed_size / sizeof(T));
+        auto decompress_res = decompress_function(
+            cpptrace::detail::bspan(reinterpret_cast<char*>(buffer.data()), decompressed_size),
+            *file,
+            static_cast<off_t>(section.sh_offset + sizeof(CHeader)),
+            section.sh_size - sizeof(CHeader)
+        );
+        if(!decompress_res) {
+            return decompress_res.unwrap_error();
+        }
+        return buffer;
+    }
+
     Result<const std::vector<char>&, internal_error> elf::get_strtab(std::size_t index) {
         auto res = strtab_entries.insert({index, {}});
         auto it = res.first;
@@ -330,12 +387,21 @@ namespace detail {
         if(section.sh_type != SHT_STRTAB) {
             return internal_error("requested strtab section not a strtab (requested {} of {})", index, file->path());
         }
-        entry.data.resize(section.sh_size + 1);
-        auto read_res = file->read_bytes(span<char>{entry.data.data(), section.sh_size}, section.sh_offset);
-        if(!read_res) {
-            return read_res.unwrap_error();
+        if(section.sh_flags & SHF_COMPRESSED) {
+            auto decompressed = read_compressed_section<char>(section);
+            if(decompressed.is_error()) {
+                return std::move(decompressed).unwrap_error();
+            }
+            entry.data = std::move(decompressed.unwrap_value());
+            entry.data.push_back(0); // null-terminate for safety
+        } else {
+            entry.data.resize(section.sh_size + 1);
+            auto read_res = file->read_bytes(span<char>{entry.data.data(), section.sh_size}, section.sh_offset);
+            if(!read_res) {
+                return read_res.unwrap_error();
+            }
+            entry.data[section.sh_size] = 0; // just out of an abundance of caution
         }
-        entry.data[section.sh_size] = 0; // just out of an abundance of caution
         entry.did_load_strtab = true;
         return entry.data;
     }
@@ -416,13 +482,22 @@ namespace detail {
                 if(section.sh_entsize != sizeof(SymEntry)) {
                     return internal_error("elf seems corrupted, sym entry mismatch {}", file->path());
                 }
-                if(section.sh_size % section.sh_entsize != 0) {
-                    return internal_error("elf seems corrupted, sym entry vs section size mismatch {}", file->path());
-                }
-                std::vector<SymEntry> buffer(section.sh_size / section.sh_entsize);
-                auto res = file->read_span(make_span(buffer.begin(), buffer.end()), section.sh_offset);
-                if(!res) {
-                    return res.unwrap_error();
+                std::vector<SymEntry> buffer;
+                if(section.sh_flags & SHF_COMPRESSED) {
+                    auto decompressed = read_compressed_section<SymEntry>(section);
+                    if(!decompressed) {
+                        return decompressed.unwrap_error();
+                    }
+                    buffer = std::move(decompressed.unwrap_value());
+                } else {
+                    if(section.sh_size % section.sh_entsize != 0) {
+                        return internal_error("elf seems corrupted, sym entry vs section size mismatch {}", file->path());
+                    }
+                    buffer.resize(section.sh_size / section.sh_entsize);
+                    auto res = file->read_span(make_span(buffer.begin(), buffer.end()), static_cast<off_t>(section.sh_offset));
+                    if(!res) {
+                        return res.unwrap_error();
+                    }
                 }
                 symbol_table = symtab_info{};
                 symbol_table.unwrap().entries.reserve(buffer.size());

src/binary/elf.hpp

@@ -36,6 +36,7 @@ namespace detail {
         struct section_info {
             uint32_t sh_name;
             uint32_t sh_type;
+            uint32_t sh_flags;
             uint64_t sh_addr;
             uint64_t sh_offset;
             uint64_t sh_size;
@@ -127,6 +128,11 @@ namespace detail {
         template<std::size_t Bits>
         Result<const std::vector<section_info>&, internal_error> get_sections_impl();
 
+        template<typename T>
+        Result<std::vector<T>, internal_error> read_compressed_section(const section_info& section);
+        template<typename T, std::size_t Bits>
+        Result<std::vector<T>, internal_error> read_compressed_section_impl(const section_info& section);
+
         Result<const std::vector<char>&, internal_error> get_strtab(std::size_t index);
 
         Result<const optional<symtab_info>&, internal_error> get_symtab();

src/utils/decompress/decompress_zlib.cpp

@@ -0,0 +1,66 @@
+#include "utils/decompress/decompress_zlib.h"
+
+#include <zlib.h>
+#include "utils/utils.hpp"
+
+namespace cpptrace {
+namespace detail {
+
+Result<monostate, internal_error> decompress_zlib(
+    bspan decompressed_data,
+    base_file& compressed_file,
+    off_t offset,
+    size_t compressed_size
+) {
+    // zlib docs provide an example of 16K and also says:
+    // Larger buffer sizes would be more efficient, especially for inflate(). If the memory is
+    // available, buffers sizes on the order of 128K or 256K bytes should be used.
+    constexpr size_t kChunkSize = 262144; // 256K
+    z_stream strm{};
+    int ret = inflateInit(&strm);
+    if(ret != Z_OK) {
+        return internal_error("zlib inflateInit failed");
+    }
+    std::unique_ptr<z_stream, decltype(&inflateEnd)> strm_raii(&strm, inflateEnd);
+    size_t total_read = 0;
+    size_t total_written = 0;
+    std::vector<char> chunk_buffer(kChunkSize);
+    strm.next_out = reinterpret_cast<Bytef*>(decompressed_data.data());
+    strm.avail_out = static_cast<uInt>(decompressed_data.size());
+    while(total_read < compressed_size) {
+        size_t to_read = std::min(kChunkSize, compressed_size - total_read);
+        auto read_res = compressed_file.read_span(
+            cpptrace::detail::make_span(chunk_buffer.begin(), chunk_buffer.begin() + static_cast<std::ptrdiff_t>(to_read)),
+            static_cast<off_t>(offset + total_read)
+        );
+        if(!read_res) {
+            return read_res.unwrap_error();
+        }
+        strm.next_in = reinterpret_cast<Bytef*>(chunk_buffer.data());
+        strm.avail_in = static_cast<uInt>(to_read);
+        while(strm.avail_in > 0) {
+            ret = inflate(&strm, Z_NO_FLUSH);
+            if(ret == Z_STREAM_END) {
+                break;
+            }
+            if(ret != Z_OK) {
+                return internal_error("zlib inflate failed");
+            }
+        }
+        total_read += to_read;
+        total_written = strm.total_out;
+        if(ret == Z_STREAM_END) {
+            break;
+        }
+    }
+    if(ret != Z_STREAM_END) {
+        return internal_error("zlib did not reach stream end");
+    }
+    if(total_written != decompressed_data.size()) {
+        return internal_error("zlib decompressed size mismatch");
+    }
+    return monostate{};
+}
+
+}  // namespace detail
+}

src/utils/decompress/decompress_zlib.h

@@ -0,0 +1,22 @@
+#ifndef DECOMPRESS_ZLIB_HPP
+#define DECOMPRESS_ZLIB_HPP
+
+#include "utils/error.hpp"
+#include "utils/io/base_file.hpp"
+#include "utils/result.hpp"
+#include "utils/span.hpp"
+
+namespace cpptrace {
+namespace detail {
+
+Result<monostate, internal_error> decompress_zlib(
+    bspan decompressed_data,
+    base_file& compressed_file,
+    off_t offset,
+    size_t compressed_size
+);
+
+}
+}
+
+#endif

src/utils/decompress/decompress_zstd.cpp

@@ -0,0 +1,70 @@
+#include "utils/decompress/decompress_zstd.h"
+
+#include <zstd.h> // For ZSTD_*, ZSTD_DStream, etc.
+
+#include <vector>
+#include <memory>
+
+namespace cpptrace {
+namespace detail {
+
+Result<monostate, internal_error> decompress_zstd(
+    bspan decompressed_data,
+    base_file& compressed_file,
+    off_t offset,
+    size_t compressed_size
+) {
+    std::unique_ptr<ZSTD_DStream, decltype(&ZSTD_freeDStream)> dstream(ZSTD_createDStream(), ZSTD_freeDStream);
+    if(!dstream) {
+        return internal_error("ZSTD_createDStream failed");
+    }
+    size_t init_ret = ZSTD_initDStream(dstream.get());
+    if(ZSTD_isError(init_ret)) {
+        return internal_error(std::string("ZSTD_initDStream failed: ") + ZSTD_getErrorName(init_ret));
+    }
+
+    static const size_t CHUNK_SIZE = ZSTD_DStreamInSize();
+    std::vector<char> chunk_buffer(CHUNK_SIZE);
+
+    ZSTD_outBuffer output = {};
+    output.dst = decompressed_data.data();
+    output.size = decompressed_data.size();
+    output.pos = 0;
+
+    size_t total_read = 0;
+    while(total_read < compressed_size) {
+        size_t to_read = std::min(CHUNK_SIZE, compressed_size - total_read);
+        auto read_res = compressed_file.read_span(
+            cpptrace::detail::make_span(chunk_buffer.begin(), chunk_buffer.begin() + static_cast<std::ptrdiff_t>(to_read)),
+            static_cast<off_t>(offset + total_read)
+        );
+        if(!read_res) {
+            return read_res.unwrap_error();
+        }
+
+        ZSTD_inBuffer input = {};
+        input.src = chunk_buffer.data();
+        input.size = to_read;
+        input.pos = 0;
+
+        while(input.pos < input.size) {
+            size_t decompress_ret = ZSTD_decompressStream(dstream.get(), &output, &input);
+            if(ZSTD_isError(decompress_ret)) {
+                return internal_error(std::string("ZSTD_decompressStream failed: ") + ZSTD_getErrorName(decompress_ret));
+            }
+            if(decompress_ret == 0) {
+                break; 
+            }
+        }
+        total_read += to_read;
+    }
+
+    if(output.pos != decompressed_data.size()) {
+        return internal_error("zstd decompressed size mismatch");
+    }
+
+    return monostate{};
+}
+
+} // namespace detail
+} // namespace cpptrace

src/utils/decompress/decompress_zstd.h

@@ -0,0 +1,22 @@
+#ifndef DECOMPRESS_ZSTD_HPP
+#define DECOMPRESS_ZSTD_HPP
+
+#include "utils/error.hpp"
+#include "utils/result.hpp"
+#include "utils/io/base_file.hpp"
+#include "utils/span.hpp" // For bspan
+
+namespace cpptrace {
+namespace detail {
+
+Result<monostate, internal_error> decompress_zstd(
+    bspan decompressed_data,
+    base_file& compressed_file,
+    off_t offset,
+    size_t compressed_size
+);
+
+}
+}
+
+#endif