From b635776825e47a936da67485856340345ffc3648 Mon Sep 17 00:00:00 2001 From: Hennadiy Brych Date: Wed, 6 Dec 2023 21:16:55 -0500 Subject: [PATCH] Revert "sources reorg" This reverts commit 70e6f21d0768ff37ed3474467902a7be41aeb214. --- CMakeLists.txt | 5 +- cd/cd_dump.ixx | 4 +- cd/protection.ixx | 269 ---------------- cd/toc.ixx | 1 - drive.ixx | 1 - dump.ixx | 103 ------ dvd/dvd_key.ixx | 2 +- cd/offset_manager.ixx => offset_manager.ixx | 2 +- redumper.ixx | 1 - cd/split.ixx => split.ixx | 292 ++++++++++++++++- tests/CMakeLists.txt | 1 - tests/tests.cc | 1 - utils/misc.ixx | 328 +++++++++++++++++++- utils/strings.ixx | 239 +------------- 14 files changed, 618 insertions(+), 631 deletions(-) delete mode 100644 cd/protection.ixx rename cd/offset_manager.ixx => offset_manager.ixx (95%) rename cd/split.ixx => split.ixx (82%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 248f44c..d325db4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -128,10 +128,7 @@ target_sources(redumper "cd/cdrom.ixx" "cd/ecc.ixx" "cd/edc.ixx" - "cd/offset_manager.ixx" - "cd/protection.ixx" "cd/scrambler.ixx" - "cd/split.ixx" "cd/subcode.ixx" "cd/toc.ixx" "crc/crc.ixx" @@ -179,11 +176,13 @@ target_sources(redumper "drive.ixx" "hash.ixx" "info.ixx" + "offset_manager.ixx" "options.ixx" "redumper.ixx" "rings.ixx" "rom_entry.ixx" "skeleton.ixx" + "split.ixx" "version.ixx" ) diff --git a/cd/cd_dump.ixx b/cd/cd_dump.ixx index 6ad8a4e..7f47abf 100644 --- a/cd/cd_dump.ixx +++ b/cd/cd_dump.ixx @@ -1,6 +1,5 @@ module; #include -#include #include #include #include @@ -32,7 +31,6 @@ import utils.file_io; import utils.logger; import utils.misc; import utils.signal; -import utils.strings; @@ -365,7 +363,7 @@ uint32_t state_from_c2(std::vector &state, const uint8_t *c2_data) if(c2_quad) { state[i] = State::ERROR_C2; - c2_count += std::popcount(c2_quad); + c2_count += bits_count(c2_quad); } } diff --git a/cd/protection.ixx b/cd/protection.ixx deleted file mode 100644 index 95d3986..0000000 --- a/cd/protection.ixx +++ /dev/null @@ -1,269 +0,0 @@ -module; - -#include -#include -#include -#include -#include -#include -#include "throw_line.hh" - -export module cd.protection; - -import cd.cd; -import cd.toc; -import dump; -import filesystem.iso9660; -import options; -import readers.image_bin_form1_reader; -import utils.file_io; -import utils.logger; -import utils.strings; - - - -namespace gpsxre -{ - -export void redumper_protection(Context &ctx, Options &options) -{ - if(options.image_name.empty()) - throw_line("image name is not provided"); - - auto image_prefix = (std::filesystem::path(options.image_path) / options.image_name).string(); - - std::filesystem::path scm_path(image_prefix + ".scram"); - std::filesystem::path scp_path(image_prefix + ".scrap"); - std::filesystem::path sub_path(image_prefix + ".subcode"); - std::filesystem::path state_path(image_prefix + ".state"); - std::filesystem::path toc_path(image_prefix + ".toc"); - std::filesystem::path fulltoc_path(image_prefix + ".fulltoc"); - - bool scrap = !std::filesystem::exists(scm_path) && std::filesystem::exists(scp_path); - auto scra_path(scrap ? scp_path : scm_path); - - //TODO: rework - uint32_t sectors_count = check_file(state_path, CD_DATA_SIZE_SAMPLES); - - // TOC - std::vector toc_buffer = read_vector(toc_path); - TOC toc(toc_buffer, false); - - // FULL TOC - if(std::filesystem::exists(fulltoc_path)) - { - std::vector fulltoc_buffer = read_vector(fulltoc_path); - TOC toc_full(fulltoc_buffer, true); - if(toc_full.sessions.size() > 1) - toc = toc_full; - } - - { - auto &t = toc.sessions.back().tracks.back(); - - // fake TOC - if(t.lba_end < 0) - { - LOG("warning: fake TOC detected, using default 74min disc size"); - t.lba_end = MSF_to_LBA(MSF{74, 0, 0}); - } - - // incomplete dump (dumped with --stop-lba) - if(t.lba_end > (int32_t)sectors_count + LBA_START) - { - LOG("warning: incomplete dump detected, using available dump size"); - t.lba_end = (int32_t)sectors_count + LBA_START; - } - } - - std::fstream scm_fs(scra_path, std::fstream::in | std::fstream::binary); - if(!scm_fs.is_open()) - throw_line("unable to open file ({})", scra_path.filename().string()); - - std::fstream state_fs(state_path, std::fstream::in | std::fstream::binary); - if(!state_fs.is_open()) - throw_line("unable to open file ({})", state_path.filename().string()); - - std::string protection("N/A"); - - // SafeDisc - // first track is data - if(toc.sessions.size() == 1 && toc.sessions.front().tracks.size() >= 2) - { - auto &t = toc.sessions.front().tracks[0]; - auto &t_next = toc.sessions.front().tracks[1]; - - if(t.control & (uint8_t)ChannelQ::Control::DATA) - { - int32_t write_offset = track_offset_by_sync(t.lba_start, t_next.lba_start, state_fs, scm_fs); - - if(write_offset != std::numeric_limits::max()) - { - uint32_t file_offset = (t.lba_start - LBA_START) * CD_DATA_SIZE + write_offset * CD_SAMPLE_SIZE; - auto form1_reader = std::make_unique(scm_fs, file_offset, t_next.lba_start - t.lba_start, !scrap); - - iso9660::PrimaryVolumeDescriptor pvd; - if(iso9660::Browser::findDescriptor((iso9660::VolumeDescriptor &)pvd, form1_reader.get(), iso9660::VolumeDescriptorType::PRIMARY)) - { - auto root_directory = iso9660::Browser::rootDirectory(form1_reader.get(), pvd); - auto entry = root_directory->subEntry("00000001.TMP"); - if(entry) - { - std::vector state(CD_DATA_SIZE_SAMPLES); - - int32_t lba_start = entry->sectorsOffset() + entry->sectorsSize(); - int32_t lba_end = lba_start; - auto entries = root_directory->entries(); - for(auto &e : entries) - { - if(e->isDirectory()) - continue; - - auto entry_offset = e->sectorsOffset(); - if(entry_offset <= lba_start) - continue; - - if(lba_end == lba_start || entry_offset < lba_end) - lba_end = entry_offset; - } - - // verified: 78 - const std::set safedisc_c2_samples = { 60, 66, 72, 78 }; - - std::vector errors; - for(int32_t lba = lba_start; lba < lba_end; ++lba) - { - read_entry(state_fs, (uint8_t *)state.data(), CD_DATA_SIZE_SAMPLES, lba - LBA_START, 1, -write_offset, (uint8_t)State::ERROR_SKIP); - - uint32_t error_count = 0; - for(auto const &s : state) - if(s == State::ERROR_C2) - ++error_count; - - if(safedisc_c2_samples.find(error_count) != safedisc_c2_samples.end()) - errors.push_back(lba); - } - - if(!errors.empty()) - { - protection = std::format("SafeDisc {}, C2: {}, gap range: {}-{}", entry->name(), errors.size(), lba_start, lba_end - 1); - - auto skip_ranges = string_to_ranges(options.skip); - for(auto e : errors) - skip_ranges.emplace_back(e, e + 1); - options.skip = ranges_to_string(skip_ranges); - } - } - } - } - } - } - - // PS2 Datel - // only one data track - if(toc.sessions.size() == 1 && toc.sessions.front().tracks.size() == 2) - { - auto &t = toc.sessions.front().tracks[0]; - auto &t_next = toc.sessions.front().tracks[1]; - - if(t.control & (uint8_t)ChannelQ::Control::DATA) - { - std::vector state(CD_DATA_SIZE_SAMPLES); - - int32_t write_offset = track_offset_by_sync(t.lba_start, t_next.lba_start, state_fs, scm_fs); - if(write_offset != std::numeric_limits::max()) - { - // preliminary check - bool candidate = false; - { - constexpr int32_t lba_check = 50; - if(lba_check >= t.lba_start && lba_check < t_next.lba_start) - { - read_entry(state_fs, (uint8_t *)state.data(), CD_DATA_SIZE_SAMPLES, lba_check - LBA_START, 1, -write_offset, (uint8_t)State::ERROR_SKIP); - for(auto const &s : state) - if(s == State::ERROR_C2) - { - candidate = true; - break; - } - } - } - - if(candidate) - { - constexpr int32_t first_file_offset = 23; - - std::string protected_filename; - { - uint32_t file_offset = (t.lba_start - LBA_START) * CD_DATA_SIZE + write_offset * CD_SAMPLE_SIZE; - auto form1_reader = std::make_unique(scm_fs, file_offset, t_next.lba_start - t.lba_start, !scrap); - - iso9660::PrimaryVolumeDescriptor pvd; - if(iso9660::Browser::findDescriptor((iso9660::VolumeDescriptor &)pvd, form1_reader.get(), iso9660::VolumeDescriptorType::PRIMARY)) - { - auto root_directory = iso9660::Browser::rootDirectory(form1_reader.get(), pvd); - - static const std::string datel_files[] = { "DATA.DAT", "BIG.DAT", "DUMMY.ZIP" }; - for(auto const &f : datel_files) - { - // protection file exists - auto entry = root_directory->subEntry(f); - if(!entry) - continue; - - // first file on disc and starts from LBA 23 - if(entry->sectorsOffset() == first_file_offset) - { - protected_filename = entry->name(); - break; - } - } - } - } - - if(!protected_filename.empty()) - { - std::pair range(0, 0); - for(int32_t lba = first_file_offset, lba_end = std::min(t_next.lba_start, 5000); lba < lba_end; ++lba) - { - read_entry(state_fs, (uint8_t *)state.data(), CD_DATA_SIZE_SAMPLES, lba - LBA_START, 1, -write_offset, (uint8_t)State::ERROR_SKIP); - - bool error = false; - for(auto const &s : state) - if(s == State::ERROR_C2) - { - error = true; - break; - } - - if(error) - { - if(!range.first) - range.first = lba; - range.second = lba + 1; - } - else - { - if(range.first) - break; - } - } - - if(range.second > range.first) - { - protection = std::format("PS2/Datel {}, C2: {}, range: {}-{}", protected_filename, range.second - range.first, range.first, range.second - 1); - - auto skip_ranges = string_to_ranges(options.skip); - skip_ranges.push_back(range); - options.skip = ranges_to_string(skip_ranges); - } - } - } - } - } - } - - LOG("protection: {}", protection); -} - -} diff --git a/cd/toc.ixx b/cd/toc.ixx index 47bd9cc..39d192f 100644 --- a/cd/toc.ixx +++ b/cd/toc.ixx @@ -16,7 +16,6 @@ import crc.crc16_gsm; import scsi.mmc; import utils.endian; import utils.misc; -import utils.strings; diff --git a/drive.ixx b/drive.ixx index 7939294..a60010f 100644 --- a/drive.ixx +++ b/drive.ixx @@ -22,7 +22,6 @@ import utils.endian; import utils.file_io; import utils.logger; import utils.misc; -import utils.strings; diff --git a/dump.ixx b/dump.ixx index 2809336..6d641b2 100644 --- a/dump.ixx +++ b/dump.ixx @@ -12,8 +12,6 @@ module; export module dump; import cd.cd; -import cd.cdrom; -import cd.scrambler; import cd.subcode; import cd.toc; import drive; @@ -25,7 +23,6 @@ import utils.endian; import utils.file_io; import utils.logger; import utils.misc; -import utils.strings; @@ -286,104 +283,4 @@ export bool profile_is_hddvd(GET_CONFIGURATION_FeatureCode_ProfileList profile) || profile == GET_CONFIGURATION_FeatureCode_ProfileList::HDDVD_RW_DL; } - -export std::list> cue_get_entries(const std::filesystem::path &cue_path) -{ - std::list> entries; - - std::fstream fs(cue_path, std::fstream::in); - if(!fs.is_open()) - throw_line("unable to open file ({})", cue_path.filename().string()); - - std::pair entry; - std::string line; - while(std::getline(fs, line)) - { - auto tokens(tokenize(line, " \t", "\"\"")); - if(tokens.size() == 3) - { - if(tokens[0] == "FILE") - entry.first = tokens[1]; - else if(tokens[0] == "TRACK" && !entry.first.empty()) - { - entry.second = tokens[2] != "AUDIO"; - entries.push_back(entry); - entry.first.clear(); - } - } - } - - return entries; -} - - -//FIXME: just do regexp -export std::string track_extract_basename(std::string str) -{ - std::string basename = str; - - // strip extension - { - auto pos = basename.find_last_of('.'); - if(pos != std::string::npos) - basename = std::string(basename, 0, pos); - } - - // strip (Track X) - { - auto pos = str.find(" (Track "); - if(pos != std::string::npos) - basename = std::string(basename, 0, pos); - } - - return basename; -} - - -export int32_t track_offset_by_sync(int32_t lba_start, int32_t lba_end, std::fstream &state_fs, std::fstream &scm_fs) -{ - int32_t write_offset = std::numeric_limits::max(); - - constexpr uint32_t sectors_to_check = 2; - - std::vector data(sectors_to_check * CD_DATA_SIZE); - std::vector state(sectors_to_check * CD_DATA_SIZE_SAMPLES); - - uint32_t groups_count = (lba_end - lba_start) / sectors_to_check; - for(uint32_t i = 0; i < groups_count; ++i) - { - int32_t lba = lba_start + i * sectors_to_check; - read_entry(scm_fs, data.data(), CD_DATA_SIZE, lba - LBA_START, sectors_to_check, 0, 0); - read_entry(state_fs, (uint8_t *)state.data(), CD_DATA_SIZE_SAMPLES, lba - LBA_START, sectors_to_check, 0, (uint8_t)State::ERROR_SKIP); - - for(auto const &s : state) - if(s == State::ERROR_SKIP || s == State::ERROR_C2) - continue; - - auto it = std::search(data.begin(), data.end(), std::begin(CD_DATA_SYNC), std::end(CD_DATA_SYNC)); - if(it != data.end()) - { - auto sector_offset = (uint32_t)(it - data.begin()); - - // enough data for one sector - if(data.size() - sector_offset >= CD_DATA_SIZE) - { - Sector §or = *(Sector *)&data[sector_offset]; - Scrambler scrambler; - scrambler.descramble((uint8_t *)§or, nullptr); - - if(BCDMSF_valid(sector.header.address)) - { - int32_t sector_lba = BCDMSF_to_LBA(sector.header.address); - write_offset = ((int32_t)sector_offset - (sector_lba - lba) * (int32_t)CD_DATA_SIZE) / (int32_t)CD_SAMPLE_SIZE; - - break; - } - } - } - } - - return write_offset; -} - } diff --git a/dvd/dvd_key.ixx b/dvd/dvd_key.ixx index 2a5a388..15499e8 100644 --- a/dvd/dvd_key.ixx +++ b/dvd/dvd_key.ixx @@ -64,7 +64,7 @@ std::map> extract_vob_list(SectorRead if(e->isDirectory()) continue; - if(e->name().ends_with(".VOB")) + if(ends_with(e->name(), ".VOB")) titles[e->name()] = std::pair(e->sectorsOffset(), e->sectorsOffset() + e->sectorsSize()); } diff --git a/cd/offset_manager.ixx b/offset_manager.ixx similarity index 95% rename from cd/offset_manager.ixx rename to offset_manager.ixx index 958dee5..6b5e6fa 100644 --- a/cd/offset_manager.ixx +++ b/offset_manager.ixx @@ -4,7 +4,7 @@ module; #include #include "throw_line.hh" -export module cd.offset_manager; +export module offset_manager; diff --git a/redumper.ixx b/redumper.ixx index 8ffd389..31d0c0f 100644 --- a/redumper.ixx +++ b/redumper.ixx @@ -15,7 +15,6 @@ export module redumper; import cd.cd; import cd.dump; -import cd.protection; import cd.scrambler; import cd.split; import cd.subcode; diff --git a/cd/split.ixx b/split.ixx similarity index 82% rename from cd/split.ixx rename to split.ixx index 81eb1ef..d01d43b 100644 --- a/cd/split.ixx +++ b/split.ixx @@ -21,13 +21,13 @@ import cd.cd; import cd.cdrom; import cd.ecc; import cd.edc; -import cd.offset_manager; import cd.scrambler; import cd.subcode; import cd.toc; import dump; import filesystem.iso9660; import hash.sha1; +import offset_manager; import options; import readers.image_bin_form1_reader; import rom_entry; @@ -46,6 +46,53 @@ const uint32_t OFFSET_SHIFT_MAX_SECTORS = 4; const uint32_t OFFSET_SHIFT_SYNC_TOLERANCE = 2; +int32_t track_offset_by_sync(int32_t lba_start, int32_t lba_end, std::fstream &state_fs, std::fstream &scm_fs) +{ + int32_t write_offset = std::numeric_limits::max(); + + constexpr uint32_t sectors_to_check = 2; + + std::vector data(sectors_to_check * CD_DATA_SIZE); + std::vector state(sectors_to_check * CD_DATA_SIZE_SAMPLES); + + uint32_t groups_count = (lba_end - lba_start) / sectors_to_check; + for(uint32_t i = 0; i < groups_count; ++i) + { + int32_t lba = lba_start + i * sectors_to_check; + read_entry(scm_fs, data.data(), CD_DATA_SIZE, lba - LBA_START, sectors_to_check, 0, 0); + read_entry(state_fs, (uint8_t *)state.data(), CD_DATA_SIZE_SAMPLES, lba - LBA_START, sectors_to_check, 0, (uint8_t)State::ERROR_SKIP); + + for(auto const &s : state) + if(s == State::ERROR_SKIP || s == State::ERROR_C2) + continue; + + auto it = std::search(data.begin(), data.end(), std::begin(CD_DATA_SYNC), std::end(CD_DATA_SYNC)); + if(it != data.end()) + { + auto sector_offset = (uint32_t)(it - data.begin()); + + // enough data for one sector + if(data.size() - sector_offset >= CD_DATA_SIZE) + { + Sector §or = *(Sector *)&data[sector_offset]; + Scrambler scrambler; + scrambler.descramble((uint8_t *)§or, nullptr); + + if(BCDMSF_valid(sector.header.address)) + { + int32_t sector_lba = BCDMSF_to_LBA(sector.header.address); + write_offset = ((int32_t)sector_offset - (sector_lba - lba) * (int32_t)CD_DATA_SIZE) / (int32_t)CD_SAMPLE_SIZE; + + break; + } + } + } + } + + return write_offset; +} + + int32_t byte_offset_by_magic(int32_t lba_start, int32_t lba_end, std::fstream &state_fs, std::fstream &scm_fs, const std::vector &magic) { int32_t write_offset = std::numeric_limits::max(); @@ -854,6 +901,249 @@ void disc_offset_normalize_records(std::vector &records, s } +export void redumper_protection(Context &ctx, Options &options) +{ + if(options.image_name.empty()) + throw_line("image name is not provided"); + + auto image_prefix = (std::filesystem::path(options.image_path) / options.image_name).string(); + + std::filesystem::path scm_path(image_prefix + ".scram"); + std::filesystem::path scp_path(image_prefix + ".scrap"); + std::filesystem::path sub_path(image_prefix + ".subcode"); + std::filesystem::path state_path(image_prefix + ".state"); + std::filesystem::path toc_path(image_prefix + ".toc"); + std::filesystem::path fulltoc_path(image_prefix + ".fulltoc"); + + bool scrap = !std::filesystem::exists(scm_path) && std::filesystem::exists(scp_path); + auto scra_path(scrap ? scp_path : scm_path); + + //TODO: rework + uint32_t sectors_count = check_file(state_path, CD_DATA_SIZE_SAMPLES); + + // TOC + std::vector toc_buffer = read_vector(toc_path); + TOC toc(toc_buffer, false); + + // FULL TOC + if(std::filesystem::exists(fulltoc_path)) + { + std::vector fulltoc_buffer = read_vector(fulltoc_path); + TOC toc_full(fulltoc_buffer, true); + if(toc_full.sessions.size() > 1) + toc = toc_full; + } + + { + auto &t = toc.sessions.back().tracks.back(); + + // fake TOC + if(t.lba_end < 0) + { + LOG("warning: fake TOC detected, using default 74min disc size"); + t.lba_end = MSF_to_LBA(MSF{74, 0, 0}); + } + + // incomplete dump (dumped with --stop-lba) + if(t.lba_end > (int32_t)sectors_count + LBA_START) + { + LOG("warning: incomplete dump detected, using available dump size"); + t.lba_end = (int32_t)sectors_count + LBA_START; + } + } + + std::fstream scm_fs(scra_path, std::fstream::in | std::fstream::binary); + if(!scm_fs.is_open()) + throw_line("unable to open file ({})", scra_path.filename().string()); + + std::fstream state_fs(state_path, std::fstream::in | std::fstream::binary); + if(!state_fs.is_open()) + throw_line("unable to open file ({})", state_path.filename().string()); + + std::string protection("N/A"); + + // SafeDisc + // first track is data + if(toc.sessions.size() == 1 && toc.sessions.front().tracks.size() >= 2) + { + auto &t = toc.sessions.front().tracks[0]; + auto &t_next = toc.sessions.front().tracks[1]; + + if(t.control & (uint8_t)ChannelQ::Control::DATA) + { + int32_t write_offset = track_offset_by_sync(t.lba_start, t_next.lba_start, state_fs, scm_fs); + + if(write_offset != std::numeric_limits::max()) + { + uint32_t file_offset = (t.lba_start - LBA_START) * CD_DATA_SIZE + write_offset * CD_SAMPLE_SIZE; + auto form1_reader = std::make_unique(scm_fs, file_offset, t_next.lba_start - t.lba_start, !scrap); + + iso9660::PrimaryVolumeDescriptor pvd; + if(iso9660::Browser::findDescriptor((iso9660::VolumeDescriptor &)pvd, form1_reader.get(), iso9660::VolumeDescriptorType::PRIMARY)) + { + auto root_directory = iso9660::Browser::rootDirectory(form1_reader.get(), pvd); + auto entry = root_directory->subEntry("00000001.TMP"); + if(entry) + { + std::vector state(CD_DATA_SIZE_SAMPLES); + + int32_t lba_start = entry->sectorsOffset() + entry->sectorsSize(); + int32_t lba_end = lba_start; + auto entries = root_directory->entries(); + for(auto &e : entries) + { + if(e->isDirectory()) + continue; + + auto entry_offset = e->sectorsOffset(); + if(entry_offset <= lba_start) + continue; + + if(lba_end == lba_start || entry_offset < lba_end) + lba_end = entry_offset; + } + + // verified: 78 + const std::set safedisc_c2_samples = { 60, 66, 72, 78 }; + + std::vector errors; + for(int32_t lba = lba_start; lba < lba_end; ++lba) + { + read_entry(state_fs, (uint8_t *)state.data(), CD_DATA_SIZE_SAMPLES, lba - LBA_START, 1, -write_offset, (uint8_t)State::ERROR_SKIP); + + uint32_t error_count = 0; + for(auto const &s : state) + if(s == State::ERROR_C2) + ++error_count; + + if(safedisc_c2_samples.find(error_count) != safedisc_c2_samples.end()) + errors.push_back(lba); + } + + if(!errors.empty()) + { + protection = std::format("SafeDisc {}, C2: {}, gap range: {}-{}", entry->name(), errors.size(), lba_start, lba_end - 1); + + auto skip_ranges = string_to_ranges(options.skip); + for(auto e : errors) + skip_ranges.emplace_back(e, e + 1); + options.skip = ranges_to_string(skip_ranges); + } + } + } + } + } + } + + + // PS2 Datel + // only one data track + if(toc.sessions.size() == 1 && toc.sessions.front().tracks.size() == 2) + { + auto &t = toc.sessions.front().tracks[0]; + auto &t_next = toc.sessions.front().tracks[1]; + + if(t.control & (uint8_t)ChannelQ::Control::DATA) + { + std::vector state(CD_DATA_SIZE_SAMPLES); + + int32_t write_offset = track_offset_by_sync(t.lba_start, t_next.lba_start, state_fs, scm_fs); + if(write_offset != std::numeric_limits::max()) + { + // preliminary check + bool candidate = false; + { + constexpr int32_t lba_check = 50; + if(lba_check >= t.lba_start && lba_check < t_next.lba_start) + { + read_entry(state_fs, (uint8_t *)state.data(), CD_DATA_SIZE_SAMPLES, lba_check - LBA_START, 1, -write_offset, (uint8_t)State::ERROR_SKIP); + for(auto const &s : state) + if(s == State::ERROR_C2) + { + candidate = true; + break; + } + } + } + + if(candidate) + { + constexpr int32_t first_file_offset = 23; + + std::string protected_filename; + { + uint32_t file_offset = (t.lba_start - LBA_START) * CD_DATA_SIZE + write_offset * CD_SAMPLE_SIZE; + auto form1_reader = std::make_unique(scm_fs, file_offset, t_next.lba_start - t.lba_start, !scrap); + + iso9660::PrimaryVolumeDescriptor pvd; + if(iso9660::Browser::findDescriptor((iso9660::VolumeDescriptor &)pvd, form1_reader.get(), iso9660::VolumeDescriptorType::PRIMARY)) + { + auto root_directory = iso9660::Browser::rootDirectory(form1_reader.get(), pvd); + + static const std::string datel_files[] = { "DATA.DAT", "BIG.DAT", "DUMMY.ZIP" }; + for(auto const &f : datel_files) + { + // protection file exists + auto entry = root_directory->subEntry(f); + if(!entry) + continue; + + // first file on disc and starts from LBA 23 + if(entry->sectorsOffset() == first_file_offset) + { + protected_filename = entry->name(); + break; + } + } + } + } + + if(!protected_filename.empty()) + { + std::pair range(0, 0); + for(int32_t lba = first_file_offset, lba_end = std::min(t_next.lba_start, 5000); lba < lba_end; ++lba) + { + read_entry(state_fs, (uint8_t *)state.data(), CD_DATA_SIZE_SAMPLES, lba - LBA_START, 1, -write_offset, (uint8_t)State::ERROR_SKIP); + + bool error = false; + for(auto const &s : state) + if(s == State::ERROR_C2) + { + error = true; + break; + } + + if(error) + { + if(!range.first) + range.first = lba; + range.second = lba + 1; + } + else + { + if(range.first) + break; + } + } + + if(range.second > range.first) + { + protection = std::format("PS2/Datel {}, C2: {}, range: {}-{}", protected_filename, range.second - range.first, range.first, range.second - 1); + + auto skip_ranges = string_to_ranges(options.skip); + skip_ranges.push_back(range); + options.skip = ranges_to_string(skip_ranges); + } + } + } + } + } + } + + LOG("protection: {}", protection); +} + + export void redumper_split(Context &ctx, Options &options) { if(options.image_name.empty()) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 4cbfa81..f6317c5 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -19,7 +19,6 @@ target_sources(tests "${CMAKE_SOURCE_DIR}/crc/crc32.ixx" "${CMAKE_SOURCE_DIR}/utils/file_io.ixx" "${CMAKE_SOURCE_DIR}/utils/misc.ixx" - "${CMAKE_SOURCE_DIR}/utils/strings.ixx" ) add_test(NAME tests COMMAND tests WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}") diff --git a/tests/tests.cc b/tests/tests.cc index 515efc9..6567a2a 100644 --- a/tests/tests.cc +++ b/tests/tests.cc @@ -13,7 +13,6 @@ import crc.crc16_gsm; import crc.crc32; import utils.file_io; import utils.misc; -import utils.strings; diff --git a/utils/misc.ixx b/utils/misc.ixx index 6f9aa49..a0aee0e 100644 --- a/utils/misc.ixx +++ b/utils/misc.ixx @@ -1,18 +1,25 @@ module; #include -#include #include #include #include +#include +#include +#include #include #include +#include #include +#include +#include #include #include #include "throw_line.hh" export module utils.misc; +import cd.cd; + namespace gpsxre @@ -87,9 +94,13 @@ void clean_write(T *dst, size_t dst_offset, size_t size, T data) export template -bool is_zeroed(const T *data, size_t size) +bool is_zeroed(const T *data, uint64_t count) { - return std::all_of(data, data + size, [](T v){ return v == 0; }); + for(uint64_t i = 0; i < count; ++i) + if(data[i]) + return false; + + return true; } @@ -180,13 +191,25 @@ void bit_copy(T *dst, size_t dst_offset, const T *src, size_t src_offset, size_t } +export template +uint32_t bits_count(T value) +{ + uint32_t count = 0; + + for(; value; ++count) + value &= value - 1; + + return count; +} + + export template uint64_t bit_diff(const T *data1, const T *data2, uint64_t count) { uint64_t diff = 0; for(uint64_t i = 0; i < count; ++i) - diff += std::popcount(data1[i] ^ data2[i]); + diff += bits_count(data1[i] ^ data2[i]); return diff; } @@ -274,17 +297,6 @@ T digits_count(T value) } -export template -T sign_extend(T value) -{ - // clear extra bits - T v = value & (1 << bits) - 1; - - T m = (T)1 << bits - 1; - return (v ^ m) - m; -} - - export template bool batch_process_range(const std::pair &range, T batch_size, const std::function &func) { @@ -309,6 +321,228 @@ bool batch_process_range(const std::pair &range, T batch_size, const std:: } +export std::string normalize_string(const std::string &s) +{ + std::string ns; + + std::istringstream iss(s); + for(std::string token; std::getline(iss, token, ' '); ) + { + if(!token.empty()) + ns += token + ' '; + } + if(!ns.empty()) + ns.pop_back(); + + return ns; +} + + +export std::vector tokenize(const std::string &str, const char *delimiters, const char *quotes) +{ + std::vector tokens; + + std::set delimiter; + for(auto d = delimiters; *d != '\0'; ++d) + delimiter.insert(*d); + + bool in = false; + std::string::const_iterator s; + for(auto it = str.begin(); it < str.end(); ++it) + { + if(in) + { + // quoted + if(quotes != nullptr && *s == quotes[0]) + { + if(*it == quotes[1]) + { + ++s; + tokens.emplace_back(s, it); + in = false; + } + } + // unquoted + else + { + if(delimiter.find(*it) != delimiter.end()) + { + tokens.emplace_back(s, it); + in = false; + } + } + } + else + { + if(delimiter.find(*it) == delimiter.end()) + { + s = it; + in = true; + } + } + } + + // remaining entry + if(in) + tokens.emplace_back(s, str.end()); + + return tokens; +} + + +export std::optional str_to_uint64(std::string::const_iterator str_begin, std::string::const_iterator str_end) +{ + uint64_t value = 0; + + bool valid = false; + for(auto it = str_begin; it != str_end; ++it) + { + if(std::isdigit(*it)) + { + value = (value * 10) + (*it - '0'); + valid = true; + } + else + { + valid = false; + break; + } + } + + return valid ? std::make_optional(value) : std::nullopt; +} + + +export std::optional str_to_uint64(const std::string &str) +{ + return str_to_uint64(str.cbegin(), str.cend()); +} + + +export std::optional str_to_int64(std::string::const_iterator str_begin, std::string::const_iterator str_end) +{ + // empty string + auto it = str_begin; + if(it == str_end) + return std::nullopt; + + // preserve sign + int64_t negative = 1; + if(*it == '+') + ++it; + else if(*it == '-') + { + negative = -1; + ++it; + } + + if(auto value = str_to_uint64(it, str_end)) + return std::make_optional(negative * *value); + + return std::nullopt; +} + + +export std::optional str_to_int64(const std::string &str) +{ + return str_to_int64(str.cbegin(), str.cend()); +} + + +export int64_t str_to_int(const std::string &str) +{ + auto value = str_to_int64(str); + if(!value) + throw_line("string is not an integer number ({})", str); + + return *value; +} + + +export std::optional str_to_double(std::string::const_iterator str_begin, std::string::const_iterator str_end) +{ + // empty string + auto it = str_begin; + if(it == str_end) + return std::nullopt; + + // preserve sign + double negative = 1; + if(*it == '+') + ++it; + else if(*it == '-') + { + negative = -1; + ++it; + } + + auto dot = std::find(it, str_end, '.'); + + if(auto whole = str_to_uint64(it, dot)) + { + if(dot == str_end) + return std::make_optional(negative * *whole); + else + { + if(auto decimal = str_to_uint64(dot + 1, str_end)) + { + auto fraction = *decimal / pow(10, digits_count(*decimal)); + + return std::make_optional(negative * (*whole + fraction)); + } + } + } + + return std::nullopt; +} + +export double str_to_double(const std::string &str) +{ + auto value = str_to_double(str.cbegin(), str.cend()); + if(!value) + throw_line("string is not a double number ({})", str); + + return *value; +} + + +export std::vector> string_to_ranges(const std::string &str) +{ + std::vector> ranges; + + std::istringstream iss(str); + for(std::string range; std::getline(iss, range, ':'); ) + { + std::istringstream range_ss(range); + std::string s; + + std::getline(range_ss, s, '-'); + uint32_t lba_start = str_to_int(s); + + std::getline(range_ss, s, '-'); + uint32_t lba_end = str_to_int(s) + 1; + + ranges.emplace_back(lba_start, lba_end); + } + + return ranges; +} + + +export std::string ranges_to_string(const std::vector> &ranges) +{ + std::string str; + + for(auto const &r : ranges) + str += std::format("{}-{}:", r.first, r.second - 1); + + if(!str.empty()) + str.pop_back(); + + return str; +} + + export template const std::pair *inside_range(T value, const std::vector> &ranges) { @@ -329,6 +563,40 @@ export std::string system_date_time(std::string fmt) } +//FIXME: just do regexp +export std::string track_extract_basename(std::string str) +{ + std::string basename = str; + + // strip extension + { + auto pos = basename.find_last_of('.'); + if(pos != std::string::npos) + basename = std::string(basename, 0, pos); + } + + // strip (Track X) + { + auto pos = str.find(" (Track "); + if(pos != std::string::npos) + basename = std::string(basename, 0, pos); + } + + return basename; +} + + +export template +T sign_extend(T value) +{ + // clear extra bits + T v = value & (1 << bits) - 1; + + T m = (T)1 << bits - 1; + return (v ^ m) - m; +} + + export bool number_is_year(uint32_t year) { // reasonable unixtime range @@ -341,4 +609,34 @@ export bool number_is_month(uint32_t month) return month >= 1 && month <= 12; } + +export std::list> cue_get_entries(const std::filesystem::path &cue_path) +{ + std::list> entries; + + std::fstream fs(cue_path, std::fstream::in); + if(!fs.is_open()) + throw_line("unable to open file ({})", cue_path.filename().string()); + + std::pair entry; + std::string line; + while(std::getline(fs, line)) + { + auto tokens(tokenize(line, " \t", "\"\"")); + if(tokens.size() == 3) + { + if(tokens[0] == "FILE") + entry.first = tokens[1]; + else if(tokens[0] == "TRACK" && !entry.first.empty()) + { + entry.second = tokens[2] != "AUDIO"; + entries.push_back(entry); + entry.first.clear(); + } + } + } + + return entries; +} + } diff --git a/utils/strings.ixx b/utils/strings.ixx index 20b807b..9bf7697 100644 --- a/utils/strings.ixx +++ b/utils/strings.ixx @@ -3,17 +3,10 @@ module; #include #include #include -#include -#include -#include #include -#include -#include "throw_line.hh" export module utils.strings; -import utils.misc; - namespace gpsxre @@ -85,6 +78,15 @@ export std::string replace_all(std::string s, const std::string &from, const std } +export bool ends_with(const std::string &s, const std::string &suffix) +{ + if(suffix.size() > s.size()) + return false; + + return std::equal(suffix.rbegin(), suffix.rend(), s.rbegin()); +} + + export std::string str_uppercase(const std::string &s) { std::string str_uc; @@ -96,7 +98,6 @@ export std::string str_uppercase(const std::string &s) return str_uc; } - export std::string str_quoted_if_space(const std::string &s) { const std::string quote("\""); @@ -105,226 +106,4 @@ export std::string str_quoted_if_space(const std::string &s) } -export std::string normalize_string(const std::string &s) -{ - std::string ns; - - std::istringstream iss(s); - for(std::string token; std::getline(iss, token, ' '); ) - { - if(!token.empty()) - ns += token + ' '; - } - if(!ns.empty()) - ns.pop_back(); - - return ns; -} - - -export std::vector tokenize(const std::string &str, const char *delimiters, const char *quotes) -{ - std::vector tokens; - - std::set delimiter; - for(auto d = delimiters; *d != '\0'; ++d) - delimiter.insert(*d); - - bool in = false; - std::string::const_iterator s; - for(auto it = str.begin(); it < str.end(); ++it) - { - if(in) - { - // quoted - if(quotes != nullptr && *s == quotes[0]) - { - if(*it == quotes[1]) - { - ++s; - tokens.emplace_back(s, it); - in = false; - } - } - // unquoted - else - { - if(delimiter.find(*it) != delimiter.end()) - { - tokens.emplace_back(s, it); - in = false; - } - } - } - else - { - if(delimiter.find(*it) == delimiter.end()) - { - s = it; - in = true; - } - } - } - - // remaining entry - if(in) - tokens.emplace_back(s, str.end()); - - return tokens; -} - - -export std::optional str_to_uint64(std::string::const_iterator str_begin, std::string::const_iterator str_end) -{ - uint64_t value = 0; - - bool valid = false; - for(auto it = str_begin; it != str_end; ++it) - { - if(std::isdigit(*it)) - { - value = (value * 10) + (*it - '0'); - valid = true; - } - else - { - valid = false; - break; - } - } - - return valid ? std::make_optional(value) : std::nullopt; -} - - -export std::optional str_to_uint64(const std::string &str) -{ - return str_to_uint64(str.cbegin(), str.cend()); -} - - -export std::optional str_to_int64(std::string::const_iterator str_begin, std::string::const_iterator str_end) -{ - // empty string - auto it = str_begin; - if(it == str_end) - return std::nullopt; - - // preserve sign - int64_t negative = 1; - if(*it == '+') - ++it; - else if(*it == '-') - { - negative = -1; - ++it; - } - - if(auto value = str_to_uint64(it, str_end)) - return std::make_optional(negative * *value); - - return std::nullopt; -} - - -export std::optional str_to_int64(const std::string &str) -{ - return str_to_int64(str.cbegin(), str.cend()); -} - - -export int64_t str_to_int(const std::string &str) -{ - auto value = str_to_int64(str); - if(!value) - throw_line("string is not an integer number ({})", str); - - return *value; -} - - -export std::optional str_to_double(std::string::const_iterator str_begin, std::string::const_iterator str_end) -{ - // empty string - auto it = str_begin; - if(it == str_end) - return std::nullopt; - - // preserve sign - double negative = 1; - if(*it == '+') - ++it; - else if(*it == '-') - { - negative = -1; - ++it; - } - - auto dot = std::find(it, str_end, '.'); - - if(auto whole = str_to_uint64(it, dot)) - { - if(dot == str_end) - return std::make_optional(negative * *whole); - else - { - if(auto decimal = str_to_uint64(dot + 1, str_end)) - { - auto fraction = *decimal / pow(10, digits_count(*decimal)); - - return std::make_optional(negative * (*whole + fraction)); - } - } - } - - return std::nullopt; -} - - -export double str_to_double(const std::string &str) -{ - auto value = str_to_double(str.cbegin(), str.cend()); - if(!value) - throw_line("string is not a double number ({})", str); - - return *value; -} - - -export std::vector> string_to_ranges(const std::string &str) -{ - std::vector> ranges; - - std::istringstream iss(str); - for(std::string range; std::getline(iss, range, ':'); ) - { - std::istringstream range_ss(range); - std::string s; - - std::getline(range_ss, s, '-'); - uint32_t lba_start = str_to_int(s); - - std::getline(range_ss, s, '-'); - uint32_t lba_end = str_to_int(s) + 1; - - ranges.emplace_back(lba_start, lba_end); - } - - return ranges; -} - - -export std::string ranges_to_string(const std::vector> &ranges) -{ - std::string str; - - for(auto const &r : ranges) - str += std::format("{}-{}:", r.first, r.second - 1); - - if(!str.empty()) - str.pop_back(); - - return str; -} - }