From 1b42641018cbc958793429b53dcbf6f7adc93f12 Mon Sep 17 00:00:00 2001 From: Nerdu Date: Thu, 16 Sep 2021 14:52:17 -0400 Subject: [PATCH] Refactoring All archive repacking/unpacking logic has been moved to the tdmarchive module. The tool has been made more intuitive with command line arguments, no longer needing '--input' before the file/folder name. You can either put in just the file/folder name or an explicit 'extract' and 'repack' command. Only one JSON file is made now, condensing the info of the original two. --- README.md | 18 +- source/app.d | 594 ++++---------------------------------------- source/tdmarchive.d | 554 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 604 insertions(+), 562 deletions(-) create mode 100644 source/tdmarchive.d diff --git a/README.md b/README.md index a69fa24..36211e5 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,19 @@ # tdmextractor An Archive Extractor & Repacker for "The Denpa Men" series
This tool can replace the usage of the existing quickbms script for The Denpa Men 3 -
If you wish to extract archives from the other games, you can download the latest prerelease to do so, but compatibility with all archives is not yet 100%. +
Compatibility with all archives is not yet 100%, some archives from The Denpa Men 2 have problems with repacking. # Current Features -- Extraction of files from TDM3 archives -- Repacking to TDM3 version archives +- Extraction of files from TDM1/TDM2/TDM3/TDMF archives +- Repacking to TDM1/TDM2/TDM3/TDMF version archives # Planned Features -- Extraction of files from TDM1/TDM2/TDMF archives [Completed in Prerelease] -- Repacking to TDM1/TDM2/TDMF version archives [Completed in Prerelease] - Drag and Drop support for files/folders on Windows(Alternative is the use of batch files and the %1 operator) # Usage Run the executable in CLI/Terminal. The desired file/folder you wish to handle with the tool needs to be put in the arguments like so: -
`tdmextractor --input [name of archive/folder]` +
`tdmextractor [name of archive/folder]`
If you input a file, the archive will extract after some time, and a folder with the filename and an underscore will appear in your directory with the files. -
If you input a folder, the archive will be created after sometime and be named "output.bin", to which you can properly rename the archive for use in mods. -
If you attempt to extract an archive more than once in the same directory, it will fail because of the existing directory, meaning you will have to rename or delete it to extract again. -
This issue does not happen with repacking archives. +
If you input a folder, the archive will be created after sometime and be named "[foldername]_output", to which you can properly rename the archive for use in mods. +
If you attempt to extract an archive more than once in the same directory, you will be asked to write over existing files, you can choose to deny this and exit the program. +
Alternatively, you can make use of the 'extract' and 'repack' commands, though with these you must be exact with providing either an archive or a folder. # JSON Configuration When an archive is extracted, a JSON file is created and placed in the directory with the same name as the folder. This file contains information from the file header in the archive which is used for archive repacking. Note that these values are currently based off file headers found in The Denpa Men 3 archives. - "fileIndex": The index of the file inside the archive, its recommended to not change this value to avoid a game crash @@ -23,7 +21,7 @@ When an archive is extracted, a JSON file is created and placed in the directory - "unk": An unknown value that has an effect on how the file is read ingame, its purpose is unknown but it has important functions so its recommended to not change this value - "extension": The extension used by the particular file in the directory, its recommended to not change this value as you might crash in game - "isCompressed": Denotes whether or not this file was originally LZ77wii compressed inside the archive, there may be a relation to "unk" so you are welcome to experiment -- [PRERELEASE ONLY]"compressedUnk": In TDMF, the value after the compression mode can change depending on whether the file is compressed or not, this value is currently a placeholder in case this needs to be accurately recreated +- "compressedUnk": In TDMF, the value after the compression mode can change depending on whether the file is compressed or not, this value is currently a placeholder in case this needs to be accurately recreated # Building tdmextractor requires a D compiler(DMD is recommended), downloads can be found at https://dlang.org/.
Once installed, run `dub build` in your CLI/Terminal in the root directory of the repository to compile the project. # Contributing diff --git a/source/app.d b/source/app.d index ac310fa..8c5d8f1 100644 --- a/source/app.d +++ b/source/app.d @@ -1,243 +1,7 @@ -import binary.common; //Needed for some pack-d functions -import binary.pack; //For formatting data into specific types -import binary.reader; //For parsing data from raw byte arrays -import lz77helper; //Has helper functions for lz77wii compression -import std.algorithm; -import std.array; -import std.conv; +import tdmarchive; import std.file; -import std.format : format; -import std.getopt; import std.path; import std.stdio; -import std.string; -import vibe.data.json; //Used to write and read from the JSON helper file - -///Stores archive header information -struct ArchiveHeader { - ///Version reported by the archive - ArchiveVersion ver; -} - -///Stores file header information read from the archive -struct FileHeader { - ///Index of the file in the original archive - int fileIndex; - ///The strange first 4 bytes that seem unique to each file - string fileID; - ///A small number between 2-9 - uint unk; - ///The extension of the file - string extension; - ///True = LZ77wii compressed\nFalse = Uncompressed - bool isCompressed; - ///TDMF Exclusive: Related to compression mode(1 for uncompressed, 2 for lz77wii compressed) - uint compressedUnk = 0; -} - -///Used to determine extension of file, values refer to specific hex bytes in the header -enum KnownHeaders : ubyte[] -{ - BCH = [66, 67, 72, 0], - CGF = [67, 71, 70, 88], - ZIP = [80, 75, 3, 4], - DAR = [100, 97, 114, 99], - SAR = [83, 65, 82, 67], - NFC = [3, 0, 0, 0], //No clue what this is supposed to be but quickbms recognizes it - SDB = [2, 0, 0, 0], - VAP = [1, 0, 0, 0] - -} - -///Used to determine which game the archive belongs to -enum ArchiveVersion : uint -{ - TDM12 = 5, - TDM3 = 7, - TDMF = 10 -} - -///Grabs from a list of possible extensions and then renames them according to header data -///\nReturns a string that denotes the extension -string determineExtension(string filepath, string filename) { - try { - //Attempt to read header of file, and rename extension accordingly - auto data = cast(ubyte[]) read(filepath, 4); - if(data == KnownHeaders.BCH) { - rename(filepath, filename ~ ".bch"); - return "bch"; - } - if(data == KnownHeaders.CGF) { - rename(filepath, filename ~ ".cgf"); - return "cgf"; - } - if(data == KnownHeaders.ZIP) { - rename(filepath, filename ~ ".zip"); - return "zip"; - } - if(data == KnownHeaders.DAR) { - rename(filepath, filename ~ ".dar"); - return "dar"; - } - if(data == KnownHeaders.NFC) { - rename(filepath, filename ~ ".nfc"); - return "nfc"; - } - rename(filepath, filename ~ ".dat"); - return "dat"; - } catch (FileException ex) { - writeln("FAILED TO READ FROM FILE"); - throw ex; - } -} - -///Alternate version of lz77 compression found here https://github.com/Barubary/dsdecmp/blob/master/CSharp/DSDecmp/Formats/Nitro/LZ10.cs -ubyte[] packLZ77wiialt(File *infile) { - try { - //Setup data arrays - ubyte[] output; - auto fileLength = infile.size(); - //File cannot be larger than 16.77MB - if (fileLength > 16_777_215) { - throw new Exception("File cannot be more than 16.77MB large."); - } - ubyte[] fileData = new ubyte[fileLength]; - //Create Header for compressed file(Size of File and compression type in 4 bytes) - output ~= pack!`> 8) & 0x0F); - bufferLength++; - outbuffer[bufferLength] = cast(ubyte)(disp & 0xFF); - bufferLength++; - } else { - outbuffer[bufferLength] |= cast(ubyte)(((disp - 1) >> 8) & 0x0F); - bufferLength++; - outbuffer[bufferLength] = cast(ubyte)((disp - 1) & 0xFF); - bufferLength++; - } - } - bufferedBlocks++; - } - //Copy remaining blocks to the output - if (bufferedBlocks > 0) { - output ~= outbuffer; - compressedLength += bufferLength; - } - return output; - } catch (Exception e) { - throw new Exception("Compression Failed."); - } -} - -//Credit for Algorithm goes to Marcan at https://wiibrew.org/wiki/LZ77 -///Extract a file, decompressing it in the process -void extractLZ77wii(File *archive, FileHeader *fileheader, uint offset, string filename) { - //Begin by reading the header - ubyte[] data; - data.length = 4; - archive.seek(offset); - archive.rawRead(data); - auto reader = binaryReader(data); - const uint header = reader.read!uint(); - //Funny bit magic to read a byte and 3 bytes from uint - const auto uncompressedSize = header >> 8; //1 Byte - const auto compressionType = header >> 4 & 0xF; //3 Bytes - //Now we actually create the file and begin the decompression process - File extract = File (filename ~ ".bin", "wb"); - ubyte[] uncompressedData; - //LZ77wii compression makes use of chunks, each with a flag before them - //Flags are 8 bits where each bit represents a full byte of extracted - //data - //If a bit in the flag is 0, the corresponding byte in the chunk is - //raw data, if it is 1, it is a 2 byte reference to previous data - //where 1 Nibble is the length of data - 3 and 3 nibbles is the offset - //in uncompressed data used by the reference - while (uncompressedData.length < uncompressedSize) { - //Read Chunk Flag - data = []; - data.length = 1; - archive.rawRead(data); - reader.source(data); - auto flags = reader.read!ubyte(); - //Iterate through chunk using bitflag to determine reference or raw - for (int i = 0; i < 8; i++) { - //Is current bit set to 1? - if (flags & 0x80) { - //Read Reference - data = []; - data.length = 2; - archive.rawRead(data); - reader.source(data); - //Reference must be read BigEndian - reader.byteOrder = ByteOrder.BigEndian; - const auto info = reader.read!ushort(); - reader.byteOrder = ByteOrder.LittleEndian; - //Determine length of data - const auto num = 3 + ((info>>12)&0xF); - //TODO: Consider what to do with this since this is unused - const auto disp = info & 0xFFF; - //Determine offset in uncompressed data - auto ptr = uncompressedData.length - (info & 0xFFF) - 1; - for (int k = 0; k < num; k++) { - uncompressedData ~= uncompressedData[ptr]; - ptr += 1; - //small sanity check - if (uncompressedData.length >= uncompressedSize) - break; - } - } else { - data = []; - data.length = 1; - reader.source(data); - uncompressedData ~= archive.rawRead(data); - } - flags <<= 1; - //small sanity check - if (uncompressedData.length >= uncompressedSize) - break; - } - } - extract.rawWrite(uncompressedData); - extract.close(); - fileheader.extension = determineExtension(filename ~ ".bin", filename); -} int main(string[] args) { @@ -246,321 +10,47 @@ int main(string[] args) writeln("No arguments given. Please provide archive/folder."); return 1; } - string filename = "filename"; int workSuccess; - auto const argInfo = getopt(args, "input", &filename); - //Determine whether we are dealing with a file or a folder - if (filename.isFile()) { - //Argument is file, so begin by opening the archive file - writeln("Extracting Archive..."); - File archive = File(filename, "rb"); - workSuccess = extractArchive(archive); - } else { - //Argument is directory, so begin packing directory - writeln("Repacking Folder..."); - workSuccess = repackArchive(filename); - } - return workSuccess; -} - -///Takes a folder and repacks it into the proprietary archive format -int repackArchive(string filename) { - //Start by opening the json file inside the folder so we can determine how to go about repacking the folder - string offset = strip(filename, "_"); //Used whenever something cant be read at compile time - string jsonFilename = filename ~ "/" ~ strip(filename, "_") ~ ".json"; - string jsonArchver = filename ~ "/" ~ "version.json"; - ubyte[] compressedData; - ubyte[] headerData; - const string jsonData = readText(jsonFilename); - const string jsonVer = readText(jsonArchver); - const Json fileinfo = parseJsonString(jsonData); - const Json archinfo = parseJsonString(jsonVer); - FileHeader[] fileheaders; - ArchiveHeader archiveheader = deserializeJson!ArchiveHeader(archinfo); - writeln("Archive Version: ", archiveheader.ver); - fileheaders.length = fileinfo.length; - //Prepare output file - File outputArchive = File("output.bin", "wb"); - //Archive header needs to be written differently if we are packing a TDMF archive - if (archiveheader.ver != ArchiveVersion.TDMF) { - //Append Archive header - headerData ~= pack!` 16_777_215) { + throw new Exception("File cannot be more than 16.77MB large."); + } + ubyte[] fileData = new ubyte[fileLength]; + //Create Header for compressed file(Size of File and compression type in 4 bytes) + output ~= pack!`> 8) & 0x0F); + bufferLength++; + outbuffer[bufferLength] = cast(ubyte)(disp & 0xFF); + bufferLength++; + } else { + outbuffer[bufferLength] |= cast(ubyte)(((disp - 1) >> 8) & 0x0F); + bufferLength++; + outbuffer[bufferLength] = cast(ubyte)((disp - 1) & 0xFF); + bufferLength++; + } + } + bufferedBlocks++; + } + //Copy remaining blocks to the output + if (bufferedBlocks > 0) { + output ~= outbuffer; + compressedLength += bufferLength; + } + return output; + } catch (Exception e) { + throw new Exception("Compression Failed."); + } +} + +//Credit for Algorithm goes to Marcan at https://wiibrew.org/wiki/LZ77 +///Extract a file, decompressing it in the process +void extractLZ77wii(File archive, FileHeader fileheader, uint offset, string filename) { + //Begin by reading the header + ubyte[] data; + data.length = 4; + archive.seek(offset); + archive.rawRead(data); + auto reader = binaryReader(data); + const uint header = reader.read!uint(); + //Funny bit magic to read a byte and 3 bytes from uint + const auto uncompressedSize = header >> 8; //1 Byte + const auto compressionType = header >> 4 & 0xF; //3 Bytes + //Now we actually create the file and begin the decompression process + File extract = File (filename ~ ".bin", "wb"); + ubyte[] uncompressedData; + //LZ77wii compression makes use of chunks, each with a flag before them + //Flags are 8 bits where each bit represents a full byte of extracted + //data + //If a bit in the flag is 0, the corresponding byte in the chunk is + //raw data, if it is 1, it is a 2 byte reference to previous data + //where 1 Nibble is the length of data - 3 and 3 nibbles is the offset + //in uncompressed data used by the reference + while (uncompressedData.length < uncompressedSize) { + //Read Chunk Flag + data = []; + data.length = 1; + archive.rawRead(data); + reader.source(data); + auto flags = reader.read!ubyte(); + //Iterate through chunk using bitflag to determine reference or raw + for (int i = 0; i < 8; i++) { + //Is current bit set to 1? + if (flags & 0x80) { + //Read Reference + data = []; + data.length = 2; + archive.rawRead(data); + reader.source(data); + //Reference must be read BigEndian + reader.byteOrder = ByteOrder.BigEndian; + const auto info = reader.read!ushort(); + reader.byteOrder = ByteOrder.LittleEndian; + //Determine length of data + const auto num = 3 + ((info>>12)&0xF); + //TODO: Consider what to do with this since this is unused + const auto disp = info & 0xFFF; + //Determine offset in uncompressed data + auto ptr = uncompressedData.length - (info & 0xFFF) - 1; + for (int k = 0; k < num; k++) { + uncompressedData ~= uncompressedData[ptr]; + ptr += 1; + //small sanity check + if (uncompressedData.length >= uncompressedSize) + break; + } + } else { + data = []; + data.length = 1; + reader.source(data); + uncompressedData ~= archive.rawRead(data); + } + flags <<= 1; + //small sanity check + if (uncompressedData.length >= uncompressedSize) + break; + } + } + extract.rawWrite(uncompressedData); + extract.close(); +} + +///Takes a folder and repacks it into the proprietary archive format +int repackArchive(string filename) { + //Start by opening the json file inside the folder so we can determine how to go about repacking the folder + string offset = strip(filename, "_"); //Used whenever something cant be read at compile time + string jsonFilename = filename ~ "/" ~ strip(filename, "_") ~ ".json"; + //string jsonArchver = filename ~ "/" ~ "version.json"; + ubyte[] compressedData; + ubyte[] headerData; + const string jsonData = readText(jsonFilename); + //const string jsonVer = readText(jsonArchver); + const Json fileinfo = parseJsonString(jsonData); + //const Json archinfo = parseJsonString(jsonVer); + //FileHeader[] fileheaders; + //ArchiveHeader archiveheader = deserializeJson!ArchiveHeader(archinfo); + ArchiveInfo archInfo = deserializeJson!ArchiveInfo(fileinfo); + writeln("Archive Version: ", archInfo.archiveHeader.ver); + //fileheaders.length = fileinfo.length; + //Prepare output file + File outputArchive = File(filename ~ "output", "wb"); + //Archive header needs to be written differently if we are packing a TDMF archive + if (archInfo.archiveHeader.ver != ArchiveVersion.TDMF) { + //Append Archive header + headerData ~= pack!`