From c3d8fecad0ed9d583b9f1f79bc271e0535d87be2 Mon Sep 17 00:00:00 2001 From: Sculas Date: Thu, 4 Aug 2022 18:40:17 +0200 Subject: [PATCH] revert: feat: remove extra zipalign step This reverts commit c8e793efab8eed39b2cb564bee80ef6e0b2a7d03. --- .../app/revanced/cli/aligning/Aligning.kt | 12 +++ .../app/revanced/cli/command/MainCommand.kt | 9 +- .../app/revanced/cli/patcher/Patcher.kt | 16 +-- .../utils/filesystem/ZipFileSystemUtils.kt | 64 +++++++++++ .../utils/signing/align/ZipAligner.kt | 23 +++- .../utils/signing/align/zip/ZipFile.kt | 100 +++++------------- .../align/zip/structures/ZipEndRecord.kt | 3 +- .../signing/align/zip/structures/ZipEntry.kt | 48 +++------ 8 files changed, 154 insertions(+), 121 deletions(-) create mode 100644 src/main/kotlin/app/revanced/cli/aligning/Aligning.kt create mode 100644 src/main/kotlin/app/revanced/utils/filesystem/ZipFileSystemUtils.kt diff --git a/src/main/kotlin/app/revanced/cli/aligning/Aligning.kt b/src/main/kotlin/app/revanced/cli/aligning/Aligning.kt new file mode 100644 index 00000000..ef6d615d --- /dev/null +++ b/src/main/kotlin/app/revanced/cli/aligning/Aligning.kt @@ -0,0 +1,12 @@ +package app.revanced.cli.aligning + +import app.revanced.cli.command.MainCommand.logger +import app.revanced.utils.signing.align.ZipAligner +import java.io.File + +object Aligning { + fun align(inputFile: File, outputFile: File) { + logger.info("Aligning ${inputFile.name} to ${outputFile.name}") + ZipAligner.align(inputFile, outputFile) + } +} diff --git a/src/main/kotlin/app/revanced/cli/command/MainCommand.kt b/src/main/kotlin/app/revanced/cli/command/MainCommand.kt index 1c22cb4c..b610e61c 100644 --- a/src/main/kotlin/app/revanced/cli/command/MainCommand.kt +++ b/src/main/kotlin/app/revanced/cli/command/MainCommand.kt @@ -1,5 +1,6 @@ package app.revanced.cli.command +import app.revanced.cli.aligning.Aligning import app.revanced.cli.logging.impl.DefaultCliLogger import app.revanced.cli.patcher.Patcher import app.revanced.cli.patcher.logging.impl.PatcherLogger @@ -160,11 +161,15 @@ internal object MainCommand : Runnable { val cacheDirectory = File(pArgs.cacheDirectory) + // align the file + val alignedFile = cacheDirectory.resolve("${outputFile.nameWithoutExtension}_aligned.apk") + Aligning.align(patchedFile, alignedFile) + // sign the file val finalFile = if (!pArgs.mount) { val signedOutput = cacheDirectory.resolve("${outputFile.nameWithoutExtension}_signed.apk") Signing.sign( - patchedFile, + alignedFile, signedOutput, SigningOptions( pArgs.cn, @@ -177,7 +182,7 @@ internal object MainCommand : Runnable { signedOutput } else - patchedFile + alignedFile // finally copy to the specified output file logger.info("Copying ${finalFile.name} to ${outputFile.name}") diff --git a/src/main/kotlin/app/revanced/cli/patcher/Patcher.kt b/src/main/kotlin/app/revanced/cli/patcher/Patcher.kt index 776cb981..514ce6b8 100644 --- a/src/main/kotlin/app/revanced/cli/patcher/Patcher.kt +++ b/src/main/kotlin/app/revanced/cli/patcher/Patcher.kt @@ -2,12 +2,10 @@ package app.revanced.cli.patcher import app.revanced.cli.command.MainCommand.args import app.revanced.cli.command.MainCommand.logger +import app.revanced.utils.filesystem.ZipFileSystemUtils import app.revanced.utils.patcher.addPatchesFiltered import app.revanced.utils.patcher.applyPatchesVerbose import app.revanced.utils.patcher.mergeFiles -import app.revanced.utils.signing.align.ZipAligner -import app.revanced.utils.signing.align.zip.ZipFile -import app.revanced.utils.signing.align.zip.structures.ZipEntry import java.io.File import java.nio.file.Files @@ -25,22 +23,26 @@ internal object Patcher { // write output file if (output.exists()) Files.delete(output.toPath()) + inputFile.copyTo(output) val result = patcher.save() - ZipFile(output).use { outputFile -> + ZipFileSystemUtils(output).use { outputFileSystem -> // replace all dex files result.dexFiles.forEach { logger.info("Writing dex file ${it.name}") - outputFile.addEntryCompressData(ZipEntry.createWithName(it.name), it.dexFileInputStream.readAllBytes()) + outputFileSystem.write(it.name, it.dexFileInputStream.readAllBytes()) } if (!args.disableResourcePatching) { logger.info("Writing resources...") - outputFile.copyEntriesFromFileAligned(ZipFile(result.resourceFile!!), ZipAligner::getEntryAlignment) + ZipFileSystemUtils(result.resourceFile!!).use { resourceFileSystem -> + val resourceFiles = resourceFileSystem.getFile(File.separator) + outputFileSystem.writePathRecursively(resourceFiles) + } } - outputFile.copyEntriesFromFileAligned(ZipFile(inputFile), ZipAligner::getEntryAlignment) + result.doNotCompress?.let { outputFileSystem.uncompress(*it.toTypedArray()) } } } } diff --git a/src/main/kotlin/app/revanced/utils/filesystem/ZipFileSystemUtils.kt b/src/main/kotlin/app/revanced/utils/filesystem/ZipFileSystemUtils.kt new file mode 100644 index 00000000..cfe0c305 --- /dev/null +++ b/src/main/kotlin/app/revanced/utils/filesystem/ZipFileSystemUtils.kt @@ -0,0 +1,64 @@ +package app.revanced.utils.filesystem + +import java.io.Closeable +import java.io.File +import java.nio.file.FileSystems +import java.nio.file.Files +import java.nio.file.Path +import java.util.zip.ZipEntry + +internal class ZipFileSystemUtils( + file: File +) : Closeable { + private var zipFileSystem = FileSystems.newFileSystem(file.toPath(), mapOf("noCompression" to true)) + + private fun Path.deleteRecursively() { + if (!Files.exists(this)) { + throw IllegalStateException("File exists in real folder but not in zip file system") + } + + if (Files.isDirectory(this)) { + Files.list(this).forEach { path -> + path.deleteRecursively() + } + } + + Files.delete(this) + } + + internal fun getFile(path: String) = zipFileSystem.getPath(path) + + internal fun writePathRecursively(path: Path) { + Files.list(path).use { fileStream -> + fileStream.forEach { filePath -> + val fileSystemPath = filePath.getRelativePath(path) + fileSystemPath.deleteRecursively() + } + } + + Files.walk(path).use { fileStream -> + // don't include build directory + // by skipping the root node. + fileStream.skip(1).forEach { filePath -> + val relativePath = filePath.getRelativePath(path) + + if (Files.isDirectory(filePath)) { + Files.createDirectory(relativePath) + return@forEach + } + + Files.copy(filePath, relativePath) + } + } + } + + internal fun write(path: String, content: ByteArray) = Files.write(zipFileSystem.getPath(path), content) + + private fun Path.getRelativePath(path: Path): Path = zipFileSystem.getPath(path.relativize(this).toString()) + + // TODO: figure out why the file system is uncompressed by default and how to fix it + internal fun uncompress(vararg paths: String) = + paths.forEach { Files.setAttribute(zipFileSystem.getPath(it), "zip:method", ZipEntry.STORED) } + + override fun close() = zipFileSystem.close() +} \ No newline at end of file diff --git a/src/main/kotlin/app/revanced/utils/signing/align/ZipAligner.kt b/src/main/kotlin/app/revanced/utils/signing/align/ZipAligner.kt index be40ba1b..1a348f67 100644 --- a/src/main/kotlin/app/revanced/utils/signing/align/ZipAligner.kt +++ b/src/main/kotlin/app/revanced/utils/signing/align/ZipAligner.kt @@ -1,11 +1,28 @@ package app.revanced.utils.signing.align -import app.revanced.utils.signing.align.zip.structures.ZipEntry +import app.revanced.utils.signing.align.zip.ZipFile +import java.io.File internal object ZipAligner { private const val DEFAULT_ALIGNMENT = 4 private const val LIBRARY_ALIGNMENT = 4096 - fun getEntryAlignment(entry: ZipEntry): Int? = - if (entry.compression.toUInt() != 0u) null else if (entry.fileName.endsWith(".so")) LIBRARY_ALIGNMENT else DEFAULT_ALIGNMENT + fun align(input: File, output: File) { + val inputZip = ZipFile(input) + val outputZip = ZipFile(output) + + for (entry in inputZip.entries) { + val data = inputZip.getDataForEntry(entry) + + if (entry.compression == 0.toUShort()) { + val alignment = if (entry.fileName.endsWith(".so")) LIBRARY_ALIGNMENT else DEFAULT_ALIGNMENT + + outputZip.addEntryAligned(entry, data, alignment) + } else { + outputZip.addEntry(entry, data) + } + } + + outputZip.finish() + } } diff --git a/src/main/kotlin/app/revanced/utils/signing/align/zip/ZipFile.kt b/src/main/kotlin/app/revanced/utils/signing/align/zip/ZipFile.kt index d3a5ab54..41c63481 100644 --- a/src/main/kotlin/app/revanced/utils/signing/align/zip/ZipFile.kt +++ b/src/main/kotlin/app/revanced/utils/signing/align/zip/ZipFile.kt @@ -2,21 +2,15 @@ package app.revanced.utils.signing.align.zip import app.revanced.utils.signing.align.zip.structures.ZipEndRecord import app.revanced.utils.signing.align.zip.structures.ZipEntry -import java.io.Closeable import java.io.File import java.io.RandomAccessFile import java.nio.ByteBuffer import java.nio.channels.FileChannel -import java.util.zip.CRC32 -import java.util.zip.Deflater -class ZipFile(val file: File) : Closeable { +class ZipFile(val file: File) { var entries: MutableList = mutableListOf() private val filePointer: RandomAccessFile = RandomAccessFile(file, "rw") - private var CDNeedsRewrite = false - - private val compressionLevel = 5 init { //if file isn't empty try to load entries @@ -59,24 +53,23 @@ class ZipFile(val file: File) : Closeable { return buildList(numberOfEntries) { for (i in 1..numberOfEntries) { - add( - ZipEntry.fromCDE(filePointer).also - { - //for some reason the local extra field can be different from the central one - it.readLocalExtra( - filePointer.channel.map( - FileChannel.MapMode.READ_ONLY, - it.localHeaderOffset.toLong() + 28, - 2 - ) + add(ZipEntry.fromCDE(filePointer).also + { + //for some reason the local extra field can be different from the central one + it.readLocalExtra( + filePointer.channel.map( + FileChannel.MapMode.READ_ONLY, + it.localHeaderOffset.toLong() + 28, + 2 ) - }) + ) + }) } } } - private fun writeCD() { - val CDStart = filePointer.channel.position().toUInt() + private fun writeCDE() { + val CDEStart = filePointer.channel.position().toUInt() entries.forEach { filePointer.channel.write(it.toCDE()) @@ -89,17 +82,15 @@ class ZipFile(val file: File) : Closeable { 0u, entriesCount, entriesCount, - filePointer.channel.position().toUInt() - CDStart, - CDStart, + filePointer.channel.position().toUInt() - CDEStart, + CDEStart, "" ) filePointer.channel.write(endRecord.toECD()) } - private fun addEntry(entry: ZipEntry, data: ByteBuffer) { - CDNeedsRewrite = true - + fun addEntry(entry: ZipEntry, data: ByteBuffer) { entry.localHeaderOffset = filePointer.channel.position().toUInt() filePointer.channel.write(entry.toLFH()) @@ -108,45 +99,17 @@ class ZipFile(val file: File) : Closeable { entries.add(entry) } - fun addEntryCompressData(entry: ZipEntry, data: ByteArray) { - val compressor = Deflater(compressionLevel, true) - compressor.setInput(data) - compressor.finish() - - val uncompressedSize = data.size - val compressedData = - ByteArray(uncompressedSize) //i'm guessing compression won't make the data bigger - - val compressedDataLength = compressor.deflate(compressedData) - val compressedBuffer = - ByteBuffer.wrap(compressedData.take(compressedDataLength).toByteArray()) + fun addEntryAligned(entry: ZipEntry, data: ByteBuffer, alignment: Int) { + //calculate where data would end up + val dataOffset = filePointer.filePointer + entry.LFHSize - compressor.end() - - val crc = CRC32() - crc.update(data) - - entry.compression = 8u //deflate compression - entry.uncompressedSize = uncompressedSize.toUInt() - entry.compressedSize = compressedDataLength.toUInt() - entry.crc32 = crc.value.toUInt() - - addEntry(entry, compressedBuffer) - } + val mod = dataOffset % alignment - fun addEntryCopyData(entry: ZipEntry, data: ByteBuffer, alignment: Int? = null) { - alignment?.let { alignment -> - //calculate where data would end up - val dataOffset = filePointer.filePointer + entry.LFHSize - - val mod = dataOffset % alignment - - //wrong alignment - if (mod != 0L) { - //add padding at end of extra field - entry.localExtraField = - entry.localExtraField.copyOf((entry.localExtraField.size + (alignment - mod)).toInt()) - } + //wrong alignment + if (mod != 0L) { + //add padding at end of extra field + entry.localExtraField = + entry.localExtraField.copyOf((entry.localExtraField.size + (alignment - mod)).toInt()) } addEntry(entry, data) @@ -160,17 +123,8 @@ class ZipFile(val file: File) : Closeable { ) } - fun copyEntriesFromFileAligned(file: ZipFile, entryAlignment: (entry: ZipEntry) -> Int?) { - for (entry in file.entries) { - if (entries.any { it.fileName == entry.fileName }) continue //don't add duplicates - - val data = file.getDataForEntry(entry) - addEntryCopyData(entry, data, entryAlignment(entry)) - } - } - - override fun close() { - if (CDNeedsRewrite) writeCD() + fun finish() { + writeCDE() filePointer.close() } } diff --git a/src/main/kotlin/app/revanced/utils/signing/align/zip/structures/ZipEndRecord.kt b/src/main/kotlin/app/revanced/utils/signing/align/zip/structures/ZipEndRecord.kt index c95e6195..d26e551d 100644 --- a/src/main/kotlin/app/revanced/utils/signing/align/zip/structures/ZipEndRecord.kt +++ b/src/main/kotlin/app/revanced/utils/signing/align/zip/structures/ZipEndRecord.kt @@ -58,8 +58,7 @@ data class ZipEndRecord( fun toECD(): ByteBuffer { val commentBytes = fileComment.toByteArray(Charsets.UTF_8) - val buffer = ByteBuffer.allocate(ECD_HEADER_SIZE + commentBytes.size) - .also { it.order(ByteOrder.LITTLE_ENDIAN) } + val buffer = ByteBuffer.allocate(ECD_HEADER_SIZE + commentBytes.size).also { it.order(ByteOrder.LITTLE_ENDIAN) } buffer.putUInt(ECD_SIGNATURE) buffer.putUShort(diskNumber) diff --git a/src/main/kotlin/app/revanced/utils/signing/align/zip/structures/ZipEntry.kt b/src/main/kotlin/app/revanced/utils/signing/align/zip/structures/ZipEntry.kt index ab5f2c10..9e70f526 100644 --- a/src/main/kotlin/app/revanced/utils/signing/align/zip/structures/ZipEntry.kt +++ b/src/main/kotlin/app/revanced/utils/signing/align/zip/structures/ZipEntry.kt @@ -1,6 +1,10 @@ package app.revanced.utils.signing.align.zip.structures -import app.revanced.utils.signing.align.zip.* +import app.revanced.utils.signing.align.zip.getUShort +import app.revanced.utils.signing.align.zip.putUInt +import app.revanced.utils.signing.align.zip.putUShort +import app.revanced.utils.signing.align.zip.readUIntLE +import app.revanced.utils.signing.align.zip.readUShortLE import java.io.DataInput import java.nio.ByteBuffer import java.nio.ByteOrder @@ -9,12 +13,12 @@ data class ZipEntry( val version: UShort, val versionNeeded: UShort, val flags: UShort, - var compression: UShort, + val compression: UShort, val modificationTime: UShort, val modificationDate: UShort, - var crc32: UInt, - var compressedSize: UInt, - var uncompressedSize: UInt, + val crc32: UInt, + val compressedSize: UInt, + val uncompressedSize: UInt, val diskNumber: UShort, val internalAttributes: UShort, val externalAttributes: UInt, @@ -22,7 +26,7 @@ data class ZipEntry( val fileName: String, val extraField: ByteArray, val fileComment: String, - var localExtraField: ByteArray = ByteArray(0), //separate for alignment + var localExtraField: ByteArray = ByteArray(0), //seperate for alignment ) { val LFHSize: Int get() = LFH_HEADER_SIZE + fileName.toByteArray(Charsets.UTF_8).size + localExtraField.size @@ -37,27 +41,6 @@ data class ZipEntry( const val LFH_HEADER_SIZE = 30 const val LFH_SIGNATURE = 0x04034b50u - fun createWithName(fileName: String): ZipEntry { - return ZipEntry( - 0x1403u, //made by unix, version 20 - 0u, - 0u, - 0u, - 0x0821u, //seems to be static time google uses, no idea - 0x0221u, //same as above - 0u, - 0u, - 0u, - 0u, - 0u, - 0u, - 0u, - fileName, - ByteArray(0), - "" - ) - } - fun fromCDE(input: DataInput): ZipEntry { val signature = input.readUIntLE() @@ -84,8 +67,7 @@ data class ZipEntry( val externalAttributes = input.readUIntLE() val localHeaderOffset = input.readUIntLE() - val variableFieldsLength = - fileNameLength.toInt() + extraFieldLength.toInt() + fileCommentLength.toInt() + val variableFieldsLength = fileNameLength.toInt() + extraFieldLength.toInt() + fileCommentLength.toInt() if (variableFieldsLength > 0) { val fileNameBytes = ByteArray(fileNameLength.toInt()) @@ -99,8 +81,7 @@ data class ZipEntry( fileComment = fileCommentBytes.toString(Charsets.UTF_8) } - flags = (flags and 0b1000u.inv() - .toUShort()) //disable data descriptor flag as they are not used + flags = (flags and 0b1000u.inv().toUShort()) //disable data descriptor flag as they are not used return ZipEntry( version, @@ -157,9 +138,8 @@ data class ZipEntry( val nameBytes = fileName.toByteArray(Charsets.UTF_8) val commentBytes = fileComment.toByteArray(Charsets.UTF_8) - val buffer = - ByteBuffer.allocate(CDE_HEADER_SIZE + nameBytes.size + extraField.size + commentBytes.size) - .also { it.order(ByteOrder.LITTLE_ENDIAN) } + val buffer = ByteBuffer.allocate(CDE_HEADER_SIZE + nameBytes.size + extraField.size + commentBytes.size) + .also { it.order(ByteOrder.LITTLE_ENDIAN) } buffer.putUInt(CDE_SIGNATURE) buffer.putUShort(version)