From a942a57364777d6ab6fcb11e6171cc4b496c8540 Mon Sep 17 00:00:00 2001 From: bogadana <30848157+bogadana@users.noreply.github.com> Date: Fri, 15 Jul 2022 04:24:45 +0200 Subject: [PATCH] feat: more efficient zipalign --- .../utils/signing/align/ZipAligner.kt | 64 ++----- .../signing/align/stream/MultiOutputStream.kt | 20 --- .../signing/align/stream/PeekingFakeStream.kt | 21 --- .../utils/signing/align/zip/Extensions.kt | 33 ++++ .../utils/signing/align/zip/ZipFile.kt | 128 +++++++++++++ .../align/zip/structures/ZipEndRecord.kt | 77 ++++++++ .../signing/align/zip/structures/ZipEntry.kt | 170 ++++++++++++++++++ 7 files changed, 423 insertions(+), 90 deletions(-) delete mode 100644 src/main/kotlin/app/revanced/utils/signing/align/stream/MultiOutputStream.kt delete mode 100644 src/main/kotlin/app/revanced/utils/signing/align/stream/PeekingFakeStream.kt create mode 100644 src/main/kotlin/app/revanced/utils/signing/align/zip/Extensions.kt create mode 100644 src/main/kotlin/app/revanced/utils/signing/align/zip/ZipFile.kt create mode 100644 src/main/kotlin/app/revanced/utils/signing/align/zip/structures/ZipEndRecord.kt create mode 100644 src/main/kotlin/app/revanced/utils/signing/align/zip/structures/ZipEntry.kt diff --git a/src/main/kotlin/app/revanced/utils/signing/align/ZipAligner.kt b/src/main/kotlin/app/revanced/utils/signing/align/ZipAligner.kt index 2c441e1..29d1162 100644 --- a/src/main/kotlin/app/revanced/utils/signing/align/ZipAligner.kt +++ b/src/main/kotlin/app/revanced/utils/signing/align/ZipAligner.kt @@ -1,63 +1,29 @@ package app.revanced.utils.signing.align -import app.revanced.utils.signing.align.stream.MultiOutputStream -import app.revanced.utils.signing.align.stream.PeekingFakeStream -import java.io.BufferedOutputStream +import app.revanced.utils.signing.align.zip.ZipFile import java.io.File import java.util.* -import java.util.zip.ZipEntry -import java.util.zip.ZipFile -import java.util.zip.ZipOutputStream internal object ZipAligner { - fun align(input: File, output: File, alignment: Int = 4) { - val zipFile = ZipFile(input) + const val DEFAULT_ALIGNMENT = 4 + const val LIBRARY_ALIGNEMNT = 4096 - val entries: Enumeration = zipFile.entries() + fun align(input: File, output: File) { + val inputZip = ZipFile(input) + val outputZip = ZipFile(output) - // fake - val peekingFakeStream = PeekingFakeStream() - val fakeOutputStream = ZipOutputStream(peekingFakeStream) - // real - val zipOutputStream = ZipOutputStream(BufferedOutputStream(output.outputStream())) + for (entry in inputZip.entries) { + val data = inputZip.getDataForEntry(entry) - val multiOutputStream = MultiOutputStream( - listOf( - fakeOutputStream, // fake, used to add the data to the fake stream - zipOutputStream // real - ) - ) + if (entry.compression == 0.toUShort()) { + val alignment = if (entry.fileName.endsWith(".so")) LIBRARY_ALIGNEMNT else DEFAULT_ALIGNMENT - var bias = 0 - while (entries.hasMoreElements()) { - var padding = 0 - - val entry: ZipEntry = entries.nextElement()!! - // fake, used to calculate the file offset of the entry - fakeOutputStream.putNextEntry(entry) - - if (entry.size == entry.compressedSize) { - val fileOffset = peekingFakeStream.peek() - val newOffset = fileOffset + bias - padding = ((alignment - (newOffset % alignment)) % alignment).toInt() - - // real - entry.extra = if (entry.extra == null) ByteArray(padding) - else Arrays.copyOf(entry.extra, entry.extra.size + padding) + outputZip.addEntryAligned(entry, data, alignment) + } else { + outputZip.addEntry(entry, data) } - - zipOutputStream.putNextEntry(entry) - zipFile.getInputStream(entry).copyTo(multiOutputStream) - - // fake, used to add remaining bytes - fakeOutputStream.closeEntry() - // real - zipOutputStream.closeEntry() - - bias += padding } - zipFile.close() - zipOutputStream.close() + outputZip.finish() } -} \ No newline at end of file +} diff --git a/src/main/kotlin/app/revanced/utils/signing/align/stream/MultiOutputStream.kt b/src/main/kotlin/app/revanced/utils/signing/align/stream/MultiOutputStream.kt deleted file mode 100644 index 1ab80a9..0000000 --- a/src/main/kotlin/app/revanced/utils/signing/align/stream/MultiOutputStream.kt +++ /dev/null @@ -1,20 +0,0 @@ -package app.revanced.utils.signing.align.stream - -import java.io.OutputStream - -internal class MultiOutputStream( - private val streams: Iterable, -) : OutputStream() { - override fun write(b: ByteArray, off: Int, len: Int) = streams.forEach { - it.write(b, off, len) - } - - override fun write(b: ByteArray) = streams.forEach { - it.write(b) - } - - override fun write(b: Int) = streams.forEach { - it.write(b) - } - -} \ No newline at end of file diff --git a/src/main/kotlin/app/revanced/utils/signing/align/stream/PeekingFakeStream.kt b/src/main/kotlin/app/revanced/utils/signing/align/stream/PeekingFakeStream.kt deleted file mode 100644 index 06ee49e..0000000 --- a/src/main/kotlin/app/revanced/utils/signing/align/stream/PeekingFakeStream.kt +++ /dev/null @@ -1,21 +0,0 @@ -package app.revanced.utils.signing.align.stream - -import java.io.OutputStream - -internal class PeekingFakeStream : OutputStream() { - private var numberOfBytes: Long = 0 - - fun peek() = numberOfBytes - - override fun write(b: Int) { - numberOfBytes++ - } - - override fun write(b: ByteArray) { - numberOfBytes += b.size - } - - override fun write(b: ByteArray, offset: Int, len: Int) { - numberOfBytes += len - } -} \ No newline at end of file diff --git a/src/main/kotlin/app/revanced/utils/signing/align/zip/Extensions.kt b/src/main/kotlin/app/revanced/utils/signing/align/zip/Extensions.kt new file mode 100644 index 0000000..87f7db6 --- /dev/null +++ b/src/main/kotlin/app/revanced/utils/signing/align/zip/Extensions.kt @@ -0,0 +1,33 @@ +package app.revanced.utils.signing.align.zip + +import java.io.DataInput +import java.io.DataOutput +import java.nio.ByteBuffer + +fun UInt.toLittleEndian() = + (((this.toInt() and 0xff000000.toInt()) shr 24) or ((this.toInt() and 0x00ff0000) shr 8) or ((this.toInt() and 0x0000ff00) shl 8) or (this.toInt() shl 24)).toUInt() + +fun UShort.toLittleEndian() = (this.toUInt() shl 16).toLittleEndian().toUShort() + +fun UInt.toBigEndian() = (((this.toInt() and 0xff) shl 24) or ((this.toInt() and 0xff00) shl 8) + or ((this.toInt() and 0x00ff0000) ushr 8) or (this.toInt() ushr 24)).toUInt() + +fun UShort.toBigEndian() = (this.toUInt() shl 16).toBigEndian().toUShort() + +fun ByteBuffer.getUShort() = this.getShort().toUShort() +fun ByteBuffer.getUInt() = this.getInt().toUInt() + +fun ByteBuffer.putUShort(ushort: UShort) = this.putShort(ushort.toShort()) +fun ByteBuffer.putUInt(uint: UInt) = this.putInt(uint.toInt()) + +fun DataInput.readUShort() = this.readShort().toUShort() +fun DataInput.readUInt() = this.readInt().toUInt() + +fun DataOutput.writeUShort(ushort: UShort) = this.writeShort(ushort.toInt()) +fun DataOutput.writeUInt(uint: UInt) = this.writeInt(uint.toInt()) + +fun DataInput.readUShortLE() = this.readUShort().toBigEndian() +fun DataInput.readUIntLE() = this.readUInt().toBigEndian() + +fun DataOutput.writeUShortLE(ushort: UShort) = this.writeUShort(ushort.toLittleEndian()) +fun DataOutput.writeUIntLE(uint: UInt) = this.writeUInt(uint.toLittleEndian()) diff --git a/src/main/kotlin/app/revanced/utils/signing/align/zip/ZipFile.kt b/src/main/kotlin/app/revanced/utils/signing/align/zip/ZipFile.kt new file mode 100644 index 0000000..875a1de --- /dev/null +++ b/src/main/kotlin/app/revanced/utils/signing/align/zip/ZipFile.kt @@ -0,0 +1,128 @@ +package app.revanced.utils.signing.align.zip + +import app.revanced.utils.signing.align.zip.structures.ZipEndRecord +import app.revanced.utils.signing.align.zip.structures.ZipEntry +import java.io.File +import java.io.RandomAccessFile +import java.nio.ByteBuffer +import java.nio.channels.FileChannel + +class ZipFile(val file: File) { + var entries: MutableList = mutableListOf() + + private val filePointer: RandomAccessFile = RandomAccessFile(file, "rw") + + init { + //if file isn't empty try to load entries + if (file.length() > 0) { + val endRecord = findEndRecord() + + if (endRecord.diskNumber > 0u || endRecord.totalEntries != endRecord.diskEntries) + throw IllegalArgumentException("Multi-file archives are not supported") + + entries = readEntries(endRecord).toMutableList() + } + + //seek back to start for writing + filePointer.seek(0) + } + + private fun findEndRecord(): ZipEndRecord { + //look from end to start since end record is at the end + for (i in filePointer.length() - 1 downTo 0) { + filePointer.seek(i) + //possible beginning of signature + if (filePointer.readByte() == 0x50.toByte()) { + //seek back to get the full int + filePointer.seek(i) + val possibleSignature = filePointer.readUIntLE() + if (possibleSignature == ZipEndRecord.ECD_SIGNATURE) { + filePointer.seek(i) + return ZipEndRecord.fromECD(filePointer) + } + } + } + + throw Exception("Couldn't find end record") + } + + private fun readEntries(endRecord: ZipEndRecord): List { + filePointer.seek(endRecord.centralDirectoryStartOffset.toLong()) + + val numberOfEntries = endRecord.diskEntries.toInt() + + return buildList(numberOfEntries) { + for (i in 1..numberOfEntries) { + add(ZipEntry.fromCDE(filePointer).also + { + //for some reason the local extra field can be different from the central one + it.readLocalExtra( + filePointer.channel.map( + FileChannel.MapMode.READ_ONLY, + it.localHeaderOffset.toLong() + 28, + 2 + ) + ) + }) + } + } + } + + private fun writeCDE() { + val CDEStart = filePointer.channel.position().toUInt() + + entries.forEach { + filePointer.channel.write(it.toCDE()) + } + + val endRecord = ZipEndRecord( + 0u, + 0u, + entries.count().toUShort(), + entries.count().toUShort(), + filePointer.channel.position().toUInt() - CDEStart, + CDEStart, + "" + ) + + filePointer.channel.write(endRecord.toECD()) + } + + fun addEntry(entry: ZipEntry, data: ByteBuffer) { + entry.localHeaderOffset = filePointer.channel.position().toUInt() + + filePointer.channel.write(entry.toLFH()) + filePointer.channel.write(data) + + entries.add(entry) + } + + fun addEntryAligned(entry: ZipEntry, data: ByteBuffer, alignment: Int) { + //calculate where data would end up + val dataOffset = filePointer.filePointer + entry.LFHSize + + val mod = dataOffset % alignment + + //wrong alignment + if (mod != 0L) { + //add padding at end of extra field + entry.localExtraField = + entry.localExtraField.copyOf((entry.localExtraField.size + (alignment - mod)).toInt()) + } + + addEntry(entry, data) + } + + fun getDataForEntry(entry: ZipEntry): ByteBuffer { + return filePointer.channel.map( + FileChannel.MapMode.READ_ONLY, + entry.dataOffset.toLong(), + entry.compressedSize.toLong() + ) + } + + fun finish() { + writeCDE() + filePointer.close() + } +} diff --git a/src/main/kotlin/app/revanced/utils/signing/align/zip/structures/ZipEndRecord.kt b/src/main/kotlin/app/revanced/utils/signing/align/zip/structures/ZipEndRecord.kt new file mode 100644 index 0000000..d26e551 --- /dev/null +++ b/src/main/kotlin/app/revanced/utils/signing/align/zip/structures/ZipEndRecord.kt @@ -0,0 +1,77 @@ +package app.revanced.utils.signing.align.zip.structures + +import app.revanced.utils.signing.align.zip.putUInt +import app.revanced.utils.signing.align.zip.putUShort +import app.revanced.utils.signing.align.zip.readUIntLE +import app.revanced.utils.signing.align.zip.readUShortLE +import java.io.DataInput +import java.nio.ByteBuffer +import java.nio.ByteOrder + +data class ZipEndRecord( + val diskNumber: UShort, + val startingDiskNumber: UShort, + val diskEntries: UShort, + val totalEntries: UShort, + val centralDirectorySize: UInt, + val centralDirectoryStartOffset: UInt, + val fileComment: String, +) { + + companion object { + const val ECD_HEADER_SIZE = 22 + const val ECD_SIGNATURE = 0x06054b50u + + fun fromECD(input: DataInput): ZipEndRecord { + val signature = input.readUIntLE() + + if (signature != ECD_SIGNATURE) + throw IllegalArgumentException("Input doesn't start with end record signature") + + val diskNumber = input.readUShortLE() + val startingDiskNumber = input.readUShortLE() + val diskEntries = input.readUShortLE() + val totalEntries = input.readUShortLE() + val centralDirectorySize = input.readUIntLE() + val centralDirectoryStartOffset = input.readUIntLE() + val fileCommentLength = input.readUShortLE() + var fileComment = "" + + if (fileCommentLength > 0u) { + val fileCommentBytes = ByteArray(fileCommentLength.toInt()) + input.readFully(fileCommentBytes) + fileComment = fileCommentBytes.toString(Charsets.UTF_8) + } + + return ZipEndRecord( + diskNumber, + startingDiskNumber, + diskEntries, + totalEntries, + centralDirectorySize, + centralDirectoryStartOffset, + fileComment + ) + } + } + + fun toECD(): ByteBuffer { + val commentBytes = fileComment.toByteArray(Charsets.UTF_8) + + val buffer = ByteBuffer.allocate(ECD_HEADER_SIZE + commentBytes.size).also { it.order(ByteOrder.LITTLE_ENDIAN) } + + buffer.putUInt(ECD_SIGNATURE) + buffer.putUShort(diskNumber) + buffer.putUShort(startingDiskNumber) + buffer.putUShort(diskEntries) + buffer.putUShort(totalEntries) + buffer.putUInt(centralDirectorySize) + buffer.putUInt(centralDirectoryStartOffset) + buffer.putUShort(commentBytes.size.toUShort()) + + buffer.put(commentBytes) + + buffer.flip() + return buffer + } +} diff --git a/src/main/kotlin/app/revanced/utils/signing/align/zip/structures/ZipEntry.kt b/src/main/kotlin/app/revanced/utils/signing/align/zip/structures/ZipEntry.kt new file mode 100644 index 0000000..9e70f52 --- /dev/null +++ b/src/main/kotlin/app/revanced/utils/signing/align/zip/structures/ZipEntry.kt @@ -0,0 +1,170 @@ +package app.revanced.utils.signing.align.zip.structures + +import app.revanced.utils.signing.align.zip.getUShort +import app.revanced.utils.signing.align.zip.putUInt +import app.revanced.utils.signing.align.zip.putUShort +import app.revanced.utils.signing.align.zip.readUIntLE +import app.revanced.utils.signing.align.zip.readUShortLE +import java.io.DataInput +import java.nio.ByteBuffer +import java.nio.ByteOrder + +data class ZipEntry( + val version: UShort, + val versionNeeded: UShort, + val flags: UShort, + val compression: UShort, + val modificationTime: UShort, + val modificationDate: UShort, + val crc32: UInt, + val compressedSize: UInt, + val uncompressedSize: UInt, + val diskNumber: UShort, + val internalAttributes: UShort, + val externalAttributes: UInt, + var localHeaderOffset: UInt, + val fileName: String, + val extraField: ByteArray, + val fileComment: String, + var localExtraField: ByteArray = ByteArray(0), //seperate for alignment +) { + val LFHSize: Int + get() = LFH_HEADER_SIZE + fileName.toByteArray(Charsets.UTF_8).size + localExtraField.size + + val dataOffset: UInt + get() = localHeaderOffset + LFHSize.toUInt() + + companion object { + const val CDE_HEADER_SIZE = 46 + const val CDE_SIGNATURE = 0x02014b50u + + const val LFH_HEADER_SIZE = 30 + const val LFH_SIGNATURE = 0x04034b50u + + fun fromCDE(input: DataInput): ZipEntry { + val signature = input.readUIntLE() + + if (signature != CDE_SIGNATURE) + throw IllegalArgumentException("Input doesn't start with central directory entry signature") + + val version = input.readUShortLE() + val versionNeeded = input.readUShortLE() + var flags = input.readUShortLE() + val compression = input.readUShortLE() + val modificationTime = input.readUShortLE() + val modificationDate = input.readUShortLE() + val crc32 = input.readUIntLE() + val compressedSize = input.readUIntLE() + val uncompressedSize = input.readUIntLE() + val fileNameLength = input.readUShortLE() + var fileName = "" + val extraFieldLength = input.readUShortLE() + var extraField = ByteArray(extraFieldLength.toInt()) + val fileCommentLength = input.readUShortLE() + var fileComment = "" + val diskNumber = input.readUShortLE() + val internalAttributes = input.readUShortLE() + val externalAttributes = input.readUIntLE() + val localHeaderOffset = input.readUIntLE() + + val variableFieldsLength = fileNameLength.toInt() + extraFieldLength.toInt() + fileCommentLength.toInt() + + if (variableFieldsLength > 0) { + val fileNameBytes = ByteArray(fileNameLength.toInt()) + input.readFully(fileNameBytes) + fileName = fileNameBytes.toString(Charsets.UTF_8) + + input.readFully(extraField) + + val fileCommentBytes = ByteArray(fileCommentLength.toInt()) + input.readFully(fileCommentBytes) + fileComment = fileCommentBytes.toString(Charsets.UTF_8) + } + + flags = (flags and 0b1000u.inv().toUShort()) //disable data descriptor flag as they are not used + + return ZipEntry( + version, + versionNeeded, + flags, + compression, + modificationTime, + modificationDate, + crc32, + compressedSize, + uncompressedSize, + diskNumber, + internalAttributes, + externalAttributes, + localHeaderOffset, + fileName, + extraField, + fileComment, + ) + } + } + + fun readLocalExtra(buffer: ByteBuffer) { + buffer.order(ByteOrder.LITTLE_ENDIAN) + localExtraField = ByteArray(buffer.getUShort().toInt()) + } + + fun toLFH(): ByteBuffer { + val nameBytes = fileName.toByteArray(Charsets.UTF_8) + + val buffer = ByteBuffer.allocate(LFH_HEADER_SIZE + nameBytes.size + localExtraField.size) + .also { it.order(ByteOrder.LITTLE_ENDIAN) } + + buffer.putUInt(LFH_SIGNATURE) + buffer.putUShort(versionNeeded) + buffer.putUShort(flags) + buffer.putUShort(compression) + buffer.putUShort(modificationTime) + buffer.putUShort(modificationDate) + buffer.putUInt(crc32) + buffer.putUInt(compressedSize) + buffer.putUInt(uncompressedSize) + buffer.putUShort(nameBytes.size.toUShort()) + buffer.putUShort(localExtraField.size.toUShort()) + + buffer.put(nameBytes) + buffer.put(localExtraField) + + buffer.flip() + return buffer + } + + fun toCDE(): ByteBuffer { + val nameBytes = fileName.toByteArray(Charsets.UTF_8) + val commentBytes = fileComment.toByteArray(Charsets.UTF_8) + + val buffer = ByteBuffer.allocate(CDE_HEADER_SIZE + nameBytes.size + extraField.size + commentBytes.size) + .also { it.order(ByteOrder.LITTLE_ENDIAN) } + + buffer.putUInt(CDE_SIGNATURE) + buffer.putUShort(version) + buffer.putUShort(versionNeeded) + buffer.putUShort(flags) + buffer.putUShort(compression) + buffer.putUShort(modificationTime) + buffer.putUShort(modificationDate) + buffer.putUInt(crc32) + buffer.putUInt(compressedSize) + buffer.putUInt(uncompressedSize) + buffer.putUShort(nameBytes.size.toUShort()) + buffer.putUShort(extraField.size.toUShort()) + buffer.putUShort(commentBytes.size.toUShort()) + buffer.putUShort(diskNumber) + buffer.putUShort(internalAttributes) + buffer.putUInt(externalAttributes) + buffer.putUInt(localHeaderOffset) + + buffer.put(nameBytes) + buffer.put(extraField) + buffer.put(commentBytes) + + buffer.flip() + return buffer + } +} +