From f7364ea45641923883ea9607aa3b924ef350d598 Mon Sep 17 00:00:00 2001 From: sisong Date: Tue, 8 Feb 2022 16:22:50 +0800 Subject: [PATCH 1/9] update lzma & HDiffPatch; --- HDiffPatch | 2 +- Makefile | 2 ++ builds/codeblocks/ZipDiff.cbp | 6 ++++++ builds/vc/ZipDiff.vcxproj | 2 ++ lzma | 2 +- 5 files changed, 12 insertions(+), 2 deletions(-) diff --git a/HDiffPatch b/HDiffPatch index 524fe18..2353067 160000 --- a/HDiffPatch +++ b/HDiffPatch @@ -1 +1 @@ -Subproject commit 524fe189d4e91ffa832b8c21cef7bc8aca7b9357 +Subproject commit 2353067e9869c9070847e30de52b5a7774cab147 diff --git a/Makefile b/Makefile index edf7426..31f2532 100644 --- a/Makefile +++ b/Makefile @@ -49,6 +49,8 @@ ZIPDIFF_OBJ := \ lzma/C/LzmaEnc.o \ lzma/C/LzFind.o \ lzma/C/LzFindMt.o \ + lzma/C/LzFindOpt.o \ + lzma/C/CpuArch.o \ lzma/C/MtCoder.o \ lzma/C/MtDec.o \ lzma/C/Threads.o \ diff --git a/builds/codeblocks/ZipDiff.cbp b/builds/codeblocks/ZipDiff.cbp index 01ffd85..2736b39 100644 --- a/builds/codeblocks/ZipDiff.cbp +++ b/builds/codeblocks/ZipDiff.cbp @@ -77,12 +77,18 @@ + + + + diff --git a/builds/vc/ZipDiff.vcxproj b/builds/vc/ZipDiff.vcxproj index 2f2037d..3e1c209 100644 --- a/builds/vc/ZipDiff.vcxproj +++ b/builds/vc/ZipDiff.vcxproj @@ -33,8 +33,10 @@ + + diff --git a/lzma b/lzma index 5a215d7..f4bceec 160000 --- a/lzma +++ b/lzma @@ -1 +1 @@ -Subproject commit 5a215d7c5623357cad46f2933fe274ebad165cd3 +Subproject commit f4bceecac01afbd6562ce15c94d310e2f50372b1 From 9182c75128be36ae41ba6a7a31b0e7c4003c9816 Mon Sep 17 00:00:00 2001 From: sisong Date: Tue, 8 Feb 2022 16:24:48 +0800 Subject: [PATCH 2/9] remove apk v1 signfiles when Normalized --- src/apk_normalized.cpp | 6 ++-- src/diff/DiffData.cpp | 21 ++++++++++---- src/diff/DiffData.h | 6 ++-- src/diff/Differ.cpp | 6 ++-- src/normalized/normalized.cpp | 52 +++++++++++++++++++++++++---------- src/normalized/normalized.h | 3 +- src/patch/ZipDiffData.h | 4 +-- src/patch/Zipper.cpp | 50 +++++++++++++++++++++++++-------- src/patch/Zipper.h | 4 ++- 9 files changed, 110 insertions(+), 42 deletions(-) diff --git a/src/apk_normalized.cpp b/src/apk_normalized.cpp index 3df8bf7..a1f5cbe 100644 --- a/src/apk_normalized.cpp +++ b/src/apk_normalized.cpp @@ -172,14 +172,16 @@ int normalized_cmd_line(int argc, const char * argv[]){ const char* dstApk=arg_values[1]; printf("src: \"%s\"\nout: \"%s\"\n",srcApk,dstApk); double time0=clock_s(); - if (!ZipNormalized(srcApk,dstApk,(int)alignSize,(int)compressLevel,isNotCompressEmptyFile?true:false)){ + int apkV1SignFilesRemoved=0; + if (!ZipNormalized(srcApk,dstApk,(int)alignSize,(int)compressLevel, + isNotCompressEmptyFile?true:false,&apkV1SignFilesRemoved)){ printf("\nrun ApkNormalized ERROR!\n"); return 1; } printf("run ApkNormalized ok!\n"); //check - if (!getZipIsSame(srcApk,dstApk)){ + if (!getZipIsSame(srcApk,dstApk,apkV1SignFilesRemoved)){ printf("ApkNormalized result file check ERROR!\n"); return 1; } diff --git a/src/diff/DiffData.cpp b/src/diff/DiffData.cpp index 08e60c8..3cfb8a8 100644 --- a/src/diff/DiffData.cpp +++ b/src/diff/DiffData.cpp @@ -62,7 +62,8 @@ bool zipFileData_isSame(UnZipper* selfZip,int selfIndex,UnZipper* srcZip,int src bool getZipIsSameWithStream(const hpatch_TStreamInput* oldZipStream, - const hpatch_TStreamInput* newZipStream,bool* out_isOldHaveApkV2Sign){ + const hpatch_TStreamInput* newZipStream, + int newApkV1SignFilesRemoved,bool* out_isOldHaveApkV2Sign){ UnZipper newZip; UnZipper oldZip; std::map oldMap; @@ -80,12 +81,12 @@ bool getZipIsSameWithStream(const hpatch_TStreamInput* oldZipStream, } fileCount=UnZipper_fileCount(&oldZip); - test_clear(fileCount==UnZipper_fileCount(&newZip)); + test_clear(fileCount==UnZipper_fileCount(&newZip)+newApkV1SignFilesRemoved); for (int i=0;i::iterator it=oldMap.find(zipFile_name(&newZip,i)); test_clear(it!=oldMap.end()); int old_i=it->second; @@ -93,13 +94,22 @@ bool getZipIsSameWithStream(const hpatch_TStreamInput* oldZipStream, test_clear(UnZipper_file_crc32(&oldZip,old_i)==UnZipper_file_crc32(&newZip,i)); test_clear(zipFileData_isSame(&oldZip,old_i,&newZip,i)); } + check_clear(oldMap.size()==newApkV1SignFilesRemoved); + if (newApkV1SignFilesRemoved>0){ + for (std::map::iterator it=oldMap.begin();it!=oldMap.end();++it){ + int old_i=it->second; + check_clear(UnZipper_file_isApkV1Sign(&oldZip,old_i)); + } + } + clear: _isInClear=true; check_clear(UnZipper_close(&newZip)); check_clear(UnZipper_close(&oldZip)); return result;} -bool getZipIsSame(const char* oldZipPath,const char* newZipPath,bool* out_isOldHaveApkV2Sign){ +bool getZipIsSame(const char* oldZipPath,const char* newZipPath, + int newApkV1SignFilesRemoved,bool* out_isOldHaveApkV2Sign){ hpatch_TFileStreamInput oldZipStream; hpatch_TFileStreamInput newZipStream; bool result=true; @@ -109,7 +119,8 @@ bool getZipIsSame(const char* oldZipPath,const char* newZipPath,bool* out_isOldH hpatch_TFileStreamInput_init(&newZipStream); check_clear(hpatch_TFileStreamInput_open(&oldZipStream,oldZipPath)); check_clear(hpatch_TFileStreamInput_open(&newZipStream,newZipPath)); - result=getZipIsSameWithStream(&oldZipStream.base,&newZipStream.base); + result=getZipIsSameWithStream(&oldZipStream.base,&newZipStream.base, + newApkV1SignFilesRemoved,out_isOldHaveApkV2Sign); clear: _isInClear=true; check_clear(hpatch_TFileStreamInput_close(&newZipStream)); diff --git a/src/diff/DiffData.h b/src/diff/DiffData.h index 15df8a6..3323697 100644 --- a/src/diff/DiffData.h +++ b/src/diff/DiffData.h @@ -45,9 +45,11 @@ #define kDefaultDiffMatchScore 3 // for hdiff bool zipFileData_isSame(UnZipper* self,int selfIndex,UnZipper* srcZip,int srcIndex);//byte by byte test -bool getZipIsSame(const char* oldZipPath,const char* newZipPath,bool* out_isOldHaveApkV2Sign=0); +bool getZipIsSame(const char* oldZipPath,const char* newZipPath, + int newApkV1SignFilesRemoved=0,bool* out_isOldHaveApkV2Sign=0); bool getZipIsSameWithStream(const hpatch_TStreamInput* oldZipStream, - const hpatch_TStreamInput* newZipStream,bool* out_isOldHaveApkV2Sign=0); + const hpatch_TStreamInput* newZipStream, + int newApkV1SignFilesRemoved=0,bool* out_isOldHaveApkV2Sign=0); bool getCompressedIsNormalized(UnZipper* zip,int* out_zlibCompressLevel, int* out_zlibCompressMemLevel,bool testReCompressedByApkV2Sign=false); //只检查压缩数据是否标准化; bool getCompressedIsNormalizedBy(UnZipper* zip,int zlibCompressLevel, diff --git a/src/diff/Differ.cpp b/src/diff/Differ.cpp index da5dd11..61408a6 100644 --- a/src/diff/Differ.cpp +++ b/src/diff/Differ.cpp @@ -247,7 +247,7 @@ static bool checkZipInfo(UnZipper* oldZip,UnZipper* newZip){ bool isOk=true; if (oldZip->_isDataNormalized) printf(" NOTE: oldZip maybe normalized\n"); - if (UnZipper_isHaveApkV1_or_jarSign(oldZip)) + if (UnZipper_isHaveApkV1Sign(oldZip)) printf(" NOTE: oldZip found JarSign(ApkV1Sign)\n"); if (UnZipper_isHaveApkV2Sign(oldZip)) printf(" NOTE: oldZip found ApkV2Sign\n"); @@ -255,7 +255,7 @@ static bool checkZipInfo(UnZipper* oldZip,UnZipper* newZip){ printf(" NOTE: oldZip found ApkV3Sign\n"); if (newZip->_isDataNormalized) printf(" NOTE: newZip maybe normalized\n"); - if (UnZipper_isHaveApkV1_or_jarSign(newZip)) + if (UnZipper_isHaveApkV1Sign(newZip)) printf(" NOTE: newZip found JarSign(ApkV1Sign)\n"); bool newIsV2Sign=UnZipper_isHaveApkV2Sign(newZip); if (newIsV2Sign) @@ -341,7 +341,7 @@ TCheckZipDiffResult checkZipDiffWithStream(const hpatch_TStreamInput* oldZipStre return CHECK_BYTE_BY_BYTE_EQUAL_TRUE; bool isOldHaveApkV2Sign=false; - bool isSame=getZipIsSameWithStream(newZipStream,&temp_inNewZipStream,&isOldHaveApkV2Sign); + bool isSame=getZipIsSameWithStream(newZipStream,&temp_inNewZipStream,0,&isOldHaveApkV2Sign); if (!isSame) return CHECK_SAME_LIKE_ERROR; if (isOldHaveApkV2Sign) diff --git a/src/normalized/normalized.cpp b/src/normalized/normalized.cpp index 80c17a5..6bb7f71 100644 --- a/src/normalized/normalized.cpp +++ b/src/normalized/normalized.cpp @@ -47,13 +47,16 @@ struct TFileValue{ static bool inline isEndWith(const std::string& s,const char* sub){ return (s.find(sub)==s.size()-strlen(sub)); } + static bool inline isSignMFFile(const std::string& s){ + return (s=="META-INF/MANIFEST.MF")||(s=="META-INF\\MANIFEST.MF"); + } size_t _v(const TFileValue& x)const{ size_t xi=x.fileIndex; if (isInSignDir(x.fileName)){ xi+=_fileCount; if (isEndWith(x.fileName,".SF")) xi+=_fileCount; else if (isEndWith(x.fileName,".RSA")) xi+=_fileCount*2; - else if (isEndWith(x.fileName,".MF")) xi+=_fileCount*3; + else if (isSignMFFile(x.fileName)) xi+=_fileCount*3; } return xi; } @@ -80,13 +83,15 @@ inline static bool isCompressedEmptyFile(const UnZipper* unzipper,int fileIndex) } bool ZipNormalized(const char* srcApk,const char* dstApk, - int ZipAlignSize,int compressLevel,bool isNotCompressEmptyFile){ + int ZipAlignSize,int compressLevel,bool isNotCompressEmptyFile,int* out_apkV1SignFilesRemoved){ bool result=true; bool _isInClear=false; int fileCount=0; bool isHaveApkV2Sign=false; + bool isHaveApkV3Sign=false; int jarSignFileCount=0; std::vector fileIndexs; + std::vector removedFiles; UnZipper unzipper; Zipper zipper; UnZipper_init(&unzipper); @@ -95,21 +100,27 @@ bool ZipNormalized(const char* srcApk,const char* dstApk, check(UnZipper_openFile(&unzipper,srcApk)); fileCount=UnZipper_fileCount(&unzipper); check(Zipper_openFile(&zipper,dstApk,fileCount,ZipAlignSize,compressLevel,kDefaultZlibCompressMemLevel)); - - {//sort file + isHaveApkV2Sign=UnZipper_isHaveApkV2Sign(&unzipper); + isHaveApkV3Sign=UnZipper_isHaveApkV3Sign(&unzipper); + { std::vector files; getFiles(&unzipper,files); std::sort(files.begin(),files.end(),TFileValue::TCmp(fileCount)); for (int i=0; i0) - printf("NOTE: src found JarSign(ApkV1Sign) (%d file)\n",jarSignFileCount); - if (isHaveApkV2Sign) - printf("WARNING: src found ApkV2Sign and not save to dst(%d Byte, need re sign)\n",(int)UnZipper_ApkV2SignSize(&unzipper)); + if (jarSignFileCount>0){ + if (isHaveApkV2Sign){ + printf("WARNING: src removed JarSign(ApkV1Sign) (%d file, need re sign)\n",jarSignFileCount); + for (size_t i=0;i=0; --i) { + if (UnZipper_file_isApkV1Sign(self,i)) + return true; + } + return false; +} bool UnZipper_file_is_sameName(const UnZipper* self,int fileIndex,const char* fileName,int fileNameLen){ return (UnZipper_file_nameLen(self,fileIndex)==fileNameLen)&& @@ -241,16 +249,24 @@ int UnZipper_searchFileIndexByName(const UnZipper* self,const char* fileName,int return '2'==(*posBegin); } + static bool _fileName_isEndWith(const UnZipper* self,int fileIndex,const char* tag){ + const size_t tagLen=strlen(tag); + int fnameLen=UnZipper_file_nameLen(self,fileIndex); + if (fnameLen<(int)tagLen) return false; + const char* fnameBegin=UnZipper_file_nameBegin(self,fileIndex); + return (0==memcmp(fnameBegin+fnameLen-tagLen,tag,tagLen)); + } + static bool _fileName_is(const UnZipper* self,int fileIndex,const char* tag){ + return (strlen(tag)==UnZipper_file_nameLen(self,fileIndex)) + && _fileName_isEndWith(self,fileIndex,tag); + } + + static const char* kJarSignSF=".SF"; static bool _UnZipper_file_isHaveApkV2orV3SignTag_in_ApkV1SignFile(UnZipper* self,int fileIndex){ - if (!UnZipper_file_isApkV1_or_jarSign(self,fileIndex)) return false; + if (!UnZipper_file_isApkV1Sign(self,fileIndex)) return false; //match fileType - const char* kJarSignSF=".SF"; - const size_t kJarSignSFLen=strlen(kJarSignSF); - int fnameLen=UnZipper_file_nameLen(self,fileIndex); - if (fnameLen<(int)kJarSignSFLen) return false; - const char* fnameBegin=UnZipper_file_nameBegin(self,fileIndex); - if (0!=memcmp(fnameBegin+fnameLen-kJarSignSFLen,kJarSignSF,kJarSignSFLen)) return false; + if (!_fileName_isEndWith(self,fileIndex,kJarSignSF)) return false; //match V2 tag in txt file size_t fsize=UnZipper_file_uncompressedSize(self,fileIndex); TByte* data=(TByte*)malloc(fsize+1); //+1 for '\0' @@ -275,16 +291,28 @@ bool UnZipper_isHaveApkV2orV3SignTag_in_ApkV1SignFile(UnZipper* self){ } bool UnZipper_file_isApkV1_or_jarSign(const UnZipper* self,int fileIndex){ - const char* kJarSignPath="META-INF/"; + const char* kJarSignPath0="META-INF/"; + const char* kJarSignPath1="META-INF/"; const size_t kJarSignPathLen=8+1; - return (UnZipper_file_nameLen(self,fileIndex)>=(int)kJarSignPathLen) - && (0==memcmp(UnZipper_file_nameBegin(self,fileIndex),kJarSignPath,kJarSignPathLen)); + if (UnZipper_file_nameLen(self,fileIndex)<(int)kJarSignPathLen) return false; + const char* fn=UnZipper_file_nameBegin(self,fileIndex); + return (0==memcmp(fn,kJarSignPath0,kJarSignPathLen)) + ||(0==memcmp(fn,kJarSignPath1,kJarSignPathLen)); +} + +bool UnZipper_file_isApkV1Sign(const UnZipper* self,int fileIndex){ + if (!UnZipper_file_isApkV1_or_jarSign(self,fileIndex)) return false; + if (_fileName_isEndWith(self,fileIndex,kJarSignSF)) return true; + if (_fileName_isEndWith(self,fileIndex,".RSA")) return true; + if (_fileName_is(self,fileIndex,"META-INF/MANIFEST.MF")) return true; + if (_fileName_is(self,fileIndex,"META-INF\\MANIFEST.MF")) return true; + return false; } bool UnZipper_file_isReCompressedByApkV2Sign(const UnZipper* self,int fileIndex){ return UnZipper_isHaveApkV2Sign(self) && UnZipper_file_isCompressed(self,fileIndex) - && UnZipper_file_isApkV1_or_jarSign(self,fileIndex); + && UnZipper_file_isApkV1Sign(self,fileIndex); } bool UnZipper_isHaveApkV3Sign(const UnZipper* self){ diff --git a/src/patch/Zipper.h b/src/patch/Zipper.h index ded0cd8..0481373 100644 --- a/src/patch/Zipper.h +++ b/src/patch/Zipper.h @@ -107,9 +107,11 @@ static inline const unsigned char* UnZipper_CEData(const UnZipper* self) { retur bool UnZipper_searchApkV2Sign(const hpatch_TStreamInput* stream,hpatch_StreamPos_t centralDirectory_pos, ZipFilePos_t* v2sign_topPos,ZipFilePos_t* out_blockSize=0,bool* out_isHaveV3Sign=0); bool UnZipper_isHaveApkV1_or_jarSign(const UnZipper* self); +bool UnZipper_isHaveApkV1Sign(const UnZipper* self); bool UnZipper_isHaveApkV3Sign(const UnZipper* self); bool UnZipper_isHaveApkV2orV3SignTag_in_ApkV1SignFile(UnZipper* self); //found true; not found or unknown or error false -bool UnZipper_file_isApkV1_or_jarSign(const UnZipper* self,int fileIndex); +bool UnZipper_file_isApkV1_or_jarSign(const UnZipper* self,int fileIndex); // file is in ApkV1Sign Dir +bool UnZipper_file_isApkV1Sign(const UnZipper* self,int fileIndex); // file is ApkV1Sign File bool UnZipper_file_isReCompressedByApkV2Sign(const UnZipper* self,int fileIndex); ZipFilePos_t UnZipper_fileEntry_offset_unsafe(const UnZipper* self,int fileIndex); int UnZipper_searchFileIndexByName(const UnZipper* self,const char* fileName,int fileNameLen); From c05932279a6c914b23d64f8cec83ed8f791919e7 Mon Sep 17 00:00:00 2001 From: sisong Date: Mon, 31 Oct 2022 13:14:34 +0800 Subject: [PATCH 3/9] update HDiffPatch; udate version; --- HDiffPatch | 2 +- README.md | 2 +- builds/vc/ZipDiff.vcxproj | 5 +++-- lzma | 2 +- src/patch/patch_types.h | 4 ++-- 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/HDiffPatch b/HDiffPatch index 2353067..680c86d 160000 --- a/HDiffPatch +++ b/HDiffPatch @@ -1 +1 @@ -Subproject commit 2353067e9869c9070847e30de52b5a7774cab147 +Subproject commit 680c86d97e7a1a66d45ea1246c550efc8051c3d5 diff --git a/README.md b/README.md index e765add..51e378c 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # [ApkDiffPatch] -[![release](https://img.shields.io/badge/release-v1.4.1-blue.svg)](https://github.com/sisong/ApkDiffPatch/releases) +[![release](https://img.shields.io/badge/release-v1.5.0-blue.svg)](https://github.com/sisong/ApkDiffPatch/releases) [![license](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/sisong/ApkDiffPatch/blob/master/LICENSE) [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-blue.svg)](https://github.com/sisong/ApkDiffPatch/pulls) [![+issue Welcome](https://img.shields.io/github/issues-raw/sisong/ApkDiffPatch?color=green&label=%2Bissue%20welcome)](https://github.com/sisong/ApkDiffPatch/issues) diff --git a/builds/vc/ZipDiff.vcxproj b/builds/vc/ZipDiff.vcxproj index 3e1c209..339fb95 100644 --- a/builds/vc/ZipDiff.vcxproj +++ b/builds/vc/ZipDiff.vcxproj @@ -23,12 +23,13 @@ - - + + + diff --git a/lzma b/lzma index f4bceec..ec6ab5a 160000 --- a/lzma +++ b/lzma @@ -1 +1 @@ -Subproject commit f4bceecac01afbd6562ce15c94d310e2f50372b1 +Subproject commit ec6ab5acf2fb90653cbb94e64c86884b13e5db1f diff --git a/src/patch/patch_types.h b/src/patch/patch_types.h index ff1ef44..8e98d67 100644 --- a/src/patch/patch_types.h +++ b/src/patch/patch_types.h @@ -32,8 +32,8 @@ #include //uint32_t uint16_t #define APKDIFFPATCH_VERSION_MAJOR 1 -#define APKDIFFPATCH_VERSION_MINOR 4 -#define APKDIFFPATCH_VERSION_RELEASE 1 +#define APKDIFFPATCH_VERSION_MINOR 5 +#define APKDIFFPATCH_VERSION_RELEASE 0 #define _APKDIFFPATCH_VERSION APKDIFFPATCH_VERSION_MAJOR.APKDIFFPATCH_VERSION_MINOR.APKDIFFPATCH_VERSION_RELEASE #define _APKDIFFPATCH_QUOTE(str) #str From 80791b8a5ba78bdcfb776f679bfc647fa3b472b6 Mon Sep 17 00:00:00 2001 From: sisong Date: Mon, 31 Oct 2022 14:01:43 +0800 Subject: [PATCH 4/9] used blockmatch & muti-thread up diff speed; --- builds/vc/ZipDiff.vcxproj | 1 + src/diff/Differ.cpp | 58 ++++++++++++++++++++++++++------------- src/diff/Differ.h | 4 +-- src/zip_diff.cpp | 2 +- 4 files changed, 43 insertions(+), 22 deletions(-) diff --git a/builds/vc/ZipDiff.vcxproj b/builds/vc/ZipDiff.vcxproj index 339fb95..8e6fa67 100644 --- a/builds/vc/ZipDiff.vcxproj +++ b/builds/vc/ZipDiff.vcxproj @@ -21,6 +21,7 @@ + diff --git a/src/diff/Differ.cpp b/src/diff/Differ.cpp index 61408a6..d62470f 100644 --- a/src/diff/Differ.cpp +++ b/src/diff/Differ.cpp @@ -29,7 +29,7 @@ #include #include #include //sort -#include "../../HDiffPatch/libHDiffPatch/HDiff/diff.h" //https://github.com/sisong/HDiffPatch +#include "../../HDiffPatch/libHDiffPatch/HDiff/match_block.h" //https://github.com/sisong/HDiffPatch #include "../../HDiffPatch/libHDiffPatch/HPatch/patch.h" #include "../../HDiffPatch/file_for_patch.h" #include "../../HDiffPatch/_clock_for_demo.h" @@ -41,8 +41,10 @@ #include "../patch/patch_types.h" -static bool HDiffZ(const std::vector& oldData,const std::vector& newData,std::vector& out_diffData, - const hdiff_TCompress* compressPlugin,hpatch_TDecompress* decompressPlugin,int myBestMatchScore); +static bool HDiffZ(std::vector& oldData,std::vector& newData,std::vector& out_diffData, + const hdiff_TCompress* compressPlugin,int myBestMatchScore,int threadNum); +static bool HPatchZ_check(const std::vector& oldData,const std::vector& newData, + const std::vector& diffData,hpatch_TDecompress* decompressPlugin); static bool checkZipInfo(UnZipper* oldZip,UnZipper* newZip); #define check(value) { \ @@ -52,7 +54,7 @@ static bool checkZipInfo(UnZipper* oldZip,UnZipper* newZip); bool ZipDiff(const char* oldZipPath,const char* newZipPath,const char* outDiffFileName, const hdiff_TCompress* compressPlugin,hpatch_TDecompress* decompressPlugin, - int diffMatchScore,bool* out_isNewZipApkV2SignNoError){ + int diffMatchScore,bool* out_isNewZipApkV2SignNoError,int threadNum){ hpatch_TFileStreamInput oldZipStream; hpatch_TFileStreamInput newZipStream; hpatch_TFileStreamOutput outDiffStream; @@ -67,7 +69,8 @@ bool ZipDiff(const char* oldZipPath,const char* newZipPath,const char* outDiffFi check(hpatch_TFileStreamOutput_open(&outDiffStream,outDiffFileName,(hpatch_StreamPos_t)(-1))); hpatch_TFileStreamOutput_setRandomOut(&outDiffStream,hpatch_TRUE); result=ZipDiffWithStream(&oldZipStream.base,&newZipStream.base,&outDiffStream.base, - compressPlugin,decompressPlugin,diffMatchScore,out_isNewZipApkV2SignNoError); + compressPlugin,decompressPlugin,diffMatchScore, + out_isNewZipApkV2SignNoError,threadNum); clear: _isInClear=true; check(hpatch_TFileStreamOutput_close(&outDiffStream)); @@ -105,7 +108,7 @@ TCheckZipDiffResult checkZipDiff(const char* oldZipPath,const char* newZipPath,c bool ZipDiffWithStream(const hpatch_TStreamInput* oldZipStream,const hpatch_TStreamInput* newZipStream, const hpatch_TStreamOutput* outDiffStream, const hdiff_TCompress* compressPlugin,hpatch_TDecompress* decompressPlugin, - int diffMatchScore,bool* out_isNewZipApkV2SignNoError){ + int diffMatchScore,bool* out_isNewZipApkV2SignNoError,int threadNum){ UnZipper oldZip; UnZipper newZip; std::vector newData; @@ -190,13 +193,17 @@ bool ZipDiffWithStream(const hpatch_TStreamInput* oldZipStream,const hpatch_TStr //for (int i=0; i<(int)newRefOtherCompressedList.size(); ++i) std::cout<(),newData)); - }else{ - check(readZipStreamData(&newZip,newRefList,newRefOtherCompressedList,newData)); - } - check(readZipStreamData(&oldZip,oldRefList,std::vector(),oldData)); - check(HDiffZ(oldData,newData,hdiffzData,compressPlugin,decompressPlugin,diffMatchScore)); + #define _loadNewAndOldData() \ + if ((newZip_otherCompressLevel|newZip_otherCompressMemLevel)!=0){ \ + check(readZipStreamData(&newZip,newRefDecompressList,std::vector(),newData)); \ + }else{ \ + check(readZipStreamData(&newZip,newRefList,newRefOtherCompressedList,newData)); \ + } \ + check(readZipStreamData(&oldZip,oldRefList,std::vector(),oldData)); + _loadNewAndOldData(); + check(HDiffZ(oldData,newData,hdiffzData,compressPlugin,diffMatchScore,threadNum)); + _loadNewAndOldData(); + check(HPatchZ_check(oldData,newData,hdiffzData,decompressPlugin)); { std::vector _empty; oldData.swap(_empty); } { std::vector _empty; newData.swap(_empty); } @@ -215,8 +222,8 @@ bool ZipDiffWithStream(const hpatch_TStreamInput* oldZipStream,const hpatch_TStr return result; } -static bool HDiffZ(const std::vector& oldData,const std::vector& newData,std::vector& out_diffData, - const hdiff_TCompress* compressPlugin,hpatch_TDecompress* decompressPlugin,int myBestMatchScore){ +static bool HDiffZ(std::vector& oldData,std::vector& newData,std::vector& out_diffData, + const hdiff_TCompress* compressPlugin,int myBestMatchScore,int threadNum){ double time0=clock_s(); const size_t oldDataSize=oldData.size(); const size_t newDataSize=newData.size(); @@ -224,13 +231,26 @@ static bool HDiffZ(const std::vector& oldData,const std::vector& n std::cout<<" oldDataSize : "<& diffData=out_diffData; - const TByte* newData0=newData.data(); - const TByte* oldData0=oldData.data(); - create_compressed_diff(newData0,newData0+newDataSize,oldData0,oldData0+oldDataSize, - diffData,compressPlugin,myBestMatchScore); + TByte* newData0=newData.data(); + TByte* oldData0=oldData.data(); + const bool isUseBigCacheMatch=true; + const size_t matchBlockSize=1024*4; + create_compressed_diff_block(newData0,newData0+newDataSize,oldData0,oldData0+oldDataSize, + diffData,compressPlugin,myBestMatchScore, + isUseBigCacheMatch,matchBlockSize,threadNum); double time1=clock_s(); std::cout<<" diffDataSize: "<& oldData,const std::vector& newData, + const std::vector& diffData,hpatch_TDecompress* decompressPlugin){ + const size_t oldDataSize=oldData.size(); + const size_t newDataSize=newData.size(); + + const TByte* newData0=newData.data(); + const TByte* oldData0=oldData.data(); + double time1=clock_s(); if (!check_compressed_diff(newData0,newData0+newDataSize,oldData0,oldData0+oldDataSize, diffData.data(),diffData.data()+diffData.size(),decompressPlugin)){ std::cout<<"\n hpatchz check hdiffz result error!!!\n"; diff --git a/src/diff/Differ.h b/src/diff/Differ.h index a918481..f1b73e1 100644 --- a/src/diff/Differ.h +++ b/src/diff/Differ.h @@ -42,11 +42,11 @@ typedef enum TCheckZipDiffResult { bool ZipDiff(const char* oldZipPath,const char* newZipPath,const char* outDiffFileName, const hdiff_TCompress* compressPlugin,hpatch_TDecompress* decompressPlugin, - int diffMatchScore,bool* out_isNewZipApkV2SignNoError=0); + int diffMatchScore,bool* out_isNewZipApkV2SignNoError=0,int threadNum=1); bool ZipDiffWithStream(const hpatch_TStreamInput* oldZipStream,const hpatch_TStreamInput* newZipStream, const hpatch_TStreamOutput* outDiffStream, const hdiff_TCompress* compressPlugin,hpatch_TDecompress* decompressPlugin, - int diffMatchScore,bool* out_isNewZipApkV2SignNoError=0); + int diffMatchScore,bool* out_isNewZipApkV2SignNoError=0,int threadNum=1); TCheckZipDiffResult checkZipDiff(const char* oldZipPath,const char* newZipPath,const char* diffFileName,int threadNum); TCheckZipDiffResult checkZipDiffWithStream(const hpatch_TStreamInput* oldZipStream, diff --git a/src/zip_diff.cpp b/src/zip_diff.cpp index 59c84fb..3b1052a 100644 --- a/src/zip_diff.cpp +++ b/src/zip_diff.cpp @@ -340,7 +340,7 @@ int zipdiff_cmd_line(int argc, const char * argv[]) { bool isNewZipApkV2SignNoError=true; if (isDiff){ if (!ZipDiff(oldZipPath,newZipPath,outDiffFileName,compressPlugin,decompressPlugin, - (int)diffMatchScore,&isNewZipApkV2SignNoError)){ + (int)diffMatchScore,&isNewZipApkV2SignNoError,(int)threadNum)){ printf("ZipDiff error!\n"); return 1; }//else From 8cdf95d63ef8fec9c446554aa3b19596abde05b8 Mon Sep 17 00:00:00 2001 From: sisong Date: Mon, 31 Oct 2022 17:18:06 +0800 Subject: [PATCH 5/9] makefile, android ndk, macos xcode, linux code::blocks, build ok! --- Makefile | 41 ++++++++++++++++--- builds/android_ndk_jni_mk/Android.mk | 11 +++-- builds/android_ndk_jni_mk/Android_DEBUG.mk | 12 +++++- .../android_ndk_jni_mk/Application_symbol.mk | 8 ---- builds/codeblocks/ZipDiff.cbp | 12 +++--- builds/codeblocks/ZipPatch.cbp | 1 - .../HDiffPatch.xcodeproj/project.pbxproj | 32 +++++++-------- .../xcode/ZipPatch.xcodeproj/project.pbxproj | 4 ++ builds/xcode/lzma.xcodeproj/project.pbxproj | 20 +++++++++ src/diff/Differ.cpp | 1 + 10 files changed, 100 insertions(+), 42 deletions(-) delete mode 100644 builds/android_ndk_jni_mk/Application_symbol.mk diff --git a/Makefile b/Makefile index 31f2532..adddff8 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,12 @@ +# args +STATIC_CPP := 0 +# used clang? +CL := 0 +# build with -m32? +M32 := 0 +# build for out min size +MINS := 0 + ZLIB_OBJ := \ zlib1.2.11/adler32.o \ zlib1.2.11/compress.o \ @@ -36,7 +45,9 @@ ZIPDIFF_OBJ := \ src/diff/DiffData.o \ src/diff/Differ.o \ src/diff/OldRef.o \ + HDiffPatch/libHDiffPatch/HPatchLite/hpatch_lite.o \ HDiffPatch/libHDiffPatch/HDiff/diff.o \ + HDiffPatch/libHDiffPatch/HDiff/match_block.o \ HDiffPatch/libHDiffPatch/HDiff/private_diff/bytes_rle.o \ HDiffPatch/libHDiffPatch/HDiff/private_diff/suffix_string.o \ HDiffPatch/libHDiffPatch/HDiff/private_diff/compress_detect.o \ @@ -56,19 +67,39 @@ ZIPDIFF_OBJ := \ lzma/C/Threads.o \ $(ZIPPATCH_OBJ) -CFLAGS += -O2 -DNDEBUG -D_IS_USED_PTHREAD=1 -CXXFLAGS += -O2 -DNDEBUG -D_IS_USED_PTHREAD=1 +DEF_FLAGS := -O3 -DNDEBUG -D_IS_USED_MULTITHREAD=1 -D_IS_USED_PTHREAD=1 +CFLAGS += $(DEF_FLAGS) +CXXFLAGS += $(DEF_FLAGS) -std=c++11 + +LINK_LIB := -lpthread # link pthread +ifeq ($(M32),0) +else + LINK_LIB += -m32 +endif +ifeq ($(MINS),0) +else + LINK_LIB += -Wl,--gc-sections,--as-needed +endif +ifeq ($(CL),1) + CXX := clang++ + CC := clang +endif +ifeq ($(STATIC_CPP),0) + LINK_LIB += -lstdc++ +else + LINK_LIB += -static-libstdc++ +endif .PHONY: all clean all: ApkNormalized ZipDiff ZipPatch ApkNormalized: $(APKNORM_OBJ) - $(CXX) $(APKNORM_OBJ) -lpthread -o ApkNormalized + $(CXX) $(APKNORM_OBJ) $(LINK_LIB) -o ApkNormalized ZipDiff: $(ZIPDIFF_OBJ) - $(CXX) $(ZIPDIFF_OBJ) -lpthread -o ZipDiff + $(CXX) $(ZIPDIFF_OBJ) $(LINK_LIB) -o ZipDiff ZipPatch: src/zip_patch.o $(ZIPPATCH_OBJ) - $(CXX) src/zip_patch.o $(ZIPPATCH_OBJ) -lpthread -o ZipPatch + $(CXX) src/zip_patch.o $(ZIPPATCH_OBJ) $(LINK_LIB) -o ZipPatch clean: -rm -f ApkNormalized ZipDiff ZipPatch src/zip_patch.o $(ZIPDIFF_OBJ) $(APKNORM_OBJ) \ No newline at end of file diff --git a/builds/android_ndk_jni_mk/Android.mk b/builds/android_ndk_jni_mk/Android.mk index c22757e..fea4bd2 100644 --- a/builds/android_ndk_jni_mk/Android.mk +++ b/builds/android_ndk_jni_mk/Android.mk @@ -37,12 +37,15 @@ Src_Files := $(LOCAL_PATH)/apk_patch_jni.cpp \ LOCAL_SRC_FILES := $(Src_Files) $(Lzma_Files) $(Zlib_Files) $(Hdp_Files) $(Adp_Files) -DEF_FLAGS := -O2 -D_7ZIP_ST -D_IS_USED_MULTITHREAD=1 -D_IS_USED_PTHREAD=1 +LOCAL_LDLIBS := -llog +LOCAL_CFLAGS := -Os -DANDROID_NDK -DNDEBUG -D_IS_USED_MULTITHREAD=1 -D_IS_USED_PTHREAD=1 -D_IS_NEED_CACHE_OLD_BY_COVERS=0 +LOCAL_CFLAGS += -D_IS_NEED_atomic_func=0 +ifneq ($(TARGET_ARCH_ABI),armeabi) +LOCAL_CFLAGS += -DUNALIGNED_OK +endif ifeq ($(TARGET_ARCH_ABI),arm64-v8a) - DEF_FLAGS += -D_LZMA_DEC_OPT +LOCAL_CFLAGS += -D_LZMA_DEC_OPT endif -LOCAL_LDLIBS := -llog -landroid -LOCAL_CFLAGS := -DANDROID_NDK $(DEF_FLAGS) include $(BUILD_SHARED_LIBRARY) diff --git a/builds/android_ndk_jni_mk/Android_DEBUG.mk b/builds/android_ndk_jni_mk/Android_DEBUG.mk index fcfc8a9..6c540fd 100644 --- a/builds/android_ndk_jni_mk/Android_DEBUG.mk +++ b/builds/android_ndk_jni_mk/Android_DEBUG.mk @@ -43,8 +43,16 @@ ifeq ($(TARGET_ARCH_ABI),arm64-v8a) DEF_FLAGS += -D_LZMA_DEC_OPT endif -LOCAL_LDLIBS := -llog -landroid -LOCAL_CFLAGS := -DANDROID_NDK $(DEF_FLAGS) +LOCAL_LDLIBS := -llog +LOCAL_CFLAGS := -Os -DANDROID_NDK -DNDEBUG -D_IS_USED_MULTITHREAD=1 -D_IS_USED_PTHREAD=1 -D_IS_NEED_CACHE_OLD_BY_COVERS=0 +LOCAL_CFLAGS += -D_IS_NEED_atomic_func=0 +ifneq ($(TARGET_ARCH_ABI),armeabi) +LOCAL_CFLAGS += -DUNALIGNED_OK +endif +ifeq ($(TARGET_ARCH_ABI),arm64-v8a) +LOCAL_CFLAGS += -D_LZMA_DEC_OPT +endif + LOCAL_SANITIZE := address include $(BUILD_SHARED_LIBRARY) diff --git a/builds/android_ndk_jni_mk/Application_symbol.mk b/builds/android_ndk_jni_mk/Application_symbol.mk deleted file mode 100644 index df1869d..0000000 --- a/builds/android_ndk_jni_mk/Application_symbol.mk +++ /dev/null @@ -1,8 +0,0 @@ -APP_PLATFORM := android-14 -APP_CFLAGS += -Wno-error=format-security -APP_CFLAGS += -ffunction-sections -fdata-sections -APP_LDFLAGS += -Wl,--gc-sections -APP_STL := stlport_static -APP_CPPFLAGS := -fexceptions -APP_BUILD_SCRIPT := Android.mk -APP_ABI := armeabi-v7a arm64-v8a x86 x86_64 \ No newline at end of file diff --git a/builds/codeblocks/ZipDiff.cbp b/builds/codeblocks/ZipDiff.cbp index 2736b39..da35612 100644 --- a/builds/codeblocks/ZipDiff.cbp +++ b/builds/codeblocks/ZipDiff.cbp @@ -55,14 +55,11 @@