From 7ce394e4d946bcfb455390550abae1dcd93dfc0c Mon Sep 17 00:00:00 2001 From: Dhinak G <17605561+dhinakg@users.noreply.github.com> Date: Sat, 5 Aug 2023 15:06:24 +0530 Subject: [PATCH 01/21] dsce with a makefile --- .clang-format | 15 ++ .clangd | 16 ++ Address.h | 2 + Address.m | 61 -------- Address.mm | 55 +++++++ CacheFile.h | 30 ++-- CacheFile.m | 273 -------------------------------- CacheFile.mm | 234 ++++++++++++++++++++++++++++ CacheImage.h | 21 ++- CacheImage.m | 164 -------------------- CacheImage.mm | 146 ++++++++++++++++++ CacheSet.h | 21 ++- CacheSet.m | 180 ---------------------- CacheSet.mm | 158 +++++++++++++++++++ ImageHeader.h | 4 + ImageHeader.m | 317 -------------------------------------- ImageHeader.mm | 275 +++++++++++++++++++++++++++++++++ Location.h | 24 +-- Location.m => Location.mm | 2 + Main.mm | 273 ++++++++++++++------------------ Makefile | 16 ++ Output.h | 30 ++-- Output.m => Output.mm | 4 + Selector.h | 2 + Selector.m => Selector.mm | 2 + Util.h | 14 ++ dyld | 1 + objc4 | 1 + 28 files changed, 1147 insertions(+), 1194 deletions(-) create mode 100644 .clang-format create mode 100644 .clangd delete mode 100644 Address.m create mode 100644 Address.mm delete mode 100644 CacheFile.m create mode 100644 CacheFile.mm delete mode 100644 CacheImage.m create mode 100644 CacheImage.mm delete mode 100644 CacheSet.m create mode 100644 CacheSet.mm delete mode 100644 ImageHeader.m create mode 100644 ImageHeader.mm rename Location.m => Location.mm (98%) create mode 100644 Makefile rename Output.m => Output.mm (99%) rename Selector.m => Selector.mm (56%) create mode 100644 Util.h create mode 160000 dyld create mode 160000 objc4 diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..0a2d35a --- /dev/null +++ b/.clang-format @@ -0,0 +1,15 @@ +BasedOnStyle: Google +IndentWidth: 4 +ColumnLimit: 140 +DerivePointerAlignment: false +PointerAlignment: Left +ExperimentalAutoDetectBinPacking: true +SpaceBeforeCpp11BracedList: true +--- +Language: ObjC +# Force pointers to the type for C++. +AlignArrayOfStructures: None +SpacesInContainerLiterals: false +BreakStringLiterals: false +ObjCBreakBeforeNestedBlockParam: false +ObjCBlockIndentWidth: 4 diff --git a/.clangd b/.clangd new file mode 100644 index 0000000..0f890dd --- /dev/null +++ b/.clangd @@ -0,0 +1,16 @@ +CompileFlags: + Add: + [ + -fmodules, + -fcxx-modules, + -Wno-unused-getter-return-value, + -mmacosx-version-min=12, + -I.., + -I../dyld/common, + -DDSCE_VERSION=6, + ] +--- +If: + PathMatch: [.*\.cpp, .*\.hpp, .*\.mm] +CompileFlags: + Add: [-std=c++17] diff --git a/Address.h b/Address.h index a35e249..fd44514 100644 --- a/Address.h +++ b/Address.h @@ -1,3 +1,5 @@ +@import Foundation; + #define ADDRESS_REBASE 1 #define ADDRESS_BIND 2 #define ADDRESS_EXPORT 3 diff --git a/Address.m b/Address.m deleted file mode 100644 index 895a735..0000000 --- a/Address.m +++ /dev/null @@ -1,61 +0,0 @@ -@implementation Address - -+(instancetype)rebaseWithAddress:(long)address -{ - Address* result=Address.alloc.init.autorelease; - result.type=ADDRESS_REBASE; - result.address=address; - return result; -} - -+(instancetype)bindWithAddress:(long)address ordinal:(int)ordinal name:(NSString*)name addend:(int)addend -{ - Address* result=Address.alloc.init.autorelease; - result.type=ADDRESS_BIND; - result.address=address; - result.dylibOrdinal=ordinal; - result.name=name; - result.addend=addend; - return result; -} - -+(instancetype)exportWithAddress:(long)address name:(NSString*)name -{ - Address* result=Address.alloc.init.autorelease; - result.type=ADDRESS_EXPORT; - result.address=address; - result.name=name; - return result; -} - -+(instancetype)reexportWithName:(NSString*)name importName:(NSString*)importName importOrdinal:(int)ordinal -{ - Address* result=Address.alloc.init.autorelease; - result.type=ADDRESS_REEXPORT; - result.name=name; - result.importName=importName; - result.dylibOrdinal=ordinal; - return result; -} - --(BOOL)isRebase -{ - return self.type==ADDRESS_REBASE; -} - --(BOOL)isBind -{ - return self.type==ADDRESS_BIND; -} - --(BOOL)isExport -{ - return self.type==ADDRESS_EXPORT; -} - --(BOOL)isReexport -{ - return self.type==ADDRESS_REEXPORT; -} - -@end \ No newline at end of file diff --git a/Address.mm b/Address.mm new file mode 100644 index 0000000..5bde39a --- /dev/null +++ b/Address.mm @@ -0,0 +1,55 @@ +#import "Address.h" + +@implementation Address + ++ (instancetype)rebaseWithAddress:(long)address { + Address* result = Address.alloc.init.autorelease; + result.type = ADDRESS_REBASE; + result.address = address; + return result; +} + ++ (instancetype)bindWithAddress:(long)address ordinal:(int)ordinal name:(NSString*)name addend:(int)addend { + Address* result = Address.alloc.init.autorelease; + result.type = ADDRESS_BIND; + result.address = address; + result.dylibOrdinal = ordinal; + result.name = name; + result.addend = addend; + return result; +} + ++ (instancetype)exportWithAddress:(long)address name:(NSString*)name { + Address* result = Address.alloc.init.autorelease; + result.type = ADDRESS_EXPORT; + result.address = address; + result.name = name; + return result; +} + ++ (instancetype)reexportWithName:(NSString*)name importName:(NSString*)importName importOrdinal:(int)ordinal { + Address* result = Address.alloc.init.autorelease; + result.type = ADDRESS_REEXPORT; + result.name = name; + result.importName = importName; + result.dylibOrdinal = ordinal; + return result; +} + +- (BOOL)isRebase { + return self.type == ADDRESS_REBASE; +} + +- (BOOL)isBind { + return self.type == ADDRESS_BIND; +} + +- (BOOL)isExport { + return self.type == ADDRESS_EXPORT; +} + +- (BOOL)isReexport { + return self.type == ADDRESS_REEXPORT; +} + +@end \ No newline at end of file diff --git a/CacheFile.h b/CacheFile.h index 98368b6..afce6c8 100644 --- a/CacheFile.h +++ b/CacheFile.h @@ -1,23 +1,31 @@ // TODO: properly tune this +@import Foundation; + +#import "Extern.h" +#import "LocationBase.h" + +@class CacheImage; + #define FAST_CHUNK_SIZE 0x10000 -@interface CacheFile:NSObject +@interface CacheFile : NSObject @property(retain) NSMutableData* data; @property(retain) NSArray* images; -@property(retain) NSDictionary*>* fastRebasesByChunk; -@property(retain) NSDictionary*>* fastImagesByChunk; -@property(retain) NSDictionary* fastImagesByPath; +@property(retain) NSDictionary*>* fastRebasesByChunk; +@property(retain) NSDictionary*>* fastImagesByChunk; +@property(retain) NSDictionary* fastImagesByPath; --(instancetype)initWithPath:(NSString*)path; +- (instancetype)initWithPath:(NSString*)path; --(long)maxConstDataMappingAddress; --(long)maxConstDataSegmentAddress; --(CacheImage*)imageWithPath:(NSString*)path; --(NSArray*)imagesWithPathPrefix:(NSString*)path; --(CacheImage*)imageWithAddress:(long)address; --(NSArray*)rebasesWithStartAddress:(long)start endAddress:(long)end; +- (long)maxConstDataMappingAddress; +- (long)maxConstDataSegmentAddress; +- (CacheImage*)imageWithPath:(NSString*)path; +- (NSArray*)imagesWithPathPrefix:(NSString*)path; +- (CacheImage*)imageWithAddress:(long)address; +- (NSArray*)rebasesWithStartAddress:(long)start endAddress:(long)end; +- (struct dyld_cache_header*)header; @end \ No newline at end of file diff --git a/CacheFile.m b/CacheFile.m deleted file mode 100644 index 9e424f7..0000000 --- a/CacheFile.m +++ /dev/null @@ -1,273 +0,0 @@ -@implementation CacheFile - --(instancetype)initWithPath:(NSString*)path -{ - self=super.init; - - self.data=[NSMutableData dataWithContentsOfFile:path]; - if(!self.data) - { - return nil; - } - - NSMutableArray* images=NSMutableArray.alloc.init.autorelease; - - struct dyld_cache_image_info* infos=(struct dyld_cache_image_info*)wrapOffset(self,self.header->imagesOffset).pointer; - - for(int index=0;indeximagesCount;index++) - { - CacheImage* image=[CacheImage.alloc initWithCacheFile:self info:&infos[index]].autorelease; - if(image) - { - [images addObject:image]; - } - } - - self.images=images; - - NSMutableDictionary*>* rebasesByChunk=NSMutableDictionary.alloc.init.autorelease; - - [self forEachMapping:^(struct dyld_cache_mapping_and_slide_info* mapping) - { - if(mapping->slideInfoFileOffset) - { - dyld_cache_slide_info2* slide=(struct dyld_cache_slide_info2*)wrapOffset(self,mapping->slideInfoFileOffset).pointer; - assert(slide->version==2); - - // dyld_cache_format.h - - unsigned long valueMask=~(slide->delta_mask); - int deltaShift=__builtin_ctzll(slide->delta_mask)-2; - - short* starts=(short*)((char*)slide+slide->page_starts_offset); - for(long pageIndex=0;pageIndexpage_starts_count;pageIndex++) - { - if(starts[pageIndex]==DYLD_CACHE_SLIDE_PAGE_ATTR_NO_REBASE) - { - continue; - } - - assert((starts[pageIndex]&DYLD_CACHE_SLIDE_PAGE_ATTR_EXTRA)==0); - - unsigned long startAddress=mapping->address+pageIndex*slide->page_size+starts[pageIndex]*4; - char* pointer=wrapAddress(self,startAddress).pointer; - - int delta=1; - while(delta) - { - unsigned long* valuePointer=(unsigned long*)pointer; - delta=(*valuePointer&slide->delta_mask)>>deltaShift; - - unsigned long value=(*valuePointer&valueMask)+slide->value_add; - *valuePointer=value; - - unsigned long address=wrapPointer(self,pointer).address; - NSNumber* chunk=[NSNumber numberWithLong:address/FAST_CHUNK_SIZE]; - if(!rebasesByChunk[chunk]) - { - rebasesByChunk[chunk]=NSMutableArray.alloc.init.autorelease; - } - [rebasesByChunk[chunk] addObject:[NSNumber numberWithLong:address]]; - - pointer+=delta; - } - } - } - }]; - self.fastRebasesByChunk=rebasesByChunk; - - NSMutableDictionary*>* imagesByChunk=NSMutableDictionary.alloc.init.autorelease; - for(CacheImage* image in self.images) - { - NSArray* chunks=[image enclosingChunksWithSize:FAST_CHUNK_SIZE]; - for(NSNumber* chunk in chunks) - { - if(!imagesByChunk[chunk]) - { - imagesByChunk[chunk]=NSMutableArray.alloc.init.autorelease; - } - [imagesByChunk[chunk] addObject:image]; - } - } - self.fastImagesByChunk=imagesByChunk; - - NSMutableDictionary* imagesByPath=NSMutableDictionary.alloc.init.autorelease; - for(CacheImage* image in self.images) - { - imagesByPath[image.path]=image; - } - self.fastImagesByPath=imagesByPath; - - return self; -} - --(long)maxConstDataMappingAddress -{ - __block long max=0; - - [self forEachMapping:^(struct dyld_cache_mapping_and_slide_info* info) - { - if(info->flags==DYLD_CACHE_MAPPING_CONST_DATA) - { - assert(max==0); - max=info->address+info->size; - } - }]; - - assert(max!=0); - return max; -} - --(long)maxConstDataSegmentAddress -{ - long mappingEnd=self.maxConstDataMappingAddress; - - __block long max=0; - - for(CacheImage* image in self.images) - { - [image.header forEachSegmentCommand:^(struct segment_command_64* command) - { - long end=command->vmaddr+command->vmsize; - if(endmappingWithSlideOffset); - for(int index=0;indexmappingWithSlideCount;index++) - { - block(&infos[index]); - } -} - --(long)addressWithOffset:(long)offset -{ - __block dyld_cache_mapping_and_slide_info* info=NULL; - - [self forEachMapping:^(struct dyld_cache_mapping_and_slide_info* mapping) - { - if(offset>=mapping->fileOffset&&offsetfileOffset+mapping->size) - { - assert(!info); - info=mapping; - } - }]; - - if(!info) - { - return -1; - } - - return info->address+offset-info->fileOffset; -} - --(long)addressWithPointer:(char*)pointer -{ - return [self addressWithOffset:pointer-(char*)self.data.mutableBytes]; -} - --(long)offsetWithAddress:(long)address -{ - __block dyld_cache_mapping_and_slide_info* info=NULL; - - [self forEachMapping:^(struct dyld_cache_mapping_and_slide_info* mapping) - { - if(address>=mapping->address&&addressaddress+mapping->size) - { - assert(!info); - info=mapping; - } - }]; - - if(!info) - { - return -1; - } - - return info->fileOffset+address-info->address; -} - --(char*)pointerWithAddress:(long)address -{ - long offset=[self offsetWithAddress:address]; - if(offset==-1) - { - return NULL; - } - - return (char*)self.data.mutableBytes+offset; -} - --(CacheImage*)imageWithPath:(NSString*)path -{ - return self.fastImagesByPath[path]; -} - --(NSArray*)imagesWithPathPrefix:(NSString*)path -{ - NSMutableArray* result=NSMutableArray.alloc.init.autorelease; - - for(CacheImage* image in self.images) - { - if([image.path hasPrefix:path]) - { - [result addObject:image]; - } - } - - return result; -} - --(CacheImage*)imageWithAddress:(long)address -{ - NSNumber* chunk=[NSNumber numberWithLong:address/FAST_CHUNK_SIZE]; - NSArray* candidates=self.fastImagesByChunk[chunk]; - - for(CacheImage* image in candidates) - { - if([image.header segmentCommandWithAddress:address indexOut:NULL]) - { - return image; - } - } - - return nil; -} - --(NSArray*)rebasesWithStartAddress:(long)start endAddress:(long)end -{ - NSMutableArray* result=NSMutableArray.alloc.init.autorelease; - - long startChunk=start/FAST_CHUNK_SIZE; - long endChunk=(end-1)/FAST_CHUNK_SIZE; - for(long chunk=startChunk;chunk<=endChunk;chunk++) - { - for(NSNumber* rebase in self.fastRebasesByChunk[[NSNumber numberWithLong:chunk]]) - { - if(rebase.longValue>=start&&rebase.longValue* images = NSMutableArray.alloc.init.autorelease; + + struct dyld_cache_image_info* infos = (struct dyld_cache_image_info*)wrapOffset(self, self.header->imagesOffset).pointer; + + for (int index = 0; index < self.header->imagesCount; index++) { + CacheImage* image = [CacheImage.alloc initWithCacheFile:self info:&infos[index]].autorelease; + if (image) { + [images addObject:image]; + } + } + + self.images = images; + + NSMutableDictionary*>* rebasesByChunk = NSMutableDictionary.alloc.init.autorelease; + + [self forEachMapping:^(struct dyld_cache_mapping_and_slide_info* mapping) { + if (mapping->slideInfoFileOffset) { + dyld_cache_slide_info2* slide = (struct dyld_cache_slide_info2*)wrapOffset(self, mapping->slideInfoFileOffset).pointer; + assert(slide->version == 2); + + // dyld_cache_format.h + + unsigned long valueMask = ~(slide->delta_mask); + int deltaShift = __builtin_ctzll(slide->delta_mask) - 2; + + short* starts = (short*)((char*)slide + slide->page_starts_offset); + for (long pageIndex = 0; pageIndex < slide->page_starts_count; pageIndex++) { + if (starts[pageIndex] == DYLD_CACHE_SLIDE_PAGE_ATTR_NO_REBASE) { + continue; + } + + assert((starts[pageIndex] & DYLD_CACHE_SLIDE_PAGE_ATTR_EXTRA) == 0); + + unsigned long startAddress = mapping->address + pageIndex * slide->page_size + starts[pageIndex] * 4; + char* pointer = wrapAddress(self, startAddress).pointer; + + int delta = 1; + while (delta) { + unsigned long* valuePointer = (unsigned long*)pointer; + delta = (*valuePointer & slide->delta_mask) >> deltaShift; + + unsigned long value = (*valuePointer & valueMask) + slide->value_add; + *valuePointer = value; + + unsigned long address = wrapPointer(self, pointer).address; + NSNumber* chunk = [NSNumber numberWithLong:address / FAST_CHUNK_SIZE]; + if (!rebasesByChunk[chunk]) { + rebasesByChunk[chunk] = NSMutableArray.alloc.init.autorelease; + } + [rebasesByChunk[chunk] addObject:[NSNumber numberWithLong:address]]; + + pointer += delta; + } + } + } + }]; + self.fastRebasesByChunk = rebasesByChunk; + + NSMutableDictionary*>* imagesByChunk = NSMutableDictionary.alloc.init.autorelease; + for (CacheImage* image in self.images) { + NSArray* chunks = [image enclosingChunksWithSize:FAST_CHUNK_SIZE]; + for (NSNumber* chunk in chunks) { + if (!imagesByChunk[chunk]) { + imagesByChunk[chunk] = NSMutableArray.alloc.init.autorelease; + } + [imagesByChunk[chunk] addObject:image]; + } + } + self.fastImagesByChunk = imagesByChunk; + + NSMutableDictionary* imagesByPath = NSMutableDictionary.alloc.init.autorelease; + for (CacheImage* image in self.images) { + imagesByPath[image.path] = image; + } + self.fastImagesByPath = imagesByPath; + + return self; +} + +- (long)maxConstDataMappingAddress { + __block long max = 0; + + [self forEachMapping:^(struct dyld_cache_mapping_and_slide_info* info) { + if (info->flags == DYLD_CACHE_MAPPING_CONST_DATA) { + assert(max == 0); + max = info->address + info->size; + } + }]; + + assert(max != 0); + return max; +} + +- (long)maxConstDataSegmentAddress { + long mappingEnd = self.maxConstDataMappingAddress; + + __block long max = 0; + + for (CacheImage* image in self.images) { + [image.header forEachSegmentCommand:^(struct segment_command_64* command) { + long end = command->vmaddr + command->vmsize; + if (end < mappingEnd) { + max = MAX(max, end); + } + }]; + } + + assert(max != 0); + return max; +} + +- (struct dyld_cache_header*)header { + return (struct dyld_cache_header*)self.data.bytes; +} + +- (void)forEachMapping:(void (^)(struct dyld_cache_mapping_and_slide_info*))block { + // using Location here would create a circular dependency + + struct dyld_cache_mapping_and_slide_info* infos = + (struct dyld_cache_mapping_and_slide_info*)((char*)self.data.mutableBytes + self.header->mappingWithSlideOffset); + for (int index = 0; index < self.header->mappingWithSlideCount; index++) { + block(&infos[index]); + } +} + +- (long)addressWithOffset:(long)offset { + __block dyld_cache_mapping_and_slide_info* info = NULL; + + [self forEachMapping:^(struct dyld_cache_mapping_and_slide_info* mapping) { + if (offset >= mapping->fileOffset && offset < mapping->fileOffset + mapping->size) { + assert(!info); + info = mapping; + } + }]; + + if (!info) { + return -1; + } + + return info->address + offset - info->fileOffset; +} + +- (long)addressWithPointer:(char*)pointer { + return [self addressWithOffset:pointer - (char*)self.data.mutableBytes]; +} + +- (long)offsetWithAddress:(long)address { + __block dyld_cache_mapping_and_slide_info* info = NULL; + + [self forEachMapping:^(struct dyld_cache_mapping_and_slide_info* mapping) { + if (address >= mapping->address && address < mapping->address + mapping->size) { + assert(!info); + info = mapping; + } + }]; + + if (!info) { + return -1; + } + + return info->fileOffset + address - info->address; +} + +- (char*)pointerWithAddress:(long)address { + long offset = [self offsetWithAddress:address]; + if (offset == -1) { + return NULL; + } + + return (char*)self.data.mutableBytes + offset; +} + +- (CacheImage*)imageWithPath:(NSString*)path { + return self.fastImagesByPath[path]; +} + +- (NSArray*)imagesWithPathPrefix:(NSString*)path { + NSMutableArray* result = NSMutableArray.alloc.init.autorelease; + + for (CacheImage* image in self.images) { + if ([image.path hasPrefix:path]) { + [result addObject:image]; + } + } + + return result; +} + +- (CacheImage*)imageWithAddress:(long)address { + NSNumber* chunk = [NSNumber numberWithLong:address / FAST_CHUNK_SIZE]; + NSArray* candidates = self.fastImagesByChunk[chunk]; + + for (CacheImage* image in candidates) { + if ([image.header segmentCommandWithAddress:address indexOut:NULL]) { + return image; + } + } + + return nil; +} + +- (NSArray*)rebasesWithStartAddress:(long)start endAddress:(long)end { + NSMutableArray* result = NSMutableArray.alloc.init.autorelease; + + long startChunk = start / FAST_CHUNK_SIZE; + long endChunk = (end - 1) / FAST_CHUNK_SIZE; + for (long chunk = startChunk; chunk <= endChunk; chunk++) { + for (NSNumber* rebase in self.fastRebasesByChunk[[NSNumber numberWithLong:chunk]]) { + if (rebase.longValue >= start && rebase.longValue < end) { + [result addObject:rebase]; + } + } + } + + return result; +} + +@end \ No newline at end of file diff --git a/CacheImage.h b/CacheImage.h index bc15fdd..1166267 100644 --- a/CacheImage.h +++ b/CacheImage.h @@ -1,18 +1,23 @@ -@interface CacheImage:NSObject +#import "Address.h" +#import "ImageHeader.h" +@import Foundation; +#import "CacheFile.h" + +@interface CacheImage : NSObject @property(assign) CacheFile* file; @property(assign) long baseAddress; @property(retain) ImageHeader* header; @property(retain) NSString* path; @property(retain) NSMutableArray* exports; -@property(retain) NSMutableDictionary* fastExportsByAddress; -@property(retain) NSMutableDictionary* fastReexportsByName; +@property(retain) NSMutableDictionary* fastExportsByAddress; +@property(retain) NSMutableDictionary* fastReexportsByName; --(instancetype)initWithCacheFile:(CacheFile*)file info:(struct dyld_cache_image_info*)info; +- (instancetype)initWithCacheFile:(CacheFile*)file info:(struct dyld_cache_image_info*)info; --(void)forEachLegacySymbol:(void (^)(struct nlist_64*,char*))block; --(Address*)exportWithAddress:(long)address; --(Address*)reexportWithName:(NSString*)name; --(NSArray*)enclosingChunksWithSize:(long)chunkSize; +- (void)forEachLegacySymbol:(void (^)(struct nlist_64*, char*))block; +- (Address*)exportWithAddress:(long)address; +- (Address*)reexportWithName:(NSString*)name; +- (NSArray*)enclosingChunksWithSize:(long)chunkSize; @end \ No newline at end of file diff --git a/CacheImage.m b/CacheImage.m deleted file mode 100644 index e281379..0000000 --- a/CacheImage.m +++ /dev/null @@ -1,164 +0,0 @@ -@implementation CacheImage - --(instancetype)initWithCacheFile:(CacheFile*)file info:(struct dyld_cache_image_info*)info -{ - self=super.init; - - Location* headerLocation=wrapAddressUnsafe(file,info->address); - if(!headerLocation) - { - return nil; - } - - // TODO: make this fully conform to LocationBase...? - - self.baseAddress=headerLocation.address; - - self.header=[ImageHeader.alloc initWithPointer:headerLocation.pointer].autorelease; - - self.file=file; - self.path=[NSString stringWithUTF8String:wrapOffset(file,info->pathFileOffset).pointer]; - - self.loadSymbols; - - return self; -} - -// TODO: bit weird to put this here, move to ImageHeader - --(void)forEachLegacySymbol:(void (^)(struct nlist_64*,char*))block -{ - struct symtab_command* symtab=(struct symtab_command*)[self.header commandWithType:LC_SYMTAB]; - assert(symtab); - - struct nlist_64* symbols=(struct nlist_64*)wrapOffset(self.file,symtab->symoff).pointer; - char* strings=wrapOffset(self.file,symtab->stroff).pointer; - - for(int index=0;indexnsyms;index++) - { - block(&symbols[index],strings+symbols[index].n_un.n_strx); - } -} - --(void)loadSymbols -{ - // now that i noticed dyld_info_command has exports in some images - // scanning legacy symtab is never necessary 🤦🏻‍♀️ - - NSMutableArray* exports=NSMutableArray.alloc.init.autorelease; - - char* trieData=NULL; - long trieSize; - - struct linkedit_data_command* trieCommand=(struct linkedit_data_command*)[self.header commandWithType:LC_DYLD_EXPORTS_TRIE]; - if(trieCommand&&trieCommand->datasize!=0) - { - trieData=wrapOffset(self.file,trieCommand->dataoff).pointer; - trieSize=trieCommand->datasize; - } - - struct dyld_info_command* infoCommand=(struct dyld_info_command*)[self.header commandWithType:LC_DYLD_INFO]; - struct dyld_info_command* infoCommandOnly=(struct dyld_info_command*)[self.header commandWithType:LC_DYLD_INFO_ONLY]; - if(infoCommandOnly) - { - assert(!infoCommand); - infoCommand=infoCommandOnly; - } - - if(infoCommand&&infoCommand->export_size!=0) - { - assert(!trieData); - trieData=wrapOffset(self.file,infoCommand->export_off).pointer; - trieSize=infoCommand->export_size; - } - - if(trieData) - { - std::vector entries; - assert(ExportInfoTrie::parseTrie((const unsigned char*)trieData,(const unsigned char*)trieData+trieSize,entries)); - - for(ExportInfoTrie::Entry entry:entries) - { - long address=0; - if(entry.info.address) - { - address=self.baseAddress+entry.info.address; - } - - Address* item; - NSString* name=[NSString stringWithUTF8String:entry.name.c_str()]; - - if(entry.info.flags&EXPORT_SYMBOL_FLAGS_REEXPORT) - { - assert(address==0); - - // blank if unchanged - - NSString* importName=[NSString stringWithUTF8String:entry.info.importName.c_str()]; - item=[Address reexportWithName:name importName:importName importOrdinal:entry.info.other]; - } - else - { - // TODO: sufficient to bind against these, but can't extract images with them - - item=[Address exportWithAddress:address name:name]; - } - - [exports addObject:item]; - } - } - - self.exports=exports; - - NSMutableDictionary* byAddress=NSMutableDictionary.alloc.init.autorelease; - NSMutableDictionary* byName=NSMutableDictionary.alloc.init.autorelease; - - for(Address* item in exports) - { - if(item.address!=0) - { - byAddress[[NSNumber numberWithLong:item.address]]=item; - } - - if(item.isReexport) - { - byName[item.name]=item; - if(item.importName&&item.importName.length!=0) - { - byName[item.importName]=item; - } - } - } - - self.fastExportsByAddress=byAddress; - self.fastReexportsByName=byName; -} - --(Address*)exportWithAddress:(long)address -{ - return self.fastExportsByAddress[[NSNumber numberWithLong:address]]; -} - --(Address*)reexportWithName:(NSString*)name -{ - return self.fastReexportsByName[name]; -} - --(NSArray*)enclosingChunksWithSize:(long)chunkSize -{ - NSMutableArray* result=NSMutableArray.alloc.init.autorelease; - - [self.header forEachSegmentCommand:^(struct segment_command_64* command) - { - long startChunk=command->vmaddr/chunkSize; - long endChunk=(command->vmaddr+command->vmsize-1)/chunkSize; - for(long chunk=startChunk;chunk<=endChunk;chunk++) - { - [result addObject:[NSNumber numberWithLong:chunk]]; - } - }]; - - return result; -} - -@end \ No newline at end of file diff --git a/CacheImage.mm b/CacheImage.mm new file mode 100644 index 0000000..f6597c5 --- /dev/null +++ b/CacheImage.mm @@ -0,0 +1,146 @@ +#import "CacheImage.h" + +#import "Extern.h" +#import "Location.h" + +@implementation CacheImage + +- (instancetype)initWithCacheFile:(CacheFile*)file info:(struct dyld_cache_image_info*)info { + self = super.init; + + Location* headerLocation = wrapAddressUnsafe(file, info->address); + if (!headerLocation) { + return nil; + } + + // TODO: make this fully conform to LocationBase...? + + self.baseAddress = headerLocation.address; + + self.header = [ImageHeader.alloc initWithPointer:headerLocation.pointer].autorelease; + + self.file = file; + self.path = [NSString stringWithUTF8String:wrapOffset(file, info->pathFileOffset).pointer]; + + self.loadSymbols; + + return self; +} + +// TODO: bit weird to put this here, move to ImageHeader + +- (void)forEachLegacySymbol:(void (^)(struct nlist_64*, char*))block { + struct symtab_command* symtab = (struct symtab_command*)[self.header commandWithType:LC_SYMTAB]; + assert(symtab); + + struct nlist_64* symbols = (struct nlist_64*)wrapOffset(self.file, symtab->symoff).pointer; + char* strings = wrapOffset(self.file, symtab->stroff).pointer; + + for (int index = 0; index < symtab->nsyms; index++) { + block(&symbols[index], strings + symbols[index].n_un.n_strx); + } +} + +- (void)loadSymbols { + // now that i noticed dyld_info_command has exports in some images + // scanning legacy symtab is never necessary 🤦🏻‍♀️ + + NSMutableArray* exports = NSMutableArray.alloc.init.autorelease; + + char* trieData = NULL; + long trieSize; + + struct linkedit_data_command* trieCommand = (struct linkedit_data_command*)[self.header commandWithType:LC_DYLD_EXPORTS_TRIE]; + if (trieCommand && trieCommand->datasize != 0) { + trieData = wrapOffset(self.file, trieCommand->dataoff).pointer; + trieSize = trieCommand->datasize; + } + + struct dyld_info_command* infoCommand = (struct dyld_info_command*)[self.header commandWithType:LC_DYLD_INFO]; + struct dyld_info_command* infoCommandOnly = (struct dyld_info_command*)[self.header commandWithType:LC_DYLD_INFO_ONLY]; + if (infoCommandOnly) { + assert(!infoCommand); + infoCommand = infoCommandOnly; + } + + if (infoCommand && infoCommand->export_size != 0) { + assert(!trieData); + trieData = wrapOffset(self.file, infoCommand->export_off).pointer; + trieSize = infoCommand->export_size; + } + + if (trieData) { + std::vector entries; + assert(ExportInfoTrie::parseTrie((const unsigned char*)trieData, (const unsigned char*)trieData + trieSize, entries)); + + for (ExportInfoTrie::Entry entry : entries) { + long address = 0; + if (entry.info.address) { + address = self.baseAddress + entry.info.address; + } + + Address* item; + NSString* name = [NSString stringWithUTF8String:entry.name.c_str()]; + + if (entry.info.flags & EXPORT_SYMBOL_FLAGS_REEXPORT) { + assert(address == 0); + + // blank if unchanged + + NSString* importName = [NSString stringWithUTF8String:entry.info.importName.c_str()]; + item = [Address reexportWithName:name importName:importName importOrdinal:entry.info.other]; + } else { + // TODO: sufficient to bind against these, but can't extract images with them + + item = [Address exportWithAddress:address name:name]; + } + + [exports addObject:item]; + } + } + + self.exports = exports; + + NSMutableDictionary* byAddress = NSMutableDictionary.alloc.init.autorelease; + NSMutableDictionary* byName = NSMutableDictionary.alloc.init.autorelease; + + for (Address* item in exports) { + if (item.address != 0) { + byAddress[[NSNumber numberWithLong:item.address]] = item; + } + + if (item.isReexport) { + byName[item.name] = item; + if (item.importName && item.importName.length != 0) { + byName[item.importName] = item; + } + } + } + + self.fastExportsByAddress = byAddress; + self.fastReexportsByName = byName; +} + +- (Address*)exportWithAddress:(long)address { + return self.fastExportsByAddress[[NSNumber numberWithLong:address]]; +} + +- (Address*)reexportWithName:(NSString*)name { + return self.fastReexportsByName[name]; +} + +- (NSArray*)enclosingChunksWithSize:(long)chunkSize { + NSMutableArray* result = NSMutableArray.alloc.init.autorelease; + + [self.header forEachSegmentCommand:^(struct segment_command_64* command) { + long startChunk = command->vmaddr / chunkSize; + long endChunk = (command->vmaddr + command->vmsize - 1) / chunkSize; + for (long chunk = startChunk; chunk <= endChunk; chunk++) { + [result addObject:[NSNumber numberWithLong:chunk]]; + } + }]; + + return result; +} + +@end \ No newline at end of file diff --git a/CacheSet.h b/CacheSet.h index 186de02..3427a92 100644 --- a/CacheSet.h +++ b/CacheSet.h @@ -1,12 +1,23 @@ -@interface CacheSet:NSObject +@import Foundation; + +#import "CacheFile.h" +#import "CacheSet.h" + +@interface CacheSet : NSObject @property(retain) NSArray* files; @property(assign) long magicSelAddress; --(instancetype)initWithPathPrefix:(NSString*)prefix; +- (instancetype)initWithPathPrefix:(NSString*)prefix; + +- (CacheImage*)imageWithPath:(NSString*)path; +- (NSArray*)imagesWithPathPrefix:(NSString*)path; +- (CacheImage*)imageWithAddress:(long)address; + +- (int)majorVersion; + +- (int)minorVersion; --(CacheImage*)imageWithPath:(NSString*)path; --(NSArray*)imagesWithPathPrefix:(NSString*)path; --(CacheImage*)imageWithAddress:(long)address; +- (int)subMinorVersion; @end \ No newline at end of file diff --git a/CacheSet.m b/CacheSet.m deleted file mode 100644 index 1e68ce1..0000000 --- a/CacheSet.m +++ /dev/null @@ -1,180 +0,0 @@ -@implementation CacheSet - --(instancetype)initWithPathPrefix:(NSString*)prefix -{ - trace(@"read %@",prefix); - - NSMutableArray* files=NSMutableArray.alloc.init.autorelease; - - CacheFile* file=[CacheFile.alloc initWithPath:prefix].autorelease; - if(!file) - { - return nil; - } - - [files addObject:file]; - - // TODO: silly, can probably use dyld_subcache_entry - - for(NSString* format in @[@"%@.%d",@"%@.%02d"]) - { - for(int index=1;;index++) - { - NSString* path=[NSString stringWithFormat:format,prefix,index]; - - CacheFile* file=[CacheFile.alloc initWithPath:path].autorelease; - if(!file) - { - break; - } - - [files addObject:file]; - } - } - - if(files.count==0) - { - return nil; - } - - self.files=files; - - trace(@"os version %d.%d.%d, subcache count %x",self.majorVersion,self.minorVersion,self.subMinorVersion,files.count); - - self.findMagicSel; - - return self; -} - --(int)majorVersion -{ - return self.files[0].header->osVersion/0x10000; -} - --(int)minorVersion -{ - return (self.files[0].header->osVersion/0x100)%0x100; -} - --(int)subMinorVersion -{ - return self.files[0].header->osVersion%0x100; -} - --(long)addressWithOffset:(long)offset -{ - // lacks context of which cache file - - abort(); -} - --(long)addressWithPointer:(char*)pointer -{ - for(CacheFile* file in self.files) - { - Location* location=wrapPointerUnsafe(file,pointer); - if(location) - { - return location.address; - } - } - - return -1; -} - --(long)offsetWithAddress:(long)address -{ - for(CacheFile* file in self.files) - { - Location* location=wrapAddressUnsafe(file,address); - if(location) - { - return location.offset; - } - } - - return -1; -} - --(char*)pointerWithAddress:(long)address -{ - for(CacheFile* file in self.files) - { - Location* location=wrapAddressUnsafe(file,address); - if(location) - { - return location.pointer; - } - } - - return NULL; -} - --(CacheImage*)imageWithPath:(NSString*)path -{ - for(CacheFile* file in self.files) - { - CacheImage* image=[file imageWithPath:path]; - if(image) - { - return image; - } - } - - return nil; -} - --(NSArray*)imagesWithPathPrefix:(NSString*)path -{ - NSMutableArray* result=NSMutableArray.alloc.init.autorelease; - - for(CacheFile* file in self.files) - { - NSArray* images=[file imagesWithPathPrefix:path]; - [result addObjectsFromArray:images]; - } - - return result; -} - --(CacheImage*)imageWithAddress:(long)address -{ - for(CacheFile* file in self.files) - { - CacheImage* image=[file imageWithAddress:address]; - if(image) - { - return image; - } - } - - return nil; -} - --(void)findMagicSel -{ - CacheImage* image=[self imagesWithPathPrefix:@"/usr/lib/libobjc.A.dylib"].firstObject; - assert(image); - - struct section_64* section=[image.header sectionCommandWithName:(char*)"__objc_selrefs"]; - assert(section); - - long* refs=(long*)wrapOffset(image.file,section->offset).pointer; - int count=section->size/sizeof(long*); - - for(int index=0;index* files = NSMutableArray.alloc.init.autorelease; + + CacheFile* file = [CacheFile.alloc initWithPath:prefix].autorelease; + if (!file) { + return nil; + } + + [files addObject:file]; + + // TODO: silly, can probably use dyld_subcache_entry + + for (NSString* format in @[@"%@.%d", @"%@.%02d"]) { + for (int index = 1;; index++) { + NSString* path = [NSString stringWithFormat:format, prefix, index]; + + CacheFile* file = [CacheFile.alloc initWithPath:path].autorelease; + if (!file) { + break; + } + + [files addObject:file]; + } + } + + if (files.count == 0) { + return nil; + } + + self.files = files; + + trace(@"os version %d.%d.%d, subcache count %x", self.majorVersion, self.minorVersion, self.subMinorVersion, files.count); + + self.findMagicSel; + + return self; +} + +- (int)majorVersion { + return self.files[0].header->osVersion / 0x10000; +} + +- (int)minorVersion { + return (self.files[0].header->osVersion / 0x100) % 0x100; +} + +- (int)subMinorVersion { + return self.files[0].header->osVersion % 0x100; +} + +- (long)addressWithOffset:(long)offset { + // lacks context of which cache file + + abort(); +} + +- (long)addressWithPointer:(char*)pointer { + for (CacheFile* file in self.files) { + Location* location = wrapPointerUnsafe(file, pointer); + if (location) { + return location.address; + } + } + + return -1; +} + +- (long)offsetWithAddress:(long)address { + for (CacheFile* file in self.files) { + Location* location = wrapAddressUnsafe(file, address); + if (location) { + return location.offset; + } + } + + return -1; +} + +- (char*)pointerWithAddress:(long)address { + for (CacheFile* file in self.files) { + Location* location = wrapAddressUnsafe(file, address); + if (location) { + return location.pointer; + } + } + + return NULL; +} + +- (CacheImage*)imageWithPath:(NSString*)path { + for (CacheFile* file in self.files) { + CacheImage* image = [file imageWithPath:path]; + if (image) { + return image; + } + } + + return nil; +} + +- (NSArray*)imagesWithPathPrefix:(NSString*)path { + NSMutableArray* result = NSMutableArray.alloc.init.autorelease; + + for (CacheFile* file in self.files) { + NSArray* images = [file imagesWithPathPrefix:path]; + [result addObjectsFromArray:images]; + } + + return result; +} + +- (CacheImage*)imageWithAddress:(long)address { + for (CacheFile* file in self.files) { + CacheImage* image = [file imageWithAddress:address]; + if (image) { + return image; + } + } + + return nil; +} + +- (void)findMagicSel { + CacheImage* image = [self imagesWithPathPrefix:@"/usr/lib/libobjc.A.dylib"].firstObject; + assert(image); + + struct section_64* section = [image.header sectionCommandWithName:(char*)"__objc_selrefs"]; + assert(section); + + long* refs = (long*)wrapOffset(image.file, section->offset).pointer; + int count = section->size / sizeof(long*); + + for (int index = 0; index < count; index++) { + char* name = wrapAddress(self, refs[index]).pointer; + + if (name && !strcmp(name, "\xf0\x9f\xa4\xaf")) { + self.magicSelAddress = refs[index]; + trace(@"found magic selector at %lx", self.magicSelAddress); + break; + } + } + + assert(self.magicSelAddress); +} + +@end \ No newline at end of file diff --git a/ImageHeader.h b/ImageHeader.h index eac9f76..6a81c0d 100644 --- a/ImageHeader.h +++ b/ImageHeader.h @@ -1,3 +1,7 @@ +@import Foundation; + +@class CacheSet; + @interface ImageHeader:NSObject @property(retain) NSMutableData* data; diff --git a/ImageHeader.m b/ImageHeader.m deleted file mode 100644 index 886db7c..0000000 --- a/ImageHeader.m +++ /dev/null @@ -1,317 +0,0 @@ -@implementation ImageHeader - --(instancetype)init; -{ - // TODO: some Output operations use addCommand: without reloading pointers - // which causes crashes when the NSMutableData backing gets enlarged - // preallocating should prevent this, but it's ugly and not explicitly documented - - self.data=[NSMutableData dataWithCapacity:0x10000]; - - return self; -} - --(instancetype)initWithPointer:(char*)pointer -{ - self=self.init; - - int size=sizeof(struct mach_header_64)+((struct mach_header_64*)pointer)->sizeofcmds; - [self.data appendBytes:pointer length:size]; - - return self; -} - --(instancetype)initEmpty -{ - self=self.init; - - [self.data increaseLengthBy:sizeof(struct mach_header_64)]; - - // TODO: check - - self.header->magic=MH_MAGIC_64; - self.header->cputype=CPU_TYPE_X86_64; - self.header->cpusubtype=CPU_SUBTYPE_X86_64_ALL; - self.header->filetype=MH_DYLIB; - - return self; -} - --(struct mach_header_64*)header -{ - return (struct mach_header_64*)self.data.mutableBytes; -} - --(void)forEachCommand:(void (^)(struct load_command*))block -{ - struct load_command* command=(struct load_command*)(self.header+1); - - for(int commandIndex=0;commandIndexncmds;commandIndex++) - { - block(command); - command=(struct load_command*)(((char*)command)+command->cmdsize); - } -} - --(struct load_command*)commandWithType:(int)type -{ - __block struct load_command* result=NULL; - - [self forEachCommand:^(struct load_command* command) - { - if(command->cmd==type) - { - assert(!result); - result=command; - } - }]; - - return result; -} - --(void)forEachSegmentCommand:(void (^)(struct segment_command_64*))block -{ - [self forEachCommand:^(struct load_command* command) - { - if(command->cmd==LC_SEGMENT_64) - { - struct segment_command_64* segment=(struct segment_command_64*)command; - block(segment); - } - }]; -} - --(struct segment_command_64*)segmentCommandWithName:(char*)name -{ - __block struct segment_command_64* output=NULL; - - [self forEachSegmentCommand:^(struct segment_command_64* command) - { - int length=MIN(16,MAX(strlen(command->segname),strlen(name))); - if(!strncmp(command->segname,name,length)) - { - assert(!output); - output=command; - } - }]; - - return output; -} - -// TODO: slow and ugly, may be worth abstracting Segment and creating a map - --(struct segment_command_64*)segmentCommandWithAddress:(long)address indexOut:(int*)indexOut -{ - __block struct segment_command_64* result=NULL; - __block int index=0; - - [self forEachSegmentCommand:^(struct segment_command_64* command) - { - if(address>=command->vmaddr&&addressvmaddr+command->vmsize) - { - if(result) - { - trace(@"multiple segments cover address %lx",address); - self.dumpSegments; - abort(); - } - - result=command; - - if(indexOut) - { - *indexOut=index; - } - } - - index++; - }]; - - return result; -} - --(struct segment_command_64*)segmentCommandWithOffset:(long)offset indexOut:(int*)indexOut -{ - __block struct segment_command_64* result=NULL; - __block int index=0; - - [self forEachSegmentCommand:^(struct segment_command_64* command) - { - if(offset>=command->fileoff&&offsetfileoff+command->filesize) - { - if(result) - { - trace(@"multiple segments cover offset %lx",offset); - self.dumpSegments; - abort(); - } - - result=command; - - if(indexOut) - { - *indexOut=index; - } - } - - index++; - }]; - - return result; -} - --(void)forEachSectionCommand:(void (^)(struct segment_command_64*,struct section_64*))block -{ - [self forEachSegmentCommand:^(struct segment_command_64* command) - { - struct section_64* sections=(struct section_64*)(command+1); - for(int index=0;indexnsects;index++) - { - block(command,§ions[index]); - } - }]; -} - --(struct section_64*)sectionCommandWithName:(char*)name -{ - __block struct section_64* output=NULL; - - [self forEachSectionCommand:^(struct segment_command_64* segment,struct section_64* command) - { - // hack to avoid matching subsets or overrunning into segname - - int length=MIN(16,MAX(strlen(command->sectname),strlen(name))); - if(!strncmp(command->sectname,name,length)) - { - assert(!output); - output=command; - } - }]; - - return output; -} - --(void)addCommand:(struct load_command*)command -{ - // TODO: confirm Ventura requires this, and pad instead of crashing - - assert(command->cmdsize%8==0); - - [self.data appendBytes:(char*)command length:command->cmdsize]; - - self.header->ncmds++; - self.header->sizeofcmds+=command->cmdsize; -} - -// TODO: everything below here is a bit weird, refactor? move to other classes? - --(NSArray*)dylibPathsReexportOnly:(BOOL)reexportOnly -{ - NSMutableArray* result=NSMutableArray.alloc.init.autorelease; - - [self forEachCommand:^(struct load_command* command) - { - if(command->cmd==LC_LOAD_DYLIB||command->cmd==LC_LOAD_WEAK_DYLIB||command->cmd==LC_LOAD_UPWARD_DYLIB||command->cmd==LC_REEXPORT_DYLIB) - { - if(!reexportOnly||command->cmd==LC_REEXPORT_DYLIB) - { - int nameOffset=((struct dylib_command*)command)->dylib.name.offset; - NSString* name=[NSString stringWithUTF8String:(char*)command+nameOffset]; - [result addObject:name]; - } - } - }]; - - return result; -} - --(NSArray*)dylibPaths -{ - // order matters here (bind ordinal) - - if(!self.fastShallowPaths) - { - self.fastShallowPaths=[self dylibPathsReexportOnly:false]; - } - - return self.fastShallowPaths; -} - --(NSSet*)reexportedDylibPathsRecursiveWithCache:(CacheSet*)cache -{ - if(!self.fastRecursivePaths) - { - // order doesn't matter here, since we're getting children of one import - - NSArray* reexports=[self dylibPathsReexportOnly:true]; - - NSMutableSet* result=NSMutableSet.alloc.init.autorelease; - [result addObjectsFromArray:reexports]; - - for(NSString* path in reexports) - { - CacheImage* image=[cache imageWithPath:path]; - assert(image); - [result unionSet:[image.header reexportedDylibPathsRecursiveWithCache:cache]]; - } - - self.fastRecursivePaths=result; - } - - return self.fastRecursivePaths; -} - --(int)ordinalWithDylibPath:(NSString*)target cache:(CacheSet*)cache symbol:(NSString*)symbol newSymbolOut:(NSString**)newSymbolOut -{ - NSArray* shallowPaths=self.dylibPaths; - long shallowFound=[shallowPaths indexOfObject:target]; - if(shallowFound!=NSNotFound) - { - return shallowFound+1; - } - - for(int index=0;index* deepPaths=[shallowImage.header reexportedDylibPathsRecursiveWithCache:cache]; - if([deepPaths containsObject:target]) - { - return index+1; - } - - // TODO: check if symbol actually comes from the dylib we originally matched - // name collisions are theoretically possible... - - Address* reexport=[shallowImage reexportWithName:symbol]; - if(reexport) - { - *newSymbolOut=reexport.name; - return index+1; - } - - for(NSString* deepPath in deepPaths) - { - CacheImage* deepImage=[cache imageWithPath:deepPath]; - Address* reexport=[deepImage reexportWithName:symbol]; - if(reexport) - { - *newSymbolOut=reexport.name; - return index+1; - } - } - } - - return -1; -} - --(void)dumpSegments -{ - [self forEachSegmentCommand:^(struct segment_command_64* seg) - { - trace(@"segment %s address %lx offset %lx memory size %lx file size %lx",seg->segname,seg->vmaddr,seg->fileoff,seg->vmsize,seg->filesize); - }]; -} - -@end \ No newline at end of file diff --git a/ImageHeader.mm b/ImageHeader.mm new file mode 100644 index 0000000..5481e6a --- /dev/null +++ b/ImageHeader.mm @@ -0,0 +1,275 @@ +#import "ImageHeader.h" +#import "CacheImage.h" +#import "Util.h" +@import MachO.loader; + +@implementation ImageHeader + +- (instancetype)init; +{ + // TODO: some Output operations use addCommand: without reloading pointers + // which causes crashes when the NSMutableData backing gets enlarged + // preallocating should prevent this, but it's ugly and not explicitly documented + + self.data = [NSMutableData dataWithCapacity:0x10000]; + + return self; +} + +- (instancetype)initWithPointer:(char*)pointer { + self = self.init; + + int size = sizeof(struct mach_header_64) + ((struct mach_header_64*)pointer)->sizeofcmds; + [self.data appendBytes:pointer length:size]; + + return self; +} + +- (instancetype)initEmpty { + self = self.init; + + [self.data increaseLengthBy:sizeof(struct mach_header_64)]; + + // TODO: check + + self.header->magic = MH_MAGIC_64; + self.header->cputype = CPU_TYPE_X86_64; + self.header->cpusubtype = CPU_SUBTYPE_X86_64_ALL; + self.header->filetype = MH_DYLIB; + + return self; +} + +- (struct mach_header_64*)header { + return (struct mach_header_64*)self.data.mutableBytes; +} + +- (void)forEachCommand:(void (^)(struct load_command*))block { + struct load_command* command = (struct load_command*)(self.header + 1); + + for (int commandIndex = 0; commandIndex < self.header->ncmds; commandIndex++) { + block(command); + command = (struct load_command*)(((char*)command) + command->cmdsize); + } +} + +- (struct load_command*)commandWithType:(int)type { + __block struct load_command* result = NULL; + + [self forEachCommand:^(struct load_command* command) { + if (command->cmd == type) { + assert(!result); + result = command; + } + }]; + + return result; +} + +- (void)forEachSegmentCommand:(void (^)(struct segment_command_64*))block { + [self forEachCommand:^(struct load_command* command) { + if (command->cmd == LC_SEGMENT_64) { + struct segment_command_64* segment = (struct segment_command_64*)command; + block(segment); + } + }]; +} + +- (struct segment_command_64*)segmentCommandWithName:(char*)name { + __block struct segment_command_64* output = NULL; + + [self forEachSegmentCommand:^(struct segment_command_64* command) { + int length = MIN(16, MAX(strlen(command->segname), strlen(name))); + if (!strncmp(command->segname, name, length)) { + assert(!output); + output = command; + } + }]; + + return output; +} + +// TODO: slow and ugly, may be worth abstracting Segment and creating a map + +- (struct segment_command_64*)segmentCommandWithAddress:(long)address indexOut:(int*)indexOut { + __block struct segment_command_64* result = NULL; + __block int index = 0; + + [self forEachSegmentCommand:^(struct segment_command_64* command) { + if (address >= command->vmaddr && address < command->vmaddr + command->vmsize) { + if (result) { + trace(@"multiple segments cover address %lx", address); + self.dumpSegments; + abort(); + } + + result = command; + + if (indexOut) { + *indexOut = index; + } + } + + index++; + }]; + + return result; +} + +- (struct segment_command_64*)segmentCommandWithOffset:(long)offset indexOut:(int*)indexOut { + __block struct segment_command_64* result = NULL; + __block int index = 0; + + [self forEachSegmentCommand:^(struct segment_command_64* command) { + if (offset >= command->fileoff && offset < command->fileoff + command->filesize) { + if (result) { + trace(@"multiple segments cover offset %lx", offset); + self.dumpSegments; + abort(); + } + + result = command; + + if (indexOut) { + *indexOut = index; + } + } + + index++; + }]; + + return result; +} + +- (void)forEachSectionCommand:(void (^)(struct segment_command_64*, struct section_64*))block { + [self forEachSegmentCommand:^(struct segment_command_64* command) { + struct section_64* sections = (struct section_64*)(command + 1); + for (int index = 0; index < command->nsects; index++) { + block(command, §ions[index]); + } + }]; +} + +- (struct section_64*)sectionCommandWithName:(char*)name { + __block struct section_64* output = NULL; + + [self forEachSectionCommand:^(struct segment_command_64* segment, struct section_64* command) { + // hack to avoid matching subsets or overrunning into segname + + int length = MIN(16, MAX(strlen(command->sectname), strlen(name))); + if (!strncmp(command->sectname, name, length)) { + assert(!output); + output = command; + } + }]; + + return output; +} + +- (void)addCommand:(struct load_command*)command { + // TODO: confirm Ventura requires this, and pad instead of crashing + + assert(command->cmdsize % 8 == 0); + + [self.data appendBytes:(char*)command length:command->cmdsize]; + + self.header->ncmds++; + self.header->sizeofcmds += command->cmdsize; +} + +// TODO: everything below here is a bit weird, refactor? move to other classes? + +- (NSArray*)dylibPathsReexportOnly:(BOOL)reexportOnly { + NSMutableArray* result = NSMutableArray.alloc.init.autorelease; + + [self forEachCommand:^(struct load_command* command) { + if (command->cmd == LC_LOAD_DYLIB || command->cmd == LC_LOAD_WEAK_DYLIB || command->cmd == LC_LOAD_UPWARD_DYLIB || + command->cmd == LC_REEXPORT_DYLIB) { + if (!reexportOnly || command->cmd == LC_REEXPORT_DYLIB) { + int nameOffset = ((struct dylib_command*)command)->dylib.name.offset; + NSString* name = [NSString stringWithUTF8String:(char*)command + nameOffset]; + [result addObject:name]; + } + } + }]; + + return result; +} + +- (NSArray*)dylibPaths { + // order matters here (bind ordinal) + + if (!self.fastShallowPaths) { + self.fastShallowPaths = [self dylibPathsReexportOnly:false]; + } + + return self.fastShallowPaths; +} + +- (NSSet*)reexportedDylibPathsRecursiveWithCache:(CacheSet*)cache { + if (!self.fastRecursivePaths) { + // order doesn't matter here, since we're getting children of one import + + NSArray* reexports = [self dylibPathsReexportOnly:true]; + + NSMutableSet* result = NSMutableSet.alloc.init.autorelease; + [result addObjectsFromArray:reexports]; + + for (NSString* path in reexports) { + CacheImage* image = [cache imageWithPath:path]; + assert(image); + [result unionSet:[image.header reexportedDylibPathsRecursiveWithCache:cache]]; + } + + self.fastRecursivePaths = result; + } + + return self.fastRecursivePaths; +} + +- (int)ordinalWithDylibPath:(NSString*)target cache:(CacheSet*)cache symbol:(NSString*)symbol newSymbolOut:(NSString**)newSymbolOut { + NSArray* shallowPaths = self.dylibPaths; + long shallowFound = [shallowPaths indexOfObject:target]; + if (shallowFound != NSNotFound) { + return shallowFound + 1; + } + + for (int index = 0; index < shallowPaths.count; index++) { + CacheImage* shallowImage = [cache imageWithPath:shallowPaths[index]]; + assert(shallowImage); + + NSSet* deepPaths = [shallowImage.header reexportedDylibPathsRecursiveWithCache:cache]; + if ([deepPaths containsObject:target]) { + return index + 1; + } + + // TODO: check if symbol actually comes from the dylib we originally matched + // name collisions are theoretically possible... + + Address* reexport = [shallowImage reexportWithName:symbol]; + if (reexport) { + *newSymbolOut = reexport.name; + return index + 1; + } + + for (NSString* deepPath in deepPaths) { + CacheImage* deepImage = [cache imageWithPath:deepPath]; + Address* reexport = [deepImage reexportWithName:symbol]; + if (reexport) { + *newSymbolOut = reexport.name; + return index + 1; + } + } + } + + return -1; +} + +- (void)dumpSegments { + [self forEachSegmentCommand:^(struct segment_command_64* seg) { + trace(@"segment %s address %lx offset %lx memory size %lx file size %lx", seg->segname, seg->vmaddr, seg->fileoff, seg->vmsize, + seg->filesize); + }]; +} + +@end \ No newline at end of file diff --git a/Location.h b/Location.h index 0f63eb2..827e64a 100644 --- a/Location.h +++ b/Location.h @@ -1,21 +1,25 @@ -@interface Location:NSObject +@import Foundation; -@property(assign) NSObject* base; +#import "LocationBase.h" + +@interface Location : NSObject + +@property(assign) NSObject *base; @property(assign) long address; --(long)offset; --(char*)pointer; +- (long)offset; +- (char *)pointer; @end // return nil on error -Location* wrapAddressUnsafe(NSObject* base,long address); -Location* wrapOffsetUnsafe(NSObject* base,long offset); -Location* wrapPointerUnsafe(NSObject* base,char* pointer); +Location *wrapAddressUnsafe(NSObject *base, long address); +Location *wrapOffsetUnsafe(NSObject *base, long offset); +Location *wrapPointerUnsafe(NSObject *base, char *pointer); // abort on error -Location* wrapAddress(NSObject* base,long address); -Location* wrapOffset(NSObject* base,long offset); -Location* wrapPointer(NSObject* base,char* pointer); \ No newline at end of file +Location *wrapAddress(NSObject *base, long address); +Location *wrapOffset(NSObject *base, long offset); +Location *wrapPointer(NSObject *base, char *pointer); \ No newline at end of file diff --git a/Location.m b/Location.mm similarity index 98% rename from Location.m rename to Location.mm index 35f72a4..611e278 100644 --- a/Location.m +++ b/Location.mm @@ -1,3 +1,5 @@ +#import "Location.h" + @implementation Location +(instancetype)locationWithBase:(NSObject*)base address:(long)address diff --git a/Main.mm b/Main.mm index b0efc1a..f32c3c9 100644 --- a/Main.mm +++ b/Main.mm @@ -2,167 +2,130 @@ #import "Extern.h" -void trace(NSString* format,...) -{ - va_list args; - va_start(args,format); - NSString* message=[NSString.alloc initWithFormat:format arguments:args].autorelease; - va_end(args); - - printf("\e[%dm%s\e[0m\n",31+DSCE_VERSION%6,message.UTF8String); -} - -BOOL flagPad=false; +BOOL flagPad = false; -#import "LocationBase.h" -#import "Location.h" -@class CacheImage; -#import "CacheFile.h" -@class CacheSet; -#import "ImageHeader.h" #import "Address.h" +#import "CacheFile.h" #import "CacheImage.h" #import "CacheSet.h" -#import "Selector.h" +#import "ImageHeader.h" +#import "Location.h" +#import "LocationBase.h" #import "Output.h" +#import "Selector.h" +#import "Util.h" + +void process(NSMutableArray* args) { + NSString* cachePath = args[0]; + [args removeObjectAtIndex:0]; + + CacheSet* cache = [CacheSet.alloc initWithPathPrefix:cachePath].autorelease; + assert(cache); + + if ([args containsObject:@"list"]) { + assert(args.count == 1); + + NSArray* images = [cache imagesWithPathPrefix:@"/"]; + + trace(@"list %x images", images.count); + + for (CacheImage* image in images) { + trace(@"%@", image.path); + } + + return; + } + + if ([args containsObject:@"search"]) { + assert(args.count == 2); + + NSData* target = [args[1] dataUsingEncoding:NSUTF8StringEncoding]; + NSMutableSet* seen = NSMutableSet.alloc.init.autorelease; -#import "Location.m" -#import "CacheFile.m" -#import "ImageHeader.m" -#import "Address.m" -#import "CacheImage.m" -#import "CacheSet.m" -#import "Selector.m" -#import "Output.m" - -void process(NSMutableArray* args) -{ - NSString* cachePath=args[0]; - [args removeObjectAtIndex:0]; - - CacheSet* cache=[CacheSet.alloc initWithPathPrefix:cachePath].autorelease; - assert(cache); - - if([args containsObject:@"list"]) - { - assert(args.count==1); - - NSArray* images=[cache imagesWithPathPrefix:@"/"]; - - trace(@"list %x images",images.count); - - for(CacheImage* image in images) - { - trace(@"%@",image.path); - } - - return; - } - - if([args containsObject:@"search"]) - { - assert(args.count==2); - - NSData* target=[args[1] dataUsingEncoding:NSUTF8StringEncoding]; - NSMutableSet* seen=NSMutableSet.alloc.init.autorelease; - - trace(@"search %@",target); - - for(CacheFile* file in cache.files) - { - long offset=0; - while(true) - { - NSRange range=[file.data rangeOfData:target options:0 range:NSMakeRange(offset,file.data.length-offset)]; - if(range.location==NSNotFound) - { - break; - } - else - { - CacheImage* image=[cache imageWithAddress:wrapOffset(file,range.location).address]; - - if(![seen containsObject:image.path]) - { - trace(@"%@",image.path); - if(image.path) - { - [seen addObject:image.path]; - } - } - - offset=range.location+range.length; - } - } - } - - return; - } - - flagPad=[args containsObject:@"pad"]; - [args removeObject:@"pad"]; - - NSMutableArray* images=NSMutableArray.alloc.init.autorelease; - for(NSString* arg in args) - { - NSArray* subset=[cache imagesWithPathPrefix:arg]; - - if(subset.count==0) - { - trace(@"no images found for %@*",arg); - abort(); - } - - [images addObjectsFromArray:subset]; - } - - for(int index=0;index* images = NSMutableArray.alloc.init.autorelease; + for (NSString* arg in args) { + NSArray* subset = [cache imagesWithPathPrefix:arg]; + + if (subset.count == 0) { + trace(@"no images found for %@*", arg); + abort(); + } + + [images addObjectsFromArray:subset]; + } + + for (int index = 0; index < images.count; index++) { + trace(@"extract %@ (%x/%x)", images[index].path, index + 1, images.count); + + // TODO: check for leaks between images + + @autoreleasepool { + double startTime = NSDate.date.timeIntervalSince1970; + + NSString* outPath = [@"Out" stringByAppendingString:images[index].path]; + + NSString* outFolder = outPath.stringByDeletingLastPathComponent; + assert([NSFileManager.defaultManager createDirectoryAtPath:outFolder withIntermediateDirectories:true attributes:nil + error:nil]); + + [Output runWithCache:cache image:images[index] outPath:outPath]; + + trace(@"image took %.2lf seconds", NSDate.date.timeIntervalSince1970 - startTime); + } + } } -int main(int argc,char** argv) -{ - NSString* edge=[@"" stringByPaddingToLength:9+log10(DSCE_VERSION) withString:@"─" startingAtIndex:0]; - trace(@"┌%@┐",edge); - trace(@"│ dsce v%d │",DSCE_VERSION); - trace(@"└%@┘",edge); - - if(argc<3) - { - trace(@"usage: dsce ( list | search | [pad] ... )"); - return 1; - } - - double startTime=NSDate.date.timeIntervalSince1970; - - NSMutableArray* args=NSMutableArray.alloc.init.autorelease; - for(int index=1;index ( list | search | [pad] ... )"); + return 1; + } + + double startTime = NSDate.date.timeIntervalSince1970; + + NSMutableArray* args = NSMutableArray.alloc.init.autorelease; + for (int index = 1; index < argc; index++) { + NSString* arg = [NSString stringWithUTF8String:argv[index]]; + [args addObject:arg]; + } + + process(args); + + trace(@"total %.2lf seconds", NSDate.date.timeIntervalSince1970 - startTime); + + return 0; } \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..fe250d0 --- /dev/null +++ b/Makefile @@ -0,0 +1,16 @@ +VERSION := 6 + +# This will force a rebuild every time, in case dyld changes or headers change +.PHONY: all clean + +SRC=$(wildcard *.m *.mm) +HEADERS=$(wildcard *.h) + +all: dsce + +# This is a bit of a hack to get the binary to rebuild when the headers change +dsce: $(HEADERS) + clang++ -fmodules -fcxx-modules -std=c++17 -Wno-unused-getter-return-value -mmacosx-version-min=12 -Idyld/common -DDSCE_VERSION="$(VERSION)" $(SRC) -o $@ + +clean: + rm -f $(OBJ) dsce diff --git a/Output.h b/Output.h index 9ee5a5e..5149e26 100644 --- a/Output.h +++ b/Output.h @@ -1,14 +1,24 @@ -@interface Output:NSObject - -@property(assign) CacheSet* cache; -@property(assign) CacheImage* cacheImage; -@property(retain) ImageHeader* header; -@property(retain) NSMutableData* data; -@property(retain) NSMutableDictionary* fixups; -@property(retain) NSMutableArray* exports; -@property(retain) NSMutableDictionary* sels; +@import Foundation; + +#import "Address.h" +#import "CacheImage.h" +#import "CacheSet.h" +#import "ImageHeader.h" +#import "Selector.h" + +@interface Output : NSObject + +@property(assign) CacheSet *cache; +@property(assign) CacheImage *cacheImage; +@property(retain) ImageHeader *header; +@property(retain) NSMutableData *data; +@property(retain) NSMutableDictionary *fixups; +@property(retain) NSMutableArray
*exports; +@property(retain) NSMutableDictionary *sels; @property(assign) long baseAddressDelta; -+(void)runWithCache:(CacheSet*)cache image:(CacheImage*)image outPath:(NSString*)outPath; ++ (void)runWithCache:(CacheSet *)cache + image:(CacheImage *)image + outPath:(NSString *)outPath; @end \ No newline at end of file diff --git a/Output.m b/Output.mm similarity index 99% rename from Output.m rename to Output.mm index 7c08e91..6ea2edd 100644 --- a/Output.m +++ b/Output.mm @@ -1,3 +1,7 @@ +#import "Output.h" +#import "Location.h" +#import "Util.h" + #define IMPOSTOR_OBJC_TEMP "dsce.objc" #define IMPOSTOR_OBJC_OLD "dsce.objc.old" #define IMPOSTOR_GOT "dsce.got" diff --git a/Selector.h b/Selector.h index f9a649b..ecb07e6 100644 --- a/Selector.h +++ b/Selector.h @@ -1,3 +1,5 @@ +@import Foundation; + @interface Selector:NSObject @property(assign) long stringAddress; diff --git a/Selector.m b/Selector.mm similarity index 56% rename from Selector.m rename to Selector.mm index e50fc74..31b7a63 100644 --- a/Selector.m +++ b/Selector.mm @@ -1,2 +1,4 @@ +#import "Selector.h" + @implementation Selector @end \ No newline at end of file diff --git a/Util.h b/Util.h new file mode 100644 index 0000000..96a66d7 --- /dev/null +++ b/Util.h @@ -0,0 +1,14 @@ + +@import Foundation; + +inline void trace(NSString *format, ...) { + va_list args; + va_start(args, format); + NSString *message = + [NSString.alloc initWithFormat:format arguments:args].autorelease; + va_end(args); + + printf("\e[%dm%s\e[0m\n", 31 + DSCE_VERSION % 6, message.UTF8String); +} + +extern BOOL flagPad; \ No newline at end of file diff --git a/dyld b/dyld new file mode 160000 index 0000000..f73171c --- /dev/null +++ b/dyld @@ -0,0 +1 @@ +Subproject commit f73171cf0a177f453fdb124952908fe83864acab diff --git a/objc4 b/objc4 new file mode 160000 index 0000000..8701d56 --- /dev/null +++ b/objc4 @@ -0,0 +1 @@ +Subproject commit 8701d5672d3fd3cd817aeb84db1077aafe1a1604 From 5c0792aba7d0274b8bb506579149ce186d13b152 Mon Sep 17 00:00:00 2001 From: Dhinak G <17605561+dhinakg@users.noreply.github.com> Date: Mon, 7 Aug 2023 09:16:34 +0530 Subject: [PATCH 02/21] Add workflows --- .github/workflows/build.yml | 14 ++++++++++++++ .github/workflows/format.yml | 31 +++++++++++++++++++++++++++++++ Makefile | 1 + 3 files changed, 46 insertions(+) create mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/format.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..87aa615 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,14 @@ +name: Build + +on: + push: + pull_request: + workflow_dispatch: + +jobs: + format: + name: Build + runs-on: macos-latest + steps: + - uses: actions/checkout@v3 + - run: make diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml new file mode 100644 index 0000000..f913d00 --- /dev/null +++ b/.github/workflows/format.yml @@ -0,0 +1,31 @@ +name: Check code style + +on: + push: + pull_request: + workflow_dispatch: + +jobs: + format: + name: Check code style + runs-on: macos-latest + steps: + - uses: actions/checkout@v3 + - uses: cpp-linter/cpp-linter-action@v2 + id: linter + with: + style: file + extensions: 'c,h,m,C,H,cpp,mm,hpp,cc,hh,c++,h++,cxx,hxx' + tidy-checks: '-*' + version: '16' + files-changed-only: ${{ github.event_name == 'pull_request' }} + thread-comments: true + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Check for lint failure + if: steps.linter.outputs.checks-failed > 0 + run: | + echo "::error::Code style check failed." + exit 1 + diff --git a/Makefile b/Makefile index fe250d0..b308e08 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,7 @@ all: dsce # This is a bit of a hack to get the binary to rebuild when the headers change dsce: $(HEADERS) +dsce: $(SRC) clang++ -fmodules -fcxx-modules -std=c++17 -Wno-unused-getter-return-value -mmacosx-version-min=12 -Idyld/common -DDSCE_VERSION="$(VERSION)" $(SRC) -o $@ clean: From 4afc7622edf22fea2b90b92f82672ae8bc588078 Mon Sep 17 00:00:00 2001 From: Dhinak G <17605561+dhinakg@users.noreply.github.com> Date: Mon, 7 Aug 2023 09:19:02 +0530 Subject: [PATCH 03/21] Switch to submodules --- dyld | 1 - objc4 | 1 - 2 files changed, 2 deletions(-) delete mode 160000 dyld delete mode 160000 objc4 diff --git a/dyld b/dyld deleted file mode 160000 index f73171c..0000000 --- a/dyld +++ /dev/null @@ -1 +0,0 @@ -Subproject commit f73171cf0a177f453fdb124952908fe83864acab diff --git a/objc4 b/objc4 deleted file mode 160000 index 8701d56..0000000 --- a/objc4 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 8701d5672d3fd3cd817aeb84db1077aafe1a1604 From 12c215a1ec1ced0df35d62cca52bbe2aaced717c Mon Sep 17 00:00:00 2001 From: Dhinak G <17605561+dhinakg@users.noreply.github.com> Date: Mon, 7 Aug 2023 09:19:27 +0530 Subject: [PATCH 04/21] Add submodules --- .gitmodules | 6 ++++++ dyld | 1 + objc4 | 1 + 3 files changed, 8 insertions(+) create mode 100644 .gitmodules create mode 160000 dyld create mode 160000 objc4 diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..159d9eb --- /dev/null +++ b/.gitmodules @@ -0,0 +1,6 @@ +[submodule "dyld"] + path = dyld + url = https://github.com/apple-oss-distributions/dyld.git +[submodule "objc4"] + path = objc4 + url = https://github.com/apple-oss-distributions/objc4.git diff --git a/dyld b/dyld new file mode 160000 index 0000000..ce1cc20 --- /dev/null +++ b/dyld @@ -0,0 +1 @@ +Subproject commit ce1cc2088ef390df1c48a1648075bbd51c5bbc6a diff --git a/objc4 b/objc4 new file mode 160000 index 0000000..c3f0025 --- /dev/null +++ b/objc4 @@ -0,0 +1 @@ +Subproject commit c3f002513d195ef564f3c7e9496c2606360e144a From af1a11a007a670f0db9e10adac3b1cb583412571 Mon Sep 17 00:00:00 2001 From: Dhinak G <17605561+dhinakg@users.noreply.github.com> Date: Mon, 7 Aug 2023 09:20:15 +0530 Subject: [PATCH 05/21] recursive checkout --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 87aa615..f1a30d0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -11,4 +11,6 @@ jobs: runs-on: macos-latest steps: - uses: actions/checkout@v3 + with: + submodules: recursive - run: make From 1b67fc205c641af40ce3c9d5b048e63a1fed3ee7 Mon Sep 17 00:00:00 2001 From: Dhinak G <17605561+dhinakg@users.noreply.github.com> Date: Mon, 7 Aug 2023 09:21:06 +0530 Subject: [PATCH 06/21] use ubuntu for format --- .github/workflows/format.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index f913d00..fd23f0e 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -8,7 +8,7 @@ on: jobs: format: name: Check code style - runs-on: macos-latest + runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - uses: cpp-linter/cpp-linter-action@v2 From c6945045a4c9b9b32fd563090ece292ea60b1f82 Mon Sep 17 00:00:00 2001 From: Dhinak G <17605561+dhinakg@users.noreply.github.com> Date: Mon, 7 Aug 2023 09:22:09 +0530 Subject: [PATCH 07/21] disable thread comments --- .github/workflows/format.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index fd23f0e..838a8ec 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -19,7 +19,6 @@ jobs: tidy-checks: '-*' version: '16' files-changed-only: ${{ github.event_name == 'pull_request' }} - thread-comments: true env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 149dd2e905502fb9127abf46b72dc0f2791d02dd Mon Sep 17 00:00:00 2001 From: Dhinak G <17605561+dhinakg@users.noreply.github.com> Date: Tue, 8 Aug 2023 11:59:13 +0530 Subject: [PATCH 08/21] Cache LLVM --- .github/workflows/build.yml | 8 ++++---- .github/workflows/format.yml | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f1a30d0..cae3224 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -10,7 +10,7 @@ jobs: name: Build runs-on: macos-latest steps: - - uses: actions/checkout@v3 - with: - submodules: recursive - - run: make + - uses: actions/checkout@v3 + with: + submodules: recursive + - run: make diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 838a8ec..329799a 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -5,12 +5,32 @@ on: pull_request: workflow_dispatch: +env: + LLVM_VERSION: 16 + jobs: format: name: Check code style runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + + - name: Add LLVM repo + run: | + if [[ ! -f /etc/apt/trusted.gpg.d/apt.llvm.org.asc ]]; then + # download GPG key once + wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc + fi + source /etc/os-release + sudo add-apt-repository "deb http://apt.llvm.org/${UBUNTU_CODENAME}/ llvm-toolchain-${UBUNTU_CODENAME}-${LLVM_VERSION} main" + sudo apt-get update + + - name: Install LLVM + uses: awalsh128/cache-apt-pkgs-action@latest + with: + packages: clang-format-${LLVM_VERSION} clang-tidy-${LLVM_VERSION} + version: ${UBUNTU_CODENAME}-${LLVM_VERSION} + - uses: cpp-linter/cpp-linter-action@v2 id: linter with: From 2218056e4deeede47f881f15a351c1588d1e861b Mon Sep 17 00:00:00 2001 From: Dhinak G <17605561+dhinakg@users.noreply.github.com> Date: Tue, 8 Aug 2023 12:01:45 +0530 Subject: [PATCH 09/21] fix issues --- .github/workflows/format.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 329799a..56cfe8f 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -22,14 +22,15 @@ jobs: wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc fi source /etc/os-release + echo "UBUNTU_CODENAME=${UBUNTU_CODENAME}" >> $GITHUB_ENV sudo add-apt-repository "deb http://apt.llvm.org/${UBUNTU_CODENAME}/ llvm-toolchain-${UBUNTU_CODENAME}-${LLVM_VERSION} main" sudo apt-get update - name: Install LLVM uses: awalsh128/cache-apt-pkgs-action@latest with: - packages: clang-format-${LLVM_VERSION} clang-tidy-${LLVM_VERSION} - version: ${UBUNTU_CODENAME}-${LLVM_VERSION} + packages: clang-format-${{ env.LLVM_VERSION}} clang-tidy-${{ env.LLVM_VERSION }} + version: ${{ env.UBUNTU_CODENAME }}-${{ env.LLVM_VERSION }} - uses: cpp-linter/cpp-linter-action@v2 id: linter From ef3ede4624b067c5d35e30661b9bdfa7f2486806 Mon Sep 17 00:00:00 2001 From: Dhinak G <17605561+dhinakg@users.noreply.github.com> Date: Tue, 8 Aug 2023 13:15:42 +0530 Subject: [PATCH 10/21] add comment --- .github/workflows/format.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 56cfe8f..03c2167 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -13,6 +13,7 @@ jobs: name: Check code style runs-on: ubuntu-latest steps: + # We do not want to check submodules - uses: actions/checkout@v3 - name: Add LLVM repo From a448ed5bc950aee0955188ae90108cd35824351f Mon Sep 17 00:00:00 2001 From: Dhinak G <17605561+dhinakg@users.noreply.github.com> Date: Tue, 8 Aug 2023 13:41:23 +0530 Subject: [PATCH 11/21] fix typo --- .github/workflows/format.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 03c2167..4fadd4f 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -24,7 +24,7 @@ jobs: fi source /etc/os-release echo "UBUNTU_CODENAME=${UBUNTU_CODENAME}" >> $GITHUB_ENV - sudo add-apt-repository "deb http://apt.llvm.org/${UBUNTU_CODENAME}/ llvm-toolchain-${UBUNTU_CODENAME}-${LLVM_VERSION} main" + sudo add-apt-repository "deb http://apt.llvm.org/${UBUNTU_CODENAME}/ llvm-toolchain-${UBUNTU_CODENAME}-${{ env.LLVM_VERSION }} main" sudo apt-get update - name: Install LLVM From c460c03fe9ce9ad24e700a9f854434132e551553 Mon Sep 17 00:00:00 2001 From: Dhinak G <17605561+dhinakg@users.noreply.github.com> Date: Tue, 8 Aug 2023 13:46:16 +0530 Subject: [PATCH 12/21] Better caching --- .github/workflows/format.yml | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 4fadd4f..4b74b9e 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -16,6 +16,28 @@ jobs: # We do not want to check submodules - uses: actions/checkout@v3 + - name: Set permissions + run: chmod -R a+rwx /var/cache/apt/archives + + - name: Get codename + run: | + source /etc/os-release + echo "UBUNTU_CODENAME=${UBUNTU_CODENAME}" >> $GITHUB_ENV + + - name: Restore cache + uses: actions/cache@v3 + with: + key: ${{ env.UBUNTU_CODENAME }}-apt-${{ env.LLVM_VERSION }} + path: | + /var/cache/apt/archives/**.deb + !/var/cache/apt/archives/partial + !/var/cache/apt/archives/lock + /etc/apt/trusted.gpg.d/apt.llvm.org.asc + /etc/apt/sources.list.d/llvm.list + /var/lib/apt/lists/** + !/var/lib/apt/lists/partial + !/var/lib/apt/lists/lock + - name: Add LLVM repo run: | if [[ ! -f /etc/apt/trusted.gpg.d/apt.llvm.org.asc ]]; then @@ -23,15 +45,9 @@ jobs: wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc fi source /etc/os-release - echo "UBUNTU_CODENAME=${UBUNTU_CODENAME}" >> $GITHUB_ENV - sudo add-apt-repository "deb http://apt.llvm.org/${UBUNTU_CODENAME}/ llvm-toolchain-${UBUNTU_CODENAME}-${{ env.LLVM_VERSION }} main" + echo "deb http://apt.llvm.org/${{ env.UBUNTU_CODENAME }}/ llvm-toolchain-${{ env.UBUNTU_CODENAME }}-${{ env.LLVM_VERSION}} main" | sudo tee /etc/apt/sources.list.d/llvm.list sudo apt-get update - - - name: Install LLVM - uses: awalsh128/cache-apt-pkgs-action@latest - with: - packages: clang-format-${{ env.LLVM_VERSION}} clang-tidy-${{ env.LLVM_VERSION }} - version: ${{ env.UBUNTU_CODENAME }}-${{ env.LLVM_VERSION }} + sudo apt-get install clang-format-${{ env.LLVM_VERSION}} clang-tidy-${{ env.LLVM_VERSION }} - uses: cpp-linter/cpp-linter-action@v2 id: linter From c51d119750379b0f2cbed59212d1f9f1b508e11d Mon Sep 17 00:00:00 2001 From: Dhinak G <17605561+dhinakg@users.noreply.github.com> Date: Tue, 8 Aug 2023 13:47:22 +0530 Subject: [PATCH 13/21] use sudo --- .github/workflows/format.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 4b74b9e..72b001b 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/checkout@v3 - name: Set permissions - run: chmod -R a+rwx /var/cache/apt/archives + run: sudo chmod -R a+rwx /var/cache/apt/archives /etc/apt/trusted.gpg.d/apt.llvm.org.asc /etc/apt/sources.list.d/llvm.list /var/lib/apt/lists - name: Get codename run: | From 7d6c4f247100fc081408815be2684253bfff7e51 Mon Sep 17 00:00:00 2001 From: Dhinak G <17605561+dhinakg@users.noreply.github.com> Date: Tue, 8 Aug 2023 13:48:19 +0530 Subject: [PATCH 14/21] use directories instead of files --- .github/workflows/format.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 72b001b..5f52ac8 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/checkout@v3 - name: Set permissions - run: sudo chmod -R a+rwx /var/cache/apt/archives /etc/apt/trusted.gpg.d/apt.llvm.org.asc /etc/apt/sources.list.d/llvm.list /var/lib/apt/lists + run: sudo chmod -R a+rwx /var/cache/apt/archives /etc/apt/trusted.gpg.d /etc/apt/sources.list.d /var/lib/apt/lists - name: Get codename run: | From a6bf1253ea04a9c4b881027bb73b476f4d6890a9 Mon Sep 17 00:00:00 2001 From: Dhinak G <17605561+dhinakg@users.noreply.github.com> Date: Tue, 8 Aug 2023 13:50:39 +0530 Subject: [PATCH 15/21] temporarily disable lint failure step to test caching --- .github/workflows/format.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 5f52ac8..373d29f 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -61,7 +61,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Check for lint failure - if: steps.linter.outputs.checks-failed > 0 + if: steps.linter.outputs.checks-failed > 0 && false run: | echo "::error::Code style check failed." exit 1 From ed27fe90db631d0bf62addf8261048fd8342c966 Mon Sep 17 00:00:00 2001 From: Dhinak G <17605561+dhinakg@users.noreply.github.com> Date: Tue, 8 Aug 2023 13:53:12 +0530 Subject: [PATCH 16/21] update caching --- .github/workflows/format.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 373d29f..7dcbe47 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -26,6 +26,7 @@ jobs: - name: Restore cache uses: actions/cache@v3 + id: cache with: key: ${{ env.UBUNTU_CODENAME }}-apt-${{ env.LLVM_VERSION }} path: | @@ -39,6 +40,7 @@ jobs: !/var/lib/apt/lists/lock - name: Add LLVM repo + if: steps.cache.outputs.cache-hit != 'true' run: | if [[ ! -f /etc/apt/trusted.gpg.d/apt.llvm.org.asc ]]; then # download GPG key once @@ -46,8 +48,6 @@ jobs: fi source /etc/os-release echo "deb http://apt.llvm.org/${{ env.UBUNTU_CODENAME }}/ llvm-toolchain-${{ env.UBUNTU_CODENAME }}-${{ env.LLVM_VERSION}} main" | sudo tee /etc/apt/sources.list.d/llvm.list - sudo apt-get update - sudo apt-get install clang-format-${{ env.LLVM_VERSION}} clang-tidy-${{ env.LLVM_VERSION }} - uses: cpp-linter/cpp-linter-action@v2 id: linter @@ -55,7 +55,7 @@ jobs: style: file extensions: 'c,h,m,C,H,cpp,mm,hpp,cc,hh,c++,h++,cxx,hxx' tidy-checks: '-*' - version: '16' + version: ${{ env.LLVM_VERSION }} files-changed-only: ${{ github.event_name == 'pull_request' }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 5cecb67b1bf3c235ad15fe1a8c927ad66f294c89 Mon Sep 17 00:00:00 2001 From: Dhinak G <17605561+dhinakg@users.noreply.github.com> Date: Tue, 8 Aug 2023 13:55:52 +0530 Subject: [PATCH 17/21] do not try to access lists/partial --- .github/workflows/format.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 7dcbe47..8b3756b 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -35,7 +35,7 @@ jobs: !/var/cache/apt/archives/lock /etc/apt/trusted.gpg.d/apt.llvm.org.asc /etc/apt/sources.list.d/llvm.list - /var/lib/apt/lists/** + /var/lib/apt/lists/* !/var/lib/apt/lists/partial !/var/lib/apt/lists/lock From 20b1c5c7c7e0239a738e020cc9a84e65335608e1 Mon Sep 17 00:00:00 2001 From: Dhinak G <17605561+dhinakg@users.noreply.github.com> Date: Tue, 8 Aug 2023 13:57:40 +0530 Subject: [PATCH 18/21] set permissions, again --- .github/workflows/format.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 8b3756b..a9809ab 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -66,3 +66,6 @@ jobs: echo "::error::Code style check failed." exit 1 + - name: Set permissions + run: sudo chmod -R a+rwx /var/cache/apt/archives /etc/apt/trusted.gpg.d /etc/apt/sources.list.d /var/lib/apt/lists + From 353d88283f041bf8626052722444ea09af03aab6 Mon Sep 17 00:00:00 2001 From: Dhinak G <17605561+dhinakg@users.noreply.github.com> Date: Tue, 8 Aug 2023 14:00:57 +0530 Subject: [PATCH 19/21] remove extraneous source --- .github/workflows/format.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index a9809ab..85d3a3f 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -46,7 +46,6 @@ jobs: # download GPG key once wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc fi - source /etc/os-release echo "deb http://apt.llvm.org/${{ env.UBUNTU_CODENAME }}/ llvm-toolchain-${{ env.UBUNTU_CODENAME }}-${{ env.LLVM_VERSION}} main" | sudo tee /etc/apt/sources.list.d/llvm.list - uses: cpp-linter/cpp-linter-action@v2 From d32c393e57fc101bd7bf7f13d857242fec43aac5 Mon Sep 17 00:00:00 2001 From: Dhinak G <17605561+dhinakg@users.noreply.github.com> Date: Tue, 8 Aug 2023 14:04:21 +0530 Subject: [PATCH 20/21] don't bother caching lists --- .github/workflows/format.yml | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 85d3a3f..8bb9fca 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/checkout@v3 - name: Set permissions - run: sudo chmod -R a+rwx /var/cache/apt/archives /etc/apt/trusted.gpg.d /etc/apt/sources.list.d /var/lib/apt/lists + run: sudo chmod -R a+rwx /var/cache/apt/archives /etc/apt/keyrings /etc/apt/sources.list.d - name: Get codename run: | @@ -33,20 +33,17 @@ jobs: /var/cache/apt/archives/**.deb !/var/cache/apt/archives/partial !/var/cache/apt/archives/lock - /etc/apt/trusted.gpg.d/apt.llvm.org.asc + /etc/apt/keyrings/apt.llvm.org.asc /etc/apt/sources.list.d/llvm.list - /var/lib/apt/lists/* - !/var/lib/apt/lists/partial - !/var/lib/apt/lists/lock - name: Add LLVM repo if: steps.cache.outputs.cache-hit != 'true' run: | - if [[ ! -f /etc/apt/trusted.gpg.d/apt.llvm.org.asc ]]; then + if [[ ! -f /etc/apt/keyrings/apt.llvm.org.asc ]]; then # download GPG key once - wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc + wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/keyrings/apt.llvm.org.asc fi - echo "deb http://apt.llvm.org/${{ env.UBUNTU_CODENAME }}/ llvm-toolchain-${{ env.UBUNTU_CODENAME }}-${{ env.LLVM_VERSION}} main" | sudo tee /etc/apt/sources.list.d/llvm.list + echo "deb [signed-by=/etc/apt/keyrings/apt.llvm.org.asc] http://apt.llvm.org/${{ env.UBUNTU_CODENAME }}/ llvm-toolchain-${{ env.UBUNTU_CODENAME }}-${{ env.LLVM_VERSION}} main" | sudo tee /etc/apt/sources.list.d/llvm.list - uses: cpp-linter/cpp-linter-action@v2 id: linter @@ -66,5 +63,4 @@ jobs: exit 1 - name: Set permissions - run: sudo chmod -R a+rwx /var/cache/apt/archives /etc/apt/trusted.gpg.d /etc/apt/sources.list.d /var/lib/apt/lists - + run: sudo chmod -R a+rwx /var/cache/apt/archives /etc/apt/keyrings /etc/apt/sources.list.d From 8d64d80e30b6fa9ca8707e639ecabe1b777f5c79 Mon Sep 17 00:00:00 2001 From: Dhinak G <17605561+dhinakg@users.noreply.github.com> Date: Tue, 8 Aug 2023 14:06:38 +0530 Subject: [PATCH 21/21] Remove key downloaded check --- .github/workflows/format.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 8bb9fca..0b689de 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -39,10 +39,7 @@ jobs: - name: Add LLVM repo if: steps.cache.outputs.cache-hit != 'true' run: | - if [[ ! -f /etc/apt/keyrings/apt.llvm.org.asc ]]; then - # download GPG key once - wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/keyrings/apt.llvm.org.asc - fi + wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/keyrings/apt.llvm.org.asc echo "deb [signed-by=/etc/apt/keyrings/apt.llvm.org.asc] http://apt.llvm.org/${{ env.UBUNTU_CODENAME }}/ llvm-toolchain-${{ env.UBUNTU_CODENAME }}-${{ env.LLVM_VERSION}} main" | sudo tee /etc/apt/sources.list.d/llvm.list - uses: cpp-linter/cpp-linter-action@v2