WIP: pe: Add decoder

format/all/all.go

@@ -40,6 +40,7 @@ import (
 	_ "github.com/wader/fq/format/ogg"
 	_ "github.com/wader/fq/format/opus"
 	_ "github.com/wader/fq/format/pcap"
+	_ "github.com/wader/fq/format/pe"
 	_ "github.com/wader/fq/format/png"
 	_ "github.com/wader/fq/format/postgres"
 	_ "github.com/wader/fq/format/prores"

format/format.go

@@ -145,6 +145,9 @@ var (
 	Opus_Packet         = &decode.Group{Name: "opus_packet"}
 	PCAP                = &decode.Group{Name: "pcap"}
 	PCAPNG              = &decode.Group{Name: "pcapng"}
+	PE                  = &decode.Group{Name: "pe"}
+	PE_COFF             = &decode.Group{Name: "pe_coff"}
+	PE_MSDOS_Stub       = &decode.Group{Name: "pe_msdos_stub"}
 	Pg_BTree            = &decode.Group{Name: "pg_btree"}
 	Pg_Control          = &decode.Group{Name: "pg_control"}
 	Pg_Heap             = &decode.Group{Name: "pg_heap"}

format/pe/pe.go

@@ -0,0 +1,37 @@
+package pe
+
+// https://osandamalith.com/2020/07/19/exploring-the-ms-dos-stub/
+
+import (
+	"github.com/wader/fq/format"
+	"github.com/wader/fq/pkg/decode"
+	"github.com/wader/fq/pkg/interp"
+)
+
+// TODO: probe?
+// TODO: not pe_ prefix for format names?
+
+var peMSDosStubGroup decode.Group
+var peCOFFGroup decode.Group
+
+func init() {
+	interp.RegisterFormat(
+		format.PE,
+		&decode.Format{
+			Description: "Portable Executable",
+			Groups:      []*decode.Group{format.Probe},
+			Dependencies: []decode.Dependency{
+				{Groups: []*decode.Group{format.PE_MSDOS_Stub}, Out: &peMSDosStubGroup},
+				{Groups: []*decode.Group{format.PE_COFF}, Out: &peCOFFGroup},
+			},
+			DecodeFn: peDecode,
+		})
+}
+
+func peDecode(d *decode.D) any {
+
+	d.FieldFormat("ms_dos_stub", &peMSDosStubGroup, nil)
+	d.FieldFormat("coff", &peCOFFGroup, nil)
+
+	return nil
+}

format/pe/pe_coff.go

@@ -0,0 +1,238 @@
+package pe
+
+// https://osandamalith.com/2020/07/19/exploring-the-ms-dos-stub/
+
+import (
+	"time"
+
+	"github.com/wader/fq/format"
+	"github.com/wader/fq/pkg/decode"
+	"github.com/wader/fq/pkg/interp"
+	"github.com/wader/fq/pkg/scalar"
+)
+
+// TODO: probe?
+
+func init() {
+	interp.RegisterFormat(
+		format.PE_COFF,
+		&decode.Format{
+			Description: "Common Object File Format",
+			DecodeFn:    peCoffStubDecode,
+		})
+}
+
+const (
+	peFormat32     = 0x10b
+	peFormat32Plus = 0x20b
+)
+
+var peFormatNames = scalar.UintMapSymStr{
+	peFormat32:     "pe32",
+	peFormat32Plus: "pe32+",
+}
+
+const (
+	MachineTypeUNKNOWN     = 0x0
+	MachineTypeALPHA       = 0x184
+	MachineTypeALPHA64     = 0x284
+	MachineTypeAM33        = 0x1d3
+	MachineTypeAMD64       = 0x8664
+	MachineTypeARM         = 0x1c0
+	MachineTypeARM64       = 0xaa64
+	MachineTypeARMNT       = 0x1c4
+	MachineTypeAXP64       = 0x284
+	MachineTypeEBC         = 0xebc
+	MachineTypeI386        = 0x14c
+	MachineTypeIA64        = 0x200
+	MachineTypeLOONGARCH32 = 0x6232
+	MachineTypeLOONGARCH64 = 0x6264
+	MachineTypeM32R        = 0x9041
+	MachineTypeMIPS16      = 0x266
+	MachineTypeMIPSFPU     = 0x366
+	MachineTypeMIPSFPU16   = 0x466
+	MachineTypePOWERPC     = 0x1f0
+	MachineTypePOWERPCFP   = 0x1f1
+	MachineTypeR4000       = 0x166
+	MachineTypeRISCV32     = 0x5032
+	MachineTypeRISCV64     = 0x5064
+	MachineTypeRISCV128    = 0x5128
+	MachineTypeSH3         = 0x1a2
+	MachineTypeSH3DSP      = 0x1a3
+	MachineTypeSH4         = 0x1a6
+	MachineTypeSH5         = 0x1a8
+	MachineTypeTHUMB       = 0x1c2
+	MachineTypeWCEMIPSV2   = 0x169
+)
+
+var MachineTypeNames = scalar.UintMap{
+	MachineTypeUNKNOWN: {Sym: "UNKNOWN", Description: "The content of this field is assumed to be applicable to any machine type"},
+	MachineTypeALPHA:   {Sym: "ALPHA", Description: "Alpha AXP, 32-bit address space"},
+	MachineTypeALPHA64: {Sym: "ALPHA64", Description: "Alpha 64, 64-bit address space"},
+	MachineTypeAM33:    {Sym: "AM33", Description: "Matsushita AM33"},
+	MachineTypeAMD64:   {Sym: "AMD64", Description: "x64"},
+	MachineTypeARM:     {Sym: "ARM", Description: "ARM little endian"},
+	MachineTypeARM64:   {Sym: "ARM64", Description: "ARM64 little endian"},
+	MachineTypeARMNT:   {Sym: "ARMNT", Description: "ARM Thumb-2 little endian"},
+	//MachineTypeAXP64:       {Sym: "AXP64", Description: "AXP 64 (Same as Alpha 64)"},
+	MachineTypeEBC:         {Sym: "EBC", Description: "EFI byte code"},
+	MachineTypeI386:        {Sym: "I386", Description: "Intel 386 or later processors and compatible processors"},
+	MachineTypeIA64:        {Sym: "IA64", Description: "Intel Itanium processor family"},
+	MachineTypeLOONGARCH32: {Sym: "LOONGARCH32", Description: "LoongArch 32-bit processor family"},
+	MachineTypeLOONGARCH64: {Sym: "LOONGARCH64", Description: "LoongArch 64-bit processor family"},
+	MachineTypeM32R:        {Sym: "M32R", Description: "Mitsubishi M32R little endian"},
+	MachineTypeMIPS16:      {Sym: "MIPS16", Description: "MIPS16"},
+	MachineTypeMIPSFPU:     {Sym: "MIPSFPU", Description: "MIPS with FPU"},
+	MachineTypeMIPSFPU16:   {Sym: "MIPSFPU16", Description: "MIPS16 with FPU"},
+	MachineTypePOWERPC:     {Sym: "POWERPC", Description: "Power PC little endian"},
+	MachineTypePOWERPCFP:   {Sym: "POWERPCFP", Description: "Power PC with floating point support"},
+	MachineTypeR4000:       {Sym: "R4000", Description: "MIPS little endian"},
+	MachineTypeRISCV32:     {Sym: "RISCV32", Description: "RISC-V 32-bit address space"},
+	MachineTypeRISCV64:     {Sym: "RISCV64", Description: "RISC-V 64-bit address space"},
+	MachineTypeRISCV128:    {Sym: "RISCV128", Description: "RISC-V 128-bit address space"},
+	MachineTypeSH3:         {Sym: "SH3", Description: "Hitachi SH3"},
+	MachineTypeSH3DSP:      {Sym: "SH3DSP", Description: "Hitachi SH3 DSP"},
+	MachineTypeSH4:         {Sym: "SH4", Description: "Hitachi SH4"},
+	MachineTypeSH5:         {Sym: "SH5", Description: "Hitachi SH5"},
+	MachineTypeTHUMB:       {Sym: "THUMB", Description: "Thumb"},
+	MachineTypeWCEMIPSV2:   {Sym: "WCEMIPSV2", Description: "MIPS little-endian WCE v2"},
+}
+
+const (
+	SubSystemUNKNOWN                  = 0
+	SubSystemNATIVE                   = 1
+	SubSystemWINDOWS_GUI              = 2
+	SubSystemWINDOWS_CUI              = 3
+	SubSystemOS2_CUI                  = 5
+	SubSystemPOSIX_CUI                = 7
+	SubSystemNATIVE_WINDOWS           = 8
+	SubSystemWINDOWS_CE_GUI           = 9
+	SubSystemEFI_APPLICATION          = 10
+	SubSystemEFI_BOOT_SERVICE_DRIVER  = 11
+	SubSystemEFI_RUNTIME_DRIVER       = 12
+	SubSystemEFI_ROM                  = 13
+	SubSystemXBOX                     = 14
+	SubSystemWINDOWS_BOOT_APPLICATION = 16
+)
+
+var subSystemNames = scalar.UintMap{
+	SubSystemUNKNOWN:                  {Sym: "UNKNOWN", Description: "An unknown subsystem"},
+	SubSystemNATIVE:                   {Sym: "NATIVE", Description: "Device drivers and native Windows processes"},
+	SubSystemWINDOWS_GUI:              {Sym: "WINDOWS_GUI", Description: "The Windows graphical user interface (GUI) subsystem"},
+	SubSystemWINDOWS_CUI:              {Sym: "WINDOWS_CUI", Description: "The Windows character subsystem"},
+	SubSystemOS2_CUI:                  {Sym: "OS2_CUI", Description: "The OS/2 character subsystem"},
+	SubSystemPOSIX_CUI:                {Sym: "POSIX_CUI", Description: "The Posix character subsystem"},
+	SubSystemNATIVE_WINDOWS:           {Sym: "NATIVE_WINDOWS", Description: "Native Win9x driver"},
+	SubSystemWINDOWS_CE_GUI:           {Sym: "WINDOWS_CE_GUI", Description: "Windows CE"},
+	SubSystemEFI_APPLICATION:          {Sym: "EFI_APPLICATION", Description: "An Extensible Firmware Interface (EFI) application"},
+	SubSystemEFI_BOOT_SERVICE_DRIVER:  {Sym: "EFI_BOOT_SERVICE_DRIVER", Description: "An EFI driver with boot services"},
+	SubSystemEFI_RUNTIME_DRIVER:       {Sym: "EFI_RUNTIME_DRIVER", Description: "An EFI driver with run-time services"},
+	SubSystemEFI_ROM:                  {Sym: "EFI_ROM", Description: "An EFI ROM image"},
+	SubSystemXBOX:                     {Sym: "XBOX", Description: "XBOX"},
+	SubSystemWINDOWS_BOOT_APPLICATION: {Sym: "WINDOWS_BOOT_APPLICATION", Description: "Windows boot application."},
+}
+
+func peCoffStubDecode(d *decode.D) any {
+	d.Endian = decode.LittleEndian
+
+	d.FieldRawLen("signature", 4*8, d.AssertBitBuf([]byte("PE\x00\x00")))
+	d.FieldU16("machine", MachineTypeNames, scalar.UintHex)
+	d.FieldU16("number_of_sections")
+	d.FieldU32("time_date_stamp", scalar.UintActualUnixTime(time.RFC3339))
+	d.FieldU32("pointer_to_symbol_table", scalar.UintHex)
+	d.FieldU32("number_of_symbol_table")
+	sizeOfOptionalHeader := d.FieldU16("size_of_optional_header")
+	d.FieldStruct("characteristics", func(d *decode.D) {
+		// TODO: wrong byte order
+		d.FieldBool("BYTES_REVERSED_HI")   //  0x8000 // Big endian: the MSB precedes the LSB in memory. This flag is deprecated and should be zero.
+		d.FieldBool("UP_SYSTEM_ONLY")      //  0x4000 // The file should be run only on a uniprocessor machine.
+		d.FieldBool("DLL")                 //  0x2000 // The image file is a dynamic-link library (DLL). Such files are considered executable files for almost all purposes, although they cannot be directly run.
+		d.FieldBool("SYSTEM")              //  0x1000 // The image file is a system file, not a user program.
+		d.FieldBool("NET_RUN_FROM_SWAP")   //  0x0800 // If the image is on network media, fully load it and copy it to the swap file.
+		d.FieldBool("REMOVABLE_RUN_")      //  FROM_SWAP 0x0400 // If the image is on removable media, fully load it and copy it to the swap file.
+		d.FieldBool("DEBUG_STRIPPED")      //  0x0200 // Debugging information is removed from the image file.
+		d.FieldBool("32BIT_MACHINE")       //  0x0100 // Machine is based on a 32-bit-word architecture.
+		d.FieldBool("BYTES_REVERSED_LO")   //  0x0080 // Little endian: the least significant bit (LSB) precedes the most significant bit (MSB) in memory. This flag is deprecated and should be zero.
+		d.FieldBool("RESERVED")            //  0x0040 // This flag is reserved for future use.
+		d.FieldBool("LARGE_ADDRESS_")      //  AWARE 0x0020 // Application can handle > 2-GB addresses.
+		d.FieldBool("AGGRESSIVE_WS_TRIM")  //  0x0010 // Obsolete. Aggressively trim working set. This flag is deprecated for Windows 2000 and later and must be zero.
+		d.FieldBool("LOCAL_SYMS_STRIPPED") //  0x0008 // COFF symbol table entries for local symbols have been removed. This flag is deprecated and should be zero.
+		d.FieldBool("LINE_NUMS_STRIPPED")  //  0x0004 // COFF line numbers have been removed. This flag is deprecated and should be zero.
+		d.FieldBool("EXECUTABLE_IMAGE")    //  0x0002 // Image only. This indicates that the image file is valid and can be run. If this flag is not set, it indicates a linker error.
+		d.FieldBool("RELOCS_STRIPPED")     //  0x0001 // Image only, Windows CE, and Microsoft Windows NT and later. This indicates that the file does not contain base relocations and must therefore be loaded at its preferred base address. If the base address is not available, the loader reports an error. The default behavior of the linker is to strip base relocations from executable (EXE) files.
+	})
+
+	// how to know if image only? windows specific?
+	if sizeOfOptionalHeader > 0 {
+		d.FieldStruct("optional_header", func(d *decode.D) {
+			d.FramedFn(int64(sizeOfOptionalHeader)*8, func(d *decode.D) {
+				peFormat := d.FieldU16("format", peFormatNames, scalar.UintHex)
+				d.FieldU8("major_linker_version")
+				d.FieldU8("minor_linker_version")
+				d.FieldU32("size_of_code")
+				d.FieldU32("size_of_initialized_data")
+				d.FieldU32("size_of_uninitialized_data")
+				d.FieldU32("address_of_entry_point", scalar.UintHex)
+				d.FieldU32("base_of_code", scalar.UintHex)
+				addrSize := 64
+				if peFormat == peFormat32 {
+					d.FieldU32("base_of_data", scalar.UintHex)
+					addrSize = 32
+				}
+
+				d.FieldU("image_base", addrSize, scalar.UintHex)
+				d.FieldU32("section_alignment")
+				d.FieldU32("file_alignment")
+				d.FieldU16("major_os_version")
+				d.FieldU16("minor_os_version")
+				d.FieldU16("major_image_version")
+				d.FieldU16("minor_image_version")
+				d.FieldU16("major_subsystem_version")
+				d.FieldU16("minor_subsystem_version")
+				d.FieldU32("win32_version")
+				d.FieldU32("size_of_image")
+				d.FieldU32("size_of_headers")
+				d.FieldU32("chunk_sum", scalar.UintHex)
+				d.FieldU16("subsystem", subSystemNames)
+				d.FieldStruct("dll_characteristics", func(d *decode.D) {
+					d.FieldBool("force_integrity") // Code Integrity checks are enforced.
+					d.FieldBool("dynamic_base")    // DLL can be relocated at load time.
+					d.FieldBool("high_entropy_va") // Image can handle a high entropy 64-bit virtual address space.
+					d.FieldBool("reserved0")       // ??
+					d.FieldBool("reserved1")
+					d.FieldBool("reserved2")
+					d.FieldBool("reserved3")
+					d.FieldBool("reserved4")
+
+					d.FieldBool("terminal_server_aware") // Terminal Server aware.
+					d.FieldBool("guard_cf")              // Image supports Control Flow Guard.
+					d.FieldBool("wdm_driver")            // A WDM driver.
+					d.FieldBool("appcontainer")          // Image must execute in an AppContainer.
+					d.FieldBool("no_bind")               // Do not bind the image.
+					d.FieldBool("no_seh")                // Does not use structured exception (SE) handling. No SE handler may be called in this image.
+					d.FieldBool("no_isolation")          // Isolation aware, but do not isolate the image.
+					d.FieldBool("nx_compat")             // Image is NX compatible.
+				})
+				d.FieldU("size_of_track_reserve", addrSize)
+				d.FieldU("size_of_stack_commit", addrSize)
+				d.FieldU("size_of_heap_reserve", addrSize)
+				d.FieldU("size_of_heap_commit", addrSize)
+				d.FieldU32("loader_flags")
+				numberOfRvaAndSizes := d.FieldU32("number_of_rva_and_sizes")
+				d.FieldArray("data_directories", func(d *decode.D) {
+					for i := 0; i < int(numberOfRvaAndSizes); i++ {
+						d.FieldStruct("data_directory", func(d *decode.D) {
+							d.FieldU32("virtual_address", scalar.UintHex)
+							d.FieldU32("size")
+						})
+					}
+				})
+
+				//d.FieldRawLen("unknown", d.BitsLeft())
+			})
+		})
+
+	}
+
+	return nil
+}

format/pe/pe_msdos_stub.go

@@ -0,0 +1,56 @@
+package pe
+
+// https://osandamalith.com/2020/07/19/exploring-the-ms-dos-stub/
+
+import (
+	"github.com/wader/fq/format"
+	"github.com/wader/fq/pkg/decode"
+	"github.com/wader/fq/pkg/interp"
+	"github.com/wader/fq/pkg/scalar"
+)
+
+// TODO: probe?
+
+func init() {
+	interp.RegisterFormat(
+		format.PE_MSDOS_Stub,
+		&decode.Format{
+			Description: "MS-DOS Stub",
+			DecodeFn:    msDosStubDecode,
+		})
+}
+
+func msDosStubDecode(d *decode.D) any {
+	d.Endian = decode.LittleEndian
+
+	d.FieldU16("e_magic", scalar.UintDescription("Magic number"), d.UintAssert(0x5a4d), scalar.UintHex)
+	d.FieldU16("e_cblp", scalar.UintDescription("Bytes on last page of file"))
+	d.FieldU16("e_cp", scalar.UintDescription("Pages in file"))
+	d.FieldU16("e_crlc", scalar.UintDescription("Relocations"))
+	d.FieldU16("e_cparhdr", scalar.UintDescription("Size of header in paragraphs"))
+	d.FieldU16("e_minalloc", scalar.UintDescription("Minimum extra paragraphs needed"))
+	d.FieldU16("e_maxalloc", scalar.UintDescription("Maximum extra paragraphs needed"))
+	d.FieldU16("e_ss", scalar.UintDescription("Initial (relative) SS value"))
+	d.FieldU16("e_sp", scalar.UintDescription("Initial SP value"))
+	d.FieldU16("e_csum", scalar.UintDescription("Checksum"))
+	d.FieldU16("e_ip", scalar.UintDescription("Initial IP value"))
+	d.FieldU16("e_cs", scalar.UintDescription("Initial (relative) CS value"))
+	d.FieldU16("e_lfarlc", scalar.UintDescription("File address of relocation table"))
+	d.FieldU16("e_ovno", scalar.UintDescription("Overlay number"))
+	d.FieldRawLen("e_res", 4*16, scalar.BitBufDescription("Reserved words"))
+	d.FieldU16("e_oemid", scalar.UintDescription("OEM identifier (for e_oeminfo)"))
+	d.FieldU16("e_oeminfo", scalar.UintDescription("OEM information; e_oemid specific"))
+	d.FieldRawLen("e_res2", 10*16, scalar.BitBufDescription("Reserved words"))
+	lfanew := d.FieldU32("e_lfanew", scalar.UintDescription("File address of new exe header"))
+
+	// TODO: x86 format in the future
+	d.FieldRawLen("stub", 64*8, scalar.BitBufDescription("Sub program"))
+
+	subEndPos := d.Pos()
+
+	// TODO: is not padding i guess?
+	padding := lfanew*8 - uint64(subEndPos)
+	d.FieldRawLen("padding", int64(padding))
+
+	return nil
+}