pe,pe_msdos_stub,coff: Add decoders

format/all/all.fqtest

@@ -23,6 +23,7 @@ $ fq -n _registry.groups.probe
   "ogg",
   "pcap",
   "pcapng",
+  "pe",
   "png",
   "tar",
   "tiff",
@@ -76,6 +77,7 @@ bytes                Raw bytes
 bzip2                bzip2 compression
 caff                 Live2D Cubism archive
 cbor                 Concise Binary Object Representation
+coff                 Common Object File Format
 csv                  Comma separated values
 dns                  DNS packet
 dns_tcp              DNS packet (TCP)
@@ -127,12 +129,14 @@ mpeg_pes             MPEG Packetized elementary stream
 mpeg_pes_packet      MPEG Packetized elementary stream packet
 mpeg_spu             Sub Picture Unit (DVD subtitle)
 mpeg_ts              MPEG Transport Stream
+msdos_stub           MS-DOS Stub
 msgpack              MessagePack
 ogg                  OGG file
 ogg_page             OGG page
 opus_packet          Opus packet
 pcap                 PCAP packet capture
 pcapng               PCAPNG packet capture
+pe                   Portable Executable
 pg_btree             PostgreSQL btree index file
 pg_control           PostgreSQL control file
 pg_heap              PostgreSQL heap file

format/all/all.go

@@ -42,6 +42,7 @@ import (
 	_ "github.com/wader/fq/format/ogg"
 	_ "github.com/wader/fq/format/opus"
 	_ "github.com/wader/fq/format/pcap"
+	_ "github.com/wader/fq/format/pe"
 	_ "github.com/wader/fq/format/png"
 	_ "github.com/wader/fq/format/postgres"
 	_ "github.com/wader/fq/format/prores"

format/format.go

@@ -89,6 +89,7 @@ var (
 	Bzip2               = &decode.Group{Name: "bzip2"}
 	CAFF                = &decode.Group{Name: "caff"}
 	CBOR                = &decode.Group{Name: "cbor"}
+	COFF                = &decode.Group{Name: "coff"}
 	CSV                 = &decode.Group{Name: "csv"}
 	DNS                 = &decode.Group{Name: "dns"}
 	DNS_TCP             = &decode.Group{Name: "dns_tcp"}
@@ -137,16 +138,18 @@ var (
 	MP4                 = &decode.Group{Name: "mp4"}
 	MPEG_ASC            = &decode.Group{Name: "mpeg_asc"}
 	MPEG_ES             = &decode.Group{Name: "mpeg_es"}
-	MPES_PES            = &decode.Group{Name: "mpeg_pes"}
+	MPEG_PES            = &decode.Group{Name: "mpeg_pes"}
 	MPEG_PES_Packet     = &decode.Group{Name: "mpeg_pes_packet"}
 	MPEG_SPU            = &decode.Group{Name: "mpeg_spu"}
 	MPEG_TS             = &decode.Group{Name: "mpeg_ts"}
+	MSDOS_Stub          = &decode.Group{Name: "msdos_stub"}
 	MsgPack             = &decode.Group{Name: "msgpack"}
 	Ogg                 = &decode.Group{Name: "ogg"}
 	Ogg_Page            = &decode.Group{Name: "ogg_page"}
 	Opus_Packet         = &decode.Group{Name: "opus_packet"}
 	PCAP                = &decode.Group{Name: "pcap"}
 	PCAPNG              = &decode.Group{Name: "pcapng"}
+	PE                  = &decode.Group{Name: "pe"}
 	Pg_BTree            = &decode.Group{Name: "pg_btree"}
 	Pg_Control          = &decode.Group{Name: "pg_control"}
 	Pg_Heap             = &decode.Group{Name: "pg_heap"}
@@ -396,3 +399,11 @@ type Pg_Heap_In struct {
 type Pg_BTree_In struct {
 	Page int `doc:"First page number in file, default is 0"`
 }
+
+type MS_DOS_Out struct {
+	LFANew int // logical file address for the New Executable header
+}
+
+type COFF_In struct {
+	FilePointerOffset int `doc:"File pointer offset"`
+}

format/mpeg/mpeg_pes.go

@@ -16,7 +16,7 @@ var mpegSpuGroup decode.Group
 
 func init() {
 	interp.RegisterFormat(
-		format.MPES_PES,
+		format.MPEG_PES,
 		&decode.Format{
 			Description: "MPEG Packetized elementary stream",
 			DecodeFn:    pesDecode,

format/pe/msdos_stub.go

@@ -0,0 +1,63 @@
+package pe
+
+// https://osandamalith.com/2020/07/19/exploring-the-ms-dos-stub/
+
+import (
+	"github.com/wader/fq/format"
+	"github.com/wader/fq/internal/mathex"
+	"github.com/wader/fq/pkg/decode"
+	"github.com/wader/fq/pkg/interp"
+	"github.com/wader/fq/pkg/scalar"
+)
+
+// TODO: probe?
+
+func init() {
+	interp.RegisterFormat(
+		format.MSDOS_Stub,
+		&decode.Format{
+			Description: "MS-DOS Stub",
+			DecodeFn:    msDosStubDecode,
+		})
+}
+
+func msDosStubDecode(d *decode.D) any {
+	d.Endian = decode.LittleEndian
+
+	d.FieldU16("e_magic", scalar.UintDescription("Magic number"), d.UintAssert(0x5a4d), scalar.UintHex)
+	d.FieldU16("e_cblp", scalar.UintDescription("Bytes on last page of file"))
+	d.FieldU16("e_cp", scalar.UintDescription("Pages in file"))
+	d.FieldU16("e_crlc", scalar.UintDescription("Relocations"))
+	d.FieldU16("e_cparhdr", scalar.UintDescription("Size of header in paragraphs"))
+	d.FieldU16("e_minalloc", scalar.UintDescription("Minimum extra paragraphs needed"))
+	d.FieldU16("e_maxalloc", scalar.UintDescription("Maximum extra paragraphs needed"))
+	d.FieldU16("e_ss", scalar.UintDescription("Initial (relative) SS value"))
+	d.FieldU16("e_sp", scalar.UintDescription("Initial SP value"))
+	d.FieldU16("e_csum", scalar.UintDescription("Checksum"))
+	d.FieldU16("e_ip", scalar.UintDescription("Initial IP value"))
+	d.FieldU16("e_cs", scalar.UintDescription("Initial (relative) CS value"))
+	d.FieldU16("e_lfarlc", scalar.UintDescription("File address of relocation table"))
+	d.FieldU16("e_ovno", scalar.UintDescription("Overlay number"))
+	d.FieldRawLen("e_res", 4*16, scalar.BitBufDescription("Reserved words"))
+	d.FieldU16("e_oemid", scalar.UintDescription("OEM identifier (for e_oeminfo)"))
+	d.FieldU16("e_oeminfo", scalar.UintDescription("OEM information; e_oemid specific"))
+	d.FieldRawLen("e_res2", 10*16, scalar.BitBufDescription("Reserved words"))
+	lfanew := d.FieldU32("e_lfanew", scalar.UintDescription("File address of new exe header"))
+
+	// TODO: how to detect UEFI?
+
+	subEndPos := mathex.Min(d.Pos()+64*8, int64(lfanew)*8)
+
+	// TODO: x86 format in the future
+	d.FieldRawLen("stub", subEndPos-d.Pos(), scalar.BitBufDescription("Sub program"))
+
+	// TODO: is not padding i guess?
+	padding := lfanew*8 - uint64(subEndPos)
+	if padding > 0 {
+		d.FieldRawLen("padding", int64(padding))
+	}
+
+	return format.MS_DOS_Out{
+		LFANew: int(lfanew),
+	}
+}

format/pe/pe.go

@@ -0,0 +1,42 @@
+package pe
+
+// https://osandamalith.com/2020/07/19/exploring-the-ms-dos-stub/
+
+import (
+	"fmt"
+
+	"github.com/wader/fq/format"
+	"github.com/wader/fq/pkg/decode"
+	"github.com/wader/fq/pkg/interp"
+)
+
+// TODO: probe?
+// TODO: not pe_ prefix for format names?
+
+var msDosStubGroup decode.Group
+var coffGroup decode.Group
+
+func init() {
+	interp.RegisterFormat(
+		format.PE,
+		&decode.Format{
+			Description: "Portable Executable",
+			Groups:      []*decode.Group{format.Probe},
+			Dependencies: []decode.Dependency{
+				{Groups: []*decode.Group{format.MSDOS_Stub}, Out: &msDosStubGroup},
+				{Groups: []*decode.Group{format.COFF}, Out: &coffGroup},
+			},
+			DecodeFn: peDecode,
+		})
+}
+
+func peDecode(d *decode.D) any {
+	_, v := d.FieldFormat("ms_dos_stub", &msDosStubGroup, nil)
+	msDOSOut, ok := v.(format.MS_DOS_Out)
+	if !ok {
+		panic(fmt.Sprintf("expected MS_DOS_Out got %#+v", v))
+	}
+	d.FieldFormat("coff", &coffGroup, format.COFF_In{FilePointerOffset: msDOSOut.LFANew})
+
+	return nil
+}

format/pe/testdata/README.md

@@ -0,0 +1,7 @@
+PE sample from https://github.com/JonathanSalwan/binary-samples under MIT license
+
+```sh
+rm -f *.fqtest; for i in pe-* *.efi; do echo "\$ fq dv $i" > $i.fqtest ; done
+```
+
+EFI sample from https://github.com/badcf00d/UEFI-helloworld under BSD license