From c006864fd5faf7060c22c5ec1e4f541376006f3e Mon Sep 17 00:00:00 2001 From: elicn Date: Fri, 10 Mar 2023 11:57:53 +0200 Subject: [PATCH 01/42] Remove unnecessary property elf_emu_start --- qiling/extensions/idaplugin/qilingida.py | 2 +- qiling/loader/elf.py | 1 - qiling/os/freebsd/freebsd.py | 1 - qiling/os/linux/linux.py | 1 - qiling/os/qnx/qnx.py | 1 - 5 files changed, 1 insertion(+), 5 deletions(-) diff --git a/qiling/extensions/idaplugin/qilingida.py b/qiling/extensions/idaplugin/qilingida.py index 2d68a258a..f3524ec94 100644 --- a/qiling/extensions/idaplugin/qilingida.py +++ b/qiling/extensions/idaplugin/qilingida.py @@ -899,7 +899,7 @@ def start(self, *args, **kwargs): elffile = ELFFile(f) elf_header = elffile.header if elf_header['e_type'] == 'ET_EXEC': - self.baseaddr = self.ql.os.elf_mem_start + self.baseaddr = self.ql.loader.images[0].base elif elf_header['e_type'] == 'ET_DYN': if self.ql.arch.bits == 32: self.baseaddr = int(self.ql.os.profile.get("OS32", "load_address"), 16) diff --git a/qiling/loader/elf.py b/qiling/loader/elf.py index 5b44ef570..639a32a12 100644 --- a/qiling/loader/elf.py +++ b/qiling/loader/elf.py @@ -353,7 +353,6 @@ def __push_str(top: int, s: str) -> int: self.init_sp = self.ql.arch.regs.arch_sp self.ql.os.entry_point = self.entry_point = entry_point - self.ql.os.elf_mem_start = mem_start self.ql.os.elf_entry = self.elf_entry self.ql.os.function_hook = FunctionHook(self.ql, elf_phdr, elf_phnum, elf_phent, load_address, mem_end) diff --git a/qiling/os/freebsd/freebsd.py b/qiling/os/freebsd/freebsd.py index e0218d4da..b578198ae 100644 --- a/qiling/os/freebsd/freebsd.py +++ b/qiling/os/freebsd/freebsd.py @@ -16,7 +16,6 @@ class QlOsFreebsd(QlOsPosix): def __init__(self, ql): super(QlOsFreebsd, self).__init__(ql) - self.elf_mem_start = 0x0 self.load() diff --git a/qiling/os/linux/linux.py b/qiling/os/linux/linux.py index 959cf5ab5..d10bb9033 100644 --- a/qiling/os/linux/linux.py +++ b/qiling/os/linux/linux.py @@ -48,7 +48,6 @@ def __init__(self, ql: Qiling): self.futexm = None self.fh = None self.function_after_load_list = [] - self.elf_mem_start = 0x0 self.load() def load(self): diff --git a/qiling/os/qnx/qnx.py b/qiling/os/qnx/qnx.py index 9f79f1d95..1d6172dd1 100644 --- a/qiling/os/qnx/qnx.py +++ b/qiling/os/qnx/qnx.py @@ -45,7 +45,6 @@ def __init__(self, ql: Qiling): self.futexm = None self.fh = None self.function_after_load_list = [] - self.elf_mem_start = 0x0 self.load() # use counters to get free Ids From 28e1f3323da5a10beab807c92e080f57a3f38640 Mon Sep 17 00:00:00 2001 From: elicn Date: Fri, 10 Mar 2023 12:07:49 +0200 Subject: [PATCH 02/42] Fix unsafe ELF interpreter loading --- qiling/loader/elf.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/qiling/loader/elf.py b/qiling/loader/elf.py index 639a32a12..f3df83b99 100644 --- a/qiling/loader/elf.py +++ b/qiling/loader/elf.py @@ -230,10 +230,15 @@ def load_elf_segments(elffile: ELFFile, load_address: int, info: str): # load the interpreter, if there is one if interp_path: - interp_local_path = os.path.normpath(self.ql.rootfs + interp_path) - self.ql.log.debug(f'Interpreter path: {interp_local_path}') + interp_vpath = self.ql.os.path.virtual_abspath(interp_path) + interp_hpath = self.ql.os.path.virtual_to_host_path(interp_path) - with open(interp_local_path, 'rb') as infile: + self.ql.log.debug(f'Interpreter path: {interp_vpath}') + + if not self.ql.os.path.is_safe_host_path(interp_hpath): + raise PermissionError(f'unsafe path: {interp_hpath}') + + with open(interp_hpath, 'rb') as infile: interp = ELFFile(infile) min_vaddr = min(seg['p_vaddr'] for seg in interp.iter_segments(type='PT_LOAD')) @@ -244,10 +249,10 @@ def load_elf_segments(elffile: ELFFile, load_address: int, info: str): self.ql.log.debug(f'Interpreter addr: {interp_address:#x}') # load interpreter segments data to memory - interp_start, interp_end = load_elf_segments(interp, interp_address, interp_local_path) + interp_start, interp_end = load_elf_segments(interp, interp_address, interp_vpath) # add interpreter to the loaded images list - self.images.append(Image(interp_start, interp_end, os.path.abspath(interp_local_path))) + self.images.append(Image(interp_start, interp_end, interp_hpath)) # determine entry point entry_point = interp_address + interp['e_entry'] From f6200acda4548cff22ba4e3fe70b2ea4058481af Mon Sep 17 00:00:00 2001 From: elicn Date: Fri, 10 Mar 2023 12:47:29 +0200 Subject: [PATCH 03/42] Fix wrapper decoration --- qiling/core_hooks.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/qiling/core_hooks.py b/qiling/core_hooks.py index f142684fd..411ce5941 100644 --- a/qiling/core_hooks.py +++ b/qiling/core_hooks.py @@ -8,7 +8,7 @@ # handling hooks # ############################################## -import functools +from functools import wraps from typing import Any, Callable, MutableMapping, MutableSequence, Protocol from typing import TYPE_CHECKING @@ -87,8 +87,7 @@ def __call__(self, __ql: 'Qiling', intno: int, *__context: Any) -> Any: def hookcallback(ql: 'Qiling', callback: Callable): - - functools.wraps(callback) + @wraps(callback) def wrapper(*args, **kwargs): try: return callback(*args, **kwargs) From 4b4ed620f04ad45af533fc32027aeaad25a9260a Mon Sep 17 00:00:00 2001 From: elicn Date: Fri, 10 Mar 2023 12:49:38 +0200 Subject: [PATCH 04/42] Reduce module dependencies --- qiling/core_hooks.py | 44 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/qiling/core_hooks.py b/qiling/core_hooks.py index 411ce5941..49dbcebe4 100644 --- a/qiling/core_hooks.py +++ b/qiling/core_hooks.py @@ -8,22 +8,52 @@ # handling hooks # ############################################## +from __future__ import annotations + from functools import wraps from typing import Any, Callable, MutableMapping, MutableSequence, Protocol from typing import TYPE_CHECKING -from unicorn import Uc -from unicorn.unicorn_const import * +from unicorn.unicorn_const import ( + UC_HOOK_INTR, + UC_HOOK_INSN, + UC_HOOK_CODE, + UC_HOOK_BLOCK, + + UC_HOOK_MEM_READ_UNMAPPED, # attempt to read from an unmapped memory location + UC_HOOK_MEM_WRITE_UNMAPPED, # attempt to write to an unmapped memory location + UC_HOOK_MEM_FETCH_UNMAPPED, # attempt to fetch from an unmapped memory location + UC_HOOK_MEM_UNMAPPED, # any of the 3 above + + UC_HOOK_MEM_READ_PROT, # attempt to read from a non-readable memory location + UC_HOOK_MEM_WRITE_PROT, # attempt to write to a write-protected memory location + UC_HOOK_MEM_FETCH_PROT, # attempt to fetch from a non-executable memory location + UC_HOOK_MEM_PROT, # any of the 3 above + + UC_HOOK_MEM_READ_INVALID, # UC_HOOK_MEM_READ_UNMAPPED | UC_HOOK_MEM_READ_PROT + UC_HOOK_MEM_WRITE_INVALID, # UC_HOOK_MEM_WRITE_UNMAPPED | UC_HOOK_MEM_WRITE_INVALID + UC_HOOK_MEM_FETCH_INVALID, # UC_HOOK_MEM_FETCH_UNMAPPED | UC_HOOK_MEM_FETCH_INVALID + UC_HOOK_MEM_INVALID, # any of the 3 above + + UC_HOOK_MEM_READ, # valid memory read + UC_HOOK_MEM_WRITE, # valid memory write + UC_HOOK_MEM_FETCH, # valid instruction fetch + UC_HOOK_MEM_VALID, # any of the 3 above + + UC_HOOK_MEM_READ_AFTER, + UC_HOOK_INSN_INVALID +) from .core_hooks_types import Hook, HookAddr, HookIntr, HookRet from .const import QL_HOOK_BLOCK from .exception import QlErrorCoreHook if TYPE_CHECKING: + from unicorn import Uc from qiling import Qiling class MemHookCallback(Protocol): - def __call__(self, __ql: 'Qiling', __access: int, __address: int, __size: int, __value: int, *__context: Any) -> Any: + def __call__(self, __ql: Qiling, __access: int, __address: int, __size: int, __value: int, *__context: Any) -> Any: """Memory access hook callback. Args: @@ -41,7 +71,7 @@ def __call__(self, __ql: 'Qiling', __access: int, __address: int, __size: int, _ pass class TraceHookCalback(Protocol): - def __call__(self, __ql: 'Qiling', __address: int, __size: int, *__context: Any) -> Any: + def __call__(self, __ql: Qiling, __address: int, __size: int, *__context: Any) -> Any: """Execution hook callback. Args: @@ -57,7 +87,7 @@ def __call__(self, __ql: 'Qiling', __address: int, __size: int, *__context: Any) pass class AddressHookCallback(Protocol): - def __call__(self, __ql: 'Qiling', *__context: Any) -> Any: + def __call__(self, __ql: Qiling, *__context: Any) -> Any: """Address hook callback. Args: @@ -71,7 +101,7 @@ def __call__(self, __ql: 'Qiling', *__context: Any) -> Any: pass class InterruptHookCallback(Protocol): - def __call__(self, __ql: 'Qiling', intno: int, *__context: Any) -> Any: + def __call__(self, __ql: Qiling, intno: int, *__context: Any) -> Any: """Interrupt hook callback. Args: @@ -86,7 +116,7 @@ def __call__(self, __ql: 'Qiling', intno: int, *__context: Any) -> Any: pass -def hookcallback(ql: 'Qiling', callback: Callable): +def hookcallback(ql: Qiling, callback: Callable): @wraps(callback) def wrapper(*args, **kwargs): try: From b9f61a9cfde5c0ae370e32bee0870026130a7689 Mon Sep 17 00:00:00 2001 From: elicn Date: Fri, 10 Mar 2023 12:53:07 +0200 Subject: [PATCH 05/42] Have gdbserver report back Uc errors --- qiling/debugger/gdb/gdb.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/qiling/debugger/gdb/gdb.py b/qiling/debugger/gdb/gdb.py index 6dd7b3614..22feaebae 100644 --- a/qiling/debugger/gdb/gdb.py +++ b/qiling/debugger/gdb/gdb.py @@ -293,11 +293,11 @@ def handle_m(subcmd: str) -> Reply: addr, size = (int(p, 16) for p in subcmd.split(',')) try: - data = self.ql.mem.read(addr, size).hex() - except UcError: - return 'E14' + data = self.ql.mem.read(addr, size) + except UcError as ex: + return f'E{ex.errno:02d}' else: - return data + return data.hex() def handle_M(subcmd: str) -> Reply: """Write target memory. @@ -313,8 +313,8 @@ def handle_M(subcmd: str) -> Reply: try: self.ql.mem.write(addr, data) - except UcError: - return 'E01' + except UcError as ex: + return f'E{ex.errno:02d}' else: return REPLY_OK @@ -693,8 +693,8 @@ def handle_X(subcmd: str) -> Reply: try: if data: self.ql.mem.write(addr, data.encode(ENCODING)) - except UcError: - return 'E01' + except UcError as ex: + return f'E{ex.errno:02d}' else: return REPLY_OK From c89b43a41153e9f28719b9806f15911919d9613b Mon Sep 17 00:00:00 2001 From: elicn Date: Fri, 10 Mar 2023 13:01:30 +0200 Subject: [PATCH 06/42] Opportunistic PEP8 fixes --- qiling/core_hooks.py | 35 +++++++------------------------- qiling/os/freebsd/freebsd.py | 8 +++----- qiling/os/os.py | 4 ++-- qiling/os/posix/const_mapping.py | 7 ++++--- qiling/os/posix/posix.py | 10 ++++----- qiling/os/qnx/qnx.py | 11 ++++------ 6 files changed, 25 insertions(+), 50 deletions(-) diff --git a/qiling/core_hooks.py b/qiling/core_hooks.py index 49dbcebe4..58d4d736e 100644 --- a/qiling/core_hooks.py +++ b/qiling/core_hooks.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# +# # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # @@ -52,6 +52,7 @@ from unicorn import Uc from qiling import Qiling + class MemHookCallback(Protocol): def __call__(self, __ql: Qiling, __access: int, __address: int, __size: int, __value: int, *__context: Any) -> Any: """Memory access hook callback. @@ -70,6 +71,7 @@ def __call__(self, __ql: Qiling, __access: int, __address: int, __size: int, __v """ pass + class TraceHookCalback(Protocol): def __call__(self, __ql: Qiling, __address: int, __size: int, *__context: Any) -> Any: """Execution hook callback. @@ -86,6 +88,7 @@ def __call__(self, __ql: Qiling, __address: int, __size: int, *__context: Any) - """ pass + class AddressHookCallback(Protocol): def __call__(self, __ql: Qiling, *__context: Any) -> Any: """Address hook callback. @@ -100,6 +103,7 @@ def __call__(self, __ql: Qiling, *__context: Any) -> Any: """ pass + class InterruptHookCallback(Protocol): def __call__(self, __ql: Qiling, intno: int, *__context: Any) -> Any: """Interrupt hook callback. @@ -142,10 +146,10 @@ def __init__(self, uc: Uc): self._addr_hook: MutableMapping[int, MutableSequence[HookAddr]] = {} self._addr_hook_fuc: MutableMapping[int, int] = {} - ######################## # Callback definitions # ######################## + def _hook_intr_cb(self, uc: Uc, intno: int, pack_data) -> None: """Interrupt hooks dispatcher. """ @@ -171,7 +175,6 @@ def _hook_intr_cb(self, uc: Uc, intno: int, pack_data) -> None: if not handled: raise QlErrorCoreHook("_hook_intr_cb : not handled") - def _hook_insn_cb(self, uc: Uc, *args): """Instruction hooks dispatcher. """ @@ -195,7 +198,6 @@ def _hook_insn_cb(self, uc: Uc, *args): # use the last return value received return retval - def _hook_trace_cb(self, uc: Uc, addr: int, size: int, pack_data) -> None: """Code and block hooks dispatcher. """ @@ -212,7 +214,6 @@ def _hook_trace_cb(self, uc: Uc, addr: int, size: int, pack_data) -> None: if type(ret) is int and ret & QL_HOOK_BLOCK: break - def _hook_mem_cb(self, uc: Uc, access: int, addr: int, size: int, value: int, pack_data): """Memory access hooks dispatcher. """ @@ -236,7 +237,6 @@ def _hook_mem_cb(self, uc: Uc, access: int, addr: int, size: int, value: int, pa return True - def _hook_insn_invalid_cb(self, uc: Uc, pack_data) -> None: """Invalid instruction hooks dispatcher. """ @@ -257,7 +257,6 @@ def _hook_insn_invalid_cb(self, uc: Uc, pack_data) -> None: if not handled: raise QlErrorCoreHook("_hook_insn_invalid_cb : not handled") - def _hook_addr_cb(self, uc: Uc, addr: int, size: int, pack_data): """Address hooks dispatcher. """ @@ -276,18 +275,17 @@ def _hook_addr_cb(self, uc: Uc, addr: int, size: int, pack_data): ############### # Class Hooks # ############### + def _ql_hook_internal(self, hook_type: int, callback: Callable, context: Any, *args) -> int: _callback = hookcallback(self, callback) return self._h_uc.hook_add(hook_type, _callback, (self, context), 1, 0, *args) - def _ql_hook_addr_internal(self, callback: Callable, address: int) -> int: _callback = hookcallback(self, callback) return self._h_uc.hook_add(UC_HOOK_CODE, _callback, self, address, address) - def _ql_hook(self, hook_type: int, h: Hook, *args) -> None: def __handle_intr(t: int) -> None: @@ -359,7 +357,6 @@ def __handle_invalid_insn(t: int) -> None: if hook_type & t: handler(t) - def ql_hook(self, hook_type: int, callback: Callable, user_data: Any = None, begin: int = 1, end: int = 0, *args) -> HookRet: """Intercept certain emulation events within a specified range. @@ -384,7 +381,6 @@ def ql_hook(self, hook_type: int, callback: Callable, user_data: Any = None, beg return HookRet(self, hook_type, hook) - def hook_code(self, callback: TraceHookCalback, user_data: Any = None, begin: int = 1, end: int = 0) -> HookRet: """Intercept assembly instructions before they get executed. @@ -403,12 +399,10 @@ def hook_code(self, callback: TraceHookCalback, user_data: Any = None, begin: in return self.ql_hook(UC_HOOK_CODE, callback, user_data, begin, end) - # TODO: remove; this is a special case of hook_intno(-1) def hook_intr(self, callback, user_data=None, begin=1, end=0): return self.ql_hook(UC_HOOK_INTR, callback, user_data, begin, end) - def hook_block(self, callback: TraceHookCalback, user_data: Any = None, begin: int = 1, end: int = 0) -> HookRet: """Intercept landings in new basic blocks in a specified range. @@ -427,7 +421,6 @@ def hook_block(self, callback: TraceHookCalback, user_data: Any = None, begin: i return self.ql_hook(UC_HOOK_BLOCK, callback, user_data, begin, end) - def hook_mem_unmapped(self, callback: MemHookCallback, user_data: Any = None, begin: int = 1, end: int = 0) -> HookRet: """Intercept illegal accesses to unmapped memory in a specified range. @@ -446,7 +439,6 @@ def hook_mem_unmapped(self, callback: MemHookCallback, user_data: Any = None, be return self.ql_hook(UC_HOOK_MEM_UNMAPPED, callback, user_data, begin, end) - def hook_mem_read_invalid(self, callback: MemHookCallback, user_data: Any = None, begin: int = 1, end: int = 0) -> HookRet: """Intercept illegal reading attempts from a specified range. @@ -465,7 +457,6 @@ def hook_mem_read_invalid(self, callback: MemHookCallback, user_data: Any = None return self.ql_hook(UC_HOOK_MEM_READ_INVALID, callback, user_data, begin, end) - def hook_mem_write_invalid(self, callback: MemHookCallback, user_data: Any = None, begin: int = 1, end: int = 0) -> HookRet: """Intercept illegal writing attempts to a specified range. @@ -484,7 +475,6 @@ def hook_mem_write_invalid(self, callback: MemHookCallback, user_data: Any = Non return self.ql_hook(UC_HOOK_MEM_WRITE_INVALID, callback, user_data, begin, end) - def hook_mem_fetch_invalid(self, callback: MemHookCallback, user_data: Any = None, begin: int = 1, end: int = 0) -> HookRet: """Intercept illegal code fetching attempts from a specified range. @@ -503,7 +493,6 @@ def hook_mem_fetch_invalid(self, callback: MemHookCallback, user_data: Any = Non return self.ql_hook(UC_HOOK_MEM_FETCH_INVALID, callback, user_data, begin, end) - def hook_mem_valid(self, callback: MemHookCallback, user_data: Any = None, begin: int = 1, end: int = 0) -> HookRet: """Intercept benign memory accesses within a specified range. This is equivalent to hooking memory reads, writes and fetches. @@ -523,7 +512,6 @@ def hook_mem_valid(self, callback: MemHookCallback, user_data: Any = None, begin return self.ql_hook(UC_HOOK_MEM_VALID, callback, user_data, begin, end) - def hook_mem_invalid(self, callback: MemHookCallback, user_data: Any = None, begin: int = 1, end: int = 0) -> HookRet: """Intercept invalid memory accesses within a specified range. This is equivalent to hooking invalid memory reads, writes and fetches. @@ -543,7 +531,6 @@ def hook_mem_invalid(self, callback: MemHookCallback, user_data: Any = None, beg return self.ql_hook(UC_HOOK_MEM_INVALID, callback, user_data, begin, end) - def hook_address(self, callback: AddressHookCallback, address: int, user_data: Any = None) -> HookRet: """Intercept execution from a certain memory address. @@ -569,7 +556,6 @@ def hook_address(self, callback: AddressHookCallback, address: int, user_data: A # note: assuming 0 is not a valid hook type return HookRet(self, 0, hook) - def hook_intno(self, callback: InterruptHookCallback, intno: int, user_data: Any = None) -> HookRet: """Intercept interrupts. @@ -587,7 +573,6 @@ def hook_intno(self, callback: InterruptHookCallback, intno: int, user_data: Any return HookRet(self, UC_HOOK_INTR, hook) - def hook_mem_read(self, callback: MemHookCallback, user_data: Any = None, begin: int = 1, end: int = 0) -> HookRet: """Intercept benign memory reads from a specified range. @@ -606,7 +591,6 @@ def hook_mem_read(self, callback: MemHookCallback, user_data: Any = None, begin: return self.ql_hook(UC_HOOK_MEM_READ, callback, user_data, begin, end) - def hook_mem_write(self, callback: MemHookCallback, user_data: Any = None, begin: int = 1, end: int = 0) -> HookRet: """Intercept benign memory writes to a specified range. @@ -625,7 +609,6 @@ def hook_mem_write(self, callback: MemHookCallback, user_data: Any = None, begin return self.ql_hook(UC_HOOK_MEM_WRITE, callback, user_data, begin, end) - def hook_mem_fetch(self, callback: MemHookCallback, user_data: Any = None, begin: int = 1, end: int = 0) -> HookRet: """Intercept benign code fetches from a specified range. @@ -644,7 +627,6 @@ def hook_mem_fetch(self, callback: MemHookCallback, user_data: Any = None, begin return self.ql_hook(UC_HOOK_MEM_FETCH, callback, user_data, begin, end) - def hook_insn(self, callback, insn_type: int, user_data: Any = None, begin: int = 1, end: int = 0) -> HookRet: """Intercept execution of a certain instruction type within a specified range. @@ -666,7 +648,6 @@ def hook_insn(self, callback, insn_type: int, user_data: Any = None, begin: int return self.ql_hook(UC_HOOK_INSN, callback, user_data, begin, end, insn_type) - def hook_del(self, hret: HookRet) -> None: """Unregister an existing hook and release its resources. @@ -721,7 +702,6 @@ def __remove(hooks_map, handles_map, key: int) -> None: if hook_type & t: handler(t) - def clear_hooks(self): for ptr in self._hook_fuc.values(): self._h_uc.hook_del(ptr) @@ -734,7 +714,6 @@ def clear_hooks(self): self.clear_ql_hooks() - def clear_ql_hooks(self): self._hook = {} self._hook_fuc = {} diff --git a/qiling/os/freebsd/freebsd.py b/qiling/os/freebsd/freebsd.py index b578198ae..5bcb920d0 100644 --- a/qiling/os/freebsd/freebsd.py +++ b/qiling/os/freebsd/freebsd.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# +# # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # @@ -10,6 +10,7 @@ from qiling.const import QL_OS from qiling.os.posix.posix import QlOsPosix + class QlOsFreebsd(QlOsPosix): type = QL_OS.FREEBSD @@ -18,7 +19,6 @@ def __init__(self, ql): self.load() - def load(self): gdtm = GDTManager(self.ql) @@ -28,16 +28,14 @@ def load(self): self.ql.hook_insn(self.hook_syscall, UC_X86_INS_SYSCALL) - def hook_syscall(self, ql): return self.load_syscall() - def run(self): if self.ql.exit_point is not None: self.exit_point = self.ql.exit_point - if self.ql.entry_point is not None: + if self.ql.entry_point is not None: self.ql.loader.elf_entry = self.ql.entry_point try: diff --git a/qiling/os/os.py b/qiling/os/os.py index 006d094f9..19553ff2a 100644 --- a/qiling/os/os.py +++ b/qiling/os/os.py @@ -48,9 +48,9 @@ def __init__(self, ql: Qiling, resolvers: Mapping[Any, Resolver] = {}): self.fs_mapper = QlFsMapper(self.path) self.user_defined_api = { - QL_INTERCEPT.CALL : {}, + QL_INTERCEPT.CALL: {}, QL_INTERCEPT.ENTER: {}, - QL_INTERCEPT.EXIT : {} + QL_INTERCEPT.EXIT: {} } # IDAPython has some hack on standard io streams and thus they don't have corresponding fds. diff --git a/qiling/os/posix/const_mapping.py b/qiling/os/posix/const_mapping.py index 2fcd2d3c2..9b32b8c4a 100644 --- a/qiling/os/posix/const_mapping.py +++ b/qiling/os/posix/const_mapping.py @@ -156,12 +156,13 @@ def socket_domain_mapping(p: int, archtype: QL_ARCH, ostype: QL_OS) -> str: def socket_tcp_option_mapping(t: int, archtype: QL_ARCH) -> str: socket_option_map = { - QL_ARCH.X86: linux_socket_tcp_options, + QL_ARCH.X86: linux_socket_tcp_options, QL_ARCH.X8664: linux_socket_tcp_options, - QL_ARCH.ARM: linux_socket_tcp_options, + QL_ARCH.ARM: linux_socket_tcp_options, QL_ARCH.ARM64: linux_socket_tcp_options, - QL_ARCH.MIPS: linux_socket_tcp_options, + QL_ARCH.MIPS: linux_socket_tcp_options, }[archtype] + return _constant_mapping(t, socket_option_map) diff --git a/qiling/os/posix/posix.py b/qiling/os/posix/posix.py index a4f88f052..bce85b7ed 100644 --- a/qiling/os/posix/posix.py +++ b/qiling/os/posix/posix.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# +# # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # @@ -110,9 +110,9 @@ def __init__(self, ql: Qiling): self.ifrname_ovr = conf.get('ifrname_override') self.posix_syscall_hooks = { - QL_INTERCEPT.CALL : {}, + QL_INTERCEPT.CALL: {}, QL_INTERCEPT.ENTER: {}, - QL_INTERCEPT.EXIT : {} + QL_INTERCEPT.EXIT: {} } self.__syscall_id_reg = { @@ -191,7 +191,7 @@ def root(self, enabled: bool) -> None: self.euid = 0 if enabled else self.uid self.egid = 0 if enabled else self.gid - def set_syscall(self, target: Union[int, str], handler: Callable, intercept: QL_INTERCEPT=QL_INTERCEPT.CALL): + def set_syscall(self, target: Union[int, str], handler: Callable, intercept: QL_INTERCEPT = QL_INTERCEPT.CALL): """Either hook or replace a system call with a custom one. Args: @@ -298,7 +298,7 @@ def __get_os_module(osname: str): raise e # print out log entry - syscall_basename = syscall_name[len(SYSCALL_PREF) if syscall_name.startswith(SYSCALL_PREF) else 0:] + syscall_basename = syscall_name[len(SYSCALL_PREF) if syscall_name.startswith(SYSCALL_PREF) else 0:] args = [] diff --git a/qiling/os/qnx/qnx.py b/qiling/os/qnx/qnx.py index 1d6172dd1..cb1e59a61 100644 --- a/qiling/os/qnx/qnx.py +++ b/qiling/os/qnx/qnx.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# +# # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # @@ -20,6 +20,7 @@ from qiling.os.const import * from qiling.os.posix.posix import QlOsPosix + class QlOsQnx(QlOsPosix): type = QL_OS.QNX @@ -46,7 +47,7 @@ def __init__(self, ql: Qiling): self.fh = None self.function_after_load_list = [] self.load() - + # use counters to get free Ids self.channel_id = 1 # TODO: replace 0x400 with NR_OPEN from Qiling 1.25 @@ -74,26 +75,22 @@ def load(self): 'get_tls': 0xffff0fe0 }) - def hook_syscall(self, ql, intno): return self.load_syscall() - def register_function_after_load(self, function): if function not in self.function_after_load_list: self.function_after_load_list.append(function) - def run_function_after_load(self): for f in self.function_after_load_list: f() - def run(self): if self.ql.exit_point is not None: self.exit_point = self.ql.exit_point - if self.ql.entry_point is not None: + if self.ql.entry_point is not None: self.ql.loader.elf_entry = self.ql.entry_point self.cpupage_addr = int(self.ql.os.profile.get("OS32", "cpupage_address"), 16) From c67a64eba829753899d084e81bc4b2c4088e1c55 Mon Sep 17 00:00:00 2001 From: elicn Date: Fri, 10 Mar 2023 13:06:05 +0200 Subject: [PATCH 07/42] Properly load QNS profile values --- qiling/os/qnx/qnx.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/qiling/os/qnx/qnx.py b/qiling/os/qnx/qnx.py index cb1e59a61..7187aefcf 100644 --- a/qiling/os/qnx/qnx.py +++ b/qiling/os/qnx/qnx.py @@ -3,8 +3,6 @@ # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # -import os - from unicorn import UcError from qiling import Qiling @@ -93,15 +91,17 @@ def run(self): if self.ql.entry_point is not None: self.ql.loader.elf_entry = self.ql.entry_point - self.cpupage_addr = int(self.ql.os.profile.get("OS32", "cpupage_address"), 16) - self.cpupage_tls_addr = int(self.ql.os.profile.get("OS32", "cpupage_tls_address"), 16) - self.tls_data_addr = int(self.ql.os.profile.get("OS32", "tls_data_address"), 16) - self.syspage_addr = int(self.ql.os.profile.get("OS32", "syspage_address"), 16) - syspage_path = os.path.join(self.ql.rootfs, "syspage.bin") + profile = self.ql.os.profile['OS32'] + + self.cpupage_addr = profile.getint('cpupage_address') + self.cpupage_tls_addr = profile.getint('cpupage_tls_address') + self.tls_data_addr = profile.getint('tls_data_address') + self.syspage_addr = profile.getint('syspage_address') self.ql.mem.map(self.syspage_addr, 0x4000, info="[syspage_mem]") - - with open(syspage_path, "rb") as sp: + syspage_hpath = self.ql.os.path.virtual_to_host_path("/syspage.bin") + + with open(syspage_hpath, "rb") as sp: self.ql.mem.write(self.syspage_addr, sp.read()) # Address of struct _thread_local_storage for our thread From 5e9738c904c4092fb8a400fae355ba3d9d6a1a33 Mon Sep 17 00:00:00 2001 From: elicn Date: Fri, 10 Mar 2023 13:09:06 +0200 Subject: [PATCH 08/42] Minor code rearrangements in gdb --- qiling/debugger/gdb/gdb.py | 58 +++++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 26 deletions(-) diff --git a/qiling/debugger/gdb/gdb.py b/qiling/debugger/gdb/gdb.py index 22feaebae..e8c501181 100644 --- a/qiling/debugger/gdb/gdb.py +++ b/qiling/debugger/gdb/gdb.py @@ -78,36 +78,42 @@ def __init__(self, ql: Qiling, ip: str = '127.0.0.1', port: int = 9999): self.ip = ip self.port = port - if ql.baremetal: - load_address = ql.loader.load_address - exit_point = load_address + os.path.getsize(ql.path) - elif ql.code: - load_address = ql.os.entry_point - exit_point = load_address + len(ql.code) - else: - load_address = ql.loader.load_address - exit_point = load_address + os.path.getsize(ql.path) + def __get_attach_addr() -> int: + if ql.baremetal: + entry_point = ql.loader.entry_point - if ql.baremetal: - entry_point = ql.loader.entry_point - elif ql.os.type in (QL_OS.LINUX, QL_OS.FREEBSD) and not ql.code: - entry_point = ql.os.elf_entry - else: - entry_point = ql.os.entry_point + elif ql.os.type in (QL_OS.LINUX, QL_OS.FREEBSD) and not ql.code: + entry_point = ql.os.elf_entry + + else: + entry_point = ql.os.entry_point + + # though linkers set the entry point LSB to indicate arm thumb mode, the + # effective entry point address is aligned. make sure we have it aligned + if hasattr(ql.arch, 'is_thumb'): + entry_point &= ~0b1 - # though linkers set the entry point LSB to indicate arm thumb mode, the - # effective entry point address is aligned. make sure we have it aligned - if hasattr(ql.arch, 'is_thumb'): - entry_point &= ~0b1 + return entry_point + + def __get_detach_addr() -> int: + if ql.baremetal: + base = ql.loader.load_address + size = os.path.getsize(ql.path) + + elif ql.code: + base = ql.os.entry_point + size = len(ql.code) + + else: + base = ql.loader.load_address + size = os.path.getsize(ql.path) - # Only part of the binary file will be debugged. - if ql.entry_point is not None: - entry_point = ql.entry_point + return base + size - if ql.exit_point is not None: - exit_point = ql.exit_point + attach_addr = __get_attach_addr() if ql.entry_point is None else ql.entry_point + detach_addr = __get_detach_addr() if ql.exit_point is None else ql.exit_point - self.gdb = QlGdbUtils(ql, entry_point, exit_point) + self.gdb = QlGdbUtils(ql, attach_addr, detach_addr) self.features = QlGdbFeatures(self.ql.arch.type, self.ql.os.type) self.regsmap = self.features.regsmap @@ -837,7 +843,7 @@ def readpackets(self) -> Iterator[bytes]: buffer += incoming # discard incoming acks - if buffer[0:1] == REPLY_ACK: + if buffer.startswith(REPLY_ACK): del buffer[0] packet = pattern.match(buffer) From f58c4942cb7ee14ebb96e47abd98e9964c1dd86e Mon Sep 17 00:00:00 2001 From: elicn Date: Mon, 27 Mar 2023 20:22:53 +0300 Subject: [PATCH 09/42] Revamp FS mapper --- qiling/core.py | 4 +- qiling/loader/dos.py | 2 +- qiling/os/linux/linux.py | 10 +- qiling/os/mapper.py | 226 ++++++++++++++++++++++++++------------- 4 files changed, 162 insertions(+), 80 deletions(-) diff --git a/qiling/core.py b/qiling/core.py index 2cb5aff75..70c2ecad4 100644 --- a/qiling/core.py +++ b/qiling/core.py @@ -697,11 +697,11 @@ def restore(self, saved_states: Mapping[str, Any] = {}, *, snapshot: Optional[st # Map "ql_path" to any objects which implements QlFsMappedObject. def add_fs_mapper(self, ql_path: Union["PathLike", str], real_dest): - self.os.fs_mapper.add_fs_mapping(ql_path, real_dest) + self.os.fs_mapper.add_mapping(ql_path, real_dest) # Remove "ql_path" mapping. def remove_fs_mapper(self, ql_path: Union["PathLike", str]): - self.os.fs_mapper.remove_fs_mapping(ql_path) + self.os.fs_mapper.remove_mapping(ql_path) # push to stack bottom, and update stack register def stack_push(self, data): diff --git a/qiling/loader/dos.py b/qiling/loader/dos.py index 5e3db8bbd..847a51c49 100644 --- a/qiling/loader/dos.py +++ b/qiling/loader/dos.py @@ -84,7 +84,7 @@ def run(self): # https://en.wikipedia.org/wiki/Master_boot_record#BIOS_to_MBR_interface if not self.ql.os.fs_mapper.has_mapping(0x80): - self.ql.os.fs_mapper.add_fs_mapping(0x80, QlDisk(path, 0x80)) + self.ql.os.fs_mapper.add_mapping(0x80, QlDisk(path, 0x80)) # 0x80 -> first drive self.ql.arch.regs.dx = 0x80 diff --git a/qiling/os/linux/linux.py b/qiling/os/linux/linux.py index d10bb9033..3f7196924 100644 --- a/qiling/os/linux/linux.py +++ b/qiling/os/linux/linux.py @@ -121,11 +121,11 @@ def load(self): self.fd[i] = None def setup_procfs(self): - self.fs_mapper.add_fs_mapping(r'/proc/self/auxv', partial(QlProcFS.self_auxv, self)) - self.fs_mapper.add_fs_mapping(r'/proc/self/cmdline', partial(QlProcFS.self_cmdline, self)) - self.fs_mapper.add_fs_mapping(r'/proc/self/environ', partial(QlProcFS.self_environ, self)) - self.fs_mapper.add_fs_mapping(r'/proc/self/exe', partial(QlProcFS.self_exe, self)) - self.fs_mapper.add_fs_mapping(r'/proc/self/maps', partial(QlProcFS.self_map, self.ql.mem)) + self.fs_mapper.add_mapping(r'/proc/self/auxv', partial(QlProcFS.self_auxv, self), force=True) + self.fs_mapper.add_mapping(r'/proc/self/cmdline', partial(QlProcFS.self_cmdline, self), force=True) + self.fs_mapper.add_mapping(r'/proc/self/environ', partial(QlProcFS.self_environ, self), force=True) + self.fs_mapper.add_mapping(r'/proc/self/exe', partial(QlProcFS.self_exe, self), force=True) + self.fs_mapper.add_mapping(r'/proc/self/maps', partial(QlProcFS.self_map, self.ql.mem), force=True) def hook_syscall(self, ql, intno = None): return self.load_syscall() diff --git a/qiling/os/mapper.py b/qiling/os/mapper.py index cc0d840c5..583081097 100644 --- a/qiling/os/mapper.py +++ b/qiling/os/mapper.py @@ -4,11 +4,15 @@ # import os +from os import PathLike from typing import Any, Callable, MutableMapping, Union from .path import QlOsPath from .filestruct import ql_file +QlPath = Union['PathLike[str]', str, 'PathLike[bytes]', bytes] + + # All mapped objects should inherit this class. # Note this object is compatible with ql_file. # Q: Why not derive from ql_file directly? @@ -63,115 +67,193 @@ def __init__(self, path: QlOsPath): self._mapping: MutableMapping[str, Any] = {} self.path = path - def _open_mapping_ql_file(self, ql_path: str, openflags: int, openmode: int): - real_dest = self._mapping[ql_path] + def __contains__(self, vpath: str) -> bool: + # canonicalize the path first + absvpath = self.path.virtual_abspath(vpath) - if isinstance(real_dest, str): - obj = ql_file.open(real_dest, openflags, openmode) + return absvpath in self._mapping - elif callable(real_dest): - obj = real_dest() + def has_mapping(self, vpath: str) -> bool: + """Check whether a specific virtrual path has a binding. - else: - obj = real_dest + Args: + vpath: virtual path name to check - return obj + Returns: `True` if the specified virtual path has been bound, `False` otherwise. + """ - def _open_mapping(self, ql_path: str, openmode: str): - real_dest = self._mapping[ql_path] + return vpath in self - if isinstance(real_dest, str): - obj = open(real_dest, openmode) + def __len__(self) -> int: + return len(self._mapping) - elif callable(real_dest): - obj = real_dest() + def mapping_count(self) -> int: + """Count of currently existing bindings. + """ + + return len(self) + def __open_mapped(self, absvpath: str, opener: Callable, *args) -> Any: + """Internal method user for opening an existing mapped object. + + Args: + absvpath: absolute virtual path name + opener: a method to use to open the target host path + *args: arguments to the opener method + """ + + mapped = self._mapping[absvpath] + + # mapped to a file name on the host file system + if isinstance(mapped, str): + obj = opener(mapped, *args) + + # mapped to a class or a method + elif callable(mapped): + obj = mapped() + + # mapped to another kind of object else: - obj = real_dest + obj = mapped return obj - def has_mapping(self, fm: str) -> bool: - return fm in self._mapping + def __open_new(self, absvpath: str, opener: Callable, *args) -> Any: + hpath = self.path.virtual_to_host_path(absvpath) - def mapping_count(self) -> int: - return len(self._mapping) + if not self.path.is_safe_host_path(hpath): + raise PermissionError(f'unsafe path: {hpath}') - def open_ql_file(self, path: str, openflags: int, openmode: int): - if self.has_mapping(path): - return self._open_mapping_ql_file(path, openflags, openmode) + return opener(hpath, *args) - host_path = self.path.virtual_to_host_path(path) + def open_ql_file(self, vpath: str, flags: int, mode: int): + absvpath = self.path.virtual_abspath(vpath) + opener = self.__open_mapped if self.has_mapping(absvpath) else self.__open_new + + return opener(absvpath, ql_file.open, flags, mode) + + def open(self, vpath: str, mode: str): + absvpath = self.path.virtual_abspath(vpath) + opener = self.__open_mapped if self.has_mapping(absvpath) else self.__open_new + + return opener(absvpath, open, mode) + + def file_exists(self, vpath: str) -> bool: + """Check whether a file exists on the virtual file system. + + Args: + vpath: virtual path name to check - if not self.path.is_safe_host_path(host_path): - raise PermissionError(f'unsafe path: {host_path}') + Returns: `True` if the specified virtual path has an existing mapping or + resolves to an existing file on the virtual file system. `False` otherwise. + """ - return ql_file.open(host_path, openflags, openmode) - def file_exists(self, path:str) -> bool: - # check if file exists - if self.has_mapping(path): + if self.has_mapping(vpath): return True - host_path = self.path.virtual_to_host_path(path) - if not self.path.is_safe_host_path(host_path): - raise PermissionError(f'unsafe path: {host_path}') - return os.path.isfile(host_path) - - def create_empty_file(self, path:str)->bool: - if not self.file_exists(path): - try: - f = self.open(path, "w+") - f.close() - return True + hpath = self.path.virtual_to_host_path(vpath) + + if not self.path.is_safe_host_path(hpath): + raise PermissionError(f'unsafe path: {hpath}') - except Exception as e: + return os.path.isfile(hpath) + + def create_empty_file(self, vpath: str) -> bool: + if not self.file_exists(vpath): + try: + f = self.open(vpath, "w+") + except OSError: # for some reason, we could not create an empty file. return False + else: + f.close() + return True - - def open(self, path: str, openmode: str): - if self.has_mapping(path): - return self._open_mapping(path, openmode) - host_path = self.path.virtual_to_host_path(path) + def __fspath(self, path: QlPath) -> str: + """Similar to os.fspath, this method takes a path-like object and returns + its string representation. + """ + + if isinstance(path, PathLike): + path = path.__fspath__() - if not self.path.is_safe_host_path(host_path): - raise PermissionError(f'unsafe path: {host_path}') + if isinstance(path, str): + return path - return open(host_path, openmode) + elif isinstance(path, bytes): + return path.decode('utf-8') - def _parse_path(self, p: Union[os.PathLike, str]) -> str: - fspath = getattr(p, '__fspath__', None) + raise TypeError(path) - # p is an `os.PathLike` object - if fspath is not None: - p = fspath() + def add_mapping(self, vpath: QlPath, binding: Union[QlPath, QlFsMappedObject, Callable], *, force: bool = False) -> None: + """Create a new mapping in the virtual filesystem. - if isinstance(p, bytes): # os.PathLike.__fspath__ may return bytes. - p = p.decode("utf-8") + Args: + vpath: a virtual path to bind - return p + binding: a target to use whenever the bound virtual path is referenced. such a target can be + either a path on the host filesystem, an object instance or a class. the behavior of the mapping + is determined by the bound object type: + [*] a string: bind a path on the host filesystem (e.g. "/dev/urandom"). use with caution! + [*] an object: bind an object instance which will be returned each time the virtual path is opened + [*] a class: bind a class that will be instantiated each time the virtual path is opened - def add_fs_mapping(self, ql_path: Union[os.PathLike, str], real_dest: Union[str, QlFsMappedObject, Callable]) -> None: - """Map an object to Qiling emulated file system. + force: when set to `True`, re-mapping an existing vpath becomes possible. In such case, the + old mapping will be discarded - Args: - ql_path: Emulated path which should be convertable to a string or a hashable object. e.g. pathlib.Path - real_dest: Mapped object, can be a string, an object or a callable(class). - string: mapped path in the host machine, e.g. '/dev/urandom' -> '/dev/urandom' - object: mapped object, will be returned each time the emulated path has been opened - class: mapped callable, will be used to create a new instance each time the emulated path has been opened + Raises: + `KeyError`: in case the specified vpath has already been mapped (default behavior). """ - ql_path = self._parse_path(ql_path) - real_dest = self._parse_path(real_dest) + vpath = self.__fspath(vpath) + absvpath = self.path.virtual_abspath(vpath) + + if self.has_mapping(absvpath) and not force: + raise KeyError(f'mapping already exists: "{absvpath}"') - self._mapping[ql_path] = real_dest + if isinstance(binding, (str, bytes, PathLike)): + binding = self.__fspath(binding) - def remove_fs_mapping(self, ql_path: Union[os.PathLike, str]): + self._mapping[absvpath] = binding + + def remove_mapping(self, vpath: QlPath) -> None: """Remove a mapping from the fs mapper. Args: - ql_path (Union[os.PathLike, str]): The mapped path. + vpath: bound virtual path to remove + + Raises: + `KeyError`: in case the specified vpath has no mapping """ - del self._mapping[self._parse_path(ql_path)] + + vpath = self.__fspath(vpath) + absvpath = self.path.virtual_abspath(vpath) + + if not self.has_mapping(absvpath): + raise KeyError(absvpath) + + del self._mapping[absvpath] + + def rename_mapping(self, old_vpath: str, new_vpath: str) -> None: + old_absvpath = self.path.virtual_abspath(old_vpath) + + # vpath to rename does not exist + if not self.has_mapping(old_absvpath): + raise KeyError(old_vpath) + + new_absvpath = self.path.virtual_abspath(new_vpath) + + # new vpath already exists + if self.has_mapping(new_absvpath): + raise KeyError(new_vpath) + + # avoid renaming to the same vapth + if old_absvpath == new_absvpath: + return + + binding = self._mapping[old_absvpath] + + # remove old mapping and add a new one instead + self._mapping[new_absvpath] = binding + del self._mapping[old_absvpath] From 263d5237b362a2b345351c4c9505de331a818596 Mon Sep 17 00:00:00 2001 From: elicn Date: Mon, 27 Mar 2023 20:25:08 +0300 Subject: [PATCH 10/42] Revamp POSIX fcntl --- qiling/os/posix/syscall/fcntl.py | 141 +++++++++++++++++-------------- 1 file changed, 78 insertions(+), 63 deletions(-) diff --git a/qiling/os/posix/syscall/fcntl.py b/qiling/os/posix/syscall/fcntl.py index bca208863..a6c3c1f0f 100644 --- a/qiling/os/posix/syscall/fcntl.py +++ b/qiling/os/posix/syscall/fcntl.py @@ -13,10 +13,9 @@ from qiling.os.posix.const_mapping import ql_open_flag_mapping from qiling.os.posix.filestruct import ql_socket + def ql_syscall_open(ql: Qiling, filename: int, flags: int, mode: int): - path = ql.os.utils.read_cstring(filename) - real_path = ql.os.path.transform_to_real_path(path) - relative_path = ql.os.path.transform_to_relative_path(path) + vpath = ql.os.utils.read_cstring(filename) flags &= 0xffffffff mode &= 0xffffffff @@ -26,62 +25,68 @@ def ql_syscall_open(ql: Qiling, filename: int, flags: int, mode: int): if idx == -1: regreturn = -EMFILE else: - try: - if ql.arch.type == QL_ARCH.ARM and ql.os.type != QL_OS.QNX: - mode = 0 + if ql.arch.type == QL_ARCH.ARM and ql.os.type != QL_OS.QNX: + mode = 0 + try: flags = ql_open_flag_mapping(ql, flags) - ql.os.fd[idx] = ql.os.fs_mapper.open_ql_file(path, flags, mode) - regreturn = idx + ql.os.fd[idx] = ql.os.fs_mapper.open_ql_file(vpath, flags, mode) except QlSyscallError as e: - regreturn = - e.errno + regreturn = -e.errno + else: + regreturn = idx + hpath = ql.os.path.virtual_to_host_path(vpath) + absvpath = ql.os.path.virtual_abspath(vpath) - ql.log.debug("open(%s, 0o%o) = %d" % (relative_path, mode, regreturn)) + ql.log.debug(f'open("{absvpath}", {mode:#o}) = {regreturn}') if regreturn >= 0 and regreturn != 2: - ql.log.debug(f'File found: {real_path:s}') + ql.log.debug(f'File found: {hpath:s}') else: - ql.log.debug(f'File not found {real_path:s}') + ql.log.debug(f'File not found {hpath:s}') return regreturn -def ql_syscall_creat(ql: Qiling, filename: int, mode: int): - flags = posix_open_flags["O_WRONLY"] | posix_open_flags["O_CREAT"] | posix_open_flags["O_TRUNC"] - path = ql.os.utils.read_cstring(filename) - real_path = ql.os.path.transform_to_real_path(path) - relative_path = ql.os.path.transform_to_relative_path(path) +def ql_syscall_creat(ql: Qiling, filename: int, mode: int): + vpath = ql.os.utils.read_cstring(filename) - flags &= 0xffffffff + # FIXME: this is broken + flags = posix_open_flags["O_WRONLY"] | posix_open_flags["O_CREAT"] | posix_open_flags["O_TRUNC"] mode &= 0xffffffff idx = next((i for i in range(NR_OPEN) if ql.os.fd[i] is None), -1) if idx == -1: - regreturn = -ENOMEM + regreturn = -ENOMEM else: - try: - if ql.arch.type == QL_ARCH.ARM: - mode = 0 + if ql.arch.type == QL_ARCH.ARM: + mode = 0 + try: flags = ql_open_flag_mapping(ql, flags) - ql.os.fd[idx] = ql.os.fs_mapper.open_ql_file(path, flags, mode) - regreturn = idx + ql.os.fd[idx] = ql.os.fs_mapper.open_ql_file(vpath, flags, mode) except QlSyscallError as e: regreturn = -e.errno + else: + regreturn = idx - ql.log.debug("creat(%s, 0o%o) = %d" % (relative_path, mode, regreturn)) + hpath = ql.os.path.virtual_to_host_path(vpath) + absvpath = ql.os.path.virtual_abspath(vpath) + + ql.log.debug(f'creat("{absvpath}", {mode:#o}) = {regreturn}') if regreturn >= 0 and regreturn != 2: - ql.log.debug(f'File found: {real_path:s}') + ql.log.debug(f'File found: {hpath:s}') else: - ql.log.debug(f'File not found {real_path:s}') + ql.log.debug(f'File not found {hpath:s}') return regreturn + def ql_syscall_openat(ql: Qiling, fd: int, path: int, flags: int, mode: int): - file_path = ql.os.utils.read_cstring(path) + vpath = ql.os.utils.read_cstring(path) # real_path = ql.os.path.transform_to_real_path(path) # relative_path = ql.os.path.transform_to_relative_path(path) @@ -93,27 +98,29 @@ def ql_syscall_openat(ql: Qiling, fd: int, path: int, flags: int, mode: int): if idx == -1: regreturn = -EMFILE else: - try: - if ql.arch.type == QL_ARCH.ARM: - mode = 0 + fd = ql.unpacks(ql.pack(fd)) + if ql.arch.type == QL_ARCH.ARM: + mode = 0 + + try: flags = ql_open_flag_mapping(ql, flags) - fd = ql.unpacks(ql.pack(fd)) if 0 <= fd < NR_OPEN: fobj = ql.os.fd[fd] + # ql_file object or QlFsMappedObject if hasattr(fobj, "fileno") and hasattr(fobj, "name"): - if not Path.is_absolute(Path(file_path)): - file_path = Path(fobj.name) / Path(file_path) - - ql.os.fd[idx] = ql.os.fs_mapper.open_ql_file(file_path, flags, mode) + if not Path.is_absolute(Path(vpath)): + vpath = str(Path(fobj.name) / Path(vpath)) - regreturn = idx + ql.os.fd[idx] = ql.os.fs_mapper.open_ql_file(vpath, flags, mode) except QlSyscallError as e: regreturn = -e.errno - - ql.log.debug(f'openat(fd = {fd:d}, path = {file_path}, mode = {mode:#o}) = {regreturn:d}') + else: + regreturn = idx + + ql.log.debug(f'openat(fd = {fd:d}, path = {vpath}, mode = {mode:#o}) = {regreturn:d}') return regreturn @@ -209,28 +216,36 @@ def ql_syscall_flock(ql: Qiling, fd: int, operation: int): def ql_syscall_rename(ql: Qiling, oldname_buf: int, newname_buf: int): - """ - rename(const char *oldpath, const char *newpath) - description: change the name or location of a file - ret value: On success, zero is returned. On error, -1 is returned - """ - regreturn = 0 # default value is success - oldpath = ql.os.utils.read_cstring(oldname_buf) - newpath = ql.os.utils.read_cstring(newname_buf) - - ql.log.debug(f"rename() path: {oldpath} -> {newpath}") - - old_realpath = ql.os.path.transform_to_real_path(oldpath) - new_realpath = ql.os.path.transform_to_real_path(newpath) - - if old_realpath == new_realpath: - # do nothing, just return success - return regreturn - - try: - os.rename(old_realpath, new_realpath) - except OSError: - ql.log.exception(f"rename(): {newpath} exists!") - regreturn = -1 + old_vpath = ql.os.utils.read_cstring(oldname_buf) + new_vpath = ql.os.utils.read_cstring(newname_buf) + + old_absvpath = ql.os.path.virtual_abspath(old_vpath) - return regreturn \ No newline at end of file + # if has a mapping, rename the mapped vpath + if ql.os.fs_mapper.has_mapping(old_absvpath): + try: + ql.os.fs_mapper.rename_mapping(old_vpath, new_vpath) + except KeyError: + regreturn = -1 + else: + regreturn = 0 + + # otherwise, rename the actual files + else: + old_hpath = ql.os.path.virtual_to_host_path(old_vpath) + new_hpath = ql.os.path.virtual_to_host_path(new_vpath) + + # if source and target paths are identical, do nothing + if old_hpath == new_hpath: + return 0 + + try: + os.rename(old_hpath, new_hpath) + except OSError: + regreturn = -1 + else: + regreturn = 0 + + ql.log.debug(f'rename("{old_vpath}", "{new_vpath}") = {regreturn}') + + return regreturn From 5aedbaf1662c2b564c3b442323e89e02135d01c1 Mon Sep 17 00:00:00 2001 From: elicn Date: Mon, 27 Mar 2023 20:26:35 +0300 Subject: [PATCH 11/42] Fix close_on_exec type --- qiling/os/filestruct.py | 6 +++--- qiling/os/linux/linux.py | 2 +- qiling/os/posix/syscall/fcntl.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/qiling/os/filestruct.py b/qiling/os/filestruct.py index 050d749f7..1d836c58f 100644 --- a/qiling/os/filestruct.py +++ b/qiling/os/filestruct.py @@ -21,7 +21,7 @@ def __init__(self, path: AnyStr, fd: int): # information for syscall mmap self._is_map_shared = False self._mapped_offset = -1 - self._close_on_exec = 0 + self._close_on_exec = False @classmethod def open(cls, open_path: AnyStr, open_flags: int, open_mode: int, dir_fd: int = None): @@ -88,9 +88,9 @@ def name(self): return self.__path @property - def close_on_exec(self) -> int: + def close_on_exec(self) -> bool: return self._close_on_exec @close_on_exec.setter - def close_on_exec(self, value: int) -> None: + def close_on_exec(self, value: bool) -> None: self._close_on_exec = value diff --git a/qiling/os/linux/linux.py b/qiling/os/linux/linux.py index 3f7196924..9f6359333 100644 --- a/qiling/os/linux/linux.py +++ b/qiling/os/linux/linux.py @@ -117,7 +117,7 @@ def load(self): # on fork or execve, do not inherit opened files tagged as 'close on exec' for i in range(len(self.fd)): - if getattr(self.fd[i], 'close_on_exec', 0): + if getattr(self.fd[i], 'close_on_exec', False): self.fd[i] = None def setup_procfs(self): diff --git a/qiling/os/posix/syscall/fcntl.py b/qiling/os/posix/syscall/fcntl.py index a6c3c1f0f..6904cba92 100644 --- a/qiling/os/posix/syscall/fcntl.py +++ b/qiling/os/posix/syscall/fcntl.py @@ -147,10 +147,10 @@ def ql_syscall_fcntl(ql: Qiling, fd: int, cmd: int, arg: int): regreturn = -EMFILE elif cmd == F_GETFD: - regreturn = getattr(f, "close_on_exec", 0) + regreturn = getattr(f, "close_on_exec", False) elif cmd == F_SETFD: - f.close_on_exec = 1 if arg & FD_CLOEXEC else 0 + f.close_on_exec = bool(arg & FD_CLOEXEC) regreturn = 0 elif cmd == F_GETFL: From 82610397acf3e331d0ac6613a4b3a8dcda3b2ffe Mon Sep 17 00:00:00 2001 From: elicn Date: Thu, 30 Mar 2023 23:37:08 +0300 Subject: [PATCH 12/42] Avoid overwriting custom procfs mappings --- qiling/os/linux/linux.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/qiling/os/linux/linux.py b/qiling/os/linux/linux.py index 9f6359333..7b706077a 100644 --- a/qiling/os/linux/linux.py +++ b/qiling/os/linux/linux.py @@ -121,11 +121,18 @@ def load(self): self.fd[i] = None def setup_procfs(self): - self.fs_mapper.add_mapping(r'/proc/self/auxv', partial(QlProcFS.self_auxv, self), force=True) - self.fs_mapper.add_mapping(r'/proc/self/cmdline', partial(QlProcFS.self_cmdline, self), force=True) - self.fs_mapper.add_mapping(r'/proc/self/environ', partial(QlProcFS.self_environ, self), force=True) - self.fs_mapper.add_mapping(r'/proc/self/exe', partial(QlProcFS.self_exe, self), force=True) - self.fs_mapper.add_mapping(r'/proc/self/maps', partial(QlProcFS.self_map, self.ql.mem), force=True) + files = ( + (r'/proc/self/auxv', lambda: partial(QlProcFS.self_auxv, self)), + (r'/proc/self/cmdline', lambda: partial(QlProcFS.self_cmdline, self)), + (r'/proc/self/environ', lambda: partial(QlProcFS.self_environ, self)), + (r'/proc/self/exe', lambda: partial(QlProcFS.self_exe, self)), + (r'/proc/self/maps', lambda: partial(QlProcFS.self_map, self.ql.mem)) + ) + + for filename, wrapper in files: + # add mapping only if the user has not already mapped it + if not self.fs_mapper.has_mapping(filename): + self.fs_mapper.add_mapping(filename, wrapper()) def hook_syscall(self, ql, intno = None): return self.load_syscall() From b8c78c686467b6bcbb215d181bd39a7d4c4c91de Mon Sep 17 00:00:00 2001 From: elicn Date: Thu, 30 Mar 2023 23:43:01 +0300 Subject: [PATCH 13/42] Turned close_on_exec into a simple member --- qiling/os/filestruct.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/qiling/os/filestruct.py b/qiling/os/filestruct.py index 1d836c58f..5230d7f06 100644 --- a/qiling/os/filestruct.py +++ b/qiling/os/filestruct.py @@ -21,7 +21,7 @@ def __init__(self, path: AnyStr, fd: int): # information for syscall mmap self._is_map_shared = False self._mapped_offset = -1 - self._close_on_exec = False + self.close_on_exec = False @classmethod def open(cls, open_path: AnyStr, open_flags: int, open_mode: int, dir_fd: int = None): @@ -87,10 +87,4 @@ def readline(self, end: bytes = b'\n') -> bytes: def name(self): return self.__path - @property - def close_on_exec(self) -> bool: - return self._close_on_exec - @close_on_exec.setter - def close_on_exec(self, value: bool) -> None: - self._close_on_exec = value From 0fd62090060ae5a385f1cdbf6d653584af124c6d Mon Sep 17 00:00:00 2001 From: elicn Date: Thu, 30 Mar 2023 23:43:50 +0300 Subject: [PATCH 14/42] Added closed property to ql_file --- qiling/os/filestruct.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/qiling/os/filestruct.py b/qiling/os/filestruct.py index 5230d7f06..6c93ed825 100644 --- a/qiling/os/filestruct.py +++ b/qiling/os/filestruct.py @@ -18,6 +18,8 @@ class ql_file: def __init__(self, path: AnyStr, fd: int): self.__path = path self.__fd = fd + self.__closed = False + # information for syscall mmap self._is_map_shared = False self._mapped_offset = -1 @@ -52,6 +54,8 @@ def lseek(self, lseek_offset: int, lseek_origin: int = os.SEEK_SET) -> int: def close(self) -> None: os.close(self.__fd) + self.__closed = True + def fstat(self): return Fstat(self.__fd) @@ -87,4 +91,6 @@ def readline(self, end: bytes = b'\n') -> bytes: def name(self): return self.__path - + @property + def closed(self) -> bool: + return self.__closed From 056969fe58d73797dc22868d123333c9639b4d6c Mon Sep 17 00:00:00 2001 From: elicn Date: Thu, 30 Mar 2023 23:48:46 +0300 Subject: [PATCH 15/42] Patched unistd functions --- qiling/os/path.py | 20 +- qiling/os/posix/syscall/unistd.py | 546 +++++++++++++++++------------- 2 files changed, 337 insertions(+), 229 deletions(-) diff --git a/qiling/os/path.py b/qiling/os/path.py index 61c5b8db1..61b3c8cd5 100644 --- a/qiling/os/path.py +++ b/qiling/os/path.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# +# # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # @@ -10,6 +10,7 @@ AnyPurePath = Union[PurePosixPath, PureWindowsPath] + class QlOsPath: """Virtual to host path manipulations helper. """ @@ -60,6 +61,10 @@ def __strip_parent_refs(path: AnyPurePath) -> AnyPurePath: return path + @property + def root(self) -> str: + return str(self._cwd_anchor) + @property def cwd(self) -> str: return str(self._cwd_anchor / self._cwd_vpath) @@ -274,6 +279,18 @@ def host_to_virtual_path(self, hostpath: str) -> str: return str(virtpath) + def is_virtual_abspath(self, virtpath: str) -> bool: + """Determine whether a given virtual path is absolute. + + Args: + virtpath : virtual path to query + + Returns: `True` if `virtpath` is an absolute path, `False` if relative + """ + + vpath = self.PureVirtualPath(virtpath) + + return vpath.is_absolute() def virtual_abspath(self, virtpath: str) -> str: """Convert a relative virtual path to an absolute virtual path based @@ -346,4 +363,3 @@ def host_casefold_path(self, hostpath: str) -> Optional[str]: return QlOsPath.__host_casefold_path(hostpath) return hostpath - diff --git a/qiling/os/posix/syscall/unistd.py b/qiling/os/posix/syscall/unistd.py index b52f5e4b7..6f413cf3d 100644 --- a/qiling/os/posix/syscall/unistd.py +++ b/qiling/os/posix/syscall/unistd.py @@ -3,23 +3,26 @@ # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # +from __future__ import annotations + import os -import stat import itertools import pathlib -from typing import Iterator -from multiprocessing import Process +from typing import TYPE_CHECKING, Iterator, Optional from qiling import Qiling from qiling.const import QL_ARCH, QL_OS from qiling.os.posix.filestruct import ql_pipe from qiling.os.posix.const import * -from qiling.os.posix.stat import Stat from qiling.core_hooks import QlCoreHooks +if TYPE_CHECKING: + from qiling.os.posix.posix import QlOsPosix + + def ql_syscall_exit(ql: Qiling, code: int): - if ql.os.child_processes == True: + if ql.os.child_processes: os._exit(0) if ql.multithread: @@ -37,7 +40,7 @@ def _sched_cb_exit(cur_thread): def ql_syscall_exit_group(ql: Qiling, code: int): - if ql.os.child_processes == True: + if ql.os.child_processes: os._exit(0) if ql.multithread: @@ -135,99 +138,163 @@ def ql_syscall_setgroups(ql: Qiling, gidsetsize: int, grouplist: int): def ql_syscall_setresuid(ql: Qiling): return 0 + def ql_syscall_setresgid(ql: Qiling): return 0 + def ql_syscall_capget(ql: Qiling, hdrp: int, datap: int): return 0 + def ql_syscall_capset(ql: Qiling, hdrp: int, datap: int): return 0 + def ql_syscall_kill(ql: Qiling, pid: int, sig: int): return 0 +def get_opened_fd(os: QlOsPosix, fd: int): + if fd not in range(NR_OPEN): + # TODO: set errno to EBADF + return None + + f = os.fd[fd] + + if f is None: + # TODO: set errno to EBADF + return None + + return f + + def ql_syscall_fsync(ql: Qiling, fd: int): - try: - os.fsync(ql.os.fd[fd].fileno()) - regreturn = 0 - except: + f = get_opened_fd(ql.os, fd) + + if f is None: regreturn = -1 - ql.log.debug("fsync(%d) = %d" % (fd, regreturn)) + + else: + try: + os.fsync(f.fileno()) + except OSError: + regreturn = -1 + else: + regreturn = 0 + + ql.log.debug(f'fsync({fd:d}) = {regreturn}') + return regreturn def ql_syscall_fdatasync(ql: Qiling, fd: int): try: os.fdatasync(ql.os.fd[fd].fileno()) - regreturn = 0 - except: + except OSError: regreturn = -1 - ql.log.debug("fdatasync(%d) = %d" % (fd, regreturn)) + else: + regreturn = 0 + + ql.log.debug(f'fdatasync({fd:d}) = {regreturn}') + return regreturn -def ql_syscall_faccessat(ql: Qiling, dfd: int, filename: int, mode: int): - access_path = ql.os.utils.read_cstring(filename) - real_path = ql.os.path.transform_to_real_path(access_path) +def virtual_abspath_at(ql: Qiling, vpath: str, dirfd: int) -> Optional[str]: + if ql.os.path.is_virtual_abspath(vpath): + return vpath - if not os.path.exists(real_path): - regreturn = -1 - else: - regreturn = 0 + # + def __as_signed(value: int, nbits: int) -> int: + msb = (1 << (nbits - 1)) + + return -(((value & msb) << 1) - value) + + # syscall params are read as unsigned int by default. until we fix that + # broadly, this is a workaround to turn fd into a signed value + dirfd = __as_signed(dirfd, ql.arch.bits) + # + + if dirfd == AT_FDCWD: + basedir = ql.os.path.cwd - if regreturn == -1: - ql.log.debug(f'File not found or skipped: {access_path}') else: - ql.log.debug(f'File found: {access_path}') + f = get_opened_fd(ql.os, dirfd) - return regreturn + if f is None or not hasattr(f, 'name'): + # EBADF + return None + hpath = f.name -def ql_syscall_lseek(ql: Qiling, fd: int, offset: int, origin: int): - if fd not in range(NR_OPEN): - return -EBADF + if not os.path.isdir(hpath): + # ENOTDIR + return None - f = ql.os.fd[fd] + basedir = ql.os.path.host_to_virtual_path(hpath) - if f is None: - return -EBADF + return str(ql.os.path.PureVirtualPath(basedir, vpath)) - offset = ql.unpacks(ql.pack(offset)) - try: - regreturn = f.seek(offset, origin) - except OSError: +def ql_syscall_faccessat(ql: Qiling, dirfd: int, filename: int, mode: int): + vpath = ql.os.utils.read_cstring(filename) + vpath_at = virtual_abspath_at(ql, vpath, dirfd) + + if vpath_at is None: regreturn = -1 - # ql.log.debug("lseek(fd = %d, ofset = 0x%x, origin = 0x%x) = %d" % (fd, offset, origin, regreturn)) + else: + hpath = ql.os.path.virtual_to_host_path(vpath_at) + + if not ql.os.path.is_safe_host_path(hpath): + raise PermissionError(f'unsafe path: {hpath}') + + regreturn = 0 if os.path.exists(hpath) else -1 + + ql.log.debug(f'faccessat({dirfd:d}, "{vpath}", {mode:d}) = {regreturn}') return regreturn -def ql_syscall__llseek(ql: Qiling, fd: int, offset_high: int, offset_low: int, result: int, whence: int): - if fd not in range(NR_OPEN): - return -EBADF +def ql_syscall_lseek(ql: Qiling, fd: int, offset: int, origin: int): + offset = ql.unpacks(ql.pack(offset)) - f = ql.os.fd[fd] + f = get_opened_fd(ql.os, fd) if f is None: - return -EBADF + regreturn = -1 + + else: + try: + regreturn = f.seek(offset, origin) + except OSError: + regreturn = -1 + + ql.log.debug(f'lseek({fd:d}, {offset:#x}, {origin}) = {regreturn}') + + return regreturn + +def ql_syscall__llseek(ql: Qiling, fd: int, offset_high: int, offset_low: int, result: int, whence: int): # treat offset as a signed value offset = ql.unpack64s(ql.pack64((offset_high << 32) | offset_low)) - origin = whence - try: - ret = f.seek(offset, origin) - except OSError: + f = get_opened_fd(ql.os, fd) + + if f is None: regreturn = -1 + else: - ql.mem.write_ptr(result, ret, 8) - regreturn = 0 + try: + ret = f.seek(offset, whence) + except OSError: + regreturn = -1 + else: + ql.mem.write_ptr(result, ret, 8) + regreturn = 0 - # ql.log.debug("_llseek(%d, 0x%x, 0x%x, 0x%x) = %d" % (fd, offset_high, offset_low, origin, regreturn)) + ql.log.debug(f'_llseek({fd:d}, {offset_high:#x}, {offset_low:#x}, {result:#x}, {whence}) = {regreturn}') return regreturn @@ -251,75 +318,68 @@ def ql_syscall_brk(ql: Qiling, inp: int): def ql_syscall_access(ql: Qiling, path: int, mode: int): - file_path = ql.os.utils.read_cstring(path) - real_path = ql.os.path.transform_to_real_path(file_path) - relative_path = ql.os.path.transform_to_relative_path(file_path) + vpath = ql.os.utils.read_cstring(path) + hpath = ql.os.path.virtual_to_host_path(vpath) - regreturn = 0 if os.path.exists(real_path) else -1 + if not ql.os.path.is_safe_host_path(hpath): + raise PermissionError(f'unsafe path: {hpath}') - # ql.log.debug("access(%s, 0x%x) = %d " % (relative_path, access_mode, regreturn)) + regreturn = 0 if os.path.exists(hpath) else -1 - if regreturn == 0: - ql.log.debug(f'File found: {relative_path}') - else: - ql.log.debug(f'No such file or directory: {relative_path}') + ql.log.debug(f'access("{vpath}", 0{mode:o}) = {regreturn}') return regreturn def ql_syscall_close(ql: Qiling, fd: int): - if fd not in range(NR_OPEN): - return -1 - - f = ql.os.fd[fd] + f = get_opened_fd(ql.os, fd) if f is None: - return -1 + regreturn = -1 - f.close() - ql.os.fd[fd] = None + else: + f.close() + ql.os.fd[fd] = None + regreturn = 0 - return 0 + ql.log.debug(f'close({fd:d}) = {regreturn}') + return regreturn -def ql_syscall_pread64(ql: Qiling, fd: int, buf: int, length: int, offt: int): - if fd not in range(NR_OPEN): - return -1 - f = ql.os.fd[fd] +def ql_syscall_pread64(ql: Qiling, fd: int, buf: int, length: int, offt: int): + f = get_opened_fd(ql.os, fd) if f is None: - return -1 + regreturn = -1 - # https://chromium.googlesource.com/linux-syscall-support/+/2c73abf02fd8af961e38024882b9ce0df6b4d19b - # https://chromiumcodereview.appspot.com/10910222 - if ql.arch.type == QL_ARCH.MIPS: - offt = ql.mem.read_ptr(ql.arch.regs.arch_sp + 0x10, 8) + else: + # https://chromium.googlesource.com/linux-syscall-support/+/2c73abf02fd8af961e38024882b9ce0df6b4d19b + # https://chromiumcodereview.appspot.com/10910222 + if ql.arch.type == QL_ARCH.MIPS: + offt = ql.mem.read_ptr(ql.arch.regs.arch_sp + 0x10, 8) - try: - pos = f.tell() - f.seek(offt) + try: + pos = f.tell() + f.seek(offt) - data = f.read(length) - f.seek(pos) + data = f.read(length) + f.seek(pos) + except OSError: + regreturn = -1 + else: + ql.mem.write(buf, data) - ql.mem.write(buf, data) - except: - regreturn = -1 - else: - regreturn = len(data) + regreturn = len(data) return regreturn def ql_syscall_read(ql: Qiling, fd, buf: int, length: int): - if fd not in range(NR_OPEN): - return -EBADF - - f = ql.os.fd[fd] + f = get_opened_fd(ql.os, fd) if f is None: - return -EBADF + return -1 try: data = f.read(length) @@ -334,13 +394,10 @@ def ql_syscall_read(ql: Qiling, fd, buf: int, length: int): def ql_syscall_write(ql: Qiling, fd: int, buf: int, count: int): - if fd not in range(NR_OPEN): - return -EBADF - - f = ql.os.fd[fd] + f = get_opened_fd(ql.os, fd) if f is None: - return -EBADF + return -1 try: data = ql.mem.read(buf, count) @@ -357,90 +414,95 @@ def ql_syscall_write(ql: Qiling, fd: int, buf: int, count: int): ql.log.warning(f'write failed since fd {fd:d} does not have a write method') regreturn = -1 - return regreturn -def ql_syscall_readlink(ql: Qiling, path_name: int, path_buff: int, path_buffsize: int): - pathname = ql.os.utils.read_cstring(path_name) - # pathname = str(pathname, 'utf-8', errors="ignore") - host_path = ql.os.path.virtual_to_host_path(pathname) - virt_path = ql.os.path.virtual_abspath(pathname) +def __do_readlink(ql: Qiling, absvpath: str, outbuf: int) -> int: + target = None - # cover procfs psaudo files first - # TODO: /proc/self/root, /proc/self/cwd - if virt_path == r'/proc/self/exe': - p = ql.os.path.host_to_virtual_path(ql.path) - p = p.encode('utf-8') + # cover a few procfs pseudo files first + if absvpath == r'/proc/self/exe': + # note this would raise an exception if the binary is not under rootfs + target = ql.os.path.host_to_virtual_path(ql.path) - ql.mem.write(path_buff, p + b'\x00') - regreturn = len(p) + elif absvpath == r'/proc/self/cwd': + target = ql.os.path.cwd - elif os.path.exists(host_path): - regreturn = 0 + elif absvpath == r'/proc/self/root': + target = ql.os.path.root else: - regreturn = -1 + hpath = ql.os.path.virtual_to_host_path(absvpath) - ql.log.debug('readlink("%s", 0x%x, 0x%x) = %d' % (virt_path, path_buff, path_buffsize, regreturn)) + if not ql.os.path.is_safe_host_path(hpath): + raise PermissionError(f'unsafe path: {hpath}') - return regreturn + # FIXME: we do not really know how to emulated links, so we do not read them + if os.path.exists(hpath): + target = '' + if target is None: + return -1 + + cstr = target.encode('utf-8') + + if cstr: + ql.mem.write(outbuf, cstr + b'\x00') + + return len(cstr) -def ql_syscall_getcwd(ql: Qiling, path_buff: int, path_buffsize: int): - localpath = ql.os.path.transform_to_relative_path('./') - localpath = bytes(localpath, 'utf-8') + b'\x00' - ql.mem.write(path_buff, localpath) - regreturn = len(localpath) +def ql_syscall_readlink(ql: Qiling, pathname: int, buf: int, bufsize: int): + vpath = ql.os.utils.read_cstring(pathname) + absvpath = ql.os.path.virtual_abspath(vpath) - pathname = ql.os.utils.read_cstring(path_buff) - # pathname = str(pathname, 'utf-8', errors="ignore") + regreturn = __do_readlink(ql, absvpath, buf) - ql.log.debug("getcwd(%s, 0x%x) = %d" % (pathname, path_buffsize, regreturn)) + ql.log.debug(f'readlink("{vpath}", {buf:#x}, {bufsize:#x}) = {regreturn}') return regreturn -def ql_syscall_chdir(ql: Qiling, path_name: int): - pathname = ql.os.utils.read_cstring(path_name) - host_path = ql.os.path.virtual_to_host_path(pathname) - virt_path = ql.os.path.virtual_abspath(pathname) +def ql_syscall_readlinkat(ql: Qiling, dirfd: int, pathname: int, buf: int, bufsize: int): + vpath = ql.os.utils.read_cstring(pathname) + absvpath = virtual_abspath_at(ql, vpath, dirfd) - if os.path.exists(host_path) and os.path.isdir(host_path): - ql.os.path.cwd = virt_path + regreturn = -1 if absvpath is None else __do_readlink(ql, absvpath, buf) - regreturn = 0 - ql.log.debug("chdir(%s) = %d"% (virt_path, regreturn)) - else: - regreturn = -1 - ql.log.warning("chdir(%s) = %d : not found" % (virt_path, regreturn)) + ql.log.debug(f'readlinkat({dirfd:d}, "{vpath}", {buf:#x}, {bufsize:#x}) = {regreturn}') return regreturn -def ql_syscall_readlinkat(ql: Qiling, dfd: int, path: int, buf: int, bufsize: int): - pathname = ql.os.utils.read_cstring(path) - # pathname = str(pathname, 'utf-8', errors="ignore") - host_path = ql.os.path.virtual_to_host_path(pathname) - virt_path = ql.os.path.virtual_abspath(pathname) +def ql_syscall_getcwd(ql: Qiling, path_buff: int, path_buffsize: int): + cwd = ql.os.path.cwd - # cover procfs psaudo files first - # TODO: /proc/self/root, /proc/self/cwd - if virt_path == r'/proc/self/exe': - p = ql.os.path.host_to_virtual_path(ql.path) - p = p.encode('utf-8') + cwd_bytes = cwd.encode('utf-8') + b'\x00' + ql.mem.write(path_buff, cwd_bytes) + regreturn = len(cwd_bytes) - ql.mem.write(buf, p + b'\x00') - regreturn = len(p) + ql.log.debug(f'getcwd("{cwd}", {path_buffsize}) = {regreturn}') - elif os.path.exists(host_path): - regreturn = 0 + return regreturn + + +def ql_syscall_chdir(ql: Qiling, path_name: int): + vpath = ql.os.utils.read_cstring(path_name) + hpath = ql.os.path.virtual_to_host_path(vpath) + + if not ql.os.path.is_safe_host_path(hpath): + raise PermissionError(f'unsafe path: {hpath}') + + absvpath = ql.os.path.virtual_abspath(vpath) + + if os.path.isdir(hpath): + ql.os.path.cwd = absvpath + regreturn = 0 else: regreturn = -1 - ql.log.debug('readlinkat(%d, "%s", 0x%x, 0x%x) = %d' % (dfd, virt_path, buf, bufsize, regreturn)) + ql.log.debug(f'chdir("{absvpath}") = {regreturn}') return regreturn @@ -455,6 +517,8 @@ def ql_syscall_getppid(ql: Qiling): def ql_syscall_vfork(ql: Qiling): if ql.host.os == QL_OS.WINDOWS: + from multiprocessing import Process + try: pid = Process() pid = 0 @@ -479,6 +543,7 @@ def ql_syscall_vfork(ql: Qiling): def ql_syscall_fork(ql: Qiling): return ql_syscall_vfork(ql) + def ql_syscall_setsid(ql: Qiling): return os.getpid() @@ -539,51 +604,72 @@ def __read_str_array(addr: int) -> Iterator[str]: def ql_syscall_dup(ql: Qiling, oldfd: int): - if oldfd not in range(NR_OPEN): - return -EBADF - - f = ql.os.fd[oldfd] + f = get_opened_fd(ql.os, oldfd) if f is None: - return -EBADF + return -1 - idx = next((i for i in range(NR_OPEN) if ql.os.fd[i] is None), -1) + newfd = next((i for i in range(NR_OPEN) if ql.os.fd[i] is None), -1) - if idx == -1: + if newfd == -1: return -EMFILE - ql.os.fd[idx] = f.dup() + ql.os.fd[newfd] = f.dup() - return idx + ql.log.debug(f'dup({oldfd:d}) = {newfd:d}') + return newfd -def ql_syscall_dup2(ql: Qiling, fd: int, newfd: int): - if fd not in range(NR_OPEN) or newfd not in range(NR_OPEN): - return -EBADF - f = ql.os.fd[fd] +def ql_syscall_dup2(ql: Qiling, oldfd: int, newfd: int): + f = get_opened_fd(ql.os, oldfd) if f is None: - return -EBADF + return -1 + + if newfd not in range(NR_OPEN): + return -1 + + newslot = ql.os.fd[newfd] + + if newslot is not None: + newslot.close() ql.os.fd[newfd] = f.dup() + ql.log.debug(f'dup2({oldfd:d}, {newfd:d}) = {newfd:d}') + return newfd -def ql_syscall_dup3(ql: Qiling, fd: int, newfd: int, flags: int): - if fd not in range(NR_OPEN) or newfd not in range(NR_OPEN): - return -1 +def ql_syscall_dup3(ql: Qiling, oldfd: int, newfd: int, flags: int): + O_CLOEXEC = 0o2000000 - f = ql.os.fd[fd] + f = get_opened_fd(ql.os, oldfd) if f is None: return -1 - ql.os.fd[newfd] = f.dup() + if newfd not in range(NR_OPEN): + return -1 + + newslot = ql.os.fd[newfd] + + if newslot is not None: + newslot.close() + + newf = f.dup() + + if flags & O_CLOEXEC: + newf.close_on_exec = True + + ql.os.fd[newfd] = newf + + ql.log.debug(f'dup3({oldfd:d}, {newfd:d}, 0{flags:o}) = {newfd:d}') return newfd + def ql_syscall_set_tid_address(ql: Qiling, tidptr: int): if ql.os.thread_management: regreturn = ql.os.thread_management.cur_thread.id @@ -621,102 +707,107 @@ def ql_syscall_nice(ql: Qiling, inc: int): return 0 -def ql_syscall_truncate(ql: Qiling, path: int, length: int): - file_path = ql.os.utils.read_cstring(path) - real_path = ql.os.path.transform_to_real_path(file_path) - st_size = Stat(real_path).st_size +def __do_truncate(ql: Qiling, hpath: str, length: int) -> int: + if not ql.os.path.is_safe_host_path(hpath): + raise PermissionError(f'unsafe path: {hpath}') try: - if st_size >= length: - os.truncate(real_path, length) + st_size = os.path.getsize(hpath) - else: + if st_size > length: + os.truncate(hpath, length) + + elif st_size < length: padding = length - st_size - with open(real_path, 'a+b') as ofile: + with open(hpath, 'a+b') as ofile: ofile.write(b'\x00' * padding) - except: - regreturn = -1 + except OSError: + return -1 else: - regreturn = 0 + return 0 + - ql.log.debug('truncate(%s, 0x%x) = %d' % (file_path, length, regreturn)) +def ql_syscall_truncate(ql: Qiling, path: int, length: int): + vpath = ql.os.utils.read_cstring(path) + hpath = ql.os.path.virtual_to_host_path(vpath) + + regreturn = __do_truncate(ql, hpath, length) + + ql.log.debug(f'truncate("{vpath}", {length:#x}) = {regreturn}') return regreturn def ql_syscall_ftruncate(ql: Qiling, fd: int, length: int): - real_path = ql.os.fd[fd].name - st_size = Stat(real_path).st_size + f = get_opened_fd(ql.os, fd) - try: - if st_size >= length: - os.truncate(real_path, length) + regreturn = -1 if f is None else __do_truncate(ql, f.name, length) - else: - padding = length - st_size + ql.log.debug(f'ftruncate({fd}, {length:#x}) = {regreturn}') - with open(real_path, 'a+b') as ofile: - ofile.write(b'\x00' * padding) - except: - regreturn = -1 - else: - regreturn = 0 + return regreturn - ql.log.debug("ftruncate(%d, 0x%x) = %d" % (fd, length, regreturn)) - return regreturn +def __do_unlink(ql: Qiling, absvpath: str) -> int: + def __has_opened_fd(hpath: str) -> bool: + opened_fds = (ql.os.fd[i] for i in range(NR_OPEN) if ql.os.fd[i] is not None) + f = next((fd for fd in opened_fds if getattr(fd, 'name', '') == hpath), None) -def ql_syscall_unlink(ql: Qiling, pathname: int): - file_path = ql.os.utils.read_cstring(pathname) - real_path = ql.os.path.transform_to_real_path(file_path) + return f is not None and f.closed - opened_fds = [getattr(ql.os.fd[i], 'name', None) for i in range(NR_OPEN) if ql.os.fd[i] is not None] - path = pathlib.Path(real_path) + hpath = ql.os.path.virtual_to_host_path(absvpath) - if any((real_path not in opened_fds, path.is_block_device(), path.is_fifo(), path.is_socket(), path.is_symlink())): - try: - os.unlink(real_path) - except FileNotFoundError: - ql.log.debug('No such file or directory') - regreturn = -1 - except: - regreturn = -1 - else: - regreturn = 0 + if ql.os.fs_mapper.has_mapping(absvpath): + if __has_opened_fd(hpath): + return -1 + + ql.os.fs_mapper.remove_mapping(absvpath) else: - regreturn = -1 + if not ql.os.path.is_safe_host_path(hpath): + raise PermissionError(f'unsafe path: {hpath}') - ql.log.debug("unlink(%s) = %d" % (file_path, regreturn)) + # NOTE: no idea why these are always ok to remove + def __ok_to_remove(hpath: str) -> bool: + path = pathlib.Path(hpath) - return regreturn + return any((path.is_block_device(), path.is_fifo(), path.is_socket(), path.is_symlink())) + if __has_opened_fd(hpath) and not __ok_to_remove(hpath): + return -1 -def ql_syscall_unlinkat(ql: Qiling, fd: int, pathname: int): - file_path = ql.os.utils.read_cstring(pathname) - real_path = ql.os.path.transform_to_real_path(file_path) + try: + os.unlink(hpath) + except OSError: + return -1 - try: - dir_fd = ql.os.fd[fd].fileno() - except: - dir_fd = None + return 0 - try: - if dir_fd is None: - os.unlink(real_path) - else: - os.unlink(file_path, dir_fd=dir_fd) - except OSError as e: - regreturn = -e.errno - else: - regreturn = 0 - ql.log.debug("unlinkat(fd = %d, path = '%s') = %d" % (fd, file_path, regreturn)) +def ql_syscall_unlink(ql: Qiling, pathname: int): + vpath = ql.os.utils.read_cstring(pathname) + absvpath = ql.os.path.virtual_abspath(vpath) + + regreturn = __do_unlink(ql, absvpath) + + ql.log.debug(f'unlink("{vpath}") = {regreturn}') return regreturn + +def ql_syscall_unlinkat(ql: Qiling, dirfd: int, pathname: int, flags: int): + vpath = ql.os.utils.read_cstring(pathname) + absvpath = virtual_abspath_at(ql, vpath, dirfd) + + regreturn = -1 if absvpath is None else __do_unlink(ql, absvpath) + + ql.log.debug(f'unlinkat({dirfd}, "{vpath}") = {regreturn}') + + return regreturn + + # https://man7.org/linux/man-pages/man2/getdents.2.html # struct linux_dirent { # unsigned long d_ino; /* Inode number */ @@ -827,5 +918,6 @@ def _type_mapping(ent): def ql_syscall_getdents(ql: Qiling, fd: int, dirp: int, count: int): return __getdents_common(ql, fd, dirp, count, is_64=False) + def ql_syscall_getdents64(ql: Qiling, fd: int, dirp: int, count: int): return __getdents_common(ql, fd, dirp, count, is_64=True) From 0be462013ad16d333dd04dbd9dd332589da9123a Mon Sep 17 00:00:00 2001 From: elicn Date: Thu, 30 Mar 2023 23:49:32 +0300 Subject: [PATCH 16/42] Patched fcntl functions --- qiling/os/linux/procfs.py | 4 ++ qiling/os/posix/posix.py | 2 +- qiling/os/posix/syscall/fcntl.py | 101 +++++++++++-------------------- 3 files changed, 40 insertions(+), 67 deletions(-) diff --git a/qiling/os/linux/procfs.py b/qiling/os/linux/procfs.py index 11bd623f1..dc51e0611 100644 --- a/qiling/os/linux/procfs.py +++ b/qiling/os/linux/procfs.py @@ -16,6 +16,10 @@ class FsMappedStream(io.BytesIO): def __init__(self, fname: str, *args) -> None: super().__init__(*args) + # note that the name property should reflect the actual file name + # on the host file system, and here we get a virtual file name + # instead. we should be fine, however, since there is no file + # backing this object anyway self.name = fname diff --git a/qiling/os/posix/posix.py b/qiling/os/posix/posix.py index bce85b7ed..25335d409 100644 --- a/qiling/os/posix/posix.py +++ b/qiling/os/posix/posix.py @@ -72,7 +72,7 @@ def __init__(self): def __len__(self): return len(self.__fds) - def __getitem__(self, idx: Union[slice, int]): + def __getitem__(self, idx: int): return self.__fds[idx] def __setitem__(self, idx: int, val: Optional[IO]): diff --git a/qiling/os/posix/syscall/fcntl.py b/qiling/os/posix/syscall/fcntl.py index 6904cba92..5b1f2a9c3 100644 --- a/qiling/os/posix/syscall/fcntl.py +++ b/qiling/os/posix/syscall/fcntl.py @@ -4,7 +4,6 @@ # import os -from pathlib import Path from qiling import Qiling from qiling.const import QL_OS, QL_ARCH @@ -13,38 +12,51 @@ from qiling.os.posix.const_mapping import ql_open_flag_mapping from qiling.os.posix.filestruct import ql_socket +from .unistd import virtual_abspath_at, get_opened_fd -def ql_syscall_open(ql: Qiling, filename: int, flags: int, mode: int): - vpath = ql.os.utils.read_cstring(filename) +def __do_open(ql: Qiling, absvpath: str, flags: int, mode: int) -> int: flags &= 0xffffffff mode &= 0xffffffff + # look for the next available fd slot idx = next((i for i in range(NR_OPEN) if ql.os.fd[i] is None), -1) if idx == -1: - regreturn = -EMFILE - else: - if ql.arch.type == QL_ARCH.ARM and ql.os.type != QL_OS.QNX: - mode = 0 + return -EMFILE - try: - flags = ql_open_flag_mapping(ql, flags) - ql.os.fd[idx] = ql.os.fs_mapper.open_ql_file(vpath, flags, mode) - except QlSyscallError as e: - regreturn = -e.errno - else: - regreturn = idx + if ql.arch.type is QL_ARCH.ARM and ql.os.type is not QL_OS.QNX: + mode = 0 - hpath = ql.os.path.virtual_to_host_path(vpath) + # translate emulated os open flags into host os open flags + flags = ql_open_flag_mapping(ql, flags) + + try: + ql.os.fd[idx] = ql.os.fs_mapper.open_ql_file(absvpath, flags, mode) + except QlSyscallError: + return -1 + + return idx + + +def ql_syscall_open(ql: Qiling, filename: int, flags: int, mode: int): + vpath = ql.os.utils.read_cstring(filename) absvpath = ql.os.path.virtual_abspath(vpath) - ql.log.debug(f'open("{absvpath}", {mode:#o}) = {regreturn}') + regreturn = __do_open(ql, absvpath, flags, mode) - if regreturn >= 0 and regreturn != 2: - ql.log.debug(f'File found: {hpath:s}') - else: - ql.log.debug(f'File not found {hpath:s}') + ql.log.debug(f'open("{absvpath}", {flags:#x}, 0{mode:o}) = {regreturn}') + + return regreturn + + +def ql_syscall_openat(ql: Qiling, fd: int, path: int, flags: int, mode: int): + vpath = ql.os.utils.read_cstring(path) + absvpath = virtual_abspath_at(ql, vpath, fd) + + regreturn = -1 if absvpath is None else __do_open(ql, absvpath, flags, mode) + + ql.log.debug(f'openat({fd:d}, "{vpath}", {flags:#x}, 0{mode:o}) = {regreturn:d}') return regreturn @@ -85,54 +97,11 @@ def ql_syscall_creat(ql: Qiling, filename: int, mode: int): return regreturn -def ql_syscall_openat(ql: Qiling, fd: int, path: int, flags: int, mode: int): - vpath = ql.os.utils.read_cstring(path) - # real_path = ql.os.path.transform_to_real_path(path) - # relative_path = ql.os.path.transform_to_relative_path(path) - - flags &= 0xffffffff - mode &= 0xffffffff - - idx = next((i for i in range(NR_OPEN) if ql.os.fd[i] is None), -1) - - if idx == -1: - regreturn = -EMFILE - else: - fd = ql.unpacks(ql.pack(fd)) - - if ql.arch.type == QL_ARCH.ARM: - mode = 0 - - try: - flags = ql_open_flag_mapping(ql, flags) - - if 0 <= fd < NR_OPEN: - fobj = ql.os.fd[fd] - - # ql_file object or QlFsMappedObject - if hasattr(fobj, "fileno") and hasattr(fobj, "name"): - if not Path.is_absolute(Path(vpath)): - vpath = str(Path(fobj.name) / Path(vpath)) - - ql.os.fd[idx] = ql.os.fs_mapper.open_ql_file(vpath, flags, mode) - except QlSyscallError as e: - regreturn = -e.errno - else: - regreturn = idx - - ql.log.debug(f'openat(fd = {fd:d}, path = {vpath}, mode = {mode:#o}) = {regreturn:d}') - - return regreturn - - def ql_syscall_fcntl(ql: Qiling, fd: int, cmd: int, arg: int): - if fd not in range(NR_OPEN): - return -EBADF - - f = ql.os.fd[fd] + f = get_opened_fd(ql.os, fd) if f is None: - return -EBADF + return -1 if cmd == F_DUPFD: if arg not in range(NR_OPEN): @@ -147,7 +116,7 @@ def ql_syscall_fcntl(ql: Qiling, fd: int, cmd: int, arg: int): regreturn = -EMFILE elif cmd == F_GETFD: - regreturn = getattr(f, "close_on_exec", False) + regreturn = int(getattr(f, "close_on_exec", False)) elif cmd == F_SETFD: f.close_on_exec = bool(arg & FD_CLOEXEC) From 24305784ea00009724ff4c1b77141bea226a4e76 Mon Sep 17 00:00:00 2001 From: elicn Date: Thu, 30 Mar 2023 23:50:32 +0300 Subject: [PATCH 17/42] Patched some stat functions --- qiling/os/posix/syscall/stat.py | 81 +++++++++++++++++++++++---------- 1 file changed, 56 insertions(+), 25 deletions(-) diff --git a/qiling/os/posix/syscall/stat.py b/qiling/os/posix/syscall/stat.py index 0a6da9e4f..d4e1be375 100644 --- a/qiling/os/posix/syscall/stat.py +++ b/qiling/os/posix/syscall/stat.py @@ -1225,27 +1225,45 @@ def transform_path(ql: Qiling, dirfd: int, path: int, flags: int = 0): def ql_syscall_chmod(ql: Qiling, filename: int, mode: int): - file_path = ql.os.utils.read_cstring(filename) - real_path = ql.os.path.transform_to_real_path(file_path) + vpath = ql.os.utils.read_cstring(filename) + hpath = ql.os.path.virtual_to_host_path(vpath) + + if not ql.os.path.is_safe_host_path(hpath): + raise PermissionError(f'unsafe path: {hpath}') + try: - os.chmod(real_path, mode) - regreturn = 0 - except: + os.chmod(hpath, mode) + except OSError: regreturn = -1 - ql.log.debug(f'chmod("{ql.os.utils.read_cstring(filename)}", {mode:d}) = 0') + else: + regreturn = 0 + + ql.log.debug(f'chmod("{vpath}", 0{mode:o}) = {regreturn}') + return regreturn + def ql_syscall_fchmod(ql: Qiling, fd: int, mode: int): - if fd not in range(NR_OPEN) or ql.os.fd[fd] is None: - return -EBADF + if fd not in range(NR_OPEN): + return -1 + + f = ql.os.fd[fd] + + if f is None: + return -1 + try: - os.fchmod(ql.os.fd[fd].fileno(), mode) - regreturn = 0 - except: + os.fchmod(f.fileno(), mode) + except OSError: regreturn = -1 - ql.log.debug("fchmod(%d, %d) = %d" % (fd, mode, regreturn)) + else: + regreturn = 0 + + ql.log.debug(f'fchmod({fd}, 0{mode:o}) = {regreturn}') + return regreturn + def ql_syscall_fstatat64(ql: Qiling, dirfd: int, path: int, buf_ptr: int, flags: int): dirfd, real_path = transform_path(ql, dirfd, path, flags) @@ -1476,32 +1494,45 @@ def ql_syscall_mknodat(ql: Qiling, dirfd: int, path: int, mode: int, dev: int): def ql_syscall_mkdir(ql: Qiling, pathname: int, mode: int): - file_path = ql.os.utils.read_cstring(pathname) - real_path = ql.os.path.transform_to_real_path(file_path) - regreturn = 0 + vpath = ql.os.utils.read_cstring(pathname) + hpath = ql.os.path.virtual_to_host_path(vpath) + + if not ql.os.path.is_safe_host_path(hpath): + raise PermissionError(f'unsafe path: {hpath}') try: - if not os.path.exists(real_path): - os.mkdir(real_path, mode) - except: + if not os.path.exists(hpath): + os.mkdir(hpath, mode) + except OSError: regreturn = -1 + else: + regreturn = 0 + + ql.log.debug(f'mkdir("{vpath}", 0{mode:o}) = {regreturn}') - ql.log.debug("mkdir(%s, 0%o) = %d" % (real_path, mode, regreturn)) return regreturn + def ql_syscall_rmdir(ql: Qiling, pathname: int): - file_path = ql.os.utils.read_cstring(pathname) - real_path = ql.os.path.transform_to_real_path(file_path) - regreturn = 0 + vpath = ql.os.utils.read_cstring(pathname) + hpath = ql.os.path.virtual_to_host_path(vpath) + + if not ql.os.path.is_safe_host_path(hpath): + raise PermissionError(f'unsafe path: {hpath}') try: - if os.path.exists(real_path): - os.rmdir(real_path) - except: + if os.path.exists(hpath): + os.rmdir(hpath) + except OSError: regreturn = -1 + else: + regreturn = 0 + + ql.log.debug(f'rmdir("{vpath}") = {regreturn}') return regreturn + def ql_syscall_fstatfs(ql: Qiling, fd: int, buf: int): data = b"0" * (12 * 8) # for now, just return 0s regreturn = 0 From 63c92f1a0941627e8f3e34e0d102b4181917ae08 Mon Sep 17 00:00:00 2001 From: elicn Date: Thu, 30 Mar 2023 23:51:34 +0300 Subject: [PATCH 18/42] Fix a bug in FindFirstFileA --- qiling/os/windows/dlls/kernel32/fileapi.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/qiling/os/windows/dlls/kernel32/fileapi.py b/qiling/os/windows/dlls/kernel32/fileapi.py index 1176462f6..60465ff87 100644 --- a/qiling/os/windows/dlls/kernel32/fileapi.py +++ b/qiling/os/windows/dlls/kernel32/fileapi.py @@ -52,21 +52,13 @@ def hook_FindFirstFileA(ql: Qiling, address: int, params): if len(filename) >= MAX_PATH: return ERROR_INVALID_PARAMETER - host_path = ql.os.path.virtual_to_host_path(filename) - - # Verify the directory is in ql.rootfs to ensure no path traversal has taken place - if not ql.os.path.is_safe_host_path(host_path): - ql.os.last_error = ERROR_FILE_NOT_FOUND - - return INVALID_HANDLE_VALUE - # Check if path exists filesize = 0 try: - f = ql.os.fs_mapper.open(host_path, "r") + f = ql.os.fs_mapper.open(filename, "r") - filesize = os.path.getsize(host_path) + filesize = os.path.getsize(f.name) except FileNotFoundError: ql.os.last_error = ERROR_FILE_NOT_FOUND From 629454f2de10b0ebabbe16a9003061188d17514e Mon Sep 17 00:00:00 2001 From: elicn Date: Sun, 2 Apr 2023 20:49:08 +0300 Subject: [PATCH 19/42] Re-implemented POSIX shm syscalls --- qiling/os/posix/const.py | 29 +++++--- qiling/os/posix/posix.py | 18 ++++- qiling/os/posix/syscall/__init__.py | 1 + qiling/os/posix/syscall/mman.py | 30 -------- qiling/os/posix/syscall/shm.py | 104 ++++++++++++++++++++++++++++ 5 files changed, 142 insertions(+), 40 deletions(-) create mode 100644 qiling/os/posix/syscall/shm.py diff --git a/qiling/os/posix/const.py b/qiling/os/posix/const.py index 632a95ffa..13508058a 100644 --- a/qiling/os/posix/const.py +++ b/qiling/os/posix/const.py @@ -1003,11 +1003,24 @@ class qnx_mmap_flags(Flag): } # shm syscall -IPC_CREAT = 8**3 -IPC_EXCL = 2*(8**3) -IPC_NOWAIT = 4*(8**3) - -SHM_RDONLY = 8**4 -SHM_RND = 2*(8**4) -SHM_REMAP= 4*(8**4) -SHM_EXEC = 1*(8**5) +IPC_PRIVATE = 0 + +# see: https://elixir.bootlin.com/linux/v5.19.17/source/include/uapi/linux/ipc.h +IPC_CREAT = 0o0001000 # create if key is nonexistent +IPC_EXCL = 0o0002000 # fail if key exists +IPC_NOWAIT = 0o0004000 # return error on wait + +# see: https://elixir.bootlin.com/linux/v5.19.17/source/include/uapi/linux/shm.h +SHM_W = 0o000200 +SHM_R = 0o000400 +SHM_HUGETLB = 0o004000 # segment will use huge TLB pages +SHM_RDONLY = 0o010000 # read-only access +SHM_RND = 0o020000 # round attach address to SHMLBA boundary +SHM_REMAP = 0o040000 # take-over region on attach +SHM_EXEC = 0o100000 # execution access + +SHMMNI = 4096 # max num of segs system wide + +# see: https://elixir.bootlin.com/linux/v5.19.17/source/include/uapi/asm-generic/hugetlb_encode.h +HUGETLB_FLAG_ENCODE_SHIFT = 26 +HUGETLB_FLAG_ENCODE_MASK = 0x3f diff --git a/qiling/os/posix/posix.py b/qiling/os/posix/posix.py index 25335d409..1d639ce13 100644 --- a/qiling/os/posix/posix.py +++ b/qiling/os/posix/posix.py @@ -4,7 +4,7 @@ # from inspect import signature, Parameter -from typing import TextIO, Union, Callable, IO, List, Optional +from typing import Dict, NamedTuple, TextIO, Union, Callable, IO, List, Optional from unicorn.arm64_const import UC_ARM64_REG_X8, UC_ARM64_REG_X16 from unicorn.arm_const import ( @@ -91,6 +91,16 @@ def restore(self, fds): self.__fds = fds +# vaguely reflects a shmid_ds structure +class QlShmId(NamedTuple): + segsz: int + uid: int + gid: int + # cuid: int + # cgid: int + mode: int + + class QlOsPosix(QlOs): def __init__(self, ql: Qiling): @@ -157,7 +167,7 @@ def __init__(self, ql: Qiling): self.stdout = self._stdout self.stderr = self._stderr - self._shms = {} + self._shm: Dict[int, QlShmId] = {} def __get_syscall_mapper(self, archtype: QL_ARCH): qlos_path = f'.os.{self.type.name.lower()}.map_syscall' @@ -349,3 +359,7 @@ def set_syscall_return(self, retval: int): @property def fd(self): return self._fd + + @property + def shm(self): + return self._shm \ No newline at end of file diff --git a/qiling/os/posix/syscall/__init__.py b/qiling/os/posix/syscall/__init__.py index 4e87d9b65..3477a9479 100644 --- a/qiling/os/posix/syscall/__init__.py +++ b/qiling/os/posix/syscall/__init__.py @@ -16,6 +16,7 @@ from .sched import * from .select import * from .sendfile import * +from .shm import * from .signal import * from .socket import * from .stat import * diff --git a/qiling/os/posix/syscall/mman.py b/qiling/os/posix/syscall/mman.py index 6d80a74e1..4990b3c51 100755 --- a/qiling/os/posix/syscall/mman.py +++ b/qiling/os/posix/syscall/mman.py @@ -225,33 +225,3 @@ def ql_syscall_mmap2(ql: Qiling, addr: int, length: int, prot: int, flags: int, pgoffset *= ql.mem.pagesize return syscall_mmap_impl(ql, addr, length, prot, flags, fd, pgoffset, 2) - - -def ql_syscall_shmget(ql: Qiling, key: int, size: int, shmflg: int): - if shmflg & IPC_CREAT: - if shmflg & IPC_EXCL: - if key in ql.os._shms: - return EEXIST - else: - #addr = ql.mem.map_anywhere(size) - ql.os._shms[key] = (key, size) - return key - else: - if key not in ql.os._shms: - return ENOENT - - -def ql_syscall_shmat(ql: Qiling, shmid: int, shmaddr: int, shmflg: int): - # shmid == key - # dummy implementation - if shmid not in ql.os._shms: - return EINVAL - - key, size = ql.os._shms[shmid] - - if shmaddr == 0: - addr = ql.mem.map_anywhere(size) - else: - addr = ql.mem.map(shmaddr, size, info="[shm]") - - return addr diff --git a/qiling/os/posix/syscall/shm.py b/qiling/os/posix/syscall/shm.py new file mode 100644 index 000000000..926f72e49 --- /dev/null +++ b/qiling/os/posix/syscall/shm.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +# +# Cross Platform and Multi Architecture Advanced Binary Emulation Framework +# + +from unicorn.unicorn_const import UC_PROT_WRITE, UC_PROT_READ + +from qiling import Qiling +from qiling.exception import QlOutOfMemory +from qiling.os.posix.const import * +from qiling.os.posix.posix import QlShmId + + +def ql_syscall_shmget(ql: Qiling, key: int, size: int, shmflg: int): + + def __create_shm(size: int, flags: int) -> int: + perms = flags & ((1 << 9) - 1) + + posix_to_uc = ( + (SHM_W, UC_PROT_WRITE), + (SHM_R, UC_PROT_READ) + ) + + # convert posix permissions to unicorn memory access bits + uc_perms = sum(u_perm for p_perm, u_perm in posix_to_uc if perms & p_perm) + + # determine size alignment: either normal or huge page + if flags & SHM_HUGETLB: + shiftsize = (flags >> HUGETLB_FLAG_ENCODE_SHIFT) & HUGETLB_FLAG_ENCODE_MASK + pagesize = (1 << shiftsize) + else: + pagesize = ql.mem.pagesize + + size = ql.mem.align_up(size, pagesize) + + if len(ql.os.shm) < SHMMNI: + try: + key = ql.mem.map_anywhere(size, perms=uc_perms, info='[shm]') + except QlOutOfMemory: + return -1 # ENOMEM + + ql.os.shm[key] = QlShmId(size, ql.os.uid, ql.os.gid, perms) + + else: + return -1 # ENOSPC + + return key + + # create new shared memory segment + if key == IPC_PRIVATE: + key = __create_shm(size, shmflg) + + # a shm with the specified key exists + elif key in ql.os.shm: + # ... but the user requested to create a new one + if shmflg & (IPC_CREAT | IPC_EXCL): + return -1 # EEXIST + + shmid = ql.os.shm[key] + + # check whether the user has permissions to access this shm + # FIXME: should probably use ql.os.cuid instead, but we don't support it yet + if (ql.os.uid == shmid.uid) and (shmid.mode & (SHM_W | SHM_R)): + return key + + else: + return -1 # EACCES + + # a shm with the specified key does not exist + else: + if shmflg & IPC_CREAT: + key = __create_shm(size, shmflg) + + else: + return -1 # ENOENT + + return key + + +def ql_syscall_shmat(ql: Qiling, shmid: int, shmaddr: int, shmflg: int): + if shmid not in ql.os.shm: + return -1 # EINVAL + + if shmaddr == 0: + # system may choose any suitable page-aligned address, so just use the key + addr = shmid + + elif shmflg & SHM_RND: + # note: should align to SHMLBA, but usually its value is just a page + addr = ql.mem.align(shmaddr) + + else: + if shmaddr & (ql.mem.pagesize - 1): + return -1 # EINVAL + + addr = shmaddr + + return addr + + +__all__ = [ + 'ql_syscall_shmget', + 'ql_syscall_shmat' +] From e76b8ab95e1c66178dd5cc3d8c2d1cffb61a0233 Mon Sep 17 00:00:00 2001 From: elicn Date: Sun, 2 Apr 2023 20:50:38 +0300 Subject: [PATCH 20/42] Partialy implemented POSIX IPC syscall --- qiling/os/posix/const.py | 14 ++++++++++ qiling/os/posix/syscall/__init__.py | 1 + qiling/os/posix/syscall/syscall.py | 41 +++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+) create mode 100644 qiling/os/posix/syscall/syscall.py diff --git a/qiling/os/posix/const.py b/qiling/os/posix/const.py index 13508058a..103559f02 100644 --- a/qiling/os/posix/const.py +++ b/qiling/os/posix/const.py @@ -1024,3 +1024,17 @@ class qnx_mmap_flags(Flag): # see: https://elixir.bootlin.com/linux/v5.19.17/source/include/uapi/asm-generic/hugetlb_encode.h HUGETLB_FLAG_ENCODE_SHIFT = 26 HUGETLB_FLAG_ENCODE_MASK = 0x3f + +# ipc syscall +SEMOP = 1 +SEMGET = 2 +SEMCTL = 3 +SEMTIMEDOP = 4 +MSGSND = 11 +MSGRCV = 12 +MSGGET = 13 +MSGCTL = 14 +SHMAT = 21 +SHMDT = 22 +SHMGET = 23 +SHMCTL = 24 diff --git a/qiling/os/posix/syscall/__init__.py b/qiling/os/posix/syscall/__init__.py index 3477a9479..a8c25d18f 100644 --- a/qiling/os/posix/syscall/__init__.py +++ b/qiling/os/posix/syscall/__init__.py @@ -21,6 +21,7 @@ from .socket import * from .stat import * from .sysctl import * +from .syscall import * from .sysinfo import * from .time import * from .types import * diff --git a/qiling/os/posix/syscall/syscall.py b/qiling/os/posix/syscall/syscall.py new file mode 100644 index 000000000..53b3dc0e9 --- /dev/null +++ b/qiling/os/posix/syscall/syscall.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +# +# Cross Platform and Multi Architecture Advanced Binary Emulation Framework +# + +from qiling import Qiling +from qiling.os.posix.const import * + +from .shm import * + + +def ql_syscall_ipc(ql: Qiling, call: int, first: int, second: int, third: int, ptr: int, fifth: int): + version = call >> 16 # hi word + call &= 0xffff # lo word + + # FIXME: this is an incomplete implementation. + # see: https://elixir.bootlin.com/linux/v5.19.17/source/ipc/syscall.c + + def __call_shmat(*args: int) -> int: + if version == 1: + return -1 # EINVAL + + return ql_syscall_shmget(ql, args[0], args[3], args[1]) + + def __call_shmget(*args: int) -> int: + return ql_syscall_shmget(ql, args[0], args[1], args[2]) + + ipc_call = { + SHMAT: __call_shmat, + SHMGET: __call_shmget + } + + if call not in ipc_call: + return -1 # ENOSYS + + return ipc_call[call](first, second, third, ptr, fifth) + + +__all__ = [ + 'ql_syscall_ipc' +] From 2347a9b9db2e0e7fe2d31825da837889545dd991 Mon Sep 17 00:00:00 2001 From: elicn Date: Mon, 3 Apr 2023 16:38:57 +0300 Subject: [PATCH 21/42] Adjust ELF shellcode tests --- tests/test_shellcode.py | 119 +++++++++++++++++++++++++++------------- 1 file changed, 82 insertions(+), 37 deletions(-) diff --git a/tests/test_shellcode.py b/tests/test_shellcode.py index 8d01e4e59..95b414ca5 100644 --- a/tests/test_shellcode.py +++ b/tests/test_shellcode.py @@ -1,97 +1,142 @@ #!/usr/bin/env python3 -# +# # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # -import sys, unittest -from binascii import unhexlify +import sys +import unittest sys.path.append("..") -from qiling import * -from qiling.exception import * +from qiling import Qiling from qiling.const import QL_VERBOSE -test = unhexlify('cccc') -X86_LIN = unhexlify('31c050682f2f7368682f62696e89e3505389e1b00bcd80') -X8664_LIN = unhexlify('31c048bbd19d9691d08c97ff48f7db53545f995257545eb03b0f05') -MIPS32EL_LIN = unhexlify('ffff0628ffffd004ffff05280110e4270ff08424ab0f02240c0101012f62696e2f7368') -X86_WIN = unhexlify('fce8820000006089e531c0648b50308b520c8b52148b72280fb74a2631ffac3c617c022c20c1cf0d01c7e2f252578b52108b4a3c8b4c1178e34801d1518b592001d38b4918e33a498b348b01d631ffacc1cf0d01c738e075f6037df83b7d2475e4588b582401d3668b0c4b8b581c01d38b048b01d0894424245b5b61595a51ffe05f5f5a8b12eb8d5d6a01eb2668318b6f87ffd5bbf0b5a25668a695bd9dffd53c067c0a80fbe07505bb4713726f6a0053ffd5e8d5ffffff63616c6300') -X8664_WIN = unhexlify('fc4881e4f0ffffffe8d0000000415141505251564831d265488b52603e488b52183e488b52203e488b72503e480fb74a4a4d31c94831c0ac3c617c022c2041c1c90d4101c1e2ed5241513e488b52203e8b423c4801d03e8b80880000004885c0746f4801d0503e8b48183e448b40204901d0e35c48ffc93e418b34884801d64d31c94831c0ac41c1c90d4101c138e075f13e4c034c24084539d175d6583e448b40244901d0663e418b0c483e448b401c4901d03e418b04884801d0415841585e595a41584159415a4883ec204152ffe05841595a3e488b12e949ffffff5d49c7c1000000003e488d95fe0000003e4c8d850f0100004831c941ba45835607ffd54831c941baf0b5a256ffd548656c6c6f2c2066726f6d204d534621004d657373616765426f7800') -ARM_LIN = unhexlify('01308fe213ff2fe178460e300190491a921a0827c251033701df2f62696e2f2f7368') -ARM_THUMB = unhexlify('401c01464fea011200bf') -ARM64_LIN = unhexlify('420002ca210080d2400080d2c81880d2010000d4e60300aa01020010020280d2681980d2010000d4410080d2420002cae00306aa080380d2010000d4210400f165ffff54e0000010420002ca210001caa81b80d2010000d4020004d27f0000012f62696e2f736800') -X8664_FBSD = unhexlify('6a61586a025f6a015e990f054897baff02aaaa80f2ff524889e699046680c2100f05046a0f05041e4831f6990f0548976a035852488d7424f080c2100f0548b8523243427730637257488d3e48af74084831c048ffc00f055f4889d04889fe48ffceb05a0f0575f799043b48bb2f62696e2f2f73685253545f5257545e0f05') -X8664_macos = unhexlify('4831f65648bf2f2f62696e2f7368574889e74831d24831c0b00248c1c828b03b0f05') + +# test = bytes.fromhex('cccc') + +X86_LIN = bytes.fromhex('31c050682f2f7368682f62696e89e3505389e1b00bcd80') +X8664_LIN = bytes.fromhex('31c048bbd19d9691d08c97ff48f7db53545f995257545eb03b0f05') + +MIPS32EL_LIN = bytes.fromhex(''' + ffff0628ffffd004ffff05280110e4270ff08424ab0f02240c0101012f62696e + 2f7368 +''') + +X86_WIN = bytes.fromhex(''' + fce8820000006089e531c0648b50308b520c8b52148b72280fb74a2631ffac3c + 617c022c20c1cf0d01c7e2f252578b52108b4a3c8b4c1178e34801d1518b5920 + 01d38b4918e33a498b348b01d631ffacc1cf0d01c738e075f6037df83b7d2475 + e4588b582401d3668b0c4b8b581c01d38b048b01d0894424245b5b61595a51ff + e05f5f5a8b12eb8d5d6a01eb2668318b6f87ffd5bbf0b5a25668a695bd9dffd5 + 3c067c0a80fbe07505bb4713726f6a0053ffd5e8d5ffffff63616c6300 +''') + +X8664_WIN = bytes.fromhex(''' + fc4881e4f0ffffffe8d0000000415141505251564831d265488b52603e488b52 + 183e488b52203e488b72503e480fb74a4a4d31c94831c0ac3c617c022c2041c1 + c90d4101c1e2ed5241513e488b52203e8b423c4801d03e8b80880000004885c0 + 746f4801d0503e8b48183e448b40204901d0e35c48ffc93e418b34884801d64d + 31c94831c0ac41c1c90d4101c138e075f13e4c034c24084539d175d6583e448b + 40244901d0663e418b0c483e448b401c4901d03e418b04884801d0415841585e + 595a41584159415a4883ec204152ffe05841595a3e488b12e949ffffff5d49c7 + c1000000003e488d95fe0000003e4c8d850f0100004831c941ba45835607ffd5 + 4831c941baf0b5a256ffd548656c6c6f2c2066726f6d204d534621004d657373 + 616765426f7800 +''') + +ARM_LIN = bytes.fromhex(''' + 01308fe213ff2fe178460e300190491a921a0827c251033701df2f62696e2f2f + 7368 +''') + +ARM_THUMB = bytes.fromhex('401c01464fea011200bf') + +ARM64_LIN = bytes.fromhex(''' + 420002ca210080d2400080d2c81880d2010000d4e60300aa01020010020280d2 + 681980d2010000d4410080d2420002cae00306aa080380d2010000d4210400f1 + 65ffff54e0000010420002ca210001caa81b80d2010000d4020004d27f000001 + 2f62696e2f736800 +''') + +X8664_FBSD = bytes.fromhex(''' + 6a61586a025f6a015e990f054897baff02aaaa80f2ff524889e699046680c210 + 0f05046a0f05041e4831f6990f0548976a035852488d7424f080c2100f0548b8 + 523243427730637257488d3e48af74084831c048ffc00f055f4889d04889fe48 + ffceb05a0f0575f799043b48bb2f62696e2f2f73685253545f5257545e0f05 +''') + +X8664_MACOS = bytes.fromhex(''' + 4831f65648bf2f2f62696e2f7368574889e74831d24831c0b00248c1c828b03b + 0f05 +''') + class TestShellcode(unittest.TestCase): def test_linux_x86(self): print("Linux X86 32bit Shellcode") - ql = Qiling(code = X86_LIN, archtype = "x86", ostype = "linux", verbose=QL_VERBOSE.OFF) + ql = Qiling(code=X86_LIN, archtype="x86", ostype="linux", verbose=QL_VERBOSE.OFF) ql.run() def test_linux_x64(self): print("Linux X86 64bit Shellcode") - ql = Qiling(code = X8664_LIN, archtype = "x8664", ostype = "linux", verbose=QL_VERBOSE.OFF) + ql = Qiling(code=X8664_LIN, archtype="x8664", ostype="linux", verbose=QL_VERBOSE.OFF) ql.run() def test_linux_mips32(self): print("Linux MIPS 32bit EL Shellcode") - ql = Qiling(code = MIPS32EL_LIN, archtype = "mips", ostype = "linux", verbose=QL_VERBOSE.OFF) + ql = Qiling(code=MIPS32EL_LIN, archtype="mips", ostype="linux", verbose=QL_VERBOSE.OFF) ql.run() - #This shellcode needs to be changed to something non-blocking + # This shellcode needs to be changed to something non-blocking def test_linux_arm(self): - print("Linux ARM 32bit Shellcode") - ql = Qiling(code = ARM_LIN, archtype = "arm", ostype = "linux", verbose=QL_VERBOSE.OFF) - ql.run() - + print("Linux ARM 32bit Shellcode") + ql = Qiling(code=ARM_LIN, archtype="arm", ostype="linux", verbose=QL_VERBOSE.OFF) + ql.run() def test_linux_arm_thumb(self): print("Linux ARM Thumb Shllcode") - ql = Qiling(code = ARM_THUMB, archtype = "arm", ostype = "linux", verbose=QL_VERBOSE.OFF, thumb = True) + ql = Qiling(code=ARM_THUMB, archtype="arm", ostype="linux", verbose=QL_VERBOSE.OFF, thumb=True) ql.run() - def test_linux_arm64(self): print("Linux ARM 64bit Shellcode") - ql = Qiling(code = ARM64_LIN, archtype = "arm64", ostype = "linux", verbose=QL_VERBOSE.OFF) + ql = Qiling(code=ARM64_LIN, archtype="arm64", ostype="linux", verbose=QL_VERBOSE.OFF) ql.run() # #This shellcode needs to be changed to something simpler not requiring rootfs # def test_windows_x86(self): # print("Windows X86 32bit Shellcode") - # ql = Qiling(code = X86_WIN, archtype = "x86", ostype = "windows", rootfs="../examples/rootfs/x86_reactos", verbose=QL_VERBOSE.OFF) + # ql = Qiling(code=X86_WIN, archtype="x86", ostype="windows", rootfs="../examples/rootfs/x86_reactos", verbose=QL_VERBOSE.OFF) # ql.run() # #This shellcode needs to be changed to something simpler not requiring rootfs # def test_windows_x64(self): # print("\nWindows X8664 64bit Shellcode") - # ql = Qiling(code = X8664_WIN, archtype = "x8664", ostype = "windows", rootfs="../examples/rootfs/x86_reactos", verbose=QL_VERBOSE.OFF) + # ql = Qiling(code=X8664_WIN, archtype="x8664", ostype="windows", rootfs="../examples/rootfs/x86_reactos", verbose=QL_VERBOSE.OFF) # ql.run() - #This shellcode needs to be changed to something simpler, listen is blocking - #def test_freebsd_x64(self): + # #This shellcode needs to be changed to something simpler, listen is blocking + # def test_freebsd_x64(self): # print("FreeBSD X86 64bit Shellcode") - # ql = Qiling(code = X8664_FBSD, archtype = "x8664", ostype = "freebsd", verbose=QL_VERBOSE.OFF) + # ql = Qiling(code=X8664_FBSD, archtype="x8664", ostype="freebsd", verbose=QL_VERBOSE.OFF) # ql.run() # def test_macos_x64(self): # print("macos X86 64bit Shellcode") - # ql = Qiling(code = X8664_macos, archtype = "x8664", ostype = "macos", verbose=QL_VERBOSE.OFF) + # ql = Qiling(code=X8664_macos, archtype="x8664", ostype="macos", verbose=QL_VERBOSE.OFF) # ql.run() # def test_invalid_os(self): # print("Testing Unknown OS") - # self.assertRaises(QlErrorOsType, Qiling, code = test, archtype = "arm64", ostype = "qilingos", verbose=QL_VERBOSE.DEFAULT ) + # self.assertRaises(QlErrorOsType, Qiling, code=test, archtype="arm64", ostype="qilingos", verbose=QL_VERBOSE.DEFAULT ) # def test_invalid_arch(self): # print("Testing Unknown Arch") - # self.assertRaises(QlErrorArch, Qiling, code = test, archtype = "qilingarch", ostype = "linux", verbose=QL_VERBOSE.DEFAULT ) + # self.assertRaises(QlErrorArch, Qiling, code=test, archtype="qilingarch", ostype="linux", verbose=QL_VERBOSE.DEFAULT ) # def test_invalid_output(self): # print("Testing Invalid output") - # self.assertRaises(QlErrorOutput, Qiling, code = test, archtype = "arm64", ostype = "linux", verbose=QL_VERBOSE.DEFAULT ) - + # self.assertRaises(QlErrorOutput, Qiling, code=test, archtype="arm64", ostype="linux", verbose=QL_VERBOSE.DEFAULT ) + if __name__ == "__main__": unittest.main() From 32666805cf0199a8e6725d50201684dc7cebc04f Mon Sep 17 00:00:00 2001 From: elicn Date: Mon, 3 Apr 2023 16:39:42 +0300 Subject: [PATCH 22/42] Allow shellcode execve fail gracefully --- tests/test_shellcode.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/tests/test_shellcode.py b/tests/test_shellcode.py index 95b414ca5..3aa0adb9c 100644 --- a/tests/test_shellcode.py +++ b/tests/test_shellcode.py @@ -8,7 +8,7 @@ sys.path.append("..") from qiling import Qiling -from qiling.const import QL_VERBOSE +from qiling.const import QL_INTERCEPT, QL_VERBOSE # test = bytes.fromhex('cccc') @@ -70,6 +70,22 @@ ''') +# some shellcodes call execve, which under normal circumstences, does not return. +# however, those shellcodes attempt to run a non-exsting '/bin/sh' binary and do +# not bother to handle failures gracefully. +# +# the execution then continues to the next bytes, which are usually the '/bin/sh' +# string and not valid code. that causes Qiling to raise an exception, and this is +# why we need a way to thwart those execve failures and end the emulation gracefully +def graceful_execve(ql: Qiling, pathname: int, argv: int, envp: int, retval: int): + assert retval != 0, f'execve is not expected to return on success' + + vpath = ql.os.utils.read_cstring(pathname) + + ql.log.debug(f'failed to call execve("{vpath}"), ending emulation gracefully') + ql.stop() + + class TestShellcode(unittest.TestCase): def test_linux_x86(self): print("Linux X86 32bit Shellcode") @@ -84,6 +100,8 @@ def test_linux_x64(self): def test_linux_mips32(self): print("Linux MIPS 32bit EL Shellcode") ql = Qiling(code=MIPS32EL_LIN, archtype="mips", ostype="linux", verbose=QL_VERBOSE.OFF) + + ql.os.set_syscall('execve', graceful_execve, QL_INTERCEPT.EXIT) ql.run() # This shellcode needs to be changed to something non-blocking @@ -100,6 +118,8 @@ def test_linux_arm_thumb(self): def test_linux_arm64(self): print("Linux ARM 64bit Shellcode") ql = Qiling(code=ARM64_LIN, archtype="arm64", ostype="linux", verbose=QL_VERBOSE.OFF) + + ql.os.set_syscall('execve', graceful_execve, QL_INTERCEPT.EXIT) ql.run() # #This shellcode needs to be changed to something simpler not requiring rootfs From d9ff19d113a749747bc7ec38cf32988121b82d61 Mon Sep 17 00:00:00 2001 From: elicn Date: Mon, 3 Apr 2023 16:40:09 +0300 Subject: [PATCH 23/42] Patch POSIX execve --- qiling/os/posix/syscall/unistd.py | 40 +++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/qiling/os/posix/syscall/unistd.py b/qiling/os/posix/syscall/unistd.py index 6f413cf3d..901199b07 100644 --- a/qiling/os/posix/syscall/unistd.py +++ b/qiling/os/posix/syscall/unistd.py @@ -549,10 +549,18 @@ def ql_syscall_setsid(ql: Qiling): def ql_syscall_execve(ql: Qiling, pathname: int, argv: int, envp: int): - file_path = ql.os.utils.read_cstring(pathname) - real_path = ql.os.path.transform_to_real_path(file_path) + vpath = ql.os.utils.read_cstring(pathname) + hpath = ql.os.path.virtual_to_host_path(vpath) - def __read_str_array(addr: int) -> Iterator[str]: + # is it safe to run? + if not ql.os.path.is_safe_host_path(hpath): + return -1 # EACCES + + # is it a file? does it exist? + if not os.path.isfile(hpath): + return -1 # EACCES + + def __read_ptr_array(addr: int) -> Iterator[int]: if addr: while True: elem = ql.mem.read_ptr(addr) @@ -560,25 +568,28 @@ def __read_str_array(addr: int) -> Iterator[str]: if elem == 0: break - yield ql.os.utils.read_cstring(elem) + yield elem addr += ql.arch.pointersize - args = [s for s in __read_str_array(argv)] + def __read_str_array(addr: int) -> Iterator[str]: + yield from (ql.os.utils.read_cstring(ptr) for ptr in __read_ptr_array(addr)) + + args = list(__read_str_array(argv)) env = {} for s in __read_str_array(envp): k, _, v = s.partition('=') env[k] = v - ql.emu_stop() + ql.stop() + ql.clear_ql_hooks() + ql.mem.unmap_all() - ql.log.debug(f'execve({file_path}, [{", ".join(args)}], [{", ".join(f"{k}={v}" for k, v in env.items())}])') + ql.log.debug(f'execve("{vpath}", [{", ".join(args)}], [{", ".join(f"{k}={v}" for k, v in env.items())}])') ql.loader.argv = args ql.loader.env = env - ql._argv = [real_path] + args - ql.mem.map_info = [] - ql.clear_ql_hooks() + ql._argv = [hpath] + args # Clean debugger to prevent port conflicts # ql.debugger = None @@ -599,6 +610,15 @@ def __read_str_array(addr: int) -> Iterator[str]: QlCoreHooks.__init__(ql, uc) ql.os.load() + + # close all open fd marked with 'close_on_exec' + for i in range(NR_OPEN): + f = ql.os.fd[i] + + if f and f.close_on_exec and not f.closed: + f.close() + ql.os.fd[i] = None + ql.loader.run() ql.run() From b3de208edb3888ee8a45462fa0a9df2bc9302f90 Mon Sep 17 00:00:00 2001 From: elicn Date: Mon, 3 Apr 2023 16:43:01 +0300 Subject: [PATCH 24/42] Prevent emulation from closing host std streams --- qiling/os/filestruct.py | 16 ++++++++++++++++ qiling/os/os.py | 25 +++++++++++++++++-------- 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/qiling/os/filestruct.py b/qiling/os/filestruct.py index 6c93ed825..01858c2f6 100644 --- a/qiling/os/filestruct.py +++ b/qiling/os/filestruct.py @@ -94,3 +94,19 @@ def name(self): @property def closed(self) -> bool: return self.__closed + + +class PersistentQlFile(ql_file): + """A persistent variation of the ql_file class, which silently drops + attempts to close its udnerlying file. This is useful when using host + environment resources, which should not be closed when their wrapping + ql_file gets closed. + + For example, stdout and stderr might be closed by the emulated program + by calling POSIX dup2 or dup3 system calls, and then replaced by another + file or socket. this class prevents the emulated program from closing + shared resources on the hosting system. + """ + + def close(self): + pass diff --git a/qiling/os/os.py b/qiling/os/os.py index 19553ff2a..6d759bf59 100644 --- a/qiling/os/os.py +++ b/qiling/os/os.py @@ -4,6 +4,7 @@ # import sys +from io import UnsupportedOperation from typing import Any, Hashable, Iterable, Optional, Callable, Mapping, Sequence, TextIO, Tuple from unicorn import UcError @@ -13,7 +14,7 @@ from qiling.os.const import STRING, WSTRING, GUID from qiling.os.fcall import QlFunctionCall, TypedArg -from .filestruct import ql_file +from .filestruct import PersistentQlFile from .mapper import QlFsMapper from .stats import QlOsStats from .utils import QlOsUtils @@ -53,17 +54,25 @@ def __init__(self, ql: Qiling, resolvers: Mapping[Any, Resolver] = {}): QL_INTERCEPT.EXIT: {} } - # IDAPython has some hack on standard io streams and thus they don't have corresponding fds. try: - import ida_idaapi - except ImportError: - self._stdin = ql_file('stdin', sys.stdin.fileno()) - self._stdout = ql_file('stdout', sys.stdout.fileno()) - self._stderr = ql_file('stderr', sys.stderr.fileno()) - else: + # Qiling may be used on interactive shells (ex: IDLE) or embedded python + # interpreters (ex: IDA Python). such environments use their own version + # for the standard streams which usually do not support certain operations, + # such as fileno(). here we use this to determine how we are going to use + # the environment standard streams + sys.stdin.fileno() + except UnsupportedOperation: + # Qiling is used on an interactive shell or embedded python interpreter. + # if the internal stream buffer is accessible, we should use it self._stdin = getattr(sys.stdin, 'buffer', sys.stdin) self._stdout = getattr(sys.stdout, 'buffer', sys.stdout) self._stderr = getattr(sys.stderr, 'buffer', sys.stderr) + else: + # Qiling is used in a script, or on an environment that supports ordinary + # stanard streams + self._stdin = PersistentQlFile('stdin', sys.stdin.fileno()) + self._stdout = PersistentQlFile('stdout', sys.stdout.fileno()) + self._stderr = PersistentQlFile('stderr', sys.stderr.fileno()) # defult exit point self.exit_point = { From 32a8588ff69bc5c8321c6967b39d0013114cd7ad Mon Sep 17 00:00:00 2001 From: elicn Date: Mon, 3 Apr 2023 16:44:52 +0300 Subject: [PATCH 25/42] Insignificant styling and typo fixes --- qiling/os/filestruct.py | 13 +++++++------ qiling/os/linux/linux.py | 2 ++ qiling/os/os.py | 6 +++--- qiling/os/posix/syscall/socket.py | 2 +- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/qiling/os/filestruct.py b/qiling/os/filestruct.py index 01858c2f6..c57bbc54e 100644 --- a/qiling/os/filestruct.py +++ b/qiling/os/filestruct.py @@ -1,10 +1,10 @@ #!/usr/bin/env python3 -# +# # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # import os -from typing import AnyStr +from typing import AnyStr, Optional from qiling.exception import * from qiling.os.posix.stat import * @@ -14,6 +14,7 @@ except ImportError: pass + class ql_file: def __init__(self, path: AnyStr, fd: int): self.__path = path @@ -26,15 +27,15 @@ def __init__(self, path: AnyStr, fd: int): self.close_on_exec = False @classmethod - def open(cls, open_path: AnyStr, open_flags: int, open_mode: int, dir_fd: int = None): - open_mode &= 0x7fffffff + def open(cls, path: AnyStr, flags: int, mode: int, dir_fd: Optional[int] = None): + mode &= 0x7fffffff try: - fd = os.open(open_path, open_flags, open_mode, dir_fd=dir_fd) + fd = os.open(path, flags, mode, dir_fd=dir_fd) except OSError as e: raise QlSyscallError(e.errno, e.args[1] + ' : ' + e.filename) - return cls(open_path, fd) + return cls(path, fd) def read(self, read_len: int) -> bytes: return os.read(self.__fd, read_len) diff --git a/qiling/os/linux/linux.py b/qiling/os/linux/linux.py index 7b706077a..0313218d6 100644 --- a/qiling/os/linux/linux.py +++ b/qiling/os/linux/linux.py @@ -167,12 +167,14 @@ def run(self): if self.ql.entry_point is not None: self.ql.loader.elf_entry = self.ql.entry_point + # do we have an interp? elif self.ql.loader.elf_entry != self.ql.loader.entry_point: entry_address = self.ql.loader.elf_entry if self.ql.arch.type == QL_ARCH.ARM: entry_address &= ~1 + # start running interp, but stop when elf entry point is reached self.ql.emu_start(self.ql.loader.entry_point, entry_address, self.ql.timeout) self.ql.do_lib_patch() self.run_function_after_load() diff --git a/qiling/os/os.py b/qiling/os/os.py index 6d759bf59..7982fe946 100644 --- a/qiling/os/os.py +++ b/qiling/os/os.py @@ -5,7 +5,7 @@ import sys from io import UnsupportedOperation -from typing import Any, Hashable, Iterable, Optional, Callable, Mapping, Sequence, TextIO, Tuple +from typing import Any, Dict, Iterable, Optional, Callable, Mapping, Sequence, TextIO, Tuple, Union from unicorn import UcError @@ -48,7 +48,7 @@ def __init__(self, ql: Qiling, resolvers: Mapping[Any, Resolver] = {}): self.path = QlOsPath(ql.rootfs, cwd, self.type) self.fs_mapper = QlFsMapper(self.path) - self.user_defined_api = { + self.user_defined_api: Dict[QL_INTERCEPT, Dict[Union[int, str], Callable]] = { QL_INTERCEPT.CALL: {}, QL_INTERCEPT.ENTER: {}, QL_INTERCEPT.EXIT: {} @@ -216,7 +216,7 @@ def call(self, pc: int, func: Callable, proto: Mapping[str, Any], onenter: Optio return retval - def set_api(self, target: Hashable, handler: Callable, intercept: QL_INTERCEPT = QL_INTERCEPT.CALL): + def set_api(self, target: Union[int, str], handler: Callable, intercept: QL_INTERCEPT = QL_INTERCEPT.CALL): """Either hook or replace an OS API with a custom one. Args: diff --git a/qiling/os/posix/syscall/socket.py b/qiling/os/posix/syscall/socket.py index 4fd56dafb..01068aacc 100644 --- a/qiling/os/posix/syscall/socket.py +++ b/qiling/os/posix/syscall/socket.py @@ -266,7 +266,7 @@ def ql_syscall_connect(ql: Qiling, sockfd: int, addr: int, addrlen: int): port = ntohs(ql, sockaddr_obj.sin_port) host = inet6_htoa(ql, sockaddr_obj.sin6_addr.s6_addr) - ql.log.debug(f'Conecting to {host}:{port}') + ql.log.debug(f'Connecting to {host}:{port}') dest = (host, port) if dest is not None: From 48cc58d0953925896934b4ed54dd7118e4564006 Mon Sep 17 00:00:00 2001 From: elicn Date: Tue, 4 Apr 2023 18:58:33 +0300 Subject: [PATCH 26/42] Properly set emu_state --- qiling/core.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/qiling/core.py b/qiling/core.py index 70c2ecad4..536ddbd54 100644 --- a/qiling/core.py +++ b/qiling/core.py @@ -757,14 +757,16 @@ def emu_start(self, begin: int, end: int, timeout: int = 0, count: int = 0): if getattr(self.arch, '_init_thumb', False): begin |= 0b1 - self._state = QL_STATE.STARTED - # reset exception status before emulation starts self._internal_exception = None + self._state = QL_STATE.STARTED + # effectively start the emulation. this returns only after uc.emu_stop is called self.uc.emu_start(begin, end, timeout, count) + self._state = QL_STATE.STOPPED + # if an exception was raised during emulation, propagate it up if self.internal_exception is not None: raise self.internal_exception From c069a102de642e152e630c6b3db56c57b756e31f Mon Sep 17 00:00:00 2001 From: elicn Date: Tue, 4 Apr 2023 18:58:59 +0300 Subject: [PATCH 27/42] Collect new vcruntime140 DLLs --- examples/scripts/dllscollector.bat | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/scripts/dllscollector.bat b/examples/scripts/dllscollector.bat index 048d5f9c3..cd115093e 100644 --- a/examples/scripts/dllscollector.bat +++ b/examples/scripts/dllscollector.bat @@ -118,6 +118,8 @@ CALL :collect_dll64 shlwapi.dll CALL :collect_dll64 user32.dll CALL :collect_dll64 vcruntime140.dll CALL :collect_dll64 vcruntime140d.dll +CALL :collect_dll64 vcruntime140_1.dll +CALL :collect_dll64 vcruntime140_1d.dll CALL :collect_dll64 win32u.dll CALL :collect_dll64 winhttp.dll CALL :collect_dll64 wininet.dll From 412bcafde16be9a6e95b09817cda6000080f6efe Mon Sep 17 00:00:00 2001 From: elicn Date: Tue, 4 Apr 2023 19:15:58 +0300 Subject: [PATCH 28/42] Use yaml safe loader --- qiling/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiling/utils.py b/qiling/utils.py index 14528b689..ccdd970ee 100644 --- a/qiling/utils.py +++ b/qiling/utils.py @@ -402,7 +402,7 @@ def profile_setup(ostype: QL_OS, user_config: Optional[Union[str, dict]]): if user_config: with open(user_config) as f: - config = yaml.load(f, Loader=yaml.Loader) + config = yaml.load(f, Loader=yaml.SafeLoader) else: config = {} From 85430f08704a6b27d86509f3f07067fbf55986ac Mon Sep 17 00:00:00 2001 From: elicn Date: Fri, 7 Apr 2023 16:35:41 +0300 Subject: [PATCH 29/42] Use mmap min address for shm allocations --- qiling/os/posix/syscall/shm.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/qiling/os/posix/syscall/shm.py b/qiling/os/posix/syscall/shm.py index 926f72e49..4d2f5e162 100644 --- a/qiling/os/posix/syscall/shm.py +++ b/qiling/os/posix/syscall/shm.py @@ -31,20 +31,25 @@ def __create_shm(size: int, flags: int) -> int: else: pagesize = ql.mem.pagesize - size = ql.mem.align_up(size, pagesize) - if len(ql.os.shm) < SHMMNI: + shm_size = ql.mem.align_up(size, pagesize) + try: - key = ql.mem.map_anywhere(size, perms=uc_perms, info='[shm]') + shm_addr = ql.mem.find_free_space(shm_size, ql.loader.mmap_address, align=pagesize) except QlOutOfMemory: return -1 # ENOMEM + else: + ql.mem.map(shm_addr, shm_size, uc_perms, '[shm]') + + # for simplicity, the shm key is defined to be its base address + shm_key = shm_addr - ql.os.shm[key] = QlShmId(size, ql.os.uid, ql.os.gid, perms) + ql.os.shm[shm_key] = QlShmId(shm_size, ql.os.uid, ql.os.gid, perms) else: return -1 # ENOSPC - return key + return shm_key # create new shared memory segment if key == IPC_PRIVATE: @@ -52,7 +57,7 @@ def __create_shm(size: int, flags: int) -> int: # a shm with the specified key exists elif key in ql.os.shm: - # ... but the user requested to create a new one + # user asked to create a new one? if shmflg & (IPC_CREAT | IPC_EXCL): return -1 # EEXIST @@ -68,6 +73,7 @@ def __create_shm(size: int, flags: int) -> int: # a shm with the specified key does not exist else: + # user asked to create a new one? if shmflg & IPC_CREAT: key = __create_shm(size, shmflg) @@ -82,7 +88,9 @@ def ql_syscall_shmat(ql: Qiling, shmid: int, shmaddr: int, shmflg: int): return -1 # EINVAL if shmaddr == 0: - # system may choose any suitable page-aligned address, so just use the key + # system may choose any suitable page-aligned address. since existing segments are + # guaranteed to be aligned and key is defined to be shm base address, we can just + # use the key addr = shmid elif shmflg & SHM_RND: From 06cd3f09a980c41054f0e7075edea5ba1f7fe2d4 Mon Sep 17 00:00:00 2001 From: elicn Date: Fri, 7 Apr 2023 16:58:47 +0300 Subject: [PATCH 30/42] Typo bugfix --- qiling/os/posix/syscall/syscall.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiling/os/posix/syscall/syscall.py b/qiling/os/posix/syscall/syscall.py index 53b3dc0e9..3fe7cbe33 100644 --- a/qiling/os/posix/syscall/syscall.py +++ b/qiling/os/posix/syscall/syscall.py @@ -20,7 +20,7 @@ def __call_shmat(*args: int) -> int: if version == 1: return -1 # EINVAL - return ql_syscall_shmget(ql, args[0], args[3], args[1]) + return ql_syscall_shmat(ql, args[0], args[3], args[1]) def __call_shmget(*args: int) -> int: return ql_syscall_shmget(ql, args[0], args[1], args[2]) From 0dd8545e25c8c70a4c0850e8b81bdabdccb22d20 Mon Sep 17 00:00:00 2001 From: elicn Date: Thu, 13 Apr 2023 22:04:55 +0300 Subject: [PATCH 31/42] Re-implement POSIX shm --- qiling/os/posix/posix.py | 55 +++++++++--- qiling/os/posix/syscall/shm.py | 147 ++++++++++++++++++++------------- 2 files changed, 135 insertions(+), 67 deletions(-) diff --git a/qiling/os/posix/posix.py b/qiling/os/posix/posix.py index 1d639ce13..e7e609b98 100644 --- a/qiling/os/posix/posix.py +++ b/qiling/os/posix/posix.py @@ -4,7 +4,7 @@ # from inspect import signature, Parameter -from typing import Dict, NamedTuple, TextIO, Union, Callable, IO, List, Optional +from typing import Dict, TextIO, Tuple, Union, Callable, IO, List, Optional from unicorn.arm64_const import UC_ARM64_REG_X8, UC_ARM64_REG_X16 from unicorn.arm_const import ( @@ -91,14 +91,49 @@ def restore(self, fds): self.__fds = fds -# vaguely reflects a shmid_ds structure -class QlShmId(NamedTuple): - segsz: int - uid: int - gid: int - # cuid: int - # cgid: int - mode: int +# vaguely reflects a shmid64_ds structure +class QlShmId: + + def __init__(self, key: int, uid: int, gid: int, mode: int, segsz: int) -> None: + # ipc64_perm + self.key = key + self.uid = uid + self.gid = gid + self.mode = mode + + self.segsz = segsz + + # track the memory locations this segment is currently attached to + self.attach: List[int] = [] + + +class QlShm: + def __init__(self) -> None: + self.__shm: Dict[int, QlShmId] = {} + self.__id: int = 0x0F000000 + + def __len__(self) -> int: + return len(self.__shm) + + def add(self, shm: QlShmId) -> int: + shmid = self.__id + self.__shm[shmid] = shm + + self.__id += 0x1000 + + return shmid + + def remove(self, shmid: int) -> None: + del self.__shm[shmid] + + def get_by_key(self, key: int) -> Tuple[int, Optional[QlShmId]]: + return next(((shmid, shmobj) for shmid, shmobj in self.__shm.items() if shmobj.key == key), (-1, None)) + + def get_by_id(self, shmid: int) -> Optional[QlShmId]: + return self.__shm.get(shmid, None) + + def get_by_attaddr(self, shmaddr: int) -> Optional[QlShmId]: + return next((shmobj for shmobj in self.__shm.values() if shmobj.attach.count(shmaddr) > 0), None) class QlOsPosix(QlOs): @@ -167,7 +202,7 @@ def __init__(self, ql: Qiling): self.stdout = self._stdout self.stderr = self._stderr - self._shm: Dict[int, QlShmId] = {} + self._shm = QlShm() def __get_syscall_mapper(self, archtype: QL_ARCH): qlos_path = f'.os.{self.type.name.lower()}.map_syscall' diff --git a/qiling/os/posix/syscall/shm.py b/qiling/os/posix/syscall/shm.py index 4d2f5e162..760094427 100644 --- a/qiling/os/posix/syscall/shm.py +++ b/qiling/os/posix/syscall/shm.py @@ -3,107 +3,140 @@ # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # -from unicorn.unicorn_const import UC_PROT_WRITE, UC_PROT_READ +from unicorn.unicorn_const import UC_PROT_READ, UC_PROT_WRITE, UC_PROT_EXEC from qiling import Qiling -from qiling.exception import QlOutOfMemory +from qiling.const import QL_ARCH +from qiling.exception import QlMemoryMappedError from qiling.os.posix.const import * from qiling.os.posix.posix import QlShmId def ql_syscall_shmget(ql: Qiling, key: int, size: int, shmflg: int): - def __create_shm(size: int, flags: int) -> int: - perms = flags & ((1 << 9) - 1) + def __create_shm(key: int, size: int, flags: int) -> int: + """Create a new shared memory segment for the specified key. - posix_to_uc = ( - (SHM_W, UC_PROT_WRITE), - (SHM_R, UC_PROT_READ) - ) + Returns: shmid of the newly created segment, -1 if an error has occured + """ - # convert posix permissions to unicorn memory access bits - uc_perms = sum(u_perm for p_perm, u_perm in posix_to_uc if perms & p_perm) + if len(ql.os.shm) >= SHMMNI: + return -1 # ENOSPC + + mode = flags & ((1 << 9) - 1) # determine size alignment: either normal or huge page if flags & SHM_HUGETLB: shiftsize = (flags >> HUGETLB_FLAG_ENCODE_SHIFT) & HUGETLB_FLAG_ENCODE_MASK - pagesize = (1 << shiftsize) + alignment = (1 << shiftsize) else: - pagesize = ql.mem.pagesize + alignment = ql.mem.pagesize - if len(ql.os.shm) < SHMMNI: - shm_size = ql.mem.align_up(size, pagesize) - - try: - shm_addr = ql.mem.find_free_space(shm_size, ql.loader.mmap_address, align=pagesize) - except QlOutOfMemory: - return -1 # ENOMEM - else: - ql.mem.map(shm_addr, shm_size, uc_perms, '[shm]') + shm_size = ql.mem.align_up(size, alignment) - # for simplicity, the shm key is defined to be its base address - shm_key = shm_addr + shmid = ql.os.shm.add(QlShmId(key, ql.os.uid, ql.os.gid, mode, shm_size)) - ql.os.shm[shm_key] = QlShmId(shm_size, ql.os.uid, ql.os.gid, perms) - - else: - return -1 # ENOSPC + ql.log.debug(f'created a new shm: key = {key:#x}, mode = 0{mode:o}, size = {shm_size:#x}. assigned id: {shmid:#x}') - return shm_key + return shmid # create new shared memory segment if key == IPC_PRIVATE: - key = __create_shm(size, shmflg) + shmid = __create_shm(key, size, shmflg) - # a shm with the specified key exists - elif key in ql.os.shm: - # user asked to create a new one? - if shmflg & (IPC_CREAT | IPC_EXCL): - return -1 # EEXIST + else: + shmid, shm = ql.os.shm.get_by_key(key) - shmid = ql.os.shm[key] + # a shm with the specified key does not exist + if shm is None: + # the user asked to create a new one? + if shmflg & IPC_CREAT: + shmid = __create_shm(key, size, shmflg) - # check whether the user has permissions to access this shm - # FIXME: should probably use ql.os.cuid instead, but we don't support it yet - if (ql.os.uid == shmid.uid) and (shmid.mode & (SHM_W | SHM_R)): - return key + else: + return -1 # ENOENT + # a shm with the specified key exists else: - return -1 # EACCES + # the user asked to create a new one? + if shmflg & (IPC_CREAT | IPC_EXCL): + return -1 # EEXIST - # a shm with the specified key does not exist - else: - # user asked to create a new one? - if shmflg & IPC_CREAT: - key = __create_shm(size, shmflg) + # check whether the user has permissions to access this shm + # FIXME: should probably use ql.os.cuid instead, but we don't support it yet + if (ql.os.uid == shm.uid) and (shm.mode & (SHM_W | SHM_R)): + return shmid - else: - return -1 # ENOENT + else: + return -1 # EACCES - return key + return shmid def ql_syscall_shmat(ql: Qiling, shmid: int, shmaddr: int, shmflg: int): - if shmid not in ql.os.shm: + shm = ql.os.shm.get_by_id(shmid) + + # a shm with the specified key does not exist + if shm is None: return -1 # EINVAL if shmaddr == 0: - # system may choose any suitable page-aligned address. since existing segments are - # guaranteed to be aligned and key is defined to be shm base address, we can just - # use the key - addr = shmid + # system may choose any suitable page-aligned address + attaddr = ql.mem.find_free_space(shm.segsz, ql.loader.mmap_address) elif shmflg & SHM_RND: - # note: should align to SHMLBA, but usually its value is just a page - addr = ql.mem.align(shmaddr) + # select the appropriate SHMLBA value, based on the platform + shmlba = { + QL_ARCH.MIPS: 0x40000, + QL_ARCH.ARM: ql.mem.pagesize * 4, + QL_ARCH.ARM64: ql.mem.pagesize * 4, + QL_ARCH.X86: ql.mem.pagesize, + QL_ARCH.X8664: ql.mem.pagesize + } + + # align the address specified by shmaddr to platform SHMLBA + attaddr = ql.mem.align(shmaddr, shmlba[ql.arch.type]) else: + # shmaddr is expected to be aligned if shmaddr & (ql.mem.pagesize - 1): return -1 # EINVAL - addr = shmaddr + attaddr = shmaddr + + perms = UC_PROT_READ + + if shmflg & SHM_RDONLY == 0: + perms |= UC_PROT_WRITE + + if shmflg & SHM_EXEC: + perms |= UC_PROT_EXEC + + # user asked to attached the seg as readable; is it allowed? + if (perms & UC_PROT_READ) and not (shm.mode & SHM_R): + return -1 # EACCES + + # user asked to attached the seg as writable; is it allowed? + if (perms & UC_PROT_WRITE) and not (shm.mode & SHM_W): + return -1 # EACCES + + # TODO: if segment is already attached, there is no need to map another memory for it. + # if we do, data changes will not be reflected between the segment attachments. we could + # use a mmio map for additional attachments, and have writes and reads directed to the + # first attachment mapping + + try: + # attach the segment at shmaddr + ql.mem.map(attaddr, shm.segsz, perms, '[shm]') + except QlMemoryMappedError: + return -1 # EINVAL + + # track attachment + shm.attach.append(attaddr) + + ql.log.debug(f'shm {shmid:#x} attached at {attaddr:#010x}') - return addr + return attaddr __all__ = [ From 817bdb744df6b1e73363afa23c6b7de3a9b58fb9 Mon Sep 17 00:00:00 2001 From: elicn Date: Thu, 13 Apr 2023 22:10:25 +0300 Subject: [PATCH 32/42] Add POSIX shmdt syscall --- qiling/os/posix/syscall/shm.py | 12 ++++++++++++ qiling/os/posix/syscall/syscall.py | 4 ++++ 2 files changed, 16 insertions(+) diff --git a/qiling/os/posix/syscall/shm.py b/qiling/os/posix/syscall/shm.py index 760094427..7bd8228ac 100644 --- a/qiling/os/posix/syscall/shm.py +++ b/qiling/os/posix/syscall/shm.py @@ -139,7 +139,19 @@ def ql_syscall_shmat(ql: Qiling, shmid: int, shmaddr: int, shmflg: int): return attaddr +def ql_syscall_shmdt(ql: Qiling, shmaddr: int): + shm = ql.os.shm.get_by_attaddr(shmaddr) + + if shm is None: + return -1 # EINVAL + + shm.attach.remove(shmaddr) + + return 0 + + __all__ = [ 'ql_syscall_shmget', + 'ql_syscall_shmdt', 'ql_syscall_shmat' ] diff --git a/qiling/os/posix/syscall/syscall.py b/qiling/os/posix/syscall/syscall.py index 3fe7cbe33..3aba66931 100644 --- a/qiling/os/posix/syscall/syscall.py +++ b/qiling/os/posix/syscall/syscall.py @@ -22,11 +22,15 @@ def __call_shmat(*args: int) -> int: return ql_syscall_shmat(ql, args[0], args[3], args[1]) + def __call_shmdt(*args: int) -> int: + return ql_syscall_shmdt(ql, args[3]) + def __call_shmget(*args: int) -> int: return ql_syscall_shmget(ql, args[0], args[1], args[2]) ipc_call = { SHMAT: __call_shmat, + SHMDT: __call_shmdt, SHMGET: __call_shmget } From a95b22bdaf3b6da04b29c9b7801f3251ae441455 Mon Sep 17 00:00:00 2001 From: elicn Date: Fri, 14 Apr 2023 14:53:10 +0300 Subject: [PATCH 33/42] Make argv and code mutually exclusive --- qiling/core.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/qiling/core.py b/qiling/core.py index 536ddbd54..6cc5c4706 100644 --- a/qiling/core.py +++ b/qiling/core.py @@ -32,10 +32,10 @@ class Qiling(QlCoreHooks, QlCoreStructs): def __init__( self, - argv: Sequence[str] = None, + argv: Sequence[str] = [], rootfs: str = r'.', env: MutableMapping[AnyStr, AnyStr] = {}, - code: bytes = None, + code: Optional[bytes] = None, ostype: Union[str, QL_OS] = None, archtype: Union[str, QL_ARCH] = None, verbose: QL_VERBOSE = QL_VERBOSE.DEFAULT, @@ -90,18 +90,26 @@ def __init__( ############## # argv setup # ############## - if argv is None: - argv = ['qilingcode'] + if argv: + if code: + raise AttributeError('argv and code are mutually execlusive') - elif not os.path.exists(argv[0]): - raise QlErrorFileNotFound(f'Target binary not found: "{argv[0]}"') + target = argv[0] + + if not os.path.isfile(target): + raise QlErrorFileNotFound(f'Target binary not found: "{target}"') + else: + # an empty argv list means we are going to execute a shellcode. to keep + # the 'path' api compatible, we insert a dummy placeholder + + argv = [''] self._argv = argv ################ # rootfs setup # ################ - if not os.path.exists(rootfs): + if not os.path.isdir(rootfs): raise QlErrorFileNotFound(f'Target rootfs not found: "{rootfs}"') self._rootfs = rootfs From 609ea313be41df516f799fb9788cbfc5a7631704 Mon Sep 17 00:00:00 2001 From: elicn Date: Fri, 14 Apr 2023 15:08:06 +0300 Subject: [PATCH 34/42] Opportunistic PEP8 fixes --- qiling/loader/loader.py | 4 ++- qiling/log.py | 40 ++++++++++++++++++++--------- qiling/utils.py | 57 ++++++++++++++++++++++++++++------------- 3 files changed, 70 insertions(+), 31 deletions(-) diff --git a/qiling/loader/loader.py b/qiling/loader/loader.py index 6be0ccf1d..3ad3a3d74 100644 --- a/qiling/loader/loader.py +++ b/qiling/loader/loader.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# +# # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # @@ -8,11 +8,13 @@ from qiling import Qiling + class Image(NamedTuple): base: int end: int path: str + class QlLoader: def __init__(self, ql: Qiling): self.ql = ql diff --git a/qiling/log.py b/qiling/log.py index d085266ed..46026b8db 100644 --- a/qiling/log.py +++ b/qiling/log.py @@ -3,20 +3,28 @@ # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # +from __future__ import annotations + import copy import logging import os import re import weakref -from typing import Optional, TextIO +from typing import TYPE_CHECKING, Optional, TextIO +from logging import Filter, Formatter, LogRecord, Logger, NullHandler, StreamHandler, FileHandler from qiling.const import QL_VERBOSE +if TYPE_CHECKING: + from qiling import Qiling + + QL_INSTANCE_ID = 114514 FMT_STR = '%(levelname)s\t%(message)s' + class COLOR: WHITE = '\033[37m' CRIMSON = '\033[31m' @@ -28,7 +36,8 @@ class COLOR: CYAN = '\033[96m' ENDC = '\033[0m' -class QlBaseFormatter(logging.Formatter): + +class QlBaseFormatter(Formatter): __level_tag = { 'WARNING' : '[!]', 'INFO' : '[=]', @@ -37,7 +46,7 @@ class QlBaseFormatter(logging.Formatter): 'ERROR' : '[x]' } - def __init__(self, ql, *args, **kwargs): + def __init__(self, ql: Qiling, *args, **kwargs): super().__init__(*args, **kwargs) self.ql = weakref.proxy(ql) @@ -47,7 +56,7 @@ def get_level_tag(self, level: str) -> str: def get_thread_tag(self, thread: str) -> str: return thread - def format(self, record: logging.LogRecord): + def format(self, record: LogRecord): # In case we have multiple formatters, we have to keep a copy of the record. record = copy.copy(record) @@ -64,6 +73,7 @@ def format(self, record: logging.LogRecord): return super().format(record) + class QlColoredFormatter(QlBaseFormatter): __level_color = { 'WARNING' : COLOR.YELLOW, @@ -83,15 +93,17 @@ def get_thread_tag(self, tid: str) -> str: return f'{COLOR.GREEN}{s}{COLOR.ENDC}' -class RegexFilter(logging.Filter): + +class RegexFilter(Filter): def update_filter(self, regexp: str): self._filter = re.compile(regexp) - def filter(self, record: logging.LogRecord): + def filter(self, record: LogRecord): msg = record.getMessage() return self._filter.match(msg) is not None + def resolve_logger_level(verbose: QL_VERBOSE) -> int: return { QL_VERBOSE.DISABLED : logging.CRITICAL, @@ -102,6 +114,7 @@ def resolve_logger_level(verbose: QL_VERBOSE) -> int: QL_VERBOSE.DUMP : logging.DEBUG }[verbose] + def __is_color_terminal(stream: TextIO) -> bool: """Determine whether standard output is attached to a color terminal. @@ -142,7 +155,8 @@ def __default(_: int) -> bool: return handler(stream.fileno()) -def setup_logger(ql, log_file: Optional[str], console: bool, log_override: Optional[logging.Logger], log_plain: bool): + +def setup_logger(ql: Qiling, log_file: Optional[str], console: bool, log_override: Optional[Logger], log_plain: bool): global QL_INSTANCE_ID # If there is an override for our logger, then use it. @@ -155,13 +169,14 @@ def setup_logger(ql, log_file: Optional[str], console: bool, log_override: Optio # Disable propagation to avoid duplicate output. log.propagate = False + # Clear all handlers and filters. - log.handlers = [] - log.filters = [] + log.handlers.clear() + log.filters.clear() # Do we have console output? if console: - handler = logging.StreamHandler() + handler = StreamHandler() if log_plain or not __is_color_terminal(handler.stream): formatter = QlBaseFormatter(ql, FMT_STR) @@ -171,12 +186,12 @@ def setup_logger(ql, log_file: Optional[str], console: bool, log_override: Optio handler.setFormatter(formatter) log.addHandler(handler) else: - handler = logging.NullHandler() + handler = NullHandler() log.addHandler(handler) # Do we have to write log to a file? if log_file is not None: - handler = logging.FileHandler(log_file) + handler = FileHandler(log_file) formatter = QlBaseFormatter(ql, FMT_STR) handler.setFormatter(formatter) log.addHandler(handler) @@ -185,4 +200,5 @@ def setup_logger(ql, log_file: Optional[str], console: bool, log_override: Optio return log + __all__ = ['RegexFilter', 'setup_logger', 'resolve_logger_level'] diff --git a/qiling/utils.py b/qiling/utils.py index ccdd970ee..05cb0a43e 100644 --- a/qiling/utils.py +++ b/qiling/utils.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# +# # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # @@ -8,11 +8,13 @@ thoughout the qiling framework """ -from functools import partial -from pathlib import Path -import importlib, inspect, os +import importlib +import inspect +import os +from functools import partial from configparser import ConfigParser +from pathlib import Path from types import ModuleType from typing import TYPE_CHECKING, Any, Callable, Mapping, Optional, Tuple, TypeVar, Union @@ -38,27 +40,32 @@ def __name_to_enum(name: str, mapping: Mapping[str, T], aliases: Mapping[str, st return mapping.get(aliases.get(key) or key) + def os_convert(os: str) -> Optional[QL_OS]: alias_map = { - 'darwin' : 'macos' + 'darwin': 'macos' } return __name_to_enum(os, os_map, alias_map) + def arch_convert(arch: str) -> Optional[QL_ARCH]: alias_map = { - 'x86_64' : 'x8664', - 'riscv32' : 'riscv' + 'x86_64': 'x8664', + 'riscv32': 'riscv' } return __name_to_enum(arch, arch_map, alias_map) + def debugger_convert(debugger: str) -> Optional[QL_DEBUGGER]: return __name_to_enum(debugger, debugger_map) + def arch_os_convert(arch: QL_ARCH) -> Optional[QL_OS]: return arch_os_map.get(arch) + def ql_get_module(module_name: str) -> ModuleType: try: module = importlib.import_module(module_name, 'qiling') @@ -67,6 +74,7 @@ def ql_get_module(module_name: str) -> ModuleType: return module + def ql_get_module_function(module_name: str, member_name: str): module = ql_get_module(module_name) @@ -77,6 +85,7 @@ def ql_get_module_function(module_name: str, member_name: str): return member + def __emu_env_from_pathname(path: str) -> Tuple[Optional[QL_ARCH], Optional[QL_OS], Optional[QL_ENDIAN]]: if os.path.isdir(path) and path.endswith('.kext'): return QL_ARCH.X8664, QL_OS.MACOS, QL_ENDIAN.EL @@ -89,6 +98,7 @@ def __emu_env_from_pathname(path: str) -> Tuple[Optional[QL_ARCH], Optional[QL_O return None, None, None + def __emu_env_from_elf(path: str) -> Tuple[Optional[QL_ARCH], Optional[QL_OS], Optional[QL_ENDIAN]]: # instead of using full-blown elffile parsing, we perform a simple parsing to avoid # external dependencies for target systems that do not need them. @@ -99,7 +109,7 @@ def __emu_env_from_elf(path: str) -> Tuple[Optional[QL_ARCH], Optional[QL_OS], O ELFCLASS32 = 1 # 32-bit ELFCLASS64 = 2 # 64-bit - #ei_data + # ei_data ELFDATA2LSB = 1 # little-endian ELFDATA2MSB = 2 # big-endian @@ -121,8 +131,8 @@ def __emu_env_from_elf(path: str) -> Tuple[Optional[QL_ARCH], Optional[QL_OS], O EM_PPC = 20 endianess = { - ELFDATA2LSB : (QL_ENDIAN.EL, 'little'), - ELFDATA2MSB : (QL_ENDIAN.EB, 'big') + ELFDATA2LSB: (QL_ENDIAN.EL, 'little'), + ELFDATA2MSB: (QL_ENDIAN.EB, 'big') } machines32 = { @@ -140,8 +150,8 @@ def __emu_env_from_elf(path: str) -> Tuple[Optional[QL_ARCH], Optional[QL_OS], O } classes = { - ELFCLASS32 : machines32, - ELFCLASS64 : machines64 + ELFCLASS32: machines32, + ELFCLASS64: machines64 } abis = { @@ -192,6 +202,7 @@ def __emu_env_from_elf(path: str) -> Tuple[Optional[QL_ARCH], Optional[QL_OS], O return archtype, ostype, archendian + def __emu_env_from_macho(path: str) -> Tuple[Optional[QL_ARCH], Optional[QL_OS], Optional[QL_ENDIAN]]: macho_macos_sig64 = b'\xcf\xfa\xed\xfe' macho_macos_sig32 = b'\xce\xfa\xed\xfe' @@ -220,6 +231,7 @@ def __emu_env_from_macho(path: str) -> Tuple[Optional[QL_ARCH], Optional[QL_OS], return arch, ostype, endian + def __emu_env_from_pe(path: str) -> Tuple[Optional[QL_ARCH], Optional[QL_OS], Optional[QL_ENDIAN]]: import pefile @@ -260,6 +272,7 @@ def __emu_env_from_pe(path: str) -> Tuple[Optional[QL_ARCH], Optional[QL_OS], Op return arch, ostype, archendian + def ql_guess_emu_env(path: str) -> Tuple[Optional[QL_ARCH], Optional[QL_OS], Optional[QL_ENDIAN]]: guessing_methods = ( __emu_env_from_pathname, @@ -278,9 +291,10 @@ def ql_guess_emu_env(path: str) -> Tuple[Optional[QL_ARCH], Optional[QL_OS], Opt return arch, ostype, endian + def select_loader(ostype: QL_OS, libcache: bool) -> QlClassInit['QlLoader']: if ostype == QL_OS.WINDOWS: - kwargs = {'libcache' : libcache} + kwargs = {'libcache': libcache} else: kwargs = {} @@ -305,6 +319,7 @@ def select_loader(ostype: QL_OS, libcache: bool) -> QlClassInit['QlLoader']: return partial(obj, **kwargs) + def select_component(component_type: str, component_name: str, **kwargs) -> QlClassInit[Any]: component_path = f'.{component_type}.{component_name}' component_class = f'Ql{component_name.capitalize()}Manager' @@ -313,6 +328,7 @@ def select_component(component_type: str, component_name: str, **kwargs) -> QlCl return partial(obj, **kwargs) + def select_debugger(options: Union[str, bool]) -> Optional[QlClassInit['QlDebugger']]: if options is True: options = 'gdb' @@ -353,14 +369,15 @@ def __int_nothrow(v: str, /) -> Optional[int]: return None + def select_arch(archtype: QL_ARCH, endian: QL_ENDIAN, thumb: bool) -> QlClassInit['QlArch']: # set endianess and thumb mode for arm-based archs - if archtype == QL_ARCH.ARM: - kwargs = {'endian' : endian, 'thumb' : thumb} + if archtype is QL_ARCH.ARM: + kwargs = {'endian': endian, 'thumb': thumb} # set endianess for mips arch - elif archtype == QL_ARCH.MIPS: - kwargs = {'endian' : endian} + elif archtype is QL_ARCH.MIPS: + kwargs = {'endian': endian} else: kwargs = {} @@ -386,6 +403,7 @@ def select_arch(archtype: QL_ARCH, endian: QL_ENDIAN, thumb: bool) -> QlClassIni return partial(obj, **kwargs) + def select_os(ostype: QL_OS) -> QlClassInit['QlOs']: qlos_name = ostype.name qlos_path = f'.os.{qlos_name.lower()}.{qlos_name.lower()}' @@ -395,9 +413,10 @@ def select_os(ostype: QL_OS) -> QlClassInit['QlOs']: return partial(obj) + def profile_setup(ostype: QL_OS, user_config: Optional[Union[str, dict]]): # mcu uses a yaml-based config - if ostype == QL_OS.MCU: + if ostype is QL_OS.MCU: import yaml if user_config: @@ -426,6 +445,7 @@ def profile_setup(ostype: QL_OS, user_config: Optional[Union[str, dict]]): return config + # verify if emulator returns properly def verify_ret(ql: 'Qiling', err): # init_sp location is not consistent; this is here to work around that @@ -463,6 +483,7 @@ def verify_ret(ql: 'Qiling', err): else: raise + __all__ = [ 'os_convert', 'arch_convert', From 717899b55f98fdddce67ec4a2839d1fe54d98a1f Mon Sep 17 00:00:00 2001 From: elicn Date: Fri, 14 Apr 2023 15:12:22 +0300 Subject: [PATCH 35/42] Decouple runtime dependencies --- qiling/loader/loader.py | 7 +++++-- qiling/loader/pe.py | 16 +++++++++++++--- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/qiling/loader/loader.py b/qiling/loader/loader.py index 3ad3a3d74..32475f5fe 100644 --- a/qiling/loader/loader.py +++ b/qiling/loader/loader.py @@ -3,10 +3,13 @@ # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # +from __future__ import annotations + import os -from typing import Any, Mapping, MutableSequence, NamedTuple, Optional +from typing import TYPE_CHECKING, Any, Mapping, MutableSequence, NamedTuple, Optional -from qiling import Qiling +if TYPE_CHECKING: + from qiling import Qiling class Image(NamedTuple): diff --git a/qiling/loader/pe.py b/qiling/loader/pe.py index 439a6ae13..9234bbafa 100644 --- a/qiling/loader/pe.py +++ b/qiling/loader/pe.py @@ -3,13 +3,18 @@ # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # -import os, pefile, pickle, secrets, ntpath -from typing import Any, Dict, MutableMapping, NamedTuple, Optional, Mapping, Sequence, Tuple, Union +from __future__ import annotations + +import os +import pefile +import pickle +import secrets +import ntpath +from typing import TYPE_CHECKING, Any, Dict, MutableMapping, NamedTuple, Optional, Mapping, Sequence, Tuple, Union from unicorn import UcError from unicorn.x86_const import UC_X86_REG_CR4, UC_X86_REG_CR8 -from qiling import Qiling from qiling.arch.x86_const import FS_SEGMENT_ADDR, GS_SEGMENT_ADDR from qiling.const import QL_ARCH, QL_STATE from qiling.exception import QlErrorArch @@ -20,6 +25,10 @@ from qiling.os.windows.structs import * from .loader import QlLoader, Image +if TYPE_CHECKING: + from qiling import Qiling + + class QlPeCacheEntry(NamedTuple): ba: int data: bytearray @@ -859,6 +868,7 @@ def load(self, pe: Optional[pefile.PE]): self.ql.os.entry_point = self.entry_point self.init_sp = self.ql.arch.regs.arch_sp + class ShowProgress: """Display a progress animation while performing a time consuming task. From ace56d890dea2dcce3c05442ce58aabe9a5878ad Mon Sep 17 00:00:00 2001 From: elicn Date: Sun, 16 Apr 2023 19:48:31 +0300 Subject: [PATCH 36/42] Rearrange and fix ELF MT test suite --- tests/test_elf_multithread.py | 687 ++++++++++++++++++++++------------ 1 file changed, 448 insertions(+), 239 deletions(-) diff --git a/tests/test_elf_multithread.py b/tests/test_elf_multithread.py index 2121deb46..c6a7fd4ef 100644 --- a/tests/test_elf_multithread.py +++ b/tests/test_elf_multithread.py @@ -3,353 +3,550 @@ # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # -import http.client, platform, socket, sys, os, threading, time, unittest +import http.client +import platform +import re +import sys +import os +import threading +import time +import unittest + +from typing import List sys.path.append("..") from qiling import Qiling from qiling.const import * from qiling.exception import * from qiling.os.filestruct import ql_file +from qiling.os.stats import QlOsNullStats + + +BASE_ROOTFS = r'../examples/rootfs' +X86_LINUX_ROOTFS = fr'{BASE_ROOTFS}/x86_linux' +X64_LINUX_ROOTFS = fr'{BASE_ROOTFS}/x8664_linux' +ARM_LINUX_ROOTFS = fr'{BASE_ROOTFS}/arm_linux' +ARMEB_LINUX_ROOTFS = fr'{BASE_ROOTFS}/armeb_linux' +ARM64_LINUX_ROOTFS = fr'{BASE_ROOTFS}/arm64_linux' +MIPSEB_LINUX_ROOTFS = fr'{BASE_ROOTFS}/mips32_linux' +MIPSEL_LINUX_ROOTFS = fr'{BASE_ROOTFS}/mips32el_linux' + class ELFTest(unittest.TestCase): @unittest.skipIf(platform.system() == "Darwin" and platform.machine() == "arm64", 'darwin host') def test_elf_linux_execve_x8664(self): - ql = Qiling(["../examples/rootfs/x8664_linux/bin/posix_syscall_execve"], "../examples/rootfs/x8664_linux", verbose=QL_VERBOSE.DEBUG) + ql = Qiling([fr'{X64_LINUX_ROOTFS}/bin/posix_syscall_execve'], X64_LINUX_ROOTFS, verbose=QL_VERBOSE.DEBUG) ql.run() - for key, value in ql.loader.env.items(): - QL_TEST=value + env = ql.loader.env - self.assertEqual("TEST_QUERY", QL_TEST) - self.assertEqual("child", ql.loader.argv[0]) - - del QL_TEST - del ql + self.assertIn('QL_TEST', env) + self.assertEqual('TEST_QUERY', env['QL_TEST']) + self.assertEqual('child', ql.loader.argv[0]) def test_elf_linux_cloexec_x8664(self): - ql = Qiling(["../examples/rootfs/x8664_linux/bin/x8664_cloexec_test"], "../examples/rootfs/x8664_linux", verbose=QL_VERBOSE.DEBUG, multithread=True) + ql = Qiling([fr'{X64_LINUX_ROOTFS}/bin/x8664_cloexec_test'], X64_LINUX_ROOTFS, multithread=True, verbose=QL_VERBOSE.DEBUG) - filename = 'output.txt' - err = ql_file.open(filename, os.O_RDWR | os.O_CREAT, 0o777) + filename = 'stderr.txt' + err = ql_file.open(filename, os.O_RDWR | os.O_CREAT, 0o644) + ql.os.stats = QlOsNullStats() ql.os.stderr = err + ql.run() err.close() - with open(filename, 'rb') as f: - content = f.read() + with open(filename, 'r') as f: + contents = f.readlines() # cleanup os.remove(filename) - self.assertIn(b'fail', content) - - del ql + self.assertGreaterEqual(len(contents), 4) + self.assertIn('Operation not permitted', contents[-2]) + self.assertIn('Operation not permitted', contents[-1]) def test_multithread_elf_linux_x86(self): - def check_write(ql, write_fd, write_buf, write_count, *args, **kw): - nonlocal buf_out - try: - buf = ql.mem.read(write_buf, write_count) - buf = buf.decode() - buf_out = buf - except: - pass - buf_out = None - ql = Qiling(["../examples/rootfs/x86_linux/bin/x86_multithreading"], "../examples/rootfs/x86_linux", multithread=True, verbose=QL_VERBOSE.DEBUG) + logged: List[str] = [] + + def check_write(ql: Qiling, fd: int, write_buf, count: int): + if fd == 1: + content = ql.mem.read(write_buf, count) + + logged.append(content.decode()) + + ql = Qiling([fr'{X86_LINUX_ROOTFS}/bin/x86_multithreading'], X86_LINUX_ROOTFS, multithread=True, verbose=QL_VERBOSE.DEBUG) + + ql.os.stats = QlOsNullStats() ql.os.set_syscall("write", check_write, QL_INTERCEPT.ENTER) ql.run() - self.assertTrue("thread 2 ret val is" in buf_out) - - del ql + self.assertGreaterEqual(len(logged), 2) + self.assertTrue(logged[-2].startswith('thread 1 ret val is')) + self.assertTrue(logged[-1].startswith('thread 2 ret val is')) def test_multithread_elf_linux_arm64(self): - def check_write(ql, write_fd, write_buf, write_count, *args, **kw): - nonlocal buf_out - try: - buf = ql.mem.read(write_buf, write_count) - buf = buf.decode() - buf_out = buf - except: - pass - buf_out = None - ql = Qiling(["../examples/rootfs/arm64_linux/bin/arm64_multithreading"], "../examples/rootfs/arm64_linux", multithread=True, verbose=QL_VERBOSE.DEBUG) + logged: List[str] = [] + + def check_write(ql: Qiling, fd: int, write_buf, count: int): + if fd == 1: + content = ql.mem.read(write_buf, count) + + logged.append(content.decode()) + + ql = Qiling([fr'{ARM64_LINUX_ROOTFS}/bin/arm64_multithreading'], ARM64_LINUX_ROOTFS, multithread=True, verbose=QL_VERBOSE.DEBUG) + + ql.os.stats = QlOsNullStats() ql.os.set_syscall("write", check_write, QL_INTERCEPT.ENTER) ql.run() - self.assertTrue("thread 2 ret val is" in buf_out) - - del ql + self.assertGreaterEqual(len(logged), 2) + self.assertTrue(logged[-2].startswith('thread 1 ret val is')) + self.assertTrue(logged[-1].startswith('thread 2 ret val is')) def test_multithread_elf_linux_x8664(self): - def check_write(ql, write_fd, write_buf, write_count, *args, **kw): - nonlocal buf_out - try: - buf = ql.mem.read(write_buf, write_count) - buf = buf.decode() - buf_out = buf - except: - pass - buf_out = None - ql = Qiling(["../examples/rootfs/x8664_linux/bin/x8664_multithreading"], "../examples/rootfs/x8664_linux", multithread=True) + logged: List[str] = [] + + def check_write(ql: Qiling, fd: int, write_buf, count: int): + if fd == 1: + content = ql.mem.read(write_buf, count) + + logged.append(content.decode()) + + ql = Qiling([fr'{X64_LINUX_ROOTFS}/bin/x8664_multithreading'], X64_LINUX_ROOTFS, multithread=True, verbose=QL_VERBOSE.DEBUG) + + ql.os.stats = QlOsNullStats() ql.os.set_syscall("write", check_write, QL_INTERCEPT.ENTER) ql.run() - self.assertTrue("thread 2 ret val is" in buf_out) - - del ql + self.assertGreaterEqual(len(logged), 2) + self.assertTrue(logged[-2].startswith('thread 1 ret val is')) + self.assertTrue(logged[-1].startswith('thread 2 ret val is')) def test_multithread_elf_linux_mips32eb(self): - def check_write(ql, write_fd, write_buf, write_count, *args, **kw): - nonlocal buf_out - try: - buf = ql.mem.read(write_buf, write_count) - buf = buf.decode() - buf_out = buf - except: - pass - buf_out = None - ql = Qiling(["../examples/rootfs/mips32_linux/bin/mips32_multithreading"], "../examples/rootfs/mips32_linux", verbose=QL_VERBOSE.DEBUG, multithread=True) + logged: List[str] = [] + + def check_write(ql: Qiling, fd: int, write_buf, count: int): + if fd == 1: + content = ql.mem.read(write_buf, count) + + logged.append(content.decode()) + + ql = Qiling([fr'{MIPSEB_LINUX_ROOTFS}/bin/mips32_multithreading'], MIPSEB_LINUX_ROOTFS, multithread=True, verbose=QL_VERBOSE.DEBUG) + + ql.os.stats = QlOsNullStats() ql.os.set_syscall("write", check_write, QL_INTERCEPT.ENTER) ql.run() - self.assertTrue("thread 2 ret val is" in buf_out) - - del ql + self.assertGreaterEqual(len(logged), 2) + self.assertTrue(logged[-2].startswith('thread 1 ret val is')) + self.assertTrue(logged[-1].startswith('thread 2 ret val is')) def test_multithread_elf_linux_mips32el(self): - def check_write(ql, write_fd, write_buf, write_count, *args, **kw): - nonlocal buf_out - try: - buf = ql.mem.read(write_buf, write_count) - buf = buf.decode() - buf_out = buf - except: - pass - buf_out = None - ql = Qiling(["../examples/rootfs/mips32el_linux/bin/mips32el_multithreading"], "../examples/rootfs/mips32el_linux", multithread=True, verbose=QL_VERBOSE.DEBUG) + logged: List[str] = [] + + def check_write(ql: Qiling, fd: int, write_buf, count: int): + if fd == 1: + content = ql.mem.read(write_buf, count) + + logged.append(content.decode()) + + ql = Qiling([fr'{MIPSEL_LINUX_ROOTFS}/bin/mips32el_multithreading'], MIPSEL_LINUX_ROOTFS, multithread=True, verbose=QL_VERBOSE.DEBUG) + + ql.os.stats = QlOsNullStats() ql.os.set_syscall("write", check_write, QL_INTERCEPT.ENTER) ql.run() - self.assertTrue("thread 2 ret val is" in buf_out) - - del ql + self.assertGreaterEqual(len(logged), 2) + self.assertTrue(logged[-2].startswith('thread 1 ret val is')) + self.assertTrue(logged[-1].startswith('thread 2 ret val is')) def test_multithread_elf_linux_arm(self): - def check_write(ql, write_fd, write_buf, write_count, *args, **kw): - nonlocal buf_out - try: - buf = ql.mem.read(write_buf, write_count) - buf = buf.decode() - buf_out = buf - except: - pass - buf_out = None - ql = Qiling(["../examples/rootfs/arm_linux/bin/arm_multithreading"], "../examples/rootfs/arm_linux", multithread=True, verbose=QL_VERBOSE.DEBUG) + logged: List[str] = [] + + def check_write(ql: Qiling, fd: int, write_buf, count: int): + if fd == 1: + content = ql.mem.read(write_buf, count) + + logged.append(content.decode()) + + ql = Qiling([fr'{ARM_LINUX_ROOTFS}/bin/arm_multithreading'], ARM_LINUX_ROOTFS, multithread=True, verbose=QL_VERBOSE.DEBUG) + + ql.os.stats = QlOsNullStats() ql.os.set_syscall("write", check_write, QL_INTERCEPT.ENTER) ql.run() - self.assertTrue("thread 2 ret val is" in buf_out) + self.assertGreaterEqual(len(logged), 2) + self.assertTrue(logged[-2].startswith('thread 1 ret val is')) + self.assertTrue(logged[-1].startswith('thread 2 ret val is')) - del ql + @unittest.skip('broken: unicorn.unicorn.UcError: Invalid instruction (UC_ERR_INSN_INVALID)') + def test_multithread_elf_linux_armeb(self): + logged: List[str] = [] - # unicorn.unicorn.UcError: Invalid instruction (UC_ERR_INSN_INVALID) - # def test_multithread_elf_linux_armeb(self): - # def check_write(ql, write_fd, write_buf, write_count, *args, **kw): - # nonlocal buf_out - # try: - # buf = ql.mem.read(write_buf, write_count) - # buf = buf.decode() - # buf_out = buf - # except: - # pass - # buf_out = None - # ql = Qiling(["../examples/rootfs/armeb_linux/bin/armeb_multithreading"], "../examples/rootfs/armeb_linux", multithread=True, verbose=QL_VERBOSE.DEBUG) - # ql.os.set_syscall("write", check_write, QL_INTERCEPT.ENTER) - # ql.run() + def check_write(ql: Qiling, fd: int, write_buf, count: int): + if fd == 1: + content = ql.mem.read(write_buf, count) - # self.assertTrue("thread 2 ret val is" in buf_out) + logged.append(content.decode()) + + ql = Qiling([fr'{ARMEB_LINUX_ROOTFS}/bin/armeb_multithreading'], ARMEB_LINUX_ROOTFS, multithread=True, verbose=QL_VERBOSE.DEBUG) + + ql.os.stats = QlOsNullStats() + ql.os.set_syscall("write", check_write, QL_INTERCEPT.ENTER) + ql.run() - # del ql + self.assertGreaterEqual(len(logged), 2) + self.assertTrue(logged[-2].startswith('thread 1 ret val is')) + self.assertTrue(logged[-1].startswith('thread 2 ret val is')) def test_tcp_elf_linux_x86(self): - def check_write(ql, write_fd, write_buf, write_count, *args, **kw): - try: - buf = ql.mem.read(write_buf, write_count) - buf = buf.decode() - if buf.startswith("server send()"): - ql.buf_out = buf - except: - pass - ql = Qiling(["../examples/rootfs/x86_linux/bin/x86_tcp_test", "20001"], "../examples/rootfs/x86_linux", multithread=True, verbose=QL_VERBOSE.DEBUG) + logged: List[str] = [] + + def check_write(ql: Qiling, fd: int, write_buf, count: int): + if fd == 2: + content = ql.mem.read(write_buf, count) + + logged.append(content.decode()) + + ql = Qiling([fr'{X86_LINUX_ROOTFS}/bin/x86_tcp_test', '20000'], X86_LINUX_ROOTFS, multithread=True, verbose=QL_VERBOSE.DEBUG) + + ql.os.stats = QlOsNullStats() ql.os.set_syscall("write", check_write, QL_INTERCEPT.ENTER) ql.run() - self.assertEqual("server send() 14 return 14.\n", ql.buf_out) + self.assertGreaterEqual(len(logged), 2) + self.assertTrue(logged[-2].startswith('server recv()')) + self.assertTrue(logged[-1].startswith('server send()')) - del ql + # the server is expected to send the value it received, for example: + # 'server recv() return 14.\n' + # 'server send() 14 return 14.\n' + + m = re.search(r'(?P\d+)\.\s+\Z', logged[-2]) + self.assertIsNotNone(m, 'could not extract numeric value from log message') + + num = m.group('num') + msg = logged[-1].strip() + + self.assertTrue(msg.endswith(f'{num} return {num}.')) def test_tcp_elf_linux_x8664(self): - def check_write(ql, write_fd, write_buf, write_count, *args, **kw): - try: - buf = ql.mem.read(write_buf, write_count) - buf = buf.decode() - if buf.startswith("server send()"): - ql.buf_out = buf - except: - pass - ql = Qiling(["../examples/rootfs/x8664_linux/bin/x8664_tcp_test", "20002"], "../examples/rootfs/x8664_linux", multithread=True, verbose=QL_VERBOSE.DEBUG) + logged: List[str] = [] + + def check_write(ql: Qiling, fd: int, write_buf, count: int): + if fd == 2: + content = ql.mem.read(write_buf, count) + + logged.append(content.decode()) + + ql = Qiling([fr'{X64_LINUX_ROOTFS}/bin/x8664_tcp_test', '20001'], X64_LINUX_ROOTFS, multithread=True, verbose=QL_VERBOSE.DEBUG) + + ql.os.stats = QlOsNullStats() ql.os.set_syscall("write", check_write, QL_INTERCEPT.ENTER) ql.run() - self.assertEqual("server send() 14 return 14.\n", ql.buf_out) + self.assertGreaterEqual(len(logged), 2) + self.assertTrue(logged[-2].startswith('server recv()')) + self.assertTrue(logged[-1].startswith('server send()')) + + # the server is expected to send the value it received, for example: + # 'server recv() return 14.\n' + # 'server send() 14 return 14.\n' - del ql + m = re.search(r'(?P\d+)\.\s+\Z', logged[-2]) + self.assertIsNotNone(m, 'could not extract numeric value from log message') + + num = m.group('num') + msg = logged[-1].strip() + + self.assertTrue(msg.endswith(f'{num} return {num}.')) def test_tcp_elf_linux_arm(self): - def check_write(ql, write_fd, write_buf, write_count, *args, **kw): - try: - buf = ql.mem.read(write_buf, write_count) - buf = buf.decode() - if buf.startswith("server write()"): - ql.buf_out = buf - except: - pass - ql = Qiling(["../examples/rootfs/arm_linux/bin/arm_tcp_test", "20003"], "../examples/rootfs/arm_linux", multithread=True, verbose=QL_VERBOSE.DEBUG) + logged: List[str] = [] + + def check_write(ql: Qiling, fd: int, write_buf, count: int): + if fd == 2: + content = ql.mem.read(write_buf, count) + + logged.append(content.decode()) + + ql = Qiling([fr'{ARM_LINUX_ROOTFS}/bin/arm_tcp_test', '20002'], ARM_LINUX_ROOTFS, multithread=True, verbose=QL_VERBOSE.DEBUG) + + ql.os.stats = QlOsNullStats() ql.os.set_syscall("write", check_write, QL_INTERCEPT.ENTER) ql.run() - self.assertEqual("server write() 14 return 14.\n", ql.buf_out) + self.assertGreaterEqual(len(logged), 2) + self.assertTrue(logged[-2].startswith('server read()')) + self.assertTrue(logged[-1].startswith('server write()')) + + # the server is expected to send the value it received, for example: + # 'server read() return 14.\n' + # 'server write() 14 return 14.\n' + + m = re.search(r'(?P\d+)\.\s+\Z', logged[-2]) + self.assertIsNotNone(m, 'could not extract numeric value from log message') - del ql + num = m.group('num') + msg = logged[-1].strip() + self.assertTrue(msg.endswith(f'{num} return {num}.')) def test_tcp_elf_linux_arm64(self): - def check_write(ql, write_fd, write_buf, write_count, *args, **kw): - try: - buf = ql.mem.read(write_buf, write_count) - buf = buf.decode() - if buf.startswith("server send()"): - ql.buf_out = buf - except: - pass - ql = Qiling(["../examples/rootfs/arm64_linux/bin/arm64_tcp_test", "20004"], "../examples/rootfs/arm64_linux", multithread=True, verbose=QL_VERBOSE.DEBUG) + logged: List[str] = [] + + def check_write(ql: Qiling, fd: int, write_buf, count: int): + if fd == 2: + content = ql.mem.read(write_buf, count) + + logged.append(content.decode()) + + ql = Qiling([fr'{ARM64_LINUX_ROOTFS}/bin/arm64_tcp_test', '20003'], ARM64_LINUX_ROOTFS, multithread=True, verbose=QL_VERBOSE.DEBUG) + + ql.os.stats = QlOsNullStats() ql.os.set_syscall("write", check_write, QL_INTERCEPT.ENTER) ql.run() - self.assertEqual("server send() 14 return 14.\n", ql.buf_out) + self.assertGreaterEqual(len(logged), 2) + self.assertTrue(logged[-2].startswith('server recv()')) + self.assertTrue(logged[-1].startswith('server send()')) - del ql + # the server is expected to send the value it received, for example: + # 'server recv() return 14.\n' + # 'server send() 14 return 14.\n' + m = re.search(r'(?P\d+)\.\s+\Z', logged[-2]) + self.assertIsNotNone(m, 'could not extract numeric value from log message') + + num = m.group('num') + msg = logged[-1].strip() + + self.assertTrue(msg.endswith(f'{num} return {num}.')) def test_tcp_elf_linux_armeb(self): - def check_write(ql, write_fd, write_buf, write_count, *args, **kw): - try: - buf = ql.mem.read(write_buf, write_count) - buf = buf.decode() - if buf.startswith("server send()"): - ql.buf_out = buf - except: - pass - ql = Qiling(["../examples/rootfs/armeb_linux/bin/armeb_tcp_test", "20003"], "../examples/rootfs/armeb_linux", multithread=True, verbose=QL_VERBOSE.DEBUG) + logged: List[str] = [] + + def check_write(ql: Qiling, fd: int, write_buf, count: int): + if fd == 2: + content = ql.mem.read(write_buf, count) + + logged.append(content.decode()) + + ql = Qiling([fr'{ARMEB_LINUX_ROOTFS}/bin/armeb_tcp_test', '20004'], ARMEB_LINUX_ROOTFS, multithread=True, verbose=QL_VERBOSE.DEBUG) + + ql.os.stats = QlOsNullStats() ql.os.set_syscall("write", check_write, QL_INTERCEPT.ENTER) ql.run() - self.assertEqual("server send() 14 return 14.\n", ql.buf_out) + self.assertGreaterEqual(len(logged), 2) + self.assertTrue(logged[-2].startswith('server recv()')) + self.assertTrue(logged[-1].startswith('server send()')) + + # the server is expected to send the value it received, for example: + # 'server recv() return 14.\n' + # 'server send() 14 return 14.\n' + + m = re.search(r'(?P\d+)\.\s+\Z', logged[-2]) + self.assertIsNotNone(m, 'could not extract numeric value from log message') - del ql + num = m.group('num') + msg = logged[-1].strip() + self.assertTrue(msg.endswith(f'{num} return {num}.')) def test_tcp_elf_linux_mips32eb(self): - ql = Qiling(["../examples/rootfs/mips32_linux/bin/mips32_tcp_test", "20005"], "../examples/rootfs/mips32_linux", multithread=True, verbose=QL_VERBOSE.DEBUG) + logged: List[str] = [] + + def check_write(ql: Qiling, fd: int, write_buf, count: int): + if fd == 2: + content = ql.mem.read(write_buf, count) + + logged.append(content.decode()) + + ql = Qiling([fr'{MIPSEB_LINUX_ROOTFS}/bin/mips32_tcp_test', '20005'], MIPSEB_LINUX_ROOTFS, multithread=True, verbose=QL_VERBOSE.DEBUG) + + ql.os.stats = QlOsNullStats() + ql.os.set_syscall("write", check_write, QL_INTERCEPT.ENTER) ql.run() - del ql + self.assertGreaterEqual(len(logged), 2) + self.assertTrue(logged[-2].startswith('server recv()')) + self.assertTrue(logged[-1].startswith('server send()')) + + # the server is expected to send the value it received, for example: + # 'server recv() return 14.\n' + # 'server send() 14 return 14.\n' + + m = re.search(r'(?P\d+)\.\s+\Z', logged[-2]) + self.assertIsNotNone(m, 'could not extract numeric value from log message') + + num = m.group('num') + msg = logged[-1].strip() + + self.assertTrue(msg.endswith(f'{num} return {num}.')) def test_tcp_elf_linux_mips32el(self): - ql = Qiling(["../examples/rootfs/mips32el_linux/bin/mips32el_tcp_test", "20005"], "../examples/rootfs/mips32el_linux", multithread=True, verbose=QL_VERBOSE.DEBUG) + logged: List[str] = [] + + def check_write(ql: Qiling, fd: int, write_buf, count: int): + if fd == 2: + content = ql.mem.read(write_buf, count) + + logged.append(content.decode()) + + ql = Qiling([fr'{MIPSEL_LINUX_ROOTFS}/bin/mips32el_tcp_test', '20006'], MIPSEL_LINUX_ROOTFS, multithread=True, verbose=QL_VERBOSE.DEBUG) + + ql.os.stats = QlOsNullStats() + ql.os.set_syscall("write", check_write, QL_INTERCEPT.ENTER) ql.run() - del ql + self.assertGreaterEqual(len(logged), 2) + self.assertTrue(logged[-2].startswith('server read()')) + self.assertTrue(logged[-1].startswith('server write()')) + + # the server is expected to send the value it received, for example: + # 'server read() return 14.\n' + # 'server write() 14 return 14.\n' + + m = re.search(r'(?P\d+)\.\s+\Z', logged[-2]) + self.assertIsNotNone(m, 'could not extract numeric value from log message') + + num = m.group('num') + msg = logged[-1].strip() + + self.assertTrue(msg.endswith(f'{num} return {num}.')) def test_udp_elf_linux_x86(self): - def check_write(ql, write_fd, write_buf, write_count, *args, **kw): - try: - buf = ql.mem.read(write_buf, write_count) - buf = buf.decode() - if buf.startswith("server sendto()"): - ql.buf_out = buf - except: - pass - - ql = Qiling(["../examples/rootfs/x86_linux/bin/x86_udp_test", "20007"], "../examples/rootfs/x86_linux", multithread=True, verbose=QL_VERBOSE.DEBUG) + logged: List[str] = [] + + def check_write(ql: Qiling, fd: int, write_buf, count: int): + if fd == 2: + content = ql.mem.read(write_buf, count) + + logged.append(content.decode()) + + ql = Qiling([fr'{X86_LINUX_ROOTFS}/bin/x86_udp_test', '20010'], X86_LINUX_ROOTFS, multithread=True, verbose=QL_VERBOSE.DEBUG) + + ql.os.stats = QlOsNullStats() ql.os.set_syscall("write", check_write, QL_INTERCEPT.ENTER) ql.run() - self.assertEqual("server sendto() 14 return 14.\n", ql.buf_out) + self.assertGreaterEqual(len(logged), 2) + self.assertTrue(logged[-2].startswith('server recvfrom()')) + self.assertTrue(logged[-1].startswith('server sendto()')) + + # the server is expected to send the value it received, for example: + # 'server recvfrom() return 14.\n' + # 'server sendto() 14 return 14.\n' + + m = re.search(r'(?P\d+)\.\s+\Z', logged[-2]) + self.assertIsNotNone(m, 'could not extract numeric value from log message') - del ql + num = m.group('num') + msg = logged[-1].strip() + self.assertTrue(msg.endswith(f'{num} return {num}.')) def test_udp_elf_linux_x8664(self): - def check_write(ql, write_fd, write_buf, write_count, *args, **kw): - try: - buf = ql.mem.read(write_buf, write_count) - buf = buf.decode() - if buf.startswith("server sendto()"): - ql.buf_out = buf - except: - pass - - ql = Qiling(["../examples/rootfs/x8664_linux/bin/x8664_udp_test", "20008"], "../examples/rootfs/x8664_linux", multithread=True, verbose=QL_VERBOSE.DEBUG) + logged: List[str] = [] + + def check_write(ql: Qiling, fd: int, write_buf, count: int): + if fd == 2: + content = ql.mem.read(write_buf, count) + + logged.append(content.decode()) + + ql = Qiling([fr'{X64_LINUX_ROOTFS}/bin/x8664_udp_test', '20011'], X64_LINUX_ROOTFS, multithread=True, verbose=QL_VERBOSE.DEBUG) + + ql.os.stats = QlOsNullStats() ql.os.set_syscall("write", check_write, QL_INTERCEPT.ENTER) ql.run() - self.assertEqual("server sendto() 14 return 14.\n", ql.buf_out) + self.assertGreaterEqual(len(logged), 2) + self.assertTrue(logged[-2].startswith('server recvfrom()')) + self.assertTrue(logged[-1].startswith('server sendto()')) - del ql + # the server is expected to send the value it received, for example: + # 'server recvfrom() return 14.\n' + # 'server sendto() 14 return 14.\n' + + m = re.search(r'(?P\d+)\.\s+\Z', logged[-2]) + self.assertIsNotNone(m, 'could not extract numeric value from log message') + + num = m.group('num') + msg = logged[-1].strip() + + self.assertTrue(msg.endswith(f'{num} return {num}.')) def test_udp_elf_linux_arm64(self): - def check_write(ql, write_fd, write_buf, write_count, *args, **kw): - try: - buf = ql.mem.read(write_buf, write_count) - buf = buf.decode() - if buf.startswith("server sendto()"): - ql.buf_out = buf - except: - pass - - ql = Qiling(["../examples/rootfs/arm64_linux/bin/arm64_udp_test", "20009"], "../examples/rootfs/arm64_linux", multithread=True, verbose=QL_VERBOSE.DEBUG) + logged: List[str] = [] + + def check_write(ql: Qiling, fd: int, write_buf, count: int): + if fd == 2: + content = ql.mem.read(write_buf, count) + + logged.append(content.decode()) + + ql = Qiling([fr'{ARM64_LINUX_ROOTFS}/bin/arm64_udp_test', '20013'], ARM64_LINUX_ROOTFS, multithread=True, verbose=QL_VERBOSE.DEBUG) + + ql.os.stats = QlOsNullStats() ql.os.set_syscall("write", check_write, QL_INTERCEPT.ENTER) ql.run() - self.assertEqual("server sendto() 14 return 14.\n", ql.buf_out) + self.assertGreaterEqual(len(logged), 2) + self.assertTrue(logged[-2].startswith('server recvfrom()')) + self.assertTrue(logged[-1].startswith('server sendto()')) + + # the server is expected to send the value it received, for example: + # 'server recvfrom() return 14.\n' + # 'server sendto() 14 return 14.\n' - del ql + m = re.search(r'(?P\d+)\.\s+\Z', logged[-2]) + self.assertIsNotNone(m, 'could not extract numeric value from log message') + + num = m.group('num') + msg = logged[-1].strip() + + self.assertTrue(msg.endswith(f'{num} return {num}.')) def test_udp_elf_linux_armeb(self): - def check_write(ql, write_fd, write_buf, write_count, *args, **kw): - try: - buf = ql.mem.read(write_buf, write_count) - buf = buf.decode() - if buf.startswith("server sendto()"): - ql.buf_out = buf - except: - pass - - ql = Qiling(["../examples/rootfs/armeb_linux/bin/armeb_udp_test", "20010"], "../examples/rootfs/armeb_linux", multithread=True, verbose=QL_VERBOSE.DEBUG) + logged: List[str] = [] + + def check_write(ql: Qiling, fd: int, write_buf, count: int): + if fd == 2: + content = ql.mem.read(write_buf, count) + + logged.append(content.decode()) + + ql = Qiling([fr'{ARMEB_LINUX_ROOTFS}/bin/armeb_udp_test', '20014'], ARMEB_LINUX_ROOTFS, multithread=True, verbose=QL_VERBOSE.DEBUG) + + ql.os.stats = QlOsNullStats() ql.os.set_syscall("write", check_write, QL_INTERCEPT.ENTER) ql.run() - self.assertEqual("server sendto() 14 return 14.\n", ql.buf_out) + self.assertGreaterEqual(len(logged), 2) + self.assertTrue(logged[-2].startswith('server recvfrom()')) + self.assertTrue(logged[-1].startswith('server sendto()')) + + # the server is expected to send the value it received, for example: + # 'server recvfrom() return 14.\n' + # 'server sendto() 14 return 14.\n' + + m = re.search(r'(?P\d+)\.\s+\Z', logged[-2]) + self.assertIsNotNone(m, 'could not extract numeric value from log message') - del ql + num = m.group('num') + msg = logged[-1].strip() + + self.assertTrue(msg.endswith(f'{num} return {num}.')) def test_http_elf_linux_x8664(self): + PORT = 20020 + def picohttpd(): - ql = Qiling(["../examples/rootfs/x8664_linux/bin/picohttpd", "12911"], "../examples/rootfs/x8664_linux", multithread=True, verbose=QL_VERBOSE.DEBUG) + ql = Qiling([fr'{X64_LINUX_ROOTFS}/bin/picohttpd', f'{PORT:d}'], X64_LINUX_ROOTFS, multithread=True, verbose=QL_VERBOSE.DEBUG) ql.run() picohttpd_therad = threading.Thread(target=picohttpd, daemon=True) @@ -357,14 +554,18 @@ def picohttpd(): time.sleep(1) - f = http.client.HTTPConnection('localhost', 12911, timeout=10) - f.request("GET", "/") - response = f.getresponse() - self.assertEqual("httpd_test_successful", response.read().decode()) + conn = http.client.HTTPConnection('localhost', PORT, timeout=10) + conn.request('GET', '/') + + response = conn.getresponse() + feedback = response.read() + self.assertEqual('httpd_test_successful', feedback.decode()) def test_http_elf_linux_arm(self): + PORT = 20021 + def picohttpd(): - ql = Qiling(["../examples/rootfs/arm_linux/bin/picohttpd", "12912"], "../examples/rootfs/arm_linux", multithread=True, verbose=QL_VERBOSE.DEBUG) + ql = Qiling([fr'{ARM_LINUX_ROOTFS}/bin/picohttpd', f'{PORT:d}'], ARM_LINUX_ROOTFS, multithread=True, verbose=QL_VERBOSE.DEBUG) ql.run() picohttpd_therad = threading.Thread(target=picohttpd, daemon=True) @@ -372,14 +573,18 @@ def picohttpd(): time.sleep(1) - f = http.client.HTTPConnection('localhost', 12912, timeout=10) - f.request("GET", "/") - response = f.getresponse() - self.assertEqual("httpd_test_successful", response.read().decode()) + conn = http.client.HTTPConnection('localhost', PORT, timeout=10) + conn.request('GET', '/') + + response = conn.getresponse() + feedback = response.read() + self.assertEqual('httpd_test_successful', feedback.decode()) def test_http_elf_linux_armeb(self): + PORT = 20022 + def picohttpd(): - ql = Qiling(["../examples/rootfs/armeb_linux/bin/picohttpd", "12913"], "../examples/rootfs/armeb_linux", multithread=True, verbose=QL_VERBOSE.DEBUG) + ql = Qiling([fr'{ARMEB_LINUX_ROOTFS}/bin/picohttpd', f'{PORT:d}'], ARMEB_LINUX_ROOTFS, multithread=True, verbose=QL_VERBOSE.DEBUG) ql.run() picohttpd_thread = threading.Thread(target=picohttpd, daemon=True) @@ -387,19 +592,23 @@ def picohttpd(): time.sleep(1) - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.connect(("localhost", 12913)) - s.sendall(b"GET / HTTP/1.1\r\nHost: 127.0.0.1:12913\r\nUser-Agent: curl/7.74.0\r\nAccept: */*\r\n\r\n") - data = s.recv(1024) + # armeb libc uses statx to query stdout stats, but fails because 'stdout' is not a valid + # path on the hosting paltform. it prints out the "Server started" message, but stdout is + # not found and the message is kept buffered in. + # + # later on, picohttpd dups the client socket into stdout fd and uses ordinary printf to + # send data out. however, when the "successful" message is sent, it is sent along with + # the buffered message, which arrives first and raises a http.client.BadStatusLine exception + # as it reads as a malformed http response. + # + # here we use a raw 'recv' method instead of 'getresponse' to work around that. - res = data.decode("UTF-8",'replace') - self.assertIn("httpd_test_successful", res) + conn = http.client.HTTPConnection('localhost', PORT, timeout=10) + conn.request('GET', '/') + + feedback = conn.sock.recv(96).decode() + self.assertTrue(feedback.endswith('httpd_test_successful')) if __name__ == "__main__": unittest.main() - - - - - From 073950a3e851eb4563dc19070e2345ee03453dbf Mon Sep 17 00:00:00 2001 From: elicn Date: Sun, 23 Apr 2023 21:04:51 +0300 Subject: [PATCH 37/42] Fix bugs in IPv6 socket impl --- qiling/os/posix/syscall/socket.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/qiling/os/posix/syscall/socket.py b/qiling/os/posix/syscall/socket.py index 01068aacc..d02b8cfec 100644 --- a/qiling/os/posix/syscall/socket.py +++ b/qiling/os/posix/syscall/socket.py @@ -27,10 +27,10 @@ def inet_aton(ipaddr: str) -> int: return int.from_bytes(ipdata, byteorder='big') -def inet6_aton(ipaddr: str) -> int: - ipdata = ipaddress.IPv6Address(ipaddr).packed +def inet6_aton(ipaddr: str) -> Tuple[int, ...]: + abytes = ipaddress.IPv6Address(ipaddr).packed - return int.from_bytes(ipdata, byteorder='big') + return tuple(abytes) def inet_htoa(ql: Qiling, addr: int) -> str: @@ -52,6 +52,10 @@ def inet6_htoa(ql: Qiling, addr: bytes) -> str: def inet6_ntoa(addr: bytes) -> str: + # if addr arg is not strictly a bytes object, convert it to bytes + if not isinstance(addr, bytes): + addr = bytes(addr) + return ipaddress.IPv6Address(addr).compressed @@ -260,10 +264,10 @@ def ql_syscall_connect(ql: Qiling, sockfd: int, addr: int, addrlen: int): dest = (host, port) elif sock.family == AF_INET6 and ql.os.ipv6: - sockaddr_in6 = make_sockaddr_in(abits, endian) + sockaddr_in6 = make_sockaddr_in6(abits, endian) sockaddr_obj = sockaddr_in6.from_buffer(data) - port = ntohs(ql, sockaddr_obj.sin_port) + port = ntohs(ql, sockaddr_obj.sin6_port) host = inet6_htoa(ql, sockaddr_obj.sin6_addr.s6_addr) ql.log.debug(f'Connecting to {host}:{port}') @@ -409,10 +413,10 @@ def ql_syscall_bind(ql: Qiling, sockfd: int, addr: int, addrlen: int): dest = (host, port) elif sa_family == AF_INET6 and ql.os.ipv6: - sockaddr_in6 = make_sockaddr_in(abits, endian) + sockaddr_in6 = make_sockaddr_in6(abits, endian) sockaddr_obj = sockaddr_in6.from_buffer(data) - port = ntohs(ql, sockaddr_obj.sin_port) + port = ntohs(ql, sockaddr_obj.sin6_port) host = inet6_ntoa(sockaddr_obj.sin6_addr.s6_addr) if ql.os.bindtolocalhost: @@ -879,10 +883,10 @@ def ql_syscall_sendto(ql: Qiling, sockfd: int, buf: int, length: int, flags: int dest = (host, port) elif sa_family == AF_INET6 and ql.os.ipv6: - sockaddr_in6 = make_sockaddr_in(abits, endian) + sockaddr_in6 = make_sockaddr_in6(abits, endian) sockaddr_obj = sockaddr_in6.from_buffer(data) - port = ntohs(ql, sockaddr_obj.sin_port) + port = ntohs(ql, sockaddr_obj.sin6_port) host = inet6_ntoa(sockaddr_obj.sin6_addr.s6_addr) ql.log.debug(f'Sending to {host}:{port}') From 062bb576585dd1a288624a616a2e65d8dabda4c7 Mon Sep 17 00:00:00 2001 From: elicn Date: Fri, 28 Apr 2023 17:03:43 +0300 Subject: [PATCH 38/42] Remove unused log colors --- qiling/log.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/qiling/log.py b/qiling/log.py index 46026b8db..239174ce4 100644 --- a/qiling/log.py +++ b/qiling/log.py @@ -26,14 +26,12 @@ class COLOR: - WHITE = '\033[37m' CRIMSON = '\033[31m' RED = '\033[91m' GREEN = '\033[92m' YELLOW = '\033[93m' BLUE = '\033[94m' MAGENTA = '\033[95m' - CYAN = '\033[96m' ENDC = '\033[0m' @@ -48,7 +46,8 @@ class QlBaseFormatter(Formatter): def __init__(self, ql: Qiling, *args, **kwargs): super().__init__(*args, **kwargs) - self.ql = weakref.proxy(ql) + + self.ql: Qiling = weakref.proxy(ql) def get_level_tag(self, level: str) -> str: return self.__level_tag[level] From 26e4786e37cd9cd7df346f8d10138cd4bb39d3b7 Mon Sep 17 00:00:00 2001 From: elicn Date: Fri, 28 Apr 2023 17:05:25 +0300 Subject: [PATCH 39/42] Switch back to default color instead reset entirely --- qiling/log.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qiling/log.py b/qiling/log.py index 239174ce4..66feb6bae 100644 --- a/qiling/log.py +++ b/qiling/log.py @@ -32,7 +32,7 @@ class COLOR: YELLOW = '\033[93m' BLUE = '\033[94m' MAGENTA = '\033[95m' - ENDC = '\033[0m' + DEFAULT = '\033[39m' class QlBaseFormatter(Formatter): @@ -85,12 +85,12 @@ class QlColoredFormatter(QlBaseFormatter): def get_level_tag(self, level: str) -> str: s = super().get_level_tag(level) - return f'{self.__level_color[level]}{s}{COLOR.ENDC}' + return f'{self.__level_color[level]}{s}{COLOR.DEFAULT}' def get_thread_tag(self, tid: str) -> str: s = super().get_thread_tag(tid) - return f'{COLOR.GREEN}{s}{COLOR.ENDC}' + return f'{COLOR.GREEN}{s}{COLOR.DEFAULT}' class RegexFilter(Filter): From 45056c99f46e27ed56abf748211d06d4eae392bf Mon Sep 17 00:00:00 2001 From: elicn Date: Fri, 28 Apr 2023 17:05:50 +0300 Subject: [PATCH 40/42] Adhere to the NO_COLOR convention --- qiling/log.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/qiling/log.py b/qiling/log.py index 66feb6bae..fd2a00a60 100644 --- a/qiling/log.py +++ b/qiling/log.py @@ -177,7 +177,10 @@ def setup_logger(ql: Qiling, log_file: Optional[str], console: bool, log_overrid if console: handler = StreamHandler() - if log_plain or not __is_color_terminal(handler.stream): + # adhere to the NO_COLOR convention (see: https://no-color.org/) + no_color = os.getenv('NO_COLOR', False) + + if no_color or log_plain or not __is_color_terminal(handler.stream): formatter = QlBaseFormatter(ql, FMT_STR) else: formatter = QlColoredFormatter(ql, FMT_STR) From 8a364a51dbfd40301e25e73330b18d6be2616462 Mon Sep 17 00:00:00 2001 From: elicn Date: Fri, 28 Apr 2023 17:06:12 +0300 Subject: [PATCH 41/42] Slightly optimize logging for speed --- qiling/log.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/qiling/log.py b/qiling/log.py index fd2a00a60..7303afeeb 100644 --- a/qiling/log.py +++ b/qiling/log.py @@ -200,6 +200,12 @@ def setup_logger(ql: Qiling, log_file: Optional[str], console: bool, log_overrid log.setLevel(logging.INFO) + # optimize logging speed by avoiding the collection of unnecesary logging properties + logging._srcfile = None + logging.logThreads = False + logging.logProcesses = False + logging.logMultiprocessing = False + return log From 4c53a432c4943f4861f9298aa0f9be162346a3ac Mon Sep 17 00:00:00 2001 From: elicn Date: Fri, 28 Apr 2023 17:08:12 +0300 Subject: [PATCH 42/42] Handle fds that lack the close_on_exec property --- qiling/os/posix/syscall/unistd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiling/os/posix/syscall/unistd.py b/qiling/os/posix/syscall/unistd.py index 901199b07..a70e83a28 100644 --- a/qiling/os/posix/syscall/unistd.py +++ b/qiling/os/posix/syscall/unistd.py @@ -615,7 +615,7 @@ def __read_str_array(addr: int) -> Iterator[str]: for i in range(NR_OPEN): f = ql.os.fd[i] - if f and f.close_on_exec and not f.closed: + if f and getattr(f, 'close_on_exec', False) and not f.closed: f.close() ql.os.fd[i] = None