From ad4263134c1733b9c768e0c7381fefb8b5ff8bbe Mon Sep 17 00:00:00 2001 From: hugsy Date: Wed, 20 Mar 2024 09:49:53 -0700 Subject: [PATCH 1/3] new install directive for pyproject --- .github/workflows/build.yml | 3 +-- pyproject.toml | 6 +++++- tests/requirements.txt | 6 ------ 3 files changed, 6 insertions(+), 9 deletions(-) delete mode 100644 tests/requirements.txt diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1462c50..f936a84 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -57,7 +57,7 @@ jobs: python --version python -m pip --version python -m pip install --upgrade pip setuptools wheel - python -m pip install --user --upgrade . + python -m pip install --user --upgrade .[all] - name: "Post build actions for Windows" if: matrix.os == 'windows-latest' @@ -79,7 +79,6 @@ jobs: - name: "Run tests" run: | - python -m pip install --user --upgrade -r tests/requirements.txt python -m pytest tests/ - name: Publish artifact diff --git a/pyproject.toml b/pyproject.toml index 50285ee..fae5e51 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,12 +44,13 @@ dependencies = [ "Pygments", "lief", "loguru", - "pre-commit", "prompt_toolkit", "PyQt6", ] [project.optional-dependencies] +dev = ["pre-commit", "debugpy", "black"] + tests = [ "pytest", "pytest-cov", @@ -59,6 +60,9 @@ tests = [ "coverage", ] +all = ["cemu[dev,tests]"] + + [project.entry-points.console_scripts] cemu = "cemu.__main__:main" diff --git a/tests/requirements.txt b/tests/requirements.txt deleted file mode 100644 index ef7798f..0000000 --- a/tests/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -pytest -pytest-cov -pytest-xdist -pytest-benchmark -pytest-forked -coverage From 17b6d7eed93e15691425ec8374b094e03a5c888f Mon Sep 17 00:00:00 2001 From: hugsy Date: Wed, 20 Mar 2024 09:51:00 -0700 Subject: [PATCH 2/3] correct reset emulator to restore default memory mapping when switching architecture (+ minor type hint) --- cemu/__main__.py | 21 +++++++++-- cemu/cli/repl.py | 9 +---- cemu/const.py | 9 ++++- cemu/core.py | 11 +++--- cemu/emulator.py | 76 ++++++++++++++++++++++++--------------- cemu/exceptions.py | 2 ++ cemu/memory.py | 15 +++++--- cemu/ui/highlighter.py | 19 +++++----- cemu/ui/mapping.py | 82 ++++++++++++++---------------------------- 9 files changed, 131 insertions(+), 113 deletions(-) create mode 100644 cemu/exceptions.py diff --git a/cemu/__main__.py b/cemu/__main__.py index 3fff75c..6babc41 100644 --- a/cemu/__main__.py +++ b/cemu/__main__.py @@ -2,19 +2,36 @@ import pathlib import sys +import os import cemu.const import cemu.core import cemu.log +def setup_remote_debug(port: int = cemu.const.DEBUG_DEBUGPY_PORT): + assert cemu.const.DEBUG + import debugpy + + debugpy.listen(port) + cemu.log.dbg("Waiting for debugger attach") + debugpy.wait_for_client() + cemu.log.dbg("Client connected, resuming session") + + def main(): + if bool(os.getenv("DEBUG", False)) or "--debug" in sys.argv: + cemu.const.DEBUG = True + if cemu.const.DEBUG: cemu.log.register_sink(print) cemu.log.dbg("Starting in Debug Mode") - if len(sys.argv) >= 2 and sys.argv[1] == "cli": - cemu.core.CemuCli(sys.argv[2:]) + if "--attach" in sys.argv: + setup_remote_debug() + + if "--cli" in sys.argv: + cemu.core.CemuCli(sys.argv) return cemu.core.CemuGui(sys.argv) diff --git a/cemu/cli/repl.py b/cemu/cli/repl.py index fadf805..a4b197d 100644 --- a/cemu/cli/repl.py +++ b/cemu/cli/repl.py @@ -15,7 +15,7 @@ import cemu.const import cemu.core import cemu.memory -from cemu.emulator import EmulatorState +from cemu.emulator import MEMORY_MAP_DEFAULT_LAYOUT, EmulatorState from cemu.log import dbg, error, info, ok, warn from cemu.utils import hexdump @@ -23,13 +23,6 @@ TEXT_EDITOR = os.getenv("EDITOR") or "nano -c" -MEMORY_MAP_DEFAULT_LAYOUT: list[cemu.memory.MemorySection] = [ - cemu.memory.MemorySection(".text", 0x00004000, 0x1000, "READ|EXEC"), - cemu.memory.MemorySection(".data", 0x00005000, 0x1000, "READ|WRITE"), - cemu.memory.MemorySection(".stack", 0x00006000, 0x4000, "READ|WRITE"), - cemu.memory.MemorySection(".misc", 0x0000A000, 0x1000, "READ|WRITE|EXEC"), -] - @bindings.add("c-c") def _(event): diff --git a/cemu/const.py b/cemu/const.py index 0df476d..ec15bd4 100644 --- a/cemu/const.py +++ b/cemu/const.py @@ -1,6 +1,6 @@ import pathlib -DEBUG = True +DEBUG = False PROGNAME = "cemu" AUTHOR = "hugsy" @@ -30,6 +30,8 @@ CONFIG_FILEPATH = HOME / ".cemu.ini" DEFAULT_STYLE_PATH = STYLE_PATH / "default.qss" +DEBUG_DEBUGPY_PORT = 5678 + LOG_INSERT_TIMESTAMP = False LOG_DEFAULT_TIMESTAMP_FORMAT = "%Y/%m/%d - %H:%M:%S" @@ -49,3 +51,8 @@ DEFAULT_FONT: str = "Courier" DEFAULT_FONT_SIZE: int = 10 + +MEMORY_MAX_SECTION_SIZE: int = 4294967296 # 4GB +MEMORY_TEXT_SECTION_NAME: str = ".text" +MEMORY_STACK_SECTION_NAME: str = ".stack" +MEMORY_DATA_SECTION_NAME: str = ".data" diff --git a/cemu/core.py b/cemu/core.py index d77b648..f235093 100644 --- a/cemu/core.py +++ b/cemu/core.py @@ -5,8 +5,10 @@ import sys from typing import TYPE_CHECKING, Union +import cemu.ui.main + if TYPE_CHECKING: - import cemu.ui.main + from cemu.emulator import Emulator import cemu.arch import cemu.cli.repl @@ -44,7 +46,7 @@ def architecture(self, new_arch: cemu.arch.Architecture): return @property - def emulator(self) -> cemu.emulator.Emulator: + def emulator(self) -> "Emulator": return self.__emulator @property @@ -88,15 +90,13 @@ def CemuGui(args: list[str]) -> None: from PyQt6.QtGui import QIcon from PyQt6.QtWidgets import QApplication - from cemu.ui.main import CEmuWindow - cemu.log.dbg("Creating GUI context") context = GlobalGuiContext() app = QApplication(args) app.setStyleSheet(cemu.const.DEFAULT_STYLE_PATH.open().read()) app.setWindowIcon(QIcon(str(cemu.const.ICON_PATH.absolute()))) - context.root = CEmuWindow(app) + context.root = cemu.ui.main.CEmuWindow(app) sys.exit(app.exec()) @@ -124,4 +124,3 @@ def CemuCli(argv: list[str]) -> None: instance = cemu.cli.repl.CEmuRepl(args) instance.run_forever() - return diff --git a/cemu/emulator.py b/cemu/emulator.py index 2305d5d..2d278fe 100644 --- a/cemu/emulator.py +++ b/cemu/emulator.py @@ -7,24 +7,46 @@ import cemu.const import cemu.core +from cemu.exceptions import CemuEmulatorMissingRequiredSection import cemu.os +from cemu.ui.utils import popup import cemu.utils + from cemu.log import dbg, error, info, warn +from cemu.const import ( + MEMORY_TEXT_SECTION_NAME, + MEMORY_DATA_SECTION_NAME, + MEMORY_STACK_SECTION_NAME, +) + from .arch import is_x86, is_x86_32, x86 from .memory import MemorySection @unique class EmulatorState(IntEnum): - # fmt: off - STARTING = 0 # CEmu is starting - NOT_RUNNING = 1 # CEmu is started, but no emulation context is initialized - IDLE = 2 # The VM is running but stopped: used for stepping mode - RUNNING = 3 # The VM is running - TEARDOWN = 5 # Emulation is finishing - FINISHED = 6 # The VM has reached the end of the execution - # fmt: on + INVALID = 0 + """An invalid state, ideally should never be here""" + STARTING = 1 + """CEmu is starting""" + NOT_RUNNING = 2 + """CEmu is started, but no emulation context is initialized""" + IDLE = 3 + """The VM is running but stopped: used for stepping mode""" + RUNNING = 4 + """The VM is running""" + TEARDOWN = 5 + """Emulation is finishing""" + FINISHED = 6 + """The VM has reached the end of the execution""" + + +MEMORY_MAP_DEFAULT_LAYOUT: list[MemorySection] = [ + MemorySection(MEMORY_TEXT_SECTION_NAME, 0x00004000, 0x1000, "READ|EXEC"), + MemorySection(MEMORY_DATA_SECTION_NAME, 0x00005000, 0x1000, "READ|WRITE"), + MemorySection(MEMORY_STACK_SECTION_NAME, 0x00006000, 0x4000, "READ|WRITE"), +] class EmulationRegisters(collections.UserDict): @@ -83,7 +105,7 @@ def reset(self): self.vm = None self.code = b"" self.codelines = "" - self.sections = [] + self.sections = MEMORY_MAP_DEFAULT_LAYOUT[:] self.registers = EmulationRegisters( {name: 0 for name in cemu.core.context.architecture.registers} ) @@ -158,10 +180,10 @@ def setup(self) -> None: ) if not self.__populate_memory(): - raise Exception("populate_memory() failed") + raise RuntimeError("populate_memory() failed") if not self.__populate_vm_registers(): - raise Exception("populate_registers() failed") + raise RuntimeError("populate_registers() failed") if not self.__populate_text_section(): raise Exception("populate_text_section() failed") @@ -176,7 +198,7 @@ def __populate_memory(self) -> bool: error("VM is not initalized") return False - if len(self.sections) < 0: + if len(self.sections) < 1: error("No section declared") return False @@ -213,7 +235,7 @@ def __populate_vm_registers(self) -> bool: # Set the initial IP if unspecified # if self.registers[arch.pc] == 0: - section = self.find_section(".text") + section = self.find_section(cemu.const.MEMORY_TEXT_SECTION_NAME) self.registers[arch.pc] = section.address warn( f"No value specified for PC register, setting to {self.registers[arch.pc]:#x}" @@ -223,7 +245,7 @@ def __populate_vm_registers(self) -> bool: # Set the initial SP if unspecified, in the middle of the stack section # if self.registers[arch.sp] == 0: - section = self.find_section(".stack") + section = self.find_section(MEMORY_STACK_SECTION_NAME) self.registers[arch.sp] = section.address + (section.size // 2) warn( f"No value specified for SP register, setting to {self.registers[arch.sp]:#x}" @@ -235,7 +257,7 @@ def __populate_vm_registers(self) -> bool: if is_x86_32(arch): # create fake selectors ## required - text = self.find_section(".text") + text = self.find_section(MEMORY_TEXT_SECTION_NAME) self.registers["CS"] = int( x86.X86_32.SegmentDescriptor( text.address >> 8, @@ -246,7 +268,7 @@ def __populate_vm_registers(self) -> bool: ) ) - data = self.find_section(".data") + data = self.find_section(MEMORY_DATA_SECTION_NAME) self.registers["DS"] = int( x86.X86_32.SegmentDescriptor( data.address >> 8, @@ -257,7 +279,7 @@ def __populate_vm_registers(self) -> bool: ) ) - stack = self.find_section(".stack") + stack = self.find_section(MEMORY_STACK_SECTION_NAME) self.registers["SS"] = int( x86.X86_32.SegmentDescriptor( stack.address >> 8, @@ -275,7 +297,6 @@ def __populate_vm_registers(self) -> bool: self.registers["ES"] = 0 for regname, regvalue in self.registers.items(): - # TODO figure out segmentation on unicorn if regname in x86.X86_32.selector_registers: continue @@ -332,16 +353,15 @@ def __populate_text_section(self) -> bool: if not self.vm: return False - try: - text_section = self.find_section(".text") - except KeyError: - # - # Try to get the 1st executable section. Let the exception propagage if it fails - # - matches = [ - section for section in self.sections if section.permission.executable - ] - text_section = matches[0] + for secname in ( + MEMORY_TEXT_SECTION_NAME, + MEMORY_DATA_SECTION_NAME, + MEMORY_STACK_SECTION_NAME, + ): + try: + text_section = self.find_section(secname) + except KeyError: + raise CemuEmulatorMissingRequiredSection(secname) info(f"Using text section {text_section}") diff --git a/cemu/exceptions.py b/cemu/exceptions.py new file mode 100644 index 0000000..afa0278 --- /dev/null +++ b/cemu/exceptions.py @@ -0,0 +1,2 @@ +class CemuEmulatorMissingRequiredSection(Exception): + pass diff --git a/cemu/memory.py b/cemu/memory.py index 04042cf..faf1676 100644 --- a/cemu/memory.py +++ b/cemu/memory.py @@ -4,6 +4,8 @@ import unicorn +from cemu.const import MEMORY_MAX_SECTION_SIZE + MemoryLayoutEntryType = tuple[str, int, int, str, Optional[pathlib.Path]] MEMORY_FIELD_SEPARATOR = "|" @@ -180,22 +182,27 @@ def __init__( name: str, addr: int, size: int, - perm: str, + perm: str | MemoryPermission, data_file: Optional[pathlib.Path] = None, ): - if addr < 0 or addr >= 2**64: + if not (0 <= addr < 2**64): raise ValueError("address") if len(name.strip()) == 0: raise ValueError("name") - if size < 0: + if not (0 < size < MEMORY_MAX_SECTION_SIZE): raise ValueError("size") self.name = name.strip().lower() self.address = addr self.size = size - self.permission = MemoryPermission.from_string(perm) + if isinstance(perm, str): + self.permission = MemoryPermission.from_string(perm) + elif isinstance(perm, MemoryPermission): + self.permission = perm + else: + raise TypeError("permission") self.file_source = data_file if data_file and data_file.is_file() else None return diff --git a/cemu/ui/highlighter.py b/cemu/ui/highlighter.py index 4191369..782fe83 100644 --- a/cemu/ui/highlighter.py +++ b/cemu/ui/highlighter.py @@ -5,6 +5,8 @@ from pygments.lexers import get_lexer_by_name from PyQt6.QtGui import QColor, QFont, QSyntaxHighlighter, QTextCharFormat +import cemu.log + class QFormatter(Formatter): def __init__(self, *args, **kwargs): @@ -26,10 +28,10 @@ def __init__(self, *args, **kwargs): self.styles[str(token)] = qtf return - def hex2QColor(self, c): - red = int(c[0:2], 16) - green = int(c[2:4], 16) - blue = int(c[4:6], 16) + def hex2QColor(self, color: str): + red = int(color[0:2], 16) + green = int(color[2:4], 16) + blue = int(color[4:6], 16) return QColor(red, green, blue) def format(self, tokensource, outfile): @@ -56,13 +58,14 @@ def __init__(self, parent, mode): def highlightBlock(self, text): cb = self.currentBlock() - p = cb.position() + pos = cb.position() text = self.document().toPlainText() + "\n" highlight(text, self.lexer, self.formatter) for i in range(len(text)): try: - self.setFormat(i, 1, self.formatter.data[p + i]) - except IndexError: - pass + self.setFormat(i, 1, self.formatter.data[pos + i]) + except IndexError as ie: + cemu.log.dbg(f"setFormat() failed, reason: {ie}") + break self.tstamp = time.time() return diff --git a/cemu/ui/mapping.py b/cemu/ui/mapping.py index 19c41f7..433c594 100644 --- a/cemu/ui/mapping.py +++ b/cemu/ui/mapping.py @@ -1,5 +1,3 @@ -from __future__ import annotations - from typing import TYPE_CHECKING from PyQt6.QtWidgets import ( @@ -19,7 +17,7 @@ import cemu.core from cemu.emulator import Emulator, EmulatorState -from cemu.log import error +from cemu.log import error, info from cemu.memory import MemorySection from cemu.utils import format_address @@ -29,18 +27,9 @@ from cemu.ui.main import CEmuWindow -MEMORY_MAP_DEFAULT_LAYOUT: list[MemorySection] = [ - MemorySection(".text", 0x00004000, 0x1000, "READ|EXEC"), - MemorySection(".data", 0x00005000, 0x1000, "READ|WRITE"), - MemorySection(".stack", 0x00006000, 0x4000, "READ|WRITE"), - MemorySection(".misc", 0x0000A000, 0x1000, "READ|WRITE|EXEC"), -] - - class MemoryMappingWidget(QDockWidget): - def __init__(self, parent: CEmuWindow): + def __init__(self, parent: "CEmuWindow"): super().__init__("Memory Map", parent) - self.memory_sections = MEMORY_MAP_DEFAULT_LAYOUT layout = QVBoxLayout() @@ -74,55 +63,34 @@ def __init__(self, parent: CEmuWindow): # # Emulator state callback # - emu: Emulator = cemu.core.context.emulator - emu.add_state_change_cb( + self.emu: Emulator = cemu.core.context.emulator + self.emu.add_state_change_cb( EmulatorState.NOT_RUNNING, self.onNotRunningUpdateMemoryMap ) - emu.add_state_change_cb( + self.emu.add_state_change_cb( EmulatorState.RUNNING, self.onRunningDisableMemoryMapGrid ) - emu.add_state_change_cb(EmulatorState.IDLE, self.onIdleEnableMemoryMapGrid) - emu.add_state_change_cb( + self.emu.add_state_change_cb(EmulatorState.IDLE, self.onIdleEnableMemoryMapGrid) + self.emu.add_state_change_cb( EmulatorState.FINISHED, self.onFinishedEnableMemoryMapGrid ) - return def onNotRunningUpdateMemoryMap(self) -> None: - self.SynchronizeMemoryMap() - return + self.redraw_memory_map_table() def onRunningDisableMemoryMapGrid(self) -> None: self.MemoryMapTableWidget.setDisabled(True) - return def onIdleEnableMemoryMapGrid(self) -> None: self.MemoryMapTableWidget.setDisabled(False) - return onFinishedEnableMemoryMapGrid = onIdleEnableMemoryMapGrid - def SynchronizeMemoryMap(self) -> None: - # - # If unset, use a default layout - # - if not self.memory_sections: - self.memory_sections = MEMORY_MAP_DEFAULT_LAYOUT - - # - # Propagate the view change to the emulator - # - cemu.core.context.emulator.sections = self.memory_sections - - # - # Apply the values to the grid - # - self.UpdateMemoryMapGrid() - return - - def UpdateMemoryMapGrid(self) -> None: + def redraw_memory_map_table(self) -> None: + """Re-draw the memory table widget from the emulator sections""" self.MemoryMapTableWidget.clearContents() - for idx, section in enumerate(self.memory_sections): + for idx, section in enumerate(self.emu.sections): self.MemoryMapTableWidget.insertRow(idx) name = QTableWidgetItem(section.name) start_address = QTableWidgetItem(format_address(section.address)) @@ -135,7 +103,7 @@ def UpdateMemoryMapGrid(self) -> None: self.MemoryMapTableWidget.setItem(idx, 2, name) self.MemoryMapTableWidget.setItem(idx, 3, permission) - self.MemoryMapTableWidget.setRowCount(len(self.memory_sections)) + self.MemoryMapTableWidget.setRowCount(len(self.emu.sections)) return def onAddSectionButtonClicked(self) -> None: @@ -152,12 +120,13 @@ def onDeleteSectionButtonClicked(self) -> None: selection = self.MemoryMapTableWidget.selectionModel() if not selection.hasSelection(): return + indexes = [x.row() for x in selection.selectedRows()] - for idx in range(len(self.memory_sections) - 1, 0, -1): + for idx in range(len(self.emu.sections) - 1, 0, -1): if idx in indexes: - del self.memory_sections[idx] - self.UpdateMemoryMapGrid() + del cemu.core.context.emulator.sections[idx] + self.redraw_memory_map_table() return def add_or_edit_section_popup(self) -> None: @@ -179,9 +148,9 @@ def add_or_edit_section_popup(self) -> None: perm = QLabel("Permissions") permCheck = QWidget() permCheckLayout = QHBoxLayout() - perm_read_btn = QCheckBox("R") - perm_write_btn = QCheckBox("W") - perm_exec_btn = QCheckBox("X") + perm_read_btn = QCheckBox("Read") + perm_write_btn = QCheckBox("Write") + perm_exec_btn = QCheckBox("eXecute") permCheckLayout.addWidget(perm_read_btn) permCheckLayout.addWidget(perm_write_btn) permCheckLayout.addWidget(perm_exec_btn) @@ -221,12 +190,13 @@ def add_or_edit_section_popup(self) -> None: address = int(startAddressEdit.text(), 0) size = int(sizeEdit.text(), 0) - if name in (x.name for x in self.memory_sections): + if name in (x.name for x in cemu.core.context.emulator.sections): error("section name already exists") return memory_set = ( - set(range(x.address, x.address + x.size)) for x in self.memory_sections + set(range(x.address, x.address + x.size)) + for x in cemu.core.context.emulator.sections ) current_set = set(range(address, address + size)) for m in memory_set: @@ -243,8 +213,8 @@ def add_or_edit_section_popup(self) -> None: section_perm.append("EXEC") try: section = MemorySection(name, address, size, "|".join(section_perm)) - self.memory_sections.append(section) - self.UpdateMemoryMapGrid() + cemu.core.context.emulator.sections.append(section) + self.redraw_memory_map_table() except ValueError as ve: popup(f"MemorySection is invalid, reason: Invalid {str(ve)}") return @@ -253,8 +223,8 @@ def add_or_edit_section_popup(self) -> None: section_perm.append("EXEC") try: section = MemorySection(name, address, size, "|".join(section_perm)) - self.memory_sections.append(section) - self.UpdateMemoryMapGrid() + cemu.core.context.emulator.sections.append(section) + self.redraw_memory_map_table() except ValueError as ve: popup(f"MemorySection is invalid, reason: Invalid {str(ve)}") From a7c765c509fdfcb0360717301bb7a25cecd7219b Mon Sep 17 00:00:00 2001 From: hugsy Date: Fri, 22 Mar 2024 08:52:53 -0700 Subject: [PATCH 3/3] [emulator] Make sure text section is properly fetched --- .github/workflows/build.yml | 7 ++++++- cemu/emulator.py | 3 ++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2ec0409..0f4ce39 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -48,7 +48,12 @@ jobs: run: | sudo apt update sudo apt upgrade -y - sudo apt install -y build-essential python3-dev python3-pip python3-wheel python3-setuptools + sudo apt install -y build-essential libegl1 libgl1-mesa-glx python3-dev python3-pip python3-wheel python3-setuptools + + - name: "Install Pre-requisite (macOS)" + if: matrix.os == 'macos-latest' + run: | + env - name: "Install Pre-requisite (Windows)" if: matrix.os == 'windows-latest' diff --git a/cemu/emulator.py b/cemu/emulator.py index 2d278fe..893b6fd 100644 --- a/cemu/emulator.py +++ b/cemu/emulator.py @@ -359,10 +359,11 @@ def __populate_text_section(self) -> bool: MEMORY_STACK_SECTION_NAME, ): try: - text_section = self.find_section(secname) + self.find_section(secname) except KeyError: raise CemuEmulatorMissingRequiredSection(secname) + text_section = self.find_section(MEMORY_TEXT_SECTION_NAME) info(f"Using text section {text_section}") if not self.__generate_text_bytecode():