CI setup with Justfiles (#29)

* CI setup with Justfiles * Update Justfile * Update Justfile * Update ci.yml * Update struct_parser.py * Fixed mypy errors
charmoniumQ · Jul 16, 2024 · dc0f8a3 · dc0f8a3
1 parent c685077
commit dc0f8a3
Show file tree

Hide file tree

Showing 5 changed files with 114 additions and 60 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -0,0 +1,45 @@
+name: CI
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    - name: Set up Nix
+      uses: cachix/install-nix-action@v14
+      with:
+        extra_nix_config: |
+          experimental-features = nix-command flakes
+
+    - name: Run Nix development shell
+      run: nix develop --command just format-nix
+
+    - name: Format Python code with Black
+      run: nix develop --command just format-python
+
+    - name: Check Python code with Ruff
+      run: nix develop --command just check-ruff
+
+    - name: Check Python code with Mypy
+      run: nix develop --command just check-mypy
+
+    - name: Run tests
+      run: nix develop --command just test
+
+    - name: Run flake checks (optional)
+      run: nix develop --command just flake-check
+      continue-on-error: true
+
+    - name: Run developer tests
+      if: github.event_name == 'push' && github.event.head_commit.message == 'run dev tests'
+      run: nix develop --command just test-dev
diff --git a/Justfile b/Justfile
@@ -0,0 +1,20 @@
+format-nix:
+    alejandra .
+
+format-python:
+    black probe_src/probe_py
+
+check-ruff:
+    ruff check probe_src/probe_py
+
+check-mypy:
+    (cd probe_src && mypy --package probe_py --strict)
+
+test:
+    python -m pytest probe_src/probe_py
+
+test-dev:
+    python -m pytest probe_src/probe_py --failed-first --maxfail=1
+
+flake-check:
+    nix flake check --all-systems
diff --git a/flake.nix b/flake.nix
@@ -14,17 +14,19 @@
           configureFlags = oldAttrs.configureFlags ++ ["--with-pydebug"];
           # patches = oldAttrs.patches ++ [ ./python.patch ];
         });
-        export-and-rename = pkg: file-pairs: pkgs.stdenv.mkDerivation {
-          pname = "${pkg.pname}-only-bin";
-          dontUnpack = true;
-          version = pkg.version;
-          buildInputs = [ pkg ];
-          buildPhase = builtins.concatStringsSep
-            "\n"
-            (builtins.map
-              (pairs: "install -D ${pkg}/${builtins.elemAt pairs 0} $out/${builtins.elemAt pairs 1}")
-              file-pairs);
-        };
+        export-and-rename = pkg: file-pairs:
+          pkgs.stdenv.mkDerivation {
+            pname = "${pkg.pname}-only-bin";
+            dontUnpack = true;
+            version = pkg.version;
+            buildInputs = [pkg];
+            buildPhase =
+              builtins.concatStringsSep
+              "\n"
+              (builtins.map
+                (pairs: "install -D ${pkg}/${builtins.elemAt pairs 0} $out/${builtins.elemAt pairs 1}")
+                file-pairs);
+          };
       in {
         packages = {
           python-dbg = python312-debug;
@@ -49,6 +51,9 @@
                 pkgs.bash
                 pkgs.alejandra
                 pkgs.hyperfine
+                pkgs.just
+                pkgs.black
+                pkgs.ruff
               ]
               ++ (
                 # gdb broken on apple silicon

diff --git a/probe_src/arena/parse_arena.py b/probe_src/arena/parse_arena.py
@@ -4,8 +4,7 @@
 import dataclasses
 import pathlib
 import ctypes
-import typing
-
+from typing import Sequence
 
 @dataclasses.dataclass(frozen=True)
 class MemorySegment:
@@ -24,19 +23,15 @@ def _check(self) -> None:
     def length(self) -> int:
         return self.stop - self.start
 
-    @typing.overload
-    def __getitem__(self, idx: slice) -> bytes: ...
-
-    @typing.overload
-    def __getitem__(self, idx: int) -> int: ...
-
     def __getitem__(self, idx: slice | int) -> bytes | int:
         if isinstance(idx, slice):
             if not (self.start <= idx.start <= idx.stop <= self.stop):
                 raise IndexError()
             return self.buffr[idx.start - self.start : idx.stop - self.start : idx.step]
         elif isinstance(idx, int):
             return self.buffr[idx - self.start]
+        else:
+            raise TypeError("Invalid index type")
 
     def __contains__(self, idx: int) -> bool:
         return self.start <= idx < self.stop
@@ -55,38 +50,28 @@ def __repr__(self) -> str:
 
 @dataclasses.dataclass(frozen=True)
 class MemorySegments:
-    segments: typing.Sequence[MemorySegment]
+    segments: Sequence[MemorySegment]
 
     def __post_init__(self) -> None:
         self._check()
 
     def _check(self) -> None:
-        # Memory segments *are* allowed to overlap,
-        # Since we can de-allocate arenas,
-        # we can potentially reuse those addresses.
-        # for a, segment_a in enumerate(self.segments):
-        #     for segment_b in self.segments[a + 1:]:
-        #         assert not segment_a.overlaps(segment_b), (segment_a, "overlaps", segment_b)
         assert sorted(self.segments, key=lambda segment: segment.start) == self.segments
 
-    @typing.override
-    def __getitem__(self, idx: slice) -> bytes: ...
-
-    @typing.override
-    def __getitem__(self, idx: int) -> int: ...
-
     def __getitem__(self, idx: slice | int) -> bytes | int:
         if isinstance(idx, slice):
             buffr = b''
             for segment in self.segments:
                 buffr += segment.buffr[max(idx.start, segment.start) - segment.start : min(idx.stop, segment.stop) - segment.start]
             return buffr[::idx.step]
-        else:
+        elif isinstance(idx, int):
             for segment in self.segments:
                 if idx in segment:
                     return segment[idx]
             else:
                 raise IndexError(idx)
+        else:
+            raise TypeError("Invalid index type")
 
     def __contains__(self, idx: int) -> bool:
         return any(idx in segment for segment in self.segments)
@@ -96,7 +81,6 @@ class CArena(ctypes.Structure):
     _fields_ = [
         ("instantiation", ctypes.c_size_t),
         ("base_address", ctypes.c_void_p),
-        # echo -e '#include <stdint.h>\n' | cpp | grep uinptr_t
         ("capacity", ctypes.c_ulong),
         ("used", ctypes.c_ulong),
     ]
@@ -109,8 +93,8 @@ def parse_arena_buffer(buffr: bytes) -> MemorySegment:
     return MemorySegment(buffr[ctypes.sizeof(CArena) : c_arena.used], start, stop)
 
 
-def parse_arena_dir(arena_dir: pathlib.Path) -> typing.Sequence[MemorySegment]:
-    memory_segments = list[MemorySegment]()
+def parse_arena_dir(arena_dir: pathlib.Path) -> Sequence[MemorySegment]:
+    memory_segments = []
     for path in sorted(arena_dir.iterdir()):
         assert path.name.endswith(".dat")
         buffr = path.read_bytes()
@@ -121,8 +105,8 @@ def parse_arena_dir(arena_dir: pathlib.Path) -> typing.Sequence[MemorySegment]:
 def parse_arena_dir_tar(
         arena_dir_tar: tarfile.TarFile,
         prefix: pathlib.Path = pathlib.Path(),
-) -> typing.Sequence[MemorySegment]:
-    memory_segments = list[MemorySegment]()
+) -> Sequence[MemorySegment]:
+    memory_segments = []
     for member in sorted(arena_dir_tar, key=lambda member: member.name):
         member_path = pathlib.Path(member.name)
         if member_path.is_relative_to(prefix) and member_path.relative_to(prefix) != pathlib.Path("."):

diff --git a/probe_src/probe_py/struct_parser.py b/probe_src/probe_py/struct_parser.py
@@ -6,7 +6,7 @@
 import enum
 import textwrap
 import typing
-import pycparser # type: ignore
+import pycparser 
 
 
 _T = typing.TypeVar("_T")
@@ -34,7 +34,7 @@
     ("unsigned", "__int64"): ctypes.c_ulonglong,
     ("size_t",): ctypes.c_size_t,
     ("ssize_t",): ctypes.c_ssize_t,
-    ("time_t",): ctypes.c_time_t, # type: ignore
+    ("time_t",): ctypes.c_time_t, 
     ("float",): ctypes.c_float,
     ("double",): ctypes.c_double,
     ("long", "double",): ctypes.c_longdouble,
@@ -72,7 +72,7 @@ class PyUnionBase:
     ("unsigned", "__int64"): int,
     ("size_t",): int,
     ("ssize_t",): int,
-    ("time_t",): int, # type: ignore
+    ("time_t",): int, 
     ("float",): float,
     ("double",): float,
     ("long", "double",): int,
@@ -177,14 +177,14 @@ def ast_to_cpy_type(
         if isinstance(inner_c_type, Exception):
             c_type = inner_c_type
         else:
-            c_type = int_representing_pointer(inner_c_type)  # type: ignore
+            c_type = int_representing_pointer(inner_c_type)  
         if isinstance(inner_py_type, Exception):
             c_type = inner_py_type
         else:
             if inner_c_type == ctypes.c_char:
                 py_type = str
             else:
-                py_type = list[inner_py_type] # type: ignore
+                py_type = list[inner_py_type] 
         return c_type, py_type
     elif isinstance(typ, pycparser.c_ast.ArrayDecl):
         repetitions = eval_compile_time_int(c_types, py_types, typ.dim, name)
@@ -194,7 +194,7 @@ def ast_to_cpy_type(
         if isinstance(inner_c_type, Exception):
             array_c_type = inner_c_type
         else:
-            array_c_type = inner_c_type * repetitions # type: ignore
+            array_c_type = inner_c_type * repetitions 
         if isinstance(inner_py_type, Exception):
             array_py_type = inner_py_type
         else:
@@ -268,7 +268,7 @@ def parse_struct_or_union(
         zip(field_names, field_py_types),
         bases=(PyStructBase if is_struct else PyUnionBase,),
         frozen=True,
-    ) # type: ignore
+    ) 
 
     if c_type_error is None:
         c_types[(keyword, name)] = type(
@@ -355,7 +355,7 @@ def c_type_to_c_source(c_type: CType, top_level: bool = True) -> str:
             return "\n".join([
                 keyword + " " + c_type.__name__ + " " + "{",
                 *[
-                    textwrap.indent(c_type_to_c_source(field[1], False), "  ") + " " + field[0] + ";"  # type: ignore
+                    textwrap.indent(c_type_to_c_source(field[1], False), "  ") + " " + field[0] + ";"  
                     for field in c_type._fields_
                 ],
                 "}",
@@ -365,9 +365,9 @@ def c_type_to_c_source(c_type: CType, top_level: bool = True) -> str:
     elif isinstance(c_type, CArrayType):
         return c_type_to_c_source(c_type._type_, False) + "[" + str(c_type._length_) + "]"
     elif isinstance(c_type, type(ctypes._Pointer)):
-        return c_type_to_c_source(c_type._type_, False) + "*" # type: ignore
+        return c_type_to_c_source(c_type._type_, False) + "*" 
     elif isinstance(c_type, type(ctypes._SimpleCData)):
-        name = c_type.__name__  # type: ignore
+        name = c_type.__name__  
         return {
             # Ints
             "c_byte": "byte",
@@ -431,35 +431,35 @@ def convert_c_obj_to_py_obj(
     elif c_obj.__class__.__name__ == "PointerStruct":
         assert py_type.__name__ == "list" or py_type is str, (type(c_obj), py_type)
         if py_type.__name__ == "list":
-            inner_py_type = py_type.__args__[0]  # type: ignore
+            inner_py_type = py_type.__args__[0]  
         else:
             inner_py_type = str
         inner_c_type = c_obj.inner_c_type
         size = ctypes.sizeof(inner_c_type)
         pointer_int = _expect_type(int, c_obj.value)
         if pointer_int == 0:
-            return None  # type: ignore
+            return None  
         if pointer_int not in memory:
             raise ValueError(f"Pointer {pointer_int:08x} is outside of memory {memory!s}")
-        lst: inner_py_type = []  # type: ignore
+        lst: inner_py_type = []  
         idx = 0
         while True:
             cont, sub_info = (memory[pointer_int : pointer_int + 1] != b'\0', None) if info is None else info[0](memory, pointer_int)
             if cont:
                 inner_c_obj = inner_c_type.from_buffer_copy(memory[pointer_int : pointer_int + size])
-                inner_py_obj = convert_c_obj_to_py_obj(  # type: ignore
+                inner_py_obj = convert_c_obj_to_py_obj(  
                     inner_c_obj,
                     inner_py_type,
                     sub_info,
                     memory,
                     depth + 1,
                 )
-                lst.append(inner_py_obj)  # type: ignore
+                lst.append(inner_py_obj)  
                 pointer_int += size
             else:
                 break
         if py_type is str:
-            return "".join(lst) # type: ignore
+            return "".join(lst) 
         else:
             return lst
     elif isinstance(c_obj, ctypes.Array):
@@ -490,7 +490,7 @@ def convert_c_obj_to_py_obj(
                 memory,
                 depth + 1,
             )
-        return py_type(**fields)  # type: ignore
+        return py_type(**fields)  
     elif isinstance(c_obj, ctypes.Union):
         if not dataclasses.is_dataclass(py_type):
             raise TypeError(f"If {type(c_obj)} is a union, then {py_type} should be a dataclass")
@@ -509,16 +509,16 @@ def convert_c_obj_to_py_obj(
     elif isinstance(c_obj, ctypes._SimpleCData):
         if isinstance(py_type, enum.EnumType):
             assert isinstance(c_obj.value, int)
-            return py_type(c_obj.value)  # type: ignore
+            return py_type(c_obj.value)  
         elif py_type is str:
             assert isinstance(c_obj, ctypes.c_char)
-            return c_obj.value.decode()  # type: ignore
+            return c_obj.value.decode() 
         else:
             ret = c_obj.value
-            return _expect_type(py_type, ret)  # type: ignore
+            return _expect_type(py_type, ret)  
     elif isinstance(c_obj, py_type):
-        return c_obj  # type: ignore
+        return c_obj  
     elif isinstance(c_obj, int) and isinstance(py_type, enum.EnumType):
-        return py_type(c_obj)  # type: ignore
+        return py_type(c_obj)  
     else:
         raise TypeError(f"{c_obj!r} of c_type {type(c_obj)!r} cannot be converted to py_type {py_type!r}")