Merge pull request #165 from Wenzel/python_bindings

pyapi: generate basic python bindings with PyO3

.github/workflows/ci.yml

@@ -222,6 +222,51 @@ jobs:
           token: ${{ secrets.GITHUB_TOKEN }}
           args: --examples -- -D warnings
 
+  python:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: install stable toolchain with clippy
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          components: clippy
+
+      - name: Set up Python 3.7 🐍
+        uses: actions/setup-python@v1
+        with:
+          python-version: '3.7'
+
+      - uses: actions/checkout@v1
+
+      - name: install nox and check formatting, linting, type checking and testing
+        run: |
+          pip install nox
+          nox -r
+        working-directory: python
+
+      - name: build Python extension
+        uses: actions-rs/cargo@v1
+        with:
+          command: build
+          args: -p pymicrovmi
+
+      - name: annotate commit with clippy warnings
+        uses: actions-rs/clippy-check@v1
+        with:
+          name: clippy python
+          token: ${{ secrets.GITHUB_TOKEN }}
+          args: -p pymicrovmi -- -D warnings
+
+      - name: install Python package
+        run: |
+          python -m pip install -r requirements.txt
+          ./setup.py install
+        working-directory: python
+
+      - name: smoke test
+        run: python3 -c 'from microvmi import Microvmi, DriverType, DriverInitParam'
+
   debian_package:
     # create a Debian package with cargo deb
     needs: [xen, kvm, virtualbox, hyper-v]
@@ -290,7 +335,7 @@ jobs:
     # only when
     # - push on master
     # - tag starts with 'v*'
-    needs: [format, xen, kvm, virtualbox, hyper-v, c_api, examples]
+    needs: [format, xen, kvm, virtualbox, hyper-v, c_api, examples, python]
     runs-on: ubuntu-20.04
     # output these value to be used by other jobs so they can add assets
     outputs:
@@ -413,3 +458,25 @@ jobs:
         shell: bash
         run: |
           cargo publish --features xen --token ${{ secrets.CRATES_TOKEN }}
+
+
+  publish_pypi:
+    needs: release
+    runs-on: ubuntu-20.04
+
+    steps:
+      - uses: actions/checkout@v1
+
+      - name: install nox
+        run: pip install nox
+
+      - name: Build Wheels with manylinux
+        run: nox -r -s generate_wheels -- --features xen,kvm,virtualbox --release
+        working-directory: python
+
+      - name: Publish on PyPI 🚀
+        uses: pypa/gh-action-pypi-publish@v1.3.1
+        with:
+          user: __token__
+          password: ${{ secrets.PYPI_ACCESS_TOKEN }}
+          packages_dir: python/dist/manylinux

.gitignore

@@ -1,6 +1,6 @@
 # IntelliJ IDEA
-/.idea/
-/*.iml
+.idea/
+*.iml
 
 # Generated by Cargo
 # will have compiled files and executables
@@ -20,3 +20,6 @@ Cargo.lock
 
 /target
 #**/*.rs.bk
+#
+# venvs
+venv

Cargo.toml

@@ -45,7 +45,6 @@ widestring = { version = "0.4", optional = true }
 ntapi = { version = "0.3", optional = true }
 vid-sys = { version = "=0.3.0", features = ["deprecated-apis"], optional = true }
 
-
 [dev-dependencies]
 ctrlc = "3.1"
 clap = "2.33"
@@ -71,3 +70,8 @@ assets = [
     ["target/release/libmicrovmi.so", "usr/lib/libmicrovmi.so", "644"],
     ["target/release/capi/libmicrovmi.h", "usr/include/libmicrovmi.h", "644"],
 ]
+
+[workspace]
+members = [
+    "python"
+]

c_examples/mem-dump.c

@@ -24,7 +24,7 @@ void dump_memory(void* driver, const char* vm_name) {
     uint8_t buffer[PAGE_SIZE];
     for (int i = 0; i <= max_address / PAGE_SIZE; i++) {
         memset(buffer, 0, PAGE_SIZE);
-        if (microvmi_read_physical(driver, i * PAGE_SIZE, buffer, PAGE_SIZE)) {
+        if (microvmi_read_physical(driver, i * PAGE_SIZE, buffer, PAGE_SIZE, NULL)) {
             fwrite(buffer, sizeof(uint8_t), PAGE_SIZE, dump_file);
         }
     }

doc/src/SUMMARY.md

@@ -1,18 +1,30 @@
 - [Introduction](./README.md)
 
+
 # Tutorial
 
-- [Introduction](./tutorial/intro.md)
-- [Memory dump example in Rust](./tutorial/rust.md)
-- [Memory dump example in C](./tutorial/c.md)
+- [Installation](./tutorial/installation.md)
+  - [Packaged](./tutorial/packaged.md)
+  - [Source](./tutorial/source.md)
+- [Memory dump example on Xen](./tutorial/mem-dump-example.md)
+  - [Rust](./tutorial/rust.md)
+  - [C](./tutorial/c.md)
+  - [Python](./tutorial/python.md)
+- [Integration](./tutorial/integration.md)
+  - [volatility3](./tutorial/integration/volatility3.md)
 
 # Reference
 
 - [API](./reference/api.md)
-- [Build Options](./reference/build_options.md)
+- [Python API](./reference/python_api.md)
+- [Drivers](./reference/drivers.md)
 - [Integration Status](./reference/integration_status.md)
 
 # Explanation
 
 - [VMI API](./explanation/vmi_api.md)
 - [VMI Fragmentation](./explanation/vmi_ecosystem.md)
+
+# Developer
+
+- [Python](./developer/python.md)

doc/src/developer/python.md

@@ -0,0 +1,42 @@
+# Python Bindings
+
+## Nox
+
+The project uses [Nox](https://nox.thea.codes/en/stable/) to facilite and automate the developer workflow.
+Please install this tool before you start
+
+Running `nox` without any argument will run the default sessions.
+
+## Generating the Wheels
+
+Distributing a Python native extension compatible with many systems and a large set of Python interpreters is a challenging task.
+
+The [manylinux](https://github.com/pypa/manylinux) project comes to the rescue here.
+
+The extension is built based on the [`manylinux2014`](https://www.python.org/dev/peps/pep-0599/) platform tag.
+
+Generation of the wheels is managed by `nox` and requires [`Docker`](https://www.docker.com/) to build a custom manylinux2014 CentOS image, and
+execute the script inside it.
+
+To start the generation:
+
+~~~
+$ cd libmicrovmi/python
+$ nox -r -s generate_wheels -- --features xen
+~~~
+
+you can activate more drivers
+
+~~~
+$ nox -r -s generate_wheels -- --features xen,kvm,virtualbox
+~~~
+
+and enable the release mode as well
+
+~~~
+nox -r -s generate_wheels -- --features xen --release
+~~~
+
+After the execution, the wheels will be available in `libmicrovmi/python/dist/manylinux`.
+
+

doc/src/reference/drivers.md

@@ -0,0 +1,51 @@
+# Drivers
+
+This section documents the drivers available and the requirements to compile them.
+
+## Features
+
+| Feature       | Description                 |
+|--------------|-----------------------------|
+| `xen`        | Build the Xen driver        |
+| `kvm`        | Build the KVM driver        |
+| `virtualbox` | Build the VirtualBox driver |
+| `hyper-v`    | Build the Hyper-V driver    |
+
+Example
+~~~
+$ cargo build --features xen,kvm
+~~~
+
+## Xen
+
+~~~
+$ sudo apt install clang libxen-dev
+~~~
+
+Compatibility: Xen >= 4.11.0
+
+## KVM
+
+The KVM driver depends on [libkvmi](https://github.com/bitdefender/libkvmi)
+
+~~~
+$ git clone https://github.com/bitdefender/libkvmi.git
+$ cd libkvmi
+$ git checkout bf5776319e1801b59125c994c459446f0ed6837e
+$ ./bootstrap
+$ ./configure
+$ make
+$ sudo make install
+~~~
+
+## VirtualBox
+
+The VirtualBox driver depends on [libFDP](https://github.com/thalium/icebox/tree/master/src/FDP)
+
+~~~
+$ git clone --depth 1 https://github.com/thalium/icebox
+$ cd icebox/src/FDP
+$ g++ -std=c++11 -shared -fPIC FDP.cpp -o libFDP.so
+$ sudo mv include/* /usr/local/include/
+$ sudo mv libFDP.so /usr/local/lib/
+~~~

doc/src/reference/integration_status.md

@@ -3,11 +3,17 @@
 This section gives a status overview of libmicrovmi's integration into
 other applications and libraries based on _virtual machine introspection_.
 
+|   Project   | Supported |
+|:-----------:|:---------:|
+|    LibVMI   |     🟠    |
+| volatility3 |     ✅     |
+
 ## LibVMI
 
 - [project](https://github.com/libvmi/libvmi)
 - [issue](https://github.com/Wenzel/libmicrovmi/issues/137)
 - [fork](https://github.com/Wenzel/libvmi/tree/libmicrovmi) (Note: use the `libmicrovmi` branch)
+- compatibility: 🟠
 
 ### API Compatibility Status
 
@@ -72,3 +78,14 @@ The API is used in the following files:
 | `driver_set_access_required()`            |            |              |
 | `get_data()`                              |     ✅     |              |
 | `release_data()`                          |     ✅     |              |
+
+## volatility3
+
+volatility3 could inspect and run its forensics plugins on live memory by
+adding a new URL handler.
+
+- [project](https://github.com/volatilityfoundation/volatility3)
+- [issue](https://github.com/Wenzel/libmicrovmi/issues/161)
+- [fork](https://github.com/Wenzel/volatility3/tree/vm_introspection_handler) (Note: use the `vm_introspection_handler` branch)
+- compatibility: ✅
+

doc/src/reference/python_api.md

@@ -0,0 +1,26 @@
+# Python API
+
+### Initializing libmicrovmi
+
+~~~Python
+from microvmi import Microvmi
+
+micro = Microvmi("Windows10")
+~~~
+
+### Specifying the hypervisor
+
+~~~Python
+from microvmi import Microvmi, DriverType
+
+micro = Microvmi("Windows10", DriverType.XEN)
+~~~
+
+### Adding driver initialization parameters
+
+~~~Python
+from microvmi import Microvmi, DriverType, DriverInitParam
+
+init_param = DriverInitParam.kvmi_unix_socket("/tmp/introspector")
+micro = Microvmi("Windows10", DriverType.KVM, init_param)
+~~~

doc/src/tutorial/installation.md

@@ -0,0 +1,5 @@
+# Installation
+
+This section will teach you how to:
+
+- install libmicrovmi either from official repositories or source code

doc/src/tutorial/integration.md

@@ -0,0 +1,7 @@
+# Integration
+
+libmicrovmi aims to be combined with a great diversity of VMI projects.
+
+In this section you will find detailed tutorials to integrate and exploit your favorite tools on top of libmicrovmi.
+
+Please refer to the [Integration Status](https://wenzel.github.io/libmicrovmi/reference/integration_status.html) page for an overview of libmicrovmi's compatibility with existing integrations.