Add BPF support & C-based BPF tic-tac-toe (#1422)

Add initial support for BPF and a C port of tictactoe

Cargo.toml

@@ -66,12 +66,13 @@ path = "src/bin/wallet.rs"
 codecov = { repository = "solana-labs/solana", branch = "master", service = "github" }
 
 [features]
-unstable = []
-ipv6 = []
+bpf_c = []
+chacha = []
 cuda = []
 erasure = []
+ipv6 = []
 test = []
-chacha = []
+unstable = []
 
 [dependencies]
 atty = "0.2"
@@ -82,6 +83,7 @@ bytes = "0.4"
 chrono = { version = "0.4.0", features = ["serde"] }
 clap = "2.31"
 dirs = "1.0.2"
+elf = "0.0.10"
 env_logger = "0.5.12"
 generic-array = { version = "0.12.0", default-features = false, features = ["serde"] }
 getopts = "0.2"
@@ -98,6 +100,7 @@ matches = "0.1.6"
 nix = "0.11.0"
 pnet_datalink = "0.21.0"
 rand = "0.5.1"
+rbpf = { git = "https://github.com/solana-labs/rbpf" }
 rayon = "1.0.0"
 reqwest = "0.9.0"
 ring = "0.13.2"
@@ -114,9 +117,8 @@ tokio-codec = "0.1"
 untrusted = "0.6.2"
 
 [dev-dependencies]
-noop = { path = "programs/noop" }
-print = { path = "programs/print" }
-move_funds = { path = "programs/move_funds" }
+move_funds = { path = "programs/native/move_funds" }
+noop = { path = "programs/native/noop" }
 
 [[bench]]
 name = "bank"
@@ -141,8 +143,7 @@ name = "chacha"
 members = [
     ".",
     "common",
-    "programs/noop",
-    "programs/print",
-    "programs/move_funds",
+    "programs/native/move_funds",
+    "programs/native/noop",
+    "programs/bpf/noop_rust",
 ]
-

build.rs

@@ -1,5 +1,6 @@
 use std::env;
 use std::fs;
+use std::process::Command;
 
 fn main() {
     println!("cargo:rerun-if-changed=target/perf-libs");
@@ -14,11 +15,33 @@ fn main() {
         }
     });
 
+    let bpf_c = !env::var("CARGO_FEATURE_BPF_C").is_err();
+    let chacha = !env::var("CARGO_FEATURE_CHACHA").is_err();
     let cuda = !env::var("CARGO_FEATURE_CUDA").is_err();
     let erasure = !env::var("CARGO_FEATURE_ERASURE").is_err();
-    let chacha = !env::var("CARGO_FEATURE_CHACHA").is_err();
 
-    if cuda || erasure || chacha {
+    if bpf_c {
+        let out_dir = "target/".to_string() + &env::var("PROFILE").unwrap();
+
+        println!("cargo:rerun-if-changed=programs/bpf/move_funds_c/build.sh");
+        println!("cargo:rerun-if-changed=programs/bpf/move_funds_c/src/move_funds.c");
+        println!("cargo:warning=(not a warning) Compiling move_funds_c");
+        let status = Command::new("programs/bpf/move_funds_c/build.sh")
+            .arg(&out_dir)
+            .status()
+            .expect("Failed to call move_funds_c build script");
+        assert!(status.success());
+
+        println!("cargo:rerun-if-changed=programs/bpf/tictactoe_c/build.sh");
+        println!("cargo:rerun-if-changed=programs/bpf/tictactoe_c/src/tictactoe.c");
+        println!("cargo:warning=(not a warning) Compiling tictactoe_c");
+        let status = Command::new("programs/bpf/tictactoe_c/build.sh")
+            .arg(&out_dir)
+            .status()
+            .expect("Failed to call tictactoe_c build script");
+        assert!(status.success());
+    }
+    if chacha || cuda || erasure {
         println!("cargo:rustc-link-search=native=target/perf-libs");
     }
     if cuda {

common/src/account.rs

@@ -1,6 +1,7 @@
 use pubkey::Pubkey;
 
 /// An Account with userdata that is stored on chain
+#[repr(C)]
 #[derive(Serialize, Deserialize, Debug, Clone, Default)]
 pub struct Account {
     /// tokens in the account
@@ -22,6 +23,7 @@ impl Account {
     }
 }
 
+#[repr(C)]
 #[derive(Debug)]
 pub struct KeyedAccount<'a> {
     pub key: &'a Pubkey,

common/src/pubkey.rs

@@ -3,6 +3,7 @@ use generic_array::typenum::U32;
 use generic_array::GenericArray;
 use std::fmt;
 
+#[repr(C)]
 #[derive(Serialize, Deserialize, Clone, Copy, Default, Eq, PartialEq, Ord, PartialOrd, Hash)]
 pub struct Pubkey(GenericArray<u8, U32>);
 

programs/bpf/move_funds_c/build.sh

@@ -0,0 +1,9 @@
+#!/bin/bash -ex
+
+OUTDIR="${1:-../../../target/release/}"
+THISDIR=$(dirname "$0")
+mkdir -p "$OUTDIR"
+/usr/local/opt/llvm/bin/clang -Werror -target bpf -O2 -emit-llvm -fno-builtin -o "$OUTDIR"/move_funds_c.bc -c "$THISDIR"/src/move_funds.c
+/usr/local/opt/llvm/bin/llc -march=bpf -filetype=obj -function-sections -o "$OUTDIR"/move_funds_c.o "$OUTDIR"/move_funds_c.bc
+
+#/usr/local/opt/llvm/bin/llvm-objdump -color -source -disassemble "$OUTDIR"/move_funds_c.o

programs/bpf/move_funds_c/dump.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+/usr/local/opt/llvm/bin/llvm-objdump -color -source -disassemble ../../../target/release/move_funds_c.o

programs/bpf/move_funds_c/src/move_funds.c

@@ -0,0 +1,130 @@
+
+//#include <stdint.h>
+//#include <stddef.h>
+
+#if 1
+// one way to define a helper function is with index as a fixed value
+#define BPF_TRACE_PRINTK_IDX 6
+static int (*sol_print)(int, int, int, int, int) = (void *)BPF_TRACE_PRINTK_IDX;
+#else
+// relocation is another option
+extern int sol_print(int, int, int, int, int);
+#endif
+
+typedef long long unsigned int uint64_t;
+typedef long long int int64_t;
+typedef unsigned char uint8_t;
+
+typedef enum { false = 0, true } bool;
+
+#define SIZE_PUBKEY 32
+typedef struct {
+    uint8_t x[SIZE_PUBKEY];
+} SolPubkey;
+
+typedef struct {
+    SolPubkey *key;
+    int64_t tokens;
+    uint64_t userdata_len;
+    uint8_t *userdata;
+    SolPubkey *program_id;
+} SolKeyedAccounts;
+
+// TODO support BPF function calls rather then forcing everything to be inlined
+#define SOL_FN_PREFIX __attribute__((always_inline)) static
+
+// TODO move this to a registered helper
+SOL_FN_PREFIX void sol_memcpy(void *dst, void *src, int len) {
+    for (int i = 0; i < len; i++) {
+        *((uint8_t *)dst + i) = *((uint8_t *)src + i);
+    }
+}
+
+#define sol_panic() _sol_panic(__LINE__)
+SOL_FN_PREFIX void _sol_panic(uint64_t line) {
+    sol_print(0, 0, 0xFF, 0xFF, line);
+    char *pv = (char *)1;
+    *pv = 1;
+}
+
+SOL_FN_PREFIX int sol_deserialize(uint8_t *src, uint64_t num_ka, SolKeyedAccounts *ka,
+                              uint8_t **userdata, uint64_t *userdata_len) {
+    if (num_ka != *(uint64_t *)src) {
+        return -1;
+    }
+    src += sizeof(uint64_t);
+
+    // TODO fixed iteration loops ok? unrolled?
+    for (int i = 0; i < num_ka; i++) { // TODO this should end up unrolled, confirm
+        // key
+        ka[i].key = (SolPubkey *)src;
+        src += SIZE_PUBKEY;
+
+        // tokens
+        ka[i].tokens = *(uint64_t *)src;
+        src += sizeof(uint64_t);
+
+        // account userdata
+        ka[i].userdata_len = *(uint64_t *)src;
+        src += sizeof(uint64_t);
+        ka[i].userdata = src;
+        src += ka[i].userdata_len;
+
+        // program_id
+        ka[i].program_id = (SolPubkey *)src;
+        src += SIZE_PUBKEY;
+    }
+    // tx userdata
+    *userdata_len = *(uint64_t *)src;
+    src += sizeof(uint64_t);
+    *userdata = src;
+
+    return 0;
+}
+
+
+// -- Debug --
+
+SOL_FN_PREFIX void print_key(SolPubkey *key) {
+    for (int j = 0; j < SIZE_PUBKEY; j++) {
+        sol_print(0, 0, 0, j, key->x[j]);
+    }
+}
+
+SOL_FN_PREFIX void print_userdata(uint8_t *data, int len) {
+    for (int j = 0; j < len; j++) {
+        sol_print(0, 0, 0, j, data[j]);
+    }
+}
+
+SOL_FN_PREFIX void print_params(uint64_t num_ka, SolKeyedAccounts *ka,
+                                uint8_t *userdata, uint64_t userdata_len) {
+    sol_print(0, 0, 0, 0, num_ka);
+    for (int i = 0; i < num_ka; i++) {
+        // key
+        print_key(ka[i].key);
+
+        // tokens
+        sol_print(0, 0, 0, 0, ka[i].tokens);
+
+        // account userdata
+        print_userdata(ka[i].userdata, ka[i].userdata_len);
+
+        // program_id
+        print_key(ka[i].program_id);
+    }
+    // tx userdata
+    print_userdata(userdata, userdata_len);
+}
+
+void entrypoint(char *buf) {
+    SolKeyedAccounts ka[3];
+    uint64_t userdata_len;
+    uint8_t *userdata;
+
+    if (0 != sol_deserialize((uint8_t *)buf, 3, ka, &userdata, &userdata_len)) {
+        return;
+    }
+
+    print_params(3, ka, userdata, userdata_len);
+}

programs/bpf/noop_rust/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "noop_rust"
+version = "0.1.0"
+authors = ["Jack May <jack@solana.com>"]
+
+[dependencies]
+rbpf = { git = "https://github.com/solana-labs/rbpf" }
+solana = { path = "../../.." }

programs/bpf/noop_rust/build.sh

@@ -0,0 +1,10 @@
+#!/bin/bash -ex
+
+# TODO building release flavor with rust produces a bunch of output .bc files
+INTERDIR=../../../target/release
+OUTDIR="${1:-../../../target/debug/}"
+mkdir -p "$OUTDIR"
+# cargo +nightly rustc --release -- -C panic=abort --emit=llvm-ir
+cargo +nightly rustc --release -- -C panic=abort --emit=llvm-bc
+cp "$INTERDIR"/deps/noop_rust-*.bc "$OUTDIR"/noop_rust.bc 
+/usr/local/opt/llvm/bin/llc -march=bpf -filetype=obj -o "$OUTDIR"/noop_rust.o "$OUTDIR"/noop_rust.bc

programs/bpf/noop_rust/dump.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+/usr/local/opt/llvm/bin/llvm-objdump -color -source -disassemble target/release/noop_rust.o

programs/bpf/noop_rust/src/lib.rs

@@ -0,0 +1,15 @@
+extern crate rbpf;
+
+use std::mem::transmute;
+
+#[no_mangle]
+#[link_section = ".text,entrypoint"] // TODO platform independent needed
+pub extern "C" fn entrypoint(_raw: *mut u8) {
+    let bpf_func_trace_printk = unsafe {
+        transmute::<u64, extern "C" fn(u64, u64, u64, u64, u64)>(
+            rbpf::helpers::BPF_TRACE_PRINTK_IDX as u64,
+        )
+    };
+
+    bpf_func_trace_printk(0, 0, 1, 2, 3);
+}

programs/bpf/tictactoe_c/build.sh

@@ -0,0 +1,9 @@
+#!/bin/bash -ex
+
+OUTDIR="${1:-../../../target/release/}"
+THISDIR=$(dirname "$0")
+mkdir -p "$OUTDIR"
+/usr/local/opt/llvm/bin/clang -Werror -target bpf -O2 -emit-llvm -fno-builtin -o "$OUTDIR"/tictactoe_c.bc -c "$THISDIR"/src/tictactoe.c
+/usr/local/opt/llvm/bin/llc -march=bpf -filetype=obj -function-sections -o "$OUTDIR"/tictactoe_c.o "$OUTDIR"/tictactoe_c.bc
+
+# /usr/local/opt/llvm/bin/llvm-objdump -color -source -disassemble "$OUTDIR"/tictactoe_c.o

programs/bpf/tictactoe_c/dump.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+/usr/local/opt/llvm/bin/llvm-objdump -color -source -disassemble ../../../target/release/tictactoe_c.o

programs/bpf/tictactoe_c/makefile

@@ -0,0 +1,57 @@
+# Notes:
+
+#  TODO needs more investigation into what r10 is used for
+#    -O2 adds changes the generated code from:
+#      main:
+#       0:	b7 01 00 00 00 00 00 00 	r1 = 0
+#       1:	63 1a fc ff 00 00 00 00 	*(u32 *)(r10 - 4) = r1
+#       2:	bf 10 00 00 00 00 00 00 	r0 = r1
+#       3:	95 00 00 00 00 00 00 00 	exit
+#    To:
+#      main:
+#       0:	b7 00 00 00 00 00 00 00 	r0 = 0
+#       1:	95 00 00 00 00 00 00 00 	exit
+
+# - building bpf module that includes stdint.h with clang only (no w/ llc) results in an error
+#  $(TOOLS_DIR)/clang -nostdlib -O2 -fpie -target bpf -o bpf.o -c bpf.c
+#  => n file included from /usr/include/sys/_types/_intptr_t.h:30:
+#  => /usr/include/machine/types.h:37:2: error: architecture not supported
+#  => error architecture not supported
+
+TOOLS_DIR = /usr/local/opt/llvm/bin
+
+test_x86.o: test.c
+	$(TOOLS_DIR)/clang -O2 -fpie -target x86_64 -o $@ -c $<
+
+test_x86.so: test_x86.o
+	$(TOOLS_DIR)/ld.lld --shared -o $@ $<
+
+test_x86_dylib.o: test.c
+	$(TOOLS_DIR)/clang -O2 -fpie -target x86_64-apple-darwin13.0.0 -o $@ -c $<
+
+test_x86.dylib: test_x86_dylib.o
+	/usr/local/opt/llvm/bin/ld64.lld -dylib -lc -arch x86_64 -o $@ $<
+
+# TODO does not work if pulling in stdlib, claims unsupported architecture
+bpf_clang.o: bpf.c
+	$(TOOLS_DIR)/clang -nostdlib -O2 -fpie -target bpf -o bpf.o -c bpf.c
+
+bpf.o: bpf.c
+	$(TOOLS_DIR)/clang -O2 -emit-llvm -c $< -o - | $(TOOLS_DIR)/llc -march=bpf -filetype=obj -o $@
+
+bpf_rust.o: bpf_rust.rs
+	rustc +nightly -C opt-level=2 -C panic=abort --emit llvm-bc $< 
+	$(TOOLS_DIR)/llc -march=bpf -filetype=obj -function-sections -o $@ bpf_rust.bc
+
+dumpall:
+	$(TOOLS_DIR)/llvm-objdump -color -source -disassemble *.o
+
+cleanall:
+	rm -f *.o
+	rm -f *.so
+	rm -f *.dylib
+	rm -f *.ll
+	rm -rf *.bc
+
+all:  bpf_clang.o bpf.o
+