From 302ac09126ad44122cc0087f496bb76991467514 Mon Sep 17 00:00:00 2001 From: danbugs Date: Thu, 15 May 2025 19:34:06 -0700 Subject: [PATCH 01/12] [dev/verify-msrv.sh] use 'x86_64-unknown-none' target for guest lib Now, because we no longer support building the guest library for MSVC, we directly specify to use 'x86_64-unknown-none' for it in the script. Otherwise, on Windows, as is, we'd try to use MSVC. Signed-off-by: danbugs --- dev/verify-msrv.sh | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/dev/verify-msrv.sh b/dev/verify-msrv.sh index b49e0acd5..bc72a7fa6 100755 --- a/dev/verify-msrv.sh +++ b/dev/verify-msrv.sh @@ -6,10 +6,21 @@ for CRATE in "$@"; do if ! rustup toolchain list | grep -q "$VERSION"; then rustup --quiet toolchain install "$VERSION" --no-self-update --profile minimal fi - if cargo +$VERSION check --quiet -p $CRATE; then - echo -e " \u001b[1;32m✓\u001b[0m Crate \u001b[1m$CRATE\u001b[0m builds with rust \u001b[1m$VERSION\u001b[0m" + if [[ "$CRATE" == "hyperlight-guest" ]]; then + TARGET="x86_64-unknown-none" + rustup target add "$TARGET" --toolchain "$VERSION" >/dev/null + if cargo +"$VERSION" check --quiet -p "$CRATE" --target "$TARGET"; then + echo -e " \u001b[1;32m✓\u001b[0m Crate \u001b[1m$CRATE\u001b[0m builds with rust \u001b[1m$VERSION\u001b[0m and target \u001b[1m$TARGET\u001b[0m" + else + echo -e " \u001b[1;31m✗\u001b[0m Crate \u001b[1m$CRATE\u001b[0m fails with rust \u001b[1m$VERSION\u001b[0m and target \u001b[1m$TARGET\u001b[0m" + exit 1 + fi else - echo -e " \u001b[1;31m✗\u001b[0m Crate \u001b[1m$CRATE\u001b[0m fails with rust \u001b[1m$VERSION\u001b[0m" - exit 1 + if cargo +"$VERSION" check --quiet -p "$CRATE"; then + echo -e " \u001b[1;32m✓\u001b[0m Crate \u001b[1m$CRATE\u001b[0m builds with rust \u001b[1m$VERSION\u001b[0m" + else + echo -e " \u001b[1;31m✗\u001b[0m Crate \u001b[1m$CRATE\u001b[0m fails with rust \u001b[1m$VERSION\u001b[0m" + exit 1 + fi fi done From 2757a4ad6cb24928c280a628dbb128ead0f8ee2d Mon Sep 17 00:00:00 2001 From: danbugs Date: Thu, 15 May 2025 23:22:06 +0000 Subject: [PATCH 02/12] [guest/setjmp] remove custom setjmp custom impl in favour of musl's Our implementations of setjmp and longjmp conformed to the win64 ABI. With removal of PE support, we can leverage musl's implementation instead. Signed-off-by: danbugs --- src/hyperlight_guest/build.rs | 1 - src/hyperlight_guest/include/setjmp.h | 47 -------- src/hyperlight_guest/src/lib.rs | 1 - src/hyperlight_guest/src/setjmp.rs | 110 ------------------ .../third_party/musl/include/setjmp.h | 49 ++++++++ .../musl/src/setjmp/x86_64/longjmp.s | 18 +++ .../musl/src/setjmp/x86_64/setjmp.s | 22 ++++ 7 files changed, 89 insertions(+), 159 deletions(-) delete mode 100644 src/hyperlight_guest/include/setjmp.h delete mode 100644 src/hyperlight_guest/src/setjmp.rs create mode 100644 src/hyperlight_guest/third_party/musl/include/setjmp.h create mode 100644 src/hyperlight_guest/third_party/musl/src/setjmp/x86_64/longjmp.s create mode 100644 src/hyperlight_guest/third_party/musl/src/setjmp/x86_64/setjmp.s diff --git a/src/hyperlight_guest/build.rs b/src/hyperlight_guest/build.rs index 571cf70d8..03482df79 100644 --- a/src/hyperlight_guest/build.rs +++ b/src/hyperlight_guest/build.rs @@ -121,7 +121,6 @@ fn cargo_main() { copy_includes(&include_dir, "third_party/printf/"); } if cfg!(feature = "libc") { - copy_includes(&include_dir, "include"); copy_includes(&include_dir, "third_party/musl/include"); copy_includes(&include_dir, "third_party/musl/arch/generic"); copy_includes(&include_dir, "third_party/musl/arch/x86_64"); diff --git a/src/hyperlight_guest/include/setjmp.h b/src/hyperlight_guest/include/setjmp.h deleted file mode 100644 index b20bcf1ea..000000000 --- a/src/hyperlight_guest/include/setjmp.h +++ /dev/null @@ -1,47 +0,0 @@ -/* -Copyright 2024 The Hyperlight Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -#ifndef _SETJMP_H -#define _SETJMP_H - -#ifdef __cplusplus -extern "C" { -#endif - -typedef void* jmp_buf[256/sizeof(void *)]; - -#if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1) -#define __returnstwice __attribute__((__returns_twice__)) -#else -#define __returnstwice -#endif - -#if __STDC_VERSION__ >= 201112L -#elif defined(__GNUC__) -#define _Noreturn __attribute__((__noreturn__)) -#else -#define _Noreturn -#endif - -typedef void* jmp_buf[256/sizeof(void *)]; -__returnstwice int setjmp(jmp_buf buf); -_Noreturn int longjmp(jmp_buf buf, int arg); - -#ifdef __cplusplus -} -#endif - -#endif /* _SETJMP_H */ diff --git a/src/hyperlight_guest/src/lib.rs b/src/hyperlight_guest/src/lib.rs index f150f7834..5ef3d2a0e 100644 --- a/src/hyperlight_guest/src/lib.rs +++ b/src/hyperlight_guest/src/lib.rs @@ -42,7 +42,6 @@ pub(crate) mod guest_logger; pub mod memory; pub mod print; pub(crate) mod security_check; -pub mod setjmp; pub mod chkstk; pub mod error; diff --git a/src/hyperlight_guest/src/setjmp.rs b/src/hyperlight_guest/src/setjmp.rs deleted file mode 100644 index 2d9139205..000000000 --- a/src/hyperlight_guest/src/setjmp.rs +++ /dev/null @@ -1,110 +0,0 @@ -// ; source https://raw.githubusercontent.com/koka-lang/libhandler/master/src/asm/setjmp_x64.asm -// ; ---------------------------------------------------------------------------- -// ; Copyright (c) 2016, Microsoft Research, Daan Leijen -// ; This is free software; you can redistribute it and/or modify it under the -// ; terms of the Apache License, Version 2.0. A copy of the License can be -// ; found in the file "license.txt" at the root of this distribution. -// ; ----------------------------------------------------------------------------- - -// ; ------------------------------------------------------- -// ; Code for x64 (x86_64) calling convention as used on Windows -// ; see: -// ; - https://en.wikipedia.org/wiki/X86_calling_conventions -// ; - https://msdn.microsoft.com/en-us/library/ms235286.aspx -// ; - http://www.agner.org/optimize/calling_conventions.pdf -// ; - -use core::arch::global_asm; -global_asm!( - " -.global win64_setjmp -win64_setjmp: - mov rax, [rsp] - mov [rcx+80], rax - - lea rax, [rsp+8] - mov [rcx+16], rax - - mov [rcx+ 0], edx - mov [rcx+ 8], rbx - mov [rcx+24], rbp - mov [rcx+32], rsi - mov [rcx+40], rdi - mov [rcx+48], r12 - mov [rcx+56], r13 - mov [rcx+64], r14 - mov [rcx+72], r15 - - stmxcsr [rcx+88] - fnstcw [rcx+92] - - movdqu [rcx+96], xmm6 - movdqu [rcx+112], xmm7 - movdqu [rcx+128], xmm8 - movdqu [rcx+144], xmm9 - movdqu [rcx+160], xmm10 - movdqu [rcx+176], xmm11 - movdqu [rcx+192], xmm12 - movdqu [rcx+208], xmm13 - movdqu [rcx+224], xmm14 - movdqu [rcx+240], xmm15 - - xor eax, eax - ret - " -); - -global_asm!( - " -.global win64_longjmp -win64_longjmp: - mov eax, edx - - mov rdx, [rcx+ 0] - mov rbx, [rcx+ 8] - mov rbp, [rcx+24] - mov rsi, [rcx+32] - mov rdi, [rcx+40] - mov r12, [rcx+48] - mov r13, [rcx+56] - mov r14, [rcx+64] - mov r15, [rcx+72] - - ldmxcsr [rcx+88] - fnclex - fldcw [rcx+92] - - movdqu xmm6, [rcx+96] - movdqu xmm7, [rcx+112] - movdqu xmm8, [rcx+128] - movdqu xmm9, [rcx+144] - movdqu xmm10, [rcx+160] - movdqu xmm11, [rcx+176] - movdqu xmm12, [rcx+192] - movdqu xmm13, [rcx+208] - movdqu xmm14, [rcx+224] - movdqu xmm15, [rcx+240] - - test eax, eax - jnz ok - inc eax -ok: - mov rsp, [rcx+16] - jmp qword ptr [rcx+80] -" -); - -/* Adapt the calling convention of the above to the native "C" calling - * convention. */ -extern "win64" { - fn win64_setjmp(x: u64) -> u64; - fn win64_longjmp(x: u64, y: u64) -> !; -} -#[no_mangle] -extern "C" fn setjmp(x: u64) -> u64 { - unsafe { win64_setjmp(x) } -} -#[no_mangle] -extern "C" fn longjmp(x: u64, y: u64) -> ! { - unsafe { win64_longjmp(x, y) } -} diff --git a/src/hyperlight_guest/third_party/musl/include/setjmp.h b/src/hyperlight_guest/third_party/musl/include/setjmp.h new file mode 100644 index 000000000..2ccc2ac82 --- /dev/null +++ b/src/hyperlight_guest/third_party/musl/include/setjmp.h @@ -0,0 +1,49 @@ +#ifndef _SETJMP_H +#define _SETJMP_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +#include + +typedef struct __jmp_buf_tag { + __jmp_buf __jb; + unsigned long __fl; + unsigned long __ss[128/sizeof(long)]; +} jmp_buf[1]; + +#if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1) +#define __setjmp_attr __attribute__((__returns_twice__)) +#else +#define __setjmp_attr +#endif + +#if defined(_POSIX_SOURCE) || defined(_POSIX_C_SOURCE) \ + || defined(_XOPEN_SOURCE) || defined(_GNU_SOURCE) \ + || defined(_BSD_SOURCE) +typedef jmp_buf sigjmp_buf; +int sigsetjmp (sigjmp_buf, int) __setjmp_attr; +_Noreturn void siglongjmp (sigjmp_buf, int); +#endif + +#if defined(_XOPEN_SOURCE) || defined(_GNU_SOURCE) \ + || defined(_BSD_SOURCE) +int _setjmp (jmp_buf) __setjmp_attr; +_Noreturn void _longjmp (jmp_buf, int); +#endif + +int setjmp (jmp_buf) __setjmp_attr; +_Noreturn void longjmp (jmp_buf, int); + +#define setjmp setjmp + +#undef __setjmp_attr + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/src/hyperlight_guest/third_party/musl/src/setjmp/x86_64/longjmp.s b/src/hyperlight_guest/third_party/musl/src/setjmp/x86_64/longjmp.s new file mode 100644 index 000000000..64aad91c2 --- /dev/null +++ b/src/hyperlight_guest/third_party/musl/src/setjmp/x86_64/longjmp.s @@ -0,0 +1,18 @@ +/* Copyright 2011-2012 Nicholas J. Kain, licensed under standard MIT license */ +.global _longjmp +.global longjmp +.type _longjmp,@function +.type longjmp,@function +_longjmp: +longjmp: + xor %eax,%eax + cmp $1,%esi /* CF = val ? 0 : 1 */ + adc %esi,%eax /* eax = val + !val */ + mov (%rdi),%rbx /* rdi is the jmp_buf, restore regs from it */ + mov 8(%rdi),%rbp + mov 16(%rdi),%r12 + mov 24(%rdi),%r13 + mov 32(%rdi),%r14 + mov 40(%rdi),%r15 + mov 48(%rdi),%rsp + jmp *56(%rdi) /* goto saved address without altering rsp */ \ No newline at end of file diff --git a/src/hyperlight_guest/third_party/musl/src/setjmp/x86_64/setjmp.s b/src/hyperlight_guest/third_party/musl/src/setjmp/x86_64/setjmp.s new file mode 100644 index 000000000..0cd4b3135 --- /dev/null +++ b/src/hyperlight_guest/third_party/musl/src/setjmp/x86_64/setjmp.s @@ -0,0 +1,22 @@ +/* Copyright 2011-2012 Nicholas J. Kain, licensed under standard MIT license */ +.global __setjmp +.global _setjmp +.global setjmp +.type __setjmp,@function +.type _setjmp,@function +.type setjmp,@function +__setjmp: +_setjmp: +setjmp: + mov %rbx,(%rdi) /* rdi is jmp_buf, move registers onto it */ + mov %rbp,8(%rdi) + mov %r12,16(%rdi) + mov %r13,24(%rdi) + mov %r14,32(%rdi) + mov %r15,40(%rdi) + lea 8(%rsp),%rdx /* this is our rsp WITHOUT current ret addr */ + mov %rdx,48(%rdi) + mov (%rsp),%rdx /* save return addr ptr for new rip */ + mov %rdx,56(%rdi) + xor %eax,%eax /* always return 0 */ + ret \ No newline at end of file From 0536591437f5d993c0f3788206b587d9199c2c63 Mon Sep 17 00:00:00 2001 From: danbugs Date: Thu, 15 May 2025 23:31:23 +0000 Subject: [PATCH 03/12] [guests/chkstk] remove chkstk chkstk was a dependency of the msvc toolchain. With PE support removed, we can get rid of it. Signed-off-by: danbugs --- src/hyperlight_guest/src/chkstk.rs | 64 ------------------------- src/hyperlight_guest/src/entrypoint.rs | 2 - src/hyperlight_guest/src/guest_error.rs | 7 --- src/hyperlight_guest/src/lib.rs | 1 - 4 files changed, 74 deletions(-) delete mode 100644 src/hyperlight_guest/src/chkstk.rs diff --git a/src/hyperlight_guest/src/chkstk.rs b/src/hyperlight_guest/src/chkstk.rs deleted file mode 100644 index f3cd24f64..000000000 --- a/src/hyperlight_guest/src/chkstk.rs +++ /dev/null @@ -1,64 +0,0 @@ -/* -Copyright 2024 The Hyperlight Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -use core::arch::global_asm; - -use crate::guest_error::set_stack_allocate_error; -use crate::MIN_STACK_ADDRESS; - -extern "win64" { - fn __chkstk(); - fn __chkstk_in_proc(); -} - -global_asm!( - " - .global __chkstk - __chkstk: - /* Save R10, R11 */ - push r10 - push r11 - - /* Load the minimum stack address from the PEB */ - mov r11, [rip+{min_stack_addr}] - - /* Get the current stack pointer */ - lea r10, [rsp+0x18] - - /* Calculate what the new stack pointer will be */ - sub r10, rax - - /* If result is negative, cause StackOverflow */ - js call_set_error - - /* Compare the new stack pointer with the minimum stack address */ - cmp r10, r11 - /* If the new stack pointer is greater or equal to the minimum stack address, - then we are good. Otherwise set the error code to 9 (stack overflow) call set_error and halt */ - jae cs_ret - - call_set_error: - call {set_error} - hlt - - cs_ret: - /* Restore RAX, R11 */ - pop r11 - pop r10 - ret", - min_stack_addr = sym MIN_STACK_ADDRESS, - set_error = sym set_stack_allocate_error, -); diff --git a/src/hyperlight_guest/src/entrypoint.rs b/src/hyperlight_guest/src/entrypoint.rs index 0e8506844..c03dd2bbf 100644 --- a/src/hyperlight_guest/src/entrypoint.rs +++ b/src/hyperlight_guest/src/entrypoint.rs @@ -73,8 +73,6 @@ extern "C" { static INIT: Once = Once::new(); -// Note: entrypoint cannot currently have a stackframe >4KB, as that will invoke __chkstk on msvc -// target without first having setup global `RUNNING_MODE` variable, which __chkstk relies on. #[no_mangle] pub extern "win64" fn entrypoint(peb_address: u64, seed: u64, ops: u64, max_log_level: u64) { if peb_address == 0 { diff --git a/src/hyperlight_guest/src/guest_error.rs b/src/hyperlight_guest/src/guest_error.rs index 1c8f57261..2ed5eadab 100644 --- a/src/hyperlight_guest/src/guest_error.rs +++ b/src/hyperlight_guest/src/guest_error.rs @@ -19,10 +19,8 @@ use alloc::vec::Vec; use core::ffi::{c_char, CStr}; use hyperlight_common::flatbuffer_wrappers::guest_error::{ErrorCode, GuestError}; -use hyperlight_common::outb::OutBAction; use crate::entrypoint::halt; -use crate::host_function_call::outb; use crate::shared_output_data::push_shared_output_data; pub(crate) fn write_error(error_code: ErrorCode, message: Option<&str>) { @@ -47,11 +45,6 @@ pub(crate) fn set_error_and_halt(error_code: ErrorCode, message: &str) { halt(); } -#[no_mangle] -pub(crate) extern "win64" fn set_stack_allocate_error() { - outb(OutBAction::Abort as u16, &[ErrorCode::StackOverflow as u8]); -} - /// Exposes a C API to allow the guest to set an error /// /// # Safety diff --git a/src/hyperlight_guest/src/lib.rs b/src/hyperlight_guest/src/lib.rs index 5ef3d2a0e..c38b5fddc 100644 --- a/src/hyperlight_guest/src/lib.rs +++ b/src/hyperlight_guest/src/lib.rs @@ -43,7 +43,6 @@ pub mod memory; pub mod print; pub(crate) mod security_check; -pub mod chkstk; pub mod error; pub mod gdt; pub mod idt; From 5e0fa6e87948c3ae07eebeaf25955656e79c32ba Mon Sep 17 00:00:00 2001 From: danbugs Date: Thu, 15 May 2025 23:43:15 +0000 Subject: [PATCH 04/12] [guest/exceptions] changed the ABI of exception related logic We used the sysv64 ABI for exception stuff. This ABI is not available for every target so having it there unconditionally makes the hyperlight_guest lib not be compilable to said targets. This commit changes the ABI to "C" (i.e., smt more commonly used) and gates it behind a x86_64 cfg as the assembly code is dependent on 64-bit architectures. Signed-off-by: danbugs --- src/hyperlight_guest/src/entrypoint.rs | 13 ++++++++----- src/hyperlight_guest/src/{ => exceptions}/gdt.rs | 0 .../handlers.rs} | 2 +- src/hyperlight_guest/src/{ => exceptions}/idt.rs | 4 ++-- src/hyperlight_guest/src/{ => exceptions}/idtr.rs | 2 +- .../src/{ => exceptions}/interrupt_entry.rs | 9 ++------- src/hyperlight_guest/src/lib.rs | 13 ++++++++----- 7 files changed, 22 insertions(+), 21 deletions(-) rename src/hyperlight_guest/src/{ => exceptions}/gdt.rs (100%) rename src/hyperlight_guest/src/{interrupt_handlers.rs => exceptions/handlers.rs} (96%) rename src/hyperlight_guest/src/{ => exceptions}/idt.rs (97%) rename src/hyperlight_guest/src/{ => exceptions}/idtr.rs (92%) rename src/hyperlight_guest/src/{ => exceptions}/interrupt_entry.rs (94%) diff --git a/src/hyperlight_guest/src/entrypoint.rs b/src/hyperlight_guest/src/entrypoint.rs index c03dd2bbf..1942489a8 100644 --- a/src/hyperlight_guest/src/entrypoint.rs +++ b/src/hyperlight_guest/src/entrypoint.rs @@ -22,11 +22,11 @@ use hyperlight_common::outb::OutBAction; use log::LevelFilter; use spin::Once; -use crate::gdt::load_gdt; +#[cfg(target_arch = "x86_64")] +use crate::exceptions::{gdt::load_gdt, idtr::load_idt}; use crate::guest_function_call::dispatch_function; use crate::guest_logger::init_logger; use crate::host_function_call::outb; -use crate::idtr::load_idt; use crate::{__security_cookie, HEAP_ALLOCATOR, MIN_STACK_ADDRESS, OS_PAGE_SIZE, P_PEB}; #[inline(never)] @@ -101,9 +101,12 @@ pub extern "win64" fn entrypoint(peb_address: u64, seed: u64, ops: u64, max_log_ // don't have to change the assembly code. MIN_STACK_ADDRESS = (*peb_ptr).gueststackData.minUserStackAddress; - // Setup GDT and IDT - load_gdt(); - load_idt(); + #[cfg(target_arch = "x86_64")] + { + // Setup GDT and IDT + load_gdt(); + load_idt(); + } let heap_start = (*peb_ptr).guestheapData.guestHeapBuffer as usize; let heap_size = (*peb_ptr).guestheapData.guestHeapSize as usize; diff --git a/src/hyperlight_guest/src/gdt.rs b/src/hyperlight_guest/src/exceptions/gdt.rs similarity index 100% rename from src/hyperlight_guest/src/gdt.rs rename to src/hyperlight_guest/src/exceptions/gdt.rs diff --git a/src/hyperlight_guest/src/interrupt_handlers.rs b/src/hyperlight_guest/src/exceptions/handlers.rs similarity index 96% rename from src/hyperlight_guest/src/interrupt_handlers.rs rename to src/hyperlight_guest/src/exceptions/handlers.rs index e2716ac67..c2c6950a2 100644 --- a/src/hyperlight_guest/src/interrupt_handlers.rs +++ b/src/hyperlight_guest/src/exceptions/handlers.rs @@ -24,7 +24,7 @@ use crate::entrypoint::abort_with_code_and_message; /// Exception handler #[no_mangle] -pub extern "sysv64" fn hl_exception_handler( +pub extern "C" fn hl_exception_handler( stack_pointer: u64, exception_number: u64, page_fault_address: u64, diff --git a/src/hyperlight_guest/src/idt.rs b/src/hyperlight_guest/src/exceptions/idt.rs similarity index 97% rename from src/hyperlight_guest/src/idt.rs rename to src/hyperlight_guest/src/exceptions/idt.rs index 0e1b46423..9c32471d7 100644 --- a/src/hyperlight_guest/src/idt.rs +++ b/src/hyperlight_guest/src/exceptions/idt.rs @@ -16,7 +16,7 @@ limitations under the License. use hyperlight_common::outb::Exception; -use crate::interrupt_entry::{ +use crate::exceptions::interrupt_entry::{ _do_excp0, _do_excp1, _do_excp10, _do_excp11, _do_excp12, _do_excp13, _do_excp14, _do_excp15, _do_excp16, _do_excp17, _do_excp18, _do_excp19, _do_excp2, _do_excp20, _do_excp3, _do_excp30, _do_excp4, _do_excp5, _do_excp6, _do_excp7, _do_excp8, _do_excp9, @@ -96,7 +96,7 @@ pub(crate) fn init_idt() { set_idt_entry(Exception::SecurityException as usize, _do_excp30); // Security Exception } -fn set_idt_entry(index: usize, handler: unsafe extern "sysv64" fn()) { +fn set_idt_entry(index: usize, handler: unsafe extern "C" fn()) { let handler_addr = handler as *const () as u64; unsafe { IDT[index] = IdtEntry::new(handler_addr); diff --git a/src/hyperlight_guest/src/idtr.rs b/src/hyperlight_guest/src/exceptions/idtr.rs similarity index 92% rename from src/hyperlight_guest/src/idtr.rs rename to src/hyperlight_guest/src/exceptions/idtr.rs index 598e62ff0..26bc634a9 100644 --- a/src/hyperlight_guest/src/idtr.rs +++ b/src/hyperlight_guest/src/exceptions/idtr.rs @@ -1,6 +1,6 @@ use core::ptr::addr_of; -use crate::idt::{init_idt, IdtEntry, IDT}; +use crate::exceptions::idt::{init_idt, IdtEntry, IDT}; #[repr(C, packed)] pub struct Idtr { diff --git a/src/hyperlight_guest/src/interrupt_entry.rs b/src/hyperlight_guest/src/exceptions/interrupt_entry.rs similarity index 94% rename from src/hyperlight_guest/src/interrupt_entry.rs rename to src/hyperlight_guest/src/exceptions/interrupt_entry.rs index 62ecd79f1..a04963b8c 100644 --- a/src/hyperlight_guest/src/interrupt_entry.rs +++ b/src/hyperlight_guest/src/exceptions/interrupt_entry.rs @@ -19,9 +19,9 @@ limitations under the License. use core::arch::global_asm; -use crate::interrupt_handlers::hl_exception_handler; +use crate::exceptions::handlers::hl_exception_handler; -extern "sysv64" { +extern "C" { // Exception handlers pub(crate) fn _do_excp0(); pub(crate) fn _do_excp1(); @@ -119,7 +119,6 @@ macro_rules! generate_exceptions { // Common exception handler ".global _do_excp_common\n", "_do_excp_common:\n", - // In SysV ABI, the first argument is passed in rdi // rdi is the stack pointer. " mov rdi, rsp\n", " call {hl_exception_handler}\n", @@ -173,12 +172,10 @@ macro_rules! generate_excp { stringify!($num), ":\n", context_save!(), - // In SysV ABI, the second argument is passed in rsi // rsi is the exception number. " mov rsi, ", stringify!($num), "\n", - // In SysV ABI, the third argument is passed in rdx // rdx is only used for pagefault exception and // contains the address that caused the pagefault. " mov rdx, 0\n", @@ -198,12 +195,10 @@ macro_rules! generate_excp { // stack aligned. " push 0\n", context_save!(), - // In SysV ABI, the second argument is passed in rsi // rsi is the exception number. " mov rsi, ", stringify!($num), "\n", - // In SysV ABI, the third argument is passed in rdx // rdx is only used for pagefault exception and // contains the address that caused the pagefault. " mov rdx, 0\n", diff --git a/src/hyperlight_guest/src/lib.rs b/src/hyperlight_guest/src/lib.rs index c38b5fddc..606857529 100644 --- a/src/hyperlight_guest/src/lib.rs +++ b/src/hyperlight_guest/src/lib.rs @@ -44,11 +44,14 @@ pub mod print; pub(crate) mod security_check; pub mod error; -pub mod gdt; -pub mod idt; -pub mod idtr; -pub mod interrupt_entry; -pub mod interrupt_handlers; +#[cfg(target_arch = "x86_64")] +pub mod exceptions { + pub mod gdt; + pub mod handlers; + pub mod idt; + pub mod idtr; + pub mod interrupt_entry; +} pub mod logging; // Unresolved symbols From 6f29792e8fc72bbb6cd298c9fea50e8e61a61269 Mon Sep 17 00:00:00 2001 From: danbugs Date: Thu, 15 May 2025 23:53:25 +0000 Subject: [PATCH 05/12] [guest/entrypoint,host/drivers] change the ABI of entrypoint from win64 Originally, because we only supported PE, our entrypoint used the win64 ABI. Now, with PE file support being removed, we can use an ABI that wouldn't be restrictive in compiling the guest library to targets that don't support the win64 ABI. Signed-off-by: danbugs --- src/hyperlight_guest/src/entrypoint.rs | 2 +- src/hyperlight_host/src/hypervisor/hyperv_linux.rs | 8 ++++---- src/hyperlight_host/src/hypervisor/hyperv_windows.rs | 8 ++++---- src/hyperlight_host/src/hypervisor/kvm.rs | 8 ++++---- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/hyperlight_guest/src/entrypoint.rs b/src/hyperlight_guest/src/entrypoint.rs index 1942489a8..2e7583fec 100644 --- a/src/hyperlight_guest/src/entrypoint.rs +++ b/src/hyperlight_guest/src/entrypoint.rs @@ -74,7 +74,7 @@ extern "C" { static INIT: Once = Once::new(); #[no_mangle] -pub extern "win64" fn entrypoint(peb_address: u64, seed: u64, ops: u64, max_log_level: u64) { +pub extern "C" fn entrypoint(peb_address: u64, seed: u64, ops: u64, max_log_level: u64) { if peb_address == 0 { panic!("PEB address is null"); } diff --git a/src/hyperlight_host/src/hypervisor/hyperv_linux.rs b/src/hyperlight_host/src/hypervisor/hyperv_linux.rs index 75b5cd2ec..85dc514b5 100644 --- a/src/hyperlight_host/src/hypervisor/hyperv_linux.rs +++ b/src/hyperlight_host/src/hypervisor/hyperv_linux.rs @@ -476,10 +476,10 @@ impl Hypervisor for HypervLinuxDriver { rflags: 2, //bit 1 of rlags is required to be set // function args - rcx: peb_addr.into(), - rdx: seed, - r8: page_size.into(), - r9: max_guest_log_level, + rdi: peb_addr.into(), + rsi: seed, + rdx: page_size.into(), + rcx: max_guest_log_level, ..Default::default() }; diff --git a/src/hyperlight_host/src/hypervisor/hyperv_windows.rs b/src/hyperlight_host/src/hypervisor/hyperv_windows.rs index 03ca86198..abc2b77cf 100644 --- a/src/hyperlight_host/src/hypervisor/hyperv_windows.rs +++ b/src/hyperlight_host/src/hypervisor/hyperv_windows.rs @@ -321,10 +321,10 @@ impl Hypervisor for HypervWindowsDriver { rsp: self.orig_rsp.absolute()?, // function args - rcx: peb_address.into(), - rdx: seed, - r8: page_size.into(), - r9: max_guest_log_level, + rdi: peb_address.into(), + rsi: seed, + rdx: page_size.into(), + rcx: max_guest_log_level, rflags: 1 << 1, // eflags bit index 1 is reserved and always needs to be 1 ..Default::default() diff --git a/src/hyperlight_host/src/hypervisor/kvm.rs b/src/hyperlight_host/src/hypervisor/kvm.rs index f39fb99bd..3dd1cb1fc 100644 --- a/src/hyperlight_host/src/hypervisor/kvm.rs +++ b/src/hyperlight_host/src/hypervisor/kvm.rs @@ -420,10 +420,10 @@ impl Hypervisor for KVMDriver { rsp: self.orig_rsp.absolute()?, // function args - rcx: peb_addr.into(), - rdx: seed, - r8: page_size.into(), - r9: max_guest_log_level, + rdi: peb_addr.into(), + rsi: seed, + rdx: page_size.into(), + rcx: max_guest_log_level, ..Default::default() }; From 0514111725b492ddad711aa32b60ed23671cc015 Mon Sep 17 00:00:00 2001 From: danbugs Date: Thu, 15 May 2025 23:58:43 +0000 Subject: [PATCH 06/12] [guest/fxn_call] change the ABI of the dispatch_function Similar to the entrypoint change in the previous commit, we can change the ABI of the dispatch function. Signed-off-by: danbugs --- src/hyperlight_guest/src/guest_function_call.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hyperlight_guest/src/guest_function_call.rs b/src/hyperlight_guest/src/guest_function_call.rs index 2c67cf2b5..361b9c3ab 100644 --- a/src/hyperlight_guest/src/guest_function_call.rs +++ b/src/hyperlight_guest/src/guest_function_call.rs @@ -97,7 +97,7 @@ fn internal_dispatch_function() -> Result<()> { // This is implemented as a separate function to make sure that epilogue in the internal_dispatch_function is called before the halt() // which if it were included in the internal_dispatch_function cause the epilogue to not be called because the halt() would not return // when running in the hypervisor. -pub(crate) extern "win64" fn dispatch_function() { +pub(crate) extern "C" fn dispatch_function() { let _ = internal_dispatch_function(); halt(); } From de799d7e28c383f31a2b5980dd91fce68a8ca033 Mon Sep 17 00:00:00 2001 From: danbugs Date: Fri, 16 May 2025 00:07:02 +0000 Subject: [PATCH 07/12] [guest/lib] removed unresolved symbols required by msvc toolchain __CxxFrameHandler3 and _fltused were symbols only required by the msvc toolchain and can now be removed. Signed-off-by: danbugs --- src/hyperlight_guest/src/lib.rs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/hyperlight_guest/src/lib.rs b/src/hyperlight_guest/src/lib.rs index 606857529..98e736fc2 100644 --- a/src/hyperlight_guest/src/lib.rs +++ b/src/hyperlight_guest/src/lib.rs @@ -54,15 +54,6 @@ pub mod exceptions { } pub mod logging; -// Unresolved symbols -///cbindgen:ignore -#[no_mangle] -pub(crate) extern "C" fn __CxxFrameHandler3() {} -///cbindgen:ignore -#[no_mangle] -#[clippy::allow(clippy::non_upper_case_globals)] -pub(crate) static _fltused: i32 = 0; - // It looks like rust-analyzer doesn't correctly manage no_std crates, // and so it displays an error about a duplicate panic_handler. // See more here: https://github.com/rust-lang/rust-analyzer/issues/4490 From b590cab0669bf038b77f7904cbf48c02b7d6734c Mon Sep 17 00:00:00 2001 From: danbugs Date: Fri, 16 May 2025 00:08:16 +0000 Subject: [PATCH 08/12] [guest/{security_cookie,security_check,entrypoint,lib}] remove usage of __security_cookie symbol The __security_cookie symbol and its usage were only required by the MSVC toolchain. W/ the removal of PE support, we can remove it from the guest lib. Signed-off-by: danbugs --- src/hyperlight_guest/src/entrypoint.rs | 3 +- src/hyperlight_guest/src/guest_error.rs | 5 ---- src/hyperlight_guest/src/lib.rs | 6 ---- src/hyperlight_guest/src/security_check.rs | 33 ---------------------- 4 files changed, 1 insertion(+), 46 deletions(-) delete mode 100644 src/hyperlight_guest/src/security_check.rs diff --git a/src/hyperlight_guest/src/entrypoint.rs b/src/hyperlight_guest/src/entrypoint.rs index 2e7583fec..cc559a8d2 100644 --- a/src/hyperlight_guest/src/entrypoint.rs +++ b/src/hyperlight_guest/src/entrypoint.rs @@ -27,7 +27,7 @@ use crate::exceptions::{gdt::load_gdt, idtr::load_idt}; use crate::guest_function_call::dispatch_function; use crate::guest_logger::init_logger; use crate::host_function_call::outb; -use crate::{__security_cookie, HEAP_ALLOCATOR, MIN_STACK_ADDRESS, OS_PAGE_SIZE, P_PEB}; +use crate::{HEAP_ALLOCATOR, MIN_STACK_ADDRESS, OS_PAGE_SIZE, P_PEB}; #[inline(never)] pub fn halt() { @@ -83,7 +83,6 @@ pub extern "C" fn entrypoint(peb_address: u64, seed: u64, ops: u64, max_log_leve unsafe { P_PEB = Some(peb_address as *mut HyperlightPEB); let peb_ptr = P_PEB.unwrap(); - __security_cookie = peb_address ^ seed; let srand_seed = ((peb_address << 8 ^ seed >> 4) >> 32) as u32; diff --git a/src/hyperlight_guest/src/guest_error.rs b/src/hyperlight_guest/src/guest_error.rs index 2ed5eadab..d56cd82d6 100644 --- a/src/hyperlight_guest/src/guest_error.rs +++ b/src/hyperlight_guest/src/guest_error.rs @@ -40,11 +40,6 @@ pub(crate) fn set_error(error_code: ErrorCode, message: &str) { write_error(error_code, Some(message)); } -pub(crate) fn set_error_and_halt(error_code: ErrorCode, message: &str) { - set_error(error_code, message); - halt(); -} - /// Exposes a C API to allow the guest to set an error /// /// # Safety diff --git a/src/hyperlight_guest/src/lib.rs b/src/hyperlight_guest/src/lib.rs index 98e736fc2..7997a87f0 100644 --- a/src/hyperlight_guest/src/lib.rs +++ b/src/hyperlight_guest/src/lib.rs @@ -41,7 +41,6 @@ pub mod host_function_call; pub(crate) mod guest_logger; pub mod memory; pub mod print; -pub(crate) mod security_check; pub mod error; #[cfg(target_arch = "x86_64")] @@ -74,11 +73,6 @@ fn panic(info: &core::panic::PanicInfo) -> ! { #[global_allocator] pub(crate) static HEAP_ALLOCATOR: LockedHeap<32> = LockedHeap::<32>::empty(); -///cbindgen:ignore -#[no_mangle] -#[clippy::allow(clippy::non_upper_case_globals)] -pub(crate) static mut __security_cookie: u64 = 0; - pub(crate) static mut P_PEB: Option<*mut HyperlightPEB> = None; pub static mut MIN_STACK_ADDRESS: u64 = 0; diff --git a/src/hyperlight_guest/src/security_check.rs b/src/hyperlight_guest/src/security_check.rs deleted file mode 100644 index b21a94af8..000000000 --- a/src/hyperlight_guest/src/security_check.rs +++ /dev/null @@ -1,33 +0,0 @@ -/* -Copyright 2024 The Hyperlight Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// implements security cookie used by /GS compiler option and checks value is valid -// calls report_gsfailure if value is invalid - -use hyperlight_common::flatbuffer_wrappers::guest_error::ErrorCode::GsCheckFailed; - -use crate::__security_cookie; -use crate::guest_error::set_error_and_halt; - -///cbindgen:ignore -#[no_mangle] -pub(crate) extern "C" fn __security_check_cookie(cookie: u64) { - unsafe { - if __security_cookie != cookie { - set_error_and_halt(GsCheckFailed, "GS Check Failed"); - } - } -} From 9abffc026e4696caab816c91c6e7664cdc00d2e7 Mon Sep 17 00:00:00 2001 From: danbugs Date: Fri, 16 May 2025 00:10:51 +0000 Subject: [PATCH 09/12] [rust_guests/*/.cargo/config.toml] remove msvc target config from .cargo/config.toml of test guests W/ PE support removed, these are no longer needed or correct. Signed-off-by: danbugs --- src/tests/rust_guests/callbackguest/.cargo/config.toml | 7 ------- src/tests/rust_guests/dummyguest/.cargo/config.toml | 7 ------- src/tests/rust_guests/simpleguest/.cargo/config.toml | 7 ------- 3 files changed, 21 deletions(-) diff --git a/src/tests/rust_guests/callbackguest/.cargo/config.toml b/src/tests/rust_guests/callbackguest/.cargo/config.toml index f4d7ab660..f977e97ab 100644 --- a/src/tests/rust_guests/callbackguest/.cargo/config.toml +++ b/src/tests/rust_guests/callbackguest/.cargo/config.toml @@ -1,13 +1,6 @@ [build] target = "x86_64-unknown-none" -[target.x86_64-pc-windows-msvc] -rustflags = [ - "-C", - "link-args=/RELEASE /DEBUG /NOLOGO /NXCOMPAT /SAFESEH:NO /ENTRY:entrypoint /SUBSYSTEM:NATIVE /ALIGN:4096 /FILEALIGN:4096 /NODEFAULTLIB /HEAP:131072,131072 /DYNAMICBASE /STACK:65536,65536 /MACHINE:X64", -] -linker = "rust-lld" - [target.x86_64-unknown-none] rustflags = [ "-C", diff --git a/src/tests/rust_guests/dummyguest/.cargo/config.toml b/src/tests/rust_guests/dummyguest/.cargo/config.toml index 93af0c721..4b549b837 100644 --- a/src/tests/rust_guests/dummyguest/.cargo/config.toml +++ b/src/tests/rust_guests/dummyguest/.cargo/config.toml @@ -1,13 +1,6 @@ [build] target = "x86_64-unknown-none" -[target.x86_64-pc-windows-msvc] -rustflags = [ - "-C", - "link-args=/RELEASE /DEBUG /NOLOGO /NXCOMPAT /SAFESEH:NO /ENTRY:entrypoint /SUBSYSTEM:NATIVE /ALIGN:4096 /FILEALIGN:4096 /NODEFAULTLIB /HEAP:131072,131072 /DYNAMICBASE /STACK:65536,65536 /MACHINE:X64", -] -linker = "rust-lld" - [target.x86_64-unknown-none] rustflags = [ "-C", diff --git a/src/tests/rust_guests/simpleguest/.cargo/config.toml b/src/tests/rust_guests/simpleguest/.cargo/config.toml index 7efafae98..037201bc1 100644 --- a/src/tests/rust_guests/simpleguest/.cargo/config.toml +++ b/src/tests/rust_guests/simpleguest/.cargo/config.toml @@ -1,13 +1,6 @@ [build] target = "x86_64-unknown-none" -[target.x86_64-pc-windows-msvc] -rustflags = [ - "-C", - "link-args=/RELEASE /DEBUG /NOLOGO /NXCOMPAT /SAFESEH:NO /ENTRY:entrypoint /SUBSYSTEM:NATIVE /ALIGN:4096 /FILEALIGN:4096 /NODEFAULTLIB /HEAP:131072,131072 /DYNAMICBASE /STACK:65536,65536 /MACHINE:X64", -] -linker = "rust-lld" - [target.x86_64-unknown-none] rustflags = [ "-C", From d3772ad03e945faa33aea63f9a862325c73cf38a Mon Sep 17 00:00:00 2001 From: danbugs Date: Fri, 16 May 2025 00:15:58 +0000 Subject: [PATCH 10/12] [README] update README to remove MSVC mentions In particular, I am removing the instructions related to installing the msvc target to compile the guests to PE. Signed-off-by: danbugs --- README.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 8a9788b19..e026e67f3 100644 --- a/README.md +++ b/README.md @@ -177,13 +177,10 @@ After having an environment with a hypervisor setup, running the example has the Azure Linux, run `sudo dnf install build-essential`. 2. [Rust](https://www.rust-lang.org/tools/install). Install toolchain v1.81 or later. - Also, install the `x86_64-pc-windows-msvc` and `x86_64-unknown-none` targets, these are needed to build the test - guest binaries. (Note: install both targets on either Linux or Windows: Hyperlight can load ELF or PE files on either - OS, and the tests/examples are built for both): - + Also, install the `x86_64-unknown-none` target, it is needed to build the test + guest binaries. ```sh rustup target add x86_64-unknown-none - rustup target add x86_64-pc-windows-msvc ``` 3. [just](https://github.com/casey/just). `cargo install just` On Windows you also need [pwsh](https://learn.microsoft.com/en-us/powershell/scripting/install/installing-powershell-on-windows?view=powershell-7.4). From 743ac7bc7e8ecff0a45d2d5b698a5ad6a1c692ec Mon Sep 17 00:00:00 2001 From: danbugs Date: Fri, 16 May 2025 00:17:09 +0000 Subject: [PATCH 11/12] [.devcontainer/Dockerfile] remove installation of msvc target in .devcontainer W/ PE support removed, our .devcontainer no longer needs the msvc target installed. Signed-off-by: danbugs --- .devcontainer/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 99b93e95b..37b8e7d64 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -59,7 +59,6 @@ RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y \ && rustup default ${RUST_TOOLCHAIN} \ && rustup target add x86_64-unknown-linux-gnu \ && rustup target add x86_64-unknown-none \ - && rustup target add x86_64-pc-windows-msvc \ && rustup toolchain add nightly-x86_64-unknown-linux-gnu \ && cargo install just From 43bcf6efaf3020914b38f512474cdf1bd9ff3be5 Mon Sep 17 00:00:00 2001 From: danbugs Date: Fri, 16 May 2025 00:56:13 +0000 Subject: [PATCH 12/12] [host/mgr] added note on MSVC's rsp meddling See #498 for more details. Signed-off-by: danbugs --- src/hyperlight_host/src/mem/mgr.rs | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/src/hyperlight_host/src/mem/mgr.rs b/src/hyperlight_host/src/mem/mgr.rs index 81da40030..8569c99b3 100644 --- a/src/hyperlight_host/src/mem/mgr.rs +++ b/src/hyperlight_host/src/mem/mgr.rs @@ -107,21 +107,13 @@ where mem_size: u64, regions: &mut [MemoryRegion], ) -> Result { - // For MSVC, move rsp down by 0x28. This gives the called 'main' - // function the appearance that rsp was 16 byte aligned before - // the 'call' that calls main (note we don't really have a return value - // on the stack but some assembly instructions are expecting rsp have - // started 0x8 bytes off of 16 byte alignment when 'main' is invoked. - // We do 0x28 instead of 0x8 because MSVC can expect that there are - // 0x20 bytes of space to write to by the called function. - // I am not sure if this happens with the 'main' method, but we do this - // just in case. - // - // NOTE: We do this also for GCC freestanding binaries because we - // specify __attribute__((ms_abi)) on the start method let rsp: u64 = self.layout.get_top_of_user_stack_offset() as u64 + SandboxMemoryLayout::BASE_ADDRESS as u64 + self.layout.stack_size as u64 + // TODO: subtracting 0x28 was a requirement for MSVC. It should no longer be + // necessary now, but, for some reason, without this, the `multiple_parameters` + // test from `sandbox_host_tests` fails. We should investigate this further. + // See issue #498 for more details. - 0x28; self.shared_mem.with_exclusivity(|shared_mem| {