Compressed Oops Support

.github/scripts/ci-matrix-result-check.py

@@ -36,11 +36,11 @@ def read_in_plans():
  value = m.group(1)
  else:
  raise ValueError(f"Cannot find a plan string in {prop}")
- 
+
  # Store the value in the dictionary
  key = chr(97+i)
  results[key] = value
- 
+
  return results
 
 def read_in_actual_results(line, plan_dict):
@@ -144,7 +144,7 @@ def print_log(directory, search_string):
  if expected[plan] == "ignore":
  print(f"Result for {plan} is ignored")
  continue
- 
+
  if expected[plan] != actual[plan]:
  error_no = 1
  if expected[plan] == "pass":

.github/scripts/ci-test-assertions.sh

@@ -56,4 +56,5 @@ sudo sysctl -w vm.max_map_count=655300
 export MMTK_PLAN=PageProtect
 
 build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms4G -Xmx4G -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar fop
-build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms4G -Xmx4G -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar luindex
+# Note: Disable compressed pointers for luindex as it does not work well with GC plans that uses virtual memory excessively.
+build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -Xms4G -Xmx4G -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar luindex

.github/scripts/ci-test-malloc-mark-sweep.sh

@@ -16,7 +16,7 @@ run_test() {
  # Malloc marksweep is horribly slow. We just run fop.
 
  # build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar antlr
- build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms50M -Xmx50M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar fop
+ build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms50M -Xmx50M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar fop
  # build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar luindex
  # build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar pmd
  # build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar hsqldb

.github/scripts/ci-test-only-normal.sh

@@ -151,14 +151,3 @@ build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHea
 # These benchmarks take 40s+ for slowdebug build, we may consider removing them from the CI
 build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -XX:TieredStopAtLevel=1 -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar hsqldb
 build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -XX:TieredStopAtLevel=1 -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar eclipse
-
-# --- PageProtect ---
-# Make sure this runs last in our tests unless we want to set it back to the default limit.
-sudo sysctl -w vm.max_map_count=655300
-
-export MMTK_PLAN=PageProtect
-
-build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms4G -Xmx4G -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar antlr
-build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms4G -Xmx4G -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar fop
-build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms4G -Xmx4G -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar luindex
-# build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms4G -Xmx4G -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar pmd

.github/scripts/ci-test.sh

@@ -6,6 +6,8 @@ cd $cur
 cd $cur
 ./ci-test-only-normal.sh
 cd $cur
+./ci-test-only-normal-no-compressed-oops.sh
+cd $cur
 ./ci-test-only-weak-ref.sh
 cd $cur
 ./ci-test-assertions.sh

mmtk/Cargo.toml

@@ -24,6 +24,7 @@ openjdk_version = "28e56ee32525c32c5a88391d0b01f24e5cd16c0f"
 libc = "0.2"
 lazy_static = "1.1"
 once_cell = "1.10.0"
+atomic = "0.5.1"
 # Be very careful to commit any changes to the following mmtk dependency, as our CI scripts (including mmtk-core CI)
 # rely on matching these lines to modify them: e.g. comment out the git dependency and use the local path.
 # These changes are safe:

mmtk/src/abi.rs

@@ -1,10 +1,14 @@
-use crate::UPCALLS;
+use super::UPCALLS;
+use crate::OpenJDKEdge;
+use atomic::Atomic;
+use atomic::Ordering;
 use mmtk::util::constants::*;
 use mmtk::util::conversions;
 use mmtk::util::ObjectReference;
 use mmtk::util::{Address, OpaquePointer};
 use std::ffi::CStr;
 use std::fmt;
+use std::sync::atomic::AtomicUsize;
 use std::{mem, slice};
 
 #[repr(i32)]
@@ -80,7 +84,7 @@ impl Klass {
  pub const LH_HEADER_SIZE_SHIFT: i32 = BITS_IN_BYTE as i32 * 2;
  pub const LH_HEADER_SIZE_MASK: i32 = (1 << BITS_IN_BYTE) - 1;
  pub unsafe fn cast<'a, T>(&self) -> &'a T {
- &*(self as *const _ as usize as *const T)
+ &*(self as *const Self as *const T)
  }
  /// Force slow-path for instance size calculation?
  const fn layout_helper_needs_slow_path(lh: i32) -> bool {
@@ -168,7 +172,7 @@ impl InstanceKlass {
  const VTABLE_START_OFFSET: usize = Self::HEADER_SIZE * BYTES_IN_WORD;
 
  fn start_of_vtable(&self) -> *const usize {
- unsafe { (self as *const _ as *const u8).add(Self::VTABLE_START_OFFSET) as _ }
+ unsafe { (self as *const Self as *const u8).add(Self::VTABLE_START_OFFSET) as _ }
  }
 
  fn start_of_itable(&self) -> *const usize {
@@ -263,24 +267,53 @@ impl InstanceRefKlass {
  }
  *DISCOVERED_OFFSET
  }
- pub fn referent_address(oop: Oop) -> Address {
- oop.get_field_address(Self::referent_offset())
+ pub fn referent_address<const COMPRESSED: bool>(oop: Oop) -> OpenJDKEdge<COMPRESSED> {
+ oop.get_field_address(Self::referent_offset()).into()
  }
- pub fn discovered_address(oop: Oop) -> Address {
- oop.get_field_address(Self::discovered_offset())
+ pub fn discovered_address<const COMPRESSED: bool>(oop: Oop) -> OpenJDKEdge<COMPRESSED> {
+ oop.get_field_address(Self::discovered_offset()).into()
  }
 }
 
+#[repr(C)]
+union KlassPointer {
+ /// uncompressed Klass pointer
+ klass: &'static Klass,
+ /// compressed Klass pointer
+ narrow_klass: u32,
+}
+
 #[repr(C)]
 pub struct OopDesc {
  pub mark: usize,
- pub klass: &'static Klass,
+ klass: KlassPointer,
+}
+
+static COMPRESSED_KLASS_BASE: Atomic<Address> = Atomic::new(Address::ZERO);
+static COMPRESSED_KLASS_SHIFT: AtomicUsize = AtomicUsize::new(0);
+
+/// When enabling compressed pointers, the class pointers are also compressed.
+/// The c++ part of the binding should pass the compressed klass base and shift to rust binding, as object scanning will need it.
+pub fn set_compressed_klass_base_and_shift(base: Address, shift: usize) {
+ COMPRESSED_KLASS_BASE.store(base, Ordering::Relaxed);
+ COMPRESSED_KLASS_SHIFT.store(shift, Ordering::Relaxed);
 }
 
 impl OopDesc {
  pub fn start(&self) -> Address {
  unsafe { mem::transmute(self) }
  }
+
+ pub fn klass<const COMPRESSED: bool>(&self) -> &'static Klass {
+ if COMPRESSED {
+ let compressed = unsafe { self.klass.narrow_klass };
+ let addr = COMPRESSED_KLASS_BASE.load(Ordering::Relaxed)
+ + ((compressed as usize) << COMPRESSED_KLASS_SHIFT.load(Ordering::Relaxed));
+ unsafe { &*addr.to_ptr::<Klass>() }
+ } else {
+ unsafe { self.klass.klass }
+ }
+ }
 }
 
 impl fmt::Debug for OopDesc {
@@ -294,6 +327,16 @@ impl fmt::Debug for OopDesc {
 
 pub type Oop = &'static OopDesc;
 
+#[repr(transparent)]
+#[derive(Clone, Copy)]
+pub struct NarrowOop(u32);
+
+impl NarrowOop {
+ pub fn slot(&self) -> Address {
+ Address::from_ref(self)
+ }
+}
+
 /// Convert ObjectReference to Oop
 impl From<ObjectReference> for &OopDesc {
  fn from(o: ObjectReference) -> Self {
@@ -323,8 +366,8 @@ impl OopDesc {
  }
 
  /// Calculate object instance size
- pub unsafe fn size(&self) -> usize {
- let klass = self.klass;
+ pub unsafe fn size<const COMPRESSED: bool>(&self) -> usize {
+ let klass = self.klass::<COMPRESSED>();
  let lh = klass.layout_helper;
  // The (scalar) instance size is pre-recorded in the TIB?
  if lh > Klass::LH_NEUTRAL_VALUE {
@@ -336,7 +379,7 @@ impl OopDesc {
  } else if lh <= Klass::LH_NEUTRAL_VALUE {
  if lh < Klass::LH_NEUTRAL_VALUE {
  // Calculate array size
- let array_length = self.as_array_oop().length();
+ let array_length = self.as_array_oop().length::<COMPRESSED>();
  let mut size_in_bytes: usize =
  (array_length as usize) << Klass::layout_helper_log2_element_size(lh);
  size_in_bytes += Klass::layout_helper_header_size(lh) as usize;
@@ -356,34 +399,58 @@ pub struct ArrayOopDesc(OopDesc);
 pub type ArrayOop = &'static ArrayOopDesc;
 
 impl ArrayOopDesc {
- const LENGTH_OFFSET: usize = mem::size_of::<Self>();
+ fn length_offset<const COMPRESSED: bool>() -> usize {
+ if COMPRESSED {
+ mem::size_of::<usize>() + mem::size_of::<u32>()
+ } else {
+ mem::size_of::<Self>()
+ }
+ }
 
  fn element_type_should_be_aligned(ty: BasicType) -> bool {
  ty == BasicType::T_DOUBLE || ty == BasicType::T_LONG
  }
 
- fn header_size(ty: BasicType) -> usize {
- let typesize_in_bytes =
- conversions::raw_align_up(Self::LENGTH_OFFSET + BYTES_IN_INT, BYTES_IN_LONG);
+ fn header_size<const COMPRESSED: bool>(ty: BasicType) -> usize {
+ let typesize_in_bytes = conversions::raw_align_up(
+ Self::length_offset::<COMPRESSED>() + BYTES_IN_INT,
+ BYTES_IN_LONG,
+ );
  if Self::element_type_should_be_aligned(ty) {
  conversions::raw_align_up(typesize_in_bytes / BYTES_IN_WORD, BYTES_IN_LONG)
  } else {
  typesize_in_bytes / BYTES_IN_WORD
  }
  }
- fn length(&self) -> i32 {
- unsafe { *((self as *const _ as *const u8).add(Self::LENGTH_OFFSET) as *const i32) }
+ fn length<const COMPRESSED: bool>(&self) -> i32 {
+ unsafe {
+ *((self as *const Self as *const u8).add(Self::length_offset::<COMPRESSED>())
+ as *const i32)
+ }
  }
- fn base(&self, ty: BasicType) -> Address {
- let base_offset_in_bytes = Self::header_size(ty) * BYTES_IN_WORD;
- Address::from_ptr(unsafe { (self as *const _ as *const u8).add(base_offset_in_bytes) })
+ fn base<const COMPRESSED: bool>(&self, ty: BasicType) -> Address {
+ let base_offset_in_bytes = Self::header_size::<COMPRESSED>(ty) * BYTES_IN_WORD;
+ Address::from_ptr(unsafe { (self as *const Self as *const u8).add(base_offset_in_bytes) })
  }
- // This provides an easy way to access the array data in Rust. However, the array data
- // is Java types, so we have to map Java types to Rust types. The caller needs to guarantee:
- // 1. <T> matches the actual Java type
- // 2. <T> matches the argument, BasicType `ty`
- pub unsafe fn data<T>(&self, ty: BasicType) -> &[T] {
- slice::from_raw_parts(self.base(ty).to_ptr(), self.length() as _)
+ /// This provides an easy way to access the array data in Rust. However, the array data
+ /// is Java types, so we have to map Java types to Rust types. The caller needs to guarantee:
+ /// 1. <T> matches the actual Java type
+ /// 2. <T> matches the argument, BasicType `ty`
+ pub unsafe fn data<T, const COMPRESSED: bool>(&self, ty: BasicType) -> &[T] {
+ slice::from_raw_parts(
+ self.base::<COMPRESSED>(ty).to_ptr(),
+ self.length::<COMPRESSED>() as _,
+ )
+ }
+
+ pub unsafe fn slice<const COMPRESSED: bool>(
+ &self,
+ ty: BasicType,
+ ) -> crate::OpenJDKEdgeRange<COMPRESSED> {
+ let base = self.base::<COMPRESSED>(ty);
+ let start = base;
+ let end = base + ((self.length::<COMPRESSED>() as usize) << if COMPRESSED { 2 } else { 3 });
+ (start..end).into()
  }
 }
 

mmtk/src/active_plan.rs

@@ -1,6 +1,5 @@
 use crate::MutatorClosure;
 use crate::OpenJDK;
-use crate::SINGLETON;
 use crate::UPCALLS;
 use mmtk::util::opaque_pointer::*;
 use mmtk::vm::ActivePlan;
@@ -9,12 +8,12 @@ use mmtk::Plan;
 use std::collections::VecDeque;
 use std::marker::PhantomData;
 
-struct OpenJDKMutatorIterator<'a> {
- mutators: VecDeque<&'a mut Mutator<OpenJDK>>,
+struct OpenJDKMutatorIterator<'a, const COMPRESSED: bool> {
+ mutators: VecDeque<&'a mut Mutator<OpenJDK<COMPRESSED>>>,
  phantom_data: PhantomData<&'a ()>,
 }
 
-impl<'a> OpenJDKMutatorIterator<'a> {
+impl<'a, const COMPRESSED: bool> OpenJDKMutatorIterator<'a, COMPRESSED> {
  fn new() -> Self {
  let mut mutators = VecDeque::new();
  unsafe {
@@ -29,8 +28,8 @@ impl<'a> OpenJDKMutatorIterator<'a> {
  }
 }
 
-impl<'a> Iterator for OpenJDKMutatorIterator<'a> {
- type Item = &'a mut Mutator<OpenJDK>;
+impl<'a, const COMPRESSED: bool> Iterator for OpenJDKMutatorIterator<'a, COMPRESSED> {
+ type Item = &'a mut Mutator<OpenJDK<COMPRESSED>>;
 
  fn next(&mut self) -> Option<Self::Item> {
  self.mutators.pop_front()
@@ -39,24 +38,24 @@ impl<'a> Iterator for OpenJDKMutatorIterator<'a> {
 
 pub struct VMActivePlan {}
 
-impl ActivePlan<OpenJDK> for VMActivePlan {
- fn global() -> &'static dyn Plan<VM = OpenJDK> {
- SINGLETON.get_plan()
+impl<const COMPRESSED: bool> ActivePlan<OpenJDK<COMPRESSED>> for VMActivePlan {
+ fn global() -> &'static dyn Plan<VM = OpenJDK<COMPRESSED>> {
+ crate::singleton::<COMPRESSED>().get_plan()
  }
 
  fn is_mutator(tls: VMThread) -> bool {
  unsafe { ((*UPCALLS).is_mutator)(tls) }
  }
 
- fn mutator(tls: VMMutatorThread) -> &'static mut Mutator<OpenJDK> {
+ fn mutator(tls: VMMutatorThread) -> &'static mut Mutator<OpenJDK<COMPRESSED>> {
  unsafe {
  let m = ((*UPCALLS).get_mmtk_mutator)(tls);
- &mut *m
+ &mut *(m as *mut Mutator<OpenJDK<COMPRESSED>>)
  }
  }
 
- fn mutators<'a>() -> Box<dyn Iterator<Item = &'a mut Mutator<OpenJDK>> + 'a> {
- Box::new(OpenJDKMutatorIterator::new())
+ fn mutators<'a>() -> Box<dyn Iterator<Item = &'a mut Mutator<OpenJDK<COMPRESSED>>> + 'a> {
+ Box::new(OpenJDKMutatorIterator::<COMPRESSED>::new())
  }
 
  fn number_of_mutators() -> usize {