Add missing docs for some util modules (#1024)

This PR is a step towards #309.
* Add missing docs for some `util` modules.
* Make `util::metadata::side_metadata::helpers/helpers_32` not public.
* Make `util::reference_processor` not public.
* Allow adding docs for options defined by the `options!` macro.

Author: qinsoon

.github/scripts/ci-test.sh

@@ -13,7 +13,7 @@ fi
 
 ./examples/build.py
 
-ALL_PLANS=$(sed -n '/enum PlanSelector/,/}/p' src/util/options.rs | xargs | grep -o '{.*}' | grep -o '\w\+')
+ALL_PLANS=$(sed -n '/enum PlanSelector/,/}/p' src/util/options.rs | sed -e 's;//.*;;g' -e '/^$/d' -e 's/,//g' | xargs | grep -o '{.*}' | grep -o '\w\+')
 
 # Test with DummyVM (each test in a separate run)
 cd vmbindings/dummyvm

src/memory_manager.rs

@@ -735,7 +735,7 @@ pub fn is_mapped_address(address: Address) -> bool {
 /// * `mmtk`: A reference to an MMTk instance.
 /// * `reff`: The weak reference to add.
 pub fn add_weak_candidate<VM: VMBinding>(mmtk: &MMTK<VM>, reff: ObjectReference) {
-    mmtk.reference_processors.add_weak_candidate::<VM>(reff);
+    mmtk.reference_processors.add_weak_candidate(reff);
 }
 
 /// Add a reference to the list of soft references. A binding may
@@ -745,7 +745,7 @@ pub fn add_weak_candidate<VM: VMBinding>(mmtk: &MMTK<VM>, reff: ObjectReference)
 /// * `mmtk`: A reference to an MMTk instance.
 /// * `reff`: The soft reference to add.
 pub fn add_soft_candidate<VM: VMBinding>(mmtk: &MMTK<VM>, reff: ObjectReference) {
-    mmtk.reference_processors.add_soft_candidate::<VM>(reff);
+    mmtk.reference_processors.add_soft_candidate(reff);
 }
 
 /// Add a reference to the list of phantom references. A binding may
@@ -755,7 +755,7 @@ pub fn add_soft_candidate<VM: VMBinding>(mmtk: &MMTK<VM>, reff: ObjectReference)
 /// * `mmtk`: A reference to an MMTk instance.
 /// * `reff`: The phantom reference to add.
 pub fn add_phantom_candidate<VM: VMBinding>(mmtk: &MMTK<VM>, reff: ObjectReference) {
-    mmtk.reference_processors.add_phantom_candidate::<VM>(reff);
+    mmtk.reference_processors.add_phantom_candidate(reff);
 }
 
 /// Generic hook to allow benchmarks to be harnessed. We do a full heap

src/policy/marksweepspace/malloc_ms/metadata.rs

@@ -108,7 +108,7 @@ fn map_active_chunk_metadata(chunk_start: Address) {
 /// We map the active chunk metadata (if not previously mapped), as well as the VO bit metadata
 /// and active page metadata here. Note that if [addr, addr + size) crosses multiple chunks, we
 /// will map for each chunk.
-pub fn map_meta_space(metadata: &SideMetadataContext, addr: Address, size: usize) {
+pub(super) fn map_meta_space(metadata: &SideMetadataContext, addr: Address, size: usize) {
     // In order to prevent race conditions, we synchronize on the lock first and then
     // check if we need to map the active chunk metadata for `chunk_start`
     let _lock = CHUNK_MAP_LOCK.lock().unwrap();
@@ -197,7 +197,7 @@ pub(super) unsafe fn is_page_marked_unsafe(page_addr: Address) -> bool {
     ACTIVE_PAGE_METADATA_SPEC.load::<u8>(page_addr) == 1
 }
 
-pub fn is_chunk_mapped(chunk_start: Address) -> bool {
+pub(super) fn is_chunk_mapped(chunk_start: Address) -> bool {
     // Since `address_to_meta_address` will translate a data address to a metadata address without caring
     // if it goes across metadata boundaries, we have to check if we have accidentally gone over the bounds
     // of the active chunk metadata spec before we check if the metadata has been mapped or not

src/policy/space.rs

@@ -431,7 +431,7 @@ pub struct CommonSpace<VM: VMBinding> {
     pub vm_map: &'static dyn VMMap,
     pub mmapper: &'static dyn Mmapper,
 
-    pub metadata: SideMetadataContext,
+    pub(crate) metadata: SideMetadataContext,
 
     /// This field equals to needs_log_bit in the plan constraints.
     // TODO: This should be a constant for performance.

src/util/heap/layout/vm_layout.rs

@@ -1,3 +1,5 @@
+//! The module defines virutal memory layout parameters.
+
 use std::sync::atomic::AtomicBool;
 
 use atomic::Ordering;
@@ -8,26 +10,17 @@ use crate::util::Address;
 
 use crate::util::conversions::{chunk_align_down, chunk_align_up};
 
-/**
- * log_2 of the coarsest unit of address space allocation.
- *
- * In the 32-bit VM layout, this determines the granularity of
- * allocation in a discontigouous space.  In the 64-bit layout,
- * this determines the growth factor of the large contiguous spaces
- * that we provide.
- */
+/// log_2 of the coarsest unit of address space allocation.
 pub const LOG_BYTES_IN_CHUNK: usize = 22;
-
-/** Coarsest unit of address space allocation. */
+/// Coarsest unit of address space allocation.
 pub const BYTES_IN_CHUNK: usize = 1 << LOG_BYTES_IN_CHUNK;
+/// Mask for chunk size.
 pub const CHUNK_MASK: usize = (1 << LOG_BYTES_IN_CHUNK) - 1;
-
-/** Coarsest unit of address space allocation, in pages */
+/// Coarsest unit of address space allocation, in pages
 pub const PAGES_IN_CHUNK: usize = 1 << (LOG_BYTES_IN_CHUNK - LOG_BYTES_IN_PAGE as usize);
-
-/** Granularity at which we map and unmap virtual address space in the heap */
+/// log_2 of the granularity at which we map and unmap virtual address space in the heap
 pub const LOG_MMAP_CHUNK_BYTES: usize = LOG_BYTES_IN_CHUNK;
-
+/// Granularity at which we map and unmap virtual address space in the heap
 pub const MMAP_CHUNK_BYTES: usize = 1 << LOG_MMAP_CHUNK_BYTES;
 
 /// Runtime-initialized virtual memory constants
@@ -50,9 +43,12 @@ pub struct VMLayout {
 
 impl VMLayout {
     #[cfg(target_pointer_width = "32")]
+    /// The maximum virtual memory address space that can be used on the target.
     pub const LOG_ARCH_ADDRESS_SPACE: usize = 32;
     #[cfg(target_pointer_width = "64")]
+    /// The maximum virtual memory address space that can be used on the target.
     pub const LOG_ARCH_ADDRESS_SPACE: usize = 47;
+
     /// An upper bound on the extent of any space in the
     /// current memory layout
     pub const fn max_space_extent(&self) -> usize {
@@ -189,6 +185,9 @@ static mut VM_LAYOUT: VMLayout = VMLayout::new_64bit();
 
 static VM_LAYOUT_FETCHED: AtomicBool = AtomicBool::new(false);
 
+/// Get the current virtual memory layout in use.
+/// If the binding would like to set a custom virtual memory layout ([`crate::mmtk::MMTKBuilder::set_vm_layout`]), they should not
+/// call this function before they set a custom layout.
 pub fn vm_layout() -> &'static VMLayout {
     if cfg!(debug_assertions) {
         VM_LAYOUT_FETCHED.store(true, Ordering::SeqCst);

src/util/malloc/mod.rs

@@ -1,6 +1,17 @@
+//! This module exposes a set of malloc API. They are currently implemented with
+//! the library malloc. This may change in the future, and will be replaced
+//! with a native MMTk implementation.
+
+//! We have two versions for each function:
+//! * a normal version: it has the signature that is compatible with the standard malloc library.
+//! * a counted version: the allocated/freed bytes are calculated into MMTk's heap. So extra arguments
+//!   are needed to maintain allocated bytes properly. The API is inspired by Julia's counted malloc.
+//!   The counted version is only available with the feature `malloc_counted_size`.
+
 /// Malloc provided by libraries
 pub(crate) mod library;
-/// Using malloc as mark sweep free-list allocator
+/// Using malloc as mark sweep free-list allocator.
+// This module is made public so we can test it from dummyvm. It should be pub(crate).
 pub mod malloc_ms_util;
 
 use crate::util::Address;
@@ -9,20 +20,13 @@ use crate::vm::VMBinding;
 #[cfg(feature = "malloc_counted_size")]
 use crate::MMTK;
 
-// The following expose a set of malloc API. They are currently implemented with
-// the library malloc. When we have native malloc implementation, we should change
-// their implementation to point to our native malloc.
-
-// We have two versions for each function:
-// * a normal version: it has the signature that is compatible with the standard malloc library.
-// * a counted version: the allocated/freed bytes are calculated into MMTk's heap. So extra arguments
-//   are needed to maintain allocated bytes properly. The API is inspired by Julia's counted malloc.
-//   The counted version is only available with the feature `malloc_counted_size`.
-
+/// Manually allocate memory. Similar to libc's malloc.
 pub fn malloc(size: usize) -> Address {
     Address::from_mut_ptr(unsafe { self::library::malloc(size) })
 }
 
+/// Manually allocate memory. Similar to libc's malloc.
+/// This also counts the allocated memory into the heap size of the given MMTk instance.
 #[cfg(feature = "malloc_counted_size")]
 pub fn counted_malloc<VM: VMBinding>(mmtk: &MMTK<VM>, size: usize) -> Address {
     let res = malloc(size);
@@ -32,10 +36,13 @@ pub fn counted_malloc<VM: VMBinding>(mmtk: &MMTK<VM>, size: usize) -> Address {
     res
 }
 
+/// Manually allocate memory and initialize the bytes in the allocated memory to zero. Similar to libc's calloc.
 pub fn calloc(num: usize, size: usize) -> Address {
     Address::from_mut_ptr(unsafe { self::library::calloc(num, size) })
 }
 
+/// Manually allocate memory and initialize the bytes in the allocated memory to zero. Similar to libc's calloc.
+/// This also counts the allocated memory into the heap size of the given MMTk instance.
 #[cfg(feature = "malloc_counted_size")]
 pub fn counted_calloc<VM: VMBinding>(mmtk: &MMTK<VM>, num: usize, size: usize) -> Address {
     let res = calloc(num, size);
@@ -45,10 +52,14 @@ pub fn counted_calloc<VM: VMBinding>(mmtk: &MMTK<VM>, num: usize, size: usize) -
     res
 }
 
+/// Reallocate the given area of memory. Similar to libc's realloc.
 pub fn realloc(addr: Address, size: usize) -> Address {
     Address::from_mut_ptr(unsafe { self::library::realloc(addr.to_mut_ptr(), size) })
 }
 
+/// Reallocate the given area of memory. Similar to libc's realloc.
+/// This also adjusts the allocated memory size based on the original allocation and the new allocation, and counts
+/// that into the heap size for the given MMTk instance.
 #[cfg(feature = "malloc_counted_size")]
 pub fn realloc_with_old_size<VM: VMBinding>(
     mmtk: &MMTK<VM>,
@@ -68,10 +79,13 @@ pub fn realloc_with_old_size<VM: VMBinding>(
     res
 }
 
+/// Manually free the memory that is returned from other manual allocation functions in this module.
 pub fn free(addr: Address) {
     unsafe { self::library::free(addr.to_mut_ptr()) }
 }
 
+/// Manually free the memory that is returned from other manual allocation functions in this module.
+/// This also reduces the allocated memory size.
 #[cfg(feature = "malloc_counted_size")]
 pub fn free_with_size<VM: VMBinding>(mmtk: &MMTK<VM>, addr: Address, old_size: usize) {
     free(addr);

src/util/metadata/global.rs

@@ -14,15 +14,19 @@ use atomic::Ordering;
 /// For performance reasons, objects of this struct should be constants.
 #[derive(Clone, Copy, Debug)]
 pub enum MetadataSpec {
+    /// In-header metadata uses bits from an object header.
     InHeader(HeaderMetadataSpec),
+    /// On-side metadata uses a side table.
     OnSide(SideMetadataSpec),
 }
 
 impl MetadataSpec {
+    /// Is this metadata stored in the side table?
     pub const fn is_on_side(&self) -> bool {
         matches!(self, &MetadataSpec::OnSide(_))
     }
 
+    /// Is this metadata stored in the object header?
     pub const fn is_in_header(&self) -> bool {
         matches!(self, &MetadataSpec::InHeader(_))
     }

src/util/metadata/log_bit.rs

@@ -28,6 +28,7 @@ impl VMGlobalLogBitSpec {
         }
     }
 
+    /// Check if the log bit represents the unlogged state (the bit is 1).
     pub fn is_unlogged<VM: VMBinding>(&self, object: ObjectReference, order: Ordering) -> bool {
         self.load_atomic::<VM, u8>(object, None, order) == 1
     }

src/util/metadata/metadata_val_traits.rs

@@ -6,7 +6,9 @@ use num_traits::{Unsigned, WrappingAdd, WrappingSub, Zero};
 /// Describes bits and log2 bits for the numbers.
 /// If num_traits has this, we do not need our own implementation: <https://github.com/rust-num/num-traits/issues/247>
 pub trait Bits {
+    /// The size of this atomic type in bits.
     const BITS: u32;
+    /// The size (in log2) of this atomic type in bits.
     const LOG2: u32;
 }
 macro_rules! impl_bits_trait {
@@ -26,9 +28,13 @@ impl_bits_trait!(usize);
 /// Describes bitwise operations.
 /// If num_traits has this, we do not need our own implementation: <https://github.com/rust-num/num-traits/issues/232>
 pub trait BitwiseOps {
+    /// Perform bitwise and for two values.
     fn bitand(self, other: Self) -> Self;
+    /// Perform bitwise or for two values.
     fn bitor(self, other: Self) -> Self;
+    /// Perform bitwise xor for two values.
     fn bitxor(self, other: Self) -> Self;
+    /// Perform bitwise invert (not) for the value.
     fn inv(self) -> Self;
 }
 macro_rules! impl_bitwise_ops_trait {

src/util/metadata/mod.rs

@@ -199,7 +199,7 @@
 //! When a space is created by a plan (e.g. SemiSpace::new), the plan can create its global specs by `MetadataContext::new_global_specs(&[GLOBAL_META_1, GLOBAL_META_2])`. Then,
 //! the global specs are passed to each space that the plan creates.
 //!
-//! Each space will then combine the global specs and its own local specs to create a [SideMetadataContext](crate::util::metadata::side_metadata::SideMetadataContext).
+//! Each space will then combine the global specs and its own local specs to create a SideMetadataContext.
 //! Allocating side metadata space and accounting its memory usage is done by `SideMetadata`. If a space uses `CommonSpace`, `CommonSpace` will create `SideMetadata` and manage
 //! reserving and allocating metadata space when necessary. If a space does not use `CommonSpace`, it should create `SideMetadata` itself and manage allocating metadata space
 //! as its own responsibility.