Upgrades the coverage map to Version 4

compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs

@@ -4,7 +4,7 @@ use crate::llvm;
 
 use llvm::coverageinfo::CounterMappingRegion;
 use rustc_codegen_ssa::coverageinfo::map::{Counter, CounterExpression};
-use rustc_codegen_ssa::traits::{BaseTypeMethods, ConstMethods};
+use rustc_codegen_ssa::traits::ConstMethods;
 use rustc_data_structures::fx::FxIndexSet;
 use rustc_llvm::RustString;
 use rustc_middle::mir::coverage::CodeRegion;
@@ -38,46 +38,50 @@ pub fn finalize<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) {
     let mut mapgen = CoverageMapGenerator::new();
 
     // Encode coverage mappings and generate function records
-    let mut function_records = Vec::<&'ll llvm::Value>::new();
-    let coverage_mappings_buffer = llvm::build_byte_buffer(|coverage_mappings_buffer| {
-        for (instance, function_coverage) in function_coverage_map.into_iter() {
-            debug!("Generate coverage map for: {:?}", instance);
-
-            let mangled_function_name = cx.tcx.symbol_name(instance).to_string();
-            let function_source_hash = function_coverage.source_hash();
-            let (expressions, counter_regions) =
-                function_coverage.get_expressions_and_counter_regions();
-
-            let old_len = coverage_mappings_buffer.len();
-            mapgen.write_coverage_mappings(expressions, counter_regions, coverage_mappings_buffer);
-            let mapping_data_size = coverage_mappings_buffer.len() - old_len;
-            debug_assert!(
-                mapping_data_size > 0,
-                "Every `FunctionCoverage` should have at least one counter"
-            );
-
-            let function_record = mapgen.make_function_record(
-                cx,
-                mangled_function_name,
-                function_source_hash,
-                mapping_data_size,
-            );
-            function_records.push(function_record);
-        }
-    });
+    let mut function_data = Vec::new();
+    for (instance, function_coverage) in function_coverage_map.into_iter() {
+        debug!("Generate coverage map for: {:?}", instance);
+
+        let mangled_function_name = cx.tcx.symbol_name(instance).to_string();
+        let function_source_hash = function_coverage.source_hash();
+        let (expressions, counter_regions) =
+            function_coverage.get_expressions_and_counter_regions();
+
+        let coverage_mapping_buffer = llvm::build_byte_buffer(|coverage_mapping_buffer| {
+            mapgen.write_coverage_mapping(expressions, counter_regions, coverage_mapping_buffer);
+        });
+        debug_assert!(
+            coverage_mapping_buffer.len() > 0,
+            "Every `FunctionCoverage` should have at least one counter"
+        );
+
+        function_data.push((mangled_function_name, function_source_hash, coverage_mapping_buffer));
+    }
 
     // Encode all filenames referenced by counters/expressions in this module
     let filenames_buffer = llvm::build_byte_buffer(|filenames_buffer| {
         coverageinfo::write_filenames_section_to_buffer(&mapgen.filenames, filenames_buffer);
     });
 
+    let filenames_size = filenames_buffer.len();
+    let filenames_val = cx.const_bytes(&filenames_buffer[..]);
+    let filenames_ref = coverageinfo::hash_bytes(filenames_buffer);
+
     // Generate the LLVM IR representation of the coverage map and store it in a well-known global
-    mapgen.save_generated_coverage_map(
-        cx,
-        function_records,
-        filenames_buffer,
-        coverage_mappings_buffer,
-    );
+    let cov_data_val = mapgen.generate_coverage_map(cx, filenames_size, filenames_val);
+
+    for (mangled_function_name, function_source_hash, coverage_mapping_buffer) in function_data {
+        save_function_record(
+            cx,
+            mangled_function_name,
+            function_source_hash,
+            filenames_ref,
+            coverage_mapping_buffer,
+        );
+    }
+
+    // Save the coverage data value to LLVM IR
+    coverageinfo::save_cov_data_to_mod(cx, cov_data_val);
 }
 
 struct CoverageMapGenerator {
@@ -92,12 +96,12 @@ impl CoverageMapGenerator {
     /// Using the `expressions` and `counter_regions` collected for the current function, generate
     /// the `mapping_regions` and `virtual_file_mapping`, and capture any new filenames. Then use
     /// LLVM APIs to encode the `virtual_file_mapping`, `expressions`, and `mapping_regions` into
-    /// the given `coverage_mappings` byte buffer, compliant with the LLVM Coverage Mapping format.
-    fn write_coverage_mappings(
+    /// the given `coverage_mapping` byte buffer, compliant with the LLVM Coverage Mapping format.
+    fn write_coverage_mapping(
         &mut self,
         expressions: Vec<CounterExpression>,
         counter_regions: impl Iterator<Item = (Counter, &'a CodeRegion)>,
-        coverage_mappings_buffer: &RustString,
+        coverage_mapping_buffer: &RustString,
     ) {
         let mut counter_regions = counter_regions.collect::<Vec<_>>();
         if counter_regions.is_empty() {
@@ -145,89 +149,78 @@ impl CoverageMapGenerator {
             virtual_file_mapping,
             expressions,
             mapping_regions,
-            coverage_mappings_buffer,
+            coverage_mapping_buffer,
         );
     }
 
-    /// Generate and return the function record `Value`
-    fn make_function_record(
-        &mut self,
-        cx: &CodegenCx<'ll, 'tcx>,
-        mangled_function_name: String,
-        function_source_hash: u64,
-        mapping_data_size: usize,
-    ) -> &'ll llvm::Value {
-        let name_ref = coverageinfo::compute_hash(&mangled_function_name);
-        let name_ref_val = cx.const_u64(name_ref);
-        let mapping_data_size_val = cx.const_u32(mapping_data_size as u32);
-        let func_hash_val = cx.const_u64(function_source_hash);
-        cx.const_struct(
-            &[name_ref_val, mapping_data_size_val, func_hash_val],
-            /*packed=*/ true,
-        )
-    }
-
-    /// Combine the filenames and coverage mappings buffers, construct coverage map header and the
-    /// array of function records, and combine everything into the complete coverage map. Save the
-    /// coverage map data into the LLVM IR as a static global using a specific, well-known section
-    /// and name.
-    fn save_generated_coverage_map(
+    /// Construct coverage map header and the array of function records, and combine them into the
+    /// coverage map. Save the coverage map data into the LLVM IR as a static global using a
+    /// specific, well-known section and name.
+    fn generate_coverage_map(
         self,
         cx: &CodegenCx<'ll, 'tcx>,
-        function_records: Vec<&'ll llvm::Value>,
-        filenames_buffer: Vec<u8>,
-        mut coverage_mappings_buffer: Vec<u8>,
-    ) {
-        // Concatenate the encoded filenames and encoded coverage mappings, and add additional zero
-        // bytes as-needed to ensure 8-byte alignment.
-        let mut coverage_size = coverage_mappings_buffer.len();
-        let filenames_size = filenames_buffer.len();
-        let remaining_bytes =
-            (filenames_size + coverage_size) % coverageinfo::COVMAP_VAR_ALIGN_BYTES;
-        if remaining_bytes > 0 {
-            let pad = coverageinfo::COVMAP_VAR_ALIGN_BYTES - remaining_bytes;
-            coverage_mappings_buffer.append(&mut [0].repeat(pad));
-            coverage_size += pad;
-        }
-        let filenames_and_coverage_mappings = [filenames_buffer, coverage_mappings_buffer].concat();
-        let filenames_and_coverage_mappings_val =
-            cx.const_bytes(&filenames_and_coverage_mappings[..]);
-
+        filenames_size: usize,
+        filenames_val: &'ll llvm::Value,
+    ) -> &'ll llvm::Value {
         debug!(
-            "cov map: n_records = {}, filenames_size = {}, coverage_size = {}, 0-based version = {}",
-            function_records.len(),
+            "cov map: filenames_size = {}, 0-based version = {}",
             filenames_size,
-            coverage_size,
             coverageinfo::mapping_version()
         );
 
-        // Create the coverage data header
-        let n_records_val = cx.const_u32(function_records.len() as u32);
+        // Create the coverage data header (Note, fields 0 and 2 are now always zero,
+        // as of `llvm::coverage::CovMapVersion::Version4`.
+        let zero_was_n_records_val = cx.const_u32(0);
         let filenames_size_val = cx.const_u32(filenames_size as u32);
-        let coverage_size_val = cx.const_u32(coverage_size as u32);
+        let zero_was_coverage_size_val = cx.const_u32(0 as u32);
         let version_val = cx.const_u32(coverageinfo::mapping_version());
         let cov_data_header_val = cx.const_struct(
-            &[n_records_val, filenames_size_val, coverage_size_val, version_val],
+            &[zero_was_n_records_val, filenames_size_val, zero_was_coverage_size_val, version_val],
             /*packed=*/ false,
         );
 
-        // Create the function records array
-        let name_ref_from_u64 = cx.type_i64();
-        let mapping_data_size_from_u32 = cx.type_i32();
-        let func_hash_from_u64 = cx.type_i64();
-        let function_record_ty = cx.type_struct(
-            &[name_ref_from_u64, mapping_data_size_from_u32, func_hash_from_u64],
-            /*packed=*/ true,
-        );
-        let function_records_val = cx.const_array(function_record_ty, &function_records[..]);
-
         // Create the complete LLVM coverage data value to add to the LLVM IR
-        let cov_data_val = cx.const_struct(
-            &[cov_data_header_val, function_records_val, filenames_and_coverage_mappings_val],
-            /*packed=*/ false,
-        );
-
-        // Save the coverage data value to LLVM IR
-        coverageinfo::save_map_to_mod(cx, cov_data_val);
+        cx.const_struct(&[cov_data_header_val, filenames_val], /*packed=*/ false)
     }
 }
+
+/// Construct a function record and combine it with the function's coverage mapping data.
+/// Save the function record into the LLVM IR as a static global using a
+/// specific, well-known section and name.
+fn save_function_record(
+    cx: &CodegenCx<'ll, 'tcx>,
+    mangled_function_name: String,
+    function_source_hash: u64,
+    filenames_ref: u64,
+    coverage_mapping_buffer: Vec<u8>,
+) {
+    // Concatenate the encoded coverage mappings
+    let coverage_mapping_size = coverage_mapping_buffer.len();
+    let coverage_mapping_val = cx.const_bytes(&coverage_mapping_buffer[..]);
+
+    let func_name_hash = coverageinfo::hash_str(&mangled_function_name);
+    let func_name_hash_val = cx.const_u64(func_name_hash);
+    let coverage_mapping_size_val = cx.const_u32(coverage_mapping_size as u32);
+    let func_hash_val = cx.const_u64(function_source_hash);
+    let filenames_ref_val = cx.const_u64(filenames_ref);
+    let func_record_val = cx.const_struct(
+        &[
+            func_name_hash_val,
+            coverage_mapping_size_val,
+            func_hash_val,
+            filenames_ref_val,
+            coverage_mapping_val,
+        ],
+        /*packed=*/ true,
+    );
+
+    // At the present time, the coverage map for Rust assumes every instrumented function `is_used`.
+    // Note that Clang marks functions as "unused" in `CodeGenPGO::emitEmptyCounterMapping`. (See:
+    // https://github.com/rust-lang/llvm-project/blob/de02a75e398415bad4df27b4547c25b896c8bf3b/clang%2Flib%2FCodeGen%2FCodeGenPGO.cpp#L877-L878
+    // for example.)
+    //
+    // It's not yet clear if or how this may be applied to Rust in the future, but the `is_used`
+    // argument is available and handled similarly.
+    let is_used = true;
+    coverageinfo::save_func_record_to_mod(cx, func_name_hash, func_record_val, is_used);
+}

compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs

@@ -23,7 +23,7 @@ use tracing::debug;
 
 pub mod mapgen;
 
-const COVMAP_VAR_ALIGN_BYTES: usize = 8;
+const VAR_ALIGN_BYTES: usize = 8;
 
 /// A context object for maintaining all state needed by the coverageinfo module.
 pub struct CrateCoverageContext<'tcx> {
@@ -177,17 +177,20 @@ pub(crate) fn write_mapping_to_buffer(
         );
     }
 }
+pub(crate) fn hash_str(strval: &str) -> u64 {
+    let strval = CString::new(strval).expect("null error converting hashable str to C string");
+    unsafe { llvm::LLVMRustCoverageHashCString(strval.as_ptr()) }
+}
 
-pub(crate) fn compute_hash(name: &str) -> u64 {
-    let name = CString::new(name).expect("null error converting hashable name to C string");
-    unsafe { llvm::LLVMRustCoverageComputeHash(name.as_ptr()) }
+pub(crate) fn hash_bytes(bytes: Vec<u8>) -> u64 {
+    unsafe { llvm::LLVMRustCoverageHashByteArray(bytes.as_ptr().cast(), bytes.len()) }
 }
 
 pub(crate) fn mapping_version() -> u32 {
     unsafe { llvm::LLVMRustCoverageMappingVersion() }
 }
 
-pub(crate) fn save_map_to_mod<'ll, 'tcx>(
+pub(crate) fn save_cov_data_to_mod<'ll, 'tcx>(
     cx: &CodegenCx<'ll, 'tcx>,
     cov_data_val: &'ll llvm::Value,
 ) {
@@ -198,16 +201,51 @@ pub(crate) fn save_map_to_mod<'ll, 'tcx>(
     debug!("covmap var name: {:?}", covmap_var_name);
 
     let covmap_section_name = llvm::build_string(|s| unsafe {
-        llvm::LLVMRustCoverageWriteSectionNameToString(cx.llmod, s);
+        llvm::LLVMRustCoverageWriteMapSectionNameToString(cx.llmod, s);
     })
     .expect("Rust Coverage section name failed UTF-8 conversion");
     debug!("covmap section name: {:?}", covmap_section_name);
 
     let llglobal = llvm::add_global(cx.llmod, cx.val_ty(cov_data_val), &covmap_var_name);
     llvm::set_initializer(llglobal, cov_data_val);
     llvm::set_global_constant(llglobal, true);
-    llvm::set_linkage(llglobal, llvm::Linkage::InternalLinkage);
+    llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage);
     llvm::set_section(llglobal, &covmap_section_name);
-    llvm::set_alignment(llglobal, COVMAP_VAR_ALIGN_BYTES);
+    llvm::set_alignment(llglobal, VAR_ALIGN_BYTES);
+    cx.add_used_global(llglobal);
+}
+
+pub(crate) fn save_func_record_to_mod<'ll, 'tcx>(
+    cx: &CodegenCx<'ll, 'tcx>,
+    func_name_hash: u64,
+    func_record_val: &'ll llvm::Value,
+    is_used: bool,
+) {
+    // Assign a name to the function record. This is used to merge duplicates.
+    //
+    // In LLVM, a "translation unit" (effectively, a `Crate` in Rust) can describe functions that
+    // are included-but-not-used. If (or when) Rust generates functions that are
+    // included-but-not-used, note that a dummy description for a function included-but-not-used
+    // in a Crate can be replaced by full description provided by a different Crate. The two kinds
+    // of descriptions play distinct roles in LLVM IR; therefore, assign them different names (by
+    // appending "u" to the end of the function record var name, to prevent `linkonce_odr` merging.
+    let func_record_var_name =
+        format!("__covrec_{:X}{}", func_name_hash, if is_used { "u" } else { "" });
+    debug!("function record var name: {:?}", func_record_var_name);
+
+    let func_record_section_name = llvm::build_string(|s| unsafe {
+        llvm::LLVMRustCoverageWriteFuncSectionNameToString(cx.llmod, s);
+    })
+    .expect("Rust Coverage function record section name failed UTF-8 conversion");
+    debug!("function record section name: {:?}", func_record_section_name);
+
+    let llglobal = llvm::add_global(cx.llmod, cx.val_ty(func_record_val), &func_record_var_name);
+    llvm::set_initializer(llglobal, func_record_val);
+    llvm::set_global_constant(llglobal, true);
+    llvm::set_linkage(llglobal, llvm::Linkage::LinkOnceODRLinkage);
+    llvm::set_visibility(llglobal, llvm::Visibility::Hidden);
+    llvm::set_section(llglobal, &func_record_section_name);
+    llvm::set_alignment(llglobal, VAR_ALIGN_BYTES);
+    llvm::set_comdat(cx.llmod, llglobal, &func_record_var_name);
     cx.add_used_global(llglobal);
 }

compiler/rustc_codegen_llvm/src/llvm/ffi.rs

@@ -1791,10 +1791,14 @@ extern "C" {
 
     pub fn LLVMRustCoverageCreatePGOFuncNameVar(F: &'a Value, FuncName: *const c_char)
     -> &'a Value;
-    pub fn LLVMRustCoverageComputeHash(Name: *const c_char) -> u64;
+    pub fn LLVMRustCoverageHashCString(StrVal: *const c_char) -> u64;
+    pub fn LLVMRustCoverageHashByteArray(Bytes: *const c_char, NumBytes: size_t) -> u64;
 
     #[allow(improper_ctypes)]
-    pub fn LLVMRustCoverageWriteSectionNameToString(M: &Module, Str: &RustString);
+    pub fn LLVMRustCoverageWriteMapSectionNameToString(M: &Module, Str: &RustString);
+
+    #[allow(improper_ctypes)]
+    pub fn LLVMRustCoverageWriteFuncSectionNameToString(M: &Module, Str: &RustString);
 
     #[allow(improper_ctypes)]
     pub fn LLVMRustCoverageWriteMappingVarNameToString(Str: &RustString);

compiler/rustc_codegen_llvm/src/llvm/mod.rs

@@ -220,12 +220,24 @@ pub fn set_linkage(llglobal: &Value, linkage: Linkage) {
     }
 }
 
+pub fn set_visibility(llglobal: &Value, visibility: Visibility) {
+    unsafe {
+        LLVMRustSetVisibility(llglobal, visibility);
+    }
+}
+
 pub fn set_alignment(llglobal: &Value, bytes: usize) {
     unsafe {
         ffi::LLVMSetAlignment(llglobal, bytes as c_uint);
     }
 }
 
+pub fn set_comdat(llmod: &Module, llglobal: &Value, name: &str) {
+    unsafe {
+        LLVMRustSetComdat(llmod, llglobal, name.as_ptr().cast(), name.len());
+    }
+}
+
 /// Safe wrapper around `LLVMGetParam`, because segfaults are no fun.
 pub fn get_param(llfn: &Value, index: c_uint) -> &Value {
     unsafe {