diff --git a/ChangeLog.md b/ChangeLog.md
index 71ec1fba5a206..a237bccaf90e6 100644
--- a/ChangeLog.md
+++ b/ChangeLog.md
@@ -20,6 +20,8 @@ See docs/process.md for more on how version tagging works.
4.0.11 (in development)
-----------------------
+- Add support for [Source-based Code Coverage](https://clang.llvm.org/docs/SourceBasedCodeCoverage.html)
+ To build with coverage enabled use `-fprofile-instr-generate -fcoverage-mapping`. (#24160)
- The `ENVIRONMENT` setting will now be automatically updated to include
`worker` if multi-threading is enabled. (#24525)
- Removed the `HEADLESS` option. It tried to simulate a minimal browser-like
diff --git a/system/lib/compiler-rt/include/profile/InstrProfData.inc b/system/lib/compiler-rt/include/profile/InstrProfData.inc
new file mode 100644
index 0000000000000..39613da81ecb4
--- /dev/null
+++ b/system/lib/compiler-rt/include/profile/InstrProfData.inc
@@ -0,0 +1,1023 @@
+/*===-- InstrProfData.inc - instr profiling runtime structures -*- C++ -*-=== *\
+|*
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+|* See https://llvm.org/LICENSE.txt for license information.
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+|*
+\*===----------------------------------------------------------------------===*/
+/*
+ * This is the main file that defines all the data structure, signature,
+ * constant literals that are shared across profiling runtime library,
+ * compiler (instrumentation), and host tools (reader/writer). The entities
+ * defined in this file affect the profile runtime ABI, the raw profile format,
+ * or both.
+ *
+ * The file has two identical copies. The primary copy lives in LLVM and
+ * the other one sits in compiler-rt/lib/profile directory. To make changes
+ * in this file, first modify the primary copy and copy it over to compiler-rt.
+ * Testing of any change in this file can start only after the two copies are
+ * synced up.
+ *
+ * The first part of the file includes macros that defines types, names, and
+ * initializers for the member fields of the core data structures. The field
+ * declarations for one structure is enabled by defining the field activation
+ * macro associated with that structure. Only one field activation record
+ * can be defined at one time and the rest definitions will be filtered out by
+ * the preprocessor.
+ *
+ * Examples of how the template is used to instantiate structure definition:
+ * 1. To declare a structure:
+ *
+ * struct ProfData {
+ * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
+ * Type Name;
+ * #include "llvm/ProfileData/InstrProfData.inc"
+ * };
+ *
+ * 2. To construct LLVM type arrays for the struct type:
+ *
+ * Type *DataTypes[] = {
+ * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
+ * LLVMType,
+ * #include "llvm/ProfileData/InstrProfData.inc"
+ * };
+ *
+ * 4. To construct constant array for the initializers:
+ * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
+ * Initializer,
+ * Constant *ConstantVals[] = {
+ * #include "llvm/ProfileData/InstrProfData.inc"
+ * };
+ *
+ *
+ * The second part of the file includes definitions all other entities that
+ * are related to runtime ABI and format. When no field activation macro is
+ * defined, this file can be included to introduce the definitions.
+ *
+\*===----------------------------------------------------------------------===*/
+
+/* Functions marked with INSTR_PROF_VISIBILITY must have hidden visibility in
+ * the compiler runtime. */
+#ifndef INSTR_PROF_VISIBILITY
+#define INSTR_PROF_VISIBILITY
+#endif
+
+// clang-format off:consider re-enabling clang-format if auto-formatted C macros
+// are readable (e.g., after `issue #82426` is fixed)
+/* INSTR_PROF_DATA start. */
+/* Definition of member fields of the per-function control structure. */
+#ifndef INSTR_PROF_DATA
+#define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer)
+#else
+#define INSTR_PROF_DATA_DEFINED
+#endif
+INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \
+ ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \
+ IndexedInstrProf::ComputeHash(getPGOFuncNameVarInitializer(Inc->getName()))))
+INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \
+ ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \
+ Inc->getHash()->getZExtValue()))
+INSTR_PROF_DATA(const IntPtrT, IntPtrTy, CounterPtr, RelativeCounterPtr)
+INSTR_PROF_DATA(const IntPtrT, IntPtrTy, BitmapPtr, RelativeBitmapPtr)
+/* This is used to map function pointers for the indirect call targets to
+ * function name hashes during the conversion from raw to merged profile
+ * data.
+ */
+INSTR_PROF_DATA(const IntPtrT, llvm::PointerType::getUnqual(Ctx), FunctionPointer, \
+ FunctionAddr)
+INSTR_PROF_DATA(IntPtrT, llvm::PointerType::getUnqual(Ctx), Values, \
+ ValuesPtrExpr)
+INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \
+ ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters))
+INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \
+ ConstantArray::get(Int16ArrayTy, Int16ArrayVals)) \
+INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumBitmapBytes, \
+ ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumBitmapBytes))
+#undef INSTR_PROF_DATA
+/* INSTR_PROF_DATA end. */
+
+/* For a virtual table object, record the name hash to associate profiled
+ * addresses with global variables, and record {starting address, size in bytes}
+ * to map the profiled virtual table (which usually have an offset from the
+ * starting address) back to a virtual table object. */
+#ifndef INSTR_PROF_VTABLE_DATA
+#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Initializer)
+#else
+#define INSTR_PROF_VTABLE_DATA_DEFINED
+#endif
+INSTR_PROF_VTABLE_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), \
+ VTableNameHash, ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \
+ IndexedInstrProf::ComputeHash(PGOVTableName)))
+INSTR_PROF_VTABLE_DATA(const IntPtrT, llvm::PointerType::getUnqual(Ctx), \
+ VTablePointer, VTableAddr)
+INSTR_PROF_VTABLE_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), VTableSize, \
+ ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \
+ VTableSizeVal))
+#undef INSTR_PROF_VTABLE_DATA
+/* INSTR_PROF_VTABLE_DATA end. */
+
+/* This is an internal data structure used by value profiler. It
+ * is defined here to allow serialization code sharing by LLVM
+ * to be used in unit test.
+ *
+ * typedef struct ValueProfNode {
+ * // InstrProfValueData VData;
+ * uint64_t Value;
+ * uint64_t Count;
+ * struct ValueProfNode *Next;
+ * } ValueProfNode;
+ */
+/* INSTR_PROF_VALUE_NODE start. */
+#ifndef INSTR_PROF_VALUE_NODE
+#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Initializer)
+#else
+#define INSTR_PROF_DATA_DEFINED
+#endif
+INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Value, \
+ ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0))
+INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Count, \
+ ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0))
+INSTR_PROF_VALUE_NODE(PtrToNodeT, llvm::PointerType::getUnqual(Ctx), Next, \
+ ConstantInt::get(llvm::PointerType::getUnqual(Ctx), 0))
+#undef INSTR_PROF_VALUE_NODE
+/* INSTR_PROF_VALUE_NODE end. */
+
+/* INSTR_PROF_RAW_HEADER start */
+/* Definition of member fields of the raw profile header data structure. */
+/* Please update llvm/docs/InstrProfileFormat.rst as appropriate when updating
+ raw profile format. */
+#ifndef INSTR_PROF_RAW_HEADER
+#define INSTR_PROF_RAW_HEADER(Type, Name, Initializer)
+#else
+#define INSTR_PROF_DATA_DEFINED
+#endif
+INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic())
+INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version())
+INSTR_PROF_RAW_HEADER(uint64_t, BinaryIdsSize, __llvm_write_binary_ids(NULL))
+INSTR_PROF_RAW_HEADER(uint64_t, NumData, NumData)
+INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesBeforeCounters, PaddingBytesBeforeCounters)
+INSTR_PROF_RAW_HEADER(uint64_t, NumCounters, NumCounters)
+INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterCounters, PaddingBytesAfterCounters)
+INSTR_PROF_RAW_HEADER(uint64_t, NumBitmapBytes, NumBitmapBytes)
+INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterBitmapBytes, PaddingBytesAfterBitmapBytes)
+INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize)
+INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta,
+ (uintptr_t)CountersBegin - (uintptr_t)DataBegin)
+INSTR_PROF_RAW_HEADER(uint64_t, BitmapDelta,
+ (uintptr_t)BitmapBegin - (uintptr_t)DataBegin)
+INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin)
+INSTR_PROF_RAW_HEADER(uint64_t, NumVTables, NumVTables)
+INSTR_PROF_RAW_HEADER(uint64_t, VNamesSize, VNamesSize)
+INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last)
+#undef INSTR_PROF_RAW_HEADER
+/* INSTR_PROF_RAW_HEADER end */
+
+/* VALUE_PROF_FUNC_PARAM start */
+/* Definition of parameter types of the runtime API used to do value profiling
+ * for a given value site.
+ */
+#ifndef VALUE_PROF_FUNC_PARAM
+#define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType)
+#define INSTR_PROF_COMMA
+#else
+#define INSTR_PROF_DATA_DEFINED
+#define INSTR_PROF_COMMA ,
+#endif
+VALUE_PROF_FUNC_PARAM(uint64_t, TargetValue, Type::getInt64Ty(Ctx)) \
+ INSTR_PROF_COMMA
+VALUE_PROF_FUNC_PARAM(void *, Data, PointerType::getUnqual(Ctx)) INSTR_PROF_COMMA
+VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx))
+#undef VALUE_PROF_FUNC_PARAM
+#undef INSTR_PROF_COMMA
+/* VALUE_PROF_FUNC_PARAM end */
+
+/* VALUE_PROF_KIND start */
+#ifndef VALUE_PROF_KIND
+#define VALUE_PROF_KIND(Enumerator, Value, Descr)
+#else
+#define INSTR_PROF_DATA_DEFINED
+#endif
+/* For indirect function call value profiling, the addresses of the target
+ * functions are profiled by the instrumented code. The target addresses are
+ * written in the raw profile data and converted to target function name's MD5
+ * hash by the profile reader during deserialization. Typically, this happens
+ * when the raw profile data is read during profile merging.
+ *
+ * For this remapping the ProfData is used. ProfData contains both the function
+ * name hash and the function address.
+ */
+VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target")
+/* For memory intrinsic functions size profiling. */
+VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size")
+/* For virtual table address profiling, the address point of the virtual table
+ * (i.e., the address contained in objects pointing to a virtual table) are
+ * profiled. Note this may not be the address of the per C++ class virtual table
+ * object (e.g., there might be an offset).
+ *
+ * The profiled addresses are stored in raw profile, together with the following
+ * two types of information.
+ * 1. The (starting and ending) addresses of per C++ class virtual table objects.
+ * 2. The (compressed) virtual table object names.
+ * RawInstrProfReader converts profiled virtual table addresses to virtual table
+ * objects' MD5 hash.
+ */
+VALUE_PROF_KIND(IPVK_VTableTarget, 2, "The profiled address point of the vtable")
+/* These two kinds must be the last to be
+ * declared. This is to make sure the string
+ * array created with the template can be
+ * indexed with the kind value.
+ */
+VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget, "first")
+VALUE_PROF_KIND(IPVK_Last, IPVK_VTableTarget, "last")
+
+#undef VALUE_PROF_KIND
+/* VALUE_PROF_KIND end */
+
+#undef COVMAP_V2_OR_V3
+#ifdef COVMAP_V2
+#define COVMAP_V2_OR_V3
+#endif
+#ifdef COVMAP_V3
+#define COVMAP_V2_OR_V3
+#endif
+
+/* COVMAP_FUNC_RECORD start */
+/* Definition of member fields of the function record structure in coverage
+ * map.
+ */
+#ifndef COVMAP_FUNC_RECORD
+#define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Initializer)
+#else
+#define INSTR_PROF_DATA_DEFINED
+#endif
+#ifdef COVMAP_V1
+COVMAP_FUNC_RECORD(const IntPtrT, llvm::PointerType::getUnqual(Ctx), \
+ NamePtr, llvm::ConstantExpr::getBitCast(NamePtr, \
+ llvm::PointerType::getUnqual(Ctx)))
+COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), NameSize, \
+ llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \
+ NameValue.size()))
+#endif
+#ifdef COVMAP_V2_OR_V3
+COVMAP_FUNC_RECORD(const int64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \
+ llvm::ConstantInt::get( \
+ llvm::Type::getInt64Ty(Ctx), NameHash))
+#endif
+COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), DataSize, \
+ llvm::ConstantInt::get( \
+ llvm::Type::getInt32Ty(Ctx), CoverageMapping.size()))
+COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \
+ llvm::ConstantInt::get( \
+ llvm::Type::getInt64Ty(Ctx), FuncHash))
+#ifdef COVMAP_V3
+COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FilenamesRef, \
+ llvm::ConstantInt::get( \
+ llvm::Type::getInt64Ty(Ctx), FilenamesRef))
+COVMAP_FUNC_RECORD(const char, \
+ llvm::ArrayType::get(llvm::Type::getInt8Ty(Ctx), \
+ CoverageMapping.size()), \
+ CoverageMapping,
+ llvm::ConstantDataArray::getRaw( \
+ CoverageMapping, CoverageMapping.size(), \
+ llvm::Type::getInt8Ty(Ctx)))
+#endif
+#undef COVMAP_FUNC_RECORD
+/* COVMAP_FUNC_RECORD end. */
+
+/* COVMAP_HEADER start */
+/* Definition of member fields of coverage map header.
+ */
+#ifndef COVMAP_HEADER
+#define COVMAP_HEADER(Type, LLVMType, Name, Initializer)
+#else
+#define INSTR_PROF_DATA_DEFINED
+#endif
+COVMAP_HEADER(uint32_t, Int32Ty, NRecords, \
+ llvm::ConstantInt::get(Int32Ty, NRecords))
+COVMAP_HEADER(uint32_t, Int32Ty, FilenamesSize, \
+ llvm::ConstantInt::get(Int32Ty, FilenamesSize))
+COVMAP_HEADER(uint32_t, Int32Ty, CoverageSize, \
+ llvm::ConstantInt::get(Int32Ty, CoverageMappingSize))
+COVMAP_HEADER(uint32_t, Int32Ty, Version, \
+ llvm::ConstantInt::get(Int32Ty, CovMapVersion::CurrentVersion))
+#undef COVMAP_HEADER
+/* COVMAP_HEADER end. */
+
+/* COVINIT_FUNC start */
+#ifndef COVINIT_FUNC
+#define COVINIT_FUNC(Type, LLVMType, Name, Initializer)
+#else
+#define INSTR_PROF_DATA_DEFINED
+#endif
+COVINIT_FUNC(IntPtrT, llvm::PointerType::getUnqual(Ctx), WriteoutFunction, \
+ WriteoutF)
+COVINIT_FUNC(IntPtrT, llvm::PointerType::getUnqual(Ctx), ResetFunction, \
+ ResetF)
+#undef COVINIT_FUNC
+/* COVINIT_FUNC end */
+
+#ifdef INSTR_PROF_SECT_ENTRY
+#define INSTR_PROF_DATA_DEFINED
+INSTR_PROF_SECT_ENTRY(IPSK_data, \
+ INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON), \
+ INSTR_PROF_DATA_COFF, "__DATA,")
+INSTR_PROF_SECT_ENTRY(IPSK_cnts, \
+ INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON), \
+ INSTR_PROF_CNTS_COFF, "__DATA,")
+INSTR_PROF_SECT_ENTRY(IPSK_bitmap, \
+ INSTR_PROF_QUOTE(INSTR_PROF_BITS_COMMON), \
+ INSTR_PROF_BITS_COFF, "__DATA,")
+INSTR_PROF_SECT_ENTRY(IPSK_name, \
+ INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \
+ INSTR_PROF_NAME_COFF, "__DATA,")
+INSTR_PROF_SECT_ENTRY(IPSK_vname, \
+ INSTR_PROF_QUOTE(INSTR_PROF_VNAME_COMMON), \
+ INSTR_PROF_VNAME_COFF, "__DATA,")
+INSTR_PROF_SECT_ENTRY(IPSK_vals, \
+ INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \
+ INSTR_PROF_VALS_COFF, "__DATA,")
+INSTR_PROF_SECT_ENTRY(IPSK_vnodes, \
+ INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \
+ INSTR_PROF_VNODES_COFF, "__DATA,")
+INSTR_PROF_SECT_ENTRY(IPSK_vtab, \
+ INSTR_PROF_QUOTE(INSTR_PROF_VTAB_COMMON), \
+ INSTR_PROF_VTAB_COFF, "__DATA,")
+INSTR_PROF_SECT_ENTRY(IPSK_covmap, \
+ INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \
+ INSTR_PROF_COVMAP_COFF, "__LLVM_COV,")
+INSTR_PROF_SECT_ENTRY(IPSK_covfun, \
+ INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON), \
+ INSTR_PROF_COVFUN_COFF, "__LLVM_COV,")
+INSTR_PROF_SECT_ENTRY(IPSK_orderfile, \
+ INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON), \
+ INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COFF), "__DATA,")
+INSTR_PROF_SECT_ENTRY(IPSK_covdata, \
+ INSTR_PROF_QUOTE(INSTR_PROF_COVDATA_COMMON), \
+ INSTR_PROF_COVDATA_COFF, "__LLVM_COV,")
+INSTR_PROF_SECT_ENTRY(IPSK_covname, \
+ INSTR_PROF_QUOTE(INSTR_PROF_COVNAME_COMMON), \
+ INSTR_PROF_COVNAME_COFF, "__LLVM_COV,")
+INSTR_PROF_SECT_ENTRY(IPSK_covinit, \
+ INSTR_PROF_QUOTE(INSTR_PROF_COVINIT_COMMON), \
+ INSTR_PROF_COVINIT_COFF, "__LLVM_COV,")
+
+#undef INSTR_PROF_SECT_ENTRY
+#endif
+
+
+#ifdef INSTR_PROF_VALUE_PROF_DATA
+#define INSTR_PROF_DATA_DEFINED
+
+#define INSTR_PROF_MAX_NUM_VAL_PER_SITE 255
+/*!
+ * This is the header of the data structure that defines the on-disk
+ * layout of the value profile data of a particular kind for one function.
+ */
+typedef struct ValueProfRecord {
+ /* The kind of the value profile record. */
+ uint32_t Kind;
+ /*
+ * The number of value profile sites. It is guaranteed to be non-zero;
+ * otherwise the record for this kind won't be emitted.
+ */
+ uint32_t NumValueSites;
+ /*
+ * The first element of the array that stores the number of profiled
+ * values for each value site. The size of the array is NumValueSites.
+ * Since NumValueSites is greater than zero, there is at least one
+ * element in the array.
+ */
+ uint8_t SiteCountArray[1];
+
+ /*
+ * The fake declaration is for documentation purpose only.
+ * Align the start of next field to be on 8 byte boundaries.
+ uint8_t Padding[X];
+ */
+
+ /* The array of value profile data. The size of the array is the sum
+ * of all elements in SiteCountArray[].
+ InstrProfValueData ValueData[];
+ */
+
+#ifdef __cplusplus
+ /*!
+ * Return the number of value sites.
+ */
+ uint32_t getNumValueSites() const { return NumValueSites; }
+ /*!
+ * Read data from this record and save it to Record.
+ */
+ void deserializeTo(InstrProfRecord &Record,
+ InstrProfSymtab *SymTab);
+ /*
+ * In-place byte swap:
+ * Do byte swap for this instance. \c Old is the original order before
+ * the swap, and \c New is the New byte order.
+ */
+ void swapBytes(llvm::endianness Old, llvm::endianness New);
+#endif
+} ValueProfRecord;
+
+/*!
+ * Per-function header/control data structure for value profiling
+ * data in indexed format.
+ */
+typedef struct ValueProfData {
+ /*
+ * Total size in bytes including this field. It must be a multiple
+ * of sizeof(uint64_t).
+ */
+ uint32_t TotalSize;
+ /*
+ *The number of value profile kinds that has value profile data.
+ * In this implementation, a value profile kind is considered to
+ * have profile data if the number of value profile sites for the
+ * kind is not zero. More aggressively, the implementation can
+ * choose to check the actual data value: if none of the value sites
+ * has any profiled values, the kind can be skipped.
+ */
+ uint32_t NumValueKinds;
+
+ /*
+ * Following are a sequence of variable length records. The prefix/header
+ * of each record is defined by ValueProfRecord type. The number of
+ * records is NumValueKinds.
+ * ValueProfRecord Record_1;
+ * ValueProfRecord Record_N;
+ */
+
+#if __cplusplus
+ /*!
+ * Return the total size in bytes of the on-disk value profile data
+ * given the data stored in Record.
+ */
+ static uint32_t getSize(const InstrProfRecord &Record);
+ /*!
+ * Return a pointer to \c ValueProfData instance ready to be streamed.
+ */
+ static std::unique_ptr<ValueProfData>
+ serializeFrom(const InstrProfRecord &Record);
+ /*!
+ * Check the integrity of the record.
+ */
+ Error checkIntegrity();
+ /*!
+ * Return a pointer to \c ValueProfileData instance ready to be read.
+ * All data in the instance are properly byte swapped. The input
+ * data is assumed to be in little endian order.
+ */
+ static Expected<std::unique_ptr<ValueProfData>>
+ getValueProfData(const unsigned char *SrcBuffer,
+ const unsigned char *const SrcBufferEnd,
+ llvm::endianness SrcDataEndianness);
+ /*!
+ * Swap byte order from \c Endianness order to host byte order.
+ */
+ void swapBytesToHost(llvm::endianness Endianness);
+ /*!
+ * Swap byte order from host byte order to \c Endianness order.
+ */
+ void swapBytesFromHost(llvm::endianness Endianness);
+ /*!
+ * Return the total size of \c ValueProfileData.
+ */
+ uint32_t getSize() const { return TotalSize; }
+ /*!
+ * Read data from this data and save it to \c Record.
+ */
+ void deserializeTo(InstrProfRecord &Record,
+ InstrProfSymtab *SymTab);
+ void operator delete(void *ptr) { ::operator delete(ptr); }
+#endif
+} ValueProfData;
+
+/*
+ * The closure is designed to abstact away two types of value profile data:
+ * - InstrProfRecord which is the primary data structure used to
+ * represent profile data in host tools (reader, writer, and profile-use)
+ * - value profile runtime data structure suitable to be used by C
+ * runtime library.
+ *
+ * Both sources of data need to serialize to disk/memory-buffer in common
+ * format: ValueProfData. The abstraction allows compiler-rt's raw profiler
+ * writer to share the same format and code with indexed profile writer.
+ *
+ * For documentation of the member methods below, refer to corresponding methods
+ * in class InstrProfRecord.
+ */
+typedef struct ValueProfRecordClosure {
+ const void *Record;
+ uint32_t (*GetNumValueKinds)(const void *Record);
+ uint32_t (*GetNumValueSites)(const void *Record, uint32_t VKind);
+ uint32_t (*GetNumValueData)(const void *Record, uint32_t VKind);
+ uint32_t (*GetNumValueDataForSite)(const void *R, uint32_t VK, uint32_t S);
+
+ /*
+ * After extracting the value profile data from the value profile record,
+ * this method is used to map the in-memory value to on-disk value. If
+ * the method is null, value will be written out untranslated.
+ */
+ uint64_t (*RemapValueData)(uint32_t, uint64_t Value);
+ void (*GetValueForSite)(const void *R, InstrProfValueData *Dst, uint32_t K,
+ uint32_t S);
+ ValueProfData *(*AllocValueProfData)(size_t TotalSizeInBytes);
+} ValueProfRecordClosure;
+
+INSTR_PROF_VISIBILITY ValueProfRecord *
+getFirstValueProfRecord(ValueProfData *VPD);
+INSTR_PROF_VISIBILITY ValueProfRecord *
+getValueProfRecordNext(ValueProfRecord *VPR);
+INSTR_PROF_VISIBILITY InstrProfValueData *
+getValueProfRecordValueData(ValueProfRecord *VPR);
+INSTR_PROF_VISIBILITY uint32_t
+getValueProfRecordHeaderSize(uint32_t NumValueSites);
+
+#undef INSTR_PROF_VALUE_PROF_DATA
+#endif /* INSTR_PROF_VALUE_PROF_DATA */
+
+
+#ifdef INSTR_PROF_COMMON_API_IMPL
+#define INSTR_PROF_DATA_DEFINED
+#ifdef __cplusplus
+#define INSTR_PROF_INLINE inline
+#define INSTR_PROF_NULLPTR nullptr
+#else
+#define INSTR_PROF_INLINE
+#define INSTR_PROF_NULLPTR NULL
+#endif
+
+#ifndef offsetof
+#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
+#endif
+
+// clang-format on
+
+/*!
+ * Return the \c ValueProfRecord header size including the
+ * padding bytes.
+ */
+INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint32_t
+getValueProfRecordHeaderSize(uint32_t NumValueSites) {
+ uint32_t Size = offsetof(ValueProfRecord, SiteCountArray) +
+ sizeof(uint8_t) * NumValueSites;
+ /* Round the size to multiple of 8 bytes. */
+ Size = (Size + 7) & ~7;
+ return Size;
+}
+
+/*!
+ * Return the total size of the value profile record including the
+ * header and the value data.
+ */
+INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint32_t
+getValueProfRecordSize(uint32_t NumValueSites, uint32_t NumValueData) {
+ return getValueProfRecordHeaderSize(NumValueSites) +
+ sizeof(InstrProfValueData) * NumValueData;
+}
+
+/*!
+ * Return the pointer to the start of value data array.
+ */
+INSTR_PROF_VISIBILITY INSTR_PROF_INLINE InstrProfValueData *
+getValueProfRecordValueData(ValueProfRecord *This) {
+ return (InstrProfValueData *)((char *)This + getValueProfRecordHeaderSize(
+ This->NumValueSites));
+}
+
+/*!
+ * Return the total number of value data for \c This record.
+ */
+INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint32_t
+getValueProfRecordNumValueData(ValueProfRecord *This) {
+ uint32_t NumValueData = 0;
+ uint32_t I;
+ for (I = 0; I < This->NumValueSites; I++)
+ NumValueData += This->SiteCountArray[I];
+ return NumValueData;
+}
+
+/*!
+ * Use this method to advance to the next \c This \c ValueProfRecord.
+ */
+INSTR_PROF_VISIBILITY INSTR_PROF_INLINE ValueProfRecord *
+getValueProfRecordNext(ValueProfRecord *This) {
+ uint32_t NumValueData = getValueProfRecordNumValueData(This);
+ return (ValueProfRecord *)((char *)This +
+ getValueProfRecordSize(This->NumValueSites,
+ NumValueData));
+}
+
+/*!
+ * Return the first \c ValueProfRecord instance.
+ */
+INSTR_PROF_VISIBILITY INSTR_PROF_INLINE ValueProfRecord *
+getFirstValueProfRecord(ValueProfData *This) {
+ return (ValueProfRecord *)((char *)This + sizeof(ValueProfData));
+}
+
+/* Closure based interfaces. */
+
+/*!
+ * Return the total size in bytes of the on-disk value profile data
+ * given the data stored in Record.
+ */
+INSTR_PROF_VISIBILITY uint32_t
+getValueProfDataSize(ValueProfRecordClosure *Closure) {
+ uint32_t Kind;
+ uint32_t TotalSize = sizeof(ValueProfData);
+ const void *Record = Closure->Record;
+
+ for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) {
+ uint32_t NumValueSites = Closure->GetNumValueSites(Record, Kind);
+ if (!NumValueSites)
+ continue;
+ TotalSize += getValueProfRecordSize(NumValueSites,
+ Closure->GetNumValueData(Record, Kind));
+ }
+ return TotalSize;
+}
+
+/*!
+ * Extract value profile data of a function for the profile kind \c ValueKind
+ * from the \c Closure and serialize the data into \c This record instance.
+ */
+INSTR_PROF_VISIBILITY void
+serializeValueProfRecordFrom(ValueProfRecord *This,
+ ValueProfRecordClosure *Closure,
+ uint32_t ValueKind, uint32_t NumValueSites) {
+ uint32_t S;
+ const void *Record = Closure->Record;
+ This->Kind = ValueKind;
+ This->NumValueSites = NumValueSites;
+ InstrProfValueData *DstVD = getValueProfRecordValueData(This);
+
+ for (S = 0; S < NumValueSites; S++) {
+ uint32_t ND = Closure->GetNumValueDataForSite(Record, ValueKind, S);
+ This->SiteCountArray[S] = ND;
+ Closure->GetValueForSite(Record, DstVD, ValueKind, S);
+ DstVD += ND;
+ }
+}
+
+/*!
+ * Extract value profile data of a function from the \c Closure
+ * and serialize the data into \c DstData if it is not NULL or heap
+ * memory allocated by the \c Closure's allocator method. If \c
+ * DstData is not null, the caller is expected to set the TotalSize
+ * in DstData.
+ */
+INSTR_PROF_VISIBILITY ValueProfData *
+serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
+ ValueProfData *DstData) {
+ uint32_t Kind;
+ uint32_t TotalSize =
+ DstData ? DstData->TotalSize : getValueProfDataSize(Closure);
+
+ ValueProfData *VPD =
+ DstData ? DstData : Closure->AllocValueProfData(TotalSize);
+
+ VPD->TotalSize = TotalSize;
+ VPD->NumValueKinds = Closure->GetNumValueKinds(Closure->Record);
+ ValueProfRecord *VR = getFirstValueProfRecord(VPD);
+ for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) {
+ uint32_t NumValueSites = Closure->GetNumValueSites(Closure->Record, Kind);
+ if (!NumValueSites)
+ continue;
+ serializeValueProfRecordFrom(VR, Closure, Kind, NumValueSites);
+ VR = getValueProfRecordNext(VR);
+ }
+ return VPD;
+}
+
+#undef INSTR_PROF_COMMON_API_IMPL
+#endif /* INSTR_PROF_COMMON_API_IMPL */
+
+/*============================================================================*/
+
+// clang-format off:consider re-enabling clang-format if auto-formatted C macros
+// are readable (e.g., after `issue #82426` is fixed)
+#ifndef INSTR_PROF_DATA_DEFINED
+
+#ifndef INSTR_PROF_DATA_INC
+#define INSTR_PROF_DATA_INC
+
+/* Helper macros. */
+#define INSTR_PROF_SIMPLE_QUOTE(x) #x
+#define INSTR_PROF_QUOTE(x) INSTR_PROF_SIMPLE_QUOTE(x)
+#define INSTR_PROF_SIMPLE_CONCAT(x,y) x ## y
+#define INSTR_PROF_CONCAT(x,y) INSTR_PROF_SIMPLE_CONCAT(x,y)
+
+/* Magic number to detect file format and endianness.
+ * Use 255 at one end, since no UTF-8 file can use that character. Avoid 0,
+ * so that utilities, like strings, don't grab it as a string. 129 is also
+ * invalid UTF-8, and high enough to be interesting.
+ * Use "lprofr" in the centre to stand for "LLVM Profile Raw", or "lprofR"
+ * for 32-bit platforms.
+ */
+#define INSTR_PROF_RAW_MAGIC_64 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \
+ (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \
+ (uint64_t)'f' << 16 | (uint64_t)'r' << 8 | (uint64_t)129
+#define INSTR_PROF_RAW_MAGIC_32 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \
+ (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \
+ (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129
+
+/* Raw profile format version (start from 1). */
+#define INSTR_PROF_RAW_VERSION 10
+/* Indexed profile format version (start from 1). */
+#define INSTR_PROF_INDEX_VERSION 12
+/* Coverage mapping format version (start from 0). */
+#define INSTR_PROF_COVMAP_VERSION 6
+
+/* Profile version is always of type uint64_t. Reserve the upper 32 bits in the
+ * version for other variants of profile. We set the 8th most significant bit
+ * (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentation
+ * generated profile, and 0 if this is a Clang FE generated profile.
+ * 1 in bit 57 indicates there are context-sensitive records in the profile.
+ * The 54th bit indicates whether to always instrument loop entry blocks.
+ * The 58th bit indicates whether to always instrument function entry blocks.
+ * The 59th bit indicates whether to use debug info to correlate profiles.
+ * The 60th bit indicates single byte coverage instrumentation.
+ * The 61st bit indicates function entry instrumentation only.
+ * The 62nd bit indicates whether memory profile information is present.
+ * The 63rd bit indicates if this is a temporal profile.
+ */
+#define VARIANT_MASKS_ALL 0xffffffff00000000ULL
+#define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL)
+#define VARIANT_MASK_INSTR_LOOP_ENTRIES (0x1ULL << 55)
+#define VARIANT_MASK_IR_PROF (0x1ULL << 56)
+#define VARIANT_MASK_CSIR_PROF (0x1ULL << 57)
+#define VARIANT_MASK_INSTR_ENTRY (0x1ULL << 58)
+#define VARIANT_MASK_DBG_CORRELATE (0x1ULL << 59)
+#define VARIANT_MASK_BYTE_COVERAGE (0x1ULL << 60)
+#define VARIANT_MASK_FUNCTION_ENTRY_ONLY (0x1ULL << 61)
+#define VARIANT_MASK_MEMPROF (0x1ULL << 62)
+#define VARIANT_MASK_TEMPORAL_PROF (0x1ULL << 63)
+#define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version
+#define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime
+#define INSTR_PROF_PROFILE_COUNTER_BIAS_VAR __llvm_profile_counter_bias
+#define INSTR_PROF_PROFILE_BITMAP_BIAS_VAR __llvm_profile_bitmap_bias
+#define INSTR_PROF_PROFILE_SET_TIMESTAMP __llvm_profile_set_timestamp
+#define INSTR_PROF_PROFILE_SAMPLING_VAR __llvm_profile_sampling
+
+/* The variable that holds the name of the profile data
+ * specified via command line. */
+#define INSTR_PROF_PROFILE_NAME_VAR __llvm_profile_filename
+
+/* section name strings common to all targets other
+ than WIN32 */
+#define INSTR_PROF_DATA_COMMON __llvm_prf_data
+#define INSTR_PROF_NAME_COMMON __llvm_prf_names
+#define INSTR_PROF_VNAME_COMMON __llvm_prf_vns
+#define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts
+#define INSTR_PROF_BITS_COMMON __llvm_prf_bits
+#define INSTR_PROF_VALS_COMMON __llvm_prf_vals
+#define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds
+#define INSTR_PROF_VTAB_COMMON __llvm_prf_vtab
+#define INSTR_PROF_COVMAP_COMMON __llvm_covmap
+#define INSTR_PROF_COVFUN_COMMON __llvm_covfun
+#define INSTR_PROF_COVDATA_COMMON __llvm_covdata
+#define INSTR_PROF_COVNAME_COMMON __llvm_covnames
+#define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile
+#define INSTR_PROF_COVINIT_COMMON __llvm_covinit
+
+/* Windows section names. Because these section names contain dollar characters,
+ * they must be quoted.
+ */
+#define INSTR_PROF_DATA_COFF ".lprfd$M"
+#define INSTR_PROF_NAME_COFF ".lprfn$M"
+#define INSTR_PROF_VNAME_COFF ".lprfvn$M"
+#define INSTR_PROF_CNTS_COFF ".lprfc$M"
+#define INSTR_PROF_BITS_COFF ".lprfb$M"
+#define INSTR_PROF_VALS_COFF ".lprfv$M"
+#define INSTR_PROF_VNODES_COFF ".lprfnd$M"
+#define INSTR_PROF_VTAB_COFF ".lprfvt$M"
+#define INSTR_PROF_COVMAP_COFF ".lcovmap$M"
+#define INSTR_PROF_COVFUN_COFF ".lcovfun$M"
+/* Since cov data and cov names sections are not allocated, we don't need to
+ * access them at runtime.
+ */
+#define INSTR_PROF_COVDATA_COFF ".lcovd"
+#define INSTR_PROF_COVNAME_COFF ".lcovn"
+#define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M"
+
+// FIXME: Placeholder for Windows. Windows currently does not initialize
+// the GCOV functions in the runtime.
+#define INSTR_PROF_COVINIT_COFF ".lcovd$M"
+
+#ifdef _WIN32
+/* Runtime section names and name strings. */
+#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COFF
+#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COFF
+#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COFF
+#define INSTR_PROF_BITS_SECT_NAME INSTR_PROF_BITS_COFF
+#define INSTR_PROF_VTAB_SECT_NAME INSTR_PROF_VTAB_COFF
+#define INSTR_PROF_VNAME_SECT_NAME INSTR_PROF_VNAME_COFF
+/* Array of pointers. Each pointer points to a list
+ * of value nodes associated with one value site.
+ */
+#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_VALS_COFF
+/* Value profile nodes section. */
+#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF
+#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF
+#define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_COVFUN_COFF
+#define INSTR_PROF_COVDATA_SECT_NAME INSTR_PROF_COVDATA_COFF
+#define INSTR_PROF_COVNAME_SECT_NAME INSTR_PROF_COVNAME_COFF
+#define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF
+#define INSTR_PROF_COVINIT_SECT_NAME INSTR_PROF_COVINIT_COFF
+#else
+/* Runtime section names and name strings. */
+#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON)
+#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON)
+#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON)
+#define INSTR_PROF_BITS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_BITS_COMMON)
+#define INSTR_PROF_VTAB_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VTAB_COMMON)
+#define INSTR_PROF_VNAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNAME_COMMON)
+/* Array of pointers. Each pointer points to a list
+ * of value nodes associated with one value site.
+ */
+#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON)
+/* Value profile nodes section. */
+#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON)
+#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON)
+#define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON)
+#define INSTR_PROF_COVDATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVDATA_COMMON)
+#define INSTR_PROF_COVNAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVNAME_COMMON)
+/* Order file instrumentation. */
+#define INSTR_PROF_ORDERFILE_SECT_NAME \
+ INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON)
+#define INSTR_PROF_COVINIT_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVINIT_COMMON)
+#endif
+
+#define INSTR_PROF_ORDERFILE_BUFFER_NAME _llvm_order_file_buffer
+#define INSTR_PROF_ORDERFILE_BUFFER_NAME_STR \
+ INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_NAME)
+#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME _llvm_order_file_buffer_idx
+#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME_STR \
+ INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME)
+
+/* Macros to define start/stop section symbol for a given
+ * section on Linux. For instance
+ * INSTR_PROF_SECT_START(INSTR_PROF_DATA_SECT_NAME) will
+ * expand to __start___llvm_prof_data
+ */
+#define INSTR_PROF_SECT_START(Sect) \
+ INSTR_PROF_CONCAT(__start_,Sect)
+#define INSTR_PROF_SECT_STOP(Sect) \
+ INSTR_PROF_CONCAT(__stop_,Sect)
+
+/* Value Profiling API linkage name. */
+#define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target
+#define INSTR_PROF_VALUE_PROF_FUNC_STR \
+ INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC)
+#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC __llvm_profile_instrument_memop
+#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR \
+ INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_MEMOP_FUNC)
+
+/* InstrProfile per-function control data alignment. */
+#define INSTR_PROF_DATA_ALIGNMENT 8
+
+/* The data structure that represents a tracked value by the
+ * value profiler.
+ */
+typedef struct InstrProfValueData {
+ /* Profiled value. */
+ uint64_t Value;
+ /* Number of times the value appears in the training run. */
+ uint64_t Count;
+} InstrProfValueData;
+
+#endif /* INSTR_PROF_DATA_INC */
+
+#ifndef INSTR_ORDER_FILE_INC
+/* The maximal # of functions: 128*1024 (the buffer size will be 128*4 KB). */
+#define INSTR_ORDER_FILE_BUFFER_SIZE 131072
+#define INSTR_ORDER_FILE_BUFFER_BITS 17
+#define INSTR_ORDER_FILE_BUFFER_MASK 0x1ffff
+#endif /* INSTR_ORDER_FILE_INC */
+#else
+#undef INSTR_PROF_DATA_DEFINED
+#endif
+
+#undef COVMAP_V2_OR_V3
+
+#ifdef INSTR_PROF_VALUE_PROF_MEMOP_API
+
+#ifdef __cplusplus
+#define INSTR_PROF_INLINE inline
+#else
+#define INSTR_PROF_INLINE
+#endif
+
+/* The value range buckets (22 buckets) for the memop size value profiling looks
+ * like:
+ *
+ * [0, 0]
+ * [1, 1]
+ * [2, 2]
+ * [3, 3]
+ * [4, 4]
+ * [5, 5]
+ * [6, 6]
+ * [7, 7]
+ * [8, 8]
+ * [9, 15]
+ * [16, 16]
+ * [17, 31]
+ * [32, 32]
+ * [33, 63]
+ * [64, 64]
+ * [65, 127]
+ * [128, 128]
+ * [129, 255]
+ * [256, 256]
+ * [257, 511]
+ * [512, 512]
+ * [513, UINT64_MAX]
+ *
+ * Each range has a 'representative value' which is the lower end value of the
+ * range and used to store in the runtime profile data records and the VP
+ * metadata. For example, it's 2 for [2, 2] and 64 for [65, 127].
+ */
+#define INSTR_PROF_NUM_BUCKETS 22
+
+/*
+ * Clz and Popcount. This code was copied from
+ * compiler-rt/lib/fuzzer/{FuzzerBuiltins.h,FuzzerBuiltinsMsvc.h} and
+ * llvm/include/llvm/Support/MathExtras.h.
+ */
+#if defined(_MSC_VER) && !defined(__clang__)
+
+#include <intrin.h>
+INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
+int InstProfClzll(unsigned long long X) {
+ unsigned long LeadZeroIdx = 0;
+#if !defined(_M_ARM64) && !defined(_M_X64)
+ // Scan the high 32 bits.
+ if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X >> 32)))
+ return (int)(63 - (LeadZeroIdx + 32)); // Create a bit offset
+ // from the MSB.
+ // Scan the low 32 bits.
+ if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X)))
+ return (int)(63 - LeadZeroIdx);
+#else
+ if (_BitScanReverse64(&LeadZeroIdx, X)) return 63 - LeadZeroIdx;
+#endif
+ return 64;
+}
+INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
+int InstProfPopcountll(unsigned long long X) {
+ // This code originates from https://reviews.llvm.org/rG30626254510f.
+ unsigned long long v = X;
+ v = v - ((v >> 1) & 0x5555555555555555ULL);
+ v = (v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL);
+ v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL;
+ return (int)((unsigned long long)(v * 0x0101010101010101ULL) >> 56);
+}
+
+#else
+
+INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
+int InstProfClzll(unsigned long long X) { return __builtin_clzll(X); }
+INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
+int InstProfPopcountll(unsigned long long X) { return __builtin_popcountll(X); }
+
+#endif /* defined(_MSC_VER) && !defined(__clang__) */
+
+// clang-format on
+
+/* Map an (observed) memop size value to the representative value of its range.
+ * For example, 5 -> 5, 22 -> 17, 99 -> 65, 256 -> 256, 1001 -> 513. */
+INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint64_t
+InstrProfGetRangeRepValue(uint64_t Value) {
+ if (Value <= 8)
+ // The first ranges are individually tracked. Use the value as is.
+ return Value;
+ else if (Value >= 513)
+ // The last range is mapped to its lowest value.
+ return 513;
+ else if (InstProfPopcountll(Value) == 1)
+ // If it's a power of two, use it as is.
+ return Value;
+ else
+ // Otherwise, take to the previous power of two + 1.
+ return (UINT64_C(1) << (64 - InstProfClzll(Value) - 1)) + 1;
+}
+
+/* Return true if the range that an (observed) memop size value belongs to has
+ * only a single value in the range. For example, 0 -> true, 8 -> true, 10 ->
+ * false, 64 -> true, 100 -> false, 513 -> false. */
+INSTR_PROF_VISIBILITY INSTR_PROF_INLINE unsigned
+InstrProfIsSingleValRange(uint64_t Value) {
+ if (Value <= 8)
+ // The first ranges are individually tracked.
+ return 1;
+ else if (InstProfPopcountll(Value) == 1)
+ // If it's a power of two, there's only one value.
+ return 1;
+ else
+ // Otherwise, there's more than one value in the range.
+ return 0;
+}
+
+#endif /* INSTR_PROF_VALUE_PROF_MEMOP_API */
diff --git a/system/lib/compiler-rt/include/profile/MIBEntryDef.inc b/system/lib/compiler-rt/include/profile/MIBEntryDef.inc
new file mode 100644
index 0000000000000..58c1fc4de4aba
--- /dev/null
+++ b/system/lib/compiler-rt/include/profile/MIBEntryDef.inc
@@ -0,0 +1,55 @@
+/*===-- MemEntryDef.inc - MemProf profiling runtime macros -*- C++ -*-======== *\
+|*
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+|* See https://llvm.org/LICENSE.txt for license information.
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+|*
+\*===----------------------------------------------------------------------===*/
+/*
+ * This file defines the macros for memprof profiling data structures.
+ * Eg. usage to define the memprof meminfoblock struct:
+ *
+ * struct MemInfoBlock {
+ * #define MIBEntryDef(NameTag, Name, Type) Type Name;
+ * #include MIBEntryDef.inc
+ * #undef MIBEntryDef
+ * };
+ *
+ * This file has two identical copies. The primary copy lives in LLVM and
+ * the other one sits in compiler-rt/include/profile directory. To make changes
+ * in this file, first modify the primary copy and copy it over to compiler-rt.
+ * Testing of any change in this file can start only after the two copies are
+ * synced up.
+ *
+\*===----------------------------------------------------------------------===*/
+#ifndef MIBEntryDef
+#define MIBEntryDef(NameTag, Name, Type)
+#endif
+
+MIBEntryDef(AllocCount = 1, AllocCount, uint32_t)
+MIBEntryDef(TotalAccessCount = 2, TotalAccessCount, uint64_t)
+MIBEntryDef(MinAccessCount = 3, MinAccessCount, uint64_t)
+MIBEntryDef(MaxAccessCount = 4, MaxAccessCount, uint64_t)
+MIBEntryDef(TotalSize = 5, TotalSize, uint64_t)
+MIBEntryDef(MinSize = 6, MinSize, uint32_t)
+MIBEntryDef(MaxSize = 7, MaxSize, uint32_t)
+MIBEntryDef(AllocTimestamp = 8, AllocTimestamp, uint32_t)
+MIBEntryDef(DeallocTimestamp = 9, DeallocTimestamp, uint32_t)
+MIBEntryDef(TotalLifetime = 10, TotalLifetime, uint64_t)
+MIBEntryDef(MinLifetime = 11, MinLifetime, uint32_t)
+MIBEntryDef(MaxLifetime = 12, MaxLifetime, uint32_t)
+MIBEntryDef(AllocCpuId = 13, AllocCpuId, uint32_t)
+MIBEntryDef(DeallocCpuId = 14, DeallocCpuId, uint32_t)
+MIBEntryDef(NumMigratedCpu = 15, NumMigratedCpu, uint32_t)
+MIBEntryDef(NumLifetimeOverlaps = 16, NumLifetimeOverlaps, uint32_t)
+MIBEntryDef(NumSameAllocCpu = 17, NumSameAllocCpu, uint32_t)
+MIBEntryDef(NumSameDeallocCpu = 18, NumSameDeallocCpu, uint32_t)
+MIBEntryDef(DataTypeId = 19, DataTypeId, uint64_t)
+MIBEntryDef(TotalAccessDensity = 20, TotalAccessDensity, uint64_t)
+MIBEntryDef(MinAccessDensity = 21, MinAccessDensity, uint32_t)
+MIBEntryDef(MaxAccessDensity = 22, MaxAccessDensity, uint32_t)
+MIBEntryDef(TotalLifetimeAccessDensity = 23, TotalLifetimeAccessDensity, uint64_t)
+MIBEntryDef(MinLifetimeAccessDensity = 24, MinLifetimeAccessDensity, uint32_t)
+MIBEntryDef(MaxLifetimeAccessDensity = 25, MaxLifetimeAccessDensity, uint32_t)
+MIBEntryDef(AccessHistogramSize = 26, AccessHistogramSize, uint32_t)
+MIBEntryDef(AccessHistogram = 27, AccessHistogram, uintptr_t)
\ No newline at end of file
diff --git a/system/lib/compiler-rt/include/profile/MemProfData.inc b/system/lib/compiler-rt/include/profile/MemProfData.inc
new file mode 100644
index 0000000000000..3f785bd23fce3
--- /dev/null
+++ b/system/lib/compiler-rt/include/profile/MemProfData.inc
@@ -0,0 +1,235 @@
+#ifndef MEMPROF_DATA_INC
+#define MEMPROF_DATA_INC
+/*===-- MemProfData.inc - MemProf profiling runtime structures -*- C++ -*-=== *\
+|*
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+|* See https://llvm.org/LICENSE.txt for license information.
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+|*
+\*===----------------------------------------------------------------------===*/
+/*
+ * This is the main file that defines all the data structure, signature,
+ * constant literals that are shared across profiling runtime library,
+ * and host tools (reader/writer).
+ *
+ * This file has two identical copies. The primary copy lives in LLVM and
+ * the other one sits in compiler-rt/include/profile directory. To make changes
+ * in this file, first modify the primary copy and copy it over to compiler-rt.
+ * Testing of any change in this file can start only after the two copies are
+ * synced up.
+ *
+\*===----------------------------------------------------------------------===*/
+#include <string.h>
+
+#ifdef _MSC_VER
+#define PACKED(...) __pragma(pack(push,1)) __VA_ARGS__ __pragma(pack(pop))
+#else
+#define PACKED(...) __VA_ARGS__ __attribute__((__packed__))
+#endif
+
+// A 64-bit magic number to uniquely identify the raw binary memprof profile file.
+#define MEMPROF_RAW_MAGIC_64 \
+ ((uint64_t)255 << 56 | (uint64_t)'m' << 48 | (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | \
+ (uint64_t)'o' << 24 | (uint64_t)'f' << 16 | (uint64_t)'r' << 8 | (uint64_t)129)
+
+// The version number of the raw binary format.
+#define MEMPROF_RAW_VERSION 4ULL
+
+// Currently supported versions.
+#define MEMPROF_RAW_SUPPORTED_VERSIONS \
+ { 3ULL, 4ULL }
+
+#define MEMPROF_V3_MIB_SIZE 132ULL;
+
+#define MEMPROF_BUILDID_MAX_SIZE 32ULL
+
+namespace llvm {
+namespace memprof {
+// A struct describing the header used for the raw binary memprof profile format.
+PACKED(struct Header {
+ uint64_t Magic;
+ uint64_t Version;
+ uint64_t TotalSize;
+ uint64_t SegmentOffset;
+ uint64_t MIBOffset;
+ uint64_t StackOffset;
+});
+
+// A struct describing the information necessary to describe a /proc/maps
+// segment entry for a particular binary/library identified by its build id.
+PACKED(struct SegmentEntry {
+ uint64_t Start;
+ uint64_t End;
+ uint64_t Offset;
+ uint64_t BuildIdSize;
+ uint8_t BuildId[MEMPROF_BUILDID_MAX_SIZE] = {0};
+
+ // This constructor is only used in tests so don't set the BuildId.
+ SegmentEntry(uint64_t S, uint64_t E, uint64_t O)
+ : Start(S), End(E), Offset(O), BuildIdSize(0) {}
+
+ SegmentEntry(const SegmentEntry& S) {
+ Start = S.Start;
+ End = S.End;
+ Offset = S.Offset;
+ BuildIdSize = S.BuildIdSize;
+ memcpy(BuildId, S.BuildId, S.BuildIdSize);
+ }
+
+ SegmentEntry& operator=(const SegmentEntry& S) {
+ Start = S.Start;
+ End = S.End;
+ Offset = S.Offset;
+ BuildIdSize = S.BuildIdSize;
+ memcpy(BuildId, S.BuildId, S.BuildIdSize);
+ return *this;
+ }
+
+ bool operator==(const SegmentEntry& S) const {
+ return Start == S.Start && End == S.End && Offset == S.Offset &&
+ BuildIdSize == S.BuildIdSize &&
+ memcmp(BuildId, S.BuildId, S.BuildIdSize) == 0;
+ }
+});
+
+// Packed struct definition for MSVC. We can't use the PACKED macro defined in
+// MemProfData.inc since it would mean we are embedding a directive (the
+// #include for MIBEntryDef) into the macros which is undefined behaviour.
+#ifdef _MSC_VER
+__pragma(pack(push,1))
+#endif
+
+// A struct representing the heap allocation characteristics of a particular
+// runtime context. This struct is shared between the compiler-rt runtime and
+// the raw profile reader. The indexed format uses a separate, self-describing
+// backwards compatible format.
+struct MemInfoBlock{
+
+#define MIBEntryDef(NameTag, Name, Type) Type Name;
+#include "MIBEntryDef.inc"
+#undef MIBEntryDef
+
+bool operator==(const MemInfoBlock& Other) const {
+ bool IsEqual = true;
+#define MIBEntryDef(NameTag, Name, Type) \
+ IsEqual = (IsEqual && Name == Other.Name);
+#include "MIBEntryDef.inc"
+#undef MIBEntryDef
+ return IsEqual;
+}
+
+MemInfoBlock() {
+#define MIBEntryDef(NameTag, Name, Type) Name = Type();
+#include "MIBEntryDef.inc"
+#undef MIBEntryDef
+}
+
+MemInfoBlock(uint32_t Size, uint64_t AccessCount, uint32_t AllocTs,
+ uint32_t DeallocTs, uint32_t AllocCpu, uint32_t DeallocCpu,
+ uintptr_t Histogram, uint32_t HistogramSize)
+ : MemInfoBlock() {
+ AllocCount = 1U;
+ TotalAccessCount = AccessCount;
+ MinAccessCount = AccessCount;
+ MaxAccessCount = AccessCount;
+ TotalSize = Size;
+ MinSize = Size;
+ MaxSize = Size;
+ AllocTimestamp = AllocTs;
+ DeallocTimestamp = DeallocTs;
+ TotalLifetime = DeallocTimestamp - AllocTimestamp;
+ MinLifetime = TotalLifetime;
+ MaxLifetime = TotalLifetime;
+ // Access density is accesses per byte. Multiply by 100 to include the
+ // fractional part.
+ TotalAccessDensity = AccessCount * 100 / Size;
+ MinAccessDensity = TotalAccessDensity;
+ MaxAccessDensity = TotalAccessDensity;
+ // Lifetime access density is the access density per second of lifetime.
+ // Multiply by 1000 to convert denominator lifetime to seconds (using a
+ // minimum lifetime of 1ms to avoid divide by 0. Do the multiplication first
+ // to reduce truncations to 0.
+ TotalLifetimeAccessDensity =
+ TotalAccessDensity * 1000 / (TotalLifetime ? TotalLifetime : 1);
+ MinLifetimeAccessDensity = TotalLifetimeAccessDensity;
+ MaxLifetimeAccessDensity = TotalLifetimeAccessDensity;
+ AllocCpuId = AllocCpu;
+ DeallocCpuId = DeallocCpu;
+ NumMigratedCpu = AllocCpuId != DeallocCpuId;
+ AccessHistogramSize = HistogramSize;
+ AccessHistogram = Histogram;
+}
+
+void Merge(const MemInfoBlock &newMIB) {
+ AllocCount += newMIB.AllocCount;
+
+ TotalAccessCount += newMIB.TotalAccessCount;
+ MinAccessCount = newMIB.MinAccessCount < MinAccessCount ? newMIB.MinAccessCount : MinAccessCount;
+ MaxAccessCount = newMIB.MaxAccessCount > MaxAccessCount ? newMIB.MaxAccessCount : MaxAccessCount;
+
+ TotalSize += newMIB.TotalSize;
+ MinSize = newMIB.MinSize < MinSize ? newMIB.MinSize : MinSize;
+ MaxSize = newMIB.MaxSize > MaxSize ? newMIB.MaxSize : MaxSize;
+
+ TotalLifetime += newMIB.TotalLifetime;
+ MinLifetime = newMIB.MinLifetime < MinLifetime ? newMIB.MinLifetime : MinLifetime;
+ MaxLifetime = newMIB.MaxLifetime > MaxLifetime ? newMIB.MaxLifetime : MaxLifetime;
+
+ TotalAccessDensity += newMIB.TotalAccessDensity;
+ MinAccessDensity = newMIB.MinAccessDensity < MinAccessDensity
+ ? newMIB.MinAccessDensity
+ : MinAccessDensity;
+ MaxAccessDensity = newMIB.MaxAccessDensity > MaxAccessDensity
+ ? newMIB.MaxAccessDensity
+ : MaxAccessDensity;
+
+ TotalLifetimeAccessDensity += newMIB.TotalLifetimeAccessDensity;
+ MinLifetimeAccessDensity =
+ newMIB.MinLifetimeAccessDensity < MinLifetimeAccessDensity
+ ? newMIB.MinLifetimeAccessDensity
+ : MinLifetimeAccessDensity;
+ MaxLifetimeAccessDensity =
+ newMIB.MaxLifetimeAccessDensity > MaxLifetimeAccessDensity
+ ? newMIB.MaxLifetimeAccessDensity
+ : MaxLifetimeAccessDensity;
+
+ // We know newMIB was deallocated later, so just need to check if it was
+ // allocated before last one deallocated.
+ NumLifetimeOverlaps += newMIB.AllocTimestamp < DeallocTimestamp;
+ AllocTimestamp = newMIB.AllocTimestamp;
+ DeallocTimestamp = newMIB.DeallocTimestamp;
+
+ NumSameAllocCpu += AllocCpuId == newMIB.AllocCpuId;
+ NumSameDeallocCpu += DeallocCpuId == newMIB.DeallocCpuId;
+ AllocCpuId = newMIB.AllocCpuId;
+ DeallocCpuId = newMIB.DeallocCpuId;
+
+ // For merging histograms, we always keep the longer histogram, and add
+ // values of shorter histogram to larger one.
+ uintptr_t ShorterHistogram;
+ uint32_t ShorterHistogramSize;
+ if (newMIB.AccessHistogramSize > AccessHistogramSize) {
+ ShorterHistogram = AccessHistogram;
+ ShorterHistogramSize = AccessHistogramSize;
+ // Swap histogram of current to larger histogram
+ AccessHistogram = newMIB.AccessHistogram;
+ AccessHistogramSize = newMIB.AccessHistogramSize;
+ } else {
+ ShorterHistogram = newMIB.AccessHistogram;
+ ShorterHistogramSize = newMIB.AccessHistogramSize;
+ }
+ for (size_t i = 0; i < ShorterHistogramSize; ++i) {
+ ((uint64_t *)AccessHistogram)[i] += ((uint64_t *)ShorterHistogram)[i];
+ }
+}
+
+#ifdef _MSC_VER
+} __pragma(pack(pop));
+#else
+} __attribute__((__packed__));
+#endif
+
+} // namespace memprof
+} // namespace llvm
+
+#endif
diff --git a/system/lib/compiler-rt/include/profile/instr_prof_interface.h b/system/lib/compiler-rt/include/profile/instr_prof_interface.h
new file mode 100644
index 0000000000000..be40f2685934b
--- /dev/null
+++ b/system/lib/compiler-rt/include/profile/instr_prof_interface.h
@@ -0,0 +1,92 @@
+/*===---- instr_prof_interface.h - Instrumentation PGO User Program API ----===
+ *
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ *
+ *===-----------------------------------------------------------------------===
+ *
+ * This header provides a public interface for fine-grained control of counter
+ * reset and profile dumping. These interface functions can be directly called
+ * in user programs.
+ *
+\*===---------------------------------------------------------------------===*/
+
+#ifndef COMPILER_RT_INSTR_PROFILING
+#define COMPILER_RT_INSTR_PROFILING
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifdef __LLVM_INSTR_PROFILE_GENERATE
+// Profile file reset and dump interfaces.
+// When `-fprofile[-instr]-generate`/`-fcs-profile-generate` is in effect,
+// clang defines __LLVM_INSTR_PROFILE_GENERATE to pick up the API calls.
+
+/*!
+ * \brief Set the filename for writing instrumentation data.
+ *
+ * Sets the filename to be used for subsequent calls to
+ * \a __llvm_profile_write_file().
+ *
+ * \c Name is not copied, so it must remain valid. Passing NULL resets the
+ * filename logic to the default behaviour.
+ *
+ * Note: There may be multiple copies of the profile runtime (one for each
+ * instrumented image/DSO). This API only modifies the filename within the
+ * copy of the runtime available to the calling image.
+ *
+ * Warning: This is a no-op if continuous mode (\ref
+ * __llvm_profile_is_continuous_mode_enabled) is on. The reason for this is
+ * that in continuous mode, profile counters are mmap()'d to the profile at
+ * program initialization time. Support for transferring the mmap'd profile
+ * counts to a new file has not been implemented.
+ */
+void __llvm_profile_set_filename(const char *Name);
+
+/*!
+ * \brief Interface to set all PGO counters to zero for the current process.
+ *
+ */
+void __llvm_profile_reset_counters(void);
+
+/*!
+ * \brief this is a wrapper interface to \c __llvm_profile_write_file.
+ * After this interface is invoked, an already dumped flag will be set
+ * so that profile won't be dumped again during program exit.
+ * Invocation of interface __llvm_profile_reset_counters will clear
+ * the flag. This interface is designed to be used to collect profile
+ * data from user selected hot regions. The use model is
+ * __llvm_profile_reset_counters();
+ * ... hot region 1
+ * __llvm_profile_dump();
+ * .. some other code
+ * __llvm_profile_reset_counters();
+ * ... hot region 2
+ * __llvm_profile_dump();
+ *
+ * It is expected that on-line profile merging is on with \c %m specifier
+ * used in profile filename . If merging is not turned on, user is expected
+ * to invoke __llvm_profile_set_filename to specify different profile names
+ * for different regions before dumping to avoid profile write clobbering.
+ */
+int __llvm_profile_dump(void);
+
+// Interface to dump the current process' order file to disk.
+int __llvm_orderfile_dump(void);
+
+#else
+
+#define __llvm_profile_set_filename(Name)
+#define __llvm_profile_reset_counters()
+#define __llvm_profile_dump() (0)
+#define __llvm_orderfile_dump() (0)
+
+#endif
+
+#ifdef __cplusplus
+} // extern "C"
+#endif
+
+#endif
diff --git a/system/lib/compiler-rt/lib/profile/GCDAProfiling.c b/system/lib/compiler-rt/lib/profile/GCDAProfiling.c
new file mode 100644
index 0000000000000..ac01805e70adc
--- /dev/null
+++ b/system/lib/compiler-rt/lib/profile/GCDAProfiling.c
@@ -0,0 +1,656 @@
+/*===- GCDAProfiling.c - Support library for GCDA file emission -----------===*\
+|*
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+|* See https://llvm.org/LICENSE.txt for license information.
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+|*
+|*===----------------------------------------------------------------------===*|
+|*
+|* This file implements the call back routines for the gcov profiling
+|* instrumentation pass. Link against this library when running code through
+|* the -insert-gcov-profiling LLVM pass.
+|*
+|* We emit files in a corrupt version of GCOV's "gcda" file format. These files
+|* are only close enough that LCOV will happily parse them. Anything that lcov
+|* ignores is missing.
+|*
+|* TODO: gcov is multi-process safe by having each exit open the existing file
+|* and append to it. We'd like to achieve that and be thread-safe too.
+|*
+\*===----------------------------------------------------------------------===*/
+
+#if !defined(__Fuchsia__)
+
+#include <errno.h>
+#include <fcntl.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#if defined(_WIN32)
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+#include "WindowsMMap.h"
+#else
+#include <sys/file.h>
+#include <sys/mman.h>
+#include <sys/types.h>
+#include <unistd.h>
+#endif
+
+#include "InstrProfiling.h"
+#include "InstrProfilingUtil.h"
+
+/* #define DEBUG_GCDAPROFILING */
+
+enum {
+ GCOV_DATA_MAGIC = 0x67636461, // "gcda"
+
+ GCOV_TAG_FUNCTION = 0x01000000,
+ GCOV_TAG_COUNTER_ARCS = 0x01a10000,
+ // GCOV_TAG_OBJECT_SUMMARY superseded GCOV_TAG_PROGRAM_SUMMARY in GCC 9.
+ GCOV_TAG_OBJECT_SUMMARY = 0xa1000000,
+ GCOV_TAG_PROGRAM_SUMMARY = 0xa3000000,
+};
+
+/*
+ * --- GCOV file format I/O primitives ---
+ */
+
+/*
+ * The current file name we're outputting. Used primarily for error logging.
+ */
+static char *filename = NULL;
+
+/*
+ * The current file we're outputting.
+ */
+static FILE *output_file = NULL;
+
+/*
+ * Buffer that we write things into.
+ */
+#define WRITE_BUFFER_SIZE (128 * 1024)
+static unsigned char *write_buffer = NULL;
+static uint64_t cur_buffer_size = 0;
+static uint64_t cur_pos = 0;
+static uint64_t file_size = 0;
+static int new_file = 0;
+static int gcov_version;
+#if defined(_WIN32)
+static HANDLE mmap_handle = NULL;
+#endif
+static int fd = -1;
+
+typedef void (*fn_ptr)(void);
+
+typedef void* dynamic_object_id;
+// The address of this variable identifies a given dynamic object.
+static dynamic_object_id current_id;
+#define CURRENT_ID (¤t_id)
+
+struct fn_node {
+ dynamic_object_id id;
+ fn_ptr fn;
+ struct fn_node* next;
+};
+
+struct fn_list {
+ struct fn_node *head, *tail;
+};
+
+/*
+ * A list of functions to write out the data, shared between all dynamic objects.
+ */
+struct fn_list writeout_fn_list;
+
+/*
+ * A list of reset functions, shared between all dynamic objects.
+ */
+struct fn_list reset_fn_list;
+
+static void fn_list_insert(struct fn_list* list, fn_ptr fn) {
+ struct fn_node* new_node = malloc(sizeof(struct fn_node));
+ new_node->fn = fn;
+ new_node->next = NULL;
+ new_node->id = CURRENT_ID;
+
+ if (!list->head) {
+ list->head = list->tail = new_node;
+ } else {
+ list->tail->next = new_node;
+ list->tail = new_node;
+ }
+}
+
+static void fn_list_remove(struct fn_list* list) {
+ struct fn_node* curr = list->head;
+ struct fn_node* prev = NULL;
+ struct fn_node* next = NULL;
+
+ while (curr) {
+ next = curr->next;
+
+ if (curr->id == CURRENT_ID) {
+ if (curr == list->head) {
+ list->head = next;
+ }
+
+ if (curr == list->tail) {
+ list->tail = prev;
+ }
+
+ if (prev) {
+ prev->next = next;
+ }
+
+ free(curr);
+ } else {
+ prev = curr;
+ }
+
+ curr = next;
+ }
+}
+
+static void resize_write_buffer(uint64_t size) {
+ if (!new_file) return;
+ size += cur_pos;
+ if (size <= cur_buffer_size) return;
+ size = (size - 1) / WRITE_BUFFER_SIZE + 1;
+ size *= WRITE_BUFFER_SIZE;
+ write_buffer = realloc(write_buffer, size);
+ cur_buffer_size = size;
+}
+
+static void write_bytes(const char *s, size_t len) {
+ resize_write_buffer(len);
+ memcpy(&write_buffer[cur_pos], s, len);
+ cur_pos += len;
+}
+
+static void write_32bit_value(uint32_t i) {
+ write_bytes((char*)&i, 4);
+}
+
+static void write_64bit_value(uint64_t i) {
+ // GCOV uses a lo-/hi-word format even on big-endian systems.
+ // See also GCOVBuffer::readInt64 in LLVM.
+ uint32_t lo = (uint32_t) i;
+ uint32_t hi = (uint32_t) (i >> 32);
+ write_32bit_value(lo);
+ write_32bit_value(hi);
+}
+
+static uint32_t read_32bit_value(void) {
+ uint32_t val;
+
+ if (new_file)
+ return (uint32_t)-1;
+
+ val = *(uint32_t*)&write_buffer[cur_pos];
+ cur_pos += 4;
+ return val;
+}
+
+static uint64_t read_64bit_value(void) {
+ // GCOV uses a lo-/hi-word format even on big-endian systems.
+ // See also GCOVBuffer::readInt64 in LLVM.
+ uint32_t lo = read_32bit_value();
+ uint32_t hi = read_32bit_value();
+ return ((uint64_t)hi << 32) | ((uint64_t)lo);
+}
+
+static char *mangle_filename(const char *orig_filename) {
+ char *new_filename;
+ size_t prefix_len;
+ int prefix_strip;
+ const char *prefix = lprofGetPathPrefix(&prefix_strip, &prefix_len);
+
+ if (prefix == NULL)
+ return strdup(orig_filename);
+
+ new_filename = malloc(prefix_len + 1 + strlen(orig_filename) + 1);
+ lprofApplyPathPrefix(new_filename, orig_filename, prefix, prefix_len,
+ prefix_strip);
+
+ return new_filename;
+}
+
+static int map_file(void) {
+ fseek(output_file, 0L, SEEK_END);
+ file_size = ftell(output_file);
+
+ /* A size of 0 means the file has been created just now (possibly by another
+ * process in lock-after-open race condition). No need to mmap. */
+ if (file_size == 0)
+ return -1;
+
+#if defined(_WIN32)
+ HANDLE mmap_fd;
+ if (fd == -1)
+ mmap_fd = INVALID_HANDLE_VALUE;
+ else
+ mmap_fd = (HANDLE)_get_osfhandle(fd);
+
+ mmap_handle = CreateFileMapping(mmap_fd, NULL, PAGE_READWRITE, DWORD_HI(file_size), DWORD_LO(file_size), NULL);
+ if (mmap_handle == NULL) {
+ fprintf(stderr, "profiling: %s: cannot create file mapping: %lu\n",
+ filename, GetLastError());
+ return -1;
+ }
+
+ write_buffer = MapViewOfFile(mmap_handle, FILE_MAP_WRITE, 0, 0, file_size);
+ if (write_buffer == NULL) {
+ fprintf(stderr, "profiling: %s: cannot map: %lu\n", filename,
+ GetLastError());
+ CloseHandle(mmap_handle);
+ return -1;
+ }
+#else
+ write_buffer = mmap(0, file_size, PROT_READ | PROT_WRITE,
+ MAP_FILE | MAP_SHARED, fd, 0);
+ if (write_buffer == (void *)-1) {
+ int errnum = errno;
+ fprintf(stderr, "profiling: %s: cannot map: %s\n", filename,
+ strerror(errnum));
+ return -1;
+ }
+#endif
+
+ return 0;
+}
+
+static void unmap_file(void) {
+#if defined(_WIN32)
+ if (!UnmapViewOfFile(write_buffer)) {
+ fprintf(stderr, "profiling: %s: cannot unmap mapped view: %lu\n", filename,
+ GetLastError());
+ }
+
+ if (!CloseHandle(mmap_handle)) {
+ fprintf(stderr, "profiling: %s: cannot close file mapping handle: %lu\n",
+ filename, GetLastError());
+ }
+
+ mmap_handle = NULL;
+#else
+ if (munmap(write_buffer, file_size) == -1) {
+ int errnum = errno;
+ fprintf(stderr, "profiling: %s: cannot munmap: %s\n", filename,
+ strerror(errnum));
+ }
+#endif
+
+ write_buffer = NULL;
+ file_size = 0;
+}
+
+/*
+ * --- LLVM line counter API ---
+ */
+
+/* A file in this case is a translation unit. Each .o file built with line
+ * profiling enabled will emit to a different file. Only one file may be
+ * started at a time.
+ */
+COMPILER_RT_VISIBILITY
+void llvm_gcda_start_file(const char *orig_filename, uint32_t version,
+ uint32_t checksum) {
+ const char *mode = "r+b";
+ filename = mangle_filename(orig_filename);
+
+ /* Try just opening the file. */
+ fd = open(filename, O_RDWR | O_BINARY);
+
+ if (fd == -1) {
+ /* Try creating the file. */
+ fd = open(filename, O_RDWR | O_CREAT | O_EXCL | O_BINARY, 0644);
+ if (fd != -1) {
+ mode = "w+b";
+ } else {
+ /* Try creating the directories first then opening the file. */
+ __llvm_profile_recursive_mkdir(filename);
+ fd = open(filename, O_RDWR | O_CREAT | O_EXCL | O_BINARY, 0644);
+ if (fd != -1) {
+ mode = "w+b";
+ } else {
+ /* Another process may have created the file just now.
+ * Try opening it without O_CREAT and O_EXCL. */
+ fd = open(filename, O_RDWR | O_BINARY);
+ if (fd == -1) {
+ /* Bah! It's hopeless. */
+ int errnum = errno;
+ fprintf(stderr, "profiling: %s: cannot open: %s\n", filename,
+ strerror(errnum));
+ return;
+ }
+ }
+ }
+ }
+
+ /* Try to flock the file to serialize concurrent processes writing out to the
+ * same GCDA. This can fail if the filesystem doesn't support it, but in that
+ * case we'll just carry on with the old racy behaviour and hope for the best.
+ */
+ lprofLockFd(fd);
+ output_file = fdopen(fd, mode);
+
+ /* Initialize the write buffer. */
+ new_file = 0;
+ write_buffer = NULL;
+ cur_buffer_size = 0;
+ cur_pos = 0;
+
+ if (map_file() == -1) {
+ /* The file has been created just now (file_size == 0) or mmap failed
+ * unexpectedly. In the latter case, try to recover by clobbering. */
+ new_file = 1;
+ write_buffer = NULL;
+ resize_write_buffer(WRITE_BUFFER_SIZE);
+ memset(write_buffer, 0, WRITE_BUFFER_SIZE);
+ }
+
+ /* gcda file, version, stamp checksum. */
+ {
+ uint8_t c3 = version >> 24;
+ uint8_t c2 = (version >> 16) & 255;
+ uint8_t c1 = (version >> 8) & 255;
+ gcov_version = c3 >= 'A' ? (c3 - 'A') * 100 + (c2 - '0') * 10 + c1 - '0'
+ : (c3 - '0') * 10 + c1 - '0';
+ }
+ write_32bit_value(GCOV_DATA_MAGIC);
+ write_32bit_value(version);
+ write_32bit_value(checksum);
+
+#ifdef DEBUG_GCDAPROFILING
+ fprintf(stderr, "llvmgcda: [%s]\n", orig_filename);
+#endif
+}
+
+COMPILER_RT_VISIBILITY
+void llvm_gcda_emit_function(uint32_t ident, uint32_t func_checksum,
+ uint32_t cfg_checksum) {
+ uint32_t len = 2;
+ int use_extra_checksum = gcov_version >= 47;
+
+ if (use_extra_checksum)
+ len++;
+#ifdef DEBUG_GCDAPROFILING
+ fprintf(stderr, "llvmgcda: function id=0x%08x\n", ident);
+#endif
+ if (!output_file) return;
+
+ /* function tag */
+ write_32bit_value(GCOV_TAG_FUNCTION);
+ write_32bit_value(len);
+ write_32bit_value(ident);
+ write_32bit_value(func_checksum);
+ if (use_extra_checksum)
+ write_32bit_value(cfg_checksum);
+}
+
+COMPILER_RT_VISIBILITY
+void llvm_gcda_emit_arcs(uint32_t num_counters, uint64_t *counters) {
+ uint32_t i;
+ uint64_t *old_ctrs = NULL;
+ uint32_t val = 0;
+ uint64_t save_cur_pos = cur_pos;
+
+ if (!output_file) return;
+
+ val = read_32bit_value();
+
+ if (val != (uint32_t)-1) {
+ /* There are counters present in the file. Merge them. */
+ if (val != GCOV_TAG_COUNTER_ARCS) {
+ fprintf(stderr, "profiling: %s: cannot merge previous GCDA file: "
+ "corrupt arc tag (0x%08x)\n",
+ filename, val);
+ return;
+ }
+
+ val = read_32bit_value();
+ if (val == (uint32_t)-1 || val / 2 != num_counters) {
+ fprintf(stderr, "profiling: %s: cannot merge previous GCDA file: "
+ "mismatched number of counters (%d)\n",
+ filename, val);
+ return;
+ }
+
+ old_ctrs = malloc(sizeof(uint64_t) * num_counters);
+ for (i = 0; i < num_counters; ++i)
+ old_ctrs[i] = read_64bit_value();
+ }
+
+ cur_pos = save_cur_pos;
+
+ /* Counter #1 (arcs) tag */
+ write_32bit_value(GCOV_TAG_COUNTER_ARCS);
+ write_32bit_value(num_counters * 2);
+ for (i = 0; i < num_counters; ++i) {
+ counters[i] += (old_ctrs ? old_ctrs[i] : 0);
+ write_64bit_value(counters[i]);
+ }
+
+ free(old_ctrs);
+
+#ifdef DEBUG_GCDAPROFILING
+ fprintf(stderr, "llvmgcda: %u arcs\n", num_counters);
+ for (i = 0; i < num_counters; ++i)
+ fprintf(stderr, "llvmgcda: %llu\n", (unsigned long long)counters[i]);
+#endif
+}
+
+COMPILER_RT_VISIBILITY
+void llvm_gcda_summary_info(void) {
+ uint32_t runs = 1;
+ static uint32_t run_counted = 0; // We only want to increase the run count once.
+ uint32_t val = 0;
+ uint64_t save_cur_pos = cur_pos;
+
+ if (!output_file) return;
+
+ val = read_32bit_value();
+
+ if (val != (uint32_t)-1) {
+ /* There are counters present in the file. Merge them. */
+ uint32_t gcov_tag =
+ gcov_version >= 90 ? GCOV_TAG_OBJECT_SUMMARY : GCOV_TAG_PROGRAM_SUMMARY;
+ if (val != gcov_tag) {
+ fprintf(stderr,
+ "profiling: %s: cannot merge previous run count: "
+ "corrupt object tag (0x%08x)\n",
+ filename, val);
+ return;
+ }
+
+ val = read_32bit_value(); /* length */
+ uint32_t prev_runs;
+ if (gcov_version < 90) {
+ read_32bit_value();
+ read_32bit_value();
+ prev_runs = read_32bit_value();
+ } else {
+ prev_runs = read_32bit_value();
+ read_32bit_value();
+ }
+ for (uint32_t i = gcov_version < 90 ? 3 : 2; i < val; ++i)
+ read_32bit_value();
+ /* Add previous run count to new counter, if not already counted before. */
+ runs = run_counted ? prev_runs : prev_runs + 1;
+ }
+
+ cur_pos = save_cur_pos;
+
+ if (gcov_version >= 90) {
+ write_32bit_value(GCOV_TAG_OBJECT_SUMMARY);
+ write_32bit_value(2);
+ write_32bit_value(runs);
+ write_32bit_value(0); // sum_max
+ } else {
+ // Before gcov 4.8 (r190952), GCOV_TAG_SUMMARY_LENGTH was 9. r190952 set
+ // GCOV_TAG_SUMMARY_LENGTH to 22. We simply use the smallest length which
+ // can make gcov read "Runs:".
+ write_32bit_value(GCOV_TAG_PROGRAM_SUMMARY);
+ write_32bit_value(3);
+ write_32bit_value(0);
+ write_32bit_value(0);
+ write_32bit_value(runs);
+ }
+
+ run_counted = 1;
+
+#ifdef DEBUG_GCDAPROFILING
+ fprintf(stderr, "llvmgcda: %u runs\n", runs);
+#endif
+}
+
+COMPILER_RT_VISIBILITY
+void llvm_gcda_end_file(void) {
+ /* Write out EOF record. */
+ if (output_file) {
+ write_bytes("\0\0\0\0\0\0\0\0", 8);
+
+ if (new_file) {
+ fwrite(write_buffer, cur_pos, 1, output_file);
+ free(write_buffer);
+ } else {
+ unmap_file();
+ }
+
+ fflush(output_file);
+ lprofUnlockFd(fd);
+ fclose(output_file);
+ output_file = NULL;
+ write_buffer = NULL;
+ }
+ free(filename);
+
+#ifdef DEBUG_GCDAPROFILING
+ fprintf(stderr, "llvmgcda: -----\n");
+#endif
+}
+
+COMPILER_RT_VISIBILITY
+void llvm_register_writeout_function(fn_ptr fn) {
+ fn_list_insert(&writeout_fn_list, fn);
+}
+
+COMPILER_RT_VISIBILITY
+void llvm_writeout_files(void) {
+ struct fn_node *curr = writeout_fn_list.head;
+
+ while (curr) {
+ if (curr->id == CURRENT_ID) {
+ curr->fn();
+ }
+ curr = curr->next;
+ }
+}
+
+#ifndef _WIN32
+// __attribute__((destructor)) and destructors whose priorities are greater than
+// 100 run before this function and can thus be tracked. The priority is
+// compatible with GCC 7 onwards.
+#if __GNUC__ >= 9
+#pragma GCC diagnostic ignored "-Wprio-ctor-dtor"
+#endif
+__attribute__((destructor(100)))
+#endif
+static void llvm_writeout_and_clear(void) {
+ llvm_writeout_files();
+ fn_list_remove(&writeout_fn_list);
+}
+
+COMPILER_RT_VISIBILITY
+void llvm_register_reset_function(fn_ptr fn) {
+ fn_list_insert(&reset_fn_list, fn);
+}
+
+COMPILER_RT_VISIBILITY
+void llvm_delete_reset_function_list(void) { fn_list_remove(&reset_fn_list); }
+
+COMPILER_RT_VISIBILITY
+void llvm_reset_counters(void) {
+ struct fn_node *curr = reset_fn_list.head;
+
+ while (curr) {
+ if (curr->id == CURRENT_ID) {
+ curr->fn();
+ }
+ curr = curr->next;
+ }
+}
+
+#if !defined(_WIN32) && !defined(__wasm__)
+COMPILER_RT_VISIBILITY
+pid_t __gcov_fork() {
+ pid_t parent_pid = getpid();
+ pid_t pid = fork();
+
+ if (pid == 0) {
+ pid_t child_pid = getpid();
+ if (child_pid != parent_pid) {
+ // The pid changed so we've a fork (one could have its own fork function)
+ // Just reset the counters for this child process
+ // threads.
+ llvm_reset_counters();
+ }
+ }
+ return pid;
+}
+#endif
+
+COMPILER_RT_VISIBILITY
+void llvm_gcov_init(fn_ptr wfn, fn_ptr rfn) {
+ static int atexit_ran = 0;
+
+ if (wfn)
+ llvm_register_writeout_function(wfn);
+
+ if (rfn)
+ llvm_register_reset_function(rfn);
+
+ if (atexit_ran == 0) {
+ atexit_ran = 1;
+
+ /* Make sure we write out the data and delete the data structures. */
+ lprofAtExit(llvm_delete_reset_function_list);
+#ifdef _WIN32
+ lprofAtExit(llvm_writeout_and_clear);
+#endif
+ }
+}
+
+#if defined(_AIX)
+COMPILER_RT_VISIBILITY __attribute__((constructor)) void
+__llvm_profile_gcov_initialize() {
+ const __llvm_gcov_init_func_struct *InitFuncStart =
+ __llvm_profile_begin_covinit();
+ const __llvm_gcov_init_func_struct *InitFuncEnd =
+ __llvm_profile_end_covinit();
+
+ for (const __llvm_gcov_init_func_struct *Ptr = InitFuncStart;
+ Ptr != InitFuncEnd; ++Ptr) {
+ fn_ptr wfn = (fn_ptr)Ptr->WriteoutFunction;
+ fn_ptr rfn = (fn_ptr)Ptr->ResetFunction;
+ if (!(wfn && rfn))
+ continue;
+ llvm_gcov_init(wfn, rfn);
+ }
+}
+#endif
+
+void __gcov_dump(void) {
+ for (struct fn_node *f = writeout_fn_list.head; f; f = f->next)
+ f->fn();
+}
+
+void __gcov_reset(void) {
+ for (struct fn_node *f = reset_fn_list.head; f; f = f->next)
+ f->fn();
+}
+
+#endif
diff --git a/system/lib/compiler-rt/lib/profile/InstrProfiling.c b/system/lib/compiler-rt/lib/profile/InstrProfiling.c
new file mode 100644
index 0000000000000..da04d8ebdec95
--- /dev/null
+++ b/system/lib/compiler-rt/lib/profile/InstrProfiling.c
@@ -0,0 +1,91 @@
+/*===- InstrProfiling.c - Support library for PGO instrumentation ---------===*\
+|*
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+|* See https://llvm.org/LICENSE.txt for license information.
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+|*
+\*===----------------------------------------------------------------------===*/
+
+// Note: This is linked into the Darwin kernel, and must remain compatible
+// with freestanding compilation. See `darwin_add_builtin_libraries`.
+
+#include <limits.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "InstrProfiling.h"
+#include "InstrProfilingInternal.h"
+
+#define INSTR_PROF_VALUE_PROF_DATA
+#include "profile/InstrProfData.inc"
+
+static uint32_t __llvm_profile_global_timestamp = 1;
+
+COMPILER_RT_VISIBILITY
+void INSTR_PROF_PROFILE_SET_TIMESTAMP(uint64_t *Probe) {
+ if (*Probe == 0 || *Probe == (uint64_t)-1)
+ *Probe = __llvm_profile_global_timestamp++;
+}
+
+COMPILER_RT_VISIBILITY uint64_t __llvm_profile_get_magic(void) {
+ return sizeof(void *) == sizeof(uint64_t) ? (INSTR_PROF_RAW_MAGIC_64)
+ : (INSTR_PROF_RAW_MAGIC_32);
+}
+
+COMPILER_RT_VISIBILITY void __llvm_profile_set_dumped(void) {
+ lprofSetProfileDumped(1);
+}
+
+/* Return the number of bytes needed to add to SizeInBytes to make it
+ * the result a multiple of 8.
+ */
+COMPILER_RT_VISIBILITY uint8_t
+__llvm_profile_get_num_padding_bytes(uint64_t SizeInBytes) {
+ return 7 & (sizeof(uint64_t) - SizeInBytes % sizeof(uint64_t));
+}
+
+COMPILER_RT_VISIBILITY uint64_t __llvm_profile_get_version(void) {
+ return INSTR_PROF_RAW_VERSION_VAR;
+}
+
+COMPILER_RT_VISIBILITY void __llvm_profile_reset_counters(void) {
+ if (__llvm_profile_get_version() & VARIANT_MASK_TEMPORAL_PROF)
+ __llvm_profile_global_timestamp = 1;
+
+ char *I = __llvm_profile_begin_counters();
+ char *E = __llvm_profile_end_counters();
+
+ char ResetValue =
+ (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE) ? 0xFF : 0;
+ memset(I, ResetValue, E - I);
+
+ I = __llvm_profile_begin_bitmap();
+ E = __llvm_profile_end_bitmap();
+ memset(I, 0x0, E - I);
+
+ const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
+ const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
+ const __llvm_profile_data *DI;
+ for (DI = DataBegin; DI < DataEnd; ++DI) {
+ uint64_t CurrentVSiteCount = 0;
+ uint32_t VKI, i;
+ if (!DI->Values)
+ continue;
+
+ ValueProfNode **ValueCounters = (ValueProfNode **)DI->Values;
+
+ for (VKI = IPVK_First; VKI <= IPVK_Last; ++VKI)
+ CurrentVSiteCount += DI->NumValueSites[VKI];
+
+ for (i = 0; i < CurrentVSiteCount; ++i) {
+ ValueProfNode *CurrVNode = ValueCounters[i];
+
+ while (CurrVNode) {
+ CurrVNode->Count = 0;
+ CurrVNode = CurrVNode->Next;
+ }
+ }
+ }
+ lprofSetProfileDumped(0);
+}
diff --git a/system/lib/compiler-rt/lib/profile/InstrProfiling.h b/system/lib/compiler-rt/lib/profile/InstrProfiling.h
new file mode 100644
index 0000000000000..7f0c0c194dc91
--- /dev/null
+++ b/system/lib/compiler-rt/lib/profile/InstrProfiling.h
@@ -0,0 +1,338 @@
+/*===- InstrProfiling.h- Support library for PGO instrumentation ----------===*\
+|*
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+|* See https://llvm.org/LICENSE.txt for license information.
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+|*
+\*===----------------------------------------------------------------------===*/
+
+#ifndef PROFILE_INSTRPROFILING_H_
+#define PROFILE_INSTRPROFILING_H_
+
+#include "InstrProfilingPort.h"
+#include <stdio.h>
+
+// Make sure __LLVM_INSTR_PROFILE_GENERATE is always defined before
+// including instr_prof_interface.h so the interface functions are
+// declared correctly for the runtime.
+// __LLVM_INSTR_PROFILE_GENERATE is always `#undef`ed after the header,
+// because compiler-rt does not support profiling the profiling runtime itself.
+#ifndef __LLVM_INSTR_PROFILE_GENERATE
+#define __LLVM_INSTR_PROFILE_GENERATE
+#endif
+#include "profile/instr_prof_interface.h"
+#undef __LLVM_INSTR_PROFILE_GENERATE
+
+#define INSTR_PROF_VISIBILITY COMPILER_RT_VISIBILITY
+#include "profile/InstrProfData.inc"
+
+enum ValueKind {
+#define VALUE_PROF_KIND(Enumerator, Value, Descr) Enumerator = Value,
+#include "profile/InstrProfData.inc"
+};
+
+typedef void *IntPtrT;
+typedef struct COMPILER_RT_ALIGNAS(INSTR_PROF_DATA_ALIGNMENT)
+ __llvm_profile_data {
+#define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) Type Name;
+#include "profile/InstrProfData.inc"
+} __llvm_profile_data;
+
+typedef struct __llvm_profile_header {
+#define INSTR_PROF_RAW_HEADER(Type, Name, Initializer) Type Name;
+#include "profile/InstrProfData.inc"
+} __llvm_profile_header;
+
+typedef struct ValueProfNode * PtrToNodeT;
+typedef struct ValueProfNode {
+#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Initializer) Type Name;
+#include "profile/InstrProfData.inc"
+} ValueProfNode;
+
+typedef struct COMPILER_RT_ALIGNAS(INSTR_PROF_DATA_ALIGNMENT) VTableProfData {
+#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Initializer) Type Name;
+#include "profile/InstrProfData.inc"
+} VTableProfData;
+
+typedef struct COMPILER_RT_ALIGNAS(INSTR_PROF_DATA_ALIGNMENT)
+ __llvm_gcov_init_func_struct {
+#define COVINIT_FUNC(Type, LLVMType, Name, Initializer) Type Name;
+#include "profile/InstrProfData.inc"
+} __llvm_gcov_init_func_struct;
+
+/*!
+ * \brief Return 1 if profile counters are continuously synced to the raw
+ * profile via an mmap(). This is in contrast to the default mode, in which
+ * the raw profile is written out at program exit time.
+ */
+int __llvm_profile_is_continuous_mode_enabled(void);
+
+/*!
+ * \brief Enable continuous mode.
+ *
+ * See \ref __llvm_profile_is_continuous_mode_enabled. The behavior is undefined
+ * if continuous mode is already enabled, or if it cannot be enable due to
+ * conflicting options.
+ */
+void __llvm_profile_enable_continuous_mode(void);
+
+/*!
+ * \brief Disable continuous mode.
+ *
+ */
+void __llvm_profile_disable_continuous_mode(void);
+
+/*!
+ * \brief Set the page size.
+ *
+ * This is a pre-requisite for enabling continuous mode. The buffer size
+ * calculation code inside of libprofile cannot simply call getpagesize(), as
+ * it is not allowed to depend on libc.
+ */
+void __llvm_profile_set_page_size(unsigned PageSize);
+
+/*!
+ * \brief Get number of bytes necessary to pad the argument to eight
+ * byte boundary.
+ */
+uint8_t __llvm_profile_get_num_padding_bytes(uint64_t SizeInBytes);
+
+/*!
+ * \brief Get required size for profile buffer.
+ */
+uint64_t __llvm_profile_get_size_for_buffer(void);
+
+/*!
+ * \brief Write instrumentation data to the given buffer.
+ *
+ * \pre \c Buffer is the start of a buffer at least as big as \a
+ * __llvm_profile_get_size_for_buffer().
+ */
+int __llvm_profile_write_buffer(char *Buffer);
+
+const __llvm_profile_data *__llvm_profile_begin_data(void);
+const __llvm_profile_data *__llvm_profile_end_data(void);
+const char *__llvm_profile_begin_names(void);
+const char *__llvm_profile_end_names(void);
+const char *__llvm_profile_begin_vtabnames(void);
+const char *__llvm_profile_end_vtabnames(void);
+char *__llvm_profile_begin_counters(void);
+char *__llvm_profile_end_counters(void);
+char *__llvm_profile_begin_bitmap(void);
+char *__llvm_profile_end_bitmap(void);
+ValueProfNode *__llvm_profile_begin_vnodes(void);
+ValueProfNode *__llvm_profile_end_vnodes(void);
+const VTableProfData *__llvm_profile_begin_vtables(void);
+const VTableProfData *__llvm_profile_end_vtables(void);
+uint32_t *__llvm_profile_begin_orderfile(void);
+
+/*!
+ * \brief Merge profile data from buffer.
+ *
+ * Read profile data from buffer \p Profile and merge with in-process profile
+ * counters and bitmaps. The client is expected to have checked or already
+ * know the profile data in the buffer matches the in-process counter
+ * structure before calling it. Returns 0 (success) if the profile data is
+ * valid. Upon reading invalid/corrupted profile data, returns 1 (failure).
+ */
+int __llvm_profile_merge_from_buffer(const char *Profile, uint64_t Size);
+
+/*! \brief Check if profile in buffer matches the current binary.
+ *
+ * Returns 0 (success) if the profile data in buffer \p Profile with size
+ * \p Size was generated by the same binary and therefore matches
+ * structurally the in-process counters and bitmaps. If the profile data in
+ * buffer is not compatible, the interface returns 1 (failure).
+ */
+int __llvm_profile_check_compatibility(const char *Profile,
+ uint64_t Size);
+
+/*!
+ * \brief Counts the number of times a target value is seen.
+ *
+ * Records the target value for the CounterIndex if not seen before. Otherwise,
+ * increments the counter associated w/ the target value.
+ * void __llvm_profile_instrument_target(uint64_t TargetValue, void *Data,
+ * uint32_t CounterIndex);
+ */
+void INSTR_PROF_VALUE_PROF_FUNC(
+#define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType) ArgType ArgName
+#include "profile/InstrProfData.inc"
+ );
+
+void __llvm_profile_instrument_target_value(uint64_t TargetValue, void *Data,
+ uint32_t CounterIndex,
+ uint64_t CounterValue);
+
+/*!
+ * \brief Write instrumentation data to the current file.
+ *
+ * Writes to the file with the last name given to \a *
+ * __llvm_profile_set_filename(),
+ * or if it hasn't been called, the \c LLVM_PROFILE_FILE environment variable,
+ * or if that's not set, the last name set to INSTR_PROF_PROFILE_NAME_VAR,
+ * or if that's not set, \c "default.profraw".
+ */
+int __llvm_profile_write_file(void);
+
+int __llvm_orderfile_write_file(void);
+
+/*!
+ * \brief Set the FILE object for writing instrumentation data. Return 0 if set
+ * successfully or return 1 if failed.
+ *
+ * Sets the FILE object to be used for subsequent calls to
+ * \a __llvm_profile_write_file(). The profile file name set by environment
+ * variable, command-line option, or calls to \a __llvm_profile_set_filename
+ * will be ignored.
+ *
+ * \c File will not be closed after a call to \a __llvm_profile_write_file() but
+ * it may be flushed. Passing NULL restores default behavior.
+ *
+ * If \c EnableMerge is nonzero, the runtime will always merge profiling data
+ * with the contents of the profiling file. If EnableMerge is zero, the runtime
+ * may still merge the data if it would have merged for another reason (for
+ * example, because of a %m specifier in the file name).
+ *
+ * Note: There may be multiple copies of the profile runtime (one for each
+ * instrumented image/DSO). This API only modifies the file object within the
+ * copy of the runtime available to the calling image.
+ *
+ * Warning: This is a no-op if EnableMerge is 0 in continuous mode (\ref
+ * __llvm_profile_is_continuous_mode_enabled), because disable merging requires
+ * copying the old profile file to new profile file and this function is usually
+ * used when the proess doesn't have permission to open file.
+ */
+int __llvm_profile_set_file_object(FILE *File, int EnableMerge);
+
+/*! \brief Register to write instrumentation data to file at exit. */
+int __llvm_profile_register_write_file_atexit(void);
+
+/*! \brief Initialize file handling. */
+void __llvm_profile_initialize_file(void);
+
+/*! \brief Initialize the profile runtime. */
+void __llvm_profile_initialize(void);
+
+/*! \brief Initialize the gcov profile runtime. */
+void __llvm_profile_gcov_initialize(void);
+
+/*!
+ * \brief Return path prefix (excluding the base filename) of the profile data.
+ * This is useful for users using \c -fprofile-generate=./path_prefix who do
+ * not care about the default raw profile name. It is also useful to collect
+ * more than more profile data files dumped in the same directory (Online
+ * merge mode is turned on for instrumented programs with shared libs).
+ * Side-effect: this API call will invoke malloc with dynamic memory allocation.
+ */
+const char *__llvm_profile_get_path_prefix(void);
+
+/*!
+ * \brief Return filename (including path) of the profile data. Note that if the
+ * user calls __llvm_profile_set_filename later after invoking this interface,
+ * the actual file name may differ from what is returned here.
+ * Side-effect: this API call will invoke malloc with dynamic memory allocation
+ * (the returned pointer must be passed to `free` to avoid a leak).
+ *
+ * Note: There may be multiple copies of the profile runtime (one for each
+ * instrumented image/DSO). This API only retrieves the filename from the copy
+ * of the runtime available to the calling image.
+ */
+const char *__llvm_profile_get_filename(void);
+
+/*! \brief Get the magic token for the file format. */
+uint64_t __llvm_profile_get_magic(void);
+
+/*! \brief Get the version of the file format. */
+uint64_t __llvm_profile_get_version(void);
+
+/*! \brief Get the number of entries in the profile data section. */
+uint64_t __llvm_profile_get_num_data(const __llvm_profile_data *Begin,
+ const __llvm_profile_data *End);
+
+/*! \brief Get the size of the profile data section in bytes. */
+uint64_t __llvm_profile_get_data_size(const __llvm_profile_data *Begin,
+ const __llvm_profile_data *End);
+
+/*! \brief Get the size in bytes of a single counter entry. */
+size_t __llvm_profile_counter_entry_size(void);
+
+/*! \brief Get the number of entries in the profile counters section. */
+uint64_t __llvm_profile_get_num_counters(const char *Begin, const char *End);
+
+/*! \brief Get the size of the profile counters section in bytes. */
+uint64_t __llvm_profile_get_counters_size(const char *Begin, const char *End);
+
+/*! \brief Get the number of bytes in the profile bitmap section. */
+uint64_t __llvm_profile_get_num_bitmap_bytes(const char *Begin,
+ const char *End);
+
+/*! \brief Get the size of the profile name section in bytes. */
+uint64_t __llvm_profile_get_name_size(const char *Begin, const char *End);
+
+/*! \brief Get the number of virtual table profile data entries */
+uint64_t __llvm_profile_get_num_vtable(const VTableProfData *Begin,
+ const VTableProfData *End);
+
+/*! \brief Get the size of virtual table profile data in bytes. */
+uint64_t __llvm_profile_get_vtable_section_size(const VTableProfData *Begin,
+ const VTableProfData *End);
+
+/* ! \brief Given the sizes of the data and counter information, computes the
+ * number of padding bytes before and after the counter section, as well as the
+ * number of padding bytes after other setions in the raw profile.
+ * Returns -1 upon errors and 0 upon success. Output parameters should be used
+ * iff return value is 0.
+ *
+ * Note: When mmap() mode is disabled, no padding bytes before/after counters
+ * are needed. However, in mmap() mode, the counter section in the raw profile
+ * must be page-aligned: this API computes the number of padding bytes
+ * needed to achieve that.
+ */
+int __llvm_profile_get_padding_sizes_for_counters(
+ uint64_t DataSize, uint64_t CountersSize, uint64_t NumBitmapBytes,
+ uint64_t NamesSize, uint64_t VTableSize, uint64_t VNameSize,
+ uint64_t *PaddingBytesBeforeCounters, uint64_t *PaddingBytesAfterCounters,
+ uint64_t *PaddingBytesAfterBitmap, uint64_t *PaddingBytesAfterNames,
+ uint64_t *PaddingBytesAfterVTable, uint64_t *PaddingBytesAfterVNames);
+
+/*!
+ * \brief Set the flag that profile data has been dumped to the file.
+ * This is useful for users to disable dumping profile data to the file for
+ * certain processes in case the processes don't have permission to write to
+ * the disks, and trying to do so would result in side effects such as crashes.
+ */
+void __llvm_profile_set_dumped(void);
+
+/*!
+ * This variable is defined in InstrProfilingRuntime.cpp as a hidden
+ * symbol. Its main purpose is to enable profile runtime user to
+ * bypass runtime initialization code -- if the client code explicitly
+ * define this variable, then InstProfileRuntime.o won't be linked in.
+ * Note that this variable's visibility needs to be hidden so that the
+ * definition of this variable in an instrumented shared library won't
+ * affect runtime initialization decision of the main program.
+ * __llvm_profile_profile_runtime. */
+COMPILER_RT_VISIBILITY extern int INSTR_PROF_PROFILE_RUNTIME_VAR;
+
+/*!
+ * This variable is defined in InstrProfilingVersionVar.c as a hidden symbol
+ * (except on Apple platforms where this symbol is checked by TAPI). Its main
+ * purpose is to encode the raw profile version value and other format related
+ * information such as whether the profile is from IR based instrumentation. The
+ * variable is defined as weak so that compiler can emit an overriding
+ * definition depending on user option.
+ */
+COMPILER_RT_VISIBILITY extern uint64_t
+ INSTR_PROF_RAW_VERSION_VAR; /* __llvm_profile_raw_version */
+
+/*!
+ * This variable is a weak symbol defined in InstrProfiling.c. It allows
+ * compiler instrumentation to provide overriding definition with value
+ * from compiler command line. This variable has default visibility.
+ */
+extern char INSTR_PROF_PROFILE_NAME_VAR[1]; /* __llvm_profile_filename. */
+
+const __llvm_gcov_init_func_struct *__llvm_profile_begin_covinit();
+const __llvm_gcov_init_func_struct *__llvm_profile_end_covinit();
+#endif /* PROFILE_INSTRPROFILING_H_ */
diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingBuffer.c b/system/lib/compiler-rt/lib/profile/InstrProfilingBuffer.c
new file mode 100644
index 0000000000000..1c451d7ec7563
--- /dev/null
+++ b/system/lib/compiler-rt/lib/profile/InstrProfilingBuffer.c
@@ -0,0 +1,256 @@
+/*===- InstrProfilingBuffer.c - Write instrumentation to a memory buffer --===*\
+|*
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+|* See https://llvm.org/LICENSE.txt for license information.
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+|*
+\*===----------------------------------------------------------------------===*/
+
+// Note: This is linked into the Darwin kernel, and must remain compatible
+// with freestanding compilation. See `darwin_add_builtin_libraries`.
+
+#include "InstrProfiling.h"
+#include "InstrProfilingInternal.h"
+#include "InstrProfilingPort.h"
+
+/* When continuous mode is enabled (%c), this parameter is set to 1.
+ *
+ * This parameter is defined here in InstrProfilingBuffer.o, instead of in
+ * InstrProfilingFile.o, to sequester all libc-dependent code in
+ * InstrProfilingFile.o. The test `instrprof-without-libc` will break if this
+ * layering is violated. */
+static int ContinuouslySyncProfile = 0;
+
+/* The system page size. Only valid when non-zero. If 0, the page size is
+ * unavailable. */
+static unsigned PageSize = 0;
+
+COMPILER_RT_VISIBILITY int __llvm_profile_is_continuous_mode_enabled(void) {
+ return ContinuouslySyncProfile && PageSize;
+}
+
+COMPILER_RT_VISIBILITY void __llvm_profile_enable_continuous_mode(void) {
+ ContinuouslySyncProfile = 1;
+}
+
+COMPILER_RT_VISIBILITY void __llvm_profile_disable_continuous_mode(void) {
+ ContinuouslySyncProfile = 0;
+}
+
+COMPILER_RT_VISIBILITY void __llvm_profile_set_page_size(unsigned PS) {
+ PageSize = PS;
+}
+
+COMPILER_RT_VISIBILITY
+uint64_t __llvm_profile_get_size_for_buffer(void) {
+ const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
+ const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
+ const char *CountersBegin = __llvm_profile_begin_counters();
+ const char *CountersEnd = __llvm_profile_end_counters();
+ const char *BitmapBegin = __llvm_profile_begin_bitmap();
+ const char *BitmapEnd = __llvm_profile_end_bitmap();
+ const char *NamesBegin = __llvm_profile_begin_names();
+ const char *NamesEnd = __llvm_profile_end_names();
+ const VTableProfData *VTableBegin = __llvm_profile_begin_vtables();
+ const VTableProfData *VTableEnd = __llvm_profile_end_vtables();
+ const char *VNamesBegin = __llvm_profile_begin_vtabnames();
+ const char *VNamesEnd = __llvm_profile_end_vtabnames();
+
+ return __llvm_profile_get_size_for_buffer_internal(
+ DataBegin, DataEnd, CountersBegin, CountersEnd, BitmapBegin, BitmapEnd,
+ NamesBegin, NamesEnd, VTableBegin, VTableEnd, VNamesBegin, VNamesEnd);
+}
+
+// NOTE: Caller should guarantee that `Begin` and `End` specifies a half-open
+// interval [Begin, End). Namely, `End` is one-byte past the end of the array.
+COMPILER_RT_VISIBILITY
+uint64_t __llvm_profile_get_num_data(const __llvm_profile_data *Begin,
+ const __llvm_profile_data *End) {
+ intptr_t BeginI = (intptr_t)Begin, EndI = (intptr_t)End;
+ return ((EndI + sizeof(__llvm_profile_data) - 1) - BeginI) /
+ sizeof(__llvm_profile_data);
+}
+
+COMPILER_RT_VISIBILITY
+uint64_t __llvm_profile_get_data_size(const __llvm_profile_data *Begin,
+ const __llvm_profile_data *End) {
+ return __llvm_profile_get_num_data(Begin, End) * sizeof(__llvm_profile_data);
+}
+
+// Counts the number of `VTableProfData` elements within the range of [Begin,
+// End). Caller should guarantee that End points to one byte past the inclusive
+// range.
+// FIXME: Add a compiler-rt test to make sure the number of vtables in the
+// raw profile is the same as the number of vtable elements in the instrumented
+// binary.
+COMPILER_RT_VISIBILITY
+uint64_t __llvm_profile_get_num_vtable(const VTableProfData *Begin,
+ const VTableProfData *End) {
+ // Convert pointers to intptr_t to use integer arithmetic.
+ intptr_t EndI = (intptr_t)End, BeginI = (intptr_t)Begin;
+ return (EndI - BeginI) / sizeof(VTableProfData);
+}
+
+COMPILER_RT_VISIBILITY
+uint64_t __llvm_profile_get_vtable_section_size(const VTableProfData *Begin,
+ const VTableProfData *End) {
+ return (intptr_t)(End) - (intptr_t)(Begin);
+}
+
+COMPILER_RT_VISIBILITY size_t __llvm_profile_counter_entry_size(void) {
+ if (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE)
+ return sizeof(uint8_t);
+ return sizeof(uint64_t);
+}
+
+COMPILER_RT_VISIBILITY
+uint64_t __llvm_profile_get_num_counters(const char *Begin, const char *End) {
+ intptr_t BeginI = (intptr_t)Begin, EndI = (intptr_t)End;
+ return ((EndI + __llvm_profile_counter_entry_size() - 1) - BeginI) /
+ __llvm_profile_counter_entry_size();
+}
+
+COMPILER_RT_VISIBILITY
+uint64_t __llvm_profile_get_counters_size(const char *Begin, const char *End) {
+ return __llvm_profile_get_num_counters(Begin, End) *
+ __llvm_profile_counter_entry_size();
+}
+
+COMPILER_RT_VISIBILITY
+uint64_t __llvm_profile_get_num_bitmap_bytes(const char *Begin,
+ const char *End) {
+ return (End - Begin);
+}
+
+COMPILER_RT_VISIBILITY
+uint64_t __llvm_profile_get_name_size(const char *Begin, const char *End) {
+ return End - Begin;
+}
+
+/// Calculate the number of padding bytes needed to add to \p Offset in order
+/// for (\p Offset + Padding) to be page-aligned.
+static uint64_t calculateBytesNeededToPageAlign(uint64_t Offset) {
+ uint64_t OffsetModPage = Offset % PageSize;
+ if (OffsetModPage > 0)
+ return PageSize - OffsetModPage;
+ return 0;
+}
+
+static int needsCounterPadding(void) {
+#if defined(__APPLE__)
+ return __llvm_profile_is_continuous_mode_enabled();
+#else
+ return 0;
+#endif
+}
+
+COMPILER_RT_VISIBILITY
+int __llvm_profile_get_padding_sizes_for_counters(
+ uint64_t DataSize, uint64_t CountersSize, uint64_t NumBitmapBytes,
+ uint64_t NamesSize, uint64_t VTableSize, uint64_t VNameSize,
+ uint64_t *PaddingBytesBeforeCounters, uint64_t *PaddingBytesAfterCounters,
+ uint64_t *PaddingBytesAfterBitmapBytes, uint64_t *PaddingBytesAfterNames,
+ uint64_t *PaddingBytesAfterVTable, uint64_t *PaddingBytesAfterVName) {
+ // Counter padding is needed only if continuous mode is enabled.
+ if (!needsCounterPadding()) {
+ *PaddingBytesBeforeCounters = 0;
+ *PaddingBytesAfterCounters =
+ __llvm_profile_get_num_padding_bytes(CountersSize);
+ *PaddingBytesAfterBitmapBytes =
+ __llvm_profile_get_num_padding_bytes(NumBitmapBytes);
+ *PaddingBytesAfterNames = __llvm_profile_get_num_padding_bytes(NamesSize);
+ if (PaddingBytesAfterVTable != NULL)
+ *PaddingBytesAfterVTable =
+ __llvm_profile_get_num_padding_bytes(VTableSize);
+ if (PaddingBytesAfterVName != NULL)
+ *PaddingBytesAfterVName = __llvm_profile_get_num_padding_bytes(VNameSize);
+ return 0;
+ }
+
+ // Value profiling not supported in continuous mode at profile-write time.
+ // Return -1 to alert the incompatibility.
+ if (VTableSize != 0 || VNameSize != 0)
+ return -1;
+
+ // In continuous mode, the file offsets for headers and for the start of
+ // counter sections need to be page-aligned.
+ *PaddingBytesBeforeCounters =
+ calculateBytesNeededToPageAlign(sizeof(__llvm_profile_header) + DataSize);
+ *PaddingBytesAfterCounters = calculateBytesNeededToPageAlign(CountersSize);
+ *PaddingBytesAfterBitmapBytes =
+ calculateBytesNeededToPageAlign(NumBitmapBytes);
+ *PaddingBytesAfterNames = calculateBytesNeededToPageAlign(NamesSize);
+ // Set these two variables to zero to avoid uninitialized variables
+ // even if VTableSize and VNameSize are known to be zero.
+ if (PaddingBytesAfterVTable != NULL)
+ *PaddingBytesAfterVTable = 0;
+ if (PaddingBytesAfterVName != NULL)
+ *PaddingBytesAfterVName = 0;
+ return 0;
+}
+
+COMPILER_RT_VISIBILITY
+uint64_t __llvm_profile_get_size_for_buffer_internal(
+ const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd,
+ const char *CountersBegin, const char *CountersEnd, const char *BitmapBegin,
+ const char *BitmapEnd, const char *NamesBegin, const char *NamesEnd,
+ const VTableProfData *VTableBegin, const VTableProfData *VTableEnd,
+ const char *VNamesBegin, const char *VNamesEnd) {
+ /* Match logic in __llvm_profile_write_buffer(). */
+ const uint64_t NamesSize = (NamesEnd - NamesBegin) * sizeof(char);
+ uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd);
+ uint64_t CountersSize =
+ __llvm_profile_get_counters_size(CountersBegin, CountersEnd);
+ const uint64_t NumBitmapBytes =
+ __llvm_profile_get_num_bitmap_bytes(BitmapBegin, BitmapEnd);
+ const uint64_t VTableSize =
+ __llvm_profile_get_vtable_section_size(VTableBegin, VTableEnd);
+ const uint64_t VNameSize =
+ __llvm_profile_get_name_size(VNamesBegin, VNamesEnd);
+
+ /* Determine how much padding is needed before/after the counters and after
+ * the names. */
+ uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters,
+ PaddingBytesAfterNames, PaddingBytesAfterBitmapBytes,
+ PaddingBytesAfterVTable, PaddingBytesAfterVNames;
+ __llvm_profile_get_padding_sizes_for_counters(
+ DataSize, CountersSize, NumBitmapBytes, NamesSize, 0 /* VTableSize */,
+ 0 /* VNameSize */, &PaddingBytesBeforeCounters,
+ &PaddingBytesAfterCounters, &PaddingBytesAfterBitmapBytes,
+ &PaddingBytesAfterNames, &PaddingBytesAfterVTable,
+ &PaddingBytesAfterVNames);
+
+ return sizeof(__llvm_profile_header) + __llvm_write_binary_ids(NULL) +
+ DataSize + PaddingBytesBeforeCounters + CountersSize +
+ PaddingBytesAfterCounters + NumBitmapBytes +
+ PaddingBytesAfterBitmapBytes + NamesSize + PaddingBytesAfterNames +
+ VTableSize + PaddingBytesAfterVTable + VNameSize +
+ PaddingBytesAfterVNames;
+}
+
+COMPILER_RT_VISIBILITY
+void initBufferWriter(ProfDataWriter *BufferWriter, char *Buffer) {
+ BufferWriter->Write = lprofBufferWriter;
+ BufferWriter->WriterCtx = Buffer;
+}
+
+COMPILER_RT_VISIBILITY int __llvm_profile_write_buffer(char *Buffer) {
+ ProfDataWriter BufferWriter;
+ initBufferWriter(&BufferWriter, Buffer);
+ return lprofWriteData(&BufferWriter, 0, 0);
+}
+
+COMPILER_RT_VISIBILITY int __llvm_profile_write_buffer_internal(
+ char *Buffer, const __llvm_profile_data *DataBegin,
+ const __llvm_profile_data *DataEnd, const char *CountersBegin,
+ const char *CountersEnd, const char *BitmapBegin, const char *BitmapEnd,
+ const char *NamesBegin, const char *NamesEnd) {
+ ProfDataWriter BufferWriter;
+ initBufferWriter(&BufferWriter, Buffer);
+ // Set virtual table arguments to NULL since they are not supported yet.
+ return lprofWriteDataImpl(
+ &BufferWriter, DataBegin, DataEnd, CountersBegin, CountersEnd,
+ BitmapBegin, BitmapEnd, /*VPDataReader=*/0, NamesBegin, NamesEnd,
+ /*VTableBegin=*/NULL, /*VTableEnd=*/NULL, /*VNamesBegin=*/NULL,
+ /*VNamesEnd=*/NULL, /*SkipNameDataWrite=*/0);
+}
diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingFile.c b/system/lib/compiler-rt/lib/profile/InstrProfilingFile.c
new file mode 100644
index 0000000000000..bad4cc71801ec
--- /dev/null
+++ b/system/lib/compiler-rt/lib/profile/InstrProfilingFile.c
@@ -0,0 +1,1314 @@
+/*===- InstrProfilingFile.c - Write instrumentation to a file -------------===*\
+|*
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+|* See https://llvm.org/LICENSE.txt for license information.
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+|*
+\*===----------------------------------------------------------------------===*/
+
+#if !defined(__Fuchsia__)
+
+#include <assert.h>
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#ifdef _MSC_VER
+/* For _alloca. */
+#include <malloc.h>
+#endif
+#if defined(_WIN32)
+#include "WindowsMMap.h"
+/* For _chsize_s */
+#include <io.h>
+#include <process.h>
+#else
+#include <sys/file.h>
+#include <sys/mman.h>
+#include <unistd.h>
+#if defined(__linux__)
+#include <sys/types.h>
+#endif
+#endif
+
+#include "InstrProfiling.h"
+#include "InstrProfilingInternal.h"
+#include "InstrProfilingPort.h"
+#include "InstrProfilingUtil.h"
+
+/* From where is profile name specified.
+ * The order the enumerators define their
+ * precedence. Re-order them may lead to
+ * runtime behavior change. */
+typedef enum ProfileNameSpecifier {
+ PNS_unknown = 0,
+ PNS_default,
+ PNS_command_line,
+ PNS_environment,
+ PNS_runtime_api
+} ProfileNameSpecifier;
+
+static const char *getPNSStr(ProfileNameSpecifier PNS) {
+ switch (PNS) {
+ case PNS_default:
+ return "default setting";
+ case PNS_command_line:
+ return "command line";
+ case PNS_environment:
+ return "environment variable";
+ case PNS_runtime_api:
+ return "runtime API";
+ default:
+ return "Unknown";
+ }
+}
+
+#define MAX_PID_SIZE 16
+/* Data structure holding the result of parsed filename pattern. */
+typedef struct lprofFilename {
+ /* File name string possibly with %p or %h specifiers. */
+ const char *FilenamePat;
+ /* A flag indicating if FilenamePat's memory is allocated
+ * by runtime. */
+ unsigned OwnsFilenamePat;
+ const char *ProfilePathPrefix;
+ char PidChars[MAX_PID_SIZE];
+ char *TmpDir;
+ char Hostname[COMPILER_RT_MAX_HOSTLEN];
+ unsigned NumPids;
+ unsigned NumHosts;
+ /* When in-process merging is enabled, this parameter specifies
+ * the total number of profile data files shared by all the processes
+ * spawned from the same binary. By default the value is 1. If merging
+ * is not enabled, its value should be 0. This parameter is specified
+ * by the %[0-9]m specifier. For instance %2m enables merging using
+ * 2 profile data files. %1m is equivalent to %m. Also %m specifier
+ * can only appear once at the end of the name pattern. */
+ unsigned MergePoolSize;
+ ProfileNameSpecifier PNS;
+} lprofFilename;
+
+static lprofFilename lprofCurFilename = {0, 0, 0, {0}, NULL,
+ {0}, 0, 0, 0, PNS_unknown};
+
+static int ProfileMergeRequested = 0;
+static int getProfileFileSizeForMerging(FILE *ProfileFile,
+ uint64_t *ProfileFileSize);
+
+#if defined(__APPLE__)
+static const int ContinuousModeSupported = 1;
+static const int UseBiasVar = 0;
+static const char *FileOpenMode = "a+b";
+static void *BiasAddr = NULL;
+static void *BiasDefaultAddr = NULL;
+static void *BitmapBiasAddr = NULL;
+static void *BitmapBiasDefaultAddr = NULL;
+static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) {
+ /* Get the sizes of various profile data sections. Taken from
+ * __llvm_profile_get_size_for_buffer(). */
+ const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
+ const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
+ const char *CountersBegin = __llvm_profile_begin_counters();
+ const char *CountersEnd = __llvm_profile_end_counters();
+ const char *BitmapBegin = __llvm_profile_begin_bitmap();
+ const char *BitmapEnd = __llvm_profile_end_bitmap();
+ const char *NamesBegin = __llvm_profile_begin_names();
+ const char *NamesEnd = __llvm_profile_end_names();
+ const uint64_t NamesSize = (NamesEnd - NamesBegin) * sizeof(char);
+ uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd);
+ uint64_t CountersSize =
+ __llvm_profile_get_counters_size(CountersBegin, CountersEnd);
+ uint64_t NumBitmapBytes =
+ __llvm_profile_get_num_bitmap_bytes(BitmapBegin, BitmapEnd);
+
+ /* Check that the counter, bitmap, and data sections in this image are
+ * page-aligned. */
+ unsigned PageSize = getpagesize();
+ if ((intptr_t)CountersBegin % PageSize != 0) {
+ PROF_ERR("Counters section not page-aligned (start = %p, pagesz = %u).\n",
+ CountersBegin, PageSize);
+ return 1;
+ }
+ if ((intptr_t)BitmapBegin % PageSize != 0) {
+ PROF_ERR("Bitmap section not page-aligned (start = %p, pagesz = %u).\n",
+ BitmapBegin, PageSize);
+ return 1;
+ }
+ if ((intptr_t)DataBegin % PageSize != 0) {
+ PROF_ERR("Data section not page-aligned (start = %p, pagesz = %u).\n",
+ DataBegin, PageSize);
+ return 1;
+ }
+
+ int Fileno = fileno(File);
+ /* Determine how much padding is needed before/after the counters and
+ * after the names. */
+ uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters,
+ PaddingBytesAfterNames, PaddingBytesAfterBitmapBytes,
+ PaddingBytesAfterVTable, PaddingBytesAfterVNames;
+ __llvm_profile_get_padding_sizes_for_counters(
+ DataSize, CountersSize, NumBitmapBytes, NamesSize, /*VTableSize=*/0,
+ /*VNameSize=*/0, &PaddingBytesBeforeCounters, &PaddingBytesAfterCounters,
+ &PaddingBytesAfterBitmapBytes, &PaddingBytesAfterNames,
+ &PaddingBytesAfterVTable, &PaddingBytesAfterVNames);
+
+ uint64_t PageAlignedCountersLength = CountersSize + PaddingBytesAfterCounters;
+ uint64_t FileOffsetToCounters = CurrentFileOffset +
+ sizeof(__llvm_profile_header) + DataSize +
+ PaddingBytesBeforeCounters;
+ void *CounterMmap = mmap((void *)CountersBegin, PageAlignedCountersLength,
+ PROT_READ | PROT_WRITE, MAP_FIXED | MAP_SHARED,
+ Fileno, FileOffsetToCounters);
+ if (CounterMmap != CountersBegin) {
+ PROF_ERR(
+ "Continuous counter sync mode is enabled, but mmap() failed (%s).\n"
+ " - CountersBegin: %p\n"
+ " - PageAlignedCountersLength: %" PRIu64 "\n"
+ " - Fileno: %d\n"
+ " - FileOffsetToCounters: %" PRIu64 "\n",
+ strerror(errno), CountersBegin, PageAlignedCountersLength, Fileno,
+ FileOffsetToCounters);
+ return 1;
+ }
+
+ /* Also mmap MCDC bitmap bytes. If there aren't any bitmap bytes, mmap()
+ * will fail with EINVAL. */
+ if (NumBitmapBytes == 0)
+ return 0;
+
+ uint64_t PageAlignedBitmapLength =
+ NumBitmapBytes + PaddingBytesAfterBitmapBytes;
+ uint64_t FileOffsetToBitmap =
+ FileOffsetToCounters + CountersSize + PaddingBytesAfterCounters;
+ void *BitmapMmap =
+ mmap((void *)BitmapBegin, PageAlignedBitmapLength, PROT_READ | PROT_WRITE,
+ MAP_FIXED | MAP_SHARED, Fileno, FileOffsetToBitmap);
+ if (BitmapMmap != BitmapBegin) {
+ PROF_ERR(
+ "Continuous counter sync mode is enabled, but mmap() failed (%s).\n"
+ " - BitmapBegin: %p\n"
+ " - PageAlignedBitmapLength: %" PRIu64 "\n"
+ " - Fileno: %d\n"
+ " - FileOffsetToBitmap: %" PRIu64 "\n",
+ strerror(errno), BitmapBegin, PageAlignedBitmapLength, Fileno,
+ FileOffsetToBitmap);
+ return 1;
+ }
+ return 0;
+}
+#elif defined(__ELF__) || defined(_WIN32) || defined(_AIX)
+
+#define INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR \
+ INSTR_PROF_CONCAT(INSTR_PROF_PROFILE_COUNTER_BIAS_VAR, _default)
+COMPILER_RT_VISIBILITY int64_t INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR = 0;
+#define INSTR_PROF_PROFILE_BITMAP_BIAS_DEFAULT_VAR \
+ INSTR_PROF_CONCAT(INSTR_PROF_PROFILE_BITMAP_BIAS_VAR, _default)
+COMPILER_RT_VISIBILITY int64_t INSTR_PROF_PROFILE_BITMAP_BIAS_DEFAULT_VAR = 0;
+
+/* This variable is a weak external reference which could be used to detect
+ * whether or not the compiler defined this symbol. */
+#if defined(_MSC_VER)
+COMPILER_RT_VISIBILITY extern int64_t INSTR_PROF_PROFILE_COUNTER_BIAS_VAR;
+COMPILER_RT_VISIBILITY extern int64_t INSTR_PROF_PROFILE_BITMAP_BIAS_VAR;
+#if defined(_M_IX86) || defined(__i386__)
+#define WIN_SYM_PREFIX "_"
+#else
+#define WIN_SYM_PREFIX
+#endif
+#pragma comment( \
+ linker, "/alternatename:" WIN_SYM_PREFIX INSTR_PROF_QUOTE( \
+ INSTR_PROF_PROFILE_COUNTER_BIAS_VAR) "=" WIN_SYM_PREFIX \
+ INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR))
+#pragma comment( \
+ linker, "/alternatename:" WIN_SYM_PREFIX INSTR_PROF_QUOTE( \
+ INSTR_PROF_PROFILE_BITMAP_BIAS_VAR) "=" WIN_SYM_PREFIX \
+ INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_BITMAP_BIAS_DEFAULT_VAR))
+#else
+COMPILER_RT_VISIBILITY extern int64_t INSTR_PROF_PROFILE_COUNTER_BIAS_VAR
+ __attribute__((weak, alias(INSTR_PROF_QUOTE(
+ INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR))));
+COMPILER_RT_VISIBILITY extern int64_t INSTR_PROF_PROFILE_BITMAP_BIAS_VAR
+ __attribute__((weak, alias(INSTR_PROF_QUOTE(
+ INSTR_PROF_PROFILE_BITMAP_BIAS_DEFAULT_VAR))));
+#endif
+static const int ContinuousModeSupported = 1;
+static const int UseBiasVar = 1;
+/* TODO: If there are two DSOs, the second DSO initilization will truncate the
+ * first profile file. */
+static const char *FileOpenMode = "w+b";
+/* This symbol is defined by the compiler when runtime counter relocation is
+ * used and runtime provides a weak alias so we can check if it's defined. */
+static void *BiasAddr = &INSTR_PROF_PROFILE_COUNTER_BIAS_VAR;
+static void *BiasDefaultAddr = &INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR;
+static void *BitmapBiasAddr = &INSTR_PROF_PROFILE_BITMAP_BIAS_VAR;
+static void *BitmapBiasDefaultAddr =
+ &INSTR_PROF_PROFILE_BITMAP_BIAS_DEFAULT_VAR;
+static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) {
+ /* Get the sizes of various profile data sections. Taken from
+ * __llvm_profile_get_size_for_buffer(). */
+ const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
+ const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
+ const char *CountersBegin = __llvm_profile_begin_counters();
+ const char *CountersEnd = __llvm_profile_end_counters();
+ const char *BitmapBegin = __llvm_profile_begin_bitmap();
+ const char *BitmapEnd = __llvm_profile_end_bitmap();
+ uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd);
+ uint64_t CountersSize =
+ __llvm_profile_get_counters_size(CountersBegin, CountersEnd);
+ uint64_t NumBitmapBytes =
+ __llvm_profile_get_num_bitmap_bytes(BitmapBegin, BitmapEnd);
+ /* Get the file size. */
+ uint64_t FileSize = 0;
+ if (getProfileFileSizeForMerging(File, &FileSize))
+ return 1;
+
+ int Fileno = fileno(File);
+ uint64_t PaddingBytesAfterCounters =
+ __llvm_profile_get_num_padding_bytes(CountersSize);
+ uint64_t FileOffsetToCounters =
+ sizeof(__llvm_profile_header) + __llvm_write_binary_ids(NULL) + DataSize;
+
+ /* Map the profile. */
+ char *Profile = (char *)mmap(NULL, FileSize, PROT_READ | PROT_WRITE,
+ MAP_SHARED, Fileno, 0);
+ if (Profile == MAP_FAILED) {
+ PROF_ERR("Unable to mmap profile: %s\n", strerror(errno));
+ return 1;
+ }
+ /* Update the profile fields based on the current mapping. */
+ INSTR_PROF_PROFILE_COUNTER_BIAS_VAR =
+ (intptr_t)Profile - (uintptr_t)CountersBegin + FileOffsetToCounters;
+
+ /* Return the memory allocated for counters to OS. */
+ lprofReleaseMemoryPagesToOS((uintptr_t)CountersBegin, (uintptr_t)CountersEnd);
+
+ /* Also mmap MCDC bitmap bytes. If there aren't any bitmap bytes, mmap()
+ * will fail with EINVAL. */
+ if (NumBitmapBytes == 0)
+ return 0;
+
+ /* Update profbm_bias. */
+ uint64_t FileOffsetToBitmap =
+ FileOffsetToCounters + CountersSize + PaddingBytesAfterCounters;
+ /* Update the profile fields based on the current mapping. */
+ INSTR_PROF_PROFILE_BITMAP_BIAS_VAR =
+ (uintptr_t)Profile - (uintptr_t)BitmapBegin + FileOffsetToBitmap;
+
+ /* Return the memory allocated for counters to OS. */
+ lprofReleaseMemoryPagesToOS((uintptr_t)BitmapBegin, (uintptr_t)BitmapEnd);
+ return 0;
+}
+#else
+static const int ContinuousModeSupported = 0;
+static const int UseBiasVar = 0;
+static const char *FileOpenMode = "a+b";
+static void *BiasAddr = NULL;
+static void *BiasDefaultAddr = NULL;
+static void *BitmapBiasAddr = NULL;
+static void *BitmapBiasDefaultAddr = NULL;
+static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) {
+ return 0;
+}
+#endif
+
+static int isProfileMergeRequested(void) { return ProfileMergeRequested; }
+static void setProfileMergeRequested(int EnableMerge) {
+ ProfileMergeRequested = EnableMerge;
+}
+
+static FILE *ProfileFile = NULL;
+static FILE *getProfileFile(void) { return ProfileFile; }
+static void setProfileFile(FILE *File) { ProfileFile = File; }
+
+static int getCurFilenameLength(void);
+static const char *getCurFilename(char *FilenameBuf, int ForceUseBuf);
+static unsigned doMerging(void) {
+ return lprofCurFilename.MergePoolSize || isProfileMergeRequested();
+}
+
+/* Return 1 if there is an error, otherwise return 0. */
+static uint32_t fileWriter(ProfDataWriter *This, ProfDataIOVec *IOVecs,
+ uint32_t NumIOVecs) {
+ uint32_t I;
+ FILE *File = (FILE *)This->WriterCtx;
+ char Zeroes[sizeof(uint64_t)] = {0};
+ for (I = 0; I < NumIOVecs; I++) {
+ if (IOVecs[I].Data) {
+ if (fwrite(IOVecs[I].Data, IOVecs[I].ElmSize, IOVecs[I].NumElm, File) !=
+ IOVecs[I].NumElm)
+ return 1;
+ } else if (IOVecs[I].UseZeroPadding) {
+ size_t BytesToWrite = IOVecs[I].ElmSize * IOVecs[I].NumElm;
+ while (BytesToWrite > 0) {
+ size_t PartialWriteLen =
+ (sizeof(uint64_t) > BytesToWrite) ? BytesToWrite : sizeof(uint64_t);
+ if (fwrite(Zeroes, sizeof(uint8_t), PartialWriteLen, File) !=
+ PartialWriteLen) {
+ return 1;
+ }
+ BytesToWrite -= PartialWriteLen;
+ }
+ } else {
+ if (fseek(File, IOVecs[I].ElmSize * IOVecs[I].NumElm, SEEK_CUR) == -1)
+ return 1;
+ }
+ }
+ return 0;
+}
+
+/* TODO: make buffer size controllable by an internal option, and compiler can pass the size
+ to runtime via a variable. */
+static uint32_t orderFileWriter(FILE *File, const uint32_t *DataStart) {
+ if (fwrite(DataStart, sizeof(uint32_t), INSTR_ORDER_FILE_BUFFER_SIZE, File) !=
+ INSTR_ORDER_FILE_BUFFER_SIZE)
+ return 1;
+ return 0;
+}
+
+static void initFileWriter(ProfDataWriter *This, FILE *File) {
+ This->Write = fileWriter;
+ This->WriterCtx = File;
+}
+
+COMPILER_RT_VISIBILITY ProfBufferIO *
+lprofCreateBufferIOInternal(void *File, uint32_t BufferSz) {
+ FreeHook = &free;
+ DynamicBufferIOBuffer = (uint8_t *)calloc(1, BufferSz);
+ VPBufferSize = BufferSz;
+ ProfDataWriter *fileWriter =
+ (ProfDataWriter *)calloc(1, sizeof(ProfDataWriter));
+ initFileWriter(fileWriter, File);
+ ProfBufferIO *IO = lprofCreateBufferIO(fileWriter);
+ IO->OwnFileWriter = 1;
+ return IO;
+}
+
+static void setupIOBuffer(void) {
+ const char *BufferSzStr = 0;
+ BufferSzStr = getenv("LLVM_VP_BUFFER_SIZE");
+ if (BufferSzStr && BufferSzStr[0]) {
+ VPBufferSize = atoi(BufferSzStr);
+ DynamicBufferIOBuffer = (uint8_t *)calloc(VPBufferSize, 1);
+ }
+}
+
+/* Get the size of the profile file. If there are any errors, print the
+ * message under the assumption that the profile is being read for merging
+ * purposes, and return -1. Otherwise return the file size in the inout param
+ * \p ProfileFileSize. */
+static int getProfileFileSizeForMerging(FILE *ProfileFile,
+ uint64_t *ProfileFileSize) {
+ if (fseek(ProfileFile, 0L, SEEK_END) == -1) {
+ PROF_ERR("Unable to merge profile data, unable to get size: %s\n",
+ strerror(errno));
+ return -1;
+ }
+ *ProfileFileSize = ftell(ProfileFile);
+
+ /* Restore file offset. */
+ if (fseek(ProfileFile, 0L, SEEK_SET) == -1) {
+ PROF_ERR("Unable to merge profile data, unable to rewind: %s\n",
+ strerror(errno));
+ return -1;
+ }
+
+ if (*ProfileFileSize > 0 &&
+ *ProfileFileSize < sizeof(__llvm_profile_header)) {
+ PROF_WARN("Unable to merge profile data: %s\n",
+ "source profile file is too small.");
+ return -1;
+ }
+ return 0;
+}
+
+/* mmap() \p ProfileFile for profile merging purposes, assuming that an
+ * exclusive lock is held on the file and that \p ProfileFileSize is the
+ * length of the file. Return the mmap'd buffer in the inout variable
+ * \p ProfileBuffer. Returns -1 on failure. On success, the caller is
+ * responsible for unmapping the mmap'd buffer in \p ProfileBuffer. */
+static int mmapProfileForMerging(FILE *ProfileFile, uint64_t ProfileFileSize,
+ char **ProfileBuffer) {
+ *ProfileBuffer = mmap(NULL, ProfileFileSize, PROT_READ, MAP_SHARED | MAP_FILE,
+ fileno(ProfileFile), 0);
+ if (*ProfileBuffer == MAP_FAILED) {
+ PROF_ERR("Unable to merge profile data, mmap failed: %s\n",
+ strerror(errno));
+ return -1;
+ }
+
+ if (__llvm_profile_check_compatibility(*ProfileBuffer, ProfileFileSize)) {
+ (void)munmap(*ProfileBuffer, ProfileFileSize);
+ PROF_WARN("Unable to merge profile data: %s\n",
+ "source profile file is not compatible.");
+ return -1;
+ }
+ return 0;
+}
+
+/* Read profile data in \c ProfileFile and merge with in-memory
+ profile counters. Returns -1 if there is fatal error, otheriwse
+ 0 is returned. Returning 0 does not mean merge is actually
+ performed. If merge is actually done, *MergeDone is set to 1.
+*/
+static int doProfileMerging(FILE *ProfileFile, int *MergeDone) {
+ uint64_t ProfileFileSize;
+ char *ProfileBuffer;
+
+ /* Get the size of the profile on disk. */
+ if (getProfileFileSizeForMerging(ProfileFile, &ProfileFileSize) == -1)
+ return -1;
+
+ /* Nothing to merge. */
+ if (!ProfileFileSize)
+ return 0;
+
+ /* mmap() the profile and check that it is compatible with the data in
+ * the current image. */
+ if (mmapProfileForMerging(ProfileFile, ProfileFileSize, &ProfileBuffer) == -1)
+ return -1;
+
+ /* Now start merging */
+ if (__llvm_profile_merge_from_buffer(ProfileBuffer, ProfileFileSize)) {
+ PROF_ERR("%s\n", "Invalid profile data to merge");
+ (void)munmap(ProfileBuffer, ProfileFileSize);
+ return -1;
+ }
+
+ // Truncate the file in case merging of value profile did not happen to
+ // prevent from leaving garbage data at the end of the profile file.
+ (void)COMPILER_RT_FTRUNCATE(ProfileFile,
+ __llvm_profile_get_size_for_buffer());
+
+ (void)munmap(ProfileBuffer, ProfileFileSize);
+ *MergeDone = 1;
+
+ return 0;
+}
+
+/* Create the directory holding the file, if needed. */
+static void createProfileDir(const char *Filename) {
+ size_t Length = strlen(Filename);
+ if (lprofFindFirstDirSeparator(Filename)) {
+ char *Copy = (char *)COMPILER_RT_ALLOCA(Length + 1);
+ strncpy(Copy, Filename, Length + 1);
+ __llvm_profile_recursive_mkdir(Copy);
+ }
+}
+
+/* Open the profile data for merging. It opens the file in r+b mode with
+ * file locking. If the file has content which is compatible with the
+ * current process, it also reads in the profile data in the file and merge
+ * it with in-memory counters. After the profile data is merged in memory,
+ * the original profile data is truncated and gets ready for the profile
+ * dumper. With profile merging enabled, each executable as well as any of
+ * its instrumented shared libraries dump profile data into their own data file.
+*/
+static FILE *openFileForMerging(const char *ProfileFileName, int *MergeDone) {
+ FILE *ProfileFile = getProfileFile();
+ int rc;
+ // initializeProfileForContinuousMode will lock the profile, but if
+ // ProfileFile is set by user via __llvm_profile_set_file_object, it's assumed
+ // unlocked at this point.
+ if (ProfileFile && !__llvm_profile_is_continuous_mode_enabled()) {
+ lprofLockFileHandle(ProfileFile);
+ }
+ if (!ProfileFile) {
+ createProfileDir(ProfileFileName);
+ ProfileFile = lprofOpenFileEx(ProfileFileName);
+ }
+ if (!ProfileFile)
+ return NULL;
+
+ rc = doProfileMerging(ProfileFile, MergeDone);
+ if (rc || (!*MergeDone && COMPILER_RT_FTRUNCATE(ProfileFile, 0L)) ||
+ fseek(ProfileFile, 0L, SEEK_SET) == -1) {
+ PROF_ERR("Profile Merging of file %s failed: %s\n", ProfileFileName,
+ strerror(errno));
+ fclose(ProfileFile);
+ return NULL;
+ }
+ return ProfileFile;
+}
+
+static FILE *getFileObject(const char *OutputName) {
+ FILE *File;
+ File = getProfileFile();
+ if (File != NULL) {
+ return File;
+ }
+
+ return fopen(OutputName, "ab");
+}
+
+/* Write profile data to file \c OutputName. */
+static int writeFile(const char *OutputName) {
+ int RetVal;
+ FILE *OutputFile;
+
+ int MergeDone = 0;
+ VPMergeHook = &lprofMergeValueProfData;
+ if (doMerging())
+ OutputFile = openFileForMerging(OutputName, &MergeDone);
+ else
+ OutputFile = getFileObject(OutputName);
+
+ if (!OutputFile)
+ return -1;
+
+ FreeHook = &free;
+ setupIOBuffer();
+ ProfDataWriter fileWriter;
+ initFileWriter(&fileWriter, OutputFile);
+ RetVal = lprofWriteData(&fileWriter, lprofGetVPDataReader(), MergeDone);
+
+ if (OutputFile == getProfileFile()) {
+ fflush(OutputFile);
+ if (doMerging() && !__llvm_profile_is_continuous_mode_enabled()) {
+ lprofUnlockFileHandle(OutputFile);
+ }
+ } else {
+ fclose(OutputFile);
+ }
+
+ return RetVal;
+}
+
+/* Write order data to file \c OutputName. */
+static int writeOrderFile(const char *OutputName) {
+ int RetVal;
+ FILE *OutputFile;
+
+ OutputFile = fopen(OutputName, "w");
+
+ if (!OutputFile) {
+ PROF_WARN("can't open file with mode ab: %s\n", OutputName);
+ return -1;
+ }
+
+ FreeHook = &free;
+ setupIOBuffer();
+ const uint32_t *DataBegin = __llvm_profile_begin_orderfile();
+ RetVal = orderFileWriter(OutputFile, DataBegin);
+
+ fclose(OutputFile);
+ return RetVal;
+}
+
+#define LPROF_INIT_ONCE_ENV "__LLVM_PROFILE_RT_INIT_ONCE"
+
+static void truncateCurrentFile(void) {
+ const char *Filename;
+ char *FilenameBuf;
+ FILE *File;
+ int Length;
+
+ Length = getCurFilenameLength();
+ FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1);
+ Filename = getCurFilename(FilenameBuf, 0);
+ if (!Filename)
+ return;
+
+ /* Only create the profile directory and truncate an existing profile once.
+ * In continuous mode, this is necessary, as the profile is written-to by the
+ * runtime initializer. */
+ int initialized = getenv(LPROF_INIT_ONCE_ENV) != NULL;
+ if (initialized)
+ return;
+#if defined(_WIN32)
+ _putenv(LPROF_INIT_ONCE_ENV "=" LPROF_INIT_ONCE_ENV);
+#else
+ setenv(LPROF_INIT_ONCE_ENV, LPROF_INIT_ONCE_ENV, 1);
+#endif
+
+ /* Create the profile dir (even if online merging is enabled), so that
+ * the profile file can be set up if continuous mode is enabled. */
+ createProfileDir(Filename);
+
+ /* By pass file truncation to allow online raw profile merging. */
+ if (lprofCurFilename.MergePoolSize)
+ return;
+
+ /* Truncate the file. Later we'll reopen and append. */
+ File = fopen(Filename, "w");
+ if (!File)
+ return;
+ fclose(File);
+}
+
+/* Write a partial profile to \p Filename, which is required to be backed by
+ * the open file object \p File. */
+static int writeProfileWithFileObject(const char *Filename, FILE *File) {
+ setProfileFile(File);
+ int rc = writeFile(Filename);
+ if (rc)
+ PROF_ERR("Failed to write file \"%s\": %s\n", Filename, strerror(errno));
+ setProfileFile(NULL);
+ return rc;
+}
+
+static void initializeProfileForContinuousMode(void) {
+ if (!__llvm_profile_is_continuous_mode_enabled())
+ return;
+ if (!ContinuousModeSupported) {
+ PROF_ERR("%s\n", "continuous mode is unsupported on this platform");
+ return;
+ }
+ if (UseBiasVar && BiasAddr == BiasDefaultAddr &&
+ BitmapBiasAddr == BitmapBiasDefaultAddr) {
+ PROF_ERR("%s\n", "Neither __llvm_profile_counter_bias nor "
+ "__llvm_profile_bitmap_bias is defined");
+ return;
+ }
+
+ /* Get the sizes of counter section. */
+ uint64_t CountersSize = __llvm_profile_get_counters_size(
+ __llvm_profile_begin_counters(), __llvm_profile_end_counters());
+
+ int Length = getCurFilenameLength();
+ char *FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1);
+ const char *Filename = getCurFilename(FilenameBuf, 0);
+ if (!Filename)
+ return;
+
+ FILE *File = NULL;
+ uint64_t CurrentFileOffset = 0;
+ if (doMerging()) {
+ /* We are merging profiles. Map the counter section as shared memory into
+ * the profile, i.e. into each participating process. An increment in one
+ * process should be visible to every other process with the same counter
+ * section mapped. */
+ File = lprofOpenFileEx(Filename);
+ if (!File)
+ return;
+
+ uint64_t ProfileFileSize = 0;
+ if (getProfileFileSizeForMerging(File, &ProfileFileSize) == -1) {
+ lprofUnlockFileHandle(File);
+ fclose(File);
+ return;
+ }
+ if (ProfileFileSize == 0) {
+ /* Grow the profile so that mmap() can succeed. Leak the file handle, as
+ * the file should stay open. */
+ if (writeProfileWithFileObject(Filename, File) != 0) {
+ lprofUnlockFileHandle(File);
+ fclose(File);
+ return;
+ }
+ } else {
+ /* The merged profile has a non-zero length. Check that it is compatible
+ * with the data in this process. */
+ char *ProfileBuffer;
+ if (mmapProfileForMerging(File, ProfileFileSize, &ProfileBuffer) == -1) {
+ lprofUnlockFileHandle(File);
+ fclose(File);
+ return;
+ }
+ (void)munmap(ProfileBuffer, ProfileFileSize);
+ }
+ } else {
+ File = fopen(Filename, FileOpenMode);
+ if (!File)
+ return;
+ /* Check that the offset within the file is page-aligned. */
+ CurrentFileOffset = ftell(File);
+ unsigned PageSize = getpagesize();
+ if (CurrentFileOffset % PageSize != 0) {
+ PROF_ERR("Continuous counter sync mode is enabled, but raw profile is not"
+ "page-aligned. CurrentFileOffset = %" PRIu64 ", pagesz = %u.\n",
+ (uint64_t)CurrentFileOffset, PageSize);
+ fclose(File);
+ return;
+ }
+ if (writeProfileWithFileObject(Filename, File) != 0) {
+ fclose(File);
+ return;
+ }
+ }
+
+ /* mmap() the profile counters so long as there is at least one counter.
+ * If there aren't any counters, mmap() would fail with EINVAL. */
+ if (CountersSize > 0)
+ mmapForContinuousMode(CurrentFileOffset, File);
+
+ if (doMerging()) {
+ lprofUnlockFileHandle(File);
+ }
+ if (File != NULL) {
+ fclose(File);
+ }
+}
+
+static const char *DefaultProfileName = "default.profraw";
+static void resetFilenameToDefault(void) {
+ if (lprofCurFilename.FilenamePat && lprofCurFilename.OwnsFilenamePat) {
+#ifdef __GNUC__
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wcast-qual"
+#elif defined(__clang__)
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wcast-qual"
+#endif
+ free((void *)lprofCurFilename.FilenamePat);
+#ifdef __GNUC__
+#pragma GCC diagnostic pop
+#elif defined(__clang__)
+#pragma clang diagnostic pop
+#endif
+ }
+ memset(&lprofCurFilename, 0, sizeof(lprofCurFilename));
+ lprofCurFilename.FilenamePat = DefaultProfileName;
+ lprofCurFilename.PNS = PNS_default;
+}
+
+static unsigned getMergePoolSize(const char *FilenamePat, int *I) {
+ unsigned J = 0, Num = 0;
+ for (;; ++J) {
+ char C = FilenamePat[*I + J];
+ if (C == 'm') {
+ *I += J;
+ return Num ? Num : 1;
+ }
+ if (C < '0' || C > '9')
+ break;
+ Num = Num * 10 + C - '0';
+
+ /* If FilenamePat[*I+J] is between '0' and '9', the next byte is guaranteed
+ * to be in-bound as the string is null terminated. */
+ }
+ return 0;
+}
+
+/* Assert that Idx does index past a string null terminator. Return the
+ * result of the check. */
+static int checkBounds(int Idx, int Strlen) {
+ assert(Idx <= Strlen && "Indexing past string null terminator");
+ return Idx <= Strlen;
+}
+
+/* Parses the pattern string \p FilenamePat and stores the result to
+ * lprofcurFilename structure. */
+static int parseFilenamePattern(const char *FilenamePat,
+ unsigned CopyFilenamePat) {
+ int NumPids = 0, NumHosts = 0, I;
+ char *PidChars = &lprofCurFilename.PidChars[0];
+ char *Hostname = &lprofCurFilename.Hostname[0];
+ int MergingEnabled = 0;
+ int FilenamePatLen = strlen(FilenamePat);
+
+#ifdef __GNUC__
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wcast-qual"
+#elif defined(__clang__)
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wcast-qual"
+#endif
+ /* Clean up cached prefix and filename. */
+ if (lprofCurFilename.ProfilePathPrefix)
+ free((void *)lprofCurFilename.ProfilePathPrefix);
+
+ if (lprofCurFilename.FilenamePat && lprofCurFilename.OwnsFilenamePat) {
+ free((void *)lprofCurFilename.FilenamePat);
+ }
+#ifdef __GNUC__
+#pragma GCC diagnostic pop
+#elif defined(__clang__)
+#pragma clang diagnostic pop
+#endif
+
+ memset(&lprofCurFilename, 0, sizeof(lprofCurFilename));
+
+ if (!CopyFilenamePat)
+ lprofCurFilename.FilenamePat = FilenamePat;
+ else {
+ lprofCurFilename.FilenamePat = strdup(FilenamePat);
+ lprofCurFilename.OwnsFilenamePat = 1;
+ }
+ /* Check the filename for "%p", which indicates a pid-substitution. */
+ for (I = 0; checkBounds(I, FilenamePatLen) && FilenamePat[I]; ++I) {
+ if (FilenamePat[I] == '%') {
+ ++I; /* Advance to the next character. */
+ if (!checkBounds(I, FilenamePatLen))
+ break;
+ if (FilenamePat[I] == 'p') {
+ if (!NumPids++) {
+ if (snprintf(PidChars, MAX_PID_SIZE, "%ld", (long)getpid()) <= 0) {
+ PROF_WARN("Unable to get pid for filename pattern %s. Using the "
+ "default name.",
+ FilenamePat);
+ return -1;
+ }
+ }
+ } else if (FilenamePat[I] == 'h') {
+ if (!NumHosts++)
+ if (COMPILER_RT_GETHOSTNAME(Hostname, COMPILER_RT_MAX_HOSTLEN)) {
+ PROF_WARN("Unable to get hostname for filename pattern %s. Using "
+ "the default name.",
+ FilenamePat);
+ return -1;
+ }
+ } else if (FilenamePat[I] == 't') {
+ lprofCurFilename.TmpDir = getenv("TMPDIR");
+ if (!lprofCurFilename.TmpDir) {
+ PROF_WARN("Unable to get the TMPDIR environment variable, referenced "
+ "in %s. Using the default path.",
+ FilenamePat);
+ return -1;
+ }
+ } else if (FilenamePat[I] == 'c') {
+ if (__llvm_profile_is_continuous_mode_enabled()) {
+ PROF_WARN("%%c specifier can only be specified once in %s.\n",
+ FilenamePat);
+ __llvm_profile_disable_continuous_mode();
+ return -1;
+ }
+#if defined(__APPLE__) || defined(__ELF__) || defined(_WIN32) || defined(_AIX)
+ __llvm_profile_set_page_size(getpagesize());
+ __llvm_profile_enable_continuous_mode();
+#else
+ PROF_WARN("%s", "Continous mode is currently only supported for Mach-O,"
+ " ELF and COFF formats.");
+ return -1;
+#endif
+ } else {
+ unsigned MergePoolSize = getMergePoolSize(FilenamePat, &I);
+ if (!MergePoolSize)
+ continue;
+ if (MergingEnabled) {
+ PROF_WARN("%%m specifier can only be specified once in %s.\n",
+ FilenamePat);
+ return -1;
+ }
+ MergingEnabled = 1;
+ lprofCurFilename.MergePoolSize = MergePoolSize;
+ }
+ }
+ }
+
+ lprofCurFilename.NumPids = NumPids;
+ lprofCurFilename.NumHosts = NumHosts;
+ return 0;
+}
+
+static void parseAndSetFilename(const char *FilenamePat,
+ ProfileNameSpecifier PNS,
+ unsigned CopyFilenamePat) {
+
+ const char *OldFilenamePat = lprofCurFilename.FilenamePat;
+ ProfileNameSpecifier OldPNS = lprofCurFilename.PNS;
+
+ /* The old profile name specifier takes precedence over the old one. */
+ if (PNS < OldPNS)
+ return;
+
+ if (!FilenamePat)
+ FilenamePat = DefaultProfileName;
+
+ if (OldFilenamePat && !strcmp(OldFilenamePat, FilenamePat)) {
+ lprofCurFilename.PNS = PNS;
+ return;
+ }
+
+ /* When PNS >= OldPNS, the last one wins. */
+ if (!FilenamePat || parseFilenamePattern(FilenamePat, CopyFilenamePat))
+ resetFilenameToDefault();
+ lprofCurFilename.PNS = PNS;
+
+ if (!OldFilenamePat) {
+ if (getenv("LLVM_PROFILE_VERBOSE"))
+ PROF_NOTE("Set profile file path to \"%s\" via %s.\n",
+ lprofCurFilename.FilenamePat, getPNSStr(PNS));
+ } else {
+ if (getenv("LLVM_PROFILE_VERBOSE"))
+ PROF_NOTE("Override old profile path \"%s\" via %s to \"%s\" via %s.\n",
+ OldFilenamePat, getPNSStr(OldPNS), lprofCurFilename.FilenamePat,
+ getPNSStr(PNS));
+ }
+
+ truncateCurrentFile();
+ if (__llvm_profile_is_continuous_mode_enabled())
+ initializeProfileForContinuousMode();
+}
+
+/* Return buffer length that is required to store the current profile
+ * filename with PID and hostname substitutions. */
+/* The length to hold uint64_t followed by 3 digits pool id including '_' */
+#define SIGLEN 24
+static int getCurFilenameLength(void) {
+ int Len;
+ if (!lprofCurFilename.FilenamePat || !lprofCurFilename.FilenamePat[0])
+ return 0;
+
+ if (!(lprofCurFilename.NumPids || lprofCurFilename.NumHosts ||
+ lprofCurFilename.TmpDir || lprofCurFilename.MergePoolSize))
+ return strlen(lprofCurFilename.FilenamePat);
+
+ Len = strlen(lprofCurFilename.FilenamePat) +
+ lprofCurFilename.NumPids * (strlen(lprofCurFilename.PidChars) - 2) +
+ lprofCurFilename.NumHosts * (strlen(lprofCurFilename.Hostname) - 2) +
+ (lprofCurFilename.TmpDir ? (strlen(lprofCurFilename.TmpDir) - 1) : 0);
+ if (lprofCurFilename.MergePoolSize)
+ Len += SIGLEN;
+ return Len;
+}
+
+/* Return the pointer to the current profile file name (after substituting
+ * PIDs and Hostnames in filename pattern. \p FilenameBuf is the buffer
+ * to store the resulting filename. If no substitution is needed, the
+ * current filename pattern string is directly returned, unless ForceUseBuf
+ * is enabled. */
+static const char *getCurFilename(char *FilenameBuf, int ForceUseBuf) {
+ int I, J, PidLength, HostNameLength, TmpDirLength, FilenamePatLength;
+ const char *FilenamePat = lprofCurFilename.FilenamePat;
+
+ if (!lprofCurFilename.FilenamePat || !lprofCurFilename.FilenamePat[0])
+ return 0;
+
+ if (!(lprofCurFilename.NumPids || lprofCurFilename.NumHosts ||
+ lprofCurFilename.TmpDir || lprofCurFilename.MergePoolSize ||
+ __llvm_profile_is_continuous_mode_enabled())) {
+ if (!ForceUseBuf)
+ return lprofCurFilename.FilenamePat;
+
+ FilenamePatLength = strlen(lprofCurFilename.FilenamePat);
+ memcpy(FilenameBuf, lprofCurFilename.FilenamePat, FilenamePatLength);
+ FilenameBuf[FilenamePatLength] = '\0';
+ return FilenameBuf;
+ }
+
+ PidLength = strlen(lprofCurFilename.PidChars);
+ HostNameLength = strlen(lprofCurFilename.Hostname);
+ TmpDirLength = lprofCurFilename.TmpDir ? strlen(lprofCurFilename.TmpDir) : 0;
+ /* Construct the new filename. */
+ for (I = 0, J = 0; FilenamePat[I]; ++I)
+ if (FilenamePat[I] == '%') {
+ if (FilenamePat[++I] == 'p') {
+ memcpy(FilenameBuf + J, lprofCurFilename.PidChars, PidLength);
+ J += PidLength;
+ } else if (FilenamePat[I] == 'h') {
+ memcpy(FilenameBuf + J, lprofCurFilename.Hostname, HostNameLength);
+ J += HostNameLength;
+ } else if (FilenamePat[I] == 't') {
+ memcpy(FilenameBuf + J, lprofCurFilename.TmpDir, TmpDirLength);
+ FilenameBuf[J + TmpDirLength] = DIR_SEPARATOR;
+ J += TmpDirLength + 1;
+ } else {
+ if (!getMergePoolSize(FilenamePat, &I))
+ continue;
+ char LoadModuleSignature[SIGLEN + 1];
+ int S;
+ int ProfilePoolId = getpid() % lprofCurFilename.MergePoolSize;
+ S = snprintf(LoadModuleSignature, SIGLEN + 1, "%" PRIu64 "_%d",
+ lprofGetLoadModuleSignature(), ProfilePoolId);
+ if (S == -1 || S > SIGLEN)
+ S = SIGLEN;
+ memcpy(FilenameBuf + J, LoadModuleSignature, S);
+ J += S;
+ }
+ /* Drop any unknown substitutions. */
+ } else
+ FilenameBuf[J++] = FilenamePat[I];
+ FilenameBuf[J] = 0;
+
+ return FilenameBuf;
+}
+
+/* Returns the pointer to the environment variable
+ * string. Returns null if the env var is not set. */
+static const char *getFilenamePatFromEnv(void) {
+ const char *Filename = getenv("LLVM_PROFILE_FILE");
+ if (!Filename || !Filename[0])
+ return 0;
+ return Filename;
+}
+
+COMPILER_RT_VISIBILITY
+const char *__llvm_profile_get_path_prefix(void) {
+ int Length;
+ char *FilenameBuf, *Prefix;
+ const char *Filename, *PrefixEnd;
+
+ if (lprofCurFilename.ProfilePathPrefix)
+ return lprofCurFilename.ProfilePathPrefix;
+
+ Length = getCurFilenameLength();
+ FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1);
+ Filename = getCurFilename(FilenameBuf, 0);
+ if (!Filename)
+ return "\0";
+
+ PrefixEnd = lprofFindLastDirSeparator(Filename);
+ if (!PrefixEnd)
+ return "\0";
+
+ Length = PrefixEnd - Filename + 1;
+ Prefix = (char *)malloc(Length + 1);
+ if (!Prefix) {
+ PROF_ERR("Failed to %s\n", "allocate memory.");
+ return "\0";
+ }
+ memcpy(Prefix, Filename, Length);
+ Prefix[Length] = '\0';
+ lprofCurFilename.ProfilePathPrefix = Prefix;
+ return Prefix;
+}
+
+COMPILER_RT_VISIBILITY
+const char *__llvm_profile_get_filename(void) {
+ int Length;
+ char *FilenameBuf;
+ const char *Filename;
+
+ Length = getCurFilenameLength();
+ FilenameBuf = (char *)malloc(Length + 1);
+ if (!FilenameBuf) {
+ PROF_ERR("Failed to %s\n", "allocate memory.");
+ return "\0";
+ }
+ Filename = getCurFilename(FilenameBuf, 1);
+ if (!Filename)
+ return "\0";
+
+ return FilenameBuf;
+}
+
+/* This API initializes the file handling, both user specified
+ * profile path via -fprofile-instr-generate= and LLVM_PROFILE_FILE
+ * environment variable can override this default value.
+ */
+COMPILER_RT_VISIBILITY
+void __llvm_profile_initialize_file(void) {
+ const char *EnvFilenamePat;
+ const char *SelectedPat = NULL;
+ ProfileNameSpecifier PNS = PNS_unknown;
+ int hasCommandLineOverrider = (INSTR_PROF_PROFILE_NAME_VAR[0] != 0);
+
+ EnvFilenamePat = getFilenamePatFromEnv();
+ if (EnvFilenamePat) {
+ /* Pass CopyFilenamePat = 1, to ensure that the filename would be valid
+ at the moment when __llvm_profile_write_file() gets executed. */
+ parseAndSetFilename(EnvFilenamePat, PNS_environment, 1);
+ return;
+ } else if (hasCommandLineOverrider) {
+ SelectedPat = INSTR_PROF_PROFILE_NAME_VAR;
+ PNS = PNS_command_line;
+ } else {
+ SelectedPat = NULL;
+ PNS = PNS_default;
+ }
+
+ parseAndSetFilename(SelectedPat, PNS, 0);
+}
+
+/* This method is invoked by the runtime initialization hook
+ * InstrProfilingRuntime.o if it is linked in.
+ */
+COMPILER_RT_VISIBILITY
+void __llvm_profile_initialize(void) {
+ __llvm_profile_initialize_file();
+ if (!__llvm_profile_is_continuous_mode_enabled())
+ __llvm_profile_register_write_file_atexit();
+}
+
+/* This API is directly called by the user application code. It has the
+ * highest precedence compared with LLVM_PROFILE_FILE environment variable
+ * and command line option -fprofile-instr-generate=<profile_name>.
+ */
+COMPILER_RT_VISIBILITY
+void __llvm_profile_set_filename(const char *FilenamePat) {
+ if (__llvm_profile_is_continuous_mode_enabled())
+ return;
+ parseAndSetFilename(FilenamePat, PNS_runtime_api, 1);
+}
+
+/* The public API for writing profile data into the file with name
+ * set by previous calls to __llvm_profile_set_filename or
+ * __llvm_profile_override_default_filename or
+ * __llvm_profile_initialize_file. */
+COMPILER_RT_VISIBILITY
+int __llvm_profile_write_file(void) {
+ int rc, Length;
+ const char *Filename;
+ char *FilenameBuf;
+
+ // Temporarily suspend getting SIGKILL when the parent exits.
+ int PDeathSig = lprofSuspendSigKill();
+
+ if (lprofProfileDumped() || __llvm_profile_is_continuous_mode_enabled()) {
+ PROF_NOTE("Profile data not written to file: %s.\n", "already written");
+ if (PDeathSig == 1)
+ lprofRestoreSigKill();
+ return 0;
+ }
+
+ Length = getCurFilenameLength();
+ FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1);
+ Filename = getCurFilename(FilenameBuf, 0);
+
+ /* Check the filename. */
+ if (!Filename) {
+ PROF_ERR("Failed to write file : %s\n", "Filename not set");
+ if (PDeathSig == 1)
+ lprofRestoreSigKill();
+ return -1;
+ }
+
+ /* Check if there is llvm/runtime version mismatch. */
+ if (GET_VERSION(__llvm_profile_get_version()) != INSTR_PROF_RAW_VERSION) {
+ PROF_ERR("Runtime and instrumentation version mismatch : "
+ "expected %d, but get %d\n",
+ INSTR_PROF_RAW_VERSION,
+ (int)GET_VERSION(__llvm_profile_get_version()));
+ if (PDeathSig == 1)
+ lprofRestoreSigKill();
+ return -1;
+ }
+
+ /* Write profile data to the file. */
+ rc = writeFile(Filename);
+ if (rc)
+ PROF_ERR("Failed to write file \"%s\": %s\n", Filename, strerror(errno));
+
+ // Restore SIGKILL.
+ if (PDeathSig == 1)
+ lprofRestoreSigKill();
+
+ return rc;
+}
+
+COMPILER_RT_VISIBILITY
+int __llvm_profile_dump(void) {
+ if (!doMerging())
+ PROF_WARN("Later invocation of __llvm_profile_dump can lead to clobbering "
+ " of previously dumped profile data : %s. Either use %%m "
+ "in profile name or change profile name before dumping.\n",
+ "online profile merging is not on");
+ int rc = __llvm_profile_write_file();
+ lprofSetProfileDumped(1);
+ return rc;
+}
+
+/* Order file data will be saved in a file with suffx .order. */
+static const char *OrderFileSuffix = ".order";
+
+COMPILER_RT_VISIBILITY
+int __llvm_orderfile_write_file(void) {
+ int rc, Length, LengthBeforeAppend, SuffixLength;
+ const char *Filename;
+ char *FilenameBuf;
+
+ // Temporarily suspend getting SIGKILL when the parent exits.
+ int PDeathSig = lprofSuspendSigKill();
+
+ SuffixLength = strlen(OrderFileSuffix);
+ Length = getCurFilenameLength() + SuffixLength;
+ FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1);
+ Filename = getCurFilename(FilenameBuf, 1);
+
+ /* Check the filename. */
+ if (!Filename) {
+ PROF_ERR("Failed to write file : %s\n", "Filename not set");
+ if (PDeathSig == 1)
+ lprofRestoreSigKill();
+ return -1;
+ }
+
+ /* Append order file suffix */
+ LengthBeforeAppend = strlen(Filename);
+ memcpy(FilenameBuf + LengthBeforeAppend, OrderFileSuffix, SuffixLength);
+ FilenameBuf[LengthBeforeAppend + SuffixLength] = '\0';
+
+ /* Check if there is llvm/runtime version mismatch. */
+ if (GET_VERSION(__llvm_profile_get_version()) != INSTR_PROF_RAW_VERSION) {
+ PROF_ERR("Runtime and instrumentation version mismatch : "
+ "expected %d, but get %d\n",
+ INSTR_PROF_RAW_VERSION,
+ (int)GET_VERSION(__llvm_profile_get_version()));
+ if (PDeathSig == 1)
+ lprofRestoreSigKill();
+ return -1;
+ }
+
+ /* Write order data to the file. */
+ rc = writeOrderFile(Filename);
+ if (rc)
+ PROF_ERR("Failed to write file \"%s\": %s\n", Filename, strerror(errno));
+
+ // Restore SIGKILL.
+ if (PDeathSig == 1)
+ lprofRestoreSigKill();
+
+ return rc;
+}
+
+COMPILER_RT_VISIBILITY
+int __llvm_orderfile_dump(void) {
+ int rc = __llvm_orderfile_write_file();
+ return rc;
+}
+
+static void writeFileWithoutReturn(void) { __llvm_profile_write_file(); }
+
+COMPILER_RT_VISIBILITY
+int __llvm_profile_register_write_file_atexit(void) {
+ static int HasBeenRegistered = 0;
+
+ if (HasBeenRegistered)
+ return 0;
+
+ lprofSetupValueProfiler();
+
+ HasBeenRegistered = 1;
+ return lprofAtExit(writeFileWithoutReturn);
+}
+
+COMPILER_RT_VISIBILITY int __llvm_profile_set_file_object(FILE *File,
+ int EnableMerge) {
+ if (__llvm_profile_is_continuous_mode_enabled()) {
+ if (!EnableMerge) {
+ PROF_WARN("__llvm_profile_set_file_object(fd=%d) not supported in "
+ "continuous sync mode when merging is disabled\n",
+ fileno(File));
+ return 1;
+ }
+ if (lprofLockFileHandle(File) != 0) {
+ PROF_WARN("Data may be corrupted during profile merging : %s\n",
+ "Fail to obtain file lock due to system limit.");
+ }
+ uint64_t ProfileFileSize = 0;
+ if (getProfileFileSizeForMerging(File, &ProfileFileSize) == -1) {
+ lprofUnlockFileHandle(File);
+ return 1;
+ }
+ if (ProfileFileSize == 0) {
+ FreeHook = &free;
+ setupIOBuffer();
+ ProfDataWriter fileWriter;
+ initFileWriter(&fileWriter, File);
+ if (lprofWriteData(&fileWriter, 0, 0)) {
+ lprofUnlockFileHandle(File);
+ PROF_ERR("Failed to write file \"%d\": %s\n", fileno(File),
+ strerror(errno));
+ return 1;
+ }
+ fflush(File);
+ } else {
+ /* The merged profile has a non-zero length. Check that it is compatible
+ * with the data in this process. */
+ char *ProfileBuffer;
+ if (mmapProfileForMerging(File, ProfileFileSize, &ProfileBuffer) == -1) {
+ lprofUnlockFileHandle(File);
+ return 1;
+ }
+ (void)munmap(ProfileBuffer, ProfileFileSize);
+ }
+ mmapForContinuousMode(0, File);
+ lprofUnlockFileHandle(File);
+ } else {
+ setProfileFile(File);
+ setProfileMergeRequested(EnableMerge);
+ }
+ return 0;
+}
+
+#endif
diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingInternal.c b/system/lib/compiler-rt/lib/profile/InstrProfilingInternal.c
new file mode 100644
index 0000000000000..3dd659f905102
--- /dev/null
+++ b/system/lib/compiler-rt/lib/profile/InstrProfilingInternal.c
@@ -0,0 +1,26 @@
+/*===- InstrProfilingInternal.c - Support library for PGO instrumentation -===*\
+|*
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+|* See https://llvm.org/LICENSE.txt for license information.
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+|*
+\*===----------------------------------------------------------------------===*/
+
+// Note: This is linked into the Darwin kernel, and must remain compatible
+// with freestanding compilation. See `darwin_add_builtin_libraries`.
+
+#if !defined(__Fuchsia__)
+
+#include "InstrProfilingInternal.h"
+
+static unsigned ProfileDumped = 0;
+
+COMPILER_RT_VISIBILITY unsigned lprofProfileDumped(void) {
+ return ProfileDumped;
+}
+
+COMPILER_RT_VISIBILITY void lprofSetProfileDumped(unsigned Value) {
+ ProfileDumped = Value;
+}
+
+#endif
diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingInternal.h b/system/lib/compiler-rt/lib/profile/InstrProfilingInternal.h
new file mode 100644
index 0000000000000..b100343ca04f9
--- /dev/null
+++ b/system/lib/compiler-rt/lib/profile/InstrProfilingInternal.h
@@ -0,0 +1,215 @@
+/*===- InstrProfiling.h- Support library for PGO instrumentation ----------===*\
+|*
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+|* See https://llvm.org/LICENSE.txt for license information.
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+|*
+\*===----------------------------------------------------------------------===*/
+
+#ifndef PROFILE_INSTRPROFILING_INTERNALH_
+#define PROFILE_INSTRPROFILING_INTERNALH_
+
+#include <stddef.h>
+
+#include "InstrProfiling.h"
+
+/*!
+ * \brief Write instrumentation data to the given buffer, given explicit
+ * pointers to the live data in memory. This function is probably not what you
+ * want. Use __llvm_profile_get_size_for_buffer instead. Use this function if
+ * your program has a custom memory layout.
+ */
+uint64_t __llvm_profile_get_size_for_buffer_internal(
+ const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd,
+ const char *CountersBegin, const char *CountersEnd, const char *BitmapBegin,
+ const char *BitmapEnd, const char *NamesBegin, const char *NamesEnd,
+ const VTableProfData *VTableBegin, const VTableProfData *VTableEnd,
+ const char *VNamesBegin, const char *VNamesEnd);
+
+/*!
+ * \brief Write instrumentation data to the given buffer, given explicit
+ * pointers to the live data in memory. This function is probably not what you
+ * want. Use __llvm_profile_write_buffer instead. Use this function if your
+ * program has a custom memory layout.
+ *
+ * \pre \c Buffer is the start of a buffer at least as big as \a
+ * __llvm_profile_get_size_for_buffer_internal().
+ */
+int __llvm_profile_write_buffer_internal(
+ char *Buffer, const __llvm_profile_data *DataBegin,
+ const __llvm_profile_data *DataEnd, const char *CountersBegin,
+ const char *CountersEnd, const char *BitmapBegin, const char *BitmapEnd,
+ const char *NamesBegin, const char *NamesEnd);
+
+/*!
+ * The data structure describing the data to be written by the
+ * low level writer callback function.
+ *
+ * If \ref ProfDataIOVec.Data is null, and \ref ProfDataIOVec.UseZeroPadding is
+ * 0, the write is skipped (the writer simply advances ElmSize*NumElm bytes).
+ *
+ * If \ref ProfDataIOVec.Data is null, and \ref ProfDataIOVec.UseZeroPadding is
+ * nonzero, ElmSize*NumElm zero bytes are written.
+ */
+typedef struct ProfDataIOVec {
+ const void *Data;
+ size_t ElmSize;
+ size_t NumElm;
+ int UseZeroPadding;
+} ProfDataIOVec;
+
+struct ProfDataWriter;
+typedef uint32_t (*WriterCallback)(struct ProfDataWriter *This, ProfDataIOVec *,
+ uint32_t NumIOVecs);
+
+typedef struct ProfDataWriter {
+ WriterCallback Write;
+ void *WriterCtx;
+} ProfDataWriter;
+
+/*!
+ * The data structure for buffered IO of profile data.
+ */
+typedef struct ProfBufferIO {
+ ProfDataWriter *FileWriter;
+ uint32_t OwnFileWriter;
+ /* The start of the buffer. */
+ uint8_t *BufferStart;
+ /* Total size of the buffer. */
+ uint32_t BufferSz;
+ /* Current byte offset from the start of the buffer. */
+ uint32_t CurOffset;
+} ProfBufferIO;
+
+/* The creator interface used by testing. */
+ProfBufferIO *lprofCreateBufferIOInternal(void *File, uint32_t BufferSz);
+
+/*!
+ * This is the interface to create a handle for buffered IO.
+ */
+ProfBufferIO *lprofCreateBufferIO(ProfDataWriter *FileWriter);
+
+/*!
+ * The interface to destroy the bufferIO handle and reclaim
+ * the memory.
+ */
+void lprofDeleteBufferIO(ProfBufferIO *BufferIO);
+
+/*!
+ * This is the interface to write \c Data of \c Size bytes through
+ * \c BufferIO. Returns 0 if successful, otherwise return -1.
+ */
+int lprofBufferIOWrite(ProfBufferIO *BufferIO, const uint8_t *Data,
+ uint32_t Size);
+/*!
+ * The interface to flush the remaining data in the buffer.
+ * through the low level writer callback.
+ */
+int lprofBufferIOFlush(ProfBufferIO *BufferIO);
+
+/* The low level interface to write data into a buffer. It is used as the
+ * callback by other high level writer methods such as buffered IO writer
+ * and profile data writer. */
+uint32_t lprofBufferWriter(ProfDataWriter *This, ProfDataIOVec *IOVecs,
+ uint32_t NumIOVecs);
+void initBufferWriter(ProfDataWriter *BufferWriter, char *Buffer);
+
+struct ValueProfData;
+struct ValueProfRecord;
+struct InstrProfValueData;
+struct ValueProfNode;
+
+/*!
+ * The class that defines a set of methods to read value profile
+ * data for streaming/serialization from the instrumentation runtime.
+ */
+typedef struct VPDataReaderType {
+ uint32_t (*InitRTRecord)(const __llvm_profile_data *Data,
+ uint8_t *SiteCountArray[]);
+ /* Function pointer to getValueProfRecordHeader method. */
+ uint32_t (*GetValueProfRecordHeaderSize)(uint32_t NumSites);
+ /* Function pointer to getFristValueProfRecord method. */
+ struct ValueProfRecord *(*GetFirstValueProfRecord)(struct ValueProfData *);
+ /* Return the number of value data for site \p Site. */
+ uint32_t (*GetNumValueDataForSite)(uint32_t VK, uint32_t Site);
+ /* Return the total size of the value profile data of the
+ * current function. */
+ uint32_t (*GetValueProfDataSize)(void);
+ /*!
+ * Read the next \p N value data for site \p Site and store the data
+ * in \p Dst. \p StartNode is the first value node to start with if
+ * it is not null. The function returns the pointer to the value
+ * node pointer to be used as the \p StartNode of the next batch reading.
+ * If there is nothing left, it returns NULL.
+ */
+ struct ValueProfNode *(*GetValueData)(uint32_t ValueKind, uint32_t Site,
+ struct InstrProfValueData *Dst,
+ struct ValueProfNode *StartNode,
+ uint32_t N);
+} VPDataReaderType;
+
+/* Write profile data to destination. If SkipNameDataWrite is set to 1,
+ the name data is already in destination, we just skip over it. */
+int lprofWriteData(ProfDataWriter *Writer, VPDataReaderType *VPDataReader,
+ int SkipNameDataWrite);
+int lprofWriteDataImpl(ProfDataWriter *Writer,
+ const __llvm_profile_data *DataBegin,
+ const __llvm_profile_data *DataEnd,
+ const char *CountersBegin, const char *CountersEnd,
+ const char *BitmapBegin, const char *BitmapEnd,
+ VPDataReaderType *VPDataReader, const char *NamesBegin,
+ const char *NamesEnd, const VTableProfData *VTableBegin,
+ const VTableProfData *VTableEnd, const char *VNamesBegin,
+ const char *VNamesEnd, int SkipNameDataWrite);
+
+/* Merge value profile data pointed to by SrcValueProfData into
+ * in-memory profile counters pointed by to DstData. */
+void lprofMergeValueProfData(struct ValueProfData *SrcValueProfData,
+ __llvm_profile_data *DstData);
+
+VPDataReaderType *lprofGetVPDataReader(void);
+
+/* Internal interface used by test to reset the max number of
+ * tracked values per value site to be \p MaxVals.
+ */
+void lprofSetMaxValsPerSite(uint32_t MaxVals);
+void lprofSetupValueProfiler(void);
+
+/* Return the profile header 'signature' value associated with the current
+ * executable or shared library. The signature value can be used to for
+ * a profile name that is unique to this load module so that it does not
+ * collide with profiles from other binaries. It also allows shared libraries
+ * to dump merged profile data into its own profile file. */
+uint64_t lprofGetLoadModuleSignature(void);
+
+/*
+ * Return non zero value if the profile data has already been
+ * dumped to the file.
+ */
+unsigned lprofProfileDumped(void);
+void lprofSetProfileDumped(unsigned);
+
+COMPILER_RT_VISIBILITY extern void (*FreeHook)(void *);
+COMPILER_RT_VISIBILITY extern uint8_t *DynamicBufferIOBuffer;
+COMPILER_RT_VISIBILITY extern uint32_t VPBufferSize;
+COMPILER_RT_VISIBILITY extern uint32_t VPMaxNumValsPerSite;
+/* Pointer to the start of static value counters to be allocted. */
+COMPILER_RT_VISIBILITY extern ValueProfNode *CurrentVNode;
+COMPILER_RT_VISIBILITY extern ValueProfNode *EndVNode;
+extern void (*VPMergeHook)(struct ValueProfData *, __llvm_profile_data *);
+
+/*
+ * Write binary ids into profiles if writer is given.
+ * Return -1 if an error occurs, otherwise, return total size of binary ids.
+ */
+int __llvm_write_binary_ids(ProfDataWriter *Writer);
+
+/*
+ * Write binary id length and then its data, because binary id does not
+ * have a fixed length.
+ */
+int lprofWriteOneBinaryId(ProfDataWriter *Writer, uint64_t BinaryIdLen,
+ const uint8_t *BinaryIdData,
+ uint64_t BinaryIdPadding);
+
+#endif
diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingMerge.c b/system/lib/compiler-rt/lib/profile/InstrProfilingMerge.c
new file mode 100644
index 0000000000000..92721c4fd55ea
--- /dev/null
+++ b/system/lib/compiler-rt/lib/profile/InstrProfilingMerge.c
@@ -0,0 +1,260 @@
+/*===- InstrProfilingMerge.c - Profile in-process Merging ---------------===*\
+|*
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+|* See https://llvm.org/LICENSE.txt for license information.
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+|*
+|*===----------------------------------------------------------------------===*
+|* This file defines the API needed for in-process merging of profile data
+|* stored in memory buffer.
+\*===---------------------------------------------------------------------===*/
+
+#include "InstrProfiling.h"
+#include "InstrProfilingInternal.h"
+#include "InstrProfilingUtil.h"
+
+#define INSTR_PROF_VALUE_PROF_DATA
+#include "profile/InstrProfData.inc"
+
+COMPILER_RT_VISIBILITY
+void (*VPMergeHook)(ValueProfData *, __llvm_profile_data *);
+
+COMPILER_RT_VISIBILITY
+uint64_t lprofGetLoadModuleSignature(void) {
+ /* A very fast way to compute a module signature. */
+ uint64_t Version = __llvm_profile_get_version();
+ uint64_t NumCounters = __llvm_profile_get_num_counters(
+ __llvm_profile_begin_counters(), __llvm_profile_end_counters());
+ uint64_t NumData = __llvm_profile_get_num_data(__llvm_profile_begin_data(),
+ __llvm_profile_end_data());
+ uint64_t NamesSize =
+ (uint64_t)(__llvm_profile_end_names() - __llvm_profile_begin_names());
+ uint64_t NumVnodes =
+ (uint64_t)(__llvm_profile_end_vnodes() - __llvm_profile_begin_vnodes());
+ const __llvm_profile_data *FirstD = __llvm_profile_begin_data();
+
+ return (NamesSize << 40) + (NumCounters << 30) + (NumData << 20) +
+ (NumVnodes << 10) + (NumData > 0 ? FirstD->NameRef : 0) + Version +
+ __llvm_profile_get_magic();
+}
+
+#ifdef __GNUC__
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wcast-qual"
+#elif defined(__clang__)
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wcast-qual"
+#endif
+
+/* Returns 1 if profile is not structurally compatible. */
+COMPILER_RT_VISIBILITY
+int __llvm_profile_check_compatibility(const char *ProfileData,
+ uint64_t ProfileSize) {
+ __llvm_profile_header *Header = (__llvm_profile_header *)ProfileData;
+ __llvm_profile_data *SrcDataStart, *SrcDataEnd, *SrcData, *DstData;
+ SrcDataStart =
+ (__llvm_profile_data *)(ProfileData + sizeof(__llvm_profile_header) +
+ Header->BinaryIdsSize);
+ SrcDataEnd = SrcDataStart + Header->NumData;
+
+ if (ProfileSize < sizeof(__llvm_profile_header))
+ return 1;
+
+ /* Check the header first. */
+ if (Header->Magic != __llvm_profile_get_magic() ||
+ Header->Version != __llvm_profile_get_version() ||
+ Header->NumData !=
+ __llvm_profile_get_num_data(__llvm_profile_begin_data(),
+ __llvm_profile_end_data()) ||
+ Header->NumCounters !=
+ __llvm_profile_get_num_counters(__llvm_profile_begin_counters(),
+ __llvm_profile_end_counters()) ||
+ Header->NumBitmapBytes !=
+ __llvm_profile_get_num_bitmap_bytes(__llvm_profile_begin_bitmap(),
+ __llvm_profile_end_bitmap()) ||
+ Header->NamesSize !=
+ __llvm_profile_get_name_size(__llvm_profile_begin_names(),
+ __llvm_profile_end_names()) ||
+ Header->ValueKindLast != IPVK_Last)
+ return 1;
+
+ if (ProfileSize <
+ sizeof(__llvm_profile_header) + Header->BinaryIdsSize +
+ Header->NumData * sizeof(__llvm_profile_data) + Header->NamesSize +
+ Header->NumCounters * __llvm_profile_counter_entry_size() +
+ Header->NumBitmapBytes)
+ return 1;
+
+ for (SrcData = SrcDataStart,
+ DstData = (__llvm_profile_data *)__llvm_profile_begin_data();
+ SrcData < SrcDataEnd; ++SrcData, ++DstData) {
+ if (SrcData->NameRef != DstData->NameRef ||
+ SrcData->FuncHash != DstData->FuncHash ||
+ SrcData->NumCounters != DstData->NumCounters ||
+ SrcData->NumBitmapBytes != DstData->NumBitmapBytes)
+ return 1;
+ }
+
+ /* Matched! */
+ return 0;
+}
+
+static uintptr_t signextIfWin64(void *V) {
+#ifdef _WIN64
+ return (uintptr_t)(int32_t)(uintptr_t)V;
+#else
+ return (uintptr_t)V;
+#endif
+}
+
+// Skip names section, vtable profile data section and vtable names section
+// for runtime profile merge. To merge runtime addresses from multiple
+// profiles collected from the same instrumented binary, the binary should be
+// loaded at fixed base address (e.g., build with -no-pie, or run with ASLR
+// disabled). In this set-up these three sections remain unchanged.
+static uint64_t
+getDistanceFromCounterToValueProf(const __llvm_profile_header *const Header) {
+ const uint64_t VTableSectionSize =
+ Header->NumVTables * sizeof(VTableProfData);
+ const uint64_t PaddingBytesAfterVTableSection =
+ __llvm_profile_get_num_padding_bytes(VTableSectionSize);
+ const uint64_t VNamesSize = Header->VNamesSize;
+ const uint64_t PaddingBytesAfterVNamesSize =
+ __llvm_profile_get_num_padding_bytes(VNamesSize);
+ return Header->NamesSize +
+ __llvm_profile_get_num_padding_bytes(Header->NamesSize) +
+ VTableSectionSize + PaddingBytesAfterVTableSection + VNamesSize +
+ PaddingBytesAfterVNamesSize;
+}
+
+COMPILER_RT_VISIBILITY
+int __llvm_profile_merge_from_buffer(const char *ProfileData,
+ uint64_t ProfileSize) {
+ if (__llvm_profile_get_version() & VARIANT_MASK_TEMPORAL_PROF) {
+ PROF_ERR("%s\n",
+ "Temporal profiles do not support profile merging at runtime. "
+ "Instead, merge raw profiles using the llvm-profdata tool.");
+ return 1;
+ }
+
+ __llvm_profile_header *Header = (__llvm_profile_header *)ProfileData;
+ uintptr_t CountersDelta = Header->CountersDelta;
+ uintptr_t BitmapDelta = Header->BitmapDelta;
+
+ __llvm_profile_data *SrcDataStart =
+ (__llvm_profile_data *)(ProfileData + sizeof(__llvm_profile_header) +
+ Header->BinaryIdsSize);
+ __llvm_profile_data *SrcDataEnd = SrcDataStart + Header->NumData;
+ uintptr_t SrcCountersStart = (uintptr_t)SrcDataEnd;
+ uintptr_t SrcCountersEnd =
+ SrcCountersStart +
+ Header->NumCounters * __llvm_profile_counter_entry_size();
+ uintptr_t SrcBitmapStart =
+ SrcCountersEnd +
+ __llvm_profile_get_num_padding_bytes(SrcCountersEnd - SrcCountersStart);
+ uintptr_t SrcNameStart = SrcBitmapStart + Header->NumBitmapBytes;
+ uintptr_t SrcValueProfDataStart =
+ SrcNameStart + getDistanceFromCounterToValueProf(Header);
+ if (SrcNameStart < SrcCountersStart || SrcNameStart < SrcBitmapStart)
+ return 1;
+
+ // Merge counters by iterating the entire counter section when data section is
+ // empty due to correlation.
+ if (Header->NumData == 0) {
+ for (uintptr_t SrcCounter = SrcCountersStart,
+ DstCounter = (uintptr_t)__llvm_profile_begin_counters();
+ SrcCounter < SrcCountersEnd;) {
+ if (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE) {
+ *(char *)DstCounter &= *(const char *)SrcCounter;
+ } else {
+ *(uint64_t *)DstCounter += *(const uint64_t *)SrcCounter;
+ }
+ SrcCounter += __llvm_profile_counter_entry_size();
+ DstCounter += __llvm_profile_counter_entry_size();
+ }
+ return 0;
+ }
+
+ __llvm_profile_data *SrcData, *DstData;
+ uintptr_t SrcValueProfData;
+ for (SrcData = SrcDataStart,
+ DstData = (__llvm_profile_data *)__llvm_profile_begin_data(),
+ SrcValueProfData = SrcValueProfDataStart;
+ SrcData < SrcDataEnd; ++SrcData, ++DstData) {
+ // For the in-memory destination, CounterPtr is the distance from the start
+ // address of the data to the start address of the counter. On WIN64,
+ // CounterPtr is a truncated 32-bit value due to COFF limitation. Sign
+ // extend CounterPtr to get the original value.
+ uintptr_t DstCounters =
+ (uintptr_t)DstData + signextIfWin64(DstData->CounterPtr);
+ uintptr_t DstBitmap =
+ (uintptr_t)DstData + signextIfWin64(DstData->BitmapPtr);
+ unsigned NVK = 0;
+
+ // SrcData is a serialized representation of the memory image. We need to
+ // compute the in-buffer counter offset from the in-memory address distance.
+ // The initial CountersDelta is the in-memory address difference
+ // start(__llvm_prf_cnts)-start(__llvm_prf_data), so SrcData->CounterPtr -
+ // CountersDelta computes the offset into the in-buffer counter section.
+ //
+ // On WIN64, CountersDelta is truncated as well, so no need for signext.
+ uintptr_t SrcCounters =
+ SrcCountersStart + ((uintptr_t)SrcData->CounterPtr - CountersDelta);
+ // CountersDelta needs to be decreased as we advance to the next data
+ // record.
+ CountersDelta -= sizeof(*SrcData);
+ unsigned NC = SrcData->NumCounters;
+ if (NC == 0)
+ return 1;
+ if (SrcCounters < SrcCountersStart || SrcCounters >= SrcNameStart ||
+ (SrcCounters + __llvm_profile_counter_entry_size() * NC) > SrcNameStart)
+ return 1;
+ for (unsigned I = 0; I < NC; I++) {
+ if (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE) {
+ // A value of zero signifies the function is covered.
+ ((char *)DstCounters)[I] &= ((const char *)SrcCounters)[I];
+ } else {
+ ((uint64_t *)DstCounters)[I] += ((const uint64_t *)SrcCounters)[I];
+ }
+ }
+
+ uintptr_t SrcBitmap =
+ SrcBitmapStart + ((uintptr_t)SrcData->BitmapPtr - BitmapDelta);
+ // BitmapDelta also needs to be decreased as we advance to the next data
+ // record.
+ BitmapDelta -= sizeof(*SrcData);
+ unsigned NB = SrcData->NumBitmapBytes;
+ // NumBitmapBytes may legitimately be 0. Just keep going.
+ if (NB != 0) {
+ if (SrcBitmap < SrcBitmapStart || (SrcBitmap + NB) > SrcNameStart)
+ return 1;
+ // Merge Src and Dst Bitmap bytes by simply ORing them together.
+ for (unsigned I = 0; I < NB; I++)
+ ((char *)DstBitmap)[I] |= ((const char *)SrcBitmap)[I];
+ }
+
+ /* Now merge value profile data. */
+ if (!VPMergeHook)
+ continue;
+
+ for (unsigned I = 0; I <= IPVK_Last; I++)
+ NVK += (SrcData->NumValueSites[I] != 0);
+
+ if (!NVK)
+ continue;
+
+ if (SrcValueProfData >= (uintptr_t)ProfileData + ProfileSize)
+ return 1;
+ VPMergeHook((ValueProfData *)SrcValueProfData, DstData);
+ SrcValueProfData =
+ SrcValueProfData + ((ValueProfData *)SrcValueProfData)->TotalSize;
+ }
+
+ return 0;
+}
+
+#ifdef __GNUC__
+#pragma GCC diagnostic pop
+#elif defined(__clang__)
+#pragma clang diagnostic pop
+#endif
diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingMergeFile.c b/system/lib/compiler-rt/lib/profile/InstrProfilingMergeFile.c
new file mode 100644
index 0000000000000..8923ba21cc580
--- /dev/null
+++ b/system/lib/compiler-rt/lib/profile/InstrProfilingMergeFile.c
@@ -0,0 +1,45 @@
+/*===- InstrProfilingMergeFile.c - Profile in-process Merging ------------===*\
+|*
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+|* See https://llvm.org/LICENSE.txt for license information.
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+|*
+|*===----------------------------------------------------------------------===
+|* This file defines APIs needed to support in-process merging for profile data
+|* stored in files.
+\*===----------------------------------------------------------------------===*/
+
+#if !defined(__Fuchsia__)
+
+#include "InstrProfiling.h"
+#include "InstrProfilingInternal.h"
+#include "InstrProfilingUtil.h"
+
+#define INSTR_PROF_VALUE_PROF_DATA
+#include "profile/InstrProfData.inc"
+
+/* Merge value profile data pointed to by SrcValueProfData into
+ * in-memory profile counters pointed by to DstData. */
+COMPILER_RT_VISIBILITY
+void lprofMergeValueProfData(ValueProfData *SrcValueProfData,
+ __llvm_profile_data *DstData) {
+ unsigned I, S, V, DstIndex = 0;
+ InstrProfValueData *VData;
+ ValueProfRecord *VR = getFirstValueProfRecord(SrcValueProfData);
+ for (I = 0; I < SrcValueProfData->NumValueKinds; I++) {
+ VData = getValueProfRecordValueData(VR);
+ unsigned SrcIndex = 0;
+ for (S = 0; S < VR->NumValueSites; S++) {
+ uint8_t NV = VR->SiteCountArray[S];
+ for (V = 0; V < NV; V++) {
+ __llvm_profile_instrument_target_value(VData[SrcIndex].Value, DstData,
+ DstIndex, VData[SrcIndex].Count);
+ ++SrcIndex;
+ }
+ ++DstIndex;
+ }
+ VR = getValueProfRecordNext(VR);
+ }
+}
+
+#endif
diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingNameVar.c b/system/lib/compiler-rt/lib/profile/InstrProfilingNameVar.c
new file mode 100644
index 0000000000000..407272806ba3c
--- /dev/null
+++ b/system/lib/compiler-rt/lib/profile/InstrProfilingNameVar.c
@@ -0,0 +1,17 @@
+/*===- InstrProfilingNameVar.c - profile name variable setup -------------===*\
+|*
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+|* See https://llvm.org/LICENSE.txt for license information.
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+|*
+\*===----------------------------------------------------------------------===*/
+
+#include "InstrProfiling.h"
+
+/* char __llvm_profile_filename[1]
+ *
+ * The runtime should only provide its own definition of this symbol when the
+ * user has not specified one. Set this up by moving the runtime's copy of this
+ * symbol to an object file within the archive.
+ */
+COMPILER_RT_WEAK COMPILER_RT_VISIBILITY char INSTR_PROF_PROFILE_NAME_VAR[1] = {0};
diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformAIX.c b/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformAIX.c
new file mode 100644
index 0000000000000..651f8785d0b94
--- /dev/null
+++ b/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformAIX.c
@@ -0,0 +1,224 @@
+/*===- InstrProfilingPlatformAIX.c - Profile data AIX platform ------------===*\
+|*
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+|* See https://llvm.org/LICENSE.txt for license information.
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+|*
+\*===----------------------------------------------------------------------===*/
+
+#if defined(_AIX)
+
+#ifdef __64BIT__
+#define __XCOFF64__
+#endif
+#include <errno.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/ldr.h>
+#include <xcoff.h>
+
+#include "InstrProfiling.h"
+#include "InstrProfilingInternal.h"
+
+#define BIN_ID_PREFIX "xcoff_binary_id:"
+
+// If found, write the build-id into the Result buffer.
+static size_t FindBinaryId(char *Result, size_t Size) {
+ unsigned long EntryAddr = (unsigned long)__builtin_return_address(0);
+
+ // Use loadquery to get information about loaded modules; loadquery writes
+ // its result into a buffer of unknown size.
+ char Buf[1024];
+ size_t BufSize = sizeof(Buf);
+ char *BufPtr = Buf;
+ int RC = -1;
+
+ errno = 0;
+ RC = loadquery(L_GETXINFO | L_IGNOREUNLOAD, BufPtr, (unsigned int)BufSize);
+ if (RC == -1 && errno == ENOMEM) {
+ BufSize = 64000; // should be plenty for any program.
+ BufPtr = malloc(BufSize);
+ if (BufPtr != 0)
+ RC = loadquery(L_GETXINFO | L_IGNOREUNLOAD, BufPtr, (unsigned int)BufSize);
+ }
+
+ if (RC == -1)
+ goto done;
+
+ // Locate the ld_xinfo corresponding to this module.
+ struct ld_xinfo *CurInfo = (struct ld_xinfo *)BufPtr;
+ while (1) {
+ unsigned long CurTextStart = (uint64_t)CurInfo->ldinfo_textorg;
+ unsigned long CurTextEnd = CurTextStart + CurInfo->ldinfo_textsize;
+ if (CurTextStart <= EntryAddr && EntryAddr < CurTextEnd) {
+ // Found my slot. Now search for the build-id.
+ char *p = (char *)CurInfo->ldinfo_textorg;
+
+ FILHDR *f = (FILHDR *)p;
+ AOUTHDR *a = (AOUTHDR *)(p + FILHSZ);
+ SCNHDR *s =
+ (SCNHDR *)(p + FILHSZ + f->f_opthdr + SCNHSZ * (a->o_snloader - 1));
+ LDHDR *ldhdr = (LDHDR *)(p + s->s_scnptr);
+ // This is the loader string table
+ char *lstr = (char *)ldhdr + ldhdr->l_stoff;
+
+ // If the build-id exists, it's the first entry.
+ // Each entry is comprised of a 2-byte size component, followed by the
+ // data.
+ size_t len = *(short *)lstr;
+ char *str = (char *)(lstr + 2);
+ size_t PrefixLen = sizeof(BIN_ID_PREFIX) - 1;
+ if (len > PrefixLen && (len - PrefixLen) <= Size &&
+ strncmp(str, BIN_ID_PREFIX, PrefixLen) == 0) {
+ memcpy(Result, str + PrefixLen, len - PrefixLen);
+ RC = len - PrefixLen;
+ goto done;
+ }
+ break;
+ }
+ if (CurInfo->ldinfo_next == 0u)
+ break;
+ CurInfo = (struct ld_xinfo *)((char *)CurInfo + CurInfo->ldinfo_next);
+ }
+done:
+ if (BufSize != sizeof(Buf) && BufPtr != 0)
+ free(BufPtr);
+ return RC;
+}
+
+static int StrToHexError = 0;
+static uint8_t StrToHex(char c) {
+ if (c >= '0' && c <= '9')
+ return c - '0';
+ if (c >= 'a' && c <= 'f')
+ return c - 'a' + 0xa;
+ if (c >= 'A' && c <= 'F')
+ return c - 'A' + 0xa;
+ StrToHexError = 1;
+ return 0;
+}
+
+COMPILER_RT_VISIBILITY int __llvm_write_binary_ids(ProfDataWriter *Writer) {
+ // 200 bytes should be enough for the build-id hex string.
+ static char Buf[200];
+ // Profile reading tools expect this to be 8-bytes long.
+ static int64_t BinaryIdLen = 0;
+ static uint8_t *BinaryIdData = 0;
+
+ // -1 means we already checked for a BinaryId and didn't find one.
+ if (BinaryIdLen == -1)
+ return 0;
+
+ // Are we being called for the first time?
+ if (BinaryIdLen == 0) {
+ if (getenv("LLVM_PROFILE_NO_BUILD_ID"))
+ goto fail;
+
+ int BuildIdLen = FindBinaryId(Buf, sizeof(Buf));
+ if (BuildIdLen <= 0)
+ goto fail;
+
+ if (Buf[BuildIdLen - 1] == '\0')
+ BuildIdLen--;
+
+ // assume even number of digits/chars, so 0xabc must be 0x0abc
+ if ((BuildIdLen % 2) != 0 || BuildIdLen == 0)
+ goto fail;
+
+ // The numeric ID is represented as an ascii string in the loader section,
+ // so convert it to raw binary.
+ BinaryIdLen = BuildIdLen / 2;
+ BinaryIdData = (uint8_t *)Buf;
+
+ // Skip "0x" prefix if it exists.
+ if (Buf[0] == '0' && Buf[1] == 'x') {
+ BinaryIdLen -= 1;
+ BinaryIdData += 2;
+ }
+
+ StrToHexError = 0;
+ for (int i = 0; i < BinaryIdLen; i++)
+ BinaryIdData[i] = (StrToHex(BinaryIdData[2 * i]) << 4) +
+ StrToHex(BinaryIdData[2 * i + 1]);
+
+ if (StrToHexError)
+ goto fail;
+
+ if (getenv("LLVM_PROFILE_VERBOSE")) {
+ char *StrBuf = (char *)COMPILER_RT_ALLOCA(2 * BinaryIdLen + 1);
+ for (int i = 0; i < (int)BinaryIdLen; i++)
+ sprintf(&StrBuf[2 * i], "%02x", BinaryIdData[i]);
+ PROF_NOTE("Writing binary id: %s\n", StrBuf);
+ }
+ }
+
+ uint8_t BinaryIdPadding = __llvm_profile_get_num_padding_bytes(BinaryIdLen);
+ if (Writer && lprofWriteOneBinaryId(Writer, BinaryIdLen, BinaryIdData,
+ BinaryIdPadding) == -1)
+ return -1; // Return -1 rather goto fail to match the NT_GNU_BUILD_ID path.
+
+ return sizeof(BinaryIdLen) + BinaryIdLen + BinaryIdPadding;
+
+fail:
+ if (getenv("LLVM_PROFILE_VERBOSE"))
+ fprintf(stderr, "no or invalid binary id: %.*s\n", (int)sizeof(Buf), Buf);
+ BinaryIdLen = -1;
+ return 0;
+}
+
+// Empty stubs to allow linking object files using the registration-based scheme
+COMPILER_RT_VISIBILITY
+void __llvm_profile_register_function(void *Data_) {}
+
+COMPILER_RT_VISIBILITY
+void __llvm_profile_register_names_function(void *NamesStart,
+ uint64_t NamesSize) {}
+
+// The __start_SECNAME and __stop_SECNAME symbols (for SECNAME \in
+// {"__llvm_prf_cnts", "__llvm_prf_data", "__llvm_prf_name", "__llvm_prf_vnds",
+// "__llvm_prf_vns", "__llvm_prf_vtab"})
+// are always live when linking on AIX, regardless if the .o's being linked
+// reference symbols from the profile library (for example when no files were
+// compiled with -fprofile-generate). That's because these symbols are kept
+// alive through references in constructor functions that are always live in the
+// default linking model on AIX (-bcdtors:all). The __start_SECNAME and
+// __stop_SECNAME symbols are only resolved by the linker when the SECNAME
+// section exists. So for the scenario where the user objects have no such
+// section (i.e. when they are compiled with -fno-profile-generate), we always
+// define these zero length variables in each of the above 4 sections.
+static int dummy_cnts[0] COMPILER_RT_SECTION(
+ COMPILER_RT_SEG INSTR_PROF_CNTS_SECT_NAME);
+static int dummy_bits[0] COMPILER_RT_SECTION(
+ COMPILER_RT_SEG INSTR_PROF_BITS_SECT_NAME);
+static int dummy_data[0] COMPILER_RT_SECTION(
+ COMPILER_RT_SEG INSTR_PROF_DATA_SECT_NAME);
+static const int dummy_name[0] COMPILER_RT_SECTION(
+ COMPILER_RT_SEG INSTR_PROF_NAME_SECT_NAME);
+static int dummy_vnds[0] COMPILER_RT_SECTION(
+ COMPILER_RT_SEG INSTR_PROF_VNODES_SECT_NAME);
+static int dummy_orderfile[0] COMPILER_RT_SECTION(
+ COMPILER_RT_SEG INSTR_PROF_ORDERFILE_SECT_NAME);
+static int dummy_vname[0] COMPILER_RT_SECTION(
+ COMPILER_RT_SEG INSTR_PROF_VNAME_SECT_NAME);
+static int dummy_vtab[0] COMPILER_RT_SECTION(
+ COMPILER_RT_SEG INSTR_PROF_VTAB_SECT_NAME);
+static int dummy_covinit_funcs[0] COMPILER_RT_SECTION(
+ COMPILER_RT_SEG INSTR_PROF_COVINIT_SECT_NAME);
+
+// To avoid GC'ing of the dummy variables by the linker, reference them in an
+// array and reference the array in the runtime registration code
+// (InstrProfilingRuntime.cpp)
+#ifdef __GNUC__
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wcast-qual"
+#endif
+COMPILER_RT_VISIBILITY
+void *__llvm_profile_keep[] = {(void *)&dummy_cnts, (void *)&dummy_bits,
+ (void *)&dummy_data, (void *)&dummy_name,
+ (void *)&dummy_vnds, (void *)&dummy_orderfile,
+ (void *)&dummy_vname, (void *)&dummy_vtab,
+ (void *)&dummy_covinit_funcs};
+#ifdef __GNUC__
+#pragma GCC diagnostic pop
+#endif
+#endif
diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformDarwin.c b/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformDarwin.c
new file mode 100644
index 0000000000000..6adc7f328cbf7
--- /dev/null
+++ b/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformDarwin.c
@@ -0,0 +1,107 @@
+/*===- InstrProfilingPlatformDarwin.c - Profile data on Darwin ------------===*\
+|*
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+|* See https://llvm.org/LICENSE.txt for license information.
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+|*
+\*===----------------------------------------------------------------------===*/
+
+// Note: This is linked into the Darwin kernel, and must remain compatible
+// with freestanding compilation. See `darwin_add_builtin_libraries`.
+
+#include "InstrProfiling.h"
+#include "InstrProfilingInternal.h"
+
+#if defined(__APPLE__)
+/* Use linker magic to find the bounds of the Data section. */
+COMPILER_RT_VISIBILITY
+extern __llvm_profile_data
+ DataStart __asm("section$start$__DATA$" INSTR_PROF_DATA_SECT_NAME);
+COMPILER_RT_VISIBILITY
+extern __llvm_profile_data
+ DataEnd __asm("section$end$__DATA$" INSTR_PROF_DATA_SECT_NAME);
+COMPILER_RT_VISIBILITY
+extern char
+ NamesStart __asm("section$start$__DATA$" INSTR_PROF_NAME_SECT_NAME);
+COMPILER_RT_VISIBILITY
+extern char NamesEnd __asm("section$end$__DATA$" INSTR_PROF_NAME_SECT_NAME);
+COMPILER_RT_VISIBILITY
+extern char
+ CountersStart __asm("section$start$__DATA$" INSTR_PROF_CNTS_SECT_NAME);
+COMPILER_RT_VISIBILITY
+extern char CountersEnd __asm("section$end$__DATA$" INSTR_PROF_CNTS_SECT_NAME);
+COMPILER_RT_VISIBILITY
+extern char
+ BitmapStart __asm("section$start$__DATA$" INSTR_PROF_BITS_SECT_NAME);
+COMPILER_RT_VISIBILITY
+extern char BitmapEnd __asm("section$end$__DATA$" INSTR_PROF_BITS_SECT_NAME);
+COMPILER_RT_VISIBILITY
+extern VTableProfData
+ VTableProfStart __asm("section$start$__DATA$" INSTR_PROF_VTAB_SECT_NAME);
+COMPILER_RT_VISIBILITY
+extern VTableProfData
+ VTableProfEnd __asm("section$end$__DATA$" INSTR_PROF_VTAB_SECT_NAME);
+COMPILER_RT_VISIBILITY
+extern char
+ VNameStart __asm("section$start$__DATA$" INSTR_PROF_VNAME_SECT_NAME);
+COMPILER_RT_VISIBILITY
+extern char VNameEnd __asm("section$end$__DATA$" INSTR_PROF_VNAME_SECT_NAME);
+COMPILER_RT_VISIBILITY
+extern uint32_t
+ OrderFileStart __asm("section$start$__DATA$" INSTR_PROF_ORDERFILE_SECT_NAME);
+
+COMPILER_RT_VISIBILITY
+extern ValueProfNode
+ VNodesStart __asm("section$start$__DATA$" INSTR_PROF_VNODES_SECT_NAME);
+COMPILER_RT_VISIBILITY
+extern ValueProfNode
+ VNodesEnd __asm("section$end$__DATA$" INSTR_PROF_VNODES_SECT_NAME);
+
+COMPILER_RT_VISIBILITY
+const __llvm_profile_data *__llvm_profile_begin_data(void) {
+ return &DataStart;
+}
+COMPILER_RT_VISIBILITY
+const __llvm_profile_data *__llvm_profile_end_data(void) { return &DataEnd; }
+COMPILER_RT_VISIBILITY
+const char *__llvm_profile_begin_names(void) { return &NamesStart; }
+COMPILER_RT_VISIBILITY
+const char *__llvm_profile_end_names(void) { return &NamesEnd; }
+COMPILER_RT_VISIBILITY
+char *__llvm_profile_begin_counters(void) { return &CountersStart; }
+COMPILER_RT_VISIBILITY
+char *__llvm_profile_end_counters(void) { return &CountersEnd; }
+COMPILER_RT_VISIBILITY
+char *__llvm_profile_begin_bitmap(void) { return &BitmapStart; }
+COMPILER_RT_VISIBILITY
+char *__llvm_profile_end_bitmap(void) { return &BitmapEnd; }
+COMPILER_RT_VISIBILITY
+const VTableProfData *__llvm_profile_begin_vtables(void) {
+ return &VTableProfStart;
+}
+COMPILER_RT_VISIBILITY
+const VTableProfData *__llvm_profile_end_vtables(void) {
+ return &VTableProfEnd;
+}
+COMPILER_RT_VISIBILITY
+const char *__llvm_profile_begin_vtabnames(void) { return &VNameStart; }
+COMPILER_RT_VISIBILITY
+const char *__llvm_profile_end_vtabnames(void) { return &VNameEnd; }
+COMPILER_RT_VISIBILITY
+uint32_t *__llvm_profile_begin_orderfile(void) { return &OrderFileStart; }
+
+COMPILER_RT_VISIBILITY
+ValueProfNode *__llvm_profile_begin_vnodes(void) {
+ return &VNodesStart;
+}
+COMPILER_RT_VISIBILITY
+ValueProfNode *__llvm_profile_end_vnodes(void) { return &VNodesEnd; }
+
+COMPILER_RT_VISIBILITY ValueProfNode *CurrentVNode = &VNodesStart;
+COMPILER_RT_VISIBILITY ValueProfNode *EndVNode = &VNodesEnd;
+
+COMPILER_RT_VISIBILITY int __llvm_write_binary_ids(ProfDataWriter *Writer) {
+ return 0;
+}
+
+#endif
diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformFuchsia.c b/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformFuchsia.c
new file mode 100644
index 0000000000000..65b7bdaf403da
--- /dev/null
+++ b/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformFuchsia.c
@@ -0,0 +1,190 @@
+/*===- InstrProfilingPlatformFuchsia.c - Profile data Fuchsia platform ----===*\
+|*
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+|* See https://llvm.org/LICENSE.txt for license information.
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+|*
+\*===----------------------------------------------------------------------===*/
+/*
+ * This file implements the profiling runtime for Fuchsia and defines the
+ * shared profile runtime interface. Each module (executable or DSO) statically
+ * links in the whole profile runtime to satisfy the calls from its
+ * instrumented code. Several modules in the same program might be separately
+ * compiled and even use different versions of the instrumentation ABI and data
+ * format. All they share in common is the VMO and the offset, which live in
+ * exported globals so that exactly one definition will be shared across all
+ * modules. Each module has its own independent runtime that registers its own
+ * atexit hook to append its own data into the shared VMO which is published
+ * via the data sink hook provided by Fuchsia's dynamic linker.
+ */
+
+#if defined(__Fuchsia__)
+
+#include <inttypes.h>
+#include <stdarg.h>
+#include <stdbool.h>
+#include <stdlib.h>
+
+#include <zircon/process.h>
+#include <zircon/sanitizer.h>
+#include <zircon/status.h>
+#include <zircon/syscalls.h>
+
+#include "InstrProfiling.h"
+#include "InstrProfilingInternal.h"
+#include "InstrProfilingUtil.h"
+
+/* This variable is an external reference to symbol defined by the compiler. */
+COMPILER_RT_VISIBILITY extern int64_t INSTR_PROF_PROFILE_COUNTER_BIAS_VAR;
+
+COMPILER_RT_VISIBILITY unsigned lprofProfileDumped(void) {
+ return 1;
+}
+COMPILER_RT_VISIBILITY void lprofSetProfileDumped(unsigned Value) {}
+
+static const char ProfileSinkName[] = "llvm-profile";
+
+static inline void lprofWrite(const char *fmt, ...) {
+ char s[256];
+
+ va_list ap;
+ va_start(ap, fmt);
+ int ret = vsnprintf(s, sizeof(s), fmt, ap);
+ va_end(ap);
+
+ __sanitizer_log_write(s, ret);
+}
+
+struct lprofVMOWriterCtx {
+ /* VMO that contains the profile data for this module. */
+ zx_handle_t Vmo;
+ /* Current offset within the VMO where data should be written next. */
+ uint64_t Offset;
+};
+
+static uint32_t lprofVMOWriter(ProfDataWriter *This, ProfDataIOVec *IOVecs,
+ uint32_t NumIOVecs) {
+ struct lprofVMOWriterCtx *Ctx = (struct lprofVMOWriterCtx *)This->WriterCtx;
+
+ /* Compute the total length of data to be written. */
+ size_t Length = 0;
+ for (uint32_t I = 0; I < NumIOVecs; I++)
+ Length += IOVecs[I].ElmSize * IOVecs[I].NumElm;
+
+ /* Resize the VMO to ensure there's sufficient space for the data. */
+ zx_status_t Status = _zx_vmo_set_size(Ctx->Vmo, Ctx->Offset + Length);
+ if (Status != ZX_OK)
+ return -1;
+
+ /* Copy the data into VMO. */
+ for (uint32_t I = 0; I < NumIOVecs; I++) {
+ size_t Length = IOVecs[I].ElmSize * IOVecs[I].NumElm;
+ if (IOVecs[I].Data) {
+ Status = _zx_vmo_write(Ctx->Vmo, IOVecs[I].Data, Ctx->Offset, Length);
+ if (Status != ZX_OK)
+ return -1;
+ } else if (IOVecs[I].UseZeroPadding) {
+ /* Resizing the VMO should zero fill. */
+ }
+ Ctx->Offset += Length;
+ }
+
+ /* Record the profile size as a property of the VMO. */
+ _zx_object_set_property(Ctx->Vmo, ZX_PROP_VMO_CONTENT_SIZE, &Ctx->Offset,
+ sizeof(Ctx->Offset));
+
+ return 0;
+}
+
+static void initVMOWriter(ProfDataWriter *This, struct lprofVMOWriterCtx *Ctx) {
+ This->Write = lprofVMOWriter;
+ This->WriterCtx = Ctx;
+}
+
+/* This method is invoked by the runtime initialization hook
+ * InstrProfilingRuntime.o if it is linked in. */
+COMPILER_RT_VISIBILITY
+void __llvm_profile_initialize(void) {
+ /* Check if there is llvm/runtime version mismatch. */
+ if (GET_VERSION(__llvm_profile_get_version()) != INSTR_PROF_RAW_VERSION) {
+ lprofWrite("LLVM Profile: runtime and instrumentation version mismatch: "
+ "expected %d, but got %d\n",
+ INSTR_PROF_RAW_VERSION,
+ (int)GET_VERSION(__llvm_profile_get_version()));
+ return;
+ }
+
+ const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
+ const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
+ const char *CountersBegin = __llvm_profile_begin_counters();
+ const char *CountersEnd = __llvm_profile_end_counters();
+ const uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd);
+ const uint64_t CountersOffset =
+ sizeof(__llvm_profile_header) + __llvm_write_binary_ids(NULL) + DataSize;
+ uint64_t CountersSize =
+ __llvm_profile_get_counters_size(CountersBegin, CountersEnd);
+
+ /* Don't publish a VMO if there are no counters. */
+ if (!CountersSize)
+ return;
+
+ zx_status_t Status;
+
+ /* Create a VMO to hold the profile data. */
+ zx_handle_t Vmo = ZX_HANDLE_INVALID;
+ Status = _zx_vmo_create(0, ZX_VMO_RESIZABLE, &Vmo);
+ if (Status != ZX_OK) {
+ lprofWrite("LLVM Profile: cannot create VMO: %s\n",
+ _zx_status_get_string(Status));
+ return;
+ }
+
+ /* Give the VMO a name that includes the module signature. */
+ char VmoName[ZX_MAX_NAME_LEN];
+ snprintf(VmoName, sizeof(VmoName), "%" PRIu64 ".profraw",
+ lprofGetLoadModuleSignature());
+ _zx_object_set_property(Vmo, ZX_PROP_NAME, VmoName, strlen(VmoName));
+
+ /* Write the profile data into the mapped region. */
+ ProfDataWriter VMOWriter;
+ struct lprofVMOWriterCtx Ctx = {.Vmo = Vmo, .Offset = 0};
+ initVMOWriter(&VMOWriter, &Ctx);
+ if (lprofWriteData(&VMOWriter, 0, 0) != 0) {
+ lprofWrite("LLVM Profile: failed to write data\n");
+ _zx_handle_close(Vmo);
+ return;
+ }
+
+ uint64_t Len = 0;
+ Status = _zx_vmo_get_size(Vmo, &Len);
+ if (Status != ZX_OK) {
+ lprofWrite("LLVM Profile: failed to get the VMO size: %s\n",
+ _zx_status_get_string(Status));
+ _zx_handle_close(Vmo);
+ return;
+ }
+
+ uintptr_t Mapping;
+ Status =
+ _zx_vmar_map(_zx_vmar_root_self(), ZX_VM_PERM_READ | ZX_VM_PERM_WRITE, 0,
+ Vmo, 0, Len, &Mapping);
+ if (Status != ZX_OK) {
+ lprofWrite("LLVM Profile: failed to map the VMO: %s\n",
+ _zx_status_get_string(Status));
+ _zx_handle_close(Vmo);
+ return;
+ }
+
+ /* Publish the VMO which contains profile data to the system. Note that this
+ * also consumes the VMO handle. */
+ __sanitizer_publish_data(ProfileSinkName, Vmo);
+
+ /* Update the profile fields based on the current mapping. */
+ INSTR_PROF_PROFILE_COUNTER_BIAS_VAR =
+ (intptr_t)Mapping - (uintptr_t)CountersBegin + CountersOffset;
+
+ /* Return the memory allocated for counters to OS. */
+ lprofReleaseMemoryPagesToOS((uintptr_t)CountersBegin, (uintptr_t)CountersEnd);
+}
+
+#endif
diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c b/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c
new file mode 100644
index 0000000000000..5b230c1b20062
--- /dev/null
+++ b/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c
@@ -0,0 +1,244 @@
+/*===- InstrProfilingPlatformLinux.c - Profile data Linux platform ------===*\
+|*
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+|* See https://llvm.org/LICENSE.txt for license information.
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+|*
+\*===----------------------------------------------------------------------===*/
+
+#if defined(__linux__) || defined(__FreeBSD__) || defined(__Fuchsia__) || \
+ (defined(__sun__) && defined(__svr4__)) || defined(__NetBSD__) || \
+ defined(_AIX) || defined(__wasm__) || defined(__HAIKU__)
+
+#if !defined(_AIX) && !defined(__wasm__)
+#include <elf.h>
+#include <link.h>
+#endif
+#include <stdlib.h>
+#include <string.h>
+
+#include "InstrProfiling.h"
+#include "InstrProfilingInternal.h"
+
+#define PROF_DATA_START INSTR_PROF_SECT_START(INSTR_PROF_DATA_COMMON)
+#define PROF_DATA_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_DATA_COMMON)
+#define PROF_NAME_START INSTR_PROF_SECT_START(INSTR_PROF_NAME_COMMON)
+#define PROF_NAME_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_NAME_COMMON)
+#define PROF_VNAME_START INSTR_PROF_SECT_START(INSTR_PROF_VNAME_COMMON)
+#define PROF_VNAME_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_VNAME_COMMON)
+#define PROF_CNTS_START INSTR_PROF_SECT_START(INSTR_PROF_CNTS_COMMON)
+#define PROF_CNTS_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_CNTS_COMMON)
+#define PROF_VTABLE_START INSTR_PROF_SECT_START(INSTR_PROF_VTAB_COMMON)
+#define PROF_VTABLE_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_VTAB_COMMON)
+#define PROF_BITS_START INSTR_PROF_SECT_START(INSTR_PROF_BITS_COMMON)
+#define PROF_BITS_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_BITS_COMMON)
+#define PROF_ORDERFILE_START INSTR_PROF_SECT_START(INSTR_PROF_ORDERFILE_COMMON)
+#define PROF_VNODES_START INSTR_PROF_SECT_START(INSTR_PROF_VNODES_COMMON)
+#define PROF_VNODES_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_VNODES_COMMON)
+#define PROF_COVINIT_START INSTR_PROF_SECT_START(INSTR_PROF_COVINIT_COMMON)
+#define PROF_COVINIT_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_COVINIT_COMMON)
+
+/* Declare section start and stop symbols for various sections
+ * generated by compiler instrumentation.
+ */
+extern __llvm_profile_data PROF_DATA_START COMPILER_RT_VISIBILITY
+ COMPILER_RT_WEAK;
+extern __llvm_profile_data PROF_DATA_STOP COMPILER_RT_VISIBILITY
+ COMPILER_RT_WEAK;
+extern char PROF_CNTS_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
+extern char PROF_CNTS_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
+extern VTableProfData PROF_VTABLE_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
+extern VTableProfData PROF_VTABLE_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
+extern char PROF_VNAME_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
+extern char PROF_VNAME_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
+extern char PROF_BITS_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
+extern char PROF_BITS_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
+extern uint32_t PROF_ORDERFILE_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
+extern char PROF_NAME_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
+extern char PROF_NAME_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
+extern ValueProfNode PROF_VNODES_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
+extern ValueProfNode PROF_VNODES_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
+extern __llvm_gcov_init_func_struct PROF_COVINIT_START COMPILER_RT_VISIBILITY
+ COMPILER_RT_WEAK;
+extern __llvm_gcov_init_func_struct PROF_COVINIT_STOP COMPILER_RT_VISIBILITY
+ COMPILER_RT_WEAK;
+
+COMPILER_RT_VISIBILITY const __llvm_profile_data *
+__llvm_profile_begin_data(void) {
+ return &PROF_DATA_START;
+}
+COMPILER_RT_VISIBILITY const __llvm_profile_data *
+__llvm_profile_end_data(void) {
+ return &PROF_DATA_STOP;
+}
+COMPILER_RT_VISIBILITY const char *__llvm_profile_begin_names(void) {
+ return &PROF_NAME_START;
+}
+COMPILER_RT_VISIBILITY const char *__llvm_profile_end_names(void) {
+ return &PROF_NAME_STOP;
+}
+COMPILER_RT_VISIBILITY const char *__llvm_profile_begin_vtabnames(void) {
+ return &PROF_VNAME_START;
+}
+COMPILER_RT_VISIBILITY const char *__llvm_profile_end_vtabnames(void) {
+ return &PROF_VNAME_STOP;
+}
+COMPILER_RT_VISIBILITY const VTableProfData *
+__llvm_profile_begin_vtables(void) {
+ return &PROF_VTABLE_START;
+}
+COMPILER_RT_VISIBILITY const VTableProfData *__llvm_profile_end_vtables(void) {
+ return &PROF_VTABLE_STOP;
+}
+COMPILER_RT_VISIBILITY char *__llvm_profile_begin_counters(void) {
+ return &PROF_CNTS_START;
+}
+COMPILER_RT_VISIBILITY char *__llvm_profile_end_counters(void) {
+ return &PROF_CNTS_STOP;
+}
+COMPILER_RT_VISIBILITY char *__llvm_profile_begin_bitmap(void) {
+ return &PROF_BITS_START;
+}
+COMPILER_RT_VISIBILITY char *__llvm_profile_end_bitmap(void) {
+ return &PROF_BITS_STOP;
+}
+COMPILER_RT_VISIBILITY uint32_t *__llvm_profile_begin_orderfile(void) {
+ return &PROF_ORDERFILE_START;
+}
+
+COMPILER_RT_VISIBILITY ValueProfNode *
+__llvm_profile_begin_vnodes(void) {
+ return &PROF_VNODES_START;
+}
+COMPILER_RT_VISIBILITY ValueProfNode *__llvm_profile_end_vnodes(void) {
+ return &PROF_VNODES_STOP;
+}
+COMPILER_RT_VISIBILITY ValueProfNode *CurrentVNode = &PROF_VNODES_START;
+COMPILER_RT_VISIBILITY ValueProfNode *EndVNode = &PROF_VNODES_STOP;
+
+COMPILER_RT_VISIBILITY const __llvm_gcov_init_func_struct *
+__llvm_profile_begin_covinit() {
+ return &PROF_COVINIT_START;
+}
+
+COMPILER_RT_VISIBILITY const __llvm_gcov_init_func_struct *
+__llvm_profile_end_covinit() {
+ return &PROF_COVINIT_STOP;
+}
+
+#ifdef NT_GNU_BUILD_ID
+static size_t RoundUp(size_t size, size_t align) {
+ return (size + align - 1) & ~(align - 1);
+}
+
+/*
+ * Look for the note that has the name "GNU\0" and type NT_GNU_BUILD_ID
+ * that contains build id. If build id exists, write binary id.
+ *
+ * Each note in notes section starts with a struct which includes
+ * n_namesz, n_descsz, and n_type members. It is followed by the name
+ * (whose length is defined in n_namesz) and then by the descriptor
+ * (whose length is defined in n_descsz).
+ *
+ * Note sections like .note.ABI-tag and .note.gnu.build-id are aligned
+ * to 4 bytes, so round n_namesz and n_descsz to the nearest 4 bytes.
+ */
+static int WriteBinaryIdForNote(ProfDataWriter *Writer,
+ const ElfW(Nhdr) * Note) {
+ int BinaryIdSize = 0;
+ const char *NoteName = (const char *)Note + sizeof(ElfW(Nhdr));
+ if (Note->n_type == NT_GNU_BUILD_ID && Note->n_namesz == 4 &&
+ memcmp(NoteName, "GNU\0", 4) == 0) {
+ uint64_t BinaryIdLen = Note->n_descsz;
+ const uint8_t *BinaryIdData =
+ (const uint8_t *)(NoteName + RoundUp(Note->n_namesz, 4));
+ uint8_t BinaryIdPadding = __llvm_profile_get_num_padding_bytes(BinaryIdLen);
+ if (Writer != NULL &&
+ lprofWriteOneBinaryId(Writer, BinaryIdLen, BinaryIdData,
+ BinaryIdPadding) == -1)
+ return -1;
+
+ BinaryIdSize = sizeof(BinaryIdLen) + BinaryIdLen + BinaryIdPadding;
+ }
+
+ return BinaryIdSize;
+}
+
+/*
+ * Helper function that iterates through notes section and find build ids.
+ * If writer is given, write binary ids into profiles.
+ * If an error happens while writing, return -1.
+ */
+static int WriteBinaryIds(ProfDataWriter *Writer, const ElfW(Nhdr) * Note,
+ const ElfW(Nhdr) * NotesEnd) {
+ int BinaryIdsSize = 0;
+ while (Note < NotesEnd) {
+ int OneBinaryIdSize = WriteBinaryIdForNote(Writer, Note);
+ if (OneBinaryIdSize == -1)
+ return -1;
+ BinaryIdsSize += OneBinaryIdSize;
+
+ /* Calculate the offset of the next note in notes section. */
+ size_t NoteOffset = sizeof(ElfW(Nhdr)) + RoundUp(Note->n_namesz, 4) +
+ RoundUp(Note->n_descsz, 4);
+ Note = (const ElfW(Nhdr) *)((const char *)(Note) + NoteOffset);
+ }
+
+ return BinaryIdsSize;
+}
+
+/*
+ * Write binary ids into profiles if writer is given.
+ * Return the total size of binary ids.
+ * If an error happens while writing, return -1.
+ */
+COMPILER_RT_VISIBILITY int __llvm_write_binary_ids(ProfDataWriter *Writer) {
+ extern const ElfW(Ehdr) __ehdr_start __attribute__((visibility("hidden")));
+ extern ElfW(Dyn) _DYNAMIC[] __attribute__((weak, visibility("hidden")));
+
+ const ElfW(Ehdr) *ElfHeader = &__ehdr_start;
+ const ElfW(Phdr) *ProgramHeader =
+ (const ElfW(Phdr) *)((uintptr_t)ElfHeader + ElfHeader->e_phoff);
+
+ /* Compute the added base address in case of position-independent code. */
+ uintptr_t Base = 0;
+ for (uint32_t I = 0; I < ElfHeader->e_phnum; I++) {
+ if (ProgramHeader[I].p_type == PT_PHDR)
+ Base = (uintptr_t)ProgramHeader - ProgramHeader[I].p_vaddr;
+ if (ProgramHeader[I].p_type == PT_DYNAMIC && _DYNAMIC)
+ Base = (uintptr_t)_DYNAMIC - ProgramHeader[I].p_vaddr;
+ }
+
+ int TotalBinaryIdsSize = 0;
+ /* Iterate through entries in the program header. */
+ for (uint32_t I = 0; I < ElfHeader->e_phnum; I++) {
+ /* Look for the notes segment in program header entries. */
+ if (ProgramHeader[I].p_type != PT_NOTE)
+ continue;
+
+ /* There can be multiple notes segment, and examine each of them. */
+ const ElfW(Nhdr) *Note =
+ (const ElfW(Nhdr) *)(Base + ProgramHeader[I].p_vaddr);
+ const ElfW(Nhdr) *NotesEnd =
+ (const ElfW(Nhdr) *)((const char *)(Note) + ProgramHeader[I].p_memsz);
+
+ int BinaryIdsSize = WriteBinaryIds(Writer, Note, NotesEnd);
+ if (TotalBinaryIdsSize == -1)
+ return -1;
+
+ TotalBinaryIdsSize += BinaryIdsSize;
+ }
+
+ return TotalBinaryIdsSize;
+}
+#elif !defined(_AIX) /* !NT_GNU_BUILD_ID */
+/*
+ * Fallback implementation for targets that don't support the GNU
+ * extensions NT_GNU_BUILD_ID and __ehdr_start.
+ */
+COMPILER_RT_VISIBILITY int __llvm_write_binary_ids(ProfDataWriter *Writer) {
+ return 0;
+}
+#endif
+
+#endif
diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformOther.c b/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformOther.c
new file mode 100644
index 0000000000000..29e570b9fba92
--- /dev/null
+++ b/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformOther.c
@@ -0,0 +1,129 @@
+/*===- InstrProfilingPlatformOther.c - Profile data default platform ------===*\
+|*
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+|* See https://llvm.org/LICENSE.txt for license information.
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+|*
+\*===----------------------------------------------------------------------===*/
+
+#if !defined(__APPLE__) && !defined(__linux__) && !defined(__FreeBSD__) && \
+ !defined(__Fuchsia__) && !(defined(__sun__) && defined(__svr4__)) && \
+ !defined(__NetBSD__) && !defined(_WIN32) && !defined(_AIX) && \
+ !defined(__wasm__) && !defined(__HAIKU__)
+
+#include <stdlib.h>
+#include <stdio.h>
+
+#include "InstrProfiling.h"
+#include "InstrProfilingInternal.h"
+
+static const __llvm_profile_data *DataFirst = NULL;
+static const __llvm_profile_data *DataLast = NULL;
+static const VTableProfData *VTableProfDataFirst = NULL;
+static const VTableProfData *VTableProfDataLast = NULL;
+static const char *NamesFirst = NULL;
+static const char *NamesLast = NULL;
+static const char *VNamesFirst = NULL;
+static const char *VNamesLast = NULL;
+static char *CountersFirst = NULL;
+static char *CountersLast = NULL;
+static uint32_t *OrderFileFirst = NULL;
+
+static const void *getMinAddr(const void *A1, const void *A2) {
+ return A1 < A2 ? A1 : A2;
+}
+
+static const void *getMaxAddr(const void *A1, const void *A2) {
+ return A1 > A2 ? A1 : A2;
+}
+
+/*!
+ * \brief Register an instrumented function.
+ *
+ * Calls to this are emitted by clang with -fprofile-instr-generate. Such
+ * calls are only required (and only emitted) on targets where we haven't
+ * implemented linker magic to find the bounds of the sections.
+ */
+COMPILER_RT_VISIBILITY
+void __llvm_profile_register_function(void *Data_) {
+ /* TODO: Only emit this function if we can't use linker magic. */
+ const __llvm_profile_data *Data = (__llvm_profile_data *)Data_;
+ if (!DataFirst) {
+ DataFirst = Data;
+ DataLast = Data + 1;
+ CountersFirst = (char *)((uintptr_t)Data_ + Data->CounterPtr);
+ CountersLast =
+ CountersFirst + Data->NumCounters * __llvm_profile_counter_entry_size();
+ return;
+ }
+
+ DataFirst = (const __llvm_profile_data *)getMinAddr(DataFirst, Data);
+ CountersFirst = (char *)getMinAddr(
+ CountersFirst, (char *)((uintptr_t)Data_ + Data->CounterPtr));
+
+ DataLast = (const __llvm_profile_data *)getMaxAddr(DataLast, Data + 1);
+ CountersLast = (char *)getMaxAddr(
+ CountersLast,
+ (char *)((uintptr_t)Data_ + Data->CounterPtr) +
+ Data->NumCounters * __llvm_profile_counter_entry_size());
+}
+
+COMPILER_RT_VISIBILITY
+void __llvm_profile_register_names_function(void *NamesStart,
+ uint64_t NamesSize) {
+ if (!NamesFirst) {
+ NamesFirst = (const char *)NamesStart;
+ NamesLast = (const char *)NamesStart + NamesSize;
+ return;
+ }
+ NamesFirst = (const char *)getMinAddr(NamesFirst, NamesStart);
+ NamesLast =
+ (const char *)getMaxAddr(NamesLast, (const char *)NamesStart + NamesSize);
+}
+
+COMPILER_RT_VISIBILITY
+const __llvm_profile_data *__llvm_profile_begin_data(void) { return DataFirst; }
+COMPILER_RT_VISIBILITY
+const __llvm_profile_data *__llvm_profile_end_data(void) { return DataLast; }
+COMPILER_RT_VISIBILITY const VTableProfData *
+__llvm_profile_begin_vtables(void) {
+ return VTableProfDataFirst;
+}
+COMPILER_RT_VISIBILITY const VTableProfData *__llvm_profile_end_vtables(void) {
+ return VTableProfDataLast;
+}
+COMPILER_RT_VISIBILITY
+const char *__llvm_profile_begin_names(void) { return NamesFirst; }
+COMPILER_RT_VISIBILITY
+const char *__llvm_profile_end_names(void) { return NamesLast; }
+COMPILER_RT_VISIBILITY
+const char *__llvm_profile_begin_vtabnames(void) { return VNamesFirst; }
+COMPILER_RT_VISIBILITY
+const char *__llvm_profile_end_vtabnames(void) { return VNamesLast; }
+COMPILER_RT_VISIBILITY
+char *__llvm_profile_begin_counters(void) { return CountersFirst; }
+COMPILER_RT_VISIBILITY
+char *__llvm_profile_end_counters(void) { return CountersLast; }
+COMPILER_RT_VISIBILITY
+char *__llvm_profile_begin_bitmap(void) { return BitmapFirst; }
+COMPILER_RT_VISIBILITY
+char *__llvm_profile_end_bitmap(void) { return BitmapLast; }
+/* TODO: correctly set up OrderFileFirst. */
+COMPILER_RT_VISIBILITY
+uint32_t *__llvm_profile_begin_orderfile(void) { return OrderFileFirst; }
+
+COMPILER_RT_VISIBILITY
+ValueProfNode *__llvm_profile_begin_vnodes(void) {
+ return 0;
+}
+COMPILER_RT_VISIBILITY
+ValueProfNode *__llvm_profile_end_vnodes(void) { return 0; }
+
+COMPILER_RT_VISIBILITY ValueProfNode *CurrentVNode = 0;
+COMPILER_RT_VISIBILITY ValueProfNode *EndVNode = 0;
+
+COMPILER_RT_VISIBILITY int __llvm_write_binary_ids(ProfDataWriter *Writer) {
+ return 0;
+}
+
+#endif
diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c b/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c
new file mode 100644
index 0000000000000..741b01faada4e
--- /dev/null
+++ b/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c
@@ -0,0 +1,111 @@
+/*===- InstrProfilingPlatformWindows.c - Profile data on Windows ----------===*\
+|*
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+|* See https://llvm.org/LICENSE.txt for license information.
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+|*
+\*===----------------------------------------------------------------------===*/
+
+#include <stddef.h>
+
+#include "InstrProfiling.h"
+#include "InstrProfilingInternal.h"
+
+#if defined(_WIN32)
+
+#if defined(_MSC_VER)
+/* Merge read-write sections into .data. */
+#pragma comment(linker, "/MERGE:.lprfb=.data")
+#pragma comment(linker, "/MERGE:.lprfd=.data")
+#pragma comment(linker, "/MERGE:.lprfv=.data")
+#pragma comment(linker, "/MERGE:.lprfnd=.data")
+/* Do *NOT* merge .lprfn and .lcovmap into .rdata. llvm-cov must be able to find
+ * after the fact.
+ * Do *NOT* merge .lprfc .rdata. When binary profile correlation is enabled,
+ * llvm-cov must be able to find after the fact.
+ */
+
+/* Allocate read-only section bounds. */
+#pragma section(".lprfn$A", read)
+#pragma section(".lprfn$Z", read)
+
+/* Allocate read-write section bounds. */
+#pragma section(".lprfd$A", read, write)
+#pragma section(".lprfd$Z", read, write)
+#pragma section(".lprfc$A", read, write)
+#pragma section(".lprfc$Z", read, write)
+#pragma section(".lprfb$A", read, write)
+#pragma section(".lprfb$Z", read, write)
+#pragma section(".lorderfile$A", read, write)
+#pragma section(".lprfnd$A", read, write)
+#pragma section(".lprfnd$Z", read, write)
+#endif
+
+__llvm_profile_data COMPILER_RT_SECTION(".lprfd$A") DataStart = {0};
+__llvm_profile_data COMPILER_RT_SECTION(".lprfd$Z") DataEnd = {0};
+
+const char COMPILER_RT_SECTION(".lprfn$A") NamesStart = '\0';
+const char COMPILER_RT_SECTION(".lprfn$Z") NamesEnd = '\0';
+
+char COMPILER_RT_SECTION(".lprfc$A") CountersStart;
+char COMPILER_RT_SECTION(".lprfc$Z") CountersEnd;
+char COMPILER_RT_SECTION(".lprfb$A") BitmapStart;
+char COMPILER_RT_SECTION(".lprfb$Z") BitmapEnd;
+uint32_t COMPILER_RT_SECTION(".lorderfile$A") OrderFileStart;
+
+ValueProfNode COMPILER_RT_SECTION(".lprfnd$A") VNodesStart;
+ValueProfNode COMPILER_RT_SECTION(".lprfnd$Z") VNodesEnd;
+
+const __llvm_profile_data *__llvm_profile_begin_data(void) {
+ return &DataStart + 1;
+}
+const __llvm_profile_data *__llvm_profile_end_data(void) { return &DataEnd; }
+
+// Type profiling isn't implemented under MSVC ABI, so return NULL (rather than
+// implementing linker magic on Windows) to make it more explicit. To elaborate,
+// the current type profiling implementation maps a profiled vtable address to a
+// vtable variable through vtables mangled name. Under MSVC ABI, the variable
+// name for vtables might not be the mangled name (see
+// MicrosoftCXXABI::getAddrOfVTable in MicrosoftCXXABI.cpp for more details on
+// how a vtable name is computed). Note the mangled name is still in the vtable
+// IR (just not variable name) for mapping purpose, but more implementation work
+// is required.
+const VTableProfData *__llvm_profile_begin_vtables(void) { return NULL; }
+const VTableProfData *__llvm_profile_end_vtables(void) { return NULL; }
+
+const char *__llvm_profile_begin_names(void) { return &NamesStart + 1; }
+const char *__llvm_profile_end_names(void) { return &NamesEnd; }
+
+// Type profiling isn't supported on Windows, so return NULl to make it more
+// explicit.
+const char *__llvm_profile_begin_vtabnames(void) { return NULL; }
+const char *__llvm_profile_end_vtabnames(void) { return NULL; }
+
+char *__llvm_profile_begin_counters(void) { return &CountersStart + 1; }
+char *__llvm_profile_end_counters(void) { return &CountersEnd; }
+char *__llvm_profile_begin_bitmap(void) { return &BitmapStart + 1; }
+char *__llvm_profile_end_bitmap(void) { return &BitmapEnd; }
+uint32_t *__llvm_profile_begin_orderfile(void) { return &OrderFileStart; }
+
+ValueProfNode *__llvm_profile_begin_vnodes(void) { return &VNodesStart + 1; }
+ValueProfNode *__llvm_profile_end_vnodes(void) { return &VNodesEnd; }
+
+ValueProfNode *CurrentVNode = &VNodesStart + 1;
+ValueProfNode *EndVNode = &VNodesEnd;
+
+/* lld-link provides __buildid symbol which points to the 16 bytes build id when
+ * using /build-id flag. https://lld.llvm.org/windows_support.html#lld-flags */
+#define BUILD_ID_LEN 16
+COMPILER_RT_WEAK uint8_t __buildid[BUILD_ID_LEN] = {0};
+COMPILER_RT_VISIBILITY int __llvm_write_binary_ids(ProfDataWriter *Writer) {
+ static const uint8_t zeros[BUILD_ID_LEN] = {0};
+ if (memcmp(__buildid, zeros, BUILD_ID_LEN) != 0) {
+ if (Writer &&
+ lprofWriteOneBinaryId(Writer, BUILD_ID_LEN, __buildid, 0) == -1)
+ return -1;
+ return sizeof(uint64_t) + BUILD_ID_LEN;
+ }
+ return 0;
+}
+
+#endif
diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingPort.h b/system/lib/compiler-rt/lib/profile/InstrProfilingPort.h
new file mode 100644
index 0000000000000..66d697885eaee
--- /dev/null
+++ b/system/lib/compiler-rt/lib/profile/InstrProfilingPort.h
@@ -0,0 +1,152 @@
+/*===- InstrProfilingPort.h- Support library for PGO instrumentation ------===*\
+|*
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+|* See https://llvm.org/LICENSE.txt for license information.
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+|*
+\*===----------------------------------------------------------------------===*/
+
+/* This header must be included after all others so it can provide fallback
+ definitions for stuff missing in system headers. */
+
+#ifndef PROFILE_INSTRPROFILING_PORT_H_
+#define PROFILE_INSTRPROFILING_PORT_H_
+
+#ifdef _MSC_VER
+#define COMPILER_RT_ALIGNAS(x) __declspec(align(x))
+#define COMPILER_RT_VISIBILITY
+/* FIXME: selectany does not have the same semantics as weak. */
+#define COMPILER_RT_WEAK __declspec(selectany)
+/* Need to include <windows.h> */
+#define COMPILER_RT_ALLOCA _alloca
+/* Need to include <stdio.h> and <io.h> */
+#define COMPILER_RT_FTRUNCATE(f,l) _chsize(_fileno(f),l)
+#define COMPILER_RT_ALWAYS_INLINE __forceinline
+#define COMPILER_RT_CLEANUP(x)
+#define COMPILER_RT_USED
+#elif __GNUC__
+#ifdef _WIN32
+#define COMPILER_RT_FTRUNCATE(f, l) _chsize(fileno(f), l)
+#define COMPILER_RT_VISIBILITY
+#define COMPILER_RT_WEAK __attribute__((selectany))
+#else
+#define COMPILER_RT_FTRUNCATE(f, l) ftruncate(fileno(f), l)
+#define COMPILER_RT_VISIBILITY __attribute__((visibility("hidden")))
+#define COMPILER_RT_WEAK __attribute__((weak))
+#endif
+#define COMPILER_RT_ALIGNAS(x) __attribute__((aligned(x)))
+#define COMPILER_RT_ALLOCA __builtin_alloca
+#define COMPILER_RT_ALWAYS_INLINE inline __attribute((always_inline))
+#define COMPILER_RT_CLEANUP(x) __attribute__((cleanup(x)))
+#define COMPILER_RT_USED __attribute__((used))
+#endif
+
+#if defined(__APPLE__)
+#define COMPILER_RT_SEG "__DATA,"
+#else
+#define COMPILER_RT_SEG ""
+#endif
+
+#ifdef _MSC_VER
+#define COMPILER_RT_SECTION(Sect) __declspec(allocate(Sect))
+#else
+#define COMPILER_RT_SECTION(Sect) __attribute__((section(Sect)))
+#endif
+
+#define COMPILER_RT_MAX_HOSTLEN 128
+#if defined(__ORBIS__) || defined(__wasi__)
+#define COMPILER_RT_GETHOSTNAME(Name, Len) ((void)(Name), (void)(Len), (-1))
+#else
+#define COMPILER_RT_GETHOSTNAME(Name, Len) lprofGetHostName(Name, Len)
+#endif
+
+#if COMPILER_RT_HAS_ATOMICS == 1
+#ifdef _WIN32
+#include <windows.h>
+#if defined(_MSC_VER) && _MSC_VER < 1900
+#define snprintf _snprintf
+#endif
+#if defined(_WIN64)
+#define COMPILER_RT_BOOL_CMPXCHG(Ptr, OldV, NewV) \
+ (InterlockedCompareExchange64((LONGLONG volatile *)Ptr, (LONGLONG)NewV, \
+ (LONGLONG)OldV) == (LONGLONG)OldV)
+#define COMPILER_RT_PTR_FETCH_ADD(DomType, PtrVar, PtrIncr) \
+ (DomType *)InterlockedExchangeAdd64((LONGLONG volatile *)&PtrVar, \
+ (LONGLONG)sizeof(DomType) * PtrIncr)
+#else /* !defined(_WIN64) */
+#define COMPILER_RT_BOOL_CMPXCHG(Ptr, OldV, NewV) \
+ (InterlockedCompareExchange((LONG volatile *)Ptr, (LONG)NewV, (LONG)OldV) == \
+ (LONG)OldV)
+#define COMPILER_RT_PTR_FETCH_ADD(DomType, PtrVar, PtrIncr) \
+ (DomType *)InterlockedExchangeAdd((LONG volatile *)&PtrVar, \
+ (LONG)sizeof(DomType) * PtrIncr)
+#endif
+#else /* !defined(_WIN32) */
+#define COMPILER_RT_BOOL_CMPXCHG(Ptr, OldV, NewV) \
+ __sync_bool_compare_and_swap(Ptr, OldV, NewV)
+#define COMPILER_RT_PTR_FETCH_ADD(DomType, PtrVar, PtrIncr) \
+ (DomType *)__sync_fetch_and_add((long *)&PtrVar, sizeof(DomType) * PtrIncr)
+#endif
+#else /* COMPILER_RT_HAS_ATOMICS != 1 */
+#include "InstrProfilingUtil.h"
+#define COMPILER_RT_BOOL_CMPXCHG(Ptr, OldV, NewV) \
+ lprofBoolCmpXchg((void **)Ptr, OldV, NewV)
+#define COMPILER_RT_PTR_FETCH_ADD(DomType, PtrVar, PtrIncr) \
+ (DomType *)lprofPtrFetchAdd((void **)&PtrVar, sizeof(DomType) * PtrIncr)
+#endif
+
+#if defined(_WIN32)
+#define DIR_SEPARATOR '\\'
+#define DIR_SEPARATOR_2 '/'
+#else
+#define DIR_SEPARATOR '/'
+#endif
+
+#ifndef DIR_SEPARATOR_2
+#define IS_DIR_SEPARATOR(ch) ((ch) == DIR_SEPARATOR)
+#else /* DIR_SEPARATOR_2 */
+#define IS_DIR_SEPARATOR(ch) \
+ (((ch) == DIR_SEPARATOR) || ((ch) == DIR_SEPARATOR_2))
+#endif /* DIR_SEPARATOR_2 */
+
+#if defined(_WIN32)
+#include <windows.h>
+static inline size_t getpagesize(void) {
+ SYSTEM_INFO S;
+ GetNativeSystemInfo(&S);
+ return S.dwPageSize;
+}
+#else /* defined(_WIN32) */
+#include <unistd.h>
+#endif /* defined(_WIN32) */
+
+#define PROF_ERR(Format, ...) \
+ fprintf(stderr, "LLVM Profile Error: " Format, __VA_ARGS__);
+
+#define PROF_WARN(Format, ...) \
+ fprintf(stderr, "LLVM Profile Warning: " Format, __VA_ARGS__);
+
+#define PROF_NOTE(Format, ...) \
+ fprintf(stderr, "LLVM Profile Note: " Format, __VA_ARGS__);
+
+#ifndef MAP_FILE
+#define MAP_FILE 0
+#endif
+
+#ifndef O_BINARY
+#define O_BINARY 0
+#endif
+
+#if defined(__FreeBSD__)
+
+#include <inttypes.h>
+#include <sys/types.h>
+
+#else /* defined(__FreeBSD__) */
+
+#include <inttypes.h>
+#include <stdint.h>
+
+#endif /* defined(__FreeBSD__) && defined(__i386__) */
+
+#endif /* PROFILE_INSTRPROFILING_PORT_H_ */
diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingRuntime.cpp b/system/lib/compiler-rt/lib/profile/InstrProfilingRuntime.cpp
new file mode 100644
index 0000000000000..6b2ce97001735
--- /dev/null
+++ b/system/lib/compiler-rt/lib/profile/InstrProfilingRuntime.cpp
@@ -0,0 +1,24 @@
+//===- InstrProfilingRuntime.cpp - PGO runtime initialization -------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+extern "C" {
+
+#include "InstrProfiling.h"
+
+static int RegisterRuntime() {
+ __llvm_profile_initialize();
+#ifdef _AIX
+ extern COMPILER_RT_VISIBILITY void *__llvm_profile_keep[];
+ (void)*(void *volatile *)__llvm_profile_keep;
+#endif
+ return 0;
+}
+
+/* int __llvm_profile_runtime */
+COMPILER_RT_VISIBILITY int INSTR_PROF_PROFILE_RUNTIME_VAR = RegisterRuntime();
+}
diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingUtil.c b/system/lib/compiler-rt/lib/profile/InstrProfilingUtil.c
new file mode 100644
index 0000000000000..c637b9d0b893c
--- /dev/null
+++ b/system/lib/compiler-rt/lib/profile/InstrProfilingUtil.c
@@ -0,0 +1,450 @@
+/*===- InstrProfilingUtil.c - Support library for PGO instrumentation -----===*\
+|*
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+|* See https://llvm.org/LICENSE.txt for license information.
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+|*
+\*===----------------------------------------------------------------------===*/
+
+#ifdef _WIN32
+#include <direct.h>
+#include <process.h>
+#include <windows.h>
+#include "WindowsMMap.h"
+#else
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/file.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
+#endif
+
+#ifdef COMPILER_RT_HAS_UNAME
+#include <sys/utsname.h>
+#endif
+
+#include <stdlib.h>
+#include <string.h>
+
+#if defined(__linux__)
+#include <signal.h>
+#include <sys/prctl.h>
+#endif
+
+#if defined(__Fuchsia__)
+#include <zircon/process.h>
+#include <zircon/syscalls.h>
+#endif
+
+#if defined(__FreeBSD__)
+#include <signal.h>
+#include <sys/procctl.h>
+#endif
+
+#include "InstrProfiling.h"
+#include "InstrProfilingUtil.h"
+
+COMPILER_RT_VISIBILITY unsigned lprofDirMode = 0755;
+
+COMPILER_RT_VISIBILITY
+void __llvm_profile_recursive_mkdir(char *path) {
+ int i;
+ int start = 1;
+
+#if defined(__ANDROID__) && defined(__ANDROID_API__) && \
+ defined(__ANDROID_API_FUTURE__) && \
+ __ANDROID_API__ == __ANDROID_API_FUTURE__
+ // Avoid spammy selinux denial messages in Android by not attempting to
+ // create directories in GCOV_PREFIX. These denials occur when creating (or
+ // even attempting to stat()) top-level directories like "/data".
+ //
+ // Do so by ignoring ${GCOV_PREFIX} when invoking mkdir().
+ const char *gcov_prefix = getenv("GCOV_PREFIX");
+ if (gcov_prefix != NULL) {
+ const int gcov_prefix_len = strlen(gcov_prefix);
+ if (strncmp(path, gcov_prefix, gcov_prefix_len) == 0)
+ start = gcov_prefix_len;
+ }
+#endif
+
+ for (i = start; path[i] != '\0'; ++i) {
+ char save = path[i];
+ if (!IS_DIR_SEPARATOR(path[i]))
+ continue;
+ path[i] = '\0';
+#ifdef _WIN32
+ _mkdir(path);
+#else
+ /* Some of these will fail, ignore it. */
+ mkdir(path, __llvm_profile_get_dir_mode());
+#endif
+ path[i] = save;
+ }
+}
+
+COMPILER_RT_VISIBILITY
+void __llvm_profile_set_dir_mode(unsigned Mode) { lprofDirMode = Mode; }
+
+COMPILER_RT_VISIBILITY
+unsigned __llvm_profile_get_dir_mode(void) { return lprofDirMode; }
+
+#if COMPILER_RT_HAS_ATOMICS != 1
+COMPILER_RT_VISIBILITY
+uint32_t lprofBoolCmpXchg(void **Ptr, void *OldV, void *NewV) {
+ void *R = *Ptr;
+ if (R == OldV) {
+ *Ptr = NewV;
+ return 1;
+ }
+ return 0;
+}
+COMPILER_RT_VISIBILITY
+void *lprofPtrFetchAdd(void **Mem, long ByteIncr) {
+ void *Old = *Mem;
+ *((char **)Mem) += ByteIncr;
+ return Old;
+}
+
+#endif
+
+#ifdef _WIN32
+COMPILER_RT_VISIBILITY int lprofGetHostName(char *Name, int Len) {
+ WCHAR Buffer[COMPILER_RT_MAX_HOSTLEN];
+ DWORD BufferSize = sizeof(Buffer);
+ BOOL Result =
+ GetComputerNameExW(ComputerNameDnsFullyQualified, Buffer, &BufferSize);
+ if (!Result)
+ return -1;
+ if (WideCharToMultiByte(CP_UTF8, 0, Buffer, -1, Name, Len, NULL, NULL) == 0)
+ return -1;
+ return 0;
+}
+#elif defined(COMPILER_RT_HAS_UNAME)
+COMPILER_RT_VISIBILITY int lprofGetHostName(char *Name, int Len) {
+ struct utsname N;
+ int R = uname(&N);
+ if (R >= 0) {
+ strncpy(Name, N.nodename, Len);
+ return 0;
+ }
+ return R;
+}
+#endif
+
+COMPILER_RT_VISIBILITY int lprofLockFd(int fd) {
+#ifdef COMPILER_RT_HAS_FCNTL_LCK
+ struct flock s_flock;
+
+ s_flock.l_whence = SEEK_SET;
+ s_flock.l_start = 0;
+ s_flock.l_len = 0; /* Until EOF. */
+ s_flock.l_pid = getpid();
+ s_flock.l_type = F_WRLCK;
+
+ while (fcntl(fd, F_SETLKW, &s_flock) == -1) {
+ if (errno != EINTR) {
+ if (errno == ENOLCK) {
+ return -1;
+ }
+ break;
+ }
+ }
+ return 0;
+#elif defined(COMPILER_RT_HAS_FLOCK) || defined(_WIN32)
+ // Windows doesn't have flock but WindowsMMap.h provides a shim
+ flock(fd, LOCK_EX);
+ return 0;
+#else
+ return 0;
+#endif
+}
+
+COMPILER_RT_VISIBILITY int lprofUnlockFd(int fd) {
+#ifdef COMPILER_RT_HAS_FCNTL_LCK
+ struct flock s_flock;
+
+ s_flock.l_whence = SEEK_SET;
+ s_flock.l_start = 0;
+ s_flock.l_len = 0; /* Until EOF. */
+ s_flock.l_pid = getpid();
+ s_flock.l_type = F_UNLCK;
+
+ while (fcntl(fd, F_SETLKW, &s_flock) == -1) {
+ if (errno != EINTR) {
+ if (errno == ENOLCK) {
+ return -1;
+ }
+ break;
+ }
+ }
+ return 0;
+#elif defined(COMPILER_RT_HAS_FLOCK) || defined(_WIN32)
+ // Windows doesn't have flock but WindowsMMap.h provides a shim
+ flock(fd, LOCK_UN);
+ return 0;
+#else
+ return 0;
+#endif
+}
+
+COMPILER_RT_VISIBILITY int lprofLockFileHandle(FILE *F) {
+ int fd;
+#if defined(_WIN32)
+ fd = _fileno(F);
+#else
+ fd = fileno(F);
+#endif
+ return lprofLockFd(fd);
+}
+
+COMPILER_RT_VISIBILITY int lprofUnlockFileHandle(FILE *F) {
+ int fd;
+#if defined(_WIN32)
+ fd = _fileno(F);
+#else
+ fd = fileno(F);
+#endif
+ return lprofUnlockFd(fd);
+}
+
+COMPILER_RT_VISIBILITY FILE *lprofOpenFileEx(const char *ProfileName) {
+ FILE *f;
+ int fd;
+#ifdef COMPILER_RT_HAS_FCNTL_LCK
+ fd = open(ProfileName, O_RDWR | O_CREAT, 0666);
+ if (fd < 0)
+ return NULL;
+
+ if (lprofLockFd(fd) != 0)
+ PROF_WARN("Data may be corrupted during profile merging : %s\n",
+ "Fail to obtain file lock due to system limit.");
+
+ f = fdopen(fd, "r+b");
+#elif defined(_WIN32)
+ // FIXME: Use the wide variants to handle Unicode filenames.
+ HANDLE h = CreateFileA(ProfileName, GENERIC_READ | GENERIC_WRITE,
+ FILE_SHARE_READ | FILE_SHARE_WRITE, 0, OPEN_ALWAYS,
+ FILE_ATTRIBUTE_NORMAL, 0);
+ if (h == INVALID_HANDLE_VALUE)
+ return NULL;
+
+ fd = _open_osfhandle((intptr_t)h, 0);
+ if (fd == -1) {
+ CloseHandle(h);
+ return NULL;
+ }
+
+ if (lprofLockFd(fd) != 0)
+ PROF_WARN("Data may be corrupted during profile merging : %s\n",
+ "Fail to obtain file lock due to system limit.");
+
+ f = _fdopen(fd, "r+b");
+ if (f == 0) {
+ CloseHandle(h);
+ return NULL;
+ }
+#else
+ /* Worst case no locking applied. */
+ PROF_WARN("Concurrent file access is not supported : %s\n",
+ "lack file locking");
+ fd = open(ProfileName, O_RDWR | O_CREAT, 0666);
+ if (fd < 0)
+ return NULL;
+ f = fdopen(fd, "r+b");
+#endif
+
+ return f;
+}
+
+COMPILER_RT_VISIBILITY const char *lprofGetPathPrefix(int *PrefixStrip,
+ size_t *PrefixLen) {
+ const char *Prefix = getenv("GCOV_PREFIX");
+ const char *PrefixStripStr = getenv("GCOV_PREFIX_STRIP");
+
+ *PrefixLen = 0;
+ *PrefixStrip = 0;
+ if (Prefix == NULL || Prefix[0] == '\0')
+ return NULL;
+
+ if (PrefixStripStr) {
+ *PrefixStrip = atoi(PrefixStripStr);
+
+ /* Negative GCOV_PREFIX_STRIP values are ignored */
+ if (*PrefixStrip < 0)
+ *PrefixStrip = 0;
+ } else {
+ *PrefixStrip = 0;
+ }
+ *PrefixLen = strlen(Prefix);
+
+ return Prefix;
+}
+
+COMPILER_RT_VISIBILITY void
+lprofApplyPathPrefix(char *Dest, const char *PathStr, const char *Prefix,
+ size_t PrefixLen, int PrefixStrip) {
+
+ const char *Ptr;
+ int Level;
+ const char *StrippedPathStr = PathStr;
+
+ for (Level = 0, Ptr = PathStr + 1; Level < PrefixStrip; ++Ptr) {
+ if (*Ptr == '\0')
+ break;
+
+ if (!IS_DIR_SEPARATOR(*Ptr))
+ continue;
+
+ StrippedPathStr = Ptr;
+ ++Level;
+ }
+
+ memcpy(Dest, Prefix, PrefixLen);
+
+ if (!IS_DIR_SEPARATOR(Prefix[PrefixLen - 1]))
+ Dest[PrefixLen++] = DIR_SEPARATOR;
+
+ memcpy(Dest + PrefixLen, StrippedPathStr, strlen(StrippedPathStr) + 1);
+}
+
+COMPILER_RT_VISIBILITY const char *
+lprofFindFirstDirSeparator(const char *Path) {
+ const char *Sep = strchr(Path, DIR_SEPARATOR);
+#if defined(DIR_SEPARATOR_2)
+ const char *Sep2 = strchr(Path, DIR_SEPARATOR_2);
+ if (Sep2 && (!Sep || Sep2 < Sep))
+ Sep = Sep2;
+#endif
+ return Sep;
+}
+
+COMPILER_RT_VISIBILITY const char *lprofFindLastDirSeparator(const char *Path) {
+ const char *Sep = strrchr(Path, DIR_SEPARATOR);
+#if defined(DIR_SEPARATOR_2)
+ const char *Sep2 = strrchr(Path, DIR_SEPARATOR_2);
+ if (Sep2 && (!Sep || Sep2 > Sep))
+ Sep = Sep2;
+#endif
+ return Sep;
+}
+
+COMPILER_RT_VISIBILITY int lprofSuspendSigKill(void) {
+#if defined(__linux__)
+ int PDeachSig = 0;
+ /* Temporarily suspend getting SIGKILL upon exit of the parent process. */
+ if (prctl(PR_GET_PDEATHSIG, &PDeachSig) == 0 && PDeachSig == SIGKILL)
+ prctl(PR_SET_PDEATHSIG, 0);
+ return (PDeachSig == SIGKILL);
+#elif defined(__FreeBSD__)
+ int PDeachSig = 0, PDisableSig = 0;
+ if (procctl(P_PID, 0, PROC_PDEATHSIG_STATUS, &PDeachSig) == 0 &&
+ PDeachSig == SIGKILL)
+ procctl(P_PID, 0, PROC_PDEATHSIG_CTL, &PDisableSig);
+ return (PDeachSig == SIGKILL);
+#else
+ return 0;
+#endif
+}
+
+COMPILER_RT_VISIBILITY void lprofRestoreSigKill(void) {
+#if defined(__linux__)
+ prctl(PR_SET_PDEATHSIG, SIGKILL);
+#elif defined(__FreeBSD__)
+ int PEnableSig = SIGKILL;
+ procctl(P_PID, 0, PROC_PDEATHSIG_CTL, &PEnableSig);
+#endif
+}
+
+COMPILER_RT_VISIBILITY int lprofReleaseMemoryPagesToOS(uintptr_t Begin,
+ uintptr_t End) {
+#if defined(__ve__) || defined(__wasi__)
+ // VE and WASI doesn't support madvise.
+ return 0;
+#else
+ size_t PageSize = getpagesize();
+ uintptr_t BeginAligned = lprofRoundUpTo((uintptr_t)Begin, PageSize);
+ uintptr_t EndAligned = lprofRoundDownTo((uintptr_t)End, PageSize);
+ if (BeginAligned < EndAligned) {
+#if defined(__Fuchsia__)
+ return _zx_vmar_op_range(_zx_vmar_root_self(), ZX_VMAR_OP_DECOMMIT,
+ (zx_vaddr_t)BeginAligned,
+ EndAligned - BeginAligned, NULL, 0);
+#else
+ return madvise((void *)BeginAligned, EndAligned - BeginAligned,
+ MADV_DONTNEED);
+#endif
+ }
+ return 0;
+#endif
+}
+
+#ifdef _AIX
+typedef struct fn_node {
+ AtExit_Fn_ptr func;
+ struct fn_node *next;
+} fn_node;
+typedef struct {
+ fn_node *top;
+} fn_stack;
+
+static void fn_stack_push(fn_stack *, AtExit_Fn_ptr);
+static AtExit_Fn_ptr fn_stack_pop(fn_stack *);
+/* return 1 if stack is empty, 0 otherwise */
+static int fn_stack_is_empty(fn_stack *);
+
+static fn_stack AtExit_stack = {0};
+#define ATEXIT_STACK (&AtExit_stack)
+
+/* On AIX, atexit() functions registered by a shared library do not get called
+ * when the library is dlclose'd, causing a crash when they are eventually
+ * called at main program exit. However, a destructor does get called. So we
+ * collect all atexit functions registered by profile-rt and at program
+ * termination time (normal exit, shared library unload, or dlclose) we walk
+ * the list and execute any function that is still sitting in the atexit system
+ * queue.
+ */
+__attribute__((__destructor__)) static void cleanup() {
+ while (!fn_stack_is_empty(ATEXIT_STACK)) {
+ AtExit_Fn_ptr func = fn_stack_pop(ATEXIT_STACK);
+ if (func && unatexit(func) == 0)
+ func();
+ }
+}
+
+static void fn_stack_push(fn_stack *st, AtExit_Fn_ptr func) {
+ fn_node *old_top, *n = (fn_node *)malloc(sizeof(fn_node));
+ n->func = func;
+
+ while (1) {
+ old_top = st->top;
+ n->next = old_top;
+ if (COMPILER_RT_BOOL_CMPXCHG(&st->top, old_top, n))
+ return;
+ }
+}
+static AtExit_Fn_ptr fn_stack_pop(fn_stack *st) {
+ fn_node *old_top, *new_top;
+ while (1) {
+ old_top = st->top;
+ if (old_top == 0)
+ return 0;
+ new_top = old_top->next;
+ if (COMPILER_RT_BOOL_CMPXCHG(&st->top, old_top, new_top)) {
+ AtExit_Fn_ptr func = old_top->func;
+ free(old_top);
+ return func;
+ }
+ }
+}
+
+static int fn_stack_is_empty(fn_stack *st) { return st->top == 0; }
+#endif
+
+COMPILER_RT_VISIBILITY int lprofAtExit(AtExit_Fn_ptr func) {
+#ifdef _AIX
+ fn_stack_push(ATEXIT_STACK, func);
+#endif
+ return atexit(func);
+}
diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingUtil.h b/system/lib/compiler-rt/lib/profile/InstrProfilingUtil.h
new file mode 100644
index 0000000000000..227c2aa0a7cae
--- /dev/null
+++ b/system/lib/compiler-rt/lib/profile/InstrProfilingUtil.h
@@ -0,0 +1,92 @@
+/*===- InstrProfilingUtil.h - Support library for PGO instrumentation -----===*\
+|*
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+|* See https://llvm.org/LICENSE.txt for license information.
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+|*
+\*===----------------------------------------------------------------------===*/
+
+#ifndef PROFILE_INSTRPROFILINGUTIL_H
+#define PROFILE_INSTRPROFILINGUTIL_H
+
+#include <inttypes.h>
+#include <stddef.h>
+#include <stdio.h>
+
+/*! \brief Create a directory tree. */
+void __llvm_profile_recursive_mkdir(char *Pathname);
+
+/*! Set the mode used when creating profile directories. */
+void __llvm_profile_set_dir_mode(unsigned Mode);
+
+/*! Return the directory creation mode. */
+unsigned __llvm_profile_get_dir_mode(void);
+
+int lprofLockFd(int fd);
+int lprofUnlockFd(int fd);
+int lprofLockFileHandle(FILE *F);
+int lprofUnlockFileHandle(FILE *F);
+
+/*! Open file \c Filename for read+write with write
+ * lock for exclusive access. The caller will block
+ * if the lock is already held by another process. */
+FILE *lprofOpenFileEx(const char *Filename);
+/* PS4 doesn't have setenv/getenv/fork. Define a shim. */
+#if __ORBIS__
+#include <sys/types.h>
+static inline char *getenv(const char *name) { return NULL; }
+static inline int setenv(const char *name, const char *value, int overwrite)
+{ return 0; }
+static pid_t fork() { return -1; }
+#endif /* #if __ORBIS__ */
+
+/* GCOV_PREFIX and GCOV_PREFIX_STRIP support */
+/* Return the path prefix specified by GCOV_PREFIX environment variable.
+ * If GCOV_PREFIX_STRIP is also specified, the strip level (integer value)
+ * is returned via \c *PrefixStrip. The prefix length is stored in *PrefixLen.
+ */
+const char *lprofGetPathPrefix(int *PrefixStrip, size_t *PrefixLen);
+/* Apply the path prefix specified in \c Prefix to path string in \c PathStr,
+ * and store the result to buffer pointed to by \c Buffer. If \c PrefixStrip
+ * is not zero, path prefixes are stripped from \c PathStr (the level of
+ * stripping is specified by \c PrefixStrip) before \c Prefix is added.
+ */
+void lprofApplyPathPrefix(char *Dest, const char *PathStr, const char *Prefix,
+ size_t PrefixLen, int PrefixStrip);
+
+/* Returns a pointer to the first occurrence of \c DIR_SEPARATOR char in
+ * the string \c Path, or NULL if the char is not found. */
+const char *lprofFindFirstDirSeparator(const char *Path);
+/* Returns a pointer to the last occurrence of \c DIR_SEPARATOR char in
+ * the string \c Path, or NULL if the char is not found. */
+const char *lprofFindLastDirSeparator(const char *Path);
+
+int lprofGetHostName(char *Name, int Len);
+
+unsigned lprofBoolCmpXchg(void **Ptr, void *OldV, void *NewV);
+void *lprofPtrFetchAdd(void **Mem, long ByteIncr);
+
+/* Temporarily suspend SIGKILL. Return value of 1 means a restore is needed.
+ * Other return values mean no restore is needed.
+ */
+int lprofSuspendSigKill(void);
+
+/* Restore previously suspended SIGKILL. */
+void lprofRestoreSigKill(void);
+
+static inline size_t lprofRoundUpTo(size_t x, size_t boundary) {
+ return (x + boundary - 1) & ~(boundary - 1);
+}
+
+static inline size_t lprofRoundDownTo(size_t x, size_t boundary) {
+ return x & ~(boundary - 1);
+}
+
+int lprofReleaseMemoryPagesToOS(uintptr_t Begin, uintptr_t End);
+
+typedef void (*AtExit_Fn_ptr)(void);
+
+/* Call atexit and perform other platform-specific bookkeeping. */
+int lprofAtExit(AtExit_Fn_ptr);
+
+#endif /* PROFILE_INSTRPROFILINGUTIL_H */
diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingValue.c b/system/lib/compiler-rt/lib/profile/InstrProfilingValue.c
new file mode 100644
index 0000000000000..a608d41d39e77
--- /dev/null
+++ b/system/lib/compiler-rt/lib/profile/InstrProfilingValue.c
@@ -0,0 +1,370 @@
+/*===- InstrProfilingValue.c - Support library for PGO instrumentation ----===*\
+|*
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+|* See https://llvm.org/LICENSE.txt for license information.
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+|*
+\*===----------------------------------------------------------------------===*/
+
+#include <assert.h>
+#include <limits.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "InstrProfiling.h"
+#include "InstrProfilingInternal.h"
+#include "InstrProfilingUtil.h"
+
+#define INSTR_PROF_VALUE_PROF_DATA
+#define INSTR_PROF_COMMON_API_IMPL
+#define INSTR_PROF_VALUE_PROF_MEMOP_API
+#include "profile/InstrProfData.inc"
+
+static int hasStaticCounters = 1;
+static int OutOfNodesWarnings = 0;
+static int hasNonDefaultValsPerSite = 0;
+#define INSTR_PROF_MAX_VP_WARNS 10
+#define INSTR_PROF_DEFAULT_NUM_VAL_PER_SITE 24
+#define INSTR_PROF_VNODE_POOL_SIZE 1024
+
+#ifndef _MSC_VER
+/* A shared static pool in addition to the vnodes statically
+ * allocated by the compiler. */
+COMPILER_RT_VISIBILITY ValueProfNode
+ lprofValueProfNodes[INSTR_PROF_VNODE_POOL_SIZE] COMPILER_RT_SECTION(
+ COMPILER_RT_SEG INSTR_PROF_VNODES_SECT_NAME);
+#endif
+
+COMPILER_RT_VISIBILITY uint32_t VPMaxNumValsPerSite =
+ INSTR_PROF_DEFAULT_NUM_VAL_PER_SITE;
+
+COMPILER_RT_VISIBILITY void lprofSetupValueProfiler(void) {
+ const char *Str = 0;
+ Str = getenv("LLVM_VP_MAX_NUM_VALS_PER_SITE");
+ if (Str && Str[0]) {
+ VPMaxNumValsPerSite = atoi(Str);
+ hasNonDefaultValsPerSite = 1;
+ }
+ if (VPMaxNumValsPerSite > INSTR_PROF_MAX_NUM_VAL_PER_SITE)
+ VPMaxNumValsPerSite = INSTR_PROF_MAX_NUM_VAL_PER_SITE;
+}
+
+COMPILER_RT_VISIBILITY void lprofSetMaxValsPerSite(uint32_t MaxVals) {
+ VPMaxNumValsPerSite = MaxVals;
+ hasNonDefaultValsPerSite = 1;
+}
+
+/* This method is only used in value profiler mock testing. */
+COMPILER_RT_VISIBILITY void
+__llvm_profile_set_num_value_sites(__llvm_profile_data *Data,
+ uint32_t ValueKind, uint16_t NumValueSites) {
+#ifdef __GNUC__
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wcast-qual"
+#elif defined(__clang__)
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wcast-qual"
+#endif
+ *((uint16_t *)&Data->NumValueSites[ValueKind]) = NumValueSites;
+#ifdef __GNUC__
+#pragma GCC diagnostic pop
+#elif defined(__clang__)
+#pragma clang diagnostic pop
+#endif
+}
+
+/* This method is only used in value profiler mock testing. */
+COMPILER_RT_VISIBILITY const __llvm_profile_data *
+__llvm_profile_iterate_data(const __llvm_profile_data *Data) {
+ return Data + 1;
+}
+
+/* This method is only used in value profiler mock testing. */
+COMPILER_RT_VISIBILITY void *
+__llvm_get_function_addr(const __llvm_profile_data *Data) {
+ return Data->FunctionPointer;
+}
+
+/* Allocate an array that holds the pointers to the linked lists of
+ * value profile counter nodes. The number of element of the array
+ * is the total number of value profile sites instrumented. Returns
+ * 0 if allocation fails.
+ */
+
+static int allocateValueProfileCounters(__llvm_profile_data *Data) {
+ uint64_t NumVSites = 0;
+ uint32_t VKI;
+
+ /* This function will never be called when value site array is allocated
+ statically at compile time. */
+ hasStaticCounters = 0;
+ /* When dynamic allocation is enabled, allow tracking the max number of
+ * values allowd. */
+ if (!hasNonDefaultValsPerSite)
+ VPMaxNumValsPerSite = INSTR_PROF_MAX_NUM_VAL_PER_SITE;
+
+ for (VKI = IPVK_First; VKI <= IPVK_Last; ++VKI)
+ NumVSites += Data->NumValueSites[VKI];
+
+ // If NumVSites = 0, calloc is allowed to return a non-null pointer.
+ assert(NumVSites > 0 && "NumVSites can't be zero");
+ ValueProfNode **Mem =
+ (ValueProfNode **)calloc(NumVSites, sizeof(ValueProfNode *));
+ if (!Mem)
+ return 0;
+ if (!COMPILER_RT_BOOL_CMPXCHG(&Data->Values, 0, Mem)) {
+ free(Mem);
+ return 0;
+ }
+ return 1;
+}
+
+static ValueProfNode *allocateOneNode(void) {
+ ValueProfNode *Node;
+
+ if (!hasStaticCounters)
+ return (ValueProfNode *)calloc(1, sizeof(ValueProfNode));
+
+ /* Early check to avoid value wrapping around. */
+ if (CurrentVNode + 1 > EndVNode) {
+ if (OutOfNodesWarnings++ < INSTR_PROF_MAX_VP_WARNS) {
+ PROF_WARN("Unable to track new values: %s. "
+ " Consider using option -mllvm -vp-counters-per-site=<n> to "
+ "allocate more"
+ " value profile counters at compile time. \n",
+ "Running out of static counters");
+ }
+ return 0;
+ }
+ Node = COMPILER_RT_PTR_FETCH_ADD(ValueProfNode, CurrentVNode, 1);
+ /* Due to section padding, EndVNode point to a byte which is one pass
+ * an incomplete VNode, so we need to skip the last incomplete node. */
+ if (Node + 1 > EndVNode)
+ return 0;
+
+ return Node;
+}
+
+static COMPILER_RT_ALWAYS_INLINE void
+instrumentTargetValueImpl(uint64_t TargetValue, void *Data,
+ uint32_t CounterIndex, uint64_t CountValue) {
+ __llvm_profile_data *PData = (__llvm_profile_data *)Data;
+ if (!PData)
+ return;
+ if (!CountValue)
+ return;
+ if (!PData->Values) {
+ if (!allocateValueProfileCounters(PData))
+ return;
+ }
+
+ ValueProfNode **ValueCounters = (ValueProfNode **)PData->Values;
+ ValueProfNode *PrevVNode = NULL;
+ ValueProfNode *MinCountVNode = NULL;
+ ValueProfNode *CurVNode = ValueCounters[CounterIndex];
+ uint64_t MinCount = UINT64_MAX;
+
+ uint8_t VDataCount = 0;
+ while (CurVNode) {
+ if (TargetValue == CurVNode->Value) {
+ CurVNode->Count += CountValue;
+ return;
+ }
+ if (CurVNode->Count < MinCount) {
+ MinCount = CurVNode->Count;
+ MinCountVNode = CurVNode;
+ }
+ PrevVNode = CurVNode;
+ CurVNode = CurVNode->Next;
+ ++VDataCount;
+ }
+
+ if (VDataCount >= VPMaxNumValsPerSite) {
+ /* Bump down the min count node's count. If it reaches 0,
+ * evict it. This eviction/replacement policy makes hot
+ * targets more sticky while cold targets less so. In other
+ * words, it makes it less likely for the hot targets to be
+ * prematurally evicted during warmup/establishment period,
+ * when their counts are still low. In a special case when
+ * the number of values tracked is reduced to only one, this
+ * policy will guarantee that the dominating target with >50%
+ * total count will survive in the end. Note that this scheme
+ * allows the runtime to track the min count node in an adaptive
+ * manner. It can correct previous mistakes and eventually
+ * lock on a cold target that is alread in stable state.
+ *
+ * In very rare cases, this replacement scheme may still lead
+ * to target loss. For instance, out of \c N value slots, \c N-1
+ * slots are occupied by luke warm targets during the warmup
+ * period and the remaining one slot is competed by two or more
+ * very hot targets. If those hot targets occur in an interleaved
+ * way, none of them will survive (gain enough weight to throw out
+ * other established entries) due to the ping-pong effect.
+ * To handle this situation, user can choose to increase the max
+ * number of tracked values per value site. Alternatively, a more
+ * expensive eviction mechanism can be implemented. It requires
+ * the runtime to track the total number of evictions per-site.
+ * When the total number of evictions reaches certain threshold,
+ * the runtime can wipe out more than one lowest count entries
+ * to give space for hot targets.
+ */
+ if (MinCountVNode->Count <= CountValue) {
+ CurVNode = MinCountVNode;
+ CurVNode->Value = TargetValue;
+ CurVNode->Count = CountValue;
+ } else
+ MinCountVNode->Count -= CountValue;
+
+ return;
+ }
+
+ CurVNode = allocateOneNode();
+ if (!CurVNode)
+ return;
+ CurVNode->Value = TargetValue;
+ CurVNode->Count += CountValue;
+
+ uint32_t Success = 0;
+ if (!ValueCounters[CounterIndex])
+ Success =
+ COMPILER_RT_BOOL_CMPXCHG(&ValueCounters[CounterIndex], 0, CurVNode);
+ else if (PrevVNode && !PrevVNode->Next)
+ Success = COMPILER_RT_BOOL_CMPXCHG(&(PrevVNode->Next), 0, CurVNode);
+
+ if (!Success && !hasStaticCounters) {
+ free(CurVNode);
+ return;
+ }
+}
+
+COMPILER_RT_VISIBILITY void
+__llvm_profile_instrument_target(uint64_t TargetValue, void *Data,
+ uint32_t CounterIndex) {
+ instrumentTargetValueImpl(TargetValue, Data, CounterIndex, 1);
+}
+COMPILER_RT_VISIBILITY void
+__llvm_profile_instrument_target_value(uint64_t TargetValue, void *Data,
+ uint32_t CounterIndex,
+ uint64_t CountValue) {
+ instrumentTargetValueImpl(TargetValue, Data, CounterIndex, CountValue);
+}
+
+/*
+ * The target values are partitioned into multiple ranges. The range spec is
+ * defined in InstrProfData.inc.
+ */
+COMPILER_RT_VISIBILITY void
+__llvm_profile_instrument_memop(uint64_t TargetValue, void *Data,
+ uint32_t CounterIndex) {
+ // Map the target value to the representative value of its range.
+ uint64_t RepValue = InstrProfGetRangeRepValue(TargetValue);
+ __llvm_profile_instrument_target(RepValue, Data, CounterIndex);
+}
+
+/*
+ * A wrapper struct that represents value profile runtime data.
+ * Like InstrProfRecord class which is used by profiling host tools,
+ * ValueProfRuntimeRecord also implements the abstract interfaces defined in
+ * ValueProfRecordClosure so that the runtime data can be serialized using
+ * shared C implementation.
+ */
+typedef struct ValueProfRuntimeRecord {
+ const __llvm_profile_data *Data;
+ ValueProfNode **NodesKind[IPVK_Last + 1];
+ uint8_t **SiteCountArray;
+} ValueProfRuntimeRecord;
+
+/* ValueProfRecordClosure Interface implementation. */
+
+static uint32_t getNumValueSitesRT(const void *R, uint32_t VK) {
+ return ((const ValueProfRuntimeRecord *)R)->Data->NumValueSites[VK];
+}
+
+static uint32_t getNumValueDataRT(const void *R, uint32_t VK) {
+ uint32_t S = 0, I;
+ const ValueProfRuntimeRecord *Record = (const ValueProfRuntimeRecord *)R;
+ if (Record->SiteCountArray[VK] == INSTR_PROF_NULLPTR)
+ return 0;
+ for (I = 0; I < Record->Data->NumValueSites[VK]; I++)
+ S += Record->SiteCountArray[VK][I];
+ return S;
+}
+
+static uint32_t getNumValueDataForSiteRT(const void *R, uint32_t VK,
+ uint32_t S) {
+ const ValueProfRuntimeRecord *Record = (const ValueProfRuntimeRecord *)R;
+ return Record->SiteCountArray[VK][S];
+}
+
+static ValueProfRuntimeRecord RTRecord;
+static ValueProfRecordClosure RTRecordClosure = {
+ &RTRecord, INSTR_PROF_NULLPTR, /* GetNumValueKinds */
+ getNumValueSitesRT, getNumValueDataRT, getNumValueDataForSiteRT,
+ INSTR_PROF_NULLPTR, /* RemapValueData */
+ INSTR_PROF_NULLPTR, /* GetValueForSite, */
+ INSTR_PROF_NULLPTR /* AllocValueProfData */
+};
+
+static uint32_t
+initializeValueProfRuntimeRecord(const __llvm_profile_data *Data,
+ uint8_t *SiteCountArray[]) {
+ unsigned I, J, S = 0, NumValueKinds = 0;
+ ValueProfNode **Nodes = (ValueProfNode **)Data->Values;
+ RTRecord.Data = Data;
+ RTRecord.SiteCountArray = SiteCountArray;
+ for (I = 0; I <= IPVK_Last; I++) {
+ uint16_t N = Data->NumValueSites[I];
+ if (!N)
+ continue;
+
+ NumValueKinds++;
+
+ RTRecord.NodesKind[I] = Nodes ? &Nodes[S] : INSTR_PROF_NULLPTR;
+ for (J = 0; J < N; J++) {
+ /* Compute value count for each site. */
+ uint32_t C = 0;
+ ValueProfNode *Site =
+ Nodes ? RTRecord.NodesKind[I][J] : INSTR_PROF_NULLPTR;
+ while (Site) {
+ C++;
+ Site = Site->Next;
+ }
+ if (C > UCHAR_MAX)
+ C = UCHAR_MAX;
+ RTRecord.SiteCountArray[I][J] = C;
+ }
+ S += N;
+ }
+ return NumValueKinds;
+}
+
+static ValueProfNode *getNextNValueData(uint32_t VK, uint32_t Site,
+ InstrProfValueData *Dst,
+ ValueProfNode *StartNode, uint32_t N) {
+ unsigned I;
+ ValueProfNode *VNode = StartNode ? StartNode : RTRecord.NodesKind[VK][Site];
+ for (I = 0; I < N; I++) {
+ Dst[I].Value = VNode->Value;
+ Dst[I].Count = VNode->Count;
+ VNode = VNode->Next;
+ }
+ return VNode;
+}
+
+static uint32_t getValueProfDataSizeWrapper(void) {
+ return getValueProfDataSize(&RTRecordClosure);
+}
+
+static uint32_t getNumValueDataForSiteWrapper(uint32_t VK, uint32_t S) {
+ return getNumValueDataForSiteRT(&RTRecord, VK, S);
+}
+
+static VPDataReaderType TheVPDataReader = {
+ initializeValueProfRuntimeRecord, getValueProfRecordHeaderSize,
+ getFirstValueProfRecord, getNumValueDataForSiteWrapper,
+ getValueProfDataSizeWrapper, getNextNValueData};
+
+COMPILER_RT_VISIBILITY VPDataReaderType *lprofGetVPDataReader(void) {
+ return &TheVPDataReader;
+}
diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingVersionVar.c b/system/lib/compiler-rt/lib/profile/InstrProfilingVersionVar.c
new file mode 100644
index 0000000000000..21400bfb2caa0
--- /dev/null
+++ b/system/lib/compiler-rt/lib/profile/InstrProfilingVersionVar.c
@@ -0,0 +1,18 @@
+/*===- InstrProfilingVersionVar.c - profile version variable setup -------===*\
+|*
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+|* See https://llvm.org/LICENSE.txt for license information.
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+|*
+\*===----------------------------------------------------------------------===*/
+
+#include "InstrProfiling.h"
+
+/* uint64 __llvm_profile_raw_version
+ *
+ * The runtime should only provide its own definition of this symbol when the
+ * user has not specified one. Set this up by moving the runtime's copy of this
+ * symbol to an object file within the archive.
+ */
+COMPILER_RT_VISIBILITY COMPILER_RT_WEAK uint64_t INSTR_PROF_RAW_VERSION_VAR =
+ INSTR_PROF_RAW_VERSION;
diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingWriter.c b/system/lib/compiler-rt/lib/profile/InstrProfilingWriter.c
new file mode 100644
index 0000000000000..8816a71155511
--- /dev/null
+++ b/system/lib/compiler-rt/lib/profile/InstrProfilingWriter.c
@@ -0,0 +1,379 @@
+/*===- InstrProfilingWriter.c - Write instrumentation to a file or buffer -===*\
+|*
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+|* See https://llvm.org/LICENSE.txt for license information.
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+|*
+\*===----------------------------------------------------------------------===*/
+
+// Note: This is linked into the Darwin kernel, and must remain compatible
+// with freestanding compilation. See `darwin_add_builtin_libraries`.
+
+#ifdef _MSC_VER
+/* For _alloca */
+#include <malloc.h>
+#endif
+#include <string.h>
+
+#include "InstrProfiling.h"
+#include "InstrProfilingInternal.h"
+#include "InstrProfilingPort.h"
+
+#define INSTR_PROF_VALUE_PROF_DATA
+#include "profile/InstrProfData.inc"
+
+COMPILER_RT_VISIBILITY void (*FreeHook)(void *) = NULL;
+static ProfBufferIO TheBufferIO;
+#define VP_BUFFER_SIZE 8 * 1024
+static uint8_t BufferIOBuffer[VP_BUFFER_SIZE];
+static InstrProfValueData VPDataArray[16];
+static uint32_t VPDataArraySize = sizeof(VPDataArray) / sizeof(*VPDataArray);
+
+COMPILER_RT_VISIBILITY uint8_t *DynamicBufferIOBuffer = 0;
+COMPILER_RT_VISIBILITY uint32_t VPBufferSize = 0;
+
+/* The buffer writer is responsible in keeping writer state
+ * across the call.
+ */
+COMPILER_RT_VISIBILITY uint32_t lprofBufferWriter(ProfDataWriter *This,
+ ProfDataIOVec *IOVecs,
+ uint32_t NumIOVecs) {
+ uint32_t I;
+ char **Buffer = (char **)&This->WriterCtx;
+ for (I = 0; I < NumIOVecs; I++) {
+ size_t Length = IOVecs[I].ElmSize * IOVecs[I].NumElm;
+ if (IOVecs[I].Data)
+ memcpy(*Buffer, IOVecs[I].Data, Length);
+ else if (IOVecs[I].UseZeroPadding) {
+ /* Allocating the buffer should zero fill. */
+ }
+ *Buffer += Length;
+ }
+ return 0;
+}
+
+static void llvmInitBufferIO(ProfBufferIO *BufferIO, ProfDataWriter *FileWriter,
+ uint8_t *Buffer, uint32_t BufferSz) {
+ BufferIO->FileWriter = FileWriter;
+ BufferIO->OwnFileWriter = 0;
+ BufferIO->BufferStart = Buffer;
+ BufferIO->BufferSz = BufferSz;
+ BufferIO->CurOffset = 0;
+}
+
+COMPILER_RT_VISIBILITY ProfBufferIO *
+lprofCreateBufferIO(ProfDataWriter *FileWriter) {
+ uint8_t *Buffer = DynamicBufferIOBuffer;
+ uint32_t BufferSize = VPBufferSize;
+ if (!Buffer) {
+ Buffer = &BufferIOBuffer[0];
+ BufferSize = sizeof(BufferIOBuffer);
+ }
+ llvmInitBufferIO(&TheBufferIO, FileWriter, Buffer, BufferSize);
+ return &TheBufferIO;
+}
+
+COMPILER_RT_VISIBILITY void lprofDeleteBufferIO(ProfBufferIO *BufferIO) {
+ if (BufferIO->OwnFileWriter)
+ FreeHook(BufferIO->FileWriter);
+ if (DynamicBufferIOBuffer) {
+ FreeHook(DynamicBufferIOBuffer);
+ DynamicBufferIOBuffer = 0;
+ VPBufferSize = 0;
+ }
+}
+
+COMPILER_RT_VISIBILITY int
+lprofBufferIOWrite(ProfBufferIO *BufferIO, const uint8_t *Data, uint32_t Size) {
+ /* Buffer is not large enough, it is time to flush. */
+ if (Size + BufferIO->CurOffset > BufferIO->BufferSz) {
+ if (lprofBufferIOFlush(BufferIO) != 0)
+ return -1;
+ }
+ /* Special case, bypass the buffer completely. */
+ ProfDataIOVec IO[] = {{Data, sizeof(uint8_t), Size, 0}};
+ if (Size > BufferIO->BufferSz) {
+ if (BufferIO->FileWriter->Write(BufferIO->FileWriter, IO, 1))
+ return -1;
+ } else {
+ /* Write the data to buffer */
+ uint8_t *Buffer = BufferIO->BufferStart + BufferIO->CurOffset;
+ ProfDataWriter BufferWriter;
+ initBufferWriter(&BufferWriter, (char *)Buffer);
+ lprofBufferWriter(&BufferWriter, IO, 1);
+ BufferIO->CurOffset =
+ (uint8_t *)BufferWriter.WriterCtx - BufferIO->BufferStart;
+ }
+ return 0;
+}
+
+COMPILER_RT_VISIBILITY int lprofBufferIOFlush(ProfBufferIO *BufferIO) {
+ if (BufferIO->CurOffset) {
+ ProfDataIOVec IO[] = {
+ {BufferIO->BufferStart, sizeof(uint8_t), BufferIO->CurOffset, 0}};
+ if (BufferIO->FileWriter->Write(BufferIO->FileWriter, IO, 1))
+ return -1;
+ BufferIO->CurOffset = 0;
+ }
+ return 0;
+}
+
+/* Write out value profile data for function specified with \c Data.
+ * The implementation does not use the method \c serializeValueProfData
+ * which depends on dynamic memory allocation. In this implementation,
+ * value profile data is written out to \c BufferIO piecemeal.
+ */
+static int writeOneValueProfData(ProfBufferIO *BufferIO,
+ VPDataReaderType *VPDataReader,
+ const __llvm_profile_data *Data) {
+ unsigned I, NumValueKinds = 0;
+ ValueProfData VPHeader;
+ uint8_t *SiteCountArray[IPVK_Last + 1];
+
+ for (I = 0; I <= IPVK_Last; I++) {
+ if (!Data->NumValueSites[I])
+ SiteCountArray[I] = 0;
+ else {
+ uint32_t Sz =
+ VPDataReader->GetValueProfRecordHeaderSize(Data->NumValueSites[I]) -
+ offsetof(ValueProfRecord, SiteCountArray);
+ /* Only use alloca for this small byte array to avoid excessive
+ * stack growth. */
+ SiteCountArray[I] = (uint8_t *)COMPILER_RT_ALLOCA(Sz);
+ memset(SiteCountArray[I], 0, Sz);
+ }
+ }
+
+ /* If NumValueKinds returned is 0, there is nothing to write, report
+ success and return. This should match the raw profile reader's behavior. */
+ if (!(NumValueKinds = VPDataReader->InitRTRecord(Data, SiteCountArray)))
+ return 0;
+
+ /* First write the header structure. */
+ VPHeader.TotalSize = VPDataReader->GetValueProfDataSize();
+ VPHeader.NumValueKinds = NumValueKinds;
+ if (lprofBufferIOWrite(BufferIO, (const uint8_t *)&VPHeader,
+ sizeof(ValueProfData)))
+ return -1;
+
+ /* Make sure nothing else needs to be written before value profile
+ * records. */
+ if ((void *)VPDataReader->GetFirstValueProfRecord(&VPHeader) !=
+ (void *)(&VPHeader + 1))
+ return -1;
+
+ /* Write out the value profile record for each value kind
+ * one by one. */
+ for (I = 0; I <= IPVK_Last; I++) {
+ uint32_t J;
+ ValueProfRecord RecordHeader;
+ /* The size of the value prof record header without counting the
+ * site count array .*/
+ uint32_t RecordHeaderSize = offsetof(ValueProfRecord, SiteCountArray);
+ uint32_t SiteCountArraySize;
+
+ if (!Data->NumValueSites[I])
+ continue;
+
+ /* Write out the record header. */
+ RecordHeader.Kind = I;
+ RecordHeader.NumValueSites = Data->NumValueSites[I];
+ if (lprofBufferIOWrite(BufferIO, (const uint8_t *)&RecordHeader,
+ RecordHeaderSize))
+ return -1;
+
+ /* Write out the site value count array including padding space. */
+ SiteCountArraySize =
+ VPDataReader->GetValueProfRecordHeaderSize(Data->NumValueSites[I]) -
+ RecordHeaderSize;
+ if (lprofBufferIOWrite(BufferIO, SiteCountArray[I], SiteCountArraySize))
+ return -1;
+
+ /* Write out the value profile data for each value site. */
+ for (J = 0; J < Data->NumValueSites[I]; J++) {
+ uint32_t NRead, NRemain;
+ ValueProfNode *NextStartNode = 0;
+ NRemain = VPDataReader->GetNumValueDataForSite(I, J);
+ if (!NRemain)
+ continue;
+ /* Read and write out value data in small chunks till it is done. */
+ do {
+ NRead = (NRemain > VPDataArraySize ? VPDataArraySize : NRemain);
+ NextStartNode =
+ VPDataReader->GetValueData(I, /* ValueKind */
+ J, /* Site */
+ &VPDataArray[0], NextStartNode, NRead);
+ if (lprofBufferIOWrite(BufferIO, (const uint8_t *)&VPDataArray[0],
+ NRead * sizeof(InstrProfValueData)))
+ return -1;
+ NRemain -= NRead;
+ } while (NRemain != 0);
+ }
+ }
+ /* All done report success. */
+ return 0;
+}
+
+static int writeValueProfData(ProfDataWriter *Writer,
+ VPDataReaderType *VPDataReader,
+ const __llvm_profile_data *DataBegin,
+ const __llvm_profile_data *DataEnd) {
+ ProfBufferIO *BufferIO;
+ const __llvm_profile_data *DI = 0;
+
+ if (!VPDataReader)
+ return 0;
+
+ BufferIO = lprofCreateBufferIO(Writer);
+
+ for (DI = DataBegin; DI < DataEnd; DI++) {
+ if (writeOneValueProfData(BufferIO, VPDataReader, DI))
+ return -1;
+ }
+
+ if (lprofBufferIOFlush(BufferIO) != 0)
+ return -1;
+ lprofDeleteBufferIO(BufferIO);
+
+ return 0;
+}
+
+COMPILER_RT_VISIBILITY int lprofWriteData(ProfDataWriter *Writer,
+ VPDataReaderType *VPDataReader,
+ int SkipNameDataWrite) {
+ /* Match logic in __llvm_profile_write_buffer(). */
+ const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
+ const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
+ const char *CountersBegin = __llvm_profile_begin_counters();
+ const char *CountersEnd = __llvm_profile_end_counters();
+ const char *BitmapBegin = __llvm_profile_begin_bitmap();
+ const char *BitmapEnd = __llvm_profile_end_bitmap();
+ const char *NamesBegin = __llvm_profile_begin_names();
+ const char *NamesEnd = __llvm_profile_end_names();
+ const VTableProfData *VTableBegin = __llvm_profile_begin_vtables();
+ const VTableProfData *VTableEnd = __llvm_profile_end_vtables();
+ const char *VNamesBegin = __llvm_profile_begin_vtabnames();
+ const char *VNamesEnd = __llvm_profile_end_vtabnames();
+ return lprofWriteDataImpl(Writer, DataBegin, DataEnd, CountersBegin,
+ CountersEnd, BitmapBegin, BitmapEnd, VPDataReader,
+ NamesBegin, NamesEnd, VTableBegin, VTableEnd,
+ VNamesBegin, VNamesEnd, SkipNameDataWrite);
+}
+
+COMPILER_RT_VISIBILITY int
+lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin,
+ const __llvm_profile_data *DataEnd,
+ const char *CountersBegin, const char *CountersEnd,
+ const char *BitmapBegin, const char *BitmapEnd,
+ VPDataReaderType *VPDataReader, const char *NamesBegin,
+ const char *NamesEnd, const VTableProfData *VTableBegin,
+ const VTableProfData *VTableEnd, const char *VNamesBegin,
+ const char *VNamesEnd, int SkipNameDataWrite) {
+ /* Calculate size of sections. */
+ const uint64_t DataSectionSize =
+ __llvm_profile_get_data_size(DataBegin, DataEnd);
+ const uint64_t NumData = __llvm_profile_get_num_data(DataBegin, DataEnd);
+ const uint64_t CountersSectionSize =
+ __llvm_profile_get_counters_size(CountersBegin, CountersEnd);
+ const uint64_t NumCounters =
+ __llvm_profile_get_num_counters(CountersBegin, CountersEnd);
+ const uint64_t NumBitmapBytes =
+ __llvm_profile_get_num_bitmap_bytes(BitmapBegin, BitmapEnd);
+ const uint64_t NamesSize = __llvm_profile_get_name_size(NamesBegin, NamesEnd);
+ const uint64_t NumVTables =
+ __llvm_profile_get_num_vtable(VTableBegin, VTableEnd);
+ const uint64_t VTableSectionSize =
+ __llvm_profile_get_vtable_section_size(VTableBegin, VTableEnd);
+ const uint64_t VNamesSize =
+ __llvm_profile_get_name_size(VNamesBegin, VNamesEnd);
+
+ /* Create the header. */
+ __llvm_profile_header Header;
+
+ /* Determine how much padding is needed before/after the counters and after
+ * the names. */
+ uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters,
+ PaddingBytesAfterBitmapBytes, PaddingBytesAfterNames,
+ PaddingBytesAfterVTable, PaddingBytesAfterVNames;
+ if (__llvm_profile_get_padding_sizes_for_counters(
+ DataSectionSize, CountersSectionSize, NumBitmapBytes, NamesSize,
+ VTableSectionSize, VNamesSize, &PaddingBytesBeforeCounters,
+ &PaddingBytesAfterCounters, &PaddingBytesAfterBitmapBytes,
+ &PaddingBytesAfterNames, &PaddingBytesAfterVTable,
+ &PaddingBytesAfterVNames) == -1)
+ return -1;
+
+ {
+/* Initialize header structure. */
+#define INSTR_PROF_RAW_HEADER(Type, Name, Init) Header.Name = Init;
+#include "profile/InstrProfData.inc"
+ }
+
+ /* On WIN64, label differences are truncated 32-bit values. Truncate
+ * CountersDelta to match. */
+#ifdef _WIN64
+ Header.CountersDelta = (uint32_t)Header.CountersDelta;
+ Header.BitmapDelta = (uint32_t)Header.BitmapDelta;
+#endif
+
+ /* The data and names sections are omitted in lightweight mode. */
+ if (NumData == 0 && NamesSize == 0) {
+ Header.CountersDelta = 0;
+ Header.NamesDelta = 0;
+ }
+
+ /* Write the profile header. */
+ ProfDataIOVec IOVec[] = {{&Header, sizeof(__llvm_profile_header), 1, 0}};
+ if (Writer->Write(Writer, IOVec, sizeof(IOVec) / sizeof(*IOVec)))
+ return -1;
+
+ /* Write the binary id lengths and data. */
+ if (__llvm_write_binary_ids(Writer) == -1)
+ return -1;
+
+ /* Write the profile data. */
+ ProfDataIOVec IOVecData[] = {
+ {DataBegin, sizeof(uint8_t), DataSectionSize, 0},
+ {NULL, sizeof(uint8_t), PaddingBytesBeforeCounters, 1},
+ {CountersBegin, sizeof(uint8_t), CountersSectionSize, 0},
+ {NULL, sizeof(uint8_t), PaddingBytesAfterCounters, 1},
+ {BitmapBegin, sizeof(uint8_t), NumBitmapBytes, 0},
+ {NULL, sizeof(uint8_t), PaddingBytesAfterBitmapBytes, 1},
+ {SkipNameDataWrite ? NULL : NamesBegin, sizeof(uint8_t), NamesSize, 0},
+ {NULL, sizeof(uint8_t), PaddingBytesAfterNames, 1},
+ {VTableBegin, sizeof(uint8_t), VTableSectionSize, 0},
+ {NULL, sizeof(uint8_t), PaddingBytesAfterVTable, 1},
+ {SkipNameDataWrite ? NULL : VNamesBegin, sizeof(uint8_t), VNamesSize, 0},
+ {NULL, sizeof(uint8_t), PaddingBytesAfterVNames, 1}};
+ if (Writer->Write(Writer, IOVecData, sizeof(IOVecData) / sizeof(*IOVecData)))
+ return -1;
+
+ /* Value profiling is not yet supported in continuous mode and profile
+ * correlation mode. */
+ if (__llvm_profile_is_continuous_mode_enabled() ||
+ (NumData == 0 && NamesSize == 0))
+ return 0;
+
+ return writeValueProfData(Writer, VPDataReader, DataBegin, DataEnd);
+}
+
+/*
+ * Write binary id length and then its data, because binary id does not
+ * have a fixed length.
+ */
+COMPILER_RT_VISIBILITY
+int lprofWriteOneBinaryId(ProfDataWriter *Writer, uint64_t BinaryIdLen,
+ const uint8_t *BinaryIdData,
+ uint64_t BinaryIdPadding) {
+ ProfDataIOVec BinaryIdIOVec[] = {
+ {&BinaryIdLen, sizeof(uint64_t), 1, 0},
+ {BinaryIdData, sizeof(uint8_t), BinaryIdLen, 0},
+ {NULL, sizeof(uint8_t), BinaryIdPadding, 1},
+ };
+ if (Writer->Write(Writer, BinaryIdIOVec,
+ sizeof(BinaryIdIOVec) / sizeof(*BinaryIdIOVec)))
+ return -1;
+
+ /* Successfully wrote binary id, report success. */
+ return 0;
+}
diff --git a/system/lib/compiler-rt/lib/profile/WindowsMMap.c b/system/lib/compiler-rt/lib/profile/WindowsMMap.c
new file mode 100644
index 0000000000000..9d7da835b1ed3
--- /dev/null
+++ b/system/lib/compiler-rt/lib/profile/WindowsMMap.c
@@ -0,0 +1,187 @@
+/*
+ * This code is derived from uClibc (original license follows).
+ * https://git.uclibc.org/uClibc/tree/utils/mmap-windows.c
+ */
+ /* mmap() replacement for Windows
+ *
+ * Author: Mike Frysinger <vapier@gentoo.org>
+ * Placed into the public domain
+ */
+
+/* References:
+ * CreateFileMapping: http://msdn.microsoft.com/en-us/library/aa366537(VS.85).aspx
+ * CloseHandle: http://msdn.microsoft.com/en-us/library/ms724211(VS.85).aspx
+ * MapViewOfFile: http://msdn.microsoft.com/en-us/library/aa366761(VS.85).aspx
+ * UnmapViewOfFile: http://msdn.microsoft.com/en-us/library/aa366882(VS.85).aspx
+ */
+
+#if defined(_WIN32)
+
+#include "WindowsMMap.h"
+
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+
+#include "InstrProfiling.h"
+
+COMPILER_RT_VISIBILITY
+void *mmap(void *start, size_t length, int prot, int flags, int fd, off_t offset)
+{
+ if (prot & ~(PROT_READ | PROT_WRITE | PROT_EXEC))
+ return MAP_FAILED;
+ if (fd == -1) {
+ if (!(flags & MAP_ANON) || offset)
+ return MAP_FAILED;
+ } else if (flags & MAP_ANON)
+ return MAP_FAILED;
+
+ DWORD flProtect;
+ if (prot & PROT_WRITE) {
+ if (prot & PROT_EXEC)
+ flProtect = PAGE_EXECUTE_READWRITE;
+ else
+ flProtect = PAGE_READWRITE;
+ } else if (prot & PROT_EXEC) {
+ if (prot & PROT_READ)
+ flProtect = PAGE_EXECUTE_READ;
+ else if (prot & PROT_EXEC)
+ flProtect = PAGE_EXECUTE;
+ } else
+ flProtect = PAGE_READONLY;
+
+ off_t end = length + offset;
+ HANDLE mmap_fd, h;
+ if (fd == -1)
+ mmap_fd = INVALID_HANDLE_VALUE;
+ else
+ mmap_fd = (HANDLE)_get_osfhandle(fd);
+ h = CreateFileMapping(mmap_fd, NULL, flProtect, DWORD_HI(end), DWORD_LO(end), NULL);
+ if (h == NULL)
+ return MAP_FAILED;
+
+ DWORD dwDesiredAccess;
+ if (prot & PROT_WRITE)
+ dwDesiredAccess = FILE_MAP_WRITE;
+ else
+ dwDesiredAccess = FILE_MAP_READ;
+ if (prot & PROT_EXEC)
+ dwDesiredAccess |= FILE_MAP_EXECUTE;
+ if (flags & MAP_PRIVATE)
+ dwDesiredAccess |= FILE_MAP_COPY;
+ void *ret = MapViewOfFile(h, dwDesiredAccess, DWORD_HI(offset), DWORD_LO(offset), length);
+ if (ret == NULL) {
+ CloseHandle(h);
+ ret = MAP_FAILED;
+ }
+ return ret;
+}
+
+COMPILER_RT_VISIBILITY
+void munmap(void *addr, size_t length)
+{
+ UnmapViewOfFile(addr);
+ /* ruh-ro, we leaked handle from CreateFileMapping() ... */
+}
+
+COMPILER_RT_VISIBILITY
+int msync(void *addr, size_t length, int flags)
+{
+ if (flags & MS_INVALIDATE)
+ return -1; /* Not supported. */
+
+ /* Exactly one of MS_ASYNC or MS_SYNC must be specified. */
+ switch (flags & (MS_ASYNC | MS_SYNC)) {
+ case MS_SYNC:
+ case MS_ASYNC:
+ break;
+ default:
+ return -1;
+ }
+
+ if (!FlushViewOfFile(addr, length))
+ return -1;
+
+ if (flags & MS_SYNC) {
+ /* FIXME: No longer have access to handle from CreateFileMapping(). */
+ /*
+ * if (!FlushFileBuffers(h))
+ * return -1;
+ */
+ }
+
+ return 0;
+}
+
+COMPILER_RT_VISIBILITY
+int madvise(void *addr, size_t length, int advice)
+{
+ if (advice != MADV_DONTNEED)
+ return -1; /* Not supported. */
+
+ if (!VirtualUnlock(addr, length))
+ return -1;
+
+ return 0;
+}
+
+static int lock(HANDLE handle, DWORD lockType, BOOL blocking) {
+ DWORD flags = lockType;
+ if (!blocking)
+ flags |= LOCKFILE_FAIL_IMMEDIATELY;
+
+ OVERLAPPED overlapped;
+ ZeroMemory(&overlapped, sizeof(OVERLAPPED));
+ overlapped.hEvent = CreateEvent(NULL, FALSE, FALSE, NULL);
+ BOOL result = LockFileEx(handle, flags, 0, MAXDWORD, MAXDWORD, &overlapped);
+ if (!result) {
+ DWORD dw = GetLastError();
+
+ // In non-blocking mode, return an error if the file is locked.
+ if (!blocking && dw == ERROR_LOCK_VIOLATION)
+ return -1; // EWOULDBLOCK
+
+ // If the error is ERROR_IO_PENDING, we need to wait until the operation
+ // finishes. Otherwise, we return an error.
+ if (dw != ERROR_IO_PENDING)
+ return -1;
+
+ DWORD dwNumBytes;
+ if (!GetOverlappedResult(handle, &overlapped, &dwNumBytes, TRUE))
+ return -1;
+ }
+
+ return 0;
+}
+
+COMPILER_RT_VISIBILITY
+int flock(int fd, int operation) {
+ HANDLE handle = (HANDLE)_get_osfhandle(fd);
+ if (handle == INVALID_HANDLE_VALUE)
+ return -1;
+
+ BOOL blocking = (operation & LOCK_NB) == 0;
+ int op = operation & ~LOCK_NB;
+
+ switch (op) {
+ case LOCK_EX:
+ return lock(handle, LOCKFILE_EXCLUSIVE_LOCK, blocking);
+
+ case LOCK_SH:
+ return lock(handle, 0, blocking);
+
+ case LOCK_UN:
+ if (!UnlockFile(handle, 0, 0, MAXDWORD, MAXDWORD))
+ return -1;
+ break;
+
+ default:
+ return -1;
+ }
+
+ return 0;
+}
+
+#undef DWORD_HI
+#undef DWORD_LO
+
+#endif /* _WIN32 */
diff --git a/system/lib/compiler-rt/lib/profile/WindowsMMap.h b/system/lib/compiler-rt/lib/profile/WindowsMMap.h
new file mode 100644
index 0000000000000..1df1a0be0b02b
--- /dev/null
+++ b/system/lib/compiler-rt/lib/profile/WindowsMMap.h
@@ -0,0 +1,82 @@
+/*===- WindowsMMap.h - Support library for PGO instrumentation ------------===*\
+|*
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+|* See https://llvm.org/LICENSE.txt for license information.
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+|*
+\*===----------------------------------------------------------------------===*/
+
+#ifndef PROFILE_INSTRPROFILING_WINDOWS_MMAP_H
+#define PROFILE_INSTRPROFILING_WINDOWS_MMAP_H
+
+#if defined(_WIN32)
+
+#include <basetsd.h>
+#include <io.h>
+#include <sys/types.h>
+
+/*
+ * mmap() flags
+ */
+#define PROT_READ 0x1
+#define PROT_WRITE 0x2
+#define PROT_EXEC 0x0
+
+#define MAP_FILE 0x00
+#define MAP_SHARED 0x01
+#define MAP_PRIVATE 0x02
+#define MAP_ANONYMOUS 0x20
+#define MAP_ANON MAP_ANONYMOUS
+#define MAP_FAILED ((void *) -1)
+
+/*
+ * msync() flags
+ */
+#define MS_ASYNC 0x0001 /* return immediately */
+#define MS_INVALIDATE 0x0002 /* invalidate all cached data */
+#define MS_SYNC 0x0010 /* msync synchronously */
+
+/*
+ * madvise() flags
+ */
+
+#define MADV_NORMAL 0 /* no special treatment */
+#define MADV_WILLNEED 3 /* expect access in the near future */
+#define MADV_DONTNEED 4 /* do not expect access in the near future */
+
+/*
+ * flock() operations
+ */
+#define LOCK_SH 1 /* shared lock */
+#define LOCK_EX 2 /* exclusive lock */
+#define LOCK_NB 4 /* don't block when locking */
+#define LOCK_UN 8 /* unlock */
+
+#ifdef __USE_FILE_OFFSET64
+# define DWORD_HI(x) (x >> 32)
+# define DWORD_LO(x) ((x) & 0xffffffff)
+#else
+# define DWORD_HI(x) (0)
+# define DWORD_LO(x) (x)
+#endif
+
+#define mmap __llvm_profile_mmap
+#define munmap __llvm_profile_munmap
+#define msync __llvm_profile_msync
+#define madvise __llvm_profile_madvise
+#define flock __llvm_profile_flock
+
+void *mmap(void *start, size_t length, int prot, int flags, int fd,
+ off_t offset);
+
+void munmap(void *addr, size_t length);
+
+int msync(void *addr, size_t length, int flags);
+
+int madvise(void *addr, size_t length, int advice);
+
+int flock(int fd, int operation);
+
+#endif /* _WIN32 */
+
+#endif /* PROFILE_INSTRPROFILING_WINDOWS_MMAP_H */
diff --git a/system/lib/update_compiler_rt.py b/system/lib/update_compiler_rt.py
index 6a157d42fe108..0dabc5068154c 100755
--- a/system/lib/update_compiler_rt.py
+++ b/system/lib/update_compiler_rt.py
@@ -23,6 +23,7 @@
('lib', 'lsan'),
('lib', 'ubsan'),
('lib', 'ubsan_minimal'),
+ ('lib', 'profile'),
]
preserve_files = ('readme.txt',)
diff --git a/test/core/test_dlfcn_self.exports b/test/core/test_dlfcn_self.exports
index 778003379ab0f..7c1ea900be697 100644
--- a/test/core/test_dlfcn_self.exports
+++ b/test/core/test_dlfcn_self.exports
@@ -28,9 +28,11 @@ optopt
optreset
program_invocation_name
program_invocation_short_name
+reset_fn_list
signgam
stderr
stdin
stdout
timezone
tzname
+writeout_fn_list
diff --git a/test/test_core.py b/test/test_core.py
index 6f5967539b544..8df653901ed17 100644
--- a/test/test_core.py
+++ b/test/test_core.py
@@ -19,7 +19,7 @@
raise Exception('do not run this file directly; do something like: test/runner')
from tools.shared import PIPE
-from tools.shared import EMCC, EMAR, EMXX, FILE_PACKAGER
+from tools.shared import EMCC, EMAR, EMXX, FILE_PACKAGER, LLVM_PROFDATA, LLVM_COV
from tools.utils import WINDOWS, MACOS, LINUX, write_file, delete_file
from tools import shared, building, config, utils, webassembly
import common
@@ -9809,6 +9809,32 @@ def test_modularize_instance_embind(self):
self.assertContained('main\nfoo\nbar\n', self.run_js('runner.mjs'))
+ @no_esm_integration('fcoverage is not compatible with WASM_ESM_INTEGRATION')
+ @no_wasm64('TODO: fcoverage in memory64')
+ @no_wasm2js('wasm binary required to produce code coverage results with llvm-cov')
+ def test_fcoverage_mapping(self):
+ expected = '''\
+ 1| |/*
+ 2| | * Copyright 2016 The Emscripten Authors. All rights reserved.
+ 3| | * Emscripten is available under two separate licenses, the MIT license and the
+ 4| | * University of Illinois/NCSA Open Source License. Both these licenses can be
+ 5| | * found in the LICENSE file.
+ 6| | */
+ 7| |
+ 8| |#include <stdio.h>
+ 9| 1|int main() {
+ 10| 1| printf("hello, world!\\n");
+ 11| 1| return 0;
+ 12| 1|}
+
+'''
+ self.set_setting('NODERAWFS')
+ self.set_setting('EXIT_RUNTIME')
+ self.do_core_test('test_hello_world.c', emcc_args=['-fprofile-instr-generate', '-fcoverage-mapping', '-g'])
+ self.assertExists('default.profraw')
+ self.run_process([LLVM_PROFDATA, 'merge', '-sparse', 'default.profraw', '-o', 'out.profdata'])
+ self.assertExists('out.profdata')
+ self.assertEqual(expected, self.run_process([LLVM_COV, 'show', 'test_hello_world.wasm', '-instr-profile=out.profdata'], stdout=PIPE).stdout)
# Generate tests for everything
def make_run(name, emcc_args=None, settings=None, env=None, # noqa
diff --git a/tools/shared.py b/tools/shared.py
index fa967b3b7fe6a..8ecaf55019080 100644
--- a/tools/shared.py
+++ b/tools/shared.py
@@ -752,6 +752,8 @@ def init():
LLVM_OBJCOPY = os.path.expanduser(build_llvm_tool_path(exe_suffix('llvm-objcopy')))
LLVM_STRIP = os.path.expanduser(build_llvm_tool_path(exe_suffix('llvm-strip')))
WASM_LD = os.path.expanduser(build_llvm_tool_path(exe_suffix('wasm-ld')))
+LLVM_PROFDATA = os.path.expanduser(build_llvm_tool_path(exe_suffix('llvm-profdata')))
+LLVM_COV = os.path.expanduser(build_llvm_tool_path(exe_suffix('llvm-cov')))
EMCC = bat_suffix(path_from_root('emcc'))
EMXX = bat_suffix(path_from_root('em++'))
diff --git a/tools/system_libs.py b/tools/system_libs.py
index 1d5c09bcbba4b..36874e986ed47 100644
--- a/tools/system_libs.py
+++ b/tools/system_libs.py
@@ -939,9 +939,10 @@ class libcompiler_rt(MTLibrary, SjLjLibrary):
# restriction soon: https://reviews.llvm.org/D71738
force_object_files = True
- cflags = ['-fno-builtin', '-DNDEBUG']
+ cflags = ['-fno-builtin', '-DNDEBUG', '-DCOMPILER_RT_HAS_UNAME=1']
src_dir = 'system/lib/compiler-rt/lib/builtins'
- includes = ['system/lib/libc']
+ profile_src_dir = 'system/lib/compiler-rt/lib/profile'
+ includes = ['system/lib/libc', 'system/lib/compiler-rt/include']
excludes = [
# gcc_personality_v0.c depends on libunwind, which don't include by default.
'gcc_personality_v0.c',
@@ -973,6 +974,8 @@ class libcompiler_rt(MTLibrary, SjLjLibrary):
'truncxfhf2.c',
]
src_files = glob_in_path(src_dir, '*.c', excludes=excludes)
+ src_files += glob_in_path(profile_src_dir, '*.c')
+ src_files += glob_in_path(profile_src_dir, '*.cpp')
src_files += files_in_path(
path='system/lib/compiler-rt',
filenames=[