From 0e895d0d62dc897ec590455bb6650c1ed6e28ae9 Mon Sep 17 00:00:00 2001 From: Christian Despres Date: Thu, 22 Sep 2022 20:31:53 +0000 Subject: [PATCH 1/5] Define JITServer AOT cache serialization interface The basic interface will support reading and writing whole AOT caches to a file stream. An interface to an eventual linked list traversal of the maps holding the AOT record types has also been added. This will allow for concurrent access to a cache while serialization is in progress. Signed-off-by: Christian Despres --- .../compiler/runtime/JITServerAOTCache.cpp | 15 ++++++++ .../compiler/runtime/JITServerAOTCache.hpp | 34 ++++++++++++++++++- 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/runtime/compiler/runtime/JITServerAOTCache.cpp b/runtime/compiler/runtime/JITServerAOTCache.cpp index b0109fa62f7..358a0d13f0c 100644 --- a/runtime/compiler/runtime/JITServerAOTCache.cpp +++ b/runtime/compiler/runtime/JITServerAOTCache.cpp @@ -207,6 +207,7 @@ CachedAOTMethod::CachedAOTMethod(const AOTCacheClassChainRecord *definingClassCh TR_Hotness optLevel, const AOTCacheAOTHeaderRecord *aotHeaderRecord, const Vector> &records, const void *code, size_t codeSize, const void *data, size_t dataSize) : + _nextRecord(NULL), _data(definingClassChainRecord->data().id(), index, optLevel, aotHeaderRecord->data().id(), records.size(), code, codeSize, data, dataSize), _definingClassChainRecord(definingClassChainRecord) @@ -346,24 +347,38 @@ freeMapValues(const PersistentUnorderedMap &map) JITServerAOTCache::JITServerAOTCache(const std::string &name) : _name(name), _classLoaderMap(decltype(_classLoaderMap)::allocator_type(TR::Compiler->persistentGlobalAllocator())), + _classLoaderHead(NULL), + _classLoaderTail(NULL), _nextClassLoaderId(1),// ID 0 is invalid _classLoaderMonitor(TR::Monitor::create("JIT-JITServerAOTCacheClassLoaderMonitor")), _classMap(decltype(_classMap)::allocator_type(TR::Compiler->persistentGlobalAllocator())), + _classHead(NULL), + _classTail(NULL), _nextClassId(1),// ID 0 is invalid _classMonitor(TR::Monitor::create("JIT-JITServerAOTCacheClassMonitor")), _methodMap(decltype(_methodMap)::allocator_type(TR::Compiler->persistentGlobalAllocator())), + _methodHead(NULL), + _methodTail(NULL), _nextMethodId(1),// ID 0 is invalid _methodMonitor(TR::Monitor::create("JIT-JITServerAOTCacheMethodMonitor")), _classChainMap(decltype(_classChainMap)::allocator_type(TR::Compiler->persistentGlobalAllocator())), + _classChainHead(NULL), + _classChainTail(NULL), _nextClassChainId(1),// ID 0 is invalid _classChainMonitor(TR::Monitor::create("JIT-JITServerAOTCacheClassChainMonitor")), _wellKnownClassesMap(decltype(_wellKnownClassesMap)::allocator_type(TR::Compiler->persistentGlobalAllocator())), + _wellKnownClassesHead(NULL), + _wellKnownClassesTail(NULL), _nextWellKnownClassesId(1),// ID 0 is invalid _wellKnownClassesMonitor(TR::Monitor::create("JIT-JITServerAOTCacheWellKnownClassesMonitor")), _aotHeaderMap(decltype(_aotHeaderMap)::allocator_type(TR::Compiler->persistentGlobalAllocator())), + _aotHeaderHead(NULL), + _aotHeaderTail(NULL), _nextAOTHeaderId(1),// ID 0 is invalid _aotHeaderMonitor(TR::Monitor::create("JIT-JITServerAOTCacheAOTHeaderMonitor")), _cachedMethodMap(decltype(_cachedMethodMap)::allocator_type(TR::Compiler->persistentGlobalAllocator())), + _cachedMethodHead(NULL), + _cachedMethodTail(NULL), _cachedMethodMonitor(TR::Monitor::create("JIT-JITServerAOTCacheCachedMethodMonitor")), _numCacheBypasses(0), _numCacheHits(0), _numCacheMisses(0), _numDeserializedMethods(0), _numDeserializationFailures(0) diff --git a/runtime/compiler/runtime/JITServerAOTCache.hpp b/runtime/compiler/runtime/JITServerAOTCache.hpp index 725225ea038..ecef1f21ae9 100644 --- a/runtime/compiler/runtime/JITServerAOTCache.hpp +++ b/runtime/compiler/runtime/JITServerAOTCache.hpp @@ -49,6 +49,9 @@ namespace TR { class Monitor; } // record types) in order to simplify memory management and exception handling. // The subclasses store their underlying serialization record data inline // (as well as variable-length arrays of subrecord pointers in some cases). +// +// Each AOTCacheRecord also stores a _nextRecord pointer that points to the next record in +// a traversal of all of the records of a particular subclass (used for cache persistence). class AOTCacheRecord { public: @@ -64,8 +67,14 @@ class AOTCacheRecord static void *allocate(size_t size); static void free(void *ptr); + AOTCacheRecord *getNextRecord() const { return _nextRecord; } + void setNextRecord(AOTCacheRecord *record) { _nextRecord = record; } + protected: - AOTCacheRecord() = default; + AOTCacheRecord() : _nextRecord(NULL) {} + +private: + AOTCacheRecord *_nextRecord; }; @@ -219,6 +228,9 @@ class CachedAOTMethod const Vector> &records, const void *code, size_t codeSize, const void *data, size_t dataSize); + CachedAOTMethod *getNextRecord() const { return _nextRecord; } + void setNextRecord(CachedAOTMethod *record) { _nextRecord = record; } + private: CachedAOTMethod(const AOTCacheClassChainRecord *definingClassChainRecord, uint32_t index, TR_Hotness optLevel, const AOTCacheAOTHeaderRecord *aotHeaderRecord, @@ -231,6 +243,7 @@ class CachedAOTMethod numRecords * sizeof(AOTCacheRecord *); } + CachedAOTMethod *_nextRecord; const AOTCacheClassChainRecord *const _definingClassChainRecord; SerializedAOTMethod _data; // Array of record pointers is stored inline after serialized AOT method data @@ -298,6 +311,9 @@ class JITServerAOTCache void printStats(FILE *f) const; + bool writeCache(FILE *f) const; + static JITServerAOTCache *readCache(FILE *f, const std::string &name); + private: struct ClassLoaderKey { @@ -357,32 +373,48 @@ class JITServerAOTCache const std::string _name; + // Along with each map we also store pointers to the start and end points of a traversal of all the records. + // The _nextRecord in each record points to the next record in this traversal. PersistentUnorderedMap _classLoaderMap; uintptr_t _nextClassLoaderId; + AOTCacheClassLoaderRecord *_classLoaderHead; + AOTCacheClassLoaderRecord *_classLoaderTail; TR::Monitor *const _classLoaderMonitor; PersistentUnorderedMap _classMap; uintptr_t _nextClassId; + AOTCacheClassRecord *_classHead; + AOTCacheClassRecord *_classTail; TR::Monitor *const _classMonitor; PersistentUnorderedMap _methodMap; uintptr_t _nextMethodId; + AOTCacheMethodRecord *_methodHead; + AOTCacheMethodRecord *_methodTail; TR::Monitor *const _methodMonitor; PersistentUnorderedMap _classChainMap; + AOTCacheClassChainRecord *_classChainHead; + AOTCacheClassChainRecord *_classChainTail; uintptr_t _nextClassChainId; TR::Monitor *const _classChainMonitor; PersistentUnorderedMap _wellKnownClassesMap; + AOTCacheWellKnownClassesRecord *_wellKnownClassesHead; + AOTCacheWellKnownClassesRecord *_wellKnownClassesTail; uintptr_t _nextWellKnownClassesId; TR::Monitor *const _wellKnownClassesMonitor; PersistentUnorderedMap _aotHeaderMap; + AOTCacheAOTHeaderRecord *_aotHeaderHead; + AOTCacheAOTHeaderRecord *_aotHeaderTail; uintptr_t _nextAOTHeaderId; TR::Monitor *const _aotHeaderMonitor; PersistentUnorderedMap _cachedMethodMap; + CachedAOTMethod *_cachedMethodHead; + CachedAOTMethod *_cachedMethodTail; TR::Monitor *const _cachedMethodMonitor; // Statistics From 2965a7376238eff53791a0e63131ea0543e6b0ec Mon Sep 17 00:00:00 2001 From: Christian Despres Date: Thu, 22 Sep 2022 20:32:18 +0000 Subject: [PATCH 2/5] Define JITServer AOT cache file header structure The file header contains version data relevant to the consistency of the serialized records with the currently-running server (including cache file format version) as well as data related to the structure of the AOT record maps themselves. Signed-off-by: Christian Despres --- .../compiler/runtime/JITServerAOTCache.hpp | 34 ++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/runtime/compiler/runtime/JITServerAOTCache.hpp b/runtime/compiler/runtime/JITServerAOTCache.hpp index ecef1f21ae9..144e86636e7 100644 --- a/runtime/compiler/runtime/JITServerAOTCache.hpp +++ b/runtime/compiler/runtime/JITServerAOTCache.hpp @@ -29,8 +29,41 @@ #include "env/PersistentCollections.hpp" #include "runtime/JITServerAOTSerializationRecords.hpp" +static const uint32_t JITSERVER_AOTCACHE_VERSION = 0; +static const char JITSERVER_AOTCACHE_EYECATCHER[] = "AOTCACHE"; +// the eye-catcher is not null-terminated in the snapshot files +static const size_t JITSERVER_AOTCACHE_EYECATCHER_LENGTH = sizeof(JITSERVER_AOTCACHE_EYECATCHER) - 1; + namespace TR { class Monitor; } +// Information relevant to the compatibility of a cache snapshot with the server. +struct JITServerAOTCacheVersion + { + char _eyeCatcher[JITSERVER_AOTCACHE_EYECATCHER_LENGTH]; + uint32_t _snapshotVersion; + uint32_t _padding; // explicit padding, currently unused + uint64_t _jitserverVersion; + }; + +// The header information for an AOT cache snapshot. +struct JITServerAOTCacheHeader + { + JITServerAOTCacheVersion _version; + uint64_t _serverUID; + size_t _numClassLoaderRecords; + size_t _numClassRecords; + size_t _numMethodRecords; + size_t _numClassChainRecords; + size_t _numWellKnownClassesRecords; + size_t _numAOTHeaderRecords; + size_t _numCachedAOTMethods; + size_t _nextClassLoaderId; + size_t _nextClassId; + size_t _nextMethodId; + size_t _nextClassChainId; + size_t _nextWellKnownClassesId; + size_t _nextAOTHeaderId; + }; // Base class for serialization record "wrappers" stored at the server. // @@ -452,5 +485,4 @@ class JITServerAOTCacheMap static bool _cacheIsFull; }; - #endif /* defined(JITSERVER_AOTCACHE_H) */ From 9c998acfba381933ff4bfea9946124b5c7c34755 Mon Sep 17 00:00:00 2001 From: Christian Despres Date: Tue, 27 Sep 2022 17:11:35 +0000 Subject: [PATCH 3/5] Keep the JITServer AOT cache traversals updated Signed-off-by: Christian Despres --- .../compiler/runtime/JITServerAOTCache.cpp | 31 ++++++++++++++----- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/runtime/compiler/runtime/JITServerAOTCache.cpp b/runtime/compiler/runtime/JITServerAOTCache.cpp index 358a0d13f0c..c0b83cb0f92 100644 --- a/runtime/compiler/runtime/JITServerAOTCache.cpp +++ b/runtime/compiler/runtime/JITServerAOTCache.cpp @@ -317,8 +317,12 @@ JITServerAOTCache::AOTHeaderKey::Hash::operator()(const AOTHeaderKey &k) const n // Insert the value (which must be allocated with AOTCacheRecord::allocate()) // with the key into the map, avoiding memory leaks in case of exceptions. +// Also insert it into the linked list traversal of the map defined by the +// given head and tail. template static void addToMap(PersistentUnorderedMap &map, + V *&traversalHead, + V *&traversalTail, const typename PersistentUnorderedMap::const_iterator &it, const K &key, V *value) { @@ -331,6 +335,19 @@ addToMap(PersistentUnorderedMap &map, AOTCacheRecord::free(value); throw; } + + // Normally we would need a write barrier here to ensure that the record was fully written to memory before + // adding it to this traversal. However, since we save the number of records to be written in writeRecordList, + // we will never encounter such a partial record in the serializer, and so the write barrier is unnecessary. + if (traversalTail == NULL) + { + traversalHead = value; + } + else + { + traversalTail->setNextRecord(value); + } + traversalTail = value; } // Free all the values (which must be allocated with AOTCacheRecord::allocate()) in the map. @@ -433,7 +450,7 @@ JITServerAOTCache::getClassLoaderRecord(const uint8_t *name, size_t nameLength) } auto record = AOTCacheClassLoaderRecord::create(_nextClassLoaderId, name, nameLength); - addToMap(_classLoaderMap, it, { record->data().name(), record->data().nameLength() }, record); + addToMap(_classLoaderMap, _classLoaderHead, _classLoaderTail, it, { record->data().name(), record->data().nameLength() }, record); ++_nextClassLoaderId; if (TR::Options::getVerboseOption(TR_VerboseJITServer)) @@ -466,7 +483,7 @@ JITServerAOTCache::getClassRecord(const AOTCacheClassLoaderRecord *classLoaderRe } auto record = AOTCacheClassRecord::create(_nextClassId, classLoaderRecord, hash, romClass); - addToMap(_classMap, it, { classLoaderRecord, &record->data().hash() }, record); + addToMap(_classMap, _classHead, _classTail, it, { classLoaderRecord, &record->data().hash() }, record); ++_nextClassId; if (TR::Options::getVerboseOption(TR_VerboseJITServer)) @@ -499,7 +516,7 @@ JITServerAOTCache::getMethodRecord(const AOTCacheClassRecord *definingClassRecor } auto record = AOTCacheMethodRecord::create(_nextMethodId, definingClassRecord, index); - addToMap(_methodMap, it, key, record); + addToMap(_methodMap, _methodHead, _methodTail, it, key, record); ++_nextMethodId; if (TR::Options::getVerboseOption(TR_VerboseJITServer)) @@ -529,7 +546,7 @@ JITServerAOTCache::getClassChainRecord(const AOTCacheClassRecord *const *classRe } auto record = AOTCacheClassChainRecord::create(_nextClassChainId, classRecords, length); - addToMap(_classChainMap, it, { record->records(), length }, record); + addToMap(_classChainMap, _classChainHead, _classChainTail, it, { record->records(), length }, record); ++_nextClassChainId; if (TR::Options::getVerboseOption(TR_VerboseJITServer)) @@ -560,7 +577,7 @@ JITServerAOTCache::getWellKnownClassesRecord(const AOTCacheClassChainRecord *con } auto record = AOTCacheWellKnownClassesRecord::create(_nextWellKnownClassesId, chainRecords, length, includedClasses); - addToMap(_wellKnownClassesMap, it, { record->records(), length, includedClasses }, record); + addToMap(_wellKnownClassesMap, _wellKnownClassesHead, _wellKnownClassesTail, it, { record->records(), length, includedClasses }, record); ++_nextWellKnownClassesId; if (TR::Options::getVerboseOption(TR_VerboseJITServer)) @@ -594,7 +611,7 @@ JITServerAOTCache::getAOTHeaderRecord(const TR_AOTHeader *header, uint64_t clien } auto record = AOTCacheAOTHeaderRecord::create(_nextAOTHeaderId, header); - addToMap(_aotHeaderMap, it, { record->data().header() }, record); + addToMap(_aotHeaderMap, _aotHeaderHead, _aotHeaderTail, it, { record->data().header() }, record); ++_nextAOTHeaderId; if (TR::Options::getVerboseOption(TR_VerboseJITServer)) @@ -646,7 +663,7 @@ JITServerAOTCache::storeMethod(const AOTCacheClassChainRecord *definingClassChai auto method = CachedAOTMethod::create(definingClassChainRecord, index, optLevel, aotHeaderRecord, records, code, codeSize, data, dataSize); - addToMap(_cachedMethodMap, it, key, method); + addToMap(_cachedMethodMap, _cachedMethodHead, _cachedMethodTail, it, key, method); if (TR::Options::getVerboseOption(TR_VerboseJITServer)) TR_VerboseLog::writeLineLocked(TR_Vlog_JITServer, From d58264e244ee4f9903268505cc74a47d9c63cf31 Mon Sep 17 00:00:00 2001 From: Christian Despres Date: Thu, 29 Sep 2022 20:35:26 +0000 Subject: [PATCH 4/5] Add full JITServer AOT cache file writing Signed-off-by: Christian Despres --- .../compiler/runtime/JITServerAOTCache.cpp | 121 ++++++++++++++++++ 1 file changed, 121 insertions(+) diff --git a/runtime/compiler/runtime/JITServerAOTCache.cpp b/runtime/compiler/runtime/JITServerAOTCache.cpp index c0b83cb0f92..a4337f5fb6c 100644 --- a/runtime/compiler/runtime/JITServerAOTCache.cpp +++ b/runtime/compiler/runtime/JITServerAOTCache.cpp @@ -25,6 +25,7 @@ #include "infra/CriticalSection.hpp" #include "runtime/JITServerAOTCache.hpp" #include "runtime/JITServerSharedROMClassCache.hpp" +#include "net/CommunicationStream.hpp" size_t JITServerAOTCacheMap::_cacheMaxBytes = 300 * 1024 * 1024; bool JITServerAOTCacheMap::_cacheIsFull = false; @@ -777,6 +778,126 @@ JITServerAOTCache::printStats(FILE *f) const ); } +// Write at most numRecordsToWrite to the given stream from the linked list starting at head. +static bool +writeRecordList(FILE *f, const AOTCacheRecord *head, size_t numRecordsToWrite) + { + const AOTCacheRecord *current = head; + size_t recordsWritten = 0; + while (current && (recordsWritten < numRecordsToWrite)) + { + const AOTSerializationRecord *record = current->dataAddr(); + if (1 != fwrite(record, record->size(), 1, f)) + { + return false; + } + ++recordsWritten; + current = current->getNextRecord(); + } + TR_ASSERT(recordsWritten == numRecordsToWrite, "Expected to write %zu records, wrote %zu", numRecordsToWrite, recordsWritten); + + return true; + } + +static bool +writeCachedMethodList(FILE *f, const CachedAOTMethod *head, size_t numRecordsToWrite) + { + const CachedAOTMethod *current = head; + size_t recordsWritten = 0; + while (current && (recordsWritten < numRecordsToWrite)) + { + const SerializedAOTMethod *record = ¤t->data(); + if (1 != fwrite(record, record->size(), 1, f)) + { + return false; + } + ++recordsWritten; + current = current->getNextRecord(); + } + TR_ASSERT(recordsWritten == numRecordsToWrite, "Expected to write %zu records, wrote %zu", numRecordsToWrite, recordsWritten); + + return true; + } + +static void getCurrentAOTCacheVersion(JITServerAOTCacheVersion &version) + { + memcpy(version._eyeCatcher, JITSERVER_AOTCACHE_EYECATCHER, JITSERVER_AOTCACHE_EYECATCHER_LENGTH); + version._snapshotVersion = JITSERVER_AOTCACHE_VERSION; + version._jitserverVersion = JITServer::CommunicationStream::getJITServerFullVersion(); + } + +// Write a full AOT cache snapshot to a stream. After the header information, the +// AOTSerializationRecord or SerializedAOTMethod data (depending on record type) in each +// record traversal is written directly to the stream in sections, since the full AOT record +// can be reconstructed from only this information. These sections are ordered so that, when +// reading the snapshot, the dependencies of each record will already have been read by the +// time we get to that record. +bool +JITServerAOTCache::writeCache(FILE *f) const + { + JITServerAOTCacheHeader header = {0}; + getCurrentAOTCacheVersion(header._version); + header._serverUID = TR::CompilationInfo::get()->getPersistentInfo()->getServerUID(); + + // It is possible for a record and its dependencies to be added between .size() calls, + // so we must reverse the order in which we read the map sizes (compared to their write order) + // to ensure that those dependencies are not excluded from serialization. + { + OMR::CriticalSection cs(_cachedMethodMonitor); + header._numCachedAOTMethods = _cachedMethodMap.size(); + } + { + OMR::CriticalSection cs(_aotHeaderMonitor); + header._numAOTHeaderRecords = _aotHeaderMap.size(); + header._nextAOTHeaderId = _nextAOTHeaderId; + } + { + OMR::CriticalSection cs(_wellKnownClassesMonitor); + header._numWellKnownClassesRecords = _wellKnownClassesMap.size(); + header._nextWellKnownClassesId = _nextWellKnownClassesId; + } + { + OMR::CriticalSection cs(_classChainMonitor); + header._numClassChainRecords = _classChainMap.size(); + header._nextClassChainId = _nextClassChainId; + } + { + OMR::CriticalSection cs(_methodMonitor); + header._numMethodRecords = _methodMap.size(); + header._nextMethodId = _nextMethodId; + } + { + OMR::CriticalSection cs(_classMonitor); + header._numClassRecords = _classMap.size(); + header._nextClassId = _nextClassId; + } + { + OMR::CriticalSection cs(_classLoaderMonitor); + header._numClassLoaderRecords = _classLoaderMap.size(); + header._nextClassLoaderId = _nextClassLoaderId; + } + + if (1 != fwrite(&header, sizeof(JITServerAOTCacheHeader), 1, f)) + return false; + + if (!writeRecordList(f, _classLoaderHead, header._numClassLoaderRecords)) + return false; + if (!writeRecordList(f, _classHead, header._numClassRecords)) + return false; + if (!writeRecordList(f, _methodHead, header._numMethodRecords)) + return false; + if (!writeRecordList(f, _classChainHead, header._numClassChainRecords)) + return false; + if (!writeRecordList(f, _wellKnownClassesHead, header._numWellKnownClassesRecords)) + return false; + if (!writeRecordList(f, _aotHeaderHead, header._numAOTHeaderRecords)) + return false; + if (!writeCachedMethodList(f, _cachedMethodHead, header._numCachedAOTMethods)) + return false; + + return true; + } + bool JITServerAOTCacheMap::cacheHasSpace() { From 43604b4482acfc11062844da297afb9c4acb6328 Mon Sep 17 00:00:00 2001 From: Christian Despres Date: Thu, 6 Oct 2022 20:43:41 +0000 Subject: [PATCH 5/5] Add full JITServer AOT cache file reading Signed-off-by: Christian Despres --- .../compiler/runtime/JITServerAOTCache.cpp | 500 +++++++++++++++++- .../compiler/runtime/JITServerAOTCache.hpp | 110 +++- .../JITServerAOTSerializationRecords.hpp | 17 +- 3 files changed, 617 insertions(+), 10 deletions(-) diff --git a/runtime/compiler/runtime/JITServerAOTCache.cpp b/runtime/compiler/runtime/JITServerAOTCache.cpp index a4337f5fb6c..d3543768466 100644 --- a/runtime/compiler/runtime/JITServerAOTCache.cpp +++ b/runtime/compiler/runtime/JITServerAOTCache.cpp @@ -46,6 +46,18 @@ AOTCacheRecord::free(void *ptr) TR::Compiler->persistentGlobalMemory()->freePersistentMemory(ptr); } +bool +AOTSerializationRecord::isValid(AOTSerializationRecordType type) const + { + return (type == this->type()) && + (0 != this->id()); + } + +ClassLoaderSerializationRecord::ClassLoaderSerializationRecord() : + AOTSerializationRecord(0, 0, AOTSerializationRecordType::ClassLoader), + _nameLength(0) + { + } ClassLoaderSerializationRecord::ClassLoaderSerializationRecord(uintptr_t id, const uint8_t *name, size_t nameLength) : AOTSerializationRecord(size(nameLength), id, AOTSerializationRecordType::ClassLoader), @@ -54,6 +66,7 @@ ClassLoaderSerializationRecord::ClassLoaderSerializationRecord(uintptr_t id, con memcpy(_name, name, nameLength); } + AOTCacheClassLoaderRecord::AOTCacheClassLoaderRecord(uintptr_t id, const uint8_t *name, size_t nameLength) : _data(id, name, nameLength) { @@ -66,6 +79,32 @@ AOTCacheClassLoaderRecord::create(uintptr_t id, const uint8_t *name, size_t name return new (ptr) AOTCacheClassLoaderRecord(id, name, nameLength); } +AOTCacheClassLoaderRecord * +AOTCacheClassLoaderRecord::read(FILE *f, + const Vector &classLoaderRecords, + const Vector &classRecords, + const Vector &methodRecords, + const Vector &classChainRecords, + const Vector &wellKnownClassesRecords, + const Vector &aotHeaderRecords) + { + ClassLoaderSerializationRecord header; + if (1 != fread(&header, sizeof(header), 1, f)) + return NULL; + + if (!header.isValid()) + return NULL; + + auto record = new (AOTCacheRecord::allocate(size(header.nameLength()))) AOTCacheClassLoaderRecord(); + memcpy((void *)record->dataAddr(), &header, sizeof(header)); + if (1 != fread((uint8_t *)record->dataAddr() + sizeof(header), header.AOTSerializationRecord::size() - sizeof(header), 1, f)) + { + AOTCacheRecord::free(record); + return NULL; + } + + return record; + } ClassSerializationRecord::ClassSerializationRecord(uintptr_t id, uintptr_t classLoaderId, const JITServerROMClassHash &hash, const J9ROMClass *romClass) : @@ -76,6 +115,12 @@ ClassSerializationRecord::ClassSerializationRecord(uintptr_t id, uintptr_t class memcpy(_name, J9UTF8_DATA(J9ROMCLASS_CLASSNAME(romClass)), _nameLength); } +ClassSerializationRecord::ClassSerializationRecord() : + AOTSerializationRecord(0, 0, AOTSerializationRecordType::Class), + _classLoaderId(0), _hash(), _romClassSize(0), _nameLength(0) + { + } + AOTCacheClassRecord::AOTCacheClassRecord(uintptr_t id, const AOTCacheClassLoaderRecord *classLoaderRecord, const JITServerROMClassHash &hash, const J9ROMClass *romClass) : _classLoaderRecord(classLoaderRecord), @@ -83,6 +128,11 @@ AOTCacheClassRecord::AOTCacheClassRecord(uintptr_t id, const AOTCacheClassLoader { } +AOTCacheClassRecord::AOTCacheClassRecord(const AOTCacheClassLoaderRecord *classLoaderRecord) : + _classLoaderRecord(classLoaderRecord) + { + } + AOTCacheClassRecord * AOTCacheClassRecord::create(uintptr_t id, const AOTCacheClassLoaderRecord *classLoaderRecord, const JITServerROMClassHash &hash, const J9ROMClass *romClass) @@ -97,6 +147,34 @@ AOTCacheClassRecord::subRecordsDo(const std::function &classLoaderRecords, + const Vector &classRecords, + const Vector &methodRecords, + const Vector &classChainRecords, + const Vector &wellKnownClassesRecords, + const Vector &aotHeaderRecords) + { + ClassSerializationRecord header; + if (1 != fread(&header, sizeof(header), 1, f)) + return NULL; + + if (!header.isValid() || + (header.classLoaderId() >= classLoaderRecords.size()) || + !classLoaderRecords[header.classLoaderId()]) + return NULL; + + auto record = new (AOTCacheRecord::allocate(size(header.nameLength()))) AOTCacheClassRecord(classLoaderRecords[header.classLoaderId()]); + memcpy((void *)record->dataAddr(), &header, sizeof(header)); + if (1 != fread((uint8_t *)record->dataAddr() + sizeof(header), header.AOTSerializationRecord::size() - sizeof(header), 1, f)) + { + AOTCacheRecord::free(record); + return NULL; + } + + return record; + } MethodSerializationRecord::MethodSerializationRecord(uintptr_t id, uintptr_t definingClassId, uint32_t index) : AOTSerializationRecord(sizeof(*this), id, AOTSerializationRecordType::Method), @@ -104,6 +182,12 @@ MethodSerializationRecord::MethodSerializationRecord(uintptr_t id, uintptr_t def { } +MethodSerializationRecord::MethodSerializationRecord() : + AOTSerializationRecord(0, 0, AOTSerializationRecordType::Method), + _definingClassId(0), _index(0) + { + } + AOTCacheMethodRecord::AOTCacheMethodRecord(uintptr_t id, const AOTCacheClassRecord *definingClassRecord, uint32_t index) : _definingClassRecord(definingClassRecord), @@ -111,6 +195,11 @@ AOTCacheMethodRecord::AOTCacheMethodRecord(uintptr_t id, const AOTCacheClassReco { } +AOTCacheMethodRecord::AOTCacheMethodRecord(const AOTCacheClassRecord *definingClassRecord) : + _definingClassRecord(definingClassRecord) + { + } + AOTCacheMethodRecord * AOTCacheMethodRecord::create(uintptr_t id, const AOTCacheClassRecord *definingClassRecord, uint32_t index) { @@ -124,6 +213,29 @@ AOTCacheMethodRecord::subRecordsDo(const std::function &classLoaderRecords, + const Vector &classRecords, + const Vector &methodRecords, + const Vector &classChainRecords, + const Vector &wellKnownClassesRecords, + const Vector &aotHeaderRecords) + { + MethodSerializationRecord header; + if (1 != fread(&header, sizeof(header), 1, f)) + return NULL; + + if (!header.isValid() || + (header.definingClassId() >= classRecords.size()) || + !classRecords[header.definingClassId()]) + return NULL; + + auto record = new (AOTCacheRecord::allocate(sizeof(AOTCacheMethodRecord))) AOTCacheMethodRecord(classRecords[header.definingClassId()]); + memcpy((void *)record->dataAddr(), &header, sizeof(header)); + + return record; + } template AOTCacheListRecord::AOTCacheListRecord(uintptr_t id, const R *const *records, @@ -142,6 +254,38 @@ AOTCacheListRecord::subRecordsDo(const std::function AOTCacheListRecord* +AOTCacheListRecord::read(FILE *f, const Vector &cacheRecords) + { + D header; + if (1 != fread(&header, sizeof(header), 1, f)) + return NULL; + + if (!header.isValid()) + return NULL; + + auto record = new (AOTCacheRecord::allocate(size(header.list().length()))) AOTCacheListRecord(); + memcpy((void *)record->dataAddr(), &header, sizeof(header)); + + if (1 != fread((uint8_t *)record->dataAddr() + sizeof(header), header.AOTSerializationRecord::size() - sizeof(header), 1, f)) + { + AOTCacheRecord::free(record); + return NULL; + } + + for (size_t i = 0; i < header.list().length(); ++i) + { + uintptr_t id = record->data().list().ids()[i]; + if ((id >= cacheRecords.size()) || !cacheRecords[id]) + { + AOTCacheRecord::free(record); + return NULL; + } + record->records()[i] = cacheRecords[id]; + } + + return record; + } ClassChainSerializationRecord::ClassChainSerializationRecord(uintptr_t id, size_t length) : AOTSerializationRecord(size(length), id, AOTSerializationRecordType::ClassChain), @@ -149,6 +293,12 @@ ClassChainSerializationRecord::ClassChainSerializationRecord(uintptr_t id, size_ { } +ClassChainSerializationRecord::ClassChainSerializationRecord() : + AOTSerializationRecord(0, 0, AOTSerializationRecordType::ClassChain), + _list(0) + { + } + AOTCacheClassChainRecord * AOTCacheClassChainRecord::create(uintptr_t id, const AOTCacheClassRecord *const *records, size_t length) { @@ -164,6 +314,12 @@ WellKnownClassesSerializationRecord::WellKnownClassesSerializationRecord(uintptr { } +WellKnownClassesSerializationRecord::WellKnownClassesSerializationRecord() : + AOTSerializationRecord(0, 0, AOTSerializationRecordType::WellKnownClasses), + _includedClasses(0), _list(0) + { + } + AOTCacheWellKnownClassesRecord * AOTCacheWellKnownClassesRecord::create(uintptr_t id, const AOTCacheClassChainRecord *const *records, size_t length, uintptr_t includedClasses) @@ -179,6 +335,12 @@ AOTHeaderSerializationRecord::AOTHeaderSerializationRecord(uintptr_t id, const T { } +AOTHeaderSerializationRecord::AOTHeaderSerializationRecord() : + AOTSerializationRecord(0, 0, AOTSerializationRecordType::AOTHeader), + _header({0}) + { + } + AOTCacheAOTHeaderRecord::AOTCacheAOTHeaderRecord(uintptr_t id, const TR_AOTHeader *header) : _data(id, header) { @@ -191,6 +353,27 @@ AOTCacheAOTHeaderRecord::create(uintptr_t id, const TR_AOTHeader *header) return new (ptr) AOTCacheAOTHeaderRecord(id, header); } +AOTCacheAOTHeaderRecord * +AOTCacheAOTHeaderRecord::read(FILE *f, + const Vector &classLoaderRecords, + const Vector &classRecords, + const Vector &methodRecords, + const Vector &classChainRecords, + const Vector &wellKnownClassesRecords, + const Vector &aotHeaderRecords) + { + AOTHeaderSerializationRecord header; + if (1 != fread(&header, sizeof(header), 1, f)) + return NULL; + + if (!header.isValid()) + return NULL; + + auto record = new (AOTCacheRecord::allocate(sizeof(AOTCacheAOTHeaderRecord))) AOTCacheAOTHeaderRecord(); + memcpy((void *)record->dataAddr(), &header, sizeof(header)); + + return record; + } SerializedAOTMethod::SerializedAOTMethod(uintptr_t definingClassChainId, uint32_t index, TR_Hotness optLevel, uintptr_t aotHeaderId, size_t numRecords, @@ -204,6 +387,20 @@ SerializedAOTMethod::SerializedAOTMethod(uintptr_t definingClassChainId, uint32_ memcpy((void *)this->data(), data, dataSize); } +SerializedAOTMethod::SerializedAOTMethod() : + _size(0), + _definingClassChainId(0), _index(0), + _optLevel(TR_Hotness::numHotnessLevels), _aotHeaderId(0), + _numRecords(0), _codeSize(0), _dataSize(0) + { + } + +bool +SerializedAOTMethod::isValid() const + { + return _optLevel < TR_Hotness::numHotnessLevels; + } + CachedAOTMethod::CachedAOTMethod(const AOTCacheClassChainRecord *definingClassChainRecord, uint32_t index, TR_Hotness optLevel, const AOTCacheAOTHeaderRecord *aotHeaderRecord, const Vector> &records, @@ -221,6 +418,12 @@ CachedAOTMethod::CachedAOTMethod(const AOTCacheClassChainRecord *definingClassCh } } +CachedAOTMethod::CachedAOTMethod(const AOTCacheClassChainRecord *definingClassChainRecord) : + _nextRecord(NULL), + _definingClassChainRecord(definingClassChainRecord) + { + } + CachedAOTMethod * CachedAOTMethod::create(const AOTCacheClassChainRecord *definingClassChainRecord, uint32_t index, TR_Hotness optLevel, const AOTCacheAOTHeaderRecord *aotHeaderRecord, @@ -232,6 +435,75 @@ CachedAOTMethod::create(const AOTCacheClassChainRecord *definingClassChainRecord records, code, codeSize, data, dataSize); } +CachedAOTMethod * +CachedAOTMethod::read(FILE *f, + const Vector &classLoaderRecords, + const Vector &classRecords, + const Vector &methodRecords, + const Vector &classChainRecords, + const Vector &wellKnownClassesRecords, + const Vector &aotHeaderRecords) + { + SerializedAOTMethod header; + if (1 != fread(&header, sizeof(header), 1, f)) + return NULL; + + if ((!header.isValid()) || + (header.definingClassChainId() >= classChainRecords.size()) || + (header.aotHeaderId() >= aotHeaderRecords.size()) || + !classChainRecords[header.definingClassChainId()]) + return NULL; + + auto record = new (AOTCacheRecord::allocate(size(header.numRecords(), header.codeSize(), header.dataSize()))) + CachedAOTMethod(classChainRecords[header.definingClassChainId()]); + memcpy(&record->_data, &header, sizeof(header)); + + if (1 != fread(record->data().offsets(), sizeof(SerializedSCCOffset) * header.numRecords() + header.codeSize() + header.dataSize(), 1, f)) + goto error; + + for (size_t i = 0; i < header.numRecords(); ++i) + { + const SerializedSCCOffset &sccOffset = record->data().offsets()[i]; + + switch (sccOffset.recordType()) + { + case AOTSerializationRecordType::ClassLoader: + if ((sccOffset.recordId() >= classLoaderRecords.size()) || !classLoaderRecords[sccOffset.recordId()]) + goto error; + record->records()[i] = classLoaderRecords[sccOffset.recordId()]; + break; + case AOTSerializationRecordType::Class: + if ((sccOffset.recordId() >= classRecords.size()) || !classRecords[sccOffset.recordId()]) + goto error; + record->records()[i] = classRecords[sccOffset.recordId()]; + break; + case AOTSerializationRecordType::Method: + if ((sccOffset.recordId() >= methodRecords.size()) || !methodRecords[sccOffset.recordId()]) + goto error; + record->records()[i] = methodRecords[sccOffset.recordId()]; + break; + case AOTSerializationRecordType::ClassChain: + if ((sccOffset.recordId() >= classChainRecords.size()) || !classChainRecords[sccOffset.recordId()]) + goto error; + record->records()[i] = classChainRecords[sccOffset.recordId()]; + break; + case AOTSerializationRecordType::WellKnownClasses: + if ((sccOffset.recordId() >= wellKnownClassesRecords.size()) || !wellKnownClassesRecords[sccOffset.recordId()]) + goto error; + record->records()[i] = wellKnownClassesRecords[sccOffset.recordId()]; + break; + case AOTSerializationRecordType::AOTHeader: // never associated with an SCC offset + default: + goto error; + } + } + + return record; + +error: + AOTCacheRecord::free(record); + return NULL; + } bool JITServerAOTCache::ClassLoaderKey::operator==(const ClassLoaderKey &k) const @@ -351,6 +623,43 @@ addToMap(PersistentUnorderedMap &map, traversalTail = value; } +// Insert the value (which must be allocated with AOTCacheRecord::allocate()) +// with the key into the map, avoiding memory leaks in case of exceptions. +// If the map insertion was successful, also insert it into the linked list traversal of +// the map defined by the given head and tail. +template static bool +addToMap(PersistentUnorderedMap &map, + V *&traversalHead, + V *&traversalTail, + const K &key, V *value) + { + bool insertSuccess = false; + try + { + insertSuccess = map.insert({ key, value }).second; + } + catch (...) + { + AOTCacheRecord::free(value); + throw; + } + + if (!insertSuccess) + return false; + + if (traversalTail == NULL) + { + traversalHead = value; + } + else + { + traversalTail->setNextRecord(value); + } + traversalTail = value; + + return true; + } + // Free all the values (which must be allocated with AOTCacheRecord::allocate()) in the map. // NOTE: This function can only be used in the destructor of the object containing the map. // The now invalid pointers stay in the map, so it must be destroyed after this call. @@ -451,7 +760,7 @@ JITServerAOTCache::getClassLoaderRecord(const uint8_t *name, size_t nameLength) } auto record = AOTCacheClassLoaderRecord::create(_nextClassLoaderId, name, nameLength); - addToMap(_classLoaderMap, _classLoaderHead, _classLoaderTail, it, { record->data().name(), record->data().nameLength() }, record); + addToMap(_classLoaderMap, _classLoaderHead, _classLoaderTail, it, getRecordKey(record), record); ++_nextClassLoaderId; if (TR::Options::getVerboseOption(TR_VerboseJITServer)) @@ -484,7 +793,7 @@ JITServerAOTCache::getClassRecord(const AOTCacheClassLoaderRecord *classLoaderRe } auto record = AOTCacheClassRecord::create(_nextClassId, classLoaderRecord, hash, romClass); - addToMap(_classMap, _classHead, _classTail, it, { classLoaderRecord, &record->data().hash() }, record); + addToMap(_classMap, _classHead, _classTail, it, getRecordKey(record), record); ++_nextClassId; if (TR::Options::getVerboseOption(TR_VerboseJITServer)) @@ -504,10 +813,9 @@ const AOTCacheMethodRecord * JITServerAOTCache::getMethodRecord(const AOTCacheClassRecord *definingClassRecord, uint32_t index, const J9ROMMethod *romMethod) { - MethodKey key(definingClassRecord, index); OMR::CriticalSection cs(_methodMonitor); - auto it = _methodMap.find(key); + auto it = _methodMap.find({ definingClassRecord, index }); if (it != _methodMap.end()) return it->second; @@ -517,7 +825,7 @@ JITServerAOTCache::getMethodRecord(const AOTCacheClassRecord *definingClassRecor } auto record = AOTCacheMethodRecord::create(_nextMethodId, definingClassRecord, index); - addToMap(_methodMap, _methodHead, _methodTail, it, key, record); + addToMap(_methodMap, _methodHead, _methodTail, it, getRecordKey(record), record); ++_nextMethodId; if (TR::Options::getVerboseOption(TR_VerboseJITServer)) @@ -547,7 +855,7 @@ JITServerAOTCache::getClassChainRecord(const AOTCacheClassRecord *const *classRe } auto record = AOTCacheClassChainRecord::create(_nextClassChainId, classRecords, length); - addToMap(_classChainMap, _classChainHead, _classChainTail, it, { record->records(), length }, record); + addToMap(_classChainMap, _classChainHead, _classChainTail, it, getRecordKey(record), record); ++_nextClassChainId; if (TR::Options::getVerboseOption(TR_VerboseJITServer)) @@ -578,7 +886,7 @@ JITServerAOTCache::getWellKnownClassesRecord(const AOTCacheClassChainRecord *con } auto record = AOTCacheWellKnownClassesRecord::create(_nextWellKnownClassesId, chainRecords, length, includedClasses); - addToMap(_wellKnownClassesMap, _wellKnownClassesHead, _wellKnownClassesTail, it, { record->records(), length, includedClasses }, record); + addToMap(_wellKnownClassesMap, _wellKnownClassesHead, _wellKnownClassesTail, it, getRecordKey(record), record); ++_nextWellKnownClassesId; if (TR::Options::getVerboseOption(TR_VerboseJITServer)) @@ -612,7 +920,7 @@ JITServerAOTCache::getAOTHeaderRecord(const TR_AOTHeader *header, uint64_t clien } auto record = AOTCacheAOTHeaderRecord::create(_nextAOTHeaderId, header); - addToMap(_aotHeaderMap, _aotHeaderHead, _aotHeaderTail, it, { record->data().header() }, record); + addToMap(_aotHeaderMap, _aotHeaderHead, _aotHeaderTail, it, getRecordKey(record), record); ++_nextAOTHeaderId; if (TR::Options::getVerboseOption(TR_VerboseJITServer)) @@ -898,6 +1206,182 @@ JITServerAOTCache::writeCache(FILE *f) const return true; } +// Tests whether or not the given AOT snapshot is compatible with the server. +static bool +isCompatibleSnapshotVersion(const JITServerAOTCacheVersion &version) + { + JITServerAOTCacheVersion currentVersion = {0}; + getCurrentAOTCacheVersion(currentVersion); + + return (0 == memcmp(version._eyeCatcher, currentVersion._eyeCatcher, sizeof(currentVersion._eyeCatcher))) && + (version._snapshotVersion == currentVersion._snapshotVersion) && + (version._jitserverVersion == currentVersion._jitserverVersion); + } + +// Read an AOT cache snapshot, returning NULL if the cache is ill-formed or +// incompatible with the running server. +JITServerAOTCache * +JITServerAOTCache::readCache(FILE *f, const std::string &name, TR_Memory &trMemory) + { + if (!JITServerAOTCacheMap::cacheHasSpace()) + return NULL; + + JITServerAOTCacheHeader header = {0}; + if (1 != fread(&header, sizeof(JITServerAOTCacheHeader), 1, f)) + return NULL; + + if (!isCompatibleSnapshotVersion(header._version)) + return NULL; + + JITServerAOTCache *cache = NULL; + try + { + cache = new (TR::Compiler->persistentGlobalMemory()) JITServerAOTCache(name); + } + catch (const std::exception &e) + { + if (TR::Options::getVerboseOption(TR_VerboseJITServer)) + { + TR_VerboseLog::writeLineLocked(TR_Vlog_JITServer, "AOT cache allocation failed with exception: %s", e.what()); + } + } + + if (!cache) + return NULL; + + bool readSuccess = false; + try + { + readSuccess = cache->readCache(f, header, trMemory); + } + catch (const std::exception &e) + { + if (TR::Options::getVerboseOption(TR_VerboseJITServer)) + { + TR_VerboseLog::writeLineLocked(TR_Vlog_JITServer, "AOT cache reading failed with exception: %s", e.what()); + } + } + + if (!readSuccess) + { + cache->~JITServerAOTCache(); + TR::Compiler->persistentGlobalMemory()->freePersistentMemory(cache); + cache = NULL; + } + + return cache; + } + +// Read numRecordsToRead records of an AOTSerializationRecord subclass V from a stream, also +// updating the map, record traversal, and scratch Vector associated with V. +template bool +JITServerAOTCache::readRecords(FILE *f, + size_t numRecordsToRead, + PersistentUnorderedMap &map, + V *&traversalHead, + V *&traversalTail, + Vector &records, + const Vector &classLoaderRecords, + const Vector &classRecords, + const Vector &methodRecords, + const Vector &classChainRecords, + const Vector &wellKnownClassesRecords, + const Vector &aotHeaderRecords) + { + for (size_t i = 0; i < numRecordsToRead; ++i) + { + if (!JITServerAOTCacheMap::cacheHasSpace()) + return false; + + V *record = V::read(f, classLoaderRecords, classRecords, methodRecords, classChainRecords, wellKnownClassesRecords, aotHeaderRecords); + if (!record) + return false; + + if ((record->data().id() >= records.size() || + records[record->data().id()]) || + !addToMap(map, traversalHead, traversalTail, getRecordKey(record), record)) + { + AOTCacheRecord::free(record); + return false; + } + + records[record->data().id()] = record; + } + + return true; + } + +bool +JITServerAOTCache::readCache(FILE *f, const JITServerAOTCacheHeader &header, TR_Memory &trMemory) + { + _classLoaderMap.reserve(header._numClassLoaderRecords); + _classMap.reserve(header._numClassRecords); + _methodMap.reserve(header._numMethodRecords); + _classChainMap.reserve(header._numClassChainRecords); + _wellKnownClassesMap.reserve(header._numWellKnownClassesRecords); + _aotHeaderMap.reserve(header._numAOTHeaderRecords); + _cachedMethodMap.reserve(header._numCachedAOTMethods); + + _nextClassLoaderId = header._nextClassLoaderId; + _nextClassId = header._nextClassId; + _nextMethodId = header._nextMethodId; + _nextClassChainId = header._nextClassChainId; + _nextWellKnownClassesId = header._nextWellKnownClassesId; + _nextAOTHeaderId = header._nextAOTHeaderId; + + TR::StackMemoryRegion stackMemoryRegion(trMemory); + Vector classLoaderRecords(header._nextClassLoaderId, NULL, stackMemoryRegion); + Vector classRecords(header._nextClassId, NULL, stackMemoryRegion); + Vector methodRecords(header._nextMethodId, NULL, stackMemoryRegion); + Vector classChainRecords(header._nextClassChainId, NULL, stackMemoryRegion); + Vector wellKnownClassesRecords(header._nextWellKnownClassesId, NULL, stackMemoryRegion); + Vector aotHeaderRecords(header._nextAOTHeaderId, NULL, stackMemoryRegion); + + if (!readRecords(f, header._numClassLoaderRecords, _classLoaderMap, _classLoaderHead, _classLoaderTail, classLoaderRecords, + classLoaderRecords, classRecords, methodRecords, classChainRecords, wellKnownClassesRecords, aotHeaderRecords)) + return false; + if (!readRecords(f, header._numClassRecords, _classMap, _classHead, _classTail, classRecords, + classLoaderRecords, classRecords, methodRecords, classChainRecords, wellKnownClassesRecords, aotHeaderRecords)) + return false; + if (!readRecords(f, header._numMethodRecords, _methodMap, _methodHead, _methodTail, methodRecords, + classLoaderRecords, classRecords, methodRecords, classChainRecords, wellKnownClassesRecords, aotHeaderRecords)) + return false; + if (!readRecords(f, header._numClassChainRecords, _classChainMap, _classChainHead, _classChainTail, classChainRecords, + classLoaderRecords, classRecords, methodRecords, classChainRecords, wellKnownClassesRecords, aotHeaderRecords)) + return false; + if (!readRecords(f, header._numWellKnownClassesRecords, _wellKnownClassesMap, _wellKnownClassesHead, _wellKnownClassesTail, wellKnownClassesRecords, + classLoaderRecords, classRecords, methodRecords, classChainRecords, wellKnownClassesRecords, aotHeaderRecords)) + return false; + if (!readRecords(f, header._numAOTHeaderRecords, _aotHeaderMap, _aotHeaderHead, _aotHeaderTail, aotHeaderRecords, + classLoaderRecords, classRecords, methodRecords, classChainRecords, wellKnownClassesRecords, aotHeaderRecords)) + return false; + + for (size_t i = 0; i < header._numCachedAOTMethods; ++i) + { + if (!JITServerAOTCacheMap::cacheHasSpace()) + return false; + + auto record = CachedAOTMethod::read(f, classLoaderRecords, classRecords, methodRecords, + classChainRecords, wellKnownClassesRecords, aotHeaderRecords); + + if (!record || !aotHeaderRecords[record->data().aotHeaderId()]) + return false; + + CachedMethodKey key(record->definingClassChainRecord(), + record->data().index(), + record->data().optLevel(), + aotHeaderRecords[record->data().aotHeaderId()]); + + if (!addToMap(_cachedMethodMap, _cachedMethodHead, _cachedMethodTail, key, record)) + { + AOTCacheRecord::free(record); + return false; + } + } + + return true; + } + bool JITServerAOTCacheMap::cacheHasSpace() { diff --git a/runtime/compiler/runtime/JITServerAOTCache.hpp b/runtime/compiler/runtime/JITServerAOTCache.hpp index 144e86636e7..35018af1f2c 100644 --- a/runtime/compiler/runtime/JITServerAOTCache.hpp +++ b/runtime/compiler/runtime/JITServerAOTCache.hpp @@ -65,6 +65,13 @@ struct JITServerAOTCacheHeader size_t _nextAOTHeaderId; }; +struct AOTCacheClassLoaderRecord; +struct AOTCacheClassRecord; +struct AOTCacheMethodRecord; +struct AOTCacheClassChainRecord; +struct AOTCacheWellKnownClassesRecord; +struct AOTCacheAOTHeaderRecord; + // Base class for serialization record "wrappers" stored at the server. // // When a cached serialized method is sent to a client, the server needs @@ -119,8 +126,17 @@ class AOTCacheClassLoaderRecord final : public AOTCacheRecord static AOTCacheClassLoaderRecord *create(uintptr_t id, const uint8_t *name, size_t nameLength); + static AOTCacheClassLoaderRecord *read(FILE *f, + const Vector &classLoaderRecords, + const Vector &classRecords, + const Vector &methodRecords, + const Vector &classChainRecords, + const Vector &wellKnownClassesRecords, + const Vector &aotHeaderRecords); + private: AOTCacheClassLoaderRecord(uintptr_t id, const uint8_t *name, size_t nameLength); + AOTCacheClassLoaderRecord() {} static size_t size(size_t nameLength) { @@ -142,9 +158,18 @@ class AOTCacheClassRecord final : public AOTCacheRecord const JITServerROMClassHash &hash, const J9ROMClass *romClass); void subRecordsDo(const std::function &f) const override; + static AOTCacheClassRecord *read(FILE *f, + const Vector &classLoaderRecords, + const Vector &classRecords, + const Vector &methodRecords, + const Vector &classChainRecords, + const Vector &wellKnownClassesRecords, + const Vector &aotHeaderRecords); + private: AOTCacheClassRecord(uintptr_t id, const AOTCacheClassLoaderRecord *classLoaderRecord, const JITServerROMClassHash &hash, const J9ROMClass *romClass); + AOTCacheClassRecord(const AOTCacheClassLoaderRecord *classLoaderRecord); static size_t size(size_t nameLength) { @@ -166,8 +191,17 @@ class AOTCacheMethodRecord final : public AOTCacheRecord static AOTCacheMethodRecord *create(uintptr_t id, const AOTCacheClassRecord *definingClassRecord, uint32_t index); void subRecordsDo(const std::function &f) const override; + static AOTCacheMethodRecord *read(FILE *f, + const Vector &classLoaderRecords, + const Vector &classRecords, + const Vector &methodRecords, + const Vector &classChainRecords, + const Vector &wellKnownClassesRecords, + const Vector &aotHeaderRecords); + private: AOTCacheMethodRecord(uintptr_t id, const AOTCacheClassRecord *definingClassRecord, uint32_t index); + AOTCacheMethodRecord(const AOTCacheClassRecord *definingClassRecord); const AOTCacheClassRecord *const _definingClassRecord; const MethodSerializationRecord _data; @@ -185,11 +219,15 @@ class AOTCacheListRecord : public AOTCacheRecord const AOTSerializationRecord *dataAddr() const override { return &_data; } // Array of record pointers is stored inline after the array of IDs that is stored inline after struct D header const R *const *records() const { return (const R *const *)_data.end(); } + R **records() { return (R **)_data.end(); } void subRecordsDo(const std::function &f) const override; + static AOTCacheListRecord *read(FILE *f, const Vector &cacheRecords); + protected: AOTCacheListRecord(uintptr_t id, const R *const *records, size_t length, Args... args); + AOTCacheListRecord() {} static size_t size(size_t length) { @@ -209,6 +247,15 @@ class AOTCacheClassChainRecord final : public AOTCacheListRecordclassLoaderRecord(); } + static AOTCacheClassChainRecord *read(FILE *f, + const Vector &classLoaderRecords, + const Vector &classRecords, + const Vector &methodRecords, + const Vector &classChainRecords, + const Vector &wellKnownClassesRecords, + const Vector &aotHeaderRecords) + { return (AOTCacheClassChainRecord *)AOTCacheListRecord::read(f, classRecords); } + private: using AOTCacheListRecord::AOTCacheListRecord; }; @@ -221,6 +268,16 @@ class AOTCacheWellKnownClassesRecord final : static AOTCacheWellKnownClassesRecord *create(uintptr_t id, const AOTCacheClassChainRecord *const *records, size_t length, uintptr_t includedClasses); + static AOTCacheWellKnownClassesRecord *read(FILE *f, + const Vector &classLoaderRecords, + const Vector &classRecords, + const Vector &methodRecords, + const Vector &classChainRecords, + const Vector &wellKnownClassesRecords, + const Vector &aotHeaderRecords) + { return (AOTCacheWellKnownClassesRecord *) + AOTCacheListRecord::read(f, classChainRecords); } + private: using AOTCacheListRecord::AOTCacheListRecord; }; @@ -234,8 +291,17 @@ class AOTCacheAOTHeaderRecord final : public AOTCacheRecord static AOTCacheAOTHeaderRecord *create(uintptr_t id, const TR_AOTHeader *header); + static AOTCacheAOTHeaderRecord *read(FILE *f, + const Vector &classLoaderRecords, + const Vector &classRecords, + const Vector &methodRecords, + const Vector &classChainRecords, + const Vector &wellKnownClassesRecords, + const Vector &aotHeaderRecords); + private: AOTCacheAOTHeaderRecord(uintptr_t id, const TR_AOTHeader *header); + AOTCacheAOTHeaderRecord() {} const AOTHeaderSerializationRecord _data; }; @@ -255,12 +321,23 @@ class CachedAOTMethod const SerializedAOTMethod &data() const { return _data; } SerializedAOTMethod &data() { return _data; } const AOTCacheRecord *const *records() const { return (const AOTCacheRecord *const *)_data.end(); } + AOTCacheRecord **records() { return (AOTCacheRecord **)_data.end(); } static CachedAOTMethod *create(const AOTCacheClassChainRecord *definingClassChainRecord, uint32_t index, TR_Hotness optLevel, const AOTCacheAOTHeaderRecord *aotHeaderRecord, const Vector> &records, const void *code, size_t codeSize, const void *data, size_t dataSize); + static CachedAOTMethod *read(FILE *f, + const Vector &classLoaderRecords, + const Vector &classRecords, + const Vector &methodRecords, + const Vector &classChainRecords, + const Vector &wellKnownClassesRecords, + const Vector &aotHeaderRecords); + + static CachedAOTMethod *read(FILE *f, const Vector &classChainRecords); + CachedAOTMethod *getNextRecord() const { return _nextRecord; } void setNextRecord(CachedAOTMethod *record) { _nextRecord = record; } @@ -269,6 +346,7 @@ class CachedAOTMethod TR_Hotness optLevel, const AOTCacheAOTHeaderRecord *aotHeaderRecord, const Vector> &records, const void *code, size_t codeSize, const void *data, size_t dataSize); + CachedAOTMethod(const AOTCacheClassChainRecord *definingClassChainRecord); static size_t size(size_t numRecords, size_t codeSize, size_t dataSize) { @@ -345,7 +423,7 @@ class JITServerAOTCache void printStats(FILE *f) const; bool writeCache(FILE *f) const; - static JITServerAOTCache *readCache(FILE *f, const std::string &name); + static JITServerAOTCache *readCache(FILE *f, const std::string &name, TR_Memory &trMemory); private: struct ClassLoaderKey @@ -357,6 +435,9 @@ class JITServerAOTCache const size_t _nameLength; }; + static ClassLoaderKey getRecordKey(const AOTCacheClassLoaderRecord *record) + { return { record->data().name(), record->data().nameLength() }; } + struct ClassKey { bool operator==(const ClassKey &k) const; @@ -366,8 +447,14 @@ class JITServerAOTCache const JITServerROMClassHash *const _hash; }; + static ClassKey getRecordKey(const AOTCacheClassRecord *record) + { return { record->classLoaderRecord(), &record->data().hash() }; } + using MethodKey = std::pair; + static MethodKey getRecordKey(const AOTCacheMethodRecord *record) + { return { record->definingClassRecord(), record->data().index() }; } + struct ClassChainKey { bool operator==(const ClassChainKey &k) const; @@ -377,6 +464,9 @@ class JITServerAOTCache const size_t _length; }; + static ClassChainKey getRecordKey(const AOTCacheClassChainRecord *record) + { return { record->records(), record->data().list().length() }; } + struct WellKnownClassesKey { bool operator==(const WellKnownClassesKey &k) const; @@ -387,6 +477,9 @@ class JITServerAOTCache const uintptr_t _includedClasses; }; + static WellKnownClassesKey getRecordKey(const AOTCacheWellKnownClassesRecord *record) + { return { record->records(), record->data().list().length(), record->data().includedClasses() }; } + struct AOTHeaderKey { bool operator==(const AOTHeaderKey &k) const; @@ -395,6 +488,9 @@ class JITServerAOTCache const TR_AOTHeader *const _header; }; + static AOTHeaderKey getRecordKey(const AOTCacheAOTHeaderRecord *record) + { return { record->data().header() }; } + //NOTE: Current implementation doesn't support compatible differences in AOT headers. // A cached method can only be sent to a client with the exact same AOT header. using CachedMethodKey = std::tuple &result, UnorderedSet &newRecords, const KnownIdSet &knownIds) const; + // Read a cache snapshot into an empty cache + bool readCache(FILE *f, const JITServerAOTCacheHeader &header, TR_Memory &trMemory); + + template + static bool readRecords(FILE *f, size_t numRecordsToRead, PersistentUnorderedMap &map, V *&traversalHead, V *&traversalTail, + Vector &records, + const Vector &classLoaderRecords, + const Vector &classRecords, + const Vector &methodRecords, + const Vector &classChainRecords, + const Vector &wellKnownClassesRecords, + const Vector &aotHeaderRecords); const std::string _name; diff --git a/runtime/compiler/runtime/JITServerAOTSerializationRecords.hpp b/runtime/compiler/runtime/JITServerAOTSerializationRecords.hpp index 343f7dd0f96..3afcfd244f5 100644 --- a/runtime/compiler/runtime/JITServerAOTSerializationRecords.hpp +++ b/runtime/compiler/runtime/JITServerAOTSerializationRecords.hpp @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright (c) 2021, 2021 IBM Corp. and others + * Copyright (c) 2021, 2022 IBM Corp. and others * * This program and the accompanying materials are made available under * the terms of the Eclipse Public License 2.0 which accompanies this @@ -64,6 +64,7 @@ struct AOTSerializationRecord //NOTE: 0 signifies an invalid record ID uintptr_t id() const { return getId(_idAndType); } AOTSerializationRecordType type() const { return getType(_idAndType); } + bool isValid(AOTSerializationRecordType type) const; const uint8_t *end() const { return (const uint8_t *)this + size(); } static const AOTSerializationRecord *get(const std::string &str) @@ -110,11 +111,13 @@ struct ClassLoaderSerializationRecord : public AOTSerializationRecord public: size_t nameLength() const { return _nameLength; } const uint8_t *name() const { return _name; } + bool isValid() const { return AOTSerializationRecord::isValid(AOTSerializationRecordType::ClassLoader); } private: friend class AOTCacheClassLoaderRecord; ClassLoaderSerializationRecord(uintptr_t id, const uint8_t *name, size_t nameLength); + ClassLoaderSerializationRecord(); static size_t size(size_t nameLength) { @@ -135,12 +138,14 @@ struct ClassSerializationRecord : public AOTSerializationRecord uint32_t romClassSize() const { return _romClassSize; } size_t nameLength() const { return _nameLength; } const uint8_t *name() const { return _name; } + bool isValid() const { return AOTSerializationRecord::isValid(AOTSerializationRecordType::Class); } private: friend class AOTCacheClassRecord; ClassSerializationRecord(uintptr_t id, uintptr_t classLoaderId, const JITServerROMClassHash &hash, const J9ROMClass *romClass); + ClassSerializationRecord(); static size_t size(size_t nameLength) { @@ -162,11 +167,13 @@ struct MethodSerializationRecord : public AOTSerializationRecord public: uintptr_t definingClassId() const { return _definingClassId; } uint32_t index() const { return _index; } + bool isValid() const { return AOTSerializationRecord::isValid(AOTSerializationRecordType::Method); } private: friend class AOTCacheMethodRecord; MethodSerializationRecord(uintptr_t id, uintptr_t definingClassId, uint32_t index); + MethodSerializationRecord(); const uintptr_t _definingClassId; // Index in the array of methods of the defining class @@ -195,11 +202,13 @@ struct ClassChainSerializationRecord : public AOTSerializationRecord { public: const IdList &list() const { return _list; } + bool isValid() const { return AOTSerializationRecord::isValid(AOTSerializationRecordType::ClassChain); } private: template friend class AOTCacheListRecord; ClassChainSerializationRecord(uintptr_t id, size_t length); + ClassChainSerializationRecord(); IdList &list() { return _list; } @@ -218,11 +227,13 @@ struct WellKnownClassesSerializationRecord : public AOTSerializationRecord public: uintptr_t includedClasses() const { return _includedClasses; } const IdList &list() const { return _list; } + bool isValid() const { return AOTSerializationRecord::isValid(AOTSerializationRecordType::WellKnownClasses); } private: template friend class AOTCacheListRecord; WellKnownClassesSerializationRecord(uintptr_t id, size_t length, uintptr_t includedClasses); + WellKnownClassesSerializationRecord(); IdList &list() { return _list; } @@ -242,11 +253,13 @@ struct AOTHeaderSerializationRecord : public AOTSerializationRecord { public: const TR_AOTHeader *header() const { return &_header; } + bool isValid() const { return AOTSerializationRecord::isValid(AOTSerializationRecordType::AOTHeader); } private: friend class AOTCacheAOTHeaderRecord; AOTHeaderSerializationRecord(uintptr_t id, const TR_AOTHeader *header); + AOTHeaderSerializationRecord(); const TR_AOTHeader _header; }; @@ -294,6 +307,7 @@ struct SerializedAOTMethod const uint8_t *data() const { return code() + _codeSize; } uint8_t *data() { return (uint8_t *)(code() + _codeSize); } const uint8_t *end() const { return (const uint8_t *)this + size(); } + bool isValid() const; static SerializedAOTMethod *get(std::string &str) { @@ -308,6 +322,7 @@ struct SerializedAOTMethod SerializedAOTMethod(uintptr_t definingClassChainId, uint32_t index, TR_Hotness optLevel, uintptr_t aotHeaderId, size_t numRecords, const void *code, size_t codeSize, const void *data, size_t dataSize); + SerializedAOTMethod(); static size_t size(size_t numRecords, size_t codeSize, size_t dataSize) {