diff --git a/runtime/compiler/runtime/JITServerAOTCache.cpp b/runtime/compiler/runtime/JITServerAOTCache.cpp index a4337f5fb6c..d3543768466 100644 --- a/runtime/compiler/runtime/JITServerAOTCache.cpp +++ b/runtime/compiler/runtime/JITServerAOTCache.cpp @@ -46,6 +46,18 @@ AOTCacheRecord::free(void *ptr) TR::Compiler->persistentGlobalMemory()->freePersistentMemory(ptr); } +bool +AOTSerializationRecord::isValid(AOTSerializationRecordType type) const + { + return (type == this->type()) && + (0 != this->id()); + } + +ClassLoaderSerializationRecord::ClassLoaderSerializationRecord() : + AOTSerializationRecord(0, 0, AOTSerializationRecordType::ClassLoader), + _nameLength(0) + { + } ClassLoaderSerializationRecord::ClassLoaderSerializationRecord(uintptr_t id, const uint8_t *name, size_t nameLength) : AOTSerializationRecord(size(nameLength), id, AOTSerializationRecordType::ClassLoader), @@ -54,6 +66,7 @@ ClassLoaderSerializationRecord::ClassLoaderSerializationRecord(uintptr_t id, con memcpy(_name, name, nameLength); } + AOTCacheClassLoaderRecord::AOTCacheClassLoaderRecord(uintptr_t id, const uint8_t *name, size_t nameLength) : _data(id, name, nameLength) { @@ -66,6 +79,32 @@ AOTCacheClassLoaderRecord::create(uintptr_t id, const uint8_t *name, size_t name return new (ptr) AOTCacheClassLoaderRecord(id, name, nameLength); } +AOTCacheClassLoaderRecord * +AOTCacheClassLoaderRecord::read(FILE *f, + const Vector &classLoaderRecords, + const Vector &classRecords, + const Vector &methodRecords, + const Vector &classChainRecords, + const Vector &wellKnownClassesRecords, + const Vector &aotHeaderRecords) + { + ClassLoaderSerializationRecord header; + if (1 != fread(&header, sizeof(header), 1, f)) + return NULL; + + if (!header.isValid()) + return NULL; + + auto record = new (AOTCacheRecord::allocate(size(header.nameLength()))) AOTCacheClassLoaderRecord(); + memcpy((void *)record->dataAddr(), &header, sizeof(header)); + if (1 != fread((uint8_t *)record->dataAddr() + sizeof(header), header.AOTSerializationRecord::size() - sizeof(header), 1, f)) + { + AOTCacheRecord::free(record); + return NULL; + } + + return record; + } ClassSerializationRecord::ClassSerializationRecord(uintptr_t id, uintptr_t classLoaderId, const JITServerROMClassHash &hash, const J9ROMClass *romClass) : @@ -76,6 +115,12 @@ ClassSerializationRecord::ClassSerializationRecord(uintptr_t id, uintptr_t class memcpy(_name, J9UTF8_DATA(J9ROMCLASS_CLASSNAME(romClass)), _nameLength); } +ClassSerializationRecord::ClassSerializationRecord() : + AOTSerializationRecord(0, 0, AOTSerializationRecordType::Class), + _classLoaderId(0), _hash(), _romClassSize(0), _nameLength(0) + { + } + AOTCacheClassRecord::AOTCacheClassRecord(uintptr_t id, const AOTCacheClassLoaderRecord *classLoaderRecord, const JITServerROMClassHash &hash, const J9ROMClass *romClass) : _classLoaderRecord(classLoaderRecord), @@ -83,6 +128,11 @@ AOTCacheClassRecord::AOTCacheClassRecord(uintptr_t id, const AOTCacheClassLoader { } +AOTCacheClassRecord::AOTCacheClassRecord(const AOTCacheClassLoaderRecord *classLoaderRecord) : + _classLoaderRecord(classLoaderRecord) + { + } + AOTCacheClassRecord * AOTCacheClassRecord::create(uintptr_t id, const AOTCacheClassLoaderRecord *classLoaderRecord, const JITServerROMClassHash &hash, const J9ROMClass *romClass) @@ -97,6 +147,34 @@ AOTCacheClassRecord::subRecordsDo(const std::function &classLoaderRecords, + const Vector &classRecords, + const Vector &methodRecords, + const Vector &classChainRecords, + const Vector &wellKnownClassesRecords, + const Vector &aotHeaderRecords) + { + ClassSerializationRecord header; + if (1 != fread(&header, sizeof(header), 1, f)) + return NULL; + + if (!header.isValid() || + (header.classLoaderId() >= classLoaderRecords.size()) || + !classLoaderRecords[header.classLoaderId()]) + return NULL; + + auto record = new (AOTCacheRecord::allocate(size(header.nameLength()))) AOTCacheClassRecord(classLoaderRecords[header.classLoaderId()]); + memcpy((void *)record->dataAddr(), &header, sizeof(header)); + if (1 != fread((uint8_t *)record->dataAddr() + sizeof(header), header.AOTSerializationRecord::size() - sizeof(header), 1, f)) + { + AOTCacheRecord::free(record); + return NULL; + } + + return record; + } MethodSerializationRecord::MethodSerializationRecord(uintptr_t id, uintptr_t definingClassId, uint32_t index) : AOTSerializationRecord(sizeof(*this), id, AOTSerializationRecordType::Method), @@ -104,6 +182,12 @@ MethodSerializationRecord::MethodSerializationRecord(uintptr_t id, uintptr_t def { } +MethodSerializationRecord::MethodSerializationRecord() : + AOTSerializationRecord(0, 0, AOTSerializationRecordType::Method), + _definingClassId(0), _index(0) + { + } + AOTCacheMethodRecord::AOTCacheMethodRecord(uintptr_t id, const AOTCacheClassRecord *definingClassRecord, uint32_t index) : _definingClassRecord(definingClassRecord), @@ -111,6 +195,11 @@ AOTCacheMethodRecord::AOTCacheMethodRecord(uintptr_t id, const AOTCacheClassReco { } +AOTCacheMethodRecord::AOTCacheMethodRecord(const AOTCacheClassRecord *definingClassRecord) : + _definingClassRecord(definingClassRecord) + { + } + AOTCacheMethodRecord * AOTCacheMethodRecord::create(uintptr_t id, const AOTCacheClassRecord *definingClassRecord, uint32_t index) { @@ -124,6 +213,29 @@ AOTCacheMethodRecord::subRecordsDo(const std::function &classLoaderRecords, + const Vector &classRecords, + const Vector &methodRecords, + const Vector &classChainRecords, + const Vector &wellKnownClassesRecords, + const Vector &aotHeaderRecords) + { + MethodSerializationRecord header; + if (1 != fread(&header, sizeof(header), 1, f)) + return NULL; + + if (!header.isValid() || + (header.definingClassId() >= classRecords.size()) || + !classRecords[header.definingClassId()]) + return NULL; + + auto record = new (AOTCacheRecord::allocate(sizeof(AOTCacheMethodRecord))) AOTCacheMethodRecord(classRecords[header.definingClassId()]); + memcpy((void *)record->dataAddr(), &header, sizeof(header)); + + return record; + } template AOTCacheListRecord::AOTCacheListRecord(uintptr_t id, const R *const *records, @@ -142,6 +254,38 @@ AOTCacheListRecord::subRecordsDo(const std::function AOTCacheListRecord* +AOTCacheListRecord::read(FILE *f, const Vector &cacheRecords) + { + D header; + if (1 != fread(&header, sizeof(header), 1, f)) + return NULL; + + if (!header.isValid()) + return NULL; + + auto record = new (AOTCacheRecord::allocate(size(header.list().length()))) AOTCacheListRecord(); + memcpy((void *)record->dataAddr(), &header, sizeof(header)); + + if (1 != fread((uint8_t *)record->dataAddr() + sizeof(header), header.AOTSerializationRecord::size() - sizeof(header), 1, f)) + { + AOTCacheRecord::free(record); + return NULL; + } + + for (size_t i = 0; i < header.list().length(); ++i) + { + uintptr_t id = record->data().list().ids()[i]; + if ((id >= cacheRecords.size()) || !cacheRecords[id]) + { + AOTCacheRecord::free(record); + return NULL; + } + record->records()[i] = cacheRecords[id]; + } + + return record; + } ClassChainSerializationRecord::ClassChainSerializationRecord(uintptr_t id, size_t length) : AOTSerializationRecord(size(length), id, AOTSerializationRecordType::ClassChain), @@ -149,6 +293,12 @@ ClassChainSerializationRecord::ClassChainSerializationRecord(uintptr_t id, size_ { } +ClassChainSerializationRecord::ClassChainSerializationRecord() : + AOTSerializationRecord(0, 0, AOTSerializationRecordType::ClassChain), + _list(0) + { + } + AOTCacheClassChainRecord * AOTCacheClassChainRecord::create(uintptr_t id, const AOTCacheClassRecord *const *records, size_t length) { @@ -164,6 +314,12 @@ WellKnownClassesSerializationRecord::WellKnownClassesSerializationRecord(uintptr { } +WellKnownClassesSerializationRecord::WellKnownClassesSerializationRecord() : + AOTSerializationRecord(0, 0, AOTSerializationRecordType::WellKnownClasses), + _includedClasses(0), _list(0) + { + } + AOTCacheWellKnownClassesRecord * AOTCacheWellKnownClassesRecord::create(uintptr_t id, const AOTCacheClassChainRecord *const *records, size_t length, uintptr_t includedClasses) @@ -179,6 +335,12 @@ AOTHeaderSerializationRecord::AOTHeaderSerializationRecord(uintptr_t id, const T { } +AOTHeaderSerializationRecord::AOTHeaderSerializationRecord() : + AOTSerializationRecord(0, 0, AOTSerializationRecordType::AOTHeader), + _header({0}) + { + } + AOTCacheAOTHeaderRecord::AOTCacheAOTHeaderRecord(uintptr_t id, const TR_AOTHeader *header) : _data(id, header) { @@ -191,6 +353,27 @@ AOTCacheAOTHeaderRecord::create(uintptr_t id, const TR_AOTHeader *header) return new (ptr) AOTCacheAOTHeaderRecord(id, header); } +AOTCacheAOTHeaderRecord * +AOTCacheAOTHeaderRecord::read(FILE *f, + const Vector &classLoaderRecords, + const Vector &classRecords, + const Vector &methodRecords, + const Vector &classChainRecords, + const Vector &wellKnownClassesRecords, + const Vector &aotHeaderRecords) + { + AOTHeaderSerializationRecord header; + if (1 != fread(&header, sizeof(header), 1, f)) + return NULL; + + if (!header.isValid()) + return NULL; + + auto record = new (AOTCacheRecord::allocate(sizeof(AOTCacheAOTHeaderRecord))) AOTCacheAOTHeaderRecord(); + memcpy((void *)record->dataAddr(), &header, sizeof(header)); + + return record; + } SerializedAOTMethod::SerializedAOTMethod(uintptr_t definingClassChainId, uint32_t index, TR_Hotness optLevel, uintptr_t aotHeaderId, size_t numRecords, @@ -204,6 +387,20 @@ SerializedAOTMethod::SerializedAOTMethod(uintptr_t definingClassChainId, uint32_ memcpy((void *)this->data(), data, dataSize); } +SerializedAOTMethod::SerializedAOTMethod() : + _size(0), + _definingClassChainId(0), _index(0), + _optLevel(TR_Hotness::numHotnessLevels), _aotHeaderId(0), + _numRecords(0), _codeSize(0), _dataSize(0) + { + } + +bool +SerializedAOTMethod::isValid() const + { + return _optLevel < TR_Hotness::numHotnessLevels; + } + CachedAOTMethod::CachedAOTMethod(const AOTCacheClassChainRecord *definingClassChainRecord, uint32_t index, TR_Hotness optLevel, const AOTCacheAOTHeaderRecord *aotHeaderRecord, const Vector> &records, @@ -221,6 +418,12 @@ CachedAOTMethod::CachedAOTMethod(const AOTCacheClassChainRecord *definingClassCh } } +CachedAOTMethod::CachedAOTMethod(const AOTCacheClassChainRecord *definingClassChainRecord) : + _nextRecord(NULL), + _definingClassChainRecord(definingClassChainRecord) + { + } + CachedAOTMethod * CachedAOTMethod::create(const AOTCacheClassChainRecord *definingClassChainRecord, uint32_t index, TR_Hotness optLevel, const AOTCacheAOTHeaderRecord *aotHeaderRecord, @@ -232,6 +435,75 @@ CachedAOTMethod::create(const AOTCacheClassChainRecord *definingClassChainRecord records, code, codeSize, data, dataSize); } +CachedAOTMethod * +CachedAOTMethod::read(FILE *f, + const Vector &classLoaderRecords, + const Vector &classRecords, + const Vector &methodRecords, + const Vector &classChainRecords, + const Vector &wellKnownClassesRecords, + const Vector &aotHeaderRecords) + { + SerializedAOTMethod header; + if (1 != fread(&header, sizeof(header), 1, f)) + return NULL; + + if ((!header.isValid()) || + (header.definingClassChainId() >= classChainRecords.size()) || + (header.aotHeaderId() >= aotHeaderRecords.size()) || + !classChainRecords[header.definingClassChainId()]) + return NULL; + + auto record = new (AOTCacheRecord::allocate(size(header.numRecords(), header.codeSize(), header.dataSize()))) + CachedAOTMethod(classChainRecords[header.definingClassChainId()]); + memcpy(&record->_data, &header, sizeof(header)); + + if (1 != fread(record->data().offsets(), sizeof(SerializedSCCOffset) * header.numRecords() + header.codeSize() + header.dataSize(), 1, f)) + goto error; + + for (size_t i = 0; i < header.numRecords(); ++i) + { + const SerializedSCCOffset &sccOffset = record->data().offsets()[i]; + + switch (sccOffset.recordType()) + { + case AOTSerializationRecordType::ClassLoader: + if ((sccOffset.recordId() >= classLoaderRecords.size()) || !classLoaderRecords[sccOffset.recordId()]) + goto error; + record->records()[i] = classLoaderRecords[sccOffset.recordId()]; + break; + case AOTSerializationRecordType::Class: + if ((sccOffset.recordId() >= classRecords.size()) || !classRecords[sccOffset.recordId()]) + goto error; + record->records()[i] = classRecords[sccOffset.recordId()]; + break; + case AOTSerializationRecordType::Method: + if ((sccOffset.recordId() >= methodRecords.size()) || !methodRecords[sccOffset.recordId()]) + goto error; + record->records()[i] = methodRecords[sccOffset.recordId()]; + break; + case AOTSerializationRecordType::ClassChain: + if ((sccOffset.recordId() >= classChainRecords.size()) || !classChainRecords[sccOffset.recordId()]) + goto error; + record->records()[i] = classChainRecords[sccOffset.recordId()]; + break; + case AOTSerializationRecordType::WellKnownClasses: + if ((sccOffset.recordId() >= wellKnownClassesRecords.size()) || !wellKnownClassesRecords[sccOffset.recordId()]) + goto error; + record->records()[i] = wellKnownClassesRecords[sccOffset.recordId()]; + break; + case AOTSerializationRecordType::AOTHeader: // never associated with an SCC offset + default: + goto error; + } + } + + return record; + +error: + AOTCacheRecord::free(record); + return NULL; + } bool JITServerAOTCache::ClassLoaderKey::operator==(const ClassLoaderKey &k) const @@ -351,6 +623,43 @@ addToMap(PersistentUnorderedMap &map, traversalTail = value; } +// Insert the value (which must be allocated with AOTCacheRecord::allocate()) +// with the key into the map, avoiding memory leaks in case of exceptions. +// If the map insertion was successful, also insert it into the linked list traversal of +// the map defined by the given head and tail. +template static bool +addToMap(PersistentUnorderedMap &map, + V *&traversalHead, + V *&traversalTail, + const K &key, V *value) + { + bool insertSuccess = false; + try + { + insertSuccess = map.insert({ key, value }).second; + } + catch (...) + { + AOTCacheRecord::free(value); + throw; + } + + if (!insertSuccess) + return false; + + if (traversalTail == NULL) + { + traversalHead = value; + } + else + { + traversalTail->setNextRecord(value); + } + traversalTail = value; + + return true; + } + // Free all the values (which must be allocated with AOTCacheRecord::allocate()) in the map. // NOTE: This function can only be used in the destructor of the object containing the map. // The now invalid pointers stay in the map, so it must be destroyed after this call. @@ -451,7 +760,7 @@ JITServerAOTCache::getClassLoaderRecord(const uint8_t *name, size_t nameLength) } auto record = AOTCacheClassLoaderRecord::create(_nextClassLoaderId, name, nameLength); - addToMap(_classLoaderMap, _classLoaderHead, _classLoaderTail, it, { record->data().name(), record->data().nameLength() }, record); + addToMap(_classLoaderMap, _classLoaderHead, _classLoaderTail, it, getRecordKey(record), record); ++_nextClassLoaderId; if (TR::Options::getVerboseOption(TR_VerboseJITServer)) @@ -484,7 +793,7 @@ JITServerAOTCache::getClassRecord(const AOTCacheClassLoaderRecord *classLoaderRe } auto record = AOTCacheClassRecord::create(_nextClassId, classLoaderRecord, hash, romClass); - addToMap(_classMap, _classHead, _classTail, it, { classLoaderRecord, &record->data().hash() }, record); + addToMap(_classMap, _classHead, _classTail, it, getRecordKey(record), record); ++_nextClassId; if (TR::Options::getVerboseOption(TR_VerboseJITServer)) @@ -504,10 +813,9 @@ const AOTCacheMethodRecord * JITServerAOTCache::getMethodRecord(const AOTCacheClassRecord *definingClassRecord, uint32_t index, const J9ROMMethod *romMethod) { - MethodKey key(definingClassRecord, index); OMR::CriticalSection cs(_methodMonitor); - auto it = _methodMap.find(key); + auto it = _methodMap.find({ definingClassRecord, index }); if (it != _methodMap.end()) return it->second; @@ -517,7 +825,7 @@ JITServerAOTCache::getMethodRecord(const AOTCacheClassRecord *definingClassRecor } auto record = AOTCacheMethodRecord::create(_nextMethodId, definingClassRecord, index); - addToMap(_methodMap, _methodHead, _methodTail, it, key, record); + addToMap(_methodMap, _methodHead, _methodTail, it, getRecordKey(record), record); ++_nextMethodId; if (TR::Options::getVerboseOption(TR_VerboseJITServer)) @@ -547,7 +855,7 @@ JITServerAOTCache::getClassChainRecord(const AOTCacheClassRecord *const *classRe } auto record = AOTCacheClassChainRecord::create(_nextClassChainId, classRecords, length); - addToMap(_classChainMap, _classChainHead, _classChainTail, it, { record->records(), length }, record); + addToMap(_classChainMap, _classChainHead, _classChainTail, it, getRecordKey(record), record); ++_nextClassChainId; if (TR::Options::getVerboseOption(TR_VerboseJITServer)) @@ -578,7 +886,7 @@ JITServerAOTCache::getWellKnownClassesRecord(const AOTCacheClassChainRecord *con } auto record = AOTCacheWellKnownClassesRecord::create(_nextWellKnownClassesId, chainRecords, length, includedClasses); - addToMap(_wellKnownClassesMap, _wellKnownClassesHead, _wellKnownClassesTail, it, { record->records(), length, includedClasses }, record); + addToMap(_wellKnownClassesMap, _wellKnownClassesHead, _wellKnownClassesTail, it, getRecordKey(record), record); ++_nextWellKnownClassesId; if (TR::Options::getVerboseOption(TR_VerboseJITServer)) @@ -612,7 +920,7 @@ JITServerAOTCache::getAOTHeaderRecord(const TR_AOTHeader *header, uint64_t clien } auto record = AOTCacheAOTHeaderRecord::create(_nextAOTHeaderId, header); - addToMap(_aotHeaderMap, _aotHeaderHead, _aotHeaderTail, it, { record->data().header() }, record); + addToMap(_aotHeaderMap, _aotHeaderHead, _aotHeaderTail, it, getRecordKey(record), record); ++_nextAOTHeaderId; if (TR::Options::getVerboseOption(TR_VerboseJITServer)) @@ -898,6 +1206,182 @@ JITServerAOTCache::writeCache(FILE *f) const return true; } +// Tests whether or not the given AOT snapshot is compatible with the server. +static bool +isCompatibleSnapshotVersion(const JITServerAOTCacheVersion &version) + { + JITServerAOTCacheVersion currentVersion = {0}; + getCurrentAOTCacheVersion(currentVersion); + + return (0 == memcmp(version._eyeCatcher, currentVersion._eyeCatcher, sizeof(currentVersion._eyeCatcher))) && + (version._snapshotVersion == currentVersion._snapshotVersion) && + (version._jitserverVersion == currentVersion._jitserverVersion); + } + +// Read an AOT cache snapshot, returning NULL if the cache is ill-formed or +// incompatible with the running server. +JITServerAOTCache * +JITServerAOTCache::readCache(FILE *f, const std::string &name, TR_Memory &trMemory) + { + if (!JITServerAOTCacheMap::cacheHasSpace()) + return NULL; + + JITServerAOTCacheHeader header = {0}; + if (1 != fread(&header, sizeof(JITServerAOTCacheHeader), 1, f)) + return NULL; + + if (!isCompatibleSnapshotVersion(header._version)) + return NULL; + + JITServerAOTCache *cache = NULL; + try + { + cache = new (TR::Compiler->persistentGlobalMemory()) JITServerAOTCache(name); + } + catch (const std::exception &e) + { + if (TR::Options::getVerboseOption(TR_VerboseJITServer)) + { + TR_VerboseLog::writeLineLocked(TR_Vlog_JITServer, "AOT cache allocation failed with exception: %s", e.what()); + } + } + + if (!cache) + return NULL; + + bool readSuccess = false; + try + { + readSuccess = cache->readCache(f, header, trMemory); + } + catch (const std::exception &e) + { + if (TR::Options::getVerboseOption(TR_VerboseJITServer)) + { + TR_VerboseLog::writeLineLocked(TR_Vlog_JITServer, "AOT cache reading failed with exception: %s", e.what()); + } + } + + if (!readSuccess) + { + cache->~JITServerAOTCache(); + TR::Compiler->persistentGlobalMemory()->freePersistentMemory(cache); + cache = NULL; + } + + return cache; + } + +// Read numRecordsToRead records of an AOTSerializationRecord subclass V from a stream, also +// updating the map, record traversal, and scratch Vector associated with V. +template bool +JITServerAOTCache::readRecords(FILE *f, + size_t numRecordsToRead, + PersistentUnorderedMap &map, + V *&traversalHead, + V *&traversalTail, + Vector &records, + const Vector &classLoaderRecords, + const Vector &classRecords, + const Vector &methodRecords, + const Vector &classChainRecords, + const Vector &wellKnownClassesRecords, + const Vector &aotHeaderRecords) + { + for (size_t i = 0; i < numRecordsToRead; ++i) + { + if (!JITServerAOTCacheMap::cacheHasSpace()) + return false; + + V *record = V::read(f, classLoaderRecords, classRecords, methodRecords, classChainRecords, wellKnownClassesRecords, aotHeaderRecords); + if (!record) + return false; + + if ((record->data().id() >= records.size() || + records[record->data().id()]) || + !addToMap(map, traversalHead, traversalTail, getRecordKey(record), record)) + { + AOTCacheRecord::free(record); + return false; + } + + records[record->data().id()] = record; + } + + return true; + } + +bool +JITServerAOTCache::readCache(FILE *f, const JITServerAOTCacheHeader &header, TR_Memory &trMemory) + { + _classLoaderMap.reserve(header._numClassLoaderRecords); + _classMap.reserve(header._numClassRecords); + _methodMap.reserve(header._numMethodRecords); + _classChainMap.reserve(header._numClassChainRecords); + _wellKnownClassesMap.reserve(header._numWellKnownClassesRecords); + _aotHeaderMap.reserve(header._numAOTHeaderRecords); + _cachedMethodMap.reserve(header._numCachedAOTMethods); + + _nextClassLoaderId = header._nextClassLoaderId; + _nextClassId = header._nextClassId; + _nextMethodId = header._nextMethodId; + _nextClassChainId = header._nextClassChainId; + _nextWellKnownClassesId = header._nextWellKnownClassesId; + _nextAOTHeaderId = header._nextAOTHeaderId; + + TR::StackMemoryRegion stackMemoryRegion(trMemory); + Vector classLoaderRecords(header._nextClassLoaderId, NULL, stackMemoryRegion); + Vector classRecords(header._nextClassId, NULL, stackMemoryRegion); + Vector methodRecords(header._nextMethodId, NULL, stackMemoryRegion); + Vector classChainRecords(header._nextClassChainId, NULL, stackMemoryRegion); + Vector wellKnownClassesRecords(header._nextWellKnownClassesId, NULL, stackMemoryRegion); + Vector aotHeaderRecords(header._nextAOTHeaderId, NULL, stackMemoryRegion); + + if (!readRecords(f, header._numClassLoaderRecords, _classLoaderMap, _classLoaderHead, _classLoaderTail, classLoaderRecords, + classLoaderRecords, classRecords, methodRecords, classChainRecords, wellKnownClassesRecords, aotHeaderRecords)) + return false; + if (!readRecords(f, header._numClassRecords, _classMap, _classHead, _classTail, classRecords, + classLoaderRecords, classRecords, methodRecords, classChainRecords, wellKnownClassesRecords, aotHeaderRecords)) + return false; + if (!readRecords(f, header._numMethodRecords, _methodMap, _methodHead, _methodTail, methodRecords, + classLoaderRecords, classRecords, methodRecords, classChainRecords, wellKnownClassesRecords, aotHeaderRecords)) + return false; + if (!readRecords(f, header._numClassChainRecords, _classChainMap, _classChainHead, _classChainTail, classChainRecords, + classLoaderRecords, classRecords, methodRecords, classChainRecords, wellKnownClassesRecords, aotHeaderRecords)) + return false; + if (!readRecords(f, header._numWellKnownClassesRecords, _wellKnownClassesMap, _wellKnownClassesHead, _wellKnownClassesTail, wellKnownClassesRecords, + classLoaderRecords, classRecords, methodRecords, classChainRecords, wellKnownClassesRecords, aotHeaderRecords)) + return false; + if (!readRecords(f, header._numAOTHeaderRecords, _aotHeaderMap, _aotHeaderHead, _aotHeaderTail, aotHeaderRecords, + classLoaderRecords, classRecords, methodRecords, classChainRecords, wellKnownClassesRecords, aotHeaderRecords)) + return false; + + for (size_t i = 0; i < header._numCachedAOTMethods; ++i) + { + if (!JITServerAOTCacheMap::cacheHasSpace()) + return false; + + auto record = CachedAOTMethod::read(f, classLoaderRecords, classRecords, methodRecords, + classChainRecords, wellKnownClassesRecords, aotHeaderRecords); + + if (!record || !aotHeaderRecords[record->data().aotHeaderId()]) + return false; + + CachedMethodKey key(record->definingClassChainRecord(), + record->data().index(), + record->data().optLevel(), + aotHeaderRecords[record->data().aotHeaderId()]); + + if (!addToMap(_cachedMethodMap, _cachedMethodHead, _cachedMethodTail, key, record)) + { + AOTCacheRecord::free(record); + return false; + } + } + + return true; + } + bool JITServerAOTCacheMap::cacheHasSpace() { diff --git a/runtime/compiler/runtime/JITServerAOTCache.hpp b/runtime/compiler/runtime/JITServerAOTCache.hpp index 144e86636e7..717a34d0e02 100644 --- a/runtime/compiler/runtime/JITServerAOTCache.hpp +++ b/runtime/compiler/runtime/JITServerAOTCache.hpp @@ -65,6 +65,13 @@ struct JITServerAOTCacheHeader size_t _nextAOTHeaderId; }; +struct AOTCacheClassLoaderRecord; +struct AOTCacheClassRecord; +struct AOTCacheMethodRecord; +struct AOTCacheClassChainRecord; +struct AOTCacheWellKnownClassesRecord; +struct AOTCacheAOTHeaderRecord; + // Base class for serialization record "wrappers" stored at the server. // // When a cached serialized method is sent to a client, the server needs @@ -119,8 +126,17 @@ class AOTCacheClassLoaderRecord final : public AOTCacheRecord static AOTCacheClassLoaderRecord *create(uintptr_t id, const uint8_t *name, size_t nameLength); + static AOTCacheClassLoaderRecord *read(FILE *f, + const Vector &classLoaderRecords, + const Vector &classRecords, + const Vector &methodRecords, + const Vector &classChainRecords, + const Vector &wellKnownClassesRecords, + const Vector &aotHeaderRecords); + private: AOTCacheClassLoaderRecord(uintptr_t id, const uint8_t *name, size_t nameLength); + AOTCacheClassLoaderRecord() {} static size_t size(size_t nameLength) { @@ -142,9 +158,18 @@ class AOTCacheClassRecord final : public AOTCacheRecord const JITServerROMClassHash &hash, const J9ROMClass *romClass); void subRecordsDo(const std::function &f) const override; + static AOTCacheClassRecord *read(FILE *f, + const Vector &classLoaderRecords, + const Vector &classRecords, + const Vector &methodRecords, + const Vector &classChainRecords, + const Vector &wellKnownClassesRecords, + const Vector &aotHeaderRecords); + private: AOTCacheClassRecord(uintptr_t id, const AOTCacheClassLoaderRecord *classLoaderRecord, const JITServerROMClassHash &hash, const J9ROMClass *romClass); + AOTCacheClassRecord(const AOTCacheClassLoaderRecord *classLoaderRecord); static size_t size(size_t nameLength) { @@ -166,8 +191,17 @@ class AOTCacheMethodRecord final : public AOTCacheRecord static AOTCacheMethodRecord *create(uintptr_t id, const AOTCacheClassRecord *definingClassRecord, uint32_t index); void subRecordsDo(const std::function &f) const override; + static AOTCacheMethodRecord *read(FILE *f, + const Vector &classLoaderRecords, + const Vector &classRecords, + const Vector &methodRecords, + const Vector &classChainRecords, + const Vector &wellKnownClassesRecords, + const Vector &aotHeaderRecords); + private: AOTCacheMethodRecord(uintptr_t id, const AOTCacheClassRecord *definingClassRecord, uint32_t index); + AOTCacheMethodRecord(const AOTCacheClassRecord *definingClassRecord); const AOTCacheClassRecord *const _definingClassRecord; const MethodSerializationRecord _data; @@ -185,11 +219,15 @@ class AOTCacheListRecord : public AOTCacheRecord const AOTSerializationRecord *dataAddr() const override { return &_data; } // Array of record pointers is stored inline after the array of IDs that is stored inline after struct D header const R *const *records() const { return (const R *const *)_data.end(); } + R **records() { return (R **)_data.end(); } void subRecordsDo(const std::function &f) const override; + static AOTCacheListRecord *read(FILE *f, const Vector &cacheRecords); + protected: AOTCacheListRecord(uintptr_t id, const R *const *records, size_t length, Args... args); + AOTCacheListRecord() {} static size_t size(size_t length) { @@ -209,6 +247,15 @@ class AOTCacheClassChainRecord final : public AOTCacheListRecordclassLoaderRecord(); } + static AOTCacheClassChainRecord *read(FILE *f, + const Vector &classLoaderRecords, + const Vector &classRecords, + const Vector &methodRecords, + const Vector &classChainRecords, + const Vector &wellKnownClassesRecords, + const Vector &aotHeaderRecords) + { return (AOTCacheClassChainRecord *)AOTCacheListRecord::read(f, classRecords); } + private: using AOTCacheListRecord::AOTCacheListRecord; }; @@ -221,6 +268,16 @@ class AOTCacheWellKnownClassesRecord final : static AOTCacheWellKnownClassesRecord *create(uintptr_t id, const AOTCacheClassChainRecord *const *records, size_t length, uintptr_t includedClasses); + static AOTCacheWellKnownClassesRecord *read(FILE *f, + const Vector &classLoaderRecords, + const Vector &classRecords, + const Vector &methodRecords, + const Vector &classChainRecords, + const Vector &wellKnownClassesRecords, + const Vector &aotHeaderRecords) + { return (AOTCacheWellKnownClassesRecord *) + AOTCacheListRecord::read(f, classChainRecords); } + private: using AOTCacheListRecord::AOTCacheListRecord; }; @@ -234,8 +291,17 @@ class AOTCacheAOTHeaderRecord final : public AOTCacheRecord static AOTCacheAOTHeaderRecord *create(uintptr_t id, const TR_AOTHeader *header); + static AOTCacheAOTHeaderRecord *read(FILE *f, + const Vector &classLoaderRecords, + const Vector &classRecords, + const Vector &methodRecords, + const Vector &classChainRecords, + const Vector &wellKnownClassesRecords, + const Vector &aotHeaderRecords); + private: AOTCacheAOTHeaderRecord(uintptr_t id, const TR_AOTHeader *header); + AOTCacheAOTHeaderRecord() {} const AOTHeaderSerializationRecord _data; }; @@ -255,12 +321,23 @@ class CachedAOTMethod const SerializedAOTMethod &data() const { return _data; } SerializedAOTMethod &data() { return _data; } const AOTCacheRecord *const *records() const { return (const AOTCacheRecord *const *)_data.end(); } + AOTCacheRecord **records() { return (AOTCacheRecord **)_data.end(); } static CachedAOTMethod *create(const AOTCacheClassChainRecord *definingClassChainRecord, uint32_t index, TR_Hotness optLevel, const AOTCacheAOTHeaderRecord *aotHeaderRecord, const Vector> &records, const void *code, size_t codeSize, const void *data, size_t dataSize); + static CachedAOTMethod *read(FILE *f, + const Vector &classLoaderRecords, + const Vector &classRecords, + const Vector &methodRecords, + const Vector &classChainRecords, + const Vector &wellKnownClassesRecords, + const Vector &aotHeaderRecords); + + static CachedAOTMethod *read(FILE *f, const Vector &classChainRecords); + CachedAOTMethod *getNextRecord() const { return _nextRecord; } void setNextRecord(CachedAOTMethod *record) { _nextRecord = record; } @@ -269,6 +346,7 @@ class CachedAOTMethod TR_Hotness optLevel, const AOTCacheAOTHeaderRecord *aotHeaderRecord, const Vector> &records, const void *code, size_t codeSize, const void *data, size_t dataSize); + CachedAOTMethod(const AOTCacheClassChainRecord *definingClassChainRecord); static size_t size(size_t numRecords, size_t codeSize, size_t dataSize) { @@ -345,7 +423,7 @@ class JITServerAOTCache void printStats(FILE *f) const; bool writeCache(FILE *f) const; - static JITServerAOTCache *readCache(FILE *f, const std::string &name); + static JITServerAOTCache *readCache(FILE *f, const std::string &name, TR_Memory &trMemory); private: struct ClassLoaderKey @@ -357,6 +435,9 @@ class JITServerAOTCache const size_t _nameLength; }; + static ClassLoaderKey getRecordKey(const AOTCacheClassLoaderRecord *record) + { return { record->data().name(), record->data().nameLength() }; } + struct ClassKey { bool operator==(const ClassKey &k) const; @@ -366,8 +447,14 @@ class JITServerAOTCache const JITServerROMClassHash *const _hash; }; + static ClassKey getRecordKey(const AOTCacheClassRecord *record) + { return { record->classLoaderRecord(), &record->data().hash() }; } + using MethodKey = std::pair; + static MethodKey getRecordKey(const AOTCacheMethodRecord *record) + { return { record->definingClassRecord(), record->data().index() }; } + struct ClassChainKey { bool operator==(const ClassChainKey &k) const; @@ -377,6 +464,9 @@ class JITServerAOTCache const size_t _length; }; + static ClassChainKey getRecordKey(const AOTCacheClassChainRecord *record) + { return { record->records(), record->data().list().length() }; } + struct WellKnownClassesKey { bool operator==(const WellKnownClassesKey &k) const; @@ -387,6 +477,9 @@ class JITServerAOTCache const uintptr_t _includedClasses; }; + static WellKnownClassesKey getRecordKey(const AOTCacheWellKnownClassesRecord *record) + { return { record->records(), record->data().list().length() }; } + struct AOTHeaderKey { bool operator==(const AOTHeaderKey &k) const; @@ -395,6 +488,9 @@ class JITServerAOTCache const TR_AOTHeader *const _header; }; + static AOTHeaderKey getRecordKey(const AOTCacheAOTHeaderRecord *record) + { return { record->data().header() }; } + //NOTE: Current implementation doesn't support compatible differences in AOT headers. // A cached method can only be sent to a client with the exact same AOT header. using CachedMethodKey = std::tuple &result, UnorderedSet &newRecords, const KnownIdSet &knownIds) const; + // Read a cache snapshot into an empty cache + bool readCache(FILE *f, const JITServerAOTCacheHeader &header, TR_Memory &trMemory); + + template + static bool readRecords(FILE *f, size_t numRecordsToRead, PersistentUnorderedMap &map, V *&traversalHead, V *&traversalTail, + Vector &records, + const Vector &classLoaderRecords, + const Vector &classRecords, + const Vector &methodRecords, + const Vector &classChainRecords, + const Vector &wellKnownClassesRecords, + const Vector &aotHeaderRecords); const std::string _name; diff --git a/runtime/compiler/runtime/JITServerAOTSerializationRecords.hpp b/runtime/compiler/runtime/JITServerAOTSerializationRecords.hpp index 343f7dd0f96..3afcfd244f5 100644 --- a/runtime/compiler/runtime/JITServerAOTSerializationRecords.hpp +++ b/runtime/compiler/runtime/JITServerAOTSerializationRecords.hpp @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright (c) 2021, 2021 IBM Corp. and others + * Copyright (c) 2021, 2022 IBM Corp. and others * * This program and the accompanying materials are made available under * the terms of the Eclipse Public License 2.0 which accompanies this @@ -64,6 +64,7 @@ struct AOTSerializationRecord //NOTE: 0 signifies an invalid record ID uintptr_t id() const { return getId(_idAndType); } AOTSerializationRecordType type() const { return getType(_idAndType); } + bool isValid(AOTSerializationRecordType type) const; const uint8_t *end() const { return (const uint8_t *)this + size(); } static const AOTSerializationRecord *get(const std::string &str) @@ -110,11 +111,13 @@ struct ClassLoaderSerializationRecord : public AOTSerializationRecord public: size_t nameLength() const { return _nameLength; } const uint8_t *name() const { return _name; } + bool isValid() const { return AOTSerializationRecord::isValid(AOTSerializationRecordType::ClassLoader); } private: friend class AOTCacheClassLoaderRecord; ClassLoaderSerializationRecord(uintptr_t id, const uint8_t *name, size_t nameLength); + ClassLoaderSerializationRecord(); static size_t size(size_t nameLength) { @@ -135,12 +138,14 @@ struct ClassSerializationRecord : public AOTSerializationRecord uint32_t romClassSize() const { return _romClassSize; } size_t nameLength() const { return _nameLength; } const uint8_t *name() const { return _name; } + bool isValid() const { return AOTSerializationRecord::isValid(AOTSerializationRecordType::Class); } private: friend class AOTCacheClassRecord; ClassSerializationRecord(uintptr_t id, uintptr_t classLoaderId, const JITServerROMClassHash &hash, const J9ROMClass *romClass); + ClassSerializationRecord(); static size_t size(size_t nameLength) { @@ -162,11 +167,13 @@ struct MethodSerializationRecord : public AOTSerializationRecord public: uintptr_t definingClassId() const { return _definingClassId; } uint32_t index() const { return _index; } + bool isValid() const { return AOTSerializationRecord::isValid(AOTSerializationRecordType::Method); } private: friend class AOTCacheMethodRecord; MethodSerializationRecord(uintptr_t id, uintptr_t definingClassId, uint32_t index); + MethodSerializationRecord(); const uintptr_t _definingClassId; // Index in the array of methods of the defining class @@ -195,11 +202,13 @@ struct ClassChainSerializationRecord : public AOTSerializationRecord { public: const IdList &list() const { return _list; } + bool isValid() const { return AOTSerializationRecord::isValid(AOTSerializationRecordType::ClassChain); } private: template friend class AOTCacheListRecord; ClassChainSerializationRecord(uintptr_t id, size_t length); + ClassChainSerializationRecord(); IdList &list() { return _list; } @@ -218,11 +227,13 @@ struct WellKnownClassesSerializationRecord : public AOTSerializationRecord public: uintptr_t includedClasses() const { return _includedClasses; } const IdList &list() const { return _list; } + bool isValid() const { return AOTSerializationRecord::isValid(AOTSerializationRecordType::WellKnownClasses); } private: template friend class AOTCacheListRecord; WellKnownClassesSerializationRecord(uintptr_t id, size_t length, uintptr_t includedClasses); + WellKnownClassesSerializationRecord(); IdList &list() { return _list; } @@ -242,11 +253,13 @@ struct AOTHeaderSerializationRecord : public AOTSerializationRecord { public: const TR_AOTHeader *header() const { return &_header; } + bool isValid() const { return AOTSerializationRecord::isValid(AOTSerializationRecordType::AOTHeader); } private: friend class AOTCacheAOTHeaderRecord; AOTHeaderSerializationRecord(uintptr_t id, const TR_AOTHeader *header); + AOTHeaderSerializationRecord(); const TR_AOTHeader _header; }; @@ -294,6 +307,7 @@ struct SerializedAOTMethod const uint8_t *data() const { return code() + _codeSize; } uint8_t *data() { return (uint8_t *)(code() + _codeSize); } const uint8_t *end() const { return (const uint8_t *)this + size(); } + bool isValid() const; static SerializedAOTMethod *get(std::string &str) { @@ -308,6 +322,7 @@ struct SerializedAOTMethod SerializedAOTMethod(uintptr_t definingClassChainId, uint32_t index, TR_Hotness optLevel, uintptr_t aotHeaderId, size_t numRecords, const void *code, size_t codeSize, const void *data, size_t dataSize); + SerializedAOTMethod(); static size_t size(size_t numRecords, size_t codeSize, size_t dataSize) {