Skip to content

Commit

Permalink
Support mutable locations (#295)
Browse files Browse the repository at this point in the history
[jacodb-core] Minor: reduce number of downloads

Two versions of the Keap lib are downloaded twice, not three times.

[jacodb-core] Make code model with ERS backend mutable incremental

As of this commit, ErsPersistenceImpl has got the following additional features:
- For any location and class name, there can exist several entities.
- Amongst those entities, only one is active (current), other are logically
deleted, i.e. marked with "isDeleted" property.
- Any location identified by its id can be forced to be re-indexed. Only changed
classes are getting the new version during re-indexing.
- For binding Persistence layer with the layer of JcDatabase, a new ClassSource
implementation introduced. For getting class' byte code, this implementation
checks if class id which it holds is up-to-date.
- Code model re-indexes changed locations after JcDatabase.refresh() is invoked.

TODO. PersistentLocationsRegistry looks overcomplicated. JcLocation implementations
look as hacks in order to get JcLocation.isChanged() working, and in order to
reindex mutated locations in-place without adding/removing them. So this part
looks as a good candidate for complete rewriting.

[jacodb-core] Ensure that jar file is closed

Two resource leaks fixed: a jar file wasn't closed after
- its hash was computed;
- its classes were indexed.
  • Loading branch information
Saloed authored Jan 29, 2025

Unverified

The committer email address is not verified.
1 parent ddbea5d commit ea3c7c0
Showing 16 changed files with 412 additions and 115 deletions.
3 changes: 3 additions & 0 deletions jacodb-benchmarks/build.gradle.kts
Original file line number Diff line number Diff line change
@@ -85,6 +85,9 @@ benchmark {
include("RAMEntityRelationshipStorageMutableBenchmarks")
include("RAMEntityRelationshipStorageImmutableBenchmarks")
}
register("hash") {
include("HashBenchmarks")
}
}
}

Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
/*
* Copyright 2022 UnitTestBot contributors (utbot.org)
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

@file:Suppress("UnstableApiUsage", "DEPRECATION")

package org.jacodb.testing.performance.hash

import com.google.common.hash.Hasher
import com.google.common.hash.Hashing
import kotlinx.benchmark.Benchmark
import org.openjdk.jmh.annotations.BenchmarkMode
import org.openjdk.jmh.annotations.Measurement
import org.openjdk.jmh.annotations.Mode
import org.openjdk.jmh.annotations.OutputTimeUnit
import org.openjdk.jmh.annotations.Scope
import org.openjdk.jmh.annotations.State
import org.openjdk.jmh.annotations.Warmup
import java.security.SecureRandom
import java.util.concurrent.TimeUnit

@State(Scope.Benchmark)
@Warmup(iterations = 2, time = 1)
@BenchmarkMode(Mode.AverageTime)
@Measurement(iterations = 5, time = 1)
class HashBenchmarks {

companion object {
val array = ByteArray(1_000_000).also { SecureRandom().nextBytes(it) }
}

@Benchmark
@OutputTimeUnit(TimeUnit.NANOSECONDS)
fun createSip24Hasher(): Hasher {
return Hashing.sipHash24().newHasher()
}

@Benchmark
@OutputTimeUnit(TimeUnit.NANOSECONDS)
fun createMurMur3Hasher(): Hasher {
return Hashing.murmur3_128().newHasher()
}

@Benchmark
@OutputTimeUnit(TimeUnit.NANOSECONDS)
fun createMd5Hasher(): Hasher {
return Hashing.md5().newHasher()
}

@Benchmark
@OutputTimeUnit(TimeUnit.NANOSECONDS)
fun createSha256Hasher(): Hasher {
return Hashing.sha256().newHasher()
}

@Benchmark
@OutputTimeUnit(TimeUnit.MICROSECONDS)
fun sip24Hash(): Long {
return Hashing.sipHash24().newHasher().putBytes(array).hash().asLong()
}

@Benchmark
@OutputTimeUnit(TimeUnit.MICROSECONDS)
fun murMur3Hash(): Long {
return Hashing.murmur3_128().newHasher().putBytes(array).hash().asLong()
}

@Benchmark
@OutputTimeUnit(TimeUnit.MICROSECONDS)
fun sha256Hash(): Long {
return Hashing.sha256().newHasher().putBytes(array).hash().asLong()
}

@Benchmark
@OutputTimeUnit(TimeUnit.MICROSECONDS)
fun md5Hash(): Long {
return Hashing.md5().newHasher().putBytes(array).hash().asLong()
}
}
35 changes: 17 additions & 18 deletions jacodb-core/src/main/kotlin/org/jacodb/impl/features/Builders.kt
Original file line number Diff line number Diff line change
@@ -30,6 +30,9 @@ import org.jacodb.api.storage.ers.compressed
import org.jacodb.api.storage.ers.nonSearchable
import org.jacodb.impl.fs.PersistenceClassSource
import org.jacodb.impl.fs.className
import org.jacodb.impl.storage.ers.filterDeleted
import org.jacodb.impl.storage.ers.filterLocations
import org.jacodb.impl.storage.ers.toClassSource
import org.jacodb.impl.storage.execute
import org.jacodb.impl.storage.executeQueries
import org.jacodb.impl.storage.jooq.tables.references.BUILDERS
@@ -288,25 +291,21 @@ object Builders : JcFeature<Set<String>, BuildersResponse> {
val builderClassNameId: Long =
builder.getCompressedBlob<Long>(BuilderEntity.BUILDER_CLASS_NAME_ID)!!
txn.find("Class", "nameId", builderClassNameId.compressed)
.mapNotNull { builderClass ->
if (builderClass.getCompressed<Long>("locationId") != builderLocationId) {
null
} else {
BuildersResponse(
source = PersistenceClassSource(
db = classpath.db,
locationId = builderLocationId,
classId = builderClass.id.instanceId,
className = persistence.findSymbolName(builderClassNameId),
),
methodOffset = builder.getCompressedBlob<Int>(BuilderEntity.METHOD_OFFSET_PROPERTY)!!,
priority = builder.getCompressedBlob<Int>(BuilderEntity.PRIORITY_PROPERTY)
?: 0
)
}
.filterLocations(builderLocationId)
.filterDeleted()
.map { builderClass ->
BuildersResponse(
source = builderClass.toClassSource(
persistence = persistence,
className = persistence.findSymbolName(builderClassNameId),
nameId = builderClassNameId
),
methodOffset = builder.getCompressedBlob<Int>(BuilderEntity.METHOD_OFFSET_PROPERTY)!!,
priority = builder.getCompressedBlob<Int>(BuilderEntity.PRIORITY_PROPERTY)
?: 0
)
}
}
.toList()
}.toList()
}
)
}.sortedByDescending { it.priority }
Original file line number Diff line number Diff line change
@@ -21,6 +21,7 @@ package org.jacodb.impl.features
import kotlinx.coroutines.GlobalScope
import kotlinx.coroutines.future.future
import org.jacodb.api.jvm.JcClasspath
import org.jacodb.impl.storage.ers.filterDeleted
import org.jacodb.impl.storage.execute
import org.jacodb.impl.storage.jooq.tables.references.CLASSES
import org.jacodb.impl.storage.jooq.tables.references.SYMBOLS
@@ -48,7 +49,7 @@ suspend fun JcClasspath.duplicatedClasses(): Map<String, Int> {
},
noSqlAction = { txn ->
val result = mutableMapOf<String, Int>().also { result ->
txn.all("Class").forEach { clazz ->
txn.all("Class").filterDeleted().forEach { clazz ->
val className = persistence.findSymbolName(clazz.getCompressed<Long>("nameId")!!)
result[className] = result.getOrDefault(className, 0) + 1
}
Original file line number Diff line number Diff line change
@@ -38,6 +38,8 @@ import org.jacodb.impl.fs.PersistenceClassSource
import org.jacodb.impl.storage.BatchedSequence
import org.jacodb.impl.storage.defaultBatchSize
import org.jacodb.impl.storage.dslContext
import org.jacodb.impl.storage.ers.filterDeleted
import org.jacodb.impl.storage.ers.filterLocations
import org.jacodb.impl.storage.ers.toClassSourceSequence
import org.jacodb.impl.storage.execute
import org.jacodb.impl.storage.isSqlContext
@@ -95,12 +97,14 @@ internal fun JcClasspath.allClassesExceptObject(context: StorageContext, direct:
}
},
noSqlAction = { txn ->
val objectNameId = db.persistence.findSymbolId(JAVA_OBJECT)
txn.all("Class").filter { clazz ->
(!direct || clazz.getCompressed<Long>("inherits") == null) &&
clazz.getCompressed<Long>("locationId") in locationIds &&
clazz.getCompressed<Long>("nameId") != objectNameId
}.toClassSourceSequence(db).toList().asSequence()
val objectNameId by lazy(LazyThreadSafetyMode.NONE) { db.persistence.findSymbolId(JAVA_OBJECT) }
txn.all("Class")
.filterLocations(locationIds)
.filterDeleted()
.filter { clazz ->
(!direct || clazz.getCompressed<Long>("inherits") == null) &&
clazz.getCompressed<Long>("nameId") != objectNameId
}.toClassSourceSequence(db).toList().asSequence()
}
)
}
Original file line number Diff line number Diff line change
@@ -34,6 +34,9 @@ import org.jacodb.impl.fs.PersistenceClassSource
import org.jacodb.impl.fs.className
import org.jacodb.impl.storage.BatchedSequence
import org.jacodb.impl.storage.defaultBatchSize
import org.jacodb.impl.storage.ers.filterDeleted
import org.jacodb.impl.storage.ers.filterLocations
import org.jacodb.impl.storage.ers.toClassSource
import org.jacodb.impl.storage.execute
import org.jacodb.impl.storage.jooq.tables.references.CLASSES
import org.jacodb.impl.storage.jooq.tables.references.CLASSHIERARCHIES
@@ -112,7 +115,7 @@ object InMemoryHierarchy : JcFeature<InMemoryHierarchyReq, ClassSource> {
}
},
noSqlAction = { txn ->
txn.all("Class").forEach { clazz ->
txn.all("Class").filterDeleted().forEach { clazz ->
val locationId: Long? = clazz.getCompressed("locationId")
val classSymbolId: Long? = clazz.getCompressed("nameId")
val superClasses = mutableListOf<Long>()
@@ -234,17 +237,16 @@ object InMemoryHierarchy : JcFeature<InMemoryHierarchyReq, ClassSource> {
allSubclasses.asSequence()
.flatMap { classNameId ->
txn.find("Class", "nameId", classNameId.compressed)
.filter { clazz ->
clazz.getCompressed<Long>("locationId") in locationIds
}
.filterLocations(locationIds)
.filterDeleted()
}
.map { clazz ->
val nameId = clazz.getCompressed<Long>("nameId")!!
val classId: Long = clazz.id.instanceId
PersistenceClassSource(
db = classpath.db,
className = persistence.findSymbolName(clazz.getCompressed<Long>("nameId")!!),
classId = classId,
locationId = clazz.getCompressed<Long>("locationId")!!,
clazz.toClassSource(
persistence = persistence,
className = persistence.findSymbolName(nameId),
nameId = nameId,
cachedByteCode = if (req.full) persistence.findBytecode(classId) else null
)
}
14 changes: 9 additions & 5 deletions jacodb-core/src/main/kotlin/org/jacodb/impl/features/Usages.kt
Original file line number Diff line number Diff line change
@@ -33,6 +33,8 @@ import org.jacodb.impl.storage.BatchedSequence
import org.jacodb.impl.storage.defaultBatchSize
import org.jacodb.impl.storage.dslContext
import org.jacodb.impl.storage.eqOrNull
import org.jacodb.impl.storage.ers.filterDeleted
import org.jacodb.impl.storage.ers.toClassSource
import org.jacodb.impl.storage.execute
import org.jacodb.impl.storage.executeQueries
import org.jacodb.impl.storage.isSqlContext
@@ -275,13 +277,15 @@ object Usages : JcFeature<UsageFeatureRequest, UsageFeatureResponse> {
.map { (call, locationId) ->
val callerId = call.getCompressedBlob<Long>("callerId")!!
val caller = symbolInterner.findSymbolName(callerId)!!
val classId = txn.find("Class", "nameId", callerId.compressed).first().id.instanceId
val clazz = txn.find("Class", "nameId", callerId.compressed)
.filterDeleted()
.first()
val classId = clazz.id.instanceId
UsageFeatureResponse(
source = PersistenceClassSource(
db = classpath.db,
source = clazz.toClassSource(
persistence = persistence,
className = caller,
classId = classId,
locationId = locationId,
nameId = callerId,
cachedByteCode = persistence.findBytecode(classId)
),
offsets = call.getRawBlob("offsets")!!.toShortArray()
Original file line number Diff line number Diff line change
@@ -62,7 +62,7 @@ private fun File.classPath(classpath: MutableCollection<File>) {
}
}

private fun File.isJar() = isFile && name.endsWith(".jar") || name.endsWith(".jmod")
fun File.isJar() = isFile && name.endsWith(".jar") || name.endsWith(".jmod")

private const val file = "file:"

Original file line number Diff line number Diff line change
@@ -39,13 +39,15 @@ open class JarLocation(
get() {
val jarFile = jarFile() ?: return BigInteger.ZERO
return Hashing.sha256().newHasher().let { h ->
jarFile.entries().asSequence().filter { !it.isDirectory }.sortedBy { it.name }.forEach { entry ->
h.putString(entry.name, UTF_8)
h.putLong(entry.crc)
h.putLong(entry.size)
h.putLong(entry.compressedSize)
jarFile.use {
it.entries().asSequence().filter { !it.isDirectory }.sortedBy { it.name }.forEach { entry ->
h.putString(entry.name, UTF_8)
h.putLong(entry.crc)
h.putLong(entry.size)
h.putLong(entry.compressedSize)
}
BigInteger(h.hash().asBytes())
}
BigInteger(h.hash().asBytes())
}
}

2 changes: 1 addition & 1 deletion jacodb-core/src/main/kotlin/org/jacodb/impl/fs/Jars.kt
Original file line number Diff line number Diff line change
@@ -85,7 +85,7 @@ class JarFacade(private val runtimeVersion: Int, private val getter: () -> JarFi
val jarFile = getter() ?: return emptyMap()
return jarFile.use {
val buffer = ByteArray(DEFAULT_BUFFER_SIZE * 8)
classes.map { it.key to jarFile.getInputStream(it.value).readBytes(buffer) }
classes.map { it.key to jarFile.getInputStream(it.value).use { it.readBytes(buffer) } }
}.toMap()
}

Original file line number Diff line number Diff line change
@@ -23,10 +23,13 @@ import org.jacodb.api.jvm.JcDatabasePersistence
import org.jacodb.api.jvm.RegisteredLocation
import org.jacodb.api.storage.ers.Entity
import org.jacodb.api.storage.ers.getEntityOrNull
import org.jacodb.impl.fs.asByteCodeLocation
import org.jacodb.impl.fs.BuildFolderLocation
import org.jacodb.impl.fs.JarLocation
import org.jacodb.impl.fs.isJar
import org.jacodb.impl.storage.jooq.tables.records.BytecodelocationsRecord
import org.jacodb.impl.storage.jooq.tables.references.BYTECODELOCATIONS
import java.io.File
import java.math.BigInteger

data class PersistentByteCodeLocationData(
val id: Long,
@@ -67,14 +70,6 @@ class PersistentByteCodeLocation(
location
)

constructor(db: JcDatabase, locationId: Long) : this(
db.persistence,
db.runtimeVersion,
locationId,
null,
null
)

val data by lazy {
cachedData ?: persistence.read { context ->
context.execute(
@@ -116,14 +111,28 @@ class PersistentByteCodeLocation(
}

private fun PersistentByteCodeLocationData.toJcLocation(): JcByteCodeLocation? {
try {
val newOne = File(path).asByteCodeLocation(runtimeVersion, isRuntime = runtime).singleOrNull()
if (newOne?.fileSystemId != fileSystemId) {
return null
return try {
with(File(path)) {
if (!exists()) {
null
} else if (isJar()) {
// NB! This JarLocation inheritor is necessary for hacking PersistentLocationsRegistry
// so that isChanged() would work properly in PersistentLocationsRegistry.refresh()
val fsId = fileSystemId
object : JarLocation(this@with, isRuntime, runtimeVersion) {
override val fileSystemIdHash: BigInteger
get() {
return BigInteger(fsId, Character.MAX_RADIX)
}
}
} else if (isDirectory) {
BuildFolderLocation(this)
} else {
error("$absolutePath is nether a jar file nor a build directory")
}
}
return newOne
} catch (e: Exception) {
return null
null
}
}
}
Loading

0 comments on commit ea3c7c0

Please sign in to comment.