Skip to content

Commit

Permalink
Update memory handling and pointer size calculations
Browse files Browse the repository at this point in the history
Replaced string-based memory allocation with UTF-8 encoded byte arrays for better compatibility and precision. Adjusted key and value length calculations to use the size of pointers instead of string lengths, ensuring correctness when working with native libraries. These changes improve robustness and prevent potential errors in memory operations.
  • Loading branch information
lamba92 committed Dec 29, 2024
1 parent 91b9a96 commit 2a4785a
Show file tree
Hide file tree
Showing 7 changed files with 151 additions and 30 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,5 @@ actual val DATABASE_PATH: String
.filesDir
.resolve("testdb")
.absolutePath

actual typealias Test = org.junit.Test
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,26 @@ import com.github.lamba92.leveldb.jvm.JvmLevelDB
import com.github.lamba92.leveldb.jvm.LibLevelDB
import com.github.lamba92.leveldb.jvm.toNative
import com.sun.jna.ptr.PointerByReference
import java.nio.charset.Charset

public actual fun LevelDB(
path: String,
options: LevelDBOptions,
): LevelDB = LevelDB(path, options, Charsets.UTF_8)

/**
* Creates a new instance of a LevelDB database at the specified file path.
*
* @param path The file system path where the LevelDB database is located or will be created. It has to a directory.
* If the directory does not exist, it will be created only if [LevelDBOptions.createIfMissing] is set to `true`.
* @param options Configuration options for creating and managing the LevelDB instance, defaults to [LevelDBOptions.DEFAULT].
* @param charset The character set to use for encoding and decoding strings. Defaults to [Charsets.UTF_8].
* @return A `LevelDB` instance for interacting with the database.
*/
public fun LevelDB(
path: String,
options: LevelDBOptions,
charset: Charset
): LevelDB {
val nativeOptions = options.toNative()
val errPtr = PointerByReference()
Expand All @@ -17,7 +33,7 @@ public actual fun LevelDB(
LibLevelDB.leveldb_free(errPtr.value)
error("Failed to open database: $errorValue")
}
return JvmLevelDB(nativeDelegate, nativeOptions)
return JvmLevelDB(nativeDelegate, nativeOptions, charset)
}

public actual fun repairDatabase(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,12 @@ import com.github.lamba92.leveldb.LevelDBBatchOperation
import com.github.lamba92.leveldb.LevelDBReader
import com.github.lamba92.leveldb.LevelDBSnapshot
import com.sun.jna.ptr.PointerByReference
import java.nio.charset.Charset

public class JvmLevelDB internal constructor(
private val nativeDatabase: LibLevelDB.leveldb_t,
private val nativeOptions: LibLevelDB.leveldb_options_t,
private val charset: Charset,
) : LevelDB {
override fun put(
key: String,
Expand All @@ -21,15 +23,15 @@ public class JvmLevelDB internal constructor(
val errPtr = PointerByReference()
val writeOptions = leveldb_writeoptions_create()
leveldb_writeoptions_set_sync(writeOptions, sync.toByte())
val keyPointer = key.toPointer()
val valuePointer = value.toPointer()
val keyPointer = key.toPointer(charset)
val valuePointer = value.toPointer(charset)
leveldb_put(
db = nativeDatabase,
options = writeOptions,
key = keyPointer,
keylen = key.length.toNativeLong(),
keylen = keyPointer.size().toNativeLong(),
value = valuePointer,
vallen = value.length.toNativeLong(),
vallen = valuePointer.size().toNativeLong(),
errptr = errPtr,
)
valuePointer.close()
Expand All @@ -46,7 +48,7 @@ public class JvmLevelDB internal constructor(
key: String,
verifyChecksums: Boolean,
fillCache: Boolean,
): String? = nativeDatabase.get(verifyChecksums, fillCache, key)
): String? = nativeDatabase.get(verifyChecksums, fillCache, key, charset = charset)

override fun delete(
key: String,
Expand All @@ -56,12 +58,12 @@ public class JvmLevelDB internal constructor(
val errPtr = PointerByReference()
val writeOptions = leveldb_writeoptions_create()
leveldb_writeoptions_set_sync(writeOptions, sync.toByte())
val keyPointer = key.toPointer()
val keyPointer = key.toPointer(charset)
leveldb_delete(
db = nativeDatabase,
options = writeOptions,
key = keyPointer,
keylen = key.length.toNativeLong(),
keylen = keyPointer.size().toNativeLong(),
errptr = errPtr,
)
keyPointer.clear(key.length.toLong())
Expand All @@ -83,25 +85,25 @@ public class JvmLevelDB internal constructor(
for (operation in operations) {
when (operation) {
is LevelDBBatchOperation.Put -> {
val keyPointer = operation.key.toPointer()
val valuePointer = operation.value.toPointer()
val keyPointer = operation.key.toPointer(charset)
val valuePointer = operation.value.toPointer(charset)
leveldb_writebatch_put(
batch = nativeBatch,
key = keyPointer,
klen = operation.key.length.toNativeLong(),
klen = keyPointer.size().toNativeLong(),
value = valuePointer,
vlen = operation.value.length.toNativeLong(),
vlen = valuePointer.size().toNativeLong(),
)
keyPointer.close()
valuePointer.close()
}

is LevelDBBatchOperation.Delete -> {
val keyPointer = operation.key.toPointer()
val keyPointer = operation.key.toPointer(charset)
leveldb_writebatch_delete(
batch = nativeBatch,
key = keyPointer,
klen = operation.key.length.toNativeLong(),
klen = keyPointer.size().toNativeLong(),
)
keyPointer.close()
}
Expand All @@ -128,13 +130,18 @@ public class JvmLevelDB internal constructor(
from: String?,
verifyChecksums: Boolean,
fillCache: Boolean,
): CloseableSequence<LevelDBReader.LazyEntry> = nativeDatabase.asSequence(verifyChecksums, fillCache, from)
): CloseableSequence<LevelDBReader.LazyEntry> = nativeDatabase.asSequence(
verifyChecksums,
fillCache,
from,
charset = charset
)

@BrokenNativeAPI
override fun <T> withSnapshot(action: LevelDBSnapshot.() -> T): T {
val nativeSnapshot = LibLevelDB.leveldb_create_snapshot(nativeDatabase)
return try {
action(JvmLevelDBSnapshot(nativeDatabase, nativeSnapshot))
action(JvmLevelDBSnapshot(nativeDatabase, nativeSnapshot, charset))
} finally {
LibLevelDB.leveldb_release_snapshot(nativeDatabase, nativeSnapshot)
}
Expand All @@ -147,11 +154,11 @@ public class JvmLevelDB internal constructor(
val startPointer =
start
.takeIf { it.isNotEmpty() }
?.toPointer()
?.toPointer(charset)
val endPointer =
end
.takeIf { it.isNotEmpty() }
?.toPointer()
?.toPointer(charset)
LibLevelDB.leveldb_compact_range(
db = nativeDatabase,
start_key = startPointer,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,24 +3,38 @@ package com.github.lamba92.leveldb.jvm
import com.github.lamba92.leveldb.CloseableSequence
import com.github.lamba92.leveldb.LevelDBReader
import com.github.lamba92.leveldb.LevelDBSnapshot
import java.nio.charset.Charset
import kotlinx.datetime.Clock
import kotlinx.datetime.Instant

public class JvmLevelDBSnapshot internal constructor(
private val nativeDatabase: LibLevelDB.leveldb_t,
private val nativeSnapshot: LibLevelDB.leveldb_snapshot_t,
private val charset: Charset,
) : LevelDBSnapshot {
override val createdAt: Instant = Clock.System.now()

override fun get(
key: String,
verifyChecksums: Boolean,
fillCache: Boolean,
): String? = nativeDatabase.get(verifyChecksums, fillCache, key, nativeSnapshot)
): String? = nativeDatabase.get(
verifyChecksums = verifyChecksums,
fillCache = fillCache,
key = key,
snapshot = nativeSnapshot,
charset = charset
)

override fun scan(
from: String?,
verifyChecksums: Boolean,
fillCache: Boolean,
): CloseableSequence<LevelDBReader.LazyEntry> = nativeDatabase.asSequence(verifyChecksums, fillCache, from, nativeSnapshot)
): CloseableSequence<LevelDBReader.LazyEntry> = nativeDatabase.asSequence(
verifyChecksums = verifyChecksums,
fillCache = fillCache,
from = from,
snapshot = nativeSnapshot,
charset = charset
)
}
Original file line number Diff line number Diff line change
Expand Up @@ -6,16 +6,24 @@ import com.github.lamba92.leveldb.LevelDBReader
import com.github.lamba92.leveldb.asCloseable
import com.sun.jna.Memory
import com.sun.jna.NativeLong
import com.sun.jna.Pointer
import com.sun.jna.ptr.LongByReference
import com.sun.jna.ptr.PointerByReference
import java.nio.charset.Charset

internal fun String.toPointer() = Memory(length.toLong() + 1L).also { it.setString(0L, this) }
internal fun String.toPointer(charset: Charset): Memory {
val data = toByteArray(charset)
val mem = Memory(data.size.toLong())
mem.write(0, data, 0, data.size)
return mem
}

internal fun LibLevelDB.leveldb_t.get(
verifyChecksums: Boolean,
fillCache: Boolean,
key: String,
snapshot: LibLevelDB.leveldb_snapshot_t? = null,
charset: Charset,
) = with(LibLevelDB) {
val errPtr = PointerByReference()
val nativeReadOptions = leveldb_readoptions_create()
Expand All @@ -24,14 +32,14 @@ internal fun LibLevelDB.leveldb_t.get(
if (snapshot != null) {
leveldb_readoptions_set_snapshot(nativeReadOptions, snapshot)
}
val keyPointer = key.toPointer()
val keyPointer = key.toPointer(charset)
val valueLengthPointer = LongByReference()
val value =
leveldb_get(
db = this@get,
options = nativeReadOptions,
key = keyPointer,
keylen = key.length.toNativeLong(),
keylen = keyPointer.size().toNativeLong(),
vallen = valueLengthPointer,
errptr = errPtr,
)
Expand All @@ -42,8 +50,7 @@ internal fun LibLevelDB.leveldb_t.get(
if (errorValue != null) {
error("Failed to get value: $errorValue")
}
value?.getByteArray(0, valueLength.toInt())
?.toString(Charsets.UTF_8)
value?.toString(valueLength.toInt(), charset)
}

internal fun LevelDBOptions.toNative() =
Expand Down Expand Up @@ -71,6 +78,7 @@ internal fun LibLevelDB.leveldb_t.asSequence(
fillCache: Boolean,
from: String? = null,
snapshot: LibLevelDB.leveldb_snapshot_t? = null,
charset: Charset,
): CloseableSequence<LevelDBReader.LazyEntry> =
with(LibLevelDB) {
val nativeOptions = leveldb_readoptions_create()
Expand All @@ -85,7 +93,7 @@ internal fun LibLevelDB.leveldb_t.asSequence(
when (from) {
null -> leveldb_iter_seek_to_first(iterator)
else -> {
val fromPointer = from.toPointer()
val fromPointer = from.toPointer(charset)
leveldb_iter_seek(iterator, fromPointer, from.length.toNativeLong())
fromPointer.close()
}
Expand All @@ -99,15 +107,13 @@ internal fun LibLevelDB.leveldb_t.asSequence(
val key =
lazy {
val keyPointer = leveldb_iter_key(iterator, keyLengthPointer)
keyPointer.getByteArray(0, keyLengthPointer.value.toInt())
?.toString(Charsets.UTF_8)
keyPointer.toString(keyLengthPointer.value.toInt(), charset)
?: error("Failed to read key")
}
val value =
lazy {
val valuePointer = leveldb_iter_value(iterator, valueLengthPointer)
valuePointer.getByteArray(0, valueLengthPointer.value.toInt())
?.toString(Charsets.UTF_8)
valuePointer.toString(valueLengthPointer.value.toInt(), charset)
?: error("Failed to read value for key '${key.value}'")
}
yield(LevelDBReader.LazyEntry(key, value))
Expand All @@ -121,6 +127,11 @@ internal fun LibLevelDB.leveldb_t.asSequence(
}
}

private fun Pointer.toString(length: Int, charset: Charset): String? =
getByteArray(0, length)
?.toString(charset)
?: error("Failed to read string")

internal fun Boolean.toByte(): Byte = if (this) 1 else 0

internal fun Number.toNativeLong() = NativeLong(toLong())
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
@file:Suppress("EXPECT_ACTUAL_CLASSIFIERS_ARE_IN_BETA_WARNING", "unused")

package com.github.lamba92.leveldb.tests

import kotlin.test.assertEquals

@Target(AnnotationTarget.FUNCTION)
expect annotation class Test()

class UTF16Tests {
@Test
fun emoji() = withDatabase { db ->
val key = "👋🌍"
val value = "👋🌍"
db.put(key, value)
assertEquals(value, db.get(key))
}

@Test
fun accentedChars() = withDatabase { db ->
val key = "áéíóú"
val value = "áéíóú"
db.put(key, value)
assertEquals(value, db.get(key))
}

@Test
fun chineseChars() = withDatabase { db ->
val key = "你好"
val value = "你好"
db.put(key, value)
assertEquals(value, db.get(key))
}

@Test
fun japaneseChars() = withDatabase { db ->
val key = "こんにちは"
val value = "こんにちは"
db.put(key, value)
assertEquals(value, db.get(key))
}

@Test
fun cyrillicChars() = withDatabase { db ->
val key = "Здравствуйте"
val value = "Здравствуйте"
db.put(key, value)
assertEquals(value, db.get(key))
}

@Test
fun dieresis() = withDatabase { db ->
val key = "äëïöü"
val value = "äëïöü"
db.put(key, value)
assertEquals(value, db.get(key))
}

@Test
fun greekChars() = withDatabase { db ->
val key = "Γειά σας"
val value = "Γειά σας"
db.put(key, value)
assertEquals(value, db.get(key))
}
}

Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
@file:Suppress("EXPECT_ACTUAL_CLASSIFIERS_ARE_IN_BETA_WARNING")

package com.github.lamba92.leveldb.tests

actual val DATABASE_PATH: String
get() =
System.getenv("LEVELDB_LOCATION")
?: error("LEVELDB_LOCATION environment variable not set")

actual typealias Test = org.junit.jupiter.api.Test

0 comments on commit 2a4785a

Please sign in to comment.