Skip to content
This repository has been archived by the owner on Sep 16, 2024. It is now read-only.

Commit

Permalink
feat(kv): introduce KVBufferStore interface and implement it with lmdb
Browse files Browse the repository at this point in the history
  • Loading branch information
dtfiedler committed Mar 21, 2024
1 parent 8740e59 commit d827090
Show file tree
Hide file tree
Showing 6 changed files with 198 additions and 0 deletions.
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
"express-async-handler": "^1.2.0",
"express-openapi-validator": "^5.0.5",
"express-prometheus-middleware": "^1.2.0",
"lmdb": "^3.0.0",
"middleware-async": "^1.3.5",
"prom-client": "^14.0.1",
"swagger-ui-express": "^5.0.0",
Expand Down
51 changes: 51 additions & 0 deletions src/cache/lmdb-kv-store.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
/**
* AR.IO ArNS Resolver
* Copyright (C) 2023 Permanent Data Solutions, Inc. All Rights Reserved.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { RootDatabase, open } from 'lmdb';

import { KVBufferStore } from '../types.js';

export class LmdbKVStore implements KVBufferStore {
private db: RootDatabase<Buffer, string>;

constructor({ dbPath }: { dbPath: string }) {
this.db = open({
path: dbPath,
encoding: 'binary',
commitDelay: 100, // 100ms delay - increases writes per transaction to reduce I/O
});
}

async get(key: string): Promise<Buffer | undefined> {
const value = this.db.get(key);
return value;
}

async has(key: string): Promise<boolean> {
return this.db.doesExist(key);
}

async del(key: string): Promise<void> {
if (await this.has(key)) {
await this.db.remove(key);
}
}

async set(key: string, buffer: Buffer): Promise<void> {
await this.db.put(key, buffer);
}
}
4 changes: 4 additions & 0 deletions src/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,4 +38,8 @@ export const CONTRACT_TX_ID = env.varOrDefault(
'CONTRACT_TX_ID',
'bLAgYxAdX2Ry-nt6aH2ixgvJXbpsEYm28NgJgyqfs-U',
);
export const ARNS_CACHE_PATH = env.varOrDefault(
'ARNS_CACHE_PATH',
'./data/arns',
);
export const PORT = +env.varOrDefault('PORT', '6000');
5 changes: 5 additions & 0 deletions src/system.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
*/
import { ArIO, RemoteContract } from '@ar.io/sdk/node';

import { LmdbKVStore } from './cache/lmdb-kv-store.js';
import * as config from './config.js';
import log from './log.js';

Expand All @@ -27,6 +28,10 @@ export const network = new ArIO({
}),
});

export const arnsCache = new LmdbKVStore({
dbPath: config.ARNS_CACHE_PATH,
});

export async function evaluateArNSNames() {
log.info('Evaluating arns names');
// TODO: add evaluation logic here
Expand Down
6 changes: 6 additions & 0 deletions src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,3 +15,9 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
export type KVBufferStore = {
get(key: string): Promise<Buffer | undefined>;
set(key: string, buffer: Buffer): Promise<void>;
del(key: string): Promise<void>;
has(key: string): Promise<boolean>;
};
131 changes: 131 additions & 0 deletions yarn.lock
Original file line number Diff line number Diff line change
Expand Up @@ -394,6 +394,66 @@
resolved "https://registry.yarnpkg.com/@jsdevtools/ono/-/ono-7.1.3.tgz#9df03bbd7c696a5c58885c34aa06da41c8543796"
integrity sha512-4JQNk+3mVzK3xh2rqd6RB4J46qUR19azEHBneZyTZM+c456qOrbbM/5xcR8huNCCcbVt7+UmizG6GuUvPvKUYg==

"@lmdb/lmdb-darwin-arm64@3.0.0":
version "3.0.0"
resolved "https://registry.yarnpkg.com/@lmdb/lmdb-darwin-arm64/-/lmdb-darwin-arm64-3.0.0.tgz#146133d513c62492977956af6d740723294108e9"
integrity sha512-iuJO2xaqKFw1AhHaFTZajHE2VaN53tAZO3gUkT7sbOX0MVDSYvmdLNMLEehsrz2Yt5j4QyO/0p1xkFYqeUUxjA==

"@lmdb/lmdb-darwin-x64@3.0.0":
version "3.0.0"
resolved "https://registry.yarnpkg.com/@lmdb/lmdb-darwin-x64/-/lmdb-darwin-x64-3.0.0.tgz#50659fc68d39dee9f61fdc1d690b3a5013b8c8c1"
integrity sha512-zPXTjgGfAm/kVYbnhwhsi4fU9qkeJWi+Bo6CY8yWCWnNRZLNDdQVXKpzgpHfWC2BB0V9JTNMf6ls562vv3s+lQ==

"@lmdb/lmdb-linux-arm64@3.0.0":
version "3.0.0"
resolved "https://registry.yarnpkg.com/@lmdb/lmdb-linux-arm64/-/lmdb-linux-arm64-3.0.0.tgz#373070e212a3273602ffa5a9a5b69697e8d45d6c"
integrity sha512-UEZ/nV8dGjRf3FvlTfZhZqgLxz+Lofsb45hxRAaiflv/3rJdtuGeQgGHl1AO8lKJG0J7zmlTx+HaFsThkGxsHQ==

"@lmdb/lmdb-linux-arm@3.0.0":
version "3.0.0"
resolved "https://registry.yarnpkg.com/@lmdb/lmdb-linux-arm/-/lmdb-linux-arm-3.0.0.tgz#b1ce6d23a59db07d455f8a077a52e8d00f060d55"
integrity sha512-QNXKEKXRu7BH0t1iImaDJ62ix1LnR2HUGPBpcgeFCASCJxL4vrAUrNccLwZX9G1yfQDI6nlj6NGei/siz05UOQ==

"@lmdb/lmdb-linux-x64@3.0.0":
version "3.0.0"
resolved "https://registry.yarnpkg.com/@lmdb/lmdb-linux-x64/-/lmdb-linux-x64-3.0.0.tgz#559e21a4c4ccaa09124530101805f83e6123ff72"
integrity sha512-AW+/J/PFExlGTjD3Cf0lE6Wor/dNRKT9ZbfZQHHqezT+NhdplKYmGI0D1ogyL1OCn++zrE3tL2jICqkgH6JqaA==

"@lmdb/lmdb-win32-x64@3.0.0":
version "3.0.0"
resolved "https://registry.yarnpkg.com/@lmdb/lmdb-win32-x64/-/lmdb-win32-x64-3.0.0.tgz#5bab50f23cb084619748c6fbf9b39a1b4eda56a7"
integrity sha512-j7CaqYtpICn+tApohDfVolBbBdBFHe0vhq5s6HBkCN62LqvaIaFkeMUy8tKdCKa5ELw1zsnKESTzRpPQS/h8cw==

"@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.2":
version "3.0.2"
resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.2.tgz#44d752c1a2dc113f15f781b7cc4f53a307e3fa38"
integrity sha512-9bfjwDxIDWmmOKusUcqdS4Rw+SETlp9Dy39Xui9BEGEk19dDwH0jhipwFzEff/pFg95NKymc6TOTbRKcWeRqyQ==

"@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.2":
version "3.0.2"
resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.2.tgz#f954f34355712212a8e06c465bc06c40852c6bb3"
integrity sha512-lwriRAHm1Yg4iDf23Oxm9n/t5Zpw1lVnxYU3HnJPTi2lJRkKTrps1KVgvL6m7WvmhYVt/FIsssWay+k45QHeuw==

"@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.2":
version "3.0.2"
resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.2.tgz#45c63037f045c2b15c44f80f0393fa24f9655367"
integrity sha512-FU20Bo66/f7He9Fp9sP2zaJ1Q8L9uLPZQDub/WlUip78JlPeMbVL8546HbZfcW9LNciEXc8d+tThSJjSC+tmsg==

"@msgpackr-extract/msgpackr-extract-linux-arm@3.0.2":
version "3.0.2"
resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.2.tgz#35707efeafe6d22b3f373caf9e8775e8920d1399"
integrity sha512-MOI9Dlfrpi2Cuc7i5dXdxPbFIgbDBGgKR5F2yWEa6FVEtSWncfVNKW5AKjImAQ6CZlBK9tympdsZJ2xThBiWWA==

"@msgpackr-extract/msgpackr-extract-linux-x64@3.0.2":
version "3.0.2"
resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.2.tgz#091b1218b66c341f532611477ef89e83f25fae4f"
integrity sha512-gsWNDCklNy7Ajk0vBBf9jEx04RUxuDQfBse918Ww+Qb9HCPoGzS+XJTLe96iN3BVK7grnLiYghP/M4L8VsaHeA==

"@msgpackr-extract/msgpackr-extract-win32-x64@3.0.2":
version "3.0.2"
resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.2.tgz#0f164b726869f71da3c594171df5ebc1c4b0a407"
integrity sha512-O+6Gs8UeDbyFpbSh2CPEz/UOrrdWPTBYNblZK5CxxLisYt4kGX3Sc+czffFonyjiGSq3jWLwJS/CCJc7tBr4sQ==

"@nodelib/fs.scandir@2.1.5":
version "2.1.5"
resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz#7619c2eb21b25483f6d167548b4cfd5a7488c3d5"
Expand Down Expand Up @@ -1521,6 +1581,11 @@ detect-libc@^1.0.2:
resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-1.0.3.tgz#fa137c4bd698edf55cd5cd02ac559f91a4c4ba9b"
integrity sha512-pGjwhsmsp4kL2RTz08wcOlGN83otlqHeD/Z5T8GXZB+/YcpQ/dgo+lbU8ZsGxV0HIvqqxo9l7mqYwyYMD9bKDg==

detect-libc@^2.0.1:
version "2.0.3"
resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-2.0.3.tgz#f0cd503b40f9939b894697d19ad50895e30cf700"
integrity sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==

diff@^4.0.1:
version "4.0.2"
resolved "https://registry.yarnpkg.com/diff/-/diff-4.0.2.tgz#60f3aecb89d5fae520c11aa19efc2bb982aade7d"
Expand Down Expand Up @@ -2434,6 +2499,24 @@ lines-and-columns@^1.1.6:
resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.2.4.tgz#eca284f75d2965079309dc0ad9255abb2ebc1632"
integrity sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==

lmdb@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/lmdb/-/lmdb-3.0.0.tgz#f9c0fe9eb1ed29722a209d6be26731753cc6dd4e"
integrity sha512-rjuwTQ7bGrow+HvMDuQFX/Hex8JzR72mpgKOqBY7cA4PWs5HyNBeF3Q/buc7zvjFKVYY2FahxmsP7zcBDfhKpg==
dependencies:
msgpackr "^1.9.9"
node-addon-api "^6.1.0"
node-gyp-build-optional-packages "5.1.1"
ordered-binary "^1.4.1"
weak-lru-cache "^1.2.2"
optionalDependencies:
"@lmdb/lmdb-darwin-arm64" "3.0.0"
"@lmdb/lmdb-darwin-x64" "3.0.0"
"@lmdb/lmdb-linux-arm" "3.0.0"
"@lmdb/lmdb-linux-arm64" "3.0.0"
"@lmdb/lmdb-linux-x64" "3.0.0"
"@lmdb/lmdb-win32-x64" "3.0.0"

locate-path@^5.0.0:
version "5.0.0"
resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-5.0.0.tgz#1afba396afd676a6d42504d0a67a3a7eb9f62aa0"
Expand Down Expand Up @@ -2676,6 +2759,27 @@ ms@2.1.3, ms@^2.1.1:
resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2"
integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==

msgpackr-extract@^3.0.2:
version "3.0.2"
resolved "https://registry.yarnpkg.com/msgpackr-extract/-/msgpackr-extract-3.0.2.tgz#e05ec1bb4453ddf020551bcd5daaf0092a2c279d"
integrity sha512-SdzXp4kD/Qf8agZ9+iTu6eql0m3kWm1A2y1hkpTeVNENutaB0BwHlSvAIaMxwntmRUAUjon2V4L8Z/njd0Ct8A==
dependencies:
node-gyp-build-optional-packages "5.0.7"
optionalDependencies:
"@msgpackr-extract/msgpackr-extract-darwin-arm64" "3.0.2"
"@msgpackr-extract/msgpackr-extract-darwin-x64" "3.0.2"
"@msgpackr-extract/msgpackr-extract-linux-arm" "3.0.2"
"@msgpackr-extract/msgpackr-extract-linux-arm64" "3.0.2"
"@msgpackr-extract/msgpackr-extract-linux-x64" "3.0.2"
"@msgpackr-extract/msgpackr-extract-win32-x64" "3.0.2"

msgpackr@^1.9.9:
version "1.10.1"
resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.10.1.tgz#51953bb4ce4f3494f0c4af3f484f01cfbb306555"
integrity sha512-r5VRLv9qouXuLiIBrLpl2d5ZvPt8svdQTl5/vMvE4nzDMyEX4sgW5yWhuBBj5UmgwOTWj8CIdSXn5sAfsHAWIQ==
optionalDependencies:
msgpackr-extract "^3.0.2"

multer@^1.4.5-lts.1:
version "1.4.5-lts.1"
resolved "https://registry.yarnpkg.com/multer/-/multer-1.4.5-lts.1.tgz#803e24ad1984f58edffbc79f56e305aec5cfd1ac"
Expand Down Expand Up @@ -2728,6 +2832,23 @@ negotiator@0.6.3:
resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.3.tgz#58e323a72fedc0d6f9cd4d31fe49f51479590ccd"
integrity sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==

node-addon-api@^6.1.0:
version "6.1.0"
resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-6.1.0.tgz#ac8470034e58e67d0c6f1204a18ae6995d9c0d76"
integrity sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==

node-gyp-build-optional-packages@5.0.7:
version "5.0.7"
resolved "https://registry.yarnpkg.com/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.0.7.tgz#5d2632bbde0ab2f6e22f1bbac2199b07244ae0b3"
integrity sha512-YlCCc6Wffkx0kHkmam79GKvDQ6x+QZkMjFGrIMxgFNILFvGSbCp2fCBC55pGTT9gVaz8Na5CLmxt/urtzRv36w==

node-gyp-build-optional-packages@5.1.1:
version "5.1.1"
resolved "https://registry.yarnpkg.com/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.1.1.tgz#52b143b9dd77b7669073cbfe39e3f4118bfc603c"
integrity sha512-+P72GAjVAbTxjjwUmwjVrqrdZROD4nf8KgpBoDxqXXTiYZZt/ud60dE5yvCSr9lRO8e8yv6kgJIC0K0PfZFVQw==
dependencies:
detect-libc "^2.0.1"

node-gyp-build@^4.3.0:
version "4.6.1"
resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-4.6.1.tgz#24b6d075e5e391b8d5539d98c7fc5c210cac8a3e"
Expand Down Expand Up @@ -2896,6 +3017,11 @@ optionator@^0.9.3:
prelude-ls "^1.2.1"
type-check "^0.4.0"

ordered-binary@^1.4.1:
version "1.5.1"
resolved "https://registry.yarnpkg.com/ordered-binary/-/ordered-binary-1.5.1.tgz#94ccbf14181711081ee23931db0dc3f58aaa0df6"
integrity sha512-5VyHfHY3cd0iza71JepYG50My+YUbrFtGoUz2ooEydPyPM7Aai/JW098juLr+RG6+rDJuzNNTsEQu2DZa1A41A==

os-homedir@^1.0.0:
version "1.0.2"
resolved "https://registry.yarnpkg.com/os-homedir/-/os-homedir-1.0.2.tgz#ffbc4988336e0e833de0c168c7ef152121aa7fb3"
Expand Down Expand Up @@ -3849,6 +3975,11 @@ warp-wasm-metering@1.0.1:
leb128 "^0.0.4"
warp-wasm-json-toolkit "1.0.2"

weak-lru-cache@^1.2.2:
version "1.2.2"
resolved "https://registry.yarnpkg.com/weak-lru-cache/-/weak-lru-cache-1.2.2.tgz#fdbb6741f36bae9540d12f480ce8254060dccd19"
integrity sha512-DEAoo25RfSYMuTGc9vPJzZcZullwIqRDSI9LOy+fkCJPi6hykCnfKaXTuPBDuXAUcqHXyOgFtHNp/kB2FjYHbw==

which@^2.0.1:
version "2.0.2"
resolved "https://registry.yarnpkg.com/which/-/which-2.0.2.tgz#7c6a8dd0a636a0327e10b59c9286eee93f3f51b1"
Expand Down

0 comments on commit d827090

Please sign in to comment.