Skip to content

Commit

Permalink
Clients: add modify and clear (#23)
Browse files Browse the repository at this point in the history
* add modify and clear

* test blyss service via python client
  • Loading branch information
neilmovva authored Apr 20, 2023
1 parent 1f5c056 commit 3b28c30
Show file tree
Hide file tree
Showing 16 changed files with 377 additions and 105 deletions.
12 changes: 12 additions & 0 deletions .github/workflows/build-python.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
name: Build Python SDK

env:
BLYSS_STAGING_SERVER: https://dev2.api.blyss.dev
BLYSS_STAGING_API_KEY: Gh1pz1kEiNa1npEdDaRRvM1LsVypM1u2x1YbGb54

on:
push:
branches: [ "main" ]
Expand All @@ -26,6 +30,14 @@ jobs:
- uses: actions/setup-python@v4
with:
python-version: '3.10'
- name: Install Python SDK
working-directory: python
shell: bash
run: pip install .
- name: Test Python SDK
working-directory: python
shell: bash
run: python tests/test_service.py
- name: Build wheels
uses: PyO3/maturin-action@v1
with:
Expand Down
3 changes: 2 additions & 1 deletion .vscode/settings.json
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
{
"python.analysis.typeCheckingMode": "basic"
"python.analysis.typeCheckingMode": "basic",
"editor.formatOnSave": true
}
129 changes: 129 additions & 0 deletions e2e-tests/api.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
import type { Bucket, Client } from '@blyss/sdk';
const blyss = require('@blyss/sdk/node');

async function keyToValue(key: string, len: number): Promise<Uint8Array> {
const keyBytes = new TextEncoder().encode(key);
const value = new Uint8Array(len);
let i = 0;
// fill the value with the hash.
// if the hash is smaller than the value, we hash the hash again.
while (i < len) {
const hash = await crypto.subtle.digest('SHA-1', keyBytes);
const hashBytes = new Uint8Array(hash);
const toCopy = Math.min(hashBytes.length, len - i);
value.set(hashBytes.slice(0, toCopy), i);
i += toCopy;
}
return value;
}

async function verifyRead(key: string, value: Uint8Array): Promise<void> {
const expected = await keyToValue(key, value.length);
if (expected.toString() !== value.toString()) {
throw new Error('Incorrect value for key ' + key);
}
}

function generateKeys(n: number, seed: number = 0): string[] {
return new Array(n).fill(0).map(
(_, i) => seed.toString() + '-' + i.toString()
);
}

function generateBucketName(): string {
return 'api-tester-' + Math.random().toString(16).substring(2, 10);
}

async function testBlyssService(endpoint: string = 'https://dev2.api.blyss.dev') {
const apiKey = process.env.BLYSS_API_KEY;
if (!apiKey) {
throw new Error('BLYSS_API_KEY environment variable is not set');
}
console.log('Using key: ' + apiKey + ' to connect to ' + endpoint);
const client: Client = await new blyss.Client(
{
endpoint: endpoint,
apiKey: apiKey
}
);
// generate random string for bucket name
const bucketName = generateBucketName();
await client.create(bucketName);
const bucket: Bucket = await client.connect(bucketName);
console.log(bucket.metadata);

// generate N random keys
const N = 100;
const itemSize = 32;
let localKeys = generateKeys(N);
function getRandomKey(): string {
return localKeys[Math.floor(Math.random() * localKeys.length)];
}
// write all N keys
await bucket.write(
await Promise.all(localKeys.map(
async (k) => ({
k: await keyToValue(k, itemSize)
})
))
);
console.log(`Wrote ${N} keys`);

// read a random key
let testKey = getRandomKey();
let value = await bucket.privateRead(testKey);
await verifyRead(testKey, value);
console.log(`Read key ${testKey}`);

// delete testKey from the bucket, and localData.
await bucket.deleteKey(testKey);
localKeys.splice(localKeys.indexOf(testKey), 1);
console.log(`Deleted key ${testKey}`);

// write a new value
testKey = 'newKey0';
await bucket.write({ testKey: keyToValue(testKey, itemSize) });
localKeys.push(testKey);
console.log(`Wrote key ${testKey}`);

// clear all keys
await bucket.clearEntireBucket();
localKeys = [];
console.log('Cleared bucket');

// write a new set of N keys
localKeys = generateKeys(N, 1);
await bucket.write(
await Promise.all(localKeys.map(
async (k) => ({
k: await keyToValue(k, itemSize)
})
))
);
console.log(`Wrote ${N} keys`);

// rename the bucket
const newBucketName = bucketName + '-rn';
await bucket.rename(newBucketName);
console.log(`Renamed bucket`);
console.log(await bucket.info());

// random read
testKey = getRandomKey();
value = await bucket.privateRead(testKey);
await verifyRead(testKey, value);
console.log(`Read key ${testKey}`);

// destroy the bucket
await bucket.destroyEntireBucket();
console.log(`Destroyed bucket ${bucket.name}`);
}

async function main() {
const endpoint = "https://dev2.api.blyss.dev"
console.log('Testing Blyss service at URL ' + endpoint);
await testBlyssService(endpoint);
console.log('All tests completed successfully.');
}

main();
69 changes: 33 additions & 36 deletions js/bucket/bucket.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ import { base64ToBytes, getRandomSeed } from '../client/seed';
import { decompress } from '../compression/bz2_decompress';
import { bloomLookup } from '../data/bloom';
import {
DataWithMetadata,
concatBytes,
deserialize,
deserializeChunks,
Expand Down Expand Up @@ -40,7 +39,7 @@ export class Bucket {
readonly api: Api;

/** The name of this bucket. */
readonly name: string;
name: string;

/**
* The secret seed for this instance of the client, which can be saved and
Expand Down Expand Up @@ -110,7 +109,7 @@ export class Bucket {
try {
decompressedResult = decompress(decryptedResult);
} catch (e) {
console.error('decompress error', e);
console.error(`key ${key} not found (decompression failed)`);
}
if (decompressedResult === null) {
return null;
Expand All @@ -120,7 +119,7 @@ export class Bucket {
try {
extractedResult = this.lib.extractResult(key, decompressedResult);
} catch (e) {
console.error('extraction error', e);
console.error(`key ${key} not found (extraction failed)`);
}
if (extractedResult === null) {
return null;
Expand Down Expand Up @@ -151,7 +150,7 @@ export class Bucket {

private async performPrivateReads(
keys: string[]
): Promise<DataWithMetadata[]> {
): Promise<any[]> {
if (!this.uuid || !this.check(this.uuid)) {
await this.setup();
}
Expand Down Expand Up @@ -185,7 +184,7 @@ export class Bucket {
return endResults;
}

private async performPrivateRead(key: string): Promise<DataWithMetadata> {
private async performPrivateRead(key: string): Promise<any> {
return (await this.performPrivateReads([key]))[0];
}

Expand Down Expand Up @@ -320,6 +319,15 @@ export class Bucket {
return await this.api.meta(this.name);
}

/** Renames this bucket, leaving all data and other bucket settings intact. */
async rename(newBucketName: string): Promise<BucketMetadata> {
const bucketCreateReq = {
name: newBucketName
};
await this.api.modify(this.name, JSON.stringify(bucketCreateReq));
this.name = newBucketName;
}

/** Gets info on all keys in this bucket. */
async listKeys(): Promise<KeyInfo[]> {
this.ensureSpiral();
Expand All @@ -333,32 +341,28 @@ export class Bucket {
* key-value pairs to write. Keys must be strings, and values may be any
* JSON-serializable value or a Uint8Array. The maximum size of a key is
* 1024 UTF-8 bytes.
* @param {{ [key: string]: any }} [metadata] - An optional object containing
* metadata. Each key of this object should also be a key of
* `keyValuePairs`, and the value should be some metadata object to store
* with the values being written.
*/
async write(
keyValuePairs: { [key: string]: any },
metadata?: { [key: string]: any }
keyValuePairs: { [key: string]: any }
) {
this.ensureSpiral();

const data = [];
for (const key in keyValuePairs) {
if (Object.prototype.hasOwnProperty.call(keyValuePairs, key)) {
const value = keyValuePairs[key];
let valueMetadata = undefined;
if (metadata && Object.prototype.hasOwnProperty.call(metadata, key)) {
valueMetadata = metadata[key];
}
const valueBytes = serialize(value, valueMetadata);
const valueBytes = serialize(value);
const keyBytes = new TextEncoder().encode(key);
const serializedKeyValue = wrapKeyValue(keyBytes, valueBytes);
data.push(serializedKeyValue);
// const kv = {
// key: key,
// value: Buffer.from(valueBytes).toString('base64')
// }
}
}
const concatenatedData = concatBytes(data);
// const concatenatedData = serialize(data);
await this.api.write(this.name, concatenatedData);
}

Expand All @@ -385,6 +389,14 @@ export class Bucket {
await this.api.destroy(this.name);
}

/**
* Clears the contents of the entire bucket, and all data inside of it. This action is
* permanent and irreversible.
*/
async clearEntireBucket() {
await this.api.clear(this.name);
}

/**
* Privately reads the supplied key from the bucket, returning the value
* corresponding to the key.
Expand All @@ -398,28 +410,13 @@ export class Bucket {
this.ensureSpiral();

if (Array.isArray(key)) {
return (await this.performPrivateReads(key)).map(r => r.data);
return (await this.performPrivateReads(key));
} else {
const result = await this.performPrivateRead(key);
return result ? result.data : null;
return result ? result : null;
}
}

/**
* Privately reads the supplied key from the bucket, returning the value and
* metadata corresponding to the key.
*
* No entity, including the Blyss service, should be able to determine which
* key this method was called for.
*
* @param {string} key - The key to _privately_ retrieve the value of.
*/
async privateReadWithMetadata(key: string): Promise<DataWithMetadata> {
this.ensureSpiral();

return await this.performPrivateRead(key);
}

/**
* Privately intersects the given set of keys with the keys in this bucket,
* returning the keys that intersected and their values. This is generally
Expand All @@ -437,7 +434,7 @@ export class Bucket {
this.ensureSpiral();

if (keys.length < BLOOM_CUTOFF) {
return (await this.performPrivateReads(keys)).map(x => x.data);
return (await this.performPrivateReads(keys));
}

const bloomFilter = await this.api.bloom(this.name);
Expand All @@ -451,7 +448,7 @@ export class Bucket {
if (!retrieveValues) {
return matches;
}
return (await this.performPrivateReads(matches)).map(x => x.data);
return (await this.performPrivateReads(matches));
}

/**
Expand Down
24 changes: 24 additions & 0 deletions js/client/api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ import { gzip } from '../compression/pako';
import { BloomFilter, bloomFilterFromBytes } from '../data/bloom';

const CREATE_PATH = '/create';
const MODIFY_PATH = '/modify';
const CLEAR_PATH = '/clear';
const DESTROY_PATH = '/destroy';
const CHECK_PATH = '/check';
const DELETE_PATH = '/delete';
Expand Down Expand Up @@ -215,6 +217,18 @@ class Api {
return await getData(this.apiKey, this.urlFor(bucketName, META_PATH), true);
}

/**
* Modify a bucket's properties.
*
* @param bucketName The name of the bucket.
* @param dataJson A JSON-encoded string of the bucket metadata. Supports the same fields as `create()`.
* @returns Bucket metadata after update.
*/
async modify(bucketName: string, dataJson: string): Promise<BucketMetadata> {
return await postData(this.apiKey, this.urlFor(bucketName, MODIFY_PATH), dataJson, true);
}


/**
* Get the Bloom filter for keys in this bucket. The Bloom filter contains all
* keys ever inserted into this bucket; it does not remove deleted keys.
Expand Down Expand Up @@ -304,6 +318,16 @@ class Api {
);
}

/** Clear contents of this bucket. */
async clear(bucketName: string) {
await postData(
this.apiKey,
this.urlFor(bucketName, CLEAR_PATH),
'',
false
);
}

/** Write to this bucket. */
async write(bucketName: string, data: Uint8Array) {
await postData(
Expand Down
Loading

1 comment on commit 3b28c30

@vercel
Copy link

@vercel vercel bot commented on 3b28c30 Apr 20, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Successfully deployed to the following URLs:

docs – ./

docs-nu-ruddy.vercel.app
docs.blyss.dev
docs-blyss.vercel.app
docs-git-main-blyss.vercel.app

Please sign in to comment.