-
Notifications
You must be signed in to change notification settings - Fork 116
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(lib/erasure): implement Go binding over rust for erasure coding (#…
- Loading branch information
1 parent
5bd66b0
commit 6f864af
Showing
13 changed files
with
4,584 additions
and
148 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,6 @@ | ||
| # Building Rust code Binary | ||
|
|
||
| - Generate rust binary | ||
| ``` | ||
| cargo build --release --manifest-path=lib/erasure/rustlib/Cargo.toml | ||
| ``` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,70 +1,101 @@ | ||
| // Copyright 2021 ChainSafe Systems (ON) | ||
| // Copyright 2023 ChainSafe Systems (ON) | ||
| // SPDX-License-Identifier: LGPL-3.0-only | ||
|
|
||
| package erasure | ||
|
|
||
| // #cgo LDFLAGS: -Wl,-rpath,${SRCDIR}/rustlib/target/release -L${SRCDIR}/rustlib/target/release -lerasure | ||
| // #include "./erasure.h" | ||
| import ( | ||
| "C" | ||
| ) | ||
| import ( | ||
| "bytes" | ||
| "errors" | ||
| "fmt" | ||
|
|
||
| "github.com/klauspost/reedsolomon" | ||
| "unsafe" | ||
| ) | ||
|
|
||
| // ErrNotEnoughValidators cannot encode something for zero or one validator | ||
| var ErrNotEnoughValidators = errors.New("expected at least 2 validators") | ||
| var ( | ||
| ErrZeroSizedData = errors.New("data can't be zero sized") | ||
| ErrZeroSizedChunks = errors.New("chunks can't be zero sized") | ||
| ) | ||
|
|
||
| // ObtainChunks obtains erasure-coded chunks, divides data into number of validatorsQty chunks and | ||
| // creates parity chunks for reconstruction | ||
| func ObtainChunks(validatorsQty int, data []byte) ([][]byte, error) { | ||
| recoveryThres, err := recoveryThreshold(validatorsQty) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| enc, err := reedsolomon.New(validatorsQty, recoveryThres) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("creating new reed solomon failed: %w", err) | ||
| // ObtainChunks obtains erasure-coded chunks, one for each validator. | ||
| // This works only up to 65536 validators, and `n_validators` must be non-zero and accepts | ||
| // number of validators and scale encoded data. | ||
| func ObtainChunks(nValidators uint, data []byte) ([][]byte, error) { | ||
| if len(data) == 0 { | ||
| return nil, ErrZeroSizedData | ||
| } | ||
| shards, err := enc.Split(data) | ||
| if err != nil { | ||
| return nil, err | ||
|
|
||
| var cFlattenedChunks *C.uchar | ||
| var cFlattenedChunksLen C.size_t | ||
|
|
||
| cnValidators := C.size_t(nValidators) | ||
| cData := (*C.uchar)(unsafe.Pointer(&data[0])) | ||
| cLen := C.size_t(len(data)) | ||
|
|
||
| cErr := C.obtain_chunks(cnValidators, cData, cLen, &cFlattenedChunks, &cFlattenedChunksLen) | ||
| errStr := C.GoString(cErr) | ||
| C.free(unsafe.Pointer(cErr)) | ||
|
|
||
| if len(errStr) > 0 { | ||
| return nil, errors.New(errStr) | ||
| } | ||
| err = enc.Encode(shards) | ||
| if err != nil { | ||
| return nil, err | ||
|
|
||
| resData := C.GoBytes(unsafe.Pointer(cFlattenedChunks), C.int(cFlattenedChunksLen)) | ||
| C.free(unsafe.Pointer(cFlattenedChunks)) | ||
|
|
||
| chunkSize := uint(len(resData)) / nValidators | ||
| chunks := make([][]byte, nValidators) | ||
|
|
||
| start := uint(0) | ||
| for i := start; i < nValidators; i++ { | ||
| end := start + chunkSize | ||
| chunks[i] = resData[start:end] | ||
| start = end | ||
| } | ||
|
|
||
| return shards, nil | ||
| return chunks, nil | ||
| } | ||
|
|
||
| // Reconstruct the missing data from a set of chunks | ||
| func Reconstruct(validatorsQty, originalDataLen int, chunks [][]byte) ([]byte, error) { | ||
| recoveryThres, err := recoveryThreshold(validatorsQty) | ||
| if err != nil { | ||
| return nil, err | ||
| // Reconstruct decodable data from a set of chunks. | ||
| // | ||
| // Provide an iterator containing chunk data and the corresponding index. | ||
| // The indices of the present chunks must be indicated. If too few chunks | ||
| // are provided, recovery is not possible. | ||
| // | ||
| // Works only up to 65536 validators, and `n_validators` must be non-zero | ||
| func Reconstruct(nValidators uint, chunks [][]byte) ([]byte, error) { | ||
| if len(chunks) == 0 { | ||
| return nil, ErrZeroSizedChunks | ||
| } | ||
|
|
||
| enc, err := reedsolomon.New(validatorsQty, recoveryThres) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| err = enc.Reconstruct(chunks) | ||
| if err != nil { | ||
| return nil, err | ||
| var cReconstructedData *C.uchar | ||
| var cReconstructedDataLen C.size_t | ||
| var flattenedChunks []byte | ||
|
|
||
| for _, chunk := range chunks { | ||
| flattenedChunks = append(flattenedChunks, chunk...) | ||
| } | ||
| buf := new(bytes.Buffer) | ||
| err = enc.Join(buf, chunks, originalDataLen) | ||
| return buf.Bytes(), err | ||
| } | ||
|
|
||
| // recoveryThreshold gives the max number of shards/chunks that we can afford to lose and still construct | ||
| // the full initial data. Total number of chunks will be validatorQty + recoveryThreshold | ||
| func recoveryThreshold(validators int) (int, error) { | ||
| if validators <= 1 { | ||
| return 0, ErrNotEnoughValidators | ||
| cChunkSize := C.size_t(len(chunks[0])) | ||
| cFlattenedChunks := (*C.uchar)(unsafe.Pointer(&flattenedChunks[0])) | ||
| cFlattenedChunksLen := C.size_t(len(flattenedChunks)) | ||
|
|
||
| cErr := C.reconstruct( | ||
| C.size_t(nValidators), | ||
| cFlattenedChunks, cFlattenedChunksLen, | ||
| cChunkSize, | ||
| &cReconstructedData, &cReconstructedDataLen, | ||
| ) | ||
| errStr := C.GoString(cErr) | ||
| C.free(unsafe.Pointer(cErr)) | ||
|
|
||
| if len(errStr) > 0 { | ||
| return nil, errors.New(errStr) | ||
| } | ||
|
|
||
| needed := (validators - 1) / 3 | ||
| res := C.GoBytes(unsafe.Pointer(cReconstructedData), C.int(cReconstructedDataLen)) | ||
| C.free(unsafe.Pointer(cReconstructedData)) | ||
|
|
||
| return needed + 1, nil | ||
| return res, nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| /* | ||
| * Copyright 2023 ChainSafe Systems (ON) | ||
| * SPDX-License-Identifier: LGPL-3.0-only | ||
| */ | ||
|
|
||
| #include <stdlib.h> | ||
| #include <stddef.h> | ||
|
|
||
| int32_t add(int32_t a, int32_t b); | ||
| const char* obtain_chunks(size_t n_validators, unsigned char *data, size_t len, unsigned char **flattened_chunks, size_t *flattened_chunks_len); | ||
| const char* reconstruct(size_t n_validators, unsigned char *flattened_chunks, size_t flattened_chunks_len, size_t chunk_size, unsigned char **res_data, size_t *res_len); |
Oops, something went wrong.