Skip to content

Commit

Permalink
sort out EDS
Browse files Browse the repository at this point in the history
  • Loading branch information
fl0rek committed Jan 7, 2024
1 parent 2282960 commit fb4611c
Show file tree
Hide file tree
Showing 10 changed files with 156 additions and 143 deletions.
10 changes: 9 additions & 1 deletion tools/update-proto-vendor.sh
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,8 @@ extract_urls ../target/proto-vendor-src \
https://github.com/cosmos/cosmos-proto/archive/refs/tags/v1.0.0-alpha4.tar.gz \
https://github.com/cosmos/gogoproto/archive/refs/tags/v1.4.11.tar.gz \
https://github.com/celestiaorg/go-header/archive/refs/heads/main.tar.gz \
https://github.com/googleapis/googleapis/archive/refs/heads/master.tar.gz
https://github.com/googleapis/googleapis/archive/refs/heads/master.tar.gz \
https://codeload.github.com/celestiaorg/celestia-node/zip/refs/heads/hlib/v2-prototype

mkdir -p vendor

Expand Down Expand Up @@ -70,6 +71,13 @@ for pb_dir in ../target/proto-vendor-src/celestia-node-main/share/*/*/pb; do
cp -r "$pb_dir" "$out_dir"
done

# TODO: replace with version from main, once it is merged
# https://github.com/celestiaorg/celestia-node/pull/2675
rm -rf vendor/share/p2p/shwap
mkdir -p vendor/share/p2p/shwap/pb
cp ../target/proto-vendor-src/celestia-node-hlib-v2-prototype/share/shwap/pb/shwap_pb.proto \
vendor/share/p2p/shwap/pb/shwap.proto

rm -rf vendor/tendermint
cp -r ../target/proto-vendor-src/celestia-core-0.34.x-celestia/proto/tendermint vendor

Expand Down
2 changes: 1 addition & 1 deletion types/src/byzantine.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,12 @@ use tendermint_proto::Protobuf;

use crate::bail_validation;
use crate::consts::appconsts;
use crate::extended_data_square::AxisType;
use crate::fraud_proof::FraudProof;
use crate::nmt::{
Namespace, NamespaceProof, NamespacedHash, NamespacedHashExt, NMT_CODEC, NMT_ID_SIZE,
NMT_MULTIHASH_CODE, NS_SIZE,
};
use crate::rsmt2d::AxisType;
use crate::{Error, ExtendedHeader, Result, Share};

type Cid = CidGeneric<NMT_ID_SIZE>;
Expand Down
2 changes: 1 addition & 1 deletion types/src/data_availability_header.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@ use tendermint_proto::Protobuf;
use crate::consts::data_availability_header::{
MAX_EXTENDED_SQUARE_WIDTH, MIN_EXTENDED_SQUARE_WIDTH,
};
use crate::extended_data_square::AxisType;
use crate::hash::Hash;
use crate::nmt::{NamespacedHash, NamespacedHashExt};
use crate::rsmt2d::AxisType;
use crate::{bail_validation, Error, Result, ValidateBasic, ValidationError};

#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
Expand Down
3 changes: 3 additions & 0 deletions types/src/error.rs
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,9 @@ pub enum Error {
#[error("Data square index out of range: {0}")]
EdsIndexOutOfRange(usize),

#[error("Invalid dimensions of EDS")]
EdsInvalidDimentions,

#[error("Invalid zero block height")]
ZeroBlockHeight,
}
Expand Down
136 changes: 136 additions & 0 deletions types/src/extended_data_square.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
use std::result::Result as StdResult;

use nmt_rs::NamespaceMerkleHasher;

use crate::namespaced_data::{NamespacedData, NamespacedDataId};
use crate::nmt::{Namespace, NamespacedSha2Hasher, Nmt};
use crate::{DataAvailabilityHeader, Share};
use crate::{Error, Result};

/// Represents either Column or Row of the Data Square.
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
#[repr(u8)]
pub enum AxisType {
Row = 0,
Col,
}

impl TryFrom<u8> for AxisType {
type Error = Error;

fn try_from(value: u8) -> StdResult<Self, Self::Error> {
match value {
0 => Ok(AxisType::Row),
1 => Ok(AxisType::Col),
n => Err(Error::InvalidAxis(n.into())),
}
}
}

/// Represents valid ExtendedDataSquare
pub struct ExtendedDataSquare {
shares: Vec<Share>,
square_len: usize,
}

impl ExtendedDataSquare {
pub fn new(shares: Vec<Share>) -> Result<Self> {
let square_len = f64::sqrt(shares.len() as f64) as usize;
if square_len * square_len != shares.len() {
return Err(Error::EdsInvalidDimentions);
}

Ok(Self { shares, square_len })
}

pub fn row(&self, index: usize, square_len: usize) -> Vec<Share> {
self.shares[index * square_len..(index + 1) * square_len].to_vec()
}

pub fn column(&self, mut index: usize, square_len: usize) -> Vec<Share> {
let mut r = Vec::with_capacity(square_len);
while index < self.shares.len() {
r.push(self.shares[index].clone());
index += square_len;
}
r
}

pub fn axis(&self, axis: AxisType, index: usize, square_len: usize) -> Vec<Share> {
match axis {
AxisType::Col => self.column(index, square_len),
AxisType::Row => self.row(index, square_len),
}
}

pub fn square_len(&self) -> usize {
self.square_len
}

pub fn get_namespaced_data(
&self,
namespace: Namespace,
dah: &DataAvailabilityHeader,
height: u64,
) -> Result<Vec<NamespacedData>> {
let square_len = dah.square_len();

let mut data = Vec::new();

for i in 0..square_len {
let row_root = dah.row_root(i).unwrap();
if !row_root.contains::<NamespacedSha2Hasher>(*namespace) {
continue;
}

let mut shares = Vec::with_capacity(square_len);

let mut tree = Nmt::with_hasher(NamespacedSha2Hasher::with_ignore_max_ns(true));
for s in &self.shares {
tree.push_leaf(s.data(), *s.namespace())
.map_err(Error::Nmt)?;
if s.namespace() == namespace {
shares.push(s.clone());
}
}

let proof = tree.get_namespace_proof(*namespace);
let namespaced_data_id = NamespacedDataId {
row_index: i.try_into().unwrap(),
namespace,
hash: dah.hash().as_ref().try_into().unwrap(),
block_height: height,
};

data.push(NamespacedData {
namespaced_data_id,
proof: proof.into(),
shares,
})
}

Ok(data)
}
}

#[cfg(test)]
mod tests {
use super::*;

#[test]
fn axis_type_serialization() {
assert_eq!(AxisType::Row as u8, 0);
assert_eq!(AxisType::Col as u8, 1);
}

#[test]
fn axis_type_deserialization() {
assert_eq!(AxisType::try_from(0).unwrap(), AxisType::Row);
assert_eq!(AxisType::try_from(1).unwrap(), AxisType::Col);

let axis_type_err = AxisType::try_from(2).unwrap_err();
assert!(matches!(axis_type_err, Error::InvalidAxis(2)));
let axis_type_err = AxisType::try_from(99).unwrap_err();
assert!(matches!(axis_type_err, Error::InvalidAxis(99)));
}
}
1 change: 1 addition & 0 deletions types/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ mod byzantine;
pub mod consts;
mod data_availability_header;
mod error;
mod extended_data_square;
mod extended_header;
pub mod fraud_proof;
pub mod hash;
Expand Down
2 changes: 1 addition & 1 deletion types/src/namespaced_data.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@ use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use tendermint_proto::Protobuf;

use crate::extended_data_square::AxisType;
use crate::nmt::{Namespace, NamespaceProof, NamespacedHashExt, HASH_SIZE, NS_SIZE};
use crate::rsmt2d::AxisType;
use crate::{DataAvailabilityHeader, Error, Result, Share};

const NAMESPACED_DATA_ID_SIZE: usize = NamespacedDataId::size();
Expand Down
6 changes: 3 additions & 3 deletions types/src/row.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@ use serde::{Deserialize, Serialize};
use tendermint::Hash;
use tendermint_proto::Protobuf;

use crate::extended_data_square::ExtendedDataSquare;
use crate::nmt::{NamespacedSha2Hasher, Nmt};
use crate::ExtendedDataSquare;
use crate::{Error, Result, Share};

const ROW_ID_SIZE: usize = RowId::size();
Expand Down Expand Up @@ -215,8 +215,8 @@ mod tests {
#[test]
fn index_calculation() {
let height = 100;
let shares = vec![vec![0; SHARE_SIZE]; 8 * 8];
let eds = ExtendedDataSquare::new(shares, "".to_string());
let shares = vec![Share::from_raw(&[0; SHARE_SIZE]).unwrap(); 8 * 8];
let eds = ExtendedDataSquare::new(shares).unwrap();

Row::new(1, &eds, height).unwrap();
Row::new(7, &eds, height).unwrap();
Expand Down
134 changes: 0 additions & 134 deletions types/src/rsmt2d.rs
Original file line number Diff line number Diff line change
@@ -1,142 +1,8 @@
use std::result::Result as StdResult;

use nmt_rs::NamespaceMerkleHasher;
use serde::{Deserialize, Serialize};

use crate::namespaced_data::{NamespacedData, NamespacedDataId};
use crate::nmt::{Namespace, NamespacedSha2Hasher, Nmt};
use crate::{DataAvailabilityHeader, Share};
use crate::{Error, Result};

/// Represents either Column or Row of the Data Square.
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
#[repr(u8)]
pub enum AxisType {
Row = 0,
Col,
}

impl TryFrom<u8> for AxisType {
type Error = Error;

fn try_from(value: u8) -> StdResult<Self, Self::Error> {
match value {
0 => Ok(AxisType::Row),
1 => Ok(AxisType::Col),
n => Err(Error::InvalidAxis(n.into())),
}
}
}

#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ExtendedDataSquare {
#[serde(with = "tendermint_proto::serializers::bytes::vec_base64string")]
pub data_square: Vec<Vec<u8>>,
pub codec: String,
}

impl ExtendedDataSquare {
pub fn new(shares: Vec<Vec<u8>>, codec: String) -> Self {
// TODO: validation
Self {
data_square: shares,
codec,
}
}

pub fn row(&self, index: usize, square_len: usize) -> Vec<Share> {
self.data_square[index * square_len..(index + 1) * square_len]
.iter()
.map(|s| Share::from_raw(s))
.collect::<Result<_, _>>()
.unwrap()
}

pub fn column(&self, mut index: usize, square_len: usize) -> Vec<Share> {
let mut r = Vec::with_capacity(square_len);
while index < self.data_square.len() {
r.push(Share::from_raw(&self.data_square[index]).unwrap());
index += square_len;
}
r
}

pub fn axis(&self, axis: AxisType, index: usize, square_len: usize) -> Vec<Share> {
match axis {
AxisType::Col => self.column(index, square_len),
AxisType::Row => self.row(index, square_len),
}
}

pub fn square_len(&self) -> usize {
f64::sqrt(self.data_square.len() as f64) as usize + 1
}

pub fn get_namespaced_data(
&self,
namespace: Namespace,
dah: &DataAvailabilityHeader,
height: u64,
) -> Result<Vec<NamespacedData>> {
let square_len = dah.square_len();

let mut data = Vec::new();

for i in 0..square_len {
let row_root = dah.row_root(i).unwrap();
if !row_root.contains::<NamespacedSha2Hasher>(*namespace) {
continue;
}

let mut shares = Vec::with_capacity(square_len);

let mut tree = Nmt::with_hasher(NamespacedSha2Hasher::with_ignore_max_ns(true));
for s in &self.data_square {
let s = Share::from_raw(s).unwrap(); // TODO: swap entire tree to shares?
tree.push_leaf(s.data(), *s.namespace())
.map_err(Error::Nmt)?;
if s.namespace() == namespace {
shares.push(s);
}
}

let proof = tree.get_namespace_proof(*namespace);
let namespaced_data_id = NamespacedDataId {
row_index: i.try_into().unwrap(),
namespace,
hash: dah.hash().as_ref().try_into().unwrap(),
block_height: height,
};

data.push(NamespacedData {
namespaced_data_id,
proof: proof.into(),
shares,
})
}

Ok(data)
}
}

#[cfg(test)]
mod tests {
use super::*;

#[test]
fn axis_type_serialization() {
assert_eq!(AxisType::Row as u8, 0);
assert_eq!(AxisType::Col as u8, 1);
}

#[test]
fn axis_type_deserialization() {
assert_eq!(AxisType::try_from(0).unwrap(), AxisType::Row);
assert_eq!(AxisType::try_from(1).unwrap(), AxisType::Col);

let axis_type_err = AxisType::try_from(2).unwrap_err();
assert!(matches!(axis_type_err, Error::InvalidAxis(2)));
let axis_type_err = AxisType::try_from(99).unwrap_err();
assert!(matches!(axis_type_err, Error::InvalidAxis(99)));
}
}
3 changes: 1 addition & 2 deletions types/src/sample.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,9 @@ use serde::{Deserialize, Serialize};
use tendermint::Hash;
use tendermint_proto::Protobuf;

use crate::extended_data_square::{AxisType, ExtendedDataSquare};
use crate::nmt::{NamespaceProof, NamespacedHash, NamespacedHashExt, NamespacedSha2Hasher, Nmt};
use crate::row::RowId;
use crate::rsmt2d::AxisType;
use crate::ExtendedDataSquare;
use crate::{Error, Result, Share};

const SAMPLE_ID_SIZE: usize = SampleId::size();
Expand Down

0 comments on commit fb4611c

Please sign in to comment.