diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d6c8ec..7fd7d1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased](https://github.com/near/near-lake-framework/compare/v0.7.2...HEAD) - Simpler start boilerplate, simpler structures to deal with! +- Upgrade to latest AWS SDK version (*since beta.3*) ### Breaking changes diff --git a/Cargo.toml b/Cargo.toml index eaab7ee..704c8ff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ members = [ # cargo-workspaces [workspace.package] -version = "0.8.0-beta.2" +version = "0.8.0-beta.3" license = "MIT OR Apache-2.0" repository = "https://github.com/near/near-lake-framework-rs" description = "Library to connect to the NEAR Lake S3 and stream the data" diff --git a/lake-framework/Cargo.toml b/lake-framework/Cargo.toml index cc70fdd..75dc44d 100644 --- a/lake-framework/Cargo.toml +++ b/lake-framework/Cargo.toml @@ -7,10 +7,10 @@ license.workspace = true repository.workspace = true [dependencies] -aws-config = "0.53.0" -aws-types = "0.53.0" -aws-credential-types = "0.53.0" -aws-sdk-s3 = "0.23.0" +aws-config = { version = "1.0.0", features = ["behavior-version-latest"] } +aws-types = "1.0.0" +aws-credential-types = "1.0.0" +aws-sdk-s3 = "0.39.0" async-stream = "0.3.3" async-trait = "0.1.64" derive_builder = "0.11.2" @@ -26,7 +26,8 @@ near-lake-primitives = { path = "../lake-primitives", version = "0.8.0-beta.2" } near-lake-context-derive = { path = "../lake-context-derive", version = "0.8.0-beta.2" } [dev-dependencies] -aws-smithy-http = "0.53.0" +aws-smithy-http = "0.60.0" +aws-smithy-types = "1.0.0" # use by examples anyhow = "1.0.51" diff --git a/lake-framework/README.md b/lake-framework/README.md index 3ae153e..489f62c 100644 --- a/lake-framework/README.md +++ b/lake-framework/README.md @@ -226,19 +226,21 @@ $ mkdir -p /data/near-lake-custom && minio server /data use near_lake_framework::LakeBuilder; # #[tokio::main] -# async fn main() { -let aws_config = aws_config::from_env().load().await; -let mut s3_conf = aws_sdk_s3::config::Builder::from(&aws_config) - .endpoint_url("http://0.0.0.0:9000") - .build(); +# async fn main() -> anyhow::Result<()> { + let aws_config = aws_config::from_env().load().await; + let s3_config = aws_sdk_s3::config::Builder::from(&aws_types::SdkConfig::from(aws_config)) + .endpoint_url("http://0.0.0.0:9000") + .build(); + + LakeBuilder::default() + .s3_bucket_name("near-lake-custom") + .s3_region_name("eu-central-1") + .start_block_height(0) + .s3_config(s3_config) + .build() + .expect("Failed to build Lake"); -let lake = LakeBuilder::default() - .s3_config(s3_conf) - .s3_bucket_name("near-lake-data-custom") - .s3_region_name("eu-central-1") - .start_block_height(1) - .build() - .expect("Failed to build LakeConfig"); +# Ok(()) # } ``` diff --git a/lake-framework/src/s3_fetchers.rs b/lake-framework/src/s3_fetchers.rs index 14bdfc6..7b2607d 100644 --- a/lake-framework/src/s3_fetchers.rs +++ b/lake-framework/src/s3_fetchers.rs @@ -1,7 +1,8 @@ use async_trait::async_trait; use std::str::FromStr; -use aws_sdk_s3::output::{GetObjectOutput, ListObjectsV2Output}; +use aws_sdk_s3::operation::get_object::GetObjectOutput; +use aws_sdk_s3::operation::list_objects_v2::ListObjectsV2Output; #[async_trait] pub trait S3Client { @@ -9,7 +10,10 @@ pub trait S3Client { &self, bucket: &str, prefix: &str, - ) -> Result>; + ) -> Result< + GetObjectOutput, + aws_sdk_s3::error::SdkError, + >; async fn list_objects( &self, @@ -17,7 +21,7 @@ pub trait S3Client { start_after: &str, ) -> Result< ListObjectsV2Output, - aws_sdk_s3::types::SdkError, + aws_sdk_s3::error::SdkError, >; } @@ -38,14 +42,16 @@ impl S3Client for LakeS3Client { &self, bucket: &str, prefix: &str, - ) -> Result> - { + ) -> Result< + GetObjectOutput, + aws_sdk_s3::error::SdkError, + > { Ok(self .s3 .get_object() .bucket(bucket) .key(prefix) - .request_payer(aws_sdk_s3::model::RequestPayer::Requester) + .request_payer(aws_sdk_s3::types::RequestPayer::Requester) .send() .await?) } @@ -56,7 +62,7 @@ impl S3Client for LakeS3Client { start_after: &str, ) -> Result< ListObjectsV2Output, - aws_sdk_s3::types::SdkError, + aws_sdk_s3::error::SdkError, > { Ok(self .s3 @@ -64,7 +70,7 @@ impl S3Client for LakeS3Client { .max_keys(1000) // 1000 is the default and max value for this parameter .delimiter("/".to_string()) .start_after(start_after) - .request_payer(aws_sdk_s3::model::RequestPayer::Requester) + .request_payer(aws_sdk_s3::types::RequestPayer::Requester) .bucket(bucket) .send() .await?) @@ -218,10 +224,11 @@ mod test { use async_trait::async_trait; - use aws_sdk_s3::output::{get_object_output, list_objects_v2_output}; - use aws_sdk_s3::types::ByteStream; + use aws_sdk_s3::operation::get_object::builders::GetObjectOutputBuilder; + use aws_sdk_s3::operation::list_objects_v2::builders::ListObjectsV2OutputBuilder; + use aws_sdk_s3::primitives::ByteStream; - use aws_smithy_http::body::SdkBody; + use aws_smithy_types::body::SdkBody; #[derive(Clone, Debug)] pub struct LakeS3Client {} @@ -232,12 +239,14 @@ mod test { &self, _bucket: &str, prefix: &str, - ) -> Result> - { + ) -> Result< + GetObjectOutput, + aws_sdk_s3::error::SdkError, + > { let path = format!("{}/blocks/{}", env!("CARGO_MANIFEST_DIR"), prefix); let file_bytes = tokio::fs::read(path).await.unwrap(); let stream = ByteStream::new(SdkBody::from(file_bytes)); - Ok(get_object_output::Builder::default().body(stream).build()) + Ok(GetObjectOutputBuilder::default().body(stream).build()) } async fn list_objects( @@ -246,9 +255,9 @@ mod test { _start_after: &str, ) -> Result< ListObjectsV2Output, - aws_sdk_s3::types::SdkError, + aws_sdk_s3::error::SdkError, > { - Ok(list_objects_v2_output::Builder::default().build()) + Ok(ListObjectsV2OutputBuilder::default().build()) } } diff --git a/lake-framework/src/types.rs b/lake-framework/src/types.rs index a97feb0..51352c9 100644 --- a/lake-framework/src/types.rs +++ b/lake-framework/src/types.rs @@ -128,27 +128,28 @@ pub enum LakeError { #[from] error_message: serde_json::Error, }, - #[error("AWS S3 error")] + #[error("AWS S3 error: {error}")] AwsGetObjectError { #[from] - error: aws_sdk_s3::types::SdkError, + error: aws_sdk_s3::error::SdkError, }, - #[error("AWS S3 error")] + #[error("AWS S3 error: {error}")] AwsLisObjectsV2Error { #[from] - error: aws_sdk_s3::types::SdkError, + error: + aws_sdk_s3::error::SdkError, }, - #[error("Failed to convert integer")] + #[error("Failed to convert integer: {error}")] IntConversionError { #[from] error: std::num::TryFromIntError, }, - #[error("Join error")] + #[error("Join error: {error}")] JoinError { #[from] error: tokio::task::JoinError, }, - #[error("Failed to start runtime")] + #[error("Failed to start runtime: {error}")] RuntimeStartError { #[from] error: std::io::Error,