Add support for unsigned payloads in aws

object_store/src/aws/client.rs

@@ -204,6 +204,7 @@ pub struct S3Config {
     pub credentials: Box<dyn CredentialProvider>,
     pub retry_config: RetryConfig,
     pub client_options: ClientOptions,
+    pub sign_payload: bool,
 }
 
 impl S3Config {
@@ -256,7 +257,12 @@ impl S3Client {
         }
 
         let response = builder
-            .with_aws_sigv4(credential.as_ref(), &self.config.region, "s3")
+            .with_aws_sigv4(
+                credential.as_ref(),
+                &self.config.region,
+                "s3",
+                self.config.sign_payload,
+            )
             .send_retry(&self.config.retry_config)
             .await
             .context(GetRequestSnafu {
@@ -287,7 +293,12 @@ impl S3Client {
 
         let response = builder
             .query(query)
-            .with_aws_sigv4(credential.as_ref(), &self.config.region, "s3")
+            .with_aws_sigv4(
+                credential.as_ref(),
+                &self.config.region,
+                "s3",
+                self.config.sign_payload,
+            )
             .send_retry(&self.config.retry_config)
             .await
             .context(PutRequestSnafu {
@@ -309,7 +320,12 @@ impl S3Client {
         self.client
             .request(Method::DELETE, url)
             .query(query)
-            .with_aws_sigv4(credential.as_ref(), &self.config.region, "s3")
+            .with_aws_sigv4(
+                credential.as_ref(),
+                &self.config.region,
+                "s3",
+                self.config.sign_payload,
+            )
             .send_retry(&self.config.retry_config)
             .await
             .context(DeleteRequestSnafu {
@@ -328,7 +344,12 @@ impl S3Client {
         self.client
             .request(Method::PUT, url)
             .header("x-amz-copy-source", source)
-            .with_aws_sigv4(credential.as_ref(), &self.config.region, "s3")
+            .with_aws_sigv4(
+                credential.as_ref(),
+                &self.config.region,
+                "s3",
+                self.config.sign_payload,
+            )
             .send_retry(&self.config.retry_config)
             .await
             .context(CopyRequestSnafu {
@@ -369,7 +390,12 @@ impl S3Client {
             .client
             .request(Method::GET, &url)
             .query(&query)
-            .with_aws_sigv4(credential.as_ref(), &self.config.region, "s3")
+            .with_aws_sigv4(
+                credential.as_ref(),
+                &self.config.region,
+                "s3",
+                self.config.sign_payload,
+            )
             .send_retry(&self.config.retry_config)
             .await
             .context(ListRequestSnafu)?
@@ -407,7 +433,12 @@ impl S3Client {
         let response = self
             .client
             .request(Method::POST, url)
-            .with_aws_sigv4(credential.as_ref(), &self.config.region, "s3")
+            .with_aws_sigv4(
+                credential.as_ref(),
+                &self.config.region,
+                "s3",
+                self.config.sign_payload,
+            )
             .send_retry(&self.config.retry_config)
             .await
             .context(CreateMultipartRequestSnafu)?
@@ -446,7 +477,12 @@ impl S3Client {
             .request(Method::POST, url)
             .query(&[("uploadId", upload_id)])
             .body(body)
-            .with_aws_sigv4(credential.as_ref(), &self.config.region, "s3")
+            .with_aws_sigv4(
+                credential.as_ref(),
+                &self.config.region,
+                "s3",
+                self.config.sign_payload,
+            )
             .send_retry(&self.config.retry_config)
             .await
             .context(CompleteMultipartRequestSnafu)?;

object_store/src/aws/credential.rs

@@ -39,6 +39,7 @@ type StdError = Box<dyn std::error::Error + Send + Sync>;
 /// SHA256 hash of empty string
 static EMPTY_SHA256_HASH: &str =
     "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
+static UNSIGNED_PAYLOAD_LITERAL: &str = "UNSIGNED-PAYLOAD";
 
 #[derive(Debug)]
 pub struct AwsCredential {
@@ -72,6 +73,7 @@ struct RequestSigner<'a> {
     credential: &'a AwsCredential,
     service: &'a str,
     region: &'a str,
+    sign_payload: bool,
 }
 
 const DATE_HEADER: &str = "x-amz-date";
@@ -98,9 +100,13 @@ impl<'a> RequestSigner<'a> {
         let date_val = HeaderValue::from_str(&date_str).unwrap();
         request.headers_mut().insert(DATE_HEADER, date_val);
 
-        let digest = match request.body() {
-            None => EMPTY_SHA256_HASH.to_string(),
-            Some(body) => hex_digest(body.as_bytes().unwrap()),
+        let digest = if self.sign_payload {
+            match request.body() {
+                None => EMPTY_SHA256_HASH.to_string(),
+                Some(body) => hex_digest(body.as_bytes().unwrap()),
+            }
+        } else {
+            UNSIGNED_PAYLOAD_LITERAL.to_string()
         };
 
         let header_digest = HeaderValue::from_str(&digest).unwrap();
@@ -158,6 +164,7 @@ pub trait CredentialExt {
         credential: &AwsCredential,
         region: &str,
         service: &str,
+        sign_payload: bool,
     ) -> Self;
 }
 
@@ -167,6 +174,7 @@ impl CredentialExt for RequestBuilder {
         credential: &AwsCredential,
         region: &str,
         service: &str,
+        sign_payload: bool,
     ) -> Self {
         // Hack around lack of access to underlying request
         // https://github.com/seanmonstar/reqwest/issues/1212
@@ -182,6 +190,7 @@ impl CredentialExt for RequestBuilder {
             credential,
             service,
             region,
+            sign_payload,
         };
 
         signer.sign(&mut request);
@@ -591,7 +600,7 @@ mod tests {
 
     // Test generated using https://docs.aws.amazon.com/general/latest/gr/sigv4-signed-request-examples.html
     #[test]
-    fn test_sign() {
+    fn test_sign_with_signed_payload() {
         let client = Client::new();
 
         // Test credentials from https://docs.aws.amazon.com/AmazonS3/latest/userguide/RESTAuthentication.html
@@ -621,12 +630,51 @@ mod tests {
             credential: &credential,
             service: "ec2",
             region: "us-east-1",
+            sign_payload: true,
         };
 
         signer.sign(&mut request);
         assert_eq!(request.headers().get(AUTH_HEADER).unwrap(), "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20220806/us-east-1/ec2/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=a3c787a7ed37f7fdfbfd2d7056a3d7c9d85e6d52a2bfbec73793c0be6e7862d4")
     }
 
+    #[test]
+    fn test_sign_with_unsigned_payload() {
+        let client = Client::new();
+
+        // Test credentials from https://docs.aws.amazon.com/AmazonS3/latest/userguide/RESTAuthentication.html
+        let credential = AwsCredential {
+            key_id: "AKIAIOSFODNN7EXAMPLE".to_string(),
+            secret_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY".to_string(),
+            token: None,
+        };
+
+        // method = 'GET'
+        // service = 'ec2'
+        // host = 'ec2.amazonaws.com'
+        // region = 'us-east-1'
+        // endpoint = 'https://ec2.amazonaws.com'
+        // request_parameters = ''
+        let date = DateTime::parse_from_rfc3339("2022-08-06T18:01:34Z")
+            .unwrap()
+            .with_timezone(&Utc);
+
+        let mut request = client
+            .request(Method::GET, "https://ec2.amazon.com/")
+            .build()
+            .unwrap();
+
+        let signer = RequestSigner {
+            date,
+            credential: &credential,
+            service: "ec2",
+            region: "us-east-1",
+            sign_payload: false,
+        };
+
+        signer.sign(&mut request);
+        assert_eq!(request.headers().get(AUTH_HEADER).unwrap(), "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20220806/us-east-1/ec2/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=653c3d8ea261fd826207df58bc2bb69fbb5003e9eb3c0ef06e4a51f2a81d8699")
+    }
+
     #[test]
     fn test_sign_port() {
         let client = Client::new();
@@ -657,6 +705,7 @@ mod tests {
             credential: &credential,
             service: "s3",
             region: "us-east-1",
+            sign_payload: true,
         };
 
         signer.sign(&mut request);

object_store/src/aws/mod.rs

@@ -385,6 +385,7 @@ pub struct AmazonS3Builder {
     retry_config: RetryConfig,
     imdsv1_fallback: bool,
     virtual_hosted_style_request: bool,
+    unsigned_payload: bool,
     metadata_endpoint: Option<String>,
     profile: Option<String>,
     client_options: ClientOptions,
@@ -504,6 +505,15 @@ pub enum AmazonS3ConfigKey {
     /// - `virtual_hosted_style_request`
     VirtualHostedStyleRequest,
 
+    /// Avoid computing payload checksum when calculating signature.
+    ///
+    /// See [`AmazonS3Builder::with_unsigned_payload`] for details.
+    ///
+    /// Supported keys:
+    /// - `aws_unsigned_payload`
+    /// - `unsigned_payload`
+    UnsignedPayload,
+
     /// Set the instance metadata endpoint
     ///
     /// See [`AmazonS3Builder::with_metadata_endpoint`] for details.
@@ -535,6 +545,7 @@ impl AsRef<str> for AmazonS3ConfigKey {
             Self::DefaultRegion => "aws_default_region",
             Self::MetadataEndpoint => "aws_metadata_endpoint",
             Self::Profile => "aws_profile",
+            Self::UnsignedPayload => "aws_unsigned_payload",
         }
     }
 }
@@ -563,6 +574,7 @@ impl FromStr for AmazonS3ConfigKey {
             "aws_profile" | "profile" => Ok(Self::Profile),
             "aws_imdsv1_fallback" | "imdsv1_fallback" => Ok(Self::ImdsV1Fallback),
             "aws_metadata_endpoint" | "metadata_endpoint" => Ok(Self::MetadataEndpoint),
+            "aws_unsigned_payload" | "unsigned_payload" => Ok(Self::UnsignedPayload),
             _ => Err(Error::UnknownConfigurationKey { key: s.into() }.into()),
         }
     }
@@ -679,6 +691,9 @@ impl AmazonS3Builder {
                 self.metadata_endpoint = Some(value.into())
             }
             AmazonS3ConfigKey::Profile => self.profile = Some(value.into()),
+            AmazonS3ConfigKey::UnsignedPayload => {
+                self.unsigned_payload = str_is_truthy(&value.into())
+            }
         };
         Ok(self)
     }
@@ -822,6 +837,15 @@ impl AmazonS3Builder {
         self
     }
 
+    /// Sets if unsigned payload option has to be used.
+    /// See [unsigned payload option](https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html)
+    /// * false (default): Signed payload option is used, where the checksum for the request body is computed and included when constructing a canonical request.
+    /// * true: Unsigned payload option is used. `UNSIGNED-PAYLOAD` literal is included when constructing a canonical request,
+    pub fn with_unsigned_payload(mut self, unsigned_payload: bool) -> Self {
+        self.unsigned_payload = unsigned_payload;
+        self
+    }
+
     /// Set the [instance metadata endpoint](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html),
     /// used primarily within AWS EC2.
     ///
@@ -967,6 +991,7 @@ impl AmazonS3Builder {
             credentials,
             retry_config: self.retry_config,
             client_options: self.client_options,
+            sign_payload: !self.unsigned_payload,
         };
 
         let client = Arc::new(S3Client::new(config)?);
@@ -1125,6 +1150,7 @@ mod tests {
             "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI",
             &container_creds_relative_uri,
         );
+        env::set_var("AWS_UNSIGNED_PAYLOAD", "true");
 
         let builder = AmazonS3Builder::from_env();
         assert_eq!(builder.access_key_id.unwrap(), aws_access_key_id.as_str());
@@ -1136,9 +1162,9 @@ mod tests {
 
         assert_eq!(builder.endpoint.unwrap(), aws_endpoint);
         assert_eq!(builder.token.unwrap(), aws_session_token);
-
         let metadata_uri = format!("{METADATA_ENDPOINT}{container_creds_relative_uri}");
         assert_eq!(builder.metadata_endpoint.unwrap(), metadata_uri);
+        assert!(builder.unsigned_payload);
     }
 
     #[test]
@@ -1154,6 +1180,7 @@ mod tests {
             ("aws_default_region", aws_default_region.clone()),
             ("aws_endpoint", aws_endpoint.clone()),
             ("aws_session_token", aws_session_token.clone()),
+            ("aws_unsigned_payload", "true".to_string()),
         ]);
 
         let builder = AmazonS3Builder::new()
@@ -1166,6 +1193,7 @@ mod tests {
         assert_eq!(builder.region.unwrap(), aws_default_region);
         assert_eq!(builder.endpoint.unwrap(), aws_endpoint);
         assert_eq!(builder.token.unwrap(), aws_session_token);
+        assert!(builder.unsigned_payload);
     }
 
     #[test]
@@ -1181,6 +1209,7 @@ mod tests {
             (AmazonS3ConfigKey::DefaultRegion, aws_default_region.clone()),
             (AmazonS3ConfigKey::Endpoint, aws_endpoint.clone()),
             (AmazonS3ConfigKey::Token, aws_session_token.clone()),
+            (AmazonS3ConfigKey::UnsignedPayload, "true".to_string()),
         ]);
 
         let builder = AmazonS3Builder::new()
@@ -1193,6 +1222,7 @@ mod tests {
         assert_eq!(builder.region.unwrap(), aws_default_region);
         assert_eq!(builder.endpoint.unwrap(), aws_endpoint);
         assert_eq!(builder.token.unwrap(), aws_session_token);
+        assert!(builder.unsigned_payload);
     }
 
     #[test]
@@ -1220,6 +1250,12 @@ mod tests {
         list_with_delimiter(&integration).await;
         rename_and_copy(&integration).await;
         stream_get(&integration).await;
+
+        // run integration test with unsigned payload enabled
+        let config = maybe_skip_integration!().with_unsigned_payload(true);
+        let is_local = matches!(&config.endpoint, Some(e) if e.starts_with("http://"));
+        let integration = config.build().unwrap();
+        put_get_delete_list_opts(&integration, is_local).await;
     }
 
     #[tokio::test]