elliptic-curve: make ExpandMsg fallible

As noted on #865 there are some potential overflow problems constructing
the messages used for expansion, particularly around input lengths.

This commit changes `ExpandMsg::expand_message`, and all of its
transitive callers, to be fallible in order to handle these cases.

elliptic-curve/src/hash2curve/group_digest.rs

@@ -1,5 +1,8 @@
 use super::MapToCurve;
-use crate::hash2field::{hash_to_field, ExpandMsg, FromOkm};
+use crate::{
+    hash2field::{hash_to_field, ExpandMsg, FromOkm},
+    Result,
+};
 use group::cofactor::CofactorGroup;
 
 /// Adds hashing arbitrary byte sequences to a valid group element
@@ -34,9 +37,9 @@ pub trait GroupDigest {
     /// let pt = ProjectivePoint::hash_from_bytes::<hash2field::ExpandMsgXof<sha3::Shake256>>(b"test data", b"CURVE_XOF:SHAKE-256_SSWU_RO_");
     /// ```
     ///
-    fn hash_from_bytes<X: ExpandMsg>(msg: &[u8], dst: &'static [u8]) -> Self::Output {
+    fn hash_from_bytes<X: ExpandMsg>(msg: &[u8], dst: &'static [u8]) -> Result<Self::Output> {
         let mut u = [Self::FieldElement::default(), Self::FieldElement::default()];
-        hash_to_field::<X, _>(msg, dst, &mut u);
+        hash_to_field::<X, _>(msg, dst, &mut u)?;
         let q0 = Self::Output::map_to_curve(u[0]);
         let q1 = Self::Output::map_to_curve(u[1]);
         // Ideally we could add and then clear cofactor once
@@ -49,7 +52,7 @@ pub trait GroupDigest {
         // isogenies are different with curves like k256 and bls12-381.
         // This problem doesn't manifest for curves with no isogeny like p256.
         // For k256 and p256 clear_cofactor doesn't do anything anyway so it will be a no-op.
-        q0.clear_cofactor() + q1.clear_cofactor()
+        Ok(q0.clear_cofactor() + q1.clear_cofactor())
     }
 
     /// Computes the encode to curve routine according to
@@ -60,10 +63,10 @@ pub trait GroupDigest {
     /// uniformly random in G: the set of possible outputs of
     /// encode_to_curve is only a fraction of the points in G, and some
     /// points in this set are more likely to be output than others.
-    fn encode_from_bytes<X: ExpandMsg>(msg: &[u8], dst: &'static [u8]) -> Self::Output {
+    fn encode_from_bytes<X: ExpandMsg>(msg: &[u8], dst: &'static [u8]) -> Result<Self::Output> {
         let mut u = [Self::FieldElement::default()];
-        hash_to_field::<X, _>(msg, dst, &mut u);
+        hash_to_field::<X, _>(msg, dst, &mut u)?;
         let q0 = Self::Output::map_to_curve(u[0]);
-        q0.clear_cofactor()
+        Ok(q0.clear_cofactor())
     }
 }

elliptic-curve/src/hash2field.rs

@@ -5,6 +5,8 @@ mod expand_msg_xof;
 pub use expand_msg::*;
 pub use expand_msg_xmd::*;
 pub use expand_msg_xof::*;
+
+use crate::Result;
 use generic_array::{typenum::Unsigned, ArrayLength, GenericArray};
 
 /// The trait for helping to convert to a scalar
@@ -18,16 +20,17 @@ pub trait FromOkm {
 
 /// Convert an arbitrary byte sequence according to
 /// <https://tools.ietf.org/html/draft-irtf-cfrg-hash-to-curve-11#section-5.3>
-pub fn hash_to_field<E, T>(data: &[u8], domain: &'static [u8], out: &mut [T])
+pub fn hash_to_field<E, T>(data: &[u8], domain: &'static [u8], out: &mut [T]) -> Result<()>
 where
     E: ExpandMsg,
     T: FromOkm + Default,
 {
     let len_in_bytes = T::Length::to_usize() * out.len();
     let mut tmp = GenericArray::<u8, <T as FromOkm>::Length>::default();
-    let mut expander = E::expand_message(data, domain, len_in_bytes);
+    let mut expander = E::expand_message(data, domain, len_in_bytes)?;
     for o in out.iter_mut() {
         expander.fill_bytes(&mut tmp);
         *o = T::from_okm(&tmp);
     }
+    Ok(())
 }

elliptic-curve/src/hash2field/expand_msg.rs

@@ -1,3 +1,4 @@
+use crate::Result;
 use digest::{Digest, ExtendableOutputDirty, Update, XofReader};
 use generic_array::{ArrayLength, GenericArray};
 
@@ -7,11 +8,11 @@ const OVERSIZE_DST_SALT: &[u8] = b"H2C-OVERSIZE-DST-";
 const MAX_DST_LEN: usize = 255;
 
 /// Trait for types implementing expand_message interface for hash_to_field
-pub trait ExpandMsg {
+pub trait ExpandMsg: Sized {
     /// Expands `msg` to the required number of bytes
     /// Returns an expander that can be used to call `read` until enough
     /// bytes have been consumed
-    fn expand_message(msg: &[u8], dst: &'static [u8], len_in_bytes: usize) -> Self;
+    fn expand_message(msg: &[u8], dst: &'static [u8], len_in_bytes: usize) -> Result<Self>;
 
     /// Fill the array with the expanded bytes
     fn fill_bytes(&mut self, okm: &mut [u8]);

elliptic-curve/src/hash2field/expand_msg_xmd.rs

@@ -1,4 +1,5 @@
 use super::{Domain, ExpandMsg};
+use crate::{Error, Result};
 use digest::{
     generic_array::{
         typenum::{IsLess, IsLessOrEqual, Unsigned, U256},
@@ -56,15 +57,20 @@ where
 impl<HashT> ExpandMsg for ExpandMsgXmd<HashT>
 where
     HashT: Digest + BlockInput,
-    HashT::OutputSize: IsLess<U256>,
-    HashT::OutputSize: IsLessOrEqual<HashT::BlockSize>,
+    HashT::OutputSize: IsLess<U256> + IsLessOrEqual<HashT::BlockSize>,
 {
-    fn expand_message(msg: &[u8], dst: &'static [u8], len_in_bytes: usize) -> Self {
+    fn expand_message(msg: &[u8], dst: &'static [u8], len_in_bytes: usize) -> Result<Self> {
+        if len_in_bytes > 0xFFFF {
+            return Err(Error);
+        }
+
         let b_in_bytes = HashT::OutputSize::to_usize();
         let ell = (len_in_bytes + b_in_bytes - 1) / b_in_bytes;
-        // if ell > 255 {
-        //     panic!("ell was too big in expand_message_xmd");
-        // }
+
+        if ell > 255 {
+            return Err(Error);
+        }
+
         let domain = Domain::xmd::<HashT>(dst);
         let b_0 = HashT::new()
             .chain(GenericArray::<u8, HashT::BlockSize>::default())
@@ -81,14 +87,14 @@ where
             .chain([domain.len() as u8])
             .finalize();
 
-        Self {
+        Ok(Self {
             b_0,
             b_vals,
             domain,
             index: 1,
             offset: 0,
             ell,
-        }
+        })
     }
 
     fn fill_bytes(&mut self, okm: &mut [u8]) {

elliptic-curve/src/hash2field/expand_msg_xof.rs

@@ -1,5 +1,5 @@
 use super::ExpandMsg;
-use crate::hash2field::Domain;
+use crate::{hash2field::Domain, Result};
 use digest::{ExtendableOutput, ExtendableOutputDirty, Update, XofReader};
 use generic_array::typenum::U32;
 
@@ -16,15 +16,15 @@ impl<HashT> ExpandMsg for ExpandMsgXof<HashT>
 where
     HashT: Default + ExtendableOutput + ExtendableOutputDirty + Update,
 {
-    fn expand_message(msg: &[u8], dst: &'static [u8], len_in_bytes: usize) -> Self {
+    fn expand_message(msg: &[u8], dst: &'static [u8], len_in_bytes: usize) -> Result<Self> {
         let domain = Domain::<U32>::xof::<HashT>(dst);
         let reader = HashT::default()
             .chain(msg)
             .chain([(len_in_bytes >> 8) as u8, len_in_bytes as u8])
             .chain(domain.data())
             .chain([domain.len() as u8])
             .finalize_xof();
-        Self { reader }
+        Ok(Self { reader })
     }
 
     fn fill_bytes(&mut self, okm: &mut [u8]) {