Add Huffman encoding for HTTP2 headers. Resolves #31308

Author: Cory Nelson

src/Common/tests/System/Net/Http/GenericLoopbackServer.cs

@@ -85,14 +85,16 @@ public struct HttpHeaderData
     {
         public string Name { get; }
         public string Value { get; }
-        public bool HuffmanEncoded { get; }
+        public bool HuffmanEncodedName { get; }
+        public bool HuffmanEncodedValue { get; }
         public byte[] Raw { get; }
 
-        public HttpHeaderData(string name, string value, bool huffmanEncoded = false, byte[] raw = null)
+        public HttpHeaderData(string name, string value, bool huffmanEncodedName = false, bool huffmanEncodedValue = false, byte[] raw = null)
         {
             Name = name;
             Value = value;
-            HuffmanEncoded = huffmanEncoded;
+            HuffmanEncodedName = huffmanEncodedName;
+            HuffmanEncodedValue = huffmanEncodedValue;
             Raw = raw;
         }
 

src/Common/tests/System/Net/Http/Http2LoopbackConnection.cs

@@ -336,21 +336,26 @@ private static (int bytesConsumed, int value) DecodeInteger(ReadOnlySpan<byte> h
             return (2, prefixMask + b);
         }
 
-        private static (int bytesConsumed, string value) DecodeString(ReadOnlySpan<byte> headerBlock)
+        private static (int bytesConsumed, string value, bool huffmanEncoded) DecodeString(ReadOnlySpan<byte> headerBlock)
         {
             (int bytesConsumed, int stringLength) = DecodeInteger(headerBlock, 0b01111111);
-            if ((headerBlock[0] & 0b10000000) != 0)
+            bool isHuffmanCoded = (headerBlock[0] & 0b10000000) != 0;
+
+            headerBlock = headerBlock.Slice(bytesConsumed, stringLength);
+
+            if (isHuffmanCoded)
             {
                 // Huffman encoded
                 byte[] buffer = new byte[stringLength * 2];
-                int bytesDecoded = HuffmanDecoder.Decode(headerBlock.Slice(bytesConsumed, stringLength), buffer);
+                int bytesDecoded = HuffmanDecoder.Decode(headerBlock, buffer);
                 string value = Encoding.ASCII.GetString(buffer, 0, bytesDecoded);
-                return (bytesConsumed + stringLength, value);
+
+                return (bytesConsumed + stringLength, value, true);
             }
             else
             {
-                string value = Encoding.ASCII.GetString(headerBlock.Slice(bytesConsumed, stringLength));
-                return (bytesConsumed + stringLength, value);
+                string value = Encoding.ASCII.GetString(headerBlock);
+                return (bytesConsumed + stringLength, value, false);
             }
         }
 
@@ -432,9 +437,11 @@ private static (int bytesConsumed, HttpHeaderData headerData) DecodeLiteralHeade
             i += bytesConsumed;
 
             string name;
+            bool nameCompressed = false;
+
             if (index == 0)
             {
-                (bytesConsumed, name) = DecodeString(headerBlock.Slice(i));
+                (bytesConsumed, name, nameCompressed) = DecodeString(headerBlock.Slice(i));
                 i += bytesConsumed;
             }
             else
@@ -443,18 +450,19 @@ private static (int bytesConsumed, HttpHeaderData headerData) DecodeLiteralHeade
             }
 
             string value;
-            (bytesConsumed, value) = DecodeString(headerBlock.Slice(i));
+            bool valueCompressed;
+            (bytesConsumed, value, valueCompressed) = DecodeString(headerBlock.Slice(i));
             i += bytesConsumed;
 
-            return (i, new HttpHeaderData(name, value));
+            return (i, new HttpHeaderData(name, value, huffmanEncodedName: nameCompressed, huffmanEncodedValue: valueCompressed));
         }
 
         private static (int bytesConsumed, HttpHeaderData headerData) DecodeHeader(ReadOnlySpan<byte> headerBlock)
         {
             int i = 0;
 
             byte b = headerBlock[0];
-            if ((b & 0b10000000) != 0)
+            if ((b & 0b10000000) == 0b10000000)
             {
                 // Indexed header
                 (int bytesConsumed, int index) = DecodeInteger(headerBlock, 0b01111111);
@@ -474,7 +482,7 @@ private static (int bytesConsumed, HttpHeaderData headerData) DecodeHeader(ReadO
             }
             else
             {
-                // Literal, never indexed
+                // Literal, never indexed OR literal, without indexing.
                 return DecodeLiteralHeader(headerBlock, 0b00001111);
             }
         }
@@ -537,13 +545,14 @@ public async Task<byte[]> ReadBodyAsync()
             requestData.RequestId = streamId;
 
             Memory<byte> data = headersFrame.Data;
+
             int i = 0;
             while (i < data.Length)
             {
                 (int bytesConsumed, HttpHeaderData headerData) = DecodeHeader(data.Span.Slice(i));
 
                 byte[] headerRaw = data.Span.Slice(i, bytesConsumed).ToArray();
-                headerData = new HttpHeaderData(headerData.Name, headerData.Value, headerData.HuffmanEncoded, headerRaw);
+                headerData = new HttpHeaderData(headerData.Name, headerData.Value, headerData.HuffmanEncodedName, headerData.HuffmanEncodedValue, headerRaw);
 
                 requestData.Headers.Add(headerData);
                 i += bytesConsumed;
@@ -614,7 +623,19 @@ public async Task SendResponseHeadersAsync(int streamId, bool endStream = true,
             {
                 foreach (HttpHeaderData headerData in headers)
                 {
-                    bytesGenerated += HPackEncoder.EncodeHeader(headerData.Name, headerData.Value, headerData.HuffmanEncoded ? HPackFlags.HuffmanEncode : HPackFlags.None, headerBlock.AsSpan(bytesGenerated));
+                    HPackFlags hpackFlags = HPackFlags.None;
+
+                    if (headerData.HuffmanEncodedName)
+                    {
+                        hpackFlags |= HPackFlags.HuffmanEncodeName;
+                    }
+
+                    if (headerData.HuffmanEncodedValue)
+                    {
+                        hpackFlags |= HPackFlags.HuffmanEncodeValue;
+                    }
+
+                    bytesGenerated += HPackEncoder.EncodeHeader(headerData.Name, headerData.Value, hpackFlags, headerBlock.AsSpan(bytesGenerated));
                 }
             }
 

src/Common/tests/System/Net/Http/HuffmanEncoder.cs

@@ -8,7 +8,7 @@ public static class HuffmanEncoder
     {
         // Stolen from product code
         // See https://github.com/dotnet/corefx/blob/ae7b3970bb2c8d76004ea397083ce7ceb1238133/src/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/HPack/Huffman.cs#L12
-        private static readonly (uint code, int bitLength)[] s_encodingTable = new (uint code, int bitLength)[]
+        public static readonly (uint code, int bitLength)[] s_encodingTable = new (uint code, int bitLength)[]
         {
             (0b11111111_11000000_00000000_00000000, 13),
             (0b11111111_11111111_10110000_00000000, 23),
@@ -265,8 +265,7 @@ public static class HuffmanEncoder
             (0b11111111_11111111_11111101_11000000, 27),
             (0b11111111_11111111_11111101_11100000, 27),
             (0b11111111_11111111_11111110_00000000, 27),
-            (0b11111111_11111111_11111011_10000000, 26),
-            (0b11111111_11111111_11111111_11111100, 30)
+            (0b11111111_11111111_11111011_10000000, 26)
         };
 
         public static int GetEncodedLength(ReadOnlySpan<byte> src)
@@ -281,32 +280,52 @@ public static int GetEncodedLength(ReadOnlySpan<byte> src)
             return (bits + 7) / 8;
         }
 
-        public static int Encode(ReadOnlySpan<byte> src, Span<byte> dst)
+        // Encoded values are 30 bits at most, so are stored in the table in a uint.
+        // Convert to ulong here and put the encoded value in the most significant bits.
+        // This makes the encoding logic below simpler.
+        private static (ulong code, int bitLength) GetEncodedValue(byte b)
         {
-            ulong buffer = 0;
-            int bufferLength = 0;
-            int dstIdx = 0;
+            (uint code, int bitLength) = s_encodingTable[b];
+            return (((ulong)code) << 32, bitLength);
+        }
 
-            foreach (byte x in src)
+        public static int Encode(ReadOnlySpan<byte> source, Span<byte> destination, bool injectEOS = false)
+        {
+            ulong currentBits = 0;  // We can have 7 bits of rollover plus 30 bits for the next encoded value, so use a ulong
+            int currentBitCount = 0;
+            int dstOffset = 0;
+
+            for (int i = 0; i < source.Length; i++)
             {
-                (uint code, int codeLength) = s_encodingTable[x];
+                (ulong code, int bitLength) = GetEncodedValue(source[i]);
+
+                // inject EOS if instructed to
+                if (injectEOS)
+                {
+                    code |= (ulong)0b11111111_11111111_11111111_11111100 << (32 - bitLength);
+                    bitLength += 30;
+                    injectEOS = false;
+                }
 
-                buffer = buffer << codeLength | code >> 32 - codeLength;
-                bufferLength += codeLength;
+                currentBits |= code >> currentBitCount;
+                currentBitCount += bitLength;
 
-                while (bufferLength >= 8)
+                while (currentBitCount >= 8)
                 {
-                    bufferLength -= 8;
-                    dst[dstIdx++] = (byte)(buffer >> bufferLength);
+                    destination[dstOffset++] = (byte)(currentBits >> 56);
+                    currentBits <<= 8;
+                    currentBitCount -= 8;
                 }
             }
 
-            if (bufferLength != 0)
+            // Fill any trailing bits with ones, per RFC
+            if (currentBitCount > 0)
             {
-                dst[dstIdx++] = (byte)(buffer << (8 - bufferLength) | 0xFFu >> bufferLength);
+                currentBits |= 0xFFFFFFFFFFFFFFFF >> currentBitCount;
+                destination[dstOffset++] = (byte)(currentBits >> 56);
             }
 
-            return dstIdx;
+            return dstOffset;
         }
     }
 }