Patch for first byte flush

plugins/experimental/esi/Makefile.am

@@ -33,6 +33,7 @@ check_PROGRAMS = docnode_test parser_test processor_test utils_test vars_test
 libesi_la_SOURCES = \
 	lib/DocNode.cc \
 	lib/EsiParser.cc \
+	lib/EsiGzip.cc \
 	lib/EsiProcessor.cc \
 	lib/Expression.cc \
 	lib/FailureInfo.cc \
@@ -45,6 +46,7 @@ libesi_la_SOURCES = \
 libtest_la_SOURCES = \
 	lib/DocNode.cc \
 	lib/EsiParser.cc \
+	lib/EsiGzip.cc \
 	lib/EsiProcessor.cc \
 	lib/Expression.cc \
 	lib/FailureInfo.cc \

plugins/experimental/esi/README

@@ -47,10 +47,11 @@ Enabling ESI
 
 esi.so
 
-There are three options you can add. 
+There are four options you can add. 
   "--private-response" will add private cache control and expires header to the processed ESI document. 
   "--packed-node-support" will enable the support for using packed node, which will improve the performance of parsing cached ESI document. 
   "--disable-gzip-output" will disable gzipped output, which will NOT gzip the output anyway.
+  "--first-byte-flush" will enable the first byte flush feature, which will flush content to users as soon as the entire ESI document is received and parsed without all ESI includes fetched (the flushing will stop at the ESI include markup till that include is fetched). 
 
 2) We need a mapping for origin server response that contains the ESI markup. Assume that the ATS server is abc.com. And your origin server is xyz.com and the response containing ESI markup is http://xyz.com/esi.php. We will need the following line in /usr/local/etc/trafficserver/remap.config
 

plugins/experimental/esi/lib/EsiGzip.cc

@@ -0,0 +1,163 @@
+/** @file
+
+  A brief file description
+
+  @section license License
+
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+ */
+
+#include "EsiGzip.h"
+#include "gzip.h"
+#include <ctype.h>
+#include <stdint.h>
+
+using std::string;
+using namespace EsiLib;
+
+static const int COMPRESSION_LEVEL = 6;
+static const int ZLIB_MEM_LEVEL = 8;
+
+static const int GZIP_HEADER_SIZE = 10;
+static const int GZIP_TRAILER_SIZE = 8;
+
+static const char MAGIC_BYTE_1 = 0x1f;
+static const char MAGIC_BYTE_2 = 0x8b;
+static const char OS_TYPE = 3; // Unix
+
+static const int BUF_SIZE = 1 << 15; // 32k buffer
+
+EsiGzip::EsiGzip(const char *debug_tag,
+                           ComponentBase::Debug debug_func, ComponentBase::Error error_func)
+  : ComponentBase(debug_tag, debug_func, error_func),
+    _downstream_length(0), 
+    _total_data_length(0) {
+}
+
+template<typename T>
+inline void append(string &out, T data) {
+  for (unsigned int i = 0; i < sizeof(data); ++i) {
+    out += static_cast<char>(data & 0xff);
+    data = data >> 8;
+  }
+}
+
+inline int runDeflateLoop(z_stream &zstrm, int flush, std::string &cdata) {
+  char buf[BUF_SIZE];
+  int deflate_result = Z_OK;
+  do {
+    zstrm.next_out = reinterpret_cast<Bytef *>(buf);
+    zstrm.avail_out = BUF_SIZE;
+    deflate_result = deflate(&zstrm, flush);
+    if ((deflate_result == Z_OK) || (deflate_result == Z_STREAM_END)) {
+      cdata.append(buf, BUF_SIZE - zstrm.avail_out);
+      if ((deflate_result == Z_STREAM_END) || zstrm.avail_out > 6) {
+        break;
+      }
+    } else {
+      break;
+    }
+  } while (true);
+  return deflate_result;
+}
+
+bool
+EsiGzip::stream_encode(const char *data, int data_len, std::string &cdata) {
+  if(_downstream_length == 0) {
+    cdata.assign(GZIP_HEADER_SIZE, 0); // reserving space for the header
+    cdata[0] = MAGIC_BYTE_1;
+    cdata[1] = MAGIC_BYTE_2;
+    cdata[2] = Z_DEFLATED;
+    cdata[9] = OS_TYPE;
+
+    //_zstrm.zalloc = Z_NULL;
+    //_zstrm.zfree = Z_NULL;
+    //_zstrm.opaque = Z_NULL;
+    //if (deflateInit2(&_zstrm, COMPRESSION_LEVEL, Z_DEFLATED, -MAX_WBITS,
+    //               ZLIB_MEM_LEVEL, Z_DEFAULT_STRATEGY) != Z_OK) {
+    //  _errorLog("[%s] deflateInit2 failed!", __FUNCTION__);
+    //  return false;
+    //}
+
+    _crc = crc32(0, Z_NULL, 0);
+  }
+
+    _zstrm.zalloc = Z_NULL;
+    _zstrm.zfree = Z_NULL;
+    _zstrm.opaque = Z_NULL;
+    if (deflateInit2(&_zstrm, COMPRESSION_LEVEL, Z_DEFLATED, -MAX_WBITS,
+                   ZLIB_MEM_LEVEL, Z_DEFAULT_STRATEGY) != Z_OK) {
+      _errorLog("[%s] deflateInit2 failed!", __FUNCTION__);
+      return false;
+    }
+
+  int deflate_result = Z_OK;
+  if (data && (data_len > 0)) {
+    _zstrm.next_in = reinterpret_cast<Bytef *>(const_cast<char *>(data));
+    _zstrm.avail_in = data_len;
+    deflate_result = runDeflateLoop(_zstrm, Z_FULL_FLUSH, cdata);
+    if (deflate_result != Z_OK) {
+      _errorLog("[%s] runDeflateLoop failed!", __FUNCTION__);
+
+      deflateEnd(&_zstrm);
+
+      return false;
+    }
+    _crc = crc32(_crc, reinterpret_cast<const Bytef *>(data), data_len);
+    _downstream_length += cdata.size();
+    _total_data_length += data_len;
+  }
+
+    deflateEnd(&_zstrm);    
+
+  return true;
+}
+
+bool EsiGzip::stream_finish(std::string &cdata, int&downstream_length) {
+  char buf[BUF_SIZE];
+
+    _zstrm.zalloc = Z_NULL;
+    _zstrm.zfree = Z_NULL;
+    _zstrm.opaque = Z_NULL;
+    if (deflateInit2(&_zstrm, COMPRESSION_LEVEL, Z_DEFLATED, -MAX_WBITS,
+                   ZLIB_MEM_LEVEL, Z_DEFAULT_STRATEGY) != Z_OK) {
+      _errorLog("[%s] deflateInit2 failed!", __FUNCTION__);
+      return false;
+    }
+
+  _zstrm.next_in = reinterpret_cast<Bytef *>(buf);
+  _zstrm.avail_in = 0; 
+  // required for the "finish" loop as no data has been given so far
+  int deflate_result = runDeflateLoop(_zstrm, Z_FINISH, cdata);
+  deflateEnd(&_zstrm);
+  if (deflate_result != Z_STREAM_END) {
+    _errorLog("[%s] deflateEnd failed!", __FUNCTION__);
+    downstream_length = 0;
+    return false;
+  }
+  append(cdata, static_cast<uint32_t>(_crc));
+  append(cdata, static_cast<int32_t>(_total_data_length));
+  _downstream_length += cdata.size();
+  downstream_length = _downstream_length;
+  return true;
+}
+
+EsiGzip::~EsiGzip() {
+  _downstream_length = 0;
+  _total_data_length = 0;
+}
+

plugins/experimental/esi/lib/EsiGzip.h

@@ -0,0 +1,60 @@
+/** @file
+
+  A brief file description
+
+  @section license License
+
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+ */
+
+#ifndef _ESI_GZIP_H
+#define _ESI_GZIP_H
+
+#include "ComponentBase.h"
+#include <zlib.h>
+#include <string>
+
+class EsiGzip : private EsiLib::ComponentBase
+{
+
+public:
+
+  EsiGzip(const char *debug_tag,
+               EsiLib::ComponentBase::Debug debug_func, EsiLib::ComponentBase::Error error_func);
+
+  virtual ~EsiGzip();
+
+  bool stream_encode(const char *data, int data_len, std::string &cdata);
+
+  inline bool stream_encode(std::string data, std::string &cdata) {
+    return stream_encode(data.data(), data.size(), cdata);
+  }
+
+  bool stream_finish(std::string &cdata, int&downstream_length);
+
+private:
+
+  //int runDeflateLoop(z_stream &zstrm, int flush, std::string &cdata);
+  int _downstream_length;
+  int _total_data_length;
+  z_stream _zstrm;
+  uLong _crc;
+};
+
+
+#endif // _ESI_GZIP_H
+