apache · ezelkow1 · Mar 20, 2020 · Dec 3, 2019
diff --git a/doc/admin-guide/plugins/slice.en.rst b/doc/admin-guide/plugins/slice.en.rst
@@ -138,9 +138,9 @@ Under normal logging these slice block errors tend to show up as::
 By default more detailed stitching errors are written to ``diags.log``.
 Examples are as follows::
 
-    ERROR: [slice.cc: 288] logSliceError(): 1555705573.639 reason="Non 206 internal block response" uri="http://ats_ep/someasset.mp4" uas="curl" req_range="bytes=1000000-" norm_range="bytes 1000000-52428799/52428800" etag_exp="%221603934496%22" lm_exp="Fri, 19 Apr 2019 18:53:20 GMT" blk_range="21000000-21999999" status_got="206" cr_got="" etag_got="%221603934496%22" lm_got="" cc="no-store" via=""
+ERROR: [slice.cc: 288] logSliceError(): 1555705573.639 reason="Non 206 internal block response" uri="http://ats_ep/someasset.mp4" uas="curl" req_range="bytes=1000000-" norm_range="bytes 1000000-52428799/52428800" etag_exp="%221603934496%22" lm_exp="Fri, 19 Apr 2019 18:53:20 GMT" blk_range="21000000-21999999" status_got="206" cr_got="" etag_got="%221603934496%22" lm_got="" cc="no-store" via=""
 
-    ERROR: [server.cc: 288] logSliceError(): 1572370000.219 reason="Mismatch block Etag" uri="http://ats_ep/someasset.mp4" uas="curl" req_range="bytes=1092779033-1096299354" norm_range="bytes 1092779033-1096299354/2147483648" etag_exp="%223719843648%22" lm_exp="Tue, 29 Oct 2019 14:40:00 GMT" blk_range="1095000000-1095999999" status_got="206" cr_got="bytes 1095000000-1095999999/2147483648" etag_got="%223719853648%22" lm_got="Tue, 29 Oct 2019 17:26:40 GMT" cc="max-age=10000" via=""
+ERROR: [server.cc: 288] logSliceError(): 1572370000.219 reason="Mismatch block Etag" uri="http://ats_ep/someasset.mp4" uas="curl" req_range="bytes=1092779033-1096299354" norm_range="bytes 1092779033-1096299354/2147483648" etag_exp="%223719843648%22" lm_exp="Tue, 29 Oct 2019 14:40:00 GMT" blk_range="1095000000-1095999999" status_got="206" cr_got="bytes 1095000000-1095999999/2147483648" etag_got="%223719853648%22" lm_got="Tue, 29 Oct 2019 17:26:40 GMT" cc="max-age=10000" via=""
 
 Whether or how often these detailed log entries are written are
 configurable plugin options.

diff --git a/plugins/experimental/slice/Config.cc b/plugins/experimental/slice/Config.cc
@@ -98,13 +98,14 @@ Config::fromArgs(int const argc, char const *const argv[])
     {const_cast<char *>("remap-host"), required_argument, nullptr, 'r'},
     {const_cast<char *>("pace-errorlog"), required_argument, nullptr, 'p'},
     {const_cast<char *>("disable-errorlog"), no_argument, nullptr, 'd'},
+    {const_cast<char *>("throttle"), no_argument, nullptr, 'o'},
     {nullptr, 0, nullptr, 0},
   };
 
   // getopt assumes args start at '1' so this hack is needed
   char *const *argvp = (const_cast<char *const *>(argv) - 1);
   for (;;) {
-    int const opt = getopt_long(argc + 1, argvp, "b:t:r:p:d", longopts, nullptr);
+    int const opt = getopt_long(argc + 1, argvp, "b:t:r:p:do", longopts, nullptr);
     if (-1 == opt) {
       break;
     }
@@ -149,6 +150,10 @@ Config::fromArgs(int const argc, char const *const argv[])
     case 'd':
       m_paceerrsecs = -1;
       break;
+    case 'o':
+      m_throttle = true;
+      DEBUG_LOG("Enabling internal block throttling");
+      break;
     default:
       break;
     }

diff --git a/plugins/experimental/slice/Config.h b/plugins/experimental/slice/Config.h
@@ -30,6 +30,7 @@ struct Config {
 
   int64_t m_blockbytes{blockbytesdefault};
   std::string m_remaphost; // remap host to use for loopback slice GET
+  bool m_throttle{false};  // internal block throttling
   int m_paceerrsecs{0};    // -1 disable logging, 0 no pacing, max 60s
 
   // Convert optarg to bytes

diff --git a/plugins/experimental/slice/Data.h b/plugins/experimental/slice/Data.h
@@ -20,16 +20,13 @@
 
 #include "ts/ts.h"
 
-#include "Config.h"
 #include "HttpHeader.h"
 #include "Range.h"
 #include "Stage.h"
 
 #include <netinet/in.h>
 
-void incrData();
-
-void decrData();
+struct Config;
 
 struct Data {
   Data(Data const &) = delete;
@@ -61,7 +58,9 @@ struct Data {
   int64_t m_blockexpected; // body bytes expected
   int64_t m_blockskip;     // number of bytes to skip in this block
   int64_t m_blockconsumed; // body bytes consumed
-  bool m_iseos;            // server in EOS state
+
+  enum BlockState { Pending, Active, Done, Fail };
+  BlockState m_blockstate; // is there an active slice block
 
   int64_t m_bytestosend; // header + content bytes to send
   int64_t m_bytessent;   // number of bytes written to the client
@@ -86,33 +85,32 @@ struct Data {
       m_etaglen(0),
       m_lastmodifiedlen(0),
       m_statustype(TS_HTTP_STATUS_NONE),
-      m_bail(false),
       m_req_range(-1, -1),
-      m_contentlen(-1)
-
-      ,
+      m_contentlen(-1),
       m_blocknum(-1),
       m_blockexpected(0),
       m_blockskip(0),
       m_blockconsumed(0),
-      m_iseos(false)
-
-      ,
+      m_blockstate(Pending),
       m_bytestosend(0),
       m_bytessent(0),
       m_server_block_header_parsed(false),
       m_server_first_header_parsed(false),
       m_http_parser(nullptr)
   {
-    // incrData();
     m_hostname[0]     = '\0';
     m_lastmodified[0] = '\0';
     m_etag[0]         = '\0';
+#if defined(COLLECT_STATS)
+    TSStatIntIncrement(stats::DataCreate, 1);
+#endif
   }
 
   ~Data()
   {
-    // decrData();
+#if defined(COLLECT_STATS)
+    TSStatIntIncrement(stats::DataDestroy, 1);
+#endif
     if (nullptr != m_urlbuf) {
       if (nullptr != m_urlloc) {
         TSHandleMLocRelease(m_urlbuf, TS_NULL_MLOC, m_urlloc);

diff --git a/plugins/experimental/slice/HttpHeader.cc b/plugins/experimental/slice/HttpHeader.cc
@@ -311,7 +311,8 @@ HttpHeader::toString() const
 /////// HdrMgr
 
 TSParseResult
-HdrMgr::populateFrom(TSHttpParser const http_parser, TSIOBufferReader const reader, HeaderParseFunc const parsefunc)
+HdrMgr::populateFrom(TSHttpParser const http_parser, TSIOBufferReader const reader, HeaderParseFunc const parsefunc,
+                     int64_t *const bytes)
 {
   TSParseResult parse_res = TS_PARSE_CONT;
 
@@ -322,14 +323,14 @@ HdrMgr::populateFrom(TSHttpParser const http_parser, TSIOBufferReader const read
     m_lochdr = TSHttpHdrCreate(m_buffer);
   }
 
-  int64_t read_avail = TSIOBufferReaderAvail(reader);
-  if (0 < read_avail) {
+  int64_t avail = TSIOBufferReaderAvail(reader);
+  if (0 < avail) {
     TSIOBufferBlock block = TSIOBufferReaderStart(reader);
     int64_t consumed      = 0;
 
     parse_res = TS_PARSE_CONT;
 
-    while (nullptr != block && 0 < read_avail) {
+    while (nullptr != block && 0 < avail) {
       int64_t blockbytes       = 0;
       char const *const bstart = TSIOBufferBlockReadStart(block, reader, &blockbytes);
 
@@ -341,7 +342,7 @@ HdrMgr::populateFrom(TSHttpParser const http_parser, TSIOBufferReader const read
       int64_t const bytes_parsed(ptr - bstart);
 
       consumed += bytes_parsed;
-      read_avail -= bytes_parsed;
+      avail -= bytes_parsed;
 
       if (TS_PARSE_CONT == parse_res) {
         block = TSIOBufferBlockNext(block);
@@ -350,6 +351,12 @@ HdrMgr::populateFrom(TSHttpParser const http_parser, TSIOBufferReader const read
       }
     }
     TSIOBufferReaderConsume(reader, consumed);
+
+    if (nullptr != bytes) {
+      *bytes = consumed;
+    }
+  } else if (nullptr != bytes) {
+    *bytes = 0;
   }
 
   return parse_res;

diff --git a/plugins/experimental/slice/HttpHeader.h b/plugins/experimental/slice/HttpHeader.h
@@ -53,6 +53,16 @@ struct HttpHeader {
     return nullptr != m_buffer && nullptr != m_lochdr;
   }
 
+  int
+  byteSize() const
+  {
+    if (isValid()) {
+      return TSHttpHdrLengthGet(m_buffer, m_lochdr);
+    } else {
+      return 0;
+    }
+  }
+
   // TS_HTTP_TYPE_UNKNOWN, TS_HTTP_TYPE_REQUEST, TS_HTTP_TYPE_RESPONSE
   TSHttpType type() const;
 
@@ -205,7 +215,8 @@ struct HdrMgr {
      TSHttpHdrParseResp
     Call this multiple times if necessary.
   */
-  TSParseResult populateFrom(TSHttpParser const http_parser, TSIOBufferReader const reader, HeaderParseFunc const parsefunc);
+  TSParseResult populateFrom(TSHttpParser const http_parser, TSIOBufferReader const reader, HeaderParseFunc const parsefunc,
+                             int64_t *const consumed);
 
   bool
   isValid() const

diff --git a/plugins/experimental/slice/Makefile.inc b/plugins/experimental/slice/Makefile.inc
@@ -23,7 +23,6 @@ experimental_slice_slice_la_SOURCES = \
   experimental/slice/Config.h \
   experimental/slice/ContentRange.cc \
   experimental/slice/ContentRange.h \
-  experimental/slice/Data.cc \
   experimental/slice/Data.h \
   experimental/slice/HttpHeader.cc \
   experimental/slice/HttpHeader.h \
@@ -39,7 +38,9 @@ experimental_slice_slice_la_SOURCES = \
   experimental/slice/slice.h \
   experimental/slice/Stage.h \
   experimental/slice/transfer.cc \
-  experimental/slice/transfer.h
+  experimental/slice/transfer.h \
+  experimental/slice/util.cc \
+  experimental/slice/util.h
 
 check_PROGRAMS += experimental/slice/test_content_range
 

diff --git a/plugins/experimental/slice/Makefile.tsxs b/plugins/experimental/slice/Makefile.tsxs
@@ -22,7 +22,6 @@ all: $(PLUGIN).so
 SOURCES = \
 	Config.cc \
 	ContentRange.cc \
-	Data.cc \
 	HttpHeader.cc \
 	Range.cc \
 	client.cc \

diff --git a/plugins/experimental/slice/Range.cc b/plugins/experimental/slice/Range.cc
@@ -152,6 +152,16 @@ Range::firstBlockFor(int64_t const blocksize) const
   }
 }
 
+int64_t
+Range::lastBlockFor(int64_t const blocksize) const
+{
+  if (0 < blocksize && isValid()) {
+    return std::max((int64_t)0, (m_end - 1) / blocksize);
+  } else {
+    return -1;
+  }
+}
+
 Range
 Range::intersectedWith(Range const &other) const
 {
@@ -162,7 +172,6 @@ bool
 Range::blockIsInside(int64_t const blocksize, int64_t const blocknum) const
 {
   Range const blockrange(blocksize * blocknum, blocksize * (blocknum + 1));
-
   Range const isec(blockrange.intersectedWith(*this));
 
   return isec.isValid();

diff --git a/plugins/experimental/slice/Range.h b/plugins/experimental/slice/Range.h
@@ -56,6 +56,10 @@ struct Range {
    */
   int64_t firstBlockFor(int64_t const blockbytes) const;
 
+  /** block number of last (inclusive) range block
+   */
+  int64_t lastBlockFor(int64_t const blockbytes) const;
+
   /** block intersection
    */
   Range intersectedWith(Range const &other) const;