diff --git a/doc/admin-guide/plugins/cache_range_requests.en.rst b/doc/admin-guide/plugins/cache_range_requests.en.rst index eb01960f707..e677bc528dd 100644 --- a/doc/admin-guide/plugins/cache_range_requests.en.rst +++ b/doc/admin-guide/plugins/cache_range_requests.en.rst @@ -118,6 +118,8 @@ X-Crr-Ims header support .. option:: --consider-ims .. option:: -c +.. option:: --ims-header=[header name] (default: X-Crr-Ims) +.. option:: -i To support slice plugin self healing an option to force revalidation after cache lookup complete was added. This option is triggered by a @@ -137,6 +139,11 @@ In order for this to properly work in a CDN each cache in the chain *SHOULD* also contain a remap rule with the :program:`cache_range_requests` plugin with this option set. +When used with the :program:`slice` plugin its `--crr-ims-header` +option must have the same value (or not be defined) in order to work. + +Presence of the `--ims-header` automatically sets the `--consider-ims` option. + Don't modify the Cache Key -------------------------- diff --git a/doc/admin-guide/plugins/slice.en.rst b/doc/admin-guide/plugins/slice.en.rst index edb338906f2..0246fa78f49 100644 --- a/doc/admin-guide/plugins/slice.en.rst +++ b/doc/admin-guide/plugins/slice.en.rst @@ -113,6 +113,19 @@ The slice plugin supports the following options:: Requires setting up an intermediate loopback remap rule. -r for short + --skip-header=
(default: X-Slicer-Info) + Header name used by the slice plugin after the loopback + to indicate that the slice plugin should be skipped. + -s for short + + --crr-ims-header=
(default: X-Crr-Ims) + Header name used by the slice plugin to tell the + `cache_range_requests` plugin that a request should + be marked as STALE. Used for self healing. + This must match the `--ims-header` option used by the + `cache_range_requests` plugin. + -i for short + Examples:: @plugin=slice.so @pparam=--blockbytes=1000000 @plugin=cache_range_requests.so diff --git a/plugins/cache_range_requests/cache_range_requests.cc b/plugins/cache_range_requests/cache_range_requests.cc index 622352724de..23f458a8d4a 100644 --- a/plugins/cache_range_requests/cache_range_requests.cc +++ b/plugins/cache_range_requests/cache_range_requests.cc @@ -47,11 +47,14 @@ using parent_select_mode_t = enum parent_select_mode { PS_CACHEKEY_URL, // Set parent selection url to cache_key url }; +constexpr std::string_view DefaultImsHeader = {"X-Crr-Ims"}; + struct pluginconfig { parent_select_mode_t ps_mode{PS_DEFAULT}; bool consider_ims_header{false}; bool modify_cache_key{true}; bool verify_cacheability{false}; + std::string ims_header; }; struct txndata { @@ -61,9 +64,6 @@ struct txndata { bool verify_cacheability{false}; }; -// Header for optional revalidation -constexpr std::string_view X_IMS_HEADER = {"X-Crr-Ims"}; - // pluginconfig struct (global plugin only) pluginconfig *gPluginConfig = {nullptr}; @@ -99,9 +99,10 @@ create_pluginconfig(int argc, char *const argv[]) } static const struct option longopts[] = { - {const_cast("ps-cachekey"), no_argument, nullptr, 'p'}, {const_cast("consider-ims"), no_argument, nullptr, 'c'}, + {const_cast("ims-header"), required_argument, nullptr, 'i'}, {const_cast("no-modify-cachekey"), no_argument, nullptr, 'n'}, + {const_cast("ps-cachekey"), no_argument, nullptr, 'p'}, {const_cast("verify-cacheability"), no_argument, nullptr, 'v'}, {nullptr, 0, nullptr, 0}, }; @@ -111,24 +112,29 @@ create_pluginconfig(int argc, char *const argv[]) --argv; for (;;) { - int const opt = getopt_long(argc, argv, "", longopts, nullptr); + int const opt = getopt_long(argc, argv, "i:", longopts, nullptr); if (-1 == opt) { break; } switch (opt) { - case 'p': { - DEBUG_LOG("Plugin modifies parent selection key"); - pc->ps_mode = PS_CACHEKEY_URL; - } break; case 'c': { - DEBUG_LOG("Plugin considers the '%.*s' header", (int)X_IMS_HEADER.size(), X_IMS_HEADER.data()); + DEBUG_LOG("Plugin considers the ims header"); + pc->consider_ims_header = true; + } break; + case 'i': { + DEBUG_LOG("Plugin uses custom ims header: %s", optarg); + pc->ims_header.assign(optarg); pc->consider_ims_header = true; } break; case 'n': { DEBUG_LOG("Plugin doesn't modify cache key"); pc->modify_cache_key = false; } break; + case 'p': { + DEBUG_LOG("Plugin modifies parent selection key"); + pc->ps_mode = PS_CACHEKEY_URL; + } break; case 'v': { DEBUG_LOG("Plugin verifies whether the object in the transaction is cacheable"); pc->verify_cacheability = true; @@ -144,6 +150,11 @@ create_pluginconfig(int argc, char *const argv[]) pc->ps_mode = PS_CACHEKEY_URL; } + if (pc->consider_ims_header && pc->ims_header.empty()) { + pc->ims_header = DefaultImsHeader; + DEBUG_LOG("Plugin uses default ims header: %s", pc->ims_header.c_str()); + } + return pc; } @@ -244,12 +255,12 @@ range_header_check(TSHttpTxn txnp, pluginconfig *const pc) } } - // optionally consider an X-CRR-IMS header + // optionally consider an ims header if (pc->consider_ims_header) { - TSMLoc const imsloc = TSMimeHdrFieldFind(hdr_buf, hdr_loc, X_IMS_HEADER.data(), X_IMS_HEADER.size()); + TSMLoc const imsloc = TSMimeHdrFieldFind(hdr_buf, hdr_loc, pc->ims_header.data(), pc->ims_header.size()); if (TS_NULL_MLOC != imsloc) { time_t const itime = TSMimeHdrFieldValueDateGet(hdr_buf, hdr_loc, imsloc); - DEBUG_LOG("Servicing the '%.*s' header", (int)X_IMS_HEADER.size(), X_IMS_HEADER.data()); + DEBUG_LOG("Servicing the '%s' header", pc->ims_header.c_str()); TSHandleMLocRelease(hdr_buf, hdr_loc, imsloc); if (0 < itime) { txn_state->ims_time = itime; @@ -303,7 +314,7 @@ handle_send_origin_request(TSCont contp, TSHttpTxn txnp, txndata *const txn_stat } if (TS_SUCCESS == TSHttpTxnServerReqGet(txnp, &hdr_buf, &hdr_loc) && !rv.empty()) { - if (set_header(hdr_buf, hdr_loc, TS_MIME_FIELD_RANGE, TS_MIME_LEN_RANGE, rv.data(), rv.length())) { + if (set_header(hdr_buf, hdr_loc, TS_MIME_FIELD_RANGE, TS_MIME_LEN_RANGE, rv.data(), rv.size())) { DEBUG_LOG("Added range header: %s", rv.c_str()); TSHttpTxnHookAdd(txnp, TS_HTTP_READ_RESPONSE_HDR_HOOK, contp); } @@ -345,7 +356,7 @@ handle_client_send_response(TSHttpTxn txnp, txndata *const txn_state) TSMLoc req_loc = TS_NULL_MLOC; if (TS_SUCCESS == TSHttpTxnClientReqGet(txnp, &req_buf, &req_loc)) { DEBUG_LOG("Adding range header: %s", rv.c_str()); - if (!set_header(req_buf, req_loc, TS_MIME_FIELD_RANGE, TS_MIME_LEN_RANGE, rv.data(), rv.length())) { + if (!set_header(req_buf, req_loc, TS_MIME_FIELD_RANGE, TS_MIME_LEN_RANGE, rv.data(), rv.size())) { DEBUG_LOG("set_header() failed."); } TSHandleMLocRelease(req_buf, TS_NULL_MLOC, req_loc); diff --git a/plugins/experimental/slice/Config.cc b/plugins/experimental/slice/Config.cc index 2cce89c1e16..2c810c69347 100644 --- a/plugins/experimental/slice/Config.cc +++ b/plugins/experimental/slice/Config.cc @@ -26,6 +26,12 @@ #include "ts/experimental.h" +namespace +{ +constexpr std::string_view DefaultSliceSkipHeader = {"X-Slicer-Info"}; +constexpr std::string_view DefaultCrrImsHeader = {"X-Crr-Ims"}; +} // namespace + Config::~Config() { if (nullptr != m_regex_extra) { @@ -108,12 +114,14 @@ Config::fromArgs(int const argc, char const *const argv[]) // standard parsing constexpr struct option longopts[] = { {const_cast("blockbytes"), required_argument, nullptr, 'b'}, + {const_cast("crr-ims-header"), required_argument, nullptr, 'c'}, {const_cast("disable-errorlog"), no_argument, nullptr, 'd'}, {const_cast("exclude-regex"), required_argument, nullptr, 'e'}, {const_cast("include-regex"), required_argument, nullptr, 'i'}, {const_cast("ref-relative"), no_argument, nullptr, 'l'}, {const_cast("pace-errorlog"), required_argument, nullptr, 'p'}, {const_cast("remap-host"), required_argument, nullptr, 'r'}, + {const_cast("skip-header"), required_argument, nullptr, 's'}, {const_cast("blockbytes-test"), required_argument, nullptr, 't'}, {nullptr, 0, nullptr, 0}, }; @@ -121,7 +129,7 @@ Config::fromArgs(int const argc, char const *const argv[]) // getopt assumes args start at '1' so this hack is needed char *const *argvp = (const_cast(argv) - 1); for (;;) { - int const opt = getopt_long(argc + 1, argvp, "b:de:i:lp:r:t:", longopts, nullptr); + int const opt = getopt_long(argc + 1, argvp, "b:dc:e:i:lp:r:s:t:", longopts, nullptr); if (-1 == opt) { break; } @@ -138,6 +146,10 @@ Config::fromArgs(int const argc, char const *const argv[]) ERROR_LOG("Invalid blockbytes: %s", optarg); } } break; + case 'c': { + m_crr_ims_header.assign(optarg); + DEBUG_LOG("Using override crr ims header %s", optarg); + } break; case 'd': { m_paceerrsecs = -1; } break; @@ -193,6 +205,10 @@ Config::fromArgs(int const argc, char const *const argv[]) m_remaphost = optarg; DEBUG_LOG("Using loopback remap host override: %s", m_remaphost.c_str()); } break; + case 's': { + m_skip_header.assign(optarg); + DEBUG_LOG("Using slice skip header %s", optarg); + } break; case 't': { if (0 == blockbytes) { int64_t const bytesread = bytesFrom(optarg); @@ -225,6 +241,14 @@ Config::fromArgs(int const argc, char const *const argv[]) } else { DEBUG_LOG("Block stitching error logs at most every %d sec(s)", m_paceerrsecs); } + if (m_crr_ims_header.empty()) { + m_crr_ims_header = DefaultCrrImsHeader; + DEBUG_LOG("Using default crr ims header %s", m_crr_ims_header.c_str()); + } + if (m_skip_header.empty()) { + m_skip_header = DefaultSliceSkipHeader; + DEBUG_LOG("Using default slice skip header %s", m_skip_header.c_str()); + } return true; } diff --git a/plugins/experimental/slice/Config.h b/plugins/experimental/slice/Config.h index 033ac7da671..7188c6ea934 100644 --- a/plugins/experimental/slice/Config.h +++ b/plugins/experimental/slice/Config.h @@ -45,6 +45,9 @@ struct Config { enum RefType { First, Relative }; RefType m_reftype{First}; // reference slice is relative to request + std::string m_skip_header; + std::string m_crr_ims_header; + // Convert optarg to bytes static int64_t bytesFrom(char const *const valstr); diff --git a/plugins/experimental/slice/HttpHeader.h b/plugins/experimental/slice/HttpHeader.h index e22b83d4bf6..15a1b158ef1 100644 --- a/plugins/experimental/slice/HttpHeader.h +++ b/plugins/experimental/slice/HttpHeader.h @@ -36,8 +36,6 @@ #include -static char const *const SLICER_MIME_FIELD_INFO = "X-Slicer-Info"; - /** Designed to be a cheap throwaway struct which allows a consumer to make various calls to manipulate headers. diff --git a/plugins/experimental/slice/client.cc b/plugins/experimental/slice/client.cc index c1a9b59ef94..3d19372440a 100644 --- a/plugins/experimental/slice/client.cc +++ b/plugins/experimental/slice/client.cc @@ -57,12 +57,13 @@ handle_client_req(TSCont contp, TSEvent event, Data *const data) Range rangebe; char rangestr[1024]; - int rangelen = sizeof(rangestr); - bool const hasRange = header.valueForKey(TS_MIME_FIELD_RANGE, TS_MIME_LEN_RANGE, rangestr, &rangelen, + int rangelen = sizeof(rangestr); + bool const hasRange = header.valueForKey(TS_MIME_FIELD_RANGE, TS_MIME_LEN_RANGE, rangestr, &rangelen, 0); // <-- first range only + Config const *const conf = data->m_config; if (hasRange) { // write parsed header into slicer meta tag - header.setKeyVal(SLICER_MIME_FIELD_INFO, strlen(SLICER_MIME_FIELD_INFO), rangestr, rangelen); + header.setKeyVal(conf->m_skip_header.c_str(), conf->m_skip_header.size(), rangestr, rangelen); bool const isRangeGood = rangebe.fromStringClosed(rangestr); if (isRangeGood) { @@ -74,21 +75,21 @@ handle_client_req(TSCont contp, TSEvent event, Data *const data) data->m_statustype = TS_HTTP_STATUS_REQUESTED_RANGE_NOT_SATISFIABLE; // First block will give Content-Length - rangebe = Range(0, data->m_config->m_blockbytes); + rangebe = Range(0, conf->m_blockbytes); } } else { DEBUG_LOG("%p Full content request", data); static char const *const valstr = "-"; static size_t const vallen = strlen(valstr); - header.setKeyVal(SLICER_MIME_FIELD_INFO, strlen(SLICER_MIME_FIELD_INFO), valstr, vallen); + header.setKeyVal(conf->m_skip_header.data(), conf->m_skip_header.size(), valstr, vallen); data->m_statustype = TS_HTTP_STATUS_OK; rangebe = Range(0, Range::maxval); } - if (Config::RefType::First == data->m_config->m_reftype) { + if (Config::RefType::First == conf->m_reftype) { data->m_blocknum = 0; } else { - data->m_blocknum = rangebe.firstBlockFor(data->m_config->m_blockbytes); + data->m_blocknum = rangebe.firstBlockFor(conf->m_blockbytes); } data->m_req_range = rangebe; diff --git a/plugins/experimental/slice/server.cc b/plugins/experimental/slice/server.cc index 510edab73ac..ddf0cc9ad2f 100644 --- a/plugins/experimental/slice/server.cc +++ b/plugins/experimental/slice/server.cc @@ -223,9 +223,9 @@ handleFirstServerHeader(Data *const data, TSCont const contp) void logSliceError(char const *const message, Data const *const data, HttpHeader const &header_resp) { - Config *const config = data->m_config; + Config *const conf = data->m_config; - bool const logToError = config->canLogError(); + bool const logToError = conf->canLogError(); // always write block stitch errors while in debug mode if (!logToError && !TSIsDebugTagSet(PLUGIN_NAME)) { @@ -259,7 +259,7 @@ logSliceError(char const *const message, Data const *const data, HttpHeader cons // raw range request char rangestr[1024]; int rangelen = sizeof(rangestr); - header_req.valueForKey(SLICER_MIME_FIELD_INFO, strlen(SLICER_MIME_FIELD_INFO), rangestr, &rangelen); + header_req.valueForKey(conf->m_skip_header.data(), conf->m_skip_header.size(), rangestr, &rangelen); // Normalized range request ContentRange const crange(data->m_req_range.m_beg, data->m_req_range.m_end, data->m_contentlen); @@ -268,8 +268,8 @@ logSliceError(char const *const message, Data const *const data, HttpHeader cons crange.toStringClosed(normstr, &normlen); // block range request - int64_t const blockbeg = data->m_blocknum * data->m_config->m_blockbytes; - int64_t const blockend = std::min(blockbeg + data->m_config->m_blockbytes, data->m_contentlen); + int64_t const blockbeg = data->m_blocknum * conf->m_blockbytes; + int64_t const blockend = std::min(blockbeg + conf->m_blockbytes, data->m_contentlen); // Block response data TSHttpStatus const statusgot = header_resp.status(); @@ -416,8 +416,9 @@ handleNextServerHeader(Data *const data, TSCont const contp) // add special CRR IMS header to the request HttpHeader headerreq(data->m_req_hdrmgr.m_buffer, data->m_req_hdrmgr.m_lochdr); - if (!headerreq.setKeyTime(X_CRR_IMS_HEADER.data(), X_CRR_IMS_HEADER.size(), dateims)) { - ERROR_LOG("Failed setting '%.*s'", (int)X_CRR_IMS_HEADER.size(), X_CRR_IMS_HEADER.data()); + Config const *const conf = data->m_config; + if (!headerreq.setKeyTime(conf->m_crr_ims_header.data(), conf->m_crr_ims_header.size(), dateims)) { + ERROR_LOG("Failed setting '%s'", conf->m_crr_ims_header.c_str()); return false; } @@ -438,8 +439,9 @@ handleNextServerHeader(Data *const data, TSCont const contp) // add special CRR IMS header to the request HttpHeader headerreq(data->m_req_hdrmgr.m_buffer, data->m_req_hdrmgr.m_lochdr); - if (!headerreq.setKeyTime(X_CRR_IMS_HEADER.data(), X_CRR_IMS_HEADER.size(), dateims)) { - ERROR_LOG("Failed setting '%.*s'", (int)X_CRR_IMS_HEADER.size(), X_CRR_IMS_HEADER.data()); + Config const *const conf = data->m_config; + if (!headerreq.setKeyTime(conf->m_crr_ims_header.data(), conf->m_crr_ims_header.size(), dateims)) { + ERROR_LOG("Failed setting '%s'", conf->m_crr_ims_header.c_str()); return false; } diff --git a/plugins/experimental/slice/slice.cc b/plugins/experimental/slice/slice.cc index fbcd6434854..abb1630fc1d 100644 --- a/plugins/experimental/slice/slice.cc +++ b/plugins/experimental/slice/slice.cc @@ -42,8 +42,7 @@ read_request(TSHttpTxn txnp, Config *const config) HttpHeader const header(hdrmgr.m_buffer, hdrmgr.m_lochdr); if (TS_HTTP_METHOD_GET == header.method()) { - static int const SLICER_MIME_LEN_INFO = strlen(SLICER_MIME_FIELD_INFO); - if (!header.hasKey(SLICER_MIME_FIELD_INFO, SLICER_MIME_LEN_INFO)) { + if (!header.hasKey(config->m_skip_header.data(), config->m_skip_header.size())) { // check if any previous plugin has monkeyed with the transaction status TSHttpStatus const txnstat = TSHttpTxnStatusGet(txnp); if (TS_HTTP_STATUS_NONE != txnstat) { @@ -233,9 +232,7 @@ TSReturnCode TSRemapNewInstance(int argc, char *argv[], void **ih, char * /* errbuf */, int /* errbuf_size */) { Config *const config = new Config; - if (2 < argc) { - config->fromArgs(argc - 2, argv + 2); - } + config->fromArgs(argc - 2, argv + 2); *ih = static_cast(config); return TS_SUCCESS; } @@ -274,9 +271,7 @@ TSPluginInit(int argc, char const *argv[]) return; } - if (1 < argc) { - globalConfig.fromArgs(argc - 1, argv + 1); - } + globalConfig.fromArgs(argc - 1, argv + 1); TSCont const contp(TSContCreate(global_read_request_hook, nullptr)); diff --git a/plugins/experimental/slice/slice.h b/plugins/experimental/slice/slice.h index ed86dd6771d..4f9df58a382 100644 --- a/plugins/experimental/slice/slice.h +++ b/plugins/experimental/slice/slice.h @@ -32,8 +32,6 @@ #define PLUGIN_NAME "slice" #endif -constexpr std::string_view X_CRR_IMS_HEADER = {"X-Crr-Ims"}; - #if !defined(UNITTEST) #define __FILENAME__ (strrchr(__FILE__, '/') ? strrchr(__FILE__, '/') + 1 : __FILE__) diff --git a/plugins/experimental/slice/util.cc b/plugins/experimental/slice/util.cc index 4ca86c3d1a4..90ef9d8b525 100644 --- a/plugins/experimental/slice/util.cc +++ b/plugins/experimental/slice/util.cc @@ -125,14 +125,16 @@ request_block(TSCont contp, Data *const data) case BlockState::Pending: data->m_blockstate = BlockState::Active; break; - case BlockState::PendingInt: - data->m_blockstate = BlockState::ActiveInt; - header.removeKey(X_CRR_IMS_HEADER.data(), X_CRR_IMS_HEADER.size()); - break; - case BlockState::PendingRef: - data->m_blockstate = BlockState::ActiveRef; - header.removeKey(X_CRR_IMS_HEADER.data(), X_CRR_IMS_HEADER.size()); - break; + case BlockState::PendingInt: { + data->m_blockstate = BlockState::ActiveInt; + Config const *const conf = data->m_config; + header.removeKey(conf->m_crr_ims_header.c_str(), conf->m_crr_ims_header.size()); + } break; + case BlockState::PendingRef: { + data->m_blockstate = BlockState::ActiveRef; + Config const *const conf = data->m_config; + header.removeKey(conf->m_crr_ims_header.c_str(), conf->m_crr_ims_header.size()); + } break; default: ERROR_LOG("Invalid blockstate"); return false; diff --git a/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_ims.test.py b/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_ims.test.py index 22f795839ab..eb96261e78c 100644 --- a/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_ims.test.py +++ b/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests_ims.test.py @@ -88,10 +88,13 @@ server.addResponse("sessionlog.json", req_full, res_full) # cache range requests plugin remap -ts.Disk.remap_config.AddLine( - 'map http://www.example.com http://127.0.0.1:{}'.format(server.Variables.Port) + +ts.Disk.remap_config.AddLines([ + f'map http://ims http://127.0.0.1:{server.Variables.Port}' + ' @plugin=cache_range_requests.so @pparam=--consider-ims', -) + f'map http://imsheader http://127.0.0.1:{server.Variables.Port}' + + ' @plugin=cache_range_requests.so @pparam=--consider-ims' + + ' @pparam=--ims-header=CrrIms', +]) # cache debug ts.Disk.plugin_config.AddLine('xdebug.so') @@ -109,29 +112,40 @@ ps = tr.Processes.Default ps.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) ps.StartBefore(Test.Processes.ts) -ps.Command = curl_and_args + ' http://www.example.com/path -r 0-' +ps.Command = curl_and_args + ' http://ims/path -r 0-' ps.ReturnCode = 0 ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: miss", "expected cache miss for load") tr.StillRunningAfter = ts -# set up the IMS date field (go in the future) RFC 2616 -futuretime = time.time() + 100 # seconds -futurestr = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(futuretime)) - # test inner range # 1 Test - Fetch range into cache tr = Test.AddTestRun("0- cache hit check") ps = tr.Processes.Default -ps.Command = curl_and_args + ' http://www.example.com/path -r 0-'.format(futurestr) +ps.Command = curl_and_args + ' http://ims/path -r 0-' ps.ReturnCode = 0 ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit", "expected cache hit") tr.StillRunningAfter = ts +# set up the IMS date field (go in the future) RFC 2616 +futuretime = time.time() + 100 # seconds +futurestr = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(futuretime)) + # 2 Test - Ensure X-CRR-IMS header results in hit-stale tr = Test.AddTestRun("0- range X-CRR-IMS check") ps = tr.Processes.Default -ps.Command = curl_and_args + ' http://www.example.com/path -r 0- -H "X-CRR-IMS: {}"'.format(futurestr) +ps.Command = curl_and_args + ' http://ims/path -r 0- -H "X-CRR-IMS: {}"'.format(futurestr) +ps.ReturnCode = 0 +ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-stale", "expected cache hit-stale") +tr.StillRunningAfter = ts + +futuretime += 10 # seconds +futurestr = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(futuretime)) + +# 3 Test - Ensure CrrIms header results in hit-stale +tr = Test.AddTestRun("0- range CrrIms check, override header") +ps = tr.Processes.Default +ps.Command = curl_and_args + ' http://imsheader/path -r 0- -H "CrrIms: {}"'.format(futurestr) ps.ReturnCode = 0 ps.Streams.stdout.Content = Testers.ContainsExpression("X-Cache: hit-stale", "expected cache hit-stale") tr.StillRunningAfter = ts diff --git a/tests/gold_tests/pluginTest/slice/slice.test.py b/tests/gold_tests/pluginTest/slice/slice.test.py index 7dfadbfe294..d3e09c4fda3 100644 --- a/tests/gold_tests/pluginTest/slice/slice.test.py +++ b/tests/gold_tests/pluginTest/slice/slice.test.py @@ -82,9 +82,13 @@ # set up whole asset fetch into cache ts.Disk.remap_config.AddLines([ - 'map http://preload/ http://127.0.0.1:{}'.format(server.Variables.Port), - 'map http://slice/ http://127.0.0.1:{}'.format(server.Variables.Port) + - ' @plugin=slice.so @pparam=--blockbytes-test={}'.format(block_bytes) + f'map http://preload/ http://127.0.0.1:{server.Variables.Port}', + f'map http://slice_only/ http://127.0.0.1:{server.Variables.Port}', + f'map http://slice/ http://127.0.0.1:{server.Variables.Port}' + + f' @plugin=slice.so @pparam=--blockbytes-test={block_bytes}', + f'map http://slicehdr/ http://127.0.0.1:{server.Variables.Port}' + + f' @plugin=slice.so @pparam=--blockbytes-test={block_bytes}' + + ' @pparam=--skip-header=SkipSlice', ]) ts.Disk.records_config.update({ @@ -180,3 +184,14 @@ ps.Command = curl_and_args + ' http://slice/path' + ' -r {}-{}'.format(beg, end) ps.Streams.stdout.Content = Testers.ContainsExpression("416 Requested Range Not Satisfiable", "expected 416 response") tr.StillRunningAfter = ts + +# 9 Test - First complete slice using override header +# if this fails it will infinite loop +tr = Test.AddTestRun("Fetch first slice range") +ps = tr.Processes.Default +ps.Command = curl_and_args + ' http://slicehdr/path' + ' -r 0-6' +ps.ReturnCode = 0 +ps.Streams.stderr = "gold/slice_first.stderr.gold" +ps.Streams.stdout.Content = Testers.ContainsExpression("206 Partial Content", "expected 206 response") +ps.Streams.stdout.Content += Testers.ContainsExpression("Content-Range: bytes 0-6/18", "mismatch byte content response") +tr.StillRunningAfter = ts diff --git a/tests/gold_tests/pluginTest/slice/slice_selfhealing.test.py b/tests/gold_tests/pluginTest/slice/slice_selfhealing.test.py index 9d3ca27682e..2a3af17be0e 100644 --- a/tests/gold_tests/pluginTest/slice/slice_selfhealing.test.py +++ b/tests/gold_tests/pluginTest/slice/slice_selfhealing.test.py @@ -15,16 +15,27 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import time +import datetime Test.Summary = ''' Slice selfhealing test ''' + +def to_httpdate(dt): + # string representation of a date according to RFC 1123 (HTTP/1.1). + weekday = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"][dt.weekday()] + month = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"][dt.month - 1] + return "%s, %02d %s %04d %02d:%02d:%02d GMT" % (weekday, dt.day, month, + dt.year, dt.hour, dt.minute, dt.second) + # Test description: # Preload the cache with the entire asset to be range requested. # Reload remap rule with slice plugin # Request content through the slice plugin + Test.SkipUnless( Condition.PluginExists('slice.so'), Condition.PluginExists('cache_range_requests.so'), @@ -60,10 +71,15 @@ # set up slice plugin with remap host into cache_range_requests ts.Disk.remap_config.AddLines([ - 'map http://slice/ http://127.0.0.1:{}/'.format(server.Variables.Port) + - ' @plugin=slice.so @pparam=--blockbytes-test=3 @pparam=--remap-host=cache_range_requests', - 'map http://cache_range_requests/ http://127.0.0.1:{}/'.format(server.Variables.Port) + + f'map http://slice/ http://127.0.0.1:{server.Variables.Port}/' + + ' @plugin=slice.so @pparam=--blockbytes-test=3 @pparam=--remap-host=crr', + f'map http://crr/ http://127.0.0.1:{server.Variables.Port}/' + ' @plugin=cache_range_requests.so @pparam=--consider-ims', + f'map http://slicehdr/ http://127.0.0.1:{server.Variables.Port}/' + + ' @plugin=slice.so @pparam=--blockbytes-test=3' + + ' @pparam=--remap-host=crrhdr @pparam=--crr-ims-header=crr-foo', + f'map http://crrhdr/ http://127.0.0.1:{server.Variables.Port}/' + ' @plugin=cache_range_requests.so @pparam=--ims-header=crr-foo', ]) ts.Disk.plugin_config.AddLine('xdebug.so') @@ -154,16 +170,16 @@ ps = tr.Processes.Default ps.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) ps.StartBefore(Test.Processes.ts) -ps.Command = curl_and_args + ' http://cache_range_requests/second -r 0-2 -H "uuid: etagnew-0"' +ps.Command = curl_and_args + ' http://crr/second -r 0-2 -H "uuid: etagnew-0"' ps.ReturnCode = 0 ps.Streams.stderr = "gold/bbb.gold" ps.Streams.stdout.Content = Testers.ContainsExpression("etagnew", "expected etagnew") tr.StillRunningAfter = ts -# 1 Test - Preload reference etagold-1 +# 1 Test - Preload slice etagold-1 tr = Test.AddTestRun("Preload slice etagold-1") ps = tr.Processes.Default -ps.Command = curl_and_args + ' http://cache_range_requests/second -r 3-5 -H "uuid: etagold-1"' +ps.Command = curl_and_args + ' http://crr/second -r 3-5 -H "uuid: etagold-1"' ps.ReturnCode = 0 ps.Streams.stderr = "gold/aa.gold" ps.Streams.stdout.Content = Testers.ContainsExpression("etagold", "expected etagold") @@ -264,7 +280,7 @@ # 4 Test - Preload reference etagold-0 tr = Test.AddTestRun("Preload reference etagold-0") ps = tr.Processes.Default -ps.Command = curl_and_args + ' http://cache_range_requests/reference -r 0-2 -H "uuid: etagold-0"' +ps.Command = curl_and_args + ' http://crr/reference -r 0-2 -H "uuid: etagold-0"' ps.ReturnCode = 0 ps.Streams.stderr = "gold/aaa.gold" ps.Streams.stdout.Content = Testers.ContainsExpression("etagold", "expected etagold") @@ -273,7 +289,7 @@ # 5 Test - Preload reference etagnew-1 tr = Test.AddTestRun("Preload slice etagnew-1") ps = tr.Processes.Default -ps.Command = curl_and_args + ' http://cache_range_requests/reference -r 3-5 -H "uuid: etagnew-1"' +ps.Command = curl_and_args + ' http://crr/reference -r 3-5 -H "uuid: etagnew-1"' ps.ReturnCode = 0 ps.Streams.stderr = "gold/bb.gold" ps.Streams.stdout.Content = Testers.ContainsExpression("etagnew", "expected etagnew") @@ -386,5 +402,28 @@ ps.Streams.stdout.Content = Testers.ContainsExpression("404 Not Found", "Expected 404") tr.StillRunningAfter = ts +# custom headers + +edt = datetime.datetime.fromtimestamp(time.time() + 100) +edate = to_httpdate(edt) + +# 12 Test - Preload reference etagold-1 +tr = Test.AddTestRun("Preload slice etagold-1") +ps = tr.Processes.Default +ps.Command = curl_and_args + f' http://crrhdr/second -r 3-5 -H "uuid: etagold-1" -H "crr-foo: {edate}"' +ps.ReturnCode = 0 +ps.Streams.stderr = "gold/aa.gold" +ps.Streams.stdout.Content = Testers.ContainsExpression("etagold", "expected etagold") +tr.StillRunningAfter = ts + +# 13 Test - Request second slice via slice plugin, with instructions to fetch new 2nd slice +tr = Test.AddTestRun("Request 2nd slice (expect refetch)") +ps = tr.Processes.Default +ps.Command = curl_and_args + ' http://slicehdr/second -r 3- -H "uuid: etagnew-1"' +ps.ReturnCode = 0 +ps.Streams.stderr = "gold/bb.gold" +ps.Streams.stdout.Content = Testers.ContainsExpression("etagnew", "expected etagnew") +tr.StillRunningAfter = ts + # Over riding the built in ERROR check since we expect to see logSliceErrors ts.Disk.diags_log.Content = Testers.ContainsExpression("logSliceError", "logSliceErrors generated")