diff --git a/doc/admin-guide/plugins/slice.en.rst b/doc/admin-guide/plugins/slice.en.rst index 383d30fdc7d..9fa2832d8a3 100644 --- a/doc/admin-guide/plugins/slice.en.rst +++ b/doc/admin-guide/plugins/slice.en.rst @@ -292,6 +292,21 @@ is faster since it does not visit any slices outside those needed to fulfill a request. However this may still cause problems if the requested range was calculated from a newer version of the asset. +Purge Requests +-------------- + +The slice plugin supports PURGE requests, discarding the requested object from cache. +If a range is given in the client request, only the slice blocks from the +requested range will be purged (if in cache). If not, all of the blocks will be discarded +from the cache. + +If a block receives a 404, indicating the requested block to be purged is not in the cache, +slice will not continue to purge the following blocks. + +The functionality works with `--ref-relative` both enabled and disabled. If `--ref-relative` is +disabled (using slice 0 as the reference block), requesting to PURGE a block that does not have +slice 0 in its range will still PURGE the slice 0 block, as the reference block is always processed. + Important Notes =============== diff --git a/plugins/experimental/slice/Config.h b/plugins/experimental/slice/Config.h index a63716ee967..430495333a6 100644 --- a/plugins/experimental/slice/Config.h +++ b/plugins/experimental/slice/Config.h @@ -45,9 +45,9 @@ struct Config { int m_paceerrsecs{0}; // -1 disable logging, 0 no pacing, max 60s int m_prefetchcount{0}; // 0 disables prefetching enum RefType { First, Relative }; - RefType m_reftype{First}; // reference slice is relative to request - bool m_head_req{false}; // HEAD request - bool m_head_strip_range{false}; // strip range header for head requests + RefType m_reftype{First}; // reference slice is relative to request + const char *m_method_type{nullptr}; // type of header request + bool m_head_strip_range{false}; // strip range header for head requests std::string m_skip_header; std::string m_crr_ims_header; @@ -71,6 +71,13 @@ struct Config { return None != m_regex_type; } + // Check if response only expects header + bool + onlyHeader() const + { + return (m_method_type == TS_HTTP_METHOD_HEAD || m_method_type == TS_HTTP_METHOD_PURGE); + } + // If no null reg, true, otherwise check against regex bool matchesRegex(char const *const url, int const urllen) const; diff --git a/plugins/experimental/slice/server.cc b/plugins/experimental/slice/server.cc index 25baf75080b..b0ebb291e92 100644 --- a/plugins/experimental/slice/server.cc +++ b/plugins/experimental/slice/server.cc @@ -111,8 +111,10 @@ handleFirstServerHeader(Data *const data, TSCont const contp) // Should run TSVIONSetBytes(output_io, hlen + bodybytes); int64_t const hlen = TSHttpHdrLengthGet(header.m_buffer, header.m_lochdr); int64_t const clen = contentLengthFrom(header); - if (data->m_config->m_head_req && TS_HTTP_STATUS_OK == header.status()) { - DEBUG_LOG("HEAD request stripped Range header: expects 200"); + if (TS_HTTP_STATUS_OK == header.status() && data->m_config->onlyHeader()) { + DEBUG_LOG("HEAD/PURGE request stripped Range header: expects 200"); + data->m_bytestosend = hlen; + data->m_blockexpected = 0; TSVIONBytesSet(output_vio, hlen); TSHttpHdrPrint(header.m_buffer, header.m_lochdr, output_buf); data->m_bytessent = hlen; @@ -218,7 +220,7 @@ handleFirstServerHeader(Data *const data, TSCont const contp) int const hbytes = TSHttpHdrLengthGet(header.m_buffer, header.m_lochdr); // HEAD request only sends header - if (data->m_config->m_head_req) { + if (data->m_config->onlyHeader()) { data->m_bytestosend = hbytes; data->m_blockexpected = 0; } else { @@ -362,6 +364,9 @@ handleNextServerHeader(Data *const data, TSCont const contp) switch (header.status()) { case TS_HTTP_STATUS_NOT_FOUND: + if (data->m_config->onlyHeader()) { + return false; + } // need to reissue reference slice logSliceError("404 internal block response (asset gone)", data, header); same = false; @@ -369,6 +374,9 @@ handleNextServerHeader(Data *const data, TSCont const contp) case TS_HTTP_STATUS_PARTIAL_CONTENT: break; default: + if (data->m_config->onlyHeader() && header.status() == TS_HTTP_STATUS_OK) { + return true; + } DEBUG_LOG("Non 206/404 internal block response encountered"); return false; break; @@ -632,7 +640,7 @@ handle_server_resp(TSCont contp, TSEvent event, Data *const data) // corner condition, good source header + 0 length aborted content // results in no header being read, just an EOS. // trying to delete the upstream will crash ATS (??) - if (0 == data->m_blockexpected && !data->m_config->m_head_req) { + if (0 == data->m_blockexpected && !data->m_config->onlyHeader()) { shutdown(contp, data); // this will crash if first block return; } @@ -667,9 +675,8 @@ handle_server_resp(TSCont contp, TSEvent event, Data *const data) // Don't immediately request the next slice if the client // isn't keeping up + bool start_next_block = true; if (data->m_dnstream.m_write.isOpen()) { - bool start_next_block = true; - // check throttle condition TSVIO const output_vio = data->m_dnstream.m_write.m_vio; int64_t const output_done = TSVIONDoneGet(output_vio); @@ -677,17 +684,19 @@ handle_server_resp(TSCont contp, TSEvent event, Data *const data) int64_t const threshout = data->m_config->m_blockbytes; int64_t const buffered = output_sent - output_done; - if (threshout < buffered) { + if (threshout < buffered && !data->m_config->onlyHeader()) { start_next_block = false; DEBUG_LOG("%p handle_server_resp: throttling %" PRId64, data, buffered); } - - if (start_next_block) { - if (!request_block(contp, data)) { - data->m_blockstate = BlockState::Fail; - abort(contp, data); - return; - } + } else if (!data->m_config->onlyHeader()) { + // client doesn't need to accept server response for PURGE requests + start_next_block = false; + } + if (start_next_block) { + if (!request_block(contp, data)) { + data->m_blockstate = BlockState::Fail; + abort(contp, data); + return; } } } else { diff --git a/plugins/experimental/slice/slice.cc b/plugins/experimental/slice/slice.cc index 90477ce2f93..09d2847383f 100644 --- a/plugins/experimental/slice/slice.cc +++ b/plugins/experimental/slice/slice.cc @@ -41,7 +41,7 @@ read_request(TSHttpTxn txnp, Config *const config) hdrmgr.populateFrom(txnp, TSHttpTxnClientReqGet); HttpHeader const header(hdrmgr.m_buffer, hdrmgr.m_lochdr); - if (TS_HTTP_METHOD_GET == header.method() || TS_HTTP_METHOD_HEAD == header.method()) { + if (TS_HTTP_METHOD_GET == header.method() || TS_HTTP_METHOD_HEAD == header.method() || TS_HTTP_METHOD_PURGE == header.method()) { if (!header.hasKey(config->m_skip_header.data(), config->m_skip_header.size())) { // check if any previous plugin has monkeyed with the transaction status TSHttpStatus const txnstat = TSHttpTxnStatusGet(txnp); @@ -50,8 +50,8 @@ read_request(TSHttpTxn txnp, Config *const config) return false; } - // set HEAD config to only expect header response - config->m_head_req = (TS_HTTP_METHOD_HEAD == header.method()); + // set header method config to only expect header response + config->m_method_type = header.method(); if (config->hasRegex()) { int urllen = 0; diff --git a/plugins/experimental/slice/transfer.h b/plugins/experimental/slice/transfer.h index a63bb243a2d..27337f80502 100644 --- a/plugins/experimental/slice/transfer.h +++ b/plugins/experimental/slice/transfer.h @@ -19,6 +19,7 @@ #pragma once #include "Data.h" +#include "Config.h" /** Functions to deal with the connection to the client. * Body content transfers are handled by the client. diff --git a/plugins/experimental/slice/util.cc b/plugins/experimental/slice/util.cc index b91636b02ca..71d99b98698 100644 --- a/plugins/experimental/slice/util.cc +++ b/plugins/experimental/slice/util.cc @@ -80,7 +80,7 @@ request_block(TSCont contp, Data *const data) HttpHeader header(data->m_req_hdrmgr.m_buffer, data->m_req_hdrmgr.m_lochdr); // if configured, remove range header from head requests - if (data->m_config->m_head_req && data->m_config->m_head_strip_range) { + if (data->m_config->m_method_type == TS_HTTP_METHOD_HEAD && data->m_config->m_head_strip_range) { header.removeKey(TS_MIME_FIELD_RANGE, TS_MIME_LEN_RANGE); } else { // add/set sub range key and add slicer tag diff --git a/tests/gold_tests/pluginTest/slice/replay/slice_purge.replay.yaml b/tests/gold_tests/pluginTest/slice/replay/slice_purge.replay.yaml new file mode 100644 index 00000000000..33e2167ce8e --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/replay/slice_purge.replay.yaml @@ -0,0 +1,136 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +meta: + version: "1.0" + +sessions: +- transactions: + - client-request: + method: "GET" + version: "1.1" + url: /ref/block + headers: + fields: + - [ Host, example.com ] + - [ uuid, 1 ] + - [ Range, bytes=0-9 ] + server-response: + status: 206 + reason: OK + headers: + fields: + - [ Content-Length, 10 ] + - [ Content-Range, { value: "bytes 0-9/16", as: equal}] + - [ Cache-Control, max-age=300 ] + - [ X-Response, cached_response ] + proxy-response: + status: 206 + headers: + fields: + - [ Content-Length, { value: "10", as: equal} ] + - [ Content-Range, { value: "bytes 0-9/16", as: equal}] + - [ X-Response, { value: cached_response, as: equal} ] + + - client-request: + method: "GET" + version: "1.1" + url: /ref/block + headers: + fields: + - [ Host, example.com ] + - [ uuid, 2 ] + - [ Range, bytes=10-19 ] + server-response: + status: 206 + reason: OK + headers: + fields: + - [ Content-Length, 6 ] + - [ Content-Range, { value: "bytes 10-15/16", as: equal}] + - [ Cache-Control, max-age=300 ] + - [ X-Response, cached_response ] + proxy-response: + status: 206 + headers: + fields: + - [ Content-Length, { value: "6", as: equal} ] + - [ Content-Range, { value: "bytes 10-15/16", as: equal}] + - [ X-Response, { value: cached_response, as: equal} ] + + - client-request: + method: "GET" + version: "1.1" + url: /ref/block + headers: + fields: + - [ Host, example.com ] + - [ Range, bytes=10-15 ] + - [ uuid, 3 ] + server-response: + status: 500 + headers: + fields: + - [ Content-Length, 0 ] + - [ X-Response, internal_server_error ] + proxy-response: + status: 206 + headers: + fields: + - [ Content-Range, { value: "bytes 10-15/16", as: equal}] + - [ X-Response, { value: cached_response, as: equal} ] + + - client-request: + method: "PURGE" + version: "1.1" + url: /ref/block + headers: + fields: + - [ Host, example.com ] + - [ uuid, 4 ] + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Content-Length, 0 ] + proxy-response: + status: 200 + headers: + fields: + - [ Content-Length, { value: "0", as: equal} ] + + - client-request: + method: "GET" + version: "1.1" + url: /ref/block + headers: + fields: + - [ Host, example.com ] + - [ Range, bytes=10-15 ] + - [ uuid, 5 ] + server-response: + status: 404 + headers: + fields: + - [ Content-Length, 0 ] + - [ X-Response, internal_server_error ] + proxy-response: + status: 404 + headers: + fields: + - [ Content-Length, { value: "0", as: equal} ] + - [ X-Response, { value: internal_server_error, as: equal} ] \ No newline at end of file diff --git a/tests/gold_tests/pluginTest/slice/replay/slice_purge_no_ref.replay.yaml b/tests/gold_tests/pluginTest/slice/replay/slice_purge_no_ref.replay.yaml new file mode 100644 index 00000000000..9b755a85205 --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/replay/slice_purge_no_ref.replay.yaml @@ -0,0 +1,115 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +meta: + version: "1.0" + +sessions: +- transactions: + - client-request: + method: "GET" + version: "1.1" + url: /ref/block + headers: + fields: + - [ Host, example.com ] + - [ uuid, 1 ] + - [ Range, bytes=0-9 ] + server-response: + status: 206 + reason: OK + headers: + fields: + - [ Content-Length, 10 ] + - [ Content-Range, { value: "bytes 0-9/16", as: equal}] + - [ Cache-Control, max-age=300 ] + - [ X-Response, cached_response ] + proxy-response: + status: 206 + headers: + fields: + - [ Content-Length, { value: "10", as: equal} ] + - [ Content-Range, { value: "bytes 0-9/16", as: equal}] + - [ X-Response, { value: cached_response, as: equal} ] + + - client-request: + method: "GET" + version: "1.1" + url: /ref/block + headers: + fields: + - [ Host, example.com ] + - [ uuid, 2 ] + - [ Range, bytes=10-19 ] + server-response: + status: 206 + reason: OK + headers: + fields: + - [ Content-Length, 6 ] + - [ Content-Range, { value: "bytes 10-15/16", as: equal}] + - [ Cache-Control, max-age=300 ] + - [ X-Response, cached_response ] + proxy-response: + status: 206 + headers: + fields: + - [ Content-Length, { value: "6", as: equal} ] + - [ Content-Range, { value: "bytes 10-15/16", as: equal}] + - [ X-Response, { value: cached_response, as: equal} ] + + - client-request: + method: "PURGE" + version: "1.1" + url: /ref/block + headers: + fields: + - [ Host, example.com ] + - [ Range, bytes=10-15 ] + - [ uuid, 3 ] + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Content-Length, 0 ] + proxy-response: + status: 200 + headers: + fields: + - [ Content-Length, { value: "0", as: equal} ] + + - client-request: + method: "GET" + version: "1.1" + url: /ref/block + headers: + fields: + - [ Host, example.com ] + - [ Range, bytes=0-5 ] + - [ uuid, 4 ] + server-response: + status: 404 + headers: + fields: + - [ Content-Length, 0 ] + - [ X-Response, internal_server_error ] + proxy-response: + status: 206 + headers: + fields: + - [ Content-Range, { value: "bytes 0-5/16", as: equal}] + - [ X-Response, { value: cached_response, as: equal} ] \ No newline at end of file diff --git a/tests/gold_tests/pluginTest/slice/replay/slice_purge_ref.replay.yaml b/tests/gold_tests/pluginTest/slice/replay/slice_purge_ref.replay.yaml new file mode 100644 index 00000000000..1d14f051008 --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/replay/slice_purge_ref.replay.yaml @@ -0,0 +1,115 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +meta: + version: "1.0" + +sessions: +- transactions: + - client-request: + method: "GET" + version: "1.1" + url: /ref/block + headers: + fields: + - [ Host, example.com ] + - [ uuid, 1 ] + - [ Range, bytes=0-9 ] + server-response: + status: 206 + reason: OK + headers: + fields: + - [ Content-Length, 10 ] + - [ Content-Range, { value: "bytes 0-9/16", as: equal}] + - [ Cache-Control, max-age=300 ] + - [ X-Response, cached_response ] + proxy-response: + status: 206 + headers: + fields: + - [ Content-Length, { value: "10", as: equal} ] + - [ Content-Range, { value: "bytes 0-9/16", as: equal}] + - [ X-Response, { value: cached_response, as: equal} ] + + - client-request: + method: "GET" + version: "1.1" + url: /ref/block + headers: + fields: + - [ Host, example.com ] + - [ uuid, 2 ] + - [ Range, bytes=10-19 ] + server-response: + status: 206 + reason: OK + headers: + fields: + - [ Content-Length, 6 ] + - [ Content-Range, { value: "bytes 10-15/16", as: equal}] + - [ Cache-Control, max-age=300 ] + - [ X-Response, cached_response ] + proxy-response: + status: 206 + headers: + fields: + - [ Content-Length, { value: "6", as: equal} ] + - [ Content-Range, { value: "bytes 10-15/16", as: equal}] + - [ X-Response, { value: cached_response, as: equal} ] + + - client-request: + method: "PURGE" + version: "1.1" + url: /ref/block + headers: + fields: + - [ Host, example.com ] + - [ Range, bytes=10-15 ] + - [ uuid, 3 ] + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Content-Length, 0 ] + proxy-response: + status: 200 + headers: + fields: + - [ Content-Length, { value: "0", as: equal} ] + + - client-request: + method: "GET" + version: "1.1" + url: /ref/block + headers: + fields: + - [ Host, example.com ] + - [ Range, bytes=0-9 ] + - [ uuid, 5 ] + server-response: + status: 404 + headers: + fields: + - [ Content-Length, 0 ] + - [ X-Response, internal_server_error ] + proxy-response: + status: 404 + headers: + fields: + - [ Content-Length, { value: "0", as: equal} ] + - [ X-Response, { value: internal_server_error, as: equal} ] \ No newline at end of file diff --git a/tests/gold_tests/pluginTest/slice/slice_purge.test.py b/tests/gold_tests/pluginTest/slice/slice_purge.test.py new file mode 100644 index 00000000000..5e865f17afc --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/slice_purge.test.py @@ -0,0 +1,100 @@ +''' +Verify ATS slice plugin config accepts PURGE requests +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +Verify ATS slice plugin config accepts PURGE requests +''' +Test.SkipUnless( + Condition.PluginExists('slice.so'), + Condition.PluginExists('cache_range_requests.so'), +) + + +class SlicePurgeRequestTest: + replay_file = "replay/slice_purge.replay.yaml" + replay_ref_file = "replay/slice_purge_ref.replay.yaml" + replay_no_ref_file = "replay/slice_purge_no_ref.replay.yaml" + + def __init__(self, ref_block, num): + """Initialize the Test processes for the test runs.""" + self._ref_block = ref_block + self._num = num + self._server = Test.MakeVerifierServerProcess(f"server_{self._num}", SlicePurgeRequestTest.replay_file) + self._configure_trafficserver() + + def _configure_trafficserver(self): + """Configure Traffic Server.""" + self._ts = Test.MakeATSProcess(f"ts_{self._num}", enable_cache=True) + + if (self._ref_block): + self._ts.Disk.remap_config.AddLines([ + f"map /ref/block http://127.0.0.1:{self._server.Variables.http_port} \ + @plugin=slice.so @pparam=--blockbytes-test=10 \ + @plugin=cache_range_requests.so", + ]) + else: + self._ts.Disk.remap_config.AddLines([ + f"map /ref/block http://127.0.0.1:{self._server.Variables.http_port} \ + @plugin=slice.so @pparam=--blockbytes-test=10 @pparam=--ref-relative \ + @plugin=cache_range_requests.so", + ]) + + self._ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'http|slice|cache_range_requests', + }) + + def slice_purge(self): + tr = Test.AddTestRun() + + tr.AddVerifierClientProcess( + f"client_{self._num}", + SlicePurgeRequestTest.replay_file, + http_ports=[self._ts.Variables.port]) + + tr.Processes.Default.StartBefore(self._server) + tr.Processes.Default.StartBefore(self._ts) + + def slice_purge_ref(self): + tr = Test.AddTestRun() + + tr.AddVerifierClientProcess( + "client_ref", + SlicePurgeRequestTest.replay_ref_file, + http_ports=[self._ts.Variables.port]) + + tr.Processes.Default.StartBefore(self._server) + tr.Processes.Default.StartBefore(self._ts) + + def slice_purge_no_ref(self): + tr = Test.AddTestRun() + + tr.AddVerifierClientProcess( + "client_no_ref", + SlicePurgeRequestTest.replay_no_ref_file, + http_ports=[self._ts.Variables.port]) + + tr.Processes.Default.StartBefore(self._server) + tr.Processes.Default.StartBefore(self._ts) + + +SlicePurgeRequestTest(True, 0).slice_purge() +SlicePurgeRequestTest(True, 1).slice_purge_ref() +SlicePurgeRequestTest(False, 2).slice_purge() +SlicePurgeRequestTest(False, 3).slice_purge_no_ref()