Allow slice plugin to purge requests

doc/admin-guide/plugins/slice.en.rst

@@ -292,6 +292,21 @@ is faster since it does not visit any slices outside those needed to
 fulfill a request.  However this may still cause problems if the
 requested range was calculated from a newer version of the asset.
 
+Purge Requests
+--------------
+
+The slice plugin supports PURGE requests, discarding the requested object from cache.
+If a range is given in the client request, only the slice blocks from the
+requested range will be purged (if in cache). If not, all of the blocks will be discarded
+from the cache.
+
+If a block receives a 404, indicating the requested block to be purged is not in the cache,
+slice will not continue to purge the following blocks.
+
+The functionality works with `--ref-relative` both enabled and disabled. If `--ref-relative` is
+disabled (using slice 0 as the reference block), requesting to PURGE a block that does not have
+slice 0 in its range will still PURGE the slice 0 block, as the reference block is always processed.
+
 Important Notes
 ===============
 

plugins/experimental/slice/Config.h

@@ -45,9 +45,9 @@ struct Config {
   int m_paceerrsecs{0};   // -1 disable logging, 0 no pacing, max 60s
   int m_prefetchcount{0}; // 0 disables prefetching
   enum RefType { First, Relative };
-  RefType m_reftype{First};       // reference slice is relative to request
-  bool m_head_req{false};         // HEAD request
-  bool m_head_strip_range{false}; // strip range header for head requests
+  RefType m_reftype{First};           // reference slice is relative to request
+  const char *m_method_type{nullptr}; // type of header request
+  bool m_head_strip_range{false};     // strip range header for head requests
 
   std::string m_skip_header;
   std::string m_crr_ims_header;
@@ -71,6 +71,13 @@ struct Config {
     return None != m_regex_type;
   }
 
+  // Check if response only expects header
+  bool
+  onlyHeader() const
+  {
+    return (m_method_type == TS_HTTP_METHOD_HEAD || m_method_type == TS_HTTP_METHOD_PURGE);
+  }
+
   // If no null reg, true, otherwise check against regex
   bool matchesRegex(char const *const url, int const urllen) const;
 

plugins/experimental/slice/server.cc

@@ -111,8 +111,10 @@ handleFirstServerHeader(Data *const data, TSCont const contp)
     // Should run TSVIONSetBytes(output_io, hlen + bodybytes);
     int64_t const hlen = TSHttpHdrLengthGet(header.m_buffer, header.m_lochdr);
     int64_t const clen = contentLengthFrom(header);
-    if (data->m_config->m_head_req && TS_HTTP_STATUS_OK == header.status()) {
-      DEBUG_LOG("HEAD request stripped Range header: expects 200");
+    if (TS_HTTP_STATUS_OK == header.status() && data->m_config->onlyHeader()) {
+      DEBUG_LOG("HEAD/PURGE request stripped Range header: expects 200");
+      data->m_bytestosend   = hlen;
+      data->m_blockexpected = 0;
       TSVIONBytesSet(output_vio, hlen);
       TSHttpHdrPrint(header.m_buffer, header.m_lochdr, output_buf);
       data->m_bytessent = hlen;
@@ -218,7 +220,7 @@ handleFirstServerHeader(Data *const data, TSCont const contp)
   int const hbytes = TSHttpHdrLengthGet(header.m_buffer, header.m_lochdr);
 
   // HEAD request only sends header
-  if (data->m_config->m_head_req) {
+  if (data->m_config->onlyHeader()) {
     data->m_bytestosend   = hbytes;
     data->m_blockexpected = 0;
   } else {
@@ -362,13 +364,19 @@ handleNextServerHeader(Data *const data, TSCont const contp)
 
   switch (header.status()) {
   case TS_HTTP_STATUS_NOT_FOUND:
+    if (data->m_config->onlyHeader()) {
+      return false;
+    }
     // need to reissue reference slice
     logSliceError("404 internal block response (asset gone)", data, header);
     same = false;
     break;
   case TS_HTTP_STATUS_PARTIAL_CONTENT:
     break;
   default:
+    if (data->m_config->onlyHeader() && header.status() == TS_HTTP_STATUS_OK) {
+      return true;
+    }
     DEBUG_LOG("Non 206/404 internal block response encountered");
     return false;
     break;
@@ -632,7 +640,7 @@ handle_server_resp(TSCont contp, TSEvent event, Data *const data)
     // corner condition, good source header + 0 length aborted content
     // results in no header being read, just an EOS.
     // trying to delete the upstream will crash ATS (??)
-    if (0 == data->m_blockexpected && !data->m_config->m_head_req) {
+    if (0 == data->m_blockexpected && !data->m_config->onlyHeader()) {
       shutdown(contp, data); // this will crash if first block
       return;
     }
@@ -667,27 +675,28 @@ handle_server_resp(TSCont contp, TSEvent event, Data *const data)
       // Don't immediately request the next slice if the client
       // isn't keeping up
 
+      bool start_next_block = true;
       if (data->m_dnstream.m_write.isOpen()) {
-        bool start_next_block = true;
-
         // check throttle condition
         TSVIO const output_vio    = data->m_dnstream.m_write.m_vio;
         int64_t const output_done = TSVIONDoneGet(output_vio);
         int64_t const output_sent = data->m_bytessent;
         int64_t const threshout   = data->m_config->m_blockbytes;
         int64_t const buffered    = output_sent - output_done;
 
-        if (threshout < buffered) {
+        if (threshout < buffered && !data->m_config->onlyHeader()) {
           start_next_block = false;
           DEBUG_LOG("%p handle_server_resp: throttling %" PRId64, data, buffered);
         }
-
-        if (start_next_block) {
-          if (!request_block(contp, data)) {
-            data->m_blockstate = BlockState::Fail;
-            abort(contp, data);
-            return;
-          }
+      } else if (!data->m_config->onlyHeader()) {
+        // client doesn't need to accept server response for PURGE requests
+        start_next_block = false;
+      }
+      if (start_next_block) {
+        if (!request_block(contp, data)) {
+          data->m_blockstate = BlockState::Fail;
+          abort(contp, data);
+          return;
         }
       }
     } else {

plugins/experimental/slice/slice.cc

@@ -41,7 +41,7 @@ read_request(TSHttpTxn txnp, Config *const config)
   hdrmgr.populateFrom(txnp, TSHttpTxnClientReqGet);
   HttpHeader const header(hdrmgr.m_buffer, hdrmgr.m_lochdr);
 
-  if (TS_HTTP_METHOD_GET == header.method() || TS_HTTP_METHOD_HEAD == header.method()) {
+  if (TS_HTTP_METHOD_GET == header.method() || TS_HTTP_METHOD_HEAD == header.method() || TS_HTTP_METHOD_PURGE == header.method()) {
     if (!header.hasKey(config->m_skip_header.data(), config->m_skip_header.size())) {
       // check if any previous plugin has monkeyed with the transaction status
       TSHttpStatus const txnstat = TSHttpTxnStatusGet(txnp);
@@ -50,8 +50,8 @@ read_request(TSHttpTxn txnp, Config *const config)
         return false;
       }
 
-      // set HEAD config to only expect header response
-      config->m_head_req = (TS_HTTP_METHOD_HEAD == header.method());
+      // set header method config to only expect header response
+      config->m_method_type = header.method();
 
       if (config->hasRegex()) {
         int urllen         = 0;

plugins/experimental/slice/transfer.h

@@ -19,6 +19,7 @@
 #pragma once
 
 #include "Data.h"
+#include "Config.h"
 
 /** Functions to deal with the connection to the client.
  * Body content transfers are handled by the client.

plugins/experimental/slice/util.cc

@@ -80,7 +80,7 @@ request_block(TSCont contp, Data *const data)
   HttpHeader header(data->m_req_hdrmgr.m_buffer, data->m_req_hdrmgr.m_lochdr);
 
   // if configured, remove range header from head requests
-  if (data->m_config->m_head_req && data->m_config->m_head_strip_range) {
+  if (data->m_config->m_method_type == TS_HTTP_METHOD_HEAD && data->m_config->m_head_strip_range) {
     header.removeKey(TS_MIME_FIELD_RANGE, TS_MIME_LEN_RANGE);
   } else {
     // add/set sub range key and add slicer tag

tests/gold_tests/pluginTest/slice/replay/slice_purge.replay.yaml

@@ -0,0 +1,136 @@
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+meta:
+  version: "1.0"
+
+sessions:
+- transactions:
+  - client-request:
+      method: "GET"
+      version: "1.1"
+      url: /ref/block
+      headers:
+        fields:
+          - [ Host, example.com ]
+          - [ uuid, 1 ]
+          - [ Range, bytes=0-9 ]
+    server-response:
+      status: 206
+      reason: OK
+      headers:
+        fields:
+          - [ Content-Length, 10 ]
+          - [ Content-Range, { value: "bytes 0-9/16", as: equal}]
+          - [ Cache-Control, max-age=300 ]
+          - [ X-Response, cached_response ]
+    proxy-response:
+      status: 206
+      headers:
+        fields:
+          - [ Content-Length, { value: "10", as: equal} ]
+          - [ Content-Range, { value: "bytes 0-9/16", as: equal}]
+          - [ X-Response, { value: cached_response, as: equal} ]
+
+  - client-request:
+      method: "GET"
+      version: "1.1"
+      url: /ref/block
+      headers:
+        fields:
+          - [ Host, example.com ]
+          - [ uuid, 2 ]
+          - [ Range, bytes=10-19 ]
+    server-response:
+      status: 206
+      reason: OK
+      headers:
+        fields:
+          - [ Content-Length, 6 ]
+          - [ Content-Range, { value: "bytes 10-15/16", as: equal}]
+          - [ Cache-Control, max-age=300 ]
+          - [ X-Response, cached_response ]
+    proxy-response:
+      status: 206
+      headers:
+        fields:
+          - [ Content-Length, { value: "6", as: equal} ]
+          - [ Content-Range, { value: "bytes 10-15/16", as: equal}]
+          - [ X-Response, { value: cached_response, as: equal} ]
+
+  - client-request:
+      method: "GET"
+      version: "1.1"
+      url: /ref/block
+      headers:
+        fields:
+          - [ Host, example.com ]
+          - [ Range, bytes=10-15 ]
+          - [ uuid, 3 ]
+    server-response:
+      status: 500
+      headers:
+        fields:
+          - [ Content-Length, 0 ]
+          - [ X-Response, internal_server_error ]
+    proxy-response:
+      status: 206
+      headers:
+        fields:
+          - [ Content-Range, { value: "bytes 10-15/16", as: equal}]
+          - [ X-Response, { value: cached_response, as: equal} ]
+
+  - client-request:
+      method: "PURGE"
+      version: "1.1"
+      url: /ref/block
+      headers:
+        fields:
+          - [ Host, example.com ]
+          - [ uuid, 4 ]
+    server-response:
+      status: 200
+      reason: OK
+      headers:
+        fields:
+          - [ Content-Length, 0 ]
+    proxy-response:
+      status: 200
+      headers:
+        fields:
+          - [ Content-Length, { value: "0", as: equal} ]
+
+  - client-request:
+      method: "GET"
+      version: "1.1"
+      url: /ref/block
+      headers:
+        fields:
+          - [ Host, example.com ]
+          - [ Range, bytes=10-15 ]
+          - [ uuid, 5 ]
+    server-response:
+      status: 404
+      headers:
+        fields:
+          - [ Content-Length, 0 ]
+          - [ X-Response, internal_server_error ]
+    proxy-response:
+      status: 404
+      headers:
+        fields:
+          - [ Content-Length, { value: "0", as: equal} ]
+          - [ X-Response, { value: internal_server_error, as: equal} ]