Add cachekey latest features to 8.1.x

doc/admin-guide/plugins/index.en.rst

@@ -75,8 +75,8 @@ Plugins that are considered stable are installed by default in |TS| releases.
 :doc:`Background Fetch <background_fetch.en>`
    Proactively fetch content from Origin in a way that it will fill the object into cache.
 
-:doc:`Cache Key Manipulation <cachekey.en>`
-   Allows some common cache key manipulations based on various HTTP request elements.
+:doc:`Cache Key and Parent Selection URL Manipulation <cachekey.en>`
+   Allows some common cache key or parent selection URL manipulations based on various HTTP request elements.
 
 :doc:`Cache Promotion Policies <cache_promote.en>`
    Allows for control over which assets should be written to cache, or not.

plugins/cachekey/README.md

@@ -1,7 +1,7 @@
 # Description
 This plugin allows some common cache key manipulations based on various HTTP request elements.  It can
 
-* sort query parameters to prevent query parameters reordereding from being a cache miss
+* sort query parameters to prevent query parameters reordering from being a cache miss
 * ignore specific query parameters from the cache key by name or regular expression
 * ignore all query parameters from the cache key
 * only use specific query parameters in the cache key by name or regular expression

plugins/cachekey/cachekey.cc

@@ -23,6 +23,7 @@
 
 #include <cstring> /* strlen() */
 #include <sstream> /* istringstream */
+#include <utility>
 #include "cachekey.h"
 
 static void
@@ -187,15 +188,60 @@ getUri(TSMBuffer buf, TSMLoc url)
   return uri;
 }
 
+static String
+getCanonicalUrl(TSMBuffer buf, TSMLoc url, bool canonicalPrefix, bool provideDefaultKey)
+{
+  String canonicalUrl;
+
+  String scheme;
+  int schemeLen;
+  const char *schemePtr = TSUrlSchemeGet(buf, url, &schemeLen);
+  if (nullptr != schemePtr && 0 != schemeLen) {
+    scheme.assign(schemePtr, schemeLen);
+  } else {
+    CacheKeyError("failed to get scheme");
+    return canonicalUrl;
+  }
+
+  String host;
+  int hostLen;
+  const char *hostPtr = TSUrlHostGet(buf, url, &hostLen);
+  if (nullptr != hostPtr && 0 != hostLen) {
+    host.assign(hostPtr, hostLen);
+  } else {
+    CacheKeyError("failed to get host");
+    return canonicalUrl;
+  }
+
+  String port;
+  int portInt = TSUrlPortGet(buf, url);
+  ::append(port, portInt);
+
+  if (canonicalPrefix) {
+    /* return the same for both regex input or default key, results in 'scheme://host:port' */
+    canonicalUrl.assign(scheme).append("://").append(host).append(":").append(port);
+  } else {
+    if (provideDefaultKey) {
+      /* return the key default - results in '/host/port' */
+      canonicalUrl.assign("/").append(host).append("/").append(port);
+    } else {
+      /* return regex input string - results in 'host:port' (use-case kept for compatibility reasons) */
+      canonicalUrl.assign(host).append(":").append(port);
+    }
+  }
+
+  return canonicalUrl;
+}
+
 /**
  * @brief Constructor setting up the cache key prefix, initializing request info.
  * @param txn transaction handle.
  * @param separator cache key elements separator
  * @param uriType type of the URI used to create the cachekey ("remap" or "pristine")
  * @param rri remap request info
  */
-CacheKey::CacheKey(TSHttpTxn txn, String separator, CacheKeyUriType uriType, TSRemapRequestInfo *rri)
-  : _txn(txn), _separator(separator), _uriType(uriType)
+CacheKey::CacheKey(TSHttpTxn txn, String separator, CacheKeyUriType uriType, CacheKeyKeyType keyType, TSRemapRequestInfo *rri)
+  : _txn(txn), _separator(std::move(separator)), _uriType(uriType), _keyType(keyType)
 {
   _key.reserve(512);
 
@@ -204,8 +250,9 @@ CacheKey::CacheKey(TSHttpTxn txn, String separator, CacheKeyUriType uriType, TSR
   /* Get the URI and header to base the cachekey on.
    * @TODO it might make sense to add more supported URI types */
 
+  CacheKeyDebug("setting %s from a %s plugin", getCacheKeyKeyTypeName(_keyType), _remap ? "remap" : "global");
+
   if (_remap) {
-    CacheKeyDebug("setting cache key from a remap plugin");
     if (PRISTINE == _uriType) {
       if (TS_SUCCESS != TSHttpTxnPristineUrlGet(_txn, &_buf, &_url)) {
         /* Failing here is unlikely. No action seems the only reasonable thing to do from within this plug-in */
@@ -220,7 +267,6 @@ CacheKey::CacheKey(TSHttpTxn txn, String separator, CacheKeyUriType uriType, TSR
     }
     _hdrs = rri->requestHdrp;
   } else {
-    CacheKeyDebug("setting cache key from a global plugin");
     if (TS_SUCCESS != TSHttpTxnClientReqGet(_txn, &_buf, &_hdrs)) {
       /* Failing here is unlikely. No action seems the only reasonable thing to do from within this plug-in */
       CacheKeyError("failed to get client request handle");
@@ -288,6 +334,16 @@ CacheKey::append(const String &s)
   ::appendEncoded(_key, s.data(), s.size());
 }
 
+void
+CacheKey::append(const String &s, bool useSeparator)
+{
+  if (useSeparator) {
+    append(s);
+  } else {
+    _key.append(s);
+  }
+}
+
 /**
  * @brief Append null-terminated C-style string to the key.
  * @param s null-terminated C-style string.
@@ -318,42 +374,31 @@ CacheKey::append(const char *s, unsigned n)
  * @param prefix if not empty string will append the static prefix to the cache key.
  * @param prefixCapture if not empty will append regex capture/replacement from the host:port.
  * @param prefixCaptureUri if not empty will append regex capture/replacement from the whole URI.
+ * @param canonicalPrefix false - use 'host:port' as starting point of all transformations, true - use 'scheme://host:port'
  * @note if both prefix and pattern are not empty prefix will be added first, followed by the results from pattern.
  */
 void
-CacheKey::appendPrefix(const String &prefix, Pattern &prefixCapture, Pattern &prefixCaptureUri)
+CacheKey::appendPrefix(const String &prefix, Pattern &prefixCapture, Pattern &prefixCaptureUri, bool canonicalPrefix)
 {
-  // "true" would mean that the plugin config meant to override the default prefix (host:port).
+  // "true" would mean that the plugin config meant to override the default prefix, "false" means use default.
   bool customPrefix = false;
-  String host;
-  int port = 0;
+
+  /* For all the following operations if a canonical prefix is required then appned to the key with no separator
+   * to leave the door open for potential valid host name formed in the final resulting cache key. */
 
   if (!prefix.empty()) {
     customPrefix = true;
-    append(prefix);
+    append(prefix, /* useSeparator */ !canonicalPrefix);
     CacheKeyDebug("added static prefix, key: '%s'", _key.c_str());
   }
 
-  int hostLen;
-  const char *hostPtr = TSUrlHostGet(_buf, _url, &hostLen);
-  if (nullptr != hostPtr && 0 != hostLen) {
-    host.assign(hostPtr, hostLen);
-  } else {
-    CacheKeyError("failed to get host");
-  }
-  port = TSUrlPortGet(_buf, _url);
-
   if (!prefixCapture.empty()) {
     customPrefix = true;
 
-    String hostAndPort;
-    hostAndPort.append(host).append(":");
-    ::append(hostAndPort, port);
-
     StringVector captures;
-    if (prefixCapture.process(hostAndPort, captures)) {
+    if (prefixCapture.process(getCanonicalUrl(_buf, _url, canonicalPrefix, /* provideDefaultKey */ false), captures)) {
       for (auto &capture : captures) {
-        append(capture);
+        append(capture, /* useSeparator */ !canonicalPrefix);
       }
       CacheKeyDebug("added host:port capture prefix, key: '%s'", _key.c_str());
     }
@@ -367,16 +412,16 @@ CacheKey::appendPrefix(const String &prefix, Pattern &prefixCapture, Pattern &pr
       StringVector captures;
       if (prefixCaptureUri.process(uri, captures)) {
         for (auto &capture : captures) {
-          append(capture);
+          append(capture, /* useSeparator */ !canonicalPrefix);
         }
         CacheKeyDebug("added URI capture prefix, key: '%s'", _key.c_str());
       }
     }
   }
 
   if (!customPrefix) {
-    append(host);
-    append(port);
+    /* nothing was customized => default prefix */
+    append(getCanonicalUrl(_buf, _url, canonicalPrefix, /* provideDefaultKey */ true), /* useSeparator */ false);
     CacheKeyDebug("added default prefix, key: '%s'", _key.c_str());
   }
 }
@@ -700,24 +745,67 @@ CacheKey::appendUaClass(Classifier &classifier)
 bool
 CacheKey::finalize() const
 {
-  bool res = true;
-  CacheKeyDebug("finalizing cache key '%s' from a %s plugin", _key.c_str(), (_remap ? "remap" : "global"));
-  if (TS_SUCCESS != TSCacheUrlSet(_txn, &(_key[0]), _key.size())) {
-    int len;
-    char *url = TSHttpTxnEffectiveUrlStringGet(_txn, &len);
-    if (nullptr != url) {
+  bool res = false;
+  String msg;
+
+  CacheKeyDebug("finalizing %s '%s' from a %s plugin", getCacheKeyKeyTypeName(_keyType), _key.c_str(),
+                (_remap ? "remap" : "global"));
+  switch (_keyType) {
+  case CACHE_KEY: {
+    if (TS_SUCCESS == TSCacheUrlSet(_txn, &(_key[0]), _key.size())) {
+      /* Set cache key succesfully */
+      msg.assign("set cache key to ").append(_key);
+      res = true;
+    } else {
       if (_remap) {
         /* Remap instance. Always runs first by design (before TS_HTTP_POST_REMAP_HOOK) */
-        CacheKeyError("failed to set cache key for url %.*s", len, url);
+        msg.assign("failed to set cache key");
       } else {
         /* Global instance. We would fail and get here if a per-remap instance has already set the cache key
          * (currently TSCacheUrlSet() can be called only once successfully). Don't error, just debug.
          * @todo avoid the consecutive attempts and error only on unexpected failures. */
-        CacheKeyDebug("failed to set cache key for url %.*s", len, url);
+        msg.assign("failed to set cache key");
       }
+    }
+  } break;
+  case PARENT_SELECTION_URL: {
+    /* parent selection */
+    const char *start = _key.c_str();
+    const char *end   = _key.c_str() + _key.length();
+    TSMLoc new_url_loc;
+    if (TS_SUCCESS == TSUrlCreate(_buf, &new_url_loc)) {
+      if (TS_PARSE_DONE == TSUrlParse(_buf, new_url_loc, &start, end)) {
+        if (TS_SUCCESS == TSHttpTxnParentSelectionUrlSet(_txn, _buf, new_url_loc)) {
+          msg.assign("set parent selection URL to ").append(_key);
+          res = true;
+        } else {
+          msg.assign("failed to set parent selection URL");
+        }
+      } else {
+        msg.assign("failed to parse parent selection URL");
+      }
+      TSHandleMLocRelease(_buf, TS_NULL_MLOC, new_url_loc);
+    } else {
+      msg.assign("failed to create parent selection URL");
+    }
+  } break;
+  default: {
+    msg.assign("unknown target URI type");
+  } break;
+  }
+
+  /* Report status - debug level in case of success, error in case of failure.
+   * Since getting effective URI is expensive add it only in case of failure */
+  if (res) {
+    CacheKeyDebug("%.*s", static_cast<int>(msg.length()), msg.c_str());
+  } else {
+    int len;
+    char *url = TSHttpTxnEffectiveUrlStringGet(_txn, &len);
+    if (nullptr != url) {
+      msg.append(" for url ").append(url, len);
       TSfree(url);
     }
-    res = false;
+    CacheKeyError("%.*s", static_cast<int>(msg.length()), msg.c_str());
   }
   return res;
 }

plugins/cachekey/cachekey.h

@@ -50,14 +50,16 @@
 class CacheKey
 {
 public:
-  CacheKey(TSHttpTxn txn, String separator, CacheKeyUriType urlType, TSRemapRequestInfo *rri = nullptr);
+  CacheKey(TSHttpTxn txn, String separator, CacheKeyUriType urlType, CacheKeyKeyType targetUrlType,
+           TSRemapRequestInfo *rri = nullptr);
   ~CacheKey();
 
   void append(unsigned number);
   void append(const String &);
+  void append(const String &s, bool useSeparator);
   void append(const char *s);
   void append(const char *n, unsigned s);
-  void appendPrefix(const String &prefix, Pattern &prefixCapture, Pattern &prefixCaptureUri);
+  void appendPrefix(const String &prefix, Pattern &prefixCapture, Pattern &prefixCaptureUri, bool canonicalPrefix);
   void appendPath(Pattern &pathCapture, Pattern &pathCaptureUri);
   void appendHeaders(const ConfigHeaders &config);
   void appendQuery(const ConfigQuery &config);
@@ -85,7 +87,8 @@ class CacheKey
   bool _valid = false; /**< @brief shows if the constructor discovered the input correctly */
   bool _remap = false; /**< @brief shows if the input URI was from remap info */
 
-  String _key;              /**< @brief cache key */
-  String _separator;        /**< @brief a separator used to separate the cache key elements extracted from the URI */
-  CacheKeyUriType _uriType; /**< @brief the URI type used as a cachekey base: pristine, remap, etc. */
+  String _key;                          /**< @brief cache key */
+  String _separator;                    /**< @brief a separator used to separate the cache key elements extracted from the URI */
+  CacheKeyUriType _uriType = REMAP;     /**< @brief the URI type used as a cachekey base: pristine, remap, etc. */
+  CacheKeyKeyType _keyType = CACHE_KEY; /**< @brief the target URI type: cache key, parent selection, etc. */
 };