TS-4959: Remove remnants of old UA configurations and handling

doc/admin-guide/files/records.config.en.rst

@@ -1130,13 +1130,6 @@ ip-resolve
    Controls wether new POST requests re-use keep-alive sessions (``1``) or
    create new connections per request (``0``).
 
-.. ts:cv:: CONFIG proxy.config.http.accept_encoding_filter_enabled INT 0
-
-   Enables (``1``) or disables (``0``) additional handling of ``Accept-encoding``
-   header on incoming requests based on the ``User-Agent`` header, to account
-   for known deficiencies in legacy browsers which mis-report the encodings
-   they are able to accept.
-
 .. ts:cv:: CONFIG proxy.config.http.disallow_post_100_continue INT 0
 
    Allows you to return a 405 Method Not Supported with Posts also

doc/admin-guide/plugins/ts_lua.en.rst

@@ -2917,7 +2917,6 @@ Http config constants
     TS_LUA_CONFIG_HTTP_RESPONSE_HEADER_MAX_SIZE
     TS_LUA_CONFIG_HTTP_NEGATIVE_REVALIDATING_ENABLED
     TS_LUA_CONFIG_HTTP_NEGATIVE_REVALIDATING_LIFETIME
-    TS_LUA_CONFIG_HTTP_ACCEPT_ENCODING_FILTER_ENABLED
     TS_LUA_CONFIG_SSL_HSTS_MAX_AGE
     TS_LUA_CONFIG_SSL_HSTS_INCLUDE_SUBDOMAINS
     TS_LUA_CONFIG_HTTP_CACHE_OPEN_READ_RETRY_TIME

doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst

@@ -140,7 +140,6 @@ The following configurations (from ``records.config``) are overridable.
 |   :ts:cv:`proxy.config.http.response_header_max_size`
 |   :ts:cv:`proxy.config.http.negative_revalidating_enabled`
 |   :ts:cv:`proxy.config.http.negative_revalidating_lifetime`
-|   :ts:cv:`proxy.config.http.accept_encoding_filter_enabled`
 |   :ts:cv:`proxy.config.http.cache.range.write`
 |   :ts:cv:`proxy.config.http.global_user_agent_header`
 |   :ts:cv:`proxy.config.http.slow.log.threshold`

doc/developer-guide/api/types/TSOverridableConfigKey.en.rst

@@ -189,8 +189,6 @@ Enumeration Members
 
 .. c:member:: TSOverridableConfigKey TS_CONFIG_HTTP_NEGATIVE_REVALIDATING_LIFETIME
 
-.. c:member:: TSOverridableConfigKey TS_CONFIG_HTTP_ACCEPT_ENCODING_FILTER_ENABLED
-
 .. c:member:: TSOverridableConfigKey TS_CONFIG_SSL_HSTS_MAX_AGE
 
 .. c:member:: TSOverridableConfigKey TS_CONFIG_SSL_HSTS_INCLUDE_SUBDOMAINS

doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpOverridableConfig.en.po

@@ -388,10 +388,6 @@ msgstr ""
 msgid ":ts:cv:`proxy.config.http.negative_revalidating_lifetime`"
 msgstr ""
 
-#: ../../../developer-guide/api/functions/TSHttpOverridableConfig.en.rst:143
-msgid ":ts:cv:`proxy.config.http.accept_encoding_filter_enabled`"
-msgstr ""
-
 #: ../../../developer-guide/api/functions/TSHttpOverridableConfig.en.rst:144
 msgid ":ts:cv:`proxy.config.http.cache.range.write`"
 msgstr ""

lib/ts/apidefs.h.in

@@ -717,7 +717,6 @@ typedef enum {
   TS_CONFIG_HTTP_RESPONSE_HEADER_MAX_SIZE,
   TS_CONFIG_HTTP_NEGATIVE_REVALIDATING_ENABLED,
   TS_CONFIG_HTTP_NEGATIVE_REVALIDATING_LIFETIME,
-  TS_CONFIG_HTTP_ACCEPT_ENCODING_FILTER_ENABLED,
   TS_CONFIG_SSL_HSTS_MAX_AGE,
   TS_CONFIG_SSL_HSTS_INCLUDE_SUBDOMAINS,
   TS_CONFIG_HTTP_CACHE_OPEN_READ_RETRY_TIME,
@@ -1154,7 +1153,6 @@ extern tsapi const char *const TS_PROTO_TAG_UDP;
 extern tsapi const char *const TS_PROTO_TAG_IPV4;
 extern tsapi const char *const TS_PROTO_TAG_IPV6;
 
-
 /* --------------------------------------------------------------------------
    MLoc Constants */
 /**

plugins/experimental/ts_lua/ts_lua_http_config.c

@@ -95,7 +95,6 @@ typedef enum {
   TS_LUA_CONFIG_HTTP_RESPONSE_HEADER_MAX_SIZE                 = TS_CONFIG_HTTP_RESPONSE_HEADER_MAX_SIZE,
   TS_LUA_CONFIG_HTTP_NEGATIVE_REVALIDATING_ENABLED            = TS_CONFIG_HTTP_NEGATIVE_REVALIDATING_ENABLED,
   TS_LUA_CONFIG_HTTP_NEGATIVE_REVALIDATING_LIFETIME           = TS_CONFIG_HTTP_NEGATIVE_REVALIDATING_LIFETIME,
-  TS_LUA_CONFIG_HTTP_ACCEPT_ENCODING_FILTER_ENABLED           = TS_CONFIG_HTTP_ACCEPT_ENCODING_FILTER_ENABLED,
   TS_LUA_CONFIG_SSL_HSTS_MAX_AGE                              = TS_CONFIG_SSL_HSTS_MAX_AGE,
   TS_LUA_CONFIG_SSL_HSTS_INCLUDE_SUBDOMAINS                   = TS_CONFIG_SSL_HSTS_INCLUDE_SUBDOMAINS,
   TS_LUA_CONFIG_HTTP_CACHE_OPEN_READ_RETRY_TIME               = TS_CONFIG_HTTP_CACHE_OPEN_READ_RETRY_TIME,
@@ -207,7 +206,6 @@ ts_lua_var_item ts_lua_http_config_vars[] = {
   TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_RESPONSE_HEADER_MAX_SIZE),
   TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_NEGATIVE_REVALIDATING_ENABLED),
   TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_NEGATIVE_REVALIDATING_LIFETIME),
-  TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_ACCEPT_ENCODING_FILTER_ENABLED),
   TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_SSL_HSTS_MAX_AGE),
   TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_SSL_HSTS_INCLUDE_SUBDOMAINS),
   TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_CACHE_OPEN_READ_RETRY_TIME),

proxy/InkAPI.cc

@@ -8052,9 +8052,6 @@ _conf_to_memberp(TSOverridableConfigKey conf, OverridableHttpConfigParams *overr
     typ = OVERRIDABLE_TYPE_INT;
     ret = &overridableHttpConfig->negative_revalidating_lifetime;
     break;
-  case TS_CONFIG_HTTP_ACCEPT_ENCODING_FILTER_ENABLED:
-    ret = &overridableHttpConfig->accept_encoding_filter_enabled;
-    break;
   case TS_CONFIG_SSL_HSTS_MAX_AGE:
     typ = OVERRIDABLE_TYPE_INT;
     ret = &overridableHttpConfig->proxy_response_hsts_max_age;
@@ -8767,11 +8764,6 @@ TSHttpTxnConfigFind(const char *name, int length, TSOverridableConfigKey *conf,
 
   case 48:
     switch (name[length - 1]) {
-    case 'd':
-      if (!strncmp(name, "proxy.config.http.accept_encoding_filter_enabled", length)) {
-        cnf = TS_CONFIG_HTTP_ACCEPT_ENCODING_FILTER_ENABLED;
-      }
-      break;
     case 'e':
       if (!strncmp(name, "proxy.config.http.cache.ignore_client_cc_max_age", length)) {
         cnf = TS_CONFIG_HTTP_CACHE_IGNORE_CLIENT_CC_MAX_AGE;

proxy/InkAPITest.cc

@@ -7592,7 +7592,6 @@ const char *SDK_Overridable_Configs[TS_CONFIG_LAST_ENTRY] = {
   "proxy.config.http.response_header_max_size",
   "proxy.config.http.negative_revalidating_enabled",
   "proxy.config.http.negative_revalidating_lifetime",
-  "proxy.config.http.accept_encoding_filter_enabled",
   "proxy.config.ssl.hsts_max_age",
   "proxy.config.ssl.hsts_include_subdomains",
   "proxy.config.http.cache.open_read_retry_time",

proxy/http/HttpConfig.cc

@@ -120,7 +120,6 @@ static const ConfigEnumPair<TSServerSessionSharingPoolType> SessionSharingPoolSt
 ////////////////////////////////////////////////////////////////
 int HttpConfig::m_id = 0;
 HttpConfigParams HttpConfig::m_master;
-HttpUserAgent_RegxEntry *HttpConfig::user_agent_list = NULL;
 
 static volatile int http_config_changes = 1;
 static HttpConfigCont *http_config_cont = NULL;
@@ -1353,8 +1352,6 @@ HttpConfig::reconfigure()
 
   params->strict_uri_parsing = INT_TO_BOOL(m_master.strict_uri_parsing);
 
-  params->oride.accept_encoding_filter_enabled = INT_TO_BOOL(m_master.oride.accept_encoding_filter_enabled);
-
   params->oride.down_server_timeout    = m_master.oride.down_server_timeout;
   params->oride.client_abort_threshold = m_master.oride.client_abort_threshold;
 

proxy/http/HttpConfig.h

@@ -402,7 +402,6 @@ struct OverridableHttpConfigParams {
       insert_response_via_string(0),
       doc_in_cache_skip_dns(1),
       flow_control_enabled(0),
-      accept_encoding_filter_enabled(0),
       normalize_ae_gzip(0),
       srv_enabled(0),
       cache_open_write_fail_action(0),
@@ -543,11 +542,6 @@ struct OverridableHttpConfigParams {
   MgmtByte doc_in_cache_skip_dns;
   MgmtByte flow_control_enabled;
 
-  ////////////////////////////////////////////////////////
-  // HTTP Accept-Encoding filtering based on User-Agent //
-  ////////////////////////////////////////////////////////
-  MgmtByte accept_encoding_filter_enabled;
-
   ////////////////////////////////
   // Optimize gzip alternates   //
   ////////////////////////////////
@@ -819,39 +813,6 @@ struct HttpConfigParams : public ConfigInfo {
   HttpConfigParams &operator=(const HttpConfigParams &);
 };
 
-/////////////////////////////////////////////////////////////
-//
-// class HttpUserAgent_RegxEntry
-//
-// configuration entry for specific User-Agent
-// Created at startup time only and never changed
-// The main purpose of the User-Agent filtering is to find "bad" user agents
-// and modify Accept-Encoding to prevent compression for such "bad" guys
-/////////////////////////////////////////////////////////////
-
-class HttpUserAgent_RegxEntry
-{
-public:
-  typedef enum { // for more details, please see comments in "ae_ua.config" file
-    STRTYPE_UNKNOWN = 0,
-    STRTYPE_SUBSTR_CASE,  /* .substring, .string */
-    STRTYPE_SUBSTR_NCASE, /* .substring_ncase, .string_ncase */
-    STRTYPE_REGEXP        /* .regexp POSIX regular expression */
-  } StrType;
-
-  HttpUserAgent_RegxEntry *next;
-  int user_agent_str_size;
-  char *user_agent_str;
-  bool regx_valid;
-  StrType stype;
-  pcre *regx;
-
-  HttpUserAgent_RegxEntry();
-  ~HttpUserAgent_RegxEntry();
-
-  bool create(char *refexp_str = NULL, char *errmsgbuf = NULL, int errmsgbuf_size = 0);
-};
-
 /////////////////////////////////////////////////////////////
 //
 // class HttpConfig
@@ -879,7 +840,6 @@ class HttpConfig
 public:
   static int m_id;
   static HttpConfigParams m_master;
-  static HttpUserAgent_RegxEntry *user_agent_list;
 };
 
 // DI's request to disable ICP on the fly

proxy/http/HttpTransact.cc

@@ -647,94 +647,6 @@ HttpTransact::HandleBlindTunnel(State *s)
   HandleRequest(s);
 }
 
-bool
-HttpTransact::perform_accept_encoding_filtering(State *s)
-{
-  HttpUserAgent_RegxEntry *uae;
-  HTTPHdr *client_request;
-  MIMEField *accept_field;
-  MIMEField *usragent_field;
-  char tmp_ua_buf[1024], *c;
-  char const *u_agent = NULL;
-  int u_agent_len     = 0;
-  bool retcode        = false;
-  bool ua_match       = false;
-
-  client_request = &s->hdr_info.client_request;
-
-  // Make sense to check Accept-Encoding if UserAgent is present (and matches)
-  if ((usragent_field = client_request->field_find(MIME_FIELD_USER_AGENT, MIME_LEN_USER_AGENT)) != 0 &&
-      (u_agent = usragent_field->value_get(&u_agent_len)) != 0 && u_agent_len > 0) {
-    if (u_agent_len >= (int)sizeof(tmp_ua_buf)) {
-      u_agent_len = (int)(sizeof(tmp_ua_buf) - 1);
-    }
-    memcpy(tmp_ua_buf, u_agent, u_agent_len);
-    tmp_ua_buf[u_agent_len] = '\0';
-
-    // TODO: Do we really want to do these hardcoded checks still?
-    // Check hardcoded case MSIE>6 & Mozilla>4
-    if ((c = strstr(tmp_ua_buf, "MSIE")) != NULL) {
-      if (c[5] >= '7' && c[5] <= '9') {
-        return false; // Don't change anything for IE > 6
-      }
-      ua_match = true;
-    } else if (!strncasecmp(tmp_ua_buf, "mozilla", 7)) {
-      if (tmp_ua_buf[8] >= '5' && tmp_ua_buf[8] <= '9') {
-        return false; // Don't change anything for Mozilla > 4
-      }
-      ua_match = true;
-    }
-
-    // Check custom filters
-    if (!ua_match && HttpConfig::user_agent_list) {
-      for (uae = HttpConfig::user_agent_list; uae && !ua_match; uae = uae->next) {
-        switch (uae->stype) {
-        case HttpUserAgent_RegxEntry::STRTYPE_SUBSTR_CASE: /* .substring, .string */
-          if (u_agent_len >= uae->user_agent_str_size && !memcmp(tmp_ua_buf, uae->user_agent_str, uae->user_agent_str_size)) {
-            ua_match = true;
-          }
-          break;
-        case HttpUserAgent_RegxEntry::STRTYPE_SUBSTR_NCASE: /* .substring_ncase, .string_ncase */
-          if (u_agent_len >= uae->user_agent_str_size && !strncasecmp(uae->user_agent_str, tmp_ua_buf, uae->user_agent_str_size)) {
-            ua_match = true;
-          }
-          break;
-        case HttpUserAgent_RegxEntry::STRTYPE_REGEXP: /* .regexp POSIX regular expression */
-          if (uae->regx_valid && !pcre_exec(uae->regx, NULL, tmp_ua_buf, u_agent_len, 0, 0, NULL, 0)) {
-            ua_match = true;
-          }
-          break;
-        default: /* unknown type in the structure - bad initialization - impossible bug! */
-          /* I can use ink_error() here since we should shutdown TS immediately */
-          ink_error("[HttpTransact::perform_accept_encoding_filtering] - get unknown User-Agent string type - bad initialization");
-        };
-      }
-    }
-
-    /* If we have correct User-Agent header ....
-       Just set Accept-Encoding: identity or .... do nothing because
-       "If no Accept-Encoding field is present in a request, the server MAY assume that the client
-       will accept any content coding. In this case, if "identity" is one of the available content-codings,
-       then the server SHOULD use the "identity" content-coding, unless it has additional information that
-       a different content-coding is meaningful to the client." */
-    if (ua_match) {
-      DebugTxn("http_trans", "HttpTransact::ModifyRequest, insert identity Accept-Encoding");
-      accept_field = client_request->field_find(MIME_FIELD_ACCEPT_ENCODING, MIME_LEN_ACCEPT_ENCODING);
-      if (!accept_field) {
-        accept_field = client_request->field_create(MIME_FIELD_ACCEPT_ENCODING, MIME_LEN_ACCEPT_ENCODING);
-        if (accept_field) {
-          client_request->field_attach(accept_field);
-        }
-      }
-      if (accept_field) {
-        client_request->field_value_set(accept_field, HTTP_VALUE_IDENTITY, HTTP_LEN_IDENTITY);
-      }
-    }
-    retcode = true;
-  } // end of 'user-agent'
-  return retcode;
-}
-
 void
 HttpTransact::StartRemapRequest(State *s)
 {
@@ -1212,16 +1124,9 @@ HttpTransact::ModifyRequest(State *s)
     }
   }
 
-  /////////////////////////////////////////////////////////
-  // Modify Accept-Encoding for several specific User-Agent
-  /////////////////////////////////////////////////////////
-  if (s->txn_conf->accept_encoding_filter_enabled) {
-    perform_accept_encoding_filtering(s);
-  }
-
   DebugTxn("http_trans", "END HttpTransact::ModifyRequest");
-
   DebugTxn("http_trans", "Checking if transaction wants to upgrade");
+
   if (handle_upgrade_request(s)) {
     // everything should be handled by the upgrade handler.
     DebugTxn("http_trans", "Transaction will be upgraded by the appropriate upgrade handler.");
@@ -8231,34 +8136,7 @@ HttpTransact::build_error_response(State *s, HTTPStatus status_code, const char
       }
     }
   }
-  if (status_code == HTTP_STATUS_PROXY_AUTHENTICATION_REQUIRED && s->method == HTTP_WKSIDX_CONNECT &&
-      s->hdr_info.client_response.presence(MIME_PRESENCE_PROXY_CONNECTION)) {
-    int has_ua_msie = 0;
-    int user_agent_value_len, slen;
-    const char *user_agent_value, *c, *e;
-
-    user_agent_value = s->hdr_info.client_request.value_get(MIME_FIELD_USER_AGENT, MIME_LEN_USER_AGENT, &user_agent_value_len);
-    if (user_agent_value && user_agent_value_len >= 4) {
-      c = user_agent_value;
-      e = c + user_agent_value_len - 4;
-      while (1) {
-        slen = (int)(e - c);
-        c    = (const char *)memchr(c, 'M', slen);
-        if (c == NULL || (e - c) < 3) {
-          break;
-        }
-        if ((c[1] == 'S') && (c[2] == 'I') && (c[3] == 'E')) {
-          has_ua_msie = 1;
-          break;
-        }
-        c++;
-      }
-    }
 
-    if (has_ua_msie) {
-      s->hdr_info.client_response.value_set(MIME_FIELD_PROXY_CONNECTION, MIME_LEN_PROXY_CONNECTION, "close", 5);
-    }
-  }
   // Add a bunch of headers to make sure that caches between
   // the Traffic Server and the client do not cache the error
   // page.