diff --git a/doc/admin-guide/files/records.config.en.rst b/doc/admin-guide/files/records.config.en.rst index 199ffaad822..a196c47efba 100644 --- a/doc/admin-guide/files/records.config.en.rst +++ b/doc/admin-guide/files/records.config.en.rst @@ -1130,13 +1130,6 @@ ip-resolve Controls wether new POST requests re-use keep-alive sessions (``1``) or create new connections per request (``0``). -.. ts:cv:: CONFIG proxy.config.http.accept_encoding_filter_enabled INT 0 - - Enables (``1``) or disables (``0``) additional handling of ``Accept-encoding`` - header on incoming requests based on the ``User-Agent`` header, to account - for known deficiencies in legacy browsers which mis-report the encodings - they are able to accept. - .. ts:cv:: CONFIG proxy.config.http.disallow_post_100_continue INT 0 Allows you to return a 405 Method Not Supported with Posts also diff --git a/doc/admin-guide/plugins/ts_lua.en.rst b/doc/admin-guide/plugins/ts_lua.en.rst index 7e13f60c375..7c8027a5aa7 100644 --- a/doc/admin-guide/plugins/ts_lua.en.rst +++ b/doc/admin-guide/plugins/ts_lua.en.rst @@ -2917,7 +2917,6 @@ Http config constants TS_LUA_CONFIG_HTTP_RESPONSE_HEADER_MAX_SIZE TS_LUA_CONFIG_HTTP_NEGATIVE_REVALIDATING_ENABLED TS_LUA_CONFIG_HTTP_NEGATIVE_REVALIDATING_LIFETIME - TS_LUA_CONFIG_HTTP_ACCEPT_ENCODING_FILTER_ENABLED TS_LUA_CONFIG_SSL_HSTS_MAX_AGE TS_LUA_CONFIG_SSL_HSTS_INCLUDE_SUBDOMAINS TS_LUA_CONFIG_HTTP_CACHE_OPEN_READ_RETRY_TIME diff --git a/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst b/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst index 10f4aa18a38..2d8095fc979 100644 --- a/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst +++ b/doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst @@ -140,7 +140,6 @@ The following configurations (from ``records.config``) are overridable. | :ts:cv:`proxy.config.http.response_header_max_size` | :ts:cv:`proxy.config.http.negative_revalidating_enabled` | :ts:cv:`proxy.config.http.negative_revalidating_lifetime` -| :ts:cv:`proxy.config.http.accept_encoding_filter_enabled` | :ts:cv:`proxy.config.http.cache.range.write` | :ts:cv:`proxy.config.http.global_user_agent_header` | :ts:cv:`proxy.config.http.slow.log.threshold` diff --git a/doc/developer-guide/api/types/TSOverridableConfigKey.en.rst b/doc/developer-guide/api/types/TSOverridableConfigKey.en.rst index abe104551b8..e106bc55c58 100644 --- a/doc/developer-guide/api/types/TSOverridableConfigKey.en.rst +++ b/doc/developer-guide/api/types/TSOverridableConfigKey.en.rst @@ -189,8 +189,6 @@ Enumeration Members .. c:member:: TSOverridableConfigKey TS_CONFIG_HTTP_NEGATIVE_REVALIDATING_LIFETIME -.. c:member:: TSOverridableConfigKey TS_CONFIG_HTTP_ACCEPT_ENCODING_FILTER_ENABLED - .. c:member:: TSOverridableConfigKey TS_CONFIG_SSL_HSTS_MAX_AGE .. c:member:: TSOverridableConfigKey TS_CONFIG_SSL_HSTS_INCLUDE_SUBDOMAINS diff --git a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpOverridableConfig.en.po b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpOverridableConfig.en.po index 7c961d9933f..41f9ceac893 100644 --- a/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpOverridableConfig.en.po +++ b/doc/locale/ja/LC_MESSAGES/developer-guide/api/functions/TSHttpOverridableConfig.en.po @@ -388,10 +388,6 @@ msgstr "" msgid ":ts:cv:`proxy.config.http.negative_revalidating_lifetime`" msgstr "" -#: ../../../developer-guide/api/functions/TSHttpOverridableConfig.en.rst:143 -msgid ":ts:cv:`proxy.config.http.accept_encoding_filter_enabled`" -msgstr "" - #: ../../../developer-guide/api/functions/TSHttpOverridableConfig.en.rst:144 msgid ":ts:cv:`proxy.config.http.cache.range.write`" msgstr "" diff --git a/lib/ts/apidefs.h.in b/lib/ts/apidefs.h.in index 885a7f874c9..637e5c5b272 100644 --- a/lib/ts/apidefs.h.in +++ b/lib/ts/apidefs.h.in @@ -717,7 +717,6 @@ typedef enum { TS_CONFIG_HTTP_RESPONSE_HEADER_MAX_SIZE, TS_CONFIG_HTTP_NEGATIVE_REVALIDATING_ENABLED, TS_CONFIG_HTTP_NEGATIVE_REVALIDATING_LIFETIME, - TS_CONFIG_HTTP_ACCEPT_ENCODING_FILTER_ENABLED, TS_CONFIG_SSL_HSTS_MAX_AGE, TS_CONFIG_SSL_HSTS_INCLUDE_SUBDOMAINS, TS_CONFIG_HTTP_CACHE_OPEN_READ_RETRY_TIME, @@ -1154,7 +1153,6 @@ extern tsapi const char *const TS_PROTO_TAG_UDP; extern tsapi const char *const TS_PROTO_TAG_IPV4; extern tsapi const char *const TS_PROTO_TAG_IPV6; - /* -------------------------------------------------------------------------- MLoc Constants */ /** diff --git a/plugins/experimental/ts_lua/ts_lua_http_config.c b/plugins/experimental/ts_lua/ts_lua_http_config.c index 6252face899..4b65b714ab2 100644 --- a/plugins/experimental/ts_lua/ts_lua_http_config.c +++ b/plugins/experimental/ts_lua/ts_lua_http_config.c @@ -95,7 +95,6 @@ typedef enum { TS_LUA_CONFIG_HTTP_RESPONSE_HEADER_MAX_SIZE = TS_CONFIG_HTTP_RESPONSE_HEADER_MAX_SIZE, TS_LUA_CONFIG_HTTP_NEGATIVE_REVALIDATING_ENABLED = TS_CONFIG_HTTP_NEGATIVE_REVALIDATING_ENABLED, TS_LUA_CONFIG_HTTP_NEGATIVE_REVALIDATING_LIFETIME = TS_CONFIG_HTTP_NEGATIVE_REVALIDATING_LIFETIME, - TS_LUA_CONFIG_HTTP_ACCEPT_ENCODING_FILTER_ENABLED = TS_CONFIG_HTTP_ACCEPT_ENCODING_FILTER_ENABLED, TS_LUA_CONFIG_SSL_HSTS_MAX_AGE = TS_CONFIG_SSL_HSTS_MAX_AGE, TS_LUA_CONFIG_SSL_HSTS_INCLUDE_SUBDOMAINS = TS_CONFIG_SSL_HSTS_INCLUDE_SUBDOMAINS, TS_LUA_CONFIG_HTTP_CACHE_OPEN_READ_RETRY_TIME = TS_CONFIG_HTTP_CACHE_OPEN_READ_RETRY_TIME, @@ -207,7 +206,6 @@ ts_lua_var_item ts_lua_http_config_vars[] = { TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_RESPONSE_HEADER_MAX_SIZE), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_NEGATIVE_REVALIDATING_ENABLED), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_NEGATIVE_REVALIDATING_LIFETIME), - TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_ACCEPT_ENCODING_FILTER_ENABLED), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_SSL_HSTS_MAX_AGE), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_SSL_HSTS_INCLUDE_SUBDOMAINS), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_CACHE_OPEN_READ_RETRY_TIME), diff --git a/proxy/InkAPI.cc b/proxy/InkAPI.cc index b767e26dbf1..669b567c7e8 100644 --- a/proxy/InkAPI.cc +++ b/proxy/InkAPI.cc @@ -8052,9 +8052,6 @@ _conf_to_memberp(TSOverridableConfigKey conf, OverridableHttpConfigParams *overr typ = OVERRIDABLE_TYPE_INT; ret = &overridableHttpConfig->negative_revalidating_lifetime; break; - case TS_CONFIG_HTTP_ACCEPT_ENCODING_FILTER_ENABLED: - ret = &overridableHttpConfig->accept_encoding_filter_enabled; - break; case TS_CONFIG_SSL_HSTS_MAX_AGE: typ = OVERRIDABLE_TYPE_INT; ret = &overridableHttpConfig->proxy_response_hsts_max_age; @@ -8767,11 +8764,6 @@ TSHttpTxnConfigFind(const char *name, int length, TSOverridableConfigKey *conf, case 48: switch (name[length - 1]) { - case 'd': - if (!strncmp(name, "proxy.config.http.accept_encoding_filter_enabled", length)) { - cnf = TS_CONFIG_HTTP_ACCEPT_ENCODING_FILTER_ENABLED; - } - break; case 'e': if (!strncmp(name, "proxy.config.http.cache.ignore_client_cc_max_age", length)) { cnf = TS_CONFIG_HTTP_CACHE_IGNORE_CLIENT_CC_MAX_AGE; diff --git a/proxy/InkAPITest.cc b/proxy/InkAPITest.cc index fe91445d9f1..1f447487e72 100644 --- a/proxy/InkAPITest.cc +++ b/proxy/InkAPITest.cc @@ -7592,7 +7592,6 @@ const char *SDK_Overridable_Configs[TS_CONFIG_LAST_ENTRY] = { "proxy.config.http.response_header_max_size", "proxy.config.http.negative_revalidating_enabled", "proxy.config.http.negative_revalidating_lifetime", - "proxy.config.http.accept_encoding_filter_enabled", "proxy.config.ssl.hsts_max_age", "proxy.config.ssl.hsts_include_subdomains", "proxy.config.http.cache.open_read_retry_time", diff --git a/proxy/http/HttpConfig.cc b/proxy/http/HttpConfig.cc index df7d49c4133..a447f14ef1f 100644 --- a/proxy/http/HttpConfig.cc +++ b/proxy/http/HttpConfig.cc @@ -120,7 +120,6 @@ static const ConfigEnumPair SessionSharingPoolSt //////////////////////////////////////////////////////////////// int HttpConfig::m_id = 0; HttpConfigParams HttpConfig::m_master; -HttpUserAgent_RegxEntry *HttpConfig::user_agent_list = NULL; static volatile int http_config_changes = 1; static HttpConfigCont *http_config_cont = NULL; @@ -1353,8 +1352,6 @@ HttpConfig::reconfigure() params->strict_uri_parsing = INT_TO_BOOL(m_master.strict_uri_parsing); - params->oride.accept_encoding_filter_enabled = INT_TO_BOOL(m_master.oride.accept_encoding_filter_enabled); - params->oride.down_server_timeout = m_master.oride.down_server_timeout; params->oride.client_abort_threshold = m_master.oride.client_abort_threshold; diff --git a/proxy/http/HttpConfig.h b/proxy/http/HttpConfig.h index 03901836bbc..7e4fdd8c182 100644 --- a/proxy/http/HttpConfig.h +++ b/proxy/http/HttpConfig.h @@ -402,7 +402,6 @@ struct OverridableHttpConfigParams { insert_response_via_string(0), doc_in_cache_skip_dns(1), flow_control_enabled(0), - accept_encoding_filter_enabled(0), normalize_ae_gzip(0), srv_enabled(0), cache_open_write_fail_action(0), @@ -543,11 +542,6 @@ struct OverridableHttpConfigParams { MgmtByte doc_in_cache_skip_dns; MgmtByte flow_control_enabled; - //////////////////////////////////////////////////////// - // HTTP Accept-Encoding filtering based on User-Agent // - //////////////////////////////////////////////////////// - MgmtByte accept_encoding_filter_enabled; - //////////////////////////////// // Optimize gzip alternates // //////////////////////////////// @@ -819,39 +813,6 @@ struct HttpConfigParams : public ConfigInfo { HttpConfigParams &operator=(const HttpConfigParams &); }; -///////////////////////////////////////////////////////////// -// -// class HttpUserAgent_RegxEntry -// -// configuration entry for specific User-Agent -// Created at startup time only and never changed -// The main purpose of the User-Agent filtering is to find "bad" user agents -// and modify Accept-Encoding to prevent compression for such "bad" guys -///////////////////////////////////////////////////////////// - -class HttpUserAgent_RegxEntry -{ -public: - typedef enum { // for more details, please see comments in "ae_ua.config" file - STRTYPE_UNKNOWN = 0, - STRTYPE_SUBSTR_CASE, /* .substring, .string */ - STRTYPE_SUBSTR_NCASE, /* .substring_ncase, .string_ncase */ - STRTYPE_REGEXP /* .regexp POSIX regular expression */ - } StrType; - - HttpUserAgent_RegxEntry *next; - int user_agent_str_size; - char *user_agent_str; - bool regx_valid; - StrType stype; - pcre *regx; - - HttpUserAgent_RegxEntry(); - ~HttpUserAgent_RegxEntry(); - - bool create(char *refexp_str = NULL, char *errmsgbuf = NULL, int errmsgbuf_size = 0); -}; - ///////////////////////////////////////////////////////////// // // class HttpConfig @@ -879,7 +840,6 @@ class HttpConfig public: static int m_id; static HttpConfigParams m_master; - static HttpUserAgent_RegxEntry *user_agent_list; }; // DI's request to disable ICP on the fly diff --git a/proxy/http/HttpTransact.cc b/proxy/http/HttpTransact.cc index a16f9e8f746..f7dd4e96240 100644 --- a/proxy/http/HttpTransact.cc +++ b/proxy/http/HttpTransact.cc @@ -647,94 +647,6 @@ HttpTransact::HandleBlindTunnel(State *s) HandleRequest(s); } -bool -HttpTransact::perform_accept_encoding_filtering(State *s) -{ - HttpUserAgent_RegxEntry *uae; - HTTPHdr *client_request; - MIMEField *accept_field; - MIMEField *usragent_field; - char tmp_ua_buf[1024], *c; - char const *u_agent = NULL; - int u_agent_len = 0; - bool retcode = false; - bool ua_match = false; - - client_request = &s->hdr_info.client_request; - - // Make sense to check Accept-Encoding if UserAgent is present (and matches) - if ((usragent_field = client_request->field_find(MIME_FIELD_USER_AGENT, MIME_LEN_USER_AGENT)) != 0 && - (u_agent = usragent_field->value_get(&u_agent_len)) != 0 && u_agent_len > 0) { - if (u_agent_len >= (int)sizeof(tmp_ua_buf)) { - u_agent_len = (int)(sizeof(tmp_ua_buf) - 1); - } - memcpy(tmp_ua_buf, u_agent, u_agent_len); - tmp_ua_buf[u_agent_len] = '\0'; - - // TODO: Do we really want to do these hardcoded checks still? - // Check hardcoded case MSIE>6 & Mozilla>4 - if ((c = strstr(tmp_ua_buf, "MSIE")) != NULL) { - if (c[5] >= '7' && c[5] <= '9') { - return false; // Don't change anything for IE > 6 - } - ua_match = true; - } else if (!strncasecmp(tmp_ua_buf, "mozilla", 7)) { - if (tmp_ua_buf[8] >= '5' && tmp_ua_buf[8] <= '9') { - return false; // Don't change anything for Mozilla > 4 - } - ua_match = true; - } - - // Check custom filters - if (!ua_match && HttpConfig::user_agent_list) { - for (uae = HttpConfig::user_agent_list; uae && !ua_match; uae = uae->next) { - switch (uae->stype) { - case HttpUserAgent_RegxEntry::STRTYPE_SUBSTR_CASE: /* .substring, .string */ - if (u_agent_len >= uae->user_agent_str_size && !memcmp(tmp_ua_buf, uae->user_agent_str, uae->user_agent_str_size)) { - ua_match = true; - } - break; - case HttpUserAgent_RegxEntry::STRTYPE_SUBSTR_NCASE: /* .substring_ncase, .string_ncase */ - if (u_agent_len >= uae->user_agent_str_size && !strncasecmp(uae->user_agent_str, tmp_ua_buf, uae->user_agent_str_size)) { - ua_match = true; - } - break; - case HttpUserAgent_RegxEntry::STRTYPE_REGEXP: /* .regexp POSIX regular expression */ - if (uae->regx_valid && !pcre_exec(uae->regx, NULL, tmp_ua_buf, u_agent_len, 0, 0, NULL, 0)) { - ua_match = true; - } - break; - default: /* unknown type in the structure - bad initialization - impossible bug! */ - /* I can use ink_error() here since we should shutdown TS immediately */ - ink_error("[HttpTransact::perform_accept_encoding_filtering] - get unknown User-Agent string type - bad initialization"); - }; - } - } - - /* If we have correct User-Agent header .... - Just set Accept-Encoding: identity or .... do nothing because - "If no Accept-Encoding field is present in a request, the server MAY assume that the client - will accept any content coding. In this case, if "identity" is one of the available content-codings, - then the server SHOULD use the "identity" content-coding, unless it has additional information that - a different content-coding is meaningful to the client." */ - if (ua_match) { - DebugTxn("http_trans", "HttpTransact::ModifyRequest, insert identity Accept-Encoding"); - accept_field = client_request->field_find(MIME_FIELD_ACCEPT_ENCODING, MIME_LEN_ACCEPT_ENCODING); - if (!accept_field) { - accept_field = client_request->field_create(MIME_FIELD_ACCEPT_ENCODING, MIME_LEN_ACCEPT_ENCODING); - if (accept_field) { - client_request->field_attach(accept_field); - } - } - if (accept_field) { - client_request->field_value_set(accept_field, HTTP_VALUE_IDENTITY, HTTP_LEN_IDENTITY); - } - } - retcode = true; - } // end of 'user-agent' - return retcode; -} - void HttpTransact::StartRemapRequest(State *s) { @@ -1212,16 +1124,9 @@ HttpTransact::ModifyRequest(State *s) } } - ///////////////////////////////////////////////////////// - // Modify Accept-Encoding for several specific User-Agent - ///////////////////////////////////////////////////////// - if (s->txn_conf->accept_encoding_filter_enabled) { - perform_accept_encoding_filtering(s); - } - DebugTxn("http_trans", "END HttpTransact::ModifyRequest"); - DebugTxn("http_trans", "Checking if transaction wants to upgrade"); + if (handle_upgrade_request(s)) { // everything should be handled by the upgrade handler. DebugTxn("http_trans", "Transaction will be upgraded by the appropriate upgrade handler."); @@ -8231,34 +8136,7 @@ HttpTransact::build_error_response(State *s, HTTPStatus status_code, const char } } } - if (status_code == HTTP_STATUS_PROXY_AUTHENTICATION_REQUIRED && s->method == HTTP_WKSIDX_CONNECT && - s->hdr_info.client_response.presence(MIME_PRESENCE_PROXY_CONNECTION)) { - int has_ua_msie = 0; - int user_agent_value_len, slen; - const char *user_agent_value, *c, *e; - - user_agent_value = s->hdr_info.client_request.value_get(MIME_FIELD_USER_AGENT, MIME_LEN_USER_AGENT, &user_agent_value_len); - if (user_agent_value && user_agent_value_len >= 4) { - c = user_agent_value; - e = c + user_agent_value_len - 4; - while (1) { - slen = (int)(e - c); - c = (const char *)memchr(c, 'M', slen); - if (c == NULL || (e - c) < 3) { - break; - } - if ((c[1] == 'S') && (c[2] == 'I') && (c[3] == 'E')) { - has_ua_msie = 1; - break; - } - c++; - } - } - if (has_ua_msie) { - s->hdr_info.client_response.value_set(MIME_FIELD_PROXY_CONNECTION, MIME_LEN_PROXY_CONNECTION, "close", 5); - } - } // Add a bunch of headers to make sure that caches between // the Traffic Server and the client do not cache the error // page.