Merge branch 'hotfix/1.27.2'

ledge-1.27.1-1.rockspec → ledge-1.27.2-1.rockspec

@@ -1,8 +1,8 @@
 package = "ledge"
-version = "1.27.1-1"
+version = "1.27.2-1"
 source = {
    url = "git://github.com/pintsized/ledge",
-   tag = "v1.27.1"
+   tag = "v1.27.2"
 }
 description = {
    summary = "An ESI capable HTTP cache module for OpenResty",

lib/ledge/esi.lua

@@ -44,7 +44,7 @@ end
 
 
 local _M = {
-    _VERSION = '1.27.1',
+    _VERSION = '1.27.2',
 }
 
 

lib/ledge/header_util.lua

@@ -6,7 +6,7 @@ local tbl_concat = table.concat
 
 
 local _M = {
-    _VERSION = '1.27.1'
+    _VERSION = '1.27.2'
 }
 
 local mt = {

lib/ledge/jobs/collect_entity.lua

@@ -5,7 +5,7 @@ local ngx_ERR = ngx.ERR
 
 
 local _M = {
-    _VERSION = '1.27.1',
+    _VERSION = '1.27.2',
 }
 
 

lib/ledge/jobs/purge.lua

@@ -11,7 +11,7 @@ local ngx_null = ngx.null
 local ngx_md5 = ngx.md5
 
 local _M = {
-    _VERSION = '1.27.1',
+    _VERSION = '1.27.2',
 }
 
 

lib/ledge/jobs/revalidate.lua

@@ -5,7 +5,7 @@ local ngx_null = ngx.null
 local str_match = string.match
 
 local _M = {
-    _VERSION = '1.27.1',
+    _VERSION = '1.27.2',
 }
 
 

lib/ledge/ledge.lua

@@ -179,7 +179,7 @@ end
 
 
 local _M = {
-    _VERSION = '1.27.1',
+    _VERSION = '1.27.2',
 
     ORIGIN_MODE_BYPASS = 1, -- Never go to the origin, serve from cache or 503.
     ORIGIN_MODE_AVOID  = 2, -- Avoid the origin, serve from cache where possible.
@@ -2281,7 +2281,14 @@ function _M.read_from_cache(self)
     -- TODO: From Redis 3.2.1 this can be one TOUCH command
     local _ = redis:hlen(key_chain.reval_params)
     local _ = redis:hlen(key_chain.reval_req_headers)
-    local _ = redis:zcard(key_chain.entities)
+    local entities, err = redis:zcard(key_chain.entities)
+    if not entities or entities == ngx_null then
+        ngx_log(ngx_ERR, "could not read entities set: ", err)
+        return nil
+    elseif entities == 0 then
+        -- Entities set is perhaps evicted
+        return nil
+    end
 
     local ttl = nil
     local time_in_cache = 0
@@ -2316,31 +2323,38 @@ function _M.read_from_cache(self)
 
     -- Read headers
     local headers = redis:hgetall(key_chain.headers)
-    if headers then
-        local headers_len = tbl_getn(headers)
-
-        for i = 1, headers_len, 2 do
-            local header = headers[i]
-            if str_find(header, ":") then
-                -- We have multiple headers with the same field name
-                local index, key = unpack(str_split(header, ":"))
-                if not res.header[key] then
-                    res.header[key] = {}
-                end
-                tbl_insert(res.header[key], headers[i + 1])
-            else
-                res.header[header] = headers[i + 1]
+    if not headers or headers == ngx_null then
+        ngx_log(ngx_ERR, "could not read headers: ", err)
+        return nil
+    end
+
+    local headers_len = tbl_getn(headers)
+    if headers_len == 0 then
+        -- Headers have likely been evicted
+        return nil
+    end
+
+    for i = 1, headers_len, 2 do
+        local header = headers[i]
+        if str_find(header, ":") then
+            -- We have multiple headers with the same field name
+            local index, key = unpack(str_split(header, ":"))
+            if not res.header[key] then
+                res.header[key] = {}
             end
+            tbl_insert(res.header[key], headers[i + 1])
+        else
+            res.header[header] = headers[i + 1]
         end
+    end
 
-        -- Calculate the Age header
-        if res.header["Age"] then
-            -- We have end-to-end Age headers, add our time_in_cache.
-            res.header["Age"] = tonumber(res.header["Age"]) + time_in_cache
-        elseif res.header["Date"] then
-            -- We have no advertised Age, use the generated timestamp.
-            res.header["Age"] = time_since_generated
-        end
+    -- Calculate the Age header
+    if res.header["Age"] then
+        -- We have end-to-end Age headers, add our time_in_cache.
+        res.header["Age"] = tonumber(res.header["Age"]) + time_in_cache
+    elseif res.header["Date"] then
+        -- We have no advertised Age, use the generated timestamp.
+        res.header["Age"] = time_since_generated
     end
 
     self:emit("cache_accessed", res)

lib/ledge/response.lua

@@ -36,7 +36,7 @@ end
 
 
 local _M = {
-    _VERSION = '1.27.1'
+    _VERSION = '1.27.2'
 }
 
 local mt = {

t/27-memory_pressure.t

@@ -0,0 +1,183 @@
+use Test::Nginx::Socket;
+use Cwd qw(cwd);
+
+plan tests => 36;
+
+my $pwd = cwd();
+
+$ENV{TEST_LEDGE_REDIS_DATABASE} |= 2;
+$ENV{TEST_LEDGE_REDIS_QLESS_DATABASE} |= 3;
+$ENV{TEST_USE_RESTY_CORE} ||= 'nil';
+$ENV{TEST_COVERAGE} ||= 0;
+
+our $HttpConfig = qq{
+lua_package_path "$pwd/../lua-ffi-zlib/lib/?.lua;$pwd/../lua-resty-redis-connector/lib/?.lua;$pwd/../lua-resty-qless/lib/?.lua;$pwd/../lua-resty-http/lib/?.lua;$pwd/../lua-resty-cookie/lib/?.lua;$pwd/lib/?.lua;/usr/local/share/lua/5.1/?.lua;;";
+    init_by_lua_block {
+        if $ENV{TEST_COVERAGE} == 1 then
+            jit.off()
+            require("luacov.runner").init()
+        end
+
+        local use_resty_core = $ENV{TEST_USE_RESTY_CORE}
+        if use_resty_core then
+            require 'resty.core'
+        end
+		ledge_mod = require 'ledge.ledge'
+        ledge = ledge_mod:new()
+		ledge:config_set('redis_database', $ENV{TEST_LEDGE_REDIS_DATABASE})
+        ledge:config_set('redis_qless_database', $ENV{TEST_LEDGE_REDIS_QLESS_DATABASE})
+        ledge:config_set('upstream_host', '127.0.0.1')
+        ledge:config_set('upstream_port', 1984)
+        ledge:config_set("esi_enabled", true)
+        redis_socket = '$ENV{TEST_LEDGE_REDIS_SOCKET}'
+    }
+
+    init_worker_by_lua_block {
+        if $ENV{TEST_COVERAGE} == 1 then
+            jit.off()
+        end
+        ledge:run_workers()
+    }
+};
+
+no_long_string();
+run_tests();
+
+__DATA__
+=== TEST 1: Prime some cache
+--- http_config eval: $::HttpConfig
+--- config
+	location "/mem_pressure_1_prx" {
+        rewrite ^(.*)_prx$ $1 break;
+        content_by_lua_block {
+            ledge:run()
+        }
+    }
+    location "/mem_pressure_1" {
+        default_type text/html;
+        content_by_lua_block {
+            ngx.header["Cache-Control"] = "max-age=3600"
+            ngx.header["Surrogate-Control"] = [[content="ESI/1.0"]]
+            ngx.print("<esi:vars></esi:vars>Key: ", ngx.req.get_uri_args()["key"])
+        }
+    }
+--- request eval
+["GET /mem_pressure_1_prx?key=main",
+"GET /mem_pressure_1_prx?key=headers",
+"GET /mem_pressure_1_prx?key=entities",
+"GET /mem_pressure_1_prx?key=body",
+"GET /mem_pressure_1_prx?key=body_esi"]
+--- response_body eval
+["Key: main",
+"Key: headers",
+"Key: entities",
+"Key: body",
+"Key: body_esi"]
+--- no_error_log
+[error]
+
+
+=== TEST 1b: Break each key, in a different way for each, then try to serve
+--- http_config eval: $::HttpConfig
+--- config
+	location "/mem_pressure_1_prx" {
+        rewrite ^(.*)_prx$ $1 break;
+        content_by_lua_block {
+            local redis_mod = require "resty.redis"
+            local redis = redis_mod.new()
+            redis:connect("127.0.0.1", 6379)
+            redis:select(ledge:config_get("redis_database"))
+            ledge:ctx().redis = redis
+
+            local key_chain = ledge:cache_key_chain()
+
+            local cache_key_chain = ledge:cache_key_chain()
+            local entity_key_chain = ledge:entity_key_chain(false)
+
+            local evict = ngx.req.get_uri_args()["key"]
+            local key
+            if string.sub(evict, 1, 4) == "body" then
+                key = entity_key_chain[evict]
+            else
+                key = cache_key_chain[evict]
+            end
+            ngx.log(ngx.DEBUG, "will evict: ", key)
+            local res, err = redis:del(key)
+            ngx.log(ngx.DEBUG, tostring(res))
+
+            ledge:ctx().redis:close()
+
+            ledge:run()
+        }
+    }
+
+    location "/mem_pressure_1" {
+        content_by_lua_block {
+            ngx.header["Cache-Control"] = "max-age=3600"
+            ngx.print("MISSED: ", ngx.req.get_uri_args()["key"])
+        }
+    }
+--- request eval
+["GET /mem_pressure_1_prx?key=main",
+"GET /mem_pressure_1_prx?key=headers",
+"GET /mem_pressure_1_prx?key=entities",
+"GET /mem_pressure_1_prx?key=body",
+"GET /mem_pressure_1_prx?key=body_esi"]
+--- response_body eval
+["MISSED: main",
+"MISSED: headers",
+"MISSED: entities",
+"MISSED: body",
+"MISSED: body_esi"]
+--- no_error_log
+[error]
+
+
+=== TEST 2: Prime and break ::main before transaction completes (leaves it partial)
+--- http_config eval: $::HttpConfig
+--- config
+	location "/mem_pressure_2_prx" {
+        rewrite ^(.*)_prx$ $1 break;
+        content_by_lua_block {
+            ledge:bind("response_ready", function(res)
+                local redis = ledge:ctx().redis
+                local main = ledge:cache_key_chain().main
+                redis:del(main)
+            end)
+            ledge:run()
+        }
+    }
+    location "/mem_pressure_2" {
+        default_type text/html;
+        content_by_lua_block {
+            ngx.header["Cache-Control"] = "max-age=3600"
+            ngx.print("ORIGIN")
+        }
+    }
+--- request
+GET /mem_pressure_2_prx
+--- response_body: ORIGIN
+--- no_error_log
+[error]
+
+
+=== TEST 2b: Confirm broken ::main doesn't get served
+--- http_config eval: $::HttpConfig
+--- config
+	location "/mem_pressure_2_prx" {
+        rewrite ^(.*)_prx$ $1 break;
+        content_by_lua_block {
+            ledge:run()
+        }
+    }
+    location "/mem_pressure_2" {
+        default_type text/html;
+        content_by_lua_block {
+            ngx.print("ORIGIN")
+        }
+    }
+--- request
+GET /mem_pressure_2_prx
+--- response_body: ORIGIN
+--- no_error_log
+[error]