Merge branch 'release/1.27'

.gitignore

@@ -2,3 +2,4 @@ t/servroot/
 t/error.log
 dump.rdb
 stdout
+luacov.*

.luacov

@@ -0,0 +1,6 @@
+modules = {
+   ["ledge.ledge"] = "lib/ledge/ledge.lua",
+   ["ledge.esi"] = "lib/ledge/esi.lua",
+   ["ledge.response"] = "lib/ledge/response.lua",
+   ["ledge.header_util"] = "lib/ledge/header_util.lua",
+}

Makefile

@@ -22,7 +22,7 @@ REDIS_FIRST_PORT                    := $(firstword $(TEST_LEDGE_REDIS_PORTS))
 REDIS_SLAVE_ARG                     := --slaveof 127.0.0.1 $(REDIS_FIRST_PORT)
 REDIS_CLI                           := redis-cli -p $(REDIS_FIRST_PORT)
 
-# Override ledge socket for running make test on its' own 
+# Override ledge socket for running make test on its' own
 # (make test TEST_LEDGE_REDIS_SOCKET=/path/to/sock.sock)
 TEST_LEDGE_REDIS_SOCKET             ?= $(REDIS_PREFIX)$(REDIS_FIRST_PORT)$(REDIS_SOCK)
 
@@ -66,7 +66,7 @@ INSTALL         ?= install
 .PHONY: all install test test_all start_redis_instances stop_redis_instances \
 	start_redis_instance stop_redis_instance cleanup_redis_instance flush_db \
 	create_sentinel_config delete_sentinel_config check_ports test_ledge \
-	test_sentinel
+	test_sentinel coverage
 
 all: ;
 
@@ -153,3 +153,10 @@ test_sentinel: flush_db
 
 test_leak: flush_db
 	$(TEST_LEDGE_REDIS_VARS) TEST_NGINX_CHECK_LEAK=1 $(PROVE) $(TEST_FILE)
+
+coverage: flush_db
+	-@echo "Cleaning stats"
+	@rm -f luacov.stats.out
+	$(TEST_LEDGE_REDIS_VARS) TEST_COVERAGE=1 $(PROVE) $(TEST_FILE)
+	@luacov
+	@tail -12 luacov.report.out

README.md

@@ -54,13 +54,22 @@ end)
 
 Content is considered "stale" when its age is beyond its TTL. However, depending on the value of [keep_cache_for](#keep_cache_for) (which defaults to 1 month), we don't actually expire content in Redis straight away.
 
-This allows us to continue to serve expired content in the event of upstream failures (see [stale_if_error](#stale_if_error)), or scheduled maintenance (see [origin_mode](#origin_mode)).
+This allows us to implement the stale cache control extensions described in [RFC5861](https://tools.ietf.org/html/rfc5861), which provides request and response header semantics for describing how stale something can be served, when it should be revalidated in the background, and how long we can serve stale content in the event of upstream errors.
 
-In addition, by configuring [max_stale](#max_stale), expired content will continue to be served from cache in normal conditions (advertised as "stale" with the `Warning` response header), but immediately triggering a background revalidation.
+This can be very effective in ensuring a fast user experience. For example, if your content has a `max-age` of 24 hours, consider changing this to 1 hour, and adding `stale-while-revalidate` for 23 hours. The net TTL is therefore the same, but the first request after the first hour will trigger backgrounded revalidation, extending the TTL for a further 1 hour + 23 hours.
 
-This can be very effective in ensuring a fast user experience. For example, if you have a TTL of 24 hours, consider changing this TTL to 1 hour, and specifying `max_stale` as 23 hours. The net TTL is therefore the same, but the first request after the first hour will trigger backgrounded revalidation, extending the TTL for a further 1 hour + 23 hours.
+If your origin server cannot be configured in this way, you can always override by [binding](#events) to the `before_save` event.
 
-In other words, set the TTL to the highest comfortable frequency of requests at the origin, and `max_stale` to the longest comfortable TTL, to increase the chances of background revalidation occurring. Note that the first stale request will obviously get stale content, and so very long `max_stale` values can result in very out of data content for one request.
+```lua
+ledge:bind("before_save", function(res)
+	-- Valid for 1 hour, stale-while-revalidate for 23 hours, stale-if-error for three days
+	res.header["Cache-Control"] = "max-age=3600, stale-while-revalidate=82800, stale-if-error=259200"
+end)
+```
+
+In other words, set the TTL to the highest comfortable frequency of requests at the origin, and `stale-while-revalidate` to the longest comfortable TTL, to increase the chances of background revalidation occurring. Note that the first stale request will obviously get stale content, and so very long values can result in very out of data content for one request.
+
+All stale behaviours are constrained by normal cache control semantics. For example, if the origin is down, and the response could be served stale due to the upstream error, but the request contains `Cache-Control: no-cache` or even `Cache-Control: max-age=60` where the content is older than 60 seconds, they will be served the error, rather than the stale content.
 
 
 ### PURGE API
@@ -287,7 +296,7 @@ Ledge is a Lua module for OpenResty. It is not designed to work in a pure Lua en
 Download and install:
 
 * [OpenResty](http://openresty.org/) >= 1.9.x *(With LuaJIT enabled)*
-* [Redis](http://redis.io/download) >= 2.8.x *(Note: Redis 3.2 is is currently not supported)*
+* [Redis](http://redis.io/download) >= 2.8.x *(Note: Redis 3.2.x is not yet supported)*
 * [LuaRocks](https://luarocks.org/) *(Not required, but simplifies installation)*
 
 ```
@@ -379,8 +388,6 @@ Config set in `content_by_lua_block` will only affect that specific location, an
  * [redis_sentinels](#redis_sentinels)
  * [keep_cache_for](#keep_cache_for)
  * [minimum_old_entity_download_rate](#minimum_old_entity_download_rate)
- * [max_stale](#max_stale)
- * [stale_if_error](#stale_if_error)
  * [cache_key_spec](#cache_key_spec)
  * [enable_collapsed_forwarding](#enable_collapsed_forwarding)
  * [collapsed_forwarding_window](#collapsed_forwarding_window)
@@ -392,7 +399,6 @@ Config set in `content_by_lua_block` will only affect that specific location, an
  * [esi_args_prefix](#esi_args_prefix)
  * [gunzip_enabled](#gunzip_enabled)
  * [keyspace_scan_count](#keyspace_scan_count)
- * [revalidate_parent_headers](#revalidate_parent_headers)
 
 
 ### origin_mode
@@ -640,30 +646,6 @@ Lowering this is fairer on slow clients, but widens the potential window for mul
 This design favours high availability (since there are no read-locks, we can serve cache from Redis slaves in the event of failure) on the assumption that the chances of this causing incomplete resources to be served are quite low.
 
 
-### max_stale
-
-syntax: `ledge:config_set("max_stale", 300)`
-
-default: `nil`
-
-Specifies, in seconds, how far past expiry we can serve cached content. If a value is specified by the `Cache-Control: max-stale=xx` request header, then this setting is ignored, placing control in the client's hands.
-
-When content is served stale because of this, a background revalidation job is scheduled immediately.
-
-**WARNING:** Any setting other than `nil` may violate the HTTP specification (i.e. if the client does not override it with a valid request header value).
-
-
-### stale_if_error
-
-syntax: `ledge:config_set("stale_if_error", 86400)`
-
-default: `nil`
-
-Specifies, in seconds, how far past expiry to serve stale cached content if the origin returns an error.
-
-This can be overridden by the request using the [stale-if-error](http://tools.ietf.org/html/rfc5861) `Cache-Control` extension.
-
-
 ### cache_key_spec
 
 `syntax: ledge:config_set("cache_key_spec", { ngx.var.host, ngx.var.uri, ngx.var.args })`
@@ -722,6 +704,7 @@ Toggles [ESI](http://www.w3.org/TR/esi-lang) scanning and processing, though beh
 
 ESI instructions are detected on the slow path (i.e. when fetching from the origin), so only instructions which are known to be present are processed on cache HITs.
 
+
 ### esi_content_types
 
 syntax: `ledge:config_set("esi_content_types", { "text/html", "text/javascript" })`
@@ -791,14 +774,6 @@ Tunes the behaviour of keyspace scans, which occur when sending a PURGE request
 A higher number may be better if latency to Redis is high and the keyspace is large.
 
 
-### revalidate_parent_headers
-
-syntax: `ledge:config_set("revalidate_parent_headers", {"x-real-ip", "authorization"})`
-
-default: {"authorization", "cookie"}
-
-Defines which headers from the parent request are passed through to a background revalidation. Useful when upstreams require authentication.
-
 
 ## Events
 
@@ -827,7 +802,7 @@ end)
 * [origin_fetched](#origin_fetched)
 * [before_save](#before_save)
 * [response_ready](#response_ready)
-* [set_revalidation_headers](#set_revalidation_headers)
+* [before_save_revalidation_data](#before_save_revalidation_data)
 
 ### cache_accessed
 

ledge-1.26.1-1.rockspec → ledge-1.27-1.rockspec

@@ -1,8 +1,8 @@
 package = "ledge"
-version = "1.26.1-1"
+version = "1.27-1"
 source = {
    url = "git://github.com/pintsized/ledge",
-   tag = "v1.26.1"
+   tag = "v1.27"
 }
 description = {
    summary = "An ESI capable HTTP cache module for OpenResty",
@@ -12,9 +12,9 @@ description = {
 }
 dependencies = {
    "lua ~> 5.1",
-   "lua-resty-http >= 0.09",
+   "lua-resty-http >= 0.10",
    "lua-resty-redis-connector >= 0.03",
-   "lua-resty-qless >= 0.07",
+   "lua-resty-qless >= 0.08",
    "lua-resty-cookie >= 0.1",
    "lua-ffi-zlib >= 0.1"
 }

lib/ledge/esi.lua

@@ -44,7 +44,7 @@ end
 
 
 local _M = {
-    _VERSION = '1.26.1',
+    _VERSION = '1.27',
 }
 
 
@@ -377,6 +377,7 @@ local function esi_fetch_include(include_tag, buffer_size, pre_include_callback,
                 local ok, err = httpc:ssl_handshake(false, host, false)
                 if not ok then
                     ngx_log(ngx_ERR, "ssl handshake failed: ", err)
+                    return nil
                 end
             end
 

lib/ledge/header_util.lua

@@ -1,11 +1,12 @@
 local ngx_re_match = ngx.re.match
+local ngx_re_find = ngx.re.find
 local str_find = string.find
 local str_gsub = string.gsub
 local tbl_concat = table.concat
 
 
 local _M = {
-    _VERSION = '1.26.1'
+    _VERSION = '1.27'
 }
 
 local mt = {
@@ -18,7 +19,11 @@ function _M.header_has_directive(header, directive)
         if type(header) == "table" then header = tbl_concat(header, ", ") end
 
         -- Just checking the directive appears in the header, e.g. no-cache, private etc.
-        return (str_find(header, directive, 1, true) ~= nil)
+        return ngx_re_find(
+            header,
+            [[(?:\s*|,?)(]] .. directive .. [[)\s*(?:$|=|,)]],
+            "ioj"
+        ) ~= nil
     end
     return false
 end
@@ -30,8 +35,9 @@ function _M.get_header_token(header, directive)
 
         -- Want the string value from a token
         local value = ngx_re_match(
-            header, 
-            str_gsub(directive, '-','\\-').."=\"?([a-z0-9_~!#%&/',`\\$\\*\\+\\-\\|\\^\\.]+)\"?", "ioj"
+            header,
+            directive .. [[="?([a-z0-9_~!#%&/',`\$\*\+\-\|\^\.]+)"?]],
+            "ioj"
         )
         if value ~= nil then
             return value[1]
@@ -48,8 +54,8 @@ function _M.get_numeric_header_token(header, directive)
 
         -- Want the numeric value from a token
         local value = ngx_re_match(
-        header, 
-        str_gsub(directive, '-','\\-').."=\"?(\\d+)\"?", "ioj"
+            header,
+            directive .. [[="?(\d+)"?]], "ioj"
         )
         if value ~= nil then
             return tonumber(value[1])

lib/ledge/jobs/collect_entity.lua

@@ -1,9 +1,11 @@
 local pairs, unpack = pairs, unpack
 local tbl_insert = table.insert
+local ngx_log = ngx.log
+local ngx_ERR = ngx.ERR
 
 
 local _M = {
-    _VERSION = '1.26.1',
+    _VERSION = '1.27',
 }
 
 
@@ -17,7 +19,7 @@ function _M.perform(job)
     local ok = redis:multi()
 
     local del_keys = {}
-    for _, key in pairs(job.data.entity_keys) do
+    for _, key in pairs(job.data.entity_key_chain) do
         tbl_insert(del_keys, key)
     end
 
@@ -28,22 +30,23 @@ function _M.perform(job)
     -- Params: key, decrement
     -- Return: (integer): the value of the key after the operation.
     local POSDECRBYX = [[
-        local value = redis.call("GET", KEYS[1])
+        local value = redis.call("HGET", KEYS[1], ARGV[1])
         if value and tonumber(value) > 0 then
-            return redis.call("DECRBY", KEYS[1], ARGV[1])
+            return redis.call("HINCRBY", KEYS[1], ARGV[1], -ARGV[2])
         else
             return 0
         end
     ]]
 
-    res, err = redis:eval(POSDECRBYX, 1, job.data.cache_key_chain.memused, job.data.size)
-    res, err = redis:zrem(job.data.cache_key_chain.entities, job.data.entity_keys.main)
+    res, err = redis:eval(POSDECRBYX, 1, job.data.cache_key_chain.main, "memused", job.data.size)
+    if not res then ngx_log(ngx_ERR, err) end
+
+    res, err = redis:zrem(job.data.cache_key_chain.entities, job.data.entity_id)
+    if not res then ngx_log(ngx_ERR, err) end
 
     res, err = redis:exec()
 
-    if res then
-        return true, nil
-    else
+    if not res then
         return nil, "redis-error", err
     end
 end