Showing preview only (767K chars total). Download the full file or copy to clipboard to get everything.
Repository: pintsized/ledge
Branch: master
Commit: 2a872096e73c
Files: 94
Total size: 733.6 KB
Directory structure:
gitextract_rst1uaiw/
├── .gitattributes
├── .github/
│ └── FUNDING.yml
├── .gitignore
├── .luacheckrc
├── .luacov
├── .travis.yml
├── Makefile
├── README.md
├── dist.ini
├── docker/
│ └── tests/
│ └── docker-compose.yml
├── lib/
│ ├── ledge/
│ │ ├── background.lua
│ │ ├── cache_key.lua
│ │ ├── collapse.lua
│ │ ├── esi/
│ │ │ ├── processor_1_0.lua
│ │ │ └── tag_parser.lua
│ │ ├── esi.lua
│ │ ├── gzip.lua
│ │ ├── handler.lua
│ │ ├── header_util.lua
│ │ ├── jobs/
│ │ │ ├── collect_entity.lua
│ │ │ ├── purge.lua
│ │ │ └── revalidate.lua
│ │ ├── purge.lua
│ │ ├── range.lua
│ │ ├── request.lua
│ │ ├── response.lua
│ │ ├── stale.lua
│ │ ├── state_machine/
│ │ │ ├── actions.lua
│ │ │ ├── events.lua
│ │ │ ├── pre_transitions.lua
│ │ │ └── states.lua
│ │ ├── state_machine.lua
│ │ ├── storage/
│ │ │ └── redis.lua
│ │ ├── util.lua
│ │ ├── validation.lua
│ │ └── worker.lua
│ └── ledge.lua
├── migrations/
│ └── 1.26-1.27.lua
├── t/
│ ├── 01-unit/
│ │ ├── cache_key.t
│ │ ├── esi.t
│ │ ├── events.t
│ │ ├── handler.t
│ │ ├── jobs.t
│ │ ├── ledge.t
│ │ ├── processor_1_0.t
│ │ ├── purge.t
│ │ ├── range.t
│ │ ├── request.t
│ │ ├── response.t
│ │ ├── stale.t
│ │ ├── state_machine.t
│ │ ├── storage.t
│ │ ├── tag_parser.t
│ │ ├── util.t
│ │ ├── validation.t
│ │ └── worker.t
│ ├── 02-integration/
│ │ ├── age.t
│ │ ├── cache.t
│ │ ├── collapsed_forwarding.t
│ │ ├── esi.t
│ │ ├── events.t
│ │ ├── gc.t
│ │ ├── gzip.t
│ │ ├── hop_by_hop_headers.t
│ │ ├── max-stale.t
│ │ ├── max_size.t
│ │ ├── memory_pressure.t
│ │ ├── multiple_headers.t
│ │ ├── on_abort.t
│ │ ├── origin_mode.t
│ │ ├── purge.t
│ │ ├── range.t
│ │ ├── req_body.t
│ │ ├── req_method.t
│ │ ├── request_leak.t
│ │ ├── response.t
│ │ ├── ssl.t
│ │ ├── stale-if-error.t
│ │ ├── stale-while-revalidate.t
│ │ ├── upstream.t
│ │ ├── upstream_client.t
│ │ ├── validation.t
│ │ ├── vary.t
│ │ └── via_header.t
│ ├── 03-sentinel/
│ │ ├── 01-master_up.t
│ │ ├── 02-master_down.t
│ │ └── 03-slave_promoted.t
│ ├── LedgeEnv.pm
│ └── cert/
│ ├── example.com.crt
│ ├── example.com.key
│ ├── rootCA.key
│ ├── rootCA.pem
│ └── rootCA.srl
└── util/
└── lua-releng
================================================
FILE CONTENTS
================================================
================================================
FILE: .gitattributes
================================================
*.t linguist-language=lua
================================================
FILE: .github/FUNDING.yml
================================================
github: pintsized
================================================
FILE: .gitignore
================================================
t/servroot/
t/error.log
dump.rdb
stdout
luacov.*
*.src.rock
================================================
FILE: .luacheckrc
================================================
std = "ngx_lua"
redefined = false
================================================
FILE: .luacov
================================================
modules = {
["ledge"] = "lib/ledge.lua",
["ledge.esi.*"] = "lib/",
["ledge.jobs.*"] = "lib/",
["ledge.state_machine.*"] = "lib/",
["ledge.storage.*"] = "lib/",
["ledge.*"] = "lib/"
}
================================================
FILE: .travis.yml
================================================
services:
- docker
script:
- cd docker/tests
- docker-compose run --rm runner
================================================
FILE: Makefile
================================================
SHELL := /bin/bash # Cheat by using bash :)
OPENRESTY_PREFIX = /usr/local/openresty
TEST_FILE ?= t/01-unit t/02-integration
SENTINEL_TEST_FILE ?= t/03-sentinel
TEST_LEDGE_REDIS_HOST ?= 127.0.0.1
TEST_LEDGE_REDIS_PORT ?= 6379
TEST_LEDGE_REDIS_DATABASE ?= 2
TEST_LEDGE_REDIS_QLESS_DATABASE ?= 3
TEST_NGINX_HOST ?= 127.0.0.1
# Command line arguments for ledge tests
TEST_LEDGE_REDIS_VARS = PATH=$(OPENRESTY_PREFIX)/nginx/sbin:$(PATH) \
TEST_LEDGE_REDIS_HOST=$(TEST_LEDGE_REDIS_HOST) \
TEST_LEDGE_REDIS_PORT=$(TEST_LEDGE_REDIS_PORT) \
TEST_LEDGE_REDIS_SOCKET=unix://$(TEST_LEDGE_REDIS_SOCKET) \
TEST_LEDGE_REDIS_DATABASE=$(TEST_LEDGE_REDIS_DATABASE) \
TEST_LEDGE_REDIS_QLESS_DATABASE=$(TEST_LEDGE_REDIS_QLESS_DATABASE) \
TEST_NGINX_HOST=$(TEST_NGINX_HOST) \
TEST_NGINX_NO_SHUFFLE=1
REDIS_CLI := redis-cli -h $(TEST_LEDGE_REDIS_HOST) -p $(TEST_LEDGE_REDIS_PORT)
###############################################################################
# Deprecated, ues docker copose to run Redis instead
###############################################################################
REDIS_CMD = redis-server
SENTINEL_CMD = $(REDIS_CMD) --sentinel
REDIS_SOCK = /redis.sock
REDIS_PID = /redis.pid
REDIS_LOG = /redis.log
REDIS_PREFIX = /tmp/redis-
# Overrideable ledge test variables
TEST_LEDGE_REDIS_PORTS ?= 6379 6380
REDIS_FIRST_PORT := $(firstword $(TEST_LEDGE_REDIS_PORTS))
REDIS_SLAVE_ARG := --slaveof 127.0.0.1 $(REDIS_FIRST_PORT)
# Override ledge socket for running make test on its' own
# (make test TEST_LEDGE_REDIS_SOCKET=/path/to/sock.sock)
TEST_LEDGE_REDIS_SOCKET ?= $(REDIS_PREFIX)$(REDIS_FIRST_PORT)$(REDIS_SOCK)
# Overrideable ledge + sentinel test variables
TEST_LEDGE_SENTINEL_PORTS ?= 26379 26380 26381
TEST_LEDGE_SENTINEL_MASTER_NAME ?= mymaster
TEST_LEDGE_SENTINEL_PROMOTION_TIME ?= 20
# Command line arguments for ledge + sentinel tests
TEST_LEDGE_SENTINEL_VARS = PATH=$(OPENRESTY_PREFIX)/nginx/sbin:$(PATH) \
TEST_LEDGE_SENTINEL_PORT=$(firstword $(TEST_LEDGE_SENTINEL_PORTS)) \
TEST_LEDGE_SENTINEL_MASTER_NAME=$(TEST_LEDGE_SENTINEL_MASTER_NAME) \
TEST_LEDGE_REDIS_DATABASE=$(TEST_LEDGE_REDIS_DATABASE) \
TEST_NGINX_NO_SHUFFLE=1
# Sentinel configuration can only be set by a config file
define TEST_LEDGE_SENTINEL_CONFIG
sentinel monitor $(TEST_LEDGE_SENTINEL_MASTER_NAME) 127.0.0.1 $(REDIS_FIRST_PORT) 2
sentinel down-after-milliseconds $(TEST_LEDGE_SENTINEL_MASTER_NAME) 2000
sentinel failover-timeout $(TEST_LEDGE_SENTINEL_MASTER_NAME) 10000
sentinel parallel-syncs $(TEST_LEDGE_SENTINEL_MASTER_NAME) 5
endef
export TEST_LEDGE_SENTINEL_CONFIG
SENTINEL_CONFIG_PREFIX = /tmp/sentinel
###############################################################################
PREFIX ?= /usr/local
LUA_INCLUDE_DIR ?= $(PREFIX)/include
LUA_LIB_DIR ?= $(PREFIX)/lib/lua/$(LUA_VERSION)
PROVE ?= prove -I ../test-nginx/lib
INSTALL ?= install
.PHONY: all install test test_all start_redis_instances stop_redis_instances \
start_redis_instance stop_redis_instance cleanup_redis_instance flush_db \
check_ports test_ledge test_sentinel coverage delete_sentinel_config check
all: ;
install: all
$(INSTALL) -d $(DESTDIR)/$(LUA_LIB_DIR)/ledge
$(INSTALL) lib/ledge/*.lua $(DESTDIR)/$(LUA_LIB_DIR)/ledge
test: test_ledge
test_all: start_redis_instances test_ledge test_sentinel stop_redis_instances
###############################################################################
# Deprecated, ues docker copose to run Redis instead
##############################################################################
start_redis_instances: check_ports
@$(foreach port,$(TEST_LEDGE_REDIS_PORTS), \
[[ "$(port)" != "$(REDIS_FIRST_PORT)" ]] && \
SLAVE="$(REDIS_SLAVE_ARG)" || \
SLAVE="" && \
$(MAKE) start_redis_instance args="$$SLAVE" port=$(port) \
prefix=$(REDIS_PREFIX)$(port) && \
) true
@$(foreach port,$(TEST_LEDGE_SENTINEL_PORTS), \
echo "port $(port)" > "$(SENTINEL_CONFIG_PREFIX)-$(port).conf"; \
echo "$$TEST_LEDGE_SENTINEL_CONFIG" >> "$(SENTINEL_CONFIG_PREFIX)-$(port).conf"; \
$(MAKE) start_redis_instance \
port=$(port) args="$(SENTINEL_CONFIG_PREFIX)-$(port).conf --sentinel" \
prefix=$(REDIS_PREFIX)$(port) && \
) true
stop_redis_instances: delete_sentinel_config
-@$(foreach port,$(TEST_LEDGE_REDIS_PORTS) $(TEST_LEDGE_SENTINEL_PORTS), \
$(MAKE) stop_redis_instance cleanup_redis_instance port=$(port) \
prefix=$(REDIS_PREFIX)$(port) && \
) true 2>&1 > /dev/null
start_redis_instance:
-@echo "Starting redis on port $(port) with args: \"$(args)\""
-@mkdir -p $(prefix)
@$(REDIS_CMD) $(args) \
--pidfile $(prefix)$(REDIS_PID) \
--bind 127.0.0.1 --port $(port) \
--unixsocket $(prefix)$(REDIS_SOCK) \
--unixsocketperm 777 \
--dir $(prefix) \
--logfile $(prefix)$(REDIS_LOG) \
--loglevel debug \
--daemonize yes
stop_redis_instance:
-@echo "Stopping redis on port $(port)"
-@[[ -f "$(prefix)$(REDIS_PID)" ]] && kill -QUIT \
`cat $(prefix)$(REDIS_PID)` 2>&1 > /dev/null || true
cleanup_redis_instance: stop_redis_instance
-@echo "Cleaning up redis files in $(prefix)"
-@rm -rf $(prefix)
delete_sentinel_config:
-@echo "Cleaning up sentinel config files"
-@rm -f $(SENTINEL_CONFIG_PREFIX)-*.conf
check_ports:
-@echo "Checking ports $(REDIS_PORTS)"
@$(foreach port,$(REDIS_PORTS),! lsof -i :$(port) &&) true 2>&1 > /dev/null
###############################################################################
releng:
@util/lua-releng -eL
flush_db:
@$(REDIS_CLI) flushall
test_ledge: releng flush_db
@$(TEST_LEDGE_REDIS_VARS) $(PROVE) $(TEST_FILE)
-@echo "Qless errors:"
@$(REDIS_CLI) -n $(TEST_LEDGE_REDIS_QLESS_DATABASE) llen ql:f:job-error
test_sentinel: releng flush_db
$(TEST_LEDGE_SENTINEL_VARS) $(PROVE) $(SENTINEL_TEST_FILE)/01-master_up.t
$(REDIS_CLI) shutdown
$(TEST_LEDGE_SENTINEL_VARS) $(PROVE) $(SENTINEL_TEST_FILE)/02-master_down.t
sleep $(TEST_LEDGE_SENTINEL_PROMOTION_TIME)
$(TEST_LEDGE_SENTINEL_VARS) $(PROVE) $(SENTINEL_TEST_FILE)/03-slave_promoted.t
test_leak: releng flush_db
$(TEST_LEDGE_REDIS_VARS) TEST_NGINX_CHECK_LEAK=1 $(PROVE) $(TEST_FILE)
coverage: releng flush_db
@rm -f luacov.stats.out
@$(TEST_LEDGE_REDIS_VARS) TEST_COVERAGE=1 $(PROVE) $(TEST_FILE)
@luacov
@tail -30 luacov.report.out
-@echo "Qless errors:"
@$(REDIS_CLI) -n $(TEST_LEDGE_REDIS_QLESS_DATABASE) llen ql:f:job-error
check:
luacheck lib
================================================
FILE: README.md
================================================
# Ledge
[](https://travis-ci.org/ledgetech/ledge)
An RFC compliant and [ESI](https://www.w3.org/TR/esi-lang) capable HTTP cache for [Nginx](http://nginx.org) / [OpenResty](https://openresty.org), backed by [Redis](http://redis.io).
Ledge can be utilised as a fast, robust and scalable alternative to Squid / Varnish etc, either installed standalone or integrated into an existing Nginx server or load balancer.
Moreover, it is particularly suited to applications where the origin is expensive or distant, making it desirable to serve from cache as optimistically as possible.
## Table of Contents
* [Installation](#installation)
* [Philosophy and Nomenclature](#philosophy-and-nomenclature)
* [Cache keys](#cache-keys)
* [Streaming design](#streaming-design)
* [Collapsed forwarding](#collapsed-forwarding)
* [Advanced cache patterns](#advanced-cache-patterns)
* [Minimal configuration](#minimal-configuration)
* [Config systems](#config-systems)
* [Events system](#events-system)
* [Caching basics](#caching-basics)
* [Purging](#purging)
* [Serving stale](#serving-stale)
* [Edge Side Includes](#edge-side-includes)
* [API](#api)
* [ledge.configure](#ledgeconfigure)
* [ledge.set_handler_defaults](#ledgeset_handler_defaults)
* [ledge.create\_handler](#ledgecreate_handler)
* [ledge.create\_worker](#ledgecreate_worker)
* [ledge.bind](#ledgebind)
* [handler.bind](#handlerbind)
* [handler.run](#handlerrun)
* [worker.run](#workerrun)
* [Handler configuration options](#handler-configuration-options)
* [Events](#events)
* [Administration](#administration)
* [Managing Qless](#managing-qless)
* [Licence](#licence)
## Installation
[OpenResty](http://openresty.org/) is a superset of [Nginx](http://nginx.org), bundling [LuaJIT](http://luajit.org/) and the [lua-nginx-module](https://github.com/openresty/lua-nginx-module) as well as many other things. Whilst it is possible to build all of these things into Nginx yourself, we recommend using the latest OpenResty.
### 1. Download and install:
* [OpenResty](http://openresty.org/) >= 1.11.x
* [Redis](http://redis.io/download) >= 2.8.x
* [LuaRocks](https://luarocks.org/)
### 2. Install Ledge using LuaRocks:
```
luarocks install ledge
```
This will install the latest stable release, and all other Lua module dependencies, which if installing manually without LuaRocks are:
* [lua-resty-http](https://github.com/pintsized/lua-resty-http)
* [lua-resty-redis-connector](https://github.com/pintsized/lua-resty-redis-connector)
* [lua-resty-qless](https://github.com/pintsized/lua-resty-qless)
* [lua-resty-cookie](https://github.com/cloudflare/lua-resty-cookie)
* [lua-ffi-zlib](https://github.com/hamishforbes/lua-ffi-zlib)
* [lua-resty-upstream](https://github.com/hamishforbes/lua-resty-upstream) *(optional, for load balancing / healthchecking upstreams)*
### 3. Review OpenResty documentation
If you are new to OpenResty, it's quite important to review the [lua-nginx-module](https://github.com/openresty/lua-nginx-module) documentation on how to run Lua code in Nginx, as the environment is unusual. Specifcally, it's useful to understand the meaning of the different Nginx phase hooks such as `init_by_lua` and `content_by_lua`, as well as how the `lua-nginx-module` locates Lua modules with the [lua_package_path](https://github.com/openresty/lua-nginx-module#lua_package_path) directive.
[Back to TOC](#table-of-contents)
## Philosophy and Nomenclature
The central module is called `ledge`, and provides factory methods for creating `handler` instances (for handling a request) and `worker` instances (for running background tasks). The `ledge` module is also where global configuration is managed.
A `handler` is short lived. It is typically created at the beginning of the Nginx `content` phase for a request, and when its [run()](#handlerrun) method is called, takes responsibility for processing the current request and delivering a response. When [run()](#handlerrun) has completed, HTTP status, headers and body will have been delivered to the client.
A `worker` is long lived, and there is one per Nginx worker process. It is created when Nginx starts a worker process, and dies when the Nginx worker dies. The `worker` pops queued background jobs and processes them.
An `upstream` is the only thing which must be manually configured, and points to another HTTP host where actual content lives. Typically one would use DNS to resolve client connections to the Nginx server running Ledge, and tell Ledge where to fetch from with the `upstream` configuration. As such, Ledge isn't designed to work as a forwarding proxy.
[Redis](http://redis.io) is used for much more than cache storage. We rely heavily on its data structures to maintain cache `metadata`, as well as embedded Lua scripts for atomic task management and so on. By default, all cache body data and `metadata` will be stored in the same Redis instance. The location of cache `metadata` is global, set when Nginx starts up.
Cache body data is handled by the `storage` system, and as mentioned, by default shares the same Redis instance as the `metadata`. However, `storage` is abstracted via a [driver system](#storage_driver) making it possible to store cache body data in a separate Redis instance, or a group of horizontally scalable Redis instances via a [proxy](https://github.com/twitter/twemproxy), or to roll your own `storage` driver, for example targeting PostreSQL or even simply a filesystem. It's perhaps important to consider that by default all cache storage uses Redis, and as such is bound by system memory.
[Back to TOC](#table-of-contents)
### Cache keys
A goal of any caching system is to safely maximise the HIT potential. That is, normalise factors which would split the cache wherever possible, in order to share as much cache as possible.
This is tricky to generalise, and so by default Ledge puts sane defaults from the request URI into the cache key, and provides a means for this to be customised by altering the [cache\_key\_spec](#cache_key_spec).
URI arguments are sorted alphabetically by default, so `http://example.com?a=1&b=2` would hit the same cache entry as `http://example.com?b=2&a=1`.
[Back to TOC](#table-of-contents)
### Streaming design
HTTP response sizes can be wildly different, sometimes tiny and sometimes huge, and it's not always possible to know the total size up front.
To guarantee predictable memory usage regardless of response sizes Ledge operates a streaming design, meaning it only ever operates on a single `buffer` per request at a time. This is equally true when fetching upstream to when reading from cache or serving to the client request.
It's also true (mostly) when processing [ESI](#edge-size-includes) instructions, except for in the case where an instruction is found to span multiple buffers. In this case, we continue buffering until a complete instruction can be understood, up to a [configurable limit](#esi_max_size).
This streaming design also improves latency, since we start serving the first `buffer` to the client request as soon as we're done with it, rather than fetching and saving an entire resource prior to serving. The `buffer` size can be [tuned](#buffer_size) even on a per `location` basis.
[Back to TOC](#table-of-contents)
### Collapsed forwarding
Ledge can attempt to collapse concurrent origin requests for known (previously) cacheable resources into a single upstream request. That is, if an upstream request for a resource is in progress, subsequent concurrent requests for the same resource will not bother the upstream, and instead wait for the first request to finish.
This is particularly useful to reduce upstream load if a spike of traffic occurs for expired and expensive content (since the chances of concurrent requests is higher on slower content).
[Back to TOC](#table-of-contents)
### Advanced cache patterns
Beyond standard RFC compliant cache behaviours, Ledge has many features designed to maximise cache HIT rates and to reduce latency for requests. See the sections on [Edge Side Includes](#edge-side-includes), [serving stale](#serving-stale) and [revalidating on purge](#purging) for more information.
[Back to TOC](#table-of-contents)
## Minimal configuration
Assuming you have Redis running on `localhost:6379`, and your upstream is at `localhost:8080`, add the following to the `nginx.conf` file in your OpenResty installation.
```lua
http {
if_modified_since Off;
lua_check_client_abort On;
init_by_lua_block {
require("ledge").configure({
redis_connector_params = {
url = "redis://127.0.0.1:6379/0",
},
})
require("ledge").set_handler_defaults({
upstream_host = "127.0.0.1",
upstream_port = 8080,
})
}
init_worker_by_lua_block {
require("ledge").create_worker():run()
}
server {
server_name example.com;
listen 80;
location / {
content_by_lua_block {
require("ledge").create_handler():run()
}
}
}
}
```
[Back to TOC](#table-of-contents)
## Config systems
There are four different layers to the configuration system. Firstly there is the main [Redis config](#ledgeconfigure) and [handler defaults](#ledgeset_handler_defaults) config, which are global and must be set during the Nginx `init` phase.
Beyond this, you can specify [handler instance config](#ledgecreate_handler) on an Nginx `location` block basis, and finally there are some performance tuning config options for the [worker](#ledgecreate_worker) instances.
In addition, there is an [events system](#events-system) for binding Lua functions to mid-request events, proving opportunities to dynamically alter configuration.
[Back to TOC](#table-of-contents)
## Events system
Ledge makes most of its decisions based on the content it is working with. HTTP request and response headers drive the semantics for content delivery, and so rather than having countless configuration options to change this, we instead provide opportunities to alter the given semantics when necessary.
For example, if an `upstream` fails to set a long enough cache expiry, rather than inventing an option such as "extend\_ttl", we instead would `bind` to the `after_upstream_request` event, and adjust the response headers to include the ttl we're hoping for.
```lua
handler:bind("after_upstream_request", function(res)
res.header["Cache-Control"] = "max-age=86400"
end)
```
This particular event fires after we've fetched upstream, but before Ledge makes any decisions about whether the content can be cached or not. Once we've adjustead our headers, Ledge will read them as if they came from the upstream itself.
Note that multiple functions can be bound to a single event, either globally or per handler, and they will be called in the order they were bound. There is also currently no means to inspect which functions have been bound, or to unbind them.
See the [events](#events) section for a complete list of events and their definitions.
[Back to TOC](#table-of-contents)
### Binding globally
Binding a function globally means it will fire for the given event, on all requests. This is perhaps useful if you have many different `location` blocks, but need to always perform the same logic.
```lua
init_by_lua_block {
require("ledge").bind("before_serve", function(res)
res.header["X-Foo"] = "bar" -- always set X-Foo to bar
end)
}
```
[Back to TOC](#table-of-contents)
### Binding to handlers
More commonly, we just want to alter behaviour for a given Nginx `location`.
```lua
location /foo_location {
content_by_lua_block {
local handler = require("ledge").create_handler()
handler:bind("before_serve", function(res)
res.header["X-Foo"] = "bar" -- only set X-Foo for this location
end)
handler:run()
}
}
```
[Back to TOC](#table-of-contents)
### Performance implications
Writing simple logic for events is not expensive at all (and in many cases will be JIT compiled). If you need to consult service endpoints during an event then obviously consider that this will affect your overall latency, and make sure you do everything in a **non-blocking** way, e.g. using [cosockets](https://github.com/openresty/lua-nginx-module#ngxsockettcp) provided by OpenResty, or a driver based upon this.
If you have lots of event handlers, consider that creating closures in Lua is relatively expensive. A good solution would be to make your own module, and pass the defined functions in.
```lua
location /foo_location {
content_by_lua_block {
local handler = require("ledge").create_handler()
handler:bind("before_serve", require("my.handler.hooks").add_foo_header)
handler:run()
}
}
```
[Back to TOC](#table-of-contents)
## Caching basics
For normal HTTP caching operation, no additional configuration is required. If the HTTP response indicates the resource can be cached, then it will cache it. If the HTTP request indicates it accepts cache, it will be served cache. Note that these two conditions aren't mutually exclusive - a request could specify `no-cache`, and this will indeed trigger a fetch upstream, but if the response is cacheable then it will be saved and served to subsequent cache-accepting requests.
For more information on the myriad factors affecting this, including end-to-end revalidation and so on, please refer to [RFC 7234](https://tools.ietf.org/html/rfc7234).
The goal is to be 100% RFC compliant, but with some extensions to allow more agressive caching in certain cases. If something doesn't work as you expect, please do feel free to [raise an issue](https://github.com/pintsized/ledge).
[Back to TOC](#table-of-contents)
## Purging
To manually invalidate a cache item (or purge), we support the non-standard `PURGE` method familiar to users of Squid. Send a HTTP request to the URI with the method set, and Ledge will attempt to invalidate the item, returning status `200` on success and `404` if the URI was not found in cache, along with a JSON body for more details.
A purge request will affect all representations associated with the cache key, for example compressed and uncompressed responses separated by the `Vary: Accept-Encoding` response header will all be purged.
`$> curl -X PURGE -H "Host: example.com" http://cache.example.com/page1 | jq .`
```json
{
"purge_mode": "invalidate",
"result": "nothing to purge"
}
```
There are three purge modes, selectable by setting the `X-Purge` request header with one or more of the following values:
* `invalidate`: (default) marks the item as expired, but doesn't delete anything.
* `delete`: hard removes the item from cache
* `revalidate`: invalidates but then schedules a background revalidation to re-prime the cache.
`$> curl -X PURGE -H "X-Purge: revalidate" -H "Host: example.com" http://cache.example.com/page1 | jq .`
```json
{
"purge_mode": "revalidate",
"qless_job": {
"options": {
"priority": 4,
"jid": "5eeabecdc75571d1b93e9c942dfcebcb",
"tags": [
"revalidate"
]
},
"jid": "5eeabecdc75571d1b93e9c942dfcebcb",
"klass": "ledge.jobs.revalidate"
},
"result": "already expired"
}
```
Background revalidation jobs can be tracked in the qless metadata. See [managing qless](#managing-qless) for more information.
In general, `PURGE` is considered an administration task and probably shouldn't be allowed from the internet. Consider limiting it by IP address for example:
```nginx
limit_except GET POST PUT DELETE {
allow 127.0.0.1;
deny all;
}
```
[Back to TOC](#table-of-contents)
### JSON API
A JSON based API is also available for purging cache multiple cache items at once.
This requires a `PURGE` request with a `Content-Type` header set to `application/json` and a valid JSON request body.
Valid parameters
* `uris` - Array of URIs to purge, can contain wildcard URIs
* `purge_mode` - As the `X-Purge` header in a normal purge request
* `headers` - Hash of additional headers to include in the purge request
Returns a results hash keyed by URI or a JSON error response
`$> curl -X PURGE -H "Content-Type: Application/JSON" http://cache.example.com/ -d '{"uris": ["http://www.example.com/1", "http://www.example.com/2"]}' | jq .`
```json
{
"purge_mode": "invalidate",
"result": {
"http://www.example.com/1": {
"result": "purged"
},
"http://www.example.com/2":{
"result": "nothing to purge"
}
}
}
```
[Back to TOC](#table-of-contents)
### Wildcard purging
Wildcard (\*) patterns are also supported in `PURGE` URIs, which will always return a status of `200` and a JSON body detailing a background job. Wildcard purges involve scanning the entire keyspace, and so can take a little while. See [keyspace\_scan\_count](#keyspace_scan_count) for tuning help.
In addition, the `X-Purge` mode will propagate to all URIs purged as a result of the wildcard, making it possible to trigger site / section wide revalidation for example. Be careful what you wish for.
`$> curl -v -X PURGE -H "X-Purge: revalidate" -H "Host: example.com" http://cache.example.com/* | jq .`
```json
{
"purge_mode": "revalidate",
"qless_job": {
"options": {
"priority": 5,
"jid": "b2697f7cb2e856cbcad1f16682ee20b0",
"tags": [
"purge"
]
},
"jid": "b2697f7cb2e856cbcad1f16682ee20b0",
"klass": "ledge.jobs.purge"
},
"result": "scheduled"
}
```
[Back to TOC](#table-of-contents)
## Serving stale
Content is considered "stale" when its age is beyond its TTL. However, depending on the value of [keep_cache_for](#keep_cache_for) (which defaults to 1 month), we don't actually expire content in Redis straight away.
This allows us to implement the stale cache control extensions described in [RFC5861](https://tools.ietf.org/html/rfc5861), which provides request and response header semantics for describing how stale something can be served, when it should be revalidated in the background, and how long we can serve stale content in the event of upstream errors.
This can be very effective in ensuring a fast user experience. For example, if your content has a genuine `max-age` of 24 hours, consider changing this to 1 hour, and adding `stale-while-revalidate` for 23 hours. The total TTL is therefore the same, but the first request after the first hour will trigger backgrounded revalidation, extending the TTL for a further 1 hour + 23 hours.
If your origin server cannot be configured in this way, you can always override by [binding](#events) to the [before_save](#before_save) event.
```lua
handler:bind("before_save", function(res)
-- Valid for 1 hour, stale-while-revalidate for 23 hours, stale-if-error for three days
res.header["Cache-Control"] = "max-age=3600, stale-while-revalidate=82800, stale-if-error=259200"
end)
```
In other words, set the TTL to the highest comfortable frequency of requests at the origin, and `stale-while-revalidate` to the longest comfortable TTL, to increase the chances of background revalidation occurring. Note that the first stale request will obviously get stale content, and so very long values can result in very out of date content for one request.
All stale behaviours are constrained by normal cache control semantics. For example, if the origin is down, and the response could be served stale due to the upstream error, but the request contains `Cache-Control: no-cache` or even `Cache-Control: max-age=60` where the content is older than 60 seconds, they will be served the error, rather than the stale content.
[Back to TOC](#table-of-contents)
## Edge Side Includes
Almost complete support for the [ESI 1.0 Language Specification](https://www.w3.org/TR/esi-lang) is included, with a few exceptions, and a few enhancements.
```html
<html>
<esi:include="/header" />
<body>
<esi:choose>
<esi:when test="$(QUERY_STRING{foo}) == 'bar'">
Hi
</esi:when>
<esi:otherwise>
<esi:choose>
<esi:when test="$(HTTP_COOKIE{mycookie}) == 'yep'">
<esi:include src="http://example.com/_fragments/fragment1" />
</esi:when>
</esi:choose>
</esi:otherwise>
</esi:choose>
</body>
</html>
```
[Back to TOC](#table-of-contents)
### Enabling ESI
Note that simply [enabling](#esi_enabled) ESI might not be enough. We also check the [content type](#esi_content_types) against the allowed types specified, but more importantly ESI processing is contingent upon the [Edge Architecture Specification](https://www.w3.org/TR/edge-arch/). When enabled, Ledge will advertise capabilities upstream with the `Surrogate-Capability` request header, and expect the upstream response to include a `Surrogate-Control` header delegating ESI processing to Ledge.
If your upstream is not ESI aware, a common approach is to bind to the [after\_upstream\_request](#after_upstream_request) event in order to add the `Surrogate-Control` header manually. E.g.
```lua
handler:bind("after_upstream_request", function(res)
-- Don't enable ESI on redirect responses
-- Don't override Surrogate Control if it already exists
local status = res.status
if not res.header["Surrogate-Control"] and not (status > 300 and status < 303) then
res.header["Surrogate-Control"] = 'content="ESI/1.0"'
end
end)
```
Note that if ESI is processed, downstream cache-ability is automatically dropped since you don't want other intermediaries or browsers caching the result.
It's therefore best to only set `Surrogate-Control` for content which you know has ESI instructions. Whilst Ledge will detect the presence of ESI instructions when saving (and do nothing on cache HITs if no instructions are present), on a cache MISS it will have already dropped downstream cache headers before reading / saving the body. This is a side-effect of the [streaming design](#streaming-design).
[Back to TOC](#table-of-contents)
### Regular expressions in conditions
In addition to the operators defined in the
[ESI specification](https://www.w3.org/TR/esi-lang), we also support regular
expressions in conditions (as string literals), using the `=~` operator.
```html
<esi:choose>
<esi:when test="$(QUERY_STRING{name}) =~ '/james|john/i'">
Hi James or John
</esi:when>
</esi:choose>
```
Supported modifiers are as per the [ngx.re.\*](https://github.com/openresty/lua-nginx-module#ngxrematch) documentation.
[Back to TOC](#table-of-contents)
### Custom ESI variables
In addition to the variables defined in the [ESI specification](https://www.w3.org/TR/esi-lang), it is possible to provide run time custom variables using the [esi_custom_variables](#esi_custom_variables) handler config option.
```lua
content_by_lua_block {
require("ledge").create_handler({
esi_custom_variables = {
messages = {
foo = "bar",
},
},
}):run()
}
```
```html
<esi:vars>$(MESSAGES{foo})</esi:vars>
```
[Back to TOC](#table-of-contents)
### ESI Args
It can be tempting to use URI arguments to pages using ESI in order to change layout dynamically, but this comes at the cost of generating multiple cache items - one for each permutation of URI arguments.
ESI args is a neat feature to get around this, by using a configurable [prefix](#esi_args_prefix), which defaults to `esi_`. URI arguments with this prefix are removed from the cache key and also from upstream requests, and instead stuffed into the `$(ESI_ARGS{foo})` variable for use in ESI, typically in conditions. That is, think of them as magic URI arguments which have meaning for the ESI processor only, and should never affect cacheability or upstream content generation.
`$> curl -H "Host: example.com" http://cache.example.com/page1?esi_display_mode=summary`
```html
<esi:choose>
<esi:when test="$(ESI_ARGS{display_mode}) == 'summary'">
<!-- SUMMARY -->
</esi:when>
<esi:when test="$(ESI_ARGS{display_mode}) == 'details'">
<!-- DETAILS -->
</esi:when>
</esi:choose>
```
In this example, the `esi_display_mode` values of `summary` or `details` will return the same cache HIT, but display different content.
If `$(ESI_ARGS)` is used without a field key, it renders the original query string arguments, e.g. `esi_foo=bar&esi_display_mode=summary`, URL encoded.
[Back to TOC](#table-of-contents)
### Variable Escaping
ESI variables are minimally escaped by default in order to prevent user's injecting additional ESI tags or XSS exploits.
Unescaped variables are available by prefixing the variable name with `RAW_`. This should be used with care.
```html
# /esi/test.html?a=<script>alert()</script>
<esi:vars>
$(QUERY_STRING{a}) <!-- <script>alert()</script> -->
$(RAW_QUERY_STRING{a}) <!-- <script>alert()</script> -->
</esi:vars>
```
[Back to TOC](#table-of-contents)
### Missing ESI features
The following parts of the [ESI specification](https://www.w3.org/TR/esi-lang) are not supported, but could be in due course if a need is identified.
* `<esi:inline>` not implemented (or advertised as a capability).
* No support for the `onerror` or `alt` attributes for `<esi:include>`. Instead, we "continue" on error by default.
* `<esi:try | attempt | except>` not implemented.
* The "dictionary (special)" substructure variable type for `HTTP_USER_AGENT` is not implemented.
[Back to TOC](#table-of-contents)
## API
### ledge.configure
syntax: `ledge.configure(config)`
This function provides Ledge with Redis connection details for all cache `metadata` and background jobs. This is global and cannot be specified or adjusted outside the Nginx `init` phase.
```lua
init_by_lua_block {
require("ledge").configure({
redis_connector_params = {
url = "redis://mypassword@127.0.0.1:6380/3",
}
qless_db = 4,
})
}
```
`config` is a table with the following options (unrecognised config will error hard on start up).
[Back to TOC](#table-of-contents)
#### redis_connector_params
`default: {}`
Ledge uses [lua-resty-redis-connector](https://github.com/pintsized/lua-resty-redis-connector) to handle all Redis connections. It simply passes anything given in `redis_connector_params` straight to [lua-resty-redis-connector](https://github.com/pintsized/lua-resty-redis-connector), so review the documentation there for options, including how to use [Redis Sentinel](https://redis.io/topics/sentinel).
#### qless_db
`default: 1`
Specifies the Redis DB number to store [qless](https://github.com/pintsized/lua-resty-qless) background job data.
[Back to TOC](#table-of-contents)
### ledge.set\_handler\_defaults
syntax: `ledge.set_handler_defaults(config)`
This method overrides the default configuration used for all spawned request `handler` instances. This is global and cannot be specified or adjusted outside the Nginx `init` phase, but these defaults can be overriden on a per `handler` basis. See [below](#handler-configuration-options) for a complete list of configuration options.
```lua
init_by_lua_block {
require("ledge").set_handler_defaults({
upstream_host = "127.0.0.1",
upstream_port = 8080,
})
}
```
[Back to TOC](#table-of-contents)
### ledge.create\_handler
syntax: `local handler = ledge.create_handler(config)`
Creates a `handler` instance for the current reqiest. Config given here will be merged with the defaults, allowing certain options to be adjusted on a per Nginx `location` basis.
```lua
server {
server_name example.com;
listen 80;
location / {
content_by_lua_block {
require("ledge").create_handler({
upstream_port = 8081,
}):run()
}
}
}
```
[Back to TOC](#table-of-contents)
### ledge.create\_worker
syntax: `local worker = ledge.create_worker(config)`
Creates a `worker` instance inside the current Nginx worker process, for processing background jobs. You only need to call this once inside a single `init_worker` block, and it will be called for each Nginx worker that is configured.
Job queues can be run at varying amounts of concurrency per worker, which can be set by providing `config` here. See [managing qless](#managing-qless) for more details.
```lua
init_worker_by_lua_block {
require("ledge").create_worker({
interval = 1,
gc_queue_concurrency = 1,
purge_queue_concurrency = 2,
revalidate_queue_concurrency = 5,
}):run()
}
```
[Back to TOC](#table-of-contents)
### ledge.bind
syntax: `ledge.bind(event_name, callback)`
Binds the `callback` function to the event given in `event_name`, globally for all requests on this system. Arguments to `callback` vary based on the event. See [below](#events) for event definitions.
[Back to TOC](#table-of-contents)
### handler.bind
syntax: `handler:bind(event_name, callback)`
Binds the `callback` function to the event given in `event_name` for this handler only. Note the `:` in `handler:bind()` which differs to the global `ledge.bind()`.
Arguments to `callback` vary based on the event. See [below](#events) for event definitions.
[Back to TOC](#table-of-contents)
### handler.run
syntax: `handler:run()`
Must be called during the `content_by_lua` phase. It processes the current request and serves a response. If you fail to call this method in your `location` block, nothing will happen.
[Back to TOC](#table-of-contents)
### worker.run
syntax: `handler:run()`
Must be called during the `init_worker` phase, otherwise background tasks will not be run, including garbage collection which is very importatnt.
[Back to TOC](#table-of-contents)
### Handler configuration options
* [storage_driver](#storage_driver)
* [storage_driver_config](#storage_driver_config)
* [origin_mode](#origin_mode)
* [upstream_connect_timeout](#upstream_connect_timeout)
* [upstream_send_timeout](#upstream_send_timeout)
* [upstream_read_timeout](#upstream_read_timeout)
* [upstream_keepalive_timeout](#upstream_keepalive_timeout)
* [upstream_keepalive_poolsize](#upstream_keepalive_poolsize)
* [upstream_host](#upstream_host)
* [upstream_port](#upstream_port)
* [upstream_use_ssl](#upstream_use_ssl)
* [upstream_ssl_server_name](#upstream_ssl_server_name)
* [upstream_ssl_verify](#upstream_ssl_verify)
* [buffer_size](#buffer_size)
* [advertise_ledge](#buffer_size)
* [keep_cache_for](#buffer_size)
* [minimum_old_entity_download_rate](#minimum_old_entity_download_rate)
* [esi_enabled](#esi_enabled)
* [esi_content_types](#esi_content_types)
* [esi_allow_surrogate_delegation](#esi_allow_surrogate_delegation)
* [esi_recursion_limit](#esi_recursion_limit)
* [esi_args_prefix](#esi_args_prefix)
* [esi_custom_variables](#esi_custom_variables)
* [esi_max_size](#esi_max_size)
* [esi_attempt_loopback](#esi_attempt_loopback)
* [esi_vars_cookie_blacklist](#esi_vars_cookie_blacklist)
* [esi_disable_third_party_includes](#esi_disable_third_party_includes)
* [esi_third_party_includes_domain_whitelist](#esi_third_party_includes_domain_whitelist)
* [enable_collapsed_forwarding](#enable_collapsed_forwarding)
* [collapsed_forwarding_window](#collapsed_forwarding_window)
* [gunzip_enabled](#gunzip_enabled)
* [keyspace_scan_count](#keyspace_scan_count)
* [cache_key_spec](#cache_key_spec)
* [max_uri_args](#max_uri_args)
#### storage_driver
default: `ledge.storage.redis`
This is a `string` value, which will be used to attempt to load a storage driver. Any third party driver here can accept its own config options (see below), but must provide the following interface:
* `bool new()`
* `bool connect()`
* `bool close()`
* `number get_max_size()` *(return nil for no max)*
* `bool exists(string entity_id)`
* `bool delete(string entity_id)`
* `bool set_ttl(string entity_id, number ttl)`
* `number get_ttl(string entity_id)`
* `function get_reader(object response)`
* `function get_writer(object response, number ttl, function onsuccess, function onfailure)`
*Note, whilst it is possible to configure storage drivers on a per `location` basis, it is **strongly** recommended that you never do this, and consider storage drivers to be system wide, much like the main Redis config. If you really need differenet storage driver configurations for different locations, then it will work, but features such as purging using wildcards will silently not work. YMMV.*
[Back to TOC](#handler-configuration-options)
#### storage_driver_config
`default: {}`
Storage configuration can vary based on the driver. Currently we only have a Redis driver.
[Back to TOC](#handler-configuration-options)
##### Redis storage driver config
* `redis_connector_params` Redis params table, as per [lua-resty-redis-connector](https://github.com/pintsized/lua-resty-redis-connector)
* `max_size` (bytes), defaults to `1MB`
* `supports_transactions` defaults to `true`, set to false if using a Redis proxy.
If `supports_transactions` is set to `false`, cache bodies are not written atomically. However, if there is an error writing, the main Redis system will be notified and the overall transaction will be aborted. The result being potentially orphaned body entities in the storage system, which will hopefully eventually expire. The only reason to turn this off is if you are using a Redis proxy, as any transaction related commands will break the connection.
[Back to TOC](#handler-configuration-options)
#### upstream_connect_timeout
default: `1000 (ms)`
Maximum time to wait for an upstream connection (in milliseconds). If it is exceeded, we send a `503` status code, unless [stale_if_error](#stale_if_error) is configured.
[Back to TOC](#handler-configuration-options)
#### upstream_send_timeout
default: `2000 (ms)`
Maximum time to wait sending data on a connected upstream socket (in milliseconds). If it is exceeded, we send a `503` status code, unless [stale_if_error](#stale_if_error) is configured.
[Back to TOC](#handler-configuration-options)
#### upstream_read_timeout
default: `10000 (ms)`
Maximum time to wait on a connected upstream socket (in milliseconds). If it is exceeded, we send a `503` status code, unless [stale_if_error](#stale_if_error) is configured.
[Back to TOC](#handler-configuration-options)
#### upstream_keepalive_timeout
default: `75000`
[Back to TOC](#handler-configuration-options)
#### upstream_keepalive_poolsize
default: `64`
[Back to TOC](#handler-configuration-options)
#### upstream_host
default: `""`
Specifies the hostname or IP address of the upstream host. If a hostname is specified, you must configure the Nginx [resolver](http://nginx.org/en/docs/http/ngx_http_core_module.html#resolver) somewhere, for example:
```nginx
resolver 8.8.8.8;
```
[Back to TOC](#handler-configuration-options)
#### upstream_port
default: `80`
Specifies the port of the upstream host.
[Back to TOC](#handler-configuration-options)
#### upstream_use_ssl
default: `false`
Toggles the use of SSL on the upstream connection. Other `upstream_ssl_*` options will be ignored if this is not set to `true`.
[Back to TOC](#handler-configuration-options)
#### upstream_ssl_server_name
default: `""`
Specifies the SSL server name used for Server Name Indication (SNI). See [sslhandshake](https://github.com/openresty/lua-nginx-module#tcpsocksslhandshake) for more information.
[Back to TOC](#handler-configuration-options)
#### upstream_ssl_verify
default: `true`
Toggles SSL verification. See [sslhandshake](https://github.com/openresty/lua-nginx-module#tcpsocksslhandshake) for more information.
[Back to TOC](#handler-configuration-options)
#### cache_key_spec
`default: cache_key_spec = { "scheme", "host", "uri", "args" },`
Specifies the format for creating cache keys. The default spec above will create keys in Redis similar to:
```
ledge:cache:http:example.com:/about::
ledge:cache:http:example.com:/about:p=2&q=foo:
```
The list of available string identifiers in the spec is:
* `scheme` either http or https
* `host` the hostname of the current request
* `port` the public port of the current request
* `uri` the URI (without args)
* `args` the URI args, sorted alphabetically
In addition to these string identifiers, dynamic parameters can be added to the cache key by providing functions. Any functions given must expect no arguments and return a string value.
```lua
local function get_device_type()
-- dynamically work out device type
return "tablet"
end
require("ledge").create_handler({
cache_key_spec = {
get_device_type,
"scheme",
"host",
"uri",
"args",
}
}):run()
```
Consider leveraging vary, via the [before_vary_selection](#before_vary_selection) event, for separating cache entries rather than modifying the main `cache_key_spec` directly.
[Back to TOC](#handler-configuration-options)
#### origin_mode
default: `ledge.ORIGIN_MODE_NORMAL`
Determines the overall behaviour for connecting to the origin. `ORIGIN_MODE_NORMAL` will assume the origin is up, and connect as necessary.
`ORIGIN_MODE_AVOID` is similar to Squid's `offline_mode`, where any retained cache (expired or not) will be served rather than trying the origin, regardless of cache-control headers, but the origin will be tried if there is no cache to serve.
`ORIGIN_MODE_BYPASS` is the same as `AVOID`, except if there is no cache to serve we send a `503 Service Unavailable` status code to the client and never attempt an upstream connection.
[Back to TOC](#handler-configuration-options)
#### keep_cache_for
default: `86400 * 30 (1 month in seconds)`
Specifies how long to retain cache data past its expiry date. This allows us to serve stale cache in the event of upstream failure with [stale_if_error](#stale_if_error) or [origin_mode](#origin_mode) settings.
Items will be evicted when under memory pressure provided you are using one of the Redis [volatile eviction policies](http://redis.io/topics/lru-cache), so there should generally be no real need to lower this for space reasons.
Items at the extreme end of this (i.e. nearly a month old) are clearly very rarely requested, or more likely, have been removed at the origin.
[Back to TOC](#handler-configuration-options)
#### minimum_old_entity_download_rate
default: `56 (kbps)`
Clients reading slower than this who are also unfortunate enough to have started reading from an entity which has been replaced (due to another client causing a revalidation for example), may have their entity garbage collected before they finish, resulting in an incomplete resource being delivered.
Lowering this is fairer on slow clients, but widens the potential window for multiple old entities to stack up, which in turn could threaten Redis storage space and force evictions.
[Back to TOC](#handler-configuration-options)
#### enable_collapsed_forwarding
default: `false`
[Back to TOC](#handler-configuration-options)
#### collapsed_forwarding_window
When collapsed forwarding is enabled, if a fatal error occurs during the origin request, the collapsed requests may never receive the response they are waiting for. This setting puts a limit on how long they will wait, and how long before new requests will decide to try the origin for themselves.
If this is set shorter than your origin takes to respond, then you may get more upstream requests than desired. Fatal errors (server reboot etc) may result in hanging connections for up to the maximum time set. Normal errors (such as upstream timeouts) work independently of this setting.
[Back to TOC](#handler-configuration-options)
#### gunzip_enabled
default: `true`
With this enabled, gzipped responses will be uncompressed on the fly for clients that do not set `Accept-Encoding: gzip`. Note that if we receive a gzipped response for a resource containing ESI instructions, we gunzip whilst saving and store uncompressed, since we need to read the ESI instructions.
Also note that `Range` requests for gzipped content must be ignored - the full response will be returned.
[Back to TOC](#handler-configuration-options)
#### buffer_size
default: `2^16 (64KB in bytes)`
Specifies the internal buffer size (in bytes) used for data to be read/written/served. Upstream responses are read in chunks of this maximum size, preventing allocation of large amounts of memory in the event of receiving large files. Data is also stored internally as a list of chunks, and delivered to the Nginx output chain buffers in the same fashion.
The only exception is if ESI is configured, and Ledge has determined there are ESI instructions to process, and any of these instructions span a given chunk. In this case, buffers are concatenated until a complete instruction is found, and then ESI operates on this new buffer, up to a maximum of [esi_max_size](#esi_max_size).
[Back to TOC](#handler-configuration-options)
#### keyspace_scan_count
default: `1000`
Tunes the behaviour of keyspace scans, which occur when sending a PURGE request with wildcard syntax. A higher number may be better if latency to Redis is high and the keyspace is large.
[Back to TOC](#handler-configuration-options)
#### max_uri_args
default: `100`
Limits the number of URI arguments returned in calls to [ngx.req.get_uri_args()](https://github.com/openresty/lua-nginx-module#ngxreqget_uri_args), to protect against DOS attacks.
[Back to TOC](#handler-configuration-options)
#### esi_enabled
default: `false`
Toggles [ESI](http://www.w3.org/TR/esi-lang) scanning and processing, though behaviour is also contingent upon [esi_content_types](#esi_content_types) and [esi_surrogate_delegation](#esi_surrogate_delegation) settings, as well as `Surrogate-Control` / `Surrogate-Capability` headers.
ESI instructions are detected on the slow path (i.e. when fetching from the origin), so only instructions which are known to be present are processed on cache HITs.
[Back to TOC](#handler-configuration-options)
#### esi_content_types
default: `{ text/html }`
Specifies content types to perform ESI processing on. All other content types will not be considered for processing.
[Back to TOC](#handler-configuration-options)
#### esi_allow_surrogate_delegation
default: false
[ESI Surrogate Delegation](http://www.w3.org/TR/edge-arch) allows for downstream intermediaries to advertise a capability to process ESI instructions nearer to the client. By setting this to `true` any downstream offering this will disable ESI processing in Ledge, delegating it downstream.
When set to a Lua table of IP address strings, delegation will only be allowed to this specific hosts. This may be important if ESI instructions contain sensitive data which must be removed.
[Back to TOC](#handler-configuration-options)
#### esi_recursion_limit
default: 10
Limits fragment inclusion nesting, to avoid accidental infinite recursion.
[Back to TOC](#handler-configuration-options)
#### esi_args_prefix
default: "esi\_"
URI args prefix for parameters to be ignored from the cache key (and not proxied upstream), for use exclusively with ESI rendering logic. Set to nil to disable the feature.
[Back to TOC](#handler-configuration-options)
#### esi_custom_variables
defualt: `{}`
Any variables supplied here will be available anywhere ESI vars can be used evaluated. See [Custom ESI variables](#custom-esi-variables).
[Back to TOC](#handler-configuration-options)
#### esi_max_size
default: `1024 * 1024 (bytes)`
[Back to TOC](#handler-configuration-options)
#### esi_attempt_loopback
default: `true`
If an ESI subrequest has the same `scheme` and `host` as the parent request, we loopback the connection to the current
`server_addr` and `server_port` in order to avoid going over network.
[Back to TOC](#handler-configuration-options)
#### esi_vars_cookie_blacklist
default: `{}`
Cookie names given here will not be expandable as ESI variables: e.g. `$(HTTP_COOKIE)` or `$(HTTP_COOKIE{foo})`. However they
are not removed from the request data, and will still be propagated to `<esi:include>` subrequests.
This is useful if your client is sending a sensitive cookie that you don't ever want to accidentally evaluate in server output.
```lua
require("ledge").create_handler({
esi_vars_cookie_blacklist = {
secret = true,
["my-secret-cookie"] = true,
}
}):run()
```
Cookie names are given as the table key with a truthy value, for O(1) runtime lookup.
[Back to TOC](#handler-configuration-options)
#### esi_disable_third_party_includes
default: `false`
`<esi:include>` tags can make requests to any arbitrary URI. Turn this on to ensure the URI domain must match the URI of the current request.
[Back to TOC](#handler-configuration-options)
#### esi_third_party_includes_domain_whitelist
default: `{}`
If third party includes are disabled, you can also explicitly provide a whitelist of allowed third party domains.
```lua
require("ledge").create_handler({
esi_disable_third_party_includes = true,
esi_third_party_includes_domain_whitelist = {
["example.com"] = true,
}
}):run()
```
Hostnames are given as the table key with a truthy value, for O(1) lookup.
*Note; This behaviour was introduced in v2.2*
[Back to TOC](#handler-configuration-options)
#### advertise_ledge
default `true`
If set to false, disables advertising the software name and version, e.g. `(ledge/2.01)` from the `Via` response header.
[Back to TOC](#handler-configuration-options)
### Events
* [after_cache_read](#after_cache_read)
* [before_upstream_connect](#before_upstream_connect)
* [before_upstream_request](#before_upstream_request)
* [before_esi_inclulde_request"](#before_esi_include_request)
* [after_upstream_request](#after_upstream_request)
* [before_save](#before_save)
* [before_serve](#before_serve)
* [before_save_revalidation_data](#before_save_revalidation_data)
* [before_vary_selection](#before_vary_selection)
#### after_cache_read
syntax: `bind("after_cache_read", function(res) -- end)`
params: `res`. The cached response table.
Fires directly after the response was successfully loaded from cache.
The `res` table given contains:
* `res.header` the table of case-insenitive HTTP response headers
* `res.status` the HTTP response status code
*Note; there are other fields and methods attached, but it is strongly advised to never adjust anything other than the above*
[Back to TOC](#events)
#### before_upstream_connect
syntax: `bind("before_upstream_connect", function(handler) -- end)`
params: `handler`. The current handler instance.
Fires before the default `handler.upstream_client` is created, allowing a pre-connected HTTP client to be externally provided. The client must be API compatible with [lua-resty-http](https://github.com/pintsized/lua-resty-http). For example, using [lua-resty-upstream](https://github.com/hamishforbes/lua-resty-upstream) for load balancing.
[Back to TOC](#events)
#### before_upstream_request
syntax: `bind("before_upstream_request", function(req_params) -- end)`
params: `req_params`. The table of request params about to send to the [request](https://github.com/pintsized/lua-resty-http#request) method.
Fires when about to perform an upstream request.
[Back to TOC](#events)
#### before_esi_include_request
syntax: `bind("before_esi_include_request", function(req_params) -- end)`
params: `req_params`. The table of request params about to be used for an ESI include, via the [request](https://github.com/pintsized/lua-resty-http#request) method.
Fires when about to perform a HTTP request on behalf of an ESI include instruction.
[Back to TOC](#events)
#### after_upstream_request
syntax: `bind("after_upstream_request", function(res) -- end)`
params: `res` The response table.
Fires when the status / headers have been fetched, but before the body it is stored. Typically used to override cache headers before we decide what to do with this response.
The `res` table given contains:
* `res.header` the table of case-insenitive HTTP response headers
* `res.status` the HTTP response status code
*Note; there are other fields and methods attached, but it is strongly advised to never adjust anything other than the above*
*Note: unlike `before_save` below, this fires for all fetched content, not just cacheable content.*
[Back to TOC](#events)
#### before_save
syntax: `bind("before_save", function(res) -- end)`
params: `res` The response table.
Fires when we're about to save the response.
The `res` table given contains:
* `res.header` the table of case-insenitive HTTP response headers
* `res.status` the HTTP response status code
*Note; there are other fields and methods attached, but it is strongly advised to never adjust anything other than the above*
[Back to TOC](#events)
#### before_serve
syntax: `ledge:bind("before_serve", function(res) -- end)`
params: `res` The `ledge.response` object.
Fires when we're about to serve. Often used to modify downstream headers.
The `res` table given contains:
* `res.header` the table of case-insenitive HTTP response headers
* `res.status` the HTTP response status code
*Note; there are other fields and methods attached, but it is strongly advised to never adjust anything other than the above*
[Back to TOC](#events)
#### before_save_revalidation_data
syntax: `bind("before_save_revalidation_data", function(reval_params, reval_headers) -- end)`
params: `reval_params`. Table of revalidation params.
params: `reval_headers`. Table of revalidation HTTP headers.
Fires when a background revalidation is triggered or when cache is being saved. Allows for modifying the headers and paramters (such as connection parameters) which are inherited by the background revalidation.
The `reval_params` are values derived from the current running configuration for:
* server_addr
* server_port
* scheme
* uri
* connect_timeout
* read_timeout
* ssl_server_name
* ssl_verify
[Back to TOC](#events)
#### before_vary_selection
syntax: `bind("before_vary_selection", function(vary_key) -- end)`
params: `vary_key` A table of selecting headers
Fires when we're about to generate the vary key, used to select the correct cache representation.
The `vary_key` table is a hash of header field names (lowercase) to values.
A field name which exists in the Vary response header but does not exist in the current request header will have a value of `ngx.null`.
```
Request Headers:
Accept-Encoding: gzip
X-Test: abc
X-test: def
Response Headers:
Vary: Accept-Encoding, X-Test
Vary: X-Foo
vary_key table:
{
["accept-encoding"] = "gzip",
["x-test"] = "abc,def",
["x-foo"] = ngx.null
}
```
[Back to TOC](#events)
## Administration
### X-Cache
Ledge adds the non-standard `X-Cache` header, familiar to users of other caches. It indicates simply `HIT` or `MISS` and the host name in question, preserving upstream values when more than one cache server is in play.
If a resource is considered not cacheable, the `X-Cache` header will not be present in the response.
For example:
* `X-Cache: HIT from ledge.tld` *A cache hit, with no (known) cache layer upstream.*
* `X-Cache: HIT from ledge.tld, HIT from proxy.upstream.tld` *A cache hit, also hit upstream.*
* `X-Cache: MISS from ledge.tld, HIT from proxy.upstream.tld` *A cache miss, but hit upstream.*
* `X-Cache: MISS from ledge.tld, MISS from proxy.upstream.tld` *Regenerated at the origin.*
[Back to TOC](#table-of-contents)
### Logging
It's often useful to add some extra headers to your Nginx logs, for example
```
log_format ledge '$remote_addr - $remote_user [$time_local] '
'"$request" $status $body_bytes_sent '
'"$http_referer" "$http_user_agent" '
'"Cache:$sent_http_x_cache" "Age:$sent_http_age" "Via:$sent_http_via"'
;
access_log /var/log/nginx/access_log ledge;
```
Will give log lines such as:
```
192.168.59.3 - - [23/May/2016:22:22:18 +0000] "GET /x/y/z HTTP/1.1" 200 57840 "-" "curl/7.37.1""Cache:HIT from 159e8241f519:8080" "Age:724"
```
[Back to TOC](#table-of-contents)
### Managing Qless
Ledge uses [lua-resty-qless](https://github.com/pintsized/lua-resty-qless) to schedule and process background tasks, which are stored in Redis.
Jobs are scheduled for background revalidation requests as well as wildcard PURGE requests, but most importantly for garbage collection of replaced body entities.
That is, it's very important that jobs are being run properly and in a timely fashion.
Installing the [web user interface](https://github.com/hamishforbes/lua-resty-qless-web) can be very helpful to check this.
You may also wish to tweak the [qless job history](https://github.com/pintsized/lua-resty-qless#configuration-options) settings if it takes up too much space.
[Back to TOC](#table-of-contents)
## Author
James Hurst <james@pintsized.co.uk>
## Licence
This module is licensed under the 2-clause BSD license.
Copyright (c) James Hurst <james@pintsized.co.uk>
All rights reserved.
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
================================================
FILE: dist.ini
================================================
name=ledge
abstract=An RFC compliant and ESI capable HTTP cache for Nginx / OpenResty, backed by Redis
author=James Hurst, Hamish Forbes
is_original=yes
license=2bsd
lib_dir=lib
repo_link=https://github.com/pintsized/ledge
main_module=lib/ledge.lua
requires = pintsized/lua-resty-http >= 0.11, pintsized/lua-resty-redis-connector >= 0.06, pintsized/lua-resty-qless >= 0.11, p0pr0ck5/lua-resty-cookie >= 0.01, hamishforbes/lua-ffi-zlib >= 0.3.0
================================================
FILE: docker/tests/docker-compose.yml
================================================
version: '3'
services:
runner:
image: "ledgetech/test-runner:latest"
volumes:
- ../../:/code
# Use this to mount any local Lua dependencies, overriding
# published versions
- ${EXTLIB-../../lib}:/code/extlib
environment:
- TEST_FILE
command: /bin/bash -c "TEST_LEDGE_REDIS_HOST=redis make coverage"
working_dir: /code
depends_on:
- redis
redis:
image: "redis:alpine"
================================================
FILE: lib/ledge/background.lua
================================================
local require = require
local math_ceil = math.ceil
local qless = require("resty.qless")
local _M = {
_VERSION = "2.3.0",
}
local function put_background_job( queue, klass, data, options)
local q = qless.new({
get_redis_client = require("ledge").create_qless_connection
})
-- If we've been specified a jid (i.e. a non random jid), putting this
-- job will overwrite any existing job with the same jid.
-- We test for a "running" state, and if so we silently drop this job.
if options.jid then
local existing = q.jobs:get(options.jid)
if existing and existing.state == "running" then
return nil, "Job with the same jid is currently running"
end
end
-- Put the job
local res, err = q.queues[queue]:put(klass, data, options)
q:redis_close()
if res then
return {
jid = res,
klass = klass,
options = options,
}
else
return res, err
end
end
_M.put_background_job = put_background_job
-- Calculate when to GC an entity based on its size and the minimum download
-- rate setting, plus 1 second of arbitrary latency for good measure.
local function gc_wait(entity_size, minimum_download_rate)
local dl_rate_Bps = minimum_download_rate * 128
return math_ceil((entity_size / dl_rate_Bps)) + 1
end
_M.gc_wait = gc_wait
return _M
================================================
FILE: lib/ledge/cache_key.lua
================================================
local ipairs, next, type, pcall, setmetatable =
ipairs, next, type, pcall, setmetatable
local str_lower = string.lower
local ngx_log = ngx.log
local ngx_ERR = ngx.ERR
local ngx_var = ngx.var
local ngx_null = ngx.null
local tbl_insert = table.insert
local tbl_concat = table.concat
local tbl_sort = table.sort
local req_args_sorted = require("ledge.request").args_sorted
local req_default_args = require("ledge.request").default_args
local get_fixed_field_metatable_proxy =
require("ledge.util").mt.get_fixed_field_metatable_proxy
local http_headers = require("resty.http_headers")
local _M = {
_VERSION = "2.3.0",
}
-- Generates the root key. The default spec is:
-- ledge:cache_obj:http:example.com:/about:p=3&q=searchterms
local function generate_root_key(key_spec, max_args)
-- If key_spec is empty, provide a default
if not key_spec or not next(key_spec) then
key_spec = {
"scheme",
"host",
"uri",
"args",
}
end
local key = {
"ledge",
"cache",
}
for _, field in ipairs(key_spec) do
if field == "scheme" then
tbl_insert(key, ngx_var.scheme)
elseif field == "host" then
tbl_insert(key, ngx_var.host)
elseif field == "port" then
tbl_insert(key, ngx_var.server_port)
elseif field == "uri" then
tbl_insert(key, ngx_var.uri)
elseif field == "args" then
tbl_insert(
key,
req_args_sorted(max_args) or req_default_args()
)
elseif type(field) == "function" then
local ok, res = pcall(field)
if not ok then
ngx_log(ngx_ERR,
"error in function supplied to cache_key_spec: ", res
)
elseif type(res) ~= "string" then
ngx_log(ngx_ERR,
"functions supplied to cache_key_spec must " ..
"return a string"
)
else
tbl_insert(key, res)
end
end
end
return tbl_concat(key, ":")
end
_M.generate_root_key = generate_root_key
-- Read the list of vary headers from redis
local function read_vary_spec(redis, root_key)
if not redis or not next(redis) then
return nil, "Redis required"
end
if not root_key then
return nil, "Root key required"
end
local res, err = redis:smembers(root_key.."::vary")
if err then
return nil, err
end
table.sort(res)
return res
end
_M.read_vary_spec = read_vary_spec
local function vary_compare(spec_a, spec_b)
if (not spec_a or not next(spec_a)) then
if (not spec_b or not next(spec_b)) then
-- both nil or empty
return true
else
-- spec_b is set but spec_a is empty
return false
end
elseif (spec_b and next(spec_b)) then
local outer_match = true
-- Loop over all values in spec_a
for _, v in ipairs(spec_a) do
local match = false
-- Look for a match in spec_b
for _, v2 in ipairs(spec_b) do
if v == v2 then
match = true
break
end
end
-- Didn't match any values in spec_b
if match == false then
outer_match = false
break
end
end
return outer_match
end
-- spec_a is a thing but spec_b is not
return false
end
_M.vary_compare = vary_compare
local function generate_vary_key(vary_spec, callback, headers)
local vary_key = http_headers.new()
if vary_spec and next(vary_spec) then
headers = headers or ngx.req.get_headers()
for _, h in ipairs(vary_spec) do
local v = headers[h]
if type(v) == "table" then
v = tbl_concat(v, ",")
end
-- ngx.null represents a key which was in the spec
-- but has no matching request header
vary_key[h] = v or ngx_null
end
end
-- Callback allows user to modify the key
if type(callback) == "function" then
callback(vary_key)
end
if not next(vary_key) then
return ""
end
-- Extract keys and sort them
local keys = {}
for k,v in pairs(vary_key) do
if v ~= ngx_null then
tbl_insert(keys, k)
end
end
tbl_sort(keys)
-- Convert hash table to flat array
local t = {}
local i = 1
for _, k in ipairs(keys) do
t[i] = k
t[i + 1] = vary_key[k]
i = i + 2
end
return str_lower(tbl_concat(t, ":"))
end
_M.generate_vary_key = generate_vary_key
-- Returns the key chain for all cache keys, except the body entity
local function key_chain(root_key, vary_key, vary_spec)
if not root_key then
return nil, "Missing root key"
end
if not vary_key then
return nil, "Missing vary key"
end
if not vary_spec then
return nil, "Missing vary_spec"
end
local full_key = root_key .. "#" .. vary_key
-- Apply metatable
local key_chain = setmetatable({
-- hash: cache key metadata
main = full_key .. "::main",
-- sorted set: current entities score with sizes
entities = full_key .. "::entities",
-- hash: response headers
headers = full_key .. "::headers",
-- hash: request headers for revalidation
reval_params = full_key .. "::reval_params",
-- hash: request params for revalidation
reval_req_headers = full_key .. "::reval_req_headers",
}, get_fixed_field_metatable_proxy({
-- Hide these keys from iterators
-- These are not actual keys but useful to keep around
root = root_key,
full = full_key,
vary_spec = vary_spec,
-- set: headers upon which to vary
vary = root_key .. "::vary",
-- set: representations for this root key
repset = root_key .. "::repset",
-- Lock key for collapsed forwarding
fetching_lock = full_key .. "::fetching",
})
)
return key_chain
end
_M.key_chain = key_chain
local function clean_repset(redis, repset)
-- Ensure representation set only includes keys which actually exist
-- This only runs on the slow path at save time so should be ok?
-- Prevents this set from growing perpetually if there are unique variations
-- TODO use scan here incase the set is pathologically huge?
-- Has to be able to run in a transaction so maybe a housekeeping qless job?
local clean = [[
local repset = KEYS[1]
local reps = redis.call("SMEMBERS", repset)
for _, rep in ipairs(reps) do
if redis.call("EXISTS", rep.."::main") == 0 then
redis.call("SREM", repset, rep)
end
end
]]
local res, err = redis:eval(clean, 1, repset)
if not res or res == ngx_null then
return nil, err
end
return true
end
local function save_key_chain(redis, key_chain, ttl)
if not redis then
return nil, "Redis required"
end
if type(key_chain) ~= "table" or not next(key_chain) then
return nil, "Key chain required"
end
if not tonumber(ttl) then
return nil, "TTL must be a number"
end
-- Delete the current set of vary headers
local _, e = redis:del(key_chain.vary)
if e then ngx_log(ngx_ERR, e) end
local vary_spec = key_chain.vary_spec
if next(vary_spec) then
-- Always lowercase all vary fields
-- key_chain.vary is a set so will deduplicate for us
for i,v in ipairs(vary_spec) do
vary_spec[i] = str_lower(v)
end
local _, e = redis:sadd(key_chain.vary, unpack(vary_spec))
if e then ngx_log(ngx_ERR, e) end
local _, e = redis:expire(key_chain.vary, ttl)
if e then ngx_log(ngx_ERR, e) end
end
-- Add this representation to the set
local _, e = redis:sadd(key_chain.repset, key_chain.full)
if e then ngx_log(ngx_ERR, e) end
local _, e = redis:expire(key_chain.repset, ttl)
if e then ngx_log(ngx_ERR, e) end
local _, e = clean_repset(redis, key_chain.repset)
if e then ngx_log(ngx_ERR, e) end
return true
end
_M.save_key_chain = save_key_chain
return _M
================================================
FILE: lib/ledge/collapse.lua
================================================
local _M = {
_VERSION = "2.3.0",
}
-- Attempts to set a lock key in redis. The lock will expire after
-- the expiry value if it is not cleared (i.e. in case of errors).
-- Returns true if the lock was acquired, false if the lock already
-- exists, and nil, err in case of failure.
local function acquire_lock(redis, lock_key, timeout)
-- We use a Lua script to emulate SETNEX (set if not exists with expiry).
-- This avoids a race window between the GET / SETEX.
-- Params: key, expiry
-- Return: OK or BUSY
local SETNEX = [[
local lock = redis.call("GET", KEYS[1])
if not lock then
return redis.call("PSETEX", KEYS[1], ARGV[1], "locked")
else
return "BUSY"
end
]]
local res, err = redis:eval(SETNEX, 1, lock_key, timeout)
if not res then -- Lua script failed
return nil, err
elseif res == "OK" then -- We have the lock
return true
elseif res == "BUSY" then -- Lock is busy
return false
end
end
_M.acquire_lock = acquire_lock
return _M
================================================
FILE: lib/ledge/esi/processor_1_0.lua
================================================
local http = require "resty.http"
local cookie = require "resty.cookie"
local tag_parser = require "ledge.esi.tag_parser"
local util = require "ledge.util"
local tostring, type, tonumber, next, unpack, pcall, setfenv, loadstring =
tostring, type, tonumber, next, unpack, pcall, setfenv, loadstring
local str_sub = string.sub
local str_byte = string.byte
-- TODO: Find places we can use str_find over ngx_re_find
local str_find = string.find
local tbl_concat = table.concat
local tbl_insert = table.insert
local co_yield = coroutine.yield
local co_wrap = util.coroutine.wrap
local ngx_re_gsub = ngx.re.gsub
local ngx_re_sub = ngx.re.sub
local ngx_re_match = ngx.re.match
local ngx_re_find = ngx.re.find
local ngx_req_get_headers = ngx.req.get_headers
local ngx_req_get_uri_args = ngx.req.get_uri_args
local ngx_flush = ngx.flush
local ngx_var = ngx.var
local ngx_log = ngx.log
local ngx_ERR = ngx.ERR
local ngx_INFO = ngx.INFO
local get_fixed_field_metatable_proxy =
require("ledge.util").mt.get_fixed_field_metatable_proxy
local _M = {
_VERSION = "2.3.0",
}
function _M.new(handler)
return setmetatable({
handler = handler,
token = "ESI/1.0",
}, get_fixed_field_metatable_proxy(_M))
end
-- $1: variable name (e.g. QUERY_STRING)
-- $2: substructure key
-- $3: default value
-- $4: default value if quoted
local esi_var_pattern =
[[\$\(([A-Z_]+){?([a-zA-Z\.\-~_%0-9]*)}?\|?(?:([^\s\)']+)|'([^\')]+)')?\)]]
-- Evaluates a given ESI variable.
local function _esi_eval_var(var)
-- Extract variables from capture results table
local var_name = var[1] or ""
local key = var[2]
if key == "" then key = nil end
local default = var[3]
local default_quoted = var[4]
local default = default or default_quoted or ""
if var_name == "QUERY_STRING" then
if not key then
-- We don't have a key so give them the whole string
return ngx_var.args or default
else
-- Lookup the querystring component by key
local value = ngx_req_get_uri_args()[key]
if value then
if type(value) == "table" then
return tbl_concat(value, ", ")
else
return value
end
else
return default
end
end
elseif str_sub(var_name, 1, 5) == "HTTP_" then
-- Evaluate request headers. Cookie and Accept-Language are special
-- according to the spec.
local header = str_sub(var_name, 6)
if header == "COOKIE" then
local cookies = ngx.ctx.__ledge_esi_cookies or cookie:new()
local blacklist = ngx.ctx.__ledge_esi_vars_cookie_blacklist or {}
if not next(blacklist) then
if key then
return cookies:get(key) or default
end
return ngx_var.http_cookie or default
end
-- We have a blacklist to filter with
if key then
if not blacklist[key] then
return cookies:get(key) or default
end
return default
else
-- We need a full cookie string, with any blacklisted values removed
local cookies = cookies:get_all()
local value = {}
for k, v in pairs(cookies) do
if not blacklist[k] then
tbl_insert(value, k .. "=" .. v)
end
end
return tbl_concat(value, "; ") or default
end
else
local value = ngx_req_get_headers()[header]
if not value then
return default
elseif header == "ACCEPT_LANGUAGE" and key then
-- If we're a table (multilple Accept-Language headers), convert
-- to string
if type(value) == "table" then
value = tbl_concat(value, ", ")
end
if ngx_re_find(value, key, "oj") then
return "true"
else
return "false"
end
elseif type(value) == "table" then
-- For normal repeated headers, numeric indexes are supported
key = tonumber(key)
if key then
-- We can index numerically (0 indexed)
return tostring(value[key + 1] or default)
else
-- Without a numeric key, render as a comma separated list
return tbl_concat(value, ", ") or default
end
else
return value
end
end
elseif var_name == "ESI_ARGS" then
local esi_args = ngx.ctx.__ledge_esi_args
if not esi_args then
-- No ESI args in request
return default
end
if not key then
-- __tostring metamethod turns these back into encoded URI args
return tostring(esi_args)
else
local value = esi_args[key] or default
if type(value) == "table" then
return tbl_concat(value, ",")
end
return tostring(value)
end
else
local custom_variables = ngx.ctx.__ledge_esi_custom_variables
if next(custom_variables) then
local var = custom_variables[var_name]
if var then
if key then
if type(var) == "table" then
return tostring(var[key] or default)
end
else
if type(var) == "table" then
-- No sane way to stringify other tables
return default
else
-- We're a string
return var or default
end
end
end
end
return default
end
end
local function esi_eval_var(var)
local escape = true
local var_name = var[1]
-- If var name begins with RAW_ do not escape
local b1, b2, b3, b4 = str_byte(var_name, 1, 4)
if b1 == 82 and b2 == 65 and b3 == 87 and b4 == 95 then
escape = false
var[1] = str_sub(var_name, 5, -1)
end
local res = _esi_eval_var(var)
-- Always escape ESI tags in ESI variables
if escape or str_find(res, "<esi", 1, true) ~= nil then
res = ngx_re_gsub(res, "<", "<", "soj")
res = ngx_re_gsub(res, ">", ">", "soj")
end
return res
end
_M.esi_eval_var = esi_eval_var
local function esi_replace_vars(str, cb)
cb = cb or esi_eval_var
return ngx_re_gsub(str, esi_var_pattern, cb, "soj")
end
local function esi_eval_var_in_when_tag(var)
var = esi_eval_var(var)
-- Quote unless we can be considered a number
local number = tonumber(var)
if number then
return number
else
-- Strings must be enclosed in single quotes, so also backslash
-- escape single quotes within the value
return "\'" .. ngx_re_gsub(var, "'", "\\'", "oj") .. "\'"
end
end
local function _esi_condition_lexer(condition)
-- ESI to Lua operators
local op_replacements = {
["!="] = "~=",
["|"] = " or ",
["&"] = " and ",
["||"] = " or ",
["&&"] = " and ",
["!"] = " not ",
}
-- Mapping of types to types they are allowed to follow
local lexer_rules = {
number = {
["nil"] = true,
["operator"] = true,
},
string = {
["nil"] = true,
["operator"] = true,
},
operator = {
["nil"] = true,
["number"] = true,
["string"] = true,
["operator"] = true,
},
}
-- $1: number
-- $2: string
-- $3: operator
local p =[[(\d+(?:\.\d+)?)|(?:'(.*?)(?<!\\)')|(\!=|!|\|{1,2}|&{1,2}|={2}|=~|\(|\)|<=|>=|>|<)]]
local ctx = {}
local tokens = {}
local prev_type
local expecting_pattern = false
repeat
local token, err = ngx_re_match(condition, p, "", ctx)
if err then ngx_log(ngx_ERR, err) end
if token then
local number, string, operator = token[1], token[2], token[3]
local token_type
if number then
token_type = "number"
tbl_insert(tokens, number)
elseif string then
token_type = "string"
-- Check to see if we're expecing a regex pattern
if expecting_pattern then
-- Extract the pattern and options
local re = ngx_re_match(
string,
[[\/(.*?)(?<!\\)\/([a-z]*)]],
"oj"
)
if not re then
ngx_log(ngx_INFO,
"Parse error: could not parse regular expression",
"in: \"", condition, "\""
)
return nil
else
local pattern, options = re[1], re[2]
-- The last item in tokens is the compare string.
-- Override this with a function call
local cmp_string = tokens[#tokens]
tokens[#tokens] =
"find(" .. cmp_string .. ", '" ..
pattern .. "', '" .. options .. "oj')"
end
expecting_pattern = false
else
-- Plain string literal
tbl_insert(tokens, "'" .. string .. "'")
end
elseif operator then
token_type = "operator"
-- Look for the regexp op
if operator == "=~" then
if prev_type == "operator" then
ngx_log(ngx_INFO,
"Parse error: regular expression attempting ",
"against non-string in: \"", condition, "\""
)
return nil
else
-- Don't insert this operator, just set this flag and
-- look for the pattern in the next string
expecting_pattern = true
end
else
-- Replace operators with Lua equivalents, if needed
tbl_insert(tokens, op_replacements[operator] or operator)
end
end
-- If we break the rules, log a parse error and bail
if prev_type then
if not lexer_rules[prev_type][token_type] then
ngx_log(ngx_INFO,
"Parse error: found ", token_type, " after ", prev_type,
" in: \"", condition, "\""
)
return nil
end
end
prev_type = token_type
end
until not token
return true, tbl_concat(tokens or {}, " ")
end
_M._esi_condition_lexer = _esi_condition_lexer
local function _esi_evaluate_condition(condition)
-- Evaluate variables in the condition
condition = esi_replace_vars(condition, esi_eval_var_in_when_tag)
local ok, condition = _esi_condition_lexer(condition)
if not ok then
return false
end
-- Try to parse as Lua code, place in an empty sandbox, and pcall to
-- evaluate the condition.
local eval, err = loadstring("return " .. condition)
if eval then
-- Empty environment except an re.find function
setfenv(eval, { find = ngx.re.find })
local ok, res = pcall(eval)
if ok then
return res
else
ngx_log(ngx_ERR, res)
return false
end
else
ngx_log(ngx_ERR, err)
return false
end
end
-- Assumed chunk contains a complete conditional instruction set. Handles
-- recursion for nested conditions.
local function evaluate_conditionals(chunk, res, recursion)
if not recursion then recursion = 0 end
if not res then res = {} end
local parser = tag_parser.new(chunk)
-- $1: the condition inside test=""
local esi_when_pattern = [[(?:<esi:when)\s+(?:test="(.+?)"\s*>)]]
local after -- Will contain anything after the last closing choose tag
local chunk_has_conditionals = false
repeat
local choose, ch_before, ch_after = parser:next("esi:choose")
if choose and choose.closing then
chunk_has_conditionals = true
-- Anything before this choose should just be output
if ch_before then
tbl_insert(res, ch_before)
end
-- If this ends up being the last choose tag, content after this
-- should be output
if ch_after then
after = ch_after
end
local inner_parser = tag_parser.new(choose.contents)
local when_matched = false
local otherwise
repeat
local tag = inner_parser:next("esi:when|esi:otherwise")
if tag and tag.closing then
if tag.tagname == "esi:when" and when_matched == false then
local function process_when(m_when)
-- We only show the first matching branch, others
-- must be removed even if they also match.
if when_matched then return "" end
local condition = m_when[1]
if _esi_evaluate_condition(condition) then
when_matched = true
if ngx_re_find(tag.contents, "<esi:choose>") then
-- recurse
evaluate_conditionals(
tag.contents,
res,
recursion + 1
)
else
tbl_insert(res, tag.contents)
end
end
return ""
end
local ok, err = ngx_re_sub(
tag.whole,
esi_when_pattern,
process_when
)
if not ok and err then ngx_log(ngx_ERR, err) end
-- Break after the first winning expression
elseif tag.tagname == "esi:otherwise" then
otherwise = tag.contents
end
end
until not tag
if not when_matched and otherwise then
if ngx_re_find(otherwise, "<esi:choose>") then
-- recurse
evaluate_conditionals(otherwise, res, recursion + 1)
else
tbl_insert(res, otherwise)
end
end
end
until not choose
if after then
tbl_insert(res, after)
end
-- Variables inside ESI tags should be evaluated.
-- Return hint to eval this chunk
if not chunk_has_conditionals then
return chunk, false
else
return tbl_concat(res), true
end
end
-- Used in esi_process_vars_tag. Declared locally to avoid runtime closure
local function _esi_gsub_vars(m)
return esi_replace_vars(m[2])
end
-- Replaces all variables in <esi:vars> blocks.
-- Also removes the <esi:vars> tags themselves.
local function esi_process_vars_tag(chunk)
if str_find(chunk, "esi:vars", 1, true) == nil then
return chunk
end
-- For every esi:vars block, substitute any number of variables found.
return ngx_re_gsub(chunk,
"(<esi:vars>)(.*?)(</esi:vars>)",
_esi_gsub_vars,
"soj"
)
end
_M.esi_process_vars_tag = esi_process_vars_tag
local function process_escaping(chunk, res, recursion)
if not recursion then recursion = 0 end
if not res then res = {} end
local parser = tag_parser.new(chunk)
local chunk_has_escaping = false
repeat
local tag, before, after = parser:next("!--esi")
if tag and tag.closing then
chunk_has_escaping = true
if before then
tbl_insert(res, before)
end
-- If there are more nested, recurse
if ngx_re_find(tag.contents, "<!--esi", "soj") then
return process_escaping(tag.contents, res, recursion)
else
tbl_insert(res, tag.contents)
tbl_insert(res, after)
end
end
until not tag
if chunk_has_escaping then
return tbl_concat(res)
else
return chunk
end
end
_M.process_escaping = process_escaping
local function is_include_host_on_same_domain(host)
return host == (ngx_var.http_host or ngx_var.host)
end
local function can_make_request_to_domain(config, host)
-- Third party domain requests may need to be explicitly enabled
if config.esi_disable_third_party_includes then
if not is_include_host_on_same_domain(host) then
local allowed_third_party_domains = config.esi_third_party_includes_domain_whitelist
if not next(allowed_third_party_domains) or not allowed_third_party_domains[host] then
return false
end
end
end
return true
end
-- If our esi include host matches the current host, use server_addr /
-- server_port instead. This keeps the connection local to this node
-- where possible.
local function should_loopback_request(config, scheme, host)
return config.esi_attempt_loopback and host == ngx_var.http_host and scheme == ngx_var.scheme
end
local function parse_src_attribute(include_tag)
local src, err = ngx_re_match(include_tag, [[src="([^"]+)"]], "oj")
if not src then
return nil, err
end
-- Evaluate variables in the src URI
return esi_replace_vars(src[1])
end
local function parse_include_src(src)
local scheme, host, port, path
local uri_parts = http.parse_uri(nil, src)
if not uri_parts then
-- Not a valid URI, so probably a relative path. Resolve
-- local to the current request.
scheme = ngx_var.scheme
host = ngx_var.http_host or ngx_var.host
port = ngx_var.server_port
path = src
-- No leading slash means we have a relative path. Append
-- this to the current URI.
if str_sub(path, 1, 1) ~= "/" then
path = ngx_var.uri .. "/" .. path
end
return scheme, host, port, path
end
return unpack(uri_parts)
end
local function make_esi_connection(config, upstream, scheme, host, port)
local httpc = http.new()
httpc:set_timeouts(
config.upstream_connect_timeout,
config.upstream_send_timeout,
config.upstream_read_timeout
)
local res, err
port = tonumber(port)
if port then
res, err = httpc:connect(upstream, port)
else
res, err = httpc:connect(upstream)
end
if not res then
return nil, err .. " connecting to " .. upstream .. ":" .. port
end
if scheme == "https" then
local ok, err = httpc:ssl_handshake(false, host, false)
if not ok then
return nil, "ssl handshake failed: " .. err
end
end
return httpc
end
local function make_esi_request_params(conn, host, path)
local parent_headers = ngx_req_get_headers()
local req_params = {
method = "GET",
path = ngx_re_gsub(path, "\\s", "%20", "jo"),
headers = {
["Host"] = host,
["Cache-Control"] = parent_headers["Cache-Control"],
["User-Agent"] = conn._USER_AGENT .. " ledge_esi/" .. _M._VERSION,
},
}
if is_include_host_on_same_domain(host) then
req_params.headers["Authorization"] = parent_headers["Authorization"]
req_params.headers["Cookie"] = parent_headers["Cookie"]
end
return req_params
end
function _M.esi_fetch_include(self, include_tag, buffer_size)
-- We track include recursion, and bail past the limit, yielding a special
-- "esi:abort_includes" instruction which the outer process filter checks
-- for.
local recursion_count =
tonumber(ngx_req_get_headers()["X-ESI-Recursion-Level"]) or 0
local config = self.handler.config
local recursion_limit = config.esi_recursion_limit
if recursion_count >= recursion_limit then
ngx_log(ngx_ERR, "ESI recursion limit (", recursion_limit, ") exceeded")
co_yield("<esi:abort_includes />")
return nil
end
local src, err = parse_src_attribute(include_tag)
if not src then
ngx_log(ngx_ERR, err)
return nil
end
local scheme, host, port, path = parse_include_src(src)
if not scheme then return nil end
if (not can_make_request_to_domain(config, host)) then return nil end
local upstream = host
if should_loopback_request(config, scheme, host) then
upstream = ngx_var.server_addr
port = ngx_var.server_port
end
local httpc, err = make_esi_connection(config, upstream, scheme, host, port)
if not httpc then
ngx_log(ngx_ERR, err)
return nil
end
local req_params = make_esi_request_params(httpc, host, path)
-- A chance to modify the request before we go upstream
self.handler:emit("before_esi_include_request", req_params)
-- Add these after the pre_include_callback so that they cannot be
-- accidentally overriden
req_params.headers["X-ESI-Parent-URI"] =
ngx_var.scheme .. "://" .. ngx_var.host .. ngx_var.request_uri
req_params.headers["X-ESI-Recursion-Level"] = recursion_count + 1
-- Go!
local res, err = httpc:request(req_params)
if not res then
ngx_log(ngx_ERR, err, " from ", (src or ''))
return nil
elseif res.status >= 500 then
ngx_log(ngx_ERR, res.status, " from ", (src or ''))
return nil
else
if res then
-- Stream the include fragment, yielding as we go
local reader = res.body_reader
repeat
local ch, err = reader(buffer_size)
if ch then
co_yield(ch)
elseif err then
ngx_log(ngx_ERR, err)
end
until not ch
end
end
httpc:set_keepalive(
config.upstream_keepalive_timeout,
config.upstream_keepalive_poolsize
)
end
local function esi_process_include_tags(self, chunk, esi_abort_flag, buffer_size, eval_vars)
-- Short circuit
if not chunk or str_find(chunk, "<esi:include", 1, true) == nil then
if eval_vars then
chunk = esi_replace_vars(chunk)
end
return co_yield(chunk)
end
-- Find and loop over esi:include tags
local re_ctx = { pos = 1 }
local yield_from = 1
repeat
local from, to, err = ngx_re_find(
chunk,
[[<esi:include\s*src="[^"]+"\s*/>]],
"oj",
re_ctx
)
if err then ngx_log(ngx_ERR, err) end
if from then
-- Yield up to the start of the include tag
local pre = str_sub(chunk, yield_from, from - 1)
if eval_vars then
pre = esi_replace_vars(pre)
end
co_yield(pre)
ngx_flush()
yield_from = to + 1
-- This will be true if an include has
-- previously yielded the "esi:abort_includes
-- instruction.
if esi_abort_flag == false then
-- Fetches and yields the streamed response
self:esi_fetch_include(
str_sub(chunk, from, to),
buffer_size
)
end
else
if yield_from == 1 then
-- No includes found, yield everything
if eval_vars then
chunk = esi_replace_vars(chunk)
end
co_yield(chunk)
else
-- No *more* includes, yield what's left
chunk = str_sub(chunk, re_ctx.pos, -1)
if eval_vars then
chunk = esi_replace_vars(chunk)
end
co_yield(chunk)
end
end
until not from
end
local function esi_process_comment_tags(chunk)
if str_find(chunk, "<esi:comment", 1, true) == nil then
return chunk
end
return ngx_re_gsub(chunk,
"<esi:comment (?:.*?)/>",
"",
"soj"
)
end
local function esi_process_remove_tags(chunk)
if str_find(chunk, "<esi:remove", 1, true) == nil then
return chunk
end
return ngx_re_gsub(chunk,
"(<esi:remove>.*?</esi:remove>)",
"",
"soj"
)
end
-- Reads from reader according to "buffer_size", and scans for ESI instructions.
-- Acts as a sink when ESI instructions are not complete, buffering until the
-- chunk contains a full instruction safe to process on serve.
function _M.get_scan_filter(self, res)
local reader = res.body_reader
local esi_detected = false
local max_size = self.handler.config.esi_max_size
local bailed = false
return co_wrap(function(buffer_size)
local prev_chunk = ""
local tag_hint
repeat
local chunk, err = reader(buffer_size)
if err then ngx_log(ngx_ERR, err) end
if chunk then
-- If we have a tag hint (partial opening ESI tag) from the
-- previous chunk then prepend it here.
if tag_hint then
chunk = tag_hint .. chunk
tag_hint = nil
end
-- prev_chunk will contain the last buffer if we have
-- an ESI instruction spanning buffers.
chunk = prev_chunk .. chunk
-- If we've buffered beyond max_size, give up
if bailed or #chunk > max_size then
bailed = true
prev_chunk = ""
ngx_log(ngx_INFO,
"esi scan bailed as instructions spanned buffers " ..
"larger than esi_max_size"
)
co_yield(chunk, nil, false)
else
local parser = tag_parser.new(chunk)
repeat
local tag, before, after = parser:next()
if tag and tag.whole then
-- We have a whole instruction
-- Yield anything before this tag
if before ~= "" then
co_yield(before, nil, false)
end
-- Yield the entire tag with has_esi=true
co_yield(tag.whole, nil, true)
-- On first time, set res:set_and_save("has_esi", parser)
if not esi_detected then
res:set_and_save("has_esi", self.token)
esi_detected = true
end
-- Trim chunk to what's left
chunk = after
prev_chunk = ""
elseif tag and not tag.whole then
-- Opening, but incompete. We yield up to this point
-- and buffer from the opening tag onwards, to try again.
-- This is so that we don't buffer the "before" content
-- if there turns out to be no closing tag
if before ~= "" then
co_yield(before, nil, false)
end
prev_chunk = tag.opening.tag .. after
break
else
-- No complete tag found, but look for something
-- resembling the beginning of an incomplete ESI tag
local start_from, _, err = ngx_re_find(
chunk,
"<(?:!--)?esi", "soj"
)
if err then ngx_log(ngx_ERR, err) end
if start_from then
-- Incomplete opening tag, so buffer and try again
prev_chunk = chunk
break
end
-- Check the end of the chunk for the beginning of an
-- opening tag (a hint), incase it spans to the next
-- buffer.
local hint_match, err = ngx_re_match(
str_sub(chunk, -6, -1),
"(?:<!--es|<!--e|<!--|<es|<!-|<e|<!|<)$", "soj"
)
if err then ngx_log(ngx_ERR, err) end
if hint_match then
tag_hint = hint_match[0]
-- Remove the hint from this chunk, it'll be
-- prepending to the next one.
chunk = str_sub(chunk, 1, - (#tag_hint + 1))
end
-- Nothing found, yield the whole chunk
co_yield(chunk, nil, false)
break
end
until not tag
end
elseif tag_hint then
-- We had what looked like a tag_hint but there are no more
-- chunks left.
co_yield(tag_hint)
end
until not chunk
end)
end
function _M.get_process_filter(self, res)
local recursion_count =
tonumber(ngx_req_get_headers()["X-ESI-Recursion-Level"]) or 0
local reader = res.body_reader
-- push configured custom variables into ctx to be read by regex functions
ngx.ctx.__ledge_esi_custom_variables = self.handler.config.esi_custom_variables
-- push current request cookies and blacklist into ctx for regex functions
ngx.ctx.__ledge_esi_cookies = cookie:new()
ngx.ctx.__ledge_esi_vars_cookie_blacklist = self.handler.config.esi_vars_cookie_blacklist
-- We use an outer coroutine to filter the processed output in case we have
-- to abort recursive includes.
return co_wrap(function(buffer_size)
local esi_abort_flag = false
-- This is the actual process filter coroutine
local inner_reader = co_wrap(function(buffer_size)
repeat
local chunk, err, has_esi = reader(buffer_size)
if err then ngx_log(ngx_ERR, err) end
if chunk then
if has_esi then
-- Remove <!--esi-->
chunk = process_escaping(chunk)
-- Remove comments.
chunk = esi_process_comment_tags(chunk)
-- Remove '<esi:remove' blocks
chunk = esi_process_remove_tags(chunk)
-- Evaluate and replace <esi:vars>
chunk = esi_process_vars_tag(chunk)
-- Evaluate choose / when / otherwise conditions...
local chunk, should_eval = evaluate_conditionals(chunk)
-- Process ESI includes
-- Will yield content to the outer reader
esi_process_include_tags(self, chunk, esi_abort_flag, buffer_size, should_eval)
else
co_yield(chunk)
end
end
until not chunk
end)
-- Outer filter, which checks for an esi:abort_includes instruction,
-- so that we can handle accidental recursion.
repeat
local chunk, err = inner_reader(buffer_size)
if err then ngx_log(ngx_ERR, err) end
if chunk then
-- If we see an abort instruction, we set a flag to stop
-- further esi:includes.
if str_find(chunk, "<esi:abort_includes", 1, true) then
esi_abort_flag = true
end
-- We don't wish to see abort instructions in the final output,
-- so the the top most request (recursion_count 0) is
-- responsible for removing them.
if recursion_count == 0 then
chunk = ngx_re_gsub(chunk,
"<esi:abort_includes />",
"",
"soj"
)
end
co_yield(chunk)
end
until not chunk
end)
end
return _M
================================================
FILE: lib/ledge/esi/tag_parser.lua
================================================
local setmetatable, type =
setmetatable, type
local str_sub = string.sub
local ngx_re_find = ngx.re.find
local ngx_re_match = ngx.re.match
local ngx_log = ngx.log
local ngx_ERR = ngx.ERR
local get_fixed_field_metatable_proxy =
require("ledge.util").mt.get_fixed_field_metatable_proxy
local _M = {
_VERSION = "2.3.0",
}
function _M.new(content, offset)
return setmetatable({
content = content,
pos = (offset or 0),
open_comments = 0,
}, get_fixed_field_metatable_proxy(_M))
end
function _M.next(self, tagname)
local tag = self:find_whole_tag(tagname)
local before, after
if tag then
before = str_sub(self.content, self.pos + 1, tag.opening.from - 1)
if tag.closing then
-- This is block level (with a closing tag)
after = str_sub(self.content, tag.closing.to + 1)
self.pos = tag.closing.to
else
-- Inline (no closing tag)
after = str_sub(self.content, tag.opening.to + 1)
self.pos = tag.opening.to
end
end
return tag, before, after
end
function _M.open_pattern(tag)
if tag == "!--esi" then
return "<(!--esi)"
else
-- $1: the tag name, $2 the closing characters, e.g. "/>" or ">"
return "<(" .. tag .. [[)(?:\s*(?:[a-z]+=\".+?(?<!\\)\"))?[^>]*?(?:\s*)(\/>|>)?]]
end
end
function _M.close_pattern(tag)
if tag == "!--esi" then
return "-->"
else
-- $1: the tag name
return "</(" .. tag .. ")\\s*>"
end
end
function _M.either_pattern(tag)
if tag == "!--esi" then
return "(?:<(!--esi)|(-->))"
else
-- $1: the tag name, $2 the closing characters, e.g. "/>" or ">"
return [[<[\/]?(]] .. tag .. [[)(?:\s*(?:[a-z]+=\".+?(?<!\\)\"))?[^>]*?(?:\s*)(\s*\\/>|>)?]]
end
end
-- Finds the next esi tag, accounting for nesting to find the correct
-- matching closing tag etc.
function _M.find_whole_tag(self, tag)
-- Only work on the remaining markup (after pos)
local markup = str_sub(self.content, self.pos + 1)
if not tag then
-- Look for anything (including comment syntax)
tag = "(?:!--esi)|(?:esi:[a-z]+)"
end
-- Find the first opening tag
local opening_f, opening_t, err = ngx_re_find(markup, self.open_pattern(tag), "soj")
if not opening_f then
if err then ngx_log(ngx_ERR, err) end
-- Nothing here
return nil
end
-- We found an opening tag and has its position, but need to understand it better
-- to handle comments and inline tags.
local opening_m, err = ngx_re_match(
str_sub(markup, opening_f, opening_t),
self.open_pattern(tag), "soj"
)
if not opening_m then
if err then ngx_log(ngx_ERR, err) end
return nil
end
-- We return a table with opening tag positions (absolute), as well as
-- tag contents etc. Block level tags will have "closing" data too.
local ret = {
opening = {
from = opening_f + self.pos,
to = opening_t + self.pos,
tag = str_sub(markup, opening_f, opening_t),
},
tagname = opening_m[1],
closing = nil,
contents = nil,
}
-- If this is an inline (non-block) tag, we have everything
if type(opening_m[2]) == "string" and str_sub(opening_m[2], -2) == "/>" then
ret.whole = str_sub(markup, opening_f, opening_t)
return ret
end
-- We must be block level, and could potentially be nesting
local search = opening_t -- We search from after the opening tag
local f, t, closing_f, closing_t
local depth = 1
local level = 1
repeat
-- keep looking for opening or closing tags
f, t = ngx_re_find(str_sub(markup, search + 1), self.either_pattern(ret.tagname), "soj")
if f and t then
-- Move closing markers along
closing_f = f
closing_t = t
-- Track current level and total depth
local tag = str_sub(markup, search + f, search + t)
if ngx_re_find(tag, self.open_pattern(ret.tagname)) then
depth = depth + 1
level = level + 1
elseif ngx_re_find(tag, self.close_pattern(ret.tagname)) then
level = level - 1
end
-- Move search pos along
search = search + t
end
until level == 0 or not f
if closing_t and t then
-- We have a complete block tag with the matching closing tag
-- Make closing tag absolute
closing_t = closing_t + search - t
closing_f = closing_f + search - t
ret.closing = {
from = closing_f + self.pos,
to = closing_t + self.pos,
tag = str_sub(markup, closing_f, closing_t),
}
ret.contents = str_sub(markup, opening_t + 1, closing_f - 1)
ret.whole = str_sub(markup, opening_f, closing_t)
return ret
else
-- We have an opening block tag, but not the closing part. Return
-- what we can as the filters will buffer until we find the rest.
return ret
end
end
return _M
================================================
FILE: lib/ledge/esi.lua
================================================
local h_util = require "ledge.header_util"
local type, tonumber = type, tonumber
local str_sub = string.sub
local str_find = string.find
local tbl_concat = table.concat
local tbl_insert = table.insert
local ngx_re_match = ngx.re.match
local ngx_req_get_headers = ngx.req.get_headers
local ngx_req_get_uri_args = ngx.req.get_uri_args
local ngx_encode_args = ngx.encode_args
local ngx_req_set_uri_args = ngx.req.set_uri_args
local ngx_var = ngx.var
local ngx_log = ngx.log
local ngx_ERR = ngx.ERR
local _M = {
_VERSION = "2.3.0",
}
local esi_processors = {
["ESI"] = {
["1.0"] = require "ledge.esi.processor_1_0",
-- 2.0 = require ledge.esi.processor_2_0", -- for example
},
}
function _M.split_esi_token(token)
if token then
local m = ngx_re_match(
token,
[[^([A-Za-z0-9-_]+)\/(\d+\.?\d+)$]],
"oj"
)
if m then
return m[1], tonumber(m[2])
end
end
end
function _M.esi_capabilities()
local capabilities = {}
for processor_type,processors in pairs(esi_processors) do
for version,_ in pairs(processors) do
tbl_insert(capabilities, processor_type .. "/" .. version)
end
end
return tbl_concat(capabilities, " ")
end
-- Returns a processor instance based on Surrogate-Control header
function _M.choose_esi_processor(handler)
local res = handler.response
local res_surrogate_control = res.header["Surrogate-Control"]
if res_surrogate_control then
-- Get the token value (e.g. "ESI/1.0")
local content_token =
h_util.get_header_token(res_surrogate_control, "content")
if content_token then
local processor_token, version = _M.split_esi_token(content_token)
if processor_token and version then
-- Lookup the prcoessor
local processor_type = esi_processors[processor_token]
if processor_type then
for v,processor in pairs(processor_type) do
if tonumber(version) <= tonumber(v) then
return processor.new(handler)
end
end
end
end
end
end
end
-- Returns true of res.header.Content-Type is in allowed_types
function _M.is_allowed_content_type(res, allowed_types)
if allowed_types and type(allowed_types) == "table" then
local res_content_type = res.header["Content-Type"]
if res_content_type then
for _, content_type in ipairs(allowed_types) do
local sep = str_find(res_content_type, ";")
if sep then sep = sep - 1 end
if str_sub(res_content_type, 1, sep) == content_type then
return true
end
end
end
end
end
-- Returns true if we're allowed to delegate ESI processing to a downstream
-- surrogate for the current request
function _M.can_delegate_to_surrogate(surrogates, processor_token)
local surrogate_capability = ngx_req_get_headers()["Surrogate-Capability"]
if surrogate_capability then
-- Surrogate-Capability: host.example.com="ESI/1.0"
local capability_token = h_util.get_header_token(
surrogate_capability,
"[!#\\$%&'\\*\\+\\-.\\^_`\\|~0-9a-zA-Z]+"
)
local capability_processor, capability_version =
_M.split_esi_token(capability_token)
if capability_processor and capability_version then
local control_processor, control_version =
_M.split_esi_token(processor_token)
if control_processor and control_version
and control_processor == capability_processor
and control_version <= capability_version then
if type(surrogates) == "boolean" then
if surrogates == true then
return true
end
elseif type(surrogates) == "table" then
local remote_addr = ngx_var.remote_addr
if remote_addr then
for _, ip in ipairs(surrogates) do
if ip == remote_addr then
return true
end
end
end
end
end
end
end
return false
end
function _M.filter_esi_args(handler)
local config = handler.config
local esi_args_prefix = config.esi_args_prefix
if esi_args_prefix then
local args = ngx_req_get_uri_args(config.max_uri_args)
local esi_args = {}
local has_esi_args = false
local non_esi_args = {}
for k,v in pairs(args) do
-- TODO: optimise
-- If we have the prefix, extract the suffix
local m, err = ngx_re_match(
k,
"^" .. esi_args_prefix .. "(\\S+)",
"oj"
)
if err then ngx_log(ngx_ERR, err) end
if m and m[1] then
has_esi_args = true
esi_args[m[1]] = v
else
-- Otherwise, this is a normal arg
non_esi_args[k] = v
end
end
if has_esi_args then
-- Add them to ctx to be read by the esi processor, along with a
-- __tostsring metamethod for the $(ESI_ARGS) string case
ngx.ctx.__ledge_esi_args = setmetatable(esi_args, {
__tostring = function(t)
local args = {}
for k,v in pairs(t) do
args[esi_args_prefix .. k] = v
end
return ngx_encode_args(args)
end
})
-- Set the request args to the ones left over
ngx_req_set_uri_args(non_esi_args)
end
end
end
return _M
================================================
FILE: lib/ledge/gzip.lua
================================================
local co_yield = coroutine.yield
local co_wrap = require("ledge.util").coroutine.wrap
local ngx_log = ngx.log
local ngx_ERR = ngx.ERR
local zlib = require("ffi-zlib")
local _M = {
_VERSION = "2.3.0",
}
local zlib_output = function(data)
co_yield(data)
end
local function get_gzip_decoder(reader)
return co_wrap(function(buffer_size)
local ok, err = zlib.inflateGzip(reader, zlib_output, buffer_size)
if not ok then
ngx_log(ngx_ERR, err)
end
-- zlib decides it is done when the stream is complete.
-- Call reader() one more time to resume the next coroutine in the
-- chain.
reader(buffer_size)
end)
end
_M.get_gzip_decoder = get_gzip_decoder
local function get_gzip_encoder(reader)
return co_wrap(function(buffer_size)
local ok, err = zlib.deflateGzip(reader, zlib_output, buffer_size)
if not ok then
ngx_log(ngx_ERR, err)
end
-- zlib decides it is done when the stream is complete.
-- Call reader() one more time to resume the next coroutine in the
-- chain
reader(buffer_size)
end)
end
_M.get_gzip_encoder = get_gzip_encoder
return _M
================================================
FILE: lib/ledge/handler.lua
================================================
local setmetatable, tostring, tonumber, pcall, type, ipairs, pairs, next, error =
setmetatable, tostring, tonumber, pcall, type, ipairs, pairs, next, error
local ngx_req_get_method = ngx.req.get_method
local ngx_req_get_headers = ngx.req.get_headers
local ngx_req_http_version = ngx.req.http_version
local ngx_log = ngx.log
local ngx_WARN = ngx.WARN
local ngx_ERR = ngx.ERR
local ngx_INFO = ngx.INFO
local ngx_var = ngx.var
local ngx_null = ngx.null
local ngx_flush = ngx.flush
local ngx_print = ngx.print
local ngx_on_abort = ngx.on_abort
local ngx_md5 = ngx.md5
local ngx_time = ngx.time
local ngx_http_time = ngx.http_time
local ngx_parse_http_time = ngx.parse_http_time
local str_lower = string.lower
local str_len = string.len
local tbl_insert = table.insert
local tbl_concat = table.concat
local esi_capabilities = require("ledge.esi").esi_capabilities
local append_server_port = require("ledge.util").append_server_port
local ledge_cache_key = require("ledge.cache_key")
local req_relative_uri = require("ledge.request").relative_uri
local req_full_uri = require("ledge.request").full_uri
local put_background_job = require("ledge.background").put_background_job
local gc_wait = require("ledge.background").gc_wait
local fixed_field_metatable = require("ledge.util").mt.fixed_field_metatable
local get_fixed_field_metatable_proxy =
require("ledge.util").mt.get_fixed_field_metatable_proxy
local ledge = require("ledge")
local http = require("resty.http")
local http_headers = require("resty.http_headers")
local state_machine = require("ledge.state_machine")
local response = require("ledge.response")
local _M = {
_VERSION = "2.3.0",
}
-- Creates a new handler instance.
--
-- Config defaults are provided in the ledge module, and so instances
-- should always be created with ledge.create_handler(), not directly.
--
-- @param table The complete config table
-- @return table Handler instance, or nil if no config table is provided
local function new(config, events)
if not config then return nil, "config table expected" end
config = setmetatable(config, fixed_field_metatable)
local self = setmetatable({
-- public:
config = config,
events = events,
upstream_client = {},
-- Slots for composed objects
redis = {},
redis_subscriber = {},
storage = {},
state_machine = {},
range = {},
response = {},
error_response = {},
esi_processor = {},
client_validators = {},
output_buffers_enabled = true,
esi_scan_enabled = false,
esi_process_enabled = false,
-- private:
_root_key = "",
_vary_key = ngx_null, -- empty string is not the same as not set
_vary_spec = ngx_null, -- empty table is not the same as not set
_cache_key_chain = {},
_publish_key = "",
}, get_fixed_field_metatable_proxy(_M))
return self
end
_M.new = new
local function run(self)
-- Instantiate state machine
local sm = state_machine.new(self)
self.state_machine = sm
-- Install the client abort handler
local ok, err = ngx_on_abort(function()
return self.state_machine:e "aborted"
end)
if not ok then
ngx_log(ngx_WARN, "on_abort handler could not be set: " .. err)
end
-- Create Redis connection
local redis, err = ledge.create_redis_connection()
if not redis then
return nil, "could not connect to redis, " .. tostring(err)
else
self.redis = redis
end
-- Create storage connection
local config = self.config
local storage, err = ledge.create_storage_connection(
config.storage_driver,
config.storage_driver_config
)
if not storage then
return nil, "could not connect to storage, " .. tostring(err)
else
self.storage = storage
end
return sm:e "init"
end
_M.run = run
-- Bind a user callback to an event
--
-- Callbacks will be called in the order they are bound
--
-- @param table self
-- @param string event name
-- @param function callback
-- @return bool, string success, error
local function bind(self, event, callback)
local ev = self.events[event]
if not ev then
local err = "no such event: " .. tostring(event)
ngx_log(ngx_ERR, err)
return nil, err
else
tbl_insert(ev, callback)
end
return true, nil
end
_M.bind = bind
-- Calls any registered callbacks for event, in the order they were bound
-- Hard errors if event is not specified in self.events
local function emit(self, event, ...)
local ev = self.events[event]
if not ev then
error("attempt to emit non existent event: " .. tostring(event), 2)
end
for _, handler in ipairs(ev) do
if type(handler) == "function" then
local ok, err = pcall(handler, ...)
if not ok then
ngx_log(ngx_ERR,
"error in user callback for '", event, "': ", err)
end
end
end
return true
end
_M.emit = emit
function _M.entity_id(self, key_chain)
if not key_chain or not key_chain.main then return nil end
local entity_id, err = self.redis:hget(key_chain.main, "entity")
if not entity_id or entity_id == ngx_null then
return nil, err
end
return entity_id
end
local function root_key(self)
if self._root_key == "" then
self._root_key = ledge_cache_key.generate_root_key(
self.config.cache_key_spec,
self.config.max_uri_args
)
end
return self._root_key
end
_M.root_key = root_key
local function vary_spec(self, root_key)
if self._vary_spec == ngx_null then
local vary_spec, err = ledge_cache_key.read_vary_spec(
self.redis,
root_key
)
if not vary_spec then
ngx_log(ngx_ERR, "Failed to read vary spec: ", err)
return false
end
self._vary_spec = vary_spec
end
return self._vary_spec
end
_M.vary_spec = vary_spec
local function create_vary_key_callback(self)
return function(vary_key)
-- TODO: gunzip?
emit(self, "before_vary_selection", vary_key)
end
end
_M.create_vary_key_callback = create_vary_key_callback
local function vary_key(self, vary_spec)
if self._vary_key == ngx_null then
self._vary_key = ledge_cache_key.generate_vary_key(
vary_spec,
create_vary_key_callback(self)
)
end
return self._vary_key
end
_M.vary_key = vary_key
local function cache_key_chain(self)
if not next(self._cache_key_chain) then
if not self.redis or not next(self.redis) then
ngx_log(ngx_ERR, "Cannot get cache key without a redis connection")
return nil
end
local rk = root_key(self)
local vs = vary_spec(self, rk)
local vk = vary_key(self, vs)
local chain, err = ledge_cache_key.key_chain(rk, vk, vs)
if not chain then
return nil, err
end
self._cache_key_chain = chain
end
return self._cache_key_chain
end
_M.cache_key_chain = cache_key_chain
local function reset_cache_key(self)
self._root_key = ""
self._vary_key = ngx_null
self._vary_spec = ngx_null
self._cache_key_chain = {}
end
_M.reset_cache_key = reset_cache_key
local function set_vary_spec(self, vary_spec)
reset_cache_key(self)
if vary_spec then
self._vary_spec = vary_spec
end
end
_M.set_vary_spec = set_vary_spec
local function read_from_cache(self)
local res, err = response.new(self)
if not res then return nil, err end
local ok, err = res:read()
if err then
-- Error, abort request
ngx_log(ngx_ERR, "could not read response: ", err)
return self.state_machine:e "http_internal_server_error"
end
if not ok then
return {} -- MISS
end
if res.size > 0 then
local storage = self.storage
-- Check storage has the entity, if not presume it has been evitcted
-- and clean up
if not storage:exists(res.entity_id) then
local config = self.config
put_background_job(
"ledge_gc",
"ledge.jobs.collect_entity",
{
entity_id = res.entity_id,
storage_driver = config.storage_driver,
storage_driver_config = config.storage_driver_config,
},
{
delay = gc_wait(
res.size,
config.minimum_old_entity_download_rate
),
tags = { "collect_entity" },
priority = 10,
}
)
return {} -- MISS
end
res:filter_body_reader("cache_body_reader", storage:get_reader(res))
end
emit(self, "after_cache_read", res)
return res
end
_M.read_from_cache = read_from_cache
-- http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.5.1
local hop_by_hop_headers = {
["connection"] = true,
["keep-alive"] = true,
["proxy-authenticate"] = true,
["proxy-authorization"] = true,
["te"] = true,
["trailers"] = true,
["transfer-encoding"] = true,
["upgrade"] = true,
["content-length"] = true, -- Not strictly hop-by-hop, but we
-- set dynamically downstream.
}
-- Fetches a resource from the origin server.
local function fetch_from_origin(self)
local res, err = response.new(self)
if not res then return nil, err end
local method = ngx['HTTP_' .. ngx_req_get_method()]
if not method then
res.status = ngx.HTTP_METHOD_NOT_IMPLEMENTED
return res
end
emit(self, "before_upstream_connect", self)
local config = self.config
if not next(self.upstream_client) then
local httpc = http.new()
httpc:set_timeouts(
config.upstream_connect_timeout,
config.upstream_send_timeout,
config.upstream_read_timeout
)
local port = tonumber(config.upstream_port)
local ok, err
if port then
ok, err = httpc:connect(config.upstream_host, port)
else
ok, err = httpc:connect(config.upstream_host)
end
if not ok then
ngx_log(ngx_ERR, "upstream connection failed: ", err)
if err == "timeout" then
res.status = 524 -- upstream server timeout
else
res.status = 503
end
return res
end
if config.upstream_use_ssl == true then
-- treat an empty ("") ssl_server_name as nil
local ssl_server_name = config.upstream_ssl_server_name
if type(ssl_server_name) ~= "string" or
str_len(ssl_server_name) == 0 then
ssl_server_name = nil
end
local ok, err = httpc:ssl_handshake(
false,
ssl_server_name,
config.upstream_ssl_verify
)
if not ok then
ngx_log(ngx_ERR, "ssl handshake failed: ", err)
res.status = 525 -- SSL Handshake Failed
return res
end
end
self.upstream_client = httpc
end
local upstream_client = self.upstream_client
-- Case insensitve headers so that we can safely manipulate them
local headers = http_headers.new()
for k,v in pairs(ngx_req_get_headers()) do
headers[k] = v
end
-- Advertise ESI surrogate capabilities
if config.esi_enabled then
local capability_entry = self.config.visible_hostname .. '="'
.. esi_capabilities() .. '"'
local sc = headers["Surrogate-Capability"]
if not sc then
headers["Surrogate-Capability"] = capability_entry
else
headers["Surrogate-Capability"] = sc .. ", " .. capability_entry
end
end
local client_body_reader, err =
upstream_client:get_client_body_reader(config.buffer_size)
if err then
ngx_log(ngx_ERR, "error getting client body reader: ", err)
end
local req_params = {
method = ngx_req_get_method(),
path = req_relative_uri(),
body = client_body_reader,
headers = headers,
}
-- allow request params to be customised
emit(self, "before_upstream_request", req_params)
local origin, err = upstream_client:request(req_params)
if not origin then
ngx_log(ngx_ERR, err)
res.status = 524
return res
end
res.status = origin.status
-- Merge end-to-end headers
local hop_by_hop_headers = hop_by_hop_headers
for k,v in pairs(origin.headers) do
if not hop_by_hop_headers[str_lower(k)] then
res.header[k] = v
end
end
-- May well be nil (we set to false if that's the case), but if present
-- we bail on saving large bodies to memory nice and early.
res.length = tonumber(origin.headers["Content-Length"]) or false
res.has_body = origin.has_body
res:filter_body_reader(
"upstream_body_reader",
origin.body_reader
)
if res.status < 500 then
-- http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.18
-- A received message that does not have a Date header field MUST be
-- assigned one by the recipient if the message will be cached by that
-- recipient
if type(res.header["Date"]) ~= "string" or
not ngx_parse_http_time(res.header["Date"]) then
ngx_log(ngx_WARN,
"Missing or invalid Date header from upstream, generating locally"
)
res.header["Date"] = ngx_http_time(ngx_time())
end
end
-- A nice opportunity for post-fetch / pre-save work.
emit(self, "after_upstream_request", res)
return res
end
_M.fetch_from_origin = fetch_from_origin
-- Returns data required to perform a background revalidation for this current
-- request, as two tables; reval_params and reval_headers.
local function revalidation_data(self)
-- Everything that a headless revalidation job would need to connect
local config = self.config
local reval_params = {
server_addr = ngx_var.server_addr,
server_port = ngx_var.server_port,
scheme = ngx_var.scheme,
uri = ngx_var.request_uri,
connect_timeout = config.upstream_connect_timeout,
send_timeout = config.upstream_send_timeout,
read_timeout = config.upstream_read_timeout,
keepalive_timeout = config.upstream_keepalive_timeout,
keepalive_poolsize = config.upstream_keepalive_poolsize,
}
local h = ngx_req_get_headers()
-- By default we pass through Host, and Authorization and Cookie headers
-- if present.
local reval_headers = {
host = h["Host"],
}
if h["Authorization"] then
reval_headers["Authorization"] = h["Authorization"]
end
if h["Cookie"] then
reval_headers["Cookie"] = h["Cookie"]
end
emit(self, "before_save_revalidation_data", reval_params, reval_headers)
return reval_params, reval_headers
end
local function revalidate_in_background(self, key_chain, update_revalidation_data)
local redis = self.redis
-- Revalidation data is updated if this is a proper request, but not if
-- it's a purge request.
if update_revalidation_data then
local reval_params, reval_headers = revalidation_data(self)
local ttl, err = redis:ttl(key_chain.reval_params)
if not ttl or ttl == ngx_null or ttl < 0 then
if err then ngx_log(ngx_ERR, err) end
ngx_log(ngx_INFO,
"Could not determine expiry for revalidation params. " ..
"Will fallback to 3600 seconds."
)
-- Arbitrarily expire these revalidation parameters in an hour.
ttl = 3600
end
-- Delete and update reval request headers
local _, e
_, e = redis:multi()
if e then ngx_log(ngx_ERR, e) end
_, e = redis:del(key_chain.reval_params)
if e then ngx_log(ngx_ERR, e) end
_, e = redis:hmset(key_chain.reval_params, reval_params)
if e then ngx_log(ngx_ERR, e) end
_, e = redis:expire(key_chain.reval_params, ttl)
if e then ngx_log(ngx_ERR, e) end
_, e = redis:del(key_chain.reval_req_headers)
if e then ngx_log(ngx_ERR, e) end
_, e = redis:hmset(key_chain.reval_req_headers, reval_headers)
if e then ngx_log(ngx_ERR, e) end
_, e = redis:expire(key_chain.reval_req_headers, ttl)
if e then ngx_log(ngx_ERR, e) end
local res, err = redis:exec()
if not res or res == ngx_null then
ngx_log(ngx_ERR, "Could not update revalidation params: ", err)
end
end
local uri, err = redis:hget(key_chain.main, "uri")
if not uri or uri == ngx_null then
if err then
ngx_log(ngx_ERR, "Failed to get main key while revalidating: ", err)
else
ngx_log(ngx_WARN,
"Cache key has no 'uri' field, aborting revalidation"
)
end
return nil
end
-- Schedule the background job (immediately). jid is a function of the
-- URI for automatic de-duping.
return put_background_job(
"ledge_revalidate",
"ledge.jobs.revalidate",
{ key_chain = key_chain },
{
jid = ngx_md5("revalidate:" .. uri),
tags = { "revalidate" },
priority = 4,
}
)
end
_M.revalidate_in_background = revalidate_in_background
-- Starts a "revalidation" job but maybe for brand new cache. We pass the
-- current request's revalidation data through so that the job has meaninful
-- parameters to work with (rather than using stored metadata).
local function fetch_in_background(self)
local key_chain = cache_key_chain(self)
local reval_params, reval_headers = revalidation_data(self)
return put_background_job(
"ledge_revalidate",
"ledge.jobs.revalidate",
{
key_chain = key_chain,
reval_params = reval_params,
reval_headers = reval_headers,
},
{
jid = ngx_md5("revalidate:" .. req_full_uri()),
tags = { "revalidate" },
priority = 4,
}
)
end
_M.fetch_in_background = fetch_in_background
local function save_to_cache(self, res)
if not res then return nil, "no response to save" end
emit(self, "before_save", res)
-- Length is only set if there was a Content-Length header
local length = res.length
local storage = self.storage
local max_size = storage:get_max_size()
if length and length > max_size then
-- We'll carry on serving, just not saving.
return nil, "advertised length is greated than storage max size"
end
-- Watch the main key pointer. We abort the transaction if another request
-- updates this key before we finish.
local key_chain = cache_key_chain(self)
local redis = self.redis
redis:watch(key_chain.main)
local repset_ttl = redis:ttl(key_chain.repset)
-- We'll need to mark the old entity for expiration shortly, as reads
-- could still be in progress. We need to know the previous entity keys
-- and the size.
local previous_entity_id = self:entity_id(key_chain)
local previous_entity_size, err, gc_job_spec
if previous_entity_id then
previous_entity_size, err = redis:hget(key_chain.main, "size")
if previous_entity_size == ngx_null then
previous_entity_id = nil
if err then
ngx_log(ngx_ERR, err)
end
end
-- Define GC job here, used later if required
gc_job_spec = {
"ledge_gc",
"ledge.jobs.collect_entity",
{
entity_id = previous_entity_id,
storage_driver = self.config.storage_driver,
storage_driver_config = self.config.storage_driver_config,
},
{
delay = gc_wait(
previous_entity_size,
self.config.minimum_old_entity_download_rate
),
tags = { "collect_entity" },
priority = 10,
}
}
end
-- Start the transaction
local ok, err = redis:multi()
if not ok then ngx_log(ngx_ERR, err) end
if previous_entity_id then
local ok, err = redis:srem(key_chain.entities, previous_entity_id)
if not ok then ngx_log(ngx_ERR, err) end
end
res.uri = req_full_uri()
local keep_cache_for = self.config.keep_cache_for
local ok, err = res:save(keep_cache_for)
if not ok then ngx_log(ngx_ERR, err) end
-- Set revalidation parameters from this request
local reval_params, reval_headers = revalidation_data(self)
local _, err = redis:del(key_chain.reval_params)
if err then ngx_log(ngx_ERR, err) end
_, err = redis:hmset(key_chain.reval_params, reval_params)
if err then ngx_log(ngx_ERR, err) end
_, err = redis:del(key_chain.reval_req_headers)
if err then ngx_log(ngx_ERR, err) end
_, err = redis:hmset(key_chain.reval_req_headers, reval_headers)
if err then ngx_log(ngx_ERR, err) end
local expiry = res:ttl() + keep_cache_for
redis:expire(key_chain.reval_params, expiry)
redis:expire(key_chain.reval_req_headers, expiry)
-- repset and vary TTL should be the same as the longest living represenation
if repset_ttl < expiry then
repset_ttl = expiry
end
-- Save updates to cache key
ledge_cache_key.save_key_chain(redis, key_chain, repset_ttl)
-- If we have a body, we need to attach the storage writer
-- NOTE: res.has_body is false for known bodyless repsonse types
-- (e.g. HEAD) but may be true and of zero length (commonly 301 etc).
if res.has_body then
-- Storage callback for write success
local function onsuccess(bytes_written)
-- Update size in metadata
local ok, e = redis:hset(key_chain.main, "size", bytes_written)
if not ok or ok == ngx_null then ngx_log(ngx_ERR, e) end
if bytes_written == 0 then
-- Remove the entity as it wont exist
ok, e = redis:srem(key_chain.entities, res.entity_id)
if not ok or ok == ngx_null then ngx_log(ngx_ERR, e) end
ok, e = redis:hdel(key_chain.main, "entity")
if not ok or ok == ngx_null then ngx_log(ngx_ERR, e) end
end
ok, e = redis:exec()
if not ok or ok == ngx_null then
if e then
ngx_log(ngx_ERR, "failed to complete transaction: ", e)
else
-- Transaction likely failed due to watch on main key
-- Tell storage to clean up too
ok, e = storage:delete(res.entity_id) -- luacheck: ignore ok
if e then
ngx_log(ngx_ERR, "failed to cleanup storage: ", e)
end
end
elseif previous_entity_id then
-- Everything has completed and we have an old entity
-- Schedule GC to clean it up
put_background_job(unpack(gc_job_spec))
end
end
-- Storage callback for write failure. We roll back our transaction.
local function onfailure(reason)
ngx_log(ngx_ERR, "storage failed to write: ", reason)
local ok, e = redis:discard()
if not ok or ok == ngx_null then ngx_log(ngx_ERR, e) end
end
-- Attach storage writer
local ok, writer = pcall(storage.get_writer, storage,
res,
keep_cache_for,
onsuccess,
onfailure
)
if not ok then
ngx_log(ngx_ERR, writer)
else
res:filter_body_reader("cache_body_writer", writer)
end
else
-- No body and thus no storage filter
-- We can run our transaction immediately
local ok, e = redis:exec()
if not ok or ok == ngx_null then
ngx_log(ngx_ERR, "failed to complete transaction: ", e)
elseif previous_entity_id then
-- Everything has completed and we have an old entity
-- Schedule GC to clean it up
put_background_job(unpack(gc_job_spec))
end
end
return true
end
_M.save_to_cache = save_to_cache
local function delete_from_cache(self, key_chain)
local redis = self.redis
-- Get entity_id if not already provided
local entity_id = self:entity_id(key_chain)
-- Schedule entity collection
if entity_id then
local config = self.config
local size = redis:hget(key_chain.main, "size")
put_background_job(
"ledge_gc",
"ledge.jobs.collect_entity",
{
entity_id = entity_id,
storage_driver = config.storage_driver,
storage_driver_config = config.storage_driver_config,
},
{
delay = gc_wait(
size,
config.minimum_old_entity_download_rate
),
tags = { "collect_entity" },
priority = 10,
}
)
end
-- Remove this representation from the repset
redis:srem(key_chain.repset, key_chain.full)
-- Delete everything in the keychain
local keys = {}
for _, v in pairs(key_chain) do
tbl_insert(keys, v)
end
-- If there are no more entries in the repset clean up the vary key too
local exists = redis:exists(key_chain.repset)
if exists == 0 then
tbl_insert(keys, key_chain.vary)
end
return redis:del(unpack(keys))
end
_M.delete_from_cache = delete_from_cache
-- Resumes the reader coroutine and prints the data yielded. This could be
-- via a cache read, or a save via a fetch... the interface is uniform.
local function serve_body(self, res, buffer_size)
local buffered = 0
local reader = res.body_reader
local can_flush = ngx_req_http_version() >= 1.1
repeat
local chunk, err = reader(buffer_size)
if err then ngx_log(ngx_ERR, err) end
if chunk and self.output_buffers_enabled then
local ok, err = ngx_print(chunk)
if not ok then ngx_log(ngx_INFO, err) end
-- Flush each full buffer, if we can
buffered = buffered + #chunk
if can_flush and buffered >= buffer_size then
local ok, err = ngx_flush(true)
if not ok then ngx_log(ngx_INFO, err) end
buffered = 0
end
end
until not chunk
end
local function serve(self)
if not ngx.headers_sent then
local res = self.response
local name = append_server_port(self.config.visible_hostname)
-- Via header
local via = "1.1 " .. name
if self.config.advertise_ledge then
via = via .. " (ledge/" .. _M._VERSION .. ")"
end
-- Append upstream Via
local res_via = res.header["Via"]
if (res_via ~= nil) then
-- Fix multiple upstream Via headers into list form
if (type(res_via) == "table") then
res.header["Via"] = via .. ", " .. tbl_concat(res_via, ", ")
else
res.header["Via"] = via .. ", " .. res_via
end
else
res.header["Via"] = via
end
-- X-Cache header
-- Don't set if this isn't a cacheable response. Set to MISS is we
-- fetched.
local state_history = self.state_machine.state_history
local event_history = self.state_machine.event_history
if not event_history["response_not_cacheable"] then
local x_cache = "HIT from " .. name
if not event_history["can_serve_disconnected"]
and not event_history["can_serve_stale"]
and state_history["fetching"] then
x_cache = "MISS from " .. name
end
local res_x_cache = res.header["X-Cache"]
if res_x_cache ~= nil then
res.header["X-Cache"] = x_cache .. ", " .. res_x_cache
else
res.header["X-Cache"] = x_cache
end
end
emit(self, "before_serve", res)
if res.header then
for k,v in pairs(res.header) do
ngx.header[k] = v
end
end
if res.body_reader and ngx_req_get_method() ~= "HEAD" then
local buffer_size = self.config.buffer_size
serve_body(self, res, buffer_size)
end
ngx.eof()
end
end
_M.serve = serve
local function add_warning(self, code)
return self.response:add_warning(
code,
append_server_port(self.config.visible_hostname)
)
end
_M.add_warning = add_warning
return setmetatable(_M, fixed_field_metatable)
================================================
FILE: lib/ledge/header_util.lua
================================================
local type, tonumber, setmetatable =
type, tonumber, setmetatable
local ngx_re_match = ngx.re.match
local ngx_re_find = ngx.re.find
local tbl_concat = table.concat
local _M = {
_VERSION = "2.3.0"
}
local mt = {
__index = _M,
}
-- Returns true if the directive appears in the header field value.
-- Set without_token to true to only return bare directives - i.e.
-- directives appearing with no =value part.
function _M.header_has_directive(header, directive, without_token)
if header then
if type(header) == "table" then header = tbl_concat(header, ", ") end
local pattern = [[(?:\s*|,?)(]] .. directive .. [[)\s*(?:$|=|,)]]
if without_token then
pattern = [[(?:\s*|,?)(]] .. directive .. [[)\s*(?:$|,)]]
end
return ngx_re_find(header, pattern, "ioj") ~= nil
end
return false
end
function _M.get_header_token(header, directive)
if _M.header_has_directive(header, directive) then
if type(header) == "table" then header = tbl_concat(header, ", ") end
-- Want the string value from a token
local value = ngx_re_match(
header,
directive .. [[="?([a-z0-9_~!#%&/',`\$\*\+\-\|\^\.]+)"?]],
"ioj"
)
if value ~= nil then
return value[1]
end
return nil
end
return nil
end
function _M.get_numeric_header_token(header, directive)
if _M.header_has_directive(header, directive) then
if type(header) == "table" then header = tbl_concat(header, ", ") end
-- Want the numeric value from a token
local value = ngx_re_match(
header,
directive .. [[="?(\d+)"?]], "ioj"
)
if value ~= nil then
return tonumber(value[1])
end
end
end
return setmetatable(_M, mt)
================================================
FILE: lib/ledge/jobs/collect_entity.lua
================================================
local tostring = tostring
local ngx_null = ngx.null
local create_storage_connection = require("ledge").create_storage_connection
local _M = {
_VERSION = "2.3.0",
}
-- Cleans up expired items and keeps track of memory usage.
function _M.perform(job)
local storage, err = create_storage_connection(
job.data.storage_driver,
job.data.storage_driver_config
)
if not storage then
return nil, "job-error", "could not connect to storage driver: "..tostring(err)
end
local ok, err = storage:delete(job.data.entity_id)
storage:close()
if ok == nil or ok == ngx_null then
return nil, "job-error", tostring(err)
end
end
return _M
================================================
FILE: lib/ledge/jobs/purge.lua
================================================
local ipairs, tonumber = ipairs, tonumber
local ngx_log = ngx.log
local ngx_DEBUG = ngx.DEBUG
local ngx_ERR = ngx.ERR
local ngx_null = ngx.null
local purge = require("ledge.purge").purge
local create_redis_slave_connection = require("ledge").create_redis_slave_connection
local close_redis_connection = require("ledge").close_redis_connection
local _M = {
_VERSION = "2.3.0",
}
-- Scans the keyspace for keys which match, and expires them. We do this against
-- the slave Redis instance if available.
function _M.perform(job)
if not job.redis then
return nil, "job-error", "no redis connection provided"
end
local slave, _ = create_redis_slave_connection()
if not slave then
job.redis_slave = job.redis
else
job.redis_slave = slave
end
-- Setup handler
local handler = require("ledge").create_handler()
handler.redis = job.redis
local storage, err = require("ledge").create_storage_connection(
job.data.storage_driver,
job.data.storage_driver_config
)
if not storage then
return nil, "redis-error", err
end
handler.storage = storage
-- This runs recursively using the SCAN cursor, until the entire keyspace
-- has been scanned.
local res, err = _M.expire_pattern(0, job, handler)
if slave then
close_redis_connection(slave)
end
if not res then
return nil, "redis-error", err
end
end
-- Scans the keyspace based on a pattern (asterisk), and runs a purge for each cache entry
function _M.expire_pattern(cursor, job, handler)
if job:ttl() < 10 then
if not job:heartbeat() then
return nil, "Failed to heartbeat job"
end
end
-- Scan using the "main" key to get a single key per cache entry
local res, err = job.redis_slave:scan(
cursor,
"MATCH", job.data.repset,
"COUNT", job.data.keyspace_scan_count
)
if not res or res == ngx_null then
return nil, "SCAN error: " .. tostring(err)
else
for _,key in ipairs(res[2]) do
ngx_log(ngx_DEBUG, "Purging set: ", key)
local ok, err = purge(handler, job.data.purge_mode, key)
if ok == nil and err then ngx_log(ngx_ERR, tostring(err)) end
end
local cursor = tonumber(res[1])
if cursor == 0 then
return true
end
-- If we have a valid cursor, recurse to move on.
return _M.expire_pattern(cursor, job, handler)
end
end
return _M
================================================
FILE: lib/ledge/jobs/revalidate.lua
================================================
local http = require "resty.http"
local http_headers = require "resty.http_headers"
local ngx_null = ngx.null
local _M = {
_VERSION = "2.3.0",
}
-- Utility to return all items in a Redis hash as a Lua table.
local function hgetall(redis, key)
local res, err = redis:hgetall(key)
if not res or res == ngx_null then
return nil,
"could not retrieve " .. tostring(key) .. " data:" .. tostring(err)
end
return redis:array_to_hash(res)
end
function _M.perform(job)
-- Normal background revalidation operates on stored metadata.
-- A background fetch due to partial content from upstream however, uses the
-- current request metadata for reval_headers / reval_params and passes it
-- through as job data.
local reval_params = job.data.reval_params
local reval_headers = job.data.reval_headers
-- If we don't have the metadata in job data, this is a background
-- revalidation using stored metadata.
if not reval_params and not reval_headers then
local key_chain, redis, err = job.data.key_chain, job.redis
reval_params, err = hgetall(redis, key_chain.reval_params)
if not reval_params or not next(reval_params) then
return nil, "job-error",
"Revalidation parameters are missing, presumed evicted. " ..
tostring(err)
end
reval_headers, err = hgetall(redis, key_chain.reval_req_headers)
if not reval_headers or not next(reval_headers) then
return nil, "job-error",
"Revalidation headers are missing, presumed evicted." ..
tostring(err)
end
end
-- Make outbound http request to revalidate
local httpc = http.new()
httpc:set_timeouts(
reval_params.connect_timeout,
reval_params.send_timeout,
reval_params.read_timeout
)
local port = tonumber(reval_params.server_port)
local ok, err
if port then
ok, err = httpc:connect(reval_params.server_addr, port)
else
ok, err = httpc:connect(reval_params.server_addr)
end
if not ok then
return nil, "job-error",
"could not connect to server: " .. tostring(err)
end
if reval_params.scheme == "https" then
local ok, err = httpc:ssl_handshake(false, nil, false)
if not ok then
return nil, "job-error", "ssl handshake failed: " .. tostring(err)
end
end
local headers = http_headers.new() -- Case-insensitive header table
headers["Cache-Control"] = "max-stale=0, stale-if-error=0"
headers["User-Agent"] =
httpc._USER_AGENT .. " ledge_revalidate/" .. _M._VERSION
-- Add additional headers from parent
for k,v in pairs(reval_headers) do
headers[k] = v
end
local res, err = httpc:request{
method = "GET",
path = reval_params.uri,
headers = headers,
}
if not res then
return nil, "job-error", "revalidate failed: " .. tostring(err)
else
local reader = res.body_reader
-- Read and discard the body
repeat
local chunk, _ = reader()
until not chunk
httpc:set_keepalive(
reval_params.keepalive_timeout,
reval_params.keepalive_poolsize
)
end
end
return _M
================================================
FILE: lib/ledge/purge.lua
================================================
local pcall, tonumber, tostring, pairs =
pcall, tonumber, tostring, pairs
local tbl_insert = table.insert
local ngx_var = ngx.var
local ngx_log = ngx.log
local ngx_ERR = ngx.ERR
local ngx_null = ngx.null
local ngx_time = ngx.time
local ngx_md5 = ngx.md5
local ngx_HTTP_BAD_REQUEST = ngx.HTTP_BAD_REQUEST
local str_find = string.find
local str_sub = string.sub
local str_len = string.len
local http = require("resty.http")
local cjson_encode = require("cjson").encode
local cjson_decode = require("cjson").decode
local fixed_field_metatable = require("ledge.util").mt.fixed_field_metatable
local put_background_job = require("ledge.background").put_background_job
local key_chain = require("ledge.cache_key").key_chain
local _M = {
_VERSION = "2.3.0",
}
local repset_len = -(str_len("::repset")+1)
local function create_purge_response(purge_mode, result, qless_jobs)
local d = {
purge_mode = purge_mode,
result = result,
}
if qless_jobs then d.qless_jobs = qless_jobs end
local ok, json = pcall(cjson_encode, d)
if not ok then
return nil, json
else
return json
end
end
_M.create_purge_response = create_purge_response
-- Expires the keys in key_chain and reduces the ttl in storage
local function expire_keys(redis, storage, key_chain, entity_id)
local ttl, err = redis:ttl(key_chain.main)
if not ttl or ttl == ngx_null or ttl == -1 then
return nil, "count not determine existing ttl: " .. (err or "")
end
if ttl == -2 then
-- Key doesn't exist, do nothing
return false, nil
end
local expires, err = redis:hget(key_chain.main, "expires")
expires = tonumber(expires)
if not expires or expires == ngx_null then
return nil, "could not determine existing expiry: " .. (err or "")
end
local time = ngx_time()
-- If expires is in the past then this key is stale. Nothing to do here.
if expires <= time then
return false, nil
end
local ttl_reduction = expires - time
if ttl_reduction < 0 then ttl_reduction = 0 end
local new_ttl = ttl - ttl_reduction
local _, e = redis:multi()
if e then ngx_log(ngx_ERR, e) end
-- Set the expires field of the main key to the new time, to control
-- its validity.
_, e = redis:hset(key_chain.main, "expires", tostring(time - 1))
if e then ngx_log(ngx_ERR, e) end
-- Set new TTLs for all keys in the key chain
for _,key in pairs(key_chain) do
local _, e = redis:expire(key, new_ttl)
if e then ngx_log(ngx_ERR, e) end
end
-- Reduce TTL on entity if there is one
if entity_id and entity_id ~= ngx_null then
storage:set_ttl(entity_id, new_ttl)
end
local ok, err = redis:exec() -- luacheck: ignore ok
if err then
return nil, err
else
return true, nil
end
end
_M.expire_keys = expire_keys
-- Purges the cache item according to purge_mode which defaults to "invalidate".
-- If there's nothing to do we return false which results in a 404.
-- @param table handler instance
-- @param string "invalidate" | "delete" | "revalidate
-- @param table key_chain to purge
-- @return boolean success
-- @return string message
-- @return table qless job (for revalidate only)
local function _purge(handler, purge_mode, key_chain)
local redis = handler.redis
local storage = handler.storage
local exists, err = redis:exists(key_chain.main)
if err then ngx_log(ngx_ERR, err) end
-- We 404 if we have nothing
if not exists or exists == ngx_null or exists == 0 then
return false, "nothing to purge", nil
end
-- Delete mode overrides everything else, since you can't revalidate
if purge_mode == "delete" then
local res, err = handler:delete_from_cache(key_chain)
if not res then
return nil, err, nil
else
return true, "deleted", nil
end
end
-- If we're revalidating, fire off the background job
local job
if purge_mode == "revalidate" then
job = handler:revalidate_in_background(key_chain, false)
end
-- Invalidate the keys
local ok, err = expire_keys(redis, storage, key_chain, handler:entity_id(key_chain))
if not ok and err then
return nil, err, job
elseif not ok then
return false, "already expired", job
elseif ok then
return true, "purged", job
end
end
local function key_chain_from_full_key(root_key, full_key)
local pos = str_find(full_key, "#")
if pos == nil then
return nil
end
-- Remove the root_key from the start
local vary_key = str_sub(full_key, pos+1)
local vary_spec = {} -- We don't need this
return key_chain(root_key, vary_key, vary_spec)
end
-- Purges all representatinos of the cache item
local function purge(handler, purge_mode, repset)
local representations, err = handler.redis:smembers(repset)
if err then
return nil, err
end
if #representations == 0 then
return false, "nothing to purge", nil
end
local root_key = str_sub(repset, 1, repset_len)
local res_ok, res_message
local jobs = {}
local key_chain
for _, full_key in ipairs(representations) do
key_chain = key_chain_from_full_key(root_key, full_key)
local ok, message, job = _purge(handler, purge_mode, key_chain)
-- Set the overall response if any representation was purged
if res_ok == nil or ok == true then
res_ok = ok
res_message = message
end
tbl_insert(jobs, job)
end
-- Clean up vary and repset keys if we're deleting
if purge_mode == "delete" and res_ok then
local _, e = handler.redis:del(key_chain.repset, key_chain.vary)
if e then ngx_log(ngx_ERR, e) end
end
return res_ok, res_message, jobs
end
_M.purge = purge
local function purge_in_background(handler, purge_mode)
local key_chain = handler:cache_key_chain()
local job, err = put_background_job(
"ledge_purge",
"ledge.jobs.purge",
{
repset = key_chain.repset,
keyspace_scan_count = handler.config.keyspace_scan_count,
purge_mode = purge_mode,
storage_driver = handler.config.storage_driver,
storage_driver_config = handler.config.storage_driver_config,
},
{
jid = ngx_md5("purge:" .. tostring(key_chain.root)),
tags = { "purge" },
priority = 5,
}
)
if err then ngx_log(ngx_ERR, err) end
-- Create a JSON payload for the response
local res = create_purge_response(purge_mode, "scheduled", {job})
handler.response:set_body(res)
return true
end
_M.purge_in_background = purge_in_background
local function parse_json_req()
ngx.req.read_body()
local body, err = ngx.req.get_body_data()
if not body then
return nil, "Could not read request body: " .. tostring(err)
end
local ok, req = pcall(cjson_decode, body)
if not ok then
return nil, "Could not parse request body: " .. tostring(req)
end
return req
end
local function validate_api_request(req)
local uris = req["uris"]
if not uris then
return false, "No URIs provided"
end
if type(uris) ~= "table" then
return false, "Field 'uris' must be an array"
end
if #uris == 0 then
return false, "No URIs provided"
end
local mode = req["purge_mode"]
if mode and not (
mode == "invalidate"
or mode == "revalidate"
or mode == "delete"
) then
return false, "Invalid purge_mode"
end
return true
end
local function send_purge_request(uri, purge_mode, headers)
local uri_parts, err = http:parse_uri(uri)
if not uri_parts then
return nil, err
end
local scheme, host, port, path = unpack(uri_parts)
-- TODO: timeouts
local httpc = http.new()
local ok, err = httpc:connect(ngx_var.server_addr, port)
if not ok then
return nil, "HTTP Connect ("..ngx_var.server_addr..":"..port.."): "..err
end
if scheme == "https" then
local ok, err = httpc:ssl_handshake(nil, host, false)
if not ok then
return nil, "SSL Handshake: "..err
end
end
headers = headers or {}
headers["Host"] = host
headers["X-Purge"] = purge_mode
local res, err = httpc:request({
method = "PURGE",
path = path,
headers = headers
})
if not res then
return nil, "HTTP Request: "..err
end
local body, err = res:read_body()
if not body then
return nil, "HTTP Response: "..err
end
local ok, err = httpc:set_keepalive()
if not ok then ngx_log(ngx_ERR, err) end
if res.headers["Content-Type"] == "application/json" then
body = cjson_decode(body)
else
return nil, { status = res.status, body = body, headers = res.headers}
end
return body
end
-- Run the JSON PURGE API.
-- Accepts various inputs from a JSON request body and processes purges
-- Return true on success or false on error
local function purge_api(handler)
local response = handler.response
local request, err = parse_json_req()
if not request then
response.status = ngx_HTTP_BAD_REQUEST
response:set_body(cjson_encode({["error"] = err}))
return false
end
local ok, err = validate_api_request(request)
if not ok then
response.status = ngx_HTTP_BAD_REQUEST
response:set_body(cjson_encode({["error"] = err}))
return false
end
local purge_mode = request["purge_mode"] or "invalidate" -- Default to invalidating
local api_results = {}
local uris = request["uris"]
for _, uri in ipairs(uris) do
local res, err = send_purge_request(uri, purge_mode, request["headers"])
if not res then
res = {["error"] = err}
elseif type(res) == "table" then
res["purge_mode"] = nil
end
api_results[uri] = res
end
local api_response, err = create_purge_response(purge_mode, api_results)
if not api_response then
handler.set:body(cjson_encode({["error"] = "JSON Response Error: "..tostring(err)}))
return false
end
handler.response:set_body(api_response)
return true
end
_M.purge_api = purge_api
return setmetatable(_M, fixed_field_metatable)
================================================
FILE: lib/ledge/range.lua
================================================
local setmetatable, tonumber, ipairs, type =
setmetatable, tonumber, ipairs, type
local str_match = string.match
local str_sub = string.sub
local str_randomhex = require("ledge.util").string.randomhex
local str_split = require("ledge.util").string.split
local tbl_insert = table.insert
local tbl_sort = table.sort
local tbl_remove = table.remove
local tbl_concat = table.concat
local ngx_re_match = ngx.re.match
local ngx_log = ngx.log
local ngx_ERR = ngx.ERR
local get_header_token = require("ledge.header_util").get_header_token
local co_yield = coroutine.yield
local co_wrap = require("ledge.util").coroutine.wrap
local get_fixed_field_metatable_proxy =
require("ledge.util").mt.get_fixed_field_metatable_proxy
local ngx_req_get_headers = ngx.req.get_headers
local ngx_RANGE_NOT_SATISFIABLE = 416
local ngx_PARTIAL_CONTENT = 206
local _M = {
_VERSION = "2.3.0",
}
function _M.new()
return setmetatable({
ranges = {},
boundary_end = "",
boundary = "",
}, get_fixed_field_metatable_proxy(_M))
end
-- returns a table of ranges, or nil
--
-- e.g.
-- {
-- { from = 0, to = 99 },
-- { from = 100, to = 199 },
-- }
local function req_byte_ranges()
local bytes = get_header_token(ngx_req_get_headers().range, "bytes")
local ranges = nil
if bytes then
ranges = str_split(bytes, ",")
if not ranges then ranges = { bytes } end
for i,r in ipairs(ranges) do
local from, to = str_match(r, "(%d*)%-(%d*)")
ranges[i] = { from = tonumber(from), to = tonumber(to) }
end
end
return ranges
end
_M.req_byte_ranges = req_byte_ranges
local function sort_byte_ranges(first, second)
if not first.from or not second.from then
return nil, "Attempt to compare invalid byteranges"
end
return first.from <= second.from
end
local function parse_content_range(content_range)
local m, err = ngx_re_match(
content_range,
[[bytes\s+(\d+|\*)-(\d+|\*)/(\d+)]],
"oj"
)
if err then ngx_log(ngx_ERR, err) end
if not m then
return nil
else
return tonumber(m[1]), tonumber(m[2]), tonumber(m[3])
end
end
_M.parse_content_range = parse_content_range
-- Modifies the response based on range request headers.
-- Returns the response and a flag, which if true indicates a partial response
-- should be expected, if false indicates the range could not be applied, and if
-- nil indicates no range was requested.
function _M.handle_range_request(self, res)
local range_request = req_byte_ranges()
if range_request and type(range_request) == "table" and res.size then
-- Don't attempt range filtering on non 200 responses
if res.status ~= 200 then
return res, false
end
local ranges = {}
for _,range in ipairs(range_request) do
local range_satisfiable = true
if not range.to and not range.from then
range_satisfiable = false
end
-- A missing "to" means to the "end".
if not range.to then
range.to = res.size - 1
end
-- A missing "from" means "to" is an offset from the end.
if not range.from then
range.from = res.size - (range.to)
range.to = res.size - 1
if range.from < 0 then
range_satisfiable = false
end
end
-- A "to" greater than size should be "end"
if range.to > (res.size - 1) then
range.to = res.size - 1
end
-- Check the range is satisfiable
if range.from > range.to then
range_satisfiable = false
end
if not range_satisfiable then
-- We'll return 416
res.status = ngx_RANGE_NOT_SATISFIABLE
res.body_reader = res.empty_body_reader
res.header.content_range = "bytes */" .. res.size
return res, false
else
-- We'll need the content range header value
-- for multipart boundaries: e.g. bytes 5-10/20
range.header = "bytes " .. range.from ..
"-" .. range.to ..
"/" .. res.size
tbl_insert(ranges, range)
end
end
local numranges = #ranges
if numranges > 1 then
-- Sort ranges as we cannot serve unordered.
tbl_sort(ranges, sort_byte_ranges)
-- Coalesce overlapping ranges.
for i = numranges,1,-1 do
if i > 1 then
local current_range = ranges[i]
local previous_range = ranges[i - 1]
if current_range.from <= previous_range.to then
-- extend previous range to encompass this one
previous_range.to = current_range.to
previous_range.header = "bytes " ..
previous_range.from ..
"-" ..
current_range.to ..
"/" ..
res.size
tbl_remove(ranges, i)
end
end
end
end
self.ranges = ranges
if #ranges == 1 then
-- We have a single range to serve.
local range = ranges[1]
local size = res.size
res.status = ngx_PARTIAL_CONTENT
ngx.header["Accept-Ranges"] = "bytes"
res.header["Content-Range"] = "bytes " .. range.from ..
"-" .. range.to ..
"/" .. size
return res, true
else
-- Generate boundary
local boundary_string = str_randomhex(32)
local boundary = {
"",
"--" .. boundary_string,
}
if res.header["Content-Type"] then
tbl_insert(
boundary,
"Content-Type: " .. res.header["Content-Type"]
)
end
self.boundary = tbl_concat(boundary, "\n")
self.boundary_end = "\n--" .. boundary_string .. "--"
res.status = ngx_PARTIAL_CONTENT
-- TODO: No test coverage for these headers
res.header["Accept-Ranges"] = "bytes"
res.header["Content-Type"] = "multipart/byteranges; boundary=" ..
boundary_string
return res, true
end
end
return res, nil
end
-- Filters the body reader, only yielding bytes specified in a range request.
function _M.get_range_request_filter(self, reader)
local ranges = self.ranges
local boundary_end = self.boundary_end
local boundary = self.boundary
if ranges then
return co_wrap(function(buffer_size)
local playhead = 0
local num_ranges = #ranges
while true do
local chunk, err = reader(buffer_size)
if err then ngx_log(ngx_ERR, err) end
if not chunk then break end
local chunklen = #chunk
local nextplayhead = playhead + chunklen
for _, range in ipairs(ranges) do
if range.from >= nextplayhead or range.to < playhead then -- luacheck: ignore 542
-- Skip over non matching ranges (this is
-- algorithmically simpler)
else
-- Yield the multipart byterange boundary if
-- required and only once per range.
if num_ranges > 1 and not range.boundary_printed then
co_yield(boundary)
co_yield("\nContent-Range: " .. range.header)
co_yield("\n\n")
range.boundary_printed = true
end
-- Trim range to within this chunk's context
local yield_from = range.from
local yield_to = range.to
if range.from < playhead then
yield_from = playhead
end
if range.to >= nextplayhead then
yield_to = nextplayhead - 1
end
-- Find relative points for the range within this chunk
local relative_yield_from = yield_from - playhead
local relative_yield_to = yield_to - playhead
-- Ranges are all 0 indexed, finally convert to 1 based
-- Lua indexes, and yield the range.
co_yield(
str_sub(
chunk,
relative_yield_from + 1,
relative_yield_to + 1
)
)
end
end
playhead = playhead + chunklen
end
-- Yield the multipart byterange end marker
if num_ranges > 1 then
co_yield(boundary_end)
end
end)
end
return reader
end
return _M
================================================
FILE: lib/ledge/request.lua
================================================
local hdr_has_directive = require("ledge.header_util").header_has_directive
local ngx_req_get_headers = ngx.req.get_headers
local ngx_re_gsub = ngx.re.gsub
local ngx_req_get_uri_args = ngx.req.get_uri_args
local ngx_req_get_method = ngx.req.get_method
local str_byte = string.byte
local ngx_var = ngx.var
local tbl_sort = table.sort
local tbl_insert = table.insert
local _M = {
_VERSION = "2.3.0",
}
local function purge_mode()
local x_purge = ngx_req_get_headers()["X-Purge"]
if hdr_has_directive(x_purge, "delete") then
return "delete"
elseif hdr_has_directive(x_purge, "revalidate") then
return "revalidate"
else
return "invalidate"
end
end
_M.purge_mode = purge_mode
local function relative_uri()
local uri = ngx_re_gsub(ngx_var.uri, "\\s", "%20", "jo") -- encode spaces
-- encode percentages if an encoded CRLF is in the URI
-- see: http://resources.infosecinstitute.com/http-response-splitting-attack
uri = ngx_re_gsub(uri, "%0D%0A", "%250D%250A", "ijo")
return uri .. ngx_var.is_args .. (ngx_var.query_string or "")
end
_M.relative_uri = relative_uri
local function full_uri()
return ngx_var.scheme .. '://' .. ngx_var.host .. relative_uri()
end
_M.full_uri = full_uri
local function accepts_cache()
-- Check for no-cache
local h = ngx_req_get_headers()
if hdr_has_directive(h["Pragma"], "no-cache")
or hdr_has_directive(h["Cache-Control"], "no-cache")
or hdr_has_directive(h["Cache-Control"], "no-store") then
return false
end
return true
end
_M.accepts_cache = accepts_cache
local function sort_args(a, b)
return a[1] < b[1]
end
local function args_sorted(max_args)
max_args = max_args or 100
local args = ngx_req_get_uri_args(max_args)
if not next(args) then return nil end
local sorted = {}
for k, v in pairs(args) do
tbl_insert(sorted, { k, v })
end
tbl_sort(sorted, sort_args)
local sargs = ""
local sortedln = #sorted
for i, v in ipairs(sorted) do
sargs = sargs .. ngx.encode_args({ [v[1]] = v[2] })
if i < sortedln then sargs = sargs .. "&" end
end
return sargs
end
_M.args_sorted = args_sorted
-- Used to generate a default args string for the cache key (i.e. when there are
-- no URI args present).
--
-- Returns a zero length string, unless there is an asterisk at the end of the
-- URI on a PURGE request, in which case we return the asterisk.
--
-- The purpose it to ensure trailing wildcards are greedy across both URI and
-- args portions of a cache key.
--
-- If you override the "args" field in a cache key spec with your own function,
-- you'll want to use this to ensure wildcard purges operate correctly.
local function default_args()
if ngx_req_get_method() == "PURGE" and
str_byte(ngx_var.request_uri, -1) == 42
then
return "*"
end
return ""
end
_M.default_args = default_args
return _M
================================================
FILE: lib/ledge/response.lua
================================================
local http_headers = require "resty.http_headers"
local util = require "ledge.util"
local pairs, setmetatable, tonumber, unpack =
pairs, setmetatable, tonumber, unpack
local tbl_getn = table.getn
local tbl_insert = table.insert
local tbl_concat = table.concat
local tbl_sort = table.sort
local str_lower = string.lower
local str_find = string.find
local str_sub = string.sub
local str_rep = string.rep
local str_randomhex = util.string.randomhex
local str_split = util.string.split
local ngx_null = ngx.null
local ngx_log = ngx.log
local ngx_ERR = ngx.ERR
local ngx_INFO = ngx.INFO
local ngx_DEBUG = ngx.DEBUG
local ngx_re_gmatch = ngx.re.gmatch
local ngx_parse_http_time = ngx.parse_http_time
local ngx_http_time = ngx.http_time
local ngx_time = ngx.time
local ngx_re_find = ngx.re.find
local ngx_re_gsub = ngx.re.gsub
local header_has_directive = require("ledge.header_util").header_has_directive
local get_fixed_field_metatable_proxy =
require("ledge.util").mt.get_fixed_field_metatable_proxy
local save_key_chain = require("ledge.cache_key").save_key_chain
local _DEBUG = false
local _M = {
_VERSION = "2.3.0",
set_debug = function(debug) _DEBUG = debug end,
}
-- Body reader for when the response body is missing
local function empty_body_reader()
return nil
end
_M.empty_body_reader = empty_body_reader
function _M.new(handler)
if not handler or not next(handler) then
return nil, "Handler is required"
end
if not handler.redis or not next(handler.redis) then
return nil, "Handler has no redis connection"
end
return setmetatable({
redis = handler.redis,
handler = handler, -- Cache key chain
uri = "",
status = 0,
header = http_headers.new(),
-- stored metadata
size = 0,
remaining_ttl = 0,
has_esi = false,
esi_scanned = false,
-- body
entity_id = "",
body_reader = empty_body_reader,
body_filters = {}, -- for debug logging
-- runtime metadata (not persisted)
length = 0, -- If Content-Length is present
has_body = false, -- From lua-resty-http has_body
}, get_fixed_field_metatable_proxy(_M))
end
-- Setter for a fixed body string (not streamed)
function _M.set_body(self, body_string)
local sent = false
self.body_reader = function()
if not sent then
sent = true
return body_string
else
return nil
end
end
end
function _M.filter_body_reader(self, filter_name, filter)
assert(type(filter) == "function", "filter must be a function")
if _DEBUG then
-- Keep track of the filters by name, just for debugging
ngx_log(ngx_DEBUG,
filter_name,
"(",
tbl_concat(self.body_filters,
"("), "" , str_rep(")", #self.body_filters - 1
),
")"
)
tbl_insert(self.body_filters, 1, filter_name)
end
self.body_reader = filter
end
function _M.is_cacheable(self)
-- Never cache partial content
local status = self.status
if status == 206 or status == 416 then
return false
end
local h = self.header
local directives = "(no-cache|no-store|private)"
if header_has_directive(h["Cache-Control"], directives, true) then
return false
end
if header_has_directive(h["Pragma"], "no-cache", true) then
return false
end
if h["Vary"] == "*" then
return false
end
if self:ttl() > 0 then
return true
else
return false
end
end
-- Calculates the TTL from response headers.
-- Header precedence is Cache-Control: s-maxage=NUM, Cache-Control: max-age=NUM
-- and finally Expires: HTTP_TIMESTRING.
function _M.ttl(self)
local cc = self.header["Cache-Control"]
if cc then
if type(cc) == "table" then
cc = tbl_concat(cc, ", ")
end
local max_ages = {}
for max_age in ngx_re_gmatch(cc, [[(s-maxage|max-age)=(\d+)]], "ijo") do
max_ages[max_age[1]] = max_age[2]
end
if max_ages["s-maxage"] then
return tonumber(max_ages["s-maxage"])
elseif max_ages["max-age"] then
return tonumber(max_ages["max-age"])
end
end
-- Fall back to Expires.
local expires = self.header["Expires"]
if expires then
-- If there are multiple, last one wins
if type(expires) == "table" then
expires = expires[#expires]
end
local time = ngx_parse_http_time(tostring(expires))
if time then return time - ngx_time() end
end
return 0
end
function _M.has_expired(self)
return self.remaining_ttl <= 0
end
-- Return nil and an error on an actual Redis error, this indicates that Redis
-- has failed and we aren't going to be able to proceed normally.
-- Return nil and *no* error if this is just a broken/partial cache entry
-- so we MISS and update the entry.
function _M.read(self)
local key_chain, err = self.handler:cache_key_chain()
if not key_chain then
return nil, err
end
local redis = self.redis
-- Read main metdata
local cache_parts, err = redis:hgetall(key_chain.main)
if not cache_parts or cache_parts == ngx_null then
return nil, err
end
-- No cache entry for this key
local cache_parts_len = #cache_parts
if not cache_parts_len or cache_parts_len == 0 then
return nil
end
local time_in_cache = 0
local time_since_generated = 0
-- The Redis replies is a sequence of messages, so we iterate over pairs
-- to get hash key/values.
for i = 1, cache_parts_len, 2 do
if cache_parts[i] == "uri" then
self.uri = cache_parts[i + 1]
elseif cache_parts[i] == "status" then
self.status = tonumber(cache_parts[i + 1])
elseif cache_parts[i] == "entity" then
self.entity_id = cache_parts[i + 1]
elseif cache_parts[i] == "expires" then
self.remaining_ttl = tonumber(cache_parts[i + 1]) - ngx_time()
elseif cache_parts[i] == "saved_ts" then
time_in_cache = ngx_time() - tonumber(cache_parts[i + 1])
elseif cache_parts[i] == "generated_ts" then
time_since_generated = ngx_time() - tonumber(cache_parts[i + 1])
elseif cache_parts[i] == "has_esi" then
self.has_esi = cache_parts[i + 1]
elseif cache_parts[i] == "esi_scanned" then
local scanned = cache_parts[i + 1]
if scanned == "false" then
self.esi_scanned = false
else
self.esi_scanned = true
end
elseif cache_parts[i] == "size" then
self.size = tonumber(cache_parts[i + 1])
end
end
-- Read headers
local headers, err = redis:hgetall(key_chain.headers)
if not headers or headers == ngx_null then
return nil, err
end
local headers_len = tbl_getn(headers)
if headers_len == 0 then
ngx_log(ngx_INFO, "headers missing")
return nil
end
for i = 1, headers_len, 2 do
local header = headers[i]
if str_find(header, ":") then
-- We have multiple headers with the same field name
local _, key = unpack(str_split(header, ":"))
if not self.header[key] then
self.header[key] = {}
end
tbl_insert(self.header[key], headers[i + 1])
else
self.header[header] = headers[i + 1]
end
end
-- Calculate the Age header
if self.header["Age"] then
-- We have end-to-end Age headers, add our time_in_cache.
self.header["Age"] = tonumber(self.header["Age"]) + time_in_cache
elseif self.header["Date"] then
-- We have no advertised Age, use the generated timestamp.
self.header["Age"] = time_since_generated
end
-- "touch" other keys not needed for read, so that they are
-- less likely to be unfairly evicted ahead of time
-- Note: From Redis 3.2.1 this could be one TOUCH command
local _ = redis:hlen(key_chain.reval_params)
local _ = redis:hlen(key_chain.reval_req_headers)
if self.size > 0 then
local entities, err = redis:scard(key_chain.entities)
if not entities or entities == ngx_null then
return nil, "could not read entities set: " .. err
elseif entities == 0 then
ngx_log(ngx_INFO, "entities set is empty")
return nil
end
end
-- Check this key is in the repset
local scard, err = redis:sismember(key_chain.repset, key_chain.full)
if err then
return nil, err
end
-- Got a cache entry but missing from repset or repset missing, bad...
-- Call save_key_chain which will add this rep to the repset
if scard == 0 then
local repset_ttl = redis:ttl(key_chain.main)
local ok, err = save_key_chain(redis, key_chain, repset_ttl)
if not ok then
return nil, err
end
end
return true
end
-- Takes headers from a HTTP response and returns a flat table of cacheable
-- header entries formatted for Redis.
local function prepare_cacheable_headers(headers)
-- Don't cache any headers marked as
-- Cache-Control: (no-cache|no-store|private)="header".
local uncacheable_headers = {}
local cc = headers["Cache-Control"]
if cc then
if type(cc) == "table" then cc = tbl_concat(cc, ", ") end
cc = str_lower(cc)
if str_find(cc, "=", 1, true) then
local pattern = '(?:no-cache|private)="?([0-9a-z-]+)"?'
local re_ctx = {}
repeat
local from, to, err = ngx_re_find(cc, pattern, "jo", re_ctx, 1)
if from then
uncacheable_headers[str_sub(cc, from, to)] = true
elseif err then
ngx_log(ngx_ERR, err)
end
until not from
end
end
-- Turn the headers into a flat list of pairs for the Redis query.
local h = {}
for header,header_value in pairs(headers) do
if not uncacheable_headers[str_lower(header)] then
if type(header_value) == 'table' then
-- Multiple headers are represented as a table of values
local header_value_len = tbl_getn(header_value)
for i = 1, header_value_len do
tbl_insert(h, i..':'..header)
tbl_insert(h, header_value[i])
end
else
tbl_insert(h, header)
tbl_insert(h, header_value)
end
end
end
return h
end
function _M.save(self, keep_cache_for)
if not keep_cache_for then keep_cache_for = 0 end
-- Create a new entity id
self.entity_id = str_randomhex(32)
local ttl = self:ttl()
local time = ngx_time()
local redis = self.redis
if not next(redis) then return nil, "no redis" end
local key_chain = self.handler:cache_key_chain()
if not self.header["Date"] then
self.header["Date"] = ngx_http_time(ngx_time())
end
local ok, err = redis:del(key_chain.main)
if not ok then ngx_log(ngx_ERR, err) end
local ok, err = redis:hmset(key_chain.main,
"entity", self.entity_id,
"status", self.status,
"uri", self.uri,
"expires", ttl + time,
"generated_ts", ngx_parse_http_time(self.header["Date"]),
"saved_ts", time,
"esi_scanned", tostring(self.esi_scanned) -- from bool
)
if not ok then ngx_log(ngx_ERR, err) end
local h = prepare_cacheable_headers(self.header)
ok, err = redis:del(key_chain.headers)
if not ok then ngx_log(ngx_ERR, err) end
ok, err = redis:hmset(key_chain.headers, unpack(h))
if not ok then ngx_log(ngx_ERR, err) end
-- Mark the keys as eventually volatile (the body is set by the body writer)
local expiry = ttl + tonumber(keep_cache_for)
ok, err = redis:expire(key_chain.main, expiry)
if not ok then ngx_log(ngx_ERR, err) end
ok, err = redis:expire(key_chain.headers, expiry)
if not ok then ngx_log(ngx_ERR, err) end
local ok, err = redis:sadd(key_chain.entities, self.entity_id)
if not ok then
ngx_log(ngx_ERR, "error adding entity to set: ", err)
end
ok, err = redis:expire(key_chain.entities, expiry)
if not ok then ngx_log(ngx_ERR, err) end
return true
end
function _M.set_and_save(self, field, value)
local redis = self.redis
local ok, err = redis:hset(self.handler:cache_key_chain().main, field, tostring(value))
if not ok then
if err then ngx_log(ngx_ERR, err) end
return nil, err
end
self[field] = value
return ok
end
local WARNINGS = {
["110"] = "Response is stale",
["214"] = "Transformation applied",
["112"] = "Disconnected Operation",
}
function _M.add_warning(self, code, name)
if not self.header["Warning"] then
self.header["Warning"] = {}
end
local header = code .. ' ' .. name
header = header .. ' "' .. WARNINGS[code] .. '"'
tbl_insert(self.header["Warning"], header)
end
local function deduplicate_table(table)
-- Can't have duplicates if there's 1 or 0 entries!
if #table <= 1 then
return table
end
local new_table = {}
local unique = {}
local i = 0
for _,v in ipairs(table) do
if not unique[v] then
unique[v] = true
i = i +1
new_table[i] = v
end
end
return new_table
end
function _M.parse_vary_header(self)
local vary_hdr = self.header["Vary"]
local vary_spec
if vary_hdr and vary_hdr ~= "" then
if type(vary_hdr) == "table" then
vary_hdr = tbl_concat(vary_hdr,",")
end
-- Remove whitespace around commas and lowercase
vary_hdr = ngx_re_gsub(str_lower(vary_hdr), [[\s*,\s*]], ",", "oj")
vary_spec = str_split(vary_hdr, ",")
tbl_sort(vary_spec)
vary_spec = deduplicate_table(vary_spec)
end
-- Return the new vary sepc table *and* the normalised header
return vary_spec
end
return _M
================================================
FILE: lib/ledge/stale.lua
================================================
local math_min = math.min
local ngx_req_get_headers = ngx.req.get_headers
local header_has_directive = require("ledge.header_util").header_has_directive
local get_numeric_header_token =
require("ledge.header_util").get_numeric_header_token
local _M = {
_VERSION = "2.3.0"
}
-- True if the request specifically asks for stale (req.cc.max-stale) and the
-- response doesn't explicitly forbid this res.cc.(must|proxy)-revalidate.
local function can_serve_stale(res)
local req_cc = ngx_req_get_headers()["Cache-Control"]
local req_cc_max_stale = get_numeric_header_token(req_cc, "max-stale")
if req_cc_max_stale then
local res_cc = res.header["Cache-Control"]
-- Check the response permits this at all
if header_has_directive(res_cc, "(must|proxy)-revalidate") then
return false
else
if (req_cc_max_stale * -1) <= res.remaining_ttl then
return true
end
end
end
return false
end
_M.can_serve_stale = can_serve_stale
-- Returns true if stale-while-revalidate or stale-if-error is specified, valid
-- and not constrained by other factors such as max-stale.
-- @param token "stale-while-revalidate" | "stale-if-error"
local function verify_stale_conditions(res, token)
assert(token == "stale-while-revalidate" or token
gitextract_rst1uaiw/
├── .gitattributes
├── .github/
│ └── FUNDING.yml
├── .gitignore
├── .luacheckrc
├── .luacov
├── .travis.yml
├── Makefile
├── README.md
├── dist.ini
├── docker/
│ └── tests/
│ └── docker-compose.yml
├── lib/
│ ├── ledge/
│ │ ├── background.lua
│ │ ├── cache_key.lua
│ │ ├── collapse.lua
│ │ ├── esi/
│ │ │ ├── processor_1_0.lua
│ │ │ └── tag_parser.lua
│ │ ├── esi.lua
│ │ ├── gzip.lua
│ │ ├── handler.lua
│ │ ├── header_util.lua
│ │ ├── jobs/
│ │ │ ├── collect_entity.lua
│ │ │ ├── purge.lua
│ │ │ └── revalidate.lua
│ │ ├── purge.lua
│ │ ├── range.lua
│ │ ├── request.lua
│ │ ├── response.lua
│ │ ├── stale.lua
│ │ ├── state_machine/
│ │ │ ├── actions.lua
│ │ │ ├── events.lua
│ │ │ ├── pre_transitions.lua
│ │ │ └── states.lua
│ │ ├── state_machine.lua
│ │ ├── storage/
│ │ │ └── redis.lua
│ │ ├── util.lua
│ │ ├── validation.lua
│ │ └── worker.lua
│ └── ledge.lua
├── migrations/
│ └── 1.26-1.27.lua
├── t/
│ ├── 01-unit/
│ │ ├── cache_key.t
│ │ ├── esi.t
│ │ ├── events.t
│ │ ├── handler.t
│ │ ├── jobs.t
│ │ ├── ledge.t
│ │ ├── processor_1_0.t
│ │ ├── purge.t
│ │ ├── range.t
│ │ ├── request.t
│ │ ├── response.t
│ │ ├── stale.t
│ │ ├── state_machine.t
│ │ ├── storage.t
│ │ ├── tag_parser.t
│ │ ├── util.t
│ │ ├── validation.t
│ │ └── worker.t
│ ├── 02-integration/
│ │ ├── age.t
│ │ ├── cache.t
│ │ ├── collapsed_forwarding.t
│ │ ├── esi.t
│ │ ├── events.t
│ │ ├── gc.t
│ │ ├── gzip.t
│ │ ├── hop_by_hop_headers.t
│ │ ├── max-stale.t
│ │ ├── max_size.t
│ │ ├── memory_pressure.t
│ │ ├── multiple_headers.t
│ │ ├── on_abort.t
│ │ ├── origin_mode.t
│ │ ├── purge.t
│ │ ├── range.t
│ │ ├── req_body.t
│ │ ├── req_method.t
│ │ ├── request_leak.t
│ │ ├── response.t
│ │ ├── ssl.t
│ │ ├── stale-if-error.t
│ │ ├── stale-while-revalidate.t
│ │ ├── upstream.t
│ │ ├── upstream_client.t
│ │ ├── validation.t
│ │ ├── vary.t
│ │ └── via_header.t
│ ├── 03-sentinel/
│ │ ├── 01-master_up.t
│ │ ├── 02-master_down.t
│ │ └── 03-slave_promoted.t
│ ├── LedgeEnv.pm
│ └── cert/
│ ├── example.com.crt
│ ├── example.com.key
│ ├── rootCA.key
│ ├── rootCA.pem
│ └── rootCA.srl
└── util/
└── lua-releng
Condensed preview — 94 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (797K chars).
[
{
"path": ".gitattributes",
"chars": 26,
"preview": "*.t linguist-language=lua\n"
},
{
"path": ".github/FUNDING.yml",
"chars": 18,
"preview": "github: pintsized\n"
},
{
"path": ".gitignore",
"chars": 60,
"preview": "t/servroot/\nt/error.log\ndump.rdb\nstdout\nluacov.*\n*.src.rock\n"
},
{
"path": ".luacheckrc",
"chars": 34,
"preview": "std = \"ngx_lua\"\nredefined = false\n"
},
{
"path": ".luacov",
"chars": 207,
"preview": "modules = {\n [\"ledge\"] = \"lib/ledge.lua\",\n [\"ledge.esi.*\"] = \"lib/\",\n [\"ledge.jobs.*\"] = \"lib/\",\n [\"ledge.st"
},
{
"path": ".travis.yml",
"chars": 91,
"preview": "services:\n - docker\n\nscript:\n - cd docker/tests\n - docker-compose run --rm runner\n"
},
{
"path": "Makefile",
"chars": 6534,
"preview": "SHELL := /bin/bash # Cheat by using bash :)\n\nOPENRESTY_PREFIX = /usr/local/openresty\n\nTEST_FILE ?= t/01-unit"
},
{
"path": "README.md",
"chars": 54515,
"preview": "# Ledge\n\n[](https://travis-ci.org/ledgetech/ledg"
},
{
"path": "dist.ini",
"chars": 444,
"preview": "name=ledge\nabstract=An RFC compliant and ESI capable HTTP cache for Nginx / OpenResty, backed by Redis\nauthor=James Hurs"
},
{
"path": "docker/tests/docker-compose.yml",
"chars": 504,
"preview": "version: '3'\n\nservices:\n runner:\n image: \"ledgetech/test-runner:latest\"\n volumes:\n - ../../:"
},
{
"path": "lib/ledge/background.lua",
"chars": 1396,
"preview": "local require = require\nlocal math_ceil = math.ceil\nlocal qless = require(\"resty.qless\")\n\nlocal _M = {\n _VERSION = \"2"
},
{
"path": "lib/ledge/cache_key.lua",
"chars": 8540,
"preview": "local ipairs, next, type, pcall, setmetatable =\n ipairs, next, type, pcall, setmetatable\n\nlocal str_lower = string."
},
{
"path": "lib/ledge/collapse.lua",
"chars": 1044,
"preview": "local _M = {\n _VERSION = \"2.3.0\",\n}\n\n-- Attempts to set a lock key in redis. The lock will expire after\n-- the expiry"
},
{
"path": "lib/ledge/esi/processor_1_0.lua",
"chars": 33874,
"preview": "local http = require \"resty.http\"\nlocal cookie = require \"resty.cookie\"\nlocal tag_parser = require \"ledge.esi.tag_parser"
},
{
"path": "lib/ledge/esi/tag_parser.lua",
"chars": 5205,
"preview": "local setmetatable, type =\n setmetatable, type\n\nlocal str_sub = string.sub\n\nlocal ngx_re_find = ngx.re.find\nlocal ngx"
},
{
"path": "lib/ledge/esi.lua",
"chars": 6020,
"preview": "local h_util = require \"ledge.header_util\"\n\nlocal type, tonumber = type, tonumber\n\nlocal str_sub = string.sub\nlocal str_"
},
{
"path": "lib/ledge/gzip.lua",
"chars": 1211,
"preview": "local co_yield = coroutine.yield\nlocal co_wrap = require(\"ledge.util\").coroutine.wrap\n\nlocal ngx_log = ngx.log\nlocal ngx"
},
{
"path": "lib/ledge/handler.lua",
"chars": 29567,
"preview": "local setmetatable, tostring, tonumber, pcall, type, ipairs, pairs, next, error =\n setmetatable, tostring, tonumber,"
},
{
"path": "lib/ledge/header_util.lua",
"chars": 1836,
"preview": "local type, tonumber, setmetatable =\n type, tonumber, setmetatable\n\nlocal ngx_re_match = ngx.re.match\nlocal ngx_re_fi"
},
{
"path": "lib/ledge/jobs/collect_entity.lua",
"chars": 697,
"preview": "local tostring = tostring\nlocal ngx_null = ngx.null\n\nlocal create_storage_connection = require(\"ledge\").create_storage_c"
},
{
"path": "lib/ledge/jobs/purge.lua",
"chars": 2529,
"preview": "local ipairs, tonumber = ipairs, tonumber\nlocal ngx_log = ngx.log\nlocal ngx_DEBUG = ngx.DEBUG\nlocal ngx_ERR = ngx.ERR\nlo"
},
{
"path": "lib/ledge/jobs/revalidate.lua",
"chars": 3339,
"preview": "local http = require \"resty.http\"\nlocal http_headers = require \"resty.http_headers\"\nlocal ngx_null = ngx.null\n\nlocal _M "
},
{
"path": "lib/ledge/purge.lua",
"chars": 10517,
"preview": "local pcall, tonumber, tostring, pairs =\n pcall, tonumber, tostring, pairs\n\nlocal tbl_insert = table.insert\n\nlocal ng"
},
{
"path": "lib/ledge/range.lua",
"chars": 9734,
"preview": "local setmetatable, tonumber, ipairs, type =\n setmetatable, tonumber, ipairs, type\n\nlocal str_match = string.match\nlo"
},
{
"path": "lib/ledge/request.lua",
"chars": 2966,
"preview": "local hdr_has_directive = require(\"ledge.header_util\").header_has_directive\n\nlocal ngx_req_get_headers = ngx.req.get_hea"
},
{
"path": "lib/ledge/response.lua",
"chars": 14336,
"preview": "local http_headers = require \"resty.http_headers\"\nlocal util = require \"ledge.util\"\n\nlocal pairs, setmetatable, tonumber"
},
{
"path": "lib/ledge/stale.lua",
"chars": 3376,
"preview": "local math_min = math.min\n\nlocal ngx_req_get_headers = ngx.req.get_headers\n\nlocal header_has_directive = require(\"ledge."
},
{
"path": "lib/ledge/state_machine/actions.lua",
"chars": 7398,
"preview": "local type, next = type, next\n\nlocal esi = require(\"ledge.esi\")\nlocal response = require(\"ledge.response\")\n\nlocal ngx_va"
},
{
"path": "lib/ledge/state_machine/events.lua",
"chars": 18265,
"preview": "local _M = { -- luacheck: no unused\n _VERSION = \"2.3.0\",\n}\n\n\n-- Event transition table.\n--\n-- Use \"begin\" to transiti"
},
{
"path": "lib/ledge/state_machine/pre_transitions.lua",
"chars": 1021,
"preview": "local _M = { -- luacheck: no unused\n _VERSION = \"2.3.0\",\n}\n\n\n-- Pre-transitions. These actions will *always* be perfo"
},
{
"path": "lib/ledge/state_machine/states.lua",
"chars": 17961,
"preview": "local ledge = require(\"ledge\")\nlocal esi = require(\"ledge.esi\")\nlocal range = require(\"ledge.range\")\n\nlocal ngx_log = ng"
},
{
"path": "lib/ledge/state_machine.lua",
"chars": 3154,
"preview": "local events = require(\"ledge.state_machine.events\")\nlocal pre_transitions = require(\"ledge.state_machine.pre_transition"
},
{
"path": "lib/ledge/storage/redis.lua",
"chars": 10544,
"preview": "local redis_connector = require \"resty.redis.connector\"\n\nlocal tostring, pairs, next, unpack, setmetatable =\n tostr"
},
{
"path": "lib/ledge/util.lua",
"chars": 6732,
"preview": "local ngx_var = ngx.var\nlocal ffi = require \"ffi\"\n\nlocal type, next, setmetatable, getmetatable, error, tostring =\n "
},
{
"path": "lib/ledge/validation.lua",
"chars": 2470,
"preview": "local ngx_req_get_headers = ngx.req.get_headers\nlocal ngx_req_set_header = ngx.req.set_header\nlocal ngx_parse_http_time "
},
{
"path": "lib/ledge/worker.lua",
"chars": 2221,
"preview": "local setmetatable = setmetatable\nlocal co_yield = coroutine.yield\n\nlocal ngx_get_phase = ngx.get_phase\n\nlocal tbl_copy_"
},
{
"path": "lib/ledge.lua",
"chars": 6694,
"preview": "local setmetatable, require =\n setmetatable, require\n\nlocal ngx_get_phase = ngx.get_phase\nlocal ngx_null = ngx.null\n\n"
},
{
"path": "migrations/1.26-1.27.lua",
"chars": 7624,
"preview": "local redis_connector = require(\"resty.redis.connector\").new()\nlocal math_floor = math.floor\nlocal math_ceil = math.ceil"
},
{
"path": "t/01-unit/cache_key.t",
"chars": 20770,
"preview": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::h"
},
{
"path": "t/01-unit/esi.t",
"chars": 5504,
"preview": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::h"
},
{
"path": "t/01-unit/events.t",
"chars": 3984,
"preview": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::h"
},
{
"path": "t/01-unit/handler.t",
"chars": 8898,
"preview": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::h"
},
{
"path": "t/01-unit/jobs.t",
"chars": 11068,
"preview": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::h"
},
{
"path": "t/01-unit/ledge.t",
"chars": 7861,
"preview": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::h"
},
{
"path": "t/01-unit/processor_1_0.t",
"chars": 13628,
"preview": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::h"
},
{
"path": "t/01-unit/purge.t",
"chars": 13918,
"preview": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::h"
},
{
"path": "t/01-unit/range.t",
"chars": 9813,
"preview": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::h"
},
{
"path": "t/01-unit/request.t",
"chars": 3634,
"preview": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::h"
},
{
"path": "t/01-unit/response.t",
"chars": 13445,
"preview": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::h"
},
{
"path": "t/01-unit/stale.t",
"chars": 3564,
"preview": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::h"
},
{
"path": "t/01-unit/state_machine.t",
"chars": 5411,
"preview": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::h"
},
{
"path": "t/01-unit/storage.t",
"chars": 24105,
"preview": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::h"
},
{
"path": "t/01-unit/tag_parser.t",
"chars": 9762,
"preview": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::h"
},
{
"path": "t/01-unit/util.t",
"chars": 12192,
"preview": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::h"
},
{
"path": "t/01-unit/validation.t",
"chars": 3504,
"preview": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::h"
},
{
"path": "t/01-unit/worker.t",
"chars": 2180,
"preview": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::h"
},
{
"path": "t/02-integration/age.t",
"chars": 1130,
"preview": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::h"
},
{
"path": "t/02-integration/cache.t",
"chars": 19396,
"preview": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::h"
},
{
"path": "t/02-integration/collapsed_forwarding.t",
"chars": 13051,
"preview": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::h"
},
{
"path": "t/02-integration/esi.t",
"chars": 73229,
"preview": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::h"
},
{
"path": "t/02-integration/events.t",
"chars": 1259,
"preview": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::h"
},
{
"path": "t/02-integration/gc.t",
"chars": 4497,
"preview": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::h"
},
{
"path": "t/02-integration/gzip.t",
"chars": 4137,
"preview": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::h"
},
{
"path": "t/02-integration/hop_by_hop_headers.t",
"chars": 1112,
"preview": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::h"
},
{
"path": "t/02-integration/max-stale.t",
"chars": 4857,
"preview": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::h"
},
{
"path": "t/02-integration/max_size.t",
"chars": 4083,
"preview": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::h"
},
{
"path": "t/02-integration/memory_pressure.t",
"chars": 7959,
"preview": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::h"
},
{
"path": "t/02-integration/multiple_headers.t",
"chars": 2696,
"preview": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::h"
},
{
"path": "t/02-integration/on_abort.t",
"chars": 6203,
"preview": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::h"
},
{
"path": "t/02-integration/origin_mode.t",
"chars": 3597,
"preview": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::h"
},
{
"path": "t/02-integration/purge.t",
"chars": 32063,
"preview": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::h"
},
{
"path": "t/02-integration/range.t",
"chars": 20736,
"preview": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::h"
},
{
"path": "t/02-integration/req_body.t",
"chars": 614,
"preview": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::h"
},
{
"path": "t/02-integration/req_method.t",
"chars": 2977,
"preview": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::h"
},
{
"path": "t/02-integration/request_leak.t",
"chars": 2755,
"preview": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::h"
},
{
"path": "t/02-integration/response.t",
"chars": 3841,
"preview": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::h"
},
{
"path": "t/02-integration/ssl.t",
"chars": 8504,
"preview": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\n$ENV{TEST_NGINX_HTML_DIR} ||="
},
{
"path": "t/02-integration/stale-if-error.t",
"chars": 6452,
"preview": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::h"
},
{
"path": "t/02-integration/stale-while-revalidate.t",
"chars": 15205,
"preview": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::h"
},
{
"path": "t/02-integration/upstream.t",
"chars": 2189,
"preview": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\n$ENV{TEST_NGINX_HTML_DIR} ||="
},
{
"path": "t/02-integration/upstream_client.t",
"chars": 3533,
"preview": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::h"
},
{
"path": "t/02-integration/validation.t",
"chars": 11596,
"preview": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::h"
},
{
"path": "t/02-integration/vary.t",
"chars": 15441,
"preview": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::h"
},
{
"path": "t/02-integration/via_header.t",
"chars": 2557,
"preview": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::h"
},
{
"path": "t/03-sentinel/01-master_up.t",
"chars": 3222,
"preview": "use Test::Nginx::Socket 'no_plan';\nuse Cwd qw(cwd);\n\nmy $pwd = cwd();\n\n$ENV{TEST_NGINX_PORT} ||= 1984;\n$ENV{TEST_LEDGE_R"
},
{
"path": "t/03-sentinel/02-master_down.t",
"chars": 3024,
"preview": "use Test::Nginx::Socket 'no_plan';\nuse Cwd qw(cwd);\n\nmy $pwd = cwd();\n\n$ENV{TEST_NGINX_PORT} ||= 1984;\n$ENV{TEST_LEDGE_R"
},
{
"path": "t/03-sentinel/03-slave_promoted.t",
"chars": 2531,
"preview": "use Test::Nginx::Socket 'no_plan';\nuse Cwd qw(cwd);\n\nmy $pwd = cwd();\n\n$ENV{TEST_NGINX_PORT} ||= 1984;\n$ENV{TEST_LEDGE_R"
},
{
"path": "t/LedgeEnv.pm",
"chars": 1910,
"preview": "package LedgeEnv;\nuse strict;\nuse warnings;\nuse Exporter;\n\nour $nginx_host = $ENV{TEST_NGINX_HOST} || '127.0.0.1';\nour $"
},
{
"path": "t/cert/example.com.crt",
"chars": 1237,
"preview": "-----BEGIN CERTIFICATE-----\nMIIDZDCCAkwCCQC9pPAJEKdAJTANBgkqhkiG9w0BAQsFADCBiTELMAkGA1UEBhMC\nVUsxDzANBgNVBAgMBkxvbmRvbjE"
},
{
"path": "t/cert/example.com.key",
"chars": 1679,
"preview": "-----BEGIN RSA PRIVATE KEY-----\nMIIEpQIBAAKCAQEA645crvWzzHpjJKn3tH4vh2i317GGjILics2jiiH9dUQBxuD7\nZd1x5OGujqykdzBbgjJxpjr"
},
{
"path": "t/cert/rootCA.key",
"chars": 1679,
"preview": "-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEAy6v3eQFGXrSe97M/9ullfby4bg2jF+9h9p5P/wiv+Y9dTdk8\nDRR7M40laCqnmoXSSndAvHL"
},
{
"path": "t/cert/rootCA.pem",
"chars": 1298,
"preview": "-----BEGIN CERTIFICATE-----\nMIIDkjCCAnoCCQCRNvMmzZMQezANBgkqhkiG9w0BAQsFADCBiTELMAkGA1UEBhMC\nVUsxDzANBgNVBAgMBkxvbmRvbjE"
},
{
"path": "t/cert/rootCA.srl",
"chars": 17,
"preview": "BDA4F00910A74025\n"
},
{
"path": "util/lua-releng",
"chars": 2996,
"preview": "#!/usr/bin/env perl\n\nuse strict;\nuse warnings;\n\nuse Getopt::Std;\n\nmy (@luas, @tests);\n\nmy %opts;\ngetopts('Lse', \\%opts) "
}
]
About this extraction
This page contains the full source code of the pintsized/ledge GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 94 files (733.6 KB), approximately 199.1k tokens. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.