[ { "path": ".gitattributes", "content": "*.t linguist-language=lua\n" }, { "path": ".github/FUNDING.yml", "content": "github: pintsized\n" }, { "path": ".gitignore", "content": "t/servroot/\nt/error.log\ndump.rdb\nstdout\nluacov.*\n*.src.rock\n" }, { "path": ".luacheckrc", "content": "std = \"ngx_lua\"\nredefined = false\n" }, { "path": ".luacov", "content": "modules = {\n [\"ledge\"] = \"lib/ledge.lua\",\n [\"ledge.esi.*\"] = \"lib/\",\n [\"ledge.jobs.*\"] = \"lib/\",\n [\"ledge.state_machine.*\"] = \"lib/\",\n [\"ledge.storage.*\"] = \"lib/\",\n [\"ledge.*\"] = \"lib/\"\n}\n" }, { "path": ".travis.yml", "content": "services:\n - docker\n\nscript:\n - cd docker/tests\n - docker-compose run --rm runner\n" }, { "path": "Makefile", "content": "SHELL := /bin/bash # Cheat by using bash :)\n\nOPENRESTY_PREFIX = /usr/local/openresty\n\nTEST_FILE ?= t/01-unit t/02-integration\nSENTINEL_TEST_FILE ?= t/03-sentinel\n\nTEST_LEDGE_REDIS_HOST ?= 127.0.0.1\nTEST_LEDGE_REDIS_PORT ?= 6379\nTEST_LEDGE_REDIS_DATABASE ?= 2\nTEST_LEDGE_REDIS_QLESS_DATABASE ?= 3\n\nTEST_NGINX_HOST ?= 127.0.0.1\n\n# Command line arguments for ledge tests\nTEST_LEDGE_REDIS_VARS = PATH=$(OPENRESTY_PREFIX)/nginx/sbin:$(PATH) \\\nTEST_LEDGE_REDIS_HOST=$(TEST_LEDGE_REDIS_HOST) \\\nTEST_LEDGE_REDIS_PORT=$(TEST_LEDGE_REDIS_PORT) \\\nTEST_LEDGE_REDIS_SOCKET=unix://$(TEST_LEDGE_REDIS_SOCKET) \\\nTEST_LEDGE_REDIS_DATABASE=$(TEST_LEDGE_REDIS_DATABASE) \\\nTEST_LEDGE_REDIS_QLESS_DATABASE=$(TEST_LEDGE_REDIS_QLESS_DATABASE) \\\nTEST_NGINX_HOST=$(TEST_NGINX_HOST) \\\nTEST_NGINX_NO_SHUFFLE=1\n\nREDIS_CLI := redis-cli -h $(TEST_LEDGE_REDIS_HOST) -p $(TEST_LEDGE_REDIS_PORT)\n\n###############################################################################\n# Deprecated, ues docker copose to run Redis instead\n###############################################################################\nREDIS_CMD = redis-server\nSENTINEL_CMD = $(REDIS_CMD) --sentinel\n\nREDIS_SOCK = /redis.sock\nREDIS_PID = /redis.pid\nREDIS_LOG = /redis.log\nREDIS_PREFIX = /tmp/redis-\n\n# Overrideable ledge test variables\nTEST_LEDGE_REDIS_PORTS ?= 6379 6380\n\nREDIS_FIRST_PORT := $(firstword $(TEST_LEDGE_REDIS_PORTS))\nREDIS_SLAVE_ARG := --slaveof 127.0.0.1 $(REDIS_FIRST_PORT)\n\n# Override ledge socket for running make test on its' own\n# (make test TEST_LEDGE_REDIS_SOCKET=/path/to/sock.sock)\nTEST_LEDGE_REDIS_SOCKET ?= $(REDIS_PREFIX)$(REDIS_FIRST_PORT)$(REDIS_SOCK)\n\n# Overrideable ledge + sentinel test variables\nTEST_LEDGE_SENTINEL_PORTS ?= 26379 26380 26381\nTEST_LEDGE_SENTINEL_MASTER_NAME ?= mymaster\nTEST_LEDGE_SENTINEL_PROMOTION_TIME ?= 20\n\n# Command line arguments for ledge + sentinel tests\nTEST_LEDGE_SENTINEL_VARS = PATH=$(OPENRESTY_PREFIX)/nginx/sbin:$(PATH) \\\nTEST_LEDGE_SENTINEL_PORT=$(firstword $(TEST_LEDGE_SENTINEL_PORTS)) \\\nTEST_LEDGE_SENTINEL_MASTER_NAME=$(TEST_LEDGE_SENTINEL_MASTER_NAME) \\\nTEST_LEDGE_REDIS_DATABASE=$(TEST_LEDGE_REDIS_DATABASE) \\\nTEST_NGINX_NO_SHUFFLE=1\n\n# Sentinel configuration can only be set by a config file\ndefine TEST_LEDGE_SENTINEL_CONFIG\nsentinel monitor $(TEST_LEDGE_SENTINEL_MASTER_NAME) 127.0.0.1 $(REDIS_FIRST_PORT) 2\nsentinel down-after-milliseconds $(TEST_LEDGE_SENTINEL_MASTER_NAME) 2000\nsentinel failover-timeout $(TEST_LEDGE_SENTINEL_MASTER_NAME) 10000\nsentinel parallel-syncs $(TEST_LEDGE_SENTINEL_MASTER_NAME) 5\nendef\n\nexport TEST_LEDGE_SENTINEL_CONFIG\n\nSENTINEL_CONFIG_PREFIX = /tmp/sentinel\n\n\n\n###############################################################################\n\n\nPREFIX ?= /usr/local\nLUA_INCLUDE_DIR ?= $(PREFIX)/include\nLUA_LIB_DIR ?= $(PREFIX)/lib/lua/$(LUA_VERSION)\nPROVE ?= prove -I ../test-nginx/lib\nINSTALL ?= install\n\n.PHONY: all install test test_all start_redis_instances stop_redis_instances \\\n\tstart_redis_instance stop_redis_instance cleanup_redis_instance flush_db \\\n\tcheck_ports test_ledge test_sentinel coverage delete_sentinel_config check\n\nall: ;\n\ninstall: all\n\t$(INSTALL) -d $(DESTDIR)/$(LUA_LIB_DIR)/ledge\n\t$(INSTALL) lib/ledge/*.lua $(DESTDIR)/$(LUA_LIB_DIR)/ledge\n\ntest: test_ledge\ntest_all: start_redis_instances test_ledge test_sentinel stop_redis_instances\n\n\n###############################################################################\n# Deprecated, ues docker copose to run Redis instead\n##############################################################################\nstart_redis_instances: check_ports\n\t@$(foreach port,$(TEST_LEDGE_REDIS_PORTS), \\\n\t\t[[ \"$(port)\" != \"$(REDIS_FIRST_PORT)\" ]] && \\\n\t\t\tSLAVE=\"$(REDIS_SLAVE_ARG)\" || \\\n\t\t\tSLAVE=\"\" && \\\n\t\t$(MAKE) start_redis_instance args=\"$$SLAVE\" port=$(port) \\\n\t\tprefix=$(REDIS_PREFIX)$(port) && \\\n\t) true\n\n\t@$(foreach port,$(TEST_LEDGE_SENTINEL_PORTS), \\\n\t\techo \"port $(port)\" > \"$(SENTINEL_CONFIG_PREFIX)-$(port).conf\"; \\\n\t\techo \"$$TEST_LEDGE_SENTINEL_CONFIG\" >> \"$(SENTINEL_CONFIG_PREFIX)-$(port).conf\"; \\\n\t\t$(MAKE) start_redis_instance \\\n\t\tport=$(port) args=\"$(SENTINEL_CONFIG_PREFIX)-$(port).conf --sentinel\" \\\n\t\tprefix=$(REDIS_PREFIX)$(port) && \\\n\t) true\n\nstop_redis_instances: delete_sentinel_config\n\t-@$(foreach port,$(TEST_LEDGE_REDIS_PORTS) $(TEST_LEDGE_SENTINEL_PORTS), \\\n\t\t$(MAKE) stop_redis_instance cleanup_redis_instance port=$(port) \\\n\t\tprefix=$(REDIS_PREFIX)$(port) && \\\n\t) true 2>&1 > /dev/null\n\nstart_redis_instance:\n\t-@echo \"Starting redis on port $(port) with args: \\\"$(args)\\\"\"\n\t-@mkdir -p $(prefix)\n\t@$(REDIS_CMD) $(args) \\\n\t\t--pidfile $(prefix)$(REDIS_PID) \\\n\t\t--bind 127.0.0.1 --port $(port) \\\n\t\t--unixsocket $(prefix)$(REDIS_SOCK) \\\n\t\t--unixsocketperm 777 \\\n\t\t--dir $(prefix) \\\n\t\t--logfile $(prefix)$(REDIS_LOG) \\\n\t\t--loglevel debug \\\n\t\t--daemonize yes\n\nstop_redis_instance:\n\t-@echo \"Stopping redis on port $(port)\"\n\t-@[[ -f \"$(prefix)$(REDIS_PID)\" ]] && kill -QUIT \\\n\t`cat $(prefix)$(REDIS_PID)` 2>&1 > /dev/null || true\n\ncleanup_redis_instance: stop_redis_instance\n\t-@echo \"Cleaning up redis files in $(prefix)\"\n\t-@rm -rf $(prefix)\n\ndelete_sentinel_config:\n\t-@echo \"Cleaning up sentinel config files\"\n\t-@rm -f $(SENTINEL_CONFIG_PREFIX)-*.conf\n\ncheck_ports:\n\t-@echo \"Checking ports $(REDIS_PORTS)\"\n\t@$(foreach port,$(REDIS_PORTS),! lsof -i :$(port) &&) true 2>&1 > /dev/null\n###############################################################################\n\nreleng:\n\t@util/lua-releng -eL\n\nflush_db:\n\t@$(REDIS_CLI) flushall\n\ntest_ledge: releng flush_db\n\t@$(TEST_LEDGE_REDIS_VARS) $(PROVE) $(TEST_FILE)\n\t-@echo \"Qless errors:\"\n\t@$(REDIS_CLI) -n $(TEST_LEDGE_REDIS_QLESS_DATABASE) llen ql:f:job-error\n\ntest_sentinel: releng flush_db\n\t$(TEST_LEDGE_SENTINEL_VARS) $(PROVE) $(SENTINEL_TEST_FILE)/01-master_up.t\n\t$(REDIS_CLI) shutdown\n\t$(TEST_LEDGE_SENTINEL_VARS) $(PROVE) $(SENTINEL_TEST_FILE)/02-master_down.t\n\tsleep $(TEST_LEDGE_SENTINEL_PROMOTION_TIME)\n\t$(TEST_LEDGE_SENTINEL_VARS) $(PROVE) $(SENTINEL_TEST_FILE)/03-slave_promoted.t\n\ntest_leak: releng flush_db\n\t$(TEST_LEDGE_REDIS_VARS) TEST_NGINX_CHECK_LEAK=1 $(PROVE) $(TEST_FILE)\n\ncoverage: releng flush_db\n\t@rm -f luacov.stats.out\n\t@$(TEST_LEDGE_REDIS_VARS) TEST_COVERAGE=1 $(PROVE) $(TEST_FILE)\n\t@luacov\n\t@tail -30 luacov.report.out\n\t-@echo \"Qless errors:\"\n\t@$(REDIS_CLI) -n $(TEST_LEDGE_REDIS_QLESS_DATABASE) llen ql:f:job-error\n\ncheck:\n\tluacheck lib\n" }, { "path": "README.md", "content": "# Ledge\n\n[![Build Status](https://travis-ci.org/ledgetech/ledge.svg?branch=master)](https://travis-ci.org/ledgetech/ledge)\n\nAn RFC compliant and [ESI](https://www.w3.org/TR/esi-lang) capable HTTP cache for [Nginx](http://nginx.org) / [OpenResty](https://openresty.org), backed by [Redis](http://redis.io).\n\nLedge can be utilised as a fast, robust and scalable alternative to Squid / Varnish etc, either installed standalone or integrated into an existing Nginx server or load balancer.\n\nMoreover, it is particularly suited to applications where the origin is expensive or distant, making it desirable to serve from cache as optimistically as possible.\n\n\n## Table of Contents\n\n* [Installation](#installation)\n* [Philosophy and Nomenclature](#philosophy-and-nomenclature)\n * [Cache keys](#cache-keys)\n * [Streaming design](#streaming-design)\n * [Collapsed forwarding](#collapsed-forwarding)\n * [Advanced cache patterns](#advanced-cache-patterns)\n* [Minimal configuration](#minimal-configuration)\n* [Config systems](#config-systems)\n* [Events system](#events-system)\n* [Caching basics](#caching-basics)\n* [Purging](#purging)\n* [Serving stale](#serving-stale)\n* [Edge Side Includes](#edge-side-includes)\n* [API](#api)\n * [ledge.configure](#ledgeconfigure)\n * [ledge.set_handler_defaults](#ledgeset_handler_defaults)\n * [ledge.create\\_handler](#ledgecreate_handler)\n * [ledge.create\\_worker](#ledgecreate_worker)\n * [ledge.bind](#ledgebind)\n * [handler.bind](#handlerbind)\n * [handler.run](#handlerrun)\n * [worker.run](#workerrun)\n* [Handler configuration options](#handler-configuration-options)\n* [Events](#events)\n* [Administration](#administration)\n * [Managing Qless](#managing-qless)\n* [Licence](#licence)\n\n\n## Installation\n\n[OpenResty](http://openresty.org/) is a superset of [Nginx](http://nginx.org), bundling [LuaJIT](http://luajit.org/) and the [lua-nginx-module](https://github.com/openresty/lua-nginx-module) as well as many other things. Whilst it is possible to build all of these things into Nginx yourself, we recommend using the latest OpenResty.\n\n\n### 1. Download and install:\n\n* [OpenResty](http://openresty.org/) >= 1.11.x\n* [Redis](http://redis.io/download) >= 2.8.x\n* [LuaRocks](https://luarocks.org/)\n\n\n### 2. Install Ledge using LuaRocks:\n\n```\nluarocks install ledge\n```\n\nThis will install the latest stable release, and all other Lua module dependencies, which if installing manually without LuaRocks are:\n\n* [lua-resty-http](https://github.com/pintsized/lua-resty-http)\n* [lua-resty-redis-connector](https://github.com/pintsized/lua-resty-redis-connector)\n* [lua-resty-qless](https://github.com/pintsized/lua-resty-qless)\n* [lua-resty-cookie](https://github.com/cloudflare/lua-resty-cookie)\n* [lua-ffi-zlib](https://github.com/hamishforbes/lua-ffi-zlib)\n* [lua-resty-upstream](https://github.com/hamishforbes/lua-resty-upstream) *(optional, for load balancing / healthchecking upstreams)*\n\n\n### 3. Review OpenResty documentation\n\nIf you are new to OpenResty, it's quite important to review the [lua-nginx-module](https://github.com/openresty/lua-nginx-module) documentation on how to run Lua code in Nginx, as the environment is unusual. Specifcally, it's useful to understand the meaning of the different Nginx phase hooks such as `init_by_lua` and `content_by_lua`, as well as how the `lua-nginx-module` locates Lua modules with the [lua_package_path](https://github.com/openresty/lua-nginx-module#lua_package_path) directive.\n\n[Back to TOC](#table-of-contents)\n\n\n## Philosophy and Nomenclature\n\nThe central module is called `ledge`, and provides factory methods for creating `handler` instances (for handling a request) and `worker` instances (for running background tasks). The `ledge` module is also where global configuration is managed.\n\nA `handler` is short lived. It is typically created at the beginning of the Nginx `content` phase for a request, and when its [run()](#handlerrun) method is called, takes responsibility for processing the current request and delivering a response. When [run()](#handlerrun) has completed, HTTP status, headers and body will have been delivered to the client.\n\nA `worker` is long lived, and there is one per Nginx worker process. It is created when Nginx starts a worker process, and dies when the Nginx worker dies. The `worker` pops queued background jobs and processes them.\n\nAn `upstream` is the only thing which must be manually configured, and points to another HTTP host where actual content lives. Typically one would use DNS to resolve client connections to the Nginx server running Ledge, and tell Ledge where to fetch from with the `upstream` configuration. As such, Ledge isn't designed to work as a forwarding proxy.\n\n[Redis](http://redis.io) is used for much more than cache storage. We rely heavily on its data structures to maintain cache `metadata`, as well as embedded Lua scripts for atomic task management and so on. By default, all cache body data and `metadata` will be stored in the same Redis instance. The location of cache `metadata` is global, set when Nginx starts up.\n\nCache body data is handled by the `storage` system, and as mentioned, by default shares the same Redis instance as the `metadata`. However, `storage` is abstracted via a [driver system](#storage_driver) making it possible to store cache body data in a separate Redis instance, or a group of horizontally scalable Redis instances via a [proxy](https://github.com/twitter/twemproxy), or to roll your own `storage` driver, for example targeting PostreSQL or even simply a filesystem. It's perhaps important to consider that by default all cache storage uses Redis, and as such is bound by system memory.\n\n[Back to TOC](#table-of-contents)\n\n### Cache keys\n\nA goal of any caching system is to safely maximise the HIT potential. That is, normalise factors which would split the cache wherever possible, in order to share as much cache as possible.\n\nThis is tricky to generalise, and so by default Ledge puts sane defaults from the request URI into the cache key, and provides a means for this to be customised by altering the [cache\\_key\\_spec](#cache_key_spec).\n\nURI arguments are sorted alphabetically by default, so `http://example.com?a=1&b=2` would hit the same cache entry as `http://example.com?b=2&a=1`.\n\n[Back to TOC](#table-of-contents)\n\n### Streaming design\n\nHTTP response sizes can be wildly different, sometimes tiny and sometimes huge, and it's not always possible to know the total size up front.\n\nTo guarantee predictable memory usage regardless of response sizes Ledge operates a streaming design, meaning it only ever operates on a single `buffer` per request at a time. This is equally true when fetching upstream to when reading from cache or serving to the client request.\n\nIt's also true (mostly) when processing [ESI](#edge-size-includes) instructions, except for in the case where an instruction is found to span multiple buffers. In this case, we continue buffering until a complete instruction can be understood, up to a [configurable limit](#esi_max_size).\n\nThis streaming design also improves latency, since we start serving the first `buffer` to the client request as soon as we're done with it, rather than fetching and saving an entire resource prior to serving. The `buffer` size can be [tuned](#buffer_size) even on a per `location` basis.\n\n[Back to TOC](#table-of-contents)\n\n### Collapsed forwarding\n\nLedge can attempt to collapse concurrent origin requests for known (previously) cacheable resources into a single upstream request. That is, if an upstream request for a resource is in progress, subsequent concurrent requests for the same resource will not bother the upstream, and instead wait for the first request to finish.\n\nThis is particularly useful to reduce upstream load if a spike of traffic occurs for expired and expensive content (since the chances of concurrent requests is higher on slower content).\n\n[Back to TOC](#table-of-contents)\n\n### Advanced cache patterns\n\nBeyond standard RFC compliant cache behaviours, Ledge has many features designed to maximise cache HIT rates and to reduce latency for requests. See the sections on [Edge Side Includes](#edge-side-includes), [serving stale](#serving-stale) and [revalidating on purge](#purging) for more information.\n\n[Back to TOC](#table-of-contents)\n\n\n## Minimal configuration\n\nAssuming you have Redis running on `localhost:6379`, and your upstream is at `localhost:8080`, add the following to the `nginx.conf` file in your OpenResty installation.\n\n```lua\nhttp {\n if_modified_since Off;\n lua_check_client_abort On;\n\n init_by_lua_block {\n require(\"ledge\").configure({\n redis_connector_params = {\n url = \"redis://127.0.0.1:6379/0\",\n },\n })\n\n require(\"ledge\").set_handler_defaults({\n upstream_host = \"127.0.0.1\",\n upstream_port = 8080,\n })\n }\n\n init_worker_by_lua_block {\n require(\"ledge\").create_worker():run()\n }\n\n server {\n server_name example.com;\n listen 80;\n\n location / {\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n }\n }\n}\n```\n\n[Back to TOC](#table-of-contents)\n\n\n## Config systems\n\nThere are four different layers to the configuration system. Firstly there is the main [Redis config](#ledgeconfigure) and [handler defaults](#ledgeset_handler_defaults) config, which are global and must be set during the Nginx `init` phase.\n\nBeyond this, you can specify [handler instance config](#ledgecreate_handler) on an Nginx `location` block basis, and finally there are some performance tuning config options for the [worker](#ledgecreate_worker) instances.\n\nIn addition, there is an [events system](#events-system) for binding Lua functions to mid-request events, proving opportunities to dynamically alter configuration.\n\n[Back to TOC](#table-of-contents)\n\n\n## Events system\n\nLedge makes most of its decisions based on the content it is working with. HTTP request and response headers drive the semantics for content delivery, and so rather than having countless configuration options to change this, we instead provide opportunities to alter the given semantics when necessary.\n\nFor example, if an `upstream` fails to set a long enough cache expiry, rather than inventing an option such as \"extend\\_ttl\", we instead would `bind` to the `after_upstream_request` event, and adjust the response headers to include the ttl we're hoping for.\n\n```lua\nhandler:bind(\"after_upstream_request\", function(res)\n res.header[\"Cache-Control\"] = \"max-age=86400\"\nend)\n```\n\nThis particular event fires after we've fetched upstream, but before Ledge makes any decisions about whether the content can be cached or not. Once we've adjustead our headers, Ledge will read them as if they came from the upstream itself.\n\nNote that multiple functions can be bound to a single event, either globally or per handler, and they will be called in the order they were bound. There is also currently no means to inspect which functions have been bound, or to unbind them.\n\nSee the [events](#events) section for a complete list of events and their definitions.\n\n[Back to TOC](#table-of-contents)\n\n### Binding globally\n\nBinding a function globally means it will fire for the given event, on all requests. This is perhaps useful if you have many different `location` blocks, but need to always perform the same logic.\n\n```lua\ninit_by_lua_block {\n require(\"ledge\").bind(\"before_serve\", function(res)\n res.header[\"X-Foo\"] = \"bar\" -- always set X-Foo to bar\n end)\n}\n```\n\n[Back to TOC](#table-of-contents)\n\n### Binding to handlers\n\nMore commonly, we just want to alter behaviour for a given Nginx `location`.\n\n```lua\nlocation /foo_location {\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n\n handler:bind(\"before_serve\", function(res)\n res.header[\"X-Foo\"] = \"bar\" -- only set X-Foo for this location\n end)\n\n handler:run()\n }\n}\n```\n\n[Back to TOC](#table-of-contents)\n\n### Performance implications\n\nWriting simple logic for events is not expensive at all (and in many cases will be JIT compiled). If you need to consult service endpoints during an event then obviously consider that this will affect your overall latency, and make sure you do everything in a **non-blocking** way, e.g. using [cosockets](https://github.com/openresty/lua-nginx-module#ngxsockettcp) provided by OpenResty, or a driver based upon this.\n\nIf you have lots of event handlers, consider that creating closures in Lua is relatively expensive. A good solution would be to make your own module, and pass the defined functions in.\n\n```lua\nlocation /foo_location {\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n handler:bind(\"before_serve\", require(\"my.handler.hooks\").add_foo_header)\n handler:run()\n }\n}\n```\n\n[Back to TOC](#table-of-contents)\n\n\n## Caching basics\n\nFor normal HTTP caching operation, no additional configuration is required. If the HTTP response indicates the resource can be cached, then it will cache it. If the HTTP request indicates it accepts cache, it will be served cache. Note that these two conditions aren't mutually exclusive - a request could specify `no-cache`, and this will indeed trigger a fetch upstream, but if the response is cacheable then it will be saved and served to subsequent cache-accepting requests.\n\nFor more information on the myriad factors affecting this, including end-to-end revalidation and so on, please refer to [RFC 7234](https://tools.ietf.org/html/rfc7234).\n\nThe goal is to be 100% RFC compliant, but with some extensions to allow more agressive caching in certain cases. If something doesn't work as you expect, please do feel free to [raise an issue](https://github.com/pintsized/ledge).\n\n[Back to TOC](#table-of-contents)\n\n\n## Purging\n\nTo manually invalidate a cache item (or purge), we support the non-standard `PURGE` method familiar to users of Squid. Send a HTTP request to the URI with the method set, and Ledge will attempt to invalidate the item, returning status `200` on success and `404` if the URI was not found in cache, along with a JSON body for more details.\n\nA purge request will affect all representations associated with the cache key, for example compressed and uncompressed responses separated by the `Vary: Accept-Encoding` response header will all be purged.\n\n`$> curl -X PURGE -H \"Host: example.com\" http://cache.example.com/page1 | jq .`\n\n```json\n{\n \"purge_mode\": \"invalidate\",\n \"result\": \"nothing to purge\"\n}\n```\n\nThere are three purge modes, selectable by setting the `X-Purge` request header with one or more of the following values:\n\n* `invalidate`: (default) marks the item as expired, but doesn't delete anything.\n* `delete`: hard removes the item from cache\n* `revalidate`: invalidates but then schedules a background revalidation to re-prime the cache.\n\n`$> curl -X PURGE -H \"X-Purge: revalidate\" -H \"Host: example.com\" http://cache.example.com/page1 | jq .`\n\n```json\n{\n \"purge_mode\": \"revalidate\",\n \"qless_job\": {\n \"options\": {\n \"priority\": 4,\n \"jid\": \"5eeabecdc75571d1b93e9c942dfcebcb\",\n \"tags\": [\n \"revalidate\"\n ]\n },\n \"jid\": \"5eeabecdc75571d1b93e9c942dfcebcb\",\n \"klass\": \"ledge.jobs.revalidate\"\n },\n \"result\": \"already expired\"\n}\n```\n\nBackground revalidation jobs can be tracked in the qless metadata. See [managing qless](#managing-qless) for more information.\n\nIn general, `PURGE` is considered an administration task and probably shouldn't be allowed from the internet. Consider limiting it by IP address for example:\n\n```nginx\nlimit_except GET POST PUT DELETE {\n allow 127.0.0.1;\n deny all;\n}\n```\n\n[Back to TOC](#table-of-contents)\n\n### JSON API\n\nA JSON based API is also available for purging cache multiple cache items at once.\nThis requires a `PURGE` request with a `Content-Type` header set to `application/json` and a valid JSON request body.\n\nValid parameters\n * `uris` - Array of URIs to purge, can contain wildcard URIs\n * `purge_mode` - As the `X-Purge` header in a normal purge request\n * `headers` - Hash of additional headers to include in the purge request\n\nReturns a results hash keyed by URI or a JSON error response\n\n`$> curl -X PURGE -H \"Content-Type: Application/JSON\" http://cache.example.com/ -d '{\"uris\": [\"http://www.example.com/1\", \"http://www.example.com/2\"]}' | jq .`\n\n```json\n{\n \"purge_mode\": \"invalidate\",\n \"result\": {\n \"http://www.example.com/1\": {\n \"result\": \"purged\"\n },\n \"http://www.example.com/2\":{\n \"result\": \"nothing to purge\"\n }\n }\n}\n```\n\n[Back to TOC](#table-of-contents)\n\n### Wildcard purging\n\nWildcard (\\*) patterns are also supported in `PURGE` URIs, which will always return a status of `200` and a JSON body detailing a background job. Wildcard purges involve scanning the entire keyspace, and so can take a little while. See [keyspace\\_scan\\_count](#keyspace_scan_count) for tuning help.\n\nIn addition, the `X-Purge` mode will propagate to all URIs purged as a result of the wildcard, making it possible to trigger site / section wide revalidation for example. Be careful what you wish for.\n\n`$> curl -v -X PURGE -H \"X-Purge: revalidate\" -H \"Host: example.com\" http://cache.example.com/* | jq .`\n\n```json\n{\n \"purge_mode\": \"revalidate\",\n \"qless_job\": {\n \"options\": {\n \"priority\": 5,\n \"jid\": \"b2697f7cb2e856cbcad1f16682ee20b0\",\n \"tags\": [\n \"purge\"\n ]\n },\n \"jid\": \"b2697f7cb2e856cbcad1f16682ee20b0\",\n \"klass\": \"ledge.jobs.purge\"\n },\n \"result\": \"scheduled\"\n}\n```\n\n[Back to TOC](#table-of-contents)\n\n\n## Serving stale\n\nContent is considered \"stale\" when its age is beyond its TTL. However, depending on the value of [keep_cache_for](#keep_cache_for) (which defaults to 1 month), we don't actually expire content in Redis straight away.\n\nThis allows us to implement the stale cache control extensions described in [RFC5861](https://tools.ietf.org/html/rfc5861), which provides request and response header semantics for describing how stale something can be served, when it should be revalidated in the background, and how long we can serve stale content in the event of upstream errors.\n\nThis can be very effective in ensuring a fast user experience. For example, if your content has a genuine `max-age` of 24 hours, consider changing this to 1 hour, and adding `stale-while-revalidate` for 23 hours. The total TTL is therefore the same, but the first request after the first hour will trigger backgrounded revalidation, extending the TTL for a further 1 hour + 23 hours.\n\nIf your origin server cannot be configured in this way, you can always override by [binding](#events) to the [before_save](#before_save) event.\n\n```lua\nhandler:bind(\"before_save\", function(res)\n -- Valid for 1 hour, stale-while-revalidate for 23 hours, stale-if-error for three days\n res.header[\"Cache-Control\"] = \"max-age=3600, stale-while-revalidate=82800, stale-if-error=259200\"\nend)\n```\n\nIn other words, set the TTL to the highest comfortable frequency of requests at the origin, and `stale-while-revalidate` to the longest comfortable TTL, to increase the chances of background revalidation occurring. Note that the first stale request will obviously get stale content, and so very long values can result in very out of date content for one request.\n\nAll stale behaviours are constrained by normal cache control semantics. For example, if the origin is down, and the response could be served stale due to the upstream error, but the request contains `Cache-Control: no-cache` or even `Cache-Control: max-age=60` where the content is older than 60 seconds, they will be served the error, rather than the stale content.\n\n[Back to TOC](#table-of-contents)\n\n\n## Edge Side Includes\n\nAlmost complete support for the [ESI 1.0 Language Specification](https://www.w3.org/TR/esi-lang) is included, with a few exceptions, and a few enhancements.\n\n```html\n\n\n\n\n \n \n Hi\n \n \n \n \n \n \n \n \n \n\n\n\n```\n\n[Back to TOC](#table-of-contents)\n\n### Enabling ESI\n\nNote that simply [enabling](#esi_enabled) ESI might not be enough. We also check the [content type](#esi_content_types) against the allowed types specified, but more importantly ESI processing is contingent upon the [Edge Architecture Specification](https://www.w3.org/TR/edge-arch/). When enabled, Ledge will advertise capabilities upstream with the `Surrogate-Capability` request header, and expect the upstream response to include a `Surrogate-Control` header delegating ESI processing to Ledge.\n\nIf your upstream is not ESI aware, a common approach is to bind to the [after\\_upstream\\_request](#after_upstream_request) event in order to add the `Surrogate-Control` header manually. E.g.\n\n```lua\nhandler:bind(\"after_upstream_request\", function(res)\n -- Don't enable ESI on redirect responses\n -- Don't override Surrogate Control if it already exists\n local status = res.status\n if not res.header[\"Surrogate-Control\"] and not (status > 300 and status < 303) then\n res.header[\"Surrogate-Control\"] = 'content=\"ESI/1.0\"'\n end\nend)\n```\n\nNote that if ESI is processed, downstream cache-ability is automatically dropped since you don't want other intermediaries or browsers caching the result.\n\nIt's therefore best to only set `Surrogate-Control` for content which you know has ESI instructions. Whilst Ledge will detect the presence of ESI instructions when saving (and do nothing on cache HITs if no instructions are present), on a cache MISS it will have already dropped downstream cache headers before reading / saving the body. This is a side-effect of the [streaming design](#streaming-design).\n\n[Back to TOC](#table-of-contents)\n\n### Regular expressions in conditions\n\nIn addition to the operators defined in the\n[ESI specification](https://www.w3.org/TR/esi-lang), we also support regular\nexpressions in conditions (as string literals), using the `=~` operator.\n\n```html\n\n \n Hi James or John\n \n\n```\n\nSupported modifiers are as per the [ngx.re.\\*](https://github.com/openresty/lua-nginx-module#ngxrematch) documentation.\n\n[Back to TOC](#table-of-contents)\n\n### Custom ESI variables\n\nIn addition to the variables defined in the [ESI specification](https://www.w3.org/TR/esi-lang), it is possible to provide run time custom variables using the [esi_custom_variables](#esi_custom_variables) handler config option.\n\n```lua\ncontent_by_lua_block {\n require(\"ledge\").create_handler({\n esi_custom_variables = {\n messages = {\n foo = \"bar\",\n },\n },\n }):run()\n}\n```\n\n```html\n$(MESSAGES{foo})\n```\n\n[Back to TOC](#table-of-contents)\n\n### ESI Args\n\nIt can be tempting to use URI arguments to pages using ESI in order to change layout dynamically, but this comes at the cost of generating multiple cache items - one for each permutation of URI arguments.\n\nESI args is a neat feature to get around this, by using a configurable [prefix](#esi_args_prefix), which defaults to `esi_`. URI arguments with this prefix are removed from the cache key and also from upstream requests, and instead stuffed into the `$(ESI_ARGS{foo})` variable for use in ESI, typically in conditions. That is, think of them as magic URI arguments which have meaning for the ESI processor only, and should never affect cacheability or upstream content generation.\n\n`$> curl -H \"Host: example.com\" http://cache.example.com/page1?esi_display_mode=summary`\n\n```html\n\n \n \n \n \n \n \n\n```\n\nIn this example, the `esi_display_mode` values of `summary` or `details` will return the same cache HIT, but display different content.\n\nIf `$(ESI_ARGS)` is used without a field key, it renders the original query string arguments, e.g. `esi_foo=bar&esi_display_mode=summary`, URL encoded.\n\n[Back to TOC](#table-of-contents)\n\n\n### Variable Escaping\n\nESI variables are minimally escaped by default in order to prevent user's injecting additional ESI tags or XSS exploits.\n\nUnescaped variables are available by prefixing the variable name with `RAW_`. This should be used with care.\n\n```html\n# /esi/test.html?a=\n\n$(QUERY_STRING{a}) \n$(RAW_QUERY_STRING{a}) \n\n```\n\n[Back to TOC](#table-of-contents)\n\n### Missing ESI features\n\nThe following parts of the [ESI specification](https://www.w3.org/TR/esi-lang) are not supported, but could be in due course if a need is identified.\n\n* `` not implemented (or advertised as a capability).\n* No support for the `onerror` or `alt` attributes for ``. Instead, we \"continue\" on error by default.\n* `` not implemented.\n* The \"dictionary (special)\" substructure variable type for `HTTP_USER_AGENT` is not implemented.\n\n[Back to TOC](#table-of-contents)\n\n\n## API\n\n### ledge.configure\n\nsyntax: `ledge.configure(config)`\n\nThis function provides Ledge with Redis connection details for all cache `metadata` and background jobs. This is global and cannot be specified or adjusted outside the Nginx `init` phase.\n\n```lua\ninit_by_lua_block {\n require(\"ledge\").configure({\n redis_connector_params = {\n url = \"redis://mypassword@127.0.0.1:6380/3\",\n }\n qless_db = 4,\n })\n}\n```\n\n`config` is a table with the following options (unrecognised config will error hard on start up).\n\n[Back to TOC](#table-of-contents)\n\n\n#### redis_connector_params\n\n`default: {}`\n\nLedge uses [lua-resty-redis-connector](https://github.com/pintsized/lua-resty-redis-connector) to handle all Redis connections. It simply passes anything given in `redis_connector_params` straight to [lua-resty-redis-connector](https://github.com/pintsized/lua-resty-redis-connector), so review the documentation there for options, including how to use [Redis Sentinel](https://redis.io/topics/sentinel).\n\n\n#### qless_db\n\n`default: 1`\n\nSpecifies the Redis DB number to store [qless](https://github.com/pintsized/lua-resty-qless) background job data.\n\n[Back to TOC](#table-of-contents)\n\n\n### ledge.set\\_handler\\_defaults\n\nsyntax: `ledge.set_handler_defaults(config)`\n\nThis method overrides the default configuration used for all spawned request `handler` instances. This is global and cannot be specified or adjusted outside the Nginx `init` phase, but these defaults can be overriden on a per `handler` basis. See [below](#handler-configuration-options) for a complete list of configuration options.\n\n```lua\ninit_by_lua_block {\n require(\"ledge\").set_handler_defaults({\n upstream_host = \"127.0.0.1\",\n upstream_port = 8080,\n })\n}\n```\n\n[Back to TOC](#table-of-contents)\n\n\n### ledge.create\\_handler\n\nsyntax: `local handler = ledge.create_handler(config)`\n\nCreates a `handler` instance for the current reqiest. Config given here will be merged with the defaults, allowing certain options to be adjusted on a per Nginx `location` basis.\n\n```lua\nserver {\n server_name example.com;\n listen 80;\n\n location / {\n content_by_lua_block {\n require(\"ledge\").create_handler({\n upstream_port = 8081,\n }):run()\n }\n }\n}\n```\n\n[Back to TOC](#table-of-contents)\n\n\n### ledge.create\\_worker\n\nsyntax: `local worker = ledge.create_worker(config)`\n\nCreates a `worker` instance inside the current Nginx worker process, for processing background jobs. You only need to call this once inside a single `init_worker` block, and it will be called for each Nginx worker that is configured.\n\nJob queues can be run at varying amounts of concurrency per worker, which can be set by providing `config` here. See [managing qless](#managing-qless) for more details.\n\n```lua\ninit_worker_by_lua_block {\n require(\"ledge\").create_worker({\n interval = 1,\n gc_queue_concurrency = 1,\n purge_queue_concurrency = 2,\n revalidate_queue_concurrency = 5,\n }):run()\n}\n```\n\n[Back to TOC](#table-of-contents)\n\n\n### ledge.bind\n\nsyntax: `ledge.bind(event_name, callback)`\n\nBinds the `callback` function to the event given in `event_name`, globally for all requests on this system. Arguments to `callback` vary based on the event. See [below](#events) for event definitions.\n\n[Back to TOC](#table-of-contents)\n\n\n### handler.bind\n\nsyntax: `handler:bind(event_name, callback)`\n\nBinds the `callback` function to the event given in `event_name` for this handler only. Note the `:` in `handler:bind()` which differs to the global `ledge.bind()`.\n\nArguments to `callback` vary based on the event. See [below](#events) for event definitions.\n\n[Back to TOC](#table-of-contents)\n\n\n### handler.run\n\nsyntax: `handler:run()`\n\nMust be called during the `content_by_lua` phase. It processes the current request and serves a response. If you fail to call this method in your `location` block, nothing will happen.\n\n[Back to TOC](#table-of-contents)\n\n\n### worker.run\n\nsyntax: `handler:run()`\n\nMust be called during the `init_worker` phase, otherwise background tasks will not be run, including garbage collection which is very importatnt.\n\n[Back to TOC](#table-of-contents)\n\n\n### Handler configuration options\n\n* [storage_driver](#storage_driver)\n* [storage_driver_config](#storage_driver_config)\n* [origin_mode](#origin_mode)\n* [upstream_connect_timeout](#upstream_connect_timeout)\n* [upstream_send_timeout](#upstream_send_timeout)\n* [upstream_read_timeout](#upstream_read_timeout)\n* [upstream_keepalive_timeout](#upstream_keepalive_timeout)\n* [upstream_keepalive_poolsize](#upstream_keepalive_poolsize)\n* [upstream_host](#upstream_host)\n* [upstream_port](#upstream_port)\n* [upstream_use_ssl](#upstream_use_ssl)\n* [upstream_ssl_server_name](#upstream_ssl_server_name)\n* [upstream_ssl_verify](#upstream_ssl_verify)\n* [buffer_size](#buffer_size)\n* [advertise_ledge](#buffer_size)\n* [keep_cache_for](#buffer_size)\n* [minimum_old_entity_download_rate](#minimum_old_entity_download_rate)\n* [esi_enabled](#esi_enabled)\n* [esi_content_types](#esi_content_types)\n* [esi_allow_surrogate_delegation](#esi_allow_surrogate_delegation)\n* [esi_recursion_limit](#esi_recursion_limit)\n* [esi_args_prefix](#esi_args_prefix)\n* [esi_custom_variables](#esi_custom_variables)\n* [esi_max_size](#esi_max_size)\n* [esi_attempt_loopback](#esi_attempt_loopback)\n* [esi_vars_cookie_blacklist](#esi_vars_cookie_blacklist)\n* [esi_disable_third_party_includes](#esi_disable_third_party_includes)\n* [esi_third_party_includes_domain_whitelist](#esi_third_party_includes_domain_whitelist)\n* [enable_collapsed_forwarding](#enable_collapsed_forwarding)\n* [collapsed_forwarding_window](#collapsed_forwarding_window)\n* [gunzip_enabled](#gunzip_enabled)\n* [keyspace_scan_count](#keyspace_scan_count)\n* [cache_key_spec](#cache_key_spec)\n* [max_uri_args](#max_uri_args)\n\n\n#### storage_driver\n\ndefault: `ledge.storage.redis`\n\nThis is a `string` value, which will be used to attempt to load a storage driver. Any third party driver here can accept its own config options (see below), but must provide the following interface:\n\n* `bool new()`\n* `bool connect()`\n* `bool close()`\n* `number get_max_size()` *(return nil for no max)*\n* `bool exists(string entity_id)`\n* `bool delete(string entity_id)`\n* `bool set_ttl(string entity_id, number ttl)`\n* `number get_ttl(string entity_id)`\n* `function get_reader(object response)`\n* `function get_writer(object response, number ttl, function onsuccess, function onfailure)`\n\n*Note, whilst it is possible to configure storage drivers on a per `location` basis, it is **strongly** recommended that you never do this, and consider storage drivers to be system wide, much like the main Redis config. If you really need differenet storage driver configurations for different locations, then it will work, but features such as purging using wildcards will silently not work. YMMV.*\n\n[Back to TOC](#handler-configuration-options)\n\n\n#### storage_driver_config\n\n`default: {}`\n\nStorage configuration can vary based on the driver. Currently we only have a Redis driver.\n\n[Back to TOC](#handler-configuration-options)\n\n\n##### Redis storage driver config\n\n* `redis_connector_params` Redis params table, as per [lua-resty-redis-connector](https://github.com/pintsized/lua-resty-redis-connector)\n* `max_size` (bytes), defaults to `1MB`\n* `supports_transactions` defaults to `true`, set to false if using a Redis proxy.\n\nIf `supports_transactions` is set to `false`, cache bodies are not written atomically. However, if there is an error writing, the main Redis system will be notified and the overall transaction will be aborted. The result being potentially orphaned body entities in the storage system, which will hopefully eventually expire. The only reason to turn this off is if you are using a Redis proxy, as any transaction related commands will break the connection.\n\n[Back to TOC](#handler-configuration-options)\n\n\n#### upstream_connect_timeout\n\ndefault: `1000 (ms)`\n\nMaximum time to wait for an upstream connection (in milliseconds). If it is exceeded, we send a `503` status code, unless [stale_if_error](#stale_if_error) is configured.\n\n[Back to TOC](#handler-configuration-options)\n\n\n#### upstream_send_timeout\n\ndefault: `2000 (ms)`\n\nMaximum time to wait sending data on a connected upstream socket (in milliseconds). If it is exceeded, we send a `503` status code, unless [stale_if_error](#stale_if_error) is configured.\n\n[Back to TOC](#handler-configuration-options)\n\n\n#### upstream_read_timeout\n\ndefault: `10000 (ms)`\n\nMaximum time to wait on a connected upstream socket (in milliseconds). If it is exceeded, we send a `503` status code, unless [stale_if_error](#stale_if_error) is configured.\n\n[Back to TOC](#handler-configuration-options)\n\n\n#### upstream_keepalive_timeout\n\ndefault: `75000`\n\n[Back to TOC](#handler-configuration-options)\n\n\n#### upstream_keepalive_poolsize\n\ndefault: `64`\n\n[Back to TOC](#handler-configuration-options)\n\n\n#### upstream_host\n\ndefault: `\"\"`\n\nSpecifies the hostname or IP address of the upstream host. If a hostname is specified, you must configure the Nginx [resolver](http://nginx.org/en/docs/http/ngx_http_core_module.html#resolver) somewhere, for example:\n\n```nginx\nresolver 8.8.8.8;\n```\n\n[Back to TOC](#handler-configuration-options)\n\n\n#### upstream_port\n\ndefault: `80`\n\nSpecifies the port of the upstream host.\n\n[Back to TOC](#handler-configuration-options)\n\n\n#### upstream_use_ssl\n\ndefault: `false`\n\nToggles the use of SSL on the upstream connection. Other `upstream_ssl_*` options will be ignored if this is not set to `true`.\n\n[Back to TOC](#handler-configuration-options)\n\n\n#### upstream_ssl_server_name\n\ndefault: `\"\"`\n\nSpecifies the SSL server name used for Server Name Indication (SNI). See [sslhandshake](https://github.com/openresty/lua-nginx-module#tcpsocksslhandshake) for more information.\n\n[Back to TOC](#handler-configuration-options)\n\n\n#### upstream_ssl_verify\n\ndefault: `true`\n\nToggles SSL verification. See [sslhandshake](https://github.com/openresty/lua-nginx-module#tcpsocksslhandshake) for more information.\n\n[Back to TOC](#handler-configuration-options)\n\n\n#### cache_key_spec\n\n`default: cache_key_spec = { \"scheme\", \"host\", \"uri\", \"args\" },`\n\nSpecifies the format for creating cache keys. The default spec above will create keys in Redis similar to:\n\n```\nledge:cache:http:example.com:/about::\nledge:cache:http:example.com:/about:p=2&q=foo:\n```\n\nThe list of available string identifiers in the spec is:\n\n* `scheme` either http or https\n* `host` the hostname of the current request\n* `port` the public port of the current request\n* `uri` the URI (without args)\n* `args` the URI args, sorted alphabetically\n\nIn addition to these string identifiers, dynamic parameters can be added to the cache key by providing functions. Any functions given must expect no arguments and return a string value.\n\n```lua\nlocal function get_device_type()\n -- dynamically work out device type\n return \"tablet\"\nend\n\nrequire(\"ledge\").create_handler({\n cache_key_spec = {\n get_device_type,\n \"scheme\",\n \"host\",\n \"uri\",\n \"args\",\n }\n}):run()\n```\n\nConsider leveraging vary, via the [before_vary_selection](#before_vary_selection) event, for separating cache entries rather than modifying the main `cache_key_spec` directly.\n\n[Back to TOC](#handler-configuration-options)\n\n\n#### origin_mode\n\ndefault: `ledge.ORIGIN_MODE_NORMAL`\n\nDetermines the overall behaviour for connecting to the origin. `ORIGIN_MODE_NORMAL` will assume the origin is up, and connect as necessary.\n\n`ORIGIN_MODE_AVOID` is similar to Squid's `offline_mode`, where any retained cache (expired or not) will be served rather than trying the origin, regardless of cache-control headers, but the origin will be tried if there is no cache to serve.\n\n`ORIGIN_MODE_BYPASS` is the same as `AVOID`, except if there is no cache to serve we send a `503 Service Unavailable` status code to the client and never attempt an upstream connection.\n\n[Back to TOC](#handler-configuration-options)\n\n\n#### keep_cache_for\n\ndefault: `86400 * 30 (1 month in seconds)`\n\nSpecifies how long to retain cache data past its expiry date. This allows us to serve stale cache in the event of upstream failure with [stale_if_error](#stale_if_error) or [origin_mode](#origin_mode) settings.\n\nItems will be evicted when under memory pressure provided you are using one of the Redis [volatile eviction policies](http://redis.io/topics/lru-cache), so there should generally be no real need to lower this for space reasons.\n\nItems at the extreme end of this (i.e. nearly a month old) are clearly very rarely requested, or more likely, have been removed at the origin.\n\n[Back to TOC](#handler-configuration-options)\n\n\n#### minimum_old_entity_download_rate\n\ndefault: `56 (kbps)`\n\nClients reading slower than this who are also unfortunate enough to have started reading from an entity which has been replaced (due to another client causing a revalidation for example), may have their entity garbage collected before they finish, resulting in an incomplete resource being delivered.\n\nLowering this is fairer on slow clients, but widens the potential window for multiple old entities to stack up, which in turn could threaten Redis storage space and force evictions.\n\n[Back to TOC](#handler-configuration-options)\n\n\n#### enable_collapsed_forwarding\n\ndefault: `false`\n\n[Back to TOC](#handler-configuration-options)\n\n\n#### collapsed_forwarding_window\n\nWhen collapsed forwarding is enabled, if a fatal error occurs during the origin request, the collapsed requests may never receive the response they are waiting for. This setting puts a limit on how long they will wait, and how long before new requests will decide to try the origin for themselves.\n\nIf this is set shorter than your origin takes to respond, then you may get more upstream requests than desired. Fatal errors (server reboot etc) may result in hanging connections for up to the maximum time set. Normal errors (such as upstream timeouts) work independently of this setting.\n\n[Back to TOC](#handler-configuration-options)\n\n\n#### gunzip_enabled\n\ndefault: `true`\n\nWith this enabled, gzipped responses will be uncompressed on the fly for clients that do not set `Accept-Encoding: gzip`. Note that if we receive a gzipped response for a resource containing ESI instructions, we gunzip whilst saving and store uncompressed, since we need to read the ESI instructions.\n\nAlso note that `Range` requests for gzipped content must be ignored - the full response will be returned.\n\n[Back to TOC](#handler-configuration-options)\n\n\n#### buffer_size\n\ndefault: `2^16 (64KB in bytes)`\n\nSpecifies the internal buffer size (in bytes) used for data to be read/written/served. Upstream responses are read in chunks of this maximum size, preventing allocation of large amounts of memory in the event of receiving large files. Data is also stored internally as a list of chunks, and delivered to the Nginx output chain buffers in the same fashion.\n\nThe only exception is if ESI is configured, and Ledge has determined there are ESI instructions to process, and any of these instructions span a given chunk. In this case, buffers are concatenated until a complete instruction is found, and then ESI operates on this new buffer, up to a maximum of [esi_max_size](#esi_max_size).\n\n[Back to TOC](#handler-configuration-options)\n\n\n#### keyspace_scan_count\n\ndefault: `1000`\n\nTunes the behaviour of keyspace scans, which occur when sending a PURGE request with wildcard syntax. A higher number may be better if latency to Redis is high and the keyspace is large.\n\n[Back to TOC](#handler-configuration-options)\n\n\n#### max_uri_args\n\ndefault: `100`\n\nLimits the number of URI arguments returned in calls to [ngx.req.get_uri_args()](https://github.com/openresty/lua-nginx-module#ngxreqget_uri_args), to protect against DOS attacks.\n\n[Back to TOC](#handler-configuration-options)\n\n\n#### esi_enabled\n\ndefault: `false`\n\nToggles [ESI](http://www.w3.org/TR/esi-lang) scanning and processing, though behaviour is also contingent upon [esi_content_types](#esi_content_types) and [esi_surrogate_delegation](#esi_surrogate_delegation) settings, as well as `Surrogate-Control` / `Surrogate-Capability` headers.\n\nESI instructions are detected on the slow path (i.e. when fetching from the origin), so only instructions which are known to be present are processed on cache HITs.\n\n[Back to TOC](#handler-configuration-options)\n\n\n#### esi_content_types\n\ndefault: `{ text/html }`\n\nSpecifies content types to perform ESI processing on. All other content types will not be considered for processing.\n\n[Back to TOC](#handler-configuration-options)\n\n\n#### esi_allow_surrogate_delegation\n\ndefault: false\n\n[ESI Surrogate Delegation](http://www.w3.org/TR/edge-arch) allows for downstream intermediaries to advertise a capability to process ESI instructions nearer to the client. By setting this to `true` any downstream offering this will disable ESI processing in Ledge, delegating it downstream.\n\nWhen set to a Lua table of IP address strings, delegation will only be allowed to this specific hosts. This may be important if ESI instructions contain sensitive data which must be removed.\n\n[Back to TOC](#handler-configuration-options)\n\n\n#### esi_recursion_limit\n\ndefault: 10\n\nLimits fragment inclusion nesting, to avoid accidental infinite recursion.\n\n[Back to TOC](#handler-configuration-options)\n\n\n#### esi_args_prefix\n\ndefault: \"esi\\_\"\n\nURI args prefix for parameters to be ignored from the cache key (and not proxied upstream), for use exclusively with ESI rendering logic. Set to nil to disable the feature.\n\n[Back to TOC](#handler-configuration-options)\n\n\n#### esi_custom_variables\n\ndefualt: `{}`\n\nAny variables supplied here will be available anywhere ESI vars can be used evaluated. See [Custom ESI variables](#custom-esi-variables).\n\n[Back to TOC](#handler-configuration-options)\n\n\n#### esi_max_size\n\ndefault: `1024 * 1024 (bytes)`\n\n[Back to TOC](#handler-configuration-options)\n\n\n#### esi_attempt_loopback\n\ndefault: `true`\n\nIf an ESI subrequest has the same `scheme` and `host` as the parent request, we loopback the connection to the current\n`server_addr` and `server_port` in order to avoid going over network.\n\n[Back to TOC](#handler-configuration-options)\n\n\n#### esi_vars_cookie_blacklist\n\ndefault: `{}`\n\nCookie names given here will not be expandable as ESI variables: e.g. `$(HTTP_COOKIE)` or `$(HTTP_COOKIE{foo})`. However they\nare not removed from the request data, and will still be propagated to `` subrequests.\n\nThis is useful if your client is sending a sensitive cookie that you don't ever want to accidentally evaluate in server output.\n\n```lua\nrequire(\"ledge\").create_handler({\n esi_vars_cookie_blacklist = {\n secret = true,\n [\"my-secret-cookie\"] = true,\n }\n}):run()\n```\n\nCookie names are given as the table key with a truthy value, for O(1) runtime lookup.\n\n\n[Back to TOC](#handler-configuration-options)\n\n\n#### esi_disable_third_party_includes\n\ndefault: `false`\n\n`` tags can make requests to any arbitrary URI. Turn this on to ensure the URI domain must match the URI of the current request.\n\n[Back to TOC](#handler-configuration-options)\n\n\n#### esi_third_party_includes_domain_whitelist\n\ndefault: `{}`\n\nIf third party includes are disabled, you can also explicitly provide a whitelist of allowed third party domains.\n\n```lua\nrequire(\"ledge\").create_handler({\n esi_disable_third_party_includes = true,\n esi_third_party_includes_domain_whitelist = {\n [\"example.com\"] = true,\n }\n}):run()\n```\n\nHostnames are given as the table key with a truthy value, for O(1) lookup.\n\n*Note; This behaviour was introduced in v2.2*\n\n[Back to TOC](#handler-configuration-options)\n\n\n#### advertise_ledge\n\ndefault `true`\n\nIf set to false, disables advertising the software name and version, e.g. `(ledge/2.01)` from the `Via` response header.\n\n[Back to TOC](#handler-configuration-options)\n\n\n### Events\n\n* [after_cache_read](#after_cache_read)\n* [before_upstream_connect](#before_upstream_connect)\n* [before_upstream_request](#before_upstream_request)\n* [before_esi_inclulde_request\"](#before_esi_include_request)\n* [after_upstream_request](#after_upstream_request)\n* [before_save](#before_save)\n* [before_serve](#before_serve)\n* [before_save_revalidation_data](#before_save_revalidation_data)\n* [before_vary_selection](#before_vary_selection)\n\n#### after_cache_read\n\nsyntax: `bind(\"after_cache_read\", function(res) -- end)`\n\nparams: `res`. The cached response table.\n\nFires directly after the response was successfully loaded from cache.\n\nThe `res` table given contains:\n\n* `res.header` the table of case-insenitive HTTP response headers\n* `res.status` the HTTP response status code\n\n*Note; there are other fields and methods attached, but it is strongly advised to never adjust anything other than the above*\n\n[Back to TOC](#events)\n\n\n#### before_upstream_connect\n\nsyntax: `bind(\"before_upstream_connect\", function(handler) -- end)`\n\nparams: `handler`. The current handler instance.\n\nFires before the default `handler.upstream_client` is created, allowing a pre-connected HTTP client to be externally provided. The client must be API compatible with [lua-resty-http](https://github.com/pintsized/lua-resty-http). For example, using [lua-resty-upstream](https://github.com/hamishforbes/lua-resty-upstream) for load balancing.\n\n[Back to TOC](#events)\n\n\n#### before_upstream_request\n\nsyntax: `bind(\"before_upstream_request\", function(req_params) -- end)`\n\nparams: `req_params`. The table of request params about to send to the [request](https://github.com/pintsized/lua-resty-http#request) method.\n\nFires when about to perform an upstream request.\n\n[Back to TOC](#events)\n\n\n#### before_esi_include_request\n\nsyntax: `bind(\"before_esi_include_request\", function(req_params) -- end)`\n\nparams: `req_params`. The table of request params about to be used for an ESI include, via the [request](https://github.com/pintsized/lua-resty-http#request) method.\n\nFires when about to perform a HTTP request on behalf of an ESI include instruction.\n\n[Back to TOC](#events)\n\n\n#### after_upstream_request\n\nsyntax: `bind(\"after_upstream_request\", function(res) -- end)`\n\nparams: `res` The response table.\n\nFires when the status / headers have been fetched, but before the body it is stored. Typically used to override cache headers before we decide what to do with this response.\n\nThe `res` table given contains:\n\n* `res.header` the table of case-insenitive HTTP response headers\n* `res.status` the HTTP response status code\n\n*Note; there are other fields and methods attached, but it is strongly advised to never adjust anything other than the above*\n\n*Note: unlike `before_save` below, this fires for all fetched content, not just cacheable content.*\n\n[Back to TOC](#events)\n\n\n#### before_save\n\nsyntax: `bind(\"before_save\", function(res) -- end)`\n\nparams: `res` The response table.\n\nFires when we're about to save the response.\n\nThe `res` table given contains:\n\n* `res.header` the table of case-insenitive HTTP response headers\n* `res.status` the HTTP response status code\n\n*Note; there are other fields and methods attached, but it is strongly advised to never adjust anything other than the above*\n\n[Back to TOC](#events)\n\n\n#### before_serve\n\nsyntax: `ledge:bind(\"before_serve\", function(res) -- end)`\n\nparams: `res` The `ledge.response` object.\n\nFires when we're about to serve. Often used to modify downstream headers.\n\nThe `res` table given contains:\n\n* `res.header` the table of case-insenitive HTTP response headers\n* `res.status` the HTTP response status code\n\n*Note; there are other fields and methods attached, but it is strongly advised to never adjust anything other than the above*\n\n[Back to TOC](#events)\n\n\n#### before_save_revalidation_data\n\nsyntax: `bind(\"before_save_revalidation_data\", function(reval_params, reval_headers) -- end)`\n\nparams: `reval_params`. Table of revalidation params.\n\nparams: `reval_headers`. Table of revalidation HTTP headers.\n\nFires when a background revalidation is triggered or when cache is being saved. Allows for modifying the headers and paramters (such as connection parameters) which are inherited by the background revalidation.\n\nThe `reval_params` are values derived from the current running configuration for:\n\n* server_addr\n* server_port\n* scheme\n* uri\n* connect_timeout\n* read_timeout\n* ssl_server_name\n* ssl_verify\n\n[Back to TOC](#events)\n\n\n#### before_vary_selection\n\nsyntax: `bind(\"before_vary_selection\", function(vary_key) -- end)`\n\nparams: `vary_key` A table of selecting headers\n\nFires when we're about to generate the vary key, used to select the correct cache representation.\n\nThe `vary_key` table is a hash of header field names (lowercase) to values.\nA field name which exists in the Vary response header but does not exist in the current request header will have a value of `ngx.null`.\n\n```\nRequest Headers:\n Accept-Encoding: gzip\n X-Test: abc\n X-test: def\n\nResponse Headers:\n Vary: Accept-Encoding, X-Test\n Vary: X-Foo\n\nvary_key table:\n{\n [\"accept-encoding\"] = \"gzip\",\n [\"x-test\"] = \"abc,def\",\n [\"x-foo\"] = ngx.null\n}\n```\n\n[Back to TOC](#events)\n\n\n## Administration\n\n### X-Cache\n\nLedge adds the non-standard `X-Cache` header, familiar to users of other caches. It indicates simply `HIT` or `MISS` and the host name in question, preserving upstream values when more than one cache server is in play.\n\nIf a resource is considered not cacheable, the `X-Cache` header will not be present in the response.\n\nFor example:\n\n* `X-Cache: HIT from ledge.tld` *A cache hit, with no (known) cache layer upstream.*\n* `X-Cache: HIT from ledge.tld, HIT from proxy.upstream.tld` *A cache hit, also hit upstream.*\n* `X-Cache: MISS from ledge.tld, HIT from proxy.upstream.tld` *A cache miss, but hit upstream.*\n* `X-Cache: MISS from ledge.tld, MISS from proxy.upstream.tld` *Regenerated at the origin.*\n\n[Back to TOC](#table-of-contents)\n\n\n### Logging\n\nIt's often useful to add some extra headers to your Nginx logs, for example\n\n```\nlog_format ledge '$remote_addr - $remote_user [$time_local] '\n '\"$request\" $status $body_bytes_sent '\n '\"$http_referer\" \"$http_user_agent\" '\n '\"Cache:$sent_http_x_cache\" \"Age:$sent_http_age\" \"Via:$sent_http_via\"'\n ;\n\naccess_log /var/log/nginx/access_log ledge;\n```\n\nWill give log lines such as:\n\n```\n192.168.59.3 - - [23/May/2016:22:22:18 +0000] \"GET /x/y/z HTTP/1.1\" 200 57840 \"-\" \"curl/7.37.1\"\"Cache:HIT from 159e8241f519:8080\" \"Age:724\"\n\n```\n[Back to TOC](#table-of-contents)\n\n\n### Managing Qless\n\nLedge uses [lua-resty-qless](https://github.com/pintsized/lua-resty-qless) to schedule and process background tasks, which are stored in Redis.\n\nJobs are scheduled for background revalidation requests as well as wildcard PURGE requests, but most importantly for garbage collection of replaced body entities.\n\nThat is, it's very important that jobs are being run properly and in a timely fashion.\n\nInstalling the [web user interface](https://github.com/hamishforbes/lua-resty-qless-web) can be very helpful to check this.\n\nYou may also wish to tweak the [qless job history](https://github.com/pintsized/lua-resty-qless#configuration-options) settings if it takes up too much space.\n\n\n[Back to TOC](#table-of-contents)\n\n\n## Author\n\nJames Hurst \n\n\n## Licence\n\nThis module is licensed under the 2-clause BSD license.\n\nCopyright (c) James Hurst \n\nAll rights reserved.\n\nRedistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:\n\n* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.\n\n* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" }, { "path": "dist.ini", "content": "name=ledge\nabstract=An RFC compliant and ESI capable HTTP cache for Nginx / OpenResty, backed by Redis\nauthor=James Hurst, Hamish Forbes\nis_original=yes\nlicense=2bsd\nlib_dir=lib\nrepo_link=https://github.com/pintsized/ledge\nmain_module=lib/ledge.lua\nrequires = pintsized/lua-resty-http >= 0.11, pintsized/lua-resty-redis-connector >= 0.06, pintsized/lua-resty-qless >= 0.11, p0pr0ck5/lua-resty-cookie >= 0.01, hamishforbes/lua-ffi-zlib >= 0.3.0\n" }, { "path": "docker/tests/docker-compose.yml", "content": "version: '3'\n\nservices:\n runner:\n image: \"ledgetech/test-runner:latest\"\n volumes:\n - ../../:/code\n\n # Use this to mount any local Lua dependencies, overriding\n # published versions\n - ${EXTLIB-../../lib}:/code/extlib\n environment:\n - TEST_FILE\n command: /bin/bash -c \"TEST_LEDGE_REDIS_HOST=redis make coverage\"\n working_dir: /code\n depends_on:\n - redis\n\n redis:\n image: \"redis:alpine\"\n" }, { "path": "lib/ledge/background.lua", "content": "local require = require\nlocal math_ceil = math.ceil\nlocal qless = require(\"resty.qless\")\n\nlocal _M = {\n _VERSION = \"2.3.0\",\n}\n\nlocal function put_background_job( queue, klass, data, options)\n local q = qless.new({\n get_redis_client = require(\"ledge\").create_qless_connection\n })\n\n -- If we've been specified a jid (i.e. a non random jid), putting this\n -- job will overwrite any existing job with the same jid.\n -- We test for a \"running\" state, and if so we silently drop this job.\n if options.jid then\n local existing = q.jobs:get(options.jid)\n\n if existing and existing.state == \"running\" then\n return nil, \"Job with the same jid is currently running\"\n end\n end\n\n -- Put the job\n local res, err = q.queues[queue]:put(klass, data, options)\n\n q:redis_close()\n\n if res then\n return {\n jid = res,\n klass = klass,\n options = options,\n }\n else\n return res, err\n end\nend\n_M.put_background_job = put_background_job\n\n\n-- Calculate when to GC an entity based on its size and the minimum download\n-- rate setting, plus 1 second of arbitrary latency for good measure.\nlocal function gc_wait(entity_size, minimum_download_rate)\n local dl_rate_Bps = minimum_download_rate * 128\n return math_ceil((entity_size / dl_rate_Bps)) + 1\nend\n_M.gc_wait = gc_wait\n\n\nreturn _M\n" }, { "path": "lib/ledge/cache_key.lua", "content": "local ipairs, next, type, pcall, setmetatable =\n ipairs, next, type, pcall, setmetatable\n\nlocal str_lower = string.lower\n\nlocal ngx_log = ngx.log\nlocal ngx_ERR = ngx.ERR\nlocal ngx_var = ngx.var\nlocal ngx_null = ngx.null\n\nlocal tbl_insert = table.insert\nlocal tbl_concat = table.concat\nlocal tbl_sort = table.sort\n\nlocal req_args_sorted = require(\"ledge.request\").args_sorted\nlocal req_default_args = require(\"ledge.request\").default_args\n\nlocal get_fixed_field_metatable_proxy =\n require(\"ledge.util\").mt.get_fixed_field_metatable_proxy\n\nlocal http_headers = require(\"resty.http_headers\")\n\n\nlocal _M = {\n _VERSION = \"2.3.0\",\n}\n\n\n-- Generates the root key. The default spec is:\n-- ledge:cache_obj:http:example.com:/about:p=3&q=searchterms\nlocal function generate_root_key(key_spec, max_args)\n -- If key_spec is empty, provide a default\n if not key_spec or not next(key_spec) then\n key_spec = {\n \"scheme\",\n \"host\",\n \"uri\",\n \"args\",\n }\n end\n\n local key = {\n \"ledge\",\n \"cache\",\n }\n\n for _, field in ipairs(key_spec) do\n if field == \"scheme\" then\n tbl_insert(key, ngx_var.scheme)\n elseif field == \"host\" then\n tbl_insert(key, ngx_var.host)\n elseif field == \"port\" then\n tbl_insert(key, ngx_var.server_port)\n elseif field == \"uri\" then\n tbl_insert(key, ngx_var.uri)\n elseif field == \"args\" then\n tbl_insert(\n key,\n req_args_sorted(max_args) or req_default_args()\n )\n\n elseif type(field) == \"function\" then\n local ok, res = pcall(field)\n if not ok then\n ngx_log(ngx_ERR,\n \"error in function supplied to cache_key_spec: \", res\n )\n elseif type(res) ~= \"string\" then\n ngx_log(ngx_ERR,\n \"functions supplied to cache_key_spec must \" ..\n \"return a string\"\n )\n else\n tbl_insert(key, res)\n end\n end\n end\n\n return tbl_concat(key, \":\")\nend\n_M.generate_root_key = generate_root_key\n\n\n-- Read the list of vary headers from redis\nlocal function read_vary_spec(redis, root_key)\n if not redis or not next(redis) then\n return nil, \"Redis required\"\n end\n\n if not root_key then\n return nil, \"Root key required\"\n end\n\n local res, err = redis:smembers(root_key..\"::vary\")\n if err then\n return nil, err\n end\n\n table.sort(res)\n\n return res\nend\n_M.read_vary_spec = read_vary_spec\n\n\nlocal function vary_compare(spec_a, spec_b)\n if (not spec_a or not next(spec_a)) then\n if (not spec_b or not next(spec_b)) then\n -- both nil or empty\n return true\n else\n -- spec_b is set but spec_a is empty\n return false\n end\n\n elseif (spec_b and next(spec_b)) then\n local outer_match = true\n\n -- Loop over all values in spec_a\n for _, v in ipairs(spec_a) do\n local match = false\n -- Look for a match in spec_b\n for _, v2 in ipairs(spec_b) do\n if v == v2 then\n match = true\n break\n end\n end\n\n -- Didn't match any values in spec_b\n if match == false then\n outer_match = false\n break\n end\n end\n\n return outer_match\n end\n\n -- spec_a is a thing but spec_b is not\n return false\nend\n_M.vary_compare = vary_compare\n\n\nlocal function generate_vary_key(vary_spec, callback, headers)\n local vary_key = http_headers.new()\n\n if vary_spec and next(vary_spec) then\n headers = headers or ngx.req.get_headers()\n\n for _, h in ipairs(vary_spec) do\n local v = headers[h]\n if type(v) == \"table\" then\n v = tbl_concat(v, \",\")\n end\n -- ngx.null represents a key which was in the spec\n -- but has no matching request header\n vary_key[h] = v or ngx_null\n end\n end\n\n -- Callback allows user to modify the key\n if type(callback) == \"function\" then\n callback(vary_key)\n end\n\n if not next(vary_key) then\n return \"\"\n end\n\n -- Extract keys and sort them\n local keys = {}\n for k,v in pairs(vary_key) do\n if v ~= ngx_null then\n tbl_insert(keys, k)\n end\n end\n\n tbl_sort(keys)\n\n -- Convert hash table to flat array\n local t = {}\n local i = 1\n for _, k in ipairs(keys) do\n t[i] = k\n t[i + 1] = vary_key[k]\n i = i + 2\n end\n\n return str_lower(tbl_concat(t, \":\"))\nend\n_M.generate_vary_key = generate_vary_key\n\n\n-- Returns the key chain for all cache keys, except the body entity\nlocal function key_chain(root_key, vary_key, vary_spec)\n if not root_key then\n return nil, \"Missing root key\"\n end\n if not vary_key then\n return nil, \"Missing vary key\"\n end\n if not vary_spec then\n return nil, \"Missing vary_spec\"\n end\n\n\n local full_key = root_key .. \"#\" .. vary_key\n\n -- Apply metatable\n local key_chain = setmetatable({\n -- hash: cache key metadata\n main = full_key .. \"::main\",\n\n -- sorted set: current entities score with sizes\n entities = full_key .. \"::entities\",\n\n -- hash: response headers\n headers = full_key .. \"::headers\",\n\n -- hash: request headers for revalidation\n reval_params = full_key .. \"::reval_params\",\n\n -- hash: request params for revalidation\n reval_req_headers = full_key .. \"::reval_req_headers\",\n }, get_fixed_field_metatable_proxy({\n -- Hide these keys from iterators\n\n -- These are not actual keys but useful to keep around\n root = root_key,\n full = full_key,\n vary_spec = vary_spec,\n\n -- set: headers upon which to vary\n vary = root_key .. \"::vary\",\n -- set: representations for this root key\n repset = root_key .. \"::repset\",\n -- Lock key for collapsed forwarding\n fetching_lock = full_key .. \"::fetching\",\n })\n )\n\n return key_chain\nend\n_M.key_chain = key_chain\n\n\nlocal function clean_repset(redis, repset)\n -- Ensure representation set only includes keys which actually exist\n -- This only runs on the slow path at save time so should be ok?\n -- Prevents this set from growing perpetually if there are unique variations\n -- TODO use scan here incase the set is pathologically huge?\n -- Has to be able to run in a transaction so maybe a housekeeping qless job?\n local clean = [[\n local repset = KEYS[1]\n local reps = redis.call(\"SMEMBERS\", repset)\n for _, rep in ipairs(reps) do\n if redis.call(\"EXISTS\", rep..\"::main\") == 0 then\n redis.call(\"SREM\", repset, rep)\n end\n end\n ]]\n\n local res, err = redis:eval(clean, 1, repset)\n if not res or res == ngx_null then\n return nil, err\n end\n\n return true\nend\n\n\nlocal function save_key_chain(redis, key_chain, ttl)\n if not redis then\n return nil, \"Redis required\"\n end\n\n if type(key_chain) ~= \"table\" or not next(key_chain) then\n return nil, \"Key chain required\"\n end\n\n if not tonumber(ttl) then\n return nil, \"TTL must be a number\"\n end\n\n -- Delete the current set of vary headers\n local _, e = redis:del(key_chain.vary)\n if e then ngx_log(ngx_ERR, e) end\n\n local vary_spec = key_chain.vary_spec\n\n if next(vary_spec) then\n -- Always lowercase all vary fields\n -- key_chain.vary is a set so will deduplicate for us\n for i,v in ipairs(vary_spec) do\n vary_spec[i] = str_lower(v)\n end\n\n local _, e = redis:sadd(key_chain.vary, unpack(vary_spec))\n if e then ngx_log(ngx_ERR, e) end\n\n local _, e = redis:expire(key_chain.vary, ttl)\n if e then ngx_log(ngx_ERR, e) end\n end\n\n -- Add this representation to the set\n local _, e = redis:sadd(key_chain.repset, key_chain.full)\n if e then ngx_log(ngx_ERR, e) end\n\n local _, e = redis:expire(key_chain.repset, ttl)\n if e then ngx_log(ngx_ERR, e) end\n\n\n local _, e = clean_repset(redis, key_chain.repset)\n if e then ngx_log(ngx_ERR, e) end\n\n return true\nend\n_M.save_key_chain = save_key_chain\n\n\nreturn _M\n" }, { "path": "lib/ledge/collapse.lua", "content": "local _M = {\n _VERSION = \"2.3.0\",\n}\n\n-- Attempts to set a lock key in redis. The lock will expire after\n-- the expiry value if it is not cleared (i.e. in case of errors).\n-- Returns true if the lock was acquired, false if the lock already\n-- exists, and nil, err in case of failure.\nlocal function acquire_lock(redis, lock_key, timeout)\n -- We use a Lua script to emulate SETNEX (set if not exists with expiry).\n -- This avoids a race window between the GET / SETEX.\n -- Params: key, expiry\n -- Return: OK or BUSY\n local SETNEX = [[\n local lock = redis.call(\"GET\", KEYS[1])\n if not lock then\n return redis.call(\"PSETEX\", KEYS[1], ARGV[1], \"locked\")\n else\n return \"BUSY\"\n end\n ]]\n\n local res, err = redis:eval(SETNEX, 1, lock_key, timeout)\n\n if not res then -- Lua script failed\n return nil, err\n elseif res == \"OK\" then -- We have the lock\n return true\n elseif res == \"BUSY\" then -- Lock is busy\n return false\n end\nend\n_M.acquire_lock = acquire_lock\n\nreturn _M\n" }, { "path": "lib/ledge/esi/processor_1_0.lua", "content": "local http = require \"resty.http\"\nlocal cookie = require \"resty.cookie\"\nlocal tag_parser = require \"ledge.esi.tag_parser\"\nlocal util = require \"ledge.util\"\n\nlocal tostring, type, tonumber, next, unpack, pcall, setfenv, loadstring =\n tostring, type, tonumber, next, unpack, pcall, setfenv, loadstring\n\nlocal str_sub = string.sub\nlocal str_byte = string.byte\n-- TODO: Find places we can use str_find over ngx_re_find\nlocal str_find = string.find\n\nlocal tbl_concat = table.concat\nlocal tbl_insert = table.insert\n\nlocal co_yield = coroutine.yield\nlocal co_wrap = util.coroutine.wrap\n\nlocal ngx_re_gsub = ngx.re.gsub\nlocal ngx_re_sub = ngx.re.sub\nlocal ngx_re_match = ngx.re.match\nlocal ngx_re_find = ngx.re.find\nlocal ngx_req_get_headers = ngx.req.get_headers\nlocal ngx_req_get_uri_args = ngx.req.get_uri_args\nlocal ngx_flush = ngx.flush\nlocal ngx_var = ngx.var\nlocal ngx_log = ngx.log\nlocal ngx_ERR = ngx.ERR\nlocal ngx_INFO = ngx.INFO\n\nlocal get_fixed_field_metatable_proxy =\n require(\"ledge.util\").mt.get_fixed_field_metatable_proxy\n\n\nlocal _M = {\n _VERSION = \"2.3.0\",\n}\n\n\nfunction _M.new(handler)\n return setmetatable({\n handler = handler,\n token = \"ESI/1.0\",\n }, get_fixed_field_metatable_proxy(_M))\nend\n\n\n-- $1: variable name (e.g. QUERY_STRING)\n-- $2: substructure key\n-- $3: default value\n-- $4: default value if quoted\nlocal esi_var_pattern =\n [[\\$\\(([A-Z_]+){?([a-zA-Z\\.\\-~_%0-9]*)}?\\|?(?:([^\\s\\)']+)|'([^\\')]+)')?\\)]]\n\n\n-- Evaluates a given ESI variable.\nlocal function _esi_eval_var(var)\n -- Extract variables from capture results table\n local var_name = var[1] or \"\"\n\n local key = var[2]\n if key == \"\" then key = nil end\n\n local default = var[3]\n local default_quoted = var[4]\n local default = default or default_quoted or \"\"\n\n if var_name == \"QUERY_STRING\" then\n if not key then\n -- We don't have a key so give them the whole string\n return ngx_var.args or default\n else\n -- Lookup the querystring component by key\n local value = ngx_req_get_uri_args()[key]\n if value then\n if type(value) == \"table\" then\n return tbl_concat(value, \", \")\n else\n return value\n end\n else\n return default\n end\n end\n elseif str_sub(var_name, 1, 5) == \"HTTP_\" then\n -- Evaluate request headers. Cookie and Accept-Language are special\n -- according to the spec.\n\n local header = str_sub(var_name, 6)\n\n if header == \"COOKIE\" then\n local cookies = ngx.ctx.__ledge_esi_cookies or cookie:new()\n local blacklist = ngx.ctx.__ledge_esi_vars_cookie_blacklist or {}\n\n if not next(blacklist) then\n if key then\n return cookies:get(key) or default\n end\n\n return ngx_var.http_cookie or default\n end\n\n -- We have a blacklist to filter with\n if key then\n if not blacklist[key] then\n return cookies:get(key) or default\n end\n\n return default\n else\n -- We need a full cookie string, with any blacklisted values removed\n local cookies = cookies:get_all()\n\n local value = {}\n for k, v in pairs(cookies) do\n if not blacklist[k] then\n tbl_insert(value, k .. \"=\" .. v)\n end\n end\n return tbl_concat(value, \"; \") or default\n end\n else\n local value = ngx_req_get_headers()[header]\n\n if not value then\n return default\n elseif header == \"ACCEPT_LANGUAGE\" and key then\n -- If we're a table (multilple Accept-Language headers), convert\n -- to string\n if type(value) == \"table\" then\n value = tbl_concat(value, \", \")\n end\n\n if ngx_re_find(value, key, \"oj\") then\n return \"true\"\n else\n return \"false\"\n end\n\n elseif type(value) == \"table\" then\n -- For normal repeated headers, numeric indexes are supported\n key = tonumber(key)\n if key then\n -- We can index numerically (0 indexed)\n return tostring(value[key + 1] or default)\n else\n -- Without a numeric key, render as a comma separated list\n return tbl_concat(value, \", \") or default\n end\n else\n return value\n end\n end\n elseif var_name == \"ESI_ARGS\" then\n local esi_args = ngx.ctx.__ledge_esi_args\n\n if not esi_args then\n -- No ESI args in request\n return default\n end\n\n if not key then\n -- __tostring metamethod turns these back into encoded URI args\n return tostring(esi_args)\n else\n local value = esi_args[key] or default\n if type(value) == \"table\" then\n return tbl_concat(value, \",\")\n end\n return tostring(value)\n end\n else\n local custom_variables = ngx.ctx.__ledge_esi_custom_variables\n if next(custom_variables) then\n\n local var = custom_variables[var_name]\n if var then\n if key then\n if type(var) == \"table\" then\n return tostring(var[key] or default)\n end\n else\n if type(var) == \"table\" then\n -- No sane way to stringify other tables\n return default\n else\n -- We're a string\n return var or default\n end\n end\n end\n end\n return default\n end\nend\n\n\nlocal function esi_eval_var(var)\n local escape = true\n local var_name = var[1]\n\n -- If var name begins with RAW_ do not escape\n local b1, b2, b3, b4 = str_byte(var_name, 1, 4)\n if b1 == 82 and b2 == 65 and b3 == 87 and b4 == 95 then\n escape = false\n var[1] = str_sub(var_name, 5, -1)\n end\n\n local res = _esi_eval_var(var)\n\n -- Always escape ESI tags in ESI variables\n if escape or str_find(res, \"\", \">\", \"soj\")\n end\n\n return res\nend\n_M.esi_eval_var = esi_eval_var\n\n\nlocal function esi_replace_vars(str, cb)\n cb = cb or esi_eval_var\n return ngx_re_gsub(str, esi_var_pattern, cb, \"soj\")\nend\n\n\nlocal function esi_eval_var_in_when_tag(var)\n var = esi_eval_var(var)\n -- Quote unless we can be considered a number\n local number = tonumber(var)\n if number then\n return number\n else\n -- Strings must be enclosed in single quotes, so also backslash\n -- escape single quotes within the value\n return \"\\'\" .. ngx_re_gsub(var, \"'\", \"\\\\'\", \"oj\") .. \"\\'\"\n end\nend\n\n\nlocal function _esi_condition_lexer(condition)\n -- ESI to Lua operators\n local op_replacements = {\n [\"!=\"] = \"~=\",\n [\"|\"] = \" or \",\n [\"&\"] = \" and \",\n [\"||\"] = \" or \",\n [\"&&\"] = \" and \",\n [\"!\"] = \" not \",\n }\n\n -- Mapping of types to types they are allowed to follow\n local lexer_rules = {\n number = {\n [\"nil\"] = true,\n [\"operator\"] = true,\n },\n string = {\n [\"nil\"] = true,\n [\"operator\"] = true,\n },\n operator = {\n [\"nil\"] = true,\n [\"number\"] = true,\n [\"string\"] = true,\n [\"operator\"] = true,\n },\n }\n\n -- $1: number\n -- $2: string\n -- $3: operator\n local p =[[(\\d+(?:\\.\\d+)?)|(?:'(.*?)(?=|>|<)]]\n local ctx = {}\n local tokens = {}\n local prev_type\n local expecting_pattern = false\n\n repeat\n local token, err = ngx_re_match(condition, p, \"\", ctx)\n if err then ngx_log(ngx_ERR, err) end\n if token then\n local number, string, operator = token[1], token[2], token[3]\n local token_type\n\n if number then\n token_type = \"number\"\n tbl_insert(tokens, number)\n elseif string then\n token_type = \"string\"\n\n -- Check to see if we're expecing a regex pattern\n if expecting_pattern then\n -- Extract the pattern and options\n local re = ngx_re_match(\n string,\n [[\\/(.*?)(?)]]\n local after -- Will contain anything after the last closing choose tag\n local chunk_has_conditionals = false\n repeat\n local choose, ch_before, ch_after = parser:next(\"esi:choose\")\n if choose and choose.closing then\n chunk_has_conditionals = true\n\n -- Anything before this choose should just be output\n if ch_before then\n tbl_insert(res, ch_before)\n end\n\n -- If this ends up being the last choose tag, content after this\n -- should be output\n if ch_after then\n after = ch_after\n end\n\n local inner_parser = tag_parser.new(choose.contents)\n\n local when_matched = false\n local otherwise\n repeat\n local tag = inner_parser:next(\"esi:when|esi:otherwise\")\n if tag and tag.closing then\n if tag.tagname == \"esi:when\" and when_matched == false then\n\n local function process_when(m_when)\n -- We only show the first matching branch, others\n -- must be removed even if they also match.\n if when_matched then return \"\" end\n\n local condition = m_when[1]\n\n if _esi_evaluate_condition(condition) then\n when_matched = true\n\n if ngx_re_find(tag.contents, \"\") then\n -- recurse\n evaluate_conditionals(\n tag.contents,\n res,\n recursion + 1\n )\n else\n tbl_insert(res, tag.contents)\n end\n end\n return \"\"\n end\n\n local ok, err = ngx_re_sub(\n tag.whole,\n esi_when_pattern,\n process_when\n )\n if not ok and err then ngx_log(ngx_ERR, err) end\n\n -- Break after the first winning expression\n elseif tag.tagname == \"esi:otherwise\" then\n otherwise = tag.contents\n end\n end\n until not tag\n\n if not when_matched and otherwise then\n if ngx_re_find(otherwise, \"\") then\n -- recurse\n evaluate_conditionals(otherwise, res, recursion + 1)\n else\n tbl_insert(res, otherwise)\n end\n end\n end\n\n until not choose\n\n if after then\n tbl_insert(res, after)\n end\n\n -- Variables inside ESI tags should be evaluated.\n -- Return hint to eval this chunk\n if not chunk_has_conditionals then\n return chunk, false\n else\n return tbl_concat(res), true\n end\nend\n\n\n-- Used in esi_process_vars_tag. Declared locally to avoid runtime closure\nlocal function _esi_gsub_vars(m)\n return esi_replace_vars(m[2])\nend\n\n\n-- Replaces all variables in blocks.\n-- Also removes the tags themselves.\nlocal function esi_process_vars_tag(chunk)\n if str_find(chunk, \"esi:vars\", 1, true) == nil then\n return chunk\n end\n\n -- For every esi:vars block, substitute any number of variables found.\n return ngx_re_gsub(chunk,\n \"()(.*?)()\",\n _esi_gsub_vars,\n \"soj\"\n )\nend\n_M.esi_process_vars_tag = esi_process_vars_tag\n\n\nlocal function process_escaping(chunk, res, recursion)\n if not recursion then recursion = 0 end\n if not res then res = {} end\n\n local parser = tag_parser.new(chunk)\n\n local chunk_has_escaping = false\n repeat\n local tag, before, after = parser:next(\"!--esi\")\n if tag and tag.closing then\n chunk_has_escaping = true\n if before then\n tbl_insert(res, before)\n end\n\n -- If there are more nested, recurse\n if ngx_re_find(tag.contents, \"\n chunk = process_escaping(chunk)\n\n -- Remove comments.\n chunk = esi_process_comment_tags(chunk)\n\n -- Remove '\n chunk = esi_process_vars_tag(chunk)\n\n -- Evaluate choose / when / otherwise conditions...\n local chunk, should_eval = evaluate_conditionals(chunk)\n\n -- Process ESI includes\n -- Will yield content to the outer reader\n esi_process_include_tags(self, chunk, esi_abort_flag, buffer_size, should_eval)\n\n else\n co_yield(chunk)\n end\n end\n until not chunk\n end)\n\n -- Outer filter, which checks for an esi:abort_includes instruction,\n -- so that we can handle accidental recursion.\n repeat\n local chunk, err = inner_reader(buffer_size)\n if err then ngx_log(ngx_ERR, err) end\n if chunk then\n -- If we see an abort instruction, we set a flag to stop\n -- further esi:includes.\n if str_find(chunk, \"\",\n \"\",\n \"soj\"\n )\n end\n\n co_yield(chunk)\n end\n until not chunk\n end)\nend\n\n\nreturn _M\n" }, { "path": "lib/ledge/esi/tag_parser.lua", "content": "local setmetatable, type =\n setmetatable, type\n\nlocal str_sub = string.sub\n\nlocal ngx_re_find = ngx.re.find\nlocal ngx_re_match = ngx.re.match\nlocal ngx_log = ngx.log\nlocal ngx_ERR = ngx.ERR\n\nlocal get_fixed_field_metatable_proxy =\n require(\"ledge.util\").mt.get_fixed_field_metatable_proxy\n\n\nlocal _M = {\n _VERSION = \"2.3.0\",\n}\n\n\nfunction _M.new(content, offset)\n return setmetatable({\n content = content,\n pos = (offset or 0),\n open_comments = 0,\n }, get_fixed_field_metatable_proxy(_M))\nend\n\n\nfunction _M.next(self, tagname)\n local tag = self:find_whole_tag(tagname)\n local before, after\n if tag then\n before = str_sub(self.content, self.pos + 1, tag.opening.from - 1)\n\n if tag.closing then\n -- This is block level (with a closing tag)\n after = str_sub(self.content, tag.closing.to + 1)\n self.pos = tag.closing.to\n else\n -- Inline (no closing tag)\n after = str_sub(self.content, tag.opening.to + 1)\n self.pos = tag.opening.to\n end\n end\n\n return tag, before, after\nend\n\n\nfunction _M.open_pattern(tag)\n if tag == \"!--esi\" then\n return \"<(!--esi)\"\n else\n -- $1: the tag name, $2 the closing characters, e.g. \"/>\" or \">\"\n return \"<(\" .. tag .. [[)(?:\\s*(?:[a-z]+=\\\".+?(?]*?(?:\\s*)(\\/>|>)?]]\n end\nend\n\n\nfunction _M.close_pattern(tag)\n if tag == \"!--esi\" then\n return \"-->\"\n else\n -- $1: the tag name\n return \"\"\n end\nend\n\n\nfunction _M.either_pattern(tag)\n if tag == \"!--esi\" then\n return \"(?:<(!--esi)|(-->))\"\n else\n -- $1: the tag name, $2 the closing characters, e.g. \"/>\" or \">\"\n return [[<[\\/]?(]] .. tag .. [[)(?:\\s*(?:[a-z]+=\\\".+?(?]*?(?:\\s*)(\\s*\\\\/>|>)?]]\n end\nend\n\n\n-- Finds the next esi tag, accounting for nesting to find the correct\n-- matching closing tag etc.\nfunction _M.find_whole_tag(self, tag)\n -- Only work on the remaining markup (after pos)\n local markup = str_sub(self.content, self.pos + 1)\n\n if not tag then\n -- Look for anything (including comment syntax)\n tag = \"(?:!--esi)|(?:esi:[a-z]+)\"\n end\n\n -- Find the first opening tag\n local opening_f, opening_t, err = ngx_re_find(markup, self.open_pattern(tag), \"soj\")\n if not opening_f then\n if err then ngx_log(ngx_ERR, err) end\n -- Nothing here\n return nil\n end\n\n -- We found an opening tag and has its position, but need to understand it better\n -- to handle comments and inline tags.\n local opening_m, err = ngx_re_match(\n str_sub(markup, opening_f, opening_t),\n self.open_pattern(tag), \"soj\"\n )\n if not opening_m then\n if err then ngx_log(ngx_ERR, err) end\n return nil\n end\n\n -- We return a table with opening tag positions (absolute), as well as\n -- tag contents etc. Block level tags will have \"closing\" data too.\n local ret = {\n opening = {\n from = opening_f + self.pos,\n to = opening_t + self.pos,\n tag = str_sub(markup, opening_f, opening_t),\n },\n tagname = opening_m[1],\n closing = nil,\n contents = nil,\n }\n\n -- If this is an inline (non-block) tag, we have everything\n if type(opening_m[2]) == \"string\" and str_sub(opening_m[2], -2) == \"/>\" then\n ret.whole = str_sub(markup, opening_f, opening_t)\n return ret\n end\n\n -- We must be block level, and could potentially be nesting\n\n local search = opening_t -- We search from after the opening tag\n\n local f, t, closing_f, closing_t\n local depth = 1\n local level = 1\n\n repeat\n -- keep looking for opening or closing tags\n f, t = ngx_re_find(str_sub(markup, search + 1), self.either_pattern(ret.tagname), \"soj\")\n if f and t then\n -- Move closing markers along\n closing_f = f\n closing_t = t\n\n -- Track current level and total depth\n local tag = str_sub(markup, search + f, search + t)\n if ngx_re_find(tag, self.open_pattern(ret.tagname)) then\n depth = depth + 1\n level = level + 1\n elseif ngx_re_find(tag, self.close_pattern(ret.tagname)) then\n level = level - 1\n end\n -- Move search pos along\n search = search + t\n end\n until level == 0 or not f\n\n if closing_t and t then\n -- We have a complete block tag with the matching closing tag\n\n -- Make closing tag absolute\n closing_t = closing_t + search - t\n closing_f = closing_f + search - t\n\n ret.closing = {\n from = closing_f + self.pos,\n to = closing_t + self.pos,\n tag = str_sub(markup, closing_f, closing_t),\n }\n ret.contents = str_sub(markup, opening_t + 1, closing_f - 1)\n ret.whole = str_sub(markup, opening_f, closing_t)\n\n return ret\n else\n -- We have an opening block tag, but not the closing part. Return\n -- what we can as the filters will buffer until we find the rest.\n return ret\n end\nend\n\n\nreturn _M\n" }, { "path": "lib/ledge/esi.lua", "content": "local h_util = require \"ledge.header_util\"\n\nlocal type, tonumber = type, tonumber\n\nlocal str_sub = string.sub\nlocal str_find = string.find\n\nlocal tbl_concat = table.concat\nlocal tbl_insert = table.insert\n\nlocal ngx_re_match = ngx.re.match\nlocal ngx_req_get_headers = ngx.req.get_headers\nlocal ngx_req_get_uri_args = ngx.req.get_uri_args\nlocal ngx_encode_args = ngx.encode_args\nlocal ngx_req_set_uri_args = ngx.req.set_uri_args\nlocal ngx_var = ngx.var\nlocal ngx_log = ngx.log\nlocal ngx_ERR = ngx.ERR\n\n\nlocal _M = {\n _VERSION = \"2.3.0\",\n}\n\n\nlocal esi_processors = {\n [\"ESI\"] = {\n [\"1.0\"] = require \"ledge.esi.processor_1_0\",\n -- 2.0 = require ledge.esi.processor_2_0\", -- for example\n },\n}\n\n\nfunction _M.split_esi_token(token)\n if token then\n local m = ngx_re_match(\n token,\n [[^([A-Za-z0-9-_]+)\\/(\\d+\\.?\\d+)$]],\n \"oj\"\n )\n if m then\n return m[1], tonumber(m[2])\n end\n end\nend\n\n\nfunction _M.esi_capabilities()\n local capabilities = {}\n for processor_type,processors in pairs(esi_processors) do\n for version,_ in pairs(processors) do\n tbl_insert(capabilities, processor_type .. \"/\" .. version)\n end\n end\n return tbl_concat(capabilities, \" \")\nend\n\n\n-- Returns a processor instance based on Surrogate-Control header\nfunction _M.choose_esi_processor(handler)\n local res = handler.response\n local res_surrogate_control = res.header[\"Surrogate-Control\"]\n\n if res_surrogate_control then\n -- Get the token value (e.g. \"ESI/1.0\")\n local content_token =\n h_util.get_header_token(res_surrogate_control, \"content\")\n\n if content_token then\n local processor_token, version = _M.split_esi_token(content_token)\n\n if processor_token and version then\n -- Lookup the prcoessor\n local processor_type = esi_processors[processor_token]\n\n if processor_type then\n for v,processor in pairs(processor_type) do\n if tonumber(version) <= tonumber(v) then\n return processor.new(handler)\n end\n end\n end\n end\n end\n end\nend\n\n\n-- Returns true of res.header.Content-Type is in allowed_types\nfunction _M.is_allowed_content_type(res, allowed_types)\n if allowed_types and type(allowed_types) == \"table\" then\n local res_content_type = res.header[\"Content-Type\"]\n if res_content_type then\n for _, content_type in ipairs(allowed_types) do\n local sep = str_find(res_content_type, \";\")\n if sep then sep = sep - 1 end\n if str_sub(res_content_type, 1, sep) == content_type then\n return true\n end\n end\n end\n end\nend\n\n\n-- Returns true if we're allowed to delegate ESI processing to a downstream\n-- surrogate for the current request\nfunction _M.can_delegate_to_surrogate(surrogates, processor_token)\n local surrogate_capability = ngx_req_get_headers()[\"Surrogate-Capability\"]\n\n if surrogate_capability then\n -- Surrogate-Capability: host.example.com=\"ESI/1.0\"\n local capability_token = h_util.get_header_token(\n surrogate_capability,\n \"[!#\\\\$%&'\\\\*\\\\+\\\\-.\\\\^_`\\\\|~0-9a-zA-Z]+\"\n )\n\n local capability_processor, capability_version =\n _M.split_esi_token(capability_token)\n\n if capability_processor and capability_version then\n local control_processor, control_version =\n _M.split_esi_token(processor_token)\n\n if control_processor and control_version\n and control_processor == capability_processor\n and control_version <= capability_version then\n\n if type(surrogates) == \"boolean\" then\n if surrogates == true then\n return true\n end\n elseif type(surrogates) == \"table\" then\n local remote_addr = ngx_var.remote_addr\n if remote_addr then\n for _, ip in ipairs(surrogates) do\n if ip == remote_addr then\n return true\n end\n end\n end\n end\n end\n end\n end\n\n return false\nend\n\n\nfunction _M.filter_esi_args(handler)\n local config = handler.config\n local esi_args_prefix = config.esi_args_prefix\n if esi_args_prefix then\n local args = ngx_req_get_uri_args(config.max_uri_args)\n local esi_args = {}\n local has_esi_args = false\n local non_esi_args = {}\n\n for k,v in pairs(args) do\n -- TODO: optimise\n -- If we have the prefix, extract the suffix\n local m, err = ngx_re_match(\n k,\n \"^\" .. esi_args_prefix .. \"(\\\\S+)\",\n \"oj\"\n )\n if err then ngx_log(ngx_ERR, err) end\n\n if m and m[1] then\n has_esi_args = true\n esi_args[m[1]] = v\n else\n -- Otherwise, this is a normal arg\n non_esi_args[k] = v\n end\n end\n\n if has_esi_args then\n -- Add them to ctx to be read by the esi processor, along with a\n -- __tostsring metamethod for the $(ESI_ARGS) string case\n ngx.ctx.__ledge_esi_args = setmetatable(esi_args, {\n __tostring = function(t)\n local args = {}\n for k,v in pairs(t) do\n args[esi_args_prefix .. k] = v\n end\n return ngx_encode_args(args)\n end\n })\n\n -- Set the request args to the ones left over\n ngx_req_set_uri_args(non_esi_args)\n end\n end\nend\n\n\nreturn _M\n" }, { "path": "lib/ledge/gzip.lua", "content": "local co_yield = coroutine.yield\nlocal co_wrap = require(\"ledge.util\").coroutine.wrap\n\nlocal ngx_log = ngx.log\nlocal ngx_ERR = ngx.ERR\n\nlocal zlib = require(\"ffi-zlib\")\n\n\nlocal _M = {\n _VERSION = \"2.3.0\",\n}\n\n\nlocal zlib_output = function(data)\n co_yield(data)\nend\n\n\nlocal function get_gzip_decoder(reader)\n return co_wrap(function(buffer_size)\n local ok, err = zlib.inflateGzip(reader, zlib_output, buffer_size)\n if not ok then\n ngx_log(ngx_ERR, err)\n end\n\n -- zlib decides it is done when the stream is complete.\n -- Call reader() one more time to resume the next coroutine in the\n -- chain.\n reader(buffer_size)\n end)\nend\n_M.get_gzip_decoder = get_gzip_decoder\n\n\nlocal function get_gzip_encoder(reader)\n return co_wrap(function(buffer_size)\n local ok, err = zlib.deflateGzip(reader, zlib_output, buffer_size)\n if not ok then\n ngx_log(ngx_ERR, err)\n end\n\n -- zlib decides it is done when the stream is complete.\n -- Call reader() one more time to resume the next coroutine in the\n -- chain\n reader(buffer_size)\n end)\nend\n_M.get_gzip_encoder = get_gzip_encoder\n\n\nreturn _M\n" }, { "path": "lib/ledge/handler.lua", "content": "local setmetatable, tostring, tonumber, pcall, type, ipairs, pairs, next, error =\n setmetatable, tostring, tonumber, pcall, type, ipairs, pairs, next, error\n\nlocal ngx_req_get_method = ngx.req.get_method\nlocal ngx_req_get_headers = ngx.req.get_headers\nlocal ngx_req_http_version = ngx.req.http_version\n\nlocal ngx_log = ngx.log\nlocal ngx_WARN = ngx.WARN\nlocal ngx_ERR = ngx.ERR\nlocal ngx_INFO = ngx.INFO\nlocal ngx_var = ngx.var\nlocal ngx_null = ngx.null\n\nlocal ngx_flush = ngx.flush\nlocal ngx_print = ngx.print\n\nlocal ngx_on_abort = ngx.on_abort\nlocal ngx_md5 = ngx.md5\n\nlocal ngx_time = ngx.time\nlocal ngx_http_time = ngx.http_time\nlocal ngx_parse_http_time = ngx.parse_http_time\n\nlocal str_lower = string.lower\nlocal str_len = string.len\nlocal tbl_insert = table.insert\nlocal tbl_concat = table.concat\n\nlocal esi_capabilities = require(\"ledge.esi\").esi_capabilities\n\nlocal append_server_port = require(\"ledge.util\").append_server_port\n\nlocal ledge_cache_key = require(\"ledge.cache_key\")\n\nlocal req_relative_uri = require(\"ledge.request\").relative_uri\nlocal req_full_uri = require(\"ledge.request\").full_uri\n\nlocal put_background_job = require(\"ledge.background\").put_background_job\nlocal gc_wait = require(\"ledge.background\").gc_wait\n\nlocal fixed_field_metatable = require(\"ledge.util\").mt.fixed_field_metatable\nlocal get_fixed_field_metatable_proxy =\n require(\"ledge.util\").mt.get_fixed_field_metatable_proxy\n\n\nlocal ledge = require(\"ledge\")\nlocal http = require(\"resty.http\")\nlocal http_headers = require(\"resty.http_headers\")\nlocal state_machine = require(\"ledge.state_machine\")\nlocal response = require(\"ledge.response\")\n\n\nlocal _M = {\n _VERSION = \"2.3.0\",\n}\n\n\n-- Creates a new handler instance.\n--\n-- Config defaults are provided in the ledge module, and so instances\n-- should always be created with ledge.create_handler(), not directly.\n--\n-- @param table The complete config table\n-- @return table Handler instance, or nil if no config table is provided\nlocal function new(config, events)\n if not config then return nil, \"config table expected\" end\n config = setmetatable(config, fixed_field_metatable)\n\n local self = setmetatable({\n -- public:\n config = config,\n events = events,\n upstream_client = {},\n\n -- Slots for composed objects\n redis = {},\n redis_subscriber = {},\n storage = {},\n state_machine = {},\n range = {},\n response = {},\n error_response = {},\n esi_processor = {},\n client_validators = {},\n\n output_buffers_enabled = true,\n esi_scan_enabled = false,\n esi_process_enabled = false,\n\n -- private:\n _root_key = \"\",\n _vary_key = ngx_null, -- empty string is not the same as not set\n _vary_spec = ngx_null, -- empty table is not the same as not set\n _cache_key_chain = {},\n _publish_key = \"\",\n\n }, get_fixed_field_metatable_proxy(_M))\n\n return self\nend\n_M.new = new\n\n\nlocal function run(self)\n -- Instantiate state machine\n local sm = state_machine.new(self)\n self.state_machine = sm\n\n -- Install the client abort handler\n local ok, err = ngx_on_abort(function()\n return self.state_machine:e \"aborted\"\n end)\n\n if not ok then\n ngx_log(ngx_WARN, \"on_abort handler could not be set: \" .. err)\n end\n\n -- Create Redis connection\n local redis, err = ledge.create_redis_connection()\n if not redis then\n return nil, \"could not connect to redis, \" .. tostring(err)\n else\n self.redis = redis\n end\n\n -- Create storage connection\n local config = self.config\n local storage, err = ledge.create_storage_connection(\n config.storage_driver,\n config.storage_driver_config\n )\n if not storage then\n return nil, \"could not connect to storage, \" .. tostring(err)\n else\n self.storage = storage\n end\n\n return sm:e \"init\"\nend\n_M.run = run\n\n\n-- Bind a user callback to an event\n--\n-- Callbacks will be called in the order they are bound\n--\n-- @param table self\n-- @param string event name\n-- @param function callback\n-- @return bool, string success, error\nlocal function bind(self, event, callback)\n local ev = self.events[event]\n if not ev then\n local err = \"no such event: \" .. tostring(event)\n ngx_log(ngx_ERR, err)\n return nil, err\n else\n tbl_insert(ev, callback)\n end\n return true, nil\nend\n_M.bind = bind\n\n\n-- Calls any registered callbacks for event, in the order they were bound\n-- Hard errors if event is not specified in self.events\nlocal function emit(self, event, ...)\n local ev = self.events[event]\n if not ev then\n error(\"attempt to emit non existent event: \" .. tostring(event), 2)\n end\n\n for _, handler in ipairs(ev) do\n if type(handler) == \"function\" then\n local ok, err = pcall(handler, ...)\n if not ok then\n ngx_log(ngx_ERR,\n \"error in user callback for '\", event, \"': \", err)\n end\n end\n end\n\n return true\nend\n_M.emit = emit\n\n\nfunction _M.entity_id(self, key_chain)\n if not key_chain or not key_chain.main then return nil end\n\n local entity_id, err = self.redis:hget(key_chain.main, \"entity\")\n if not entity_id or entity_id == ngx_null then\n return nil, err\n end\n\n return entity_id\nend\n\n\nlocal function root_key(self)\n if self._root_key == \"\" then\n self._root_key = ledge_cache_key.generate_root_key(\n self.config.cache_key_spec,\n self.config.max_uri_args\n )\n end\n\n return self._root_key\nend\n_M.root_key = root_key\n\n\nlocal function vary_spec(self, root_key)\n if self._vary_spec == ngx_null then\n local vary_spec, err = ledge_cache_key.read_vary_spec(\n self.redis,\n root_key\n )\n if not vary_spec then\n ngx_log(ngx_ERR, \"Failed to read vary spec: \", err)\n return false\n end\n self._vary_spec = vary_spec\n end\n\n return self._vary_spec\nend\n_M.vary_spec = vary_spec\n\n\nlocal function create_vary_key_callback(self)\n return function(vary_key)\n -- TODO: gunzip?\n emit(self, \"before_vary_selection\", vary_key)\n end\nend\n_M.create_vary_key_callback = create_vary_key_callback\n\n\nlocal function vary_key(self, vary_spec)\n if self._vary_key == ngx_null then\n self._vary_key = ledge_cache_key.generate_vary_key(\n vary_spec,\n create_vary_key_callback(self)\n )\n end\n\n return self._vary_key\nend\n_M.vary_key = vary_key\n\n\nlocal function cache_key_chain(self)\n if not next(self._cache_key_chain) then\n if not self.redis or not next(self.redis) then\n ngx_log(ngx_ERR, \"Cannot get cache key without a redis connection\")\n return nil\n end\n\n local rk = root_key(self)\n\n local vs = vary_spec(self, rk)\n\n local vk = vary_key(self, vs)\n\n local chain, err = ledge_cache_key.key_chain(rk, vk, vs)\n\n if not chain then\n return nil, err\n end\n\n self._cache_key_chain = chain\n end\n\n return self._cache_key_chain\nend\n_M.cache_key_chain = cache_key_chain\n\n\nlocal function reset_cache_key(self)\n self._root_key = \"\"\n self._vary_key = ngx_null\n self._vary_spec = ngx_null\n self._cache_key_chain = {}\nend\n_M.reset_cache_key = reset_cache_key\n\n\nlocal function set_vary_spec(self, vary_spec)\n reset_cache_key(self)\n if vary_spec then\n self._vary_spec = vary_spec\n end\nend\n_M.set_vary_spec = set_vary_spec\n\n\nlocal function read_from_cache(self)\n local res, err = response.new(self)\n if not res then return nil, err end\n\n local ok, err = res:read()\n if err then\n -- Error, abort request\n ngx_log(ngx_ERR, \"could not read response: \", err)\n return self.state_machine:e \"http_internal_server_error\"\n end\n\n if not ok then\n return {} -- MISS\n end\n\n if res.size > 0 then\n local storage = self.storage\n\n -- Check storage has the entity, if not presume it has been evitcted\n -- and clean up\n if not storage:exists(res.entity_id) then\n local config = self.config\n put_background_job(\n \"ledge_gc\",\n \"ledge.jobs.collect_entity\",\n {\n entity_id = res.entity_id,\n storage_driver = config.storage_driver,\n storage_driver_config = config.storage_driver_config,\n },\n {\n delay = gc_wait(\n res.size,\n config.minimum_old_entity_download_rate\n ),\n tags = { \"collect_entity\" },\n priority = 10,\n }\n )\n return {} -- MISS\n end\n\n res:filter_body_reader(\"cache_body_reader\", storage:get_reader(res))\n end\n\n emit(self, \"after_cache_read\", res)\n return res\nend\n_M.read_from_cache = read_from_cache\n\n\n-- http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.5.1\nlocal hop_by_hop_headers = {\n [\"connection\"] = true,\n [\"keep-alive\"] = true,\n [\"proxy-authenticate\"] = true,\n [\"proxy-authorization\"] = true,\n [\"te\"] = true,\n [\"trailers\"] = true,\n [\"transfer-encoding\"] = true,\n [\"upgrade\"] = true,\n [\"content-length\"] = true, -- Not strictly hop-by-hop, but we\n -- set dynamically downstream.\n}\n\n\n-- Fetches a resource from the origin server.\nlocal function fetch_from_origin(self)\n local res, err = response.new(self)\n if not res then return nil, err end\n\n local method = ngx['HTTP_' .. ngx_req_get_method()]\n if not method then\n res.status = ngx.HTTP_METHOD_NOT_IMPLEMENTED\n return res\n end\n\n emit(self, \"before_upstream_connect\", self)\n\n local config = self.config\n\n if not next(self.upstream_client) then\n local httpc = http.new()\n httpc:set_timeouts(\n config.upstream_connect_timeout,\n config.upstream_send_timeout,\n config.upstream_read_timeout\n )\n\n local port = tonumber(config.upstream_port)\n local ok, err\n if port then\n ok, err = httpc:connect(config.upstream_host, port)\n else\n ok, err = httpc:connect(config.upstream_host)\n end\n\n if not ok then\n ngx_log(ngx_ERR, \"upstream connection failed: \", err)\n if err == \"timeout\" then\n res.status = 524 -- upstream server timeout\n else\n res.status = 503\n end\n return res\n end\n\n if config.upstream_use_ssl == true then\n -- treat an empty (\"\") ssl_server_name as nil\n local ssl_server_name = config.upstream_ssl_server_name\n if type(ssl_server_name) ~= \"string\" or\n str_len(ssl_server_name) == 0 then\n\n ssl_server_name = nil\n end\n\n local ok, err = httpc:ssl_handshake(\n false,\n ssl_server_name,\n config.upstream_ssl_verify\n )\n\n if not ok then\n ngx_log(ngx_ERR, \"ssl handshake failed: \", err)\n res.status = 525 -- SSL Handshake Failed\n return res\n end\n end\n self.upstream_client = httpc\n end\n\n local upstream_client = self.upstream_client\n\n -- Case insensitve headers so that we can safely manipulate them\n local headers = http_headers.new()\n for k,v in pairs(ngx_req_get_headers()) do\n headers[k] = v\n end\n\n -- Advertise ESI surrogate capabilities\n if config.esi_enabled then\n local capability_entry = self.config.visible_hostname .. '=\"'\n .. esi_capabilities() .. '\"'\n\n local sc = headers[\"Surrogate-Capability\"]\n\n if not sc then\n headers[\"Surrogate-Capability\"] = capability_entry\n else\n headers[\"Surrogate-Capability\"] = sc .. \", \" .. capability_entry\n end\n end\n\n local client_body_reader, err =\n upstream_client:get_client_body_reader(config.buffer_size)\n\n if err then\n ngx_log(ngx_ERR, \"error getting client body reader: \", err)\n end\n\n local req_params = {\n method = ngx_req_get_method(),\n path = req_relative_uri(),\n body = client_body_reader,\n headers = headers,\n }\n\n -- allow request params to be customised\n emit(self, \"before_upstream_request\", req_params)\n\n local origin, err = upstream_client:request(req_params)\n\n if not origin then\n ngx_log(ngx_ERR, err)\n res.status = 524\n return res\n end\n\n res.status = origin.status\n\n -- Merge end-to-end headers\n local hop_by_hop_headers = hop_by_hop_headers\n for k,v in pairs(origin.headers) do\n if not hop_by_hop_headers[str_lower(k)] then\n res.header[k] = v\n end\n end\n\n -- May well be nil (we set to false if that's the case), but if present\n -- we bail on saving large bodies to memory nice and early.\n res.length = tonumber(origin.headers[\"Content-Length\"]) or false\n\n res.has_body = origin.has_body\n res:filter_body_reader(\n \"upstream_body_reader\",\n origin.body_reader\n )\n\n if res.status < 500 then\n -- http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.18\n -- A received message that does not have a Date header field MUST be\n -- assigned one by the recipient if the message will be cached by that\n -- recipient\n if type(res.header[\"Date\"]) ~= \"string\" or\n not ngx_parse_http_time(res.header[\"Date\"]) then\n\n ngx_log(ngx_WARN,\n \"Missing or invalid Date header from upstream, generating locally\"\n )\n res.header[\"Date\"] = ngx_http_time(ngx_time())\n end\n end\n\n -- A nice opportunity for post-fetch / pre-save work.\n emit(self, \"after_upstream_request\", res)\n\n return res\nend\n_M.fetch_from_origin = fetch_from_origin\n\n\n-- Returns data required to perform a background revalidation for this current\n-- request, as two tables; reval_params and reval_headers.\nlocal function revalidation_data(self)\n -- Everything that a headless revalidation job would need to connect\n local config = self.config\n local reval_params = {\n server_addr = ngx_var.server_addr,\n server_port = ngx_var.server_port,\n scheme = ngx_var.scheme,\n uri = ngx_var.request_uri,\n connect_timeout = config.upstream_connect_timeout,\n send_timeout = config.upstream_send_timeout,\n read_timeout = config.upstream_read_timeout,\n keepalive_timeout = config.upstream_keepalive_timeout,\n keepalive_poolsize = config.upstream_keepalive_poolsize,\n }\n\n local h = ngx_req_get_headers()\n\n -- By default we pass through Host, and Authorization and Cookie headers\n -- if present.\n local reval_headers = {\n host = h[\"Host\"],\n }\n\n if h[\"Authorization\"] then\n reval_headers[\"Authorization\"] = h[\"Authorization\"]\n end\n if h[\"Cookie\"] then\n reval_headers[\"Cookie\"] = h[\"Cookie\"]\n end\n\n emit(self, \"before_save_revalidation_data\", reval_params, reval_headers)\n\n return reval_params, reval_headers\nend\n\n\nlocal function revalidate_in_background(self, key_chain, update_revalidation_data)\n local redis = self.redis\n\n -- Revalidation data is updated if this is a proper request, but not if\n -- it's a purge request.\n if update_revalidation_data then\n local reval_params, reval_headers = revalidation_data(self)\n\n local ttl, err = redis:ttl(key_chain.reval_params)\n if not ttl or ttl == ngx_null or ttl < 0 then\n if err then ngx_log(ngx_ERR, err) end\n ngx_log(ngx_INFO,\n \"Could not determine expiry for revalidation params. \" ..\n \"Will fallback to 3600 seconds.\"\n )\n -- Arbitrarily expire these revalidation parameters in an hour.\n ttl = 3600\n end\n\n -- Delete and update reval request headers\n local _, e\n _, e = redis:multi()\n if e then ngx_log(ngx_ERR, e) end\n _, e = redis:del(key_chain.reval_params)\n if e then ngx_log(ngx_ERR, e) end\n _, e = redis:hmset(key_chain.reval_params, reval_params)\n if e then ngx_log(ngx_ERR, e) end\n _, e = redis:expire(key_chain.reval_params, ttl)\n if e then ngx_log(ngx_ERR, e) end\n\n _, e = redis:del(key_chain.reval_req_headers)\n if e then ngx_log(ngx_ERR, e) end\n _, e = redis:hmset(key_chain.reval_req_headers, reval_headers)\n if e then ngx_log(ngx_ERR, e) end\n _, e = redis:expire(key_chain.reval_req_headers, ttl)\n if e then ngx_log(ngx_ERR, e) end\n\n local res, err = redis:exec()\n if not res or res == ngx_null then\n ngx_log(ngx_ERR, \"Could not update revalidation params: \", err)\n end\n end\n\n local uri, err = redis:hget(key_chain.main, \"uri\")\n if not uri or uri == ngx_null then\n if err then\n ngx_log(ngx_ERR, \"Failed to get main key while revalidating: \", err)\n else\n ngx_log(ngx_WARN,\n \"Cache key has no 'uri' field, aborting revalidation\"\n )\n end\n return nil\n end\n\n -- Schedule the background job (immediately). jid is a function of the\n -- URI for automatic de-duping.\n return put_background_job(\n \"ledge_revalidate\",\n \"ledge.jobs.revalidate\",\n { key_chain = key_chain },\n {\n jid = ngx_md5(\"revalidate:\" .. uri),\n tags = { \"revalidate\" },\n priority = 4,\n }\n )\nend\n_M.revalidate_in_background = revalidate_in_background\n\n\n-- Starts a \"revalidation\" job but maybe for brand new cache. We pass the\n-- current request's revalidation data through so that the job has meaninful\n-- parameters to work with (rather than using stored metadata).\nlocal function fetch_in_background(self)\n local key_chain = cache_key_chain(self)\n local reval_params, reval_headers = revalidation_data(self)\n return put_background_job(\n \"ledge_revalidate\",\n \"ledge.jobs.revalidate\",\n {\n key_chain = key_chain,\n reval_params = reval_params,\n reval_headers = reval_headers,\n },\n {\n jid = ngx_md5(\"revalidate:\" .. req_full_uri()),\n tags = { \"revalidate\" },\n priority = 4,\n }\n )\nend\n_M.fetch_in_background = fetch_in_background\n\n\nlocal function save_to_cache(self, res)\n if not res then return nil, \"no response to save\" end\n emit(self, \"before_save\", res)\n\n -- Length is only set if there was a Content-Length header\n local length = res.length\n local storage = self.storage\n local max_size = storage:get_max_size()\n if length and length > max_size then\n -- We'll carry on serving, just not saving.\n return nil, \"advertised length is greated than storage max size\"\n end\n\n\n -- Watch the main key pointer. We abort the transaction if another request\n -- updates this key before we finish.\n local key_chain = cache_key_chain(self)\n local redis = self.redis\n redis:watch(key_chain.main)\n\n local repset_ttl = redis:ttl(key_chain.repset)\n\n -- We'll need to mark the old entity for expiration shortly, as reads\n -- could still be in progress. We need to know the previous entity keys\n -- and the size.\n local previous_entity_id = self:entity_id(key_chain)\n\n local previous_entity_size, err, gc_job_spec\n if previous_entity_id then\n previous_entity_size, err = redis:hget(key_chain.main, \"size\")\n if previous_entity_size == ngx_null then\n previous_entity_id = nil\n if err then\n ngx_log(ngx_ERR, err)\n end\n end\n\n -- Define GC job here, used later if required\n gc_job_spec = {\n \"ledge_gc\",\n \"ledge.jobs.collect_entity\",\n {\n entity_id = previous_entity_id,\n storage_driver = self.config.storage_driver,\n storage_driver_config = self.config.storage_driver_config,\n },\n {\n delay = gc_wait(\n previous_entity_size,\n self.config.minimum_old_entity_download_rate\n ),\n tags = { \"collect_entity\" },\n priority = 10,\n }\n }\n end\n\n -- Start the transaction\n local ok, err = redis:multi()\n if not ok then ngx_log(ngx_ERR, err) end\n\n if previous_entity_id then\n local ok, err = redis:srem(key_chain.entities, previous_entity_id)\n if not ok then ngx_log(ngx_ERR, err) end\n end\n\n res.uri = req_full_uri()\n\n local keep_cache_for = self.config.keep_cache_for\n local ok, err = res:save(keep_cache_for)\n if not ok then ngx_log(ngx_ERR, err) end\n\n -- Set revalidation parameters from this request\n local reval_params, reval_headers = revalidation_data(self)\n\n local _, err = redis:del(key_chain.reval_params)\n if err then ngx_log(ngx_ERR, err) end\n _, err = redis:hmset(key_chain.reval_params, reval_params)\n if err then ngx_log(ngx_ERR, err) end\n\n _, err = redis:del(key_chain.reval_req_headers)\n if err then ngx_log(ngx_ERR, err) end\n _, err = redis:hmset(key_chain.reval_req_headers, reval_headers)\n if err then ngx_log(ngx_ERR, err) end\n\n local expiry = res:ttl() + keep_cache_for\n redis:expire(key_chain.reval_params, expiry)\n redis:expire(key_chain.reval_req_headers, expiry)\n\n\n -- repset and vary TTL should be the same as the longest living represenation\n if repset_ttl < expiry then\n repset_ttl = expiry\n end\n\n -- Save updates to cache key\n ledge_cache_key.save_key_chain(redis, key_chain, repset_ttl)\n\n -- If we have a body, we need to attach the storage writer\n -- NOTE: res.has_body is false for known bodyless repsonse types\n -- (e.g. HEAD) but may be true and of zero length (commonly 301 etc).\n if res.has_body then\n\n -- Storage callback for write success\n local function onsuccess(bytes_written)\n -- Update size in metadata\n local ok, e = redis:hset(key_chain.main, \"size\", bytes_written)\n if not ok or ok == ngx_null then ngx_log(ngx_ERR, e) end\n\n if bytes_written == 0 then\n -- Remove the entity as it wont exist\n ok, e = redis:srem(key_chain.entities, res.entity_id)\n if not ok or ok == ngx_null then ngx_log(ngx_ERR, e) end\n\n ok, e = redis:hdel(key_chain.main, \"entity\")\n if not ok or ok == ngx_null then ngx_log(ngx_ERR, e) end\n end\n\n ok, e = redis:exec()\n if not ok or ok == ngx_null then\n if e then\n ngx_log(ngx_ERR, \"failed to complete transaction: \", e)\n else\n -- Transaction likely failed due to watch on main key\n -- Tell storage to clean up too\n ok, e = storage:delete(res.entity_id) -- luacheck: ignore ok\n if e then\n ngx_log(ngx_ERR, \"failed to cleanup storage: \", e)\n end\n end\n elseif previous_entity_id then\n -- Everything has completed and we have an old entity\n -- Schedule GC to clean it up\n put_background_job(unpack(gc_job_spec))\n end\n end\n\n -- Storage callback for write failure. We roll back our transaction.\n local function onfailure(reason)\n ngx_log(ngx_ERR, \"storage failed to write: \", reason)\n\n local ok, e = redis:discard()\n if not ok or ok == ngx_null then ngx_log(ngx_ERR, e) end\n end\n\n -- Attach storage writer\n local ok, writer = pcall(storage.get_writer, storage,\n res,\n keep_cache_for,\n onsuccess,\n onfailure\n )\n if not ok then\n ngx_log(ngx_ERR, writer)\n else\n res:filter_body_reader(\"cache_body_writer\", writer)\n end\n\n else\n -- No body and thus no storage filter\n -- We can run our transaction immediately\n local ok, e = redis:exec()\n if not ok or ok == ngx_null then\n ngx_log(ngx_ERR, \"failed to complete transaction: \", e)\n elseif previous_entity_id then\n -- Everything has completed and we have an old entity\n -- Schedule GC to clean it up\n put_background_job(unpack(gc_job_spec))\n end\n end\n return true\nend\n_M.save_to_cache = save_to_cache\n\n\nlocal function delete_from_cache(self, key_chain)\n local redis = self.redis\n\n -- Get entity_id if not already provided\n local entity_id = self:entity_id(key_chain)\n\n -- Schedule entity collection\n if entity_id then\n local config = self.config\n local size = redis:hget(key_chain.main, \"size\")\n put_background_job(\n \"ledge_gc\",\n \"ledge.jobs.collect_entity\",\n {\n entity_id = entity_id,\n storage_driver = config.storage_driver,\n storage_driver_config = config.storage_driver_config,\n },\n {\n delay = gc_wait(\n size,\n config.minimum_old_entity_download_rate\n ),\n tags = { \"collect_entity\" },\n priority = 10,\n }\n )\n end\n\n -- Remove this representation from the repset\n redis:srem(key_chain.repset, key_chain.full)\n\n -- Delete everything in the keychain\n local keys = {}\n for _, v in pairs(key_chain) do\n tbl_insert(keys, v)\n end\n\n -- If there are no more entries in the repset clean up the vary key too\n local exists = redis:exists(key_chain.repset)\n if exists == 0 then\n tbl_insert(keys, key_chain.vary)\n end\n\n return redis:del(unpack(keys))\nend\n_M.delete_from_cache = delete_from_cache\n\n\n-- Resumes the reader coroutine and prints the data yielded. This could be\n-- via a cache read, or a save via a fetch... the interface is uniform.\nlocal function serve_body(self, res, buffer_size)\n local buffered = 0\n local reader = res.body_reader\n local can_flush = ngx_req_http_version() >= 1.1\n\n repeat\n local chunk, err = reader(buffer_size)\n if err then ngx_log(ngx_ERR, err) end\n if chunk and self.output_buffers_enabled then\n local ok, err = ngx_print(chunk)\n if not ok then ngx_log(ngx_INFO, err) end\n\n -- Flush each full buffer, if we can\n buffered = buffered + #chunk\n if can_flush and buffered >= buffer_size then\n local ok, err = ngx_flush(true)\n if not ok then ngx_log(ngx_INFO, err) end\n\n buffered = 0\n end\n end\n\n until not chunk\nend\n\n\nlocal function serve(self)\n if not ngx.headers_sent then\n local res = self.response\n local name = append_server_port(self.config.visible_hostname)\n\n -- Via header\n local via = \"1.1 \" .. name\n if self.config.advertise_ledge then\n via = via .. \" (ledge/\" .. _M._VERSION .. \")\"\n end\n\n -- Append upstream Via\n local res_via = res.header[\"Via\"]\n if (res_via ~= nil) then\n -- Fix multiple upstream Via headers into list form\n if (type(res_via) == \"table\") then\n res.header[\"Via\"] = via .. \", \" .. tbl_concat(res_via, \", \")\n else\n res.header[\"Via\"] = via .. \", \" .. res_via\n end\n else\n res.header[\"Via\"] = via\n end\n\n -- X-Cache header\n -- Don't set if this isn't a cacheable response. Set to MISS is we\n -- fetched.\n local state_history = self.state_machine.state_history\n local event_history = self.state_machine.event_history\n\n if not event_history[\"response_not_cacheable\"] then\n local x_cache = \"HIT from \" .. name\n if not event_history[\"can_serve_disconnected\"]\n and not event_history[\"can_serve_stale\"]\n and state_history[\"fetching\"] then\n\n x_cache = \"MISS from \" .. name\n end\n\n local res_x_cache = res.header[\"X-Cache\"]\n\n if res_x_cache ~= nil then\n res.header[\"X-Cache\"] = x_cache .. \", \" .. res_x_cache\n else\n res.header[\"X-Cache\"] = x_cache\n end\n end\n\n emit(self, \"before_serve\", res)\n\n if res.header then\n for k,v in pairs(res.header) do\n ngx.header[k] = v\n end\n end\n\n if res.body_reader and ngx_req_get_method() ~= \"HEAD\" then\n local buffer_size = self.config.buffer_size\n serve_body(self, res, buffer_size)\n end\n\n ngx.eof()\n end\nend\n_M.serve = serve\n\n\nlocal function add_warning(self, code)\n return self.response:add_warning(\n code,\n append_server_port(self.config.visible_hostname)\n )\nend\n_M.add_warning = add_warning\n\n\nreturn setmetatable(_M, fixed_field_metatable)\n" }, { "path": "lib/ledge/header_util.lua", "content": "local type, tonumber, setmetatable =\n type, tonumber, setmetatable\n\nlocal ngx_re_match = ngx.re.match\nlocal ngx_re_find = ngx.re.find\nlocal tbl_concat = table.concat\n\n\nlocal _M = {\n _VERSION = \"2.3.0\"\n}\n\nlocal mt = {\n __index = _M,\n}\n\n\n-- Returns true if the directive appears in the header field value.\n-- Set without_token to true to only return bare directives - i.e.\n-- directives appearing with no =value part.\nfunction _M.header_has_directive(header, directive, without_token)\n if header then\n if type(header) == \"table\" then header = tbl_concat(header, \", \") end\n\n local pattern = [[(?:\\s*|,?)(]] .. directive .. [[)\\s*(?:$|=|,)]]\n if without_token then\n pattern = [[(?:\\s*|,?)(]] .. directive .. [[)\\s*(?:$|,)]]\n end\n\n return ngx_re_find(header, pattern, \"ioj\") ~= nil\n end\n return false\nend\n\n\nfunction _M.get_header_token(header, directive)\n if _M.header_has_directive(header, directive) then\n if type(header) == \"table\" then header = tbl_concat(header, \", \") end\n\n -- Want the string value from a token\n local value = ngx_re_match(\n header,\n directive .. [[=\"?([a-z0-9_~!#%&/',`\\$\\*\\+\\-\\|\\^\\.]+)\"?]],\n \"ioj\"\n )\n if value ~= nil then\n return value[1]\n end\n return nil\n end\n return nil\nend\n\n\nfunction _M.get_numeric_header_token(header, directive)\n if _M.header_has_directive(header, directive) then\n if type(header) == \"table\" then header = tbl_concat(header, \", \") end\n\n -- Want the numeric value from a token\n local value = ngx_re_match(\n header,\n directive .. [[=\"?(\\d+)\"?]], \"ioj\"\n )\n if value ~= nil then\n return tonumber(value[1])\n end\n end\nend\n\nreturn setmetatable(_M, mt)\n" }, { "path": "lib/ledge/jobs/collect_entity.lua", "content": "local tostring = tostring\nlocal ngx_null = ngx.null\n\nlocal create_storage_connection = require(\"ledge\").create_storage_connection\n\n\nlocal _M = {\n _VERSION = \"2.3.0\",\n}\n\n\n-- Cleans up expired items and keeps track of memory usage.\nfunction _M.perform(job)\n local storage, err = create_storage_connection(\n job.data.storage_driver,\n job.data.storage_driver_config\n )\n\n if not storage then\n return nil, \"job-error\", \"could not connect to storage driver: \"..tostring(err)\n end\n\n local ok, err = storage:delete(job.data.entity_id)\n storage:close()\n\n if ok == nil or ok == ngx_null then\n return nil, \"job-error\", tostring(err)\n end\nend\n\n\nreturn _M\n" }, { "path": "lib/ledge/jobs/purge.lua", "content": "local ipairs, tonumber = ipairs, tonumber\nlocal ngx_log = ngx.log\nlocal ngx_DEBUG = ngx.DEBUG\nlocal ngx_ERR = ngx.ERR\nlocal ngx_null = ngx.null\n\nlocal purge = require(\"ledge.purge\").purge\nlocal create_redis_slave_connection = require(\"ledge\").create_redis_slave_connection\nlocal close_redis_connection = require(\"ledge\").close_redis_connection\n\nlocal _M = {\n _VERSION = \"2.3.0\",\n}\n\n\n-- Scans the keyspace for keys which match, and expires them. We do this against\n-- the slave Redis instance if available.\nfunction _M.perform(job)\n if not job.redis then\n return nil, \"job-error\", \"no redis connection provided\"\n end\n\n local slave, _ = create_redis_slave_connection()\n if not slave then\n job.redis_slave = job.redis\n else\n job.redis_slave = slave\n end\n\n -- Setup handler\n local handler = require(\"ledge\").create_handler()\n handler.redis = job.redis\n\n local storage, err = require(\"ledge\").create_storage_connection(\n job.data.storage_driver,\n job.data.storage_driver_config\n )\n if not storage then\n return nil, \"redis-error\", err\n end\n\n handler.storage = storage\n\n -- This runs recursively using the SCAN cursor, until the entire keyspace\n -- has been scanned.\n local res, err = _M.expire_pattern(0, job, handler)\n\n if slave then\n close_redis_connection(slave)\n end\n\n if not res then\n return nil, \"redis-error\", err\n end\nend\n\n\n-- Scans the keyspace based on a pattern (asterisk), and runs a purge for each cache entry\nfunction _M.expire_pattern(cursor, job, handler)\n if job:ttl() < 10 then\n if not job:heartbeat() then\n return nil, \"Failed to heartbeat job\"\n end\n end\n\n -- Scan using the \"main\" key to get a single key per cache entry\n local res, err = job.redis_slave:scan(\n cursor,\n \"MATCH\", job.data.repset,\n \"COUNT\", job.data.keyspace_scan_count\n )\n\n if not res or res == ngx_null then\n return nil, \"SCAN error: \" .. tostring(err)\n else\n for _,key in ipairs(res[2]) do\n ngx_log(ngx_DEBUG, \"Purging set: \", key)\n\n local ok, err = purge(handler, job.data.purge_mode, key)\n if ok == nil and err then ngx_log(ngx_ERR, tostring(err)) end\n\n end\n\n local cursor = tonumber(res[1])\n if cursor == 0 then\n return true\n end\n\n -- If we have a valid cursor, recurse to move on.\n return _M.expire_pattern(cursor, job, handler)\n end\nend\n\n\nreturn _M\n" }, { "path": "lib/ledge/jobs/revalidate.lua", "content": "local http = require \"resty.http\"\nlocal http_headers = require \"resty.http_headers\"\nlocal ngx_null = ngx.null\n\nlocal _M = {\n _VERSION = \"2.3.0\",\n}\n\n\n-- Utility to return all items in a Redis hash as a Lua table.\nlocal function hgetall(redis, key)\n local res, err = redis:hgetall(key)\n if not res or res == ngx_null then\n return nil,\n \"could not retrieve \" .. tostring(key) .. \" data:\" .. tostring(err)\n end\n\n return redis:array_to_hash(res)\nend\n\n\nfunction _M.perform(job)\n -- Normal background revalidation operates on stored metadata.\n -- A background fetch due to partial content from upstream however, uses the\n -- current request metadata for reval_headers / reval_params and passes it\n -- through as job data.\n local reval_params = job.data.reval_params\n local reval_headers = job.data.reval_headers\n\n -- If we don't have the metadata in job data, this is a background\n -- revalidation using stored metadata.\n if not reval_params and not reval_headers then\n local key_chain, redis, err = job.data.key_chain, job.redis\n\n reval_params, err = hgetall(redis, key_chain.reval_params)\n if not reval_params or not next(reval_params) then\n return nil, \"job-error\",\n \"Revalidation parameters are missing, presumed evicted. \" ..\n tostring(err)\n end\n\n reval_headers, err = hgetall(redis, key_chain.reval_req_headers)\n if not reval_headers or not next(reval_headers) then\n return nil, \"job-error\",\n \"Revalidation headers are missing, presumed evicted.\" ..\n tostring(err)\n end\n end\n\n -- Make outbound http request to revalidate\n local httpc = http.new()\n httpc:set_timeouts(\n reval_params.connect_timeout,\n reval_params.send_timeout,\n reval_params.read_timeout\n )\n\n local port = tonumber(reval_params.server_port)\n local ok, err\n if port then\n ok, err = httpc:connect(reval_params.server_addr, port)\n else\n ok, err = httpc:connect(reval_params.server_addr)\n end\n\n if not ok then\n return nil, \"job-error\",\n \"could not connect to server: \" .. tostring(err)\n end\n\n if reval_params.scheme == \"https\" then\n local ok, err = httpc:ssl_handshake(false, nil, false)\n if not ok then\n return nil, \"job-error\", \"ssl handshake failed: \" .. tostring(err)\n end\n end\n\n local headers = http_headers.new() -- Case-insensitive header table\n headers[\"Cache-Control\"] = \"max-stale=0, stale-if-error=0\"\n headers[\"User-Agent\"] =\n httpc._USER_AGENT .. \" ledge_revalidate/\" .. _M._VERSION\n\n -- Add additional headers from parent\n for k,v in pairs(reval_headers) do\n headers[k] = v\n end\n\n local res, err = httpc:request{\n method = \"GET\",\n path = reval_params.uri,\n headers = headers,\n }\n\n if not res then\n return nil, \"job-error\", \"revalidate failed: \" .. tostring(err)\n else\n local reader = res.body_reader\n -- Read and discard the body\n repeat\n local chunk, _ = reader()\n until not chunk\n\n httpc:set_keepalive(\n reval_params.keepalive_timeout,\n reval_params.keepalive_poolsize\n )\n end\nend\n\n\nreturn _M\n" }, { "path": "lib/ledge/purge.lua", "content": "local pcall, tonumber, tostring, pairs =\n pcall, tonumber, tostring, pairs\n\nlocal tbl_insert = table.insert\n\nlocal ngx_var = ngx.var\nlocal ngx_log = ngx.log\nlocal ngx_ERR = ngx.ERR\nlocal ngx_null = ngx.null\nlocal ngx_time = ngx.time\nlocal ngx_md5 = ngx.md5\nlocal ngx_HTTP_BAD_REQUEST = ngx.HTTP_BAD_REQUEST\n\nlocal str_find = string.find\nlocal str_sub = string.sub\nlocal str_len = string.len\n\nlocal http = require(\"resty.http\")\n\nlocal cjson_encode = require(\"cjson\").encode\nlocal cjson_decode = require(\"cjson\").decode\n\nlocal fixed_field_metatable = require(\"ledge.util\").mt.fixed_field_metatable\nlocal put_background_job = require(\"ledge.background\").put_background_job\n\nlocal key_chain = require(\"ledge.cache_key\").key_chain\n\nlocal _M = {\n _VERSION = \"2.3.0\",\n}\n\nlocal repset_len = -(str_len(\"::repset\")+1)\n\n\nlocal function create_purge_response(purge_mode, result, qless_jobs)\n local d = {\n purge_mode = purge_mode,\n result = result,\n }\n if qless_jobs then d.qless_jobs = qless_jobs end\n\n local ok, json = pcall(cjson_encode, d)\n\n if not ok then\n return nil, json\n else\n return json\n end\nend\n_M.create_purge_response = create_purge_response\n\n\n-- Expires the keys in key_chain and reduces the ttl in storage\nlocal function expire_keys(redis, storage, key_chain, entity_id)\n local ttl, err = redis:ttl(key_chain.main)\n if not ttl or ttl == ngx_null or ttl == -1 then\n return nil, \"count not determine existing ttl: \" .. (err or \"\")\n end\n\n if ttl == -2 then\n -- Key doesn't exist, do nothing\n return false, nil\n end\n\n local expires, err = redis:hget(key_chain.main, \"expires\")\n expires = tonumber(expires)\n\n if not expires or expires == ngx_null then\n return nil, \"could not determine existing expiry: \" .. (err or \"\")\n end\n\n local time = ngx_time()\n\n -- If expires is in the past then this key is stale. Nothing to do here.\n if expires <= time then\n return false, nil\n end\n\n local ttl_reduction = expires - time\n if ttl_reduction < 0 then ttl_reduction = 0 end\n local new_ttl = ttl - ttl_reduction\n\n local _, e = redis:multi()\n if e then ngx_log(ngx_ERR, e) end\n\n -- Set the expires field of the main key to the new time, to control\n -- its validity.\n _, e = redis:hset(key_chain.main, \"expires\", tostring(time - 1))\n if e then ngx_log(ngx_ERR, e) end\n\n -- Set new TTLs for all keys in the key chain\n for _,key in pairs(key_chain) do\n local _, e = redis:expire(key, new_ttl)\n if e then ngx_log(ngx_ERR, e) end\n end\n\n -- Reduce TTL on entity if there is one\n if entity_id and entity_id ~= ngx_null then\n storage:set_ttl(entity_id, new_ttl)\n end\n\n local ok, err = redis:exec() -- luacheck: ignore ok\n if err then\n return nil, err\n else\n return true, nil\n end\nend\n_M.expire_keys = expire_keys\n\n-- Purges the cache item according to purge_mode which defaults to \"invalidate\".\n-- If there's nothing to do we return false which results in a 404.\n-- @param table handler instance\n-- @param string \"invalidate\" | \"delete\" | \"revalidate\n-- @param table key_chain to purge\n-- @return boolean success\n-- @return string message\n-- @return table qless job (for revalidate only)\nlocal function _purge(handler, purge_mode, key_chain)\n local redis = handler.redis\n local storage = handler.storage\n\n local exists, err = redis:exists(key_chain.main)\n if err then ngx_log(ngx_ERR, err) end\n\n -- We 404 if we have nothing\n if not exists or exists == ngx_null or exists == 0 then\n return false, \"nothing to purge\", nil\n end\n\n\n -- Delete mode overrides everything else, since you can't revalidate\n if purge_mode == \"delete\" then\n local res, err = handler:delete_from_cache(key_chain)\n if not res then\n return nil, err, nil\n else\n return true, \"deleted\", nil\n end\n end\n\n -- If we're revalidating, fire off the background job\n local job\n if purge_mode == \"revalidate\" then\n job = handler:revalidate_in_background(key_chain, false)\n end\n\n -- Invalidate the keys\n local ok, err = expire_keys(redis, storage, key_chain, handler:entity_id(key_chain))\n\n if not ok and err then\n return nil, err, job\n\n elseif not ok then\n return false, \"already expired\", job\n\n elseif ok then\n return true, \"purged\", job\n\n end\nend\n\n\nlocal function key_chain_from_full_key(root_key, full_key)\n local pos = str_find(full_key, \"#\")\n if pos == nil then\n return nil\n end\n\n -- Remove the root_key from the start\n local vary_key = str_sub(full_key, pos+1)\n local vary_spec = {} -- We don't need this\n\n return key_chain(root_key, vary_key, vary_spec)\nend\n\n\n-- Purges all representatinos of the cache item\nlocal function purge(handler, purge_mode, repset)\n local representations, err = handler.redis:smembers(repset)\n if err then\n return nil, err\n end\n\n if #representations == 0 then\n return false, \"nothing to purge\", nil\n end\n\n local root_key = str_sub(repset, 1, repset_len)\n\n local res_ok, res_message\n local jobs = {}\n\n local key_chain\n for _, full_key in ipairs(representations) do\n key_chain = key_chain_from_full_key(root_key, full_key)\n local ok, message, job = _purge(handler, purge_mode, key_chain)\n\n -- Set the overall response if any representation was purged\n if res_ok == nil or ok == true then\n res_ok = ok\n res_message = message\n end\n\n tbl_insert(jobs, job)\n end\n\n -- Clean up vary and repset keys if we're deleting\n if purge_mode == \"delete\" and res_ok then\n local _, e = handler.redis:del(key_chain.repset, key_chain.vary)\n if e then ngx_log(ngx_ERR, e) end\n end\n\n return res_ok, res_message, jobs\nend\n_M.purge = purge\n\n\nlocal function purge_in_background(handler, purge_mode)\n local key_chain = handler:cache_key_chain()\n\n local job, err = put_background_job(\n \"ledge_purge\",\n \"ledge.jobs.purge\",\n {\n repset = key_chain.repset,\n keyspace_scan_count = handler.config.keyspace_scan_count,\n purge_mode = purge_mode,\n storage_driver = handler.config.storage_driver,\n storage_driver_config = handler.config.storage_driver_config,\n },\n {\n jid = ngx_md5(\"purge:\" .. tostring(key_chain.root)),\n tags = { \"purge\" },\n priority = 5,\n }\n )\n if err then ngx_log(ngx_ERR, err) end\n\n -- Create a JSON payload for the response\n local res = create_purge_response(purge_mode, \"scheduled\", {job})\n handler.response:set_body(res)\n\n return true\nend\n_M.purge_in_background = purge_in_background\n\n\nlocal function parse_json_req()\n ngx.req.read_body()\n local body, err = ngx.req.get_body_data()\n if not body then\n return nil, \"Could not read request body: \" .. tostring(err)\n end\n\n local ok, req = pcall(cjson_decode, body)\n if not ok then\n return nil, \"Could not parse request body: \" .. tostring(req)\n end\n\n return req\nend\n\n\nlocal function validate_api_request(req)\n local uris = req[\"uris\"]\n if not uris then\n return false, \"No URIs provided\"\n end\n\n if type(uris) ~= \"table\" then\n return false, \"Field 'uris' must be an array\"\n end\n\n if #uris == 0 then\n return false, \"No URIs provided\"\n end\n\n local mode = req[\"purge_mode\"]\n if mode and not (\n mode == \"invalidate\"\n or mode == \"revalidate\"\n or mode == \"delete\"\n ) then\n return false, \"Invalid purge_mode\"\n end\n\n return true\nend\n\n\nlocal function send_purge_request(uri, purge_mode, headers)\n local uri_parts, err = http:parse_uri(uri)\n if not uri_parts then\n return nil, err\n end\n\n local scheme, host, port, path = unpack(uri_parts)\n\n -- TODO: timeouts\n local httpc = http.new()\n local ok, err = httpc:connect(ngx_var.server_addr, port)\n if not ok then\n return nil, \"HTTP Connect (\"..ngx_var.server_addr..\":\"..port..\"): \"..err\n end\n\n if scheme == \"https\" then\n local ok, err = httpc:ssl_handshake(nil, host, false)\n if not ok then\n return nil, \"SSL Handshake: \"..err\n end\n end\n\n headers = headers or {}\n headers[\"Host\"] = host\n headers[\"X-Purge\"] = purge_mode\n\n local res, err = httpc:request({\n method = \"PURGE\",\n path = path,\n headers = headers\n })\n\n if not res then\n return nil, \"HTTP Request: \"..err\n end\n\n local body, err = res:read_body()\n if not body then\n return nil, \"HTTP Response: \"..err\n end\n\n local ok, err = httpc:set_keepalive()\n if not ok then ngx_log(ngx_ERR, err) end\n\n if res.headers[\"Content-Type\"] == \"application/json\" then\n body = cjson_decode(body)\n else\n return nil, { status = res.status, body = body, headers = res.headers}\n end\n\n return body\nend\n\n\n-- Run the JSON PURGE API.\n-- Accepts various inputs from a JSON request body and processes purges\n-- Return true on success or false on error\nlocal function purge_api(handler)\n local response = handler.response\n\n local request, err = parse_json_req()\n if not request then\n response.status = ngx_HTTP_BAD_REQUEST\n response:set_body(cjson_encode({[\"error\"] = err}))\n return false\n end\n\n local ok, err = validate_api_request(request)\n if not ok then\n response.status = ngx_HTTP_BAD_REQUEST\n response:set_body(cjson_encode({[\"error\"] = err}))\n return false\n end\n\n local purge_mode = request[\"purge_mode\"] or \"invalidate\" -- Default to invalidating\n local api_results = {}\n\n local uris = request[\"uris\"]\n for _, uri in ipairs(uris) do\n local res, err = send_purge_request(uri, purge_mode, request[\"headers\"])\n if not res then\n res = {[\"error\"] = err}\n elseif type(res) == \"table\" then\n res[\"purge_mode\"] = nil\n end\n\n api_results[uri] = res\n end\n\n local api_response, err = create_purge_response(purge_mode, api_results)\n if not api_response then\n handler.set:body(cjson_encode({[\"error\"] = \"JSON Response Error: \"..tostring(err)}))\n return false\n end\n\n handler.response:set_body(api_response)\n return true\nend\n_M.purge_api = purge_api\n\n\nreturn setmetatable(_M, fixed_field_metatable)\n" }, { "path": "lib/ledge/range.lua", "content": "local setmetatable, tonumber, ipairs, type =\n setmetatable, tonumber, ipairs, type\n\nlocal str_match = string.match\nlocal str_sub = string.sub\nlocal str_randomhex = require(\"ledge.util\").string.randomhex\nlocal str_split = require(\"ledge.util\").string.split\n\nlocal tbl_insert = table.insert\nlocal tbl_sort = table.sort\nlocal tbl_remove = table.remove\nlocal tbl_concat = table.concat\n\nlocal ngx_re_match = ngx.re.match\nlocal ngx_log = ngx.log\nlocal ngx_ERR = ngx.ERR\n\nlocal get_header_token = require(\"ledge.header_util\").get_header_token\n\nlocal co_yield = coroutine.yield\nlocal co_wrap = require(\"ledge.util\").coroutine.wrap\n\nlocal get_fixed_field_metatable_proxy =\n require(\"ledge.util\").mt.get_fixed_field_metatable_proxy\n\nlocal ngx_req_get_headers = ngx.req.get_headers\nlocal ngx_RANGE_NOT_SATISFIABLE = 416\nlocal ngx_PARTIAL_CONTENT = 206\n\n\nlocal _M = {\n _VERSION = \"2.3.0\",\n}\n\n\nfunction _M.new()\n return setmetatable({\n ranges = {},\n boundary_end = \"\",\n boundary = \"\",\n }, get_fixed_field_metatable_proxy(_M))\nend\n\n\n-- returns a table of ranges, or nil\n--\n-- e.g.\n-- {\n-- { from = 0, to = 99 },\n-- { from = 100, to = 199 },\n-- }\nlocal function req_byte_ranges()\n local bytes = get_header_token(ngx_req_get_headers().range, \"bytes\")\n local ranges = nil\n\n if bytes then\n ranges = str_split(bytes, \",\")\n if not ranges then ranges = { bytes } end\n for i,r in ipairs(ranges) do\n local from, to = str_match(r, \"(%d*)%-(%d*)\")\n ranges[i] = { from = tonumber(from), to = tonumber(to) }\n end\n end\n\n return ranges\nend\n_M.req_byte_ranges = req_byte_ranges\n\n\nlocal function sort_byte_ranges(first, second)\n if not first.from or not second.from then\n return nil, \"Attempt to compare invalid byteranges\"\n end\n return first.from <= second.from\nend\n\n\nlocal function parse_content_range(content_range)\n local m, err = ngx_re_match(\n content_range,\n [[bytes\\s+(\\d+|\\*)-(\\d+|\\*)/(\\d+)]],\n \"oj\"\n )\n if err then ngx_log(ngx_ERR, err) end\n\n if not m then\n return nil\n else\n return tonumber(m[1]), tonumber(m[2]), tonumber(m[3])\n end\nend\n_M.parse_content_range = parse_content_range\n\n\n-- Modifies the response based on range request headers.\n-- Returns the response and a flag, which if true indicates a partial response\n-- should be expected, if false indicates the range could not be applied, and if\n-- nil indicates no range was requested.\nfunction _M.handle_range_request(self, res)\n local range_request = req_byte_ranges()\n\n if range_request and type(range_request) == \"table\" and res.size then\n -- Don't attempt range filtering on non 200 responses\n if res.status ~= 200 then\n return res, false\n end\n\n local ranges = {}\n\n for _,range in ipairs(range_request) do\n local range_satisfiable = true\n\n if not range.to and not range.from then\n range_satisfiable = false\n end\n\n -- A missing \"to\" means to the \"end\".\n if not range.to then\n range.to = res.size - 1\n end\n\n -- A missing \"from\" means \"to\" is an offset from the end.\n if not range.from then\n range.from = res.size - (range.to)\n range.to = res.size - 1\n\n if range.from < 0 then\n range_satisfiable = false\n end\n end\n\n -- A \"to\" greater than size should be \"end\"\n if range.to > (res.size - 1) then\n range.to = res.size - 1\n end\n\n -- Check the range is satisfiable\n if range.from > range.to then\n range_satisfiable = false\n end\n\n if not range_satisfiable then\n -- We'll return 416\n res.status = ngx_RANGE_NOT_SATISFIABLE\n res.body_reader = res.empty_body_reader\n res.header.content_range = \"bytes */\" .. res.size\n\n return res, false\n else\n -- We'll need the content range header value\n -- for multipart boundaries: e.g. bytes 5-10/20\n range.header = \"bytes \" .. range.from ..\n \"-\" .. range.to ..\n \"/\" .. res.size\n tbl_insert(ranges, range)\n end\n end\n\n local numranges = #ranges\n if numranges > 1 then\n -- Sort ranges as we cannot serve unordered.\n tbl_sort(ranges, sort_byte_ranges)\n\n -- Coalesce overlapping ranges.\n for i = numranges,1,-1 do\n if i > 1 then\n local current_range = ranges[i]\n local previous_range = ranges[i - 1]\n\n if current_range.from <= previous_range.to then\n -- extend previous range to encompass this one\n previous_range.to = current_range.to\n previous_range.header = \"bytes \" ..\n previous_range.from ..\n \"-\" ..\n current_range.to ..\n \"/\" ..\n res.size\n tbl_remove(ranges, i)\n end\n end\n end\n end\n\n self.ranges = ranges\n\n if #ranges == 1 then\n -- We have a single range to serve.\n local range = ranges[1]\n\n local size = res.size\n\n res.status = ngx_PARTIAL_CONTENT\n ngx.header[\"Accept-Ranges\"] = \"bytes\"\n res.header[\"Content-Range\"] = \"bytes \" .. range.from ..\n \"-\" .. range.to ..\n \"/\" .. size\n\n return res, true\n else\n -- Generate boundary\n local boundary_string = str_randomhex(32)\n local boundary = {\n \"\",\n \"--\" .. boundary_string,\n }\n\n if res.header[\"Content-Type\"] then\n tbl_insert(\n boundary,\n \"Content-Type: \" .. res.header[\"Content-Type\"]\n )\n end\n\n self.boundary = tbl_concat(boundary, \"\\n\")\n self.boundary_end = \"\\n--\" .. boundary_string .. \"--\"\n\n res.status = ngx_PARTIAL_CONTENT\n -- TODO: No test coverage for these headers\n res.header[\"Accept-Ranges\"] = \"bytes\"\n res.header[\"Content-Type\"] = \"multipart/byteranges; boundary=\" ..\n boundary_string\n\n return res, true\n end\n end\n\n return res, nil\nend\n\n\n-- Filters the body reader, only yielding bytes specified in a range request.\nfunction _M.get_range_request_filter(self, reader)\n local ranges = self.ranges\n local boundary_end = self.boundary_end\n local boundary = self.boundary\n\n if ranges then\n return co_wrap(function(buffer_size)\n local playhead = 0\n local num_ranges = #ranges\n\n while true do\n local chunk, err = reader(buffer_size)\n if err then ngx_log(ngx_ERR, err) end\n if not chunk then break end\n\n local chunklen = #chunk\n local nextplayhead = playhead + chunklen\n\n for _, range in ipairs(ranges) do\n if range.from >= nextplayhead or range.to < playhead then -- luacheck: ignore 542\n -- Skip over non matching ranges (this is\n -- algorithmically simpler)\n else\n -- Yield the multipart byterange boundary if\n -- required and only once per range.\n if num_ranges > 1 and not range.boundary_printed then\n co_yield(boundary)\n co_yield(\"\\nContent-Range: \" .. range.header)\n co_yield(\"\\n\\n\")\n range.boundary_printed = true\n end\n\n -- Trim range to within this chunk's context\n local yield_from = range.from\n local yield_to = range.to\n if range.from < playhead then\n yield_from = playhead\n end\n if range.to >= nextplayhead then\n yield_to = nextplayhead - 1\n end\n\n -- Find relative points for the range within this chunk\n local relative_yield_from = yield_from - playhead\n local relative_yield_to = yield_to - playhead\n\n -- Ranges are all 0 indexed, finally convert to 1 based\n -- Lua indexes, and yield the range.\n co_yield(\n str_sub(\n chunk,\n relative_yield_from + 1,\n relative_yield_to + 1\n )\n )\n end\n end\n\n playhead = playhead + chunklen\n end\n\n -- Yield the multipart byterange end marker\n if num_ranges > 1 then\n co_yield(boundary_end)\n end\n end)\n end\n\n return reader\nend\n\n\nreturn _M\n" }, { "path": "lib/ledge/request.lua", "content": "local hdr_has_directive = require(\"ledge.header_util\").header_has_directive\n\nlocal ngx_req_get_headers = ngx.req.get_headers\nlocal ngx_re_gsub = ngx.re.gsub\nlocal ngx_req_get_uri_args = ngx.req.get_uri_args\nlocal ngx_req_get_method = ngx.req.get_method\n\nlocal str_byte = string.byte\n\nlocal ngx_var = ngx.var\n\nlocal tbl_sort = table.sort\nlocal tbl_insert = table.insert\n\n\nlocal _M = {\n _VERSION = \"2.3.0\",\n}\n\n\nlocal function purge_mode()\n local x_purge = ngx_req_get_headers()[\"X-Purge\"]\n if hdr_has_directive(x_purge, \"delete\") then\n return \"delete\"\n elseif hdr_has_directive(x_purge, \"revalidate\") then\n return \"revalidate\"\n else\n return \"invalidate\"\n end\nend\n_M.purge_mode = purge_mode\n\n\nlocal function relative_uri()\n local uri = ngx_re_gsub(ngx_var.uri, \"\\\\s\", \"%20\", \"jo\") -- encode spaces\n\n -- encode percentages if an encoded CRLF is in the URI\n -- see: http://resources.infosecinstitute.com/http-response-splitting-attack\n uri = ngx_re_gsub(uri, \"%0D%0A\", \"%250D%250A\", \"ijo\")\n\n return uri .. ngx_var.is_args .. (ngx_var.query_string or \"\")\nend\n_M.relative_uri = relative_uri\n\n\nlocal function full_uri()\n return ngx_var.scheme .. '://' .. ngx_var.host .. relative_uri()\nend\n_M.full_uri = full_uri\n\n\nlocal function accepts_cache()\n -- Check for no-cache\n local h = ngx_req_get_headers()\n if hdr_has_directive(h[\"Pragma\"], \"no-cache\")\n or hdr_has_directive(h[\"Cache-Control\"], \"no-cache\")\n or hdr_has_directive(h[\"Cache-Control\"], \"no-store\") then\n return false\n end\n\n return true\nend\n_M.accepts_cache = accepts_cache\n\n\nlocal function sort_args(a, b)\n return a[1] < b[1]\nend\n\n\nlocal function args_sorted(max_args)\n max_args = max_args or 100\n local args = ngx_req_get_uri_args(max_args)\n if not next(args) then return nil end\n\n local sorted = {}\n for k, v in pairs(args) do\n tbl_insert(sorted, { k, v })\n end\n\n tbl_sort(sorted, sort_args)\n\n local sargs = \"\"\n local sortedln = #sorted\n for i, v in ipairs(sorted) do\n sargs = sargs .. ngx.encode_args({ [v[1]] = v[2] })\n if i < sortedln then sargs = sargs .. \"&\" end\n end\n\n return sargs\nend\n_M.args_sorted = args_sorted\n\n\n-- Used to generate a default args string for the cache key (i.e. when there are\n-- no URI args present).\n--\n-- Returns a zero length string, unless there is an asterisk at the end of the\n-- URI on a PURGE request, in which case we return the asterisk.\n--\n-- The purpose it to ensure trailing wildcards are greedy across both URI and\n-- args portions of a cache key.\n--\n-- If you override the \"args\" field in a cache key spec with your own function,\n-- you'll want to use this to ensure wildcard purges operate correctly.\nlocal function default_args()\n if ngx_req_get_method() == \"PURGE\" and\n str_byte(ngx_var.request_uri, -1) == 42\n then\n return \"*\"\n end\n return \"\"\nend\n_M.default_args = default_args\n\n\nreturn _M\n" }, { "path": "lib/ledge/response.lua", "content": "local http_headers = require \"resty.http_headers\"\nlocal util = require \"ledge.util\"\n\nlocal pairs, setmetatable, tonumber, unpack =\n pairs, setmetatable, tonumber, unpack\n\nlocal tbl_getn = table.getn\nlocal tbl_insert = table.insert\nlocal tbl_concat = table.concat\nlocal tbl_sort = table.sort\n\nlocal str_lower = string.lower\nlocal str_find = string.find\nlocal str_sub = string.sub\nlocal str_rep = string.rep\nlocal str_randomhex = util.string.randomhex\nlocal str_split = util.string.split\n\nlocal ngx_null = ngx.null\nlocal ngx_log = ngx.log\nlocal ngx_ERR = ngx.ERR\nlocal ngx_INFO = ngx.INFO\nlocal ngx_DEBUG = ngx.DEBUG\nlocal ngx_re_gmatch = ngx.re.gmatch\nlocal ngx_parse_http_time = ngx.parse_http_time\nlocal ngx_http_time = ngx.http_time\nlocal ngx_time = ngx.time\nlocal ngx_re_find = ngx.re.find\nlocal ngx_re_gsub = ngx.re.gsub\n\nlocal header_has_directive = require(\"ledge.header_util\").header_has_directive\n\nlocal get_fixed_field_metatable_proxy =\n require(\"ledge.util\").mt.get_fixed_field_metatable_proxy\n\nlocal save_key_chain = require(\"ledge.cache_key\").save_key_chain\n\nlocal _DEBUG = false\n\nlocal _M = {\n _VERSION = \"2.3.0\",\n set_debug = function(debug) _DEBUG = debug end,\n}\n\n\n-- Body reader for when the response body is missing\nlocal function empty_body_reader()\n return nil\nend\n_M.empty_body_reader = empty_body_reader\n\n\nfunction _M.new(handler)\n if not handler or not next(handler) then\n return nil, \"Handler is required\"\n end\n\n if not handler.redis or not next(handler.redis) then\n return nil, \"Handler has no redis connection\"\n end\n\n return setmetatable({\n redis = handler.redis,\n handler = handler, -- Cache key chain\n\n uri = \"\",\n status = 0,\n header = http_headers.new(),\n\n -- stored metadata\n size = 0,\n remaining_ttl = 0,\n has_esi = false,\n esi_scanned = false,\n\n -- body\n entity_id = \"\",\n body_reader = empty_body_reader,\n body_filters = {}, -- for debug logging\n\n -- runtime metadata (not persisted)\n length = 0, -- If Content-Length is present\n has_body = false, -- From lua-resty-http has_body\n\n }, get_fixed_field_metatable_proxy(_M))\nend\n\n\n-- Setter for a fixed body string (not streamed)\nfunction _M.set_body(self, body_string)\n local sent = false\n self.body_reader = function()\n if not sent then\n sent = true\n return body_string\n else\n return nil\n end\n end\nend\n\n\nfunction _M.filter_body_reader(self, filter_name, filter)\n assert(type(filter) == \"function\", \"filter must be a function\")\n\n if _DEBUG then\n -- Keep track of the filters by name, just for debugging\n ngx_log(ngx_DEBUG,\n filter_name,\n \"(\",\n tbl_concat(self.body_filters,\n \"(\"), \"\" , str_rep(\")\", #self.body_filters - 1\n ),\n \")\"\n )\n\n tbl_insert(self.body_filters, 1, filter_name)\n end\n\n self.body_reader = filter\nend\n\n\nfunction _M.is_cacheable(self)\n -- Never cache partial content\n local status = self.status\n if status == 206 or status == 416 then\n return false\n end\n\n local h = self.header\n local directives = \"(no-cache|no-store|private)\"\n if header_has_directive(h[\"Cache-Control\"], directives, true) then\n return false\n end\n\n if header_has_directive(h[\"Pragma\"], \"no-cache\", true) then\n return false\n end\n\n if h[\"Vary\"] == \"*\" then\n return false\n end\n\n if self:ttl() > 0 then\n return true\n else\n return false\n end\nend\n\n\n-- Calculates the TTL from response headers.\n-- Header precedence is Cache-Control: s-maxage=NUM, Cache-Control: max-age=NUM\n-- and finally Expires: HTTP_TIMESTRING.\nfunction _M.ttl(self)\n local cc = self.header[\"Cache-Control\"]\n if cc then\n if type(cc) == \"table\" then\n cc = tbl_concat(cc, \", \")\n end\n local max_ages = {}\n for max_age in ngx_re_gmatch(cc, [[(s-maxage|max-age)=(\\d+)]], \"ijo\") do\n max_ages[max_age[1]] = max_age[2]\n end\n\n if max_ages[\"s-maxage\"] then\n return tonumber(max_ages[\"s-maxage\"])\n elseif max_ages[\"max-age\"] then\n return tonumber(max_ages[\"max-age\"])\n end\n end\n\n -- Fall back to Expires.\n local expires = self.header[\"Expires\"]\n if expires then\n -- If there are multiple, last one wins\n if type(expires) == \"table\" then\n expires = expires[#expires]\n end\n\n local time = ngx_parse_http_time(tostring(expires))\n if time then return time - ngx_time() end\n end\n\n return 0\nend\n\n\nfunction _M.has_expired(self)\n return self.remaining_ttl <= 0\nend\n\n\n-- Return nil and an error on an actual Redis error, this indicates that Redis\n-- has failed and we aren't going to be able to proceed normally.\n-- Return nil and *no* error if this is just a broken/partial cache entry\n-- so we MISS and update the entry.\nfunction _M.read(self)\n local key_chain, err = self.handler:cache_key_chain()\n if not key_chain then\n return nil, err\n end\n\n local redis = self.redis\n\n -- Read main metdata\n local cache_parts, err = redis:hgetall(key_chain.main)\n if not cache_parts or cache_parts == ngx_null then\n return nil, err\n end\n\n -- No cache entry for this key\n local cache_parts_len = #cache_parts\n if not cache_parts_len or cache_parts_len == 0 then\n return nil\n end\n\n local time_in_cache = 0\n local time_since_generated = 0\n\n -- The Redis replies is a sequence of messages, so we iterate over pairs\n -- to get hash key/values.\n for i = 1, cache_parts_len, 2 do\n if cache_parts[i] == \"uri\" then\n self.uri = cache_parts[i + 1]\n\n elseif cache_parts[i] == \"status\" then\n self.status = tonumber(cache_parts[i + 1])\n\n elseif cache_parts[i] == \"entity\" then\n self.entity_id = cache_parts[i + 1]\n\n elseif cache_parts[i] == \"expires\" then\n self.remaining_ttl = tonumber(cache_parts[i + 1]) - ngx_time()\n\n elseif cache_parts[i] == \"saved_ts\" then\n time_in_cache = ngx_time() - tonumber(cache_parts[i + 1])\n\n elseif cache_parts[i] == \"generated_ts\" then\n time_since_generated = ngx_time() - tonumber(cache_parts[i + 1])\n\n elseif cache_parts[i] == \"has_esi\" then\n self.has_esi = cache_parts[i + 1]\n\n elseif cache_parts[i] == \"esi_scanned\" then\n local scanned = cache_parts[i + 1]\n if scanned == \"false\" then\n self.esi_scanned = false\n else\n self.esi_scanned = true\n end\n\n elseif cache_parts[i] == \"size\" then\n self.size = tonumber(cache_parts[i + 1])\n end\n end\n\n -- Read headers\n local headers, err = redis:hgetall(key_chain.headers)\n if not headers or headers == ngx_null then\n return nil, err\n end\n\n local headers_len = tbl_getn(headers)\n if headers_len == 0 then\n ngx_log(ngx_INFO, \"headers missing\")\n return nil\n end\n\n for i = 1, headers_len, 2 do\n local header = headers[i]\n if str_find(header, \":\") then\n -- We have multiple headers with the same field name\n local _, key = unpack(str_split(header, \":\"))\n if not self.header[key] then\n self.header[key] = {}\n end\n tbl_insert(self.header[key], headers[i + 1])\n else\n self.header[header] = headers[i + 1]\n end\n end\n\n -- Calculate the Age header\n if self.header[\"Age\"] then\n -- We have end-to-end Age headers, add our time_in_cache.\n self.header[\"Age\"] = tonumber(self.header[\"Age\"]) + time_in_cache\n elseif self.header[\"Date\"] then\n -- We have no advertised Age, use the generated timestamp.\n self.header[\"Age\"] = time_since_generated\n end\n\n -- \"touch\" other keys not needed for read, so that they are\n -- less likely to be unfairly evicted ahead of time\n -- Note: From Redis 3.2.1 this could be one TOUCH command\n local _ = redis:hlen(key_chain.reval_params)\n local _ = redis:hlen(key_chain.reval_req_headers)\n if self.size > 0 then\n local entities, err = redis:scard(key_chain.entities)\n if not entities or entities == ngx_null then\n return nil, \"could not read entities set: \" .. err\n elseif entities == 0 then\n ngx_log(ngx_INFO, \"entities set is empty\")\n return nil\n end\n end\n\n -- Check this key is in the repset\n local scard, err = redis:sismember(key_chain.repset, key_chain.full)\n if err then\n return nil, err\n end\n\n -- Got a cache entry but missing from repset or repset missing, bad...\n -- Call save_key_chain which will add this rep to the repset\n if scard == 0 then\n local repset_ttl = redis:ttl(key_chain.main)\n local ok, err = save_key_chain(redis, key_chain, repset_ttl)\n if not ok then\n return nil, err\n end\n end\n\n return true\nend\n\n\n-- Takes headers from a HTTP response and returns a flat table of cacheable\n-- header entries formatted for Redis.\nlocal function prepare_cacheable_headers(headers)\n -- Don't cache any headers marked as\n -- Cache-Control: (no-cache|no-store|private)=\"header\".\n local uncacheable_headers = {}\n local cc = headers[\"Cache-Control\"]\n if cc then\n if type(cc) == \"table\" then cc = tbl_concat(cc, \", \") end\n cc = str_lower(cc)\n if str_find(cc, \"=\", 1, true) then\n local pattern = '(?:no-cache|private)=\"?([0-9a-z-]+)\"?'\n local re_ctx = {}\n repeat\n local from, to, err = ngx_re_find(cc, pattern, \"jo\", re_ctx, 1)\n if from then\n uncacheable_headers[str_sub(cc, from, to)] = true\n elseif err then\n ngx_log(ngx_ERR, err)\n end\n until not from\n end\n end\n\n -- Turn the headers into a flat list of pairs for the Redis query.\n local h = {}\n for header,header_value in pairs(headers) do\n if not uncacheable_headers[str_lower(header)] then\n if type(header_value) == 'table' then\n -- Multiple headers are represented as a table of values\n local header_value_len = tbl_getn(header_value)\n for i = 1, header_value_len do\n tbl_insert(h, i..':'..header)\n tbl_insert(h, header_value[i])\n end\n else\n tbl_insert(h, header)\n tbl_insert(h, header_value)\n end\n end\n end\n\n return h\nend\n\n\nfunction _M.save(self, keep_cache_for)\n if not keep_cache_for then keep_cache_for = 0 end\n\n -- Create a new entity id\n self.entity_id = str_randomhex(32)\n\n local ttl = self:ttl()\n local time = ngx_time()\n\n local redis = self.redis\n if not next(redis) then return nil, \"no redis\" end\n local key_chain = self.handler:cache_key_chain()\n\n if not self.header[\"Date\"] then\n self.header[\"Date\"] = ngx_http_time(ngx_time())\n end\n\n local ok, err = redis:del(key_chain.main)\n if not ok then ngx_log(ngx_ERR, err) end\n\n local ok, err = redis:hmset(key_chain.main,\n \"entity\", self.entity_id,\n \"status\", self.status,\n \"uri\", self.uri,\n \"expires\", ttl + time,\n \"generated_ts\", ngx_parse_http_time(self.header[\"Date\"]),\n \"saved_ts\", time,\n \"esi_scanned\", tostring(self.esi_scanned) -- from bool\n )\n if not ok then ngx_log(ngx_ERR, err) end\n\n local h = prepare_cacheable_headers(self.header)\n\n ok, err = redis:del(key_chain.headers)\n if not ok then ngx_log(ngx_ERR, err) end\n\n ok, err = redis:hmset(key_chain.headers, unpack(h))\n if not ok then ngx_log(ngx_ERR, err) end\n\n -- Mark the keys as eventually volatile (the body is set by the body writer)\n local expiry = ttl + tonumber(keep_cache_for)\n\n ok, err = redis:expire(key_chain.main, expiry)\n if not ok then ngx_log(ngx_ERR, err) end\n\n ok, err = redis:expire(key_chain.headers, expiry)\n if not ok then ngx_log(ngx_ERR, err) end\n\n local ok, err = redis:sadd(key_chain.entities, self.entity_id)\n if not ok then\n ngx_log(ngx_ERR, \"error adding entity to set: \", err)\n end\n\n ok, err = redis:expire(key_chain.entities, expiry)\n if not ok then ngx_log(ngx_ERR, err) end\n\n return true\nend\n\n\nfunction _M.set_and_save(self, field, value)\n local redis = self.redis\n\n local ok, err = redis:hset(self.handler:cache_key_chain().main, field, tostring(value))\n if not ok then\n if err then ngx_log(ngx_ERR, err) end\n return nil, err\n end\n\n self[field] = value\n return ok\nend\n\n\nlocal WARNINGS = {\n [\"110\"] = \"Response is stale\",\n [\"214\"] = \"Transformation applied\",\n [\"112\"] = \"Disconnected Operation\",\n}\n\n\nfunction _M.add_warning(self, code, name)\n if not self.header[\"Warning\"] then\n self.header[\"Warning\"] = {}\n end\n\n local header = code .. ' ' .. name\n header = header .. ' \"' .. WARNINGS[code] .. '\"'\n tbl_insert(self.header[\"Warning\"], header)\nend\n\n\nlocal function deduplicate_table(table)\n -- Can't have duplicates if there's 1 or 0 entries!\n if #table <= 1 then\n return table\n end\n\n local new_table = {}\n local unique = {}\n local i = 0\n\n for _,v in ipairs(table) do\n if not unique[v] then\n unique[v] = true\n i = i +1\n new_table[i] = v\n end\n end\n\n return new_table\nend\n\n\nfunction _M.parse_vary_header(self)\n local vary_hdr = self.header[\"Vary\"]\n local vary_spec\n\n if vary_hdr and vary_hdr ~= \"\" then\n if type(vary_hdr) == \"table\" then\n vary_hdr = tbl_concat(vary_hdr,\",\")\n end\n -- Remove whitespace around commas and lowercase\n vary_hdr = ngx_re_gsub(str_lower(vary_hdr), [[\\s*,\\s*]], \",\", \"oj\")\n vary_spec = str_split(vary_hdr, \",\")\n tbl_sort(vary_spec)\n vary_spec = deduplicate_table(vary_spec)\n end\n\n -- Return the new vary sepc table *and* the normalised header\n return vary_spec\nend\n\n\nreturn _M\n" }, { "path": "lib/ledge/stale.lua", "content": "local math_min = math.min\n\nlocal ngx_req_get_headers = ngx.req.get_headers\n\nlocal header_has_directive = require(\"ledge.header_util\").header_has_directive\nlocal get_numeric_header_token =\n require(\"ledge.header_util\").get_numeric_header_token\n\n\nlocal _M = {\n _VERSION = \"2.3.0\"\n}\n\n\n-- True if the request specifically asks for stale (req.cc.max-stale) and the\n-- response doesn't explicitly forbid this res.cc.(must|proxy)-revalidate.\nlocal function can_serve_stale(res)\n local req_cc = ngx_req_get_headers()[\"Cache-Control\"]\n local req_cc_max_stale = get_numeric_header_token(req_cc, \"max-stale\")\n if req_cc_max_stale then\n local res_cc = res.header[\"Cache-Control\"]\n\n -- Check the response permits this at all\n if header_has_directive(res_cc, \"(must|proxy)-revalidate\") then\n return false\n else\n if (req_cc_max_stale * -1) <= res.remaining_ttl then\n return true\n end\n end\n end\n return false\nend\n_M.can_serve_stale = can_serve_stale\n\n\n-- Returns true if stale-while-revalidate or stale-if-error is specified, valid\n-- and not constrained by other factors such as max-stale.\n-- @param token \"stale-while-revalidate\" | \"stale-if-error\"\nlocal function verify_stale_conditions(res, token)\n assert(token == \"stale-while-revalidate\" or token == \"stale-if-error\",\n \"unknown token: \" .. tostring(token))\n\n local res_cc = res.header[\"Cache-Control\"]\n local res_cc_stale = get_numeric_header_token(res_cc, token)\n\n -- Check the response permits this at all\n if header_has_directive(res_cc, \"(must|proxy)-revalidate\") then\n return false\n end\n\n -- Get request header tokens\n local req_cc = ngx_req_get_headers()[\"Cache-Control\"]\n local req_cc_stale = get_numeric_header_token(req_cc, token)\n local req_cc_max_age = get_numeric_header_token(req_cc, \"max-age\")\n local req_cc_max_stale = get_numeric_header_token(req_cc, \"max-stale\")\n\n local stale_ttl = 0\n -- If we have both req and res stale-\" .. reason, use the lower value\n if req_cc_stale and res_cc_stale then\n stale_ttl = math_min(req_cc_stale, res_cc_stale)\n -- Otherwise return the req or res value\n elseif req_cc_stale then\n stale_ttl = req_cc_stale\n elseif res_cc_stale then\n stale_ttl = res_cc_stale\n end\n\n if stale_ttl <= 0 then\n return false -- No stale policy defined\n elseif header_has_directive(req_cc, \"min-fresh\") then\n return false -- Cannot serve stale as request demands freshness\n elseif req_cc_max_age and\n req_cc_max_age < (tonumber(res.header[\"Age\"] or 0) or 0) then\n return false -- Cannot serve stale as req max-age is less than res Age\n elseif req_cc_max_stale and req_cc_max_stale < stale_ttl then\n return false -- Cannot serve stale as req max-stale is less than S-W-R\n else\n -- We can return stale\n return true\n end\nend\n_M.verify_stale_conditions = verify_stale_conditions\n\n\nlocal function can_serve_stale_while_revalidate(res)\n return verify_stale_conditions(res, \"stale-while-revalidate\")\nend\n_M.can_serve_stale_while_revalidate = can_serve_stale_while_revalidate\n\n\nlocal function can_serve_stale_if_error(res)\n return verify_stale_conditions(res, \"stale-if-error\")\nend\n_M.can_serve_stale_if_error = can_serve_stale_if_error\n\n\nreturn _M\n" }, { "path": "lib/ledge/state_machine/actions.lua", "content": "local type, next = type, next\n\nlocal esi = require(\"ledge.esi\")\nlocal response = require(\"ledge.response\")\n\nlocal ngx_var = ngx.var\n\nlocal ngx_HTTP_NOT_MODIFIED = ngx.HTTP_NOT_MODIFIED\n\nlocal ngx_req_set_header = ngx.req.set_header\n\nlocal get_gzip_decoder = require(\"ledge.gzip\").get_gzip_decoder\n\nlocal _M = { -- luacheck: no unused\n _VERSION = \"2.3.0\",\n}\n\n\n-- Actions. Functions which can be called on transition.\nreturn {\n redis_close = function(handler)\n return require(\"ledge\").close_redis_connection(handler.redis)\n end,\n\n httpc_close = function(handler)\n local upstream_client = handler.upstream_client\n if next(upstream_client) then\n if type(upstream_client.set_keepalive) == \"function\" then\n local config = handler.config\n return upstream_client:set_keepalive(\n config.upstream_keepalive_timeout,\n config.upstream_keepalive_poolsize\n )\n end\n end\n end,\n\n httpc_close_without_keepalive = function(handler)\n local upstream_client = handler.upstream_client\n if next(upstream_client) then\n return upstream_client:close()\n end\n end,\n\n stash_error_response = function(handler)\n handler.error_response = handler.response\n end,\n\n restore_error_response = function(handler)\n local error_res = handler.error_response\n if next(error_res) then\n handler.response = error_res\n end\n end,\n\n -- If ESI is enabled and we have an esi_args prefix, weed uri args\n -- beginning with the prefix (knows as ESI_ARGS) out of the URI (and thus\n -- cache key) and stash them in the custom ESI variables table.\n filter_esi_args = function(handler)\n if handler.config.esi_enabled then\n esi.filter_esi_args(handler)\n end\n end,\n\n read_cache = function(handler)\n handler.response = handler:read_from_cache()\n end,\n\n install_no_body_reader = function(handler)\n local res = handler.response\n res.body_reader = res.empty_body_reader\n end,\n\n install_gzip_decoder = function(handler)\n local res = handler.response\n res.header[\"Content-Encoding\"] = nil\n res:filter_body_reader(\n \"gzip_decoder\",\n get_gzip_decoder(res.body_reader)\n )\n end,\n\n install_range_filter = function(handler)\n local res = handler.response\n local range = handler.range\n res:filter_body_reader(\n \"range_request_filter\",\n range:get_range_request_filter(res.body_reader)\n )\n end,\n\n set_esi_scan_enabled = function(handler)\n handler.esi_scan_enabled = true\n --handler.esi_scan_disabled = false\n handler.response.esi_scanned = true\n end,\n\n install_esi_scan_filter = function(handler)\n local res = handler.response\n local esi_processor = handler.esi_processor\n\n if next(esi_processor) then\n res:filter_body_reader(\n \"esi_scan_filter\",\n esi_processor:get_scan_filter(res)\n )\n end\n end,\n\n set_esi_scan_disabled = function(handler)\n local res = handler.response\n handler.esi_scan_enabled = false\n res.esi_scanned = false\n end,\n\n install_esi_process_filter = function(handler)\n local res = handler.response\n local esi_processor = handler.esi_processor\n\n if next(esi_processor) then\n res:filter_body_reader(\n \"esi_process_filter\",\n esi_processor:get_process_filter(res)\n )\n end\n end,\n\n set_esi_process_enabled = function(handler)\n handler.esi_process_enabled = true\n end,\n\n set_esi_process_disabled = function(handler)\n handler.esi_process_enabled = false\n end,\n\n zero_downstream_lifetime = function(handler)\n local res = handler.response\n if res.header then\n res.header[\"Cache-Control\"] = \"private, max-age=0\"\n end\n end,\n\n remove_surrogate_control_header = function(handler)\n local res = handler.response\n if res.header then\n res.header[\"Surrogate-Control\"] = nil\n end\n end,\n\n fetch = function(handler)\n local res = handler:fetch_from_origin()\n if res.status ~= ngx_HTTP_NOT_MODIFIED then\n handler.response = res\n end\n end,\n\n remove_client_validators = function(handler)\n -- Keep these in case we need to restore them (after revalidating upstream)\n local client_validators = handler.client_validators\n client_validators[\"If-Modified-Since\"] = ngx_var.http_if_modified_since\n client_validators[\"If-None-Match\"] = ngx_var.http_if_none_match\n\n ngx_req_set_header(\"If-Modified-Since\", nil)\n ngx_req_set_header(\"If-None-Match\", nil)\n end,\n\n restore_client_validators = function(handler)\n local client_validators = handler.client_validators\n ngx_req_set_header(\"If-Modified-Since\", client_validators[\"If-Modified-Since\"])\n ngx_req_set_header(\"If-None-Match\", client_validators[\"If-None-Match\"])\n end,\n\n add_stale_warning = function(handler)\n return handler:add_warning(\"110\")\n end,\n\n add_disconnected_warning = function(handler)\n return handler:add_warning(\"112\")\n end,\n\n set_json_response = function(handler)\n local res = response.new(handler)\n res.header[\"Content-Type\"] = \"application/json\"\n handler.response = res\n end,\n\n -- Updates the realidation_params key with data from the current request,\n -- and schedules a background revalidation job\n revalidate_in_background = function(handler)\n return handler:revalidate_in_background(handler:cache_key_chain(), true)\n end,\n\n -- Triggered on upstream partial content, assumes no stored\n -- revalidation metadata but since we have a rqeuest context (which isn't\n -- the case with `revalidate_in_background` we can simply fetch.\n fetch_in_background = function(handler)\n return handler:fetch_in_background()\n end,\n\n save_to_cache = function(handler)\n local res = handler.response\n return handler:save_to_cache(res)\n end,\n\n delete_from_cache = function(handler)\n return handler:delete_from_cache(handler:cache_key_chain())\n end,\n\n disable_output_buffers = function(handler)\n handler.output_buffers_enabled = false\n end,\n\n reset_cache_key = function(handler)\n handler:reset_cache_key()\n end,\n\n set_http_ok = function()\n ngx.status = ngx.HTTP_OK\n end,\n\n set_http_not_found = function()\n ngx.status = ngx.HTTP_NOT_FOUND\n end,\n\n set_http_not_modified = function()\n ngx.status = ngx_HTTP_NOT_MODIFIED\n end,\n\n set_http_service_unavailable = function()\n ngx.status = ngx.HTTP_SERVICE_UNAVAILABLE\n end,\n\n set_http_gateway_timeout = function()\n ngx.status = ngx.HTTP_GATEWAY_TIMEOUT\n end,\n\n set_http_internal_server_error = function()\n ngx.status = ngx.HTTP_INTERNAL_SERVER_ERROR\n end,\n\n set_http_status_from_response = function(handler)\n local res = handler.response\n if res and res.status then\n ngx.status = res.status\n else\n ngx.status = ngx.HTTP_INTERNAL_SERVER_ERROR\n end\n end,\n}\n" }, { "path": "lib/ledge/state_machine/events.lua", "content": "local _M = { -- luacheck: no unused\n _VERSION = \"2.3.0\",\n}\n\n\n-- Event transition table.\n--\n-- Use \"begin\" to transition based on an event. Filter transitions by current\n-- state \"when\", and/or any previous state \"after\", and/or a previously fired\n-- event \"in_case\", and run actions using \"but_first\". Transitions are processed\n-- in the order found, so place more specific entries for a given event before\n-- more generic ones.\nreturn {\n -- Initial transition (entry point). Connect to redis.\n init = {\n { begin = \"checking_method\", but_first = \"filter_esi_args\" },\n },\n\n cacheable_method = {\n { when = \"checking_origin_mode\", begin = \"checking_request\" },\n { begin = \"checking_origin_mode\" },\n },\n\n -- PURGE method detected.\n purge_requested = {\n {\n when = \"considering_wildcard_purge\",\n begin = \"purging\",\n but_first = \"set_json_response\"\n },\n {\n when = \"considering_purge_api\",\n begin = \"considering_wildcard_purge\"\n },\n { begin = \"considering_purge_api\" },\n },\n\n purge_api_requested = {\n {\n begin = \"purging_via_api\",\n but_first = \"set_json_response\"\n },\n },\n\n wildcard_purge_requested = {\n { begin = \"wildcard_purging\", but_first = \"set_json_response\" },\n },\n\n -- Succesfully purged (expired) a cache entry. Exit 200 OK.\n purged = {\n { begin = \"serving\", but_first = \"set_http_ok\" },\n },\n\n wildcard_purge_scheduled = {\n { begin = \"serving\", but_first = \"set_http_ok\" },\n },\n\n purge_api_completed = {\n { begin = \"serving\", but_first = \"set_http_ok\" },\n },\n\n purge_api_failed = {\n { begin = \"serving\", but_first = \"set_http_status_from_response\" },\n },\n\n -- URI to purge was not found. Exit 404 Not Found.\n nothing_to_purge = {\n { begin = \"serving\", but_first = \"set_http_not_found\" },\n },\n\n -- The request accepts cache. If we've already validated locally, we can\n -- think about serving. Otherwise we need to check the cache situtation.\n cache_accepted = {\n { when = \"revalidating_locally\", begin = \"considering_esi_process\" },\n { begin = \"checking_cache\" },\n },\n\n forced_cache = {\n { begin = \"accept_cache\" },\n },\n\n -- This request doesn't accept cache, so we need to see about fetching\n cache_not_accepted = {\n { begin = \"checking_can_fetch\" },\n },\n\n -- We don't know anything about this URI, so we've got to see about fetching\n cache_missing = {\n { begin = \"checking_can_fetch\" },\n },\n\n -- This URI was cacheable last time, but has expired. So see about serving\n -- stale, but failing that, see about fetching.\n cache_expired = {\n { when = \"checking_cache\", begin = \"checking_can_serve_stale\" },\n { when = \"checking_can_serve_stale\", begin = \"checking_can_fetch\" },\n { when = \"checking_can_serve_stale\", begin = \"checking_can_fetch\" },\n },\n\n -- We have a (not expired) cache entry. Lets try and validate in case we can\n -- exit 304.\n cache_valid = {\n { in_case = \"forced_cache\", begin = \"considering_esi_process\" },\n {\n in_case = \"collapsed_response_ready\",\n begin = \"considering_local_revalidation\"\n },\n { when = \"checking_cache\", begin = \"considering_revalidation\" },\n },\n\n -- We need to fetch, and there are no settings telling us we shouldn't, but\n -- collapsed forwarding is on, so if cache is accepted and in an \"expired\"\n -- state (i.e. not missing), lets try to collapse. Otherwise we just start\n -- fetching.\n can_fetch_but_try_collapse = {\n { in_case = \"cache_missing\", begin = \"fetching\" },\n { in_case = \"cache_accepted\", begin = \"requesting_collapse_lock\" },\n { begin = \"fetching\" },\n },\n\n -- We have the lock on this \"fetch\". We might be the only one. We'll never\n -- know. But we fetch as \"surrogate\" in case others are listening.\n obtained_collapsed_forwarding_lock = {\n { begin = \"fetching_as_surrogate\" },\n },\n\n -- Another request is currently fetching, so we've subscribed to updates on\n -- this URI. We need to block until we hear something (or timeout).\n subscribed_to_collapsed_forwarding_channel = {\n { begin = \"waiting_on_collapsed_forwarding_channel\" },\n },\n\n -- Another request was fetching when we asked, but by the time we subscribed\n -- the channel was closed (small window, but potentially possible). Chances\n -- are the item is now in cache, so start there.\n collapsed_forwarding_channel_closed = {\n { begin = \"checking_cache\" },\n },\n\n -- We were waiting on a collapse channel, and got a message saying the\n -- response is now ready. The item will now be fresh in cache.\n collapsed_response_ready = {\n { begin = \"checking_cache\" },\n },\n\n -- We were waiting on another request (collapsed), but it came back as a\n -- non-cacheable response (i.e. the previously cached item is no longer\n -- cacheable). So go fetch for ourselves.\n collapsed_forwarding_failed = {\n { begin = \"fetching\" },\n },\n\n -- We were waiting on another request, but it received an upstream_error\n -- (e.g. 500) Check if we can serve stale content instead\n collapsed_forwarding_upstream_error = {\n { begin = \"considering_stale_error\" },\n },\n\n -- We were waiting on another request, but the vary key changed\n -- Might still match so check the cache again\n collapsed_forwarding_vary_modified = {\n { begin = \"checking_cache\", but_first = \"reset_cache_key\" },\n },\n\n -- We need to fetch and nothing is telling us we shouldn't.\n -- Collapsed forwarding is not enabled.\n can_fetch = {\n { begin = \"fetching\" },\n },\n\n -- We've fetched and got a response status and headers. We should consider\n -- potential for ESI before doing anything else.\n response_fetched = {\n { in_case = \"vary_modified\", begin = \"considering_esi_scan\" },\n { begin = \"considering_vary\" },\n },\n\n vary_modified = {\n { begin = \"considering_esi_scan\" },\n },\n\n vary_unmodified = {\n { begin = \"considering_esi_scan\" }\n },\n\n partial_response_fetched = {\n { begin = \"considering_background_fetch\" },\n },\n\n -- We had a partial response and were able to schedule a backgroud fetch for\n -- the complete resource.\n can_fetch_in_background = {\n {\n in_case = \"partial_response_fetched\",\n begin = \"considering_esi_scan\",\n but_first = \"fetch_in_background\"\n },\n },\n\n -- We had a partial response but skipped background fetching the complete\n -- resource, most likely because it is bigger than cache_max_memory.\n background_fetch_skipped = {\n {\n in_case = \"partial_response_fetched\",\n begin = \"considering_esi_scan\"\n },\n },\n\n -- If we went upstream and errored, check if we can serve a cached copy\n -- (stale-if-error), publish the error first if we were the surrogate\n -- request\n upstream_error = {\n {\n after = \"fetching_as_surrogate\",\n begin = \"publishing_collapse_upstream_error\"\n },\n { in_case = \"cache_not_accepted\", begin = \"serving_upstream_error\" },\n { in_case = \"cache_missing\", begin = \"serving_upstream_error\" },\n { begin = \"considering_stale_error\" }\n },\n\n -- We had an error from upstream and could not serve stale content, so serve\n -- the error.\n -- Or we were collapsed and the surrogate received an error but we could not\n -- serve stale in that case, try and fetch ourselves\n can_serve_upstream_error = {\n { after = \"fetching\", begin = \"serving_upstream_error\" },\n { in_case = \"collapsed_forwarding_upstream_error\", begin = \"fetching\" },\n { begin = \"serving_upstream_error\" },\n },\n\n -- We've determined we need to scan the body for ESI.\n esi_scan_enabled = {\n {\n begin = \"considering_gzip_inflate\",\n but_first = \"set_esi_scan_enabled\"\n },\n },\n\n -- We've determined no need to scan the body for ESI.\n esi_scan_disabled = {\n { begin = \"updating_cache\", but_first = \"set_esi_scan_disabled\" },\n },\n\n gzip_inflate_enabled = {\n {\n after = \"updating_cache\",\n begin = \"preparing_response\",\n but_first = \"install_gzip_decoder\"\n },\n {\n in_case = \"esi_scan_enabled\",\n begin = \"updating_cache\",\n but_first = { \"install_gzip_decoder\", \"install_esi_scan_filter\" }\n },\n { begin = \"preparing_response\", but_first = \"install_gzip_decoder\" },\n },\n\n gzip_inflate_disabled = {\n { after = \"updating_cache\", begin = \"preparing_response\" },\n {\n after = \"considering_esi_scan\",\n in_case = \"esi_scan_enabled\",\n begin = \"updating_cache\",\n but_first = { \"install_esi_scan_filter\" }\n },\n { in_case = \"esi_process_disabled\", begin = \"checking_range_request\" },\n { begin = \"preparing_response\" },\n },\n\n range_accepted = {\n { begin = \"preparing_response\", but_first = \"install_range_filter\" },\n },\n\n range_not_accepted = {\n { begin = \"preparing_response\" },\n },\n\n range_not_requested = {\n { begin = \"preparing_response\" },\n },\n\n -- We deduced that the new response can cached. We always \"save_to_cache\".\n -- If we were fetching as a surrogate (collapsing) make sure we tell any\n -- others concerned. If we were performing a background revalidate (having\n -- served stale), we can just exit. Otherwise go back through validationg\n -- in case we can 304 to the client.\n response_cacheable = {\n {\n after = \"fetching_as_surrogate\",\n in_case = \"vary_modified\",\n begin = \"publishing_collapse_vary_modified\",\n but_first = \"save_to_cache\"\n },\n {\n after = \"fetching_as_surrogate\",\n begin = \"publishing_collapse_success\",\n but_first = \"save_to_cache\"\n },\n {\n begin = \"considering_local_revalidation\",\n but_first = \"save_to_cache\"\n },\n },\n\n -- We've deduced that the new response cannot be cached. Essentially this is\n -- as per \"response_cacheable\", except we \"delete\" rather than \"save\", and\n -- we don't try to revalidate.\n response_not_cacheable = {\n {\n after = \"fetching_as_surrogate\",\n begin = \"publishing_collapse_failure\",\n but_first = \"delete_from_cache\"\n },\n { begin = \"considering_esi_process\", but_first = \"delete_from_cache\" },\n },\n\n -- A missing response body means a HEAD request or a 304 Not Modified\n -- upstream response, for example. If we were revalidating upstream, we can\n -- now re-revalidate against local cache. If we're collapsing or background\n -- revalidating, ensure we either clean up the collapsees or exit\n -- respectively.\n response_body_missing = {\n {\n in_case = \"must_revalidate\",\n begin = \"considering_local_revalidation\"\n },\n {\n after = \"fetching_as_surrogate\",\n begin = \"publishing_collapse_failure\",\n but_first = \"delete_from_cache\"\n },\n {\n begin = \"serving\",\n but_first = {\n \"install_no_body_reader\", \"set_http_status_from_response\"\n },\n },\n },\n\n -- We were the collapser, so digressed into being a surrogate. We're done\n -- now and have published this fact, so we pick up where it would have left\n -- off - attempting to 304 to the client. Unless we received an error, in\n -- which case check if we can serve stale instead.\n published = {\n { in_case = \"upstream_error\", begin = \"considering_stale_error\" },\n { begin = \"considering_local_revalidation\" },\n },\n\n -- Client requests a max-age of 0 or stored response requires revalidation.\n must_revalidate = {\n { begin = \"checking_can_fetch\" },\n },\n\n -- We can validate locally, so do it. This doesn't imply it's valid, merely\n -- that we have the correct parameters to attempt validation.\n can_revalidate_locally = {\n { begin = \"revalidating_locally\" },\n },\n\n -- Standard non-conditional request.\n no_validator_present = {\n { begin = \"considering_esi_process\" },\n },\n\n -- The response has not been modified against the validators given. We'll\n -- exit 304 if we can but go via considering_esi_process in case of ESI work\n -- to be done.\n not_modified = {\n { when = \"revalidating_locally\", begin = \"considering_esi_process\" },\n },\n\n -- Our cache has been modified as compared to the validators. But cache is\n -- valid, so just serve it. If we've been upstream, re-compare against\n -- client validators.\n modified = {\n { when = \"revalidating_locally\", begin = \"considering_esi_process\" },\n },\n\n esi_process_enabled = {\n {\n in_case = \"can_serve_stale\",\n begin = \"serving_stale\",\n but_first = {\n \"install_esi_process_filter\",\n \"set_esi_process_enabled\",\n \"zero_downstream_lifetime\",\n \"remove_surrogate_control_header\"\n }\n },\n {\n begin = \"preparing_response\",\n but_first = {\n \"install_esi_process_filter\",\n \"set_esi_process_enabled\",\n \"zero_downstream_lifetime\",\n \"remove_surrogate_control_header\"\n }\n },\n },\n\n esi_process_disabled = {\n {\n begin = \"considering_gzip_inflate\",\n but_first = \"set_esi_process_disabled\"\n },\n },\n\n esi_process_not_required = {\n {\n begin = \"considering_gzip_inflate\",\n but_first = {\n \"set_esi_process_disabled\",\n \"remove_surrogate_control_header\"\n },\n },\n },\n\n can_serve_disconnected = {\n {\n begin = \"considering_esi_process\",\n but_first = \"add_disconnected_warning\"\n },\n },\n\n -- We've deduced we can serve a stale version of this URI. Ensure we add a\n -- warning to the response headers.\n can_serve_stale = {\n {\n after = \"considering_stale_error\",\n begin = \"considering_esi_process\",\n but_first = \"add_stale_warning\"\n },\n {\n begin = \"considering_revalidation\",\n but_first = { \"add_stale_warning\" }\n },\n },\n\n -- We can serve stale, but also trigger a background revalidation\n can_serve_stale_while_revalidate = {\n {\n begin = \"considering_esi_process\",\n but_first = { \"add_stale_warning\", \"revalidate_in_background\" }\n },\n },\n\n -- We have a response we can use. If we've already served (we are doing\n -- background work) then just exit. If it has been prepared and we were\n -- not_modified, then set 304 and serve. If it has been prepared, set\n -- status accordingly and serve. If not, prepare it.\n response_ready = {\n {\n in_case = \"served\",\n begin = \"exiting\"\n },\n {\n in_case = \"forced_cache\",\n begin = \"serving\",\n but_first = \"add_disconnected_warning\"\n },\n\n -- If we might ESI, then don't 304 downstream.\n {\n when = \"preparing_response\",\n in_case = \"esi_process_enabled\",\n begin = \"serving\",\n but_first = \"set_http_status_from_response\"\n },\n {\n when = \"preparing_response\",\n in_case = \"not_modified\",\n after = \"fetching\",\n begin = \"serving\",\n but_first = {\n \"set_http_not_modified\",\n \"disable_output_buffers\"\n }\n },\n {\n when = \"preparing_response\",\n in_case = \"not_modified\",\n begin = \"serving\",\n but_first = {\n \"set_http_not_modified\",\n \"install_no_body_reader\"\n }\n },\n {\n when = \"preparing_response\",\n begin = \"serving\",\n but_first = \"set_http_status_from_response\"\n },\n {\n begin = \"preparing_response\"\n },\n },\n\n -- We have sent the response. If it was stale, we go back around the\n -- fetching path so that a background revalidation can occur unless the\n -- upstream errored. Otherwise exit.\n served = {\n { in_case = \"upstream_error\", begin = \"exiting\" },\n { in_case = \"collapsed_forwarding_upstream_error\", begin = \"exiting\" },\n { in_case = \"response_cacheable\", begin = \"exiting\" },\n { begin = \"exiting\" },\n },\n\n -- When the client request is aborted clean up redis / http connections.\n -- If we're saving or have the collapse lock, then don't abort as we want\n -- to finish regardless.\n -- Note: this is a special entry point, triggered by ngx_lua client abort\n -- notification.\n aborted = {\n { in_case = \"response_cacheable\", begin = \"cancelling_abort_request\" },\n {\n in_case = \"obtained_collapsed_forwarding_lock\",\n begin = \"cancelling_abort_request\"\n },\n { begin = \"aborting\" },\n },\n\n\n -- Useful events for exiting with a common status. If we've already served\n -- (perhaps we're doing background work, we just exit without re-setting the\n -- status (as this errors).\n\n http_gateway_timeout = {\n { in_case = \"served\", begin = \"exiting\" },\n { begin = \"exiting\", but_first = \"set_http_gateway_timeout\" },\n },\n\n http_service_unavailable = {\n { in_case = \"served\", begin = \"exiting\" },\n { begin = \"exiting\", but_first = \"set_http_service_unavailable\" },\n },\n\n http_internal_server_error = {\n { in_case = \"served\", begin = \"exiting\" },\n { begin = \"exiting\", but_first = \"set_http_internal_server_error\" },\n },\n}\n" }, { "path": "lib/ledge/state_machine/pre_transitions.lua", "content": "local _M = { -- luacheck: no unused\n _VERSION = \"2.3.0\",\n}\n\n\n-- Pre-transitions. These actions will *always* be performed before\n-- transitioning.\nreturn {\n exiting = { \"redis_close\", \"httpc_close\" },\n aborting = { \"redis_close\", \"httpc_close_without_keepalive\" },\n checking_cache = { \"read_cache\" },\n\n -- Never fetch with client validators, but put them back afterwards.\n fetching = {\n \"remove_client_validators\", \"fetch\", \"restore_client_validators\"\n },\n\n -- Need to save the error response before reading from cache in case we\n -- need to serve it later\n considering_stale_error = {\n \"stash_error_response\",\n \"read_cache\"\n },\n\n -- Restore the saved response and set the status when serving an error page\n serving_upstream_error = {\n \"restore_error_response\",\n \"set_http_status_from_response\"\n },\n serving_stale = {\n \"set_http_status_from_response\",\n },\n cancelling_abort_request = {\n \"disable_output_buffers\"\n },\n}\n" }, { "path": "lib/ledge/state_machine/states.lua", "content": "local ledge = require(\"ledge\")\nlocal esi = require(\"ledge.esi\")\nlocal range = require(\"ledge.range\")\n\nlocal ngx_log = ngx.log\nlocal ngx_ERR = ngx.ERR\nlocal ngx_null = ngx.null\n\nlocal ngx_PARTIAL_CONTENT = 206\n\nlocal ngx_req_get_method = ngx.req.get_method\nlocal ngx_req_get_headers = ngx.req.get_headers\n\nlocal str_find = string.find\nlocal str_lower = string.lower\n\nlocal header_has_directive = require(\"ledge.header_util\").header_has_directive\n\nlocal can_revalidate_locally =\n require(\"ledge.validation\").can_revalidate_locally\nlocal must_revalidate = require(\"ledge.validation\").must_revalidate\nlocal is_valid_locally = require(\"ledge.validation\").is_valid_locally\n\nlocal can_serve_stale = require(\"ledge.stale\").can_serve_stale\nlocal can_serve_stale_if_error = require(\"ledge.stale\").can_serve_stale_if_error\nlocal can_serve_stale_while_revalidate =\n require(\"ledge.stale\").can_serve_stale_while_revalidate\n\nlocal req_accepts_cache = require(\"ledge.request\").accepts_cache\nlocal purge_mode = require(\"ledge.request\").purge_mode\n\nlocal purge = require(\"ledge.purge\").purge\nlocal purge_api = require(\"ledge.purge\").purge_api\nlocal purge_in_background = require(\"ledge.purge\").purge_in_background\nlocal create_purge_response = require(\"ledge.purge\").create_purge_response\n\nlocal acquire_lock = require(\"ledge.collapse\").acquire_lock\n\nlocal parse_content_range = require(\"ledge.range\").parse_content_range\n\nlocal vary_compare = require(\"ledge.cache_key\").vary_compare\n\n\nlocal _M = { -- luacheck: no unused\n _VERSION = \"2.3.0\",\n}\n\n\n-- Decision states.\n-- Represented as functions which should simply make a decision, and return\n-- calling state_machine:e(ev) with the event that has occurred. Place any\n-- further logic in actions triggered by the transition table.\nreturn {\n checking_method = function(sm)\n local method = ngx_req_get_method()\n if method == \"PURGE\" then\n return sm:e \"purge_requested\"\n elseif method ~= \"GET\" and method ~= \"HEAD\" then\n -- Only GET/HEAD are cacheable\n return sm:e \"cache_not_accepted\"\n else\n return sm:e \"cacheable_method\"\n end\n end,\n\n considering_purge_api = function(sm)\n local ct = ngx_req_get_headers()[\"Content-Type\"]\n if ct and str_lower(ct) == \"application/json\" then\n return sm:e \"purge_api_requested\"\n else\n return sm:e \"purge_requested\"\n end\n end,\n\n considering_wildcard_purge = function(sm, handler)\n local key_chain = handler:cache_key_chain()\n if str_find(key_chain.root, \"*\", 1, true) then\n return sm:e \"wildcard_purge_requested\"\n else\n return sm:e \"purge_requested\"\n end\n end,\n\n checking_origin_mode = function(sm, handler)\n -- Ignore the client requirements if we're not in \"NORMAL\" mode.\n if handler.config.origin_mode < ledge.ORIGIN_MODE_NORMAL then\n return sm:e \"forced_cache\"\n else\n return sm:e \"cacheable_method\"\n end\n end,\n\n accept_cache = function(sm)\n return sm:e \"cache_accepted\"\n end,\n\n checking_request = function(sm)\n if req_accepts_cache() then\n return sm:e \"cache_accepted\"\n else\n return sm:e \"cache_not_accepted\"\n end\n end,\n\n checking_cache = function(sm, handler)\n local res = handler.response\n\n if not next(res) then\n return sm:e \"cache_missing\"\n elseif res:has_expired() then\n return sm:e \"cache_expired\"\n else\n return sm:e \"cache_valid\"\n end\n end,\n\n considering_gzip_inflate = function(sm, handler)\n local res = handler.response\n local accept_encoding = ngx_req_get_headers()[\"Accept-Encoding\"] or \"\"\n\n -- If the response is gzip encoded and the client doesn't support it, then inflate\n if res.header[\"Content-Encoding\"] == \"gzip\" then\n local accepts_gzip = header_has_directive(accept_encoding, \"gzip\")\n\n if handler.esi_scan_enabled or\n (handler.config.gunzip_enabled and accepts_gzip == false) then\n return sm:e \"gzip_inflate_enabled\"\n end\n end\n\n return sm:e \"gzip_inflate_disabled\"\n end,\n\n considering_esi_scan = function(sm, handler)\n if handler.config.esi_enabled == true then\n local res = handler.response\n if not res.has_body then\n return sm:e \"esi_scan_disabled\"\n end\n\n -- Choose an ESI processor from the Surrogate-Control header\n -- (Currently there is only the ESI/1.0 processor)\n local processor = esi.choose_esi_processor(handler)\n if processor then\n if esi.is_allowed_content_type(res, handler.config.esi_content_types) then\n -- Store parser for processing\n -- TODO: Strictly this should be installed by the state machine\n handler.esi_processor = processor\n return sm:e \"esi_scan_enabled\"\n end\n end\n end\n\n return sm:e \"esi_scan_disabled\"\n end,\n\n -- We decide to process if:\n -- - We know the response has_esi (fast path)\n -- - We already decided to scan for esi (slow path)\n -- - We aren't delegating responsibility downstream, which would occur when both:\n -- - Surrogate-Capability is set with a matching parser type and version.\n -- - Delegation is enabled in configuration.\n --\n -- So essentially, if we think we may need to process, then we do. We don't want to\n -- accidentally send ESI instructions to a client, so we only delegate if we're sure.\n considering_esi_process = function(sm, handler)\n local res = handler.response\n\n -- If we know there's no esi or it hasn't been scanned, don't process\n if not res.has_esi and res.esi_scanned == false then\n return sm:e \"esi_process_disabled\"\n end\n\n if not next(handler.esi_processor) then\n -- On the fast path with ESI already detected, the processor wont have been loaded\n -- yet, so we must do that now\n -- TODO: Perhaps the state machine can load the processor to avoid this weird check\n if res.has_esi then\n local p = esi.choose_esi_processor(handler)\n if not p then\n -- This shouldn't happen\n -- if res.has_esi is set then a processor should be selectedable\n return sm:e \"esi_process_not_required\"\n else\n handler.esi_processor = p\n end\n else\n -- We know there's nothing to do\n return sm:e \"esi_process_not_required\"\n end\n end\n\n if esi.can_delegate_to_surrogate(\n handler.config.esi_allow_surrogate_delegation,\n handler.esi_processor.token\n ) then\n -- Disabled due to surrogate delegation\n return sm:e \"esi_process_disabled\"\n else\n return sm:e \"esi_process_enabled\"\n end\n end,\n\n checking_range_request = function(sm, handler)\n local res = handler.response\n\n -- TODO this should just check, not install range?\n local range = range.new()\n local res, partial_response = range:handle_range_request(res)\n\n handler.range = range\n handler.response = res\n\n if partial_response then\n return sm:e \"range_accepted\"\n elseif partial_response == false then\n return sm:e \"range_not_accepted\"\n else\n return sm:e \"range_not_requested\"\n end\n end,\n\n checking_can_fetch = function(sm, handler)\n if handler.config.origin_mode == ledge.ORIGIN_MODE_BYPASS then\n return sm:e \"http_service_unavailable\"\n end\n\n if header_has_directive(\n ngx_req_get_headers()[\"Cache-Control\"], \"only-if-cached\"\n ) then\n return sm:e \"http_gateway_timeout\"\n end\n\n if handler.config.enable_collapsed_forwarding then\n return sm:e \"can_fetch_but_try_collapse\"\n end\n\n return sm:e \"can_fetch\"\n end,\n\n requesting_collapse_lock = function(sm, handler)\n local timeout = tonumber(handler.config.collapsed_forwarding_window)\n if not timeout then\n ngx_log(ngx_ERR, \"collapsed_forwarding_window must be a number\")\n return sm:e \"collapsed_forwarding_failed\"\n end\n\n local redis = handler.redis\n local key_chain = handler:cache_key_chain()\n local lock_key = key_chain.fetching_lock\n\n local res, err = acquire_lock(redis, lock_key, timeout)\n\n if res == nil then -- Lua script failed\n if err then ngx_log(ngx_ERR, err) end\n return sm:e \"collapsed_forwarding_failed\"\n elseif res then -- We have the lock\n return sm:e \"obtained_collapsed_forwarding_lock\"\n else\n -- We didn't get the lock, try to collapse\n -- Create a new Redis connection and put it into subscribe mode\n -- Then check if the lock still exists as it may have been freed\n -- in the time between attempting to acquire and subscribing.\n -- In which case we have missed the publish event\n\n local redis_subscriber = ledge.create_redis_connection()\n local ok, err = redis_subscriber:subscribe(lock_key)\n if not ok or ok == ngx_null then\n -- Failed to enter subscribe mode\n if err then ngx_log(ngx_ERR, err) end\n return sm:e \"collapsed_forwarding_failed\"\n end\n\n local ok, err = redis:exists(lock_key)\n if ok == 1 then\n -- We subscribed before the lock was freed\n handler.redis_subscriber = redis_subscriber\n return sm:e \"subscribed_to_collapsed_forwarding_channel\"\n elseif ok == 0 then\n -- Lock was freed before we subscribed\n return sm:e \"collapsed_forwarding_channel_closed\"\n else\n -- Error checking lock still exists\n if err then ngx_log(ngx_ERR, err) end\n return sm:e \"collapsed_forwarding_failed\"\n end\n end\n end,\n\n publishing_collapse_success = function(sm, handler)\n local redis = handler.redis\n local key = handler._publish_key\n redis:del(key) -- Clear the lock\n redis:publish(key, \"collapsed_response_ready\")\n\n return sm:e \"published\"\n end,\n\n publishing_collapse_failure = function(sm, handler)\n local redis = handler.redis\n local key = handler._publish_key\n redis:del(key) -- Clear the lock\n redis:publish(key, \"collapsed_forwarding_failed\")\n\n return sm:e \"published\"\n end,\n\n publishing_collapse_upstream_error = function(sm, handler)\n local redis = handler.redis\n local key = handler._publish_key\n redis:del(key) -- Clear the lock\n redis:publish(key, \"collapsed_forwarding_upstream_error\")\n\n return sm:e \"published\"\n end,\n\n publishing_collapse_vary_modified = function(sm, handler)\n local redis = handler.redis\n local key = handler._publish_key\n redis:del(key) -- Clear the lock\n redis:publish(key, \"collapsed_forwarding_vary_modified\")\n\n return sm:e \"published\"\n end,\n\n fetching_as_surrogate = function(sm, handler)\n -- stash these because we might change the key\n -- depending on vary response\n local key_chain = handler:cache_key_chain()\n handler._publish_key = key_chain.fetching_lock\n\n return sm:e \"can_fetch\"\n end,\n\n considering_vary = function(sm, handler)\n local new_spec = handler.response:parse_vary_header()\n local key_chain = handler:cache_key_chain()\n\n if vary_compare(new_spec, key_chain.vary_spec) == false then\n handler:set_vary_spec(new_spec)\n return sm:e \"vary_modified\"\n\n else\n return sm:e \"vary_unmodified\"\n\n end\n end,\n\n waiting_on_collapsed_forwarding_channel = function(sm, handler)\n local redis = handler.redis_subscriber\n\n -- Extend the timeout to the size of the window\n redis:set_timeout(handler.config.collapsed_forwarding_window)\n local res, _ = redis:read_reply() -- block until we hear something or timeout\n if not res or res == ngx_null then\n return sm:e \"collapsed_forwarding_failed\"\n else\n -- TODO this config is now in the singleton\n redis:set_timeout(60) --handler.config.redis_read_timeout)\n redis:unsubscribe()\n ledge.close_redis_connection(redis)\n\n -- This is overly explicit for the sake of state machine introspection. That is\n -- we never call sm:e() without a literal event string.\n if res[3] == \"collapsed_response_ready\" then\n return sm:e \"collapsed_response_ready\"\n elseif res[3] == \"collapsed_forwarding_upstream_error\" then\n return sm:e \"collapsed_forwarding_upstream_error\"\n elseif res[3] == \"collapsed_forwarding_vary_modified\" then\n return sm:e \"collapsed_forwarding_vary_modified\"\n else\n return sm:e \"collapsed_forwarding_failed\"\n end\n end\n end,\n\n fetching = function(sm, handler)\n local res = handler.response\n\n if res.status >= 500 then\n return sm:e \"upstream_error\"\n elseif res.status == ngx.HTTP_NOT_MODIFIED then\n return sm:e \"response_ready\"\n elseif res.status == ngx_PARTIAL_CONTENT then\n return sm:e \"partial_response_fetched\"\n else\n return sm:e \"response_fetched\"\n end\n end,\n\n considering_background_fetch = function(sm, handler)\n local res = handler.response\n if res.status ~= ngx_PARTIAL_CONTENT then\n -- Shouldn't happen, but just in case\n return sm:e \"background_fetch_skipped\"\n else\n local content_range = res.header[\"Content-Range\"]\n if content_range then\n local _, _, size = parse_content_range(content_range)\n\n if size then\n local max_size = handler.storage:get_max_size()\n if type(max_size) == \"number\" and max_size > size then\n return sm:e \"can_fetch_in_background\"\n end\n end\n end\n\n return sm:e \"background_fetch_skipped\"\n end\n end,\n\n purging_via_api = function(sm, handler)\n local ok = purge_api(handler)\n if ok then\n return sm:e \"purge_api_completed\"\n else\n return sm:e \"purge_api_failed\"\n end\n end,\n\n purging = function(sm, handler)\n local mode = purge_mode()\n local ok, message, job = purge(handler, mode, handler:cache_key_chain().repset)\n local json = create_purge_response(mode, message, job)\n handler.response:set_body(json)\n\n if ok then\n return sm:e \"purged\"\n else\n return sm:e \"nothing_to_purge\"\n end\n end,\n\n wildcard_purging = function(sm, handler)\n purge_in_background(handler, purge_mode())\n return sm:e \"wildcard_purge_scheduled\"\n end,\n\n considering_stale_error = function(sm, handler)\n local res = handler.response\n if can_serve_stale_if_error(res) then\n return sm:e \"can_serve_disconnected\"\n else\n return sm:e \"can_serve_upstream_error\"\n end\n end,\n\n serving_upstream_error = function(sm, handler)\n handler:serve()\n return sm:e \"served\"\n end,\n\n considering_revalidation = function(sm, handler)\n if must_revalidate(handler.response) then\n return sm:e \"must_revalidate\"\n elseif can_revalidate_locally() then\n return sm:e \"can_revalidate_locally\"\n else\n return sm:e \"no_validator_present\"\n end\n end,\n\n considering_local_revalidation = function(sm)\n if can_revalidate_locally() then\n return sm:e \"can_revalidate_locally\"\n else\n return sm:e \"no_validator_present\"\n end\n end,\n\n revalidating_locally = function(sm, handler)\n if is_valid_locally(handler.response) then\n return sm:e \"not_modified\"\n else\n return sm:e \"modified\"\n end\n end,\n\n checking_can_serve_stale = function(sm, handler)\n local res = handler.response\n if handler.config.origin_mode < ledge.ORIGIN_MODE_NORMAL then\n return sm:e \"can_serve_stale\"\n elseif can_serve_stale_while_revalidate(res) then\n return sm:e \"can_serve_stale_while_revalidate\"\n elseif can_serve_stale(res) then\n return sm:e \"can_serve_stale\"\n else\n return sm:e \"cache_expired\"\n end\n end,\n\n updating_cache = function(sm, handler)\n local res = handler.response\n if res.has_body then\n if res:is_cacheable() then\n return sm:e \"response_cacheable\"\n else\n return sm:e \"response_not_cacheable\"\n end\n else\n return sm:e \"response_body_missing\"\n end\n end,\n\n preparing_response = function(sm)\n return sm:e \"response_ready\"\n end,\n\n serving = function(sm, handler)\n handler:serve()\n return sm:e \"served\"\n end,\n\n serving_stale = function(sm, handler)\n handler:serve()\n return sm:e \"served\"\n end,\n\n exiting = function()\n ngx.exit(ngx.status)\n end,\n\n cancelling_abort_request = function()\n return true\n end,\n\n aborting = function()\n ngx.exit(ngx.status)\n end,\n}\n" }, { "path": "lib/ledge/state_machine.lua", "content": "local events = require(\"ledge.state_machine.events\")\nlocal pre_transitions = require(\"ledge.state_machine.pre_transitions\")\nlocal states = require(\"ledge.state_machine.states\")\nlocal actions = require(\"ledge.state_machine.actions\")\n\nlocal ngx_log = ngx.log\nlocal ngx_DEBUG = ngx.DEBUG\nlocal ngx_ERR = ngx.ERR\n\n\nlocal get_fixed_field_metatable_proxy =\n require(\"ledge.util\").mt.get_fixed_field_metatable_proxy\n\n\nlocal _DEBUG = false\n\nlocal _M = {\n _VERSION = \"2.3.0\",\n set_debug = function(debug) _DEBUG = debug end,\n}\n\n\nlocal function new(handler)\n return setmetatable({\n handler = handler,\n state_history = {},\n event_history = {},\n current_state = \"\",\n }, get_fixed_field_metatable_proxy(_M))\nend\n_M.new = new\n\n\n-- Transition to a new state.\nlocal function t(self, state)\n -- Check for any transition pre-tasks\n local pre_t = pre_transitions[state]\n\n if pre_t then\n for _,action in ipairs(pre_t) do\n if _DEBUG then ngx_log(ngx_DEBUG, \"#a: \", action) end\n local ok, err = pcall(actions[action], self.handler)\n if not ok then\n ngx_log(ngx_ERR, \"state '\", state, \"' failed to call action '\", action, \"': \", tostring(err))\n end\n end\n end\n\n if _DEBUG then ngx_log(ngx_DEBUG, \"#t: \", state) end\n\n self.state_history[state] = true\n self.current_state = state\n return states[state](self, self.handler)\nend\n_M.t = t\n\n\n-- Process state transitions and actions based on the event fired.\nlocal function e(self, event)\n if _DEBUG then ngx_log(ngx_DEBUG, \"#e: \", event) end\n\n self.event_history[event] = true\n\n -- It's possible for states to call undefined events at run time.\n if not events[event] then\n ngx_log(ngx.CRIT, event, \" is not defined.\")\n ngx.status = ngx.HTTP_INTERNAL_SERVER_ERROR\n self:t(\"exiting\")\n end\n\n for _, trans in ipairs(events[event]) do\n local t_when = trans[\"when\"]\n if t_when == nil or t_when == self.current_state then\n local t_after = trans[\"after\"]\n if not t_after or self.state_history[t_after] then\n local t_in_case = trans[\"in_case\"]\n if not t_in_case or self.event_history[t_in_case] then\n local t_but_first = trans[\"but_first\"]\n if t_but_first then\n if type(t_but_first) == \"table\" then\n for _,action in ipairs(t_but_first) do\n if _DEBUG then\n ngx_log(ngx_DEBUG, \"#a: \", action)\n end\n actions[action](self.handler)\n end\n else\n if _DEBUG then\n ngx_log(ngx_DEBUG, \"#a: \", t_but_first)\n end\n actions[t_but_first](self.handler)\n end\n end\n\n return self:t(trans[\"begin\"])\n end\n end\n end\n end\nend\n_M.e = e\n\n\nreturn _M\n" }, { "path": "lib/ledge/storage/redis.lua", "content": "local redis_connector = require \"resty.redis.connector\"\n\nlocal tostring, pairs, next, unpack, setmetatable =\n tostring, pairs, next, unpack, setmetatable\n\nlocal ngx_null = ngx.null\nlocal ngx_log = ngx.log\nlocal ngx_ERR = ngx.ERR\nlocal ngx_WARN = ngx.WARN\n\nlocal tbl_insert = table.insert\nlocal tbl_copy_merge_defaults = require(\"ledge.util\").table.copy_merge_defaults\nlocal fixed_field_metatable = require(\"ledge.util\").mt.fixed_field_metatable\nlocal get_fixed_field_metatable_proxy =\n require(\"ledge.util\").mt.get_fixed_field_metatable_proxy\n\n\nlocal _M = {\n _VERSION = \"2.3.0\",\n}\n\n\n-- Default parameters\nlocal defaults = setmetatable({\n redis_connector_params = {},\n\n max_size = 1024 * 1024, -- Max storable size, in bytes\n\n -- Optional atomicity\n -- e.g. for use with a Redis proxy which doesn't support transactions\n supports_transactions = true,\n}, fixed_field_metatable)\n\n\n-- Redis key namespace\nlocal KEY_PREFIX = \"ledge:entity:\"\n\n\n-- Returns the Redis keys for entity_id\nlocal function entity_keys(entity_id)\n if entity_id then\n return {\n -- Both keys are lists of chunks\n body = KEY_PREFIX .. \"{\" .. entity_id .. \"}\" .. \":body\",\n body_esi = KEY_PREFIX .. \"{\" .. entity_id .. \"}\" .. \":body_esi\",\n }\n end\nend\n\n\n-- Creates a new (disconnected) storage instance\n--\n-- @return table The module instance\nfunction _M.new()\n return setmetatable({\n redis = {},\n params = {},\n\n _reader_cursor = 0,\n _keys_created = false,\n }, get_fixed_field_metatable_proxy(_M))\nend\n\n\n-- Connects to the Redis storage backend\n--\n-- @param table Module instance (self)\n-- @param table Storage params\nfunction _M.connect(self, user_params)\n -- take user_params by value and merge with defaults\n user_params = tbl_copy_merge_defaults(user_params, defaults)\n self.params = user_params\n\n local rc, err = redis_connector.new(\n user_params.redis_connector_params\n )\n if not rc then\n return nil, err\n end\n\n local redis, err = rc:connect()\n if not redis then\n return nil, err\n else\n self.redis = redis\n return self, nil\n end\nend\n\n\n-- Closes the Redis connection (placing back on the keepalive pool)\n--\n-- @param table Module instance (self)\nfunction _M.close(self)\n self._reader_cursor = 0\n self._keys_created = false\n\n local redis = self.redis\n if redis then\n local rc, err = redis_connector.new(\n self.params.redis_connector_params\n )\n if not rc then\n return nil, err\n end\n return rc:set_keepalive(redis)\n end\nend\n\n\n-- Returns the maximum size this connection is prepared to store.\n--\n-- @param table Module instance (self)\n-- @return number Size (bytes)\nfunction _M.get_max_size(self)\n return self.params.max_size\nend\n\n\n-- Returns a boolean indicating if the entity exists.\n--\n-- @param table Module instance (self)\n-- @param string The entity ID\n-- @return boolean (exists)\n-- @return string err (or nil)\nfunction _M.exists(self, entity_id)\n local keys = entity_keys(entity_id)\n if not keys then\n return nil, \"no entity id\"\n else\n local redis = self.redis\n\n redis:init_pipeline(2)\n redis:exists(keys.body)\n redis:exists(keys.body_esi)\n local res, err = redis:commit_pipeline()\n\n if not res and err then\n return nil, err\n elseif res == ngx_null or #res < 2 then\n return nil, \"expected 2 pipelined command results\"\n else\n return res[1] == 1 and res[2] == 1\n end\n end\nend\n\n\n-- Deletes an entity\n--\n-- @param table Module instance (self)\n-- @param string The entity ID\n-- @return boolean success\n-- @return string err (or nil)\nfunction _M.delete(self, entity_id)\n local key_chain = entity_keys(entity_id)\n if key_chain then\n local keys = {}\n for _, v in pairs(key_chain) do\n tbl_insert(keys, v)\n end\n local res, err = self.redis:del(unpack(keys))\n if res == 0 and not err then\n return false, nil\n else\n return res, err\n end\n end\nend\n\n\n-- Sets the time-to-live for an entity\n--\n-- @param table Module instance (self)\n-- @param string The entity ID\n-- @param number TTL (seconds)\n-- @return boolean success\n-- @return string err (or nil)\nfunction _M.set_ttl(self, entity_id, ttl)\n local key_chain = entity_keys(entity_id)\n if key_chain then\n local res, err\n for _,key in pairs(key_chain) do\n res, err = self.redis:expire(key, ttl)\n end\n if not res then\n return res, err\n elseif res == 0 then\n return false, \"entity does not exist\"\n else\n return true, nil\n end\n end\nend\n\n\n-- Gets the time-to-live for an entity\n--\n-- @param table Module instance (self)\n-- @param string The entity ID\n-- @return number ttl\n-- @return string err (or nil)\nfunction _M.get_ttl(self, entity_id)\n local key_chain = entity_keys(entity_id)\n if next(key_chain) then\n local res, err = self.redis:ttl(key_chain.body)\n if not res then\n return res, err\n elseif res == -2 then\n return false, \"entity does not exist\"\n elseif res == -1 then\n return false, \"entity does not have a ttl\"\n else\n return res, nil\n end\n end\nend\n\n\n-- Returns an iterator for reading the body chunks.\n--\n-- @param table Module instance (self)\n-- @param table Response object\n-- @return function Iterator, returning chunk, err, has_esi for each call\nfunction _M.get_reader(self, res)\n local redis = self.redis\n local entity_id = res.entity_id\n local entity_keys = entity_keys(entity_id)\n local num_chunks = redis:llen(entity_keys.body) or 0\n\n return function()\n local cursor = self._reader_cursor\n self._reader_cursor = cursor + 1\n\n if cursor < num_chunks then\n local chunk, err = redis:lindex(entity_keys.body, cursor)\n if not chunk then return nil, err, nil end\n\n local has_esi, err = redis:lindex(entity_keys.body_esi, cursor)\n if not has_esi then return nil, err, nil end\n\n if chunk == ngx_null or has_esi == ngx_null then\n ngx_log(ngx_WARN,\n \"entity removed during read, \",\n entity_keys.body\n )\n chunk = nil\n end\n\n return chunk, nil, has_esi == \"true\"\n end\n end\nend\n\n\n-- Writes a given chunk\nlocal function write_chunk(self, entity_keys, chunk, has_esi, ttl)\n local redis = self.redis\n\n -- Write chunks / has_esi onto lists\n local ok, e = redis:rpush(entity_keys.body, chunk)\n if not ok then return nil, e end\n\n ok, e = redis:rpush(entity_keys.body_esi, tostring(has_esi))\n if not ok then return nil, e end\n\n -- If this is the first write, set expiration too\n if not self._keys_created then\n self._keys_created = true\n\n ok, e = redis:expire(entity_keys.body, ttl)\n if not ok then return nil, e end\n\n ok, e = redis:expire(entity_keys.body_esi, ttl)\n if not ok then return nil, e end\n end\n\n return true, nil\nend\n\n\n-- Returns an iterator which writes chunks to cache as they are read from\n-- reader belonging to the repsonse object.\n-- If we cross the maxsize boundary, or error for any reason, we just\n-- keep yielding chunks to be served, after having removed the cache entry.\n--\n-- @param table Module instance (self)\n-- @param table Response object\n-- @param number time-to-live\n-- @param function onsuccess callback\n-- @param function onfailure callback\n-- @return function Iterator, returning chunk, err, has_esi for each call\nfunction _M.get_writer(self, res, ttl, onsuccess, onfailure)\n local redis = self.redis\n local max_size = self.params.max_size\n local supports_transactions = self.params.supports_transactions\n\n local entity_id = res.entity_id\n local entity_keys = entity_keys(entity_id)\n\n local failed = false\n local failed_reason = \"\"\n local transaction_open = false\n\n local size = 0\n local reader = res.body_reader\n\n return function(buffer_size)\n if not transaction_open and supports_transactions then\n redis:multi()\n transaction_open = true\n end\n\n local chunk, err, has_esi = reader(buffer_size)\n if not chunk and err then\n failed = true\n failed_reason = \"upstream error: \" .. err\n end\n\n if chunk and not failed then -- We have something to write\n size = size + #chunk\n\n if max_size and size > max_size then\n failed = true\n failed_reason = \"body is larger than \" .. max_size .. \" bytes\"\n else\n local ok, e = write_chunk(self,\n entity_keys,\n chunk,\n has_esi,\n ttl\n )\n if not ok then\n failed = true\n failed_reason = \"error writing: \" .. tostring(e)\n end\n end\n\n elseif not chunk and not failed then -- We're finished\n if supports_transactions then\n local ok, e = redis:exec() -- Commit\n\n if not ok or ok == ngx_null then\n -- Transaction failed\n ok, e = pcall(onfailure, e)\n if not ok then ngx_log(ngx_ERR, e) end\n end\n end\n\n -- All good, report success\n local ok, e = pcall(onsuccess, size)\n if not ok then ngx_log(ngx_ERR, e) end\n\n elseif not chunk and failed then -- We're finished, but failed\n if supports_transactions then\n redis:discard() -- Rollback\n else\n -- Attempt to clean up manually (connection could be dead)\n local ok, e = redis:del(\n entity_keys.body,\n entity_keys.body_esi\n )\n if not ok or ok == ngx_null then ngx_log(ngx_ERR, e) end\n end\n\n local ok, e = pcall(onfailure, failed_reason)\n if not ok then ngx_log(ngx_ERR, e) end\n end\n\n -- Always bubble up\n return chunk, err, has_esi\n end\nend\n\n\nreturn _M\n" }, { "path": "lib/ledge/util.lua", "content": "local ngx_var = ngx.var\nlocal ffi = require \"ffi\"\n\nlocal type, next, setmetatable, getmetatable, error, tostring =\n type, next, setmetatable, getmetatable, error, tostring\n\nlocal str_find = string.find\nlocal str_sub = string.sub\nlocal co_create = coroutine.create\nlocal co_status = coroutine.status\nlocal co_resume = coroutine.resume\nlocal math_floor = math.floor\nlocal ffi_cdef = ffi.cdef\nlocal ffi_new = ffi.new\nlocal ffi_string = ffi.string\nlocal C = ffi.C\n\n\nlocal ok, err = pcall(ffi_cdef, [[\ntypedef unsigned char u_char;\nu_char * ngx_hex_dump(u_char *dst, const u_char *src, size_t len);\nint RAND_pseudo_bytes(u_char *buf, int num);\n]])\nif not ok then ngx.log(ngx.ERR, err) end\n\nlocal ok, err = pcall(ffi_cdef, [[\nint gethostname (char *name, size_t size);\n]])\nif not ok then ngx.log(ngx.ERR, err) end\n\n\nlocal _M = {\n _VERSION = \"2.3.0\",\n string = {},\n table = {},\n mt = {},\n coroutine = {},\n}\n\n\nlocal function randomhex(len)\n local len = math_floor(len / 2)\n\n local bytes = ffi_new(\"uint8_t[?]\", len)\n C.RAND_pseudo_bytes(bytes, len)\n if not bytes then\n return nil, \"error getting random bytes via FFI\"\n end\n\n local hex = ffi_new(\"uint8_t[?]\", len * 2)\n C.ngx_hex_dump(hex, bytes, len)\n return ffi_string(hex, len * 2)\nend\n_M.string.randomhex = randomhex\n\n\nlocal function str_split(str, delim)\n local pos, endpos, prev, i = 0, 0, 0, 0 -- luacheck: ignore pos endpos\n local out = {}\n repeat\n pos, endpos = str_find(str, delim, prev, true)\n i = i+1\n if pos then\n out[i] = str_sub(str, prev, pos-1)\n else\n if prev <= #str then\n out[i] = str_sub(str, prev, -1)\n end\n break\n end\n prev = endpos +1\n until pos == nil\n\n return out\nend\n_M.string.split = str_split\n\n\n-- A metatable which prevents undefined fields from being created / accessed\nlocal fixed_field_metatable = {\n __index =\n function(t, k) -- luacheck: no unused\n error(\"field \" .. tostring(k) .. \" does not exist\", 3)\n end,\n __newindex =\n function(t, k, v) -- luacheck: no unused\n error(\"attempt to create new field \" .. tostring(k), 3)\n end,\n}\n_M.mt.fixed_field_metatable = fixed_field_metatable\n\n\n-- Returns a metatable with fixed fields (as above), which when applied to a\n-- table will provide default values via the provided `proxy`. E.g:\n--\n-- defaults = { a = 1, b = 2, c = 3 }\n-- t = setmetatable({ b = 4 }, get_fixed_field_metatable_proxy(defaults))\n--\n-- `t` now gives: { a = 1, b = 4, c = 3 }\n--\n-- @param table proxy table\n-- @return table metatable\nlocal function get_fixed_field_metatable_proxy(proxy)\n return {\n __index =\n function(t, k) -- luacheck: no unused\n return proxy[k] or\n error(\"field \" .. tostring(k) .. \" does not exist\", 2)\n end,\n __newindex =\n function(t, k, v)\n if proxy[k] then\n return rawset(t, k, v)\n else\n error(\"attempt to create new field \" .. tostring(k), 2)\n end\n end,\n }\nend\n_M.mt.get_fixed_field_metatable_proxy = get_fixed_field_metatable_proxy\n\n\n-- Returns a metatable with fixed fields (as above), which when invoked as a\n-- function will call the supplied `func`. E.g.:\n--\n-- t = setmetatable(\n-- { a = 1, b = 2, c = 3 },\n-- get_callable_fixed_field_metatable(\n-- function(t, field)\n-- print(t[field])\n-- end\n-- )\n-- )\n-- t(\"a\") -- 1\n-- t(\"b\") -- 2\n--\n-- @param function\n-- @return table callable metatable\nlocal function get_callable_fixed_field_metatable(func)\n local mt = fixed_field_metatable\n mt.__call = func\n return mt\nend\n_M.mt.get_callable_fixed_field_metatable = get_callable_fixed_field_metatable\n\n\n-- Returns a new table, recursively copied from the one given, retaining\n-- metatable assignment.\n--\n-- @param table table to be copied\n-- @return table\nlocal function tbl_copy(orig)\n local orig_type = type(orig)\n local copy\n if orig_type == \"table\" then\n copy = {}\n for orig_key, orig_value in next, orig, nil do\n copy[tbl_copy(orig_key)] = tbl_copy(orig_value)\n end\n setmetatable(copy, tbl_copy(getmetatable(orig)))\n else -- number, string, boolean, etc\n copy = orig\n end\n return copy\nend\n_M.table.copy = tbl_copy\n\n\n-- Returns a new table, recursively copied from the combination of the given\n-- table `t1`, with any missing fields copied from `defaults`.\n--\n-- If `defaults` is of type \"fixed field\" and `t1` contains a field name not\n-- present in the defults, an error will be thrown.\n--\n-- @param table t1\n-- @param table defaults\n-- @return table a new table, recursively copied and merged\nlocal function tbl_copy_merge_defaults(t1, defaults)\n if t1 == nil then t1 = {} end\n if defaults == nil then defaults = {} end\n if type(t1) == \"table\" and type(defaults) == \"table\" then\n local copy = {}\n for t1_key, t1_value in next, t1, nil do\n copy[tbl_copy(t1_key)] = tbl_copy_merge_defaults(\n t1_value, tbl_copy(defaults[t1_key])\n )\n end\n for defaults_key, defaults_value in next, defaults, nil do\n if t1[defaults_key] == nil then\n copy[tbl_copy(defaults_key)] = tbl_copy(defaults_value)\n end\n end\n return copy\n else\n return t1 -- not a table\n end\nend\n_M.table.copy_merge_defaults = tbl_copy_merge_defaults\n\n\nlocal function co_wrap(func)\n local co = co_create(func)\n if not co then\n return nil, \"could not create coroutine\"\n else\n return function(...)\n if co_status(co) == \"suspended\" then\n -- Handle errors in coroutines\n local ok, val1, val2, val3 = co_resume(co, ...)\n if ok == true then\n return val1, val2, val3\n else\n return nil, val1\n end\n else\n return nil, \"can't resume a \" .. co_status(co) .. \" coroutine\"\n end\n end\n end\nend\n_M.coroutine.wrap = co_wrap\n\n\nlocal function get_hostname()\n local name = ffi_new(\"char[?]\", 255)\n C.gethostname(name, 255)\n return ffi_string(name)\nend\n_M.get_hostname = get_hostname\n\n\nlocal function append_server_port(name)\n -- TODO: compare with scheme?\n local server_port = ngx_var.server_port\n if server_port ~= \"80\" and server_port ~= \"443\" then\n name = name .. \":\" .. server_port\n end\n return name\nend\n_M.append_server_port = append_server_port\n\n\nreturn _M\n" }, { "path": "lib/ledge/validation.lua", "content": "local ngx_req_get_headers = ngx.req.get_headers\nlocal ngx_req_set_header = ngx.req.set_header\nlocal ngx_parse_http_time = ngx.parse_http_time\n\nlocal get_numeric_header_token =\n require(\"ledge.header_util\").get_numeric_header_token\nlocal header_has_directive = require(\"ledge.header_util\").header_has_directive\n\n\nlocal _M = {\n _VERSION = \"2.3.0\",\n}\n\n\n-- True if the request or response (res) demand revalidation.\nlocal function must_revalidate(res)\n local req_cc = ngx_req_get_headers()[\"Cache-Control\"]\n local req_cc_max_age = get_numeric_header_token(req_cc, \"max-age\")\n if req_cc_max_age == 0 then\n return true\n else\n local res_age = tonumber(res.header[\"Age\"])\n local res_cc = res.header[\"Cache-Control\"]\n\n if header_has_directive(res_cc, \"(must|proxy)-revalidate\") then\n return true\n elseif req_cc_max_age and res_age then\n if req_cc_max_age < res_age then\n return true\n end\n end\n end\n return false\nend\n_M.must_revalidate = must_revalidate\n\n\n-- True if the request contains valid conditional headers.\nlocal function can_revalidate_locally()\n local req_h = ngx_req_get_headers()\n local req_ims = req_h[\"If-Modified-Since\"]\n\n if req_ims then\n if not ngx_parse_http_time(req_ims) then\n -- Bad IMS HTTP datestamp, lets remove this.\n ngx_req_set_header(\"If-Modified-Since\", nil)\n else\n return true\n end\n end\n\n if req_h[\"If-None-Match\"] and req_h[\"If-None-Match\"] ~= \"\" then\n return true\n end\n\n return false\nend\n_M.can_revalidate_locally = can_revalidate_locally\n\n\n-- True if the request conditions indicate that the response (res) can be served\nlocal function is_valid_locally(res)\n local req_h = ngx_req_get_headers()\n\n local res_lm = res.header[\"Last-Modified\"]\n local req_ims = req_h[\"If-Modified-Since\"]\n\n if res_lm and req_ims then\n local res_lm_parsed = ngx_parse_http_time(res_lm)\n local req_ims_parsed = ngx_parse_http_time(req_ims)\n\n if res_lm_parsed and req_ims_parsed then\n if res_lm_parsed <= req_ims_parsed then\n return true\n end\n end\n end\n\n local res_etag = res.header[\"Etag\"]\n local req_inm = req_h[\"If-None-Match\"]\n if res_etag and req_inm and res_etag == req_inm then\n return true\n end\n\n return false\nend\n_M.is_valid_locally = is_valid_locally\n\n\nreturn _M\n" }, { "path": "lib/ledge/worker.lua", "content": "local setmetatable = setmetatable\nlocal co_yield = coroutine.yield\n\nlocal ngx_get_phase = ngx.get_phase\n\nlocal tbl_copy_merge_defaults = require(\"ledge.util\").table.copy_merge_defaults\nlocal fixed_field_metatable = require(\"ledge.util\").mt.fixed_field_metatable\n\n\nlocal _M = {\n _VERSION = \"2.3.0\",\n}\n\n\nlocal defaults = setmetatable({\n interval = 1,\n gc_queue_concurrency = 1,\n purge_queue_concurrency = 1,\n revalidate_queue_concurrency = 1,\n}, fixed_field_metatable)\n\n\nlocal function new(config)\n assert(ngx_get_phase() == \"init_worker\",\n \"attempt to create ledge worker outside of the init_worker phase\")\n\n -- Take config by value and merge with defaults\n local config = tbl_copy_merge_defaults(config, defaults)\n return setmetatable({ config = config }, {\n __index = _M,\n })\nend\n_M.new = new\n\n\nlocal function run(self)\n assert(ngx_get_phase() == \"init_worker\",\n \"attempt to run ledge worker outside of the init_worker phase\")\n\n local ledge = require(\"ledge\")\n\n local ql_worker = assert(require(\"resty.qless.worker\").new({\n get_redis_client = ledge.create_qless_connection,\n close_redis_client = ledge.close_redis_connection\n }))\n\n -- Runs around job exectution, to instantiate necessary connections\n ql_worker.middleware = function(job)\n job.redis = ledge.create_redis_connection()\n\n co_yield() -- Perform the job\n\n ledge.close_redis_connection(job.redis)\n end\n\n -- Start a worker for each fo the queues\n\n assert(ql_worker:start({\n interval = self.config.interval,\n concurrency = self.config.gc_queue_concurrency,\n reserver = \"ordered\",\n queues = { \"ledge_gc\" },\n }))\n\n assert(ql_worker:start({\n interval = self.config.interval,\n concurrency = self.config.purge_queue_concurrency,\n reserver = \"ordered\",\n queues = { \"ledge_purge\" },\n }))\n\n assert(ql_worker:start({\n interval = self.config.interval or 1,\n concurrency = self.config.revalidate_queue_concurrency,\n reserver = \"ordered\",\n queues = { \"ledge_revalidate\" },\n }))\n\n return true\nend\n_M.run = run\n\n\nreturn setmetatable(_M, fixed_field_metatable)\n" }, { "path": "lib/ledge.lua", "content": "local setmetatable, require =\n setmetatable, require\n\nlocal ngx_get_phase = ngx.get_phase\nlocal ngx_null = ngx.null\n\nlocal tbl_insert = table.insert\n\nlocal util = require(\"ledge.util\")\nlocal tbl_copy = util.table.copy\nlocal tbl_copy_merge_defaults = util.table.copy_merge_defaults\nlocal fixed_field_metatable = util.mt.fixed_field_metatable\n\nlocal redis_connector = require(\"resty.redis.connector\")\n\n\nlocal _M = {\n _VERSION = \"2.3.0\",\n\n ORIGIN_MODE_BYPASS = 1, -- Never go to the origin, serve from cache or 503\n ORIGIN_MODE_AVOID = 2, -- Avoid the origin, serve from cache where possible\n ORIGIN_MODE_NORMAL = 4, -- Assume the origin is happy, use at will\n}\n\n\nlocal config = setmetatable({\n redis_connector_params = {\n connect_timeout = 500, -- (ms)\n read_timeout = 5000, -- (ms)\n keepalive_timeout = 60000, -- (ms)\n keepalive_poolsize = 30,\n },\n\n qless_db = 1,\n}, fixed_field_metatable)\n\n\nlocal function configure(user_config)\n assert(ngx_get_phase() == \"init\",\n \"attempt to call configure outside the 'init' phase\")\n\n config = setmetatable(\n tbl_copy_merge_defaults(user_config, config),\n fixed_field_metatable\n )\nend\n_M.configure = configure\n\n\nlocal handler_defaults = setmetatable({\n storage_driver = \"ledge.storage.redis\",\n storage_driver_config = {},\n\n origin_mode = _M.ORIGIN_MODE_NORMAL,\n\n -- Note that upstream timeout and keepalive config is shared with outbound\n -- ESI request, which are not necessarily configured to use this \"upstream\"\n upstream_connect_timeout = 1000, -- (ms)\n upstream_send_timeout = 2000, -- (ms)\n upstream_read_timeout = 10000, -- (ms)\n upstream_keepalive_timeout = 75000, -- (ms)\n upstream_keepalive_poolsize = 64,\n\n upstream_host = \"\",\n upstream_port = 80,\n upstream_use_ssl = false,\n upstream_ssl_server_name = \"\",\n upstream_ssl_verify = true,\n\n advertise_ledge = true,\n visible_hostname = util.get_hostname(),\n\n buffer_size = 2^16,\n keep_cache_for = 86400 * 30, -- (sec)\n minimum_old_entity_download_rate = 56,\n\n esi_enabled = false,\n esi_content_types = { \"text/html\" },\n esi_allow_surrogate_delegation = false,\n esi_recursion_limit = 10,\n esi_args_prefix = \"esi_\",\n esi_max_size = 1024 * 1024, -- (bytes)\n esi_custom_variables = {},\n esi_attempt_loopback = true,\n esi_vars_cookie_blacklist = {},\n\n esi_disable_third_party_includes = false,\n esi_third_party_includes_domain_whitelist = {},\n\n enable_collapsed_forwarding = false,\n collapsed_forwarding_window = 60 * 1000,\n\n gunzip_enabled = true,\n keyspace_scan_count = 10,\n\n cache_key_spec = {}, -- No default as we don't ever wish to merge it\n max_uri_args = 100,\n\n}, fixed_field_metatable)\n\n\n-- events are not fixed field to avoid runtime fatal errors from bad config\n-- ledge.bind() and handler:bind() both check validity of event names however.\nlocal event_defaults = {\n after_cache_read = {},\n before_upstream_connect = {},\n before_upstream_request = {},\n after_upstream_request = {},\n before_vary_selection = {},\n before_save = {},\n before_save_revalidation_data = {},\n before_serve = {},\n before_esi_include_request = {},\n}\n\n\nlocal function set_handler_defaults(user_config)\n assert(ngx_get_phase() == \"init\",\n \"attempt to call set_handler_defaults outside the 'init' phase\")\n\n handler_defaults = setmetatable(\n tbl_copy_merge_defaults(user_config, handler_defaults),\n fixed_field_metatable\n )\nend\n_M.set_handler_defaults = set_handler_defaults\n\n\nlocal function bind(event, callback)\n assert(ngx_get_phase() == \"init\",\n \"attempt to call bind outside the 'init' phase\")\n\n local ev = event_defaults[event]\n assert(ev, \"no such event: \" .. tostring(event))\n\n tbl_insert(ev, callback)\n return true\nend\n_M.bind = bind\n\n\nlocal function create_worker(config)\n return require(\"ledge.worker\").new(config)\nend\n_M.create_worker = create_worker\n\n\nlocal function create_handler(config)\n local config = tbl_copy_merge_defaults(config, handler_defaults)\n return require(\"ledge.handler\").new(config, tbl_copy(event_defaults))\nend\n_M.create_handler = create_handler\n\n\nlocal function create_redis_connection()\n local rc, err = redis_connector.new(config.redis_connector_params)\n if not rc then\n return nil, err\n end\n\n return rc:connect()\nend\n_M.create_redis_connection = create_redis_connection\n\n\nlocal function create_redis_slave_connection()\n local params = tbl_copy_merge_defaults(\n { role = \"slave\" },\n config.redis_connector_params\n )\n\n local rc, err = redis_connector.new(params)\n if not rc then\n return nil, err\n end\n\n return rc:connect()\nend\n_M.create_redis_slave_connection = create_redis_slave_connection\n\n\nlocal function close_redis_connection(redis)\n if not next(redis) then\n -- Possible for this to be called before we've created a redis conn\n -- Ensure we actually have a resty-redis instance to close\n return nil, \"No redis connection to close\"\n end\n\n local rc, err = redis_connector.new(config.redis_connector_params)\n if not rc then\n return nil, err\n end\n\n return rc:set_keepalive(redis)\nend\n_M.close_redis_connection = close_redis_connection\n\n\nlocal function create_qless_connection()\n local redis, err = create_redis_connection()\n if not redis then return nil, err end\n\n local ok, err = redis:select(config.qless_db)\n if not ok or ok == ngx_null then return nil, err end\n\n return redis\nend\n_M.create_qless_connection = create_qless_connection\n\n\nlocal function create_storage_connection(driver_module, storage_driver_config)\n -- Take config by value, and merge with defaults\n storage_driver_config = tbl_copy_merge_defaults(\n storage_driver_config or {},\n handler_defaults.storage_driver_config\n )\n\n if not driver_module then\n driver_module = handler_defaults.storage_driver\n end\n\n local ok, module = pcall(require, driver_module)\n if not ok then return nil, module end\n\n local ok, driver, err = pcall(module.new)\n if not ok then return nil, driver end\n if not driver then return nil, err end\n\n local ok, conn, err = pcall(driver.connect, driver, storage_driver_config)\n if not ok then return nil, conn end\n if not conn then return nil, err end\n\n return conn, nil\nend\n_M.create_storage_connection = create_storage_connection\n\n\nlocal function close_storage_connection(storage)\n return storage:close()\nend\n_M.close_storage_connection = close_storage_connection\n\n\nreturn setmetatable(_M, fixed_field_metatable)\n" }, { "path": "migrations/1.26-1.27.lua", "content": "local redis_connector = require(\"resty.redis.connector\").new()\nlocal math_floor = math.floor\nlocal math_ceil = math.ceil\nlocal ffi = require \"ffi\"\nlocal ffi_cdef = ffi.cdef\nlocal ffi_new = ffi.new\nlocal ffi_string = ffi.string\nlocal C = ffi.C\n\nffi_cdef[[\ntypedef unsigned char u_char;\nu_char * ngx_hex_dump(u_char *dst, const u_char *src, size_t len);\nint RAND_pseudo_bytes(u_char *buf, int num);\n]]\n\n\nlocal function random_hex(len)\n local len = math_floor(len / 2)\n\n local bytes = ffi_new(\"uint8_t[?]\", len)\n C.RAND_pseudo_bytes(bytes, len)\n if not bytes then\n ngx_log(ngx_ERR, \"error getting random bytes via FFI\")\n return nil\n end\n\n local hex = ffi_new(\"uint8_t[?]\", len * 2)\n C.ngx_hex_dump(hex, bytes, len)\n return ffi_string(hex, len * 2)\nend\n\n\nfunction delete(redis, cache_key, entities)\n redis:multi()\n -- Entities list is intact, so delete them too\n for _, entity in ipairs(entities) do\n delete_entity(redis, cache_key .. \"::entities\", entity)\n end\n\n local keys = {\n cache_key .. \"::key\",\n cache_key .. \"::memused\",\n cache_key .. \"::entities\",\n }\n redis:del(unpack(keys))\n return redis:exec()\nend\n\n\nfunction delete_entity(redis, set, entity)\n local keys = {\n entity,\n entity .. \":reval_req_headers\",\n entity .. \":reval_params\",\n entity .. \":headers\",\n entity .. \":body\",\n entity .. \":body_esi\",\n }\n local res, err = redis:del(unpack(keys))\n\n -- Remove from the entities set\n local res, err = redis:zrem(set, entity)\nend\n\n\nfunction delete_old_entities(redis, set, members, current_entity)\n for _, entity in ipairs(members) do\n if entity ~= current_entity then\n delete_entity(redis, set, entity)\n end\n end\nend\n\n\nfunction scan(cursor, redis)\n local res, err = redis:scan(\n cursor,\n \"MATCH\", \"ledge:cache:*::key\", -- We use the \"main\" key to single out a cache entry\n \"COUNT\", 100\n )\n\n if not res or res == ngx_null then\n return nil, \"SCAN error: \" .. tostring(err)\n else\n for _,key in ipairs(res[2]) do\n -- Strip the \"main\" suffix to find the cache key\n local cache_key = string.sub(key, 1, -(string.len(\"::key\") + 1))\n local skip = false\n\n local entity, entity_err = redis:get(cache_key .. \"::key\")\n if entity_err == nil then entity = nil end -- prevent concatentation error\n local memused, memused_err = redis:get(cache_key .. \"::memused\")\n local score = redis:zscore(cache_key .. \"::entities\", cache_key .. \"::\" .. (entity or \"\"))\n local entity_count = redis:zcard(cache_key .. \"::entities\")\n local entity_members = redis:zrange(cache_key .. \"::entities\", 0, -1)\n\n for _, val in ipairs({ entity, memused, score, entity_count, entity_members }) do\n if not val or val == ngx.null then\n -- If we're missing something we need (likely evicted) -- delete this key\n if delete(redis, cache_key, entity_members) then\n keys_deleted = keys_deleted + 1\n else\n keys_failed = keys_failed + 1\n end\n skip = true\n end\n end\n\n -- Watch the main key - if it gets created by real traffic from here on in then\n -- the transaction will simply fail.\n local res = redis:watch(cache_key .. \"::main\")\n\n -- Find out if real traffic already created this cache entry\n local new_entity = redis:hget(cache_key .. \"::main\", \"entity\")\n if new_entity and new_entity ~= ngx.null then\n -- The old entities refs will still exist, so clean them up\n delete_old_entities(redis, cache_key .. \"::entities\", entity_members, new_entity)\n keys_processed = keys_processed + 1\n skip = true\n end\n\n if not skip then\n -- Start transaction\n redis:multi()\n\n -- Move main entity to main key\n local ok, err = redis:rename(cache_key .. \"::\" .. entity, cache_key .. \"::main\")\n\n -- Rename headers etc\n for _, k in ipairs({ \"headers\", \"reval_req_headers\", \"reval_params\" }) do\n local ok, err = redis:rename(\n cache_key .. \"::\" .. entity .. \":\" .. k,\n cache_key .. \"::\" .. k\n )\n end\n\n -- Create a new entity id and rename the live entity to it\n local new_entity_id = random_hex(32)\n for _, k in ipairs({ \"body\", \"body_esi\" }) do\n local ok, err = redis:rename(\n cache_key .. \"::\" .. entity .. \":\" .. k,\n \"ledge:entity:\" .. new_entity_id .. \":\" .. k\n )\n end\n\n -- Add the entity to the entities set\n local res, err = redis:zadd(cache_key .. \"::entities\", score, new_entity_id)\n\n -- Remove the old form\n local res, err = redis:zrem(\n cache_key .. \"::entities\",\n cache_key .. \"::\" .. entity\n )\n\n -- Add the live entity pointer to the main hash, and delete the old pointer\n local ok, err = redis:hset(cache_key .. \"::main\", \"entity\", new_entity_id)\n local ok, err = redis:del(cache_key .. \"::key\")\n\n -- Add the memused to the main hash, and delete the old key\n local ok, err = redis:hset(cache_key .. \"::main\", \"memused\", memused)\n local ok, err = redis:del(cache_key .. \"::memused\")\n\n -- Delete entities scheduled for GC but will fail on new codebase\n delete_old_entities(redis, cache_key .. \"::entities\", entity_members, new_entity_id)\n\n local res, err = redis:exec()\n if not res or res == ngx.null then\n ngx.say(\"transaction failed\")\n -- Something went wrong, lets try and delete this cache entry\n if delete(redis, cache_key, entity_members) then\n keys_deleted = keys_deleted + 1\n else\n keys_failed = keys_failed + 1\n end\n else\n keys_processed = keys_processed + 1\n end\n end\n end\n end\n\n local cursor = tonumber(res[1])\n if cursor > 0 then\n -- If we have a valid cursor, recurse to move on.\n return scan(cursor, redis)\n end\n\n return true\nend\n\nlocal dsn = arg[1]\nif not dsn then\n ngx.say(\"Please provide a Redis Connector DSN as the first argument, in the form: redis://[PASSWORD@]HOST:PORT/DB\")\nelse\n local redis, err = redis_connector:connect{ url = dsn }\n if not redis then\n ngx.say(\"Could not connect to Redis with DSN: \", dsn, \" - \", err)\n return\n end\n\n keys_processed = 0\n keys_deleted = 0\n keys_failed = 0\n\n ngx.say(\"Migrating Ledge data structure from v1.26 to v1.27\\n\")\n\n local res, err = scan(0, redis)\n if not res or res == ngx.null then\n ngx.say(\"Faied to scan keyspace: \", err)\n else\n ngx.say(\"> \", keys_processed .. \" cache entries successfully updated\")\n ngx.say(\"> \", keys_deleted .. \" incomplete / broken cache entries cleaned up\")\n ngx.say(\"> \", keys_failed .. \" failures\\n\")\n end\nend\n" }, { "path": "t/01-unit/cache_key.t", "content": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::http_config();\n\nno_long_string();\nno_diff();\nrun_tests();\n\n__DATA__\n=== TEST 1: Root key is the same with nil ngx.var.args and empty string\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local ledge_cache_key = require(\"ledge.cache_key\")\n\n local key1 = ledge_cache_key.generate_root_key(nil, nil)\n\n ngx.req.set_uri_args({})\n\n local key2 = ledge_cache_key.generate_root_key(nil, nil)\n\n assert(key1 == key2, \"key1 should equal key2\")\n }\n}\n\n--- request\nGET /t\n--- no_error_log\n[error]\n\n\n=== TEST 2: Custom key spec\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local ledge_cache_key = require(\"ledge.cache_key\")\n\n local root_key = ledge_cache_key.generate_root_key(nil, nil)\n\n assert(root_key == \"ledge:cache:http:localhost:/t:a=1\",\n \"root_key should be ledge:cache:http:localhost:/t:a=1\")\n\n local cache_key_spec = {\n \"scheme\",\n \"host\",\n \"port\",\n \"uri\",\n \"args\",\n }\n local root_key = ledge_cache_key.generate_root_key(cache_key_spec, nil)\n\n assert(root_key == \"ledge:cache:http:localhost:1984:/t:a=1\",\n \"root_key should be ledge:cache:http:localhost:1984:/t:a=1\")\n\n local cache_key_spec = {\n \"host\",\n \"uri\",\n }\n local root_key = ledge_cache_key.generate_root_key(cache_key_spec, nil)\n\n assert(root_key == \"ledge:cache:localhost:/t\",\n \"root_key should be ledge:cache:localhost:/t\")\n\n\n local cache_key_spec = {\n \"host\",\n \"uri\",\n function() return \"hello\" end,\n }\n local root_key = ledge_cache_key.generate_root_key(cache_key_spec, nil)\n\n assert(root_key == \"ledge:cache:localhost:/t:hello\",\n \"root_key should be ledge:cache:localhost:/t:hello\")\n }\n}\n\n--- request\nGET /t?a=1\n--- no_error_log\n[error]\n\n\n=== TEST 3: Errors in cache key spec functions\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local ledge_cache_key = require(\"ledge.cache_key\")\n\n local cache_key_spec = {\n \"host\",\n \"uri\",\n function() return 123 end,\n }\n local root_key = ledge_cache_key.generate_root_key(cache_key_spec, nil)\n\n assert(root_key == \"ledge:cache:localhost:/t\",\n \"cache_key should be ledge:cache:localhost:/t\")\n\n\n local cache_key_spec = {\n \"host\",\n \"uri\",\n function() return foo() end,\n }\n local root_key = ledge_cache_key.generate_root_key(cache_key_spec, nil)\n\n assert(root_key == \"ledge:cache:localhost:/t\",\n \"cache_key should be ledge:cache:localhost:/t\")\n }\n}\n\n--- request\nGET /t?a=2\n--- error_log\nfunctions supplied to cache_key_spec must return a string\nerror in function supplied to cache_key_spec\n\n\n=== TEST 4: URI args are sorted (normalised)\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local ledge_cache_key = require(\"ledge.cache_key\")\n\n local root_key = ledge_cache_key.generate_root_key(nil, nil)\n ngx.print(root_key)\n }\n}\n--- request eval\n[\n \"GET /t\",\n \"GET /t?a=1\",\n \"GET /t?aba=1&aab=2\",\n \"GET /t?a=1&b=2&c=3\",\n \"GET /t?b=2&a=1&c=3\",\n \"GET /t?c=3&a=1&b=2\",\n \"GET /t?c=3&b&a=1\",\n \"GET /t?c=3&b=&a=1\",\n \"GET /t?c=3&b=2&a=1&b=4\",\n]\n--- response_body eval\n[\n \"ledge:cache:http:localhost:/t:\",\n \"ledge:cache:http:localhost:/t:a=1\",\n \"ledge:cache:http:localhost:/t:aab=2&aba=1\",\n \"ledge:cache:http:localhost:/t:a=1&b=2&c=3\",\n \"ledge:cache:http:localhost:/t:a=1&b=2&c=3\",\n \"ledge:cache:http:localhost:/t:a=1&b=2&c=3\",\n \"ledge:cache:http:localhost:/t:a=1&b&c=3\",\n \"ledge:cache:http:localhost:/t:a=1&b=&c=3\",\n \"ledge:cache:http:localhost:/t:a=1&b=2&b=4&c=3\",\n]\n--- no_error_log\n[error]\n\n\n=== TEST 5: Max URI args\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local ledge_cache_key = require(\"ledge.cache_key\")\n\n local root_key = ledge_cache_key.generate_root_key(nil, 2)\n ngx.print(root_key)\n }\n}\n--- request eval\n[\n \"GET /t\",\n \"GET /t?a=1\",\n \"GET /t?b=2&a=1\",\n \"GET /t?c=3&b=2&a=1\",\n]\n--- response_body eval\n[\n \"ledge:cache:http:localhost:/t:\",\n \"ledge:cache:http:localhost:/t:a=1\",\n \"ledge:cache:http:localhost:/t:a=1&b=2\",\n \"ledge:cache:http:localhost:/t:b=2&c=3\",\n]\n--- no_error_log\n[error]\n\n\n=== TEST 6: Wildcard purge URIs\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local ledge_cache_key = require(\"ledge.cache_key\")\n\n local root_key = ledge_cache_key.generate_root_key(nil, nil)\n ngx.print(root_key)\n }\n}\n--- request eval\n[\n \"PURGE /t*\",\n \"PURGE /t?*\",\n \"PURGE /t?a=1*\",\n \"PURGE /t?a=*\",\n]\n--- response_body eval\n[\n \"ledge:cache:http:localhost:/t*:*\",\n \"ledge:cache:http:localhost:/t:*\",\n \"ledge:cache:http:localhost:/t:a=1*\",\n \"ledge:cache:http:localhost:/t:a=*\",\n]\n--- no_error_log\n[error]\n\n=== TEST 7: Compare vary spec\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local vary_compare = require(\"ledge.cache_key\").vary_compare\n\n -- Compare vary specs\n local changed = vary_compare({}, {})\n assert(changed == true, \"empty table == empty table\")\n\n local changed = vary_compare({}, nil)\n assert(changed == true, \"empty table == nil\")\n\n local changed = vary_compare(nil, {})\n assert(changed == true, \"nil == empty table\")\n\n local changed = vary_compare({\"Foo\"}, {\"Foo\"})\n assert(changed == true, \"table == table\")\n\n local changed = vary_compare({\"Foo\", \"Bar\"}, {\"Foo\", \"Bar\"})\n assert(changed == true, \"table == table (multi-values\")\n\n local changed = vary_compare({\"Foo\", \"bar\"}, {\"foo\", \"Bar\"})\n --assert(changed == true, \"table == table (case)\")\n\n\n local changed = vary_compare({\"Foo\"}, {})\n assert(changed == false, \"table ~= empty table\")\n\n local changed = vary_compare({}, {\"Foo\"})\n assert(changed == false, \"empty table ~= table\")\n\n local changed = vary_compare({\"Foo\"}, nil)\n assert(changed == false, \"table ~= nil\")\n\n local changed = vary_compare(nil, {\"Foo\"})\n assert(changed == false, \"nil ~= table\")\n\n local changed = vary_compare({\"Foo\"}, {})\n assert(changed == false, \"table ~= empty table\")\n }\n}\n--- request\nGET /t\n--- no_error_log\n[error]\n\n=== TEST 8: Generate vary key\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local function log(...)\n ngx.log(ngx.DEBUG, ...)\n end\n\n local generate_vary_key = require(\"ledge.cache_key\").generate_vary_key\n\n local called_flag = false\n local callback = function(vary_key)\n assert(type(vary_key) == \"table\", \"callback receives vary key_table\")\n called_flag = true\n end\n\n\n -- Set headers\n ngx.req.set_header(\"Foo\", \"Bar\")\n ngx.req.set_header(\"X-Test\", \"value\")\n\n called_flag = false\n\n -- Empty/nil spec\n local vary_key = generate_vary_key(nil, nil, nil)\n log(vary_key)\n assert(vary_key == \"\", \"Nil spec generates empty string\")\n\n local vary_key = generate_vary_key({}, nil, nil)\n log(vary_key)\n assert(vary_key == \"\", \"Empty spec generates empty string\")\n\n local vary_key = generate_vary_key(nil, callback, nil)\n log(vary_key)\n assert(called_flag == true, \"Callback is called with nil spec\")\n assert(vary_key == \"\", \"Nil vary spec not modified with noop function\")\n called_flag = false\n\n local vary_key = generate_vary_key({}, callback, nil)\n log(vary_key)\n assert(called_flag == true, \"Callback is called with empty spec\")\n assert(vary_key == \"\", \"Empty vary spec not modified with noop function\")\n called_flag = false\n\n\n -- With spec\n local vary_key = generate_vary_key({\"Foo\"}, callback, nil)\n log(vary_key)\n assert(called_flag == true, \"Callback is called\")\n assert(vary_key == \"foo:bar\", \"Vary spec not modified with noop function\")\n called_flag = false\n\n local vary_key = generate_vary_key({\"Foo\", \"X-Test\"}, callback, nil)\n log(vary_key)\n assert(called_flag == true, \"Callback is called - multivalue spec\")\n assert(string.find(vary_key, \"foo:bar\"), \"Vary spec not modified with noop function - multivalue spec\")\n assert(string.find(vary_key, \"x-test:value\"), \"Vary spec not modified with noop function - multivalue spec\")\n assert(string.len(vary_key) == string.len(\"x-test:value:foo:bar\"), \"Vary spec not modified with noop function - multivalue spec only contains required headers\")\n\n called_flag = false\n\n ngx.req.set_header(\"Foo\", {\"Foo1\", \"Foo2\"})\n local vary_key = generate_vary_key({\"Foo\", \"X-Test\"}, callback, nil)\n log(vary_key)\n assert(called_flag == true, \"Callback is called - multivalue header\")\n assert(string.find(vary_key, \"foo:foo1,foo2\"), \"Vary spec - multivalue header\")\n assert(string.find(vary_key, \"x-test:value\"), \"Vary spec - multivalue header\")\n assert(string.len(vary_key) == string.len(\"x-test:value:foo:foo1,foo2\"), \"Vary spec - multivalue header only contains required headers\")\n\n called_flag = false\n ngx.req.set_header(\"Foo\", \"Bar\")\n\n\n -- Active callback\n callback = function(vary_key)\n vary_key[\"MyVal\"] = \"Arbitrary\"\n end\n local vary_key = generate_vary_key(nil, callback, nil)\n log(vary_key)\n assert(vary_key == \"myval:arbitrary\", \"Callback modifies key with nil spec\")\n\n local vary_key = generate_vary_key({}, callback, nil)\n log(vary_key)\n assert(vary_key == \"myval:arbitrary\", \"Callback modifies key with empty spec\")\n\n local vary_key = generate_vary_key({\"Foo\"}, callback, nil)\n log(vary_key)\n assert(string.find(vary_key, \"foo:bar\"), \"Callback appends key with spec\")\n assert(string.find(vary_key, \"myval:arbitrary\"), \"Callback appends key with spec\")\n assert(string.len(vary_key) == string.len(\"myval:arbitrary:foo:bar\"), \"Callback appends key with spec only contains required headers\")\n\n local vary_key = generate_vary_key({\"Foo\", \"X-Test\"}, callback, nil)\n log(vary_key)\n assert(string.find(vary_key, \"myval:arbitrary\"), \"Callback appends key with spec - multi values\")\n assert(string.find(vary_key, \"foo:bar\"), \"Callback appends key with spec - multi values\")\n assert(string.find(vary_key, \"x-test:value\"), \"Callback appends key with spec - multi values\")\n assert(string.len(vary_key) == string.len(\"myval:arbitrary:foo:bar:x-test:value\"), \"Callback appends key with spec - multi values only contains required headers\")\n\n\n callback = function(vary_key)\n vary_key[\"Foo\"] = \"Arbitrary\"\n end\n\n local vary_key = generate_vary_key({\"Foo\"}, callback, nil)\n log(vary_key)\n assert(vary_key == \"foo:arbitrary\", \"Callback overrides key spec\")\n\n\n callback = function(vary_key)\n vary_key[\"Foo\"] = nil\n end\n\n local vary_key = generate_vary_key({\"Foo\"}, callback, nil)\n log(vary_key)\n assert(vary_key == \"\", \"Callback removes from key spec\")\n\n\n callback = function(vary_key)\n assert(vary_key[\"X-None\"] == ngx.null, \"Spec values with missing headers appear as null\")\n end\n\n local vary_key = generate_vary_key({\"X-None\"}, callback, nil)\n log(vary_key)\n assert(vary_key == \"\", \"Missing values do not appear in key\")\n\n\n local vary_key = generate_vary_key({\"A\", \"B\"}, nil, {[\"A\"] = \"123\", [\"B\"] = \"xyz\"})\n log(vary_key)\n assert(string.find(vary_key, \"a:123\"), \"Vary key from arbitrary headers\")\n assert(string.find(vary_key, \"b:xyz\"), \"Vary key from arbitrary headers\")\n assert(string.len(vary_key) == string.len(\"a:123:b:xyz\"), \"Vary key from arbitrary headers only contains required headers\")\n\n local vary_key = generate_vary_key({\"Foo\", \"B\"}, nil, {[\"Foo\"] = \"123\", [\"B\"] = \"xyz\"})\n log(vary_key)\n assert(string.find(vary_key, \"foo:123\"), \"Vary key from arbitrary headers\")\n assert(string.find(vary_key, \"b:xyz\"), \"Vary key from arbitrary headers\")\n assert(string.len(vary_key) == string.len(\"foo:123:b:xyz\"), \"Vary key from arbitrary headers only contains required headers\")\n\n }\n}\n--- request\nGET /t\n--- no_error_log\n[error]\n\n=== TEST 9: Read vary spec\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local redis, err = require(\"ledge\").create_redis_connection()\n if not redis then\n error(\"redis borked: \" .. tostring(err))\n end\n\n local read_vary_spec = require(\"ledge.cache_key\").read_vary_spec\n\n local root_key = \"ledge:dummy:root:\"\n local vary_spec_key = root_key..\"::vary\"\n\n local spec, err = read_vary_spec()\n assert(spec == nil and err ~= nil, \"Redis required to read spec\")\n\n local spec, err = read_vary_spec(redis)\n assert(spec == nil and err ~= nil, \"Root key required to read spec\")\n\n redis.smembers = function() return nil, \"Redis Error\" end\n local spec, err = read_vary_spec(redis, root_key)\n assert(spec == nil and err == \"Redis Error\", \"Redis error returned\")\n redis.smembers = require(\"resty.redis\").smembers\n\n\n local exists = redis:exists(vary_spec_key)\n local spec, err = read_vary_spec(redis, root_key)\n assert(type(spec) == \"table\" and #spec == 0 and exists == 0, \"Missing key returns empty table\")\n\n\n redis:sadd(vary_spec_key, \"Foo\")\n redis:sadd(vary_spec_key, \"Bar\")\n local spec, err = read_vary_spec(redis, root_key)\n table.sort(spec)\n assert(type(spec) == \"table\" and #spec == 2 and spec[2] == \"Foo\" and spec[1] == \"Bar\", \"Spec returned\")\n\n }\n}\n--- request\nGET /t\n--- no_error_log\n[error]\n\n\n=== TEST 10: Key chain\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local key_chain = require(\"ledge.cache_key\").key_chain\n\n local root_key = \"ledge:dummy:root:\"\n local vary_key = \"foo:bar:test:value\"\n local vary_spec = {\"Foo\", \"Test\"}\n\n local expected = {\n main = \"ledge:dummy:root:#foo:bar:test:value::main\",\n entities = \"ledge:dummy:root:#foo:bar:test:value::entities\",\n headers = \"ledge:dummy:root:#foo:bar:test:value::headers\",\n reval_params = \"ledge:dummy:root:#foo:bar:test:value::reval_params\",\n reval_req_headers = \"ledge:dummy:root:#foo:bar:test:value::reval_req_headers\",\n }\n local extra = {\n vary = \"ledge:dummy:root:::vary\",\n repset = \"ledge:dummy:root:::repset\",\n root = \"ledge:dummy:root:\",\n full = \"ledge:dummy:root:#foo:bar:test:value\",\n fetching_lock = \"ledge:dummy:root:#foo:bar:test:value::fetching\",\n }\n\n local chain, err = key_chain()\n assert(chain == nil and err ~= nil, \"Root key required\")\n\n local chain, err = key_chain(root_key)\n assert(chain == nil and err ~= nil, \"Vary key required\")\n\n local chain, err = key_chain(root_key, vary_key)\n assert(chain == nil and err ~= nil, \"Vary spec required\")\n\n\n local chain, err = key_chain(root_key, vary_key, vary_spec)\n assert(type(chain) == \"table\", \"key chain returned\")\n\n local i = 0\n for k,v in pairs(chain) do\n i = i +1\n ngx.log(ngx.DEBUG, k, \": \", v, \" == \", expected[k])\n assert(expected[k] == v, k..\" chain mismatch\")\n end\n assert(i == 5, \"5 iterable keys: \"..i)\n\n for k,v in pairs(expected) do\n ngx.log(ngx.DEBUG, k,\": \", v, \" == \", chain[k])\n assert(chain[k] == v, k..\" expected mismatch\")\n end\n\n for k,v in pairs(extra) do\n ngx.log(ngx.DEBUG, k,\": \", v, \" == \", chain[k])\n assert(chain[k] == v, k..\" extra mismatch\")\n i = i +1\n end\n assert(i == 10, \"10 total chain entries: \"..i)\n\n for i,v in ipairs(vary_spec) do\n assert(chain.vary_spec[i] == v, \" Vary spec mismatch\")\n end\n\n }\n}\n--- request\nGET /t\n--- no_error_log\n[error]\n\n\n=== TEST 11: Save key chain\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local redis = require(\"ledge\").create_redis_connection()\n local key_chain = require(\"ledge.cache_key\").key_chain\n local save_key_chain = require(\"ledge.cache_key\").save_key_chain\n\n local root_key = \"ledge:dummy:root:\"\n local vary_key = \"foo:bar:test:value\"\n local vary_spec = {\"Foo\", \"Test\"}\n\n\n local chain = key_chain(root_key, vary_key, vary_spec)\n\n local ok, err = save_key_chain()\n assert(ok == nil and err ~= nil, \"Redis required\")\n\n local ok, err = save_key_chain(redis)\n assert(ok == nil and err ~= nil, \"Key chain required\")\n\n local ok, err = save_key_chain(redis, \"foo\")\n assert(ok == nil and err ~= nil, \"Key chain must be a table\")\n\n local ok, err = save_key_chain(redis, {})\n assert(ok == nil and err ~= nil, \"Key chain must not be empty\")\n\n local ok, err = save_key_chain(redis, chain)\n assert(ok == nil and err ~= nil, \"TTL required\")\n\n local ok, err = save_key_chain(redis, chain, \"foo\")\n assert(ok == nil and err ~= nil, \"TTL must be a number\")\n\n\n -- Create main key\n redis:set(chain.main, \"foobar\")\n\n local ok, err = save_key_chain(redis, chain, 3600)\n assert(ok == true , \"returns true\")\n\n assert(redis:exists(chain.vary) == 1, \"Vary spec key created\")\n assert(redis:exists(chain.repset) == 1, \"Repset created\")\n\n local vs = redis:smembers(chain.vary)\n for _, v in pairs(vs) do\n local match = false\n for _, v2 in ipairs(vary_spec) do\n if v2:lower() == v then\n match = true\n end\n end\n assert(match, \"Vary spec saved: \")\n end\n\n local vs = redis:smembers(chain.repset)\n for _, v in pairs(vs) do\n assert(v == chain.full, \"Full key added to repset\")\n end\n\n assert(redis:ttl(chain.vary) == 3600, \"Vary spec expiry set\")\n assert(redis:ttl(chain.repset) == 3600, \"Repset expiry set\")\n\n local vary_spec = {\"Baz\", \"Qux\"}\n local chain = key_chain(root_key, vary_key, vary_spec)\n local ok, err = save_key_chain(redis, chain, 3600)\n\n local vs = redis:smembers(chain.vary)\n for i, v in pairs(vs) do\n local match = false\n for _, v2 in ipairs(vary_spec) do\n if v2:lower() == v then\n match = true\n end\n end\n assert(match, \"Vary spec overwritten\")\n end\n\n redis:sadd(chain.repset, \"dummy_value\")\n local ok, err = save_key_chain(redis, chain, 3600)\n\n local vs = redis:smembers(chain.repset)\n for _, v in pairs(vs) do\n assert(v ~= \"dummy_value\", \"Missing keys are removed from repset\")\n end\n\n redis:del(chain.repset)\n\n local chain = key_chain(root_key, vary_key, {})\n local ok, err = save_key_chain(redis, chain, 3600)\n assert(redis:exists(chain.vary ) == 0, \"Empty spec removes vary key\")\n assert(redis:exists(chain.repset) == 1, \"Empty spec still creates repset\")\n\n\n local chain = key_chain(root_key, vary_key, {\"Foo\", \"Bar\", \"Foo\", \"bar\"})\n local ok, err = save_key_chain(redis, chain, 3600)\n assert(redis:scard(chain.vary) == 2, \"Deduplicate vary fields\")\n\n }\n}\n--- request\nGET /t\n--- no_error_log\n[error]\n" }, { "path": "t/01-unit/esi.t", "content": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::http_config();\n\nno_long_string();\nno_diff();\nrun_tests();\n\n__DATA__\n=== TEST 1: split_esi_token\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local esi = assert(require(\"ledge.esi\"),\n \"module should load without errors\")\n\n local capability, version = esi.split_esi_token(\"ESI/1.0\")\n assert(capability == \"ESI\" and version == 1.0,\n \"capability and version should be returned\")\n\n local ok, cap, ver = pcall(esi.split_esi_token)\n assert(ok and not cap and not ver,\n \"split_esi_token without a token should safely return nil\")\n }\n}\n\n--- request\nGET /t\n--- no_error_log\n[error]\n\n\n=== TEST 2: esi_capabilities\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n assert(require(\"ledge.esi\").esi_capabilities() == \"ESI/1.0\",\n \"capabilities should be ESI/1.0\")\n }\n}\n\n--- request\nGET /t\n--- no_error_log\n[error]\n\n\n=== TEST 3: choose_esi_processor\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n -- handler stub\n local handler = {\n response = {\n header = {\n [\"Surrogate-Control\"] = [[content=ESI/1.0]],\n }\n }\n }\n\n local processor = require(\"ledge.esi\").choose_esi_processor(handler)\n\n assert(next(processor), \"processor should be a table\")\n\n assert(type(processor.get_scan_filter) == \"function\",\n \"get_scan_filter should be a function\")\n\n assert(type(processor.get_process_filter) == \"function\",\n \"get_process_filter should be a function\")\n\n -- unknown processor\n handler.response.header[\"Surrogate-Control\"] = [[content=FOO/2.0]]\n\n assert(not require(\"ledge.esi\").choose_esi_processor(handler),\n \"processor should be nil\")\n }\n}\n\n--- request\nGET /t\n--- no_error_log\n[error]\n\n\n=== TEST 4: is_allowed_content_type\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local res = {\n header = {\n [\"Content-Type\"] = \"text/html\",\n }\n }\n\n local allowed_types = {\n \"text/html\"\n }\n\n local is_allowed_content_type =\n require(\"ledge.esi\").is_allowed_content_type\n\n assert(is_allowed_content_type(res, allowed_types),\n \"text/html is allowed\")\n\n res.header[\"Content-Type\"] = \"text/ht\"\n assert(not is_allowed_content_type(res, allowed_types),\n \"text/ht is not allowed\")\n\n res.header[\"Content-Type\"] = \"text/html_foo\"\n assert(not is_allowed_content_type(res, allowed_types),\n \"text/html_foo is not allowed\")\n\n res.header[\"Content-Type\"] = \"text/html; charset=utf-8\"\n assert(is_allowed_content_type(res, allowed_types),\n \"text/html; charset=utf-8 is allowed\")\n\n res.header[\"Content-Type\"] = \"text/json\"\n assert(not is_allowed_content_type(res, allowed_types),\n \"text/json is not allowed\")\n\n\n table.insert(allowed_types, \"text/json\")\n assert(is_allowed_content_type(res, allowed_types),\n \"text/json is allowed\")\n\n }\n}\n\n--- request\nGET /t\n--- no_error_log\n[error]\n\n\n=== TEST 5: can_delegate_to_surrogate\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local can_delegate_to_surrogate =\n require(\"ledge.esi\").can_delegate_to_surrogate\n\n assert(not can_delegate_to_surrogate(true, \"ESI/1.0\"),\n \"cannot delegate without capability\")\n\n ngx.req.set_header(\"Surrogate-Capability\", \"localhost=ESI/1.0\")\n\n assert(can_delegate_to_surrogate(true, \"ESI/1.0\"),\n \"can delegate with capability\")\n\n assert(not can_delegate_to_surrogate(true, \"FOO/1.2\"),\n \"cannnot delegate to non-supported capability\")\n\n assert(can_delegate_to_surrogate({ \"127.0.0.1\" }, \"ESI/1.0\" ),\n \"can delegate to loopback with capability\")\n\n assert(not can_delegate_to_surrogate({ \"127.0.0.2\" }, \"ESI/1.0\" ),\n \"cant delegate to non-loopback with capability\")\n\n }\n}\n--- request\nGET /t\n--- no_error_log\n[error]\n\n\n=== TEST 6: filter_esi_args\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n\n local filter_esi_args = require(\"ledge.esi\").filter_esi_args\n\n local args = ngx.req.get_uri_args()\n assert(args.a == \"1\" and args.esi_foo == \"bar bar\" and args.b == \"2\",\n \"request args should be intact\")\n\n filter_esi_args(handler)\n\n local args = ngx.req.get_uri_args()\n assert(args.a == \"1\" and not args.esi_foo and args.b == \"2\",\n \"esi args should be removed\")\n\n assert(ngx.ctx.__ledge_esi_args.foo == \"bar bar\",\n \"esi args should have foo: bar bar\")\n\n assert(tostring(ngx.ctx.__ledge_esi_args) == \"esi_foo=bar%20bar\",\n \"esi_args as a string should be foo=bar%20bar\")\n\n }\n}\n--- request\nGET /t?a=1&esi_foo=bar+bar&b=2\n--- no_error_log\n[error]\n" }, { "path": "t/01-unit/events.t", "content": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::http_config();\n\nno_long_string();\nno_diff();\nrun_tests();\n\n__DATA__\n=== TEST 1: Bind and emit\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n\n local ok, err = handler:bind(\"non_event\", function(arg) end)\n assert(not ok and err == \"no such event: non_event\",\n \"err should be set\")\n\n local function say(arg) ngx.say(arg) end\n\n local ok, err = handler:bind(\"after_cache_read\", say)\n assert(ok and not err, \"bind should return positively\")\n\n local ok, err = pcall(handler.emit, handler, \"non_event\")\n assert(not ok and err == \"attempt to emit non existent event: non_event\",\n \"emit should fail with non-event\")\n\n -- Bind and emit all events\n handler:bind(\"before_upstream_request\", say)\n handler:bind(\"after_upstream_request\", say)\n handler:bind(\"before_save\", say)\n handler:bind(\"before_save_revalidation_data\", say)\n handler:bind(\"before_serve\", say)\n handler:bind(\"before_esi_include_request\", say)\n handler:bind(\"before_vary_selection\", say)\n\n handler:emit(\"after_cache_read\", \"after_cache_read\")\n handler:emit(\"before_upstream_request\", \"before_upstream_request\")\n handler:emit(\"after_upstream_request\", \"after_upstream_request\")\n handler:emit(\"before_save\", \"before_save\")\n handler:emit(\"before_save_revalidation_data\", \"before_save_revalidation_data\")\n handler:emit(\"before_serve\", \"before_serve\")\n handler:emit(\"before_esi_include_request\", \"before_esi_include_request\")\n handler:emit(\"before_vary_selection\", \"before_vary_selection\")\n }\n}\n\n--- request\nGET /t\n--- response_body\nafter_cache_read\nbefore_upstream_request\nafter_upstream_request\nbefore_save\nbefore_save_revalidation_data\nbefore_serve\nbefore_esi_include_request\nbefore_vary_selection\n--- error_log\nno such event: non_event\n\n\n=== TEST 2: Bind multiple functions to an event\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n\n for i = 1, 3 do\n handler:bind(\"after_cache_read\", function()\n ngx.say(\"function \", i)\n end)\n end\n\n handler:emit(\"after_cache_read\")\n }\n}\n--- request\nGET /t\n--- response_body\nfunction 1\nfunction 2\nfunction 3\n--- no_error_log\n[error]\n\n\n=== TEST 3: Default binds\n--- http_config eval\nqq {\nlua_package_path \"./lib/?.lua;../lua-resty-redis-connector/lib/?.lua;../lua-resty-qless/lib/?.lua;../lua-resty-http/lib/?.lua;../lua-ffi-zlib/lib/?.lua;;\";\n\ninit_by_lua_block {\n if $LedgeEnv::test_coverage == 1 then\n require(\"luacov.runner\").init()\n end\n\n require(\"ledge\").bind(\"after_cache_read\", function(arg)\n ngx.say(\"default 1: \", arg)\n end)\n\n require(\"ledge\").bind(\"after_cache_read\", function(arg)\n ngx.say(\"default 2: \", arg)\n end)\n}\n}\n--- config\nlocation /t {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local ledge = require(\"ledge\")\n local ok, err = pcall(ledge.bind, \"after_cache_read\", function(arg)\n ngx.say(arg)\n end)\n\n assert(not ok and string.find(err, \"attempt to call bind outside the 'init' phase\"), err)\n\n local handler = require(\"ledge\").create_handler()\n\n handler:bind(\"after_cache_read\", function(arg)\n ngx.say(\"instance 1: \", arg)\n end)\n\n handler:bind(\"after_cache_read\", function(arg)\n ngx.say(\"instance 2: \", arg)\n end)\n\n handler:emit(\"after_cache_read\", \"foo\")\n }\n}\n--- request\nGET /t\n--- response_body\ndefault 1: foo\ndefault 2: foo\ninstance 1: foo\ninstance 2: foo\n--- no_error_log\n[error]\n" }, { "path": "t/01-unit/handler.t", "content": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::http_config(extra_lua_config => qq{\n -- For TEST 2\n TEST_NGINX_PORT = $LedgeEnv::nginx_port\n});\n\nno_long_string();\nno_diff();\nrun_tests();\n\n__DATA__\n=== TEST 1: Load module\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local handler, err = require(\"ledge\").create_handler()\n assert(handler,\n \"create_handler() should return postively, got: \" .. tostring(err))\n\n local ok, err = require(\"ledge.handler\").new()\n assert(not ok, \"new with empty config should return negatively\")\n assert(err == \"config table expected\",\n \"err should be 'config table expected'\")\n\n local handler = require(\"ledge.handler\")\n local ok, err = pcall(function()\n handler.foo = \"bar\"\n end)\n assert(not ok, \"setting unknown field should error\")\n assert(string.find(err, \"attempt to create new field foo\"),\n \"err should be 'attempt to create new field foo'\")\n }\n}\n--- request\nGET /t\n--- no_error_log\n[error]\n\n\n=== TEST 2: Override config defaults\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local handler = assert(require(\"ledge\").create_handler({\n upstream_host = \"example.com\",\n }), \"create_handler should return positively\")\n\n assert(handler.config.upstream_host == \"example.com\",\n \"upstream_host should be example.com\")\n\n assert(handler.config.upstream_port == TEST_NGINX_PORT,\n \"upstream_port should default to \" .. TEST_NGINX_PORT)\n\n\n -- Change config\n\n handler.config.upstream_port = 81\n assert(handler.config.upstream_port == 81,\n \"upstream_port should be 81\")\n\n\n -- Unknown config field\n\n local ok, err = pcall(function()\n handler.config.foo = \"bar\"\n end)\n assert(not ok, \"setting unknown config should error\")\n assert(string.find(err, \"attempt to create new field foo\"),\n \"err should be 'attempt to create new field foo'\")\n }\n}\n--- request\nGET /t\n--- no_error_log\n[error]\n\n\n=== TEST 3: Call run on simple request without errors\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n assert(require(\"ledge\").create_handler():run(),\n \"run should return positively\")\n }\n}\nlocation /t {\n echo \"OK\";\n}\n--- request\nGET /t_prx\n--- response_body\nOK\n--- no_error_log\n[error]\n\n\n=== TEST 4: Bind / emit\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n\n function add_header(res)\n res.header[\"X-Foo\"] = \"bar\"\n end\n\n -- Bind succeeds\n local ok, err = assert(handler:bind(\"before_serve\", add_header),\n \"bind should return positively\")\n\n -- Bad event name\n local ok, err = handler:bind(\"foo\", add_header)\n assert(not ok, \"bind should return negatively\")\n assert(err == \"no such event: foo\",\n \"err should be 'no such event: foo'\")\n\n -- Bad user event\n handler:bind(\"before_serve\", function(res) error(\"oops\", 2) end)\n\n handler:run()\n }\n}\nlocation /t {\n echo \"OK\";\n}\n--- request\nGET /t_prx\n--- response_body\nOK\n--- response_headers\nX-Foo: bar\n--- error_log\nno such event: foo\nerror in user callback for 'before_serve': oops\n\n=== TEST 5: visible hostname\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n -- Defaults to the hostname of the server\n local visible_hostname = string.lower(require(\"ledge\").create_handler().config.visible_hostname)\n local host = string.lower(ngx.var.hostname)\n assert(visible_hostname == host,\n \"visible_hostname \"..tostring(visible_hostname)..\" should be \"..host)\n\n -- Test overriding the visible_hostname\n local host = \"example.com\"\n local visible_hostname = string.lower(require(\"ledge\").create_handler({ visible_hostname = host }).config.visible_hostname)\n assert(visible_hostname == host,\n \"visible_hostname should be \" .. host)\n }\n\n}\n--- request\nGET /t\n--- no_error_log\n[error]\n\n=== TEST 6: read from cache\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n local redis = require(\"ledge\").create_redis_connection()\n\n -- Set redis and read the cache key\n handler.redis = redis\n handler:cache_key_chain()\n\n -- Unset redis again\n handler.redis = {}\n local res, err = handler:read_from_cache()\n assert(res == nil and err ~= nil,\n \"read_from_cache should error with no redis connections\")\n\n handler.redis = redis\n handler.storage = require(\"ledge\").create_storage_connection(\n handler.config.storage_driver,\n handler.config.storage_driver_config\n )\n local res, err = handler:read_from_cache()\n assert(res and not err, \"read_from_cache should return positively\")\n }\n\n}\n--- request\nGET /t\n--- no_error_log\n[error]\n\n\n=== TEST 7: Call run with bad redis details\n--- http_config eval\nqq{\nresolver local=on;\nlua_package_path \"./lib/?.lua;;\";\n\ninit_by_lua_block {\n if $LedgeEnv::test_coverage == 1 then\n require(\"luacov.runner\").init()\n end\n\n require(\"ledge\").configure({\n redis_connector_params = {\n url = \"redis://redis:0/\",\n },\n qless_db = 123,\n })\n\n require(\"ledge\").set_handler_defaults({\n upstream_host = \"$LedgeEnv::nginx_host\",\n upstream_port = $LedgeEnv::nginx_port,\n storage_driver_config = {\n redis_connector_params = {\n url = \"redis://$LedgeEnv::redis_host:$LedgeEnv::redis_port/$LedgeEnv::redis_database\"\n }\n },\n })\n\n require(\"ledge.state_machine\").set_debug(true)\n}\n}\n--- config\nlocation /t {\n lua_socket_log_errors Off;\n content_by_lua_block {\n local ok, err = require(\"ledge\").create_handler():run()\n assert(ok == nil and err ~= nil,\n \"run should return negatively with an error\")\n ngx.say(\"OK\")\n }\n}\n--- request\nGET /t\n--- response_body\nOK\n--- no_error_log\n[error]\n\n=== TEST 8: save to cache\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n local redis = require(\"ledge\").create_redis_connection()\n\n handler.redis = redis\n handler:cache_key_chain()\n handler.redis = {}\n\n local res, err = handler:save_to_cache()\n assert(res == nil and err ~= nil,\n \"read_from_cache should error with no response\")\n\n local res, err = handler:fetch_from_origin()\n assert(res == nil and err ~= nil,\n \"fetch_from_origin should error with no redis\")\n\n handler.redis = redis\n handler.storage = require(\"ledge\").create_storage_connection(\n handler.config.storage_driver,\n handler.config.storage_driver_config\n )\n local res, err = handler:fetch_from_origin()\n assert(res and not err, \"fetch_from_origin should return positively\")\n\n local res, err = handler:save_to_cache(res)\n ngx.log(ngx.DEBUG, res, \" \", err)\n assert(res and not err, \"save_to_cache should return positively\")\n\n ngx.say(\"OK\")\n }\n\n}\nlocation /t {\n echo \"origin\";\n}\n--- request\nGET /t_prx\n--- no_error_log\n[error]\n--- response_body\nOK\n\n=== TEST 8: save to cache, no body\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n\n local res, err = handler:save_to_cache()\n assert(res == nil and err ~= nil,\n \"read_from_cache should error with no response\")\n\n handler.redis = require(\"ledge\").create_redis_connection()\n handler.storage = require(\"ledge\").create_storage_connection(\n handler.config.storage_driver,\n handler.config.storage_driver_config\n )\n\n local res, err = handler:fetch_from_origin()\n assert(res and not err, \"fetch_from_origin should return positively\")\n\n res.has_body = false\n\n local res, err = handler:save_to_cache(res)\n ngx.log(ngx.DEBUG, res, \" \", err)\n assert(res and not err, \"save_to_cache should return positively\")\n\n ngx.say(\"OK\")\n }\n\n}\nlocation /t {\n echo \"origin\";\n}\n--- request\nGET /t_prx\n--- no_error_log\n[error]\n--- response_body\nOK\n" }, { "path": "t/01-unit/jobs.t", "content": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::http_config(extra_nginx_config => qq{\n lua_shared_dict ledge_test 1m;\n});\n\nno_long_string();\nno_diff();\nrun_tests();\n\n__DATA__\n=== TEST 1: Collect entity\nPrime cache then collect the entity\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n rewrite ^ /cache break;\n content_by_lua_block {\n local redis = require(\"ledge\").create_redis_connection()\n redis:flushall() -- Previous tests create some odd keys\n\n local collect_entity = require(\"ledge.jobs.collect_entity\")\n local handler = require(\"ledge\").create_handler()\n\n local entity_id = ngx.shared.ledge_test:get(\"entity_id\")\n ngx.log(ngx.DEBUG, \"Collecting: \", entity_id)\n\n local job = {\n data = {\n entity_id = entity_id,\n storage_driver = handler.config.storage_driver,\n storage_driver_config = handler.config.storage_driver_config,\n }\n }\n local ok, err, msg = collect_entity.perform(job)\n assert(err == nil, \"collect_entity should not return an error\")\n\n local storage = require(\"ledge\").create_storage_connection(\n handler.config.storage_driver,\n handler.config.storage_driver_config\n )\n local ok, err = storage:exists(entity_id)\n assert(ok == false, \"Entity should not exist\")\n\n -- Failure cases\n job.data.storage_driver = \"bad\"\n local ok, err, msg = collect_entity.perform(job)\n ngx.log(ngx.DEBUG, msg)\n assert(err == \"job-error\" and msg ~= nil, \"collect_entity should return job-error\")\n\n job.data.storage_driver = handler.config.storage_driver\n job.data.storage_driver_config = { bad_config = \"here\" }\n local ok, err, msg = collect_entity.perform(job)\n ngx.log(ngx.DEBUG, msg)\n assert(err == \"job-error\" and msg ~= nil, \"collect_entity should return job-error\")\n }\n}\nlocation /cache_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n handler:bind(\"before_serve\", function(res)\n ngx.log(ngx.DEBUG, \"primed entity: \", res.entity_id)\n ngx.shared.ledge_test:set(\"entity_id\", res.entity_id)\n end)\n handler:run()\n }\n}\n\nlocation /cache {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.say(\"TEST 1\")\n }\n}\n--- request eval\n[\n\"GET /cache_prx\",\n\"GET /t\"\n]\n--- no_error_log\n[error]\n\n\n=== TEST 2: Revalidate\nPrime, Purge, revalidate\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n rewrite ^ /cache2 break;\n content_by_lua_block {\n local revalidate = require(\"ledge.jobs.revalidate\")\n local redis = require(\"ledge\").create_redis_connection()\n\n local handler = require(\"ledge\").create_handler()\n handler.redis = redis\n\n\n local job = {\n redis = redis,\n data = {\n key_chain = handler:cache_key_chain()\n }\n }\n\n\n local ok, err, msg = revalidate.perform(job)\n assert(err == nil, \"revalidate should not return an error\")\n\n assert(ngx.shared.ledge_test:get(\"test2\") == \"Revalidate Request received\",\n \"Revalidate request was not received!\"\n )\n\n\n redis:del(job.data.key_chain.reval_req_headers)\n local ok, err, msg = revalidate.perform(job)\n assert(err == \"job-error\" and msg ~= nil, \"revalidate should return an error\")\n\n redis:del(job.data.key_chain.reval_params)\n local ok, err, msg = revalidate.perform(job)\n assert(err == \"job-error\" and msg ~= nil, \"revalidate should return an error\")\n }\n}\nlocation /cache2_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n handler:run()\n }\n}\n\nlocation /cache2 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=10\"\n ngx.print(\"TEST 2\")\n if string.find(ngx.req.get_headers().user_agent, \"revalidate\", 1, true) then\n ngx.shared.ledge_test:set(\"test2\", \"Revalidate Request received\")\n end\n }\n}\n--- request eval\n[\n\"GET /cache2_prx\",\n\"PURGE /cache2_prx\",\n\"GET /t\"\n]\n--- no_error_log\n[error]\n\n=== TEST 3: Revalidate - inline params\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local revalidate = require(\"ledge.jobs.revalidate\")\n\n local job = {\n data = {\n reval_params = {\n server_addr = ngx.var.server_addr,\n server_port = ngx.var.server_port,\n scheme = ngx.var.scheme,\n uri = \"/cache3\",\n connect_timeout = 1000,\n send_timeout = 1000,\n read_timeout = 1000,\n keepalive_timeout = 60,\n keepalive_poolsize = 10,\n },\n reval_headers = {\n [\"X-Test\"] = \"test_header\"\n }\n }\n }\n\n local ok, err, msg = revalidate.perform(job)\n assert(err == nil, \"revalidate should not return an error\")\n\n assert(ngx.shared.ledge_test:get(\"test3\") == \"test_header\",\n \"Revalidate request was not received!\"\n )\n\n local job = {\n data = {\n reval_params = {\n server_addr = ngx.var.server_addr,\n server_port = ngx.var.server_port,\n scheme = ngx.var.scheme,\n uri = \"/cache_slow\",\n connect_timeout = 1000,\n send_timeout = 100,\n read_timeout = 100,\n keepalive_timeout = 60,\n keepalive_poolsize = 10,\n },\n reval_headers = {\n [\"X-Test\"] = \"test_header\"\n }\n }\n }\n\n local ok, err, msg = revalidate.perform(job)\n assert(err == \"job-error\" and msg ~= nil, \"revalidate should return an error\")\n\n local job = {\n data = {\n reval_params = {\n server_addr = ngx.var.server_addr,\n server_port = ngx.var.server_port+1,\n scheme = ngx.var.scheme,\n uri = \"/cache3\",\n connect_timeout = 1000,\n send_timeout = 1000,\n read_timeout = 1000,\n keepalive_timeout = 60,\n keepalive_poolsize = 10,\n },\n reval_headers = {\n [\"X-Test\"] = \"test_header\"\n }\n }\n }\n\n local ok, err, msg = revalidate.perform(job)\n ngx.log(ngx.DEBUG, msg)\n assert(err == \"job-error\" and msg ~= nil, \"revalidate should return an error\")\n\n local job = {\n redis = {\n hgetall = function(...) return ngx.null end\n },\n data = {\n key_chain = {}\n }\n }\n\n local ok, err, msg = revalidate.perform(job)\n ngx.log(ngx.DEBUG, msg)\n assert(err == \"job-error\" and msg ~= nil, \"revalidate should return an error\")\n\n local job = {\n redis = {\n hgetall = function(...) return nil, \"dummy error\" end\n },\n data = {\n key_chain = {}\n }\n }\n\n local ok, err, msg = revalidate.perform(job)\n ngx.log(ngx.DEBUG, msg)\n assert(err == \"job-error\" and msg ~= nil, \"revalidate should return an error\")\n }\n}\nlocation /cache3 {\n content_by_lua_block {\n ngx.shared.ledge_test:set(\"test3\", ngx.req.get_headers()[\"X-Test\"])\n }\n}\nlocation /cache_slow {\n content_by_lua_block{\n ngx.sleep(1)\n ngx.print(\"OK\")\n }\n}\n--- request\nGET /t\n--- error_code: 200\n\n=== TEST 4: purge\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n rewrite ^ /cache break;\n content_by_lua_block {\n local purge_job = require(\"ledge.jobs.purge\")\n local redis = require(\"ledge\").create_redis_connection()\n\n local handler = require(\"ledge\").create_handler()\n handler.redis = redis\n local heartbeat_flag = false\n\n local job = {\n redis = redis,\n data = {\n repset = \"*::repset\",\n keyspace_scan_count = 2,\n purge_mode = \"invalidate\",\n storage_driver = handler.config.storage_driver,\n storage_driver_config = handler.config.storage_driver_config,\n },\n ttl = function() return 5 end,\n heartbeat = function()\n heartbeat_flag = true\n return heartbeat_flag\n end,\n }\n\n -- Failure cases\n job.data.storage_driver = \"bad\"\n local ok, err, msg = purge_job.perform(job)\n ngx.log(ngx.DEBUG, msg)\n assert(err == \"redis-error\" and msg ~= nil, \"purge should return redis-error\")\n\n job.data.storage_driver = handler.config.storage_driver\n job.data.storage_driver_config = { bad_config = \"here\" }\n local ok, err, msg = purge_job.perform(job)\n ngx.log(ngx.DEBUG, msg)\n assert(err == \"redis-error\" and msg ~= nil, \"purge should return redis-error\")\n\n -- Passing case\n job.data.storage_driver_config = handler.config.storage_driver_config\n\n local ok, err, msg = purge_job.perform(job)\n assert(err == nil, \"purge should not return an error\")\n assert(heartbeat_flag == true, \"Purge should heartbeat\")\n\n -- Heartbeat failure\n job.heartbeat = function() return false end\n local ok, err, msg = purge_job.perform(job)\n ngx.log(ngx.DEBUG, msg)\n assert(err == \"redis-error\" and msg == \"Failed to heartbeat job\", \"purge should return heartbeat error\")\n job.heartbeat = function() return true end\n\n -- Missing redis driver\n job.redis = nil\n local ok, err, msg = purge_job.perform(job)\n ngx.log(ngx.DEBUG, msg)\n assert(err == \"job-error\" and msg ~= nil, \"purge should return job-error\")\n\n\n }\n}\nlocation /cache4_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler():run()\n }\n}\n\nlocation /cache4 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.say(\"TEST 4\")\n }\n}\n--- request eval\n[\n\"GET /cache4_prx\",\"GET /cache4_prx?a=1\",\"GET /cache4_prx?a=2\",\"GET /cache4_prx?a=3\",\"GET /cache4_prx?a=4\",\"GET /cache4_prx?a=5\",\n\"GET /t\",\n\"GET /cache4_prx?a=3\"\n]\n--- response_headers_like eval\n[\"X-Cache: MISS from .*\", \"X-Cache: MISS from .*\",\"X-Cache: MISS from .*\",\"X-Cache: MISS from .*\",\"X-Cache: MISS from .*\",\"X-Cache: MISS from .*\",\n\"\",\n\"X-Cache: MISS from .*\"]\n--- no_error_log\n[error]\n" }, { "path": "t/01-unit/ledge.t", "content": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::http_config(extra_lua_config => qq{\n qless_db = $LedgeEnv::redis_qless_database\n});\n\nno_long_string();\nno_diff();\nrun_tests();\n\n__DATA__\n=== TEST 1: Load module without errors.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /ledge_1 {\n content_by_lua_block {\n assert(require(\"ledge\"), \"module should load without errors\")\n }\n}\n--- request\nGET /ledge_1\n--- no_error_log\n[error]\n\n\n=== TEST 2: Module cannot be externally modified\n--- http_config eval: $::HttpConfig\n--- config\nlocation /ledge_2 {\n content_by_lua_block {\n local ledge = require(\"ledge\")\n local ok, err = pcall(function()\n ledge.foo = \"bar\"\n end)\n assert(string.find(err, \"attempt to create new field foo\"),\n \"error 'field foo does not exist' should be thrown\")\n }\n}\n--- request\nGET /ledge_2\n--- no_error_log\n[error]\n\n\n=== TEST 3: Non existent params cannot be set\n--- http_config eval\nqq {\nlua_package_path \"./lib/?.lua;../lua-resty-redis-connector/lib/?.lua;../lua-resty-qless/lib/?.lua;;\";\ninit_by_lua_block {\n if $LedgeEnv::test_coverage == 1 then\n require(\"luacov.runner\").init()\n end\n local ok, err = pcall(require(\"ledge\").configure, { foo = \"bar\" })\n assert(string.find(err, \"field foo does not exist\"),\n \"error 'field foo does not exist' should be thrown\")\n}\n}\n--- config\nlocation /ledge_3 {\n echo \"OK\";\n}\n--- request\nGET /ledge_3\n--- no_error_log\n[error]\n\n\n=== TEST 4: Params cannot be set outside of init\n--- http_config eval: $::HttpConfig\n--- config\nlocation /ledge_4 {\n content_by_lua_block {\n require(\"ledge\").configure({ qless_db = 4 })\n }\n}\n--- request\nGET /ledge_4\n--- error_code: 500\n--- error_log\nattempt to call configure outside the 'init' phase\n\n\n=== TEST 5: Create redis connection\n--- http_config eval: $::HttpConfig\n--- config\nlocation /ledge_5 {\n content_by_lua_block {\n local redis = assert(require(\"ledge\").create_redis_connection(),\n \"create_redis_connection() should return positively\")\n\n assert(redis:set(\"ledge_5:cat\", \"dog\"),\n \"redis:set() should return positively\")\n\n local val, err = redis:get(\"ledge_5:cat\")\n ngx.say(val)\n\n assert(require(\"ledge\").close_redis_connection(redis),\n \"close_redis_connection() should return positively\")\n }\n}\n--- request\nGET /ledge_5\n--- response_body\ndog\n--- no_error_log\n[error]\n\n\n=== TEST 6: Create bad redis connection\n--- http_config eval\nqq{\nlua_package_path \"./lib/?.lua;../lua-resty-redis-connector/lib/?.lua;../lua-resty-qless/lib/?.lua;;\";\n\ninit_by_lua_block {\n if $LedgeEnv::test_coverage == 1 then\n require(\"luacov.runner\").init()\n end\n require(\"ledge\").configure({\n redis_connector_params = {\n port = 0, -- bad port\n },\n })\n}\n}\n--- config\nlocation /ledge_6 {\n content_by_lua_block {\n assert(not require(\"ledge\").create_redis_connection(),\n \"create_redis_connection() should return negatively\")\n }\n}\n--- request\nGET /ledge_6\n--- error_log eval: qr/connect\\(\\)( to 127.0.0.1:0)? failed/\n\n\n=== TEST 7: Create storage connection\n--- http_config eval: $::HttpConfig\n--- config\nlocation /ledge_7 {\n content_by_lua_block {\n local storage = assert(require(\"ledge\").create_storage_connection(),\n \"create_storage_connection should return positively\")\n\n ngx.say(storage:exists(\"ledge_7:123456\"))\n\n assert(require(\"ledge\").close_storage_connection(storage),\n \"close_storage_connection() should return positively\")\n }\n}\n--- request\nGET /ledge_7\n--- response_body\nfalse\n--- no_error_log\n[error]\n\n\n=== TEST 8: Create bad storage connection\n--- http_config eval\nqq{\nlua_package_path \"./lib/?.lua;../lua-resty-redis-connector/lib/?.lua;../lua-resty-qless/lib/?.lua;;\";\n\ninit_by_lua_block {\n if $LedgeEnv::test_coverage == 1 then\n require(\"luacov.runner\").init()\n end\n require(\"ledge\").set_handler_defaults({\n storage_driver_config = {\n redis_connector_params = {\n port = 0,\n },\n }\n })\n}\n}\n--- config\nlocation /ledge_8 {\n content_by_lua_block {\n assert(not require(\"ledge\").create_storage_connection(),\n \"create_storage_connection() should return negatively\")\n }\n}\n--- request\nGET /ledge_8\n--- error_log eval: qr/connect\\(\\)( to 127.0.0.1:0)? failed/\n\n\n=== TEST 9: Create qless connection\n--- http_config eval: $::HttpConfig\n--- config\nlocation /ledge_9 {\n content_by_lua_block {\n local redis = assert(require(\"ledge\").create_qless_connection(),\n \"create_qless_connection() should return positively\")\n\n assert(redis:set(\"ledge_9:cat\", \"dog\"),\n \"redis:set() should return positively\")\n\n assert(require(\"ledge\").close_redis_connection(redis),\n \"close_redis_connection() should return positively\")\n\n local redis = require(\"ledge\").create_redis_connection()\n assert(redis:select(qless_db), \"select() shoudl return positively\")\n\n local val, err = redis:get(\"ledge_9:cat\")\n ngx.say(val)\n\n assert(require(\"ledge\").close_redis_connection(redis),\n \"close_redis_connection() should return positively\")\n }\n}\n--- request\nGET /ledge_9\n--- response_body\ndog\n--- no_error_log\n[error]\n\n=== TEST 10: Bad redis-connector params are caught\n--- http_config eval\nqq{\nlua_package_path \"./lib/?.lua;../lua-resty-redis-connector/lib/?.lua;../lua-resty-qless/lib/?.lua;;\";\n\ninit_by_lua_block {\n if $LedgeEnv::test_coverage == 1 then\n require(\"luacov.runner\").init()\n end\n require(\"ledge\").configure({\n redis_connector_params = {\n bad_time = true\n },\n })\n require(\"ledge\").set_handler_defaults({\n storage_driver_config = {\n redis_connector_params = {\n bad_time2 = true\n },\n }\n })\n}\n}\n--- config\nlocation /ledge_10 {\n content_by_lua_block {\n local ok, err = require(\"ledge\").create_redis_connection()\n assert(ok == nil and err ~= nil,\n \"create_redis_connection() should return negatively with error\")\n\n local ok, err = require(\"ledge\").create_storage_connection()\n assert(ok == nil and err ~= nil,\n \"create_storage_connection() should return negatively with error\")\n\n local ok, err = require(\"ledge\").create_qless_connection()\n assert(ok == nil and err ~= nil,\n \"create_qless_connection() should return negatively with error\")\n\n local ok, err = require(\"ledge\").create_redis_slave_connection()\n assert(ok == nil and err ~= nil,\n \"create_redis_slave_connection() should return negatively with error\")\n\n -- Test broken redis-connector params are caught when closing redis somehow\n local ok, err = require(\"ledge\").close_redis_connection({dummy = true})\n assert(ok == nil and err ~= nil,\n \"close_redis_connection() should return negatively with error\")\n\n -- Test trying to close a non-existent redis instance\n local ok, err = require(\"ledge\").close_redis_connection({})\n assert(ok == nil and err ~= nil,\n \"close_redis_connection() should return negatively with error\")\n\n ngx.say(\"OK\")\n }\n}\n--- request\nGET /ledge_10\n--- error_code: 200\n--- response_body\nOK\n\n=== TEST 11: Closing an empty redis instance\n--- http_config eval: $::HttpConfig\n--- config\nlocation /ledge_11 {\n content_by_lua_block {\n local ok, err = require(\"ledge\").close_redis_connection({})\n assert(ok == nil,\n \"close_redis_connection() should return negatively\")\n\n ngx.say(\"OK\")\n }\n}\n--- request\nGET /ledge_11\n--- error_code: 200\n--- response_body\nOK\n" }, { "path": "t/01-unit/processor_1_0.t", "content": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::http_config();\n\nno_long_string();\nno_diff();\nrun_tests();\n\n__DATA__\n=== TEST 1: Load module\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local processor = assert(require(\"ledge.esi.processor_1_0\"),\n \"module should load without errors\")\n\n local processor = processor.new(require(\"ledge\").create_handler())\n assert(processor, \"processor_1_0.new should return positively\")\n\n ngx.say(\"OK\")\n }\n}\n\n--- request\nGET /t\n--- error_code: 200\n--- no_error_log\n[error]\n--- response_body\nOK\n\n=== TEST 2: esi_eval_var - QUERY STRING\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local processor = require(\"ledge.esi.processor_1_0\")\n local tests = {\n --{\"var_name\", \"key\", \"default\", \"default_quoted\" },\n {\"QUERY_STRING\", nil, \"default\", \"default_quoted\" },\n {\"QUERY_STRING\", nil, nil, \"default_quoted\" },\n {\"QUERY_STRING\", \"test_param\", \"default\", \"default_quoted\" },\n {\"QUERY_STRING\", \"test_param\", nil, \"default_quoted\" },\n }\n for _,test in ipairs(tests) do\n ngx.say(processor.esi_eval_var(test))\n end\n }\n}\n\n--- request eval\n[\n \"GET /t\",\n \"GET /t?test_param=test\",\n \"GET /t?other_param=test\",\n \"GET /t?test_param=test&test_param=test2\",\n]\n--- no_error_log\n[error]\n--- response_body eval\n[\n\"default\ndefault_quoted\ndefault\ndefault_quoted\n\",\n\n\"test_param=test\ntest_param=test\ntest\ntest\n\",\n\n\"other_param=test\nother_param=test\ndefault\ndefault_quoted\n\",\n\n\"test_param=test&test_param=test2\ntest_param=test&test_param=test2\ntest, test2\ntest, test2\n\",\n]\n\n\n=== TEST 3: esi_eval_var - HTTP header\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local processor = require(\"ledge.esi.processor_1_0\")\n local tests = {\n --{\"var_name\", \"key\", \"default\", \"default_quoted\" },\n {\"HTTP_X_TEST\", nil, \"default\", \"default_quoted\" },\n {\"HTTP_X_TEST\", nil, nil, \"default_quoted\" },\n }\n for _,test in ipairs(tests) do\n ngx.say(processor.esi_eval_var(test))\n end\n }\n}\n\n--- request eval\n[\n \"GET /t\",\n \"GET /t\",\n]\n--- more_headers eval\n[\n\"X-Dummy: foo\",\n\"X-TEST: test_val\"\n]\n--- no_error_log\n[error]\n--- response_body eval\n[\n\"default\ndefault_quoted\n\",\n\n\"test_val\ntest_val\n\",\n]\n\n=== TEST 4: esi_eval_var - Duplicate HTTP header\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local processor = require(\"ledge.esi.processor_1_0\")\n local tests = {\n --{\"var_name\", \"key\", \"default\", \"default_quoted\" },\n {\"HTTP_X_TEST\", nil, \"default\", \"default_quoted\" },\n {\"HTTP_X_TEST\", nil, nil, \"default_quoted\" },\n }\n for _,test in ipairs(tests) do\n ngx.say(processor.esi_eval_var(test))\n end\n }\n}\n\n--- request eval\n[\n \"GET /t\",\n \"GET /t\",\n]\n--- more_headers eval\n[\n\"X-Dummy: foo\",\n\n\"X-TEST: test_val\nX-TEST: test_val2\"\n]\n--- no_error_log\n[error]\n--- response_body eval\n[\n\"default\ndefault_quoted\n\",\n\n\"test_val, test_val2\ntest_val, test_val2\n\",\n]\n\n=== TEST 5: esi_eval_var - Cookie\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local processor = require(\"ledge.esi.processor_1_0\")\n local tests = {\n --{\"var_name\", \"key\", \"default\", \"default_quoted\" },\n {\"HTTP_COOKIE\", nil, \"default\", \"default_quoted\" },\n {\"HTTP_COOKIE\", \"test_cookie\", \"default\", \"default_quoted\" },\n {\"HTTP_COOKIE\", \"test_cookie\", nil, \"default_quoted\" },\n }\n for _,test in ipairs(tests) do\n ngx.say(processor.esi_eval_var(test))\n end\n }\n}\n\n--- request eval\n[\n \"GET /t\",\n \"GET /t\",\n \"GET /t\",\n]\n--- more_headers eval\n[\n\"\",\n\"Cookie: none=here\",\n\"Cookie: test_cookie=my_cookie\"\n]\n--- no_error_log\n[error]\n--- response_body eval\n[\n\"default\ndefault\ndefault_quoted\n\",\n\n\"none=here\ndefault\ndefault_quoted\n\",\n\n\"test_cookie=my_cookie\nmy_cookie\nmy_cookie\n\",\n]\n\n=== TEST 6: esi_eval_var - Accept-Lang\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local processor = require(\"ledge.esi.processor_1_0\")\n local tests = {\n --{\"var_name\", \"key\", \"default\", \"default_quoted\" },\n {\"HTTP_ACCEPT_LANGUAGE\", nil, \"default\", \"default_quoted\" },\n {\"HTTP_ACCEPT_LANGUAGE\", \"en\", \"default\", \"default_quoted\" },\n {\"HTTP_ACCEPT_LANGUAGE\", \"de\", nil, \"default_quoted\" },\n }\n for _,test in ipairs(tests) do\n ngx.say(processor.esi_eval_var(test))\n end\n }\n}\n\n--- request eval\n[\n \"GET /t\",\n \"GET /t\",\n \"GET /t\",\n \"GET /t\",\n]\n--- more_headers eval\n[\n\"\",\n\n\"Accept-Language: en-gb\",\n\n\"Accept-Language: en-us, blah\",\n\n\"Accept-Language: en-gb\nAccept-Language: test\"\n]\n--- no_error_log\n[error]\n--- response_body eval\n[\n\"default\ndefault\ndefault_quoted\n\",\n\n\"en-gb\ntrue\nfalse\n\",\n\n\"en-us, blah\ntrue\nfalse\n\",\n\n\"en-gb, test\ntrue\nfalse\n\",\n]\n\n=== TEST 7: esi_eval_var - ESI_ARGS\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n -- Fake ESI args\n require(\"ledge.esi\").filter_esi_args(\n require(\"ledge\").create_handler()\n )\n\n local processor = require(\"ledge.esi.processor_1_0\")\n local tests = {\n --{\"var_name\", \"key\", \"default\", \"default_quoted\" },\n {\"ESI_ARGS\", nil, \"default\", \"default_quoted\" },\n {\"ESI_ARGS\", \"var1\", \"default\", \"default_quoted\" },\n {\"ESI_ARGS\", \"var2\", nil, \"default_quoted\" },\n }\n\n local str_split = require(\"ledge.util\").string.split\n\n for _,test in ipairs(tests) do\n -- The default encoded string has a non-deterministic ordering due\n -- to being decoded and re-encoded. For test purposes, we explicitly\n -- re-order.\n local res = processor.esi_eval_var(test)\n local args = str_split (res, \"&\")\n\n if #args > 1 then\n table.sort(args)\n\n res = \"\"\n for _, v in ipairs (args) do\n res = res .. v .. \"&\"\n end\n end\n\n ngx.say(res)\n end\n }\n}\n\n--- request eval\n[\n \"GET /t\",\n \"GET /t?esi_var1=test1&esi_var2=test2&foo=bar\",\n \"GET /t?esi_var2=test2&foo=bar\",\n \"GET /t?esi_var1=test1&esi_other_var=foo&foo=bar\",\n \"GET /t?esi_var1=test1&esi_var1=test2&foo=bar\",\n]\n--- no_error_log\n[error]\n--- response_body eval\n[\n\"default\ndefault\ndefault_quoted\n\",\n\n\"esi_var1=test1&esi_var2=test2&\ntest1\ntest2\n\",\n\n\"esi_var2=test2\ndefault\ntest2\n\",\n\n\"esi_other_var=foo&esi_var1=test1&\ntest1\ndefault_quoted\n\",\n\n\"esi_var1=test1&esi_var1=test2&\ntest1,test2\ndefault_quoted\n\",\n]\n\n=== TEST 8: esi_eval_var - custom vars\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n ngx.ctx.__ledge_esi_custom_variables = ngx.req.get_uri_args() or {}\n\n if ngx.ctx.__ledge_esi_custom_variables[\"empty\"] then\n ngx.ctx.__ledge_esi_custom_variables = {}\n else\n ngx.ctx.__ledge_esi_custom_variables[\"deep\"] = {[\"table\"] = \"value!\"}\n end\n\n local processor = require(\"ledge.esi.processor_1_0\")\n local tests = {\n --{\"var_name\", \"key\", \"default\", \"default_quoted\" },\n {\"var1\", nil, \"default\", \"default_quoted\" },\n {\"var2\", nil, nil, \"default_quoted\" },\n {\"var1\", \"subvar\", nil, \"default_quoted\" },\n {\"deep\", \"table\", \"default\", \"default_quoted\" },\n }\n for _,test in ipairs(tests) do\n ngx.say(processor.esi_eval_var(test))\n end\n }\n}\n\n--- request eval\n[\n \"GET /t\",\n \"GET /t?var1=test1&var2=test2\",\n \"GET /t?var2=test2\",\n \"GET /t?empty=true\",\n]\n--- no_error_log\n[error]\n--- response_body eval\n[\n\"default\ndefault_quoted\ndefault_quoted\nvalue!\n\",\n\n\"test1\ntest2\ndefault_quoted\nvalue!\n\",\n\n\"default\ntest2\ndefault_quoted\nvalue!\n\",\n\n\"default\ndefault_quoted\ndefault_quoted\ndefault\n\",\n]\n\n=== TEST 9: esi_process_vars_tag\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n ngx.ctx.__ledge_esi_custom_variables = {\n\n [\"DANGER_ZONE\"] = ''\n }\n local processor = require(\"ledge.esi.processor_1_0\")\n local tests = {\n -- vars tags\n {\n [\"chunk\"] = [[$(QUERY_STRING)]],\n [\"res\"] = [[test_param=test]],\n [\"msg\"] = \"vars tag\"\n },\n {\n [\"chunk\"] = [[before $(QUERY_STRING) after]],\n [\"res\"] = [[before test_param=test after]],\n [\"msg\"] = \"vars tag - outside content\"\n },\n {\n [\"chunk\"] = [[before $(QUERY_STRING) after]],\n [\"res\"] = [[before test_param=test after]],\n [\"msg\"] = \"vars tag - inside content\"\n },\n {\n [\"chunk\"] = [[ $(QUERY_STRING{test_param}) ]],\n [\"res\"] = [[ test ]],\n [\"msg\"] = \"vars tag - whitespace\"\n },\n {\n [\"chunk\"] = [[

$(QUERY_STRING)

]],\n [\"res\"] = [[

test_param=test

]],\n [\"msg\"] = \"vars tag - html tags\"\n },\n {\n [\"chunk\"] = [[]],\n [\"res\"] = [[]],\n [\"msg\"] = \"empty vars tags removed\"\n },\n {\n [\"chunk\"] = [[

foo

]],\n [\"res\"] = [[

foo

]],\n [\"msg\"] = \"empty vars tags removed - content preserved\"\n },\n -- injecting ESI tags from vars\n {\n [\"chunk\"] = [[$(DANGER_ZONE)]],\n [\"res\"] = [[<esi:include src=\"/kenny\" />]],\n [\"msg\"] = \"Injected tags are escaped\"\n },\n }\n for _, t in pairs(tests) do\n local output = processor.esi_process_vars_tag(t[\"chunk\"])\n ngx.log(ngx.DEBUG, \"'\", output, \"'\")\n assert(output == t[\"res\"], \"esi_process_vars_tag mismatch: \"..t[\"msg\"] )\n end\n ngx.say(\"OK\")\n }\n}\nlocation /kenny {\n content_by_lua_block {\n ngx.print(\"Shouldn't see this\")\n }\n}\n\n--- request\nGET /t?test_param=test\n--- no_error_log\n[error]\n--- response_body\nOK\n\n\n=== TEST 12: process_escaping\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local processor = require(\"ledge.esi.processor_1_0\")\n local tests = {\n {\n [\"chunk\"] = [[Lorem ipsum dolor sit amet, consectetur adipiscing elit.]],\n [\"res\"] = [[Lorem ipsum dolor sit amet, consectetur adipiscing elit.]],\n [\"msg\"] = \"nothing to escape\"\n },\n {\n [\"chunk\"] = [[Loremconsectetur adipiscing elit.]],\n [\"res\"] = [[Lorem ipsum dolor sit amet, consectetur adipiscing elit.]],\n [\"msg\"] = \"no esi inside\"\n },\n {\n [\"chunk\"] = [[Loremconsectetur adipiscing elit.]],\n [\"res\"] = [[Lorem $(QUERY_STRING)ipsum dolor sit amet, consectetur adipiscing elit.]],\n [\"msg\"] = \"esi:vars inside\"\n },\n\n }\n for _, t in pairs(tests) do\n local output = processor.process_escaping(t[\"chunk\"])\n ngx.log(ngx.DEBUG, \"'\", output, \"'\")\n assert(output == t[\"res\"], \"process_escaping mismatch: \"..t[\"msg\"] )\n end\n ngx.say(\"OK\")\n }\n}\n\n--- request\nGET /t?test_param=test\n--- no_error_log\n[error]\n--- response_body\nOK\n\n=== TEST 13: fetch include\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n -- Override the normal coroutine.yield function\n local output\n coroutine.yield = function(chunk) output = chunk end\n\n local processor = require(\"ledge.esi.processor_1_0\")\n local handler = require(\"ledge\").create_handler()\n local self = {\n handler = handler\n }\n local buffer_size = 64*1024\n local tests = {\n {\n [\"tag\"] = [[]],\n [\"res\"] = [[fragment]],\n [\"msg\"] = \"nothing to escape\"\n },\n {\n [\"tag\"] = [[]],\n [\"res\"] = [[fragmentfoobar]],\n [\"msg\"] = \"Query string var is evaluated\"\n },\n\n }\n for _, t in pairs(tests) do\n local ret = processor.esi_fetch_include(self, t[\"tag\"], buffer_size)\n ngx.log(ngx.DEBUG, \"RET: '\", ret, \"'\")\n ngx.log(ngx.DEBUG, \"OUTPUT: '\", output, \"'\")\n assert(output == t[\"res\"], \"esi_fetch_include mismatch: \"..t[\"msg\"] )\n end\n ngx.say(\"OK\")\n }\n}\nlocation /f {\n content_by_lua_block {\n ngx.print(\"fragment\", ngx.var.args or \"\")\n }\n}\n--- request\nGET /t?test=foobar\n--- no_error_log\n[error]\n--- response_body\nOK\n\n" }, { "path": "t/01-unit/purge.t", "content": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::http_config();\n\nno_long_string();\nno_diff();\nrun_tests();\n\n__DATA__\n=== TEST 1: create_purge_response\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local cjson_decode = require(\"cjson\").decode\n\n local create_purge_response = assert(\n require(\"ledge.purge\").create_purge_response,\n \"module should load without errors\"\n )\n\n local json, err = create_purge_response(\"invalidate\", \"purged\")\n local data = cjson_decode(json)\n\n assert(not err, \"err should be nil\")\n\n assert(data.purge_mode == \"invalidate\",\n \"purge mode should be invalidate\")\n\n assert(data.result == \"purged\",\n \"result should be purged\")\n\n assert(not data.qless_jobs, \"qless_jobs should be nil\")\n\n\n local json, err = create_purge_response(\"revalidate\", \"scheduled\", {\n jid = \"12345\",\n })\n local data = cjson_decode(json)\n\n assert(not err, \"err should be nil\")\n\n assert(data.qless_jobs.jid == \"12345\",\n \"qless_job.jid should be '12345'\")\n\n\n local json, err = create_purge_response(function() end)\n assert(err == \"Cannot serialise function: type not supported\",\n \"error should be 'Cannot serialise function: type not supported\")\n }\n}\n--- request\nGET /t\n--- no_error_log\n[error]\n\n=== TEST 2: expire keys\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n rewrite ^ /cache break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n local redis = require(\"ledge\").create_redis_connection()\n handler.redis = redis\n\n local storage = require(\"ledge\").create_storage_connection(\n handler.config.storage_driver,\n handler.config.storage_driver_config\n )\n handler.storage = storage\n\n local key_chain = handler:cache_key_chain()\n local entity_id = handler:entity_id(key_chain)\n\n local ttl, err = redis:ttl(key_chain.main)\n\n local expire_keys = require(\"ledge.purge\").expire_keys\n\n local ok, err = expire_keys(redis, storage, key_chain, entity_id)\n if err then ngx.log(ngx.DEBUG, err) end\n assert(ok, \"expire_keys should return positively\")\n\n local expires, err = redis:hget(key_chain.main, \"expires\")\n ngx.log(ngx.DEBUG,\"expires: \", expires, \" <= \", ngx.now())\n assert(tonumber(expires) <= ngx.now(), \"Key not expired\")\n\n local new_ttl = redis:ttl(key_chain.main)\n ngx.log(ngx.DEBUG, \"ttl: \", tonumber(ttl), \" > \", tonumber(new_ttl))\n assert(tonumber(ttl) > tonumber(new_ttl), \"TTL not reduced\")\n\n -- non-existent key\n local ok, err = expire_keys(redis, storage, {main = \"bogus_key\"}, entity_id)\n if err then ngx.log(ngx.DEBUG, err) end\n assert(ok == false and err == nil, \"return false with no error on missing key\")\n\n -- Stub out a partial main key, no ttl\n redis:hset(\"bogus_key\", \"key\", \"value\")\n\n local ok, err = expire_keys(redis, storage, {main = \"bogus_key\"}, entity_id)\n if err then ngx.log(ngx.DEBUG, err) end\n assert(ok == nil and err ~= nil, \"return nil with no ttl\")\n\n -- Set a TTL\n redis:expire(\"bogus_key\", 9000)\n\n local ok, err = expire_keys(redis, storage, {main = \"bogus_key\"}, entity_id)\n if err then ngx.log(ngx.DEBUG, err) end\n assert(ok == nil and err ~= nil, \"return nil with error on broken key\")\n\n -- String expires value\n redis:hset(\"bogus_key\", \"expires\", \"now!\")\n\n local ok, err = expire_keys(redis, storage, {main = \"bogus_key\"}, entity_id)\n if err then ngx.log(ngx.DEBUG, err) end\n assert(ok == nil and err ~= nil, \"return nil with error on string expires\")\n }\n}\nlocation /cache_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n handler:bind(\"before_serve\", function(res)\n ngx.log(ngx.DEBUG, \"primed entity: \", res.entity_id)\n end)\n handler:run()\n }\n}\n\nlocation /cache {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.say(\"TEST 2\")\n }\n}\n--- request eval\n[\n\"GET /cache_prx\",\n\"GET /t\"\n]\n--- no_error_log\n[error]\n\n=== TEST 3: purge\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n rewrite ^ /cache3 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n local redis = require(\"ledge\").create_redis_connection()\n handler.redis = redis\n\n local storage = require(\"ledge\").create_storage_connection(\n handler.config.storage_driver,\n handler.config.storage_driver_config\n )\n handler.storage = storage\n\n local key_chain = handler:cache_key_chain()\n ngx.log(ngx.DEBUG, require(\"cjson\").encode(key_chain))\n\n local purge = require(\"ledge.purge\").purge\n\n -- invalidate - error\n local ok, err = purge(handler, \"invalidate\", \"bad_key\")\n --local ok, err = purge(handler, \"invalidate\", {main = \"bogus_key3\"})\n if err then ngx.log(ngx.DEBUG, err) end\n assert(ok == false and err == \"nothing to purge\", \"purge should return false - bad key\")\n\n -- invalidate\n local ok, err = purge(handler, \"invalidate\", key_chain.repset)\n if err then ngx.log(ngx.DEBUG, err) end\n assert(ok == true and err == \"purged\", \"purge should return true - purged\")\n\n -- revalidate\n local reval_job = false\n handler.revalidate_in_background = function()\n reval_job = true\n return \"job\"\n end\n\n local ok, err, job = purge(handler, \"revalidate\", key_chain.repset)\n if err then ngx.log(ngx.DEBUG, err) end\n assert(ok == false and err == \"already expired\", \"purge should return false - already expired\")\n assert(reval_job == true, \"revalidate should schedule job\")\n assert(job[1] == \"job\", \"revalidate should return the job \"..tostring(job))\n\n -- delete, error\n handler.delete_from_cache = function() return nil, \"delete error\" end\n local ok, err = purge(handler, \"delete\", key_chain.repset)\n if err then ngx.log(ngx.DEBUG, err) end\n assert(ok == nil and err == \"delete error\", \"purge should return nil, error\")\n handler.delete_from_cache = require(\"ledge.handler\").delete_from_cache\n\n -- delete\n local ok, err = purge(handler, \"delete\", key_chain.repset)\n if err then ngx.log(ngx.DEBUG, \"dekete: \",err) end\n assert(ok == true and err == \"deleted\", \"purge should return true - deleted\")\n\n -- delete, missing\n local ok, err = purge(handler, \"delete\", key_chain.repset)\n if err then ngx.log(ngx.DEBUG, err) end\n assert(ok == false and err == \"nothing to purge\", \"purge should return false - nothing to purge\")\n\n local keys = redis:keys(key_chain.root..\"*\")\n ngx.log(ngx.DEBUG, require(\"cjson\").encode(keys))\n\n assert(#keys == 0, \"Keys have all been removed\")\n }\n}\nlocation /cache3_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n handler:run()\n }\n}\n\nlocation /cache {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.say(\"TEST 3\")\n }\n}\n--- request eval\n[\n\"GET /cache3_prx\",\n\"GET /t\"\n]\n--- no_error_log\n[error]\n\n=== TEST 3b: purge with vary\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n rewrite ^ /cache3 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n local redis = require(\"ledge\").create_redis_connection()\n handler.redis = redis\n\n local storage = require(\"ledge\").create_storage_connection(\n handler.config.storage_driver,\n handler.config.storage_driver_config\n )\n handler.storage = storage\n\n local key_chain = handler:cache_key_chain()\n ngx.log(ngx.DEBUG, require(\"cjson\").encode(key_chain))\n\n local purge = require(\"ledge.purge\").purge\n\n -- invalidate\n local ok, err = purge(handler, \"invalidate\", key_chain.repset)\n if err then ngx.log(ngx.DEBUG, err) end\n assert(ok == true and err == \"purged\", \"purge should return true - purged\")\n\n -- revalidate\n local reval_job = false\n local jobcount = 0\n handler.revalidate_in_background = function()\n jobcount = jobcount + 1\n reval_job = true\n return \"job\"..jobcount\n end\n\n local ok, err, job = purge(handler, \"revalidate\", key_chain.repset)\n if err then ngx.log(ngx.DEBUG, err) end\n assert(ok == false and err == \"already expired\", \"purge should return false - already expired\")\n assert(reval_job == true, \"revalidate should schedule job\")\n assert(job[1] == \"job1\" and job[2] == \"job2\", \"revalidate should return the job \"..tostring(job))\n assert(jobcount == 2, \"Revalidate should schedule 1 job per representation\")\n\n -- delete\n local ok, err = purge(handler, \"delete\", key_chain.repset)\n if err then ngx.log(ngx.DEBUG, \"dekete: \",err) end\n assert(ok == true and err == \"deleted\", \"purge should return true - deleted\")\n\n\n local keys = redis:keys(key_chain.root..\"*\")\n ngx.log(ngx.DEBUG, require(\"cjson\").encode(keys))\n\n assert(#keys == 0, \"Keys have all been removed\")\n }\n}\nlocation /cache3_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n handler:run()\n }\n}\n\nlocation /cache {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.header[\"Vary\"] = \"X-Test\"\n ngx.say(\"TEST 3b\")\n }\n}\n--- request eval\n[\n\"GET /cache3_prx\", \"GET /cache3_prx\",\n\"GET /t\"\n]\n--- more_headers eval\n[\n\"X-Test: foo\", \"X-Test: bar\",\n\"\"\n]\n--- no_error_log\n[error]\n\n=== TEST 4: purge api\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n rewrite ^ /cache4 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n local redis = require(\"ledge\").create_redis_connection()\n handler.redis = redis\n\n local storage = require(\"ledge\").create_storage_connection(\n handler.config.storage_driver,\n handler.config.storage_driver_config\n )\n handler.storage = storage\n\n -- Stub out response object\n local response = {\n status = 0,\n body,\n set_body = function(self, body)\n self.body = body\n end\n }\n handler.response = response\n\n local json_body = nil\n\n ngx.req.get_body_data = function()\n return json_body\n end\n\n local purge_api = require(\"ledge.purge\").purge_api\n\n -- Nil body\n local ok, err = purge_api(handler)\n if response.body then ngx.log(ngx.DEBUG, response.body) end\n assert(ok == false and response.body ~= nil, \"nil body should return false\")\n response.body = nil\n\n -- Invalid json\n json_body = [[ foobar ]]\n local ok, err = purge_api(handler)\n if response.body then ngx.log(ngx.DEBUG, response.body) end\n assert(ok == false and response.body ~= nil, \"invalid json should return false\")\n response.body = nil\n\n -- Valid json, bad request\n json_body = [[{\"foo\": \"bar\"}]]\n local ok, err = purge_api(handler)\n if response.body then ngx.log(ngx.DEBUG, response.body) end\n assert(ok == false and response.body ~= nil, \"bad request should return false\")\n response.body = nil\n\n -- Valid API request\n json_body = require(\"cjson\").encode({\n uris = {\n \"http://\"..ngx.var.host..\":\"..ngx.var.server_port..\"/cache4_prx\"\n },\n purge_mode = \"delete\",\n headers = {\n [\"X-Test\"] = \"Test Header\"\n }\n })\n local ok, err = purge_api(handler)\n if response.body then ngx.log(ngx.DEBUG, response.body) end\n assert(ok == true and response.body ~= nil, \"valid request should return true\")\n response.body = nil\n\n local res, err = redis:exists(handler:cache_key_chain().main)\n if err then ngx_log(ngx.ERR, err) end\n assert(res == 0, \"Key should have been removed\")\n\n -- Custom headers should be added to request\n json_body = require(\"cjson\").encode({\n uris = {\n \"http://\"..ngx.var.host..\":\"..ngx.var.server_port..\"/hdr_test\"\n },\n purge_mode = \"delete\",\n headers = {\n [\"X-Test\"] = \"Test Header\"\n }\n })\n local ok, err = purge_api(handler)\n if response.body then ngx.log(ngx.DEBUG, response.body) end\n local match = response.body:find(\"X-Test: Test Header\")\n assert(ok == true and match ~= nil, \"custom header s should pass through\")\n response.body = nil\n }\n}\nlocation /cache4_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge.state_machine\").set_debug(false)\n local handler = require(\"ledge\").create_handler()\n handler:run()\n }\n}\n\nlocation /hdr_test {\n content_by_lua_block {\n ngx.print(ngx.DEBUG, \"X-Test: \", ngx.req.get_headers()[\"X-Test\"])\n }\n}\n\nlocation /cache {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=4600\"\n ngx.say(\"TEST 4\")\n }\n}\n--- request eval\n[\n\"GET /cache4_prx\",\n\"GET /t\"\n]\n--- no_error_log\n[error]\n" }, { "path": "t/01-unit/range.t", "content": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::http_config();\n\nno_long_string();\nno_diff();\nrun_tests();\n\n__DATA__\n=== TEST 1: req_byte_ranges\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local req_byte_ranges = assert(require(\"ledge.range\").req_byte_ranges,\n \"range module should load without errors\")\n\n local ranges = req_byte_ranges()\n\n local t = tonumber(ngx.req.get_uri_args().t)\n if t == 1 then\n assert(not ranges,\n \"req_byte_ranges with no range header should return nil\")\n\n elseif t == 2 then\n assert(ranges[1], \"range should exist\")\n assert(ranges[1].from == 0 and ranges[1].to == 99,\n \"req_byte_ranges should be from 0 to 99\")\n\n elseif t == 3 then\n assert(not ranges,\n \"req_byte_ranges with malformed range header should return nil\")\n\n elseif t == 4 then\n assert(ranges[1], \"range should exist\")\n assert(ranges[1].from == 0 and not ranges[1].to,\n \"req_byte_ranges should be from 0 to nil\")\n\n elseif t == 5 then\n assert(ranges[1], \"range should exist\")\n assert(not ranges[1].from and ranges[1].to == 99,\n \"req_byte_ranges should be from 0 to 99\")\n\n elseif t == 6 then\n assert(ranges[1], \"range should exist\")\n assert(not ranges[1].from and not ranges[1].to,\n \"req_byte_ranges should be from 0 to 99\")\n\n elseif t == 7 then\n assert(ranges[1] and ranges[2] and not ranges[3],\n \"two ranges should exist\")\n\n assert(ranges[1].from == 0 and ranges[1].to == 10,\n \"ranges[1] should be from 0 to 10\")\n\n assert(ranges[2].from == 20 and ranges[2].to == 30,\n \"ranges[2] should be 20 to 30\")\n\n elseif t == 8 then\n assert(ranges[1] and ranges[2] and not ranges[3],\n \"two ranges should exist\")\n\n assert(ranges[1].from == 0 and not ranges[1].to,\n \"ranges[1] should be from 0 to nil\")\n\n assert(not ranges[2].from and ranges[2].to == 30,\n \"ranges[2] should be nil to 30\")\n\n end\n }\n}\n--- more_headers eval\n[\n \"\",\n \"Range: bytes=0-99\",\n \"Range: 0-99\",\n \"Range: bytes=0-\",\n \"Range: bytes=-99\",\n \"Range: bytes=-\",\n \"Range: bytes=0-10,20-30\",\n \"Range: bytes=0-,-30\",\n]\n--- request eval\n[\n \"GET /t?t=1\",\n \"GET /t?t=2\",\n \"GET /t?t=3\",\n \"GET /t?t=4\",\n \"GET /t?t=5\",\n \"GET /t?t=6\",\n \"GET /t?t=7\",\n \"GET /t?t=8\",\n]\n--- no_error_log\n[error]\n\n\n=== TEST 2: handle_range_request\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local range = require(\"ledge.range\").new()\n local args = ngx.req.get_uri_args()\n\n -- Response stub\n local response = {\n status = tonumber(args.status),\n size = tonumber(args.size),\n header = {}\n }\n\n local range_applied = false\n response, range_applied = range:handle_range_request(response)\n\n local t = tonumber(ngx.req.get_uri_args().t)\n if t == 1 then\n assert(response and not range_applied,\n \"response should not be nil but range was not applied\")\n elseif t == 2 then\n assert(response and range_applied,\n \"response should not be nil and range should be applied\")\n\n assert(response.status == 206,\n \"status should be 206\")\n\n assert(response.header[\"Content-Range\"] == \"bytes 0-99/200\",\n \"content_range header should be set\")\n elseif t == 3 then\n assert(response and range_applied,\n \"response should not be nil and range should be applied\")\n\n assert(response.status == 206,\n \"status should be 206\")\n\n assert(response.header[\"Content-Range\"] == \"bytes 0-70/200\",\n \"content_range header should be set to coalesced ranges\")\n\n elseif t == 4 then\n assert(response and range_applied,\n \"response should not be nil and range should be applied\")\n\n assert(response.status == 206,\n \"status should be 206\")\n\n assert(response.header[\"Content-Range\"] == \"bytes 0-199/200\",\n \"Content-Range header should be expanded to size\")\n\n elseif t == 5 then\n assert(response and range_applied,\n \"response should not be nil and range should be applied\")\n\n assert(response.status == 206,\n \"status should be 206\")\n\n local ct = response.header[\"Content-Type\"]\n assert(string.find(ct, \"multipart/byteranges;\"),\n \"Content-Type header should incude multipart/byteranges\")\n\n elseif t == 6 then\n assert(response and not range_applied,\n \"response should not be nil but range was not applied\")\n\n assert(response.status == 416,\n \"status should be 416 (Not Satisfiable)\")\n\n elseif t == 7 then\n assert(response and not range_applied,\n \"response should not be nil but range was not applied\")\n\n end\n }\n}\n--- more_headers eval\n[\n \"\",\n \"Range: bytes=0-99\",\n \"Range: bytes=0-30,20-70\",\n \"Range: bytes=0-\",\n \"Range: bytes=0-10,20-30\",\n \"Range: bytes=40-20\",\n \"Range: bytes=0-10\",\n]\n--- request eval\n[\n \"GET /t?t=1&size=100&status=200\",\n \"GET /t?t=2&size=200&status=200\",\n \"GET /t?t=3&size=200&status=200\",\n \"GET /t?t=4&size=200&status=200\",\n \"GET /t?t=5&size=200&status=200\",\n \"GET /t?t=6&size=200&status=200\",\n \"GET /t?t=7&size=200&status=404\",\n]\n--- no_error_log\n[error]\n\n\n=== TEST 3: get_range_request_filter\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local range = require(\"ledge.range\").new()\n local args = ngx.req.get_uri_args()\n\n -- Response stub\n local response = {\n status = 200,\n size = 10,\n header = {},\n body_reader = coroutine.wrap(function()\n coroutine.yield(\"01234\")\n coroutine.yield(\"56789\")\n end),\n }\n\n if args[\"type\"] then\n response.header[\"Content-Type\"] = args[\"type\"]\n end\n\n local function read_body(response)\n local res = \"\"\n repeat\n local chunk, err = response.body_reader()\n if chunk then\n res = res .. chunk\n end\n until not chunk\n return res\n end\n\n local range_applied = false\n response, range_applied = range:handle_range_request(response)\n if range_applied then\n response.body_reader = range:get_range_request_filter(\n response.body_reader\n )\n end\n local body = read_body(response)\n\n local t = tonumber(ngx.req.get_uri_args().t)\n if t == 1 then\n assert(body == \"0123456789\", \"body should be un-filtered\")\n\n elseif t == 2 then\n assert(body == \"0123\", \"body should be 0123\")\n\n elseif t == 3 then\n assert(body == \"2345678\", \"body should be 2345678\")\n\n elseif t == 4 then\n assert(body == \"456789\", \"body should be 456789\")\n\n elseif t == 5 then\n assert(body == \"23456789\", \"body should be 23456789\")\n\n elseif t == 6 then\n assert(response.status == 206, \"status should be 206\")\n\n local ct = response.header[\"Content-Type\"]\n assert(string.find(ct, \"multipart/byteranges;\"),\n \"Content-Type header should incude multipart/byteranges\")\n\n assert(ngx.re.find(\n body,\n [[^(Content-Range: bytes 0-4\\/10\\n$)]],\n \"m\"\n ), \"body should contain Content-Range bytes 0-4/10\")\n\n assert(ngx.re.find(\n body,\n [[^Content-Range: bytes 6-9\\/10\\n$]],\n \"m\"\n ), \"body should contain Content-Range bytes 6-9/10\")\n\n elseif t == 7 then\n assert(body == \"3456789\", \"ranges should be coalesced\")\n\n elseif t == 8 then\n assert(body == \"0123456789\", \"body should be unfiltered\")\n assert(response.status == 206, response.status)\n assert(response.header[\"Content-Range\"] == \"bytes 0-9/10\",\n \"Content-Range header should be trimmed to size\")\n\n end\n }\n}\n--- more_headers eval\n[\n \"Range: bytes=0-9\",\n \"Range: bytes=0-3\",\n \"Range: bytes=2-8\",\n \"Range: bytes=4-\",\n \"Range: bytes=-8\",\n \"Range: bytes=0-4,6-9\",\n \"Range: bytes=0-4,6-9\",\n \"Range: bytes=3-6,6-9\",\n \"Range: bytes=0-11\",\n]\n--- request eval\n[\n \"GET /t?t=1\",\n \"GET /t?t=2\",\n \"GET /t?t=3\",\n \"GET /t?t=4\",\n \"GET /t?t=5\",\n \"GET /t?t=6\",\n \"GET /t?t=6&type=text/html\",\n \"GET /t?t=7\",\n \"GET /t?t=8\",\n]\n--- no_error_log\n[error]\n\n\n=== TEST 4: parse_content_range\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local parse_content_range = require(\"ledge.range\").parse_content_range\n\n local from, to, size = parse_content_range(\"bytes 1-2/3\")\n assert(from == 1 and to == 2 and size == 3)\n\n from, to, size = parse_content_range(\"byte 1-2/3\")\n assert(not from and not to and not size)\n\n from, to, size = parse_content_range(\"bytes 123-1234/12345\")\n assert(from == 123 and to == 1234 and size == 12345)\n }\n}\n--- request\nGET /t\n--- no_error_log\n[error]\n" }, { "path": "t/01-unit/request.t", "content": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::http_config(extra_lua_config => qq{\n TEST_NGINX_HOST = \"$LedgeEnv::nginx_host\"\n TEST_NGINX_PORT = $LedgeEnv::nginx_port\n});\n\nno_long_string();\nno_diff();\nrun_tests();\n\n__DATA__\n=== TEST 1: Purge mode\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local req_purge_mode = assert(require(\"ledge.request\").purge_mode,\n \"request module should load without errors\")\n\n local mode = ngx.req.get_uri_args()[\"p\"]\n assert(req_purge_mode() == mode,\n \"req_purge_mode should equal \" .. mode)\n\n\n }\n}\n--- more_headers eval\n[\n \"X-Purge: delete\",\n \"X-Purge: revalidate\",\n \"X-Purge: invalidate\",\n \"\"\n]\n--- request eval\n[\n \"GET /t?p=delete\",\n \"GET /t?p=revalidate\",\n \"GET /t?p=invalidate\",\n \"GET /t?p=invalidate\"\n]\n--- no_error_log\n[error]\n\n\n=== TEST 2: relative_uri - spaces encoded\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local http = require(\"resty.http\").new()\n http:connect(\n TEST_NGINX_HOST, TEST_NGINX_PORT\n )\n\n local res, err = http:request({\n path = \"/t with spaces\",\n })\n\n http:close()\n }\n\n}\n\nlocation \"/t with spaces\" {\n content_by_lua_block {\n local req_relative_uri = require(\"ledge.request\").relative_uri\n assert(req_relative_uri() == \"/t%20with%20spaces\",\n \"uri should have spaces encoded\")\n }\n}\n--- request\nGET /t\n--- no_error_log\n[error]\n\n\n=== TEST 3: relative_uri - Percent encode encoded CRLF\nhttp://resources.infosecinstitute.com/http-response-splitting-attack\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local http = require(\"resty.http\").new()\n http:connect(\n TEST_NGINX_HOST, TEST_NGINX_PORT\n )\n\n local res, err = http:request({\n path = \"/t_crlf_encoded_%250d%250A\",\n })\n\n http:close()\n }\n\n}\n\nlocation /t_crlf_encoded_ {\n content_by_lua_block {\n local req_relative_uri = require(\"ledge.request\").relative_uri\n assert(req_relative_uri() == \"/t_crlf_encoded_%250D%250A\",\n \"encoded crlf in uri should be escaped\")\n }\n}\n--- request\nGET /t\n--- no_error_log\n[error]\n\n\n=== TEST 4: full_uri\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local full_uri = require(\"ledge.request\").full_uri\n assert(full_uri() == \"http://localhost/t\",\n \"full_uri should be http://localhost/t\")\n }\n\n}\n--- request\nGET /t\n--- no_error_log\n[error]\n\n\n=== TEST 5: accepts_cache\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local accepts_cache = require(\"ledge.request\").accepts_cache\n assert(tostring(accepts_cache()) == ngx.req.get_uri_args().c,\n \"accepts_cache should be \" .. ngx.req.get_uri_args().c)\n }\n\n}\n--- more_headers eval\n[\n \"Cache-Control: no-cache\",\n \"Cache-Control: no-store\",\n \"Pragma: no-cache\",\n \"Cache-Control: no-cache, max-age=60\",\n \"Cache-Control: s-maxage=20, no-cache\",\n \"\",\n \"Cache-Control: max-age=60\",\n \"Cache-Control: max-age=0\",\n \"Pragma: cache\",\n \"Cache-Control: no-cachey\",\n]\n--- request eval\n[\n \"GET /t?c=false\",\n \"GET /t?c=false\",\n \"GET /t?c=false\",\n \"GET /t?c=false\",\n \"GET /t?c=false\",\n \"GET /t?c=true\",\n \"GET /t?c=true\",\n \"GET /t?c=true\",\n \"GET /t?c=true\",\n \"GET /t?c=true\"\n]\n--- no_error_log\n[error]\n" }, { "path": "t/01-unit/response.t", "content": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::http_config(extra_lua_config => qq{\n function read_body(res)\n repeat\n local chunk, err = res.body_reader()\n if chunk then\n ngx.print(chunk)\n end\n until not chunk\n end\n});\n\nno_long_string();\nno_diff();\nrun_tests();\n\n__DATA__\n=== TEST 1: Load module\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local res, err = require(\"ledge.response\").new()\n assert(not res, \"new with empty args should return negatively\")\n assert(err ~= nil, \"err not nil\")\n\n local res, err = require(\"ledge.response\").new({})\n assert(not res, \"new with empty handler should return negatively\")\n assert(err ~= nil, \"err not nil\")\n\n local res, err = require(\"ledge.response\").new({redis = {} })\n assert(not res, \"new with empty handler redis should return negatively\")\n assert(err ~= nil, \"err not nil\")\n\n local handler = require(\"ledge\").create_handler()\n local redis = require(\"ledge\").create_redis_connection()\n handler.redis = redis\n\n local res, err = require(\"ledge.response\").new(handler)\n\n assert(res and not err, \"response object should be created without error\")\n\n local ok, err = pcall(function()\n res.foo = \"bar\"\n end)\n assert(not ok, \"setting unknown field should error\")\n assert(string.find(err, \"attempt to create new field foo\"),\n \"err should be 'attempt to create new field foo'\")\n }\n}\n--- request\nGET /t\n--- no_error_log\n[error]\n\n\n=== TEST 2: set_body\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n local redis = require(\"ledge\").create_redis_connection()\n handler.redis = redis\n\n local res, err = require(\"ledge.response\").new(handler)\n\n read_body(res) -- will be empty\n\n res:set_body(\"foo\")\n\n read_body(res) -- will print foo\n }\n}\n--- request\nGET /t\n--- response_body: foo\n--- no_error_log\n[error]\n\n\n=== TEST 3: filter_body_reader\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n local redis = require(\"ledge\").create_redis_connection()\n handler.redis = redis\n\n require(\"ledge.response\").set_debug(true)\n local res, err = require(\"ledge.response\").new(handler)\n\n res:set_body(\"foo\")\n\n -- turns foo to moo\n function get_cow_filter(reader)\n return coroutine.wrap(function()\n repeat\n local chunk, err = reader()\n if chunk then\n coroutine.yield(ngx.re.gsub(chunk, \"f\", \"m\"))\n end\n until not chunk\n end)\n end\n\n -- turns moo to boo\n function get_sad_filter(reader)\n return coroutine.wrap(function()\n repeat\n local chunk, err = reader()\n if chunk then\n coroutine.yield(ngx.re.gsub(chunk, \"m\", \"b\"))\n end\n until not chunk\n end)\n end\n\n res:filter_body_reader(\"cow\", get_cow_filter(res.body_reader))\n res:filter_body_reader(\"sad\", get_sad_filter(res.body_reader))\n\n local ok, err = pcall(res.filter_body_reader, res, \"bad\", \"foo\")\n assert(not ok and string.find(err, \"filter must be a function\"),\n \"error shoudl contain 'filter must be a function'\")\n\n read_body(res)\n }\n}\n--- request\nGET /t\n\n\n=== TEST 4: is_cacheable\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n local redis = require(\"ledge\").create_redis_connection()\n handler.redis = redis\n\n require(\"ledge.response\").set_debug(true)\n local res, err = require(\"ledge.response\").new(handler)\n\n assert(not res:is_cacheable())\n\n res.header = {\n [\"Cache-Control\"] = \"max-age=60\",\n }\n assert(res:is_cacheable())\n\n res.header = {\n [\"Cache-Control\"] = \"max-age=60\",\n [\"Pragma\"] = \"no-cache\",\n }\n assert(not res:is_cacheable())\n\n res.header = {\n [\"Cache-Control\"] = \"s-maxage=60, private\",\n }\n assert(not res:is_cacheable())\n\n res.header = {\n [\"Cache-Control\"] = \"max-age=60, no-store\",\n }\n assert(not res:is_cacheable())\n\n res.header = {\n [\"Cache-Control\"] = \"max-age=60, no-cache\",\n }\n assert(not res:is_cacheable())\n\n res.header = {\n [\"Cache-Control\"] = \"max-age=60, no-cache=X-Foo\",\n }\n assert(res:is_cacheable())\n\n res.header = {\n [\"Cache-Control\"] = \"max-age=60\",\n [\"Vary\"] = \"*\",\n }\n assert(not res:is_cacheable())\n }\n}\n--- request\nGET /t\n--- no_error_log\n[error]\n\n\n=== TEST 5: ttl\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n local redis = require(\"ledge\").create_redis_connection()\n handler.redis = redis\n\n require(\"ledge.response\").set_debug(true)\n local res, err = require(\"ledge.response\").new(handler)\n\n assert(res:ttl() == 0, \"ttl should be 0\")\n\n res.header = {\n [\"Expires\"] = ngx.http_time(ngx.time() + 10)\n }\n assert(res:ttl() == 10, \"Expires was 10 seconds in the future\")\n\n res.header[\"Cache-Control\"] = \"max-age=20\"\n assert(res:ttl() == 20, \"max-age overrides to 20 seconds\")\n\n res.header[\"Cache-Control\"] = \"s-maxage=30\"\n assert(res:ttl() == 30, \"s-maxage overrides to 30 seconds\")\n\n res.header[\"Cache-Control\"] = \"max-age=20, s-maxage=30\"\n assert(res:ttl() == 30, \"s-maxage still overrides to 30 seconds\")\n }\n}\n--- request\nGET /t\n--- no_error_log\n[error]\n\n\n=== TEST 6: save / read / set_and_save\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n local redis = require(\"ledge\").create_redis_connection()\n handler.redis = redis\n\n local res, err = require(\"ledge.response\").new(handler)\n\n res.uri = \"http://example.com\"\n res.status = 200\n\n local ok, err = res:save(60)\n assert(ok and not err, \"res should save without err\")\n\n\n local res2, err = require(\"ledge.response\").new(handler)\n\n local ok, err = res2:read()\n assert(ok and not err, \"res2 should save without err\")\n\n assert(res2.uri == \"http://example.com\", \"res2 uri\")\n\n res2.header[\"X-Save-Me\"] = \"ok\"\n res2:save(60)\n\n local res3, err = require(\"ledge.response\").new(handler)\n res3:read()\n\n assert(res3.header[\"X-Save-Me\"] == \"ok\", \"res3 headers\")\n\n local ok, err = res3:set_and_save(\"size\", 99)\n assert(ok and not err, \"set_and_save should return positively\")\n\n assert(res3.size == 99, \"res3.size should be 99\")\n\n local res4, err = require(\"ledge.response\").new(handler)\n res4:read()\n\n assert(res4.size == 99, \"res3.size should be 99\")\n\n local ok, err = res4:set_and_save(nil, 2)\n assert(not ok and err, \"set_and_save should fail with bad params\")\n }\n}\n--- request\nGET /t\n--- error_log\nset_and_save(): ERR wrong number of arguments for 'hset' command\n\n=== TEST 7: read differentiates between redis failure and broken cache entry\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n local redis = require(\"ledge\").create_redis_connection()\n handler.redis = redis\n\n local res, err = require(\"ledge.response\").new(handler)\n\n -- Ensure entry exists\n res.uri = \"http://example.com\"\n res.status = 200\n res.size = 1\n assert(res:save(60), \"res should save without err\")\n\n -- Break entities\n redis:del(handler:cache_key_chain().entities)\n\n local ok, err = res:read()\n assert(ok == nil and not err, \"read should return no error with broken entities\")\n\n\n -- Break headers\n redis:del( handler:cache_key_chain().headers)\n\n local ok, err = res:read()\n assert(ok == nil and not err, \"read should return no error with broken headers\")\n\n -- Missing main key\n redis:del( handler:cache_key_chain().main)\n\n local ok, err = res:read()\n assert(ok == nil and not err, \"read should return no error with missing main key\")\n\n -- Break Redis instance\n res.redis.hgetall = function() return ngx.null end\n\n local ok, err = res:read()\n assert(ok or not err, \"read should return error on redis error\")\n\n\n handler.cache_key_chain = function() return nil, \"Dummy\" end\n\n local ok, err = res:read()\n assert(ok == nil and err == \"Dummy\", \"read should return error when failing to get the key chain\")\n }\n}\n--- request\nGET /t\n--- error_code: 200\n--- no_error_log\n[error]\n\n=== TEST 8: save should replace the has_esi flag\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n local redis = require(\"ledge\").create_redis_connection()\n handler.redis = redis\n\n local res, err = require(\"ledge.response\").new(handler)\n\n res.uri = \"http://example.com\"\n res.status = 200\n\n local ok, err = res:save(60)\n assert(ok and not err, \"res should save without err\")\n\n res:set_and_save(\"has_esi\", \"dummy\")\n\n local res2, err = require(\"ledge.response\").new(handler)\n\n local ok, err = res2:read()\n assert(ok and not err, \"res2 should save without err\")\n\n assert(res2.uri == \"http://example.com\", \"res2 uri\")\n assert(res2.has_esi == \"dummy\", \"res2 has_esi\")\n\n res2.header[\"X-Save-Me\"] = \"ok\"\n res2:save(60)\n\n local res3, err = require(\"ledge.response\").new(handler)\n res3:read()\n\n assert(res3.header[\"X-Save-Me\"] == \"ok\", \"res3 headers\")\n assert(res3.has_esi == false, \"res3 has_esi: \"..tostring(res3.has_esi))\n\n }\n}\n--- request\nGET /t\n--- no_error_log\n[error]\n\n\n=== TEST 9: Parse vary header\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local encode = require(\"cjson\").encode\n local handler = require(\"ledge\").create_handler()\n local redis = require(\"ledge\").create_redis_connection()\n handler.redis = redis\n\n local res, err = require(\"ledge.response\").new(handler)\n\n local tests = {\n {\n hdr = nil,\n res = nil,\n msg = \"Nil header, nil spec\",\n },\n {\n hdr = \"\",\n res = nil,\n msg = \"Empty header, nil spec\",\n },\n {\n hdr = \"foo\",\n res = {\"foo\"},\n msg = \"Single field\",\n },\n {\n hdr = \"Foo\",\n res = {\"foo\"},\n msg = \"Single field - case\",\n },\n {\n hdr = \"fOo,bar,Baz\",\n res = {\"bar\",\"baz\",\"foo\"},\n msg = \"Multi field\",\n },\n {\n hdr = \"fOo, bar , Baz\",\n res = {\"bar\",\"baz\",\"foo\"},\n msg = \"Multi field - whitespace\",\n },\n {\n hdr = \"bar,baz,foo\",\n res = {\"bar\",\"baz\",\"foo\"},\n msg = \"Multi field - sort1\",\n },\n {\n hdr = \"foo,baz,bar\",\n res = {\"bar\",\"baz\",\"foo\"},\n msg = \"Multi field - sort2\",\n },\n\n {\n hdr = \"foo, bar, bar, foo, baz\",\n res = {\"bar\",\"baz\",\"foo\"},\n msg = \"De-duplicate\",\n },\n {\n hdr = {\"foo\", \"Bar\", \"Baz, Qux\"},\n res = {\"bar\", \"baz\", \"foo\", \"qux\"},\n msg = \"Multiple vary headers\",\n },\n {\n hdr = {\"foo, bar\", \"foo\", \"bar, Qux\", \"bar, Foo\"},\n res = {\"bar\", \"foo\", \"qux\"},\n msg = \"Multiple vary headers - deduplicate\",\n },\n }\n\n for _, t in ipairs(tests) do\n res.header[\"Vary\"] = t[\"hdr\"]\n local vary_spec = res:parse_vary_header()\n ngx.log(ngx.DEBUG, \"-----------------------------------------------\")\n ngx.log(ngx.DEBUG, \"header: \", encode(t[\"hdr\"]))\n ngx.log(ngx.DEBUG, \"spec: \", encode(vary_spec))\n ngx.log(ngx.DEBUG, \"expected: \", encode(t[\"res\"]))\n\n if type(t[\"res\"]) == \"table\" then\n for i, v in ipairs(t[\"res\"]) do\n assert(vary_spec[i] == v, t[\"msg\"])\n end\n else\n\n assert(vary_spec == t[\"res\"], t[\"msg\"])\n end\n\n end\n }\n}\n--- request\nGET /t\n--- no_error_log\n[error]\n" }, { "path": "t/01-unit/stale.t", "content": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::http_config();\n\nno_long_string();\nno_diff();\nrun_tests();\n\n__DATA__\n=== TEST 1: can_serve_stale\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local can_serve_stale = require(\"ledge.stale\").can_serve_stale\n\n local args = ngx.req.get_uri_args()\n local res = {\n header = {\n [\"Cache-Control\"] = args.rescc,\n },\n remaining_ttl = tonumber(args.ttl),\n }\n\n assert(tostring(can_serve_stale(res)) == ngx.req.get_uri_args().stale,\n \"can_serve_stale should be \" .. ngx.req.get_uri_args().stale)\n\n }\n}\n--- more_headers eval\n[\n \"\",\n \"Cache-Control: max-stale=60\",\n \"Cache-Control: max-stale=60\",\n \"Cache-Control: max-stale=60\",\n \"Cache-Control: max-stale=9\",\n]\n--- request eval\n[\n \"GET /t?rescc=&ttl=0&stale=false\",\n \"GET /t?rescc=&ttl=0&stale=true\",\n \"GET /t?rescc=must-revalidate&ttl=0&stale=false\",\n \"GET /t?rescc=proxy-revalidate&ttl=0&stale=false\",\n \"GET /t?rescc=&ttl=-10&stale=false\",\n]\n--- no_error_log\n[error]\n\n\n=== TEST 2: verify_stale_conditions\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local verify_stale_conditions =\n require(\"ledge.stale\").verify_stale_conditions\n\n local args = ngx.req.get_uri_args()\n\n local res = {\n header = {\n [\"Cache-Control\"] = ngx.req.get_headers().x_res_cache_control,\n [\"Age\"] = ngx.req.get_headers().x_res_age,\n },\n remaining_ttl = tonumber(args.ttl),\n }\n\n local token = ngx.req.get_uri_args().token\n local stale = ngx.req.get_uri_args().stale\n assert(tostring(verify_stale_conditions(res, token)) == stale,\n \"verify_stale_conditions should be \" .. stale)\n\n if token == \"stale-while-revalidate\" then\n\n local can_serve_stale_while_revalidate =\n require(\"ledge.stale\").can_serve_stale_while_revalidate\n\n assert(tostring(can_serve_stale_while_revalidate(res)) == stale,\n \"can_serve_stale_while_revalidate should be \" .. stale)\n elseif token == \"stale-if-error\" then\n\n local can_serve_stale_if_error = \n require(\"ledge.stale\").can_serve_stale_if_error\n\n assert(tostring(can_serve_stale_if_error(res)) == stale,\n \"can_serve_stale_if_error should be \" .. stale)\n end\n\n }\n}\n--- more_headers eval\n[\n \"\",\n \"Cache-Control: stale-while-revalidate=60\",\n \"X-Res-Cache-Control: stale-while-revalidate=60\",\n \"Cache-Control: min-fresh=10\nX-Res-Cache-Control: stale-while-revalidate=60\",\n \"Cache-Control: max-age=10, stale-while-revalidate=60\nX-Res-Age: 5\",\n \"Cache-Control: max-age=4, stale-while-revalidate=60\nX-Res-Age: 5\",\n \"Cache-Control: max-stale=10, stale-while-revalidate=60\",\n \"Cache-Control: max-stale=60, stale-while-revalidate=60\",\n]\n--- request eval\n[\n \"GET /t?token=stale-while-revalidate&stale=false\",\n \"GET /t?token=stale-while-revalidate&stale=true\",\n \"GET /t?token=stale-while-revalidate&stale=true\",\n \"GET /t?token=stale-while-revalidate&stale=false\",\n \"GET /t?token=stale-while-revalidate&stale=true\",\n \"GET /t?token=stale-while-revalidate&stale=false\",\n \"GET /t?token=stale-while-revalidate&stale=false\",\n \"GET /t?token=stale-while-revalidate&stale=true\",\n]\n--- no_error_log\n[error]\n" }, { "path": "t/01-unit/state_machine.t", "content": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::http_config();\n\nno_long_string();\nno_diff();\nrun_tests();\n\n__DATA__\n=== TEST 1: Load module\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n assert(require(\"ledge.state_machine\"),\n \"state machine module should include without error\")\n\n assert(require(\"ledge.state_machine.events\"),\n \"events module should include without error\")\n\n assert(require(\"ledge.state_machine.pre_transitions\"),\n \"pre_transitions module should include without error\")\n\n assert(require(\"ledge.state_machine.states\"),\n \"events module should include without error\")\n }\n}\n--- request\nGET /t\n--- no_error_log\n[error]\n\n\n=== TEST 2: Prove station machine compiles\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local events = require(\"ledge.state_machine.events\")\n local pre_transitions = require(\"ledge.state_machine.pre_transitions\")\n local states = require(\"ledge.state_machine.states\")\n local actions = require(\"ledge.state_machine.actions\")\n\n for ev,t in pairs(events) do\n for _,trans in ipairs(t) do\n -- Check states\n for _,kw in ipairs { \"when\", \"after\", \"begin\" } do\n if trans[kw] then\n if \"function\" ~= type(states[trans[kw]]) then\n ngx.say(\"State '\", trans[kw], \"' requested during \",\n ev, \" is not defined\")\n end\n end\n end\n\n -- Check \"in_case\" previous event\n if trans[\"in_case\"] then\n if not events[trans[\"in_case\"]] then\n ngx.say(\"Event '\", trans[\"in_case\"],\n \"' filtered for but not in transition table\")\n end\n end\n\n -- Check actions\n if trans[\"but_first\"] then\n local action = trans[\"but_first\"]\n if type(action) == \"table\" then\n for _,ac in ipairs(action) do\n if \"function\" ~= type(actions[ac]) then\n ngx.say(\"Action '\", ac, \"' called during \", ev,\n \" is not defined\")\n end\n end\n else\n if \"function\" ~= type(actions[action]) then\n ngx.say(\"Action '\", action, \"' called during \", ev,\n \" is not defined\")\n end\n end\n end\n end\n end\n\n for t,v in pairs(pre_transitions) do\n if \"function\" ~= type(states[t]) then\n ngx.say(\"Pre-transitions defined for missing state '\", t, \"'\")\n end\n if type(v) ~= \"table\" or #v == 0 then\n ngx.say(\"No pre-transition actions defined for '\", t, \"'\")\n else\n for _,action in ipairs(v) do\n if \"function\" ~= type(actions[action]) then\n ngx.say(\"Pre-transition action '\", action,\n \"' is not defined\")\n end\n end\n end\n end\n\n for state, v in pairs(states) do\n local found = false\n for ev, t in pairs(events) do\n for _, trans in ipairs(t) do\n if trans[\"begin\"] == state then\n found = true\n end\n end\n end\n\n if found == false then\n ngx.say(\"State '\", state, \"' is never transitioned to\")\n end\n end\n\n\n local states_file = \"lib/ledge/state_machine/states.lua\"\n local handler_file = \"lib/ledge/handler.lua\"\n\n -- event in a table\n local events_called = {}\n for _, file in ipairs({ states_file, handler_file }) do\n assert(io.open(file, \"r\"),\n \"Could not find states.lua (are you running from the root dir?\")\n\n -- Run luac to extract self:e(event) calls by event name\n local cmd = \"luac -p -l \" .. file\n cmd = cmd .. [[ | grep -A2 'SELF .* \"e\"' | awk '{print $7}']]\n cmd = cmd .. [[ | grep \"\\\".*\\\"\"]]\n local f, err = io.popen(cmd, \"r\")\n\n -- For each call, check the event being triggered exists, and place the\n repeat\n local event = f:read('*l')\n if event then\n event = ngx.re.gsub(event, \"\\\"\", \"\") -- remove quotes\n events_called[event] = true\n if not events[event] then\n ngx.say(\"Event '\", event, \"' is called but does not exist\")\n end\n end\n until not event\n\n f:close()\n end\n\n for event, t_table in pairs(events) do\n if not events_called[event] then\n ngx.say(\"Event '\", event, \"' exits but is never called\")\n end\n end\n\n ngx.say(\"OK\")\n }\n}\n--- request\nGET /t\n--- response_body\nOK\n--- no_error_log\n[error]\n" }, { "path": "t/01-unit/storage.t", "content": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::http_config(extra_lua_config => qq{\n -- Define storage backends here, and add requests to each test\n -- with backend= params.\n backends = {\n redis = {\n module = \"ledge.storage.redis\",\n params = {\n redis_connector_params = {\n url = REDIS_URL,\n },\n },\n bad_params = {\n redis_connector_params = {\n url = REDIS_URL,\n foobar = \"broken!\"\n },\n }\n },\n redis_notransact = {\n module = \"ledge.storage.redis\",\n params = {\n redis_connector_params = {\n url = REDIS_URL,\n connection_is_proxied = true,\n },\n supports_transactions = false,\n },\n bad_params = {\n redis_connector_params = {\n url = REDIS_URL,\n foobar = \"broken!\"\n },\n supports_transactions = false,\n }\n },\n }\n\n function get_backend(backend)\n local config = backends[backend]\n\n if backend == \"redis_notransact\" then\n -- stub out transactional redis functions to error\n require(\"resty.redis\").multi = function(...)\n error(\"called transactional function 'multi'\")\n end\n\n require(\"resty.redis\").exec = function(...)\n error(\"called transactional function 'exec'\")\n end\n\n require(\"resty.redis\").discard = function(...)\n error(\"called transactional function 'discard'\")\n end\n end\n\n return config\n end\n\n\n -- Utility returning an iterator over given chunked data\n function get_source(data)\n local index = 0\n return function()\n index = index + 1\n if data[index] then\n return data[index][1], data[index][2], data[index][3]\n end\n end\n end\n\n\n -- Utility returning an iterator over given chunked data, but which\n -- fails (simulating storage connection failure) at fail_pos iteration.\n function get_and_fail_source(data, fail_pos, storage)\n local index = 0\n return function()\n index = index + 1\n\n if index == fail_pos then\n storage.redis:close()\n end\n\n if data[index] then\n return data[index][1], data[index][2], data[index][3]\n end\n end\n end\n\n\n -- Utility returning an iterator over given chunked data, but which\n -- fails (simulating upstream timeout) at fail_pos iteration.\n function get_and_fail_upstream_source(data, fail_pos)\n local index = 0\n return function()\n index = index + 1\n\n if index == fail_pos then return nil, \"timeout\" end\n\n if data[index] then\n return data[index][1], data[index][2], data[index][3]\n end\n end\n end\n\n\n -- Utility to read the body as is serving\n function sink(iterator)\n repeat\n local chunk, err, has_esi = iterator()\n if chunk then\n ngx.say(chunk, \":\", err, \":\", tostring(has_esi))\n end\n until not chunk\n end\n\n\n -- Utilitu to report success and the size written\n function success_handler(bytes_written)\n ngx.say(\"wrote \", bytes_written, \" bytes\")\n end\n\n\n -- Utility to report the onfailure event was called\n function failure_handler(reason)\n ngx.say(reason)\n end\n\n\n -- Response object stub\n _res = {}\n local _mt = { __index = _res }\n\n function _res.new(entity_id)\n return setmetatable({\n entity_id = entity_id,\n body_reader = function() return nil end,\n }, _mt)\n end\n});\n\nno_long_string();\nno_diff();\nrun_tests();\n\n__DATA__\n=== TEST 1: Load connect and close without errors.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /storage {\n content_by_lua_block {\n local config = get_backend(ngx.req.get_uri_args()[\"backend\"])\n local storage = require(config.module).new()\n\n assert(storage:connect(config.params),\n \"storage:connect should return positively\")\n assert(storage:close(),\n \"storage:close() should return positively\")\n\n ngx.print(ngx.req.get_uri_args()[\"backend\"], \" OK\")\n }\n}\n--- request eval\n[\n \"GET /storage?backend=redis\",\n \"GET /storage?backend=redis_notransact\",\n]\n--- response_body eval\n[\n \"redis OK\",\n \"redis_notransact OK\",\n]\n--- no_error_log\n[error]\n\n\n=== TEST 2: Write entity, read it back\n--- http_config eval: $::HttpConfig\n--- config\nlocation /storage {\n content_by_lua_block {\n local backend = ngx.req.get_uri_args()[\"backend\"]\n local config = get_backend(backend)\n\n local storage = require(config.module).new()\n\n assert(storage:connect(config.params),\n \"storage:connect should return positively\")\n\n local res = _res.new(\"00002-\" .. backend)\n res.body_reader = get_source({\n { \"CHUNK 1\", nil, false },\n { \"CHUNK 2\", nil, true },\n { \"CHUNK 3\", nil, false },\n })\n\n assert(not storage:exists(res.entity_id),\n \"entity should not exist\")\n\n -- Attach the writer, and run sink\n res.body_reader = storage:get_writer(\n res, 60,\n success_handler,\n failure_handler\n )\n sink(res.body_reader)\n\n assert(storage:exists(res.entity_id),\n \"entity should exist\")\n\n -- Attach the reader, and run sink\n res.body_reader = storage:get_reader(res)\n sink(res.body_reader)\n\n assert(storage:close(),\n \"storage:close should return positively\")\n }\n}\n--- request eval\n[\n \"GET /storage?backend=redis\",\n \"GET /storage?backend=redis_notransact\",\n]\n--- response_body eval\n[\n \"CHUNK 1:nil:false\nCHUNK 2:nil:true\nCHUNK 3:nil:false\nwrote 21 bytes\nCHUNK 1:nil:false\nCHUNK 2:nil:true\nCHUNK 3:nil:false\n\",\n \"CHUNK 1:nil:false\nCHUNK 2:nil:true\nCHUNK 3:nil:false\nwrote 21 bytes\nCHUNK 1:nil:false\nCHUNK 2:nil:true\nCHUNK 3:nil:false\n\",\n]\n--- no_error_log\n[error]\n\n\n=== TEST 3: Fail to write entity larger than max_size\n--- http_config eval: $::HttpConfig\n--- config\nlocation /storage {\n content_by_lua_block {\n local backend = ngx.req.get_uri_args()[\"backend\"]\n local config = get_backend(backend)\n\n local storage = require(config.module).new()\n config.params.max_size = 8\n\n assert(storage:connect(config.params),\n \"storage:connect should return positively\")\n\n local res = _res.new(\"00003-\" .. backend)\n res.body_reader = get_source({\n { \"123\", nil, false },\n { \"456\", nil, true },\n { \"789\", nil, false },\n })\n\n assert(not storage:exists(res.entity_id),\n \"entity should not exist\")\n\n -- Attach the writer, and run sink\n res.body_reader = storage:get_writer(\n res, 60,\n success_handler,\n failure_handler\n )\n sink(res.body_reader)\n\n -- Prove entity wasn't written\n assert(not storage:exists(res.entity_id),\n \"entity should not exist\")\n\n assert(storage:close(),\n \"storage:close should return positively\")\n }\n}\n--- request eval\n[\n \"GET /storage?backend=redis\",\n \"GET /storage?backend=redis_notransact\",\n]\n--- response_body eval\n[\n \"123:nil:false\n456:nil:true\n789:nil:false\nbody is larger than 8 bytes\n\",\n \"123:nil:false\n456:nil:true\n789:nil:false\nbody is larger than 8 bytes\n\",\n]\n--- no_error_log\n[error]\n\n\n=== TEST 4: Test zero length bodies are not written\n--- http_config eval: $::HttpConfig\n--- config\nlocation /storage {\n content_by_lua_block {\n local backend = ngx.req.get_uri_args()[\"backend\"]\n local config = get_backend(backend)\n\n local storage = require(config.module).new()\n assert(storage:connect(config.params),\n \"storage:connect should return positively\")\n\n local res = _res.new(\"00004-\" .. backend)\n\n assert(not storage:exists(res.entity_id),\n \"entity should not exist\")\n\n -- Attach the writer, and run sink\n res.body_reader = storage:get_writer(\n res, 60,\n success_handler,\n failure_handler\n )\n sink(res.body_reader)\n\n -- Prove entity wasn't written\n assert(not storage:exists(res.entity_id),\n \"entity should not exist\")\n\n assert(storage:close(),\n \"storage:close should return positively\")\n }\n}\n--- request eval\n[\n \"GET /storage?backend=redis\",\n \"GET /storage?backend=redis_notransact\",\n]\n--- response_body eval\n[\n \"wrote 0 bytes\n\",\n \"wrote 0 bytes\n\",\n]\n\n--- no_error_log\n[error]\n\n\n=== TEST 5: Test write fails and abort handler called if conn to storage is interrupted\n--- http_config eval: $::HttpConfig\n--- config\nlocation /storage {\n lua_socket_log_errors off;\n content_by_lua_block {\n -- TODO find a way to check the error log, for no transact redis,\n -- where the optimistic call to redis:del() fails due to closed conn.\n\n local backend = ngx.req.get_uri_args()[\"backend\"]\n local config = get_backend(backend)\n\n local storage = require(config.module).new()\n assert(storage:connect(config.params),\n \"storage:connect should return positively\")\n\n local res = _res.new(\"00005-\" .. backend)\n -- Load source but fail on second chunk\n res.body_reader = get_and_fail_source({\n { \"123\", nil, false },\n { \"456\", nil, true },\n { \"789\", nil, true },\n }, 2, storage)\n\n assert(not storage:exists(res.entity_id),\n \"entity should not yet exist\")\n\n -- Attach the writer, and run sink\n res.body_reader = storage:get_writer(\n res, 60,\n success_handler,\n failure_handler\n )\n sink(res.body_reader)\n\n if backend ~= \"redis_notransact\" then\n -- Prove entity wasn't written (rolled back)\n assert(not storage:exists(res.entity_id),\n \"entity should still not exist\")\n end\n }\n}\n--- request eval\n[\n \"GET /storage?backend=redis\",\n \"GET /storage?backend=redis_notransact\",\n]\n--- response_body eval\n[\n \"123:nil:false\n456:nil:true\n789:nil:true\nerror writing: closed\n\",\n \"123:nil:false\n456:nil:true\n789:nil:true\nerror writing: closed\n\",\n]\n\n\n=== TEST 5b: Test write fails and abort handler called if upstream errors\n--- http_config eval: $::HttpConfig\n--- config\nlocation /storage {\n lua_socket_log_errors off;\n content_by_lua_block {\n local backend = ngx.req.get_uri_args()[\"backend\"]\n local config = get_backend(backend)\n\n local storage = require(config.module).new()\n assert(storage:connect(config.params),\n \"storage:connect should return positively\")\n\n local res = _res.new(\"00005b-\" .. backend)\n -- Load source but fail on second chunk\n res.body_reader = get_and_fail_upstream_source({\n { \"123\", nil, false },\n { \"456\", nil, true },\n { \"789\", nil, true },\n }, 2)\n\n assert(not storage:exists(res.entity_id),\n \"entity should not yet exist\")\n\n -- Attach the writer, and run sink\n res.body_reader = storage:get_writer(\n res, 60,\n success_handler,\n failure_handler\n )\n sink(res.body_reader)\n\n if backend ~= \"redis_notransact\" then\n -- Prove entity wasn't written (rolled back)\n assert(not storage:exists(res.entity_id),\n \"entity should still not exist\")\n end\n }\n}\n--- request eval\n[\n \"GET /storage?backend=redis\",\n \"GET /storage?backend=redis_notransact\",\n]\n--- response_body eval\n[\n \"123:nil:false\nupstream error: timeout\n\",\n \"123:nil:false\nupstream error: timeout\n\",\n]\n\n\n=== TEST 6: Write entity with short exiry, test keys expire\n--- http_config eval: $::HttpConfig\n--- config\nlocation /storage {\n content_by_lua_block {\n local backend = ngx.req.get_uri_args()[\"backend\"]\n local config = get_backend(backend)\n\n local storage = require(config.module).new()\n\n assert(storage:connect(config.params),\n \"storage:connect should return positively\")\n\n local res = _res.new(\"00006-\" .. backend)\n res.body_reader = get_source({\n { \"123\", nil, false },\n { \"456\", nil, true },\n { \"789\", nil, false },\n })\n\n assert(not storage:exists(res.entity_id),\n \"entity should not exist\")\n\n -- Attach the writer, and run sink\n res.body_reader = storage:get_writer(\n res, 1,\n success_handler,\n failure_handler\n )\n sink(res.body_reader)\n\n assert(storage:exists(res.entity_id),\n \"entity should exist\")\n\n ngx.sleep(2)\n\n assert(not storage:exists(res.entity_id),\n \"entity should not exist\")\n\n assert(storage:close(),\n \"storage:close should return positively\")\n }\n}\n--- request eval\n[\n \"GET /storage?backend=redis\",\n \"GET /storage?backend=redis_notransact\",\n]\n--- response_body eval\n[\n \"123:nil:false\n456:nil:true\n789:nil:false\nwrote 9 bytes\n\",\n \"123:nil:false\n456:nil:true\n789:nil:false\nwrote 9 bytes\n\",\n]\n--- no_error_log\n[error]\n\n\n=== TEST 7: Test maxmem keys are cleaned up when transactions are not available\n--- http_config eval: $::HttpConfig\n--- config\nlocation /storage {\n content_by_lua_block {\n local backend = ngx.req.get_uri_args()[\"backend\"]\n local config = get_backend(backend)\n\n local storage = require(config.module).new()\n\n config.params.max_size = 8\n\n -- Turn off atomicity\n config.params.supports_transactions = false\n assert(storage:connect(config.params),\n \"storage:connect should return positively\")\n\n local res = _res.new(\"00007-\" .. backend)\n -- Load source but fail on second chunk\n res.body_reader = get_source({\n { \"123\", nil, false },\n { \"456\", nil, true },\n { \"789\", nil, true },\n }, storage)\n\n assert(not storage:exists(res.entity_id),\n \"entity should not exist\")\n\n -- Attach the writer, and run sink\n res.body_reader = storage:get_writer(\n res, 60,\n success_handler,\n failure_handler\n )\n sink(res.body_reader)\n\n -- Prove entity wasn't written (rolled back)\n assert(not storage:exists(res.entity_id),\n \"entity should not exist\")\n }\n}\n--- request eval\n[\n \"GET /storage?backend=redis\",\n \"GET /storage?backend=redis_notransact\",\n]\n--- response_body eval\n[\n \"123:nil:false\n456:nil:true\n789:nil:true\nbody is larger than 8 bytes\n\",\n \"123:nil:false\n456:nil:true\n789:nil:true\nbody is larger than 8 bytes\n\",\n]\n--- no_error_log\n[error]\n\n\n=== TEST 8: Keys will remain on failure when transactions are not available\n--- http_config eval: $::HttpConfig\n--- config\nlocation /storage {\n lua_socket_log_errors off;\n content_by_lua_block {\n local backend = ngx.req.get_uri_args()[\"backend\"]\n local config = get_backend(backend)\n\n local storage = require(config.module).new()\n\n config.params.supports_transactions = false\n assert(storage:connect(config.params),\n \"storage:connect should return positively\")\n\n local res = _res.new(\"00008-\" .. backend)\n -- Load source but fail on second chunk\n res.body_reader = get_and_fail_source({\n { \"123\", nil, false },\n { \"456\", nil, true },\n { \"789\", nil, true },\n }, 2, storage)\n\n assert(not storage:exists(res.entity_id),\n \"entity should not exist\")\n\n -- Attach the writer, and run sink\n res.body_reader = storage:get_writer(\n res, 60,\n success_handler,\n failure_handler\n )\n sink(res.body_reader)\n\n -- Reconnect\n assert(storage:connect(config.params),\n \"storage:connect should return positively\")\n\n -- Prove it still exists (could not be cleaned up)\n assert(storage:exists(res.entity_id),\n \"entity should exist\")\n }\n}\n--- request eval\n[\n \"GET /storage?backend=redis\",\n \"GET /storage?backend=redis_notransact\",\n]\n--- response_body eval\n[\n \"123:nil:false\n456:nil:true\n789:nil:true\nerror writing: closed\n\",\n \"123:nil:false\n456:nil:true\n789:nil:true\nerror writing: closed\n\",\n]\n--- error_log eval\n[\"closed\"]\n\n\n=== TEST 9: Close connection and then reconnect and re-read\n--- http_config eval: $::HttpConfig\n--- config\nlocation /storage {\n content_by_lua_block {\n local backend = ngx.req.get_uri_args()[\"backend\"]\n local config = get_backend(backend)\n\n local storage = require(config.module).new()\n\n assert(storage:connect(config.params),\n \"storage:connect should return positively\")\n\n local res = _res.new(\"00009-\" .. backend)\n res.body_reader = get_source({\n { \"123\", nil, false },\n { \"456\", nil, true },\n { \"789\", nil, false },\n })\n\n assert(not storage:exists(res.entity_id),\n \"entity should not exist\")\n\n -- Attach the writer, and run sink\n res.body_reader = storage:get_writer(\n res, 60,\n success_handler,\n failure_handler\n )\n sink(res.body_reader)\n\n assert(storage:exists(res.entity_id),\n \"entity should exist\")\n\n assert(storage:close(),\n \"storage:close should return positively\")\n\n assert(storage:connect(config.params),\n \"storage:connect should return positively\")\n\n assert(storage:exists(res.entity_id),\n \"entity should exist\")\n\n res.body_reader = storage:get_reader(res)\n sink(res.body_reader)\n\n assert(storage:close(),\n \"storage:close should return positively\")\n }\n}\n--- request eval\n[\n \"GET /storage?backend=redis\",\n \"GET /storage?backend=redis_notransact\",\n]\n--- response_body eval\n[\n \"123:nil:false\n456:nil:true\n789:nil:false\nwrote 9 bytes\n123:nil:false\n456:nil:true\n789:nil:false\n\",\n \"123:nil:false\n456:nil:true\n789:nil:false\nwrote 9 bytes\n123:nil:false\n456:nil:true\n789:nil:false\n\",\n]\n--- no_error_log\n[error]\n\n\n=== TEST 10: Entities can be deleted\n--- http_config eval: $::HttpConfig\n--- config\nlocation /storage {\n content_by_lua_block {\n local backend = ngx.req.get_uri_args()[\"backend\"]\n local config = get_backend(backend)\n\n local storage = require(config.module).new()\n\n assert(storage:connect(config.params),\n \"storage:connect should return positively\")\n\n local res = _res.new(\"00010-\" .. backend)\n res.body_reader = get_source({\n { \"123\", nil, false },\n { \"456\", nil, false },\n { \"789\", nil, false },\n })\n\n assert(not storage:exists(res.entity_id),\n \"entity should not exist\")\n\n -- Attach the writer, and run sink\n res.body_reader = storage:get_writer(\n res, 99,\n success_handler,\n failure_handler\n )\n sink(res.body_reader)\n\n assert(storage:exists(res.entity_id),\n \"entity should exist\")\n\n assert(storage:delete(res.entity_id),\n \"entity should delete without error\")\n\n assert(not storage:exists(res.entity_id),\n \"entity should not exist\")\n\n local ok, err = storage:delete(\"foo\")\n assert(ok == false and err == nil,\n \"deleting foo entity should return false without error\")\n\n assert(storage:close(),\n \"storage:close should return positively\")\n }\n}\n--- request eval\n[\n \"GET /storage?backend=redis\",\n \"GET /storage?backend=redis_notransact\",\n]\n--- response_body eval\n[\n \"123:nil:false\n456:nil:false\n789:nil:false\nwrote 9 bytes\n\",\n \"123:nil:false\n456:nil:false\n789:nil:false\nwrote 9 bytes\n\",\n]\n--- no_error_log\n[error]\n\n\n=== TEST 11: set_ttl / get_ttl\n--- http_config eval: $::HttpConfig\n--- config\nlocation /storage {\n content_by_lua_block {\n local backend = ngx.req.get_uri_args()[\"backend\"]\n local config = get_backend(backend)\n\n local storage = require(config.module).new()\n\n assert(storage:connect(config.params),\n \"storage:connect should return positively\")\n\n local res = _res.new(\"00011-\" .. backend)\n res.body_reader = get_source({\n { \"123\", nil, false },\n { \"456\", nil, false },\n { \"789\", nil, false },\n })\n\n assert(not storage:exists(res.entity_id),\n \"entity should not exist\")\n\n -- Attach the writer, and run sink\n res.body_reader = storage:get_writer(\n res, 99,\n success_handler,\n failure_handler\n )\n sink(res.body_reader)\n\n assert(storage:exists(res.entity_id),\n \"entity should exist\")\n\n local ttl, err = storage:get_ttl(\"foo\")\n assert(ttl == false and err == \"entity does not exist\",\n \"getting ttl on foo entity should return false without error\")\n\n local ttl = storage:get_ttl(res.entity_id)\n assert(ttl and ttl <= 99 and ttl >= 98,\n \"entity ttl should be roughly 99\")\n\n local ok, err = storage:set_ttl(\"foo\", 1)\n assert(ok == false and err == \"entity does not exist\",\n \"setting ttl on foo entity should return false without error\")\n\n assert(storage:set_ttl(res.entity_id, 1),\n \"setting ttl should return positively\")\n\n ngx.sleep(2)\n\n assert(not storage:exists(res.entity_id),\n \"entity should have expired\")\n\n assert(storage:close(),\n \"storage:close should return positively\")\n }\n}\n--- request eval\n[\n \"GET /storage?backend=redis\",\n \"GET /storage?backend=redis_notransact\",\n]\n--- response_body eval\n[\n \"123:nil:false\n456:nil:false\n789:nil:false\nwrote 9 bytes\n\",\n \"123:nil:false\n456:nil:false\n789:nil:false\nwrote 9 bytes\n\",\n]\n--- no_error_log\n[error]\n\n=== TEST 12: Bad params should return an error\n--- http_config eval: $::HttpConfig\n--- config\nlocation /storage {\n content_by_lua_block {\n local config = get_backend(ngx.req.get_uri_args()[\"backend\"])\n local storage = require(config.module).new()\n\n local ok, err = storage:connect(config.bad_params)\n\n assert(not ok,\n \"storage:connect should not return positively\")\n assert(type(err) == \"string\",\n \"storage:connect should return an error string\")\n\n ngx.log(ngx.INFO, err)\n ngx.print(ngx.req.get_uri_args()[\"backend\"], \" OK\")\n }\n}\n--- request eval\n[\n \"GET /storage?backend=redis\",\n \"GET /storage?backend=redis_notransact\",\n]\n--- response_body eval\n[\n \"redis OK\",\n \"redis_notransact OK\",\n]\n--- no_error_log\n[error]\n\n=== TEST 13: Handler run with bad config should return an error\n--- http_config eval: $::HttpConfig\n--- config\nlocation /storage {\n content_by_lua_block {\n local config = get_backend(ngx.req.get_uri_args()[\"backend\"])\n local ok, err = require(\"ledge\").create_handler({\n storage_driver = config.module,\n storage_driver_config = config.bad_params\n }):run()\n assert(ok == nil and err ~= nil,\n \"run should return negatively with an error\")\n\n ngx.print(ngx.req.get_uri_args()[\"backend\"], \" OK\")\n }\n}\n--- request eval\n[\n \"GET /storage?backend=redis\",\n \"GET /storage?backend=redis_notransact\",\n]\n--- response_body eval\n[\n \"redis OK\",\n \"redis_notransact OK\",\n]\n--- no_error_log\n[error]\n" }, { "path": "t/01-unit/tag_parser.t", "content": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::http_config(extra_lua_config => qq{\n function print_next(tag, before, after)\n if not tag.closing then\n tag.closing = {}\n end\n ngx.say(tag.closing.from)\n ngx.say(tag.closing.to)\n ngx.say(tag.closing.tag)\n ngx.say(tag.whole)\n ngx.say(tag.contents)\n ngx.say(before)\n ngx.say(after)\n end\n function strip_whitespace(content)\n return ngx.re.gsub(content, [[\\\\s*\\\\n\\\\s*]], \"\")\n end\n function check_regex(regex, content, msg)\n local to, from = ngx.re.find(content, regex, \"soj\")\n assert(from ~= nil and to ~= nil, (msg or \"regex should match\"))\n end\n});\n\nno_long_string();\nno_diff();\nrun_tests();\n\n__DATA__\n=== TEST 1: Load module\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local tag_parser = assert(require(\"ledge.esi.tag_parser\"),\n \"module should load without errors\")\n\n local parser = tag_parser.new(\"Content\")\n assert(parser, \"tag_parser.new should return positively\")\n\n ngx.say(\"OK\")\n }\n}\n\n--- request\nGET /t\n--- error_code: 200\n--- no_error_log\n[error]\n--- response_body\nOK\n\n=== TEST 2: Find next tag\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local tag_parser = assert(require(\"ledge.esi.tag_parser\"),\n \"module should load without errors\")\n\n local parser = tag_parser.new(\"content-beforeinsidecontent-after\")\n assert(parser, \"tag_parser.new should return positively\")\n\n local tag, before, after = parser:next(\"foo\")\n assert(tag, \"next should find a tag\")\n print_next(tag, before, after)\n }\n}\n\n--- request\nGET /t\n--- error_code: 200\n--- no_error_log\n[error]\n--- response_body\n26\n31\n\ninside\ninside\ncontent-before\ncontent-after\n\n=== TEST 3: Default next tag finds esi\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local tag_parser = assert(require(\"ledge.esi.tag_parser\"),\n \"module should load without errors\")\n\n local parser = tag_parser.new(\"content-beforeinsidecontent-afterlast\")\n assert(parser, \"tag_parser.new should return positively\")\n\n local tag, before, after = parser:next()\n assert(tag, \"next should find a tag\")\n print_next(tag, before, after)\n\n ngx.say(\"##########\")\n\n local tag, before, after = parser:next()\n assert(tag, \"next should find a tag\")\n print_next(tag, before, after)\n }\n}\n\n--- request\nGET /t\n--- error_code: 200\n--- no_error_log\n[error]\n--- response_body\n30\n39\n\ninside\ninside\ncontent-before\ncontent-afterlast\n##########\n68\n70\n-->\n\ncomment\ncontent-after\nlast\n\n=== TEST 4: Find tag with attributes\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local tag_parser = assert(require(\"ledge.esi.tag_parser\"),\n \"module should load without errors\")\n\n local parser = tag_parser.new(\"content-beforeinsidecontent-after\")\n assert(parser, \"tag_parser.new should return positively\")\n\n local tag, before, after = parser:next(\"foo\")\n assert(tag, \"next should find a tag\")\n print_next(tag, before, after)\n }\n}\n\n--- request\nGET /t\n--- error_code: 200\n--- no_error_log\n[error]\n--- response_body\n54\n59\n\ninside\nattr='value' attr2='value2'>inside\ncontent-before\ncontent-after\n\n=== TEST 4: Find nested tags\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local tag_parser = assert(require(\"ledge.esi.tag_parser\"),\n \"module should load without errors\")\n\n local content = strip_whitespace([[\ncontent-before\n\n inside-foo\n \n inside-bar\n \n after-bar\n \n inside-foo-2\n \n\ncontent-after\n]])\n\n local parser = tag_parser.new(content)\n assert(parser, \"tag_parser.new should return positively\")\n\n local tag, before, after = parser:next(\"foo\")\n assert(tag, \"next should find a tag\")\n print_next(tag, before, after)\n\n ngx.say(\"#######\")\n\n local parser = tag_parser.new(content)\n assert(parser, \"tag_parser.new should return positively\")\n\n local tag, before, after = parser:next(\"bar\")\n assert(tag, \"next should find a tag\")\n print_next(tag, before, after)\n }\n}\n\n--- request\nGET /t\n--- error_code: 200\n--- no_error_log\n[error]\n--- response_body\n83\n88\n\ninside-fooinside-barafter-barinside-foo-2\ninside-fooinside-barafter-barinside-foo-2\ncontent-before\ncontent-after\n#######\n45\n50\n\ninside-bar\ninside-bar\ncontent-beforeinside-foo\nafter-barinside-foo-2content-after\n\n=== TEST 5: Pattern functions return valid regex\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local tag_parser = assert(require(\"ledge.esi.tag_parser\"),\n \"module should load without errors\")\n\n local ok, err = ngx.re.find(\"\", tag_parser.open_pattern(\"tag\"))\n assert(err == nil, \"open_pattern should return a valid regex\")\n\n local ok, err = ngx.re.find(\"\", tag_parser.close_pattern(\"tag\"))\n assert(err == nil, \"open_pattern should return a valid regex\")\n\n local ok, err = ngx.re.find(\"\", tag_parser.either_pattern(\"tag\"))\n assert(err == nil, \"open_pattern should return a valid regex\")\n\n ngx.say(\"OK\")\n }\n}\n\n--- request\nGET /t\n--- error_code: 200\n--- no_error_log\n[error]\n--- response_body\nOK\n\n=== TEST 5: open pattern matches\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local tag_parser = assert(require(\"ledge.esi.tag_parser\"),\n \"module should load without errors\")\n\n local regex = tag_parser.open_pattern(\"tag\")\n ngx.log(ngx.DEBUG, regex)\n\n local checks = {\n \"start end\", \"simple tag\",\n \"start end\", \"simple closed tag\",\n \"start asdfsd end\", \"simple closed tag with content\",\n \"start end\", \"simple tag whitespace\",\n \"start end\", \"self-closing tag\",\n \"start end\", \"self-closing tag whitespace\",\n \"start end\", \"simple tag with attribute\",\n 'start end', \"simple tag with attribute (single-quote)\",\n 'start end', \"simple tag with attribute (numeric)\",\n 'start end', \"simple tag with attribute (special chars)\",\n }\n\n for i=1,#checks,2 do\n check_regex(regex, checks[i], \"open_pattern should match \"..checks[i+1])\n end\n\n ngx.say(\"OK\")\n }\n}\n\n--- request\nGET /t\n--- error_code: 200\n--- no_error_log\n[error]\n--- response_body\nOK\n\n=== TEST 6: close pattern matches\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local tag_parser = assert(require(\"ledge.esi.tag_parser\"),\n \"module should load without errors\")\n\n local regex = tag_parser.close_pattern(\"tag\")\n ngx.log(ngx.DEBUG, regex)\n\n local checks = {\n \"start end\", \"simple tag\",\n \"start end\", \"simple closed tag\",\n \"start asdfsd end\", \"simple closed tag with content\",\n \"start end\", \"simple tag with whitespace\",\n }\n\n for i=1,#checks,2 do\n check_regex(regex, checks[i], \"close_pattern should match \"..checks[i+1])\n end\n\n ngx.say(\"OK\")\n }\n}\n\n--- request\nGET /t\n--- error_code: 200\n--- no_error_log\n[error]\n--- response_body\nOK\n\n=== TEST 7: either pattern matches\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local tag_parser = assert(require(\"ledge.esi.tag_parser\"),\n \"module should load without errors\")\n\n local regex = tag_parser.either_pattern(\"tag\")\n ngx.log(ngx.DEBUG, regex)\n\n local checks = {\n \"start end\", \"simple tag\",\n \"start end\", \"simple closed tag\",\n \"start asdfsd end\", \"simple closed tag with content\",\n \"start end\", \"simple tag whitespace\",\n \"start end\", \"self-closing tag\",\n \"start end\", \"self-closing tag whitespace\",\n \"start end\", \"simple tag with attribute\",\n 'start end', \"simple tag with attribute (single-quote)\",\n 'start end', \"simple tag with attribute (numeric)\",\n 'start end', \"simple tag with attribute (special chars)\",\n\n \"start end\", \"simple tag\",\n \"start end\", \"simple closed tag\",\n \"start asdfsd end\", \"simple closed tag with content\",\n \"start end\", \"simple tag with whitespace\",\n }\n\n for i=1,#checks,2 do\n check_regex(regex, checks[i], \"either_pattern should match \"..checks[i+1])\n end\n\n ngx.say(\"OK\")\n }\n}\n\n--- request\nGET /t\n--- error_code: 200\n--- no_error_log\n[error]\n--- response_body\nOK\n" }, { "path": "t/01-unit/util.t", "content": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::http_config(extra_lua_config => qq{\n TEST_NGINX_PORT = $LedgeEnv::nginx_port\n});\n\nno_long_string();\nno_diff();\nrun_tests();\n\n__DATA__\n=== TEST 1: string.randomhex\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local randomhex = require(\"ledge.util\").string.randomhex\n\n -- lengths\n assert(#randomhex(10) == 10, \"randomhex(10) length should be 10\")\n assert(#randomhex(42) == 42, \"randomhex(42) length should be 42\")\n\n -- apparent randomness\n assert(randomhex(10) ~= randomhex(10),\n \"random hex strings should differ\")\n }\n}\n--- request\nGET /t\n--- no_error_log\n[error]\n\n\n=== TEST 2: mt.fixed_field_metatable\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local fixed_field_metatable =\n require(\"ledge.util\").mt.fixed_field_metatable\n\n -- Error if new field creation attempted\n local t = setmetatable({ a = 1, c = 3 }, fixed_field_metatable)\n local ok, err = pcall(\n function() t.b = 2 end,\n \"attempt to create new field b\"\n )\n assert(string.find(err, \"attempt to create new field b\"),\n \"err should contain 'attempt to create new field b'\")\n\n -- Error if non existent field dereferenced\n local t = setmetatable({ a = 1, c = 3 }, fixed_field_metatable)\n local ok, err = pcall(\n function() local a = t.b end,\n \"attempt to create new field b\"\n )\n assert(string.find(err, \"field b does not exist\"),\n \"err should contain 'field b does not exist'\")\n }\n}\n--- request\nGET /t\n--- no_error_log\n[error]\n\n\n=== TEST 3: mt.get_fixed_field_metatable_proxy\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local get_fixed_field_metatable_proxy =\n require(\"ledge.util\").mt.get_fixed_field_metatable_proxy\n\n local defaults = { a = 1, b = 2, c = 3 }\n\n -- Error if new field creation attempted\n local t = setmetatable(\n { b = 4 },\n get_fixed_field_metatable_proxy(defaults)\n )\n\n assert(t.a == 1, \"t.a should be 1\")\n assert(t.b == 4, \"t.b should be 4\")\n assert(t.c == 3, \"t.c should be 3\")\n }\n}\n--- request\nGET /t\n--- no_error_log\n[error]\n\n\n=== TEST 4: mt.get_callable_fixed_field_metatable\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local get_callable_fixed_field_metatable =\n require(\"ledge.util\").mt.get_callable_fixed_field_metatable\n\n local func =\n function(t, field)\n return t[field]\n end\n\n -- Error if new field creation attempted\n local t = setmetatable(\n { a = 1, b = 2, c = 3 },\n get_callable_fixed_field_metatable(func)\n )\n\n assert(t(\"a\") == 1, \"t('a') should return 1\")\n assert(t(\"b\") == 2, \"t('b') should return 2\")\n assert(t(\"c\") == 3, \"t('c') should return 3\")\n }\n}\n--- request\nGET /t\n--- no_error_log\n[error]\n\n\n=== TEST 5: table.copy\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local tbl_copy = require(\"ledge.util\").table.copy\n\n local mt = { __index = function(t, k) return \"no index\" end }\n local t = {\n a = 1,\n b = 2,\n c = {\n x = 10,\n y = 11,\n z = setmetatable({ 1, 2, 3 }, mt),\n }\n }\n\n local copy = tbl_copy(t)\n\n -- Values copied\n assert(t ~= copy, \"copy should not equal t\")\n assert(copy.a == 1, \"copy.a should be 1\")\n assert(type(copy.c) == \"table\", \"copy.c should be a table\")\n assert(copy.c ~= t.c, \"copy.c should not equal t.c\")\n assert(copy.c.x == 10, \"copy.c.x should be 10\")\n assert(type(copy.c.z) == \"table\", \"copy.c.z should be a table\")\n assert(copy.c.z ~= t.c.z, \"copy.z.a. should not equal t.c.z\")\n assert(copy.c.z[1] == 1, \"copy.c.z[1] should be 1\")\n assert(copy.c.z[3] == 3, \"copy.c.z[3] should be 3\")\n\n -- Metatables copied\n assert(getmetatable(copy) == nil, \"getmetatable(copy) should be nil\")\n assert(getmetatable(copy.c.z) ~= getmetatable(t.c.z),\n \"copy.c.z metatable should not equal t.c.z metatable\")\n assert(getmetatable(copy.c.z).__index == getmetatable(t.c.z).__index,\n \"copy.c.z __index metamethod should equal t.c.z __index metamethod\")\n assert(copy.c.z[4] == \"no index\", \"copy.c.z[3] should be 'no index'\")\n }\n}\n--- request\nGET /t\n--- no_error_log\n[error]\n\n\n=== TEST 6: table.copy_merge_defaults\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local tbl_copy_merge_defaults =\n require(\"ledge.util\").table.copy_merge_defaults\n local fixed_field_metatable =\n require(\"ledge.util\").mt.fixed_field_metatable\n\n local defaults = {\n a = 1,\n c = 3,\n d = {\n x = 10,\n z = 12,\n },\n e = {\n a = 1,\n c = 3,\n },\n }\n\n local t = {\n a = false,\n b = 2,\n e = {\n b = 2,\n },\n }\n\n local copy = tbl_copy_merge_defaults(t, defaults)\n\n -- Basic copy merge\n assert(copy ~= t, \"copy should not equal t\")\n assert(getmetatable(copy) == nil, \"copy should not have a metatable\")\n assert(copy.a == false, \"copy.a should be false\")\n assert(copy.b == 2, \"copy.b should be 2\")\n assert(copy.c == 3, \"copy.c should be 3\")\n\n -- Child table in defaults is merged\n assert(copy.d ~= defaults.d, \"copy.d should not equal defaults d\")\n assert(copy.d.x == 10, \"copy.d.x should be 10\")\n assert(copy.d.z == 12, \"copy.d.z should be 12\")\n\n -- Child table in both is merged\n assert(copy.e ~= defaults.e, \"copy.e should not equal defaults e\")\n assert(copy.e.a == 1, \"copy.e.a should be 1\")\n assert(copy.e.b == 2, \"copy.e.b should be 2\")\n assert(copy.e.c == 3, \"copy.e.c should be 3\")\n\n\n -- Same again, but with defaults being \"fixed field\"\n local defaults = setmetatable({\n a = 1,\n b = 2,\n c = 3,\n d = setmetatable({\n x = 10,\n y = 11,\n z = 12,\n }, fixed_field_metatable)\n }, fixed_field_metatable)\n\n local t_good = {\n b = 6,\n d = {\n z = 42,\n },\n }\n\n -- Copy is merged properly\n local copy = tbl_copy_merge_defaults(t_good, defaults)\n\n assert(copy.a == 1, \"copy.a should be 1\")\n assert(copy.b == 6, \"copy.b should be 6\")\n assert(copy.c == 3, \"copy.c should be 3\")\n assert(copy.d ~= defaults.d and copy.d ~= t_good.d,\n \"copy.d should not equal defaults.d or t_good.d\")\n assert(getmetatable(copy) == nil, \"getmetatable(copy) should be nil\")\n\n\n -- Copy merge should fail\n local t_bad_1 = {\n a = 4,\n foo = \"bar\",\n }\n\n local ok, err = pcall(function()\n tbl_copy_merge_defaults(t_bad_1, defaults)\n end)\n\n assert(string.find(err, \"field foo does not exist\"),\n \"error 'field foo does not exist' should be thrown\")\n\n\n -- Copy merge should fail on inner table\n local t_bad_2 = {\n a = 4,\n d = {\n x = 10,\n foo = \"bar\",\n },\n }\n\n local ok, err = pcall(function()\n tbl_copy_merge_defaults(t_bad_1, defaults)\n end)\n\n assert(string.find(err, \"field foo does not exist\"),\n \"error 'field foo does not exist' should be thrown\")\n\n\n -- Copy merge with a nil user table gives us a copy of defaults\n local t, err = tbl_copy_merge_defaults(nil, defaults)\n\n assert(t ~= nil and t.a == 1,\n \"merging a nil user table should still return defaults\")\n assert(t ~= defaults, \"defaults should be copied by value\")\n\n\n local t, err = tbl_copy_merge_defaults(t_good, nil)\n assert(t ~= nil and t.b == 6,\n \"merging with nil defaults should still return user t\")\n assert(t ~= t_good, \"user t should be copied by value\")\n\n }\n}\n--- request\nGET /t\n--- no_error_log\n[error]\n\n\n=== TEST 7: string.split\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local str_split = require(\"ledge.util\").string.split\n\n local str1 = \"comma, separated, string, \"\n local t = str_split(str1, \",\")\n\n assert(#t == 4, \"#t should be 4\")\n assert(t[1] == \"comma\", \"t[1] should be 'comma'\")\n assert(t[2] == \" separated\", \"t[2] should be ' separated'\")\n assert(t[3] == \" string\", \"t[3] should be ' string'\")\n assert(t[4] == \" \", \"t[4] should be ' '\")\n\n local t = str_split(str1, \", \")\n assert(#t == 3, \"#t should be 3\")\n assert(t[1] == \"comma\", \"t[1] should be 'comma'\")\n assert(t[2] == \"separated\", \"t[2] should be ' separated'\")\n assert(t[3] == \"string\", \"t[3] should be ' string'\")\n }\n}\n--- request\nGET /t\n--- no_error_log\n[error]\n\n\n=== TEST 8: coroutine.wrap\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local co_wrap = require(\"ledge.util\").coroutine.wrap\n\n local co = co_wrap(\n function()\n for i = 1, 10 do\n coroutine.yield(i)\n end\n end\n )\n\n function run()\n local res = \"\"\n repeat\n local num = co()\n if num then\n res = res .. num .. \"-\"\n end\n until not num\n res = res .. \"finished\"\n return res\n end\n\n assert(run() == \"1-2-3-4-5-6-7-8-9-10-finished\",\n \"run() should return 1-2-3-4-5-6-7-8-9-10-finished\")\n }\n}\n--- request\nGET /t\n--- no_error_log\n[error]\n\n=== TEST 8b: coroutine.wrap errors\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local co_wrap = require(\"ledge.util\").coroutine.wrap\n\n local co = co_wrap(\n function()\n for i = 1, 10 do\n if i == 5 then\n error(\"BOOM\")\n end\n coroutine.yield(i)\n end\n end\n )\n\n function run()\n local res = \"\"\n repeat\n local num, err = co()\n if num then\n res = res .. num .. \"-\"\n elseif err then\n ngx.log(ngx.DEBUG, \"Coroutine error: \", err)\n end\n until not num\n res = res .. \"finished\"\n return res\n end\n\n assert(run() == \"1-2-3-4-finished\", \"Error was yielded!\")\n }\n}\n--- request\nGET /t\n--- error_log\nCoroutine error:\nBOOM\n--- no_error_log\nError was yielded!\n\n\n=== TEST 9: get_hostname\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local get_hostname = require(\"ledge.util\").get_hostname\n assert(string.lower(get_hostname()) == string.lower(ngx.var.hostname),\n \"get_hostname \"..tostring(get_hostname())..\" should be \"..ngx.var.hostname)\n }\n}\n--- request\nGET /t\n--- no_error_log\n[error]\n\n=== TEST 10: append_server_port\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local append_server_port = require(\"ledge.util\").append_server_port\n local host = \"example.com\"\n local default = host..\":\"..TEST_NGINX_PORT\n assert(append_server_port(host) == default,\n \"append_server_port should be \"..default)\n }\n}\n--- request\nGET /t\n--- no_error_log\n[error]\n" }, { "path": "t/01-unit/validation.t", "content": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::http_config();\n\nno_long_string();\nno_diff();\nrun_tests();\n\n__DATA__\n=== TEST 1: must_revalidate\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local must_revalidate = require(\"ledge.validation\").must_revalidate\n\n local res = {\n header = {\n [\"Cache-Control\"] = ngx.req.get_headers().x_res_cache_control,\n [\"Age\"] = ngx.req.get_headers().x_res_age,\n },\n }\n\n local result = ngx.req.get_uri_args().result\n assert(tostring(must_revalidate(res)) == result,\n \"must_revalidate should be \" .. result)\n\n }\n}\n--- more_headers eval\n[\n \"\",\n \"Cache-Control: max-age=0\",\n \"Cache-Control: max-age=1\nX-Res-Age: 1\",\n \"Cache-Control: max-age=1\nX-Res-Age: 2\",\n \"X-Res-Cache-Control: must-revalidate\",\n \"X-Res-Cache-Control: proxy-revalidate\",\n]\n--- request eval\n[\n \"GET /t?&result=false\",\n \"GET /t?&result=true\",\n \"GET /t?&result=false\",\n \"GET /t?&result=true\",\n \"GET /t?&result=true\",\n \"GET /t?&result=true\",\n]\n--- no_error_log\n[error]\n\n\n=== TEST 2: can_revalidate_locally\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local can_revalidate_locally =\n require(\"ledge.validation\").can_revalidate_locally\n\n local result = ngx.req.get_uri_args().result\n assert(tostring(can_revalidate_locally()) == result,\n \"can_revalidate_locally should be \" .. result)\n\n }\n}\n--- more_headers eval\n[\n \"\",\n \"If-None-Match:\" ,\n \"If-None-Match: foo\",\n \"If-Modified-Since: Sun, 06 Nov 1994 08:49:37 GMT\",\n \"If-Modified-Since:\",\n \"If-Modified-Since: foo\",\n]\n--- request eval\n[\n \"GET /t?&result=false\",\n \"GET /t?&result=false\",\n \"GET /t?&result=true\",\n \"GET /t?&result=true\",\n \"GET /t?&result=false\",\n \"GET /t?&result=false\",\n]\n--- no_error_log\n[error]\n\n\n=== TEST 3: is_valid_locally\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t {\n content_by_lua_block {\n local is_valid_locally = require(\"ledge.validation\").is_valid_locally\n\n local res = {\n header = {\n [\"Last-Modified\"] = ngx.req.get_headers().x_res_last_modified,\n [\"Etag\"] = ngx.req.get_headers().x_res_etag,\n },\n }\n\n local result = ngx.req.get_uri_args().result\n assert(tostring(is_valid_locally(res)) == result,\n \"is_valid_locally should be \" .. result)\n\n }\n}\n--- more_headers eval\n[\n \"\",\n \"If-Modified-Since: Sun, 05 Nov 1994 08:49:37 GMT\nX-Res-Last-Modified: Sun, 06 Nov 1994 08:48:37 GMT\",\n \"If-Modified-Since: Sun, 06 Nov 1994 08:49:37 GMT\nX-Res-Last-Modified: Sun, 06 Nov 1994 08:48:37 GMT\",\n \"If-Modified-Since: Sun, 06 Nov 1994 08:49:38 GMT\nX-Res-Last-Modified: Sun, 06 Nov 1994 08:48:37 GMT\",\n \"If-Modified-Since: Sun, 06 Nov 1994 08:49:36 GMT\nX-Res-Last-Modified: Sun, 06 Nov 1994 08:49:37 GMT\",\n \"If-None-Match: foo\nX-Res-Etag: foo\",\n \"If-None-Match: foo\nX-Res-Etag: bar\",\n \"If-None-Match: foo\",\n \"X-Res-Etag: bar\",\n]\n--- request eval\n[\n \"GET /t?&result=false\",\n \"GET /t?&result=false\",\n \"GET /t?&result=true\",\n \"GET /t?&result=true\",\n \"GET /t?&result=false\",\n \"GET /t?&result=true\",\n \"GET /t?&result=false\",\n \"GET /t?&result=false\",\n \"GET /t?&result=false\",\n]\n--- no_error_log\n[error]\n" }, { "path": "t/01-unit/worker.t", "content": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::http_config();\n\nour $HttpConfig_Test6 = LedgeEnv::http_config(extra_lua_config => qq{\n foo = 1\n package.loaded[\"ledge.job.test\"] = {\n perform = function(job)\n foo = foo + 1\n return true\n end\n }\n}, run_worker => 1);\n\nno_long_string();\nno_diff();\nrun_tests();\n\n__DATA__\n=== TEST 1: Load module without errors.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /worker_1 {\n echo \"OK\";\n}\n--- request\nGET /worker_1\n--- no_error_log\n[error]\n\n\n=== TEST 2: Create worker with default config\n--- http_config eval: $::HttpConfig\n--- config\nlocation /worker_2 {\n echo \"OK\";\n}\n--- request\nGET /worker_2\n--- no_error_log\n[error]\n\n\n=== TEST 4: Create worker with bad config key\n--- http_config eval\nqq {\nlua_package_path \"./lib/?.lua;;\";\ninit_by_lua_block {\n if $LedgeEnv::test_coverage == 1 then\n require(\"luacov.runner\").init()\n end\n}\ninit_worker_by_lua_block {\n require(\"ledge.worker\").new({\n foo = \"one\",\n })\n}\n}\n--- config\nlocation /worker_4 {\n echo \"OK\";\n}\n--- request\nGET /worker_4\n--- error_log\nfield foo does not exist\n\n\n=== TEST 5: Run workers without errors\n--- http_config eval\nqq {\nlua_package_path \"./lib/?.lua;;\";\ninit_by_lua_block {\n if $LedgeEnv::test_coverage == 1 then\n require(\"luacov.runner\").init()\n end\n}\ninit_worker_by_lua_block {\n require(\"ledge.worker\").new():run()\n}\n}\n--- config\nlocation /worker_5 {\n echo \"OK\";\n}\n--- request\nGET /worker_5\n--- no_error_log\n[error]\n\n\n=== TEST 6: Push a job and confirm it runs\n--- http_config eval: $::HttpConfig_Test6\n--- config\nlocation /worker_6 {\n content_by_lua_block {\n local qless = assert(require(\"resty.qless\").new({\n get_redis_client = require(\"ledge\").create_qless_connection\n }))\n\n local jid = assert(qless.queues[\"ledge_gc\"]:put(\"ledge.job.test\"))\n\n ngx.sleep(2)\n ngx.say(foo)\n local job = qless.jobs:get(jid)\n ngx.say(job.state)\n }\n}\n--- request\nGET /worker_6\n--- response_body\n2\ncomplete\n--- timeout: 5\n--- no_error_log\n[error]\n" }, { "path": "t/02-integration/age.t", "content": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::http_config();\n\nno_long_string();\nno_diff();\nrun_tests();\n\n__DATA__\n=== TEST 1: No calculated Age header on cache MISS.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /age_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler({\n origin_mode = require(\"ledge\").ORIGIN_MODE_AVOID\n }):run()\n }\n}\nlocation /age {\n more_set_headers \"Cache-Control public, max-age=600\";\n echo \"OK\";\n}\n--- request\nGET /age_prx\n--- response_headers\nAge:\n--- no_error_log\n[error]\n\n\n=== TEST 2: Age header on cache HIT\n--- http_config eval: $::HttpConfig\n--- config\nlocation /age_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler({\n origin_mode = require(\"ledge\").ORIGIN_MODE_AVOID\n }):run()\n }\n}\nlocation /age {\n more_set_headers \"Cache-Control public, max-age=600\";\n echo \"OK\";\n}\n--- request\nGET /age_prx\n--- response_headers_like\nAge: \\d+\n--- no_error_log\n[error]\n" }, { "path": "t/02-integration/cache.t", "content": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::http_config(extra_nginx_config => qq{\n lua_shared_dict ledge_test 1m;\n}, run_worker => 1);\n\nno_long_string();\nno_diff();\nrun_tests();\n\n\n__DATA__\n=== TEST 1: Subzero request; X-Cache: MISS\n--- http_config eval: $::HttpConfig\n--- config\nlocation /cache_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\n\nlocation /cache {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.say(\"TEST 1\")\n }\n}\n--- request\nGET /cache_prx\n--- response_headers_like\nX-Cache: MISS from .*\n--- response_body\nTEST 1\n--- no_error_log\n[error]\n\n=== TEST 1b: Subzero request; X-Cache: MISS is prepended\n--- http_config eval: $::HttpConfig\n--- config\nlocation /cache_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\n\nlocation /cache {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.header[\"X-Cache\"] = \"HIT from example.com\"\n ngx.say(\"TEST 1\")\n }\n}\n--- request\nGET /cache_prx?append\n--- response_headers_like\nX-Cache: MISS from .+, HIT from example.com\n--- response_body\nTEST 1\n--- no_error_log\n[error]\n\n\n=== TEST 2: Hot request; X-Cache: HIT\n--- http_config eval: $::HttpConfig\n--- config\nlocation /cache_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\n--- request\nGET /cache_prx\n--- response_headers_like\nX-Cache: HIT from .*\n--- response_body\nTEST 1\n--- no_error_log\n[error]\n\n\n=== TEST 3: No-cache request; X-Cache: MISS\n--- http_config eval: $::HttpConfig\n--- config\nlocation /cache_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /cache {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.say(\"TEST 3\")\n }\n}\n--- more_headers\nCache-Control: no-cache\n--- request\nGET /cache_prx\n--- response_headers_like\nX-Cache: MISS from .*\n--- response_body\nTEST 3\n--- no_error_log\n[error]\n\n\n=== TEST 3b: No-cache request with extension; X-Cache: MISS\n--- http_config eval: $::HttpConfig\n--- config\nlocation /cache_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /cache {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.say(\"TEST 3b\")\n }\n}\n--- more_headers\nCache-Control: no-cache, stale-if-error=1234\n--- request\nGET /cache_prx\n--- response_headers_like\nX-Cache: MISS from .*\n--- response_body\nTEST 3b\n--- no_error_log\n[error]\n\n\n=== TEST 3c: No-store request; X-Cache: MISS\n--- http_config eval: $::HttpConfig\n--- config\nlocation /cache_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /cache {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.say(\"TEST 3c\")\n }\n}\n--- more_headers\nCache-Control: no-store\n--- request\nGET /cache_prx\n--- response_headers_like\nX-Cache: MISS from .*\n--- response_body\nTEST 3c\n--- no_error_log\n[error]\n\n\n=== TEST 4a: PURGE\n--- http_config eval: $::HttpConfig\n--- config\nlocation /cache_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\n--- request\nPURGE /cache_prx\n--- error_code: 200\n--- no_error_log\n[error]\n\n\n=== TEST 4b: Cold request (expired but known); X-Cache: MISS\n--- http_config eval: $::HttpConfig\n--- config\nlocation /cache_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /cache {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.say(\"TEST 4\")\n }\n}\n--- request\nGET /cache_prx\n--- response_headers_like\nX-Cache: MISS from .*\n--- response_body\nTEST 4\n--- no_error_log\n[error]\n\n\n=== TEST 4c: Clean up\n--- http_config eval: $::HttpConfig\n--- config\nlocation /cache_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\n--- more_headers\nX-Purge: delete\n--- request\nPURGE /cache_prx\n--- error_code: 200\n--- no_error_log\n[error]\n\n\n=== TEST 6a: Prime a resource into cache\n--- http_config eval: $::HttpConfig\n--- config\nlocation /cache_6_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /cache_6 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.say(\"TEST 6\")\n }\n}\n--- request\nGET /cache_6_prx\n--- response_headers_like\nX-Cache: MISS from .*\n--- response_body\nTEST 6\n--- no_error_log\n[error]\n\n\n=== TEST 6b: Revalidate - now the response is a non-cacheable 404.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /cache_6_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /cache_6 {\n content_by_lua_block {\n ngx.status = 404\n ngx.header[\"Cache-Control\"] = \"no-cache\"\n ngx.say(\"TEST 6b\")\n }\n}\n--- more_headers\nCache-Control: no-cache\n--- request\nGET /cache_6_prx\n--- response_headers_like\nX-Cache:\n--- response_body\nTEST 6b\n--- error_code: 404\n--- no_error_log\n[error]\n\n\n=== TEST 6c: Confirm all keys have been removed\n--- http_config eval: $::HttpConfig\n--- config\nlocation /cache_6 {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n local redis = require(\"ledge\").create_redis_connection()\n handler.redis = redis\n local key_chain = handler:cache_key_chain()\n\n local res, err = redis:keys(key_chain.root .. \"*\")\n if res then\n ngx.say(\"Numkeys: \", #res)\n end\n }\n}\n--- request\nGET /cache_6\n--- response_body\nNumkeys: 0\n--- no_error_log\n[error]\n\n\n=== TEST 7: only-if-cached should return 504 on cache miss\n--- http_config eval: $::HttpConfig\n--- config\nlocation /cache_7_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /cache_7 {\n content_by_lua_block {\n ngx.say(\"TEST 7\")\n }\n}\n--- more_headers\nCache-Control: only-if-cached\n--- request\nGET /cache_7_prx\n--- error_code: 504\n--- no_error_log\n[error]\n\n\n=== TEST 8: min-fresh reduces calculated ttl\n--- http_config eval: $::HttpConfig\n--- config\nlocation /cache_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /cache {\n content_by_lua_block {\n ngx.say(\"TEST 8\")\n }\n}\n--- more_headers\nCache-Control: min-fresh=9999\n--- request\nGET /cache_prx\n--- response_body\nTEST 8\n--- no_error_log\n[error]\n\n\n=== TEST 9a: Prime a 404 response into cache; X-Cache: MISS\n--- http_config eval: $::HttpConfig\n--- config\nlocation /cache_9_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /cache_9 {\n content_by_lua_block {\n ngx.status = ngx.HTTP_NOT_FOUND\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.say(\"TEST 9\")\n }\n}\n--- request\nGET /cache_9_prx\n--- response_headers_like\nX-Cache: MISS from .*\n--- response_body\nTEST 9\n--- error_code: 404\n--- no_error_log\n[error]\n\n\n=== TEST 9b: Test we still have 404; X-Cache: HIT\n--- http_config eval: $::HttpConfig\n--- config\nlocation /cache_9_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\n--- request\nGET /cache_9_prx\n--- response_headers_like\nX-Cache: HIT from .*\n--- response_body\nTEST 9\n--- error_code: 404\n--- no_error_log\n[error]\n\n\n=== TEST 11: Prime with HEAD into cache (no body); X-Cache: MISS\n--- http_config eval: $::HttpConfig\n--- config\nlocation /cache_11_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /cache_11 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n }\n}\n--- more_headers\nCache-Control: no-cache\n--- request\nHEAD /cache_11_prx\n--- response_headers_like\nX-Cache: MISS from .*\n--- response_body\n--- error_code: 200\n--- no_error_log\n[error]\n\n\n=== TEST 11b: Check HEAD request did not cache\n--- http_config eval: $::HttpConfig\n--- config\nlocation /cache_11_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /cache_11 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n }\n}\n--- request\nHEAD /cache_11_prx\n--- response_headers_like\nX-Cache: MISS from .*\n--- response_body\n--- error_code: 200\n--- no_error_log\n[error]\n\n\n=== TEST 12: Prime 301 into cache with no body; X-Cache: MISS\n--- http_config eval: $::HttpConfig\n--- config\nlocation /cache_12_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /cache_12 {\n content_by_lua_block {\n ngx.status = 301\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.header[\"Location\"] = \"http://example.com\"\n }\n}\n--- request\nGET /cache_12_prx\n--- response_headers_like\nX-Cache: MISS from .*\n--- response_body\n--- error_code: 301\n--- no_error_log\n[error]\n\n\n=== TEST 12b: Check 301 request cached with no body\n--- http_config eval: $::HttpConfig\n--- config\nlocation /cache_12_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\n--- request\nGET /cache_12_prx\n--- response_headers_like\nX-Cache: HIT from .*\n--- response_body\n--- error_code: 301\n--- no_error_log\n[error]\n\n\n=== TEST 13: Subzero request; X-Cache: MISS\n--- http_config eval: $::HttpConfig\n--- config\nlocation /cache_13_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /cache_13 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.header[\"X-Custom-Hdr\"] = \"foo\"\n ngx.say(\"TEST 13\")\n }\n}\n--- request\nGET /cache_13_prx\n--- response_headers_like\nX-Cache: MISS from .*\n--- response_headers\nX-Custom-Hdr: foo\n--- response_body\nTEST 13\n--- no_error_log\n[error]\n\n\n=== TEST 13b: Forced cache update\n--- http_config eval: $::HttpConfig\n--- config\nlocation /cache_13_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /cache_13 {\n content_by_lua_block {\n -- Should override ALL headers from TEST 13\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.header[\"X-Custom-Hdr2\"] = \"bar\"\n ngx.say(\"TEST 13b\")\n }\n}\n--- request\nGET /cache_13_prx\n--- more_headers\nCache-Control: no-cache\n--- response_headers_like\nX-Cache: MISS from .*\n--- response_headers\nX-Custom-Hdr2: bar\n--- response_body\nTEST 13b\n--- no_error_log\n[error]\n\n\n=== TEST 13c: Cache hit - Headers are overriden not appended to\n--- http_config eval: $::HttpConfig\n--- config\nlocation /cache_13_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /cache_13 {\n content_by_lua_block {\n ngx.say(\"TEST 13b\")\n ngx.log(ngx.ERR, \"Never run\")\n }\n}\n--- request\nGET /cache_13_prx\n--- response_headers_like\nX-Cache: HIT from .*\n--- response_headers\nX-Custom-Hdr2: bar\n--- raw_response_headers_unlike: .*X-Custom-Hdr: foo.*\n--- no_error_log\n[error]\n--- response_body\nTEST 13b\n\n\n=== TEST 14: Cache-Control no-cache=#field and private=#field, drop headers from cache\n--- http_config eval: $::HttpConfig\n--- config\nlocation /cache_14_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /cache_14 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = {\n 'max-age=3600, private=\"XTest\"',\n 'no-cache=\"X-Test2\"'\n }\n ngx.header[\"XTest\"] = \"foo\"\n ngx.header[\"X-test2\"] = \"bar\"\n ngx.say(\"TEST 14\")\n }\n}\n--- request\nGET /cache_14_prx\n--- response_headers_like\nX-Cache: MISS from .*\n--- response_headers\nXTest: foo\nX-Test2: bar\n--- response_body\nTEST 14\n--- no_error_log\n[error]\n\n\n=== TEST 14b: Cache hit - Headers are not returned from cache\n--- http_config eval: $::HttpConfig\n--- config\nlocation /cache_14_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /cache_14 {\n content_by_lua_block {\n ngx.say(\"TEST 14b\")\n ngx.log(ngx.ERR, \"Never run\")\n }\n}\n--- request\nGET /cache_14_prx\n--- response_headers_like\nX-Cache: HIT from .*\n--- raw_response_headers_unlike: .*(XTest: foo|X-test2: bar).*\n--- no_error_log\n[error]\n--- response_body\nTEST 14\n\n\n=== TEST 15a: Prime a resource into cache\n--- http_config eval: $::HttpConfig\n--- config\nlocation /cache_15_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler({\n keep_cache_for = 1,\n }):run()\n }\n}\nlocation /cache_15 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=60\"\n ngx.header[\"Vary\"] = \"Foobar\"\n ngx.say(\"TEST 15\")\n }\n}\n--- request\nGET /cache_15_prx\n--- response_headers_like\nX-Cache: MISS from .*\n--- response_body\nTEST 15\n--- no_error_log\n[error]\n\n\n=== TEST 15b: Confim all keys exists\n--- http_config eval: $::HttpConfig\n--- config\nlocation /cache_15_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n local redis = require(\"ledge\").create_redis_connection()\n handler.redis = redis\n local key_chain = handler:cache_key_chain()\n\n local res, err = redis:keys(key_chain.root .. \"*\")\n if res then\n ngx.say(\"Numkeys: \", #res)\n end\n\n -- Sleep longer than keep_cache_for, to prove all keys have ttl assigned\n ngx.sleep(3)\n\n local res, err = redis:keys(key_chain.root .. \"*\")\n if res then\n ngx.say(\"Numkeys: \", #res)\n end\n }\n}\n--- request\nGET /cache_15_prx\n--- timeout: 5\n--- response_body\nNumkeys: 7\nNumkeys: 7\n--- no_error_log\n[error]\n\n\n=== TEST 16: Prime a resource into cache\n--- http_config eval: $::HttpConfig\n--- config\nlocation /cache_16_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /cache_16 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=60\"\n ngx.say(\"TEST 16\")\n }\n}\n--- request\nGET /cache_16_prx\n--- response_headers_like\nX-Cache: MISS from .*\n--- response_body\nTEST 16\n--- no_error_log\n[error]\n\n=== TEST 16b: Modified main key aborts transaction and cleans up entity\n--- http_config eval: $::HttpConfig\n--- config\nlocation /cache_16_check {\n content_by_lua_block {\n local entity_id = ngx.shared.ledge_test:get(\"entity_id\")\n local redis = require(\"ledge\").create_storage_connection()\n local ok, err = redis:exists(entity_id)\n ngx.print(ok, \" \", err)\n }\n}\nlocation /cache_16_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n handler:bind(\"before_serve\", function(res)\n -- Create a new connection\n local redis = require(\"ledge\").create_redis_connection()\n -- Set a new key on the main key\n redis:hset(handler:cache_key_chain().main, \"foo\", \"bar\")\n\n ngx.shared.ledge_test:set(\"entity_id\", res.entity_id)\n end)\n\n handler:run()\n }\n}\nlocation /cache_16 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=60\"\n ngx.print(\"TEST 16b\")\n }\n}\n--- request eval\n[\"GET /cache_16_prx\", \"GET /cache_16_check\"]\n--- more_headers\nCache-Control: no-cache\n--- response_headers_like eval\n[\"X-Cache: MISS from .*\", \"\"]\n--- response_body eval\n[\"TEST 16b\", \"false nil\"]\n--- wait: 3\n--- no_error_log\n[error]\n\n=== TEST 16c: Modified main key aborts transaction - HIT\n--- http_config eval: $::HttpConfig\n--- config\nlocation /cache_16_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /cache_16 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=60\"\n ngx.say(\"TEST 16b\")\n }\n}\n--- request\nGET /cache_16_prx\n--- response_headers_like\nX-Cache: HIT from .*\n--- response_body\nTEST 16\n--- no_error_log\n[error]\n\n\n=== TEST 16d: Partial entry misses\n--- http_config eval: $::HttpConfig\n--- config\nlocation /cache_16_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n local redis = require(\"ledge\").create_redis_connection()\n handler.redis = redis\n local key_chain = handler:cache_key_chain()\n\n -- Break entities\n redis:del(handler:cache_key_chain().entities)\n\n handler:run()\n }\n}\nlocation /cache_16 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=60\"\n ngx.say(\"TEST 16d\")\n }\n}\n--- request\nGET /cache_16_prx\n--- response_headers_like\nX-Cache: MISS from .*\n--- response_body\nTEST 16d\n--- no_error_log\n[error]\n\n\n=== TEST 17: Main key is completely overriden\n--- http_config eval: $::HttpConfig\n--- config\nlocation /cache_17_modify {\n rewrite ^(.*)_modify$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n local redis = require(\"ledge\").create_redis_connection()\n handler.redis = redis\n local key = handler:cache_key_chain().main\n\n -- Add new field to main key\n redis:hset(key, \"bogus_field\", \"foobar\")\n\n -- Print result from redis\n local main, err = redis:hgetall(key)\n main = redis:array_to_hash(main)\n ngx.print(key, \" bogus_field: \", main[\"bogus_field\"])\n\n }\n}\nlocation /cache_17_check {\n rewrite ^(.*)_check$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n local redis = require(\"ledge\").create_redis_connection()\n handler.redis = redis\n local key = handler:cache_key_chain().main\n\n -- Print result from redis\n local main, err = redis:hgetall(key)\n main = redis:array_to_hash(main)\n ngx.print(key, \" bogus_field: \", main[\"bogus_field\"])\n }\n}\nlocation /cache_17_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /cache_17 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=60\"\n ngx.print(\"TEST 17\")\n }\n}\n--- request eval\n[\n\"GET /cache_17_prx\",\n\"GET /cache_17_modify\",\n\"GET /cache_17_prx\",\n\"GET /cache_17_check\",\n]\n--- more_headers eval\n[\n\"\",\n\"\",\n\"Cache-Control: no-cache\",\n\"\",\n]\n--- response_body eval\n[\n\"TEST 17\",\n\"ledge:cache:http:localhost:/cache_17:#::main bogus_field: foobar\",\n\"TEST 17\",\n\"ledge:cache:http:localhost:/cache_17:#::main bogus_field: nil\",\n]\n\n--- no_error_log\n[error]\n" }, { "path": "t/02-integration/collapsed_forwarding.t", "content": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::http_config(extra_nginx_config => qq{\n lua_shared_dict test 1m;\n lua_check_client_abort on;\n}, run_worker => 1);\n\nno_long_string();\nno_diff();\nrun_tests();\n\n__DATA__\n=== TEST 1a: Prime cache (collapsed forwardind requires having seen a previously cacheable response)\n--- http_config eval: $::HttpConfig\n--- config\nlocation /collapsed_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /collapsed {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.say(\"OK\")\n }\n}\n--- request\nGET /collapsed_prx\n--- repsonse_body\nOK\n--- no_error_log\n[error]\n\n\n=== TEST 1b: Purge cache\n--- http_config eval: $::HttpConfig\n--- config\nlocation /collapsed {\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\n--- request\nPURGE /collapsed\n--- error_code: 200\n--- no_error_log\n[error]\n\n\n=== TEST 2: Concurrent COLD requests accepting cache\n--- http_config eval: $::HttpConfig\n--- config\nlocation /concurrent_collapsed {\n rewrite_by_lua_block {\n ngx.shared.test:set(\"test_2\", 0)\n }\n\n echo_location_async \"/collapsed_prx\";\n echo_sleep 0.05;\n echo_location_async \"/collapsed_prx\";\n}\nlocation /collapsed_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler({\n enable_collapsed_forwarding = true,\n }):run()\n }\n}\nlocation /collapsed {\n content_by_lua_block {\n ngx.sleep(0.1)\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.say(\"OK \" .. ngx.shared.test:incr(\"test_2\", 1))\n }\n}\n--- request\nGET /concurrent_collapsed\n--- error_code: 200\n--- response_body\nOK 1\nOK 1\n\n\n=== TEST 3a: Purge cache\n--- http_config eval: $::HttpConfig\n--- config\nlocation /collapsed {\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\n--- request\nPURGE /collapsed\n--- error_code: 200\n--- no_error_log\n[error]\n\n\n=== TEST 3b: Concurrent COLD requests with collapsing turned off\n--- http_config eval: $::HttpConfig\n--- config\nlocation /concurrent_collapsed {\n rewrite_by_lua_block {\n ngx.shared.test:set(\"test_3\", 0)\n }\n\n echo_location_async '/collapsed_prx';\n echo_sleep 0.05;\n echo_location_async '/collapsed_prx';\n}\nlocation /collapsed_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler({\n enable_collapsed_forwarding = false,\n }):run()\n }\n}\nlocation /collapsed {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.say(\"OK \" .. ngx.shared.test:incr(\"test_3\", 1))\n ngx.sleep(0.1)\n }\n}\n--- request\nGET /concurrent_collapsed\n--- error_code: 200\n--- response_body\nOK 1\nOK 2\n\n\n=== TEST 4a: Purge cache\n--- http_config eval: $::HttpConfig\n--- config\nlocation /collapsed {\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\n--- request\nPURGE /collapsed\n--- error_code: 200\n--- no_error_log\n[error]\n\n\n=== TEST 4b: Concurrent COLD requests not accepting cache\n--- http_config eval: $::HttpConfig\n--- config\nlocation /concurrent_collapsed {\n rewrite_by_lua_block {\n ngx.shared.test:set(\"test_4\", 0)\n }\n\n echo_location_async '/collapsed_prx';\n echo_sleep 0.05;\n echo_location_async '/collapsed_prx';\n}\nlocation /collapsed_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler({\n enable_collapsed_forwarding = true,\n }):run()\n }\n}\nlocation /collapsed {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.say(\"OK \" .. ngx.shared.test:incr(\"test_4\", 1))\n ngx.sleep(0.1)\n }\n}\n--- more_headers\nCache-Control: no-cache\n--- request\nGET /concurrent_collapsed\n--- error_code: 200\n--- response_body\nOK 1\nOK 2\n\n\n=== TEST 5a: Purge cache\n--- http_config eval: $::HttpConfig\n--- config\nlocation /collapsed {\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\n--- request\nPURGE /collapsed\n--- error_code: 200\n--- no_error_log\n[error]\n\n\n=== TEST 5b: Concurrent COLD requests, response no longer cacheable\n--- http_config eval: $::HttpConfig\n--- config\nlocation /concurrent_collapsed {\n rewrite_by_lua_block {\n ngx.shared.test:set(\"test_5\", 0)\n }\n\n echo_location_async '/collapsed_prx';\n echo_sleep 0.05;\n echo_location_async '/collapsed_prx';\n}\nlocation /collpased_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler({\n enable_collapsed_forwarding = true,\n }):run()\n }\n}\nlocation /collapsed {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"no-cache\"\n ngx.say(\"OK \" .. ngx.shared.test:incr(\"test_5\", 1))\n ngx.sleep(0.1)\n }\n}\n--- request\nGET /concurrent_collapsed\n--- error_code: 200\n--- response_body\nOK 1\nOK 2\n\n\n=== TEST 6: Concurrent SUBZERO requests\n--- http_config eval: $::HttpConfig\n--- config\nlocation /concurrent_collapsed_6 {\n rewrite_by_lua_block {\n ngx.shared.test:set(\"test_6\", 0)\n }\n\n echo_location_async '/collapsed_6_prx';\n echo_sleep 0.05;\n echo_location_async '/collapsed_6_prx';\n}\nlocation /collapsed_6_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler({\n enable_collapsed_forwarding = true,\n }):run()\n }\n}\nlocation /collapsed_6 {\n content_by_lua_block {\n ngx.sleep(0.1)\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.say(\"OK \" .. ngx.shared.test:incr(\"test_6\", 1))\n }\n}\n--- request\nGET /concurrent_collapsed_6\n--- error_code: 200\n--- response_body\nOK 1\nOK 2\n\n\n=== TEST 7a: Prime cache\n--- http_config eval: $::HttpConfig\n--- config\nlocation /collapsed_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /collapsed {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.header[\"Etag\"] = \"test7a\"\n ngx.say(\"OK\")\n }\n}\n--- request\nGET /collapsed_prx\n--- repsonse_body\nOK\n--- no_error_log\n[error]\n\n\n=== TEST 7b: Concurrent conditional requests which accept cache\n (i.e. does this work with revalidation)\n--- http_config eval: $::HttpConfig\n--- config\nlocation /concurrent_collapsed {\n rewrite_by_lua_block {\n ngx.shared.test:set(\"test_7\", 0)\n }\n\n echo_location_async '/collapsed_prx';\n echo_sleep 0.05;\n echo_location_async '/collapsed_prx';\n echo_sleep 1;\n}\nlocation /collapsed_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler({\n enable_collapsed_forwarding = true,\n }):run()\n }\n}\nlocation /collapsed {\n content_by_lua_block {\n ngx.sleep(0.1)\n ngx.header[\"Etag\"] = \"test7b\"\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.say(\"OK \" .. ngx.shared.test:incr(\"test_7\", 1))\n }\n}\n--- more_headers\nCache-Control: max-age=0\nIf-None-Match: test7b\n--- request\nGET /concurrent_collapsed\n--- error_code: 200\n--- response_body\n\n=== TEST 8a: Prime cache (collapsed forwardind requires having seen a previously cacheable response)\n--- http_config eval: $::HttpConfig\n--- config\nlocation /collapsed8_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /collapsed8 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.say(\"OK\")\n }\n}\n--- request eval\n[\"GET /collapsed8_prx\", \"PURGE /collapsed8_prx\"]\n--- no_error_log\n[error]\n\n\n=== TEST 8b: Collapse window timed out\n--- http_config eval: $::HttpConfig\n--- config\nlocation /concurrent_collapsed {\n rewrite_by_lua_block {\n ngx.shared.test:set(\"test_8\", 0)\n }\n\n echo_location_async \"/collapsed8_prx\";\n echo_sleep 0.05;\n echo_location_async \"/collapsed8_prx\";\n}\nlocation /collapsed8_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler({\n enable_collapsed_forwarding = true,\n collapsed_forwarding_window = 500, -- (ms)\n }):run()\n }\n}\nlocation /collapsed8 {\n content_by_lua_block {\n ngx.sleep(0.8)\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.say(\"OK \" .. ngx.shared.test:incr(\"test_8\", 1))\n }\n}\n--- request\nGET /concurrent_collapsed\n--- error_code: 200\n--- response_body\nOK 1\nOK 2\n\n=== TEST 9: Collapsing with vary\n--- http_config eval: $::HttpConfig\n--- config\nlocation /prime {\n rewrite ^ /collapsed9 break;\n content_by_lua_block {\n require(\"ledge.state_machine\").set_debug(false)\n require(\"ledge\").create_handler({\n enable_collapsed_forwarding = true,\n }):run()\n }\n}\nlocation /concurrent_collapsed {\n rewrite_by_lua_block {\n ngx.shared.test:set(\"test_9\", 0)\n }\n\n echo_location_async \"/collapsed9_prx\";\n echo_sleep 0.05;\n echo_location_async \"/collapsed9_prx\";\n}\nlocation /collapsed9_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge.state_machine\").set_debug(true)\n require(\"ledge\").create_handler({\n enable_collapsed_forwarding = true,\n }):run()\n }\n}\nlocation /collapsed {\n content_by_lua_block {\n ngx.sleep(0.1)\n local counter = ngx.shared.test:incr(\"test_9\", 1)\n ngx.header[\"Vary\"] = \"X-Test\"\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.print(\"OK \" .. tostring(counter))\n }\n}\n--- request eval\n[\n\"GET /prime\", \"PURGE /prime\",\n\"GET /concurrent_collapsed\"\n]\n--- error_code eval\n[200, 200, 200]\n--- response_body_like eval\n[\n\"OK nil\", \".+\",\n\"OK 1OK 1\"\n]\n\n=== TEST 10: Collapsing with vary - change in spec\n--- http_config eval: $::HttpConfig\n--- config\nlocation /prime {\n rewrite ^ /collapsed10 break;\n content_by_lua_block {\n require(\"ledge.state_machine\").set_debug(false)\n require(\"ledge\").create_handler({\n enable_collapsed_forwarding = false,\n }):run()\n }\n}\nlocation /concurrent_collapsed {\n echo_location_async \"/collapsed10_prx\";\n echo_sleep 0.05;\n echo_location_async \"/collapsed10_prx\";\n}\nlocation /collapsed10_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge.state_machine\").set_debug(true)\n require(\"ledge\").create_handler({\n enable_collapsed_forwarding = true,\n }):run()\n }\n}\nlocation /collapsed {\n content_by_lua_block {\n ngx.sleep(0.1)\n local counter = ngx.shared.test:incr(\"test_10\", 1, 0)\n if counter == 1 then\n ngx.header[\"Vary\"] = \"X-Test\" -- Prime with this\n else\n ngx.header[\"Vary\"] = \"X-Test2\"\n end\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.print(\"OK \" .. tostring(counter))\n }\n}\n--- request eval\n[\n\"GET /prime\", \"PURGE /prime\",\n\"GET /concurrent_collapsed\"\n]\n--- more_headers eval\n[\n\"X-Test: Foo\",\"X-Test: Foo\",\n\"X-Test: Foo\",\n]\n--- error_code eval\n[200, 200, 200]\n--- response_body_like eval\n[\n\"OK 1\", \".+\",\n\"OK 2OK 2\"\n]\n\n=== TEST 11: Collapsing with vary - change in spec mismatch\n--- http_config eval: $::HttpConfig\n--- config\nlocation /prime {\n rewrite ^ /collapsed11 break;\n content_by_lua_block {\n require(\"ledge.state_machine\").set_debug(false)\n require(\"ledge\").create_handler({\n enable_collapsed_forwarding = false,\n }):run()\n }\n}\nlocation /concurrent_collapsed {\n echo_subrequest_async GET \"/collapsed11a_prx\"; # X-Test: Foo\n echo_sleep 0.05;\n echo_subrequest_async GET \"/collapsed11b_prx\"; # X-Test: Foo, X-Test2: Bar\n}\nlocation /collapsed11a_prx {\n rewrite ^(.*)a_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge.state_machine\").set_debug(true)\n require(\"ledge\").create_handler({\n enable_collapsed_forwarding = true,\n }):run()\n }\n}\nlocation /collapsed11b_prx {\n rewrite ^(.*)b_prx$ $1 break;\n content_by_lua_block {\n ngx.req.set_header(\"X-Test2\", \"Bar\")\n require(\"ledge.state_machine\").set_debug(true)\n require(\"ledge\").create_handler({\n enable_collapsed_forwarding = true,\n }):run()\n }\n}\nlocation /collapsed {\n content_by_lua_block {\n ngx.sleep(0.1)\n local counter = ngx.shared.test:incr(\"test_11\", 1, 0)\n if counter == 1 then\n ngx.header[\"Vary\"] = \"X-Test\" -- Prime with this\n else\n ngx.header[\"Vary\"] = \"X-Test2\"\n end\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.print(\"OK \" .. tostring(counter))\n }\n}\n--- request eval\n[\n\"GET /prime\", \"PURGE /prime\",\n\"GET /concurrent_collapsed\"\n]\n--- more_headers eval\n[\n\"X-Test: Foo\",\"X-Test: Foo\",\n\"X-Test: Foo\",\n]\n--- error_code eval\n[200, 200, 200]\n--- response_body_like eval\n[\n\"OK 1\", \".+\",\n\"OK 2OK 3\"\n]\n\n" }, { "path": "t/02-integration/esi.t", "content": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::http_config(extra_nginx_config => qq{\n lua_shared_dict test 1m;\n lua_check_client_abort on;\n if_modified_since off;\n}, extra_lua_config => qq{\n require(\"ledge\").set_handler_defaults({\n esi_enabled = true,\n buffer_size = 5, -- Try to trip scanning up with small buffers\n })\n\n -- Make all content return valid Surrogate-Control headers\n function run(handler)\n if not handler then\n handler = require(\"ledge\").create_handler()\n end\n handler:bind(\"after_upstream_request\", function(res)\n res.header[\"Surrogate-Control\"] = [[content=\"ESI/1.0\"]]\n end)\n handler:run()\n end\n}, run_worker => 1);\n\nno_long_string();\nno_diff();\nrun_tests();\n\n__DATA__\n=== TEST 0: ESI works on slow and fast paths\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_0_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge.state_machine\").set_debug(true)\n run()\n }\n}\nlocation /esi_0 {\n default_type text/html;\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=60\"\n ngx.print(\"Hello\")\n }\n}\n--- request eval\n[\n \"GET /esi_0_prx\",\n \"GET /esi_0_prx\",\n]\n--- response_body eval\n[\n \"Hello\",\n \"Hello\",\n]\n--- response_headers_like eval\n[\n \"X-Cache: MISS from .*\",\n \"X-Cache: HIT from .*\",\n]\n--- no_error_log\n[error]\n\n\n=== TEST 1: Single line comments removed\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_1_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n --ledge:config_set(\"buffer_size\", 10)\n run()\n }\n}\nlocation /esi_1 {\n default_type text/html;\n content_by_lua_block {\n ngx.say(\"\")\n ngx.say(\"\")\n }\n}\n--- request\nGET /esi_1_prx\n--- response_body\nCOMMENTED\nCOMMENTED\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- no_error_log\n[error]\n\n\n=== TEST 1b: Single line comments removed, esi instructions processed\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_1b_prx {\n rewrite ^(.*)_prx$ $1b break;\n content_by_lua_block {\n run()\n }\n}\nlocation /esi_1b {\n default_type text/html;\n content_by_lua_block {\n ngx.print(\"\")\n }\n}\n--- request\nGET /esi_1b_prx?a=1b\n--- response_body: a=1b\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- no_error_log\n[error]\n\n\n=== TEST 2: Multi line comments removed\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_2_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n run()\n }\n}\nlocation /esi_2 {\n default_type text/html;\n content_by_lua_block {\n ngx.print(\"\")\n ngx.say(\"2\")\n ngx.say(\"\")\n }\n}\n--- request\nGET /esi_2_prx\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body\n1\n2\n\n3\n--- no_error_log\n[error]\n\n\n=== TEST 2b: Multi line comments removed, ESI instructions processed\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_2_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n run()\n }\n}\nlocation /esi_2 {\n default_type text/html;\n content_by_lua_block {\n ngx.print(\"\")\n ngx.say(\"2345\")\n ngx.say(\"\")\n }\n}\nlocation /test {\n content_by_lua_block {\n ngx.print(\"OK\")\n }\n}\n--- request\nGET /esi_2_prx?a=1\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body\n1234 OK\n2345\n\na=1\n--- no_error_log\n[error]\n\n\n=== TEST 2c: Multi line escaping comments, nested.\n ESI instructions still processed\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_2c_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n run()\n }\n}\nlocation /esi_2c {\n default_type text/html;\n content_by_lua_block {\n ngx.say(\"BEFORE\")\n ngx.print(\"\")\n ngx.say(\"MIDDLE\")\n ngx.say(\"$(QUERY_STRING{c})\")\n ngx.print(\"-->\")\n ngx.say(\"AFTER\")\n }\n}\n--- request\nGET /esi_2c_prx?a=1&b=2&c=3\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body\nBEFORE\n1\n2\nMIDDLE\n3\nAFTER\n--- no_error_log\n[error]\n\n\n=== TEST 3: Single line removed.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_3_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n run()\n }\n}\nlocation /esi_3 {\n default_type text/html;\n content_by_lua_block {\n ngx.say(\"START\")\n ngx.say(\"REMOVED\")\n ngx.say(\"REMOVED\")\n ngx.say(\"END\")\n }\n}\n--- request\nGET /esi_3_prx\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body\nSTART\n\n\nEND\n--- no_error_log\n[error]\n\n\n=== TEST 4: Multi line removed.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_4_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n run()\n }\n}\nlocation /esi_4 {\n default_type text/html;\n content_by_lua_block {\n ngx.say(\"1\")\n ngx.say(\"\")\n ngx.say(\"2\")\n ngx.say(\"\")\n ngx.say(\"3\")\n ngx.say(\"4\")\n ngx.say(\"\")\n ngx.say(\"5\")\n ngx.say(\"\")\n ngx.say(\"6\")\n }\n}\n--- request\nGET /esi_4_prx\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body\n1\n\n3\n4\n\n6\n--- no_error_log\n[error]\n\n\n=== TEST 5: Include fragment\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_5_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n run()\n }\n}\nlocation /fragment_1 {\n content_by_lua_block {\n ngx.say(\"FRAGMENT: \", ngx.req.get_uri_args()[\"a\"] or \"\")\n }\n}\nlocation /esi_5 {\n default_type text/html;\n content_by_lua_block {\n ngx.say(\"1\")\n ngx.print([[]])\n ngx.say(\"2\")\n ngx.print([[]])\n ngx.print(\"3\")\n ngx.print([[]])\n }\n}\n--- request\nGET /esi_5_prx\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body\n1\nFRAGMENT: \n2\nFRAGMENT: 2\n3FRAGMENT: 3\n--- no_error_log\n[error]\n\n\n=== TEST 5b: Test fragment always issues GET and only inherits correct\n req headers\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_5b_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n run()\n }\n}\nlocation /fragment_1 {\n content_by_lua_block {\n ngx.say(\"method: \", ngx.req.get_method())\n local h = ngx.req.get_headers()\n\n local h_keys = {}\n for k,v in pairs(h) do\n table.insert(h_keys, k)\n end\n table.sort(h_keys)\n\n for _,k in ipairs(h_keys) do\n ngx.say(k, \": \", h[k])\n end\n }\n}\nlocation /esi_5b {\n default_type text/html;\n content_by_lua_block {\n ngx.print([[]])\n }\n}\n--- request\nPOST /esi_5b_prx\n--- more_headers\nCache-Control: no-cache\nCookie: foo\nAuthorization: bar\nRange: bytes=0-\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body_like\nmethod: GET\nauthorization: bar\ncache-control: no-cache\ncookie: foo\nhost: localhost\nuser-agent: lua-resty-http/\\d+\\.\\d+ \\(Lua\\) ngx_lua/\\d+ ledge_esi/\\d+\\.\\d+[\\.\\d]*\nx-esi-parent-uri: http://localhost/esi_5b_prx\nx-esi-recursion-level: 1\n--- no_error_log\n[error]\n\n\n=== TEST 5c: Include fragment with absolute URI, schemaless, and no path\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_5_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n run()\n }\n}\nlocation /fragment_1 {\n echo \"FRAGMENT\";\n}\nlocation =/ {\n echo \"ROOT FRAGMENT\";\n}\nlocation /esi_5 {\n default_type text/html;\n content_by_lua_block {\n ngx.print([[]])\n ngx.print([[]])\n ngx.print([[]])\n ngx.print([[]])\n ngx.print([[]])\n }\n}\n--- request\nGET /esi_5_prx\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body\nFRAGMENT\nFRAGMENT\nROOT FRAGMENT\nROOT FRAGMENT\nROOT FRAGMENT\n--- no_error_log\n[error]\n\n\n=== TEST 6: Include multiple fragments, in correct order.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_6_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n run()\n }\n}\nlocation /fragment_1 {\n content_by_lua_block {\n ngx.print(\"FRAGMENT_1\")\n }\n}\nlocation /fragment_2 {\n content_by_lua_block {\n ngx.print(\"FRAGMENT_2\")\n }\n}\nlocation /fragment_3 {\n content_by_lua_block {\n ngx.print(\"FRAGMENT_3\")\n }\n}\nlocation /esi_6 {\n default_type text/html;\n content_by_lua_block {\n ngx.say([[]])\n ngx.say([[MID LINE ]])\n ngx.say([[]])\n }\n}\n--- request\nGET /esi_6_prx\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body\nFRAGMENT_3\nMID LINE FRAGMENT_1\nFRAGMENT_2\n--- no_error_log\n[error]\n\n\n=== TEST 7: Leave instructions intact if ESI is not enabled.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_7_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler({\n esi_enabled = false,\n })\n run(handler)\n }\n}\nlocation /esi_7 {\n default_type text/html;\n content_by_lua_block {\n ngx.print(\"$(QUERY_STRING)\")\n }\n}\n--- request\nGET /esi_7_prx?a=1\n--- response_body: $(QUERY_STRING)\n--- no_error_log\n[error]\n\n\n=== TEST 7b: Leave instructions intact if ESI delegation is enabled - slow path.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_7b_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler({\n esi_allow_surrogate_delegation = true,\n })\n run(handler)\n }\n}\nlocation /esi_7b {\n default_type text/html;\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.print(\"$(QUERY_STRING)\")\n }\n}\n--- request\nGET /esi_7b_prx?a=1\n--- more_headers\nSurrogate-Capability: localhost=\"ESI/1.0\"\n--- response_body: $(QUERY_STRING)\n--- response_headers\nSurrogate-Control: content=\"ESI/1.0\"\n--- no_error_log\n[error]\n\n\n=== TEST 7c: Leave instructions intact if ESI delegation is enabled - fast path.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_7b_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler({\n esi_allow_surrogate_delegation = true,\n })\n run(handler)\n }\n}\n--- request\nGET /esi_7b_prx?a=1\n--- more_headers\nSurrogate-Capability: localhost=\"ESI/1.0\"\n--- response_body: $(QUERY_STRING)\n--- response_headers\nSurrogate-Control: content=\"ESI/1.0\"\n--- no_error_log\n[error]\n\n\n=== TEST 7d: Leave instructions intact if ESI delegation is enabled by IP\n on the slow path.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_7d_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler({\n esi_allow_surrogate_delegation = { \"127.0.0.1\" },\n })\n run(handler)\n }\n}\nlocation /esi_7d {\n default_type text/html;\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.print(\"$(QUERY_STRING)\")\n }\n}\n--- request\nGET /esi_7d_prx?a=1\n--- more_headers\nSurrogate-Capability: localhost=\"ESI/1.0\"\n--- response_body: $(QUERY_STRING)\n--- response_headers\nSurrogate-Control: content=\"ESI/1.0\"\n--- no_error_log\n[error]\n\n\n=== TEST 7e: Leave instructions intact if ESI delegation is enabled by IP\n on the fast path.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_7d_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler({\n esi_allow_surrogate_delegation = { \"127.0.0.1\" },\n })\n run(handler)\n }\n}\n--- request\nGET /esi_7d_prx?a=1\n--- more_headers\nSurrogate-Capability: localhost=\"ESI/1.0\"\n--- response_body: $(QUERY_STRING)\n--- response_headers\nSurrogate-Control: content=\"ESI/1.0\"\n--- no_error_log\n[error]\n\n\n=== TEST 7f: Leave instructions intact if allowed types does not match\n on the slow path\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_7f_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler({\n esi_content_types = { \"text/plain\" },\n })\n run(handler)\n }\n}\nlocation /esi_7f {\n default_type text/html;\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.print(\"$(QUERY_STRING)\")\n }\n}\n--- request\nGET /esi_7f_prx?a=1\n--- response_body: $(QUERY_STRING)\n--- response_headers\nSurrogate-Control: content=\"ESI/1.0\"\n--- no_error_log\n[error]\n\n\n=== TEST 7g: Leave instructions intact if allowed types does not match (fast path)\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_7f_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler({\n esi_content_types = { \"text/plain\" },\n })\n run(handler)\n }\n}\n--- request\nGET /esi_7f_prx?a=1\n--- more_headers\nSurrogate-Capability: localhost=\"ESI/1.0\"\n--- response_body: $(QUERY_STRING)\n--- response_headers\nSurrogate-Control: content=\"ESI/1.0\"\n--- no_error_log\n[error]\n\n\n=== TEST 8: Response downstrean cacheability is zeroed when ESI processing \n has occured.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_8_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n run()\n }\n}\nlocation /fragment_1 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=60\"\n ngx.say(\"FRAGMENT_1\")\n }\n}\nlocation /esi_8 {\n default_type text/html;\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.say([[\"]])\n }\n}\n--- request\nGET /esi_8_prx\n--- response_headers_like\nCache-Control: private, max-age=0\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- no_error_log\n[error]\n\n\n=== TEST 9: Variable evaluation\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_9_prx {\n rewrite ^(.*)_prx(.*)$ $1 break;\n content_by_lua_block {\n run()\n }\n}\nlocation /esi_9 {\n default_type text/html;\n content_by_lua_block {\n ngx.say(\"HTTP_COOKIE: $(HTTP_COOKIE)\");\n ngx.say(\"HTTP_COOKIE{SQ_SYSTEM_SESSION}: $(HTTP_COOKIE{SQ_SYSTEM_SESSION})\");\n ngx.say(\"\");\n ngx.say(\"HTTP_COOKIE: $(HTTP_COOKIE)\");\n ngx.say(\"HTTP_COOKIE{SQ_SYSTEM_SESSION}: $(HTTP_COOKIE{SQ_SYSTEM_SESSION})\");\n ngx.say(\"HTTP_COOKIE{SQ_SYSTEM_SESSION_TYPO}: $(HTTP_COOKIE{SQ_SYSTEM_SESSION_TYPO}|'default message')\");\n ngx.say(\"\");\n ngx.say(\"$(HTTP_COOKIE{SQ_SYSTEM_SESSION})$(HTTP_COOKIE)$(QUERY_STRING)\")\n ngx.say(\"$(HTTP_X_MANY_HEADERS): $(HTTP_X_MANY_HEADERS)\")\n ngx.say(\"$(HTTP_X_MANY_HEADERS{2}): $(HTTP_X_MANY_HEADERS{2})\")\n }\n}\n--- more_headers\nCookie: myvar=foo; SQ_SYSTEM_SESSION=hello\nX-Many-Headers: 1\nX-Many-Headers: 2\nX-Many-Headers: 3, 4, 5, 6=hello\n--- request\nGET /esi_9_prx?t=1\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body\nHTTP_COOKIE: myvar=foo; SQ_SYSTEM_SESSION=hello\nHTTP_COOKIE{SQ_SYSTEM_SESSION}: hello\n\nHTTP_COOKIE: myvar=foo; SQ_SYSTEM_SESSION=hello\nHTTP_COOKIE{SQ_SYSTEM_SESSION}: hello\nHTTP_COOKIE{SQ_SYSTEM_SESSION_TYPO}: default message\n\nhello$(HTTP_COOKIE)t=1\n$(HTTP_X_MANY_HEADERS): 1, 2, 3, 4, 5, 6=hello\n$(HTTP_X_MANY_HEADERS{2}): 3, 4, 5, 6=hello\n--- no_error_log\n[error]\n\n\n=== TEST 9b: Multiple Variable evaluation\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_9b_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n run()\n }\n}\nlocation /esi_9b {\n default_type text/html;\n content_by_lua_block {\n ngx.say([[ ]])\n }\n}\nlocation /fragment1b {\n content_by_lua_block {\n ngx.print(\"FRAGMENT:\"..ngx.var.args)\n }\n}\n--- request\nGET /esi_9b_prx?t=1\n--- more_headers\nX-ESI-Test: foobar\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body\nFRAGMENT:t=1&test=foobar \n--- no_error_log\n[error]\n\n\n=== TEST 9c: Dictionary variable syntax (cookie)\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_9c_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n run()\n }\n}\nlocation /esi_9c {\n default_type text/html;\n content_by_lua_block {\n ngx.say([[]])\n }\n}\nlocation /fragment1c {\n content_by_lua_block {\n ngx.print(\"FRAGMENT:\"..ngx.var.args)\n }\n}\n--- request\nGET /esi_9c_prx?t=1\n--- more_headers\nCookie: foo=bar\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body\nFRAGMENT:1&test=bar\n--- no_error_log\n[error]\n\n\n=== TEST 9d: List variable syntax (accept-language)\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_9d_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n run()\n }\n}\nlocation /esi_9d {\n default_type text/html;\n content_by_lua_block {\n ngx.say([[]])\n }\n}\nlocation /fragment1d {\n content_by_lua_block {\n ngx.print(\"FRAGMENT:\"..ngx.var.args)\n }\n}\n--- request\nGET /esi_9d_prx?t=1\n--- more_headers\nAccept-Language: da, en-gb, fr\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body\nFRAGMENT:1&en-gb=true&de=false\n--- no_error_log\n[error]\n\n\n=== TEST 9e: List variable syntax (accept-language) with multiple headers\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_9e_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n run()\n }\n}\nlocation /esi_9e {\n default_type text/html;\n content_by_lua_block {\n ngx.say([[]])\n }\n}\nlocation /fragment1d {\n content_by_lua_block {\n ngx.print(\"FRAGMENT:\"..ngx.var.args)\n }\n}\n--- request\nGET /esi_9e_prx?t=1\n--- more_headers\nAccept-Language: da, en-gb\nAccept-Language: fr\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body\nFRAGMENT:1&en-gb=true&de=false\n--- no_error_log\n[error]\n\n\n=== TEST 9e: Default variable values\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_9e_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n run()\n }\n}\nlocation /esi_9e {\n default_type text/html;\n content_by_lua_block {\n ngx.print(\"\")\n ngx.say(\"$(QUERY_STRING{a}|novalue)\")\n ngx.say(\"$(QUERY_STRING{b}|novalue)\")\n ngx.say(\"$(QUERY_STRING{c}|\\'quoted values can have spaces\\')\")\n ngx.say(\"$(QUERY_STRING{d}|unquoted values must not have spaces)\")\n ngx.print(\"\")\n }\n}\n--- request\nGET /esi_9e_prx?a=1\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body\n1\nnovalue\nquoted values can have spaces\n$(QUERY_STRING{d}|unquoted values must not have spaces)\n--- no_error_log\n[error]\n\n\n=== TEST 9f: Custom variable injection\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_9f_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler({\n esi_custom_variables = {\n [\"CUSTOM_DICTIONARY\"] = { a = 1, b = 2},\n [\"CUSTOM_STRING\"] = \"foo\"\n },\n })\n\n run(handler)\n }\n}\nlocation /esi_9f {\n default_type text/html;\n content_by_lua_block {\n ngx.print(\"\")\n ngx.say(\"$(CUSTOM_DICTIONARY|novalue)\")\n ngx.say(\"$(CUSTOM_DICTIONARY{a})\")\n ngx.say(\"$(CUSTOM_DICTIONARY{b})\")\n ngx.say(\"$(CUSTOM_DICTIONARY{c}|novalue)\")\n ngx.say(\"$(CUSTOM_STRING)\")\n ngx.say(\"$(CUSTOM_STRING{x}|novalue)\")\n ngx.print(\"\")\n }\n}\n--- request\nGET /esi_9f_prx?a=1\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body\nnovalue\n1\n2\nnovalue\nfoo\nnovalue\n--- no_error_log\n[error]\n\n\n=== TEST 10: Prime ESI in cache.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_10_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler({\n esi_enabled = true,\n cache_key_spec = {\n \"scheme\",\n \"host\",\n \"uri\",\n }\n })\n run(handler)\n }\n}\nlocation /esi_10 {\n default_type text/html;\n content_by_lua_block {\n ngx.status = 404\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.header[\"Etag\"] = \"esi10\"\n ngx.say(\"$(QUERY_STRING)\")\n }\n}\n--- request\nGET /esi_10_prx?t=1\n--- response_body\nt=1\n--- error_code: 404\n--- response_headers_like\nX-Cache: MISS from .*\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- no_error_log\n[error]\n\n\n=== TEST 10b: ESI still runs on cache HIT.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_10 {\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler({\n esi_enabled = true,\n cache_key_spec = {\n \"scheme\",\n \"host\",\n \"uri\",\n }\n })\n run(handler)\n }\n}\n--- request\nGET /esi_10?t=2\n--- response_body\nt=2\n--- error_code: 404\n--- response_headers_like\nX-Cache: HIT from .*\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- no_error_log\n[error]\n\n\n=== TEST 10c: ESI still runs on cache revalidation, upstream 200.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_10_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler({\n esi_enabled = true,\n cache_key_spec = {\n \"scheme\",\n \"host\",\n \"uri\",\n }\n })\n run(handler)\n }\n}\nlocation /esi_10 {\n default_type text/html;\n content_by_lua_block {\n ngx.status = 404\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.header[\"Etag\"] = \"esi10c\"\n ngx.say(\"$(QUERY_STRING)\")\n }\n}\n--- more_headers\nCache-Control: max-age=0\nIf-None-Match: esi10\n--- request\nGET /esi_10_prx?t=3\n--- response_body\nt=3\n--- error_code: 404\n--- response_headers_like\nX-Cache: MISS from .*\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- no_error_log\n[error]\n\n\n=== TEST 10d: ESI still runs on cache revalidation, upstream 200, locally valid.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_10_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler({\n esi_enabled = true,\n cache_key_spec = {\n \"scheme\",\n \"host\",\n \"uri\",\n }\n })\n run(handler)\n }\n}\nlocation /esi_10 {\n default_type text/html;\n content_by_lua_block {\n ngx.status = 404\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.header[\"Etag\"] = \"esi10d\"\n ngx.say(\"$(QUERY_STRING)\")\n }\n}\n--- more_headers\nCache-Control: max-age=0\nIf-None-Match: esi10c\n--- request\nGET /esi_10_prx?t=4\n--- response_body\nt=4\n--- error_code: 404\n--- response_headers_like\nX-Cache: MISS from .*\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- no_error_log\n[error]\n\n\n=== TEST 10e: ESI still runs on cache revalidation, upstream 304, locally valid.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_10_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler({\n esi_enabled = true,\n cache_key_spec = {\n \"scheme\",\n \"host\",\n \"uri\",\n }\n }) \n run(handler)\n }\n}\nlocation /esi_10 {\n content_by_lua_block {\n ngx.exit(ngx.HTTP_NOT_MODIFIED)\n }\n}\n--- more_headers\nCache-Control: max-age=0\nIf-None-Match: esi10\n--- request\nGET /esi_10_prx?t=5\n--- response_body\nt=5\n--- error_code: 404\n--- response_headers_like\nX-Cache: MISS from .*\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- no_error_log\n[error]\n\n\n=== TEST 11a: Prime fragment\n--- http_config eval: $::HttpConfig\n--- config\nlocation /fragment_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n run()\n }\n}\nlocation /fragment {\n default_type text/html;\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.say(\"FRAGMENT\")\n }\n}\n--- request\nGET /fragment_prx\n--- response_body\nFRAGMENT\n--- error_code: 200\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- no_error_log\n[error]\n\n\n=== TEST 11b: Include fragment with client validators.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_11_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n ngx.req.set_header(\"If-Modified-Since\", ngx.http_time(ngx.time() + 150))\n run()\n }\n}\nlocation /fragment_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n run()\n }\n}\nlocation /fragment {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.say(\"FRAGMENT MODIFIED\")\n }\n}\nlocation /esi_11 {\n default_type text/html;\n content_by_lua_block {\n ngx.say(\"1\")\n ngx.print([[]])\n ngx.say(\"2\")\n }\n}\n--- request\nGET /esi_11_prx\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body\n1\nFRAGMENT\n2\n--- no_error_log\n[error]\n\n\n=== TEST 11c: Include fragment with \" H\" in URI\n Bad req in Nginx unless encoded\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_11c_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n run()\n }\n}\nlocation \"/frag Hment\" {\n content_by_lua_block {\n ngx.say(\"FRAGMENT\")\n }\n}\nlocation /esi_11c {\n default_type text/html;\n content_by_lua_block {\n ngx.say(\"1\")\n ngx.print([[]])\n ngx.say(\"2\")\n }\n}\n--- request\nGET /esi_11c_prx\n--- response_body\n1\nFRAGMENT\n2\n--- error_code: 200\n--- no_error_log\n[error]\n\n\n=== TEST 11d: Use callback feature to modify fragment request params\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_11d_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n handler:bind(\"before_esi_include_request\", function(req_params)\n req_params.headers[\"X-Foo\"] = \"bar\"\n end)\n run(handler)\n }\n}\nlocation \"/fragment\" {\n content_by_lua_block {\n ngx.say(ngx.req.get_headers()[\"X-Foo\"])\n ngx.say(\"FRAGMENT\")\n }\n}\nlocation /esi_11d {\n default_type text/html;\n content_by_lua_block {\n ngx.say(\"1\")\n ngx.print([[]])\n ngx.say(\"2\")\n }\n}\n--- request\nGET /esi_11d_prx\n--- response_body\n1\nbar\nFRAGMENT\n2\n--- error_code: 200\n--- no_error_log\n[error]\n\n\n=== TEST 12: ESI processed over buffer larger than buffer_size.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_12_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler({\n buffer_size = 16,\n })\n run(handler)\n }\n}\nlocation /esi_12 {\n default_type text/html;\n content_by_lua_block {\n local junk = \"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"\n ngx.print(\"\")\n ngx.say(junk)\n ngx.say(\"$(QUERY_STRING)\")\n ngx.say(junk)\n ngx.print(\"\")\n }\n}\n--- request\nGET /esi_12_prx?a=1\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body\nAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\na=1\nAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\n--- no_error_log\n[error]\n\n\n=== TEST 12b: Incomplete ESI tag opening at the end of buffer (lookahead)\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_12b_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler({\n buffer_size = 4,\n })\n run(handler)\n }\n}\nlocation /esi_12b {\n default_type text/html;\n content_by_lua_block {\n ngx.print(\"---\")\n ngx.print(\"$(QUERY_STRING)\")\n ngx.print(\"\")\n }\n}\n--- request\nGET /esi_12b_prx?a=1\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body: ---a=1\n--- no_error_log\n[error]\n\n\n=== TEST 12c: Incomplete ESI tag opening at the end of buffer (lookahead)\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_12c_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler({\n buffer_size = 5,\n })\n run(handler)\n }\n}\nlocation /esi_12c {\n default_type text/html;\n content_by_lua_block {\n ngx.print(\"---\")\n ngx.print(\"$(QUERY_STRING)\")\n ngx.print(\"\")\n }\n}\n--- request\nGET /esi_12c_prx?a=1\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body: ---a=1\n--- no_error_log\n[error]\n\n\n=== TEST 12d: Incomplete ESI tag opening at the end of buffer (lookahead)\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_12d_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler({\n buffer_size = 6,\n })\n run(handler)\n }\n}\nlocation /esi_12d {\n default_type text/html;\n content_by_lua_block {\n ngx.print(\"---\")\n ngx.print(\"$(QUERY_STRING)\")\n ngx.print(\"\")\n }\n}\n--- request\nGET /esi_12d_prx?a=1\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body: ---a=1\n--- no_error_log\n[error]\n\n\n=== TEST 12e: Incomplete ESI tag opening at the end of response (regression)\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_12e_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler({\n buffer_size = 9,\n })\n run(handler)\n }\n}\nlocation /esi_12e {\n default_type text/html;\n content_by_lua_block {\n ngx.print(\"---\")\n ngx.print(\"$(QUERY_STRING)\")\n ngx.print(\"\")\n ngx.say(junk)\n ngx.say(\"$(QUERY_STRING)\")\n ngx.say(junk)\n ngx.say(\"\")\n }\n}\n--- request\nGET /esi_13_prx?a=1\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body\nAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\n$(QUERY_STRING)\nAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\n\n--- error_log\nesi scan bailed as instructions spanned buffers larger than esi_max_size\n\n\n=== TEST 14: choose - when - otherwise, first when matched\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_14_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n run()\n }\n}\nlocation /esi_14 {\n default_type text/html;\ncontent_by_lua_block {\nlocal content = [[Hello\n\n\nTrue\n\n\nStill true, but first match wins\n\n\nWill never happen\n\n\nGoodbye]]\n ngx.say(content)\n}\n}\n--- request\nGET /esi_14_prx?a=1\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body\nHello\n\nTrue\n\nGoodbye\n--- no_error_log\n[error]\n\n\n=== TEST 15: choose - when - otherwise, second when matched\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_15_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n run()\n }\n}\nlocation /esi_15 {\n default_type text/html;\n content_by_lua_block {\nlocal content = [[Hello\n\n\n1\n\n\n2\n\n\nStill true, but first match wins\n\n\nWill never happen\n\n\nGoodbye]]\n ngx.say(content)\n }\n}\n--- request\nGET /esi_15_prx?a=2\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body\nHello\n\n2\n\nGoodbye\n--- no_error_log\n[error]\n\n\n=== TEST 16: choose - when - otherwise, otherwise catchall\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_16_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n run()\n }\n}\nlocation /esi_16 {\n default_type text/html;\n content_by_lua_block {\nlocal content = [[Hello\n\n\n1\n\n\n2\n\n\nOtherwise\n\n\nGoodbye]]\n ngx.say(content)\n }\n}\n--- request\nGET /esi_16_prx?a=3\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body\nHello\n\nOtherwise\n\nGoodbye\n--- no_error_log\n[error]\n\n\n=== TEST 16c: multiple single line choose - when - otherwise\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_16c_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n run()\n }\n}\nlocation /esi_16c {\n default_type text/html;\n content_by_lua_block {\n local content = [[1Otherwise: 3NOPE]]\n ngx.print(content)\n }\n}\n--- request\nGET /esi_16c_prx?a=3\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body: Otherwise: 3\n--- no_error_log\n[error]\n\n\n=== TEST 17: choose - when - test, conditional syntax\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_17_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n run()\n }\n}\nlocation /esi_17 {\n default_type text/html;\n content_by_lua_block {\n local conditions = {\n \"1 == 1\",\n \"1==1\",\n \"1 != 2\",\n \"2 > 1\",\n \"1 > 2 | 3 > 2\",\n \"(1 > 2) | (3.02 > 2.4124 & 1 <= 1)\",\n \"(1>2)||(3>2&&2>1)\",\n \"! (1 < 2) | (3 > 2 & 2 >= 1)\",\n \"'hello' == 'hello'\",\n \"'hello' != 'goodbye'\",\n \"'repeat' != 'function'\", -- use of lua words in strings\n \"'repeat' != function\", -- use of lua words unquoted\n \"' repeat sentence with function in it ' == ' repeat sentence with function in it '\", -- use of lua words in strings\n \"$(QUERY_STRING{msg}) == 'hello'\",\n [['string \\' escaping' == 'string \\' escaping']],\n [['string \\\" escaping' == 'string \\\" escaping']],\n [[$(QUERY_STRING{msg2}) == 'hel\\'lo']],\n \"'hello' =~ '/llo/'\",\n [['HeL\\'\\'\\'Lo' =~ '/hel[\\']{1,3}lo/i']],\n [['http://example.com?foo=bar' =~ '/^(http[s]?)://([^:/]+)(?::(\\d+))?(.*)/']],\n [['htxtp://example.com?foo=bar' =~ '/^(http[s]?)://([^:/]+)(?::(\\d+))?(.*)/']],\n \"(1 > 2) | (3.02 > 2.4124 & 1 <= 1) && ('HeLLo' =~ '/hello/i')\",\n \"2 =~ '/[0-9]/'\",\n \"$(HTTP_ACCEPT_LANGUAGE{gb}) == 'true'\",\n \"$(HTTP_ACCEPT_LANGUAGE{fr}) == 'false'\",\n \"$(HTTP_ACCEPT_LANGUAGE{fr}) == 'true'\",\n }\n\n for _,c in ipairs(conditions) do\n ngx.say([[]], c,\n [[Failed]])\n end\n }\n}\n--- request\nGET /esi_17_prx?msg=hello&msg2=hel'lo\n--- more_headers\nAccept-Language: en-gb\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body\n1 == 1\n1==1\n1 != 2\n2 > 1\n1 > 2 | 3 > 2\n(1 > 2) | (3.02 > 2.4124 & 1 <= 1)\n(1>2)||(3>2&&2>1)\n! (1 < 2) | (3 > 2 & 2 >= 1)\n'hello' == 'hello'\n'hello' != 'goodbye'\n'repeat' != 'function'\nFailed\n' repeat sentence with function in it ' == ' repeat sentence with function in it '\nhello == 'hello'\n'string \\' escaping' == 'string \\' escaping'\n'string \\\" escaping' == 'string \\\" escaping'\nhel'lo == 'hel\\'lo'\n'hello' =~ '/llo/'\n'HeL\\'\\'\\'Lo' =~ '/hel[\\']{1,3}lo/i'\n'http://example.com?foo=bar' =~ '/^(http[s]?)://([^:/]+)(?::(\\d+))?(.*)/'\nFailed\n(1 > 2) | (3.02 > 2.4124 & 1 <= 1) && ('HeLLo' =~ '/hello/i')\n2 =~ '/[0-9]/'\ntrue == 'true'\nfalse == 'false'\nFailed\n\n\n=== TEST 17b: Lexer complains about unparseable conditions\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_17b_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n run()\n }\n}\nlocation /esi_17b {\n default_type text/html;\n content_by_lua_block {\n local content = [[\nOK\nOK\nOK\nOtherwise\n\n]]\n ngx.print(content)\n }\n}\n--- request\nGET /esi_17b_prx\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body\nOtherwise\n--- error_log\nParse error: found string after string in: \"'hello' 'there'\"\nParse error: found string after number in: \"3 'hello'\"\nParse error: found number after string in: \"'hello' 4\"\n\n\n=== TEST 18: Surrogate-Control with lower version number still works.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_18_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n handler:bind(\"after_upstream_request\", function(res)\n res.header[\"Surrogate-Control\"] = [[content=\"ESI/0.8\"]]\n end)\n handler:run()\n }\n}\nlocation /esi_18 {\n default_type text/html;\n content_by_lua_block {\n ngx.print(\"$(QUERY_STRING)\")\n }\n}\n--- request\nGET /esi_18_prx?a=1\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body: a=1\n--- no_error_log\n[error]\n\n\n=== TEST 19: Surrogate-Control with higher version fails\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_19_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n handler:bind(\"after_upstream_request\", function(res)\n res.header[\"Surrogate-Control\"] = [[content=\"ESI/1.1\"]]\n end)\n handler:run()\n }\n}\nlocation /esi_19 {\n default_type text/html;\n content_by_lua_block {\n ngx.print(\"$(QUERY_STRING)\")\n }\n}\n--- request\nGET /esi_19_prx\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body: $(QUERY_STRING)\n--- no_error_log\n[error]\n\n\n=== TEST 20: Test we advertise Surrogate-Capability\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_20_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n run()\n }\n}\nlocation /esi_20 {\n default_type text/html;\n content_by_lua_block {\n ngx.print(ngx.req.get_headers()[\"Surrogate-Capability\"])\n }\n}\n--- request\nGET /esi_20_prx\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body_like: ^(.*)=\"ESI/1.0\"$\n--- no_error_log\n[error]\n\n=== TEST 20b: Surrogate-Capability using visible_hostname\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_20_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler({\n visible_hostname = \"ledge.example.com\"\n })\n run(handler)\n }\n}\nlocation /esi_20 {\n default_type text/html;\n content_by_lua_block {\n ngx.print(ngx.req.get_headers()[\"Surrogate-Capability\"])\n }\n}\n--- request\nGET /esi_20_prx\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body: ledge.example.com=\"ESI/1.0\"\n--- no_error_log\n[error]\n\n\n=== TEST 21: Test Surrogate-Capability is appended when needed\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_21_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n run()\n }\n}\nlocation /esi_21 {\n default_type text/html;\n content_by_lua_block {\n ngx.print(ngx.req.get_headers()[\"Surrogate-Capability\"])\n }\n}\n--- request\nGET /esi_21_prx\n--- more_headers\nSurrogate-Capability: abc=\"ESI/0.8\"\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body_like: ^abc=\"ESI/0.8\", (.*)=\"ESI/1.0\"$\n--- no_error_log\n[error]\n\n\n=== TEST 22: Test comments are removed.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_22_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n run()\n }\n}\nlocation /esi_22 {\n default_type text/html;\n content_by_lua_block {\n ngx.print([[1234 5678]])\n }\n}\n--- request\nGET /esi_22_prx\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body: 1234 5678\n--- no_error_log\n[error]\n\n\n=== TEST 23a: Surrogate-Control removed when ESI enabled but no work needed\n (slow path)\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_23_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n run()\n }\n}\nlocation /esi_23 {\n default_type text/html;\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.print(\"NO ESI\")\n }\n}\n--- request\nGET /esi_23_prx?a=1\n--- response_body: NO ESI\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- no_error_log\n[error]\n\n\n=== TEST 23b: Surrogate-Control removed when ESI enabled but no work needed\n (fast path)\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_23_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n run()\n }\n}\n--- request\nGET /esi_23_prx?a=1\n--- response_body: NO ESI\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- no_error_log\n[error]\n\n\n=== TEST 24a: Fragment recursion limit\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_24_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n -- recursion limit fails on tiny buffer sizes because it can't be scanned\n local handler = require(\"ledge\").create_handler({\n buffer_size = 4096,\n })\n run(handler)\n }\n}\nlocation /fragment_24_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler({\n buffer_size = 4096,\n })\n run(handler)\n }\n}\nlocation /fragment_24 {\n default_type text/html;\n content_by_lua_block {\n ngx.say(\"c: \", ngx.req.get_headers()[\"X-ESI-Recursion-Level\"] or \"0\")\n ngx.print([[]])\n ngx.print([[]])\n }\n}\nlocation /esi_24 {\n default_type text/html;\n content_by_lua_block {\n ngx.say(\"p: \", ngx.req.get_headers()[\"X-ESI-Recursion-Level\"] or \"0\")\n ngx.print([[]])\n }\n}\n--- request\nGET /esi_24_prx\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body\np: 0\nc: 1\np: 2\nc: 3\np: 4\nc: 5\np: 6\nc: 7\np: 8\nc: 9\np: 10\n--- error_log\nESI recursion limit (10) exceeded\n\n\n=== TEST 24b: Lower fragment recursion limit\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_24_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler({\n buffer_size = 4096,\n esi_recursion_limit = 5,\n })\n run(handler)\n }\n}\nlocation /fragment_24_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler({\n buffer_size = 4096,\n esi_recursion_limit = 5,\n })\n run(handler)\n }\n}\nlocation /fragment_24 {\n default_type text/html;\n content_by_lua_block {\n ngx.say(\"c: \", ngx.req.get_headers()[\"X-ESI-Recursion-Level\"] or \"0\")\n ngx.print([[]])\n ngx.print([[]])\n }\n}\nlocation /esi_24 {\n default_type text/html;\n content_by_lua_block {\n ngx.say(\"p: \", ngx.req.get_headers()[\"X-ESI-Recursion-Level\"] or \"0\")\n ngx.print([[]])\n }\n}\n--- request\nGET /esi_24_prx\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body\np: 0\nc: 1\np: 2\nc: 3\np: 4\nc: 5\n--- error_log\nESI recursion limit (5) exceeded\n\n\n=== TEST 25: Multiple esi includes on a single line\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_25_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n run()\n }\n}\nlocation /fragment_25a {\n default_type text/html;\n content_by_lua_block {\n ngx.print(\"25a\")\n }\n}\nlocation /fragment_25b {\n default_type text/html;\n content_by_lua_block {\n ngx.print(\"25b\")\n }\n}\nlocation /esi_25 {\n default_type text/html;\n content_by_lua_block {\n ngx.print([[ ]])\n }\n}\n--- request\nGET /esi_25_prx\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body: 25a 25b\n--- no_error_log\n[error]\n\n\n=== TEST 26: Include tag whitespace\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_26_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n run()\n }\n}\nlocation /fragment_1 {\n echo \"FRAGMENT\";\n}\nlocation /esi_26 {\n default_type text/html;\n content_by_lua_block {\n ngx.say(\"1\")\n ngx.print([[]])\n ngx.say(\"2\")\n ngx.print([[]])\n }\n}\n--- request\nGET /esi_26_prx\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body\n1\nFRAGMENT\n2\nFRAGMENT\n--- no_error_log\n[error]\n\n\n=== TEST 27a: Prime cache, immediately expired\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_27_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n handler:bind(\"before_save\", function(res)\n -- immediately expire cache entries\n res.header[\"Cache-Control\"] = \"max-age=0\"\n end)\n run(handler)\n }\n}\nlocation /esi_27 {\n default_type text/html;\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=60\"\n ngx.say(\"$(QUERY_STRING)\")\n }\n}\n--- request\nGET /esi_27_prx?a=1\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body\na=1\n--- no_error_log\n[error]\n\n\n=== TEST 27b: ESI still works when serving stale\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_27_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n run()\n }\n}\n--- more_headers\nCache-Control: stale-while-revalidate=60\n--- request\nGET /esi_27_prx?a=1\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body\na=1\n--- no_error_log\n[error]\n\n\n=== TEST 27c: ESI still works when serving stale-if-error\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_27_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n run()\n }\n}\nlocation /esi_27 {\n return 500;\n}\n--- more_headers\nCache-Control: stale-if-error=9999\n--- request\nGET /esi_27_prx?a=1\n--- wait: 1\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body\na=1\n--- wait: 2\n--- no_error_log\n[error]\n\n\n=== TEST 28: Remaining parent response returned on fragment error\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_28_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n run()\n }\n}\nlocation /fragment_1 {\n return 500;\n echo \"FRAGMENT\";\n}\nlocation /esi_28 {\n default_type text/html;\n content_by_lua_block {\n ngx.say(\"1\")\n ngx.print([[]])\n ngx.say(\"2\")\n }\n}\n--- request\nGET /esi_28_prx\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body\n1\n2\n--- error_log\n500 from /fragment_1\n\n\n=== TEST 29: Remaining parent response chunks returned on fragment error\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_29_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler({\n buffer_size = 16,\n })\n run(handler)\n }\n}\nlocation /fragment_1 {\n return 500;\n echo \"FRAGMENT\";\n}\nlocation /esi_29 {\n default_type text/html;\n content_by_lua_block {\n local junk = \"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"\n ngx.say(junk)\n ngx.say(\"1\")\n ngx.print([[]])\n ngx.say(junk)\n ngx.say(\"2\")\n }\n}\n--- request\nGET /esi_29_prx\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body\nAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\n1\nAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\n2\n--- error_log\n500 from /fragment_1\n\n\n=== TEST 30: Prime with ESI args - which should not enter cache key or\n reach the origin\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_30_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler({\n esi_enabled = true,\n esi_args_prefix = \"_esi_\",\n })\n run(handler)\n }\n}\nlocation /esi_30 {\n default_type text/html;\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.print(\"$(ESI_ARGS{a}|noarg): \")\n ngx.say(ngx.req.get_uri_args()[\"_esi_a\"])\n ngx.print(\"$(ESI_ARGS{b}|noarg): \")\n ngx.say(ngx.req.get_uri_args()[\"_esi_b\"])\n ngx.say(\"$(ESI_ARGS|noarg)\")\n }\n}\n--- request\nGET /esi_30_prx?_esi_a=1&_esi_b=2\n--- response_body_like\n1: nil\n2: nil\n_esi_[ab]=[12]&_esi_[ab]=[12]\n--- error_code: 200\n--- response_headers_like\nX-Cache: MISS from .*\n--- no_error_log\n[error]\n\n\n=== TEST 30b: ESI args vary, but cache is a HIT\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_30_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n run()\n }\n}\nlocation /esi_30 {\n default_type text/html;\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.print(\"MISS\")\n }\n}\n--- request eval\n[\"GET /esi_30_prx?esi_a=2\", \"GET /esi_30_prx?esi_a=3\", \"GET /esi_30_prx?bad_esi_a=4\"]\n--- response_body eval\n[\"2: nil\nnoarg: nil\nesi_a=2\n\",\n\"3: nil\nnoarg: nil\nesi_a=3\n\",\n\"MISS\"]\n--- error_code eval\n[\"200\", \"200\", \"200\"]\n--- response_headers_like eval\n[\"X-Cache: HIT from .*\", \"X-Cache: HIT from .*\", \"X-Cache: MISS from .*\"]\n--- no_error_log\n[error]\n\n\n=== TEST 30c: As 30 but with request not accepting cache\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_30c_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n run()\n }\n}\nlocation /esi_30c {\n default_type text/html;\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.print(\"$(ESI_ARGS{a}|noarg): \")\n ngx.print(ngx.req.get_uri_args()[\"esi_a\"])\n }\n}\n--- more_headers\nCache-Control: no-cache\n--- request\nGET /esi_30c_prx?esi_a=1\n--- response_body: 1: nil\n--- error_code: 200\n--- response_headers_like\nX-Cache: MISS from .*\n--- no_error_log\n[error]\n\n\n=== TEST 31a: Multiple sibling and child conditionals, winning expressions at various depths\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_31a_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n run()\n }\n}\nlocation /esi_31a {\n default_type text/html;\n content_by_lua_block {\n local content = [[\nBEFORE CONTENT\n\n a\n\n\n b\n RANDOM ILLEGAL CONTENT\n c\n \n \n l1d\n l1e\n \n l2f\n l2 OTHERWISE\n \n \n l1 OTHERWISE\n \n l2g\n \n \n \n \n \n\nAFTER CONTENT]]\n\n ngx.print(content)\n }\n}\n--- request eval\n[\n\"GET /esi_31a_prx?a=a\",\n\"GET /esi_31a_prx?b=b\",\n\"GET /esi_31a_prx?a=a&b=b\",\n\"GET /esi_31a_prx?l1d=l1d\",\n\"GET /esi_31a_prx?c=c&l1d=l1d\",\n\"GET /esi_31a_prx?c=c&l1e=l1e&l2f=l2f\",\n\"GET /esi_31a_prx?c=c&l1e=l1e\",\n\"GET /esi_31a_prx?c=c\",\n\"GET /esi_31a_prx?c=c&l2g=l2g\",\n]\n--- response_body eval\n[\n\"BEFORE CONTENT\na\n\nAFTER CONTENT\",\n\n\"BEFORE CONTENT\n\nb\nAFTER CONTENT\",\n\n\"BEFORE CONTENT\na\nb\nAFTER CONTENT\",\n\n\"BEFORE CONTENT\n\n\nAFTER CONTENT\",\n\n\"BEFORE CONTENT\n\nc\n l1d\n \nAFTER CONTENT\",\n\n\"BEFORE CONTENT\n\nc\n l1e\n l2f\n \n \nAFTER CONTENT\",\n\n\"BEFORE CONTENT\n\nc\n l1e\n l2 OTHERWISE\n \n \nAFTER CONTENT\",\n\n\"BEFORE CONTENT\n\nc\n l1 OTHERWISE\n \n \n \nAFTER CONTENT\",\n\n\"BEFORE CONTENT\n\nc\n l1 OTHERWISE\n l2g\n \n \nAFTER CONTENT\",\n]\n--- no_error_log\n[error]\n\n\n=== TEST 31b: As above, no whitespace\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_31b_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler({\n buffer_size = 200,\n })\n run(handler)\n }\n}\nlocation /esi_31b {\n default_type text/html;\n content_by_lua_block {\n local content = [[BEFORE CONTENTabRANDOM ILLEGAL CONTENTcl1dl1el2fl2 OTHERWISEl1 OTHERWISEl2gAFTER CONTENT]]\n\n ngx.print(content)\n }\n}\n--- request eval\n[\n\"GET /esi_31b_prx?a=a\",\n\"GET /esi_31b_prx?b=b\",\n\"GET /esi_31b_prx?a=a&b=b\",\n\"GET /esi_31b_prx?l1d=l1d\",\n\"GET /esi_31b_prx?c=c&l1d=l1d\",\n\"GET /esi_31b_prx?c=c&l1e=l1e&l2f=l2f\",\n\"GET /esi_31b_prx?c=c&l1e=l1e\",\n\"GET /esi_31b_prx?c=c\",\n\"GET /esi_31b_prx?c=c&l2g=l2g\",\n]\n--- response_body eval\n[\"BEFORE CONTENTaAFTER CONTENT\",\n\"BEFORE CONTENTbAFTER CONTENT\",\n\"BEFORE CONTENTabAFTER CONTENT\",\n\"BEFORE CONTENTAFTER CONTENT\",\n\"BEFORE CONTENTcl1dAFTER CONTENT\",\n\"BEFORE CONTENTcl1el2fAFTER CONTENT\",\n\"BEFORE CONTENTcl1el2 OTHERWISEAFTER CONTENT\",\n\"BEFORE CONTENTcl1 OTHERWISEAFTER CONTENT\",\n\"BEFORE CONTENTcl1 OTHERWISEl2gAFTER CONTENT\"]\n--- no_error_log\n[error]\n\n\n=== TEST 32: Tag parsing boundaries\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_32_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler({\n buffer_size = 50,\n })\n run(handler)\n }\n}\nlocation /esi_32 {\n default_type text/html;\n content_by_lua_block {\n local content = [[\nBEFORE CONTENT\na\n\n\nAFTER CONTENT\n]]\n\n ngx.print(content)\n }\n}\nlocation /fragment {\n echo \"OK\";\n}\n--- request\nGET /esi_32_prx?a=a\n--- response_body\nBEFORE CONTENT\na\nOK\n\nAFTER CONTENT\n--- no_error_log\n[error]\n\n\n=== TEST 33: Invalid Surrogate-Capability header is ignored\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_33_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler({\n esi_allow_surrogate_delegation = true,\n })\n run(handler)\n }\n}\nlocation /esi_33 {\n default_type text/html;\n content_by_lua_block {\n ngx.header[\"Surrogate-Control\"] = 'content=\"ESI/1.0\"'\n ngx.print(\"$(QUERY_STRING)\")\n }\n}\n--- request\nGET /esi_33_prx?foo=bar\n--- more_headers\nSurrogate-capability: localhost=\"ESI/1foo\"\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body: foo=bar\n--- no_error_log\n[error]\n\n\n=== TEST 34: Leave instructions intact if surrogate-capability does not\n match http host\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_34_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler({\n esi_allow_surrogate_delegation = true,\n })\n run(handler)\n }\n}\nlocation /esi_34 {\n default_type text/html;\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.print(\"$(QUERY_STRING)\")\n }\n}\n--- request\nGET /esi_34_prx?a=1\n--- more_headers\nSurrogate-Capability: esi.example.com=\"ESI/1.0\"\n--- response_body: $(QUERY_STRING)\n--- response_headers\nSurrogate-Control: content=\"ESI/1.0\"\n--- no_error_log\n[error]\n\n\n=== TEST 35: ESI_ARGS instruction with no args in query string\n reach the origin\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_35_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler({\n esi_enabled = true,\n esi_args_prefix = \"_esi_\",\n })\n run(handler)\n }\n}\nlocation /esi_35 {\n default_type text/html;\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.say(\"$(ESI_ARGS{a}|noarg)\")\n ngx.say(\"$(ESI_ARGS{b}|noarg)\")\n ngx.say(\"$(ESI_ARGS|noarg)\")\n ngx.say(\"OK\")\n }\n}\n--- request\nGET /esi_35_prx?foo=bar\n--- response_body\nnoarg\nnoarg\nnoarg\nOK\n--- error_code: 200\n--- response_headers_like\nX-Cache: MISS from .*\n--- no_error_log\n[error]\n\n=== TEST 36: No error if res.has_esi incorrectly set_debug\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_36_break {\n rewrite ^(.*)_break$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n local redis = require(\"ledge\").create_redis_connection()\n handler.redis = redis\n local key = handler:cache_key_chain().main\n\n\n -- Incorrectly set has_esi flag on main key\n redis:hset(key, \"has_esi\", \"ESI/1.0\")\n ngx.print(\"OK\")\n }\n}\nlocation /esi_36_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge.state_machine\").set_debug(true)\n -- No surrogate control here\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /esi_36 {\n default_type text/html;\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=60\"\n ngx.print(\"Hello\")\n }\n}\n--- request eval\n[\n \"GET /esi_36_prx\",\n \"GET /esi_36_break\",\n \"GET /esi_36_prx\",\n]\n--- response_body eval\n[\n \"Hello\",\n \"OK\",\n \"Hello\",\n]\n--- response_headers_like eval\n[\n \"X-Cache: MISS from .*\",\n \"\",\n \"X-Cache: HIT from .*\",\n]\n--- no_error_log\n[error]\n\n\n=== TEST 37: SSRF\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_37_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n ngx.req.set_uri_args('evil=foo\"/>')\n run()\n }\n}\nlocation /fragment_1 {\n content_by_lua_block {\n ngx.say(\"FRAGMENT\")\n }\n}\nlocation /esi_ {\n default_type text/html;\n content_by_lua_block {\n ngx.print([[]])\n }\n}\nlocation /bad_frag {\n content_by_lua_block {\n ngx.log(ngx.ERR, \"Shouldn't be able to request this\")\n }\n}\n--- request\nGET /esi_37_prx\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- no_error_log\n[error]\n\n=== TEST 38: SSRF via \n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_38_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n ngx.req.set_uri_args('evil=')\n run()\n }\n}\nlocation /esi_ {\n default_type text/html;\n content_by_lua_block {\n ngx.say([[$(QUERY_STRING{evil})]])\n }\n}\nlocation /bad_frag {\n content_by_lua_block {\n ngx.log(ngx.ERR, \"Shouldn't be able to request this\")\n }\n}\n--- request\nGET /esi_38_prx\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body\n<esi:include src=\"/bad_frag\" />\n--- no_error_log\n[error]\n\n=== TEST 39: XSS via \n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_39_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n ngx.req.set_uri_args('evil=')\n run()\n }\n}\nlocation /esi_ {\n default_type text/html;\n content_by_lua_block {\n ngx.say([[$(QUERY_STRING{evil})]])\n ngx.say([[$(RAW_QUERY_STRING{evil})]])\n }\n}\n--- request\nGET /esi_39_prx\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body\n<script>alert(\"HAXXED\");</script>\n\n--- no_error_log\n[error]\n\n\n=== TEST 40: ESI vars in when/choose blocks are replaced\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_40_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n run()\n }\n}\nlocation /esi_40 {\n default_type text/html;\ncontent_by_lua_block {\nlocal content = [[\n$(QUERY_STRING{a})\n$(RAW_QUERY_STRING{tag})\n$(QUERY_STRING{tag})\n\n\nWill never happen\n\n]]\n ngx.print(content)\n}\n}\n--- request\nGET /esi_40_prx?a=1&tag=foobar\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body\n1\nfoobar\nfoo<script>alert(\"bad!\")</script>bar\n--- no_error_log\n[error]\n\n\n=== TEST 41: Vars inside when/choose blocks are not evaluated before esi includes\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_41_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n ngx.req.set_uri_args('a=test&evil=\"')\n run()\n }\n}\nlocation /esi_ {\n default_type text/html;\n content_by_lua_block {\nlocal content = [[BEFORE $(QUERY_STRING{a})\n\n\n$(QUERY_STRING{a})\nWill never happen\nAFTER]]\n ngx.say(content)\n }\n}\nlocation /fragment_1 {\n content_by_lua_block {\n ngx.print(\"FRAGMENT\")\n }\n}\nlocation /bad_frag {\n content_by_lua_block {\n ngx.log(ngx.ERR, \"Shouldn't be able to request this\")\n }\n}\n--- request\nGET /esi_41_prx\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body\nBEFORE $(QUERY_STRING{a})\n\nFRAGMENT\ntest\n\nAFTER\n--- no_error_log\n[error]\n\n\n=== TEST 42: By default includes to 3rd party domains are allowed\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_42_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n run()\n }\n}\nlocation /esi_42 {\n default_type text/html;\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local content = [[]]\n ngx.say(content)\n }\n}\n--- request\nGET /esi_42_prx\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body\n{\n \"userId\": 1,\n \"id\": 1,\n \"title\": \"delectus aut autem\",\n \"completed\": false\n}\n--- no_error_log\n[error]\n\n\n=== TEST 43: Disable third party includes\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_43_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler({\n esi_disable_third_party_includes = true,\n })\n run(handler)\n }\n}\nlocation /esi_43 {\n default_type text/html;\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local content = [[]]\n ngx.print(content)\n }\n}\n--- request\nGET /esi_43_prx\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body:\n--- no_error_log\n[error]\n\n\n=== TEST 44: White list third party includes\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_44_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler({\n esi_disable_third_party_includes = true,\n esi_third_party_includes_domain_whitelist = {\n [\"jsonplaceholder.typicode.com\"] = true,\n },\n })\n run(handler)\n }\n}\nlocation /esi_44 {\n default_type text/html;\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local content = [[]]\n ngx.say(content)\n }\n}\n--- request\nGET /esi_44_prx\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body\n{\n \"userId\": 1,\n \"id\": 1,\n \"title\": \"delectus aut autem\",\n \"completed\": false\n}\n--- no_error_log\n[error]\n\n\n=== TEST 45: Cookies and Authorization propagate to fragment on same domain\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_45_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n run()\n }\n}\nlocation /fragment_1 {\n content_by_lua_block {\n ngx.say(\"method: \", ngx.req.get_method())\n local h = ngx.req.get_headers()\n\n local h_keys = {}\n for k,v in pairs(h) do\n table.insert(h_keys, k)\n end\n table.sort(h_keys)\n\n for _,k in ipairs(h_keys) do\n ngx.say(k, \": \", h[k])\n end\n }\n}\nlocation /esi_45 {\n default_type text/html;\n content_by_lua_block {\n ngx.print([[]])\n }\n}\n--- request\nPOST /esi_45_prx\n--- more_headers\nCache-Control: no-cache\nCookie: foo\nAuthorization: bar\nRange: bytes=0-\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body_like\nmethod: GET\nauthorization: bar\ncache-control: no-cache\ncookie: foo\nhost: localhost\nuser-agent: lua-resty-http/\\d+\\.\\d+ \\(Lua\\) ngx_lua/\\d+ ledge_esi/\\d+\\.\\d+[\\.\\d]*\nx-esi-parent-uri: http://localhost/esi_45_prx\nx-esi-recursion-level: 1\n--- no_error_log\n[error]\n\n\n=== TEST 45b: Cookies and Authorization don't propagate to fragment on different domain\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_45_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n run()\n }\n}\nlocation /esi_45 {\n default_type text/html;\n content_by_lua_block {\n ngx.print([[]])\n }\n}\n--- request\nPOST /esi_45_prx\n--- more_headers\nCache-Control: no-cache\nCookie: foo\nAuthorization: bar\nRange: bytes=0-\nAccept: text/plain\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body_like\n(.*)\"method\": \"GET\",\n(.*)\"cache-control\": \"no-cache\",\n--- response_body_unlike\n(.*)\"authorization\": \"bar\",\n(.*)\"cookie\": \"foo\",\n--- no_error_log\n[error]\n\n\n=== TEST 46: Cookie var blacklist\n--- http_config eval: $::HttpConfig\n--- config\nlocation /esi_46_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler({\n esi_vars_cookie_blacklist = {\n not_allowed = true,\n },\n })\n run(handler)\n }\n}\nlocation /esi_46 {\n default_type text/html;\n content_by_lua_block {\n -- Blacklist should apply to expansion in vars\n ngx.say([[$(HTTP_COOKIE)]])\n\n -- And by key\n ngx.say([[$(HTTP_COOKIE{allowed}):$(HTTP_COOKIE{not_allowed})]])\n\n -- ...and also in URIs\n ngx.say([[]])\n }\n}\nlocation /fragment {\n content_by_lua_block {\n ngx.say(\"FRAGMENT:\"..ngx.var.args)\n\n -- But ALL cookies are still propagated by default to subrequests\n local cookie = require(\"resty.cookie\").new()\n ngx.print(cookie:get(\"allowed\") .. \":\" .. cookie:get(\"not_allowed\"))\n }\n}\n--- request\nGET /esi_46_prx\n--- more_headers\nCookie: allowed=yes\nCookie: also_allowed=yes\nCookie: not_allowed=no\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body_like\n(allowed=yes; also_allowed=yes)|(also_allowed=yes; allowed=yes)\nyes:\nFRAGMENT:&allowed=yes¬_allowed=\nyes:no\n--- no_error_log\n[error]\n" }, { "path": "t/02-integration/events.t", "content": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::http_config();\n\nno_long_string();\nno_diff();\nrun_tests();\n\n__DATA__\n=== TEST 1: before_serve (add response header)\n--- http_config eval: $::HttpConfig\n--- config\nlocation /events_1_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n handler:bind(\"before_serve\", function(res)\n res.header[\"X-Modified\"] = \"Modified\"\n end)\n handler:run()\n }\n}\nlocation /events_1 {\n echo \"ORIGIN\";\n}\n--- request\nGET /events_1_prx\n--- error_code: 200\n--- response_headers\nX-Modified: Modified\n--- no_error_log\n[error]\n\n\n=== TEST 2: before_upstream_request (modify request params)\n--- http_config eval: $::HttpConfig\n--- config\nlocation /events_2 {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n handler:bind(\"before_upstream_request\", function(params)\n params.path = \"/modified\"\n end)\n handler:run()\n }\n}\nlocation /modified {\n echo \"ORIGIN\";\n}\n--- request\nGET /events_2\n--- error_code: 200\n--- response_body\nORIGIN\n--- no_error_log\n[error]\n" }, { "path": "t/02-integration/gc.t", "content": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::http_config(extra_nginx_config => qq{\n lua_check_client_abort on;\n}, extra_lua_config => qq{\n require(\"ledge\").set_handler_defaults({\n keep_cache_for = 0,\n })\n}, run_worker => 1);\n\nno_long_string();\nno_diff();\nrun_tests();\n\n__DATA__\n=== TEST 1: Prime cache\n--- http_config eval: $::HttpConfig\n--- config\nlocation /gc_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /gc {\n more_set_headers \"Cache-Control: public, max-age=60\";\n echo \"OK\";\n}\n--- request\nGET /gc_prx\n--- no_error_log\n[error]\n--- response_body\nOK\n\n\n=== TEST 2: Force revaldation (creates new entity)\n--- http_config eval: $::HttpConfig\n--- config\nlocation /gc_prx {\n rewrite ^(.*)_prx$ $1 break;\n echo_location_async '/gc_a';\n echo_sleep 0.1;\n echo_location_async '/gc_b';\n echo_sleep 2.5;\n}\nlocation /gc_a {\n rewrite ^(.*)_a$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run();\n }\n}\nlocation /gc_b {\n rewrite ^(.*)_b$ $1 break;\n content_by_lua_block {\n local redis = require(\"ledge\").create_redis_connection()\n local handler = require(\"ledge\").create_handler()\n handler.redis = redis\n\n local key_chain = handler:cache_key_chain()\n local num_entities, err = redis:scard(key_chain.entities)\n ngx.say(num_entities)\n }\n}\nlocation /gc {\n more_set_headers \"Cache-Control: public, max-age=5\";\n content_by_lua_block {\n ngx.say(\"UPDATED\")\n }\n}\n--- more_headers\nCache-Control: no-cache\n--- request\nGET /gc_prx\n--- response_body\nUPDATED\n1\n--- wait: 1\n\n\n=== TEST 3: Check we now have just one entity\n--- http_config eval: $::HttpConfig\n--- config\nlocation /gc {\n content_by_lua_block {\n local redis = require(\"ledge\").create_redis_connection()\n local handler = require(\"ledge\").create_handler()\n handler.redis = redis\n\n local key_chain = handler:cache_key_chain()\n local num_entities, err = redis:scard(key_chain.entities)\n ngx.say(num_entities)\n }\n}\n--- request\nGET /gc\n--- no_error_log\n[error]\n--- response_body\n1\n--- wait: 2\n\n\n=== TEST 4: Entity will have expired, check Redis has cleaned up all keys.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /gc {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local redis = require(\"ledge\").create_redis_connection()\n local handler = require(\"ledge\").create_handler()\n handler.redis = redis\n local key_chain = handler:cache_key_chain()\n local res, err = redis:keys(key_chain.full .. \"*\")\n assert(not next(res), \"res should be empty\")\n }\n}\n--- request\nGET /gc\n--- no_error_log\n[error]\n\n\n=== TEST 5: Prime cache\n--- http_config eval: $::HttpConfig\n--- config\nlocation /gc_5_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /gc_5 {\n more_set_headers \"Cache-Control: public, max-age=60\";\n echo \"OK\";\n}\n--- request\nGET /gc_5_prx\n--- no_error_log\n[error]\n--- response_body\nOK\n\n\n=== TEST 5b: Delete one part of the key chain\nSimulate eviction under memory pressure. Will cause a MISS.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /gc_5_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local redis = require(\"ledge\").create_redis_connection()\n local handler = require(\"ledge\").create_handler()\n handler.redis = redis\n local key_chain = handler:cache_key_chain()\n redis:del(key_chain.headers)\n handler:run()\n }\n}\nlocation /gc_5 {\n more_set_headers \"Cache-Control: public, max-age=60\";\n echo \"OK 2\";\n}\n--- request\nGET /gc_5_prx\n--- wait: 3\n--- no_error_log\n[error]\n--- response_body\nOK 2\n\n\n=== TEST 5c: Missing keys should cause colleciton of the old entity.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /gc_5 {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local redis = require(\"ledge\").create_redis_connection()\n local handler = require(\"ledge\").create_handler()\n handler.redis = redis\n local key_chain = handler:cache_key_chain()\n local res, err = redis:keys(key_chain.full .. \"*\")\n if res then\n ngx.say(#res)\n end\n }\n}\n--- request\nGET /gc_5\n--- no_error_log\n[error]\n--- response_body\n5\n" }, { "path": "t/02-integration/gzip.t", "content": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::http_config();\n\nno_long_string();\nno_diff();\nrun_tests();\n\n__DATA__\n=== TEST 1: Prime gzipped response\n--- http_config eval: $::HttpConfig\n--- config\nlocation /gzip_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /gzip {\n gzip on;\n gzip_proxied any;\n gzip_min_length 1;\n gzip_http_version 1.0;\n default_type text/html;\n more_set_headers \"Cache-Control: public, max-age=600\";\n more_set_headers \"Content-Type: text/html\";\n echo \"OK\";\n}\n--- request\nGET /gzip_prx\n--- more_headers\nAccept-Encoding: gzip\n--- response_body_unlike: OK\n--- no_error_log\n[error]\n\n\n=== TEST 2: Client doesnt support gzip, gets plain response\n--- http_config eval: $::HttpConfig\n--- config\nlocation /gzip_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\n--- request\nGET /gzip_prx\n--- response_body\nOK\n--- no_error_log\n[error]\n\n\n=== TEST 2b: Client doesnt support gzip, gunzip is disabled, gets zipped response\n--- http_config eval: $::HttpConfig\n--- config\nlocation /gzip_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler({\n gunzip_enabled = false,\n }):run()\n }\n}\n--- request\nGET /gzip_prx\n--- response_body_unlike: OK\n--- no_error_log\n[error]\n\n\n=== TEST 3: Client does support gzip, gets zipped response\n--- http_config eval: $::HttpConfig\n--- config\nlocation /gzip_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\n--- request\nGET /gzip_prx\n--- more_headers\nAccept-Encoding: gzip\n--- response_body_unlike: OK\n--- no_error_log\n[error]\n\n\n=== TEST 4: Client does support gzip, but sends a range, gets plain full response\n--- http_config eval: $::HttpConfig\n--- config\nlocation /gzip_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\n--- request\nGET /gzip_prx\n--- more_headers\nAccept-Encoding: gzip\n--- more_headers\nRange: bytes=0-0\n--- error_code: 200\n--- response_body\nOK\n--- no_error_log\n[error]\n\n\n=== TEST 5: Prime gzipped response with ESI, auto unzips.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /gzip_5_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler({\n esi_enabled = true,\n }):run()\n }\n}\nlocation /gzip_5 {\n gzip on;\n gzip_proxied any;\n gzip_min_length 1;\n gzip_http_version 1.0;\n default_type text/html;\n more_set_headers \"Cache-Control: public, max-age=600\";\n more_set_headers \"Content-Type: text/html\";\n more_set_headers 'Surrogate-Control: content=\"ESI/1.0\"';\n echo \"OK\";\n}\n--- request\nGET /gzip_5_prx\n--- more_headers\nAccept-Encoding: gzip\n--- response_body\nOK\n--- no_error_log\n[error]\n\n\n=== TEST 6: Client does support gzip, but content had to be unzipped on save\n--- http_config eval: $::HttpConfig\n--- config\nlocation /gzip_5_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\n--- request\nGET /gzip_5_prx\n--- more_headers\nAccept-Encoding: gzip\n--- response_body\nOK\n--- no_error_log\n[error]\n\n\n=== TEST 7: HEAD request for gzipped response with ESI, auto unzips.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /gzip_7_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler({\n esi_enabled = true,\n }):run()\n }\n}\nlocation /gzip_7 {\n gzip on;\n gzip_proxied any;\n gzip_min_length 1;\n gzip_http_version 1.0;\n default_type text/html;\n more_set_headers \"Cache-Control: public, max-age=600\";\n more_set_headers \"Content-Type: text/html\";\n more_set_headers 'Surrogate-Control: content=\"ESI/1.0\"';\n echo \"OK\";\n}\n--- request\nHEAD /gzip_7_prx\n--- more_headers\nAccept-Encoding: gzip\n--- response_body\n--- no_error_log\n[error]\n" }, { "path": "t/02-integration/hop_by_hop_headers.t", "content": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::http_config();\n\nno_long_string();\nno_diff();\nrun_tests();\n\n__DATA__\n=== TEST 1: Test hop-by-hop headers are not passed on.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /hop_by_hop_headers_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /hop_by_hop_headers {\n more_set_headers \"Cache-Control public, max-age=600\";\n more_set_headers \"Proxy-Authenticate foo\";\n more_set_headers \"Upgrade foo\";\n echo \"OK\";\n}\n--- request\nGET /hop_by_hop_headers_prx\n--- response_headers\nProxy-Authenticate:\nUpgrade:\n--- no_error_log\n[error]\n\n\n=== TEST 2: Test hop-by-hop headers were not cached.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /hop_by_hop_headers_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\n--- request\nGET /hop_by_hop_headers_prx\n--- response_headers\nProxy-Authenticate:\nUpgrade:\n--- no_error_log\n[error]\n" }, { "path": "t/02-integration/max-stale.t", "content": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::http_config(extra_lua_config => qq{\n package.loaded[\"state\"] = {\n miss_count = 0,\n }\n}, run_worker => 1);\n\nno_long_string();\nno_diff();\nrun_tests();\n\n__DATA__\n=== TEST 1: Honour max-stale request header for an expired item\n--- http_config eval: $::HttpConfig\n--- config\nlocation /stale_1_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n handler:bind(\"before_save\", function(res)\n -- immediately expire\n res.header[\"Cache-Control\"] = \"max-age=0\"\n end)\n handler:run()\n }\n}\nlocation /stale_1 {\n content_by_lua_block {\n local state = require(\"state\")\n state.miss_count = state.miss_count + 1\n ngx.status = 404\n ngx.header[\"Cache-Control\"] = \"max-age=60\"\n ngx.print(\"TEST 1: \", state.miss_count)\n }\n}\n--- more_headers\nCache-Control: max-stale=1000\n--- request eval\n[\"GET /stale_1_prx\", \"GET /stale_1_prx\"]\n--- response_body eval\n[\"TEST 1: 1\", \"TEST 1: 1\"]\n--- response_headers_like eval\n[\"\", 'Warning: 110 (?:[^\\s]*) \"Response is stale\"']\n--- error_code eval\n[404, 404]\n--- no_error_log\n[error]\n\n\n=== TEST 1b: Confirm nothing was revalidated in the background\n--- http_config eval: $::HttpConfig\n--- config\nlocation /stale_1_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\n--- more_headers\nCache-Control: max-stale=1000\n--- request\nGET /stale_1_prx\n--- response_body: TEST 1: 1\n--- response_headers_like\nWarning: 110 (?:[^\\s]*) \"Response is stale\"\n--- error_code eval\n404\n--- no_error_log\n[error]\n\n\n=== TEST 5: proxy-revalidate must revalidate (not serve stale)\n--- http_config eval: $::HttpConfig\n--- config\nlocation /stale_5_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge.state_machine\").set_debug(true)\n local handler = require(\"ledge\").create_handler()\n handler:bind(\"before_save\", function(res)\n -- immediately expire\n res.header[\"Cache-Control\"] = \"max-age=0, proxy-revalidate\"\n end)\n handler:run()\n }\n}\nlocation /stale_5 {\n content_by_lua_block {\n local state = require(\"state\")\n state.miss_count = state.miss_count + 1\n ngx.status = 404\n ngx.header[\"Cache-Control\"] = \"max-age=3600, proxy-revalidate\"\n ngx.print(\"TEST 5: \", state.miss_count)\n }\n}\n--- more_headers\nCache-Control: max-stale=120\n--- request eval\n[\"GET /stale_5_prx\", \"GET /stale_5_prx\"]\n--- response_body eval\n[\"TEST 5: 1\", \"TEST 5: 2\"]\n--- raw_response_headers_unlike eval\n[\"Warning: 110\", \"Warning: 110\"]\n--- error_code eval\n[404, 404]\n--- no_error_log\n[error]\n\n\n=== TEST 6: must-revalidate must revalidate (not serve stale)\n--- http_config eval: $::HttpConfig\n--- config\nlocation /stale_6_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n handler:bind(\"before_save\", function(res)\n -- immediately expire\n res.header[\"Cache-Control\"] = \"max-age=0, must-revalidate\"\n end)\n handler:run()\n }\n}\nlocation /stale_6 {\n content_by_lua_block {\n local state = require(\"state\")\n state.miss_count = state.miss_count + 1\n ngx.status = 404\n ngx.header[\"Cache-Control\"] = \"max-age=3600, must-revalidate\"\n ngx.print(\"TEST 6: \", state.miss_count)\n }\n}\n--- more_headers\nCache-Control: max-stale=120\n--- request eval\n[\"GET /stale_6_prx\", \"GET /stale_6_prx\"]\n--- response_body eval\n[\"TEST 6: 1\", \"TEST 6: 2\"]\n--- raw_response_headers_unlike eval\n[\"Warning: 110\", \"Warning: 110\"]\n--- error_code eval\n[404, 404]\n--- no_error_log\n[error]\n\n\n=== TEST 7: Can serve stale but must revalidate because of Age\n--- http_config eval: $::HttpConfig\n--- config\nlocation /stale_7_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n handler:bind(\"before_save\", function(res)\n -- immediately expire\n res.header[\"Cache-Control\"] = \"max-age=0\"\n end)\n handler:run()\n }\n}\nlocation /stale_7 {\n content_by_lua_block {\n local state = require(\"state\")\n state.miss_count = state.miss_count + 1\n ngx.status = 404\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.print(\"TEST 7: \", state.miss_count)\n }\n}\n--- more_headers\nCache-Control: max-stale=120, max-age=1\n--- request eval\n[\"GET /stale_7_prx\", \"GET /stale_7_prx\"]\n--- response_body eval\n[\"TEST 7: 1\", \"TEST 7: 2\"]\n--- raw_response_headers_unlike eval\n[\"Warning: 110\", \"Warning: 110\"]\n--- error_code eval\n[404, 404]\n--- no_error_log\n[error]\n--- wait: 2\n" }, { "path": "t/02-integration/max_size.t", "content": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::http_config(extra_lua_config => qq{\n require(\"ledge\").set_handler_defaults({\n storage_driver_config = {\n max_size = 8,\n }\n })\n});\n\nno_long_string();\nno_diff();\nrun_tests();\n\n__DATA__\n=== TEST 1: Response larger than cache_max_memory.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /max_memory_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /max_memory {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.say(\"RESPONSE IS TOO LARGE TEST 1\")\n }\n}\n--- request\nGET /max_memory_prx\n--- response_body\nRESPONSE IS TOO LARGE TEST 1\n--- response_headers_like\nX-Cache: MISS from .*\n--- error_log\nstorage failed to write: body is larger than 8 bytes\n\n\n=== TEST 2: Test we did not store in previous test.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /max_memory_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /max_memory {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.say(\"TEST 2\")\n }\n}\n--- request\nGET /max_memory_prx\n--- response_body\nTEST 2\n--- response_headers_like\nX-Cache: MISS from .*\n--- no_error_log\n\n\n=== TEST 3: Non-chunked response larger than cache_max_memory.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /max_memory_3_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /max_memory_3 {\n chunked_transfer_encoding off;\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n local body = \"RESPONSE IS TOO LARGE TEST 3\\n\"\n ngx.header[\"Content-Length\"] = string.len(body)\n ngx.print(body)\n }\n}\n--- request\nGET /max_memory_3_prx\n--- response_body\nRESPONSE IS TOO LARGE TEST 3\n--- response_headers_like\nX-Cache: MISS from .*\n--- no_error_log\n\n\n=== TEST 4: Test we did not store in previous test.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /max_memory_3_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /max_memory_3 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.say(\"TEST 4\")\n }\n}\n--- request\nGET /max_memory_3_prx\n--- response_body\nTEST 4\n--- response_headers_like\nX-Cache: MISS from .*\n--- no_error_log\n\n\n=== TEST 5a: Prime cache with ok size\n--- http_config eval: $::HttpConfig\n--- config\nlocation /max_memory_5_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /max_memory_5 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.say(\"OK\")\n }\n}\n--- request\nGET /max_memory_5_prx\n--- response_body\nOK\n--- response_headers_like\nX-Cache: MISS from .*\n--- no_error_log\n[error]\n\n\n=== TEST 5b: Try to replace with a large response\n--- http_config eval: $::HttpConfig\n--- config\nlocation /max_memory_5_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /max_memory_5 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.say(\"RESPONSE IS TOO LARGE\")\n }\n}\n--- more_headers\nCache-Control: no-cache\n--- request\nGET /max_memory_5_prx\n--- response_body\nRESPONSE IS TOO LARGE\n--- response_headers_like\nX-Cache: MISS from .*\n--- error_log\nlarger than 8 bytes\n\n\n=== TEST 5c: Confirm original cache is still ok\n--- http_config eval: $::HttpConfig\n--- config\nlocation /max_memory_5_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\n--- request\nGET /max_memory_5_prx\n--- response_body\nOK\n--- response_headers_like\nX-Cache: HIT from .*\n--- no_error_log\n[error]\n" }, { "path": "t/02-integration/memory_pressure.t", "content": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::http_config(extra_lua_config => qq{\n require(\"ledge\").set_handler_defaults({\n esi_enabled = true,\n })\n}, run_worker => 1);\n\nno_long_string();\nno_diff();\nrun_tests();\n\n__DATA__\n=== TEST 1: Prime some cache\n--- http_config eval: $::HttpConfig\n--- config\nlocation \"/mem_pressure_1_prx\" {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation \"/mem_pressure_1\" {\n default_type text/html;\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.header[\"Surrogate-Control\"] = [[content=\"ESI/1.0\"]]\n ngx.print(\"Key: \", ngx.req.get_uri_args()[\"key\"])\n }\n}\n--- request eval\n[\"GET /mem_pressure_1_prx?key=main\",\n\"GET /mem_pressure_1_prx?key=headers\",\n\"GET /mem_pressure_1_prx?key=entities\"]\n--- response_body eval\n[\"Key: main\",\n\"Key: headers\",\n\"Key: entities\"]\n--- no_error_log\n[error]\n\n\n=== TEST 1b: Break each key, in a different way for each, then try to serve\n--- http_config eval: $::HttpConfig\n--- config\nlocation \"/mem_pressure_1_prx\" {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local redis = require(\"ledge\").create_redis_connection()\n local handler = require(\"ledge\").create_handler()\n handler.redis = redis\n local key_chain = handler:cache_key_chain()\n\n local evict = ngx.req.get_uri_args()[\"key\"]\n local key = key_chain[evict]\n ngx.log(ngx.DEBUG, \"will evict: \", key)\n local res, err = redis:del(key)\n if not res then\n ngx.log(ngx.ERR, \"could not evict: \", err)\n end\n redis:set(evict, \"true\")\n ngx.log(ngx.DEBUG, tostring(res))\n\n redis:close()\n\n handler:run()\n }\n}\n\nlocation \"/mem_pressure_1\" {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=0\"\n ngx.print(\"MISSED: \", ngx.req.get_uri_args()[\"key\"])\n }\n}\n--- request eval\n[\"GET /mem_pressure_1_prx?key=main\",\n\"GET /mem_pressure_1_prx?key=headers\",\n\"GET /mem_pressure_1_prx?key=entities\"]\n--- response_body eval\n[\"MISSED: main\",\n\"MISSED: headers\",\n\"MISSED: entities\"]\n--- no_error_log\n[error]\n\n\n=== TEST 2: Prime and break ::main before transaction completes\n(leaves it partial)\n--- http_config eval: $::HttpConfig\n--- config\nlocation \"/mem_pressure_2_prx\" {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n handler:bind(\"before_serve\", function(res)\n local main = handler:cache_key_chain().main\n handler.redis:del(main)\n end)\n handler:run()\n }\n}\nlocation \"/mem_pressure_2\" {\n default_type text/html;\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.print(\"ORIGIN\")\n }\n}\n--- request\nGET /mem_pressure_2_prx\n--- response_body: ORIGIN\n--- no_error_log\n[error]\n\n\n=== TEST 2b: Confirm broken ::main doesnt get served\n--- http_config eval: $::HttpConfig\n--- config\nlocation \"/mem_pressure_2_prx\" {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation \"/mem_pressure_2\" {\n default_type text/html;\n content_by_lua_block {\n ngx.print(\"ORIGIN\")\n }\n}\n--- request\nGET /mem_pressure_2_prx\n--- response_body: ORIGIN\n--- no_error_log\n[error]\n\n=== TEST 3: Prime and break active entity during read\n--- http_config eval: $::HttpConfig\n--- config\nlocation \"/mem_pressure_3_prx\" {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n if not ngx.req.get_uri_args()[\"prime\"] then\n handler:bind(\"before_serve\", function(res)\n ngx.log(ngx.DEBUG, \"Deleting: \", res.entity_id)\n handler.storage:delete(res.entity_id)\n end)\n else\n -- Dummy log for prime request\n ngx.log(ngx.DEBUG, \"entity removed during read\")\n end\n ngx.req.set_uri_args({})\n handler:run()\n }\n}\nlocation \"/mem_pressure_3\" {\n default_type text/html;\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.print(\"ORIGIN\")\n }\n}\n--- request eval\n[\"GET /mem_pressure_3_prx?prime=true\", \"GET /mem_pressure_3_prx\"]\n--- response_body eval\n[\"ORIGIN\", \"\"]\n--- response_headers_like eval\n[\"X-Cache: MISS from .*\", \"X-Cache: HIT from .*\"]\n--- no_error_log\n[error]\n--- error_log\nentity removed during read\n\n=== TEST 4: Prime some cache - stale headers\n--- http_config eval: $::HttpConfig\n--- config\nlocation \"/mem_pressure_4_prx\" {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation \"/mem_pressure_4\" {\n default_type text/html;\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600, stale-if-error=2592000, stale-while-revalidate=129600\"\n ngx.header[\"Surrogate-Control\"] = [[content=\"ESI/1.0\"]]\n ngx.print(\"Key: \", ngx.req.get_uri_args()[\"key\"])\n }\n}\n--- request eval\n[\"GET /mem_pressure_4_prx?key=main\",\n\"GET /mem_pressure_4_prx?key=headers\",\n\"GET /mem_pressure_4_prx?key=entities\"]\n--- response_body eval\n[\"Key: main\",\n\"Key: headers\",\n\"Key: entities\"]\n--- no_error_log\n[error]\n\n\n=== TEST 4b: Break each key, in a different way for each, then try to serve\n--- http_config eval: $::HttpConfig\n--- config\nlocation \"/mem_pressure_4_prx\" {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local redis = require(\"ledge\").create_redis_connection()\n local handler = require(\"ledge\").create_handler()\n handler.redis = redis\n local key_chain = handler:cache_key_chain()\n\n local evict = ngx.req.get_uri_args()[\"key\"]\n local key = key_chain[evict]\n ngx.log(ngx.DEBUG, \"will evict: \", key)\n local res, err = redis:del(key)\n if not res then\n ngx.log(ngx.ERR, \"could not evict: \", err)\n end\n redis:set(evict, \"true\")\n ngx.log(ngx.DEBUG, tostring(res))\n\n redis:close()\n\n handler:run()\n }\n}\n\nlocation \"/mem_pressure_4\" {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=0\"\n ngx.print(\"MISSED: \", ngx.req.get_uri_args()[\"key\"])\n }\n}\n--- request eval\n[\"GET /mem_pressure_4_prx?key=main\",\n\"GET /mem_pressure_4_prx?key=headers\",\n\"GET /mem_pressure_4_prx?key=entities\"]\n--- response_body eval\n[\"MISSED: main\",\n\"MISSED: headers\",\n\"MISSED: entities\"]\n--- no_error_log\n[error]\n\n=== TEST 5: Prime and break active entity during read - ESI\n--- http_config eval: $::HttpConfig\n--- config\nlocation \"/mem_pressure_5_prx\" {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n if not ngx.req.get_uri_args()[\"prime\"] then\n handler:bind(\"before_serve\", function(res)\n ngx.log(ngx.DEBUG, \"Deleting: \", res.entity_id)\n handler.storage:delete(res.entity_id)\n end)\n else\n -- Dummy log for prime request\n require(\"ledge.state_machine\").set_debug(true)\n ngx.log(ngx.DEBUG, \"entity removed during read\")\n end\n ngx.req.set_uri_args({})\n handler:run()\n }\n}\nlocation \"/mem_pressure_5\" {\n default_type text/html;\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.header[\"Surrogate-Control\"] = 'content=\"ESI/1.0\"'\n ngx.print(\"ORIGIN\")\n ngx.print(\"$(QUERY_STRING)\")\n }\n}\n--- request eval\n[\"GET /mem_pressure_5_prx?prime=true\", \"GET /mem_pressure_5_prx\"]\n--- response_body eval\n[\"ORIGIN\", \"\"]\n--- response_headers_like eval\n[\"X-Cache: MISS from .*\", \"X-Cache: HIT from .*\"]\n--- no_error_log\n[error]\n--- error_log\nentity removed during read\n" }, { "path": "t/02-integration/multiple_headers.t", "content": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::http_config();\n\nno_long_string();\nno_diff();\nrun_tests();\n\n__DATA__\n=== TEST 1: Multiple cache-control response headers, miss\n--- http_config eval: $::HttpConfig\n--- config\n location /multiple_cache_headers_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n }\n\n location /multiple_cache_headers {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = { \"public\", \"max-age=3600\"}\n ngx.say(\"TEST 1\")\n }\n }\n--- request\nGET /multiple_cache_headers_prx\n--- response_headers_like\nX-Cache: MISS from .*\n--- response_headers\nCache-Control: public, max-age=3600\n--- response_body\nTEST 1\n\n\n=== TEST 1b: Multiple cache-control response headers, hit\n--- http_config eval: $::HttpConfig\n--- config\n location /multiple_cache_headers_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n }\n\n location /multiple_cache_headers {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = { \"public\", \"max-age=3600\"}\n ngx.say(\"TEST 2\")\n }\n }\n--- request\nGET /multiple_cache_headers_prx\n--- response_headers_like\nX-Cache: HIT from .*\n--- response_headers\nCache-Control: public, max-age=3600\n--- response_body\nTEST 1\n\n=== TEST 2: Multiple Date response headers, miss\n--- http_config eval: $::HttpConfig\n--- config\n location /multiple_date_headers_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler({\n upstream_port = 12345\n }):run()\n }\n }\n--- request\nGET /multiple_date_headers_prx\n--- tcp_listen: 12345\n--- tcp_reply\nHTTP/1.1 200 OK\nDate: Mon, 24 Sep 2018 00:47:20 GMT\nServer: Apache/2\nDate: Mon, 24 Sep 2018 01:47:20 GMT\nCache-Control: public, max-age=300\n\nTEST 2\n--- response_headers_like\nX-Cache: MISS from .*\n--- response_headers_unlike\nDate: Mon, 24 Sep 2018 00:47:20 GMT\nDate: Mon, 24 Sep 2018 01:47:20 GMT\n--- response_body\nTEST 2\n\n=== TEST 2b: Multiple Date response headers, hit\n--- http_config eval: $::HttpConfig\n--- config\n location /multiple_date_headers_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n }\n--- request\nGET /multiple_date_headers_prx\n--- response_headers_like\nX-Cache: HIT from .*\n--- response_headers_unlike\nDate: Mon, 24 Sep 2018 00:47:20 GMT\nDate: Mon, 24 Sep 2018 01:47:20 GMT\n--- response_body\nTEST 2\n" }, { "path": "t/02-integration/on_abort.t", "content": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::http_config(extra_nginx_config => qq{\n lua_check_client_abort on;\n\n upstream test-upstream {\n server 127.0.0.1:1984;\n keepalive 16;\n }\n}, run_worker => 1);\n\nno_long_string();\nno_diff();\nrun_tests();\n\n__DATA__\n=== TEST 1: Warning when unable to set client abort handler\n--- http_config eval: $::HttpConfig\n--- config\nlocation /abort_prx {\n rewrite ^(.*)_prx$ $1 break;\n lua_check_client_abort off;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /abort {\n echo \"foo\";\n}\n--- request\nGET /abort_prx\n--- error_log\non_abort handler could not be set: lua_check_client_abort is off\n\n\n=== TEST 2a: Client abort mid save should still save to cache (run and abort)\n--- http_config eval: $::HttpConfig\n--- config\n location /abort_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n }\n location /abort {\n content_by_lua_block {\n ngx.status = 200\n ngx.header[\"Cache-Control\"] = \"public, max-age=3600\"\n ngx.say(\"START\")\n ngx.flush(true)\n ngx.sleep(2)\n ngx.say(\"FINISH\")\n }\n }\n--- request\nGET /abort_prx\n--- timeout: 1\n--- wait: 1.5\n--- abort\n--- ignore_response\n--- no_error_log\n[error]\n\n\n=== TEST 2b: Prove we have a complete cache entry\n--- http_config eval: $::HttpConfig\n--- config\n location /abort_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n }\n--- request\nGET /abort_prx\n--- response_body\nSTART\nFINISH\n--- error_code: 200\n--- no_error_log\n[error]\n\n\n=== TEST 3a: Client abort before save aborts fetching\n--- http_config eval: $::HttpConfig\n--- config\n location /abort_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n }\n location /abort {\n content_by_lua_block {\n ngx.sleep(2)\n ngx.status = 200\n ngx.header[\"Cache-Control\"] = \"public, max-age=3600\"\n ngx.say(\"START 2\")\n ngx.say(\"FINISH 2\")\n }\n }\n--- request\nGET /abort_prx\n--- more_headers\nCache-Control: max-age=0\n--- timeout: 1\n--- wait: 1.5\n--- abort\n--- ignore_response\n--- no_error_log\n[error]\n\n\n=== TEST 3b: Prove we still have the previous cache entry\n--- http_config eval: $::HttpConfig\n--- config\n location /abort_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n }\n--- request\nGET /abort_prx\n--- response_body\nSTART\nFINISH\n--- error_code: 200\n--- no_error_log\n[error]\n\n\n=== TEST 4a: Prime immediately expiring cache item\n--- http_config eval: $::HttpConfig\n--- config\nlocation /abort_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n handler:bind(\"before_save\", function(res)\n -- immediately expire cache entries\n res.header[\"Cache-Control\"] = \"max-age=0\"\n end)\n handler:run()\n }\n}\nlocation /abort {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.say(\"OK\")\n }\n}\n--- more_headers\nCache-Control: no-cache\n--- request\nGET /abort_prx\n--- response_body\nOK\n--- error_code: 200\n--- no_error_log\n[error]\n\n\n=== TEST 4b: Client abort before fetch with collapsed forwarding on cancels abort\n--- http_config eval: $::HttpConfig\n--- config\n location /abort_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler({\n enable_collapsed_forwarding = true,\n })\n handler:bind(\"before_upstream_request\", function(res)\n ngx.sleep(2)\n end)\n handler:run()\n }\n }\n location /abort {\n content_by_lua_block {\n ngx.status = 200\n ngx.header[\"Cache-Control\"] = \"public, max-age=3600\"\n ngx.say(\"START\")\n ngx.say(\"FINISH\")\n }\n }\n--- request\nGET /abort_prx\n--- timeout: 1\n--- wait: 1.5\n--- abort\n--- ignore_response\n--- no_error_log\n[error]\n\n\n=== TEST 4c: Prove we have the previous cache entry\n--- http_config eval: $::HttpConfig\n--- config\n location /abort_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n }\n--- request\nGET /abort_prx\n--- response_body\nSTART\nFINISH\n--- error_code: 200\n--- no_error_log\n[error]\n\n\n=== TEST 5: No error when keepalive_requests exceeded\n--- http_config eval: $::HttpConfig\n--- config\n location = /abort_top {\n content_by_lua_block {\n local http = require \"resty.http\"\n local httpc = http.new()\n local res, err = httpc:request_uri(\n \"http://\" ..\n ngx.var.server_addr .. \":\" .. ngx.var.server_port ..\n \"/abort_ngx\"\n )\n if not res then\n ngx.log(ngx.ERR, err)\n end\n\n local res, err = httpc:request_uri(\n \"http://\" ..\n ngx.var.server_addr .. \":\" .. ngx.var.server_port ..\n \"/abort_ngx\"\n )\n if not res then\n ngx.log(ngx.ERR, err)\n end\n\n ngx.say(\"OK\")\n }\n }\n location = /abort_ngx {\n rewrite ^ /abort_prx break;\n proxy_pass http://test-upstream;\n }\n location = /abort_prx {\n rewrite ^(.*)_prx$ $1 break;\n keepalive_requests 1;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n }\n location = /abort {\n content_by_lua_block {\n ngx.status = 200\n ngx.header[\"Cache-Control\"] = \"public, max-age=3600\"\n ngx.say(\"START\")\n ngx.say(\"FINISH\")\n }\n }\n--- request\nGET /abort_top\n--- response_body\nOK\n--- error_code: 200\n--- no_error_log\n[error]\n" }, { "path": "t/02-integration/origin_mode.t", "content": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::http_config();\n\nno_long_string();\nno_diff();\nrun_tests();\n\n__DATA__\n=== TEST 1: ORIGIN_MODE_NORMAL\n--- http_config eval: $::HttpConfig\n--- config\nlocation /origin_mode_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler({\n origin_mode = require(\"ledge\").ORIGIN_MODE_NORMAL\n }):run()\n }\n}\nlocation /origin_mode {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"public, max-age=60\"\n ngx.print(\"OK\")\n }\n}\n--- request eval\n[\"GET /origin_mode_prx\", \"GET /origin_mode_prx\"]\n--- response_body eval\n[\"OK\", \"OK\"]\n--- response_headers_like eval\n[\"X-Cache: MISS from .*\", \"X-Cache: HIT from .*\"]\n--- no_error_log\n[error]\n\n\n=== TEST 2: ORIGIN_MODE_AVOID (no-cache request)\n--- http_config eval: $::HttpConfig\n--- config\nlocation /origin_mode_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler({\n origin_mode = require(\"ledge\").ORIGIN_MODE_AVOID\n }):run()\n }\n}\n--- more_headers\nCache-Control: no-cache\n--- request\nGET /origin_mode_prx\n--- response_body: OK\n--- response_headers_like\nX-Cache: HIT from .*\n--- no_error_log\n[error]\n\n\n=== TEST 2a: ORIGIN_MODE_AVOID (max-age=0 request)\n--- http_config eval: $::HttpConfig\n--- config\nlocation /origin_mode_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler({\n origin_mode = require(\"ledge\").ORIGIN_MODE_AVOID\n }):run()\n }\n}\n--- more_headers\nCache-Control: max-age=0\n--- request\nGET /origin_mode_prx\n--- response_body: OK\n--- response_headers_like\nX-Cache: HIT from .*\n--- no_error_log\n[error]\n\n\n=== TEST 2b: ORIGIN_MODE_AVOID (expired cache)\n--- http_config eval: $::HttpConfig\n--- config\nlocation /origin_mode_2b_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler({\n origin_mode = require(\"ledge\").ORIGIN_MODE_AVOID\n })\n\n handler:bind(\"before_save\", function(res)\n -- immediately expire\n res.header[\"Cache-Control\"] = \"max-age=0\"\n end)\n handler:run()\n }\n}\nlocation /origin_mode_2b {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"public, max-age=60\"\n ngx.print(\"OK\")\n }\n}\n--- request eval\n[\"GET /origin_mode_2b_prx\", \"GET /origin_mode_2b_prx\"]\n--- response_body eval\n[\"OK\", \"OK\"]\n--- response_headers_like eval\n[\"X-Cache: MISS from .*\", \"X-Cache: HIT from .*\"]\n--- no_error_log\n[error]\n\n\n=== TEST 3: ORIGIN_MODE_BYPASS when cached with 112 warning\n--- http_config eval: $::HttpConfig\n--- config\nlocation /origin_mode_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler({\n origin_mode = require(\"ledge\").ORIGIN_MODE_BYPASS\n }):run()\n }\n}\n--- more_headers\nCache-Control: no-cache\n--- request\nGET /origin_mode_prx\n--- response_headers_like\nWarning: 112 .*\n--- response_body: OK\n--- no_error_log\n[error]\n\n\n=== TEST 4: ORIGIN_MODE_BYPASS when we have nothing\n--- http_config eval: $::HttpConfig\n--- config\nlocation /origin_mode_bypass_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler({\n origin_mode = require(\"ledge\").ORIGIN_MODE_BYPASS\n }):run()\n }\n}\n--- more_headers\nCache-Control: no-cache\n--- request\nGET /origin_mode_bypass_prx\n--- error_code: 503\n--- no_error_log\n[error]\n" }, { "path": "t/02-integration/purge.t", "content": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::http_config(extra_nginx_config => qq{\n lua_shared_dict ledge_test 1m;\n}, extra_lua_config => qq{\n function format_json(json, prefix)\n local decode = require(\"cjson\").decode\n if type(json) == \"string\" then\n local ok\n ok, json = pcall(decode, json)\n if not ok then return \"\" end\n end\n local keys = {}\n for k, v in pairs(json) do\n table.insert(keys, k)\n end\n table.sort(keys)\n\n local fmt = \"%s: %s\\\\n\"\n local out = \"\"\n for i, k in ipairs(keys) do\n local key = k\n if prefix then\n key = prefix..\".\"..k\n end\n if type(json[k]) == \"table\" then\n out = out .. format_json(json[k], key)\n else\n out = out .. fmt:format(key, json[k])\n end\n end\n return out\n end\n}, run_worker => 1);\n\nno_long_string();\nno_diff();\nrun_tests();\n\n__DATA__\n=== TEST 1: Prime cache for subsequent tests\n--- http_config eval: $::HttpConfig\n--- config\nlocation /purge_cached_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /purge_cached {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.say(\"TEST 1\")\n }\n}\n--- request\nGET /purge_cached_prx\n--- no_error_log\n[error]\n--- response_body\nTEST 1\n\n\n=== TEST 2: Purge cache\n--- http_config eval: $::HttpConfig\n--- config\nlocation /purge_cached {\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n body_filter_by_lua_block {\n ngx.arg[1] = format_json(ngx.arg[1])\n ngx.arg[2] = true\n }\n}\n\n--- request eval\n[\"PURGE /purge_cached\", \"PURGE /purge_cached\"]\n--- no_error_log\n[error]\n--- response_body eval\n[\n'purge_mode: invalidate\nresult: purged\n',\n'purge_mode: invalidate\nresult: already expired\n']\n--- error_code eval\n[200, 404]\n\n\n=== TEST 3: Cache has been purged\n--- http_config eval: $::HttpConfig\n--- config\nlocation /purge_cached_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /purge_cached {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.say(\"TEST 3\")\n }\n}\n--- request\nGET /purge_cached_prx\n--- no_error_log\n[error]\n--- response_body\nTEST 3\n\n\n=== TEST 4: Purge on unknown key returns 404\n--- http_config eval: $::HttpConfig\n--- config\nlocation /foobar {\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n body_filter_by_lua_block {\n ngx.arg[1] = format_json(ngx.arg[1])\n ngx.arg[2] = true\n }\n}\n\n--- request\nPURGE /foobar\n--- no_error_log\n[error]\n--- response_body\npurge_mode: invalidate\nresult: nothing to purge\n\n--- error_code: 404\n\n\n=== TEST 5a: Prime another key with args\n--- http_config eval: $::HttpConfig\n--- config\nlocation /purge_cached_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler({\n keep_cache_for = 0,\n }):run()\n }\n}\nlocation /purge_cached {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.say(\"TEST 5\")\n }\n}\n--- request\nGET /purge_cached_prx?t=1\n--- no_error_log\n[error]\n--- response_body\nTEST 5\n\n\n=== TEST 5b: Wildcard Purge\n--- http_config eval: $::HttpConfig\n--- config\nlocation /purge_cached {\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n body_filter_by_lua_block {\n ngx.arg[1] = format_json(ngx.arg[1])\n ngx.arg[2] = true\n }\n}\n--- request\nPURGE /purge_cached*\n--- wait: 1\n--- no_error_log\n[error]\n--- response_body_like\npurge_mode: invalidate\nqless_jobs.1.jid: [a-f0-9]{32}\nqless_jobs.1.klass: ledge.jobs.purge\nqless_jobs.1.options.jid: [a-f0-9]{32}\nqless_jobs.1.options.priority: 5\nqless_jobs.1.options.tags.1: purge\nresult: scheduled\n--- error_code: 200\n\n\n=== TEST 5c: Cache has been purged with args\n--- http_config eval: $::HttpConfig\n--- config\nlocation /purge_cached_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler({\n keep_cache_for = 0,\n }):run()\n }\n}\nlocation /purge_cached {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.say(\"TEST 5c\")\n }\n}\n--- request\nGET /purge_cached_prx?t=1\n--- no_error_log\n[error]\n--- response_body\nTEST 5c\n\n\n=== TEST 5d: Cache has been purged without args\n--- http_config eval: $::HttpConfig\n--- config\nlocation /purge_cached_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler({\n keep_cache_for = 0,\n }):run()\n }\n}\nlocation /purge_cached {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.say(\"TEST 5d\")\n }\n}\n--- request\nGET /purge_cached_prx\n--- no_error_log\n[error]\n--- response_body\nTEST 5d\n\n\n=== TEST 6a: Purge everything\n--- http_config eval: $::HttpConfig\n--- config\nlocation /purge_c {\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n body_filter_by_lua_block {\n ngx.arg[1] = format_json(ngx.arg[1])\n ngx.arg[2] = true\n }\n}\n--- request\nPURGE /purge_c*\n--- wait: 3\n--- error_code: 200\n--- response_body_like\npurge_mode: invalidate\nqless_jobs.1.jid: [a-f0-9]{32}\nqless_jobs.1.klass: ledge.jobs.purge\nqless_jobs.1.options.jid: [a-f0-9]{32}\nqless_jobs.1.options.priority: 5\nqless_jobs.1.options.tags.1: purge\n--- no_error_log\n[error]\n\n\n=== TEST 6: Cache keys have been collected by Redis\n--- http_config eval: $::HttpConfig\n--- config\nlocation /purge_cached {\n content_by_lua_block {\n local redis = require(\"ledge\").create_redis_connection()\n local handler = require(\"ledge\").create_handler()\n handler.redis = redis\n local key_chain = handler:cache_key_chain()\n\n local num_entities, err = redis:scard(key_chain.entities)\n ngx.say(\"entities: \", num_entities)\n }\n}\n--- request\nGET /purge_cached\n--- no_error_log\n[error]\n--- response_body\nentities: 0\n\n\n=== TEST 7a: Prime another key with args\n--- http_config eval: $::HttpConfig\n--- config\nlocation /purge_cached_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /purge_cached {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.say(\"TEST 5\")\n }\n}\n--- request\nGET /purge_cached_prx?t=1\n--- no_error_log\n[error]\n--- response_body\nTEST 5\n\n\n=== TEST 7b: Wildcard Purge, mid path (no match due to args)\n--- http_config eval: $::HttpConfig\n--- config\nlocation /purge_c {\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n body_filter_by_lua_block {\n ngx.arg[1] = format_json(ngx.arg[1])\n ngx.arg[2] = true\n }\n}\n--- request\nPURGE /purge_ca*ed\n--- wait: 1\n--- no_error_log\n[error]\n--- response_body_like\npurge_mode: invalidate\nqless_jobs.1.jid: [a-f0-9]{32}\nqless_jobs.1.klass: ledge.jobs.purge\nqless_jobs.1.options.jid: [a-f0-9]{32}\nqless_jobs.1.options.priority: 5\nqless_jobs.1.options.tags.1: purge\nresult: scheduled\n--- error_code: 200\n\n\n=== TEST 7c: Confirm purge did nothing\n--- http_config eval: $::HttpConfig\n--- config\nlocation /purge_cached_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\n--- request\nGET /purge_cached_prx?t=1\n--- no_error_log\n[error]\n--- response_body\nTEST 5\n\n\n=== TEST 8a: Prime another key - with keep_cache_for set\n--- http_config eval: $::HttpConfig\n--- config\nlocation /purge_cached_8_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler({\n keep_cache_for = 3600,\n }):run()\n }\n}\nlocation /purge_cached_8 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.say(\"TEST 8\")\n }\n}\n--- request\nGET /purge_cached_8_prx\n--- no_error_log\n[error]\n--- response_body\nTEST 8\n\n\n=== TEST 8b: Wildcard Purge (200)\n--- http_config eval: $::HttpConfig\n--- config\nlocation /purge_cached_8 {\n content_by_lua_block {\n require(\"ledge\").create_handler({\n keyspace_scan_count = 1,\n }):run()\n }\n body_filter_by_lua_block {\n ngx.arg[1] = format_json(ngx.arg[1])\n ngx.arg[2] = true\n }\n}\n--- request\nPURGE /purge_cached_8*\n--- wait: 2\n--- no_error_log\n[error]\n--- response_body_like\npurge_mode: invalidate\nqless_jobs.1.jid: [a-f0-9]{32}\nqless_jobs.1.klass: ledge.jobs.purge\nqless_jobs.1.options.jid: [a-f0-9]{32}\nqless_jobs.1.options.priority: 5\nqless_jobs.1.options.tags.1: purge\nresult: scheduled\n--- error_code: 200\n\n\n=== TEST 8d: Cache has been purged with args\n--- http_config eval: $::HttpConfig\n--- config\nlocation /purge_cached_8_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /purge_cached_8 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.say(\"TEST 8c\")\n }\n}\n--- request\nGET /purge_cached_8_prx\n--- no_error_log\n[error]\n--- response_body\nTEST 8c\n--- error_code: 200\n\n\n=== TEST 9a: Prime another key\n--- http_config eval: $::HttpConfig\n--- config\nlocation /purge_cached_9_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler({\n keep_cache_for = 3600,\n }):run()\n }\n}\nlocation /purge_cached_9 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.say(\"TEST 9: \", ngx.req.get_headers()[\"Cookie\"])\n }\n}\n--- more_headers\nCookie: primed\n--- request\nGET /purge_cached_9_prx\n--- no_error_log\n[error]\n--- response_body\nTEST 9: primed\n\n\n=== TEST 9b: Purge with X-Purge: revalidate\n--- http_config eval: $::HttpConfig\n--- config\nlocation /purge_cached_9_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n body_filter_by_lua_block {\n ngx.arg[1] = format_json(ngx.arg[1])\n ngx.arg[2] = true\n }\n}\nlocation /purge_cached_9 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.say(\"TEST 9 Revalidated: \", ngx.req.get_headers()[\"Cookie\"])\n }\n}\n--- more_headers\nX-Purge: revalidate\n--- request\nPURGE /purge_cached_9_prx\n--- no_error_log\n[error]\n--- response_body_like\npurge_mode: revalidate\nqless_jobs.1.jid: [a-f0-9]{32}\nqless_jobs.1.klass: ledge.jobs.revalidate\nqless_jobs.1.options.jid: [a-f0-9]{32}\nqless_jobs.1.options.priority: 4\nqless_jobs.1.options.tags.1: revalidate\nresult: purged\n--- error_code: 200\n\n\n=== TEST 9c: Wait for revalidation to complete\n--- http_config eval: $::HttpConfig\n--- config\nlocation /purge_cached_9_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /purge_cached_9 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.say(\"TEST 9 Revalidated: \", ngx.req.get_headers()[\"Cookie\"])\n }\n}\nlocation /waiting {\n echo \"OK\";\n}\n--- request\nGET /waiting\n--- response_body\nOK\n--- no_error_log\n[error]\n--- wait: 5\n\n\n=== TEST 9d: Confirm cache was revalidated\n--- http_config eval: $::HttpConfig\n--- config\nlocation /purge_cached_9_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\n--- request\nGET /purge_cached_9_prx\n--- wait: 3\n--- no_error_log\n[error]\n--- response_body\nTEST 9 Revalidated: primed\n\n\n=== TEST 10a: Prime two keys\n--- http_config eval: $::HttpConfig\n--- config\nlocation /purge_cached_10_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /purge_cached_10 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.print(\"TEST 10: \", ngx.req.get_uri_args()[\"a\"], \" \", ngx.req.get_headers()[\"Cookie\"])\n }\n}\n--- more_headers\nCookie: primed\n--- request eval\n[ \"GET /purge_cached_10_prx?a=1\", \"GET /purge_cached_10_prx?a=2\" ]\n--- no_error_log\n[error]\n--- response_body eval\n[ \"TEST 10: 1 primed\", \"TEST 10: 2 primed\" ]\n\n\n=== TEST 10b: Wildcard purge with X-Purge: revalidate\n--- http_config eval: $::HttpConfig\n--- config\nlocation /purge_cached_10_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n body_filter_by_lua_block {\n ngx.arg[1] = format_json(ngx.arg[1])\n ngx.arg[2] = true\n }\n}\nlocation /purge_cached_10 {\n rewrite ^(.*)$ $1_origin break;\n content_by_lua_block {\n local a = ngx.req.get_uri_args()[\"a\"]\n ngx.log(ngx.DEBUG, \"TEST 10 Revalidated: \", a, \" \", ngx.req.get_headers()[\"Cookie\"])\n }\n}\n--- more_headers\nX-Purge: revalidate\n--- request\nPURGE /purge_cached_10_prx?*\n--- wait: 2\n--- no_error_log\n[error]\n--- response_body_like\npurge_mode: revalidate\nqless_jobs.1.jid: [a-f0-9]{32}\nqless_jobs.1.klass: ledge.jobs.purge\nqless_jobs.1.options.jid: [a-f0-9]{32}\nqless_jobs.1.options.priority: 5\nqless_jobs.1.options.tags.1: purge\nresult: scheduled\n--- error_log\nTEST 10 Revalidated: 1 primed\nTEST 10 Revalidated: 2 primed\n--- error_code: 200\n\n\n=== TEST 11a: Prime a key\n--- http_config eval: $::HttpConfig\n--- config\nlocation /purge_cached_11_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler({\n keep_cache_for = 3600,\n }):run()\n }\n}\nlocation /purge_cached_11 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.print(\"TEST 11\")\n }\n}\n--- request\nGET /purge_cached_11_prx\n--- no_error_log\n[error]\n--- response_body: TEST 11\n\n\n=== TEST 11b: Purge with X-Purge: delete\n--- http_config eval: $::HttpConfig\n--- config\nlocation /purge_cached_11_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler({\n keep_cache_for = 3600,\n }):run()\n }\n body_filter_by_lua_block {\n ngx.arg[1] = format_json(ngx.arg[1])\n ngx.arg[2] = true\n }\n}\n--- more_headers\nX-Purge: delete\n--- request\nPURGE /purge_cached_11_prx\n--- no_error_log\n[error]\n--- response_body\npurge_mode: delete\nresult: deleted\n--- error_code: 200\n\n\n=== TEST 11c: Max-stale request fails as items are properly deleted\n--- http_config eval: $::HttpConfig\n--- config\nlocation /purge_cached_11_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler({\n keep_cache_for = 3600,\n }):run()\n }\n}\nlocation /purge_cached_11 {\n content_by_lua_block {\n ngx.print(\"ORIGIN\")\n }\n}\n--- more_headers\nCache-Control: max-stale=1000\n--- request\nGET /purge_cached_11_prx\n--- response_body: ORIGIN\n--- no_error_log\n[error]\n--- error_code: 200\n\n\n=== TEST 12a: Prime two keys\n--- http_config eval: $::HttpConfig\n--- config\nlocation /purge_cached_12_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler({\n keep_cache_for = 3600,\n }):run()\n }\n}\nlocation /purge_cached_12 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.print(\"TEST 12: \", ngx.req.get_uri_args()[\"a\"])\n }\n}\n--- request eval\n[ \"GET /purge_cached_12_prx?a=1\", \"GET /purge_cached_12_prx?a=2\" ]\n--- no_error_log\n[error]\n--- response_body eval\n[ \"TEST 12: 1\", \"TEST 12: 2\" ]\n\n\n=== TEST 12b: Wildcard purge with X-Purge: delete\n--- http_config eval: $::HttpConfig\n--- config\nlocation /purge_cached_12_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler({\n keep_cache_for = 3600,\n }):run()\n }\n body_filter_by_lua_block {\n ngx.arg[1] = format_json(ngx.arg[1])\n ngx.arg[2] = true\n }\n}\n--- more_headers\nX-Purge: delete\n--- request\nPURGE /purge_cached_12_prx?*\n--- wait: 2\n--- no_error_log\n[error]\n--- response_body_like\npurge_mode: delete\nqless_jobs.1.jid: [a-f0-9]{32}\nqless_jobs.1.klass: ledge.jobs.purge\nqless_jobs.1.options.jid: [a-f0-9]{32}\nqless_jobs.1.options.priority: 5\nqless_jobs.1.options.tags.1: purge\nresult: scheduled\n--- error_code: 200\n\n\n=== TEST 12c: Max-stale request fails as items are properly deleted\n--- http_config eval: $::HttpConfig\n--- config\nlocation /purge_cached_12_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler({\n keep_cache_for = 3600,\n }):run()\n }\n}\nlocation /purge_cached_12 {\n content_by_lua_block {\n ngx.print(\"ORIGIN: \", ngx.req.get_uri_args()[\"a\"])\n }\n}\n--- more_headers\nCache-Control: max-stale=1000\n--- request eval\n[ \"GET /purge_cached_12_prx?a=1\", \"GET /purge_cached_12_prx?a=2\" ]\n--- no_error_log\n[error]\n--- response_body eval\n[ \"ORIGIN: 1\", \"ORIGIN: 2\" ]\n\n\n=== TEST 13a: Prime two keys and break them\n--- http_config eval: $::HttpConfig\n--- config\nlocation /purge_cached_13_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local sabotage = ngx.req.get_uri_args()[\"sabotage\"]\n if sabotage then\n -- Set query string to match original request\n ngx.req.set_uri_args({a=1})\n\n local redis = require(\"ledge\").create_redis_connection()\n local handler = require(\"ledge\").create_handler()\n handler.redis = redis\n local key_chain = handler:cache_key_chain()\n\n if sabotage == \"uri\" then\n redis:hdel(key_chain.main, \"uri\")\n ngx.print(\"Sabotaged: uri\")\n elseif sabotage == \"body\" then\n handler.storage = require(\"ledge\").create_storage_connection()\n\n handler.storage:delete(redis:hget(key_chain.main, entity))\n\n ngx.print(\"Sabotaged: body storage\")\n end\n else\n require(\"ledge\").create_handler():run()\n end\n }\n}\nlocation /purge_cached_13 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.print(\"TEST 13: \", ngx.req.get_uri_args()[\"a\"], \" \", ngx.req.get_headers()[\"Cookie\"])\n }\n}\n--- more_headers\nCookie: primed\n--- request eval\n[ \"GET /purge_cached_13_prx?a=1\",\n\"GET /purge_cached_13_prx?a=2\",\n\"GET /purge_cached_13_prx?a=1&sabotage=body\",\n\"GET /purge_cached_13_prx?a=1&sabotage=uri\" ]\n--- no_error_log\n[error]\n--- response_body_like eval\n[ \"TEST 13: 1 primed\",\n \"TEST 13: 2 primed\",\n \"Sabotaged: body storage\",\n \"Sabotaged: uri\" ]\n\n\n=== TEST 13b: Wildcard purge broken entry with X-Purge: revalidate\n--- http_config eval: $::HttpConfig\n--- config\nlocation /purge_cached_13_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n body_filter_by_lua_block {\n ngx.arg[1] = format_json(ngx.arg[1])\n ngx.arg[2] = true\n }\n}\nlocation /purge_cached_13 {\n rewrite ^(.*)$ $1_origin break;\n content_by_lua_block {\n local a = ngx.req.get_uri_args()[\"a\"]\n ngx.log(ngx.DEBUG, \"TEST 13 Revalidated: \", a, \" \", ngx.req.get_headers()[\"Cookie\"])\n }\n}\n--- more_headers\nX-Purge: revalidate\n--- request\nPURGE /purge_cached_13_prx?*\n--- wait: 2\n--- error_log\nTEST 13 Revalidated: 2 primed\n--- response_body_like\npurge_mode: revalidate\nqless_jobs.1.jid: [a-f0-9]{32}\nqless_jobs.1.klass: ledge.jobs.purge\nqless_jobs.1.options.jid: [a-f0-9]{32}\nqless_jobs.1.options.priority: 5\nqless_jobs.1.options.tags.1: purge\nresult: scheduled\n--- error_code: 200\n\n\n=== TEST 14: Purge API runs\n--- http_config eval: $::HttpConfig\n--- config\nlocation /purge_api {\n content_by_lua_block {\n require(\"ledge.state_machine\").set_debug(true)\n require(\"ledge\").create_handler():run()\n }\n body_filter_by_lua_block {\n ngx.arg[1] = format_json(ngx.arg[1])\n ngx.arg[2] = true\n }\n}\nlocation /purge_cached_14_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge.state_machine\").set_debug(false)\n require(\"ledge\").create_handler({\n keep_cache_for = 3600,\n }):run()\n }\n}\nlocation /purge_cached_14 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.print(\"TEST 14: \", ngx.req.get_uri_args()[\"a\"])\n }\n}\n--- request eval\n[\n\"GET /purge_cached_14_prx?a=1\", \"GET /purge_cached_14_prx?a=2\",\n\nqq(PURGE /purge_api\n{\"uris\": [\"http://localhost:$LedgeEnv::nginx_port/purge_cached_14_prx?a=1\", \"http://localhost:$LedgeEnv::nginx_port/purge_cached_14_prx?a=2\"]}),\n\n\"GET /purge_cached_14_prx?a=1\", \"GET /purge_cached_14_prx?a=2\",\n]\n--- more_headers eval\n[\n\"\",\"\",\n\"Content-Type: Application/JSON\",\n\"\",\"\",\n]\n--- response_body eval\n[\n\"TEST 14: 1\", \"TEST 14: 2\",\n\nqq(purge_mode: invalidate\nresult.http://localhost:$LedgeEnv::nginx_port/purge_cached_14_prx?a=1.result: purged\nresult.http://localhost:$LedgeEnv::nginx_port/purge_cached_14_prx?a=2.result: purged\n),\n\n\"TEST 14: 1\", \"TEST 14: 2\",\n]\n--- response_headers_like eval\n[\n\"X-Cache: MISS from .+\", \"X-Cache: MISS from .+\",\n\"Content-Type: application/json\",\n\"X-Cache: MISS from .+\", \"X-Cache: MISS from .+\",\n]\n--- no_error_log\n[error]\n\n\n=== TEST 15: Purge API wildcard query string\n--- http_config eval: $::HttpConfig\n--- config\nlocation /purge_api {\n content_by_lua_block {\n require(\"ledge.state_machine\").set_debug(true)\n require(\"ledge\").create_handler():run()\n }\n body_filter_by_lua_block {\n ngx.arg[1] = format_json(ngx.arg[1])\n ngx.arg[2] = true\n }\n}\nlocation /purge_cached_15_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge.state_machine\").set_debug(false)\n require(\"ledge\").create_handler({\n keep_cache_for = 3600,\n }):run()\n }\n}\nlocation /purge_cached_15 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.print(\"TEST 15: \", ngx.req.get_uri_args()[\"a\"])\n }\n}\n--- request eval\n[\n\"GET /purge_cached_15_prx?a=1\", \"GET /purge_cached_15_prx?a=2\",\n\nqq(PURGE /purge_api\n{\"uris\": [\"http://localhost:$LedgeEnv::nginx_port/purge_cached_15_prx?a*\"]}),\n]\n--- more_headers eval\n[\n\"\",\"\",\n\"Content-Type: Application/JSON\",\n]\n--- response_body_like eval\n[\n\"TEST 15: 1\", \"TEST 15: 2\",\n\nqq(purge_mode: invalidate\nresult.http://localhost:$LedgeEnv::nginx_port/purge_cached_15_prx\\\\?a\\\\*.qless_jobs.1.jid: [a-f0-9]{32}\nresult.http://localhost:$LedgeEnv::nginx_port/purge_cached_15_prx\\\\?a\\\\*.qless_jobs.1.klass: ledge.jobs.purge\nresult.http://localhost:$LedgeEnv::nginx_port/purge_cached_15_prx\\\\?a\\\\*.qless_jobs.1.options.jid: [a-f0-9]{32}\nresult.http://localhost:$LedgeEnv::nginx_port/purge_cached_15_prx\\\\?a\\\\*.qless_jobs.1.options.priority: 5\nresult.http://localhost:$LedgeEnv::nginx_port/purge_cached_15_prx\\\\?a\\\\*.qless_jobs.1.options.tags.1: purge\nresult.http://localhost:$LedgeEnv::nginx_port/purge_cached_15_prx\\\\?a\\\\*.result: scheduled\n),\n]\n--- response_headers_like eval\n[\n\"X-Cache: MISS from .+\", \"X-Cache: MISS from .+\",\n\"Content-Type: application/json\",\n]\n--- wait: 2\n--- no_error_log\n[error]\n\n=== TEST 15b: Purge API wildcard query string\n--- http_config eval: $::HttpConfig\n--- config\nlocation /purge_cached_15_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge.state_machine\").set_debug(false)\n require(\"ledge\").create_handler({\n keep_cache_for = 3600,\n }):run()\n }\n}\nlocation /purge_cached_15 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.print(\"TEST 15b: \", ngx.req.get_uri_args()[\"a\"])\n }\n}\n--- request eval\n[\"GET /purge_cached_15_prx?a=1\", \"GET /purge_cached_15_prx?a=2\"]\n--- response_body_like eval\n[\"TEST 15b: 1\", \"TEST 15b: 2\"]\n--- response_headers_like eval\n[\"X-Cache: MISS from .+\", \"X-Cache: MISS from .+\"]\n--- no_error_log\n[error]\n\n=== TEST 16: Purge API wildcards\n--- http_config eval: $::HttpConfig\n--- config\nlocation /purge_api {\n content_by_lua_block {\n require(\"ledge.state_machine\").set_debug(true)\n require(\"ledge\").create_handler():run()\n }\n body_filter_by_lua_block {\n ngx.arg[1] = format_json(ngx.arg[1])\n ngx.arg[2] = true\n }\n}\nlocation /purge_cached_16_prx {\n rewrite ^(.*)_prx(.*)? $1$2 break;\n content_by_lua_block {\n require(\"ledge.state_machine\").set_debug(false)\n require(\"ledge\").create_handler({\n keep_cache_for = 3600,\n }):run()\n }\n}\nlocation /purge_cached_16 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.print(\"TEST 16: \", ngx.req.get_uri_args()[\"a\"])\n }\n}\n--- request eval\n[\n\"GET /purge_cached_16_prx?a=1\", \"GET /purge_cached_16_prx?a=2\",\n\nqq(PURGE /purge_api\n{\"uris\": [\"http://localhost:$LedgeEnv::nginx_port/purge_cached_16_prx*\"]}),\n]\n--- more_headers eval\n[\n\"\",\"\",\n\"Content-Type: Application/JSON\",\n]\n--- response_body_like eval\n[\n\"TEST 16: 1\", \"TEST 16: 2\",\n\nqq(purge_mode: invalidate\nresult.http://localhost:$LedgeEnv::nginx_port/purge_cached_16_prx\\\\*.qless_jobs.1.jid: [a-f0-9]{32}\nresult.http://localhost:$LedgeEnv::nginx_port/purge_cached_16_prx\\\\*.qless_jobs.1.klass: ledge.jobs.purge\nresult.http://localhost:$LedgeEnv::nginx_port/purge_cached_16_prx\\\\*.qless_jobs.1.options.jid: [a-f0-9]{32}\nresult.http://localhost:$LedgeEnv::nginx_port/purge_cached_16_prx\\\\*.qless_jobs.1.options.priority: 5\nresult.http://localhost:$LedgeEnv::nginx_port/purge_cached_16_prx\\\\*.qless_jobs.1.options.tags.1: purge\nresult.http://localhost:$LedgeEnv::nginx_port/purge_cached_16_prx\\\\*.result: scheduled\n),\n]\n--- response_headers_like eval\n[\n\"X-Cache: MISS from .+\", \"X-Cache: MISS from .+\",\n\"Content-Type: application/json\",\n]\n--- wait: 2\n--- no_error_log\n[error]\n\n=== TEST 16b: Purge API wildcard check\n--- http_config eval: $::HttpConfig\n--- config\nlocation /purge_cached_16_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge.state_machine\").set_debug(false)\n require(\"ledge\").create_handler({\n keep_cache_for = 3600,\n }):run()\n }\n}\nlocation /purge_cached_16 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.print(\"TEST 16b: \", ngx.req.get_uri_args()[\"a\"])\n }\n}\n--- request eval\n[\"GET /purge_cached_16_prx?a=1\", \"GET /purge_cached_16_prx?a=2\"]\n--- response_body_like eval\n[\"TEST 16b: 1\", \"TEST 16b: 2\"]\n--- response_headers_like eval\n[\"X-Cache: MISS from .+\", \"X-Cache: MISS from .+\"]\n--- no_error_log\n[error]\n\n=== TEST 17: Purge API - bad request\n--- http_config eval: $::HttpConfig\n--- config\nlocation /purge_api {\n content_by_lua_block {\n require(\"ledge.state_machine\").set_debug(true)\n require(\"ledge\").create_handler():run()\n }\n body_filter_by_lua_block {\n ngx.arg[1] = format_json(ngx.arg[1])\n ngx.arg[2] = true\n }\n}\n\n--- request eval\n[\n'PURGE /purge_api\n{\"uris\": [\"foobar\"]}',\n\n'PURGE /purge_api\nthis is not valid json',\n\n'PURGE /purge_api\n{\"foo\": [\"bar\"]}',\n\n'PURGE /purge_api\n{\"uris\": []}',\n\n'PURGE /purge_api\n{\"uris\": \"not an array\"}',\n\n'PURGE /purge_api\n{\"uris\": [\"http://www.example.com/\"], \"purge_mode\": \"foobar\"}'\n]\n--- more_headers\nContent-Type: Application/JSON\n--- error_code eval\n[200,400,400,400,400,400]\n--- response_body eval\n[\n\"purge_mode: invalidate\nresult.foobar.error: bad uri: foobar\n\",\n\"error: Could not parse request body: Expected value but found invalid token at character 1\n\",\n\"error: No URIs provided\n\",\n\"error: No URIs provided\n\",\n\"error: Field 'uris' must be an array\n\",\n\"error: Invalid purge_mode\n\",\n]\n--- no_error_log\n[error]\n\n\n=== TEST 17: Purge API passes through purge_mode\n--- http_config eval: $::HttpConfig\n--- config\nlocation /purge_api {\n content_by_lua_block {\n require(\"ledge.state_machine\").set_debug(true)\n require(\"ledge\").create_handler():run()\n }\n body_filter_by_lua_block {\n ngx.arg[1] = format_json(ngx.arg[1])\n ngx.arg[2] = true\n }\n}\nlocation /purge_cached_17_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge.state_machine\").set_debug(false)\n require(\"ledge\").create_handler({\n keep_cache_for = 3600,\n }):run()\n }\n}\nlocation /purge_cached_17 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.print(\"TEST 17: \", ngx.req.get_uri_args()[\"a\"])\n }\n}\n--- request eval\n[\n\"GET /purge_cached_17_prx?a=1\",\n\nqq(PURGE /purge_api\n{\"purge_mode\": \"revalidate\", \"uris\": [\"http://localhost:$LedgeEnv::nginx_port/purge_cached_17_prx?a=1\"]}),\n]\n--- more_headers eval\n[\n\"\", \"Content-Type: Application/JSON\",\n]\n--- response_body_like eval\n[\n\"TEST 17: 1\",\n\nqq(purge_mode: revalidate\nresult.http://localhost:$LedgeEnv::nginx_port/purge_cached_17_prx\\\\?a=1.qless_jobs.1.jid: [a-f0-9]{32}\nresult.http://localhost:$LedgeEnv::nginx_port/purge_cached_17_prx\\\\?a=1.qless_jobs.1.klass: ledge.jobs.revalidate\nresult.http://localhost:$LedgeEnv::nginx_port/purge_cached_17_prx\\\\?a=1.qless_jobs.1.options.jid: [a-f0-9]{32}\nresult.http://localhost:$LedgeEnv::nginx_port/purge_cached_17_prx\\\\?a=1.qless_jobs.1.options.priority: 4\nresult.http://localhost:$LedgeEnv::nginx_port/purge_cached_17_prx\\\\?a=1.qless_jobs.1.options.tags.1: revalidate\nresult.http://localhost:$LedgeEnv::nginx_port/purge_cached_17_prx\\\\?a=1.result: purged\n),\n\n]\n--- wait: 1\n\n=== TEST 18: Purge clears all representations\n--- http_config eval: $::HttpConfig\n--- config\nlocation /purge {\n rewrite ^ /purge_cached_18 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n body_filter_by_lua_block {\n ngx.arg[1] = format_json(ngx.arg[1])\n ngx.arg[2] = true\n }\n}\nlocation /purge_cached_18_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler({\n keep_cache_for = 3600,\n }):run()\n }\n}\nlocation /purge_cached_18 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.header[\"Vary\"] = \"X-Test\"\n ngx.print(\"TEST 18: \", ngx.req.get_headers()[\"X-Test\"])\n }\n}\n--- request eval\n[\n\"GET /purge_cached_18_prx\", \"GET /purge_cached_18_prx\",\n\n\"PURGE /purge\",\n\n\"GET /purge_cached_18_prx\", \"GET /purge_cached_18_prx\",\n]\n--- more_headers eval\n[\n\"X-Test: abc\", \"X-Test: xyz\",\n\"\",\n\"X-Test: abc\", \"X-Test: xyz\",\n]\n--- response_body eval\n[\n\"TEST 18: abc\", \"TEST 18: xyz\",\n\n\"purge_mode: invalidate\nresult: purged\n\",\n\n\"TEST 18: abc\", \"TEST 18: xyz\",\n]\n--- response_headers_like eval\n[\n\"X-Cache: MISS from .+\", \"X-Cache: MISS from .+\",\n\"\",\n\"X-Cache: MISS from .+\", \"X-Cache: MISS from .+\",\n]\n--- no_error_log\n[error]\n\n=== TEST 19: Purge response with no body\n--- http_config eval: $::HttpConfig\n--- config\nlocation /purge {\n rewrite ^ /purge_cached_19 break;\n content_by_lua_block {\n require(\"ledge.state_machine\").set_debug(true)\n require(\"ledge\").create_handler():run()\n }\n body_filter_by_lua_block {\n ngx.arg[1] = format_json(ngx.arg[1])\n ngx.arg[2] = true\n }\n}\nlocation /purge_cached_19_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge.state_machine\").set_debug(false)\n require(\"ledge\").create_handler({\n keep_cache_for = 3600,\n }):run()\n }\n}\nlocation /purge_cached_19 {\n content_by_lua_block {\n local incr = ngx.shared.ledge_test:incr(\"test19\", 1, 0)\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.header[\"X-Incr\"] = incr\n }\n}\n--- request eval\n[\n\"GET /purge_cached_19_prx\", \"GET /purge_cached_19_prx\",\n\n\"PURGE /purge\",\n\n\"GET /purge_cached_19_prx\"\n]\n--- error_code eval\n[200, 200, 200, 200]\n--- response_headers_like eval\n[\n\"X-Cache: MISS from .+\nX-Incr: 1\",\n\n\"X-Cache: HIT from .+\nX-Incr: 1\",\n\n\"\",\n\n\"X-Cache: MISS from .+\nX-Incr: 2\"\n]\n--- no_error_log\n[error]\n" }, { "path": "t/02-integration/range.t", "content": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::http_config(run_worker => 1);\n\nno_long_string();\nno_diff();\nrun_tests();\n\n__DATA__\n=== TEST 1: Prime cache for subsequent tests.\n--- http_config eval: $::HttpConfig\n--- config\n location /range_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n }\n\n location /range {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"public, max-age=3600\";\n ngx.print(\"0123456789\");\n }\n }\n--- request\nGET /range_prx\n--- response_body: 0123456789\n--- error_code: 200\n--- no_error_log\n[error]\n\n\n=== TEST 2: Cache HIT, get the first byte only\n--- http_config eval: $::HttpConfig\n--- config\n location /range_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n }\n--- more_headers\nRange: bytes=0-1\n--- request\nGET /range_prx\n--- response_headers_like\nX-Cache: HIT from .*\n--- response_headers\nContent-Range: bytes 0-1/10\nCache-Control: public, max-age=3600\n--- response_body: 01\n--- error_code: 206\n--- no_error_log\n[error]\n\n\n=== TEST 3: Cache HIT, get middle bytes\n--- http_config eval: $::HttpConfig\n--- config\n location /range_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n }\n--- more_headers\nRange: bytes=3-5\n--- request\nGET /range_prx\n--- response_headers_like\nX-Cache: HIT from .*\n--- response_headers\nContent-Range: bytes 3-5/10\nCache-Control: public, max-age=3600\n--- response_body: 345\n--- error_code: 206\n--- no_error_log\n[error]\n\n\n=== TEST 4: Cache HIT, get middle to end bytes\n--- http_config eval: $::HttpConfig\n--- config\n location /range_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n }\n--- more_headers\nRange: bytes=6-\n--- request\nGET /range_prx\n--- response_headers_like\nX-Cache: HIT from .*\n--- response_headers\nContent-Range: bytes 6-9/10\nCache-Control: public, max-age=3600\n--- response_body: 6789\n--- error_code: 206\n--- no_error_log\n[error]\n\n\n=== TEST 5: Cache HIT, get offset from end bytes.\n--- http_config eval: $::HttpConfig\n--- config\n location /range_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n }\n--- more_headers\nRange: bytes=-4\n--- request\nGET /range_prx\n--- response_headers_like\nX-Cache: HIT from .*\n--- response_headers\nContent-Range: bytes 6-9/10\nCache-Control: public, max-age=3600\n--- response_body: 6789\n--- error_code: 206\n--- no_error_log\n[error]\n\n\n=== TEST 5b: Cache HIT, get byte to end\n--- http_config eval: $::HttpConfig\n--- config\n location /range_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n }\n--- more_headers\nRange: bytes=2-\n--- request\nGET /range_prx\n--- response_headers_like\nX-Cache: HIT from .*\n--- response_headers\nContent-Range: bytes 2-9/10\nCache-Control: public, max-age=3600\n--- response_body: 23456789\n--- error_code: 206\n--- no_error_log\n[error]\n\n\n=== TEST 6: Cache HIT, get beginning bytes spanning buffer size\n--- http_config eval: $::HttpConfig\n--- config\n location /range_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler({\n buffer_size = 2,\n }):run()\n }\n }\n--- more_headers\nRange: bytes=0-5\n--- request\nGET /range_prx\n--- response_headers_like\nX-Cache: HIT from .*\n--- response_headers\nContent-Range: bytes 0-5/10\nCache-Control: public, max-age=3600\n--- response_body: 012345\n--- error_code: 206\n--- no_error_log\n[error]\n\n\n=== TEST 7: Cache HIT, get middle bytes spanning buffer size\n--- http_config eval: $::HttpConfig\n--- config\n location /range_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler({\n buffer_size = 4,\n }):run()\n }\n }\n--- more_headers\nRange: bytes=3-7\n--- request\nGET /range_prx\n--- response_headers_like\nX-Cache: HIT from .*\n--- response_headers\nContent-Range: bytes 3-7/10\nCache-Control: public, max-age=3600\n--- response_body: 34567\n--- error_code: 206\n--- no_error_log\n[error]\n\n\n=== TEST 8: Ask for range outside content length, last byte should be reduced to length.\n--- http_config eval: $::HttpConfig\n--- config\n location /range_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n }\n--- more_headers\nRange: bytes=3-12\n--- request\nGET /range_prx\n--- response_headers_like\nX-Cache: HIT from .*\n--- response_headers\nContent-Range: bytes 3-9/10\nCache-Control: public, max-age=3600\n--- response_body: 3456789\n--- error_code: 206\n--- no_error_log\n[error]\n\n\n=== TEST 9: Range end is smaller than range start (unsatisfiable)\n--- http_config eval: $::HttpConfig\n--- config\n location /range_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n }\n--- more_headers\nRange: bytes=12-3\n--- request\nGET /range_prx\n--- response_headers\nContent-Range: bytes */10\n--- response_body:\n--- error_code: 416\n--- no_error_log\n[error]\n\n\n=== TEST 9b: Range end offset is larger than range (unsatisfiable)\n--- http_config eval: $::HttpConfig\n--- config\n location /range_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n }\n--- more_headers\nRange: bytes=-12\n--- request\nGET /range_prx\n--- response_headers\nContent-Range: bytes */10\n--- response_body:\n--- error_code: 416\n--- no_error_log\n[error]\n\n\n=== TEST 10: Range is incompreshensible\n--- http_config eval: $::HttpConfig\n--- config\n location /range_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n }\n--- more_headers\nRange: bytes=asdfa\n--- request\nGET /range_prx\n--- response_headers\nContent-Range: bytes */10\n--- response_body:\n--- error_code: 416\n--- no_error_log\n[error]\n\n\n=== TEST 10b: Range is incompreshensible\n--- http_config eval: $::HttpConfig\n--- config\n location /range_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n }\n--- more_headers\nRange: isdfsdbytes=asdfa\n--- request\nGET /range_prx\n--- response_headers\nContent-Range: bytes */10\n--- response_body:\n--- error_code: 416\n--- no_error_log\n[error]\n\n\n=== TEST 11: Multi byte ranges\n--- http_config eval: $::HttpConfig\n--- config\n location /range_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n }\n--- more_headers\nRange: bytes=0-3,5-8\n--- request\nGET /range_prx\n--- no_error_log\n[error]\n--- response_body_like chop\n\n--[0-9a-z]+\nContent-Type: text/plain\nContent-Range: bytes 0-3/10\n\n0123\n--[0-9a-z]+\nContent-Type: text/plain\nContent-Range: bytes 5-8/10\n\n5678\n--[0-9a-z]+--\n--- error_code: 206\n--- no_error_log\n[error]\n\n\n=== TEST 12a: Prime cache with buffers smaller than range.\n--- http_config eval: $::HttpConfig\n--- config\n location /range_12_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler({\n buffer_size = 3,\n }):run()\n }\n }\n\n location /range_12 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"public, max-age=3600\";\n ngx.status = 200\n ngx.print(\"0123456789\");\n }\n }\n--- request\nGET /range_12_prx\n--- response_body: 0123456789\n--- no_error_log\n[error]\n\n\n=== TEST 12b: Multi byte ranges across chunk boundaries\n--- http_config eval: $::HttpConfig\n--- config\n location /range_12_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n }\n--- more_headers\nRange: bytes=0-3,5-8\n--- request\nGET /range_12_prx\n--- no_error_log\n[error]\n--- response_body_like chop\n\n--[0-9a-z]+\nContent-Type: text/plain\nContent-Range: bytes 0-3/10\n\n0123\n--[0-9a-z]+\nContent-Type: text/plain\nContent-Range: bytes 5-8/10\n\n5678\n--[0-9a-z]+--\n--- error_code: 206\n--- no_error_log\n[error]\n\n\n=== TEST 12c: Single range which spans chunk boundaries\n--- http_config eval: $::HttpConfig\n--- config\n location /range_12_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n }\n--- more_headers\nRange: bytes=4-7\n--- request\nGET /range_12_prx\n--- response_headers\nContent-Range: bytes 4-7/10\n--- response_body: 4567\n--- error_code: 206\n--- no_error_log\n[error]\n\n\n=== TEST 12d: Multi byte reversed ranges. Return in sane order.\n--- http_config eval: $::HttpConfig\n--- config\n location /range_12_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n }\n--- more_headers\nRange: bytes=5-8,0-3\n--- request\nGET /range_12_prx\n--- no_error_log\n[error]\n--- response_body_like chop\n\n--[0-9a-z]+\nContent-Type: text/plain\nContent-Range: bytes 0-3/10\n\n0123\n--[0-9a-z]+\nContent-Type: text/plain\nContent-Range: bytes 5-8/10\n\n5678\n--[0-9a-z]+--\n--- error_code: 206\n--- no_error_log\n[error]\n\n\n=== TEST 12d: Multi byte reversed overlapping ranges. Return in sane order and coalesced.\n--- http_config eval: $::HttpConfig\n--- config\n location /range_12_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n }\n--- more_headers\nRange: bytes=5-8,0-3,4-6\n--- request\nGET /range_12_prx\n--- no_error_log\n[error]\n--- response_body_like chop\n\n--[0-9a-z]+\nContent-Type: text/plain\nContent-Range: bytes 0-3/10\n\n0123\n--[0-9a-z]+\nContent-Type: text/plain\nContent-Range: bytes 4-8/10\n\n45678\n--[0-9a-z]+--\n--- error_code: 206\n--- no_error_log\n[error]\n\n\n=== TEST 12d: Multi byte reversed overlapping ranges. Return in sane order and coalesced to single range.\n--- http_config eval: $::HttpConfig\n--- config\n location /range_12_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n }\n--- more_headers\nRange: bytes=5-8,0-3,3-6\n--- request\nGET /range_12_prx\n--- response_headers\nContent-Range: bytes 0-8/10\n--- response_body: 012345678\n--- error_code: 206\n--- no_error_log\n[error]\n\n\n=== TEST 13a: Prime with ESI content, thus of interderminate length.\n--- http_config eval: $::HttpConfig\n--- config\n location /range_13_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler({\n esi_enabled = true,\n }):run()\n }\n }\n\n location /range_13 {\n default_type text/html;\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"public, max-age=3600\"\n ngx.header[\"Surrogate-Control\"] = 'content=\"ESI/1.0\"'\n ngx.status = 200\n ngx.print(\"01\");\n ngx.print(\"$(QUERY_STRING{a})\")\n ngx.print(\"56789\");\n }\n }\n--- request\nGET /range_13_prx?a=234\n--- response_body: 0123456789\n--- no_error_log\n[error]\n\n\n=== TEST 13b: Normal range over indeterminate length (must 200 with full reply)\n--- http_config eval: $::HttpConfig\n--- config\n location /range_13_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge.state_machine\").set_debug(true)\n require(\"ledge\").create_handler({\n esi_enabled = true,\n }):run()\n }\n }\n--- more_headers\nRange: bytes=0-5\n--- request\nGET /range_13_prx?a=234\n--- response_body: 0123456789\n--- error_code: 200\n--- no_error_log\n[error]\n\n\n=== TEST 13c: Offset to end over indeterminate length (must 200 with full reply)\n--- http_config eval: $::HttpConfig\n--- config\n location /range_13_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler({\n esi_enabled = true,\n }):run()\n }\n }\n--- more_headers\nRange: bytes=-5\n--- request\nGET /range_13_prx?a=234\n--- response_body: 0123456789\n--- error_code: 200\n--- no_error_log\n[error]\n\n\n=== TEST 13d: Range to end over indeterminate length (must 200 with full reply)\n--- http_config eval: $::HttpConfig\n--- config\n location /range_13_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler({\n esi_enabled = true,\n }):run()\n }\n }\n--- more_headers\nRange: bytes=5-\n--- request\nGET /range_13_prx?a=234\n--- response_body: 0123456789\n--- error_code: 200\n--- no_error_log\n[error]\n\n\n=== TEST 14: Confirm we do not cache 206 responses from upstream\n--- http_config eval: $::HttpConfig\n--- config\n location /range_14_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n }\n\n location /range_14 {\n content_by_lua_block {\n ngx.status = 206\n ngx.header[\"Cache-Control\"] = \"public, max-age=3600\";\n ngx.header[\"Content-Range\"] = \"bytes 0-5/10\"\n ngx.print(\"012345\");\n }\n }\n--- more_headers\nRange: bytes=0-5\n--- request eval\n[\"GET /range_14_prx\", \"GET /range_14_prx\"]\n--- raw_response_headers_unlike eval\n[\"X-Cache\", \"X-Cache\"]\n--- response_body eval\n[\"012345\", \"012345\"]\n--- wait: 1\n--- error_code eval\n[206, 206]\n--- no_error_log\n[error]\n\n\n=== TEST 15: Confirm we do not cache 416 responses from upstream\n--- http_config eval: $::HttpConfig\n--- config\n location /range_15_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n }\n\n location /range_15 {\n content_by_lua_block {\n ngx.status = 416\n ngx.header[\"Cache-Control\"] = \"public, max-age=3600\";\n ngx.header[\"Content-Range\"] = \"bytes */10\"\n }\n }\n--- more_headers\nRange: bytes=11-\n--- request eval\n[\"GET /range_15_prx\", \"GET /range_15_prx\"]\n--- raw_response_headers_unlike eval\n[\"X-Cache\", \"X-Cache\"]\n--- response_body eval\n[\"\", \"\"]\n--- error_code eval\n[416, 416]\n--- no_error_log\n[error]\n\n\n=== TEST 16: Confirm we do not attempt range processing on non-200 responses\n--- http_config eval: $::HttpConfig\n--- config\n location /range_16_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n }\n\n location /range_16 {\n content_by_lua_block {\n ngx.status = 404\n ngx.header[\"Cache-Control\"] = \"public, max-age=3600\";\n ngx.print(\"0123456789\")\n }\n }\n--- more_headers\nRange: bytes=0-5\n--- request eval\n[\"GET /range_16_prx\", \"GET /range_16_prx\"]\n--- response_headers_like eval\n[\"X-Cache: MISS from .*\", \"X-Cache: HIT from .*\"]\n--- response_body eval\n[\"0123456789\", \"0123456789\"]\n--- error_code eval\n[404, 404]\n--- no_error_log\n[error]\n\n\n=== TEST 17: Cache miss range request\nUpstream returns range, triggers background fetch\n--- http_config eval: $::HttpConfig\n--- config\n location /range_17_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge.state_machine\").set_debug(true)\n require(\"ledge\").create_handler():run()\n }\n }\n\n location /range_17 {\n content_by_lua_block {\n if ngx.req.get_headers()[\"Range\"] then\n ngx.status = 206\n ngx.header[\"Cache-Control\"] = \"public, max-age=3600\"\n ngx.header[\"Content-Range\"] = \"bytes 1-5/10\"\n ngx.print(\"012345\")\n else\n ngx.status = 200\n ngx.header[\"Cache-Control\"] = \"public, max-age=3600\"\n ngx.print(\"0123456789\")\n end\n }\n }\n--- more_headers\nRange: bytes=0-5\n--- request\nGET /range_17_prx\n--- response_body: 012345\n--- raw_response_headers_unlike\nX-Cache: .*\n--- wait: 2\n--- error_code: 206\n--- no_error_log\n[error]\n\n\n=== TEST 17b: Confirm revalidation, with a different range\n--- http_config eval: $::HttpConfig\n--- config\n location /range_17_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n }\n--- more_headers\nRange: bytes=6-\n--- request\nGET /range_17_prx\n--- response_body: 6789\n--- response_headers_like\nX-Cache: HIT from .*\n--- error_code: 206\n--- no_error_log\n[error]\n\n\n=== TEST 18: Cache miss range request, upstream returns range, but size is too big for background fetch\n--- http_config eval: $::HttpConfig\n--- config\nlocation /range_18_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n -- Set max memory on first hit, but not on background fetch.\n -- We want to test that the job is never started\n if ngx.req.get_headers()[\"Range\"] then\n local handler = require(\"ledge\").create_handler()\n handler.config.storage_driver_config.max_size = 9 -- < 10\n handler:run()\n else\n require(\"ledge\").create_handler():run()\n end\n }\n}\n\nlocation /range_18 {\n content_by_lua_block {\n if ngx.req.get_headers()[\"Range\"] then\n ngx.status = 206\n ngx.header[\"Cache-Control\"] = \"public, max-age=3600\";\n ngx.header[\"Content-Range\"] = \"bytes 0-5/10\"\n ngx.print(\"012345\");\n else\n ngx.status = 200\n ngx.header[\"Cache-Control\"] = \"public, max-age=3600\";\n ngx.print(\"0123456789\");\n end\n }\n}\n--- more_headers\nRange: bytes=0-5\n--- request\nGET /range_18_prx\n--- response_body: 012345\n--- raw_response_headers_unlike\nX-Cache: .*\n--- wait: 1\n--- error_code: 206\n--- no_error_log\n[error]\n\n\n=== TEST 18b: Confirm revalidation has not happened\n--- http_config eval: $::HttpConfig\n--- config\n location /range_18_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n }\n location /range_18 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"public, max-age=3600\";\n ngx.print(\"MISS\")\n }\n }\n--- more_headers\nRange: bytes=6-\n--- request\nGET /range_18_prx\n--- response_body: MISS\n--- response_headers_like\nX-Cache: MISS from .*\n--- error_code: 200\n--- no_error_log\n[error]\n\n\n=== TEST 19: Cache miss range request\nUpstream returns range, but size is unknown\n--- http_config eval: $::HttpConfig\n--- config\nlocation /range_19_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n -- Set max memory on first hit, but not on background fetch.\n -- We want to test that the job is never started\n if ngx.req.get_headers()[\"Range\"] then\n local handler = require(\"ledge\").create_handler()\n handler.config.storage_driver_config.max_size = 9 -- < 10\n handler:run()\n else\n require(\"ledge\").create_handler():run()\n end\n }\n}\n\nlocation /range_19 {\n content_by_lua_block {\n if ngx.req.get_headers()[\"Range\"] then\n ngx.status = 206\n ngx.header[\"Cache-Control\"] = \"public, max-age=3600\";\n ngx.header[\"Content-Range\"] = \"bytes 0-5/*\"\n ngx.print(\"012345\");\n else\n ngx.status = 200\n ngx.header[\"Cache-Control\"] = \"public, max-age=3600\";\n ngx.print(\"0123456789\");\n end\n }\n}\n--- more_headers\nRange: bytes=0-5\n--- request\nGET /range_19_prx\n--- response_body: 012345\n--- raw_response_headers_unlike\nX-Cache: .*\n--- wait: 1\n--- error_code: 206\n--- no_error_log\n[error]\n\n\n=== TEST 19b: Confirm revalidation has not happened\n--- http_config eval: $::HttpConfig\n--- config\n location /range_19_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n }\n location /range_19 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"public, max-age=3600\";\n ngx.print(\"MISS\")\n }\n }\n--- more_headers\nRange: bytes=6-\n--- request\nGET /range_19_prx\n--- response_body: MISS\n--- response_headers_like\nX-Cache: MISS from .*\n--- error_code: 200\n--- no_error_log\n[error]\n" }, { "path": "t/02-integration/req_body.t", "content": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::http_config();\n\nno_long_string();\nno_diff();\nrun_tests();\n\n__DATA__\n=== TEST 1: Should pass through request body\n--- http_config eval: $::HttpConfig\n--- config\nlocation /cached_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /cached {\n content_by_lua_block {\n ngx.req.read_body()\n ngx.say({ngx.req.get_body_data()})\n }\n}\n--- request\nPOST /cached_prx\nrequestbody\n--- response_body\nrequestbody\n" }, { "path": "t/02-integration/req_method.t", "content": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::http_config();\n\nno_long_string();\nno_diff();\nrun_tests();\n\n__DATA__\n=== TEST 1: GET\n--- http_config eval: $::HttpConfig\n--- config\nlocation /req_method_1_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /req_method_1 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.header[\"Etag\"] = \"req_method_1\"\n ngx.say(ngx.req.get_method())\n }\n}\n--- request\nGET /req_method_1_prx\n--- response_body\nGET\n--- no_error_log\n[error]\n\n\n=== TEST 2: HEAD gets GET request\n--- http_config eval: $::HttpConfig\n--- config\nlocation /req_method_1 {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\n--- request\nGET /req_method_1\n--- response_headers\nEtag: req_method_1\n--- no_error_log\n[error]\n\n\n=== TEST 3: HEAD revalidate\n--- http_config eval: $::HttpConfig\n--- config\nlocation /req_method_1_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /req_method_1 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.header[\"Etag\"] = \"req_method_1\"\n }\n}\n--- more_headers\nCache-Control: max-age=0\n--- request\nHEAD /req_method_1_prx\n--- response_headers\nEtag: req_method_1\n--- no_error_log\n[error]\n\n\n=== TEST 4: GET still has body\n--- http_config eval: $::HttpConfig\n--- config\nlocation /req_method_1 {\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\n--- request\nGET /req_method_1\n--- response_headers\nEtag: req_method_1\n--- response_body\nGET\n--- no_error_log\n[error]\n\n\n=== TEST 5: POST does not get cached copy\n--- http_config eval: $::HttpConfig\n--- config\nlocation /req_method_1_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /req_method_1 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.header[\"Etag\"] = \"req_method_posted\"\n ngx.say(ngx.req.get_method())\n }\n}\n--- request\nPOST /req_method_1_prx\n--- response_headers\nEtag: req_method_posted\n--- response_body\nPOST\n--- no_error_log\n[error]\n\n\n=== TEST 6: GET uses cached POST response.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /req_method_1 {\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\n--- request\nGET /req_method_1\n--- response_headers\nEtag: req_method_posted\n--- response_body\nPOST\n--- no_error_log\n[error]\n\n\n=== TEST 7: 501 on unrecognised method\n--- http_config eval: $::HttpConfig\n--- config\nlocation /req_method_1 {\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\n--- request\nFOOBAR /req_method_1\n--- error_code: 501\n--- no_error_log\n[error]\n" }, { "path": "t/02-integration/request_leak.t", "content": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::http_config(extra_nginx_config => qq{\n if_modified_since off;\n lua_check_client_abort on;\n});\n\nno_long_string();\nno_diff();\nrun_tests();\n\n__DATA__\n=== TEST 1: Aborted request does not leak body into subsequent request\n--- http_config eval\n\"$::HttpConfig\"\n\n--- config\n location = /trigger {\n content_by_lua_block {\n\n -- Send broken request and close socket\n local broken_sock = ngx.socket.tcp()\n broken_sock:settimeout(5000)\n local ok, err = broken_sock:connect(\"127.0.0.1\", ngx.var.server_port)\n broken_sock:send(\"POST /target?id=1 HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nContent-Length: 16\\r\\n\\r\\n123\\r\\n\")\n broken_sock:close()\n\n -- Send valid request and leave socket open\n local valid_sock = ngx.socket.tcp()\n valid_sock:settimeout(1000)\n local ok, err = valid_sock:connect(\"127.0.0.1\", ngx.var.server_port)\n valid_sock:send(\"GET /target?id=2 HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\n\\r\\n\")\n\n -- Wait and read until end of headers\n local header_reader = valid_sock:receiveuntil(\"\\r\\n\\r\\n\")\n local headers\n repeat\n headers = header_reader()\n until headers\n\n ngx.log(ngx.INFO, \"HEADERS: \", headers)\n\n -- We're expecting chunked encoding\n if not headers:find(\"chunked\") then\n ngx.log(ngx.ERR, \"Expected chunked response but no header indicating such, failed!\")\n ngx.exit(400)\n end\n\n -- Read chunk length as base16\n local chunk_len = tonumber(valid_sock:receive('*l'), 16)\n\n -- Read full chunk off wire\n local body, err, partial\n repeat\n body, err, partial = valid_sock:receive(chunk_len)\n until body or err\n\n valid_sock:close()\n\n if err then\n ngx.exit(400)\n end\n\n ngx.print(body)\n }\n }\n\n location /target {\n rewrite /target$ /origin break;\n content_by_lua_block {\n ngx.req.set_header(\"Host\", \"127.0.0.2\")\n\n require(\"ledge\").create_handler():run()\n }\n }\n\n location = /origin {\n content_by_lua_block {\n ngx.req.read_body()\n local args, err = ngx.req.get_uri_args()\n local data = ngx.req.get_body_data() or ''\n local method = ngx.req.get_method() or ''\n ngx.print(\"ORIGIN-\", args['id'], \"-\", method, \":\", data)\n ngx.exit(200)\n }\n }\n\n--- request\nGET /trigger\n--- response_body: ORIGIN-2-GET:\n" }, { "path": "t/02-integration/response.t", "content": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::http_config();\n\nno_long_string();\nno_diff();\nrun_tests();\n\n__DATA__\n=== TEST 1: Header case insensitivity\n--- http_config eval: $::HttpConfig\n--- config\nlocation /response_1_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n handler:bind(\"after_upstream_request\", function(res)\n if res.header[\"X-tesT\"] == \"1\" then\n res.header[\"x-TESt\"] = \"2\"\n end\n\n if res.header[\"X-TEST\"] == \"2\" then\n res.header[\"x-test\"] = \"3\"\n end\n end)\n handler:run()\n }\n}\nlocation /response_1 {\n content_by_lua_block {\n ngx.header[\"X-Test\"] = \"1\"\n ngx.say(\"OK\")\n }\n}\n--- request\nGET /response_1_prx\n--- response_headers\nX-Test: 3\n--- no_error_log\n[error]\n\n\n=== TEST 2: TTL from s-maxage (overrides max-age / Expires)\n--- http_config eval: $::HttpConfig\n--- config\nlocation /response_2_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n handler:bind(\"before_serve\", function(res)\n res.header[\"X-TTL\"] = res:ttl()\n end)\n handler:run()\n }\n}\nlocation /response_2 {\n content_by_lua_block {\n ngx.header[\"Expires\"] = ngx.http_time(ngx.time() + 300)\n ngx.header[\"Cache-Control\"] = \"max-age=600, s-maxage=1200\"\n ngx.say(\"OK\")\n }\n}\n--- more_headers\nCache-Control: no-cache\n--- request\nGET /response_2_prx\n--- response_headers\nX-TTL: 1200\n--- no_error_log\n[error]\n\n\n=== TEST 3: TTL from max-age (overrides Expires)\n--- http_config eval: $::HttpConfig\n--- config\nlocation /response_3_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n handler:bind(\"before_serve\", function(res)\n res.header[\"X-TTL\"] = res:ttl()\n end)\n handler:run()\n }\n}\nlocation /response_3 {\n content_by_lua_block {\n ngx.header[\"Expires\"] = ngx.http_time(ngx.time() + 300)\n ngx.header[\"Cache-Control\"] = \"max-age=600\"\n ngx.say(\"OK\")\n }\n}\n--- more_headers\nCache-Control: no-cache\n--- request\nGET /response_3_prx\n--- response_headers\nX-TTL: 600\n--- no_error_log\n[error]\n\n\n=== TEST 4: TTL from Expires\n--- http_config eval: $::HttpConfig\n--- config\nlocation /response_4_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n handler:bind(\"before_serve\", function(res)\n res.header[\"X-TTL\"] = res:ttl()\n end)\n handler:run()\n }\n}\nlocation /response_4 {\n content_by_lua_block {\n ngx.header[\"Expires\"] = ngx.http_time(ngx.time() + 300)\n ngx.say(\"OK\")\n }\n}\n--- more_headers\nCache-Control: no-cache\n--- request\nGET /response_4_prx\n--- response_headers\nX-TTL: 300\n--- no_error_log\n[error]\n\n\n=== TEST 4b: TTL from Expires, when there are multiple Expires headers\n--- http_config eval: $::HttpConfig\n--- config\nlocation /response_4b_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n handler:bind(\"before_serve\", function(res)\n res.header[\"X-TTL\"] = res:ttl()\n end)\n handler:run()\n }\n}\nlocation /response_4b {\n set $ttl_1 0;\n set $ttl_2 0;\n access_by_lua_block {\n ngx.var.ttl_1 = ngx.http_time(ngx.time() + 300)\n ngx.var.ttl_2 = ngx.http_time(ngx.time() + 100)\n }\n add_header Expires $ttl_1;\n add_header Expires $ttl_2;\n echo \"OK\";\n}\n--- more_headers\nCache-Control: no-cache\n--- request\nGET /response_4b_prx\n--- response_headers\nX-TTL: 100\n--- no_error_log\n[error]\n" }, { "path": "t/02-integration/ssl.t", "content": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\n$ENV{TEST_NGINX_HTML_DIR} ||= html_dir();\n$ENV{TEST_NGINX_SOCKET_DIR} ||= $ENV{TEST_NGINX_HTML_DIR};\n\nsub read_file {\n my $infile = shift;\n open my $in, $infile\n or die \"cannot open $infile for reading: $!\";\n my $cert = do { local $/; <$in> };\n close $in;\n $cert;\n}\n\nour $RootCACert = read_file(\"t/cert/rootCA.pem\");\nour $ExampleCert = read_file(\"t/cert/example.com.crt\");\nour $ExampleKey = read_file(\"t/cert/example.com.key\");\n\nour $HttpConfig = LedgeEnv::http_config(extra_nginx_config => qq{\n lua_ssl_trusted_certificate \"../html/rootca.pem\";\n ssl_certificate \"../html/example.com.crt\";\n ssl_certificate_key \"../html/example.com.key\";\n}, extra_lua_config => qq{\n\t-- SSL helper function\n\tfunction do_ssl(ssl_opts, params)\n\t\tlocal ssl_opts = ssl_opts or {}\n\n\t\tif not ssl_opts.verify then\n\t\t\tssl_opts.verify = false\n\t\tend\n\n\t\tif not ssl_opts.send_status_req then\n\t\t\tssl_opts.send_status_req = false\n\t\tend\n\n\t\tlocal httpc_ssl = require(\"resty.http\").new()\n\t\tlocal ok, err =\n\t\t\thttpc_ssl:connect(\"unix:$ENV{TEST_NGINX_SOCKET_DIR}/nginx-ssl.sock\")\n\n\t\tif not ok then\n\t\t\tngx.say(\"Unable to connect to sock, \", err)\n\t\t\treturn ngx.exit(ngx.status)\n\t\tend\n\n\t\tsession, err = httpc_ssl:ssl_handshake(\n\t\t\tnil,\n\t\t\tssl_opts.sni_name,\n ssl_opts.verify,\n ssl_opts.send_status_req\n )\n\n\t\tif err then\n\t\t\tngx.say(\"Unable to sslhandshake, \", err)\n\t\t\treturn ngx.exit(ngx.status)\n\t\tend\n\n\t\thttpc_ssl:set_timeout(2000)\n\n\t\tif params then\n\t\t\treturn httpc_ssl:request(params)\n\t\telse\n\t\t\treturn httpc_ssl:proxy_request()\n\t\tend\n\tend\n\n require(\"ledge\").set_handler_defaults({\n upstream_host = \"unix:$ENV{TEST_NGINX_SOCKET_DIR}/nginx-ssl.sock\",\n upstream_use_ssl = true,\n upstream_ssl_server_name = \"example.com\",\n upstream_ssl_verify = true,\n })\n}, run_worker => 1);\n\nno_long_string();\nno_diff();\nrun_tests();\n\n__DATA__\n=== TEST 1: SSL works\n--- http_config eval: $::HttpConfig\n--- config\nlisten unix:$TEST_NGINX_SOCKET_DIR/nginx-ssl.sock ssl;\nlocation /upstream_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /upstream {\n content_by_lua_block {\n ngx.say(\"OK \", ngx.var.scheme)\n }\n}\n--- user_files eval\n\">>> rootca.pem\n$::RootCACert\n>>> example.com.key\n$::ExampleKey\n>>> example.com.crt\n$::ExampleCert\"\n--- request\nGET /upstream_prx\n--- error_code: 200\n--- no_error_log\n[error]\n--- response_body\nOK https\n\n\n=== TEST 2: Bad SSL name errors\n--- http_config eval: $::HttpConfig\n--- config\nlisten unix:$TEST_NGINX_SOCKET_DIR/nginx-ssl.sock ssl;\nlocation /upstream_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler({\n upstream_ssl_server_name = \"foobar\",\n }):run()\n }\n}\nlocation /upstream {\n content_by_lua_block {\n ngx.say(\"OK \", ngx.var.scheme)\n }\n}\n--- user_files eval\n\">>> rootca.pem\n$::RootCACert\n>>> example.com.key\n$::ExampleKey\n>>> example.com.crt\n$::ExampleCert\"\n--- request\nGET /upstream_prx\n--- error_code: 525\n--- error_log\nssl handshake failed\n--- response_body:\n\n\n=== TEST 3: SSL verification can be disabled\n--- http_config eval: $::HttpConfig\n--- config\nlisten unix:$TEST_NGINX_SOCKET_DIR/nginx-ssl.sock ssl;\nlocation /upstream_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler({\n upstream_ssl_server_name = \"foobar\",\n upstream_ssl_verify = false\n }):run()\n }\n}\nlocation /upstream {\n content_by_lua_block {\n ngx.say(\"OK \", ngx.var.scheme)\n }\n}\n--- user_files eval\n\">>> rootca.pem\n$::RootCACert\n>>> example.com.key\n$::ExampleKey\n>>> example.com.crt\n$::ExampleCert\"\n--- request\nGET /upstream_prx\n--- error_code: 200\n--- no_error_log\n[error]\n--- response_body\nOK https\n\n\n=== TEST 4: Empty SSL name treated as nil\n--- http_config eval: $::HttpConfig\n--- config\nlisten unix:$TEST_NGINX_SOCKET_DIR/nginx-ssl.sock ssl;\nlocation /upstream_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler({\n upstream_ssl_server_name = \"\",\n }):run()\n }\n}\nlocation /upstream {\n content_by_lua_block {\n ngx.say(\"OK \", ngx.var.scheme)\n }\n}\n--- user_files eval\n\">>> rootca.pem\n$::RootCACert\n>>> example.com.key\n$::ExampleKey\n>>> example.com.crt\n$::ExampleCert\"\n--- request\nGET /upstream_prx\n--- error_code: 200\n--- no_error_log\n[error]\n--- response_body\nOK https\n\n\n=== TEST 9a: Prime another key\n--- http_config eval: $::HttpConfig\n--- config\nlisten unix:$TEST_NGINX_SOCKET_DIR/nginx-ssl.sock ssl;\nlocation /purge_ssl_entry {\n rewrite ^(.*)_entry$ $1_prx break;\n content_by_lua_block {\n local res, err = do_ssl(nil)\n ngx.print(res:read_body())\n }\n}\nlocation /purge_ssl_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler({\n keep_cache_for = 3600,\n }):run()\n }\n}\nlocation /purge_ssl {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.say(\"TEST 9: \", ngx.req.get_headers()[\"Cookie\"])\n }\n}\n--- user_files eval\n\">>> rootca.pem\n$::RootCACert\n>>> example.com.key\n$::ExampleKey\n>>> example.com.crt\n$::ExampleCert\"\n--- more_headers\nCookie: primed\n--- request\nGET /purge_ssl_entry\n--- no_error_log\n[error]\n--- response_body\nTEST 9: primed\n\n\n=== TEST 9b: Purge with X-Purge: revalidate\n--- http_config eval: $::HttpConfig\n--- config\nlisten unix:$TEST_NGINX_SOCKET_DIR/nginx-ssl.sock ssl;\nlocation /purge_ssl_entry {\n rewrite ^(.*)_entry$ $1_prx break;\n content_by_lua_block {\n local res, err = do_ssl(nil)\n ngx.print(res:read_body())\n }\n}\nlocation /purge_ssl_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /purge_ssl {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.say(\"TEST 9 Revalidated: \", ngx.req.get_headers()[\"Cookie\"])\n }\n}\n--- user_files eval\n\">>> rootca.pem\n$::RootCACert\n>>> example.com.key\n$::ExampleKey\n>>> example.com.crt\n$::ExampleCert\"\n--- more_headers\nX-Purge: revalidate\n--- request\nPURGE /purge_ssl_entry\n--- wait: 2\n--- no_error_log\n[error]\n--- response_body_like: \"result\":\"purged\"\n--- error_code: 200\n\n\n=== TEST 9c: Confirm cache was revalidated\n--- http_config eval: $::HttpConfig\n--- config\nlisten unix:$TEST_NGINX_SOCKET_DIR/nginx-ssl.sock ssl;\nlocation /purge_ssl_entry {\n rewrite ^(.*)_entry$ $1_prx break;\n content_by_lua_block {\n local res, err = do_ssl(nil)\n ngx.print(res:read_body())\n }\n}\nlocation /purge_ssl_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\n--- user_files eval\n\">>> rootca.pem\n$::RootCACert\n>>> example.com.key\n$::ExampleKey\n>>> example.com.crt\n$::ExampleCert\"\n--- request\nGET /purge_ssl_entry\n--- no_error_log\n[error]\n--- response_body\nTEST 9 Revalidated: primed\n\n=== TEST 10: ESI include fragment\n--- log_level: debug\n--- http_config eval: $::HttpConfig\n--- config\nlisten unix:$TEST_NGINX_SOCKET_DIR/nginx-ssl.sock ssl;\nlocation /esi_ssl_entry {\n rewrite ^(.*)_entry$ $1_prx break;\n content_by_lua_block {\n local res, err = do_ssl(nil)\n ngx.print(res:read_body())\n }\n}\nlocation /esi_ssl_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler({\n esi_enabled = true,\n }):run()\n }\n}\nlocation /fragment_1 {\n content_by_lua_block {\n ngx.say(\"FRAGMENT: \", ngx.req.get_uri_args()[\"a\"] or \"\", \"|\", ngx.var.scheme)\n }\n}\nlocation /esi_ssl {\n default_type text/html;\n content_by_lua_block {\n ngx.header[\"Surrogate-Control\"] = [[content=\"ESI/1.0\"]]\n ngx.say(\"1\")\n ngx.print([[]])\n ngx.say(\"2\")\n ngx.print([[]])\n ngx.print(\"3\")\n ngx.print([[]])\n }\n}\n--- user_files eval\n\">>> rootca.pem\n$::RootCACert\n>>> example.com.key\n$::ExampleKey\n>>> example.com.crt\n$::ExampleCert\"\n--- request\nGET /esi_ssl_entry\n--- raw_response_headers_unlike: Surrogate-Control: content=\"ESI/1.0\\\"\\r\\n\n--- response_body\n1\nFRAGMENT: |https\n2\nFRAGMENT: 2|https\n3FRAGMENT: 3|http\n--- no_error_log\n[error]\n" }, { "path": "t/02-integration/stale-if-error.t", "content": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::http_config();\n\nno_long_string();\nno_diff();\nrun_tests();\n\n__DATA__\n=== TEST 1: Prime cache for subsequent tests\n--- http_config eval: $::HttpConfig\n--- config\nlocation /stale_if_error_1_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /stale_if_error_1 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] =\n \"max-age=3600, s-maxage=60, stale-if-error=60\"\n ngx.say(\"TEST 1\")\n }\n}\n--- more_headers\nCache-Control: no-cache\n--- request\nGET /stale_if_error_1_prx\n--- response_body\nTEST 1\n--- response_headers_like\nX-Cache: MISS from .*\n--- no_error_log\n[error]\n\n\n=== TEST 1b: Assert standard non-stale behaviours are unaffected.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /stale_if_error_1_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /stale_if_error_1 {\n return 500;\n}\n--- more_headers eval\n[\n \"Cache-Control: no-cache\",\n \"Cache-Control: no-store\",\n \"Pragma: no-cache\",\n \"\"\n]\n--- request eval\n[\n \"GET /stale_if_error_1_prx\",\n \"GET /stale_if_error_1_prx\",\n \"GET /stale_if_error_1_prx\",\n \"GET /stale_if_error_1_prx\"\n]\n--- error_code eval\n[\n 500,\n 500,\n 500,\n 200\n]\n--- raw_response_headers_unlike eval\n[\n \"Warning: .*\",\n \"Warning: .*\",\n \"Warning: .*\",\n \"Warning: .*\"\n]\n--- no_error_log\n[error]\n\n\n=== TEST 2: Prime cache and expire it\n--- http_config eval: $::HttpConfig\n--- config\nlocation /stale_if_error_2_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n handler:bind(\"before_save\", function(res)\n res.header[\"Cache-Control\"] =\n \"max-age=0, s-maxage=0, stale-if-error=60\"\n end)\n handler:run()\n }\n}\nlocation /stale_if_error_2 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] =\n \"max-age=3600, s-maxage=60, stale-if-error=60\"\n ngx.print(\"TEST 2\")\n }\n}\n--- more_headers\nCache-Control: no-cache\n--- request\nGET /stale_if_error_2_prx\n--- response_body: TEST 2\n--- response_headers_like\nX-Cache: MISS from .*\n--- wait: 2\n--- no_error_log\n[error]\n\n\n=== TEST 2b: Request does not accept stale, for different reasons\n--- http_config eval: $::HttpConfig\n--- config\nlocation /stale_if_error_2_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /stale_if_error_2 {\n return 500;\n}\n--- more_headers eval\n[\n \"Cache-Control: min-fresh=5\",\n \"Cache-Control: max-age=1\",\n \"Cache-Control: max-stale=1\"\n]\n--- request eval\n[\n \"GET /stale_if_error_2_prx\",\n \"GET /stale_if_error_2_prx\",\n \"GET /stale_if_error_2_prx\"\n]\n--- error_code eval\n[\n 500,\n 500,\n 500\n]\n--- raw_response_headers_unlike eval\n[\n \"Warning: .*\",\n \"Warning: .*\",\n \"Warning: .*\"\n]\n--- response_body_unlike eval\n[\n \"TEST 2\",\n \"TEST 2\",\n \"TEST 2\",\n]\n--- no_error_log\n[error]\n\n\n=== TEST 2c: Request accepts stale\n--- http_config eval: $::HttpConfig\n--- config\nlocation /stale_if_error_2_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /stale_if_error_2 {\n return 500;\n}\n--- more_headers eval\n[\n \"Cache-Control: max-age=99999\",\n \"\"\n]\n--- request eval\n[\n \"GET /stale_if_error_2_prx\",\n \"GET /stale_if_error_2_prx\"\n]\n--- response_body eval\n[\n \"TEST 2\",\n \"TEST 2\"\n]\n--- response_headers_like eval\n[\n \"X-Cache: HIT from .*\",\n \"X-Cache: HIT from .*\"\n]\n--- raw_response_headers_like eval\n[\n \"Warning: 112 .*\",\n \"Warning: 112 .*\"\n]\n--- no_error_log\n[error]\n\n\n=== TEST 4: Prime cache and expire it\n--- http_config eval: $::HttpConfig\n--- config\nlocation /stale_if_error_4_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n handler:bind(\"before_save\", function(res)\n res.header[\"Cache-Control\"] =\n \"max-age=0, s-maxage=0, stale-if-error=60, must-revalidate\"\n end)\n handler:run()\n }\n}\nlocation /stale_if_error_4 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] =\n \"max-age=3600, s-maxage=60, stale-if-error=60, must-revalidate\"\n ngx.say(\"TEST 2\")\n }\n}\n--- more_headers\nCache-Control: no-cache\n--- request\nGET /stale_if_error_4_prx\n--- response_body\nTEST 2\n--- response_headers_like\nX-Cache: MISS from .*\n--- no_error_log\n[error]\n\n\n=== TEST 4b: Response cannot be served stale (must-revalidate)\n--- http_config eval: $::HttpConfig\n--- config\nlocation /stale_if_error_4_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /stale_if_error_4 {\n return 500;\n}\n--- request\nGET /stale_if_error_4_prx\n--- error_code: 500\n--- raw_response_headers_unlike\nWarning: .*\n--- no_error_log\n[error]\n\n\n=== TEST 4c: Prime cache (with valid stale config + proxy-revalidate) and expire\n--- http_config eval: $::HttpConfig\n--- config\nlocation /stale_if_error_4_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n handler:bind(\"before_save\", function(res)\n res.header[\"Cache-Control\"] =\n \"max-age=0, s-maxage=0, stale-if-error=60, proxy-revalidate\"\n end)\n handler:run()\n }\n}\nlocation /stale_if_error_4 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] =\n \"max-age=3600, s-maxage=60, stale-if-error=60, proxy-revalidate\"\n ngx.say(\"TEST 2\")\n }\n}\n--- more_headers\nCache-Control: no-cache\n--- request\nGET /stale_if_error_4_prx\n--- response_body\nTEST 2\n--- response_headers_like\nX-Cache: MISS from .*\n--- no_error_log\n[error]\n\n\n=== TEST 4d: Response cannot be served stale (proxy-revalidate)\n--- http_config eval: $::HttpConfig\n--- config\nlocation /stale_if_error_4_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /stale_if_error_4 {\n return 500;\n}\n--- request\nGET /stale_if_error_4_prx\n--- error_code: 500\n--- raw_response_headers_unlike\nWarning: .*\n--- no_error_log\n[error]\n" }, { "path": "t/02-integration/stale-while-revalidate.t", "content": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::http_config(extra_lua_config => qq{\n package.loaded[\"state\"] = {\n req = 1,\n }\n}, run_worker => 1);\n\nno_long_string();\nno_diff();\nrun_tests();\n\n__DATA__\n=== TEST 1: Prime cache for subsequent tests\n--- http_config eval: $::HttpConfig\n--- config\nlocation /stale_1_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /stale_1 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600, s-maxage=60, stale-while-revalidate=60\"\n ngx.say(\"TEST 1\")\n }\n}\n--- more_headers\nCache-Control: no-cache\n--- request\nGET /stale_1_prx\n--- response_body\nTEST 1\n--- response_headers_like\nX-Cache: MISS from .*\n--- no_error_log\n[error]\n\n\n=== TEST 1b: Assert standard non-stale behaviours are unaffected.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /stale_1_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /stale_1 {\n content_by_lua_block {\n local state = require(\"state\")\n\n ngx.header[\"Cache-Control\"] = \"max-age=3600, s-maxage=60, stale-while-revalidate=60\"\n ngx.print(\"ORIGIN: \", state.req)\n\n state.req = state.req + 1\n }\n}\n--- more_headers eval\n[\"Cache-Control: no-cache\", \"Cache-Control: no-store\", \"Pragma: no-cache\", \"\"]\n--- request eval\n[\"GET /stale_1_prx\", \"GET /stale_1_prx\", \"GET /stale_1_prx\", \"GET /stale_1_prx\"]\n--- response_body eval\n[\"ORIGIN: 1\", \"ORIGIN: 2\", \"ORIGIN: 3\", \"ORIGIN: 3\"]\n--- response_headers_like eval\n[\"X-Cache: MISS from .*\", \"X-Cache: MISS from .*\", \"X-Cache: MISS from .*\", \"X-Cache: HIT from .*\"]\n--- raw_response_headers_unlike eval\n[\"Warning: .*\", \"Warning: .*\", \"Warning: .*\", \"Warning: .*\"]\n--- no_error_log\n[error]\n\n\n=== TEST 2: Prime cache and expire it\n--- http_config eval: $::HttpConfig\n--- config\nlocation /stale_2_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n handler:bind(\"before_save\", function(res)\n res.header[\"Cache-Control\"] = \"max-age=0, s-maxage=0, stale-while-revalidate=60\"\n end)\n handler:run()\n }\n}\nlocation /stale_2 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600, s-maxage=60, stale-while-revalidate=60\"\n ngx.say(\"TEST 2\")\n }\n}\n--- more_headers\nCache-Control: no-cache\n--- request\nGET /stale_2_prx\n--- response_body\nTEST 2\n--- response_headers_like\nX-Cache: MISS from .*\n--- wait: 1\n--- no_error_log\n[error]\n\n\n=== TEST 2b: Request does not accept stale, for different reasons\n--- http_config eval: $::HttpConfig\n--- config\nlocation /stale_2_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n handler:bind(\"before_save\", function(res)\n res.header[\"Cache-Control\"] = \"max-age=0, s-maxage=0, stale-while-revalidate=60\"\n end)\n handler:run()\n }\n}\nlocation /stale_2 {\n content_by_lua_block {\n local state = require(\"state\")\n ngx.header[\"Cache-Control\"] = \"max-age=3600, s-maxage=60, stale-while-revalidate=60\"\n ngx.print(\"ORIGIN: \", state.req)\n\n state.req = state.req + 1\n }\n}\n--- more_headers eval\n[\"Cache-Control: min-fresh=5\", \"Cache-Control: max-age=1\", \"Cache-Control: max-stale=1\"]\n--- request eval\n[\"GET /stale_2_prx\", \"GET /stale_2_prx\", \"GET /stale_2_prx\"]\n--- response_body eval\n[\"ORIGIN: 1\", \"ORIGIN: 2\", \"ORIGIN: 3\"]\n--- response_headers_like eval\n[\"X-Cache: MISS from .*\", \"X-Cache: MISS from .*\", \"X-Cache: MISS from .*\"]\n--- raw_response_headers_unlike eval\n[\"Warning: .*\", \"Warning: .*\", \"Warning: .*\"]\n--- no_error_log\n[error]\n--- wait: 2\n\n\n=== TEST 3: Prime cache and expire it\n--- http_config eval: $::HttpConfig\n--- config\nlocation /stale_3_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n handler:bind(\"before_save\", function(res)\n res.header[\"Cache-Control\"] = \"max-age=0, s-maxage=0, stale-while-revalidate=60\"\n end)\n handler:run()\n }\n}\nlocation /stale_3 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600, s-maxage=60, stale-while-revalidate=60\"\n ngx.print(\"TEST 3\")\n }\n}\n--- more_headers\nCache-Control: no-cache\n--- request\nGET /stale_3_prx\n--- response_body: TEST 3\n--- response_headers_like\nX-Cache: MISS from .*\n--- no_error_log\n[error]\n\n\n=== TEST 3b: Request accepts stale\n--- http_config eval: $::HttpConfig\n--- config\nlocation /stale_3_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /stale_3 {\n content_by_lua_block {\n ngx.print(\"ORIGIN\")\n }\n}\n--- more_headers eval\n[\"Cache-Control: max-age=99999\", \"Cache-Control: max-stale=99999\", \"\"]\n--- request eval\n[\"GET /stale_3_prx\", \"GET /stale_3_prx\", \"GET /stale_3_prx\"]\n--- response_body eval\n[\"TEST 3\", \"TEST 3\", \"TEST 3\"]\n--- response_headers_like eval\n[\"X-Cache: HIT from .*\", \"X-Cache: HIT from .*\", \"X-Cache: HIT from .*\"]\n--- raw_response_headers_like eval\n[\"Warning: 110 .*\", \"Warning: 110 .*\", \"Warning: 110 .*\"]\n--- no_error_log\n[error]\n\n\n=== TEST 3c: Let revalidations finish to prevent errors\n--- http_config eval: $::HttpConfig\n--- config\nlocation /stale_3_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /stale_3 {\n content_by_lua_block {\n ngx.print(\"ORIGIN\")\n }\n}\n--- request\nGET /stale_3_prx\n--- response_body: TEST 3\n--- wait: 1\n--- no_error_log\n[error]\n\n\n=== TEST 4: Prime cache and expire it\n--- http_config eval: $::HttpConfig\n--- config\nlocation /stale_4_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n handler:bind(\"before_save\", function(res)\n res.header[\"Cache-Control\"] = \"max-age=0, s-maxage=0, stale-while-revalidate=60, must-revalidate\"\n end)\n handler:run()\n }\n}\nlocation /stale_4 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600, s-maxage=60, stale-while-revalidate=60, must-revalidate\"\n ngx.say(\"TEST 2\")\n }\n}\n--- more_headers\nCache-Control: no-cache\n--- request\nGET /stale_4_prx\n--- response_body\nTEST 2\n--- response_headers_like\nX-Cache: MISS from .*\n--- wait: 1\n--- no_error_log\n[error]\n\n\n=== TEST 4b: Response cannot be served stale (must-revalidate)\n--- http_config eval: $::HttpConfig\n--- config\nlocation /stale_4_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler({ visible_hostname = \"ledge.example.com\" }):run()\n }\n}\nlocation /stale_4 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600, s-maxage=60, stale-while-revalidate=60\"\n ngx.say(\"ORIGIN\")\n }\n}\n--- request\nGET /stale_4_prx\n--- response_body\nORIGIN\n--- response_headers_like\nX-Cache: MISS from .*\n--- raw_response_headers_unlike\nWarning: ledge\\.example\\.com .*\n--- no_error_log\n[error]\n\n\n=== TEST 4c: Prime cache (with valid stale config + proxy-revalidate) and expire it\n--- http_config eval: $::HttpConfig\n--- config\nlocation /stale_4_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n handler:bind(\"before_save\", function(res)\n res.header[\"Cache-Control\"] = \"max-age=0, s-maxage=0, stale-while-revalidate=60, proxy-revalidate\"\n end)\n handler:run()\n }\n}\nlocation /stale_4 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600, s-maxage=60, stale-while-revalidate=60, proxy-revalidate\"\n ngx.say(\"TEST 2\")\n }\n}\n--- more_headers\nCache-Control: no-cache\n--- request\nGET /stale_4_prx\n--- response_body\nTEST 2\n--- response_headers_like\nX-Cache: MISS from .*\n--- wait: 1\n--- no_error_log\n[error]\n\n\n=== TEST 4d: Response cannot be served stale (proxy-revalidate)\n--- http_config eval: $::HttpConfig\n--- config\nlocation /stale_4_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /stale_4 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600, s-maxage=60, stale-while-revalidate=60\"\n ngx.say(\"ORIGIN\")\n }\n}\n--- request\nGET /stale_4_prx\n--- response_body\nORIGIN\n--- response_headers_like\nX-Cache: MISS from .*\n--- raw_response_headers_unlike\nWarning: .*\n--- no_error_log\n[error]\n\n\n=== TEST 5a: Prime cache for subsequent tests\n--- http_config eval: $::HttpConfig\n--- config\nlocation /stale_5_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n handler:bind(\"before_save\", function(res)\n -- immediately expire cache entries\n res.header[\"Cache-Control\"] = \"max-age=0, s-maxage=0, stale-while-revalidate=60\"\n end)\n handler:run()\n }\n}\nlocation /stale_5 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600, s-maxage=60, stale-while-revalidate=60\"\n ngx.say(\"TEST 5\")\n }\n}\n--- more_headers\nCache-Control: no-cache\n--- request\nGET /stale_5_prx\n--- response_body\nTEST 5\n--- no_error_log\n[error]\n\n\n=== TEST 5b: Return stale\n--- http_config eval: $::HttpConfig\n--- config\nlocation /stale_5_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n handler:bind(\"before_save_revalidation_data\", function(reval_params, reval_headers)\n reval_headers[\"X-Test\"] = ngx.req.get_headers()[\"X-Test\"]\n reval_headers[\"Cookie\"] = ngx.req.get_headers()[\"Cookie\"]\n end)\n handler:run()\n }\n}\nlocation /stale_5 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.say(\"TEST 5b\")\n local hdr = ngx.req.get_headers()\n ngx.say(\"X-Test: \",hdr[\"X-Test\"])\n ngx.say(\"Cookie: \",hdr[\"Cookie\"])\n ngx.say(\"Authorization: \",hdr[\"Authorization\"])\n }\n}\n--- request\nGET /stale_5_prx\n--- more_headers\nX-Test: foobar\nCookie: baz=qux\nAuthorization: test\n--- response_body\nTEST 5\n--- wait: 1\n--- no_error_log\n[error]\n\n\n=== TEST 5c: Cache has been revalidated, custom headers\n--- http_config eval: $::HttpConfig\n--- config\nlocation /stale_5_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\n--- request\nGET /stale_5_prx\n--- response_body\nTEST 5b\nX-Test: foobar\nCookie: baz=qux\nAuthorization: test\n--- no_error_log\n[error]\n\n\n=== TEST 6: Reset cache, manually remove revalidation data\n--- http_config eval: $::HttpConfig\n--- config\nlocation /stale_reval_params_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n handler:bind(\"before_save\", function(res)\n -- immediately expire cache entries\n res.header[\"Cache-Control\"] = \"max-age=0, stale-while-revalidate=60\"\n end)\n handler:run()\n }\n}\nlocation /stale_reval_params {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600, stale-while-revalidate=60\"\n ngx.print(\"TEST 6\")\n }\n}\nlocation /stale_reval_params_remove {\n rewrite ^(.*)_remove$ $1 break;\n content_by_lua_block {\n local redis = require(\"ledge\").create_redis_connection()\n local handler = require(\"ledge\").create_handler()\n handler.redis = redis\n local key_chain = handler:cache_key_chain()\n\n redis:del(key_chain.reval_req_headers)\n redis:del(key_chain.reval_params)\n\n redis:set_keepalive()\n ngx.print(\"REMOVED\")\n }\n}\n--- more_headers\nCache-Control: no-cache\n--- request eval\n[\"GET /stale_reval_params_prx\", \"GET /stale_reval_params_remove\"]\n--- response_body eval\n[\"TEST 6\", \"REMOVED\"]\n--- no_error_log\n[error]\n\n\n=== TEST 6b: Stale revalidation does not choke on missing revalidation data.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /stale_reval_params_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /stale_reval_params {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600, stale-while-revalidate=60\"\n ngx.print(\"TEST 6: \", ngx.req.get_headers()[\"Cookie\"])\n }\n}\n--- more_headers\nCookie: mycookie\n--- request\nGET /stale_reval_params_prx\n--- response_headers_like\nWarning: 110 .*\n--- error_log\nCould not determine expiry for revalidation params. Will fallback to 3600 seconds.\n--- response_body: TEST 6\n--- wait: 1\n--- error_code: 200\n\n\n=== TEST 6c: Confirm revalidation\n--- http_config eval: $::HttpConfig\n--- config\nlocation /stale_reval_params_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\n--- request\nGET /stale_reval_params_prx\n--- no_error_log\n[error]\n--- response_body: TEST 6: mycookie\n--- error_code: 200\n\n\n=== TEST 7: Prime and immediately expire two keys\n--- http_config eval: $::HttpConfig\n--- config\nlocation /stale_reval_params_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local handler = require(\"ledge\").create_handler()\n handler:bind(\"before_save\", function(res)\n -- immediately expire cache entries\n res.header[\"Cache-Control\"] = \"max-age=0, stale-while-revalidate=60\"\n end)\n handler:run()\n }\n}\nlocation /stale_reval_params {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3700, stale-while-revalidate=60\"\n ngx.print(\"TEST 7: \", ngx.req.get_uri_args()[\"a\"])\n }\n}\n--- more_headers\nCache-Control: no-cache\n--- request eval\n[\"GET /stale_reval_params_prx?a=1\", \"GET /stale_reval_params_prx?a=2\"]\n--- response_body eval\n[\"TEST 7: 1\", \"TEST 7: 2\"]\n--- no_error_log\n[error]\n\n\n=== TEST 7b: Concurrent stale revalidation\n--- http_config eval: $::HttpConfig\n--- config\nlocation /stale_reval_params_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /stale_reval_params {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600, stale-while-revalidate=60\"\n ngx.print(\"TEST 7 Revalidated: \", ngx.req.get_uri_args()[\"a\"])\n }\n}\n--- request eval\n[\"GET /stale_reval_params_prx?a=1\", \"GET /stale_reval_params_prx?a=2\"]\n--- no_error_log\n[error]\n--- response_body eval\n[\"TEST 7: 1\", \"TEST 7: 2\"]\n--- wait: 1\n\n\n=== TEST 7c: Confirm revalidation\n--- http_config eval: $::HttpConfig\n--- config\nlocation /stale_reval_params_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\n--- request eval\n[\"GET /stale_reval_params_prx?a=1\", \"GET /stale_reval_params_prx?a=2\"]\n--- no_error_log\n[error]\n--- response_body eval\n[\"TEST 7 Revalidated: 1\", \"TEST 7 Revalidated: 2\"]\n--- no_error_log\n[error]\n" }, { "path": "t/02-integration/upstream.t", "content": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\n$ENV{TEST_NGINX_HTML_DIR} ||= html_dir();\n$ENV{TEST_NGINX_SOCKET_DIR} ||= $ENV{TEST_NGINX_HTML_DIR};\n\nour $HttpConfig = LedgeEnv::http_config();\n\nno_long_string();\nno_diff();\nrun_tests();\n\n__DATA__\n=== TEST 1: Short read timeout results in error 524.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /upstream_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler({\n upstream_send_timeout = 5000,\n upstream_connect_timeout = 5000,\n upstream_read_timeout = 100,\n }):run()\n }\n}\nlocation /upstream {\n content_by_lua_block {\n ngx.sleep(1)\n ngx.say(\"OK\")\n }\n}\n--- request\nGET /upstream_prx\n--- error_code: 524\n--- response_body\n--- error_log\ntimeout\n\n\n=== TEST 2: No upstream results in a 503.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /upstream_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler({\n upstream_host = \"\",\n }):run()\n }\n}\n--- request\nGET /upstream_prx\n--- error_code: 503\n--- response_body\n--- error_log\nupstream connection failed:\n\n\n=== TEST 3: No port results in 503\n--- http_config eval: $::HttpConfig\n--- config\nlocation /upstream_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler({\n upstream_host = \"127.0.0.1\",\n upstream_port = \"\",\n }):run()\n }\n}\n--- request\nGET /upstream_prx\n--- error_code: 503\n--- response_body\n--- error_log\nupstream connection failed:\n\n\n=== TEST 4: No port with unix socket works\n--- http_config eval: $::HttpConfig\n--- config\nlisten unix:$TEST_NGINX_SOCKET_DIR/nginx.sock;\nlocation /upstream_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler({\n upstream_host = \"unix:$TEST_NGINX_SOCKET_DIR/nginx.sock\",\n upstream_port = \"\",\n }):run()\n }\n}\nlocation /upstream {\n echo \"OK\";\n}\n--- request\nGET /upstream_prx\n--- error_code: 200\n--- response_body\nOK\n--- no_error_log\n[error]\n" }, { "path": "t/02-integration/upstream_client.t", "content": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::http_config(extra_nginx_config => qq{\n lua_shared_dict test_upstream_dict 1m;\n}, extra_lua_config => qq{\n function create_upstream_client(config)\n -- Defaults\n config = config or {}\n config[\"timeout\"] = config[\"timeout\"] or 100\n config[\"read_timeout\"] = config[\"read_timeout\"] or 500\n config[\"host\"] = config[\"host\"] or \"$LedgeEnv::nginx_host\"\n config[\"port\"] = config[\"port\"] or $LedgeEnv::nginx_port\n\n return function(handler)\n local httpc = require(\"resty.http\").new()\n httpc:set_timeout(config.timeout)\n\n local ok, err = httpc:connect(\n config.host,\n config.port\n )\n\n if not ok then\n ngx.log(ngx.ERR, \"upstream client connection failed: \", err)\n return nil\n end\n\n httpc:set_timeout(config.read_timeout)\n\n handler.upstream_client = httpc\n end\n end\n\n require(\"ledge\").bind(\"before_upstream_connect\", function(handler)\n if ngx.req.get_uri_args()[\"skip_init\"] then\n -- do nothing\n else\n -- create handler and pass through res\n create_upstream_client()(handler)\n end\n end)\n\n require(\"ledge\").set_handler_defaults({\n upstream_host = \"\",\n upstream_port = 9999,\n })\n}, run_worker => 1);\n\nno_long_string();\nno_diff();\nrun_tests();\n\n__DATA__\n=== TEST 1: Sanity, response returned with upstream_client configured\n--- http_config eval: $::HttpConfig\n--- config\nlocation /upstream_client_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /upstream_client {\n content_by_lua_block {\n ngx.say(\"OK\")\n }\n}\n--- request\nGET /upstream_client_prx\n--- no_error_log\n[error]\n--- error_code: 200\n--- response_body\nOK\n\n=== TEST 1b: Sanity, response returned with upstream_client configured at runtime\n--- http_config eval: $::HttpConfig\n--- config\nlocation /upstream_client_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local h = require(\"ledge\").create_handler()\n h:bind(\"before_upstream_connect\", create_upstream_client() )\n h:run()\n }\n}\nlocation /upstream_client {\n content_by_lua_block {\n ngx.say(\"OK\")\n }\n}\n--- request\nGET /upstream_client_prx?skip_init=true\n--- no_error_log\n[error]\n--- error_code: 200\n--- response_body\nOK\n\n=== TEST 2: Short read timeout results in error 524.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /upstream_client_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /upstream_client {\n content_by_lua_block {\n ngx.sleep(1)\n ngx.say(\"OK\")\n }\n}\n--- request\nGET /upstream_client_prx\n--- error_code: 524\n--- response_body\n--- error_log\ntimeout\n\n\n=== TEST 3: No upstream results in a 503.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /upstream_client_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local h = require(\"ledge\").create_handler()\n h:bind(\"before_upstream_connect\", function(handler)\n handler.upstream_client = {}\n end)\n h:run()\n }\n}\n--- request\nGET /upstream_client_prx\n--- error_code: 503\n--- response_body\n--- error_log\nupstream connection failed\n" }, { "path": "t/02-integration/validation.t", "content": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::http_config();\n\nno_long_string();\nno_diff();\nrun_tests();\n\n__DATA__\n=== TEST 1: Prime cache for subsequent tests\n--- http_config eval: $::HttpConfig\n--- config\nlocation /validation_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /validation {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.header[\"Etag\"] = \"test1\"\n ngx.header[\"Last-Modified\"] = ngx.http_time(ngx.time() - 100)\n ngx.say(\"TEST 1\")\n }\n}\n--- request\nGET /validation_prx\n--- response_body\nTEST 1\n--- response_headers_like\nX-Cache: MISS from .*\n--- no_error_log\n[error]\n\n\n=== TEST 2: Unspecified end-to-end revalidation\n max-age=0 + no validator, upstream 200\n--- http_config eval: $::HttpConfig\n--- config\nlocation /validation_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /validation {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.header[\"Etag\"] = \"test2\"\n ngx.header[\"Last-Modified\"] = ngx.http_time(ngx.time() - 90)\n ngx.say(\"TEST 2\")\n }\n}\n--- more_headers\nCache-Control: max-age=0\n--- request\nGET /validation_prx\n--- error_code: 200\n--- response_body\nTEST 2\n--- response_headers_like\nX-Cache: MISS from .*\n--- no_error_log\n[error]\n\n\n=== TEST 2b: Unspecified end-to-end revalidation\n max-age=0 + no validator, upstream 304\n--- http_config eval: $::HttpConfig\n--- config\nlocation /validation_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /validation {\n content_by_lua_block {\n ngx.exit(ngx.HTTP_NOT_MODIFIED)\n }\n}\n--- more_headers\nCache-Control: max-age=0\n--- request\nGET /validation_prx\n--- response_body\nTEST 2\n--- error_code: 200\n--- response_headers_like\nX-Cache: MISS from .*\n--- no_error_log\n[error]\n\n\n=== TEST 3: Revalidate against cache using IMS in the future.\n Check we still have headers with our 304, and no body.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /validation_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n ngx.req.set_header(\n \"If-Modified-Since\",\n ngx.http_time(ngx.time() + 100)\n )\n require(\"ledge\").create_handler():run()\n }\n}\n--- request\nGET /validation_prx\n--- error_code: 304\n--- response_headers\nCache-Control: max-age=3600\nEtag: test2\n--- response_body\n--- no_error_log\n[error]\n\n\n=== TEST 3b: Revalidate against cache using IMS in the past.\n Return 200 fresh cache.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /validation_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n ngx.req.set_header(\n \"If-Modified-Since\",\n ngx.http_time(ngx.time() - 100)\n )\n require(\"ledge\").create_handler():run()\n }\n}\n--- request\nGET /validation_prx\n--- error_code: 200\n--- response_body\nTEST 2\n--- response_headers_like\nX-Cache: HIT from .*\n--- no_error_log\n[error]\n\n\n=== TEST 4: Revalidate against cache using Etag.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /validation_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\n--- more_headers\nIf-None-Match: test2\n--- request\nGET /validation_prx\n--- error_code: 304\n--- response_body\n--- no_error_log\n[error]\n\n\n=== TEST 4b: Revalidate against cache using LM and Etag.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /validation_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n ngx.req.set_header(\n \"If-Modified-Since\",\n ngx.http_time(ngx.time() + 100)\n )\n require(\"ledge\").create_handler():run()\n }\n}\n--- more_headers\nIf-None-Match: test2\n--- request\nGET /validation_prx\n--- error_code: 304\n--- response_body\n--- no_error_log\n[error]\n\n\n=== TEST 5: Specific end-to-end revalidation using IMS, upstream 304.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /validation_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n ngx.req.set_header(\n \"If-Modified-Since\",\n ngx.http_time(ngx.time() - 150)\n )\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /validation {\n content_by_lua_block {\n ngx.exit(ngx.HTTP_NOT_MODIFIED)\n }\n}\n--- more_headers\nCache-Control: max-age=0\n--- request\nGET /validation_prx\n--- error_code: 200\n--- response_body\nTEST 2\n--- response_headers_like\nX-Cache: MISS from .*\n--- no_error_log\n[error]\n\n\n=== TEST 6: Specific end-to-end revalidation\n Using INM (matching), upstream 304.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /validation_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /validation {\n content_by_lua_block {\n ngx.exit(ngx.HTTP_NOT_MODIFIED)\n }\n}\n--- more_headers\nCache-Control: max-age=0\nIf-None-Match: test2\n--- request\nGET /validation_prx\n--- error_code: 304\n--- no_error_log\n[error]\n\n\n=== TEST 6b: Specific end-to-end revalidation\n Using INM (not matching), upstream 304.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /validation_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /validation {\n content_by_lua_block {\n ngx.exit(ngx.HTTP_NOT_MODIFIED)\n }\n}\n--- more_headers\nCache-Control: max-age=0\nIf-None-Match: test6b\n--- request\nGET /validation_prx\n--- error_code: 200\n--- response_body\nTEST 2\n--- response_headers_like\nX-Cache: MISS from .*\n--- no_error_log\n[error]\n\n\n=== TEST 7: Specific end-to-end revalidation using IMS, upstream 200.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /validation_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n ngx.req.set_header(\n \"If-Modified-Since\",\n ngx.http_time(ngx.time() - 150)\n )\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /validation {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.header[\"Etag\"] = \"test7\"\n ngx.header[\"Last-Modified\"] = ngx.http_time(ngx.time() - 70)\n ngx.say(\"TEST 7\")\n }\n}\n--- more_headers\nCache-Control: max-age=0\n--- request\nGET /validation_prx\n--- error_code: 200\n--- response_body\nTEST 7\n--- no_error_log\n[error]\n\n\n=== TEST 8: Specific end-to-end revalidation using INM, upstream 200.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /validation_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /validation {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.header[\"Etag\"] = \"test8\"\n ngx.say(\"TEST 8\")\n }\n}\n--- more_headers\nCache-Control: max-age=0\nIf-None-Match: test2\n--- request\nGET /validation_prx\n--- error_code: 200\n--- response_body\nTEST 8\n--- response_headers_like\nX-Cache: MISS from .*\n--- no_error_log\n[error]\n\n\n=== TEST 8b: Unspecified end-to-end revalidation\n Using INM, upstream 200, validators now match (so 304 to client).\n--- http_config eval: $::HttpConfig\n--- config\nlocation /validation_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /validation {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.header[\"Etag\"] = \"test8b\"\n ngx.say(\"TEST 8b\")\n }\n}\n--- more_headers\nCache-Control: max-age=0\nIf-None-Match: test8b\n--- request\nGET /validation_prx\n--- error_code: 304\n--- response_body\n--- no_error_log\n[error]\n\n\n=== TEST 8c: Check revalidation re-saved.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /validation_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\n--- request\nGET /validation_prx\n--- error_code: 200\n--- response_body\nTEST 8b\n--- response_headers_like\nX-Cache: HIT from .*\n--- no_error_log\n[error]\n\n\n=== TEST 9: Validators on a cache miss (should never 304).\n--- http_config eval: $::HttpConfig\n--- config\nlocation /validation_9_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /validation_9 {\n content_by_lua_block {\n if ngx.req.get_headers()[\"Cache-Control\"] == \"max-age=0\" and\n ngx.req.get_headers()[\"If-None-Match\"] == \"test9\" then\n ngx.exit(ngx.HTTP_NOT_MODIFIED)\n else\n ngx.say(\"TEST 9\")\n end\n }\n}\n--- more_headers\nIf-None-Match: test9\n--- request\nGET /validation_9_prx\n--- error_code: 200\n--- response_body\nTEST 9\n--- no_error_log\n[error]\n\n\n=== TEST 10: Re-Validation on an a cache miss using INM. Upstream 200, but valid once cached (so 304 to client).\n--- http_config eval: $::HttpConfig\n--- config\nlocation /validation10_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /validation10 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.header[\"Etag\"] = \"test10\"\n ngx.header[\"Last-Modified\"] = ngx.http_time(ngx.time() - 60)\n ngx.say(\"TEST 10\")\n }\n}\n--- more_headers\nIf-None-Match: test10\n--- request\nGET /validation10_prx\n--- error_code: 304\n--- response_body\n--- no_error_log\n[error]\n\n\n=== TEST 11: Test badly formatted IMS is ignored.\n--- http_config eval: $::HttpConfig\n--- config\nlocation /validation10_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\n--- more_headers\nIf-Modified-Since: 234qr12411224\n--- request\nGET /validation10_prx\n--- error_code: 200\n--- response_body\nTEST 10\n--- response_headers_like\nX-Cache: HIT from .*\n--- no_error_log\n[error]\n\n\n=== TEST 12: Prime cache\n--- http_config eval: $::HttpConfig\n--- config\nlocation /validation_12_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /validation_12 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"public, max-age=600\"\n ngx.say(\"Test 12\")\n }\n}\n--- request\nGET /validation_12_prx\n--- error_code: 200\n--- response_body\nTest 12\n--- no_error_log\n[error]\n\n\n=== TEST 12a: IMS in req and missing LM does not 304\n--- http_config eval: $::HttpConfig\n--- config\nlocation /validation_12_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /validation_12 {\n content_by_lua_block {\n ngx.say(\"Test 12\")\n }\n}\n--- more_headers\nIf-Modified-Since: Tue, 29 Nov 2016 23:16:59 GMT\n--- request\nGET /validation_12_prx\n--- error_code: 200\n--- response_body\nTest 12\n--- no_error_log\n[error]\n\n\n=== TEST 12b: INM in req and missing etag does not 304\n--- http_config eval: $::HttpConfig\n--- config\nlocation /validation_12_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /validation_12 {\n content_by_lua_block {\n ngx.say(\"Test 12\")\n }\n}\n--- more_headers\nIf-None-Match: 1234\n--- request\nGET /validation_12_prx\n--- error_code: 200\n--- response_body\nTest 12\n--- no_error_log\n[error]\n" }, { "path": "t/02-integration/vary.t", "content": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::http_config(extra_nginx_config => qq{\n lua_shared_dict ledge_test 1m;\n lua_check_client_abort on;\n});\n\nno_long_string();\nno_diff();\nrun_tests();\n\n__DATA__\n=== TEST 1: Vary\n--- http_config eval: $::HttpConfig\n--- config\nlocation /vary_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge.state_machine\").set_debug(true)\n require(\"ledge\").create_handler():run()\n }\n}\n\nlocation /vary {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.header[\"Vary\"] = \"X-Test\"\n ngx.print(\"TEST 1: \", ngx.req.get_headers()[\"X-Test\"])\n }\n}\n--- request eval\n[\"GET /vary_prx\", \"GET /vary_prx\", \"GET /vary_prx\", \"GET /vary_prx\"]\n--- more_headers eval\n[\n\"X-Test: testval\",\n\"X-Test: anotherval\",\n\"\",\n\"X-Test: testval\",\n]\n--- response_headers_like eval\n[\n\"X-Cache: MISS from .*\",\n\"X-Cache: MISS from .*\",\n\"X-Cache: MISS from .*\",\n\"X-Cache: HIT from .*\",\n]\n--- response_body eval\n[\n\"TEST 1: testval\",\n\"TEST 1: anotherval\",\n\"TEST 1: nil\",\n\"TEST 1: testval\",\n]\n--- no_error_log\n[error]\n\n=== TEST 2: Vary change\n--- http_config eval: $::HttpConfig\n--- config\nlocation /vary_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge.state_machine\").set_debug(true)\n require(\"ledge\").create_handler():run()\n }\n}\n\nlocation /vary {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.header[\"Vary\"] = \"X-Test2\"\n ngx.print(\"TEST 2: \", ngx.req.get_headers()[\"X-Test2\"], \" \", ngx.req.get_headers()[\"X-Test\"])\n }\n}\n--- request eval\n[\"GET /vary_prx\", \"GET /vary_prx\", \"GET /vary_prx\", \"GET /vary_prx\"]\n--- more_headers eval\n[\n\"X-Test: testval\nCache-Control: no-cache\",\n\n\"X-Test2: newval\",\n\"\",\n\n\"X-Test: testval\nX-Test2: newval\",\n]\n--- response_headers_like eval\n[\n\"X-Cache: MISS from .*\",\n\"X-Cache: MISS from .*\",\n\"X-Cache: HIT from .*\",\n\"X-Cache: HIT from .*\",\n]\n--- response_body eval\n[\n\"TEST 2: nil testval\",\n\"TEST 2: newval nil\",\n\"TEST 2: nil testval\",\n\"TEST 2: newval nil\",\n]\n--- no_error_log\n[error]\n\n\n=== TEST 3: Cache update changes 1 representation\n--- http_config eval: $::HttpConfig\n--- config\nlocation /vary3_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge.state_machine\").set_debug(true)\n require(\"ledge\").create_handler():run()\n }\n}\n\nlocation /vary {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.header[\"Vary\"] = \"X-Test\"\n ngx.print(\"TEST 3: \", ngx.req.get_headers()[\"X-Test\"])\n }\n}\n--- request eval\n[\"GET /vary3_prx\", \"GET /vary3_prx\", \"GET /vary3_prx\", \"GET /vary3_prx\"]\n--- more_headers eval\n[\n\"X-Test: testval\",\n\"X-Test: value2\",\n\n\"X-Test: testval\nCache-Control: no-cache\",\n\n\"X-Test: value2\",\n]\n--- response_headers_like eval\n[\n\"X-Cache: MISS from .*\",\n\"X-Cache: MISS from .*\",\n\"X-Cache: MISS from .*\",\n\"X-Cache: HIT from .*\",\n]\n--- response_body eval\n[\n\"TEST 3: testval\",\n\"TEST 3: value2\",\n\"TEST 3: testval\",\n\"TEST 3: value2\",\n]\n--- no_error_log\n[error]\n\n\n=== TEST 4: Missing keys are cleaned from repset\n--- http_config eval: $::HttpConfig\n--- config\nlocation /check {\n rewrite ^ /vary break;\n content_by_lua_block {\n local redis = require(\"ledge\").create_redis_connection()\n local handler = require(\"ledge\").create_handler()\n handler.redis = redis\n local res, err = redis:smembers(handler:cache_key_chain().repset)\n\n for _, v in ipairs(res) do\n assert(v ~= \"foobar\", \"Key should have been cleaned\")\n end\n ngx.print(\"OK\")\n }\n}\nlocation /vary_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge.state_machine\").set_debug(true)\n local redis = require(\"ledge\").create_redis_connection()\n local handler = require(\"ledge\").create_handler()\n handler.redis = redis\n local ok, err = redis:sadd(handler:cache_key_chain().repset, \"foobar\")\n handler:run()\n }\n}\n\nlocation /vary {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.header[\"Vary\"] = \"X-Test\"\n ngx.print(\"TEST 4\")\n }\n}\n--- request eval\n[\"GET /vary_prx\", \"GET /check\"]\n--- more_headers eval\n[\"Cache-Control: no-cache\",\"\"]\n--- response_body eval\n[\n\"TEST 4\",\n\"OK\"\n]\n--- no_error_log\n[error]\n\n\n=== TEST 5: Repset TTL maintained\n--- http_config eval: $::HttpConfig\n--- config\nlocation = /check {\n rewrite ^ /vary5 break;\n\n content_by_lua_block {\n local redis = require(\"ledge\").create_redis_connection()\n local handler = require(\"ledge\").create_handler()\n handler.redis = redis\n\n local repset_ttl, err = redis:ttl(handler:cache_key_chain().repset)\n if err then ngx.log(ngx.ERR, err) end\n\n local vary_ttl, err = redis:ttl(handler:cache_key_chain().vary)\n if err then ngx.log(ngx.ERR, err) end\n\n local count = ngx.shared.ledge_test:get(\"test5\")\n\n if count < 3 then\n if (repset_ttl - handler.config.keep_cache_for) <= 300\n or (vary_ttl - handler.config.keep_cache_for) <= 300 then\n ngx.print(\"FAIL\")\n ngx.log(ngx.ERR,\n (repset_ttl - handler.config.keep_cache_for),\n \" \",\n (vary_ttl - handler.config.keep_cache_for)\n )\n else\n ngx.print(\"OK\")\n end\n else\n\n if (repset_ttl - handler.config.keep_cache_for) < 7200\n or (vary_ttl - handler.config.keep_cache_for) < 7200 then\n ngx.print(\"FAIL 2\")\n ngx.log(ngx.ERR,\n (repset_ttl - handler.config.keep_cache_for),\n \" \",\n (vary_ttl - handler.config.keep_cache_for)\n )\n else\n ngx.print(\"OK\")\n end\n end\n }\n}\nlocation /vary5_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge.state_machine\").set_debug(false)\n require(\"ledge\").create_handler():run()\n }\n}\n\nlocation /vary {\n content_by_lua_block {\n local incr = ngx.shared.ledge_test:incr(\"test5\", 1, 0)\n if incr == 1 then\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n elseif incr == 3 then\n ngx.header[\"Cache-Control\"] = \"max-age=7200\"\n else\n ngx.header[\"Cache-Control\"] = \"max-age=300\"\n end\n ngx.header[\"Vary\"] = \"X-Test\"\n ngx.print(\"TEST 5\")\n }\n}\n--- request eval\n[\"GET /vary5_prx\", \"GET /vary5_prx\", \"GET /check\", \"GET /vary5_prx\", \"GET /check\"]\n--- more_headers eval\n[\n\"Cache-Control: no-cache\",\n\"Cache-Control: no-cache\",\n\"\",\n\"Cache-Control: no-cache\",\n\"\",\n]\n--- response_headers_like eval\n[\n\"X-Cache: MISS from .*\",\n\"X-Cache: MISS from .*\",\n\"\",\n\"X-Cache: MISS from .*\",\n\"\",\n]\n--- response_body eval\n[\n\"TEST 5\",\n\"TEST 5\",\n\"OK\",\n\"TEST 5\",\n\"OK\",\n]\n--- no_error_log\n[error]\n\n\n=== TEST 6: Vary - case insensitive\n--- http_config eval: $::HttpConfig\n--- config\nlocation /vary6_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge.state_machine\").set_debug(true)\n require(\"ledge\").create_handler():run()\n }\n}\n\nlocation /vary {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n local incr = ngx.shared.ledge_test:incr(\"test6\", 1, 0)\n if incr == 1 then\n ngx.header[\"Vary\"] = \"X-Test\"\n elseif incr == 2 then\n ngx.header[\"Vary\"] = \"X-test\"\n else\n ngx.header[\"Vary\"] = \"x-Test\"\n end\n ngx.print(\"TEST 6: \", ngx.req.get_headers()[\"X-Test\"])\n }\n}\n--- request eval\n[\"GET /vary6_prx\", \"GET /vary6_prx\", \"GET /vary6_prx\"]\n--- more_headers eval\n[\n\"X-Test: testval\",\n\"X-test: TestVAL\",\n\"X-teSt: foobar\",\n]\n--- response_headers_like eval\n[\n\"X-Cache: MISS from .*\",\n\"X-Cache: HIT from .*\",\n\"X-Cache: MISS from .*\",\n]\n--- response_body eval\n[\n\"TEST 6: testval\",\n\"TEST 6: testval\",\n\"TEST 6: foobar\",\n\n]\n--- no_error_log\n[error]\n\n=== TEST 7: Vary - sort order\n--- http_config eval: $::HttpConfig\n--- config\nlocation /vary7_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge.state_machine\").set_debug(true)\n require(\"ledge\").create_handler():run()\n }\n}\n\nlocation /vary {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3700\"\n\n local incr = ngx.shared.ledge_test:incr(\"test7\", 1, 0)\n\n if incr == 1 then\n -- Prime with 1 order\n ngx.header[\"Vary\"] = \"X-Test, X-Test2, X-Test3\"\n elseif incr == 2 then\n -- Second request, different order, different values in request\n ngx.header[\"Vary\"] = \"X-Test3, X-test, X-test2\"\n end\n\n assert (incr < 3, \"Third request should be a cache hit\")\n\n ngx.print(\"TEST 7: \", incr)\n }\n}\n--- request eval\n[\"GET /vary7_prx\", \"GET /vary7_prx\", \"GET /vary7_prx\"]\n--- more_headers eval\n[\n\"X-Test: abc\nX-Test2: 123\nX-Test3: xyz\n\",\n\n\"X-Test: abc2\nX-Test2: 123b\nX-Test3: xyz2\n\",\n\n\"X-Test: abc\nX-Test2: 123\nX-Test3: xyz\n\",\n\n]\n--- response_headers_like eval\n[\n\"X-Cache: MISS from .*\nVary: X-Test, X-Test2, X-Test3\",\n\n\"X-Cache: MISS from .*\nVary: X-Test3, X-test, X-test2\",\n\n\"X-Cache: HIT from .*\nVary: X-Test, X-Test2, X-Test3\",\n]\n--- response_body eval\n[\n\"TEST 7: 1\",\n\"TEST 7: 2\",\n\"TEST 7: 1\",\n]\n--- no_error_log\n[error]\n\n\n=== TEST 8: Vary event hook\n--- http_config eval: $::HttpConfig\n--- config\nlocation /vary8_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge.state_machine\").set_debug(true)\n local handler = require(\"ledge\").create_handler()\n\n handler:bind(\"before_vary_selection\", function(vary_key)\n local x_vary = ngx.req.get_headers()[\"X-Vary\"]\n -- Do nothing if noop set\n if x_vary ~= \"noop\" then\n vary_key[\"x-test\"] = nil\n vary_key[\"X-Test2\"] = x_vary\n end\n ngx.log(ngx.DEBUG, \"Vary Key: \", require(\"cjson\").encode(vary_key))\n end)\n\n handler:run()\n }\n}\n\nlocation /vary {\n content_by_lua_block {\n local incr = ngx.shared.ledge_test:incr(\"test8\", 1, 0)\n\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n if ngx.req.get_headers()[\"X-Vary\"] == \"noop\" then\n ngx.header[\"Vary\"] = \"X-Test2\"\n else\n ngx.header[\"Vary\"] = \"X-Test\"\n end\n ngx.print(\"TEST 8: \", incr)\n }\n}\n--- request eval\n[\"GET /vary8_prx\", \"GET /vary8_prx\", \"GET /vary8_prx\", \"GET /vary8_prx\"]\n--- more_headers eval\n[\n\"X-Test: testval\nX-Vary: foo\",\n\n\"X-Test: anotherval\nX-Vary: foo\",\n\n\"X-Test2: bar\nX-Vary: noop\",\n\n\"X-Vary: bar\",\n]\n--- response_headers_like eval\n[\n\"X-Cache: MISS from .*\",\n\"X-Cache: HIT from .*\",\n\"X-Cache: MISS from .*\",\n\"X-Cache: HIT from .*\",\n]\n--- response_body eval\n[\n\"TEST 8: 1\",\n\"TEST 8: 1\",\n\"TEST 8: 2\",\n\"TEST 8: 2\",\n]\n--- no_error_log\n[error]\n\n\n=== TEST 9: Other representations are preserved with a no-cache-response\n--- http_config eval: $::HttpConfig\n--- config\nlocation /vary_9_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge.state_machine\").set_debug(true)\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /vary_9 {\n content_by_lua_block {\n local incr = ngx.shared.ledge_test:incr(\"test9\", 1, 0)\n if incr == 3 then\n ngx.header[\"Cache-Control\"] = \"no-cache\"\n else\n ngx.header[\"Cache-Control\"] = \"max-age=60\"\n end\n ngx.header[\"Vary\"] = \"X-Test\"\n ngx.print(\"TEST 9: \", incr)\n }\n}\n--- request eval\n[\n\"GET /vary_9_prx\",\n\"GET /vary_9_prx\",\n\"GET /vary_9_prx\",\n\"GET /vary_9_prx\",\n]\n--- more_headers eval\n[\n\"X-Test: Foo\",\n\"X-Test: Bar\",\n\"X-Test: Foo\nCache-Control: no-cache\",\n\"X-Test: Bar\",\n]\n--- response_body eval\n[\n\"TEST 9: 1\",\n\"TEST 9: 2\",\n\"TEST 9: 3\",\n\"TEST 9: 2\",\n]\n--- no_error_log\n[error]\n\n=== TEST 10: Vary key cleaned up\n--- http_config eval: $::HttpConfig\n--- config\nlocation /vary_10_check {\n rewrite ^(.*)_check$ $1 break;\n content_by_lua_block {\n local redis = require(\"ledge\").create_redis_connection()\n local handler = require(\"ledge\").create_handler()\n handler.redis = redis\n\n local chain = handler:cache_key_chain()\n\n local res, err = redis:smembers(chain.repset)\n local exists, err = redis:exists(chain.vary)\n ngx.print(#res, \" \", exists)\n }\n}\nlocation /vary_10_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /vary_10 {\n content_by_lua_block {\n local incr = ngx.shared.ledge_test:incr(\"test10\", 1, 0)\n if incr < 3 then\n ngx.header[\"Cache-Control\"] = \"max-age=60\"\n else\n ngx.header[\"Cache-Control\"] = \"no-cache\"\n end\n ngx.header[\"Vary\"] = \"X-Test\"\n ngx.print(\"TEST 10: \", incr)\n }\n}\n--- request eval\n[\n\"GET /vary_10_prx\",\n\"GET /vary_10_prx\",\n\"GET /vary_10_check\",\n\"GET /vary_10_prx\",\n\"GET /vary_10_check\",\n\"GET /vary_10_prx\",\n\"GET /vary_10_check\",\n]\n--- more_headers eval\n[\n\"X-Test: Foo\",\n\"X-Test: Bar\",\n\"\",\n\"X-Test: Foo\nCache-Control: no-cache\",\n\"\",\n\"X-Test: Bar\nCache-Control: no-cache\",\n\"\",\n]\n--- response_body eval\n[\n\"TEST 10: 1\",\n\"TEST 10: 2\",\n\"2 1\",\n\"TEST 10: 3\",\n\"1 1\",\n\"TEST 10: 4\",\n\"0 0\",\n]\n--- no_error_log\n[error]\n\n=== TEST 11: Missing repset re-created on read\n--- http_config eval: $::HttpConfig\n--- config\nlocation /vary_11_break {\n rewrite ^(.*)_break $1 break;\n content_by_lua_block {\n local redis = require(\"ledge\").create_redis_connection()\n local handler = require(\"ledge\").create_handler()\n handler.redis = redis\n\n local chain = handler:cache_key_chain()\n\n local res, err = redis:del(chain.repset)\n local exists, err = redis:exists(chain.repset)\n ngx.print(exists)\n }\n}\nlocation /vary_11_check {\n rewrite ^(.*)_check$ $1 break;\n content_by_lua_block {\n local redis = require(\"ledge\").create_redis_connection()\n local handler = require(\"ledge\").create_handler()\n handler.redis = redis\n\n local chain = handler:cache_key_chain()\n\n local res, err = redis:smembers(chain.repset)\n ngx.print(#res)\n }\n}\nlocation /vary_11_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /vary_11 {\n content_by_lua_block {\n local incr = ngx.shared.ledge_test:incr(\"test11\", 1, 0)\n if incr < 3 then\n ngx.header[\"Cache-Control\"] = \"max-age=60\"\n else\n ngx.header[\"Cache-Control\"] = \"no-cache\"\n end\n ngx.header[\"Vary\"] = \"X-Test\"\n ngx.print(\"TEST 11: \", incr)\n }\n}\n--- request eval\n[\n\"GET /vary_11_prx\",\n\"GET /vary_11_prx\",\n\"GET /vary_11_break\",\n\"GET /vary_11_prx\",\n\"GET /vary_11_check\",\n\"GET /vary_11_prx\",\n\"GET /vary_11_check\",\n]\n--- more_headers eval\n[\n\"X-Test: Foo\",\n\"X-Test: Bar\",\n\"\",\n\"X-Test: Foo\",\n\"\",\n\"X-Test: Bar\",\n\"\",\n]\n--- response_body eval\n[\n\"TEST 11: 1\",\n\"TEST 11: 2\",\n\"0\",\n\"TEST 11: 1\",\n\"1\",\n\"TEST 11: 2\",\n\"2\",\n]\n--- response_headers_like eval\n[\n\"X-Cache: MISS from .*\",\n\"X-Cache: MISS from .*\",\n\"\",\n\"X-Cache: HIT from .*\",\n\"\",\n\"X-Cache: HIT from .*\",\n\"\",\n]\n--- no_error_log\n[error]\n" }, { "path": "t/02-integration/via_header.t", "content": "use Test::Nginx::Socket 'no_plan';\nuse FindBin;\nuse lib \"$FindBin::Bin/..\";\nuse LedgeEnv;\n\nour $HttpConfig = LedgeEnv::http_config();\n\nno_long_string();\nno_diff();\nrun_tests();\n\n__DATA__\n=== TEST 1: Ledge version advertised by default\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /t {\n echo \"ORIGIN\";\n}\n--- request\nGET /t_prx\n--- response_headers_like\nVia: \\d+\\.\\d+ .+ \\(ledge/\\d+\\.\\d+[\\.\\d]*\\)\n--- no_error_log\n[error]\n\n\n=== TEST 2: Ledge version not advertised\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler({\n advertise_ledge = false,\n }):run()\n }\n}\nlocation /t {\n echo \"ORIGIN\";\n}\n--- request\nGET /t_prx\n--- raw_response_headers_unlike: Via: \\d+\\.\\d+ .+ \\(ledge/\\d+\\.\\d+[\\.\\d]*\\)\n--- no_error_log\n[error]\n\n\n=== TEST 3: Via header uses visible_hostname config\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler({\n visible_hostname = \"ledge.example.com\"\n }):run()\n }\n}\nlocation /t {\n echo \"ORIGIN\";\n}\n--- request\nGET /t_prx\n--- response_headers_like\nVia: \\d+\\.\\d+ ledge.example.com:\\d+ \\(ledge/\\d+\\.\\d+[\\.\\d]*\\)\n--- no_error_log\n[error]\n\n\n=== TEST 4: Via header from upstream\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /t {\n content_by_lua_block {\n ngx.header[\"Via\"] = \"1.1 foo\"\n }\n}\n--- request\nGET /t_prx\n--- more_headers\nCache-Control: no-cache\n--- response_headers_like\nVia: \\d+\\.\\d+ .+ \\(ledge/\\d+\\.\\d+[\\.\\d]*\\), \\d+\\.\\d+ foo\n--- no_error_log\n[error]\n\n\n=== TEST 5: Erroneous multiple Via headers from upstream\n--- http_config eval: $::HttpConfig\n--- config\nlocation /t_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler({\n upstream_port = 1985,\n }):run()\n }\n}\n--- tcp_listen: 1985\n--- tcp_reply\nHTTP/1.1 200 OK\nContent-Length: 2\nContent-Type: text/plain\nVia: 1.1 foo\nVia: 1.1 foo.bar\n\nOK\n\n--- request\nGET /t_prx\n--- more_headers\nCache-Control: no-cache\n--- response_body: OK\n--- response_headers_like\nVia: 1.1 .+ \\(ledge/\\d+\\.\\d+[\\.\\d]*\\), 1.1 foo, 1.1 foo.bar\n--- no_error_log\n[error]\n" }, { "path": "t/03-sentinel/01-master_up.t", "content": "use Test::Nginx::Socket 'no_plan';\nuse Cwd qw(cwd);\n\nmy $pwd = cwd();\n\n$ENV{TEST_NGINX_PORT} ||= 1984;\n$ENV{TEST_LEDGE_REDIS_DATABASE} ||= 2;\n$ENV{TEST_LEDGE_REDIS_QLESS_DATABASE} ||= 3;\n$ENV{TEST_LEDGE_SENTINEL_MASTER_NAME} ||= 'mymaster';\n$ENV{TEST_LEDGE_SENTINEL_PORT} ||= 6381;\n$ENV{TEST_COVERAGE} ||= 0;\n\nour $HttpConfig = qq{\nlua_package_path \"./lib/?.lua;../lua-resty-redis-connector/lib/?.lua;../lua-resty-qless/lib/?.lua;../lua-resty-http/lib/?.lua;../lua-ffi-zlib/lib/?.lua;;\";\n\ninit_by_lua_block {\n if $ENV{TEST_COVERAGE} == 1 then\n require(\"luacov.runner\").init()\n end\n\n local db = $ENV{TEST_LEDGE_REDIS_DATABASE}\n local qless_db = $ENV{TEST_LEDGE_REDIS_QLESS_DATABASE}\n local master_name = '$ENV{TEST_LEDGE_SENTINEL_MASTER_NAME}'\n local sentinel_port = $ENV{TEST_LEDGE_SENTINEL_PORT}\n\n local redis_connector_params = {\n url = \"sentinel://\" .. master_name .. \":m/\" .. tostring(db),\n sentinels = {\n { host = \"127.0.0.1\", port = sentinel_port },\n },\n }\n\n require(\"ledge\").configure({\n redis_connector_params = redis_connector_params,\n qless_db = $ENV{TEST_LEDGE_REDIS_QLESS_DATABASE},\n })\n\n require(\"ledge\").set_handler_defaults({\n upstream_host = \"127.0.0.1\",\n upstream_port = $ENV{TEST_NGINX_PORT},\n storage_driver_config = {\n redis_connector_params = redis_connector_params,\n }\n })\n}\n\ninit_worker_by_lua_block {\n require(\"ledge\").create_worker():run()\n}\n\n};\n\nno_long_string();\nrun_tests();\n\n__DATA__\n=== TEST 1: Prime cache\n--- http_config eval: $::HttpConfig\n--- config\nlocation /sentinel_1_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /sentinel_1 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.say(\"OK\")\n }\n}\n--- request\nGET /sentinel_1_prx\n--- response_body\nOK\n--- no_error_log\n[error]\n\n\n=== TEST 2: create_redis_slave_connection\n--- http_config eval: $::HttpConfig\n--- config\nlocation /sentinel_1_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n local slave, err = require(\"ledge\").create_redis_slave_connection()\n assert(slave and not err,\n \"create_redis_slave_connection should return positively\")\n\n assert(slave:role()[1] == \"slave\", \"role should be slave\")\n }\n}\n--- request\nGET /sentinel_1_prx\n--- no_error_log\n[error]\n\n\n=== TEST 4a: Prime cache\n--- http_config eval: $::HttpConfig\n--- config\nlocation /sentinel_3_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /sentinel_3 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.say(\"OK\")\n }\n}\n--- request\nGET /sentinel_3_prx\n--- response_body\nOK\n--- no_error_log\n[error]\n\n\n=== TEST 3: Wildcard Purge (scan on slave)\n--- http_config eval: $::HttpConfig\n--- config\nlocation /sentinel_3 {\n content_by_lua_block {\n require(\"ledge\").create_handler({\n keyspace_scan_count = 1,\n }):run()\n }\n}\n--- request\nPURGE /sentinel_3*\n--- wait: 1\n--- no_error_log\n[error]\n--- error_code: 200\n" }, { "path": "t/03-sentinel/02-master_down.t", "content": "use Test::Nginx::Socket 'no_plan';\nuse Cwd qw(cwd);\n\nmy $pwd = cwd();\n\n$ENV{TEST_NGINX_PORT} ||= 1984;\n$ENV{TEST_LEDGE_REDIS_DATABASE} ||= 2;\n$ENV{TEST_LEDGE_REDIS_QLESS_DATABASE} ||= 3;\n$ENV{TEST_LEDGE_SENTINEL_MASTER_NAME} ||= 'mymaster';\n$ENV{TEST_LEDGE_SENTINEL_PORT} ||= 6381;\n$ENV{TEST_COVERAGE} ||= 0;\n\nour $HttpConfig = qq{\nlua_package_path \"./lib/?.lua;../lua-resty-redis-connector/lib/?.lua;../lua-resty-qless/lib/?.lua;../lua-resty-http/lib/?.lua;../lua-ffi-zlib/lib/?.lua;;\";\n\nlua_socket_log_errors Off;\ninit_by_lua_block {\n\n if $ENV{TEST_COVERAGE} == 1 then\n require(\"luacov.runner\").init()\n end\n\n local db = $ENV{TEST_LEDGE_REDIS_DATABASE}\n local qless_db = $ENV{TEST_LEDGE_REDIS_QLESS_DATABASE}\n local master_name = '$ENV{TEST_LEDGE_SENTINEL_MASTER_NAME}'\n local sentinel_port = $ENV{TEST_LEDGE_SENTINEL_PORT}\n\n local redis_connector_params = {\n url = \"sentinel://\" .. master_name .. \":s/\" .. tostring(db),\n sentinels = {\n { host = \"127.0.0.1\", port = sentinel_port },\n },\n }\n\n require(\"ledge\").configure({\n redis_connector_params = redis_connector_params,\n qless_db = $ENV{TEST_LEDGE_REDIS_QLESS_DATABASE},\n })\n\n require(\"ledge\").set_handler_defaults({\n upstream_host = \"127.0.0.1\",\n upstream_port = $ENV{TEST_NGINX_PORT},\n storage_driver_config = {\n redis_connector_params = redis_connector_params,\n }\n })\n}\n\n};\n\nno_long_string();\nrun_tests();\n\n__DATA__\n=== TEST 1: Read from cache (primed in previous test file)\n--- http_config eval: $::HttpConfig\n--- config\nlocation /sentinel_1_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n}\nlocation /sentinel_1 {\n echo \"ORIGIN\";\n}\n--- request\nGET /sentinel_1_prx\n--- response_body\nOK\n--- no_error_log\n[error]\n\n\n=== TEST 2: The write will fail, but well still get a 200 with our new content.\n--- http_config eval: $::HttpConfig\n--- config\n\tlocation /sentinel_2_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n }\n location /sentinel_2 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.say(\"TEST 2\")\n }\n }\n--- request\nGET /sentinel_2_prx\n--- response_body\nTEST 2\n--- error_log\nREADONLY You can't write against a read only slave.\n\n\n=== TEST 2b: The write will fail, but we still get a 200 with our content.\n--- http_config eval: $::HttpConfig\n--- config\n location /sentinel_2_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n }\n location /sentinel_2 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.say(\"TEST 2b\")\n }\n }\n--- request\nGET /sentinel_2_prx\n--- response_body\nTEST 2b\n--- error_log\nREADONLY You can't write against a read only slave.\n" }, { "path": "t/03-sentinel/03-slave_promoted.t", "content": "use Test::Nginx::Socket 'no_plan';\nuse Cwd qw(cwd);\n\nmy $pwd = cwd();\n\n$ENV{TEST_NGINX_PORT} ||= 1984;\n$ENV{TEST_LEDGE_REDIS_DATABASE} ||= 2;\n$ENV{TEST_LEDGE_REDIS_QLESS_DATABASE} ||= 3;\n$ENV{TEST_LEDGE_SENTINEL_MASTER_NAME} ||= 'mymaster';\n$ENV{TEST_LEDGE_SENTINEL_PORT} ||= 6381;\n$ENV{TEST_COVERAGE} ||= 0;\n\nour $HttpConfig = qq{\nlua_package_path \"./lib/?.lua;../lua-resty-redis-connector/lib/?.lua;../lua-resty-qless/lib/?.lua;../lua-resty-http/lib/?.lua;../lua-ffi-zlib/lib/?.lua;;\";\n\ninit_by_lua_block {\n if $ENV{TEST_COVERAGE} == 1 then\n require(\"luacov.runner\").init()\n end\n\n local db = $ENV{TEST_LEDGE_REDIS_DATABASE}\n local qless_db = $ENV{TEST_LEDGE_REDIS_QLESS_DATABASE}\n local master_name = '$ENV{TEST_LEDGE_SENTINEL_MASTER_NAME}'\n local sentinel_port = $ENV{TEST_LEDGE_SENTINEL_PORT}\n\n local redis_connector_params = {\n url = \"sentinel://\" .. master_name .. \":a/\" .. tostring(db),\n sentinels = {\n { host = \"127.0.0.1\", port = sentinel_port },\n },\n }\n\n require(\"ledge\").configure({\n redis_connector_params = redis_connector_params,\n qless_db = $ENV{TEST_LEDGE_REDIS_QLESS_DATABASE},\n })\n\n require(\"ledge\").set_handler_defaults({\n upstream_host = \"127.0.0.1\",\n upstream_port = $ENV{TEST_NGINX_PORT},\n storage_driver_config = {\n redis_connector_params = redis_connector_params,\n }\n })\n}\n\n};\n\nno_long_string();\nrun_tests();\n\n__DATA__\n=== TEST 1: Read from cache (primed in previous test file)\n--- http_config eval: $::HttpConfig\n--- config\n\tlocation /sentinel_1 {\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n }\n--- request\nGET /sentinel_1\n--- response_body\nOK\n--- no_error_log\n[error]\n\n\n=== TEST 2: The write will succeed, as our slave has been promoted.\n--- http_config eval: $::HttpConfig\n--- config\n\tlocation /sentinel_2_prx {\n rewrite ^(.*)_prx$ $1 break;\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n }\n location /sentinel_2 {\n content_by_lua_block {\n ngx.header[\"Cache-Control\"] = \"max-age=3600\"\n ngx.say(\"TEST 2\")\n }\n }\n--- request\nGET /sentinel_2_prx\n--- response_body\nTEST 2\n\n\n=== TEST 2b: Test for cache hit.\n--- http_config eval: $::HttpConfig\n--- config\n\tlocation /sentinel_2 {\n content_by_lua_block {\n require(\"ledge\").create_handler():run()\n }\n }\n--- request\nGET /sentinel_2\n--- response_body\nTEST 2\n" }, { "path": "t/LedgeEnv.pm", "content": "package LedgeEnv;\nuse strict;\nuse warnings;\nuse Exporter;\n\nour $nginx_host = $ENV{TEST_NGINX_HOST} || '127.0.0.1';\nour $nginx_port = $ENV{TEST_NGINX_PORT} || 1984;\nour $test_coverage = $ENV{TEST_COVERAGE} || 0;\n\nour $redis_host = $ENV{TEST_LEDGE_REDIS_HOST} || '127.0.0.1';\nour $redis_port = $ENV{TEST_LEDGE_REDIS_PORT} || 6379;\nour $redis_database = $ENV{TEST_LEDGE_REDIS_DATABASE} || 2;\nour $redis_qless_database = $ENV{TEST_LEDGE_REDIS_QLESS_DATABASE} || 3;\n\nsub http_config {\n my $extra_nginx_config = \"\";\n my $extra_lua_config = \"\";\n my $worker_config = \"\";\n\n my (%args) = @_;\n\n if (defined $args{extra_nginx_config}) {\n $extra_nginx_config = $args{extra_nginx_config};\n }\n\n if (defined $args{extra_lua_config}) {\n $extra_lua_config = $args{extra_lua_config};\n }\n\n if ($args{run_worker}) {\n $worker_config = qq{\n init_worker_by_lua_block {\n require(\"ledge\").create_worker():run()\n }\n };\n }\n\n return qq{\n $extra_nginx_config\n\n lua_package_path \"./lib/?.lua;./extlib/?.lua;;\";\n resolver local=on ipv6=off;\n\n init_by_lua_block {\n if $test_coverage == 1 then\n require(\"luacov.runner\").init()\n end\n\n local REDIS_URL = \"redis://$redis_host:$redis_port/$redis_database\"\n\n require(\"ledge\").configure({\n redis_connector_params = { url = REDIS_URL },\n qless_db = $redis_qless_database,\n })\n\n require(\"ledge\").set_handler_defaults({\n upstream_host = \"$nginx_host\",\n upstream_port = $nginx_port,\n storage_driver_config = {\n redis_connector_params = { url = REDIS_URL },\n },\n })\n\n $extra_lua_config;\n }\n\n $worker_config\n }\n}\n\nour @EXPORT = qw( http_config );\n\n1;\n" }, { "path": "t/cert/example.com.crt", "content": "-----BEGIN CERTIFICATE-----\nMIIDZDCCAkwCCQC9pPAJEKdAJTANBgkqhkiG9w0BAQsFADCBiTELMAkGA1UEBhMC\nVUsxDzANBgNVBAgMBkxvbmRvbjEPMA0GA1UEBwwGTG9uZG9uMREwDwYDVQQKDAhT\ncXVpeiBVSzEQMA4GA1UECwwHSG9zdGluZzESMBAGA1UEAwwJRWRnZSBUZXN0MR8w\nHQYJKoZIhvcNAQkBFhBlZGdlQHNxdWl6LmNvLnVrMCAXDTE5MTExMjIyNDA0MFoY\nDzIxMTkxMDE5MjI0MDQwWjBcMQswCQYDVQQGEwJVSzETMBEGA1UECAwKU29tZS1T\ndGF0ZTEPMA0GA1UEBwwGTG9uZG9uMREwDwYDVQQKDAhTcXVpeiBVSzEUMBIGA1UE\nAwwLZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDr\njlyu9bPMemMkqfe0fi+HaLfXsYaMguJyzaOKIf11RAHG4Ptl3XHk4a6OrKR3MFuC\nMnGmOuPOAJPRwGJ2PMNX6g3dI0UsEqMxdOEadJsaP/kcV22OmVTDpErdbADItk8h\ntgCvo+QWUIIFCUMbd8t2nJpgusnhyfyhipwBiBTaiflANYFfINty8D39ohgHzg6j\ntTiOLf2jBEurFJkekb8bu2kWcDxFv0lpR4VurMpvaguxuAM2XhpQVjHzqhJ28AlG\nBJY8KV4OPZb3Qz+rZnojat3QKVoDIJc42pFRAUFTyanBF/m8ayZqOOZdH90t8bcj\ndLEKMgUBHB9vnDDOOVfnAgMBAAEwDQYJKoZIhvcNAQELBQADggEBAHbAH10eDzlq\nB3QXMHidAwZhaYEwnfqFINSYThLir9o8/WtgiMvhOaD5BXZNvuoePsYjZZyYIx14\nNF+xJe0Ijnh15RnAtNBDuw+NkGKVqtszdW1SNBkU9bH4rJXJYJOkHORkIRLRWlwl\n/YR/YpXOwPSKCIgl9K8H3FsuAbjNB+sjUsaSsbyTKbOVUB67BvjDSb0e0UZwSBtc\n9wWLAdHL2gZQD+tsX/vEv1F0QdaKsBEjMfYCuLfk0Ov51hKLXfyrIG08T7Csm9Mf\nqvIku5Itl4AWNbGQZpXnbqtUHOF1OaWBe9i5xNoMtvv+WWqJcun9NQBsdMnpNMpE\nMF3NccQR5iA=\n-----END CERTIFICATE-----\n" }, { "path": "t/cert/example.com.key", "content": "-----BEGIN RSA PRIVATE KEY-----\nMIIEpQIBAAKCAQEA645crvWzzHpjJKn3tH4vh2i317GGjILics2jiiH9dUQBxuD7\nZd1x5OGujqykdzBbgjJxpjrjzgCT0cBidjzDV+oN3SNFLBKjMXThGnSbGj/5HFdt\njplUw6RK3WwAyLZPIbYAr6PkFlCCBQlDG3fLdpyaYLrJ4cn8oYqcAYgU2on5QDWB\nXyDbcvA9/aIYB84Oo7U4ji39owRLqxSZHpG/G7tpFnA8Rb9JaUeFbqzKb2oLsbgD\nNl4aUFYx86oSdvAJRgSWPCleDj2W90M/q2Z6I2rd0ClaAyCXONqRUQFBU8mpwRf5\nvGsmajjmXR/dLfG3I3SxCjIFARwfb5wwzjlX5wIDAQABAoIBADOdBgHwJG1xg7fM\n5lHONGvfLik85NZ091lgZa0mtXq0ZA9HzM4NL5+PM8hfW8oh9msY0n4x+ShyR/F1\nzh1KQyNITbFewRFfJBL6ITjCxBmEWvkyzvan8kLMBPtvZtyT1dL1JkFWD+wzx8mC\ntgmWviZHOixnwUSQFaLv1C8hujAID82wkoSiEhCgl+B69JxHG9zwsTiKmBnt8iTb\nURD38MPGxJkpGdzkWScb0nlSsbm8IZpPbEll0HU5UYjB0vkuv4tt5Ou23AqxTde0\npsC1WZYa4qyomycATX9+PykBFEA0Qkd56/oHnFMZTmPQKUevqtq3axMOeUc6vqjJ\nMEf5+8ECgYEA+MFDdhX5leLh0CxZf4PuHgaHTP9EITL9obU3Nh87fZYRlFVDjwpn\n7ZMsAJw4f7uTXYH6j5e0oQ6KmCCk65Ak89/8roq+sbymm5AGLs07PGtUYKF1goq6\nJqhMBslWPD8rkWMiUzRtlf7yQq6iA2gOQTecRKKfbjYT9vNFPqkB51sCgYEA8mqv\n3Y4ZUX6cCkAwsA+TEOqKOh52VCgJ3xtrL7iUjJ2hsfT8avM1TsvklSGob4/LWqPQ\nKN+EExaM3vULydxZ+WgSvOD73OyFSa7J25NdHdqFkyde2DC+f2aPa7zcRanuD1DD\nh6e6/Z3OGa2pZq7Ed5cW7gAljSLQJZIFNN65A2UCgYEAo+VSOX+JDoSKG8rcvPOD\n9CyBAO4/SVB7ZAwt8G7rl3dE5eK3vIsypomNOGm1oBNKqRV2rR1bWbJnBoybnMlA\nT56IsceglSKi82QVbsix+sEMuw4ming05juEvAPz2YYVgpk6iG/GtElh/SVqgawR\nmE63m1E6kjb3OIJYYUyhgHkCgYEAr9USmvFnC+V56TWGGy4wziRQ/rb5vTENd/a7\nWHHZzeTIU/wO2sRt9imOM12mfsUeCzCm2/7EHdRNearkUhaybGVAsh++kBA+3aMa\nZ1oMQIswN/xmnwk8I8yQWuUyIJWRRyqdqNfQmgTMaXO9W+2IM/Yze44/ro+Byr6P\naDnkmMECgYEAsHbqYEwD54neIKyNxlmYemybqqh63JVFp84VomJxMfHumVVW/UxQ\nPt7jKgbsvO7ayBzETtSpaQ3ajCiiKPKlrupqJDb9yXwSpdfbf6fRamQLDvVWJNkD\nZjLBiOUX5dS4Il0kEvUCeikgyFRssEFP6J5c2+Cbr5Gt12EnIQhu//I=\n-----END RSA PRIVATE KEY-----\n" }, { "path": "t/cert/rootCA.key", "content": "-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEAy6v3eQFGXrSe97M/9ullfby4bg2jF+9h9p5P/wiv+Y9dTdk8\nDRR7M40laCqnmoXSSndAvHLb+lqeHs0GDkp+ofgBg8iP8p8IX0RQ33lqcrU65FJu\nImMksswkYZLak78YpR1wqF4EMpHH6t7SKkPlFgULkg+vTpml83fFdnq2WVk8UOSx\nAsrllvVtxys8c8ny0hmBXqOWtISPQFYBNAy7N7uHLF8D/4/YMBHM2Vgp5oaMkvFX\nnsDyL6JWvnqTnz1kKjL9lbYHbxP7oziI74MIRvRzsw8Ou+OfWiYKTRZDlxfl+7Jr\nzOhxjbTpP/cOCw3ngBC6rrpdflDVuyf6r7rv2wIDAQABAoIBABtC1Uj5BrY+btiw\nwWsHKnJ+BCGW6bGWdQJRhluYihVZPx/gZ81IZIUt60faDbz9FHyrIZsXtKH55xgw\nURMwnWqIi4tcGQhciP5XYovG8JyR7WQKNHud0ZetA2GcCm2kMmRHYIDotJ8gLCYf\n1PmbRNqBql7OgqR+pFvGOEP3gNjMf66m+A0jnSrXk2zLBGeo3hfEBpsWsm2fay5T\nZ8r5PlUn8rXSBkVCK9AGGf2Ngg07Z+2wj08Htn+hq3+kTNf65D/buVpkg9kNqrxm\ncDy1v182KBiM/jXjydoBt33GKxNkG4Ds6eH0W2hFzP8oOmYq4DCZ1BryUGhR+U6c\nizj6/SECgYEA8iw2e0blUsd2Esi5iD/sDr3ZmQONEeWCTv9Twwvl0cu3h5BRjxH5\nypq/F1B1matjTFPxXY+PpVSj1dbYRGydyAfzaRwcox/ox/P+0h76zQeb3BMSnpmK\ng7dpNCARJVW9NbtDam9Y6UBgNZU3giBBaE8pKlrwwnhRTcrlfLmQIk0CgYEA10z+\nU/cXWh9SlGqz+XcRq4+Tp3oZ+gBVdartwJjjSLl2DXMOnxchhSw0pkCkAURvGEps\n+mvopXn7fmpENOyXFN/TrnH09qY1exDl/xkzPvR5akCy0HkkAERtpYMfPfYzYYrI\nG+W5olDMM5SmSU7rmKeQsTlVtNyavbx9+TB4XscCgYEAl20D6BONgzRLZTVzpXlq\nzlDxxdbNl9otn93Rb016N7OtH6wjA1XXHlOilx5tWlgrb+exLbJ9vIBvLV/4vNg5\n1ID8N8YnNezW7mhn9tT+N8PBNlwKsXcKgI/nzXsbnX++HuHoJp5XNwpU3kxeeBRZ\nMbMF54ETuFXpaL4svs99C6UCgYAuw7d+T25QEfui5yZeakF5TT9aIkhgKBBn9Y+c\nxNihZD9DHpmvbpvGTFrHPcUhzVaAJTJUlnm676rzw2s7P6R1UUSuYGw/4sw9Beef\nKD8cTofMz27Hn3h1YmeaisePctmoNzfN73EJ05j3HzObOrwrtUHVbMmz9jLaQYXv\nSVrr4wKBgQDfpZKufKQEX4q/L5OUs03hxRVEkSwtWwvIjXWFqbqvhkSEqB04+bns\n3PhhsqVGa4nGmgua6f4/GPy2OAiniIbVupoyz/i3co8usixihH6U0/JhRJO6HHU2\nte+2zL1GIHHBAA4fmBIaHXtBC7kta5Ck8RUJ2eSMQ4TFl091wvJxfg==\n-----END RSA PRIVATE KEY-----\n" }, { "path": "t/cert/rootCA.pem", "content": "-----BEGIN CERTIFICATE-----\nMIIDkjCCAnoCCQCRNvMmzZMQezANBgkqhkiG9w0BAQsFADCBiTELMAkGA1UEBhMC\nVUsxDzANBgNVBAgMBkxvbmRvbjEPMA0GA1UEBwwGTG9uZG9uMREwDwYDVQQKDAhT\ncXVpeiBVSzEQMA4GA1UECwwHSG9zdGluZzESMBAGA1UEAwwJRWRnZSBUZXN0MR8w\nHQYJKoZIhvcNAQkBFhBlZGdlQHNxdWl6LmNvLnVrMCAXDTE5MTExMjIyMzcxNloY\nDzIxMTkxMDE5MjIzNzE2WjCBiTELMAkGA1UEBhMCVUsxDzANBgNVBAgMBkxvbmRv\nbjEPMA0GA1UEBwwGTG9uZG9uMREwDwYDVQQKDAhTcXVpeiBVSzEQMA4GA1UECwwH\nSG9zdGluZzESMBAGA1UEAwwJRWRnZSBUZXN0MR8wHQYJKoZIhvcNAQkBFhBlZGdl\nQHNxdWl6LmNvLnVrMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAy6v3\neQFGXrSe97M/9ullfby4bg2jF+9h9p5P/wiv+Y9dTdk8DRR7M40laCqnmoXSSndA\nvHLb+lqeHs0GDkp+ofgBg8iP8p8IX0RQ33lqcrU65FJuImMksswkYZLak78YpR1w\nqF4EMpHH6t7SKkPlFgULkg+vTpml83fFdnq2WVk8UOSxAsrllvVtxys8c8ny0hmB\nXqOWtISPQFYBNAy7N7uHLF8D/4/YMBHM2Vgp5oaMkvFXnsDyL6JWvnqTnz1kKjL9\nlbYHbxP7oziI74MIRvRzsw8Ou+OfWiYKTRZDlxfl+7JrzOhxjbTpP/cOCw3ngBC6\nrrpdflDVuyf6r7rv2wIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQAuBBRjAQ/ZEwNa\nUXkCZPc+3QxzWrKQCXcf8WoQYb73KaQKRd5gl8QQNWRkmiFdBngDwitd1xGVn0S3\nd0jHQAvreWUztiv3fu/Uf/fGv0BWJw9ve9+Wuw4ENINR6rQbRpecXW9Ia/4Jep0w\npYFvNLBFUqPzukrdkf8UdCLyyl4H/gWENtgjgvURAxKCDJGkd3XiiBirT2837mNT\noRweVDY8gxECd+Os2OIDL4B6mon2m3oSEiJpL72bxsX0rwwc7dKdsuOrjuKyg+Jb\nokTqY3oO5UzwEDVuKwuOOdvpO11LhtQ7SfjZiMQW2NAHeBtJqNgxcYIbJPeU1Zli\naSxKnsds\n-----END CERTIFICATE-----\n" }, { "path": "t/cert/rootCA.srl", "content": "BDA4F00910A74025\n" }, { "path": "util/lua-releng", "content": "#!/usr/bin/env perl\n\nuse strict;\nuse warnings;\n\nuse Getopt::Std;\n\nmy (@luas, @tests);\n\nmy %opts;\ngetopts('Lse', \\%opts) or die \"Usage: lua-releng [-L] [-s] [-e] [files]\\n\";\n\nmy $silent = $opts{s};\nmy $stop_on_error = $opts{e};\nmy $no_long_line_check = $opts{L};\n\nmy $check_lua_ver = \"luac -v | awk '{print\\$2}'| grep 5.1\";\nmy $output = `$check_lua_ver`;\nif ($output eq '') {\n die \"ERROR: lua-releng ONLY supports Lua 5.1!\\n\";\n}\n\nif ($#ARGV != -1) {\n @luas = @ARGV;\n\n} else {\n @luas = map glob, qw{ *.lua lib/*.lua lib/*/*.lua lib/*/*/*.lua lib/*/*/*/*.lua lib/*/*/*/*/*.lua };\n if (-d 't') {\n @tests = map glob, qw{ t/*.t t/*/*.t t/*/*/*.t };\n }\n}\n\nfor my $f (sort @luas) {\n process_file($f);\n}\n\nfor my $t (@tests) {\n blank(qq{grep -H -n --color -E '\\\\--- ?(ONLY|LAST)' $t});\n}\n# p: prints a string to STDOUT appending \\n\n# w: prints a string to STDERR appending \\n\n# Both respect the $silent value\nsub p { print \"$_[0]\\n\" if (!$silent) }\nsub w { warn \"$_[0]\\n\" if (!$silent) }\n\n# blank: runs a command and looks at the output. If the output is not\n# blank it is printed (and the program dies if stop_on_error is 1)\nsub blank {\n my ($command) = @_;\n if ($stop_on_error) {\n my $output = `$command`;\n if ($output ne '') {\n die $output;\n }\n } else {\n system($command);\n }\n}\n\nmy $version;\nsub process_file {\n my $file = shift;\n # Check the sanity of each .lua file\n open my $in, $file or\n die \"ERROR: Can't open $file for reading: $!\\n\";\n my $found_ver;\n while (<$in>) {\n my ($ver, $skipping);\n if (/(?x) (?:_VERSION|version) \\s* = .*? ([\\d\\.]*\\d+) (.*? SKIP)?/) {\n my $orig_ver = $ver = $1;\n $found_ver = 1;\n $skipping = $2;\n $ver =~ s{^(\\d+)\\.(\\d{3})(\\d{3})$}{join '.', int($1), int($2), int($3)}e;\n w(\"$file: $orig_ver ($ver)\");\n last;\n\n } elsif (/(?x) (?:_VERSION|version) \\s* = \\s* ([a-zA-Z_]\\S*)/) {\n w(\"$file: $1\");\n $found_ver = 1;\n last;\n }\n\n if ($ver and $version and !$skipping) {\n if ($version ne $ver) {\n die \"$file: $ver != $version\\n\";\n }\n } elsif ($ver and !$version) {\n $version = $ver;\n }\n }\n if (!$found_ver) {\n w(\"WARNING: No \\\"_VERSION\\\" or \\\"version\\\" field found in `$file`.\");\n }\n close $in;\n\n p(\"Checking use of Lua global variables in file $file...\");\n p(\"\\top no.\\tline\\tinstruction\\targs\\t; code\");\n blank(\"luac -p -l $file | grep -E '[GS]ETGLOBAL' | grep -vE '\\\\<(require|type|tostring|error|ngx|ndk|jit|setmetatable|getmetatable|string|table|io|os|print|tonumber|math|pcall|xpcall|unpack|pairs|ipairs|assert|module|package|coroutine|[gs]etfenv|next|rawget|rawset|rawlen|select|loadstring)\\\\>'\");\n unless ($no_long_line_check) {\n p(\"Checking line length exceeding 80...\");\n blank(\"grep -H -n -E --color '.{81}' $file\");\n }\n}\n" } ]