Repository: openresty/drizzle-nginx-module Branch: master Commit: 1312cf99b711 Files: 47 Total size: 304.0 KB Directory structure: gitextract_0fqk8n_f/ ├── .gitattributes ├── .github/ │ └── workflows/ │ └── coverity.yml ├── .gitignore ├── .travis.yml ├── Changes ├── README.markdown ├── config ├── doc/ │ └── design ├── src/ │ ├── ddebug.h │ ├── ngx_http_drizzle_checker.c │ ├── ngx_http_drizzle_checker.h │ ├── ngx_http_drizzle_handler.c │ ├── ngx_http_drizzle_handler.h │ ├── ngx_http_drizzle_keepalive.c │ ├── ngx_http_drizzle_keepalive.h │ ├── ngx_http_drizzle_module.c │ ├── ngx_http_drizzle_module.h │ ├── ngx_http_drizzle_output.c │ ├── ngx_http_drizzle_output.h │ ├── ngx_http_drizzle_processor.c │ ├── ngx_http_drizzle_processor.h │ ├── ngx_http_drizzle_quoting.c │ ├── ngx_http_drizzle_quoting.h │ ├── ngx_http_drizzle_upstream.c │ ├── ngx_http_drizzle_upstream.h │ ├── ngx_http_drizzle_util.c │ ├── ngx_http_drizzle_util.h │ └── resty_dbd_stream.h ├── t/ │ ├── 000_init.t │ ├── bugs.t │ ├── charset/ │ │ ├── 000_init.t │ │ ├── errors.t │ │ ├── keepalive.t │ │ ├── methods.t │ │ ├── sanity.t │ │ └── timeout.t │ ├── errors.t │ ├── keepalive.t │ ├── methods.t │ ├── sanity.t │ ├── status.t │ └── timeout.t ├── util/ │ ├── build.sh │ ├── ngx-links │ ├── update-readme.sh │ └── wiki2pod.pl └── valgrind.suppress ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitattributes ================================================ *.t linguist-language=Text ================================================ FILE: .github/workflows/coverity.yml ================================================ name: Coverity on: schedule: - cron: "0 0 * * *" jobs: scan: runs-on: ubuntu-18.04 if: ${{ github.repository_owner == 'openresty' }} env: COVERITY_SCAN_PROJECT_NAME: 'drizzle-nginx-module' COVERITY_SCAN_BRANCH_PATTERN: '*' COVERITY_SCAN_NOTIFICATION_EMAIL: 'chipitsine@gmail.com' LUAJIT_PREFIX: '/opt/luajit21' LUAJIT_LIB: '/opt/luajit21/lib' LUAJIT_INC: '/opt/luajit21/include/luajit-2.1' LUA_INCLUDE_DIR: '/opt/luajit21/include/luajit-2.1' LUA_CMODULE_DIR: '/lib' JOBS: 3 NGX_BUILD_JOBS: 3 NGINX_VERSION: 1.19.9 CC: gcc steps: - uses: actions/checkout@v3 - name: Install apt dependencies run: | sudo apt-get update sudo apt-get install -y axel libgd-dev - name: clone OpenResty satellites run: | git clone https://github.com/openresty/nginx-devel-utils.git git clone https://github.com/openresty/openresty.git ../openresty git clone https://github.com/openresty/no-pool-nginx.git ../no-pool-nginx git clone https://github.com/simpl/ngx_devel_kit.git ../ndk-nginx-module git clone https://github.com/openresty/lua-nginx-module.git ../lua-nginx-module git clone https://github.com/openresty/lua-resty-core.git ../lua-resty-core git clone https://github.com/openresty/lua-resty-lrucache.git ../lua-resty-lrucache git clone https://github.com/openresty/nginx-eval-module.git ../eval-nginx-module git clone https://github.com/openresty/echo-nginx-module.git ../echo-nginx-module git clone https://github.com/openresty/set-misc-nginx-module.git ../set-misc-nginx-module git clone https://github.com/openresty/headers-more-nginx-module.git ../headers-more-nginx-module git clone https://github.com/openresty/rds-json-nginx-module.git ../rds-json-nginx-module git clone https://github.com/openresty/openresty.git ../ngx_openresty - name: Install libdrizzle run: | wget http://openresty.org/download/drizzle7-2011.07.21.tar.gz tar xzf drizzle7-2011.07.21.tar.gz && cd drizzle7-2011.07.21 ./configure --prefix=/usr --without-server sudo PATH=$PATH make libdrizzle-1.0 install-libdrizzle-1.0 - name: Install luajit2 run: | git clone -b v2.1-agentzh https://github.com/openresty/luajit2.git cd luajit2 make -j$JOBS CCDEBUG=-g Q= PREFIX=$LUAJIT_PREFIX CC=$CC XCFLAGS='-DLUA_USE_APICHECK -DLUA_USE_ASSERT' sudo make install PREFIX=$LUAJIT_PREFIX - name: Run Coverity Scan env: COVERITY_SCAN_TOKEN: ${{ secrets.COVERITY_SCAN_TOKEN }} run: | export COVERITY_SCAN_BUILD_COMMAND="sh util/build.sh $NGINX_VERSION" export PATH=$PWD/work/nginx/sbin:$PWD/nginx-devel-utils:$PATH export NGX_BUILD_CC=gcc curl -fsSL "https://scan.coverity.com/scripts/travisci_build_coverity_scan.sh" | bash || true ================================================ FILE: .gitignore ================================================ *.mobi genmobi.sh .libs *.swp *.slo *.la *.swo *.lo *~ *.o print.txt .rsync *.tar.gz dist build[789] build tags update-readme *.tmp test/Makefile test/blib test.sh t.sh t/t.sh test/t/servroot/ releng reset *.t_ src/handler.h src/util.c src/module.h src/module.c src/drizzle.c src/processor.h src/handler.c src/util.h src/drizzle.h src/processor.c src/output.c src/output.h libdrizzle ctags src/stream.h nginx keepalive reindex src/keepalive.c src/keepalive.h src/checker.h src/checker.c src/quoting.h src/quoting.c src/upstream.[ch] all go buildroot/ pack t/servroot/ build1[0-9] *.plist Makefile *.patch analyze *.html ================================================ FILE: .travis.yml ================================================ sudo: required dist: focal branches: only: - "master" os: linux language: c cache: apt: true directories: - download-cache addons: apt: packages: - axel - cpanminus - libtest-base-perl - libtext-diff-perl - liburi-perl - libwww-perl - libtest-longstring-perl - liblist-moreutils-perl - libgd-dev services: - mysql compiler: - gcc env: global: - LUAJIT_PREFIX=/opt/luajit21 - LUAJIT_LIB=$LUAJIT_PREFIX/lib - LIBDRIZZLE_PREFIX=/opt/drizzle - LIBDRIZZLE_INC=$LIBDRIZZLE_PREFIX/include/libdrizzle-1.0 - LIBDRIZZLE_LIB=$LIBDRIZZLE_PREFIX/lib - LUAJIT_INC=$LUAJIT_PREFIX/include/luajit-2.1 - LUA_INCLUDE_DIR=$LUAJIT_INC - LUA_CMODULE_DIR=/lib - JOBS=3 - NGX_BUILD_JOBS=$JOBS - TEST_NGINX_SLEEP=0.006 matrix: - NGINX_VERSION=1.29.8 install: - if [ ! -f download-cache/drizzle7-2011.07.21.tar.gz ]; then wget -P download-cache https://github.com/openresty/openresty-deps-prebuild/releases/download/v20230902/drizzle7-2011.07.21.tar.gz; fi - git clone https://github.com/openresty/nginx-devel-utils.git - git clone https://github.com/openresty/openresty.git ../openresty - git clone https://github.com/openresty/no-pool-nginx.git ../no-pool-nginx - git clone https://github.com/simpl/ngx_devel_kit.git ../ndk-nginx-module - git clone https://github.com/openresty/test-nginx.git - git clone -b v2.1-agentzh https://github.com/openresty/luajit2.git - git clone https://github.com/openresty/lua-nginx-module.git ../lua-nginx-module - git clone https://github.com/openresty/lua-resty-core.git ../lua-resty-core - git clone https://github.com/openresty/lua-resty-lrucache.git ../lua-resty-lrucache - git clone https://github.com/openresty/nginx-eval-module.git ../eval-nginx-module - git clone https://github.com/openresty/echo-nginx-module.git ../echo-nginx-module - git clone https://github.com/openresty/set-misc-nginx-module.git ../set-misc-nginx-module - git clone https://github.com/openresty/headers-more-nginx-module.git ../headers-more-nginx-module - git clone https://github.com/openresty/rds-json-nginx-module.git ../rds-json-nginx-module - git clone https://github.com/openresty/openresty.git ../ngx_openresty before_script: - mysql -uroot -e "create database ngx_test; CREATE USER 'ngx_test'@'%' IDENTIFIED WITH mysql_native_password BY 'ngx_test'; grant all on ngx_test.* to 'ngx_test'@'%'; flush privileges;" script: - tar xzf download-cache/drizzle7-2011.07.21.tar.gz && cd drizzle7-2011.07.21 - ./configure --prefix=$LIBDRIZZLE_PREFIX --without-server > build.log 2>&1 || (cat build.log && exit 1) - sudo PATH=$PATH make libdrizzle-1.0 install-libdrizzle-1.0 > build.log 2>&1 || (cat build.log && exit 1) - cd ../luajit2 - make -j$JOBS CCDEBUG=-g Q= PREFIX=$LUAJIT_PREFIX CC=$CC XCFLAGS='-DLUA_USE_APICHECK -DLUA_USE_ASSERT' > build.log 2>&1 || (cat build.log && exit 1) - sudo make install PREFIX=$LUAJIT_PREFIX > build.log 2>&1 || (cat build.log && exit 1) - cpanm --sudo --notest Test::Nginx IPC::Run Test2::Util > build.log 2>&1 || (cat build.log && exit 1) - cd .. - cd test-nginx && sudo cpanm . && cd .. - export PATH=$PWD/work/nginx/sbin:$PWD/nginx-devel-utils:$PATH - export NGX_BUILD_CC=$CC - sh util/build.sh $NGINX_VERSION > build.log 2>&1 || (cat build.log && exit 1) - nginx -V - ldd `which nginx`|grep luajit - prove -I. -r t ================================================ FILE: Changes ================================================ v0.1.0 - 5 July 2011 * now we require at least libdrizzle 1.0, which supports official mysql 5.5+ and is much more stable under load. thanks Taylor Weibley for pushing this. We no longer require patching libdrizzle any more. * fixed a compilation issue on Mac OS X: we should include drizzle.h prior to nginx headers, or we will not get the "bool" type properly installed on Mac OS X. * fixed the spots that trigger -Wunused-but-set-variable by gcc 4.6. * fixed the duplicate last chunk issue: we should not set the last_buf flag ourselves in drizzle_output.c because ngx_http_upstream already sends a last buf for us. * ported over Maxim Dounin's patch for ngx_http_upstream_keepalive connection pool fixes: we should have discarded stale read events for cached tcp connections in the pool. * implemented the new drizzle_status directive to provide connection pool status monitoring capability. * fixed a minor bug in the connection pool: we should resume the "name", "sockaddr", and "socklen" fields for the connection from the pool such that we can get more detailed error log messages with the "upstream: drizzle://ip.add.re.ss:port" bit. * implemented the $drizzle_thread_id variable which is automatically set when mysql/drizzle times out. * now we use the 2-clause bsd license. * report an error message when upstream name not found for drizzle_pass. * now we implemented the charset option for the "drizzle_server" diredctive which causes ngx_drizzle send "set names xxx" automatcially for every connection to that drizzle server. * added a lot of more documentation. ================================================ FILE: README.markdown ================================================ Name ==== drizzle-nginx-module - Upstream module for talking to MySQL and Drizzle directly *This module is not distributed with the Nginx source.* See [the installation instructions](#installation). Table of Contents ================= * [Status](#status) * [Version](#version) * [Synopsis](#synopsis) * [Description](#description) * [Keepalive connection pool](#keepalive-connection-pool) * [Last Insert ID](#last-insert-id) * [Directives](#directives) * [drizzle_server](#drizzle_server) * [drizzle_keepalive](#drizzle_keepalive) * [drizzle_query](#drizzle_query) * [drizzle_pass](#drizzle_pass) * [drizzle_connect_timeout](#drizzle_connect_timeout) * [drizzle_send_query_timeout](#drizzle_send_query_timeout) * [drizzle_recv_cols_timeout](#drizzle_recv_cols_timeout) * [drizzle_recv_rows_timeout](#drizzle_recv_rows_timeout) * [drizzle_buffer_size](#drizzle_buffer_size) * [drizzle_module_header](#drizzle_module_header) * [drizzle_status](#drizzle_status) * [Variables](#variables) * [$drizzle_thread_id](#drizzle_thread_id) * [Output Format](#output-format) * [RDS Header Part](#rds-header-part) * [RDS Body Part](#rds-body-part) * [Columns](#columns) * [Rows](#rows) * [Row Flag](#row-flag) * [Fields Data](#fields-data) * [RDS buffer Limitations](#rds-buffer-limitations) * [Status Code](#status-code) * [Caveats](#caveats) * [Trouble Shooting](#trouble-shooting) * [Known Issues](#known-issues) * [Installation](#installation) * [Compatibility](#compatibility) * [Community](#community) * [English Mailing List](#english-mailing-list) * [Chinese Mailing List](#chinese-mailing-list) * [Report Bugs](#report-bugs) * [Source Repository](#source-repository) * [Test Suite](#test-suite) * [TODO](#todo) * [Changes](#changes) * [Authors](#authors) * [Copyright & License](#copyright--license) * [See Also](#see-also) Status ====== This module is already production ready. Version ======= This document describes ngx_drizzle [v0.1.11](https://github.com/openresty/drizzle-nginx-module/tags) released on 19 April 2018. Synopsis ======== ```nginx http { ... upstream cluster { # simple round-robin drizzle_server 127.0.0.1:3306 dbname=test password=some_pass user=monty protocol=mysql; drizzle_server 127.0.0.1:1234 dbname=test2 password=pass user=bob protocol=drizzle; } upstream backend { drizzle_server 127.0.0.1:3306 dbname=test password=some_pass user=monty protocol=mysql; } server { location /mysql { set $my_sql 'select * from cats'; drizzle_query $my_sql; drizzle_pass backend; drizzle_connect_timeout 500ms; # default 60s drizzle_send_query_timeout 2s; # default 60s drizzle_recv_cols_timeout 1s; # default 60s drizzle_recv_rows_timeout 1s; # default 60s } ... # for connection pool monitoring location /mysql-pool-status { allow 127.0.0.1; deny all; drizzle_status; } } } ``` [Back to TOC](#table-of-contents) Description =========== This is an nginx upstream module integrating [libdrizzle](https://launchpad.net/drizzle) into Nginx in a non-blocking and streamming way. Essentially it provides a very efficient and flexible way for nginx internals to access MySQL, Drizzle, as well as other RDBMS's that support the Drizzle or MySQL wired protocol. Also it can serve as a direct REST interface to those RDBMS backends. This module does not generate human-readable outputs, rather, in a binary format called Resty-DBD-Stream (RDS) designed by ourselves. You usually need other components, like [rds-json-nginx-module](http://github.com/openresty/rds-json-nginx-module), [rds-csv-nginx-module](http://github.com/openresty/rds-csv-nginx-module), or [lua-rds-parser](http://github.com/openresty/lua-rds-parser), to work with this module. See [Output Format](#output-format) for details. [Back to TOC](#table-of-contents) Keepalive connection pool ------------------------- This module also provides a builtin per-worker connection pool mechanism for MySQL or Drizzle TCP connections. Here's a sample configuration: ```nginx upstream backend { drizzle_server 127.0.0.1:3306 dbname=test password=some_pass user=monty protocol=mysql; drizzle_keepalive max=100 mode=single overflow=reject; } ``` For now, the connection pool uses a simple LIFO algorithm to assign idle connections in the pool. That is, most recently (successfully) used connections will be reused first the next time. And new idle connections will always replace the oldest idle connections in the pool even if the pool is already full. See the [drizzle_keepalive](#drizzle_keepalive) directive for more details. [Back to TOC](#table-of-contents) Last Insert ID -------------- If you want to get LAST_INSERT_ID, then ngx_drizzle already returns that automatically for you when you're doing a SQL insert query. Consider the following sample `nginx.conf` snippet: ```nginx location /test { echo_location /mysql "drop table if exists foo"; echo; echo_location /mysql "create table foo (id serial not null, primary key (id), val real);"; echo; echo_location /mysql "insert into foo (val) values (3.1415926);"; echo; echo_location /mysql "select * from foo;"; echo; } location /mysql { drizzle_pass backend; drizzle_module_header off; drizzle_query $query_string; rds_json on; } ``` Then request `GET /test` gives the following outputs: ```javascript {"errcode":0} {"errcode":0} {"errcode":0,"insert_id":1,"affected_rows":1} [{"id":1,"val":3.1415926}] ``` You can see the `insert_id` field (as well as the `affected_rows` field in the 3rd JSON response. [Back to TOC](#table-of-contents) Directives ========== [Back to TOC](#table-of-contents) drizzle_server -------------- **syntax:** *drizzle_server <host> user=<user> password=<pass> dbname=<database>* **syntax:** *drizzle_server <host>:<port> user=<user> password=<pass> dbname=<database> protocol=<protocol> charset=<charset>* **default:** *no* **context:** *upstream* Directive assigns the name and the parameters of server. For the name it is possible to use a domain name, an address, with an optional port (default: 3306). If domain name resolves to several addresses, then all are used. The following options are supported: **user=**`` MySQL/Drizzle user name `` for login. **password=**`` Specify mysql password ``for login. If you have special characters like `#` or spaces in your password text, then you'll have to quote the whole key-value pair with either single-quotes or double-quotes, as in ```nginx drizzle_server 127.0.0.1:3306 user=monty "password=a b#1" dbname=test protocol=mysql; ``` **dbname=**`` Specify default MySQL database `` for the connection. Note that MySQL does allow referencing tables belonging to different databases by qualifying table names with database names in SQL queries. **protocol=**`` Specify which wire protocol to use, `drizzle` or `mysql`. Default to `drizzle`. **charset=**`` Explicitly specify the character set for the MySQL connections. Setting this option to a non-empty value will make this module send out a `set names ''` query right after the mysql connection is established. If the default character encoding of the MySQL connection is already what you want, you needn't set this option because it has extra runtime cost. Here is a small example: ```nginx drizzle_server foo.bar.com:3306 user=monty password=some_pass dbname=test protocol=mysql charset=utf8; ``` Please note that for the mysql server, "utf-8" is not a valid encoding name while `utf8` is. [Back to TOC](#table-of-contents) drizzle_keepalive ----------------- **syntax:** *drizzle_keepalive max=<size> mode=<mode>* **default:** *drizzle_keepalive max=0 mode=single* **context:** *upstream* Configures the keep-alive connection pool for MySQL/Drizzle connections. The following options are supported: **max=**`` Specify the capacity of the connection pool for the current upstream block. The value *must* be non-zero. If set to `0`, it effectively disables the connection pool. This option is default to `0`. **mode=**`` This supports two values, `single` and `multi`. The `single` mode means the pool does not distinguish various drizzle servers in the current upstream block while `multi` means the pool will merely reuse connections which have identical server host names and ports. Note that even under `multi`, differences between `dbname` or `user` parameters will be silently ignored. Default to `single`. **overflow=**`` This option specifies what to do when the connection pool is already full while new database connection is required. Either `reject` or `ignore` can be specified. In case of `reject`, it will reject the current request, and returns the `503 Service Unavailable` error page. For `ignore`, this module will go on creating a new database connection. [Back to TOC](#table-of-contents) drizzle_query ------------- **syntax:** *drizzle_query <sql>* **default:** *no* **context:** *http, server, location, location if* Specify the SQL queries sent to the Drizzle/MySQL backend. Nginx variable interpolation is supported, but you must be careful with SQL injection attacks. You can use the [set_quote_sql_str](http://github.com/openresty/set-misc-nginx-module#set_quote_sql_str) directive, for example, to quote values for SQL interpolation: ```nginx location /cat { set_unescape_uri $name $arg_name; set_quote_sql_str $quoted_name $name; drizzle_query "select * from cats where name = $quoted_name"; drizzle_pass my_backend; } ``` [Back to TOC](#table-of-contents) drizzle_pass ------------ **syntax:** *drizzle_pass <remote>* **default:** *no* **context:** *location, location if* **phase:** *content* This directive specifies the Drizzle or MySQL upstream name to be queried in the current location. The `` argument can be any upstream name defined with the [drizzle_server](#drizzle_server) directive. Nginx variables can also be interpolated into the `` argument, so as to do dynamic backend routing, for example: ```nginx upstream moon { drizzle_server ...; } server { location /cat { set $backend 'moon'; drizzle_query ...; drizzle_pass $backend; } } ``` [Back to TOC](#table-of-contents) drizzle_connect_timeout ----------------------- **syntax:** *drizzle_connect_time <time>* **default:** *drizzle_connect_time 60s* **context:** *http, server, location, location if* Specify the (total) timeout for connecting to a remote Drizzle or MySQL server. The `