[ { "path": ".dockerignore", "content": "# Byte-compiled / optimized / DLL files\n**/__pycache__/\n**/*.py[cod]\n\n# C extensions\n***/*.so\n\n# Distribution / packaging\n.Python\nenv/\nbuild/\ndevelop-eggs/\ndist/\ndownloads/\neggs/\n.eggs/\nlib/\nlib64/\nparts/\nsdist/\nvar/\n*.egg-info/\n.installed.cfg\n*.egg\n\n\n# Installer logs\npip-log.txt\npip-delete-this-directory.txt\n\n# Unit test / coverage reports\nhtmlcov/\n.tox/\n.coverage\n.coverage.*\n.cache\nnosetests.xml\ncoverage.xml\ncover\n\n# Translations\n**/*.mo\n**/*.pot\n\n# PyBuilder\ntarget/\n\nvenv/\nvenv3/\n.idea/\n\n# 100% unnecessary for docker image\n.*\n*.md\ndocs\nrpm\nDockerfile\n" }, { "path": ".editorconfig", "content": "root = true\n\n[*]\nend_of_file = lf\ninsert_final_newline = true\n\n[*.{py,j2}]\ncharset = utf-8\n\n[*.py]\nindent_style = space\nindent_size = 4\n\n[Makefile]\nindent_style = tab\n\n[.travis.yml]\nindent_style = space\nindent_size = 2\n" }, { "path": ".gitignore", "content": "# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n\n# C extensions\n*.so\n\n# Distribution / packaging\n.Python\nenv/\nbuild/\ndevelop-eggs/\ndist/\ndownloads/\neggs/\n.eggs/\nlib/\nlib64/\nparts/\nsdist/\nvar/\n*.egg-info/\n.installed.cfg\n*.egg\n\n# PyInstaller\n# Usually these files are written by a python script from a template\n# before PyInstaller builds the exe, so as to inject date/other infos into it.\n*.manifest\n*.spec\n!rpm/*.spec\n\n# Installer logs\npip-log.txt\npip-delete-this-directory.txt\n\n# Unit test / coverage reports\nhtmlcov/\n.tox/\n.coverage\n.coverage.*\n.cache\nnosetests.xml\ncoverage.xml\n*,cover\ncover\n\n# Translations\n*.mo\n*.pot\n\n# Django stuff:\n*.log\n\n# Sphinx documentation\ndocs/_build/\n\n# PyBuilder\ntarget/\n\nvenv/\nvenv3/\n.idea/\n" }, { "path": ".travis.yml", "content": "language: python\ndist: xenial\nsudo: false\npython:\n - \"2.7\"\n - \"3.5\"\n - \"3.6\"\n - \"3.7\"\n - \"pypy\"\n - \"pypy3\"\n\ninstall:\n - pip install -r requirements.txt\n - pip install -r requirements.dev.txt\n\nscript:\n - nosetests --with-coverage --cover-package gixy -v\n - if [[ $TRAVIS_PYTHON_VERSION != '2.6' ]]; then flake8 --max-line-length=120 setup.py gixy; fi\n" }, { "path": "AUTHORS", "content": "The following authors have created the source code of \"Gixy\"\npublished and distributed by YANDEX LLC as the owner:\n\nAndrew Krasichkov " }, { "path": "CONTRIBUTING.md", "content": "# Notice to external contributors\n\n\n\n## General info\n\nHello! In order for us (YANDEX LLC) to accept patches and other contributions from you, you will have to adopt our Yandex Contributor License Agreement (the “**CLA**”). The current version of the CLA you may find here:\n1) https://yandex.ru/legal/cla/?lang=en (in English) and \n2) https://yandex.ru/legal/cla/?lang=ru (in Russian).\n\nBy adopting the CLA, you state the following:\n\n* You obviously wish and are willingly licensing your contributions to us for our open source projects under the terms of the CLA,\n* You has read the terms and conditions of the CLA and agree with them in full,\n* You are legally able to provide and license your contributions as stated,\n* We may use your contributions for our open source projects and for any other our project too,\n* We rely on your assurances concerning the rights of third parties in relation to your contributes.\n\nIf you agree with these principles, please read and adopt our CLA. By providing us your contributions, you hereby declare that you has already read and adopt our CLA, and we may freely merge your contributions with our corresponding open source project and use it in further in accordance with terms and conditions of the CLA.\n\n## Provide contributions \n\nIf you have already adopted terms and conditions of the CLA, you are able to provide your contributes. When you submit your pull request, please add the following information into it:\n\n`\nI hereby agree to the terms of the CLA available at: [link]).\n`\n\nReplace the bracketed text as follows:\n* [link] is the link at the current version of the CLA (you may add here a link https://yandex.ru/legal/cla/?lang=en (in English) or a link https://yandex.ru/legal/cla/?lang=ru (in Russian).\n\nIt is enough to provide us such notification at once. \n\n## Other questions\n\nIf you have any questions, please mail us at opensource@yandex-team.ru.\n\n\n" }, { "path": "Dockerfile", "content": "FROM python:alpine\n\nADD . /src\n\nWORKDIR /src\n\nRUN python3 setup.py install\n\nENTRYPOINT [\"gixy\"]\n" }, { "path": "LICENSE", "content": "(C) YANDEX LLC, 2017\n\nMozilla Public License Version 2.0\n==================================\n\n1. Definitions\n--------------\n\n1.1. \"Contributor\"\n means each individual or legal entity that creates, contributes to\n the creation of, or owns Covered Software.\n\n1.2. \"Contributor Version\"\n means the combination of the Contributions of others (if any) used\n by a Contributor and that particular Contributor's Contribution.\n\n1.3. \"Contribution\"\n means Covered Software of a particular Contributor.\n\n1.4. \"Covered Software\"\n means Source Code Form to which the initial Contributor has attached\n the notice in Exhibit A, the Executable Form of such Source Code\n Form, and Modifications of such Source Code Form, in each case\n including portions thereof.\n\n1.5. \"Incompatible With Secondary Licenses\"\n means\n\n (a) that the initial Contributor has attached the notice described\n in Exhibit B to the Covered Software; or\n\n (b) that the Covered Software was made available under the terms of\n version 1.1 or earlier of the License, but not also under the\n terms of a Secondary License.\n\n1.6. \"Executable Form\"\n means any form of the work other than Source Code Form.\n\n1.7. \"Larger Work\"\n means a work that combines Covered Software with other material, in\n a separate file or files, that is not Covered Software.\n\n1.8. \"License\"\n means this document.\n\n1.9. \"Licensable\"\n means having the right to grant, to the maximum extent possible,\n whether at the time of the initial grant or subsequently, any and\n all of the rights conveyed by this License.\n\n1.10. \"Modifications\"\n means any of the following:\n\n (a) any file in Source Code Form that results from an addition to,\n deletion from, or modification of the contents of Covered\n Software; or\n\n (b) any new file in Source Code Form that contains any Covered\n Software.\n\n1.11. \"Patent Claims\" of a Contributor\n means any patent claim(s), including without limitation, method,\n process, and apparatus claims, in any patent Licensable by such\n Contributor that would be infringed, but for the grant of the\n License, by the making, using, selling, offering for sale, having\n made, import, or transfer of either its Contributions or its\n Contributor Version.\n\n1.12. \"Secondary License\"\n means either the GNU General Public License, Version 2.0, the GNU\n Lesser General Public License, Version 2.1, the GNU Affero General\n Public License, Version 3.0, or any later versions of those\n licenses.\n\n1.13. \"Source Code Form\"\n means the form of the work preferred for making modifications.\n\n1.14. \"You\" (or \"Your\")\n means an individual or a legal entity exercising rights under this\n License. For legal entities, \"You\" includes any entity that\n controls, is controlled by, or is under common control with You. For\n purposes of this definition, \"control\" means (a) the power, direct\n or indirect, to cause the direction or management of such entity,\n whether by contract or otherwise, or (b) ownership of more than\n fifty percent (50%) of the outstanding shares or beneficial\n ownership of such entity.\n\n2. License Grants and Conditions\n--------------------------------\n\n2.1. Grants\n\nEach Contributor hereby grants You a world-wide, royalty-free,\nnon-exclusive license:\n\n(a) under intellectual property rights (other than patent or trademark)\n Licensable by such Contributor to use, reproduce, make available,\n modify, display, perform, distribute, and otherwise exploit its\n Contributions, either on an unmodified basis, with Modifications, or\n as part of a Larger Work; and\n\n(b) under Patent Claims of such Contributor to make, use, sell, offer\n for sale, have made, import, and otherwise transfer either its\n Contributions or its Contributor Version.\n\n2.2. Effective Date\n\nThe licenses granted in Section 2.1 with respect to any Contribution\nbecome effective for each Contribution on the date the Contributor first\ndistributes such Contribution.\n\n2.3. Limitations on Grant Scope\n\nThe licenses granted in this Section 2 are the only rights granted under\nthis License. No additional rights or licenses will be implied from the\ndistribution or licensing of Covered Software under this License.\nNotwithstanding Section 2.1(b) above, no patent license is granted by a\nContributor:\n\n(a) for any code that a Contributor has removed from Covered Software;\n or\n\n(b) for infringements caused by: (i) Your and any other third party's\n modifications of Covered Software, or (ii) the combination of its\n Contributions with other software (except as part of its Contributor\n Version); or\n\n(c) under Patent Claims infringed by Covered Software in the absence of\n its Contributions.\n\nThis License does not grant any rights in the trademarks, service marks,\nor logos of any Contributor (except as may be necessary to comply with\nthe notice requirements in Section 3.4).\n\n2.4. Subsequent Licenses\n\nNo Contributor makes additional grants as a result of Your choice to\ndistribute the Covered Software under a subsequent version of this\nLicense (see Section 10.2) or under the terms of a Secondary License (if\npermitted under the terms of Section 3.3).\n\n2.5. Representation\n\nEach Contributor represents that the Contributor believes its\nContributions are its original creation(s) or it has sufficient rights\nto grant the rights to its Contributions conveyed by this License.\n\n2.6. Fair Use\n\nThis License is not intended to limit any rights You have under\napplicable copyright doctrines of fair use, fair dealing, or other\nequivalents.\n\n2.7. Conditions\n\nSections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted\nin Section 2.1.\n\n3. Responsibilities\n-------------------\n\n3.1. Distribution of Source Form\n\nAll distribution of Covered Software in Source Code Form, including any\nModifications that You create or to which You contribute, must be under\nthe terms of this License. You must inform recipients that the Source\nCode Form of the Covered Software is governed by the terms of this\nLicense, and how they can obtain a copy of this License. You may not\nattempt to alter or restrict the recipients' rights in the Source Code\nForm.\n\n3.2. Distribution of Executable Form\n\nIf You distribute Covered Software in Executable Form then:\n\n(a) such Covered Software must also be made available in Source Code\n Form, as described in Section 3.1, and You must inform recipients of\n the Executable Form how they can obtain a copy of such Source Code\n Form by reasonable means in a timely manner, at a charge no more\n than the cost of distribution to the recipient; and\n\n(b) You may distribute such Executable Form under the terms of this\n License, or sublicense it under different terms, provided that the\n license for the Executable Form does not attempt to limit or alter\n the recipients' rights in the Source Code Form under this License.\n\n3.3. Distribution of a Larger Work\n\nYou may create and distribute a Larger Work under terms of Your choice,\nprovided that You also comply with the requirements of this License for\nthe Covered Software. If the Larger Work is a combination of Covered\nSoftware with a work governed by one or more Secondary Licenses, and the\nCovered Software is not Incompatible With Secondary Licenses, this\nLicense permits You to additionally distribute such Covered Software\nunder the terms of such Secondary License(s), so that the recipient of\nthe Larger Work may, at their option, further distribute the Covered\nSoftware under the terms of either this License or such Secondary\nLicense(s).\n\n3.4. Notices\n\nYou may not remove or alter the substance of any license notices\n(including copyright notices, patent notices, disclaimers of warranty,\nor limitations of liability) contained within the Source Code Form of\nthe Covered Software, except that You may alter any license notices to\nthe extent required to remedy known factual inaccuracies.\n\n3.5. Application of Additional Terms\n\nYou may choose to offer, and to charge a fee for, warranty, support,\nindemnity or liability obligations to one or more recipients of Covered\nSoftware. However, You may do so only on Your own behalf, and not on\nbehalf of any Contributor. You must make it absolutely clear that any\nsuch warranty, support, indemnity, or liability obligation is offered by\nYou alone, and You hereby agree to indemnify every Contributor for any\nliability incurred by such Contributor as a result of warranty, support,\nindemnity or liability terms You offer. You may include additional\ndisclaimers of warranty and limitations of liability specific to any\njurisdiction.\n\n4. Inability to Comply Due to Statute or Regulation\n---------------------------------------------------\n\nIf it is impossible for You to comply with any of the terms of this\nLicense with respect to some or all of the Covered Software due to\nstatute, judicial order, or regulation then You must: (a) comply with\nthe terms of this License to the maximum extent possible; and (b)\ndescribe the limitations and the code they affect. Such description must\nbe placed in a text file included with all distributions of the Covered\nSoftware under this License. Except to the extent prohibited by statute\nor regulation, such description must be sufficiently detailed for a\nrecipient of ordinary skill to be able to understand it.\n\n5. Termination\n--------------\n\n5.1. The rights granted under this License will terminate automatically\nif You fail to comply with any of its terms. However, if You become\ncompliant, then the rights granted under this License from a particular\nContributor are reinstated (a) provisionally, unless and until such\nContributor explicitly and finally terminates Your grants, and (b) on an\nongoing basis, if such Contributor fails to notify You of the\nnon-compliance by some reasonable means prior to 60 days after You have\ncome back into compliance. Moreover, Your grants from a particular\nContributor are reinstated on an ongoing basis if such Contributor\nnotifies You of the non-compliance by some reasonable means, this is the\nfirst time You have received notice of non-compliance with this License\nfrom such Contributor, and You become compliant prior to 30 days after\nYour receipt of the notice.\n\n5.2. If You initiate litigation against any entity by asserting a patent\ninfringement claim (excluding declaratory judgment actions,\ncounter-claims, and cross-claims) alleging that a Contributor Version\ndirectly or indirectly infringes any patent, then the rights granted to\nYou by any and all Contributors for the Covered Software under Section\n2.1 of this License shall terminate.\n\n5.3. In the event of termination under Sections 5.1 or 5.2 above, all\nend user license agreements (excluding distributors and resellers) which\nhave been validly granted by You or Your distributors under this License\nprior to termination shall survive termination.\n\n************************************************************************\n* *\n* 6. Disclaimer of Warranty *\n* ------------------------- *\n* *\n* Covered Software is provided under this License on an \"as is\" *\n* basis, without warranty of any kind, either expressed, implied, or *\n* statutory, including, without limitation, warranties that the *\n* Covered Software is free of defects, merchantable, fit for a *\n* particular purpose or non-infringing. The entire risk as to the *\n* quality and performance of the Covered Software is with You. *\n* Should any Covered Software prove defective in any respect, You *\n* (not any Contributor) assume the cost of any necessary servicing, *\n* repair, or correction. This disclaimer of warranty constitutes an *\n* essential part of this License. No use of any Covered Software is *\n* authorized under this License except under this disclaimer. *\n* *\n************************************************************************\n\n************************************************************************\n* *\n* 7. Limitation of Liability *\n* -------------------------- *\n* *\n* Under no circumstances and under no legal theory, whether tort *\n* (including negligence), contract, or otherwise, shall any *\n* Contributor, or anyone who distributes Covered Software as *\n* permitted above, be liable to You for any direct, indirect, *\n* special, incidental, or consequential damages of any character *\n* including, without limitation, damages for lost profits, loss of *\n* goodwill, work stoppage, computer failure or malfunction, or any *\n* and all other commercial damages or losses, even if such party *\n* shall have been informed of the possibility of such damages. This *\n* limitation of liability shall not apply to liability for death or *\n* personal injury resulting from such party's negligence to the *\n* extent applicable law prohibits such limitation. Some *\n* jurisdictions do not allow the exclusion or limitation of *\n* incidental or consequential damages, so this exclusion and *\n* limitation may not apply to You. *\n* *\n************************************************************************\n\n8. Litigation\n-------------\n\nAny litigation relating to this License may be brought only in the\ncourts of a jurisdiction where the defendant maintains its principal\nplace of business and such litigation shall be governed by laws of that\njurisdiction, without reference to its conflict-of-law provisions.\nNothing in this Section shall prevent a party's ability to bring\ncross-claims or counter-claims.\n\n9. Miscellaneous\n----------------\n\nThis License represents the complete agreement concerning the subject\nmatter hereof. If any provision of this License is held to be\nunenforceable, such provision shall be reformed only to the extent\nnecessary to make it enforceable. Any law or regulation which provides\nthat the language of a contract shall be construed against the drafter\nshall not be used to construe this License against a Contributor.\n\n10. Versions of the License\n---------------------------\n\n10.1. New Versions\n\nMozilla Foundation is the license steward. Except as provided in Section\n10.3, no one other than the license steward has the right to modify or\npublish new versions of this License. Each version will be given a\ndistinguishing version number.\n\n10.2. Effect of New Versions\n\nYou may distribute the Covered Software under the terms of the version\nof the License under which You originally received the Covered Software,\nor under the terms of any subsequent version published by the license\nsteward.\n\n10.3. Modified Versions\n\nIf you create software not governed by this License, and you want to\ncreate a new license for such software, you may create and use a\nmodified version of this License if you rename the license and remove\nany references to the name of the license steward (except to note that\nsuch modified license differs from this License).\n\n10.4. Distributing Source Code Form that is Incompatible With Secondary\nLicenses\n\nIf You choose to distribute Source Code Form that is Incompatible With\nSecondary Licenses under the terms of this version of the License, the\nnotice described in Exhibit B of this License must be attached." }, { "path": "MANIFEST.in", "content": "include gixy/formatters/templates/*\ngraft tests\n" }, { "path": "Makefile", "content": ".PHONY: all build publish\n\nall: build publish\n\nbuild:\n\tpython setup.py bdist_wheel --universal sdist\n\npublish:\n\ttwine upload dist/gixy-`grep -oP \"(?<=version\\s=\\s['\\\"])[^'\\\"]*(?=['\\\"])\" gixy/__init__.py`*\n\n" }, { "path": "README.RU.md", "content": "GIXY\n====\n[![Mozilla Public License 2.0](https://img.shields.io/github/license/yandex/gixy.svg?style=flat-square)](https://github.com/yandex/gixy/blob/master/LICENSE)\n[![Build Status](https://img.shields.io/travis/yandex/gixy.svg?style=flat-square)](https://travis-ci.org/yandex/gixy)\n[![Your feedback is greatly appreciated](https://img.shields.io/maintenance/yes/2018.svg?style=flat-square)](https://github.com/yandex/gixy/issues/new)\n[![GitHub issues](https://img.shields.io/github/issues/yandex/gixy.svg?style=flat-square)](https://github.com/yandex/gixy/issues)\n[![GitHub pull requests](https://img.shields.io/github/issues-pr/yandex/gixy.svg?style=flat-square)](https://github.com/yandex/gixy/pulls)\n\n# Overview\n\n\nGixy — это утилита для анализа конфигурации Nginx.\nБольшей частью служит для обнаружения проблем безопасности, но может искать и иные ошибки.\n\nОфициально поддерживаются версии Python 2.7, 3.5, 3.6 и 3.7\n\n \n# Что умеет\nНа текущий момент Gixy способна обнаружить:\n * [[ssrf] Server Side Request Forgery](https://github.com/yandex/gixy/blob/master/docs/ru/plugins/ssrf.md)\n * [[http_splitting] HTTP Splitting](https://github.com/yandex/gixy/blob/master/docs/ru/plugins/httpsplitting.md)\n * [[origins] Проблемы валидации referrer/origin](https://github.com/yandex/gixy/blob/master/docs/ru/plugins/origins.md)\n * [[add_header_redefinition] Переопределение \"вышестоящих\" заголовков ответа директивой \"add_header\"](https://github.com/yandex/gixy/blob/master/docs/ru/plugins/addheaderredefinition.md)\n * [[host_spoofing] Подделка заголовка запроса Host](https://github.com/yandex/gixy/blob/master/docs/ru/plugins/hostspoofing.md)\n * [[valid_referers] none in valid_referers](https://github.com/yandex/gixy/blob/master/docs/ru/plugins/validreferers.md)\n * [[add_header_multiline] Многострочные заголовоки ответа](https://github.com/yandex/gixy/blob/master/docs/ru/plugins/addheadermultiline.md)\n * [[alias_traversal] Path traversal при использовании alias](https://github.com/yandex/gixy/blob/master/docs/ru/plugins/aliastraversal.md)\n\nПроблемы, которым Gixy только учится можно найти в [Issues с меткой \"new plugin\"](https://github.com/yandex/gixy/issues?q=is%3Aissue+is%3Aopen+label%3A%22new+plugin%22)\n\n# Установка\nНаиболее простой способ установки Gixy - воспользоваться pip для установки из [PyPI](https://pypi.python.org/pypi/gixy):\n```bash\npip install gixy\n```\n\n# Использование\nПосле установки должна стать доступна консольная утилита `gixy`.\nПо умолчанию Gixy ищет конфигурацию по стандартному пути `/etc/nginx/nginx.conf`, однако вы можете указать специфичное расположение:\n```\n$ gixy /etc/nginx/nginx.conf\n\n==================== Results ===================\n\nProblem: [http_splitting] Possible HTTP-Splitting vulnerability.\nDescription: Using variables that can contain \"\\n\" may lead to http injection.\nAdditional info: https://github.com/yandex/gixy/wiki/ru/httpsplitting\nReason: At least variable \"$action\" can contain \"\\n\"\nPseudo config:\ninclude /etc/nginx/sites/default.conf;\n\n\tserver {\n\n\t\tlocation ~ /v1/((?[^.]*)\\.json)?$ {\n\t\t\tadd_header X-Action $action;\n\t\t}\n\t}\n\n\n==================== Summary ===================\nTotal issues:\n Unspecified: 0\n Low: 0\n Medium: 0\n High: 1\n```\n\nGixy умеет обрабатывать директиву `include` и попробует максимально корректно обработать все зависимости, если что-то пошло не так можно попробовать запустить `gixy` с флагом `-d` для вывода дополнительной информации.\nВсе доступные опции:\n```\n$ gixy -h\nusage: gixy [-h] [-c CONFIG_FILE] [--write-config CONFIG_OUTPUT_PATH]\n [-v] [-l] [-f {console,text,json}] [-o OUTPUT_FILE] [-d]\n [--tests TESTS] [--skips SKIPS] [--disable-includes]\n [--origins-domains domains]\n [--origins-https-only https_only]\n [--add-header-redefinition-headers headers]\n [nginx.conf]\n\nGixy - a Nginx configuration [sec]analyzer\n\npositional arguments:\n nginx.conf Path to nginx.conf, e.g. /etc/nginx/nginx.conf\n\noptional arguments:\n -h, --help show this help message and exit\n -c CONFIG_FILE, --config CONFIG_FILE\n config file path\n --write-config CONFIG_OUTPUT_PATH\n takes the current command line args and writes them\n out to a config file at the given path, then exits\n -v, --version show program's version number and exit\n -l, --level Report issues of a given severity level or higher (-l\n for LOW, -ll for MEDIUM, -lll for HIGH)\n -f {console,text,json}, --format {console,text,json}\n Specify output format\n -o OUTPUT_FILE, --output OUTPUT_FILE\n Write report to file\n -d, --debug Turn on debug mode\n --tests TESTS Comma-separated list of tests to run\n --skips SKIPS Comma-separated list of tests to skip\n --disable-includes Disable \"include\" directive processing\n\nplugins options:\n --origins-domains domains\n Default: *\n --origins-https-only https_only\n Default: False\n --add-header-redefinition-headers headers\n Default: content-security-policy,x-xss-\n protection,x-frame-options,x-content-type-\n options,strict-transport-security,cache-control\n\n\navailable plugins:\n\thost_spoofing\n\tadd_header_multiline\n\thttp_splitting\n\tvalid_referers\n\torigins\n\tadd_header_redefinition\n\tssrf\n```\n\n# Contributing\nContributions to Gixy are always welcome! You can help us in different ways:\n * Open an issue with suggestions for improvements and errors you're facing;\n * Fork this repository and submit a pull request;\n * Improve the documentation.\n\nCode guidelines:\n * Python code style should follow [pep8](https://www.python.org/dev/peps/pep-0008/) standards whenever possible;\n * Pull requests with new plugins must have unit tests for it.\n" }, { "path": "README.md", "content": "GIXY\n====\n[![Mozilla Public License 2.0](https://img.shields.io/github/license/yandex/gixy.svg?style=flat-square)](https://github.com/yandex/gixy/blob/master/LICENSE)\n[![Build Status](https://img.shields.io/travis/yandex/gixy.svg?style=flat-square)](https://travis-ci.org/yandex/gixy)\n[![Your feedback is greatly appreciated](https://img.shields.io/maintenance/yes/2019.svg?style=flat-square)](https://github.com/yandex/gixy/issues/new)\n[![GitHub issues](https://img.shields.io/github/issues/yandex/gixy.svg?style=flat-square)](https://github.com/yandex/gixy/issues)\n[![GitHub pull requests](https://img.shields.io/github/issues-pr/yandex/gixy.svg?style=flat-square)](https://github.com/yandex/gixy/pulls)\n\n# Overview\n\n\nGixy is a tool to analyze Nginx configuration.\nThe main goal of Gixy is to prevent security misconfiguration and automate flaw detection.\n\nCurrently supported Python versions are 2.7, 3.5, 3.6 and 3.7.\n\nDisclaimer: Gixy is well tested only on GNU/Linux, other OSs may have some issues.\n\n# What it can do\nRight now Gixy can find:\n * [[ssrf] Server Side Request Forgery](https://github.com/yandex/gixy/blob/master/docs/en/plugins/ssrf.md)\n * [[http_splitting] HTTP Splitting](https://github.com/yandex/gixy/blob/master/docs/en/plugins/httpsplitting.md)\n * [[origins] Problems with referrer/origin validation](https://github.com/yandex/gixy/blob/master/docs/en/plugins/origins.md)\n * [[add_header_redefinition] Redefining of response headers by \"add_header\" directive](https://github.com/yandex/gixy/blob/master/docs/en/plugins/addheaderredefinition.md)\n * [[host_spoofing] Request's Host header forgery](https://github.com/yandex/gixy/blob/master/docs/en/plugins/hostspoofing.md)\n * [[valid_referers] none in valid_referers](https://github.com/yandex/gixy/blob/master/docs/en/plugins/validreferers.md)\n * [[add_header_multiline] Multiline response headers](https://github.com/yandex/gixy/blob/master/docs/en/plugins/addheadermultiline.md)\n * [[alias_traversal] Path traversal via misconfigured alias](https://github.com/yandex/gixy/blob/master/docs/en/plugins/aliastraversal.md)\n\nYou can find things that Gixy is learning to detect at [Issues labeled with \"new plugin\"](https://github.com/yandex/gixy/issues?q=is%3Aissue+is%3Aopen+label%3A%22new+plugin%22)\n\n# Installation\nGixy is distributed on [PyPI](https://pypi.python.org/pypi/gixy). The best way to install it is with pip:\n```bash\npip install gixy\n```\n\nRun Gixy and check results:\n```bash\ngixy\n```\n\n# Usage\nBy default Gixy will try to analyze Nginx configuration placed in `/etc/nginx/nginx.conf`.\n\nBut you can always specify needed path:\n```\n$ gixy /etc/nginx/nginx.conf\n\n==================== Results ===================\n\nProblem: [http_splitting] Possible HTTP-Splitting vulnerability.\nDescription: Using variables that can contain \"\\n\" may lead to http injection.\nAdditional info: https://github.com/yandex/gixy/blob/master/docs/ru/plugins/httpsplitting.md\nReason: At least variable \"$action\" can contain \"\\n\"\nPseudo config:\ninclude /etc/nginx/sites/default.conf;\n\n\tserver {\n\n\t\tlocation ~ /v1/((?[^.]*)\\.json)?$ {\n\t\t\tadd_header X-Action $action;\n\t\t}\n\t}\n\n\n==================== Summary ===================\nTotal issues:\n Unspecified: 0\n Low: 0\n Medium: 0\n High: 1\n```\n\nOr skip some tests:\n```\n$ gixy --skips http_splitting /etc/nginx/nginx.conf\n\n==================== Results ===================\nNo issues found.\n\n==================== Summary ===================\nTotal issues:\n Unspecified: 0\n Low: 0\n Medium: 0\n High: 0\n```\n\nOr something else, you can find all other `gixy` arguments with the help command: `gixy --help`\n\n## Docker usage\n\nGixy is available as a Docker image [from the Docker hub](https://hub.docker.com/r/yandex/gixy/). To\nuse it, mount the configuration that you want to analyse as a volume and provide the path to the\nconfiguration file when running the Gixy image.\n```\n$ docker run --rm -v `pwd`/nginx.conf:/etc/nginx/conf/nginx.conf yandex/gixy /etc/nginx/conf/nginx.conf\n```\n\nIf you have an image that already contains your nginx configuration, you can share the configuration\nwith the Gixy container as a volume.\n```\n$ docker run --rm --name nginx -d -v /etc/nginx\nnginx:alpinef68f2833e986ae69c0a5375f9980dc7a70684a6c233a9535c2a837189f14e905\n\n$ docker run --rm --volumes-from nginx yandex/gixy /etc/nginx/nginx.conf\n\n==================== Results ===================\nNo issues found.\n\n==================== Summary ===================\nTotal issues:\n Unspecified: 0\n Low: 0\n Medium: 0\n High: 0\n\n```\n\n# Contributing\nContributions to Gixy are always welcome! You can help us in different ways:\n * Open an issue with suggestions for improvements and errors you're facing;\n * Fork this repository and submit a pull request;\n * Improve the documentation.\n\nCode guidelines:\n * Python code style should follow [pep8](https://www.python.org/dev/peps/pep-0008/) standards whenever possible;\n * Pull requests with new plugins must have unit tests for it.\n" }, { "path": "docs/en/plugins/addheadermultiline.md", "content": "# [add_header_multiline] Multiline response headers\n\nYou should avoid using multiline response headers, because:\n * they are deprecated (see [RFC 7230](https://tools.ietf.org/html/rfc7230#section-3.2.4));\n * some HTTP-clients and web browser never supported them (e.g. IE/Edge/Nginx).\n\n## How can I find it?\nMisconfiguration example:\n```nginx\n# http://nginx.org/en/docs/http/ngx_http_headers_module.html#add_header\nadd_header Content-Security-Policy \"\n default-src: 'none';\n script-src data: https://yastatic.net;\n style-src data: https://yastatic.net;\n img-src data: https://yastatic.net;\n font-src data: https://yastatic.net;\";\n\n# https://www.nginx.com/resources/wiki/modules/headers_more/\nmore_set_headers -t 'text/html text/plain'\n 'X-Foo: Bar\n multiline';\n```\n\n## What can I do?\nThe only solution is to never use multiline response headers.\n" }, { "path": "docs/en/plugins/addheaderredefinition.md", "content": "# [add_header_redefinition] Redefining of response headers by \"add_header\" directive\n\nUnfortunately, many people don't know how the inheritance of directives works. Most often this leads to misuse of the `add_header` directive while trying to add a new response header on the nested level.\nThis feature is mentioned in Nginx [docs](http://nginx.org/en/docs/http/ngx_http_headers_module.html#add_header):\n> There could be several `add_header` directives. These directives are inherited from the previous level if and only if there are no `add_header` directives defined on the current level.\n\nThe logic is quite simple: if you set headers at one level (for example, in `server` section) and then at a lower level (let's say `location`) you set some other headers, then the first headers will discarded.\n\nIt's easy to check:\n - Configuration:\n```nginx\nserver {\n listen 80;\n add_header X-Frame-Options \"DENY\" always;\n location / {\n return 200 \"index\";\n }\n\n location /new-headers {\n # Add special cache control\n add_header Cache-Control \"no-cache, no-store, max-age=0, must-revalidate\" always;\n add_header Pragma \"no-cache\" always;\n\n return 200 \"new-headers\";\n }\n}\n```\n - Request to location `/` (`X-Frame-Options` header is in server response):\n```http\nGET / HTTP/1.0\n\nHTTP/1.1 200 OK\nServer: nginx/1.10.2\nDate: Mon, 09 Jan 2017 19:28:33 GMT\nContent-Type: application/octet-stream\nContent-Length: 5\nConnection: close\nX-Frame-Options: DENY\n\nindex\n```\n - Request to location `/new-headers` (headers `Cache-Control` and `Pragma` are present, but there's no `X-Frame-Options`):\n```http\nGET /new-headers HTTP/1.0\n\n\nHTTP/1.1 200 OK\nServer: nginx/1.10.2\nDate: Mon, 09 Jan 2017 19:29:46 GMT\nContent-Type: application/octet-stream\nContent-Length: 11\nConnection: close\nCache-Control: no-cache, no-store, max-age=0, must-revalidate\nPragma: no-cache\n\nnew-headers\n```\n\n## What can I do?\nThere are several ways to solve this problem:\n - duplicate important headers;\n - set all headers at one level (`server` section is a good choice)\n - use [ngx_headers_more](https://www.nginx.com/resources/wiki/modules/headers_more/) module.\n" }, { "path": "docs/en/plugins/aliastraversal.md", "content": "# [alias_traversal] Path traversal via misconfigured alias\n\nThe [alias](https://nginx.ru/en/docs/http/ngx_http_core_module.html#alias) directive is used to replace path of the specified location.\nFor example, with the following configuration:\n```nginx\nlocation /i/ {\n alias /data/w3/images/;\n}\n```\non request of `/i/top.gif`, the file `/data/w3/images/top.gif` will be sent.\n\nBut, if the location doesn't ends with directory separator (i.e. `/`):\n```nginx\nlocation /i {\n alias /data/w3/images/;\n}\n```\non request of `/i../app/config.py`, the file `/data/w3/app/config.py` will be sent.\n\nIn other words, the incorrect configuration of `alias` could allow an attacker to read file stored outside the target folder.\n\n## What can I do?\nIt's pretty simple:\n - you must find all the `alias` directives;\n - make sure that the parent prefixed location ends with directory separator.\n - or if you want to map a single file make sure the location starts with a `=`, e.g `=/i.gif` instead of `/i.gif`.\n" }, { "path": "docs/en/plugins/hostspoofing.md", "content": "# [host_spoofing] Request's Host header forgery\n\nOften, an application located behind Nginx needs a correct `Host` header for URL generation (redirects, resources, links in emails etc.).\nSpoofing of this header, may leads to a variety of problems, from phishing to SSRF.\n\n> Notice: your application may also use the `X-Forwarded-Host` request header for this functionality.\n> In this case you have to ensure the header is set correctly;\n\n## How can I find it?\nMost of the time it's a result of using `$http_host` variable instead of `$host`.\n\nAnd they are quite different:\n * `$host` - host in this order of precedence: host name from the request line, or host name from the “Host” request header field, or the server name matching a request;\n * `$http_host` - \"Host\" request header.\n\nConfig sample:\n```nginx\nlocation @app {\n proxy_set_header Host $http_host;\n # Other proxy params\n proxy_pass http://backend;\n}\n```\n\n## What can I do?\nLuckily, all is quite obvious:\n * list all the correct server names in `server name` directive;\n * always use `$host` instead of `$http_host`.\n\n## Additional info\n * [Host of Troubles Vulnerabilities](https://hostoftroubles.com/)\n * [Practical HTTP Host header attacks](http://www.skeletonscribe.net/2013/05/practical-http-host-header-attacks.html)\n" }, { "path": "docs/en/plugins/httpsplitting.md", "content": "# [http_splitting] HTTP Splitting\n\nHTTP Splitting - attack that use improper input validation. It usually targets web application located behind Nginx (HTTP Request Splitting) or its users (HTTP Response Splitting).\n\nVulnerability is created when an attacker can insert newline character `\\n` or `\\r` into request or into response, created by Nginx.\n\n## How can I find it?\nYou should always pay attention to:\n - variables that are used in directives, responsible for the request creation (for they may contain CRLF), e.g. `rewrite`, `return`, `add_header`, `proxy_set_header` or `proxy_pass`;\n - `$uri` and `$document_uri` variables, and in which directives they are used, because these variables contain decoded URL-encoded value;\n - variables, that are selected from an exclusive range, e.g. `(?P[^.]+)`.\n\n\nAn example of configuration that contains variable, selected from an exclusive range:\n```nginx\nserver {\n listen 80 default;\n\n location ~ /v1/((?[^.]*)\\.json)?$ {\n add_header X-Action $action;\n return 200 \"OK\";\n }\n}\n```\n\nExploitation:\n```http\nGET /v1/see%20below%0d%0ax-crlf-header:injected.json HTTP/1.0\nHost: localhost\n\nHTTP/1.1 200 OK\nServer: nginx/1.11.10\nDate: Mon, 13 Mar 2017 21:21:29 GMT\nContent-Type: application/octet-stream\nContent-Length: 2\nConnection: close\nX-Action: see below\nx-crlf-header:injected\n\nOK\n```\n\nAs you can see, an attacker could add `x-crlf-header: injected` response header. This was possible because:\n - `add_header` doesn't encode or validate input value on suggestion that author knows about the consequences;\n - the path value is normalize before location processing;\n - `$action` value was given from a regexp with an exclusive range: `[^.]*`;\n - as the result, `$action` value is equal to `see below\\r\\nx-crlf-header:injected` and on its use the response header was added.\n\n## What can I do?\n - try to use safe variables, e.g. `$request_uri` instead of `$uri`;\n - forbid the use of the new line symbol in the exclusive range by using `/some/(?[^/\\s]+)` instead of `/some/(?[^/]+`\n - it could be a good idea to validate `$uri` (only if you're sure you know what are you getting into).\n" }, { "path": "docs/en/plugins/origins.md", "content": "# [origins] Problems with referrer/origin validation\n\nIt's not unusual to use regex for `Referer` or `Origin` headers validation.\nOften it is needed for setting the `X-Frame-Options` header (ClickJacking protection) or Cross-Origin Resource Sharing.\n\nThe most common errors with this configuration are:\n - regex errors;\n - allow 3rd-party origins.\n\n > Notice: by default Gixy doesn't check regexes for 3rd-party origins matching.\n > You can pass a list of trusted domains by using the option `--origins-domains example.com,foo.bar`\n\n## How can I find it?\n\"Eazy\"-breezy:\n - you have to find all the `if` directives that are in charge of `$http_origin` or `$http_referer` check;\n - make sure your regexes are a-ok.\n\nMisconfiguration example:\n```nginx\nif ($http_origin ~* ((^https://www\\.yandex\\.ru)|(^https://ya\\.ru)$)) {\n\tadd_header 'Access-Control-Allow-Origin' \"$http_origin\";\n\tadd_header 'Access-Control-Allow-Credentials' 'true';\n}\n```\n\nTODO(buglloc): cover typical regex-writing problems\nTODO(buglloc): Regex Ninja?\n\n## What can I do?\n\n - fix your regex or toss it away :)\n - if you use regex validation for `Referer` request header, then, possibly (not 100%), you could use [ngx_http_referer_module](http://nginx.org/en/docs/http/ngx_http_referer_module.htmll);\n - sometimes it is much better to use the `map` directive without any regex at all.\n" }, { "path": "docs/en/plugins/ssrf.md", "content": "# [ssrf] Server Side Request Forgery\n\nServer Side Request Forgery - attack that forces a server to perform arbitrary requests (from Nginx in our case).\nIt's possible when an attacker controls the address of a proxied server (second argument of the `proxy_pass` directive).\n\n\n## How can I find it?\nThere are two types of errors that make a server vulnerable:\n - lack of the [internal](http://nginx.org/en/docs/http/ngx_http_core_module.html#internal) directive. It is used to point out a location that can be used for internal requests only;\n - unsafe internal redirection.\n\n### Lack of the internal directive\nClassical misconfiguration, based on lack of the `internal` directive, that makes SSRF possible:\n```nginx\nlocation ~ /proxy/(.*)/(.*)/(.*)$ {\n proxy_pass $1://$2/$3;\n}\n```\n\nAn attacker has complete control over the proxied address, that makes sending requests on behalf of Nginx possible.\n\n### Unsafe internal redirection\nLet's say you have internal location in your config and that location uses some request data as proxied server's address.\n\nE.g.:\n```nginx\nlocation ~* ^/internal-proxy/(?https?)/(?.*?)/(?.*)$ {\n internal;\n\n proxy_pass $proxy_proto://$proxy_host/$proxy_path ;\n proxy_set_header Host $proxy_host;\n}\n```\n\nAccording to Nginx docs, internal requests are the following:\n> - requests redirected by the **error_page**, index, random_index, and **try_files** directives;\n> - requests redirected by the “X-Accel-Redirect” response header field from an upstream server;\n> - subrequests formed by the “include virtual” command of the ngx_http_ssi_module module and by the ngx_http_addition_module module directives;\n> - requests changed by the **rewrite** directive\n\nAccordingly, any unsafe rewrite allows an attacker to make an internal request and control a proxied server's address.\n\nMisconfiguration example:\n```nginx\nrewrite ^/(.*)/some$ /$1/ last;\n\nlocation ~* ^/internal-proxy/(?https?)/(?.*?)/(?.*)$ {\n internal;\n\n proxy_pass $proxy_proto://$proxy_host/$proxy_path ;\n proxy_set_header Host $proxy_host;\n}\n```\n\n## What can I do?\nThere are several rules you better follow when writing such configurations:\n - use only \"internal locations\" for proxying;\n - if possible, forbid user data transmission;\n - protect proxied server's address:\n * if the quantity of proxied hosts is limited (when you have S3 or smth), you better hardcode them and choose them with `map` or do it some other way;\n * if you can' list all possible hosts to proxy, you should sign the address.\n" }, { "path": "docs/en/plugins/validreferers.md", "content": "# [valid_referers] none in valid_referers\nModule [ngx_http_referer_module](http://nginx.org/en/docs/http/ngx_http_referer_module.html) allows to block the access to service for requests with wrong `Referer` value.\nIt's often used for setting `X-Frame-Options` header (ClickJacking protection), but there may be other cases.\n\nTypical problems with this module's config:\n * use of `server_names` with bad server name (`server_name` directive);\n * too broad and/or bad regexes;\n * use of `none`.\n\n> Notice: at the moment, Gixy can only detect the use of `none` as a valid referer.\n\n## Why none is bad?\nAccording to [docs](http://nginx.org/ru/docs/http/ngx_http_referer_module.html#valid_referers):\n> `none` - the “Referer” field is missing in the request header;\n\nStill, it's important to remember that any resource can make user's browser to make a request without a `Referer` request header.\nE.g.:\n - in case of redirect from HTTPS to HTTP;\n - by setting up the [Referrer Policy](https://www.w3.org/TR/referrer-policy/);\n - a request with opaque origin, `data:` scheme, for example.\n\nSo, by using `none` as a valid referer, you nullify any attemps in refferer validation." }, { "path": "docs/ru/plugins/addheadermultiline.md", "content": "# [add_header_multiline] Многострочные заголовоки ответа\n\nМногострочных заголовков ответа стоит избегать по нескольким причинам:\n * они признаны устаревшими (см. [RFC 7230](https://tools.ietf.org/html/rfc7230#section-3.2.4));\n * они никогда не поддерживались многими HTTP-клиентами и браузерами. Например, IE/Edge/Nginx.\n\n## Как самостоятельно обнаружить?\nПример плохой конфигурации:\n```nginx\n# http://nginx.org/ru/docs/http/ngx_http_headers_module.html#add_header\nadd_header Content-Security-Policy \"\n default-src: 'none';\n script-src data: https://yastatic.net;\n style-src data: https://yastatic.net;\n img-src data: https://yastatic.net;\n font-src data: https://yastatic.net;\";\n\n# https://www.nginx.com/resources/wiki/modules/headers_more/\nmore_set_headers -t 'text/html text/plain'\n 'X-Foo: Bar\n multiline';\n```\n\n## Что делать?\nЕдинственный выход - отказ от многострочных заголовок ответа." }, { "path": "docs/ru/plugins/addheaderredefinition.md", "content": "# [add_header_redefinition] Переопределение \"вышестоящих\" заголовков ответа директивой \"add_header\"\n\nК сожалению, многие считают что с помощью директивы `add_header` можно произвольно доопределять заголовки ответа.\nЭто не так, о чем сказано в [документации](http://nginx.org/ru/docs/http/ngx_http_headers_module.html#add_header) к Nginx:\n> Директив `add_header` может быть несколько. Директивы наследуются с предыдущего уровня при условии, что на данном уровне не описаны свои директивы `add_header`.\n\nК слову, так работает наследование большинства директив в nginx'е. Если вы задаёте что-то на каком-то уровне конфигурации (например, в локейшене), то наследования с предыдущих уровней (например, с http секции) - не будет.\n\nВ этом довольно легко убедится:\n - Конфигурация:\n```nginx\nserver {\n listen 80;\n add_header X-Frame-Options \"DENY\" always;\n location / {\n return 200 \"index\";\n }\n\n location /new-headers {\n # Add special cache control\n add_header Cache-Control \"no-cache, no-store, max-age=0, must-revalidate\" always;\n add_header Pragma \"no-cache\" always;\n\n return 200 \"new-headers\";\n }\n}\n```\n - Запрос к локейшену `/` (заголовок `X-Frame-Options` есть в ответе сервера):\n```http\nGET / HTTP/1.0\n\nHTTP/1.1 200 OK\nServer: nginx/1.10.2\nDate: Mon, 09 Jan 2017 19:28:33 GMT\nContent-Type: application/octet-stream\nContent-Length: 5\nConnection: close\nX-Frame-Options: DENY\n\nindex\n```\n - Запрос к локейшену `/new-headers` (есть заголовки `Cache-Control` и `Pragma`, но нет `X-Frame-Options`):\n```http\nGET /new-headers HTTP/1.0\n\n\nHTTP/1.1 200 OK\nServer: nginx/1.10.2\nDate: Mon, 09 Jan 2017 19:29:46 GMT\nContent-Type: application/octet-stream\nContent-Length: 11\nConnection: close\nCache-Control: no-cache, no-store, max-age=0, must-revalidate\nPragma: no-cache\n\nnew-headers\n```\n\n## Что делать?\nСуществует несколько способов решить эту проблему:\n - продублировать важные заголовки;\n - устанавливать заголовки на одном уровне, например, в серверной секции;\n - использовать модуль [ngx_headers_more](https://www.nginx.com/resources/wiki/modules/headers_more/).\n\nКаждый из способов имеет свои преимущества и недостатки, какой предпочесть зависит от ваших потребностей. " }, { "path": "docs/ru/plugins/aliastraversal.md", "content": "# [alias_traversal] Path traversal при использовании alias\n\nДиректива [alias](https://nginx.ru/ru/docs/http/ngx_http_core_module.html#alias) используется для замены пути указанного локейшена.\nК примеру, для конфигурации:\n```nginx\nlocation /i/ {\n alias /data/w3/images/;\n}\n```\nна запрос `/i/top.gif` будет отдан файл `/data/w3/images/top.gif`.\n\nОднако, если локейшен не оканчивается разделителем директорий (`/`):\n```nginx\nlocation /i {\n alias /data/w3/images/;\n}\n```\nто на запрос `/i../app/config.py` будет отдан файл `/data/w3/app/config.py`.\n\nИными словами, не корректная конфигурация `alias` может позволить злоумышленнику прочесть файл за пределами целевой директории.\n\n## Что делать?\nВсе довольно просто:\n - необходимо найти все директивы `alias`;\n - убедится что вышестоящий префиксный локейшен оканчивается на `/`.\n" }, { "path": "docs/ru/plugins/hostspoofing.md", "content": "# [host_spoofing] Подделка заголовка запроса Host\n\nЗачастую, приложению, стоящему за Nginx, необходимо передать корректный заголовок `Host` для корректной генерации различных URL-адресов (редиректы, ресурсы, ссылки в письмах и т.д.).\nВозможность его подмены злоумышленником может повлечь множестве проблем от фишинговых атак до SSRF, поэтому следует избегать таких ситуаций.\n\n> Возможно, ваше приложение так же ориентируется на заголовок запроса `X-Forwarded-Host`.\n> В этом случае вам необходимо самостоятельно позаботится о его корректной установке при проксировании.\n\n## Как самостоятельно обнаружить?\nЧаще всего эта проблема возникает в результате использования переменной `$http_host` вместо `$host`.\n\nНесмотря на их схожесть, они сильно отличаются:\n * `$host` - хост в порядке приоритета: имя хоста из строки запроса, или имя хоста из заголовка `Host` заголовка запроса, или имя сервера, соответствующего запросу;\n * `$http_host` - заголовок запроса \"Host\".\n\nПример такой конфигурации:\n```nginx\nlocation @app {\n proxy_set_header Host $http_host;\n # Other proxy params\n proxy_pass http://backend;\n}\n```\n\n## Что делать?\nК счастью, все довольно очевидно:\n * перечислить корректные имена сервера в директиве `server_name`;\n * всегда использовать переменную `$host`, вместо `$http_host`.\n\n## Дополнительная информация\n * [Host of Troubles Vulnerabilities](https://hostoftroubles.com/)\n * [Practical HTTP Host header attacks](http://www.skeletonscribe.net/2013/05/practical-http-host-header-attacks.html)\n" }, { "path": "docs/ru/plugins/httpsplitting.md", "content": "# [http_splitting] HTTP Splitting\n\nHTTP Splitting - уязвимость, возникающая из-за неправильной обработки входных данных.\nЗачастую может быть для атак на приложение стоящее за Nginx (HTTP Request Splitting) или на клиентов приложения (HTTP Response Splitting).\n\nУязвимость возникает в случае, когда атакующий может внедрить символ перевода строки `\\n` или `\\r` в запрос или ответ формируемый Nginx.\n\n## Как самостоятельно обнаружить?\nПри анализе конфигурации всега стоит обращать внимание на:\n - какие переменные используются в директивах, отвечающих за формирование запросов (могут ли они содержать CRLF), например: `rewrite`, `return`, `add_header`, `proxy_set_header` или `proxy_pass`;\n - используются ли переменные `$uri` и `$document_uri` и если да, то в каких директивах, т.к. они гарантированно содержат урлдекодированное значение;\n - переменные, выделенные из групп с исключающим диапазоном: `(?P[^.]+)`.\n\nПример плохой конфигурации с переменной, полученной из группы с исключающим диапазоном:\n```nginx\nserver {\n listen 80 default;\n\n location ~ /v1/((?[^.]*)\\.json)?$ {\n add_header X-Action $action;\n return 200 \"OK\";\n }\n}\n```\n\nПример эксплуатации данной конфигурации:\n```http\nGET /v1/see%20below%0d%0ax-crlf-header:injected.json HTTP/1.0\nHost: localhost\n\nHTTP/1.1 200 OK\nServer: nginx/1.11.10\nDate: Mon, 13 Mar 2017 21:21:29 GMT\nContent-Type: application/octet-stream\nContent-Length: 2\nConnection: close\nX-Action: see below\nx-crlf-header:injected\n\nOK\n```\n\nИз примера видно, что злоумышленник смог добавить заголовок ответа `x-crlf-header: injected`. Это случилось благодаря стечению нескольких обстоятельств:\n - `add_header` не кодирует/валидирует переданные ему значения, считая что автор знает о последствиях;\n - значение пути нормализуется перед обработкой локейшена;\n - переменная `$action` была выделена из группы регулярного выражения с исключающим диапазоном: `[^.]*`;\n - таким образом, значение переменной `$action` равно `see below\\r\\nx-crlf-header:injected` и при её использовании в формировании ответа добавился заголовок.\n\n## Что делать?\n - старайтесь использовать более безопасные переменные, например, `$request_uri` вместо `$uri`;\n - запретите перевод строки в исключающем диапазоне, например, `/some/(?[^/\\s]+)` вместо `/some/(?[^/]+`;\n - возможно, хорошей идеей будет добавить валидацию `$uri` (только если вы знаете, что делаете).\n" }, { "path": "docs/ru/plugins/origins.md", "content": "# [origins] Проблемы валидации referrer/origin\n\nНередко валидация заголовка запроса `Referer` или `Origin` делается при помощи регулярного выражения.\nЗачастую, это необходимо для условного выставления заголовка `X-Frame-Options` (защита от ClickJacking) или реализации Cross-Origin Resource Sharing.\n\nНаиболее распространенно два класса ошибок конфигурации, которые приводят к этой проблеме:\n - ошибки в составлении регулярного выражения;\n - разрешение не доверенных third-party доменов.\n\n> По умолчанию Gixy не валидирует регулярные выражение на предмет матчинга third-party доменов, т.к. не знает кому можно верить.\nПередать список доверенных доменом можно при помощи опции `--origins-domains example.com,foo.bar`\n\n## Как самостоятельно обнаружить?\nВсе довольно \"просто\":\n - необходимо найти все директивы `if`, которые делают проверку переменной `$http_origin` или `$http_referer`;\n - убедится что в регулярном выражении нет проблем.\n\nПример плохой конфигурации:\n```nginx\nif ($http_origin ~* ((^https://www\\.yandex\\.ru)|(^https://ya\\.ru)$)) {\n\tadd_header 'Access-Control-Allow-Origin' \"$http_origin\";\n\tadd_header 'Access-Control-Allow-Credentials' 'true';\n}\n```\n\nTODO(buglloc): описать типичные проблемы при составлении регулярных выражений\nTODO(buglloc): Regex Ninja?\n\n## Что делать?\n - исправить регулярное выражение или отказаться от него вовсе :)\n - если вы проверяете заголовок запроса `Referer` то, возможно (имеются противопоказания), лучшим решением было бы воспользоваться модулем [ngx_http_referer_module](http://nginx.org/ru/docs/http/ngx_http_referer_module.html);\n - если вы проверяете заголовов запроса `Origin` то, зачастую, лучше использовать `map` и отказаться от регулярных выражений.\n" }, { "path": "docs/ru/plugins/ssrf.md", "content": "# [ssrf] Server Side Request Forgery\n\nServer Side Request Forgery - уязвимость, позволяющая выполнять различного рода запросы от имени веб-приложения (в нашем случае от имени Nginx).\nВозникает, когда атакующий может контролировать адрес проксируемого сервера (второй аргумент директивы `proxy_pass`).\n\n\n## Как самостоятельно обнаружить?\nНаиболее распространенно два класса ошибок конфигурации, которые приводят к этой проблеме:\n - отсутствие директивы [internal](http://nginx.org/ru/docs/http/ngx_http_core_module.html#internal). Её смысл заключается в указании того, что определенный location может использоваться только для внутренних запросов;\n - небезопасное внутреннее перенаправление.\n\n### Отсутствие директивы internal\nКлассический пример уязвимости типа SSRF в виду отсутствия директивы `internal` выглядит следующим образом:\n```nginx\nlocation ~ /proxy/(.*)/(.*)/(.*)$ {\n proxy_pass $1://$2/$3;\n}\n```\nЗлоумышленник, полностью контролируя адрес проксируемого сервера, может выполнять произвольные запросы от имени Nginx.\n\n### Небезопасное внутреннее перенаправление\nПодразумевается, что в вашей конфигурации есть internal location, которые использует какие-либо данные из запроса в качестве адреса проксируемого сервера.\n\nНапример:\n```nginx\nlocation ~* ^/internal-proxy/(?https?)/(?.*?)/(?.*)$ {\n internal;\n\n proxy_pass $proxy_proto://$proxy_host/$proxy_path ;\n proxy_set_header Host $proxy_host;\n}\n```\n\nСогласно документации Nginx внутренними запросами являются:\n> - запросы, перенаправленные директивами **error_page**, index, random_index и **try_files**;\n> - запросы, перенаправленные с помощью поля “X-Accel-Redirect” заголовка ответа вышестоящего сервера;\n> - подзапросы, формируемые командой “include virtual” модуля ngx_http_ssi_module и директивами модуля ngx_http_addition_module;\n> - запросы, изменённые директивой **rewrite**.]>\n\nСоответственно, любой \"неосторожный\" реврайт позволит злоумышленнику сделать внутренний запрос и контролировать адрес проксируемого сервера.\n\nПример плохой конфигурации:\n```nginx\nrewrite ^/(.*)/some$ /$1/ last;\n\nlocation ~* ^/internal-proxy/(?https?)/(?.*?)/(?.*)$ {\n internal;\n\n proxy_pass $proxy_proto://$proxy_host/$proxy_path ;\n proxy_set_header Host $proxy_host;\n}\n```\n\n## Что делать?\nЕсть несколько правил, которых стоит придерживаться в подобного рода конфигурациях:\n - использовать только internal location для проксирования;\n - по возможности запретить передачу пользовательских данных;\n - обезопасить адрес проксируемого сервера:\n * если количество проксируемых хостов ограниченно (например, у вас S3), то лучше их захардкодить и выбирать при помощи `map` или иным удобным для вас образом;\n * если по какой-то причине нет возможности перечислить все возможные хосты для проксирования, его стоит подписать.\n" }, { "path": "docs/ru/plugins/validreferers.md", "content": "# [valid_referers] none in valid_referers\n\nМодуль [ngx_http_referer_module](http://nginx.org/ru/docs/http/ngx_http_referer_module.html) позволяет блокировать доступ к сервису для запросов с неверными значениями заголовка запроса `Referer`.\nЗачастую используется для условного выставления заголовка `X-Frame-Options` (защита от ClickJacking), но могут быть и иные случаи.\n\nТипичные проблемы при конфигурировании этого модуля:\n * использование `server_names` при не корректном имени сервера (директива `server_name`);\n * слишком общие и/или не корректные регулярные выражения;\n * использование `none`.\n\n> На текущий момент, Gixy умеет определять только использование `none` в качестве валидного реферера.\n\n## Чем плох none?\nСогласно [документации](http://nginx.org/ru/docs/http/ngx_http_referer_module.html#valid_referers):\n> `none` - поле “Referer” в заголовке запроса отсутствует;\n\nОднако, важно помнить, что любой ресурс может заставить браузер пользователя выполнить запрос без заголовка запроса `Referer`, к примеру:\n - в случае редиректа с HTTPS на HTTP;\n - указав соответствующую [Referrer Policy](https://www.w3.org/TR/referrer-policy/);\n - обращение с opaque origin, например, используя схему `data:`.\n\nТаким образом, используя `none` в качестве валидного реферера вы сводите на нет любые попытки валидации реферера." }, { "path": "gixy/__init__.py", "content": "# flake8: noqa\n\nfrom gixy.core import severity\n\nversion = '0.1.21'\n" }, { "path": "gixy/cli/__init__.py", "content": "" }, { "path": "gixy/cli/argparser.py", "content": "# flake8: noqa\n\nfrom configargparse import *\nfrom six.moves import StringIO\n\nfrom gixy.core.plugins_manager import PluginsManager\n\n# used while parsing args to keep track of where they came from\n_COMMAND_LINE_SOURCE_KEY = 'command_line'\n_ENV_VAR_SOURCE_KEY = 'environment_variables'\n_CONFIG_FILE_SOURCE_KEY = 'config_file'\n_DEFAULTS_SOURCE_KEY = 'defaults'\n\n\nclass GixyConfigParser(DefaultConfigFileParser):\n def get_syntax_description(self):\n return ''\n\n def parse(self, stream):\n \"\"\"Parses the keys + values from a config file.\"\"\"\n\n items = OrderedDict()\n prefix = ''\n for i, line in enumerate(stream):\n line = line.strip()\n if not line or line[0] in ['#', ';'] or line.startswith('---'):\n continue\n if line[0] == '[':\n prefix = '%s-' % line[1:-1].replace('_', '-')\n continue\n\n white_space = '\\\\s*'\n key = '(?P[^:=;#\\s]+?)'\n value = white_space + '[:=\\s]' + white_space + '(?P.+?)'\n comment = white_space + '(?P\\\\s[;#].*)?'\n\n key_only_match = re.match('^' + key + comment + '$', line)\n if key_only_match:\n key = key_only_match.group('key')\n items[key] = 'true'\n continue\n\n key_value_match = re.match('^' + key + value + comment + '$', line)\n if key_value_match:\n key = key_value_match.group('key')\n value = key_value_match.group('value')\n\n if value.startswith('[') and value.endswith(']'):\n # handle special case of lists\n value = [elem.strip() for elem in value[1:-1].split(',')]\n\n items[prefix + key] = value\n continue\n\n raise ConfigFileParserException('Unexpected line %s in %s: %s' % (i,\n getattr(stream, 'name', 'stream'), line))\n return items\n\n def serialize(self, items):\n \"\"\"Does the inverse of config parsing by taking parsed values and\n converting them back to a string representing config file contents.\n \"\"\"\n r = StringIO()\n for key, value in items.items():\n if type(value) == OrderedDict:\n r.write('\\n[%s]\\n' % key)\n r.write(self.serialize(value))\n else:\n value, help = value\n if help:\n r.write('; %s\\n' % help)\n r.write('%s = %s\\n' % (key, value))\n return r.getvalue()\n\n\nclass GixyHelpFormatter(HelpFormatter):\n def format_help(self):\n manager = PluginsManager()\n help_message = super(GixyHelpFormatter, self).format_help()\n if 'plugins options:' in help_message:\n # Print available blugins _only_ if we prints options for it\n plugins = '\\n'.join('\\t' + plugin.__name__ for plugin in manager.plugins_classes)\n help_message = '{orig}\\n\\navailable plugins:\\n{plugins}\\n'.format(orig=help_message, plugins=plugins)\n return help_message\n\n\nclass ArgsParser(ArgumentParser):\n def get_possible_config_keys(self, action):\n \"\"\"This method decides which actions can be set in a config file and\n what their keys will be. It returns a list of 0 or more config keys that\n can be used to set the given action's value in a config file.\n \"\"\"\n keys = []\n for arg in action.option_strings:\n if arg in ['--config', '--write-config', '--version']:\n continue\n if any([arg.startswith(2 * c) for c in self.prefix_chars]):\n keys += [arg[2:], arg] # eg. for '--bla' return ['bla', '--bla']\n\n return keys\n\n def get_items_for_config_file_output(self, source_to_settings,\n parsed_namespace):\n \"\"\"Converts the given settings back to a dictionary that can be passed\n to ConfigFormatParser.serialize(..).\n\n Args:\n source_to_settings: the dictionary described in parse_known_args()\n parsed_namespace: namespace object created within parse_known_args()\n Returns:\n an OrderedDict where keys are strings and values are either strings\n or lists\n \"\"\"\n config_file_items = OrderedDict()\n for source, settings in source_to_settings.items():\n if source == _COMMAND_LINE_SOURCE_KEY:\n _, existing_command_line_args = settings['']\n for action in self._actions:\n config_file_keys = self.get_possible_config_keys(action)\n if config_file_keys and not action.is_positional_arg and \\\n already_on_command_line(existing_command_line_args,\n action.option_strings):\n value = getattr(parsed_namespace, action.dest, None)\n if value is not None:\n if type(value) is bool:\n value = str(value).lower()\n if ':' in action.dest:\n section, key = action.dest.split(':', 2)\n key = key.replace('_', '-')\n if section not in config_file_items:\n config_file_items[section] = OrderedDict()\n config_file_items[section][key] = (value, action.help)\n else:\n config_file_items[config_file_keys[0]] = (value, action.help)\n elif source.startswith(_CONFIG_FILE_SOURCE_KEY):\n for key, (action, value) in settings.items():\n if ':' in action.dest:\n section, key = action.dest.split(':', 2)\n key = key.replace('_', '-')\n if section not in config_file_items:\n config_file_items[section] = OrderedDict()\n config_file_items[section][key] = (value, action.help)\n else:\n config_file_items[key] = (value, action.help)\n return config_file_items\n\n\ndef create_parser():\n return ArgsParser(\n description='Gixy - a Nginx configuration [sec]analyzer\\n\\n',\n formatter_class=GixyHelpFormatter,\n config_file_parser_class=GixyConfigParser,\n auto_env_var_prefix='GIXY_',\n add_env_var_help=False,\n default_config_files=['/etc/gixy/gixy.cfg', '~/.config/gixy/gixy.conf'],\n args_for_setting_config_path=['-c', '--config'],\n args_for_writing_out_config_file=['--write-config'],\n add_config_file_help=False\n )\n" }, { "path": "gixy/cli/main.py", "content": "import os\nimport sys\nimport logging\nimport copy\n\nimport gixy\nfrom gixy.core.manager import Manager as Gixy\nfrom gixy.formatters import get_all as formatters\nfrom gixy.core.plugins_manager import PluginsManager\nfrom gixy.core.config import Config\nfrom gixy.cli.argparser import create_parser\nfrom gixy.core.exceptions import InvalidConfiguration\n\nLOG = logging.getLogger()\n\n\ndef _init_logger(debug=False):\n LOG.handlers = []\n log_level = logging.DEBUG if debug else logging.INFO\n\n LOG.setLevel(log_level)\n handler = logging.StreamHandler(sys.stderr)\n handler.setFormatter(logging.Formatter('[%(module)s]\\t%(levelname)s\\t%(message)s'))\n LOG.addHandler(handler)\n LOG.debug(\"logging initialized\")\n\n\ndef _create_plugin_help(option):\n if isinstance(option, (tuple, list, set)):\n default = ','.join(list(option))\n else:\n default = str(option)\n\n return 'Default: {0}'.format(default)\n\n\ndef _get_cli_parser():\n parser = create_parser()\n parser.add_argument('nginx_files', nargs='*', type=str, default=['/etc/nginx/nginx.conf'], metavar='nginx.conf',\n help='Path to nginx.conf, e.g. /etc/nginx/nginx.conf')\n\n parser.add_argument(\n '-v', '--version', action='version',\n version='Gixy v{0}'.format(gixy.version))\n\n parser.add_argument(\n '-l', '--level', dest='level', action='count', default=0,\n help='Report issues of a given severity level or higher (-l for LOW, -ll for MEDIUM, -lll for HIGH)')\n\n default_formatter = 'console' if sys.stdout.isatty() else 'text'\n available_formatters = formatters().keys()\n parser.add_argument(\n '-f', '--format', dest='output_format', choices=available_formatters, default=default_formatter,\n type=str, help='Specify output format')\n\n parser.add_argument(\n '-o', '--output', dest='output_file', type=str,\n help='Write report to file')\n\n parser.add_argument(\n '-d', '--debug', dest='debug', action='store_true', default=False,\n help='Turn on debug mode')\n\n parser.add_argument(\n '--tests', dest='tests', type=str,\n help='Comma-separated list of tests to run')\n\n parser.add_argument(\n '--skips', dest='skips', type=str,\n help='Comma-separated list of tests to skip')\n\n parser.add_argument(\n '--disable-includes', dest='disable_includes', action='store_true', default=False,\n help='Disable \"include\" directive processing')\n\n group = parser.add_argument_group('plugins options')\n for plugin_cls in PluginsManager().plugins_classes:\n name = plugin_cls.__name__\n if not plugin_cls.options:\n continue\n\n options = copy.deepcopy(plugin_cls.options)\n for opt_key, opt_val in options.items():\n option_name = '--{plugin}-{key}'.format(plugin=name, key=opt_key).replace('_', '-')\n dst_name = '{plugin}:{key}'.format(plugin=name, key=opt_key)\n opt_type = str if isinstance(opt_val, (tuple, list, set)) else type(opt_val)\n group.add_argument(\n option_name, metavar=opt_key, dest=dst_name, type=opt_type,\n help=_create_plugin_help(opt_val)\n )\n\n return parser\n\n\ndef main():\n parser = _get_cli_parser()\n args = parser.parse_args()\n _init_logger(args.debug)\n\n if len(args.nginx_files) == 1 and args.nginx_files[0] != '-':\n path = os.path.expanduser(args.nginx_files[0])\n if not os.path.exists(path):\n sys.stderr.write('File {path!r} was not found.\\nPlease specify correct path to configuration.\\n'.format(\n path=path))\n sys.exit(1)\n\n try:\n severity = gixy.severity.ALL[args.level]\n except IndexError:\n sys.stderr.write('Too high level filtering. Maximum level: -{0}\\n'.format('l' * (len(gixy.severity.ALL) - 1)))\n sys.exit(1)\n\n if args.tests:\n tests = [x.strip() for x in args.tests.split(',')]\n else:\n tests = None\n\n if args.skips:\n skips = [x.strip() for x in args.skips.split(',')]\n else:\n skips = None\n\n config = Config(\n severity=severity,\n output_format=args.output_format,\n output_file=args.output_file,\n plugins=tests,\n skips=skips,\n allow_includes=not args.disable_includes\n )\n\n for plugin_cls in PluginsManager().plugins_classes:\n name = plugin_cls.__name__\n options = copy.deepcopy(plugin_cls.options)\n for opt_key, opt_val in options.items():\n option_name = '{name}:{key}'.format(name=name, key=opt_key)\n if option_name not in args:\n continue\n\n val = getattr(args, option_name)\n if val is None:\n continue\n\n if isinstance(opt_val, tuple):\n val = tuple([x.strip() for x in val.split(',')])\n elif isinstance(opt_val, set):\n val = set([x.strip() for x in val.split(',')])\n elif isinstance(opt_val, list):\n val = [x.strip() for x in val.split(',')]\n options[opt_key] = val\n config.set_for(name, options)\n\n formatter = formatters()[config.output_format]()\n failed = False\n for input_path in args.nginx_files:\n path = os.path.abspath(os.path.expanduser(input_path))\n if not os.path.exists(path):\n LOG.error('File %s was not found', path)\n continue\n\n with Gixy(config=config) as yoda:\n try:\n if path == '-':\n with os.fdopen(sys.stdin.fileno(), 'rb') as fdata:\n yoda.audit('', fdata, is_stdin=True)\n else:\n with open(path, mode='rb') as fdata:\n yoda.audit(path, fdata, is_stdin=False)\n except InvalidConfiguration:\n failed = True\n formatter.feed(path, yoda)\n failed = failed or sum(yoda.stats.values()) > 0\n\n if args.output_file:\n with open(config.output_file, 'w') as f:\n f.write(formatter.flush())\n else:\n print(formatter.flush())\n\n if failed:\n # If something found - exit code must be 1, otherwise 0\n sys.exit(1)\n sys.exit(0)\n" }, { "path": "gixy/core/__init__.py", "content": "" }, { "path": "gixy/core/builtin_variables.py", "content": "from gixy.core.regexp import Regexp\nfrom gixy.core.variable import Variable\n\nBUILTIN_VARIABLES = {\n # http://nginx.org/en/docs/http/ngx_http_core_module.html#var_uri\n 'uri': r'/[^\\x20\\t]*',\n # http://nginx.org/en/docs/http/ngx_http_core_module.html#var_document_uri\n 'document_uri': r'/[^\\x20\\t]*',\n # http://nginx.org/en/docs/http/ngx_http_core_module.html#var_arg_\n 'arg_': r'[^\\s&]+',\n # http://nginx.org/en/docs/http/ngx_http_core_module.html#var_args\n 'args': r'[^\\s]+',\n # http://nginx.org/en/docs/http/ngx_http_core_module.html#var_query_string\n 'query_string': r'[^\\s]+',\n # http://nginx.org/en/docs/http/ngx_http_core_module.html#var_request_uri\n 'request_uri': r'/[^\\s]*',\n # http://nginx.org/en/docs/http/ngx_http_core_module.html#var_http_\n 'http_': r'[\\x21-\\x7e]',\n\n # http://nginx.org/en/docs/http/ngx_http_upstream_module.html#var_upstream_http_\n 'upstream_http_': '',\n # http://nginx.org/en/docs/http/ngx_http_upstream_module.html#var_upstream_cookie_\n 'upstream_cookie_': '',\n # http://nginx.org/en/docs/http/ngx_http_proxy_module.html#var_proxy_add_x_forwarded_for\n 'proxy_add_x_forwarded_for': '',\n # http://nginx.org/en/docs/http/ngx_http_proxy_module.html#var_proxy_host\n 'proxy_host': '',\n # http://nginx.org/en/docs/http/ngx_http_proxy_module.html#var_proxy_port\n 'proxy_port': '',\n # http://nginx.org/en/docs/http/ngx_http_core_module.html#var_proxy_protocol_addr\n # http://nginx.org/en/docs/stream/ngx_stream_core_module.html#var_proxy_protocol_addr\n # http://nginx.org/en/docs/http/ngx_http_core_module.html#var_proxy_protocol_port\n # http://nginx.org/en/docs/stream/ngx_stream_core_module.html#var_proxy_protocol_port\n 'proxy_protocol_port': '',\n # http://nginx.org/en/docs/http/ngx_http_fastcgi_module.html#var_fastcgi_path_info\n 'fastcgi_path_info': '',\n # http://nginx.org/en/docs/http/ngx_http_fastcgi_module.html#var_fastcgi_script_name\n 'fastcgi_script_name': '',\n # http://nginx.org/en/docs/http/ngx_http_core_module.html#var_content_type\n 'content_type': '',\n # http://nginx.org/en/docs/http/ngx_http_core_module.html#var_cookie_\n 'cookie_': '',\n # http://nginx.org/en/docs/http/ngx_http_core_module.html#var_host\n 'host': '',\n # http://nginx.org/en/docs/http/ngx_http_core_module.html#var_hostname\n # http://nginx.org/en/docs/stream/ngx_stream_core_module.html#var_hostname\n 'hostname': '',\n # http://nginx.org/en/docs/http/ngx_http_core_module.html#var_limit_rate\n 'limit_rate': '',\n # http://nginx.org/en/docs/http/ngx_http_memcached_module.html#var_memcached_key\n 'memcached_key': '',\n # http://nginx.org/en/docs/http/ngx_http_core_module.html#var_realpath_root\n 'realpath_root': '',\n # http://nginx.org/en/docs/http/ngx_http_core_module.html#var_remote_user\n 'remote_user': '',\n # http://nginx.org/en/docs/http/ngx_http_core_module.html#var_request\n 'request': '',\n # http://nginx.org/en/docs/http/ngx_http_core_module.html#var_request_body\n 'request_body': '',\n # http://nginx.org/en/docs/http/ngx_http_core_module.html#var_request_completion\n 'request_completion': '',\n # http://nginx.org/en/docs/http/ngx_http_core_module.html#var_request_filename\n 'request_filename': '',\n # http://nginx.org/en/docs/http/ngx_http_core_module.html#var_request_id\n 'request_id': '',\n # http://nginx.org/en/docs/http/ngx_http_slice_module.html#var_slice_range\n 'slice_range': '',\n # http://nginx.org/en/docs/http/ngx_http_secure_link_module.html#var_secure_link\n 'secure_link': '',\n # http://nginx.org/en/docs/http/ngx_http_secure_link_module.html#var_secure_link_expires\n 'secure_link_expires': '',\n # http://nginx.org/en/docs/http/ngx_http_core_module.html#var_sent_http_\n 'sent_http_': '',\n # http://nginx.org/en/docs/http/ngx_http_core_module.html#var_server_name\n 'server_name': '',\n\n # \"Secure\" variables that can't content or strictly limited user input\n\n # http://nginx.org/en/docs/http/ngx_http_browser_module.html#var_ancient_browser\n 'ancient_browser': None,\n # http://nginx.org/en/docs/http/ngx_http_core_module.html#var_binary_remote_addr\n # http://nginx.org/en/docs/stream/ngx_stream_core_module.html#var_binary_remote_addr\n 'binary_remote_addr': None,\n # http://nginx.org/en/docs/http/ngx_http_core_module.html#var_body_bytes_sent\n 'body_bytes_sent': None,\n # http://nginx.org/en/docs/stream/ngx_stream_core_module.html#var_bytes_received\n 'bytes_received': None,\n # http://nginx.org/en/docs/http/ngx_http_core_module.html#var_bytes_sent\n # http://nginx.org/en/docs/http/ngx_http_log_module.html#var_bytes_sent\n # http://nginx.org/en/docs/stream/ngx_stream_core_module.html#var_bytes_sent\n 'bytes_sent': None,\n # http://nginx.org/en/docs/http/ngx_http_core_module.html#var_connection\n # http://nginx.org/en/docs/http/ngx_http_log_module.html#var_connection\n # http://nginx.org/en/docs/stream/ngx_stream_core_module.html#var_connection\n 'connection': None,\n # http://nginx.org/en/docs/http/ngx_http_core_module.html#var_connection_requests\n # http://nginx.org/en/docs/http/ngx_http_log_module.html#var_connection_requests\n 'connection_requests': None,\n # http://nginx.org/en/docs/http/ngx_http_stub_status_module.html#var_connections_active\n 'connections_active': None,\n # http://nginx.org/en/docs/http/ngx_http_stub_status_module.html#var_connections_reading\n 'connections_reading': None,\n # http://nginx.org/en/docs/http/ngx_http_stub_status_module.html#var_connections_waiting\n 'connections_waiting': None,\n # http://nginx.org/en/docs/http/ngx_http_stub_status_module.html#var_connections_writing\n 'connections_writing': None,\n # http://nginx.org/en/docs/http/ngx_http_core_module.html#var_content_length\n 'content_length': None,\n # http://nginx.org/en/docs/http/ngx_http_ssi_module.html#var_date_gmt\n 'date_gmt': None,\n # http://nginx.org/en/docs/http/ngx_http_ssi_module.html#var_date_local\n 'date_local': None,\n # http://nginx.org/en/docs/http/ngx_http_core_module.html#var_document_root\n 'document_root': '/etc/nginx',\n # http://nginx.org/en/docs/http/ngx_http_geoip_module.html\n # http://nginx.org/en/docs/stream/ngx_stream_geoip_module.html\n 'geoip_': None,\n # http://nginx.org/en/docs/http/ngx_http_gzip_module.html#var_gzip_ratio\n 'gzip_ratio': None,\n # http://nginx.org/en/docs/http/ngx_http_v2_module.html#var_http2\n 'http2': None,\n # http://nginx.org/en/docs/http/ngx_http_core_module.html#var_https\n 'https': None,\n # http://nginx.org/en/docs/http/ngx_http_referer_module.html#var_invalid_referer\n 'invalid_referer': None,\n # http://nginx.org/en/docs/http/ngx_http_core_module.html#var_is_args\n 'is_args': None,\n # http://nginx.org/en/docs/http/ngx_http_auth_jwt_module.html\n 'jwt_': None,\n # http://nginx.org/en/docs/http/ngx_http_browser_module.html#var_modern_browser\n 'modern_browser': None,\n # http://nginx.org/en/docs/http/ngx_http_core_module.html#var_msec\n # http://nginx.org/en/docs/http/ngx_http_log_module.html#var_msec\n # http://nginx.org/en/docs/stream/ngx_stream_core_module.html#var_msec\n 'msec': None,\n # http://nginx.org/en/docs/http/ngx_http_browser_module.html#var_msie\n 'msie': None,\n # http://nginx.org/en/docs/http/ngx_http_core_module.html#var_nginx_version\n # http://nginx.org/en/docs/stream/ngx_stream_core_module.html#var_nginx_version\n 'nginx_version': None,\n # http://nginx.org/en/docs/http/ngx_http_core_module.html#var_pid\n # http://nginx.org/en/docs/stream/ngx_stream_core_module.html#var_pid\n 'pid': None,\n # http://nginx.org/en/docs/http/ngx_http_core_module.html#var_pipe\n # http://nginx.org/en/docs/http/ngx_http_log_module.html#var_pipe\n 'pipe': None,\n # http://nginx.org/en/docs/stream/ngx_stream_core_module.html#var_protocol\n 'protocol': None,\n # http://nginx.org/en/docs/http/ngx_http_realip_module.html#var_realip_remote_addr\n # http://nginx.org/en/docs/stream/ngx_stream_realip_module.html#var_realip_remote_addr\n # http://nginx.org/en/docs/http/ngx_http_realip_module.html#var_realip_remote_port\n # http://nginx.org/en/docs/stream/ngx_stream_realip_module.html#var_realip_remote_port\n 'realip_remote_port': None,\n # http://nginx.org/en/docs/http/ngx_http_core_module.html#var_remote_addr\n # http://nginx.org/en/docs/stream/ngx_stream_core_module.html#var_remote_addr\n 'remote_addr': None,\n # http://nginx.org/en/docs/http/ngx_http_core_module.html#var_remote_port\n # http://nginx.org/en/docs/stream/ngx_stream_core_module.html#var_remote_port\n 'remote_port': None,\n # http://nginx.org/en/docs/http/ngx_http_core_module.html#var_request_body_file\n 'request_body_file': None,\n # http://nginx.org/en/docs/http/ngx_http_core_module.html#var_request_length\n # http://nginx.org/en/docs/http/ngx_http_log_module.html#var_request_length\n 'request_length': None,\n # http://nginx.org/en/docs/http/ngx_http_core_module.html#var_request_method\n 'request_method': None,\n # http://nginx.org/en/docs/http/ngx_http_core_module.html#var_request_time\n # http://nginx.org/en/docs/http/ngx_http_log_module.html#var_request_time\n 'request_time': None,\n # http://nginx.org/en/docs/http/ngx_http_core_module.html#var_scheme\n 'scheme': None,\n # http://nginx.org/en/docs/http/ngx_http_core_module.html#var_server_addr\n # http://nginx.org/en/docs/stream/ngx_stream_core_module.html#var_server_addr\n 'server_addr': None,\n # http://nginx.org/en/docs/http/ngx_http_core_module.html#var_server_port\n # http://nginx.org/en/docs/stream/ngx_stream_core_module.html#var_server_port\n 'server_port': None,\n # http://nginx.org/en/docs/http/ngx_http_core_module.html#var_server_protocol\n 'server_protocol': None,\n # http://nginx.org/en/docs/http/ngx_http_session_log_module.html#var_session_log_binary_id\n 'session_log_binary_id': None,\n # http://nginx.org/en/docs/http/ngx_http_session_log_module.html#var_session_log_id\n 'session_log_id': None,\n # http://nginx.org/en/docs/stream/ngx_stream_core_module.html#var_session_time\n 'session_time': None,\n # http://nginx.org/en/docs/http/ngx_http_spdy_module.html#var_spdy\n 'spdy': None,\n # http://nginx.org/en/docs/http/ngx_http_spdy_module.html#var_spdy_request_priority\n 'spdy_request_priority': None,\n # http://nginx.org/en/docs/http/ngx_http_ssl_module.html\n # http://nginx.org/en/docs/stream/ngx_stream_ssl_module.html\n 'ssl_': None,\n # http://nginx.org/en/docs/http/ngx_http_core_module.html#var_status\n # http://nginx.org/en/docs/http/ngx_http_log_module.html#var_status\n # http://nginx.org/en/docs/stream/ngx_stream_core_module.html#var_status\n 'status': None,\n # http://nginx.org/en/docs/http/ngx_http_core_module.html\n 'tcpinfo_': None,\n # http://nginx.org/en/docs/http/ngx_http_core_module.html\n # http://nginx.org/en/docs/http/ngx_http_log_module.html\n # http://nginx.org/en/docs/stream/ngx_stream_core_module.html\n 'time_iso8601': None,\n # http://nginx.org/en/docs/http/ngx_http_core_module.html\n # http://nginx.org/en/docs/http/ngx_http_log_module.html\n # http://nginx.org/en/docs/stream/ngx_stream_core_module.html\n 'time_local': None,\n # http://nginx.org/en/docs/http/ngx_http_userid_module.html#var_uid_got\n 'uid_got': None,\n # http://nginx.org/en/docs/http/ngx_http_userid_module.html#var_uid_reset\n 'uid_reset': None,\n # http://nginx.org/en/docs/http/ngx_http_userid_module.html#var_uid_set\n 'uid_set': None,\n # http://nginx.org/en/docs/http/ngx_http_upstream_module.html#var_upstream_addr\n # http://nginx.org/en/docs/stream/ngx_stream_upstream_module.html#var_upstream_addr\n 'upstream_addr': None,\n # http://nginx.org/en/docs/http/ngx_http_upstream_module.html#var_upstream_bytes_received\n # http://nginx.org/en/docs/stream/ngx_stream_upstream_module.html#var_upstream_bytes_received\n 'upstream_bytes_received': None,\n # http://nginx.org/en/docs/stream/ngx_stream_upstream_module.html#var_upstream_bytes_sent\n 'upstream_bytes_sent': None,\n # http://nginx.org/en/docs/http/ngx_http_upstream_module.html#var_upstream_cache_status\n 'upstream_cache_status': None,\n # http://nginx.org/en/docs/http/ngx_http_upstream_module.html#var_upstream_connect_time\n # http://nginx.org/en/docs/stream/ngx_stream_upstream_module.html#var_upstream_connect_time\n 'upstream_connect_time': None,\n # http://nginx.org/en/docs/stream/ngx_stream_upstream_module.html#var_upstream_first_byte_time\n 'upstream_first_byte_time': None,\n # http://nginx.org/en/docs/http/ngx_http_upstream_module.html#var_upstream_header_time\n 'upstream_header_time': None,\n # http://nginx.org/en/docs/http/ngx_http_upstream_module.html#var_upstream_response_length\n 'upstream_response_length': None,\n # http://nginx.org/en/docs/http/ngx_http_upstream_module.html#var_upstream_response_time\n 'upstream_response_time': None,\n # http://nginx.org/en/docs/stream/ngx_stream_upstream_module.html#var_upstream_session_time\n 'upstream_session_time': None,\n # http://nginx.org/en/docs/http/ngx_http_upstream_module.html#var_upstream_status\n 'upstream_status': None\n}\n\n\ndef is_builtin(name):\n if isinstance(name, int):\n # Indexed variables can't be builtin\n return False\n for builtin in BUILTIN_VARIABLES:\n if builtin.endswith('_'):\n if name.startswith(builtin):\n return True\n elif name == builtin:\n return True\n return False\n\n\ndef builtin_var(name):\n for builtin, regexp in BUILTIN_VARIABLES.items():\n if builtin.endswith('_'):\n if not name.startswith(builtin):\n continue\n elif name != builtin:\n continue\n\n if regexp:\n return Variable(name=name, value=Regexp(regexp, strict=True, case_sensitive=False))\n return Variable(name=name, value='builtin', have_script=False)\n return None\n" }, { "path": "gixy/core/config.py", "content": "import gixy\n\n\nclass Config(object):\n def __init__(self,\n plugins=None,\n skips=None,\n severity=gixy.severity.UNSPECIFIED,\n output_format=None,\n output_file=None,\n allow_includes=True):\n self.severity = severity\n self.output_format = output_format\n self.output_file = output_file\n self.plugins = plugins\n self.skips = skips\n self.allow_includes = allow_includes\n self.plugins_options = {}\n\n def set_for(self, name, options):\n self.plugins_options[name] = options\n\n def get_for(self, name):\n if self.has_for(name):\n return self.plugins_options[name]\n return {}\n\n def has_for(self, name):\n return name in self.plugins_options\n" }, { "path": "gixy/core/context.py", "content": "import logging\nimport copy\n\nfrom gixy.core.utils import is_indexed_name\n\nLOG = logging.getLogger(__name__)\n\nCONTEXTS = []\n\n\ndef get_context():\n return CONTEXTS[-1]\n\n\ndef purge_context():\n del CONTEXTS[:]\n\n\ndef push_context(block):\n if len(CONTEXTS):\n context = copy.deepcopy(get_context())\n else:\n context = Context()\n context.set_block(block)\n CONTEXTS.append(context)\n return context\n\n\ndef pop_context():\n return CONTEXTS.pop()\n\n\nclass Context(object):\n def __init__(self):\n self.block = None\n self.variables = {\n 'index': {},\n 'name': {}\n }\n\n def set_block(self, directive):\n self.block = directive\n return self\n\n def clear_index_vars(self):\n self.variables['index'] = {}\n return self\n\n def add_var(self, name, var):\n if is_indexed_name(name):\n var_type = 'index'\n name = int(name)\n else:\n var_type = 'name'\n\n self.variables[var_type][name] = var\n return self\n\n def get_var(self, name):\n if is_indexed_name(name):\n var_type = 'index'\n name = int(name)\n else:\n var_type = 'name'\n\n result = None\n try:\n result = self.variables[var_type][name]\n except KeyError:\n if var_type == 'name':\n # Only named variables can be builtins\n import gixy.core.builtin_variables as builtins\n\n if builtins.is_builtin(name):\n result = builtins.builtin_var(name)\n\n if not result:\n LOG.info(\"Can't find variable '{0}'\".format(name))\n return result\n\n def __deepcopy__(self, memo):\n cls = self.__class__\n result = cls.__new__(cls)\n memo[id(self)] = result\n result.block = copy.copy(self.block)\n result.variables = {\n 'index': copy.copy(self.variables['index']),\n 'name': copy.copy(self.variables['name'])\n }\n return result\n" }, { "path": "gixy/core/exceptions.py", "content": "class InvalidConfiguration(Exception):\n pass\n" }, { "path": "gixy/core/issue.py", "content": "class Issue(object):\n def __init__(self, plugin, summary=None, description=None,\n severity=None, reason=None, help_url=None, directives=None):\n self.plugin = plugin\n self.summary = summary\n self.description = description\n self.severity = severity\n self.reason = reason\n self.help_url = help_url\n if not directives:\n self.directives = []\n elif not hasattr(directives, '__iter__'):\n self.directives = [directives]\n else:\n self.directives = directives\n" }, { "path": "gixy/core/manager.py", "content": "import os\nimport logging\n\nimport gixy\nfrom gixy.core.plugins_manager import PluginsManager\nfrom gixy.core.context import get_context, pop_context, push_context, purge_context\nfrom gixy.parser.nginx_parser import NginxParser\nfrom gixy.core.config import Config\n\nLOG = logging.getLogger(__name__)\n\n\nclass Manager(object):\n def __init__(self, config=None):\n self.root = None\n self.config = config or Config()\n self.auditor = PluginsManager(config=self.config)\n\n def audit(self, file_path, file_data, is_stdin=False):\n LOG.debug(\"Audit config file: {fname}\".format(fname=file_path))\n parser = NginxParser(\n cwd=os.path.dirname(file_path) if not is_stdin else '',\n allow_includes=self.config.allow_includes)\n self.root = parser.parse(content=file_data.read(), path_info=file_path)\n\n push_context(self.root)\n self._audit_recursive(self.root.children)\n\n @property\n def results(self):\n for plugin in self.auditor.plugins:\n if plugin.issues:\n yield plugin\n\n @property\n def stats(self):\n stats = dict.fromkeys(gixy.severity.ALL, 0)\n for plugin in self.auditor.plugins:\n base_severity = plugin.severity\n for issue in plugin.issues:\n # TODO(buglloc): encapsulate into Issue class?\n severity = issue.severity if issue.severity else base_severity\n stats[severity] += 1\n return stats\n\n def _audit_recursive(self, tree):\n for directive in tree:\n self._update_variables(directive)\n self.auditor.audit(directive)\n if directive.is_block:\n if directive.self_context:\n push_context(directive)\n self._audit_recursive(directive.children)\n if directive.self_context:\n pop_context()\n\n def _update_variables(self, directive):\n # TODO(buglloc): finish him!\n if not directive.provide_variables:\n return\n\n context = get_context()\n for var in directive.variables:\n if var.name == 0:\n # All regexps must clean indexed variables\n context.clear_index_vars()\n context.add_var(var.name, var)\n\n def __enter__(self):\n return self\n\n def __exit__(self, exc_type, exc_val, exc_tb):\n purge_context()\n" }, { "path": "gixy/core/plugins_manager.py", "content": "import os\n\nimport gixy\nfrom gixy.plugins.plugin import Plugin\n\n\nclass PluginsManager(object):\n def __init__(self, config=None):\n self.imported = False\n self.config = config\n self._plugins = []\n\n def import_plugins(self):\n if self.imported:\n return\n\n files_list = os.listdir(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'plugins'))\n for plugin_file in files_list:\n if not plugin_file.endswith('.py') or plugin_file.startswith('_'):\n continue\n __import__('gixy.plugins.' + os.path.splitext(plugin_file)[0], None, None, [''])\n\n self.imported = True\n\n def init_plugins(self):\n self.import_plugins()\n\n exclude = self.config.skips if self.config else None\n include = self.config.plugins if self.config else None\n severity = self.config.severity if self.config else None\n for plugin_cls in Plugin.__subclasses__():\n name = plugin_cls.__name__\n if include and name not in include:\n # Skip not needed plugins\n continue\n if exclude and name in exclude:\n # Skipped plugins\n continue\n if severity and not gixy.severity.is_acceptable(plugin_cls.severity, severity):\n # Skip plugin by severity level\n continue\n if self.config and self.config.has_for(name):\n options = self.config.get_for(name)\n else:\n options = plugin_cls.options\n self._plugins.append(plugin_cls(options))\n\n @property\n def plugins(self):\n if not self._plugins:\n self.init_plugins()\n return self._plugins\n\n @property\n def plugins_classes(self):\n self.import_plugins()\n return Plugin.__subclasses__()\n\n def get_plugins_descriptions(self):\n return map(lambda a: a.name, self.plugins)\n\n def audit(self, directive):\n for plugin in self.plugins:\n if plugin.directives and directive.name not in plugin.directives:\n continue\n plugin.audit(directive)\n\n def issues(self):\n result = []\n for plugin in self.plugins:\n if not plugin.issues:\n continue\n result.extend(plugin.issues)\n return result\n" }, { "path": "gixy/core/regexp.py", "content": "import six\nimport logging\nimport re\nimport random\nimport itertools\nfrom cached_property import cached_property\n\nimport gixy.core.sre_parse.sre_parse as sre_parse\n\nLOG = logging.getLogger(__name__)\n\n\ndef _build_reverse_list(original):\n result = []\n for c in range(1, 126):\n c = six.unichr(c)\n if c not in original:\n result.append(c)\n return frozenset(result)\n\n\nFIX_NAMED_GROUPS_RE = re.compile(r\"(?|')\")\n\nCATEGORIES = {\n # TODO(buglloc): unicode?\n sre_parse.CATEGORY_SPACE: sre_parse.WHITESPACE,\n sre_parse.CATEGORY_NOT_SPACE: _build_reverse_list(sre_parse.WHITESPACE),\n sre_parse.CATEGORY_DIGIT: sre_parse.DIGITS,\n sre_parse.CATEGORY_NOT_DIGIT: _build_reverse_list(sre_parse.DIGITS),\n sre_parse.CATEGORY_WORD: frozenset('abcdefghijklmnopqrstuvwxyz'\n 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'\n '0123456789_'),\n sre_parse.CATEGORY_NOT_WORD: _build_reverse_list(frozenset('abcdefghijklmnopqrstuvwxyz'\n 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'\n '0123456789_')),\n sre_parse.CATEGORY_LINEBREAK: frozenset('\\n'),\n sre_parse.CATEGORY_NOT_LINEBREAK: _build_reverse_list(frozenset('\\n')),\n 'ANY': [six.unichr(x) for x in range(1, 127) if x != 10]\n}\n\nCATEGORIES_NAMES = {\n sre_parse.CATEGORY_DIGIT: r'\\d',\n sre_parse.CATEGORY_NOT_DIGIT: r'\\D',\n sre_parse.CATEGORY_SPACE: r'\\s',\n sre_parse.CATEGORY_NOT_SPACE: r'\\S',\n sre_parse.CATEGORY_WORD: r'\\w',\n sre_parse.CATEGORY_NOT_WORD: r'\\W',\n}\n\n\ndef extract_groups(parsed, top=True):\n result = {}\n if top:\n result[0] = parsed\n for token in parsed:\n if not token:\n # Skip empty tokens\n pass\n elif token[0] == sre_parse.SUBPATTERN:\n if isinstance(token[1][0], int):\n # Captured group index can't be a string. E.g. for pattern \"(?:la)\" group name is \"None\"\n result[token[1][0]] = token[1][1]\n result.update(extract_groups(token[1][1], False))\n elif token[0] == sre_parse.MIN_REPEAT:\n result.update(extract_groups(token[1][2], False))\n elif token[0] == sre_parse.MAX_REPEAT:\n result.update(extract_groups(token[1][2], False))\n elif token[0] == sre_parse.BRANCH:\n result.update(extract_groups(token[1][1], False))\n elif token[0] == sre_parse.SUBPATTERN:\n result.update(extract_groups(token[1][1], False))\n elif token[0] == sre_parse.IN:\n result.update(extract_groups(token[1], False))\n elif isinstance(token, sre_parse.SubPattern):\n result.update(extract_groups(token, False))\n return result\n\n\ndef _gen_combinator(variants, _merge=True):\n if not hasattr(variants, '__iter__'):\n return [variants] if variants is not None else []\n\n res = []\n need_product = False\n for var in variants:\n if isinstance(var, list):\n sol = _gen_combinator(var, _merge=False)\n res.append(sol)\n need_product = True\n elif var is not None:\n res.append(var)\n\n if need_product:\n producted = itertools.product(*res)\n if _merge:\n # TODO(buglloc): ??!\n return list(six.moves.map(_merge_variants, producted))\n return producted\n elif _merge:\n return list(six.moves.map(_merge_variants, [res]))\n return res\n\n\ndef _merge_variants(variants):\n result = []\n for var in variants:\n if isinstance(var, tuple):\n result.append(_merge_variants(var))\n else:\n result.append(var)\n return ''.join(result)\n\n\nclass Token(object):\n type = None\n\n def __init__(self, token, parent, regexp):\n self.token = token\n self.childs = None\n self.parent = parent\n self.regexp = regexp\n self._parse()\n\n def parse(self):\n pass\n\n def _parse(self):\n pass\n\n def _parse_childs(self, childs):\n self.childs = parse(childs, self, regexp=self.regexp)\n\n def _get_group(self, gid):\n return self.regexp.group(gid)\n\n def _reg_group(self, gid):\n self.regexp.reg_group(gid, self)\n\n def can_contain(self, char, skip_literal=True):\n raise NotImplementedError('can_contain must be implemented')\n\n def can_startswith(self, char, strict=False):\n return self.can_contain(char, skip_literal=False)\n\n def must_contain(self, char):\n raise NotImplementedError('must_contain must be implemented')\n\n def must_startswith(self, char, strict=False):\n return self.must_contain(char)\n\n def generate(self, context):\n raise NotImplementedError('generate must be implemented')\n\n def __str__(self):\n raise NotImplementedError('__str__ must be implemented')\n\n\nclass AnyToken(Token):\n type = sre_parse.ANY\n\n def can_contain(self, char, skip_literal=True):\n return char in CATEGORIES['ANY']\n\n def must_contain(self, char, skip_literal=True):\n # Char may not be present in ANY token\n return False\n\n def generate(self, context):\n if context.char in CATEGORIES['ANY']:\n return context.char\n return 'a'\n\n def __str__(self):\n return '.'\n\n\nclass LiteralToken(Token):\n type = sre_parse.LITERAL\n\n def _parse(self):\n self.char = six.unichr(self.token[1])\n\n def can_contain(self, char, skip_literal=True):\n if skip_literal:\n return False\n return self.char == char\n\n def must_contain(self, char, skip_literal=True):\n return self.char == char\n\n def generate(self, context):\n return self.char\n\n def __str__(self):\n return re.escape(self.char)\n\n\nclass NotLiteralToken(Token):\n type = sre_parse.NOT_LITERAL\n\n def _parse(self):\n self.char = six.unichr(self.token[1])\n self.gen_char_list = list(_build_reverse_list(frozenset(self.char)))\n\n def can_contain(self, char, skip_literal=True):\n return self.char != char\n\n def must_contain(self, char):\n # Any char MAY not be present in NotLiteral, e.g.: \"a\" not present in \"[^b]\"\n return False\n\n def generate(self, context):\n if self.can_contain(context.char):\n return context.char\n\n return random.choice(self.gen_char_list)\n\n def __str__(self):\n return '[^{char}]'.format(char=self.char)\n\n\nclass RangeToken(Token):\n type = sre_parse.RANGE\n\n def _parse(self):\n self.left_code = self.token[1][0]\n self.right_code = self.token[1][1]\n self.left = six.unichr(self.left_code)\n self.right = six.unichr(self.right_code)\n\n def can_contain(self, char, skip_literal=True):\n return self.left <= char <= self.right\n\n def must_contain(self, char, skip_literal=True):\n return self.left == char == self.right\n\n def generate(self, context):\n if self.can_contain(context.char):\n return context.char\n\n return six.unichr(random.randint(self.token[1][0], self.token[1][1]))\n\n def __str__(self):\n return '{left}-{right}'.format(left=self.left, right=self.right)\n\n\nclass CategoryToken(Token):\n type = sre_parse.CATEGORY\n\n def _parse(self):\n self.char_list = CATEGORIES.get(self.token[1], [''])\n\n def can_contain(self, char, skip_literal=True):\n return char in self.char_list\n\n def must_contain(self, char, skip_literal=True):\n return frozenset([char]) == self.char_list\n\n def generate(self, context):\n if self.can_contain(context.char):\n return context.char\n\n for c in self.char_list:\n return c\n\n def __str__(self):\n return CATEGORIES_NAMES.get(self.token[1], '\\\\C')\n\n\nclass MinRepeatToken(Token):\n type = sre_parse.MIN_REPEAT\n\n def _parse(self):\n self._parse_childs(self.token[1][2])\n self.min = self.token[1][0]\n self.max = self.token[1][1]\n\n def can_contain(self, char, skip_literal=True):\n if self.max == 0:\n # [a-z]{0}\n return False\n for child in self.childs:\n if child.can_contain(char, skip_literal=skip_literal):\n return True\n return False\n\n def must_contain(self, char):\n if self.max == 0:\n # [a-z]{0}\n return False\n if self.min == 0:\n # [a-z]*?\n return False\n for child in self.childs:\n if child.must_contain(char):\n return True\n return False\n\n def can_startswith(self, char, strict=False):\n if self.max == 0:\n # [a-z]{0}\n if self.childs[0].can_startswith(char, strict):\n return False\n return None\n return self.childs[0].can_startswith(char, strict)\n\n def must_startswith(self, char, strict=False):\n if self.min == 0:\n # [a-z]*?\n return None\n if self.max == 0:\n # [a-z]{0}\n return None\n return self.childs[0].must_startswith(char, strict=strict)\n\n def generate(self, context):\n res = []\n if self.min == 0:\n # [a-z]*\n res.append('')\n if self.max == 0:\n # [a-z]{0}\n return res\n\n for child in self.childs:\n res.extend(child.generate(context))\n\n result = []\n repeat = self.max if self.max <= context.max_repeat else context.max_repeat\n for val in _gen_combinator([res]):\n result.append(val * repeat)\n return result\n\n def __str__(self):\n childs = ''.join(str(x) for x in self.childs)\n if self.min == self.max:\n return '{childs}{{{count}}}?'.format(childs=childs, count=self.min)\n if self.min == 0 and self.max == 1:\n return '{childs}?'.format(childs=childs)\n if self.min == 0 and self.max == sre_parse.MAXREPEAT:\n return '{childs}*?'.format(childs=childs)\n if self.min == 1 and self.max == sre_parse.MAXREPEAT:\n return '{childs}+?'.format(childs=childs)\n return '{childs}{{{min},{max}}}?'.format(childs=childs, min=self.min, max=self.max)\n\n\nclass MaxRepeatToken(Token):\n type = sre_parse.MAX_REPEAT\n\n def _parse(self):\n self._parse_childs(self.token[1][2])\n self.min = self.token[1][0]\n self.max = self.token[1][1]\n\n def can_contain(self, char, skip_literal=True):\n if self.max == 0:\n # [a-z]{0}\n return False\n for child in self.childs:\n if child.can_contain(char, skip_literal=skip_literal):\n return True\n return False\n\n def must_contain(self, char):\n if self.max == 0:\n # [a-z]{0}\n return False\n if self.min == 0:\n # [a-z]?\n return False\n for child in self.childs:\n if child.must_contain(char):\n return True\n return False\n\n def can_startswith(self, char, strict=False):\n if self.max == 0:\n # [a-z]{0}\n if self.childs[0].can_startswith(char, strict):\n return False\n return None\n return self.childs[0].can_startswith(char, strict)\n\n def must_startswith(self, char, strict=False):\n if self.min == 0:\n # [a-z]*\n return None\n if self.max == 0:\n # [a-z]{0}\n return None\n return self.childs[0].must_startswith(char, strict=strict)\n\n def generate(self, context):\n res = []\n if self.min == 0:\n # [a-z]*\n res.append('')\n if self.max == 0:\n # [a-z]{0}\n return res\n\n for child in self.childs:\n res.extend(child.generate(context))\n\n result = []\n repeat = self.max if self.max <= context.max_repeat else context.max_repeat\n for val in _gen_combinator([res]):\n result.append(val * repeat)\n return result\n\n def __str__(self):\n childs = ''.join(str(x) for x in self.childs)\n if self.min == self.max:\n return '{childs}{{{count}}}'.format(childs=childs, count=self.min)\n if self.min == 0 and self.max == 1:\n return '{childs}?'.format(childs=childs)\n if self.min == 0 and self.max == sre_parse.MAXREPEAT:\n return '{childs}*'.format(childs=childs)\n if self.min == 1 and self.max == sre_parse.MAXREPEAT:\n return '{childs}+'.format(childs=childs)\n return '{childs}{{{min},{max}}}'.format(childs=childs, min=self.min, max=self.max)\n\n\nclass BranchToken(Token):\n type = sre_parse.BRANCH\n\n def _parse(self):\n self.childs = []\n for token in self.token[1][1]:\n if not token:\n self.childs.append(EmptyToken(token=token, parent=self.parent, regexp=self.regexp))\n elif isinstance(token, sre_parse.SubPattern):\n self.childs.append(InternalSubpatternToken(token=token, parent=self.parent, regexp=self.regexp))\n else:\n raise RuntimeError('Unexpected token {0} in branch'.format(token))\n\n def can_contain(self, char, skip_literal=True):\n for child in self.childs:\n if child.can_contain(char, skip_literal=skip_literal):\n return True\n return False\n\n def must_contain(self, char):\n return all(child.must_contain(char) for child in self.childs)\n\n def can_startswith(self, char, strict=False):\n return any(x.can_startswith(char, strict) for x in self.childs)\n\n def must_startswith(self, char, strict=False):\n return all(x.must_startswith(char, strict) for x in self.childs)\n\n def generate(self, context):\n res = []\n for child in self.childs:\n values = child.generate(context)\n if isinstance(values, list):\n res.extend(child.generate(context))\n else:\n res.append(values)\n\n return res\n\n def __str__(self):\n return '(?:{0})'.format('|'.join(str(x) for x in self.childs))\n\n\nclass SubpatternToken(Token):\n type = sre_parse.SUBPATTERN\n\n def _parse(self):\n self._parse_childs(self.token[1][1])\n self.group = self.token[1][0]\n if isinstance(self.group, int):\n # Captured group index can't be a string. E.g. for pattern \"(?:la)\" group name is \"None\"\n self._reg_group(self.group)\n\n def can_contain(self, char, skip_literal=True):\n for child in self.childs:\n if child.can_contain(char, skip_literal=skip_literal):\n return True\n return False\n\n def must_contain(self, char):\n for child in self.childs:\n if child.must_contain(char):\n return True\n return False\n\n def can_startswith(self, char, strict=False):\n if isinstance(self.childs[0], AtToken):\n if len(self.childs) > 1:\n for child in self.childs[1:]:\n can = child.can_startswith(char, strict)\n if can is None:\n continue\n return can\n return False\n elif not strict and not isinstance(self.childs[0], (SubpatternToken, InternalSubpatternToken)):\n # Not strict regexp w/o ^ can starts with any character\n return char in CATEGORIES['ANY']\n\n for child in self.childs:\n can = child.can_startswith(char, strict)\n if can is None:\n continue\n return can\n return None\n\n def must_startswith(self, char, strict=False):\n if isinstance(self.childs[0], AtToken):\n if len(self.childs) > 1:\n for child in self.childs[1:]:\n must = child.must_startswith(char, strict=True)\n if must is None:\n continue\n return must\n return False\n elif not strict and not isinstance(self.childs[0], (SubpatternToken, InternalSubpatternToken)):\n # Not strict regexp w/o ^ MAY NOT starts with any character\n return False\n\n for child in self.childs:\n must = child.must_startswith(char, strict=strict)\n if must is None:\n continue\n return must\n return None\n\n def generate(self, context):\n res = []\n for child in self.childs:\n res.append(child.generate(context))\n\n return _gen_combinator(res)\n\n def __str__(self):\n childs = ''.join(str(x) for x in self.childs)\n if self.group is None:\n return '(?:{childs})'.format(childs=childs)\n return '({childs})'.format(childs=childs)\n\n\nclass InternalSubpatternToken(Token):\n type = sre_parse.SUBPATTERN\n\n def _parse(self):\n self._parse_childs(self.token)\n self.group = None\n\n def can_contain(self, char, skip_literal=True):\n for child in self.childs:\n if child.can_contain(char, skip_literal=skip_literal):\n return True\n return False\n\n def must_contain(self, char):\n for child in self.childs:\n if child.must_contain(char):\n return True\n return False\n\n def can_startswith(self, char, strict=False):\n if isinstance(self.childs[0], AtToken):\n if len(self.childs) > 1:\n for child in self.childs[1:]:\n can = child.can_startswith(char, strict)\n if can is None:\n continue\n return can\n return False\n elif not strict and not isinstance(self.childs[0], (SubpatternToken, InternalSubpatternToken)):\n # Not strict regexp w/o ^ can starts with any character\n return char in CATEGORIES['ANY']\n\n for child in self.childs:\n can = child.can_startswith(char, strict)\n if can is None:\n continue\n return can\n return None\n\n def must_startswith(self, char, strict=False):\n if isinstance(self.childs[0], AtToken):\n if len(self.childs) > 1:\n for child in self.childs[1:]:\n must = child.must_startswith(char, strict=True)\n if must is None:\n continue\n return must\n return False\n elif not strict and not isinstance(self.childs[0], (SubpatternToken, InternalSubpatternToken)):\n # Not strict regexp w/o ^ MAY NOT starts with any character\n return False\n\n for child in self.childs:\n must = child.must_startswith(char, strict=strict)\n if must is None:\n continue\n return must\n return None\n\n def generate(self, context):\n res = []\n for child in self.childs:\n res.append(child.generate(context))\n\n return _gen_combinator(res)\n\n def __str__(self):\n return ''.join(str(x) for x in self.childs)\n\n\nclass InToken(Token):\n type = sre_parse.IN\n\n def _parse(self):\n self.childs = parse(self.token[1], self)\n\n def can_contain(self, char, skip_literal=True):\n can = False\n negative = False\n for child in self.childs:\n if isinstance(child, NegateToken):\n negative = True\n else:\n can = child.can_contain(char, skip_literal=False)\n\n if can:\n break\n if can and not negative:\n # a in [a-z]\n return True\n if not can and negative:\n # a in [^b-z]\n return True\n return False\n\n def must_contain(self, char):\n # Any character MAY not be present in IN\n return False\n\n def _generate_positive(self, context):\n result = []\n for child in self.childs:\n if isinstance(child, (NegateToken, EmptyToken)):\n pass\n else:\n result.append(child.generate(context=context))\n return result\n\n def _generate_negative(self, context):\n blacklisted = set()\n # TODO(buglloc): move chars list into the tokens?\n for child in self.childs:\n if isinstance(child, (NegateToken, EmptyToken)):\n pass\n elif isinstance(child, LiteralToken):\n blacklisted.add(child.char)\n elif isinstance(child, RangeToken):\n blacklisted.update(six.unichr(c) for c in six.moves.range(child.left_code, child.right_code + 1))\n elif isinstance(child, CategoryToken):\n blacklisted.update(child.char_list)\n else:\n LOG.info('Unexpected child \"{0!r}\"'.format(child))\n\n for char in _build_reverse_list(set()):\n if char not in blacklisted:\n return char\n\n def generate(self, context):\n if self.can_contain(context.char, skip_literal=False):\n return context.char\n\n is_negative = self.childs and isinstance(self.childs[0], NegateToken)\n if is_negative:\n # [^a-z]\n return self._generate_negative(context)\n # [a-z]\n return self._generate_positive(context)\n\n def __str__(self):\n return '[{childs}]'.format(childs=''.join(str(x) for x in self.childs))\n\n\nclass AtToken(Token):\n type = sre_parse.AT\n\n def _parse(self):\n self.begin = self.token[1] == sre_parse.AT_BEGINNING\n self.end = self.token[1] == sre_parse.AT_END\n\n def can_contain(self, char, skip_literal=True):\n return False\n\n def must_contain(self, char):\n return False\n\n def generate(self, context):\n if context.anchored:\n if self.begin:\n return '^'\n if self.end:\n return '$'\n return None\n\n def __str__(self):\n if self.begin:\n return '^'\n if self.end:\n return '$'\n LOG.warn('unexpected AT token: %s', self.token)\n\n\nclass NegateToken(Token):\n type = sre_parse.NEGATE\n\n def can_contain(self, char, skip_literal=True):\n return False\n\n def must_contain(self, char):\n return False\n\n def can_startswith(self, char, strict=False):\n return None\n\n def must_startswith(self, char, strict=False):\n return None\n\n def generate(self, context):\n return None\n\n def __str__(self):\n return '^'\n\n\nclass GroupRefToken(Token):\n type = sre_parse.GROUPREF\n\n def _parse(self):\n self.id = self.token[1]\n self.group = self._get_group(self.id)\n\n def can_contain(self, char, skip_literal=True):\n return self.group.can_contain(char, skip_literal=skip_literal)\n\n def must_contain(self, char):\n return self.group.must_contain(char)\n\n def can_startswith(self, char, strict=False):\n return self.group.can_startswith(char, strict=strict)\n\n def must_startswith(self, char, strict=False):\n return self.group.must_startswith(char, strict=strict)\n\n def generate(self, context):\n return self.group.generate(context)\n\n def __str__(self):\n return '\\\\\\\\{0}'.format(self.id)\n\n\nclass AssertToken(Token):\n type = sre_parse.ASSERT\n\n def can_contain(self, char, skip_literal=True):\n # TODO(buglloc): Do it!\n return False\n\n def must_contain(self, char):\n # TODO(buglloc): Do it!\n return False\n\n def can_startswith(self, char, strict=False):\n return None\n\n def must_startswith(self, char, strict=False):\n return None\n\n\nclass AssertNotToken(Token):\n type = sre_parse.ASSERT_NOT\n\n def can_contain(self, char, skip_literal=True):\n # TODO(buglloc): Do it!\n return False\n\n def must_contain(self, char):\n # TODO(buglloc): Do it!\n return False\n\n def can_startswith(self, char, strict=False):\n return None\n\n def must_startswith(self, char, strict=False):\n return None\n\n\nclass EmptyToken(Token):\n type = None\n\n def can_contain(self, char, skip_literal=True):\n return False\n\n def must_contain(self, char):\n # TODO(buglloc): Do it!\n return False\n\n def can_startswith(self, char, strict=False):\n return None\n\n def must_startswith(self, char, strict=False):\n return None\n\n def generate(self, context):\n return ''\n\n def __str__(self):\n return ''\n\n\ndef parse(sre_obj, parent=None, regexp=None):\n result = []\n for token in sre_obj:\n if not token:\n result.append(EmptyToken(token=token, parent=parent, regexp=regexp))\n elif token[0] == sre_parse.ANY:\n result.append(AnyToken(token=token, parent=parent, regexp=regexp))\n elif token[0] == sre_parse.LITERAL:\n result.append(LiteralToken(token=token, parent=parent, regexp=regexp))\n elif token[0] == sre_parse.NOT_LITERAL:\n result.append(NotLiteralToken(token=token, parent=parent, regexp=regexp))\n elif token[0] == sre_parse.RANGE:\n result.append(RangeToken(token=token, parent=parent, regexp=regexp))\n elif token[0] == sre_parse.CATEGORY:\n result.append(CategoryToken(token=token, parent=parent, regexp=regexp))\n elif token[0] == sre_parse.MIN_REPEAT:\n result.append(MinRepeatToken(token=token, parent=parent, regexp=regexp))\n elif token[0] == sre_parse.MAX_REPEAT:\n result.append(MaxRepeatToken(token=token, parent=parent, regexp=regexp))\n elif token[0] == sre_parse.BRANCH:\n result.append(BranchToken(token=token, parent=parent, regexp=regexp))\n elif token[0] == sre_parse.SUBPATTERN:\n result.append(SubpatternToken(token=token, parent=parent, regexp=regexp))\n elif token[0] == sre_parse.IN:\n result.append(InToken(token=token, parent=parent, regexp=regexp))\n elif token[0] == sre_parse.NEGATE:\n result.append(NegateToken(token=token, parent=parent, regexp=regexp))\n elif token[0] == sre_parse.AT:\n result.append(AtToken(token=token, parent=parent, regexp=regexp))\n elif token[0] == sre_parse.GROUPREF:\n result.append(GroupRefToken(token=token, parent=parent, regexp=regexp))\n elif token[0] == sre_parse.ASSERT:\n pass # TODO(buglloc): Do it!\n elif token[0] == sre_parse.ASSERT_NOT:\n pass # TODO(buglloc): Do it!\n else:\n LOG.info('Unexpected token \"{0}\"'.format(token[0]))\n\n return result\n\n\nclass GenerationContext(object):\n def __init__(self, char, max_repeat=5, strict=False, anchored=True):\n self.char = char\n self.max_repeat = max_repeat\n self.strict = strict\n self.anchored = anchored\n\n\nclass Regexp(object):\n def __init__(self, source, strict=False, case_sensitive=True, _root=None, _parsed=None):\n \"\"\"\n Gixy Regexp class, parse and provide helpers to work with it.\n\n :param str source: regexp, e.g. ^foo$.\n :param bool strict: anchored or not.\n :param bool case_sensitive: case sensitive or not.\n \"\"\"\n\n self.source = source\n self.strict = strict\n self.case_sensitive = case_sensitive\n self._root = _root\n self._parsed = _parsed\n self._groups = {}\n\n def can_startswith(self, char):\n \"\"\"\n Checks if regex can starts with the specified char.\n Example:\n Regexp('[a-z][0-9]').can_startswith('s') -> True\n Regexp('[a-z][0-9]').can_startswith('0') -> True\n Regexp('^[a-z][0-9]').can_startswith('0') -> False\n Regexp('[a-z][0-9]', strict=True).can_startswith('0') -> False\n\n :param str char: character to test.\n :return bool: True if regex can starts with the specified char, False otherwise.\n \"\"\"\n\n return self.root.can_startswith(\n char=char if self.case_sensitive else char.lower(),\n strict=self.strict\n )\n\n def can_contain(self, char, skip_literal=True):\n \"\"\"\n Checks if regex can contain the specified char.\n Example:\n Regexp('[a-z][0-9]').can_contain('s') -> True\n Regexp('[a-z][0-9]').can_contain('0') -> True\n Regexp('[a-z][0-9]').can_contain('/') -> False\n Regexp('[a-z][0-9]/').can_contain('/') -> False\n Regexp('[a-z][0-9]/').can_contain('/', skip_literal=False) -> True\n\n :param str char: character to test.\n :param bool skip_literal: skip literal tokens.\n :return bool: True if regex can contain the specified char, False otherwise.\n \"\"\"\n\n return self.root.can_contain(\n char=char if self.case_sensitive else char.lower(),\n skip_literal=skip_literal\n )\n\n def must_startswith(self, char):\n \"\"\"\n Checks if regex MUST starts with the specified char.\n Example:\n Regexp('[a-z][0-9]').must_startswith('s') -> False\n Regexp('s[a-z]').must_startswith('s') -> False\n Regexp('^s[a-z]').must_startswith('s') -> True\n Regexp('s[a-z]', strict=True).must_startswith('s') -> True\n\n :param str char: character to test.\n :return bool: True if regex must starts with the specified char, False otherwise.\n \"\"\"\n\n return self.root.must_startswith(\n char=char if self.case_sensitive else char.lower(),\n strict=self.strict\n )\n\n def must_contain(self, char):\n \"\"\"\n Checks if regex MUST contain the specified char.\n Example:\n Regexp('[a-z][0-9]').must_contain('s') -> False\n Regexp('[a-z][0-9]s').must_contain('s') -> True\n\n :param str char: character to test.\n :return bool: True if regex MUST contain the specified char, False otherwise.\n \"\"\"\n\n return self.root.must_contain(\n char=char if self.case_sensitive else char.lower()\n )\n\n def generate(self, char, anchored=False, max_repeat=5):\n \"\"\"\n Generate values that match regex.\n Example:\n Regexp('.a?').generate('s') -> ['s', 'sa']\n Regexp('(?:^http|https)://.').generate('s') -> ['http://s', 'https://s']\n Regexp('(?:^http|https)://.').generate('s', anchored=True) -> ['^http://s', 'https://s']\n\n\n :param str char: \"dangerous\" character, generator try to place it wherever possible.\n :param bool anchored: place anchors in generated values.\n :param int max_repeat: maximum count of repeated group (e.g. \"a+\" provides \"aaaaa\").\n :return list of str: True if regex can contain the specified char, False otherwise.\n \"\"\"\n\n context = GenerationContext(char, anchored=anchored, max_repeat=max_repeat)\n for val in self.root.generate(context=context):\n if anchored and self.strict and not val.startswith('^'):\n yield '^' + val\n else:\n yield val\n\n def group(self, name):\n \"\"\"\n Returns group by specified name.\n\n :param name: name of the group.\n :return Regexp: Regexp object for this group.\n \"\"\"\n\n if name in self.groups:\n return self.groups[name]\n return Regexp('')\n\n def reg_group(self, gid, token):\n self._groups[gid] = token\n\n def get_group(self, gid):\n return self._groups[gid]\n\n @cached_property\n def groups(self):\n # self.root.parse()\n result = {}\n # for name, token in self._groups.items():\n # result[name] = Regexp(str(self), root=token, strict=True, case_sensitive=self.case_sensitive)\n for name, parsed in extract_groups(self.parsed).items():\n result[name] = Regexp('compiled', _parsed=parsed, strict=True, case_sensitive=self.case_sensitive)\n for name, group in self.parsed.pattern.groupdict.items():\n result[name] = result[group]\n return result\n\n @property\n def root(self):\n if self._root:\n return self._root\n\n self._root = InternalSubpatternToken(self.parsed, parent=None, regexp=self)\n self._groups[0] = self._root\n return self._root\n\n @property\n def parsed(self):\n # TODO(buglloc): Ugly hack!\n if self._parsed:\n return self._parsed\n\n try:\n self._parsed = sre_parse.parse(FIX_NAMED_GROUPS_RE.sub('(?P<\\\\1>', self.source))\n except sre_parse.error as e:\n LOG.fatal('Failed to parse regex: %s (%s)', self.source, str(e))\n raise e\n\n return self._parsed\n\n def __str__(self):\n return str(self.root)\n" }, { "path": "gixy/core/severity.py", "content": "UNSPECIFIED = 'UNSPECIFIED'\nLOW = 'LOW'\nMEDIUM = 'MEDIUM'\nHIGH = 'HIGH'\nALL = [UNSPECIFIED, LOW, MEDIUM, HIGH]\n\n\ndef is_acceptable(current_severity, min_severity):\n return ALL.index(current_severity) >= ALL.index(min_severity)\n" }, { "path": "gixy/core/sre_parse/__init__.py", "content": "" }, { "path": "gixy/core/sre_parse/sre_constants.py", "content": "# flake8: noqa\n\n#\n# Secret Labs' Regular Expression Engine\n#\n# various symbols used by the regular expression engine.\n# run this script to update the _sre include files!\n#\n# Copyright (c) 1998-2001 by Secret Labs AB. All rights reserved.\n#\n# See the sre.py file for information on usage and redistribution.\n#\n\n\"\"\"Internal support module for sre\"\"\"\n\n# update when constants are added or removed\n\nMAGIC = 20031017\n\ntry:\n from _sre import MAXREPEAT\nexcept ImportError:\n import _sre\n\n MAXREPEAT = _sre.MAXREPEAT = 65535\n\n\n# SRE standard exception (access as sre.error)\n# should this really be here?\n\nclass error(Exception):\n pass\n\n\n# operators\n\nFAILURE = \"failure\"\nSUCCESS = \"success\"\n\nANY = \"any\"\nANY_ALL = \"any_all\"\nASSERT = \"assert\"\nASSERT_NOT = \"assert_not\"\nAT = \"at\"\nBIGCHARSET = \"bigcharset\"\nBRANCH = \"branch\"\nCALL = \"call\"\nCATEGORY = \"category\"\nCHARSET = \"charset\"\nGROUPREF = \"groupref\"\nGROUPREF_IGNORE = \"groupref_ignore\"\nGROUPREF_EXISTS = \"groupref_exists\"\nIN = \"in\"\nIN_IGNORE = \"in_ignore\"\nINFO = \"info\"\nJUMP = \"jump\"\nLITERAL = \"literal\"\nLITERAL_IGNORE = \"literal_ignore\"\nMARK = \"mark\"\nMAX_REPEAT = \"max_repeat\"\nMAX_UNTIL = \"max_until\"\nMIN_REPEAT = \"min_repeat\"\nMIN_UNTIL = \"min_until\"\nNEGATE = \"negate\"\nNOT_LITERAL = \"not_literal\"\nNOT_LITERAL_IGNORE = \"not_literal_ignore\"\nRANGE = \"range\"\nREPEAT = \"repeat\"\nREPEAT_ONE = \"repeat_one\"\nSUBPATTERN = \"subpattern\"\nMIN_REPEAT_ONE = \"min_repeat_one\"\n\n# positions\nAT_BEGINNING = \"at_beginning\"\nAT_BEGINNING_LINE = \"at_beginning_line\"\nAT_BEGINNING_STRING = \"at_beginning_string\"\nAT_BOUNDARY = \"at_boundary\"\nAT_NON_BOUNDARY = \"at_non_boundary\"\nAT_END = \"at_end\"\nAT_END_LINE = \"at_end_line\"\nAT_END_STRING = \"at_end_string\"\nAT_LOC_BOUNDARY = \"at_loc_boundary\"\nAT_LOC_NON_BOUNDARY = \"at_loc_non_boundary\"\nAT_UNI_BOUNDARY = \"at_uni_boundary\"\nAT_UNI_NON_BOUNDARY = \"at_uni_non_boundary\"\n\n# categories\nCATEGORY_DIGIT = \"category_digit\"\nCATEGORY_NOT_DIGIT = \"category_not_digit\"\nCATEGORY_SPACE = \"category_space\"\nCATEGORY_NOT_SPACE = \"category_not_space\"\nCATEGORY_WORD = \"category_word\"\nCATEGORY_NOT_WORD = \"category_not_word\"\nCATEGORY_LINEBREAK = \"category_linebreak\"\nCATEGORY_NOT_LINEBREAK = \"category_not_linebreak\"\nCATEGORY_LOC_WORD = \"category_loc_word\"\nCATEGORY_LOC_NOT_WORD = \"category_loc_not_word\"\nCATEGORY_UNI_DIGIT = \"category_uni_digit\"\nCATEGORY_UNI_NOT_DIGIT = \"category_uni_not_digit\"\nCATEGORY_UNI_SPACE = \"category_uni_space\"\nCATEGORY_UNI_NOT_SPACE = \"category_uni_not_space\"\nCATEGORY_UNI_WORD = \"category_uni_word\"\nCATEGORY_UNI_NOT_WORD = \"category_uni_not_word\"\nCATEGORY_UNI_LINEBREAK = \"category_uni_linebreak\"\nCATEGORY_UNI_NOT_LINEBREAK = \"category_uni_not_linebreak\"\n\nOPCODES = [\n\n # failure=0 success=1 (just because it looks better that way :-)\n FAILURE, SUCCESS,\n\n ANY, ANY_ALL,\n ASSERT, ASSERT_NOT,\n AT,\n BRANCH,\n CALL,\n CATEGORY,\n CHARSET, BIGCHARSET,\n GROUPREF, GROUPREF_EXISTS, GROUPREF_IGNORE,\n IN, IN_IGNORE,\n INFO,\n JUMP,\n LITERAL, LITERAL_IGNORE,\n MARK,\n MAX_UNTIL,\n MIN_UNTIL,\n NOT_LITERAL, NOT_LITERAL_IGNORE,\n NEGATE,\n RANGE,\n REPEAT,\n REPEAT_ONE,\n SUBPATTERN,\n MIN_REPEAT_ONE\n\n]\n\nATCODES = [\n AT_BEGINNING, AT_BEGINNING_LINE, AT_BEGINNING_STRING, AT_BOUNDARY,\n AT_NON_BOUNDARY, AT_END, AT_END_LINE, AT_END_STRING,\n AT_LOC_BOUNDARY, AT_LOC_NON_BOUNDARY, AT_UNI_BOUNDARY,\n AT_UNI_NON_BOUNDARY\n]\n\nCHCODES = [\n CATEGORY_DIGIT, CATEGORY_NOT_DIGIT, CATEGORY_SPACE,\n CATEGORY_NOT_SPACE, CATEGORY_WORD, CATEGORY_NOT_WORD,\n CATEGORY_LINEBREAK, CATEGORY_NOT_LINEBREAK, CATEGORY_LOC_WORD,\n CATEGORY_LOC_NOT_WORD, CATEGORY_UNI_DIGIT, CATEGORY_UNI_NOT_DIGIT,\n CATEGORY_UNI_SPACE, CATEGORY_UNI_NOT_SPACE, CATEGORY_UNI_WORD,\n CATEGORY_UNI_NOT_WORD, CATEGORY_UNI_LINEBREAK,\n CATEGORY_UNI_NOT_LINEBREAK\n]\n\n\ndef makedict(list):\n d = {}\n i = 0\n for item in list:\n d[item] = i\n i = i + 1\n return d\n\n\nOPCODES = makedict(OPCODES)\nATCODES = makedict(ATCODES)\nCHCODES = makedict(CHCODES)\n\n# replacement operations for \"ignore case\" mode\nOP_IGNORE = {\n GROUPREF: GROUPREF_IGNORE,\n IN: IN_IGNORE,\n LITERAL: LITERAL_IGNORE,\n NOT_LITERAL: NOT_LITERAL_IGNORE\n}\n\nAT_MULTILINE = {\n AT_BEGINNING: AT_BEGINNING_LINE,\n AT_END: AT_END_LINE\n}\n\nAT_LOCALE = {\n AT_BOUNDARY: AT_LOC_BOUNDARY,\n AT_NON_BOUNDARY: AT_LOC_NON_BOUNDARY\n}\n\nAT_UNICODE = {\n AT_BOUNDARY: AT_UNI_BOUNDARY,\n AT_NON_BOUNDARY: AT_UNI_NON_BOUNDARY\n}\n\nCH_LOCALE = {\n CATEGORY_DIGIT: CATEGORY_DIGIT,\n CATEGORY_NOT_DIGIT: CATEGORY_NOT_DIGIT,\n CATEGORY_SPACE: CATEGORY_SPACE,\n CATEGORY_NOT_SPACE: CATEGORY_NOT_SPACE,\n CATEGORY_WORD: CATEGORY_LOC_WORD,\n CATEGORY_NOT_WORD: CATEGORY_LOC_NOT_WORD,\n CATEGORY_LINEBREAK: CATEGORY_LINEBREAK,\n CATEGORY_NOT_LINEBREAK: CATEGORY_NOT_LINEBREAK\n}\n\nCH_UNICODE = {\n CATEGORY_DIGIT: CATEGORY_UNI_DIGIT,\n CATEGORY_NOT_DIGIT: CATEGORY_UNI_NOT_DIGIT,\n CATEGORY_SPACE: CATEGORY_UNI_SPACE,\n CATEGORY_NOT_SPACE: CATEGORY_UNI_NOT_SPACE,\n CATEGORY_WORD: CATEGORY_UNI_WORD,\n CATEGORY_NOT_WORD: CATEGORY_UNI_NOT_WORD,\n CATEGORY_LINEBREAK: CATEGORY_UNI_LINEBREAK,\n CATEGORY_NOT_LINEBREAK: CATEGORY_UNI_NOT_LINEBREAK\n}\n\n# flags\nSRE_FLAG_TEMPLATE = 1 # template mode (disable backtracking)\nSRE_FLAG_IGNORECASE = 2 # case insensitive\nSRE_FLAG_LOCALE = 4 # honour system locale\nSRE_FLAG_MULTILINE = 8 # treat target as multiline string\nSRE_FLAG_DOTALL = 16 # treat target as a single string\nSRE_FLAG_UNICODE = 32 # use unicode locale\nSRE_FLAG_VERBOSE = 64 # ignore whitespace and comments\nSRE_FLAG_DEBUG = 128 # debugging\n\n# flags for INFO primitive\nSRE_INFO_PREFIX = 1 # has prefix\nSRE_INFO_LITERAL = 2 # entire pattern is literal (given by prefix)\nSRE_INFO_CHARSET = 4 # pattern starts with character from given set\n" }, { "path": "gixy/core/sre_parse/sre_parse.py", "content": "# flake8: noqa\n\n#\n# Secret Labs' Regular Expression Engine\n#\n# convert re-style regular expression to sre pattern\n#\n# Copyright (c) 1998-2001 by Secret Labs AB. All rights reserved.\n#\n# See the sre.py file for information on usage and redistribution.\n#\n\nfrom __future__ import print_function\n\n\"\"\"Internal support module for sre\"\"\"\n\nfrom sre_constants import *\n\nSPECIAL_CHARS = \".\\\\[{()*+?^$|\"\nREPEAT_CHARS = \"*+?{\"\n\nDIGITS = set(\"0123456789\")\n\nOCTDIGITS = set(\"01234567\")\nHEXDIGITS = set(\"0123456789abcdefABCDEF\")\n\nWHITESPACE = set(\" \\t\\n\\r\\v\\f\")\n\nESCAPES = {\n r\"\\a\": (LITERAL, ord(\"\\a\")),\n r\"\\b\": (LITERAL, ord(\"\\b\")),\n r\"\\f\": (LITERAL, ord(\"\\f\")),\n r\"\\n\": (LITERAL, ord(\"\\n\")),\n r\"\\r\": (LITERAL, ord(\"\\r\")),\n r\"\\t\": (LITERAL, ord(\"\\t\")),\n r\"\\v\": (LITERAL, ord(\"\\v\")),\n r\"\\\\\": (LITERAL, ord(\"\\\\\"))\n}\n\nCATEGORIES = {\n r\"\\A\": (AT, AT_BEGINNING_STRING), # start of string\n r\"\\b\": (AT, AT_BOUNDARY),\n r\"\\B\": (AT, AT_NON_BOUNDARY),\n r\"\\d\": (IN, [(CATEGORY, CATEGORY_DIGIT)]),\n r\"\\D\": (IN, [(CATEGORY, CATEGORY_NOT_DIGIT)]),\n r\"\\s\": (IN, [(CATEGORY, CATEGORY_SPACE)]),\n r\"\\S\": (IN, [(CATEGORY, CATEGORY_NOT_SPACE)]),\n r\"\\w\": (IN, [(CATEGORY, CATEGORY_WORD)]),\n r\"\\W\": (IN, [(CATEGORY, CATEGORY_NOT_WORD)]),\n r\"\\Z\": (AT, AT_END_STRING), # end of string\n}\n\nFLAGS = {\n # standard flags\n \"i\": SRE_FLAG_IGNORECASE,\n \"L\": SRE_FLAG_LOCALE,\n \"m\": SRE_FLAG_MULTILINE,\n \"s\": SRE_FLAG_DOTALL,\n \"x\": SRE_FLAG_VERBOSE,\n # extensions\n \"t\": SRE_FLAG_TEMPLATE,\n \"u\": SRE_FLAG_UNICODE,\n}\n\n\nclass Pattern:\n # master pattern object. keeps track of global attributes\n def __init__(self):\n self.flags = 0\n self.open = []\n self.groups = 1\n self.groupdict = {}\n self.lookbehind = 0\n\n def opengroup(self, name=None):\n gid = self.groups\n self.groups = gid + 1\n if name is not None:\n ogid = self.groupdict.get(name, None)\n if ogid is not None:\n raise error((\"redefinition of group name %s as group %d; \"\n \"was group %d\" % (repr(name), gid, ogid)))\n self.groupdict[name] = gid\n self.open.append(gid)\n return gid\n\n def closegroup(self, gid):\n self.open.remove(gid)\n\n def checkgroup(self, gid):\n return gid < self.groups and gid not in self.open\n\n\nclass SubPattern:\n # a subpattern, in intermediate form\n def __init__(self, pattern, data=None):\n self.pattern = pattern\n if data is None:\n data = []\n self.data = data\n self.width = None\n\n def __repr__(self):\n return repr(self.data)\n\n def __len__(self):\n return len(self.data)\n\n def __delitem__(self, index):\n del self.data[index]\n\n def __getitem__(self, index):\n if isinstance(index, slice):\n return SubPattern(self.pattern, self.data[index])\n return self.data[index]\n\n def __setitem__(self, index, code):\n self.data[index] = code\n\n def insert(self, index, code):\n self.data.insert(index, code)\n\n def append(self, code):\n self.data.append(code)\n\n def getwidth(self):\n # determine the width (min, max) for this subpattern\n if self.width:\n return self.width\n lo = hi = 0\n UNITCODES = (ANY, RANGE, IN, LITERAL, NOT_LITERAL, CATEGORY)\n REPEATCODES = (MIN_REPEAT, MAX_REPEAT)\n for op, av in self.data:\n if op is BRANCH:\n i = MAXREPEAT - 1\n j = 0\n for av in av[1]:\n l, h = av.getwidth()\n i = min(i, l)\n j = max(j, h)\n lo = lo + i\n hi = hi + j\n elif op is CALL:\n i, j = av.getwidth()\n lo = lo + i\n hi = hi + j\n elif op is SUBPATTERN:\n i, j = av[1].getwidth()\n lo = lo + i\n hi = hi + j\n elif op in REPEATCODES:\n i, j = av[2].getwidth()\n lo = lo + i * av[0]\n hi = hi + j * av[1]\n elif op in UNITCODES:\n lo = lo + 1\n hi = hi + 1\n elif op == SUCCESS:\n break\n self.width = min(lo, MAXREPEAT - 1), min(hi, MAXREPEAT)\n return self.width\n\n\nclass Tokenizer:\n def __init__(self, string):\n self.string = string\n self.index = 0\n self.__next()\n\n def __next(self):\n if self.index >= len(self.string):\n self.next = None\n return\n char = self.string[self.index]\n if char[0] == \"\\\\\":\n try:\n c = self.string[self.index + 1]\n except IndexError:\n raise error(\"bogus escape (end of line)\")\n char = char + c\n self.index = self.index + len(char)\n self.next = char\n\n def match(self, char, skip=1):\n if char == self.next:\n if skip:\n self.__next()\n return 1\n return 0\n\n def get(self):\n this = self.next\n self.__next()\n return this\n\n def tell(self):\n return self.index, self.next\n\n def seek(self, index):\n self.index, self.next = index\n\n\ndef isident(char):\n return \"a\" <= char <= \"z\" or \"A\" <= char <= \"Z\" or char == \"_\"\n\n\ndef isdigit(char):\n return \"0\" <= char <= \"9\"\n\n\ndef isname(name):\n # check that group name is a valid string\n if not isident(name[0]):\n return False\n for char in name[1:]:\n if not isident(char) and not isdigit(char):\n return False\n return True\n\n\ndef _class_escape(source, escape):\n # handle escape code inside character class\n code = ESCAPES.get(escape)\n if code:\n return code\n code = CATEGORIES.get(escape)\n if code and code[0] == IN:\n return code\n try:\n c = escape[1:2]\n if c == \"x\":\n # hexadecimal escape (exactly two digits)\n while source.next in HEXDIGITS and len(escape) < 4:\n escape = escape + source.get()\n escape = escape[2:]\n if len(escape) != 2:\n raise error(\"bogus escape: %s\" % repr(\"\\\\\" + escape))\n return LITERAL, int(escape, 16) & 0xff\n elif c in OCTDIGITS:\n # octal escape (up to three digits)\n while source.next in OCTDIGITS and len(escape) < 4:\n escape = escape + source.get()\n escape = escape[1:]\n return LITERAL, int(escape, 8) & 0xff\n elif c in DIGITS:\n raise error(\"bogus escape: %s\" % repr(escape))\n if len(escape) == 2:\n return LITERAL, ord(escape[1])\n except ValueError:\n pass\n raise error(\"bogus escape: %s\" % repr(escape))\n\n\ndef _escape(source, escape, state):\n # handle escape code in expression\n code = CATEGORIES.get(escape)\n if code:\n return code\n code = ESCAPES.get(escape)\n if code:\n return code\n try:\n c = escape[1:2]\n if c == \"x\":\n # hexadecimal escape\n while source.next in HEXDIGITS and len(escape) < 4:\n escape = escape + source.get()\n if len(escape) != 4:\n raise ValueError\n return LITERAL, int(escape[2:], 16) & 0xff\n elif c == \"0\":\n # octal escape\n while source.next in OCTDIGITS and len(escape) < 4:\n escape = escape + source.get()\n return LITERAL, int(escape[1:], 8) & 0xff\n elif c in DIGITS:\n # octal escape *or* decimal group reference (sigh)\n if source.next in DIGITS:\n escape = escape + source.get()\n if (escape[1] in OCTDIGITS and escape[2] in OCTDIGITS and\n source.next in OCTDIGITS):\n # got three octal digits; this is an octal escape\n escape = escape + source.get()\n return LITERAL, int(escape[1:], 8) & 0xff\n # not an octal escape, so this is a group reference\n group = int(escape[1:])\n if group < state.groups:\n if not state.checkgroup(group):\n raise error(\"cannot refer to open group\")\n if state.lookbehind:\n import warnings\n warnings.warn('group references in lookbehind '\n 'assertions are not supported',\n RuntimeWarning)\n return GROUPREF, group\n raise ValueError\n if len(escape) == 2:\n return LITERAL, ord(escape[1])\n except ValueError:\n pass\n raise error(\"bogus escape: %s\" % repr(escape))\n\n\ndef _parse_sub(source, state, nested=1):\n # parse an alternation: a|b|c\n\n items = []\n itemsappend = items.append\n sourcematch = source.match\n while 1:\n itemsappend(_parse(source, state))\n if sourcematch(\"|\"):\n continue\n if not nested:\n break\n if not source.next or sourcematch(\")\", 0):\n break\n else:\n raise error(\"pattern not properly closed\")\n\n if len(items) == 1:\n return items[0]\n\n subpattern = SubPattern(state)\n subpatternappend = subpattern.append\n\n # check if all items share a common prefix\n while 1:\n prefix = None\n for item in items:\n if not item:\n break\n if prefix is None:\n prefix = item[0]\n elif item[0] != prefix:\n break\n else:\n # all subitems start with a common \"prefix\".\n # move it out of the branch\n for item in items:\n del item[0]\n subpatternappend(prefix)\n continue # check next one\n break\n\n # check if the branch can be replaced by a character set\n for item in items:\n if len(item) != 1 or item[0][0] != LITERAL:\n break\n else:\n # we can store this as a character set instead of a\n # branch (the compiler may optimize this even more)\n set = []\n setappend = set.append\n for item in items:\n setappend(item[0])\n subpatternappend((IN, set))\n return subpattern\n\n subpattern.append((BRANCH, (None, items)))\n return subpattern\n\n\ndef _parse_sub_cond(source, state, condgroup):\n item_yes = _parse(source, state)\n if source.match(\"|\"):\n item_no = _parse(source, state)\n if source.match(\"|\"):\n raise error(\"conditional backref with more than two branches\")\n else:\n item_no = None\n if source.next and not source.match(\")\", 0):\n raise error(\"pattern not properly closed\")\n subpattern = SubPattern(state)\n subpattern.append((GROUPREF_EXISTS, (condgroup, item_yes, item_no)))\n return subpattern\n\n\n_PATTERNENDERS = set(\"|)\")\n_ASSERTCHARS = set(\"=!<\")\n_LOOKBEHINDASSERTCHARS = set(\"=!\")\n_REPEATCODES = set([MIN_REPEAT, MAX_REPEAT])\n\n\ndef _parse(source, state):\n # parse a simple pattern\n subpattern = SubPattern(state)\n\n # precompute constants into local variables\n subpatternappend = subpattern.append\n sourceget = source.get\n sourcematch = source.match\n _len = len\n PATTERNENDERS = _PATTERNENDERS\n ASSERTCHARS = _ASSERTCHARS\n LOOKBEHINDASSERTCHARS = _LOOKBEHINDASSERTCHARS\n REPEATCODES = _REPEATCODES\n\n while 1:\n\n if source.next in PATTERNENDERS:\n break # end of subpattern\n this = sourceget()\n if this is None:\n break # end of pattern\n\n if state.flags & SRE_FLAG_VERBOSE:\n # skip whitespace and comments\n if this in WHITESPACE:\n continue\n if this == \"#\":\n while 1:\n this = sourceget()\n if this in (None, \"\\n\"):\n break\n continue\n\n if this and this[0] not in SPECIAL_CHARS:\n subpatternappend((LITERAL, ord(this)))\n\n elif this == \"[\":\n # character set\n set = []\n setappend = set.append\n ## if sourcematch(\":\"):\n ## pass # handle character classes\n if sourcematch(\"^\"):\n setappend((NEGATE, None))\n # check remaining characters\n start = set[:]\n while 1:\n this = sourceget()\n if this == \"]\" and set != start:\n break\n elif this and this[0] == \"\\\\\":\n code1 = _class_escape(source, this)\n elif this:\n code1 = LITERAL, ord(this)\n else:\n raise error(\"unexpected end of regular expression\")\n if sourcematch(\"-\"):\n # potential range\n this = sourceget()\n if this == \"]\":\n if code1[0] is IN:\n code1 = code1[1][0]\n setappend(code1)\n setappend((LITERAL, ord(\"-\")))\n break\n elif this:\n if this[0] == \"\\\\\":\n code2 = _class_escape(source, this)\n else:\n code2 = LITERAL, ord(this)\n if code1[0] != LITERAL or code2[0] != LITERAL:\n raise error(\"bad character range\")\n lo = code1[1]\n hi = code2[1]\n if hi < lo:\n raise error(\"bad character range\")\n setappend((RANGE, (lo, hi)))\n else:\n raise error(\"unexpected end of regular expression\")\n else:\n if code1[0] is IN:\n code1 = code1[1][0]\n setappend(code1)\n\n # XXX: should move set optimization to compiler!\n if _len(set) == 1 and set[0][0] is LITERAL:\n subpatternappend(set[0]) # optimization\n elif _len(set) == 2 and set[0][0] is NEGATE and set[1][0] is LITERAL:\n subpatternappend((NOT_LITERAL, set[1][1])) # optimization\n else:\n # XXX: should add charmap optimization here\n subpatternappend((IN, set))\n\n elif this and this[0] in REPEAT_CHARS:\n # repeat previous item\n if this == \"?\":\n min, max = 0, 1\n elif this == \"*\":\n min, max = 0, MAXREPEAT\n\n elif this == \"+\":\n min, max = 1, MAXREPEAT\n elif this == \"{\":\n if source.next == \"}\":\n subpatternappend((LITERAL, ord(this)))\n continue\n here = source.tell()\n min, max = 0, MAXREPEAT\n lo = hi = \"\"\n while source.next in DIGITS:\n lo = lo + source.get()\n if sourcematch(\",\"):\n while source.next in DIGITS:\n hi = hi + sourceget()\n else:\n hi = lo\n if not sourcematch(\"}\"):\n subpatternappend((LITERAL, ord(this)))\n source.seek(here)\n continue\n if lo:\n min = int(lo)\n if min >= MAXREPEAT:\n raise OverflowError(\"the repetition number is too large\")\n if hi:\n max = int(hi)\n if max >= MAXREPEAT:\n raise OverflowError(\"the repetition number is too large\")\n if max < min:\n raise error(\"bad repeat interval\")\n else:\n raise error(\"not supported\")\n # figure out which item to repeat\n if subpattern:\n item = subpattern[-1:]\n else:\n item = None\n if not item or (_len(item) == 1 and item[0][0] == AT):\n raise error(\"nothing to repeat\")\n if item[0][0] in REPEATCODES:\n raise error(\"multiple repeat\")\n if sourcematch(\"?\"):\n subpattern[-1] = (MIN_REPEAT, (min, max, item))\n else:\n subpattern[-1] = (MAX_REPEAT, (min, max, item))\n\n elif this == \".\":\n subpatternappend((ANY, None))\n\n elif this == \"(\":\n group = 1\n name = None\n condgroup = None\n if sourcematch(\"?\"):\n group = 0\n # options\n if sourcematch(\"P\"):\n # python extensions\n if sourcematch(\"<\"):\n # named group: skip forward to end of name\n name = \"\"\n while 1:\n char = sourceget()\n if char is None:\n raise error(\"unterminated name\")\n if char == \">\":\n break\n name = name + char\n group = 1\n if not name:\n raise error(\"missing group name\")\n if not isname(name):\n raise error(\"bad character in group name %r\" %\n name)\n elif sourcematch(\"=\"):\n # named backreference\n name = \"\"\n while 1:\n char = sourceget()\n if char is None:\n raise error(\"unterminated name\")\n if char == \")\":\n break\n name = name + char\n if not name:\n raise error(\"missing group name\")\n if not isname(name):\n raise error(\"bad character in backref group name \"\n \"%r\" % name)\n gid = state.groupdict.get(name)\n if gid is None:\n msg = \"unknown group name: {0!r}\".format(name)\n raise error(msg)\n if state.lookbehind:\n import warnings\n warnings.warn('group references in lookbehind '\n 'assertions are not supported',\n RuntimeWarning)\n subpatternappend((GROUPREF, gid))\n continue\n else:\n char = sourceget()\n if char is None:\n raise error(\"unexpected end of pattern\")\n raise error(\"unknown specifier: ?P%s\" % char)\n elif sourcematch(\":\"):\n # non-capturing group\n group = 2\n elif sourcematch(\"#\"):\n # comment\n while 1:\n if source.next is None or source.next == \")\":\n break\n sourceget()\n if not sourcematch(\")\"):\n raise error(\"unbalanced parenthesis\")\n continue\n elif source.next in ASSERTCHARS:\n # lookahead assertions\n char = sourceget()\n dir = 1\n if char == \"<\":\n if source.next not in LOOKBEHINDASSERTCHARS:\n raise error(\"syntax error\")\n dir = -1 # lookbehind\n char = sourceget()\n state.lookbehind += 1\n p = _parse_sub(source, state)\n if dir < 0:\n state.lookbehind -= 1\n if not sourcematch(\")\"):\n raise error(\"unbalanced parenthesis\")\n if char == \"=\":\n subpatternappend((ASSERT, (dir, p)))\n else:\n subpatternappend((ASSERT_NOT, (dir, p)))\n continue\n elif sourcematch(\"(\"):\n # conditional backreference group\n condname = \"\"\n while 1:\n char = sourceget()\n if char is None:\n raise error(\"unterminated name\")\n if char == \")\":\n break\n condname = condname + char\n group = 2\n if not condname:\n raise error(\"missing group name\")\n if isname(condname):\n condgroup = state.groupdict.get(condname)\n if condgroup is None:\n msg = \"unknown group name: {0!r}\".format(condname)\n raise error(msg)\n else:\n try:\n condgroup = int(condname)\n except ValueError:\n raise error(\"bad character in group name\")\n if state.lookbehind:\n import warnings\n warnings.warn('group references in lookbehind '\n 'assertions are not supported',\n RuntimeWarning)\n else:\n # flags\n if not source.next in FLAGS:\n raise error(\"unexpected end of pattern\")\n while source.next in FLAGS:\n state.flags = state.flags | FLAGS[sourceget()]\n if group:\n # parse group contents\n if group == 2:\n # anonymous group\n group = None\n else:\n group = state.opengroup(name)\n if condgroup:\n p = _parse_sub_cond(source, state, condgroup)\n else:\n p = _parse_sub(source, state)\n if not sourcematch(\")\"):\n raise error(\"unbalanced parenthesis\")\n if group is not None:\n state.closegroup(group)\n subpatternappend((SUBPATTERN, (group, p)))\n else:\n while 1:\n char = sourceget()\n if char is None:\n raise error(\"unexpected end of pattern\")\n if char == \")\":\n break\n raise error(\"unknown extension\")\n\n elif this == \"^\":\n subpatternappend((AT, AT_BEGINNING))\n\n elif this == \"$\":\n subpattern.append((AT, AT_END))\n\n elif this and this[0] == \"\\\\\":\n code = _escape(source, this, state)\n subpatternappend(code)\n\n else:\n raise error(\"parser error\")\n\n return subpattern\n\n\ndef parse(str, flags=0, pattern=None):\n # parse 're' pattern into list of (opcode, argument) tuples\n\n source = Tokenizer(str)\n\n if pattern is None:\n pattern = Pattern()\n pattern.flags = flags\n pattern.str = str\n\n p = _parse_sub(source, pattern, 0)\n\n tail = source.get()\n if tail == \")\":\n raise error(\"unbalanced parenthesis\")\n elif tail:\n raise error(\"bogus characters at end of regular expression\")\n\n if not (flags & SRE_FLAG_VERBOSE) and p.pattern.flags & SRE_FLAG_VERBOSE:\n # the VERBOSE flag was switched on inside the pattern. to be\n # on the safe side, we'll parse the whole thing again...\n return parse(str, p.pattern.flags)\n\n if flags & SRE_FLAG_DEBUG:\n p.dump()\n\n return p\n\n\ndef parse_template(source, pattern):\n # parse 're' replacement string into list of literals and\n # group references\n s = Tokenizer(source)\n sget = s.get\n p = []\n a = p.append\n\n def literal(literal, p=p, pappend=a):\n if p and p[-1][0] is LITERAL:\n p[-1] = LITERAL, p[-1][1] + literal\n else:\n pappend((LITERAL, literal))\n\n sep = source[:0]\n if type(sep) is type(\"\"):\n makechar = chr\n else:\n makechar = unichr\n while 1:\n this = sget()\n if this is None:\n break # end of replacement string\n if this and this[0] == \"\\\\\":\n # group\n c = this[1:2]\n if c == \"g\":\n name = \"\"\n if s.match(\"<\"):\n while 1:\n char = sget()\n if char is None:\n raise error(\"unterminated group name\")\n if char == \">\":\n break\n name = name + char\n if not name:\n raise error(\"missing group name\")\n try:\n index = int(name)\n if index < 0:\n raise error(\"negative group number\")\n except ValueError:\n if not isname(name):\n raise error(\"bad character in group name\")\n try:\n index = pattern.groupindex[name]\n except KeyError:\n msg = \"unknown group name: {0!r}\".format(name)\n raise IndexError(msg)\n a((MARK, index))\n elif c == \"0\":\n if s.next in OCTDIGITS:\n this = this + sget()\n if s.next in OCTDIGITS:\n this = this + sget()\n literal(makechar(int(this[1:], 8) & 0xff))\n elif c in DIGITS:\n isoctal = False\n if s.next in DIGITS:\n this = this + sget()\n if (c in OCTDIGITS and this[2] in OCTDIGITS and\n s.next in OCTDIGITS):\n this = this + sget()\n isoctal = True\n literal(makechar(int(this[1:], 8) & 0xff))\n if not isoctal:\n a((MARK, int(this[1:])))\n else:\n try:\n this = makechar(ESCAPES[this][1])\n except KeyError:\n pass\n literal(this)\n else:\n literal(this)\n # convert template to groups and literals lists\n i = 0\n groups = []\n groupsappend = groups.append\n literals = [None] * len(p)\n for c, s in p:\n if c is MARK:\n groupsappend((i, s))\n # literal[i] is already None\n else:\n literals[i] = s\n i = i + 1\n return groups, literals\n\n\ndef expand_template(template, match):\n g = match.group\n sep = match.string[:0]\n groups, literals = template\n literals = literals[:]\n try:\n for index, group in groups:\n literals[index] = s = g(group)\n if s is None:\n raise error(\"unmatched group\")\n except IndexError:\n raise error(\"invalid group reference\")\n return sep.join(literals)\n" }, { "path": "gixy/core/utils.py", "content": "def is_indexed_name(name):\n return isinstance(name, int) or (len(name) == 1 and '1' <= name <= '9')\n" }, { "path": "gixy/core/variable.py", "content": "import re\nimport logging\n\nfrom gixy.core.regexp import Regexp\nfrom gixy.core.context import get_context\n\nLOG = logging.getLogger(__name__)\n# See ngx_http_script_compile in http/ngx_http_script.c\nEXTRACT_RE = re.compile(r'\\$([1-9]|[a-z_][a-z0-9_]*|\\{[a-z0-9_]+\\})', re.IGNORECASE)\n\n\ndef compile_script(script):\n \"\"\"\n Compile Nginx script to list of variables.\n Example:\n compile_script('http://$foo:$bar') ->\n [Variable('http://'), Variable($foo), Variable(':', Variable($bar).\n\n :param str script: Nginx scrip.\n :return Variable[]: list of variable.\n \"\"\"\n depends = []\n context = get_context()\n for i, var in enumerate(EXTRACT_RE.split(str(script))):\n if i % 2:\n # Variable\n var = var.strip('{}\\x20')\n var = context.get_var(var)\n if var:\n depends.append(var)\n elif var:\n # Literal\n depends.append(Variable(name=None, value=var, have_script=False))\n return depends\n\n\nclass Variable(object):\n def __init__(self, name, value=None, boundary=None, provider=None, have_script=True):\n \"\"\"\n Gixy Nginx variable class - parse and provide helpers to work with it.\n\n :param str|None name: variable name.\n :param str|Regexp value: variable value..\n :param Regexp boundary: variable boundary set.\n :param Directive provider: directive that provide variable (e.g. if, location, rewrite, etc).\n :param bool have_script: may variable have nginx script or not (mostly used to indicate a string literal).\n \"\"\"\n\n self.name = name\n self.value = value\n self.regexp = None\n self.depends = None\n self.boundary = boundary\n self.provider = provider\n if isinstance(value, Regexp):\n self.regexp = value\n elif have_script:\n self.depends = compile_script(value)\n\n def can_contain(self, char):\n \"\"\"\n Checks if variable can contain the specified char.\n\n :param str char: character to test.\n :return: True if variable can contain the specified char, False otherwise.\n \"\"\"\n\n # First of all check boundary set\n if self.boundary and not self.boundary.can_contain(char):\n return False\n\n # Then regexp\n if self.regexp:\n return self.regexp.can_contain(char, skip_literal=True)\n\n # Then dependencies\n if self.depends:\n return any(dep.can_contain(char) for dep in self.depends)\n\n # Otherwise user can't control value of this variable\n return False\n\n def can_startswith(self, char):\n \"\"\"\n Checks if variable can starts with the specified char.\n\n :param str char: character to test.\n :return: True if variable can starts with the specified char, False otherwise.\n \"\"\"\n\n # First of all check boundary set\n if self.boundary and not self.boundary.can_startswith(char):\n return False\n\n # Then regexp\n if self.regexp:\n return self.regexp.can_startswith(char)\n\n # Then dependencies\n if self.depends:\n return self.depends[0].can_startswith(char)\n\n # Otherwise user can't control value of this variable\n return False\n\n def must_contain(self, char):\n \"\"\"\n Checks if variable MUST contain the specified char.\n\n :param str char: character to test.\n :return: True if variable must contain the specified char, False otherwise.\n \"\"\"\n\n # First of all check boundary set\n if self.boundary and self.boundary.must_contain(char):\n return True\n\n # Then regexp\n if self.regexp:\n return self.regexp.must_contain(char)\n\n # Then dependencies\n if self.depends:\n return any(dep.must_contain(char) for dep in self.depends)\n\n # Otherwise checks literal\n return self.value and char in self.value\n\n def must_startswith(self, char):\n \"\"\"\n Checks if variable MUST starts with the specified char.\n\n :param str char: character to test.\n :return: True if variable must starts with the specified char.\n \"\"\"\n\n # First of all check boundary set\n if self.boundary and self.boundary.must_startswith(char):\n return True\n\n # Then regexp\n if self.regexp:\n return self.regexp.must_startswith(char)\n\n # Then dependencies\n if self.depends:\n return self.depends[0].must_startswith(char)\n\n # Otherwise checks literal\n return self.value and self.value[0] == char\n\n @property\n def providers(self):\n \"\"\"\n Returns list of variable provides.\n\n :return Directive[]: providers.\n \"\"\"\n result = []\n if self.provider:\n result.append(self.provider)\n if self.depends:\n for dep in self.depends:\n result += dep.providers\n return result\n" }, { "path": "gixy/directives/__init__.py", "content": "import os\nfrom gixy.directives.directive import Directive\n\nDIRECTIVES = {}\n\n\ndef import_directives():\n files_list = os.listdir(os.path.dirname(__file__))\n for directive_file in files_list:\n if not directive_file.endswith(\".py\") or directive_file.startswith('_'):\n continue\n __import__('gixy.directives.' + os.path.splitext(directive_file)[0], None, None, [''])\n\n\ndef get_all():\n if len(DIRECTIVES):\n return DIRECTIVES\n\n import_directives()\n for klass in Directive.__subclasses__():\n if not klass.nginx_name:\n continue\n DIRECTIVES[klass.nginx_name] = klass\n\n return DIRECTIVES\n" }, { "path": "gixy/directives/block.py", "content": "from cached_property import cached_property\n\nfrom gixy.directives.directive import Directive\nfrom gixy.core.variable import Variable\nfrom gixy.core.regexp import Regexp\n\n\ndef get_overrides():\n result = {}\n for klass in Block.__subclasses__():\n if not klass.nginx_name:\n continue\n\n if not klass.__name__.endswith('Block'):\n continue\n\n result[klass.nginx_name] = klass\n return result\n\n\nclass Block(Directive):\n nginx_name = None\n is_block = True\n self_context = True\n\n def __init__(self, name, args):\n super(Block, self).__init__(name, args)\n self.children = []\n\n def some(self, name, flat=True):\n for child in self.children:\n if child.name == name:\n return child\n if flat and child.is_block and not child.self_context:\n result = child.some(name, flat=flat)\n if result:\n return result\n return None\n\n def find(self, name, flat=False):\n result = []\n for child in self.children:\n if child.name == name:\n result.append(child)\n if flat and child.is_block and not child.self_context:\n result += child.find(name)\n return result\n\n def find_recursive(self, name):\n result = []\n for child in self.children:\n if child.name == name:\n result.append(child)\n if child.is_block:\n result += child.find_recursive(name)\n return result\n\n def append(self, directive):\n directive.set_parent(self)\n self.children.append(directive)\n\n def __str__(self):\n return '{name} {args} {{'.format(name=self.name, args=' '.join(self.args))\n\n\nclass Root(Block):\n nginx_name = None\n\n def __init__(self):\n super(Root, self).__init__(None, [])\n\n\nclass HttpBlock(Block):\n nginx_name = 'http'\n\n def __init__(self, name, args):\n super(HttpBlock, self).__init__(name, args)\n\n\nclass ServerBlock(Block):\n nginx_name = 'server'\n\n def __init__(self, name, args):\n super(ServerBlock, self).__init__(name, args)\n\n def get_names(self):\n return self.find('server_name')\n\n def __str__(self):\n server_names = [str(sn) for sn in self.find('server_name')]\n if server_names:\n return 'server {{\\n{0}'.format('\\n'.join(server_names[:2]))\n return 'server {'\n\n\nclass LocationBlock(Block):\n nginx_name = 'location'\n provide_variables = True\n\n def __init__(self, name, args):\n super(LocationBlock, self).__init__(name, args)\n if len(args) == 2:\n self.modifier, self.path = args\n else:\n self.modifier = None\n self.path = args[0]\n\n @property\n def is_internal(self):\n return self.some('internal') is not None\n\n @cached_property\n def variables(self):\n if not self.modifier or self.modifier not in ('~', '~*'):\n return []\n\n regexp = Regexp(self.path, case_sensitive=self.modifier == '~')\n result = []\n for name, group in regexp.groups.items():\n result.append(Variable(name=name, value=group, boundary=None, provider=self))\n return result\n\n\nclass IfBlock(Block):\n nginx_name = 'if'\n self_context = False\n\n def __init__(self, name, args):\n super(IfBlock, self).__init__(name, args)\n self.operand = None\n self.value = None\n self.variable = None\n\n if len(args) == 1:\n # if ($slow)\n self.variable = args[0]\n elif len(args) == 2:\n # if (!-e $foo)\n self.operand, self.value = args\n elif len(args) == 3:\n # if ($request_method = POST)\n self.variable, self.operand, self.value = args\n else:\n raise Exception('Unknown \"if\" definition, args: {0!r}'.format(args))\n\n def __str__(self):\n return '{name} ({args}) {{'.format(name=self.name, args=' '.join(self.args))\n\n\nclass IncludeBlock(Block):\n nginx_name = 'include'\n self_context = False\n\n def __init__(self, name, args):\n super(IncludeBlock, self).__init__(name, args)\n self.file_path = args[0]\n\n def __str__(self):\n return 'include {0};'.format(self.file_path)\n\n\nclass MapBlock(Block):\n nginx_name = 'map'\n self_context = False\n provide_variables = True\n\n def __init__(self, name, args):\n super(MapBlock, self).__init__(name, args)\n self.source = args[0]\n self.variable = args[1].strip('$')\n\n @cached_property\n def variables(self):\n # TODO(buglloc): Finish him!\n return [Variable(name=self.variable, value='', boundary=None, provider=self, have_script=False)]\n\n\nclass GeoBlock(Block):\n nginx_name = 'geo'\n self_context = False\n provide_variables = True\n\n def __init__(self, name, args):\n super(GeoBlock, self).__init__(name, args)\n if len(args) == 1: # geo uses $remote_addr as default source of the value\n source = '$remote_addr'\n variable = args[0].strip('$')\n else:\n source = args[0]\n variable = args[1].strip('$')\n self.source = source\n self.variable = variable\n\n @cached_property\n def variables(self):\n # TODO(buglloc): Finish him! -- same as in MapBlock\n return [Variable(name=self.variable, value='', boundary=None, provider=self, have_script=False)]\n" }, { "path": "gixy/directives/directive.py", "content": "from gixy.core.variable import Variable\nfrom gixy.core.regexp import Regexp\n\n\ndef get_overrides():\n result = {}\n for klass in Directive.__subclasses__():\n if not klass.nginx_name:\n continue\n\n if not klass.__name__.endswith('Directive'):\n continue\n\n result[klass.nginx_name] = klass\n return result\n\n\nclass Directive(object):\n nginx_name = None\n is_block = False\n provide_variables = False\n\n def __init__(self, name, args, raw=None):\n self.name = name\n self.parent = None\n self.args = args\n self._raw = raw\n\n def set_parent(self, parent):\n self.parent = parent\n\n @property\n def parents(self):\n parent = self.parent\n while parent:\n yield parent\n parent = parent.parent\n\n @property\n def variables(self):\n raise NotImplementedError()\n\n def __str__(self):\n return '{name} {args};'.format(name=self.name, args=' '.join(self.args))\n\n\nclass AddHeaderDirective(Directive):\n nginx_name = 'add_header'\n\n def __init__(self, name, args):\n super(AddHeaderDirective, self).__init__(name, args)\n self.header = args[0].lower()\n self.value = args[1]\n self.always = False\n if len(args) > 2 and args[2] == 'always':\n self.always = True\n\n\nclass SetDirective(Directive):\n nginx_name = 'set'\n provide_variables = True\n\n def __init__(self, name, args):\n super(SetDirective, self).__init__(name, args)\n self.variable = args[0].strip('$')\n self.value = args[1]\n\n @property\n def variables(self):\n return [Variable(name=self.variable, value=self.value, provider=self)]\n\n\nclass AuthRequestSetDirective(Directive):\n nginx_name = 'auth_request_set'\n provide_variables = True\n\n def __init__(self, name, args):\n super(AuthRequestSetDirective, self).__init__(name, args)\n self.variable = args[0].strip('$')\n self.value = args[1]\n\n @property\n def variables(self):\n return [Variable(name=self.variable, value=self.value, provider=self)]\n\n\nclass PerlSetDirective(Directive):\n nginx_name = 'perl_set'\n provide_variables = True\n\n def __init__(self, name, args):\n super(PerlSetDirective, self).__init__(name, args)\n self.variable = args[0].strip('$')\n self.value = args[1]\n\n @property\n def variables(self):\n return [Variable(name=self.variable, provider=self, have_script=False)]\n\n\nclass SetByLuaDirective(Directive):\n nginx_name = 'set_by_lua'\n provide_variables = True\n\n def __init__(self, name, args):\n super(SetByLuaDirective, self).__init__(name, args)\n self.variable = args[0].strip('$')\n self.value = args[1]\n\n @property\n def variables(self):\n return [Variable(name=self.variable, provider=self, have_script=False)]\n\n\nclass RewriteDirective(Directive):\n nginx_name = 'rewrite'\n provide_variables = True\n boundary = Regexp(r'[^\\s\\r\\n]')\n\n def __init__(self, name, args):\n super(RewriteDirective, self).__init__(name, args)\n self.pattern = args[0]\n self.replace = args[1]\n self.flag = None\n if len(args) > 2:\n self.flag = args[2]\n\n @property\n def variables(self):\n regexp = Regexp(self.pattern, case_sensitive=True)\n result = []\n for name, group in regexp.groups.items():\n result.append(Variable(name=name, value=group, boundary=self.boundary, provider=self))\n return result\n\n\nclass RootDirective(Directive):\n nginx_name = 'root'\n provide_variables = True\n\n def __init__(self, name, args):\n super(RootDirective, self).__init__(name, args)\n self.path = args[0]\n\n @property\n def variables(self):\n return [Variable(name='document_root', value=self.path, provider=self)]\n\n\nclass AliasDirective(Directive):\n nginx_name = 'alias'\n\n def __init__(self, name, args):\n super(AliasDirective, self).__init__(name, args)\n self.path = args[0]\n" }, { "path": "gixy/formatters/__init__.py", "content": "import os\nfrom gixy.formatters.base import BaseFormatter\n\nFORMATTERS = {}\n\n\ndef import_formatters():\n files_list = os.listdir(os.path.dirname(__file__))\n for formatter_file in files_list:\n if not formatter_file.endswith(\".py\") or formatter_file.startswith('_'):\n continue\n __import__('gixy.formatters.' + os.path.splitext(formatter_file)[0], None, None, [''])\n\n\ndef get_all():\n if len(FORMATTERS):\n return FORMATTERS\n\n import_formatters()\n for klass in BaseFormatter.__subclasses__():\n FORMATTERS[klass.__name__.replace('Formatter', '').lower()] = klass\n\n return FORMATTERS\n" }, { "path": "gixy/formatters/_jinja.py", "content": "from __future__ import absolute_import\nfrom jinja2 import Environment, PackageLoader\n\nfrom gixy.utils.text import to_text\n\n\ndef load_template(name):\n env = Environment(loader=PackageLoader('gixy', 'formatters/templates'), trim_blocks=True, lstrip_blocks=True)\n env.filters['to_text'] = to_text_filter\n return env.get_template(name)\n\n\ndef to_text_filter(text):\n try:\n return text.encode('latin1').decode('utf-8')\n except UnicodeEncodeError:\n return to_text(text)\n" }, { "path": "gixy/formatters/base.py", "content": "from __future__ import absolute_import\n\nimport gixy\nfrom gixy.directives import block\n\n\nclass BaseFormatter(object):\n skip_parents = set([block.Root, block.HttpBlock])\n\n def __init__(self):\n self.reports = {}\n self.stats = dict.fromkeys(gixy.severity.ALL, 0)\n\n def format_reports(self, reports, stats):\n raise NotImplementedError(\"Formatter must override format_reports function\")\n\n def feed(self, path, manager):\n for severity in gixy.severity.ALL:\n self.stats[severity] += manager.stats[severity]\n\n self.reports[path] = []\n for result in manager.results:\n report = self._prepare_result(manager.root,\n summary=result.summary,\n severity=result.severity,\n description=result.description,\n issues=result.issues,\n plugin=result.name,\n help_url=result.help_url)\n self.reports[path].extend(report)\n\n def flush(self):\n return self.format_reports(self.reports, self.stats)\n\n def _prepare_result(self, root, issues, severity, summary, description, plugin, help_url):\n result = {}\n for issue in issues:\n report = dict(\n plugin=plugin,\n summary=issue.summary or summary,\n severity=issue.severity or severity,\n description=issue.description or description,\n help_url=issue.help_url or help_url,\n reason=issue.reason or '',\n )\n key = ''.join(report.values())\n report['directives'] = issue.directives\n if key in result:\n result[key]['directives'].extend(report['directives'])\n else:\n result[key] = report\n\n for report in result.values():\n if report['directives']:\n config = self._resolve_config(root, report['directives'])\n else:\n config = ''\n\n del report['directives']\n report['config'] = config\n yield report\n\n def _resolve_config(self, root, directives):\n points = set()\n for directive in directives:\n points.add(directive)\n points.update(p for p in directive.parents)\n\n result = self._traverse_tree(root, points, 0)\n return '\\n'.join(result)\n\n def _traverse_tree(self, tree, points, level):\n result = []\n for leap in tree.children:\n if leap not in points:\n continue\n printable = type(leap) not in self.skip_parents\n # Special hack for includes\n # TODO(buglloc): fix me\n have_parentheses = type(leap) != block.IncludeBlock\n\n if printable:\n if leap.is_block:\n result.append('')\n directive = str(leap).replace('\\n', '\\n' + '\\t' * (level + 1))\n result.append('{indent:s}{dir:s}'.format(indent='\\t' * level, dir=directive))\n\n if leap.is_block:\n result.extend(self._traverse_tree(leap, points, level + 1 if printable else level))\n if printable and have_parentheses:\n result.append('{indent:s}}}'.format(indent='\\t' * level))\n\n return result\n" }, { "path": "gixy/formatters/console.py", "content": "from __future__ import absolute_import\n\nfrom gixy.formatters.base import BaseFormatter\nfrom gixy.formatters._jinja import load_template\n\n\nclass ConsoleFormatter(BaseFormatter):\n def __init__(self):\n super(ConsoleFormatter, self).__init__()\n self.template = load_template('console.j2')\n\n def format_reports(self, reports, stats):\n return self.template.render(reports=reports, stats=stats)\n" }, { "path": "gixy/formatters/json.py", "content": "from __future__ import absolute_import\n\nimport json\n\nfrom gixy.formatters.base import BaseFormatter\n\n\nclass JsonFormatter(BaseFormatter):\n def format_reports(self, reports, stats):\n result = []\n for path, issues in reports.items():\n for issue in issues:\n result.append(dict(\n path=path,\n plugin=issue['plugin'],\n summary=issue['summary'],\n severity=issue['severity'],\n description=issue['description'],\n reference=issue['help_url'],\n reason=issue['reason'],\n config=issue['config']\n ))\n\n return json.dumps(result, sort_keys=True, indent=2, separators=(',', ': '))\n" }, { "path": "gixy/formatters/templates/console.j2", "content": "{% set colors = {'DEF': '\\033[0m', 'TITLE': '\\033[95m', 'UNSPECIFIED': '\\033[0m', 'LOW': '\\033[94m', 'MEDIUM': '\\033[93m', 'HIGH': '\\033[91m'} %}\n\n{{ colors.TITLE }}==================== Results ==================={{ colors.DEF }}\n{% for path, issues in reports.items() %}\n{% if reports|length > 1 %}\nFile path: {{ path }}\n{% endif %}\n{% if not issues %}\nNo issues found.\n\n{% else %}\n\n{% for issue in issues|sort(attribute='severity') %}\n{{ colors[issue.severity] }}>> Problem: [{{ issue.plugin }}] {{ issue.summary }}{{ colors.DEF }}\n{% if issue.description %}\nDescription: {{ issue.description }}\n{% endif %}\n{% if issue.help_url %}\nAdditional info: {{ issue.help_url }}\n{% endif %}\n{% if issue.reason %}\nReason: {{ issue.reason }}\n{% endif %}\nPseudo config:\n{{ issue.config | to_text }}\n\n{% if not loop.last %}\n------------------------------------------------\n\n{% endif %}\n{% endfor %}\n{% endif %}\n{% if not loop.last %}\n--------8<--------8<--------8<--------8<--------\n{% endif %}\n{% endfor %}\n{% if stats %}\n{{ colors.TITLE }}==================== Summary ==================={{ colors.DEF }}\nTotal issues:\n Unspecified: {{ stats.UNSPECIFIED }}\n Low: {{ stats.LOW }}\n Medium: {{ stats.MEDIUM }}\n High: {{ stats.HIGH }}\n{% endif %}\n" }, { "path": "gixy/formatters/templates/text.j2", "content": "\n==================== Results ===================\n{% for path, issues in reports.items() %}\n{% if reports|length > 1 %}\nFile path: {{ path }}\n{% endif %}\n{% if not issues %}\nNo issues found.\n\n{% else %}\n\n{% for issue in issues|sort(attribute='severity') %}\n>> Problem: [{{ issue.plugin }}] {{ issue.summary }}\nSeverity: {{ issue.severity }}\n{% if issue.description %}\nDescription: {{ issue.description }}\n{% endif %}\n{% if issue.help_url %}\nAdditional info: {{ issue.help_url }}\n{% endif %}\n{% if issue.reason %}\nReason: {{ issue.reason }}\n{% endif %}\nPseudo config:\n{{ issue.config | to_text }}\n\n{% if not loop.last %}\n------------------------------------------------\n\n{% endif %}\n{% endfor %}\n{% endif %}\n{% if not loop.last %}\n--------8<--------8<--------8<--------8<--------\n{% endif %}\n{% endfor %}\n{% if stats %}\n==================== Summary ===================\nTotal issues:\n Unspecified: {{ stats.UNSPECIFIED }}\n Low: {{ stats.LOW }}\n Medium: {{ stats.MEDIUM }}\n High: {{ stats.HIGH }}\n{% endif %}\n" }, { "path": "gixy/formatters/text.py", "content": "from __future__ import absolute_import\n\nfrom gixy.formatters.base import BaseFormatter\nfrom gixy.formatters._jinja import load_template\n\n\nclass TextFormatter(BaseFormatter):\n def __init__(self):\n super(TextFormatter, self).__init__()\n self.template = load_template('text.j2')\n\n def format_reports(self, reports, stats):\n return self.template.render(reports=reports, stats=stats)\n" }, { "path": "gixy/parser/__init__.py", "content": "" }, { "path": "gixy/parser/nginx_parser.py", "content": "import os\nimport glob\nimport logging\nimport fnmatch\n\nfrom pyparsing import ParseException\nfrom gixy.core.exceptions import InvalidConfiguration\nfrom gixy.parser import raw_parser\nfrom gixy.directives import block, directive\nfrom gixy.utils.text import to_native\n\nLOG = logging.getLogger(__name__)\n\n\nclass NginxParser(object):\n def __init__(self, cwd='', allow_includes=True):\n self.cwd = cwd\n self.configs = {}\n self.is_dump = False\n self.allow_includes = allow_includes\n self.directives = {}\n self.parser = raw_parser.RawParser()\n self._init_directives()\n\n def parse_file(self, path, root=None):\n LOG.debug(\"Parse file: {0}\".format(path))\n content = open(path).read()\n return self.parse(content=content, root=root, path_info=path)\n\n def parse(self, content, root=None, path_info=None):\n if not root:\n root = block.Root()\n try:\n parsed = self.parser.parse(content)\n except ParseException as e:\n error_msg = 'char {char} (line:{line}, col:{col})'.format(char=e.loc, line=e.lineno, col=e.col)\n if path_info:\n LOG.error('Failed to parse config \"{file}\": {error}'.format(file=path_info, error=error_msg))\n else:\n LOG.error('Failed to parse config: {error}'.format(error=error_msg))\n raise InvalidConfiguration(error_msg)\n\n if len(parsed) and parsed[0].getName() == 'file_delimiter':\n # Were parse nginx dump\n LOG.info('Switched to parse nginx configuration dump.')\n root_filename = self._prepare_dump(parsed)\n self.is_dump = True\n self.cwd = os.path.dirname(root_filename)\n parsed = self.configs[root_filename]\n\n self.parse_block(parsed, root)\n return root\n\n def parse_block(self, parsed_block, parent):\n for parsed in parsed_block:\n parsed_type = parsed.getName()\n parsed_name = parsed[0]\n parsed_args = parsed[1:]\n if parsed_type == 'include':\n # TODO: WTF?!\n self._resolve_include(parsed_args, parent)\n else:\n directive_inst = self.directive_factory(parsed_type, parsed_name, parsed_args)\n if directive_inst:\n parent.append(directive_inst)\n\n def directive_factory(self, parsed_type, parsed_name, parsed_args):\n klass = self._get_directive_class(parsed_type, parsed_name)\n if not klass:\n return None\n\n if klass.is_block:\n args = [to_native(v).strip() for v in parsed_args[0]]\n children = parsed_args[1]\n\n inst = klass(parsed_name, args)\n self.parse_block(children, inst)\n return inst\n else:\n args = [to_native(v).strip() for v in parsed_args]\n return klass(parsed_name, args)\n\n def _get_directive_class(self, parsed_type, parsed_name):\n if parsed_type in self.directives and parsed_name in self.directives[parsed_type]:\n return self.directives[parsed_type][parsed_name]\n elif parsed_type == 'block':\n return block.Block\n elif parsed_type == 'directive':\n return directive.Directive\n elif parsed_type == 'unparsed_block':\n LOG.warning('Skip unparseable block: \"%s\"', parsed_name)\n return None\n else:\n return None\n\n def _init_directives(self):\n self.directives['block'] = block.get_overrides()\n self.directives['directive'] = directive.get_overrides()\n\n def _resolve_include(self, args, parent):\n pattern = args[0]\n # TODO(buglloc): maybe file providers?\n if self.is_dump:\n return self._resolve_dump_include(pattern=pattern, parent=parent)\n if not self.allow_includes:\n LOG.debug('Includes are disallowed, skip: {0}'.format(pattern))\n return\n\n return self._resolve_file_include(pattern=pattern, parent=parent)\n\n def _resolve_file_include(self, pattern, parent):\n path = os.path.join(self.cwd, pattern)\n exists = False\n for file_path in glob.iglob(path):\n if not os.path.exists(file_path):\n continue\n exists = True\n include = block.IncludeBlock('include', [file_path])\n parent.append(include)\n self.parse_file(file_path, include)\n\n if not exists:\n LOG.warning('File not found: {0}'.format(path))\n\n def _resolve_dump_include(self, pattern, parent):\n path = os.path.join(self.cwd, pattern)\n founded = False\n for file_path, parsed in self.configs.items():\n if fnmatch.fnmatch(file_path, path):\n founded = True\n include = block.IncludeBlock('include', [file_path])\n parent.append(include)\n self.parse_block(parsed, include)\n\n if not founded:\n LOG.warning(\"File not found: {0}\".format(path))\n\n def _prepare_dump(self, parsed_block):\n filename = ''\n root_filename = ''\n for parsed in parsed_block:\n if parsed.getName() == 'file_delimiter':\n if not filename:\n root_filename = parsed[0]\n filename = parsed[0]\n self.configs[filename] = []\n continue\n self.configs[filename].append(parsed)\n return root_filename\n" }, { "path": "gixy/parser/raw_parser.py", "content": "import logging\nimport codecs\nimport six\nfrom cached_property import cached_property\n\nfrom pyparsing import (\n Literal, Suppress, White, Word, alphanums, Forward, Group, Optional, Combine,\n Keyword, OneOrMore, ZeroOrMore, Regex, QuotedString, nestedExpr, ParseResults)\n\nLOG = logging.getLogger(__name__)\n\n\nclass NginxQuotedString(QuotedString):\n def __init__(self, quoteChar):\n super(NginxQuotedString, self).__init__(quoteChar, escChar='\\\\', multiline=True)\n # Nginx parse quoted values in special manner:\n # '^https?:\\/\\/yandex\\.ru\\/\\00\\'\\\"' -> ^https?:\\/\\/yandex\\.ru\\/\\00'\"\n # TODO(buglloc): research and find another special characters!\n\n self.escCharReplacePattern = '\\\\\\\\(\\'|\")'\n\n\nclass RawParser(object):\n \"\"\"\n A class that parses nginx configuration with pyparsing\n \"\"\"\n\n def parse(self, data):\n \"\"\"\n Returns the parsed tree.\n \"\"\"\n if isinstance(data, six.binary_type):\n if data[:3] == codecs.BOM_UTF8:\n encoding = 'utf-8-sig'\n else:\n encoding = 'latin1'\n content = data.decode(encoding).strip()\n else:\n content = data.strip()\n\n if not content:\n return ParseResults()\n\n return self.script.parseString(content, parseAll=True)\n\n @cached_property\n def script(self):\n # constants\n left_bracket = Suppress(\"{\")\n right_bracket = Suppress(\"}\")\n semicolon = Suppress(\";\")\n space = White().suppress()\n keyword = Word(alphanums + \".+-_/\")\n path = Word(alphanums + \".-_/\")\n variable = Word(\"$_-\" + alphanums)\n value_wq = Regex(r'(?:\\([^\\s;]*\\)|\\$\\{\\w+\\}|[^\\s;(){}])+')\n value_sq = NginxQuotedString(quoteChar=\"'\")\n value_dq = NginxQuotedString(quoteChar='\"')\n value = (value_dq | value_sq | value_wq)\n # modifier for location uri [ = | ~ | ~* | ^~ ]\n location_modifier = (\n Keyword(\"=\") |\n Keyword(\"~*\") | Keyword(\"~\") |\n Keyword(\"^~\"))\n # modifier for if statement\n if_modifier = Combine(Optional(\"!\") + (\n Keyword(\"=\") |\n Keyword(\"~*\") | Keyword(\"~\") |\n (Literal(\"-\") + (Literal(\"f\") | Literal(\"d\") | Literal(\"e\") | Literal(\"x\")))))\n # This ugly workaround needed to parse unquoted regex with nested parentheses\n # so we capture all content between parentheses and then parse it :(\n # TODO(buglloc): may be use something better?\n condition_body = (\n (if_modifier + Optional(space) + value) |\n (variable + Optional(space + if_modifier + Optional(space) + value))\n )\n condition = Regex(r'\\((?:[^()\\n\\r\\\\]|(?:\\(.*\\))|(?:\\\\.))+?\\)')\\\n .setParseAction(lambda s, l, t: condition_body.parseString(t[0][1:-1]))\n\n # rules\n include = (\n Keyword(\"include\") +\n space +\n value +\n semicolon\n )(\"include\")\n\n directive = (\n keyword +\n ZeroOrMore(space + value) +\n semicolon\n )(\"directive\")\n\n file_delimiter = (\n Suppress(\"# configuration file \") +\n path +\n Suppress(\":\")\n )(\"file_delimiter\")\n\n comment = (\n Regex(r\"#.*\")\n )(\"comment\").setParseAction(_fix_comment)\n\n hash_value = Group(\n value +\n ZeroOrMore(space + value) +\n semicolon\n )(\"hash_value\")\n\n generic_block = Forward()\n if_block = Forward()\n location_block = Forward()\n hash_block = Forward()\n unparsed_block = Forward()\n\n sub_block = OneOrMore(Group(if_block |\n location_block |\n hash_block |\n generic_block |\n include |\n directive |\n file_delimiter |\n comment |\n unparsed_block))\n\n if_block << (\n Keyword(\"if\") +\n Group(condition) +\n Suppress(Optional(comment)) +\n Group(\n left_bracket +\n Optional(sub_block) +\n right_bracket)\n )(\"block\")\n\n location_block << (\n Keyword(\"location\") +\n Group(\n Optional(space + location_modifier) +\n Optional(space) + value) +\n Suppress(Optional(comment)) +\n Group(\n left_bracket +\n Optional(sub_block) +\n right_bracket)\n )(\"block\")\n\n hash_block << (\n keyword +\n Group(OneOrMore(space + value)) +\n Group(\n left_bracket +\n Optional(OneOrMore(hash_value)) +\n right_bracket)\n )(\"block\")\n\n generic_block << (\n keyword +\n Group(ZeroOrMore(space + value)) +\n Suppress(Optional(comment)) +\n Group(\n left_bracket +\n Optional(sub_block) +\n right_bracket)\n )(\"block\")\n\n unparsed_block << (\n keyword +\n Group(ZeroOrMore(space + value)) +\n nestedExpr(opener=\"{\", closer=\"}\")\n )(\"unparsed_block\")\n\n return sub_block\n\n\ndef _fix_comment(string, location, tokens):\n \"\"\"\n Returns \"cleared\" comment text\n\n :param string: original parse string\n :param location: location in the string where matching started\n :param tokens: list of the matched tokens, packaged as a ParseResults_ object\n :return: list of the cleared comment tokens\n \"\"\"\n\n comment = tokens[0][1:].strip()\n return [comment]\n" }, { "path": "gixy/plugins/__init__.py", "content": "" }, { "path": "gixy/plugins/add_header_multiline.py", "content": "import gixy\nfrom gixy.plugins.plugin import Plugin\n\n\nclass add_header_multiline(Plugin):\n \"\"\"\n Insecure example:\nadd_header Content-Security-Policy \"\n default-src: 'none';\n img-src data: https://mc.yandex.ru https://yastatic.net *.yandex.net https://mc.yandex.${tld} https://mc.yandex.ru;\n font-src data: https://yastatic.net;\";\n \"\"\"\n summary = 'Found a multi-line header.'\n severity = gixy.severity.LOW\n description = ('Multi-line headers are deprecated (see RFC 7230). '\n 'Some clients never supports them (e.g. IE/Edge).')\n help_url = 'https://github.com/yandex/gixy/blob/master/docs/en/plugins/addheadermultiline.md'\n directives = ['add_header', 'more_set_headers']\n\n def audit(self, directive):\n header_values = get_header_values(directive)\n for value in header_values:\n if '\\n\\x20' in value or '\\n\\t' in value:\n self.add_issue(directive=directive)\n break\n\n\ndef get_header_values(directive):\n if directive.name == 'add_header':\n return [directive.args[1]]\n\n # See headers more documentation: https://github.com/openresty/headers-more-nginx-module#description\n result = []\n skip_next = False\n for arg in directive.args:\n if arg in ['-s', '-t']:\n # Skip next value, because it's not a header\n skip_next = True\n elif arg.startswith('-'):\n # Skip any options\n pass\n elif skip_next:\n skip_next = False\n elif not skip_next:\n result.append(arg)\n return result\n" }, { "path": "gixy/plugins/add_header_redefinition.py", "content": "import gixy\nfrom gixy.plugins.plugin import Plugin\n\n\nclass add_header_redefinition(Plugin):\n \"\"\"\n Insecure example:\n server {\n add_header X-Content-Type-Options nosniff;\n location / {\n add_header X-Frame-Options DENY;\n }\n }\n \"\"\"\n summary = 'Nested \"add_header\" drops parent headers.'\n severity = gixy.severity.MEDIUM\n description = ('\"add_header\" replaces ALL parent headers. '\n 'See documentation: http://nginx.org/en/docs/http/ngx_http_headers_module.html#add_header')\n help_url = 'https://github.com/yandex/gixy/blob/master/docs/en/plugins/addheaderredefinition.md'\n directives = ['server', 'location', 'if']\n options = {'headers': set(['x-frame-options',\n 'x-content-type-options',\n 'x-xss-protection',\n 'content-security-policy',\n 'cache-control'])\n }\n\n def __init__(self, config):\n super(add_header_redefinition, self).__init__(config)\n self.interesting_headers = self.config.get('headers')\n\n def audit(self, directive):\n if not directive.is_block:\n # Skip all not block directives\n return\n\n actual_headers = get_headers(directive)\n if not actual_headers:\n return\n\n for parent in directive.parents:\n parent_headers = get_headers(parent)\n if not parent_headers:\n continue\n\n diff = (parent_headers - actual_headers) & self.interesting_headers\n\n if len(diff):\n self._report_issue(directive, parent, diff)\n\n break\n\n def _report_issue(self, current, parent, diff):\n directives = []\n # Add headers from parent level\n directives.extend(parent.find('add_header'))\n # Add headers from current level\n directives.extend(current.find('add_header'))\n reason = 'Parent headers \"{headers}\" was dropped in current level'.format(headers='\", \"'.join(diff))\n self.add_issue(directive=directives, reason=reason)\n\n\ndef get_headers(directive):\n headers = directive.find('add_header')\n if not headers:\n return set()\n\n return set(map(lambda d: d.header, headers))\n" }, { "path": "gixy/plugins/alias_traversal.py", "content": "import gixy\nfrom gixy.plugins.plugin import Plugin\n\n\nclass alias_traversal(Plugin):\n \"\"\"\n Insecure example:\n location /files {\n alias /home/;\n }\n \"\"\"\n summary = 'Path traversal via misconfigured alias.'\n severity = gixy.severity.HIGH\n description = 'Using alias in a prefixed location that doesn\\'t ends with directory separator could lead to path ' \\\n 'traversal vulnerability. '\n help_url = 'https://github.com/yandex/gixy/blob/master/docs/en/plugins/aliastraversal.md'\n directives = ['alias']\n\n def audit(self, directive):\n for location in directive.parents:\n if location.name != 'location':\n continue\n\n if not location.modifier or location.modifier == '^~':\n # We need non-strict prefixed locations\n if not location.path.endswith('/'):\n self.add_issue(\n severity=gixy.severity.HIGH if directive.path.endswith('/') else gixy.severity.MEDIUM,\n directive=[directive, location]\n )\n break\n" }, { "path": "gixy/plugins/host_spoofing.py", "content": "import gixy\nfrom gixy.plugins.plugin import Plugin\n\n\nclass host_spoofing(Plugin):\n \"\"\"\n Insecure example:\n proxy_set_header Host $http_host\n \"\"\"\n summary = 'The proxied Host header may be spoofed.'\n severity = gixy.severity.MEDIUM\n description = 'In most cases \"$host\" variable are more appropriate, just use it.'\n help_url = 'https://github.com/yandex/gixy/blob/master/docs/en/plugins/hostspoofing.md'\n directives = ['proxy_set_header']\n\n def audit(self, directive):\n name, value = directive.args\n if name.lower() != 'host':\n # Not a \"Host\" header\n return\n\n if value == '$http_host' or value.startswith('$arg_'):\n self.add_issue(directive=directive)\n" }, { "path": "gixy/plugins/http_splitting.py", "content": "import gixy\nfrom gixy.plugins.plugin import Plugin\nfrom gixy.core.variable import compile_script\n\n\nclass http_splitting(Plugin):\n r\"\"\"\n Insecure examples:\n rewrite ^ http://$host$uri;\n return 301 http://$host$uri;\n proxy_set_header \"X-Original-Uri\" $uri;\n proxy_pass http://upstream$document_uri;\n\n location ~ /proxy/(a|b)/(\\W*)$ {\n set $path $2;\n proxy_pass http://storage/$path;\n }\n \"\"\"\n\n summary = 'Possible HTTP-Splitting vulnerability.'\n severity = gixy.severity.HIGH\n description = 'Using variables that can contain \"\\\\n\" or \"\\\\r\" may lead to http injection.'\n help_url = 'https://github.com/yandex/gixy/blob/master/docs/en/plugins/httpsplitting.md'\n directives = ['rewrite', 'return', 'add_header', 'proxy_set_header', 'proxy_pass']\n\n def audit(self, directive):\n value = _get_value(directive)\n if not value:\n return\n\n server_side = directive.name.startswith('proxy_')\n for var in compile_script(value):\n char = ''\n if var.can_contain('\\n'):\n char = '\\\\n'\n elif not server_side and var.can_contain('\\r'):\n char = '\\\\r'\n else:\n continue\n reason = 'At least variable \"${var}\" can contain \"{char}\"'.format(var=var.name, char=char)\n self.add_issue(directive=[directive] + var.providers, reason=reason)\n\n\ndef _get_value(directive):\n if directive.name == 'proxy_pass' and len(directive.args) >= 1:\n return directive.args[0]\n elif len(directive.args) >= 2:\n return directive.args[1]\n return None\n" }, { "path": "gixy/plugins/origins.py", "content": "import re\nimport logging\nimport gixy\nfrom gixy.plugins.plugin import Plugin\nfrom gixy.core.regexp import Regexp\n\nLOG = logging.getLogger(__name__)\n\n\nclass origins(Plugin):\n r\"\"\"\n Insecure example:\n if ($http_referer !~ \"^https?://([^/]+metrika.*yandex\\.ru/\"){\n add_header X-Frame-Options SAMEORIGIN;\n }\n \"\"\"\n summary = 'Validation regex for \"origin\" or \"referrer\" matches untrusted domain.'\n severity = gixy.severity.MEDIUM\n description = 'Improve the regular expression to match only trusted referrers.'\n help_url = 'https://github.com/yandex/gixy/blob/master/docs/en/plugins/origins.md'\n directives = ['if']\n options = {\n 'domains': ['*'],\n 'https_only': False\n }\n\n def __init__(self, config):\n super(origins, self).__init__(config)\n if self.config.get('domains') and self.config.get('domains')[0] and self.config.get('domains')[0] != '*':\n domains = '|'.join(re.escape(d) for d in self.config.get('domains'))\n else:\n domains = r'[^/.]*\\.[^/]{2,7}'\n\n scheme = 'https{http}'.format(http=('?' if not self.config.get('https_only') else ''))\n regex = r'^{scheme}://(?:[^/.]*\\.){{0,10}}(?P{domains})(?::\\d*)?(?:/|\\?|$)'.format(\n scheme=scheme,\n domains=domains\n )\n self.valid_re = re.compile(regex)\n\n def audit(self, directive):\n if directive.operand not in ['~', '~*', '!~', '!~*']:\n # Not regexp\n return\n\n if directive.variable not in ['$http_referer', '$http_origin']:\n # Not interesting\n return\n\n invalid_referers = set()\n regexp = Regexp(directive.value, case_sensitive=(directive.operand in ['~', '!~']))\n for value in regexp.generate('/', anchored=True):\n if value.startswith('^'):\n value = value[1:]\n else:\n value = 'http://evil.com/' + value\n\n if value.endswith('$'):\n value = value[:-1]\n elif not value.endswith('/'):\n value += '.evil.com'\n\n valid = self.valid_re.match(value)\n if not valid or valid.group('domain') == 'evil.com':\n invalid_referers.add(value)\n\n if invalid_referers:\n invalid_referers = '\", \"'.join(invalid_referers)\n name = 'origin' if directive.variable == '$http_origin' else 'referrer'\n severity = gixy.severity.HIGH if directive.variable == '$http_origin' else gixy.severity.MEDIUM\n reason = 'Regex matches \"{value}\" as a valid {name}.'.format(value=invalid_referers, name=name)\n self.add_issue(directive=directive, reason=reason, severity=severity)\n" }, { "path": "gixy/plugins/plugin.py", "content": "import gixy\nfrom gixy.core.issue import Issue\n\n\nclass Plugin(object):\n summary = ''\n description = ''\n help_url = ''\n severity = gixy.severity.UNSPECIFIED\n directives = []\n options = {}\n\n def __init__(self, config):\n self._issues = []\n self.config = config\n\n def add_issue(self, directive, summary=None, severity=None, description=None, reason=None, help_url=None):\n self._issues.append(Issue(self, directives=directive, summary=summary, severity=severity,\n description=description, reason=reason, help_url=help_url))\n\n def audit(self, directive):\n pass\n\n @property\n def issues(self):\n return self._issues\n\n @property\n def name(self):\n return self.__class__.__name__\n" }, { "path": "gixy/plugins/ssrf.py", "content": "import re\n\nimport gixy\nfrom gixy.plugins.plugin import Plugin\nfrom gixy.core.context import get_context\nfrom gixy.core.variable import compile_script\n\n\nclass ssrf(Plugin):\n \"\"\"\n Insecure examples:\n location ~ /proxy/(.*)/(.*)/(.*)$ {\n set $scheme $1;\n set $host $2;\n set $path $3;\n proxy_pass $scheme://$host/$path;\n }\n\n location /proxy/ {\n proxy_pass $arg_some;\n }\n \"\"\"\n\n summary = 'Possible SSRF (Server Side Request Forgery) vulnerability.'\n severity = gixy.severity.HIGH\n description = 'The configuration may allow attacker to create a arbitrary requests from the vulnerable server.'\n help_url = 'https://github.com/yandex/gixy/blob/master/docs/en/plugins/ssrf.md'\n directives = ['proxy_pass']\n\n def __init__(self, config):\n super(ssrf, self).__init__(config)\n self.parse_uri_re = re.compile(r'(?P[^?#/)]+://)?(?P[^?#/)]+)')\n\n def audit(self, directive):\n value = directive.args[0]\n if not value:\n return\n\n context = get_context()\n if context.block.name == 'location' and context.block.is_internal:\n # Exclude internal locations\n return\n\n parsed = self.parse_uri_re.match(value)\n if not parsed:\n return\n\n res = self._check_script(parsed.group('scheme'), directive)\n if not res:\n self._check_script(parsed.group('host'), directive)\n\n def _check_script(self, script, directive):\n for var in compile_script(script):\n if var.must_contain('/'):\n # Skip variable checks\n return False\n if var.can_contain('.'):\n # Yay! Our variable can contain any symbols!\n reason = 'At least variable \"${var}\" can contain untrusted user input'.format(var=var.name)\n self.add_issue(directive=[directive] + var.providers, reason=reason)\n return True\n return False\n" }, { "path": "gixy/plugins/valid_referers.py", "content": "import gixy\nfrom gixy.plugins.plugin import Plugin\n\n\nclass valid_referers(Plugin):\n \"\"\"\n Insecure example:\n valid_referers none server_names *.webvisor.com;\n \"\"\"\n summary = 'Used \"none\" as valid referer.'\n severity = gixy.severity.HIGH\n description = 'Never trust undefined referer.'\n help_url = 'https://github.com/yandex/gixy/blob/master/docs/en/plugins/validreferers.md'\n directives = ['valid_referers']\n\n def audit(self, directive):\n if 'none' in directive.args:\n self.add_issue(directive=directive)\n" }, { "path": "gixy/utils/__init__.py", "content": "" }, { "path": "gixy/utils/text.py", "content": "from __future__ import absolute_import\nfrom six import PY3, text_type, binary_type\n\n\ndef to_bytes(obj, encoding='latin1', errors='strict', nonstring='replace'):\n if isinstance(obj, binary_type):\n return obj\n\n if isinstance(obj, text_type):\n try:\n # Try this first as it's the fastest\n return obj.encode(encoding, errors)\n except UnicodeEncodeError:\n return b'failed_to_encode'\n\n if nonstring == 'simplerepr':\n try:\n\n value = str(obj)\n except UnicodeError:\n try:\n value = repr(obj)\n except UnicodeError:\n # Giving up\n return b'failed_to_encode'\n elif nonstring == 'passthru':\n return obj\n elif nonstring == 'replace':\n return b'failed_to_encode'\n elif nonstring == 'strict':\n raise TypeError('obj must be a string type')\n else:\n raise TypeError('Invalid value %s for to_bytes\\' nonstring parameter' % nonstring)\n\n return to_bytes(value, encoding, errors)\n\n\ndef to_text(obj, encoding='latin1', errors='strict', nonstring='replace'):\n if isinstance(obj, text_type):\n return obj\n\n if isinstance(obj, binary_type):\n try:\n return obj.decode(encoding, errors)\n except UnicodeEncodeError:\n return u'failed_to_encode'\n\n if nonstring == 'simplerepr':\n try:\n value = str(obj)\n except UnicodeError:\n try:\n value = repr(obj)\n except UnicodeError:\n # Giving up\n return u'failed_to_encode'\n elif nonstring == 'passthru':\n return obj\n elif nonstring == 'replace':\n return u'failed_to_encode'\n elif nonstring == 'strict':\n raise TypeError('obj must be a string type')\n else:\n raise TypeError('Invalid value %s for to_text\\'s nonstring parameter' % nonstring)\n\n return to_text(value, encoding, errors)\n\n\nif PY3:\n to_native = to_text\nelse:\n to_native = to_bytes\n" }, { "path": "requirements.dev.txt", "content": "nose>=1.3.7\nmock>=2.0.0\ncoverage>=4.3\nflake8>=3.2\ntox>=2.7.0" }, { "path": "requirements.txt", "content": "pyparsing>=1.5.5,<3\ncached-property>=1.2.0\nargparse>=1.4.0\nsix>=1.1.0\nJinja2>=2.8\nConfigArgParse>=0.11.0" }, { "path": "rpm/gixy.spec", "content": "########################################################################################\n\nSummary: Nginx configuration static analyzer\nName: gixy\nVersion: 0.1.5\nRelease: 0%{?dist}\nLicense: MPLv2.0\nGroup: Development/Utilities\nURL: https://github.com/yandex/gixy\n\nSource: https://github.com/yandex/%{name}/archive/v%{version}.tar.gz\n\nBuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n)\n\nBuildArch: noarch\n\nBuildRequires: python-devel python-setuptools\n\nRequires: python-setuptools python-six >= 1.1.0 python-jinja >= 2.8\nRequires: python2-cached_property >= 1.2.0 python2-configargparse >= 0.11.0\nRequires: python-argparse >= 1.4.0 pyparsing >= 1.5.5 python-markupsafe\n\nProvides: %{name} = %{verion}-%{release}\n\n########################################################################################\n\n%description\nGixy is a tool to analyze Nginx configuration. The main goal of Gixy is to prevent \nmisconfiguration and automate flaw detection.\n\n########################################################################################\n\n%prep\n%setup -qn %{name}-%{version}\n\n%clean\nrm -rf %{buildroot}\n\n%build\npython setup.py build\n\n%install\nrm -rf %{buildroot}\npython setup.py install --prefix=%{_prefix} \\\n --root=%{buildroot}\n\n########################################################################################\n\n%files\n%defattr(-,root,root,-)\n%doc LICENSE AUTHORS README.md docs/*\n%{python_sitelib}/*\n%{_bindir}/%{name}\n\n########################################################################################\n\n%changelog\n* Sun May 21 2017 Yandex Team - 0.1.5-0\n- Supported Python 2.6\n- Supported multiple config files scanning\n- Fixed summary count\n- Fixed symlink resolution\n- Minor improvements and fixes\n\n* Sun May 14 2017 Yandex Team - 0.1.4-0\n- Allow processing stdin, file descriptors\n- Fixed configuration parser\n\n* Thu May 11 2017 Yandex Team - 0.1.3-0\n- Uses english versions in plugins references\n\n* Tue May 02 2017 Yandex Team - 0.1.2-0\n- Fixed blank comments parsing\n- Added \"auth_request_set\" directive\n\n* Sat Apr 29 2017 Yandex Team - 0.1.1-0\n- Initial build\n\n" }, { "path": "rpm/python-argparse.spec", "content": "########################################################################################\n\n%{!?python_sitelib: %global python_sitelib %(%{__python} -c \"from distutils.sysconfig import get_python_lib; print(get_python_lib())\")}\n\n########################################################################################\n\n%define pkg_name argparse\n%define pkg_version r140\n\n########################################################################################\n\nSummary: Python command-line parsing library\nName: python-argparse\nVersion: 1.4.0\nRelease: 0%{?dist}\nLicense: Python License\nGroup: Development/Libraries\nURL: https://github.com/ThomasWaldmann/argparse\n\nSource: https://github.com/ThomasWaldmann/%{pkg_name}/archive/%{pkg_version}.tar.gz\n\nBuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n)\n\nBuildArch: noarch\n\nBuildRequires: python >= 2.3 python-setuptools\n\nRequires: python >= 2.3 python-setuptools\n\nProvides: %{name} = %{verion}-%{release}\n\n########################################################################################\n\n%description\nThe argparse module makes it easy to write user friendly command line interfaces.\n\nThe program defines what arguments it requires, and argparse will figure out\nhow to parse those out of sys.argv. The argparse module also automatically\ngenerates help and usage messages and issues errors when users give the program\ninvalid arguments.\n\nAs of Python >= 2.7 and >= 3.2, the argparse module is maintained within the\nPython standard library. For users who still need to support Python < 2.7 or\n< 3.2, it is also provided as a separate package, which tries to stay\ncompatible with the module in the standard library, but also supports older\nPython versions.\n\nargparse is licensed under the Python license, for details see LICENSE.txt.\n\n########################################################################################\n\n%prep\n%setup -qn %{pkg_name}-%{pkg_version}\n\n%clean\nrm -rf %{buildroot}\n\n%build\npython setup.py build\n\n%install\nrm -rf %{buildroot}\npython setup.py install --prefix=%{_prefix} \\\n --single-version-externally-managed -O1 \\\n --root=%{buildroot}\n\n########################################################################################\n\n%files\n%defattr(-,root,root,-)\n%doc LICENSE.txt NEWS.txt README.txt\n%{python_sitelib}/*\n\n########################################################################################\n\n%changelog\n* Sat Apr 29 2017 Yandex Team - 1.4.0-0\n- Initial build\n\n" }, { "path": "setup.py", "content": "import re\nfrom setuptools import setup, find_packages\n\nwith open('gixy/__init__.py', 'r') as fd:\n version = re.search(r'^version\\s*=\\s*[\\'\"]([^\\'\"]*)[\\'\"]',\n fd.read(), re.MULTILINE).group(1)\n\nif not version:\n raise RuntimeError('Cannot find version information')\n\nsetup(\n name='gixy',\n version=version,\n description='Nginx configuration [sec]analyzer',\n keywords='nginx security lint static-analysis',\n author='Yandex IS Team',\n author_email='buglloc@yandex.ru',\n url='https://github.com/yandex/gixy',\n install_requires=[\n 'pyparsing>=1.5.5,<3',\n 'cached-property>=1.2.0',\n 'argparse>=1.4.0;python_version<\"3.2\"',\n 'six>=1.1.0',\n 'Jinja2>=2.8',\n 'ConfigArgParse>=0.11.0'\n ],\n entry_points={\n 'console_scripts': ['gixy=gixy.cli.main:main'],\n },\n test_suite='nose.collector',\n packages=find_packages(exclude=['tests', 'tests.*']),\n classifiers=[\n 'Development Status :: 3 - Alpha',\n 'Environment :: Console',\n 'Intended Audience :: System Administrators',\n 'Intended Audience :: Developers',\n 'Topic :: Security',\n 'Topic :: Software Development :: Quality Assurance',\n 'Topic :: Software Development :: Testing'\n ],\n include_package_data=True\n)\n" }, { "path": "tests/__init__.py", "content": "" }, { "path": "tests/asserts.py", "content": "from nose.tools import assert_true, assert_false\n\n\n'''\nVarious nose.tools helpers that doesn't exists in Python 2.6 Unittest :(\nMust be removed with drop Python 2.6 support\n'''\n\n\ndef assert_is_instance(obj, cls, msg=None):\n \"\"\"Same as assert_true(isinstance(obj, cls)), with a nicer\n default message.\"\"\"\n if not msg:\n msg = '{orig} is not an instance of {test}'.format(orig=type(obj), test=cls)\n assert_true(isinstance(obj, cls), msg=msg)\n\n\ndef assert_is_none(obj, msg=None):\n \"\"\"Same as assert_true(obj is None), with a nicer default message.\"\"\"\n if not msg:\n msg = '{orig!r} is not None'.format(orig=obj)\n assert_true(obj is None, msg=msg)\n\n\ndef assert_is_not_none(obj, msg=None):\n \"\"\"Same as assert_false(obj is None), with a nicer default message.\"\"\"\n if not msg:\n msg = '{orig!r} is None'.format(orig=obj)\n assert_false(obj is None, msg=msg)\n\n\ndef assert_in(member, container, msg=None):\n \"\"\"Just like assert_true(a in b), but with a nicer default message.\"\"\"\n if not msg:\n msg = '{member!r} not found in {container!r}'.format(member=member, container=container)\n assert_true(member in container, msg=msg)\n" }, { "path": "tests/core/__init__.py", "content": "" }, { "path": "tests/core/test_context.py", "content": "from nose.tools import with_setup, assert_equals, assert_not_equals, assert_true\nfrom gixy.core.context import get_context, pop_context, push_context, purge_context, CONTEXTS, Context\nfrom gixy.directives.block import Root\nfrom gixy.core.variable import Variable\nfrom gixy.core.regexp import Regexp\n\n\ndef setup():\n assert_equals(len(CONTEXTS), 0)\n\n\ndef tear_down():\n purge_context()\n\n\n@with_setup(setup, tear_down)\ndef test_push_pop_context():\n root_a = Root()\n push_context(root_a)\n assert_equals(len(CONTEXTS), 1)\n root_b = Root()\n push_context(root_b)\n assert_equals(len(CONTEXTS), 2)\n\n poped = pop_context()\n assert_equals(len(CONTEXTS), 1)\n assert_equals(poped.block, root_b)\n poped = pop_context()\n assert_equals(len(CONTEXTS), 0)\n assert_equals(poped.block, root_a)\n\n\n@with_setup(setup, tear_down)\ndef test_push_get_purge_context():\n root = Root()\n push_context(root)\n assert_equals(len(CONTEXTS), 1)\n assert_equals(get_context().block, root)\n root = Root()\n push_context(root)\n assert_equals(len(CONTEXTS), 2)\n assert_equals(get_context().block, root)\n\n purge_context()\n assert_equals(len(CONTEXTS), 0)\n\n\n@with_setup(setup, tear_down)\ndef test_add_variables():\n context = push_context(Root())\n assert_equals(len(context.variables['index']), 0)\n assert_equals(len(context.variables['name']), 0)\n\n one_str_var = Variable('1')\n context.add_var('1', one_str_var)\n one_int_var = Variable(1)\n context.add_var(1, one_int_var)\n some_var = Variable('some')\n context.add_var('some', some_var)\n\n assert_equals(len(context.variables['index']), 1)\n assert_equals(context.variables['index'][1], one_int_var)\n assert_equals(len(context.variables['name']), 1)\n assert_equals(context.variables['name']['some'], some_var)\n context.clear_index_vars()\n assert_equals(len(context.variables['index']), 0)\n assert_equals(len(context.variables['name']), 1)\n assert_equals(context.variables['name']['some'], some_var)\n\n\n@with_setup(setup, tear_down)\ndef test_get_variables():\n context = push_context(Root())\n assert_equals(len(context.variables['index']), 0)\n assert_equals(len(context.variables['name']), 0)\n\n one_var = Variable(1)\n context.add_var(1, one_var)\n some_var = Variable('some')\n context.add_var('some', some_var)\n\n assert_equals(context.get_var(1), one_var)\n assert_equals(context.get_var('some'), some_var)\n # Checks not existed variables, for now context may return None\n assert_equals(context.get_var(0), None)\n assert_equals(context.get_var('not_existed'), None)\n # Checks builtins variables\n assert_true(context.get_var('uri'))\n assert_true(context.get_var('document_uri'))\n assert_true(context.get_var('arg_asdsadasd'))\n assert_true(context.get_var('args'))\n\n\n@with_setup(setup, tear_down)\ndef test_context_depend_variables():\n push_context(Root())\n assert_equals(len(get_context().variables['index']), 0)\n assert_equals(len(get_context().variables['name']), 0)\n\n get_context().add_var(1, Variable(1, value='one'))\n get_context().add_var('some', Variable('some', value='some'))\n\n assert_equals(get_context().get_var(1).value, 'one')\n assert_equals(get_context().get_var('some').value, 'some')\n\n # Checks top context variables are still exists\n push_context(Root())\n assert_equals(get_context().get_var(1).value, 'one')\n assert_equals(get_context().get_var('some').value, 'some')\n\n # Checks variable overriding\n get_context().add_var('some', Variable('some', value='some_new'))\n get_context().add_var('foo', Variable('foo', value='foo'))\n assert_not_equals(get_context().get_var('some').value, 'some')\n assert_equals(get_context().get_var('some').value, 'some_new')\n assert_equals(get_context().get_var('foo').value, 'foo')\n assert_equals(get_context().get_var(1).value, 'one')\n\n # Checks variables after restore previous context\n pop_context()\n assert_not_equals(get_context().get_var('some').value, 'some_new')\n assert_equals(get_context().get_var('some').value, 'some')\n assert_equals(get_context().get_var('foo'), None)\n assert_equals(get_context().get_var(1).value, 'one')\n\n\n@with_setup(setup, tear_down)\ndef test_push_failed_with_regexp_py35_gixy_10():\n push_context(Root())\n assert_equals(len(get_context().variables['index']), 0)\n assert_equals(len(get_context().variables['name']), 0)\n\n regexp = Regexp('^/some/(.*?)')\n for name, group in regexp.groups.items():\n get_context().add_var(name, Variable(name=name, value=group))\n\n push_context(Root())\n" }, { "path": "tests/core/test_regexp.py", "content": "from nose.tools import assert_true, assert_false, assert_equals\nfrom gixy.core.regexp import Regexp\n\n'''\nCATEGORIES:\n sre_parse.CATEGORY_SPACE\n sre_parse.CATEGORY_NOT_SPACE\n sre_parse.CATEGORY_DIGIT\n sre_parse.CATEGORY_NOT_DIGIT\n sre_parse.CATEGORY_WORD\n sre_parse.CATEGORY_NOT_WORD\n ANY\n'''\n\n\ndef test_positive_contains():\n cases = (\n (r'[a-z]', 'a'),\n (r'[a-z]*', 'a'),\n (r'[a-z]*?', 'a'),\n (r'[a-z]+?', 'a'),\n (r'[a-z]', 'z'),\n (r'(?:a|b)', 'b'),\n (r'(/|:|[a-z])', 'g'),\n (r'[^a-z]', '/'),\n (r'[^a-z]', '\\n'),\n (r'[^0]', '9'),\n (r'[^0-2]', '3'),\n (r'[^0123a-z]', '9'),\n (r'\\s', '\\x20'),\n (r'[^\\s]', 'a'),\n (r'\\d', '1'),\n (r'[^\\d]', 'b'),\n (r'\\w', '_'),\n (r'[^\\w]', '\\n'),\n (r'\\W', '\\n'),\n (r'[^\\W]', 'a'),\n (r'.', 'a')\n )\n for case in cases:\n regexp, char = case\n yield check_positive_contain, regexp, char\n\n\ndef test_negative_contains():\n cases = (\n ('[a-z]', '1'),\n ('[a-z]*', '2'),\n ('[a-z]*?', '3'),\n ('[a-z]+?', '4'),\n ('[a-z]', '\\n'),\n ('(?:a|b)', 'c'),\n ('(/|:|[a-z])', '\\n'),\n ('[^a-z]', 'a'),\n ('[^0]', '0'),\n ('[^0-2]', '0'),\n ('[^0123a-z]', 'z'),\n (r'\\s', 'a'),\n (r'[^\\s]', '\\n'),\n (r'\\d', 'f'),\n (r'[^\\d]', '2'),\n (r'\\w', '\\n'),\n (r'[^\\w]', '_'),\n (r'\\W', 'a'),\n (r'[^\\W]', '\\n'),\n (r'.', '\\n')\n )\n for case in cases:\n regexp, char = case\n yield check_negative_contain, regexp, char\n\n\ndef test_groups_names():\n cases = (\n ('foo', [0]),\n ('(1)(2)(?:3)', [0, 1, 2]),\n ('(1)((2)|(?:3))', [0, 1, 2, 3]),\n (\"(?'pcre_7'1as)(?P(?2)|(?:3))\", [0, 1, 2, 3, 'pcre_7', 'outer', 'inner']),\n ('/proxy/(?.*)$', [0, 1, 'proxy'])\n )\n for case in cases:\n regexp, groups = case\n yield check_groups_names, regexp, groups\n\n\ndef test_to_string():\n cases = (\n (r'foo', 'foo'),\n (r'(1)(2)(?:3)', '(1)(2)(?:3)'),\n (r'(1)((2)|(?:3))', '(1)((?:(2)|(?:3)))'),\n (r'\\w|1|3-5|[a-z]', '(?:[\\w]|1|3\\\\-5|[a-z])'),\n (r'(1|(?:3)|([4-6]))', '((?:1|(?:3)|([4-6])))'),\n (r'(1|(?:3)|(?P[4-6]))', '((?:1|(?:3)|([4-6])))'),\n (r'^sss', '^sss'),\n (r'(^bb|11)$', '((?:^bb|11))$'),\n (r'(http|https)', '(http(?:|s))'),\n (r'1*', '1*'),\n (r'1*?', '1*?'),\n (r'1+', '1+'),\n )\n for case in cases:\n regexp, string = case\n yield check_to_string, regexp, string\n\n\ndef test_positive_startswith():\n cases = (\n (r'foo', 'q', False),\n (r'foo', 'f', True),\n (r'^foo', 'f', False),\n (r'(^foo)', 'f', False),\n (r'(^foo)', 'f', True),\n (r'(^foo|g)', 'f', True),\n (r'(^foo|g)', 'g', True),\n (r'(^foo|g)', 'q', False),\n (r'^[^/]+', '\\n', True),\n (r'/[^/]+', '/', True),\n (r'((a))', 'a', False),\n (r'((a))', 'b', False),\n (r'^[a-z]{0}0', '0', False),\n (r'^[a-z]{1}0', 'a', False),\n )\n for case in cases:\n regexp, check, strict = case\n yield check_positive_startswith, regexp, check, strict\n\n\ndef test_negative_startswith():\n cases = (\n (r'foo', '\\n', False),\n (r'foo', 'o', True),\n (r'^foo', 'o', False),\n (r'(^foo)', 'q', False),\n (r'(^foo)', 'q', True),\n (r'(^foo|g)', 'q', True),\n (r'(^foo|g)', 'o', True),\n (r'(^foo|g)', '\\n', False),\n (r'^[^/]+', '/', True),\n (r'/[^/]+', 'a', True),\n (r'((abc)|(ss))', 'b', True),\n (r'^[a-z]{0}0', 'a', False),\n (r'^[a-z]{0}0', 'g', False),\n )\n for case in cases:\n regexp, check, strict = case\n yield check_negative_startswith, regexp, check, strict\n\n\ndef test_positive_must_contain():\n cases = (\n (r'abc', 'a'),\n (r'abc', 'b'),\n (r'abc', 'c'),\n (r'3+', '3'),\n (r'[0]', '0'),\n (r'([0])', '0'),\n (r'(?:[0])', '0'),\n (r'(?:[0])|0|((((0))))', '0'),\n )\n for case in cases:\n regexp, char = case\n yield check_positive_must_contain, regexp, char\n\n\ndef test_negative_must_contain():\n cases = (\n (r'[a-z]', '1'),\n (r'2{0}1', '2'),\n (r'3?', '3'),\n (r'3*', '3'),\n (r'3*?', '3'),\n (r'3+a', 'b'),\n (r'[a-z]', 'a'),\n (r'(?:a|b)', 'a'),\n (r'(?:a|b)', 'b'),\n (r'(/|:|[a-z])', '/'),\n (r'(/|:|[a-z])', 'z'),\n (r'[^a-z]', '\\n'),\n (r'[^0]', '0'),\n (r'[^0-2]', '0'),\n (r'[^0123a-z]', 'z'),\n (r'\\s', '\\x20'),\n (r'[^\\s]', '\\n'),\n (r'\\d', '3'),\n (r'[^\\d]', 'a'),\n (r'\\w', 'a'),\n (r'[^\\w]', '\\n'),\n (r'\\W', '\\n'),\n (r'[^\\W]', 'a'),\n (r'.', '\\n')\n )\n for case in cases:\n regexp, char = case\n yield check_negative_must_contain, regexp, char\n\n\ndef test_positive_must_startswith():\n cases = (\n (r'foo', 'f', True),\n (r'^foo', 'f', False),\n (r'(^foo)', 'f', True),\n (r'^((a))', 'a', False),\n (r'((a))', 'a', True),\n (r'^[a-z]{0}0', '0', False),\n (r'^a{1}0', 'a', False),\n )\n for case in cases:\n regexp, check, strict = case\n yield check_positive_must_startswith, regexp, check, strict\n\n\ndef test_negative_must_startswith():\n cases = (\n (r'foo', 'o', False),\n (r'^foo', 'o', False),\n (r'(^foo)', 'o', False),\n (r'[a-z]', '1', True),\n (r'[a-z]', 'a', True),\n (r'/[^/]+', 'a', True),\n (r'3?', '3', True),\n (r'3*', '3', True),\n (r'3*?', '3', True),\n (r'3+a', 'b', True),\n (r'^((a))', 'b', False),\n (r'((a))', 'a', False),\n (r'^a{0}0', 'a', False),\n )\n for case in cases:\n regexp, check, strict = case\n yield check_negative_must_startswith, regexp, check, strict\n\n\ndef test_generate():\n cases = (\n (r'foo', ['foo']),\n (r'^sss', ['^sss']),\n (r'(1)(2)(3)', ['123']),\n (r'(1)((2)|(?:3))', ['12', '13']),\n (r'(^1?2?|aa/)', ['^', '^1', '^2', '^12', 'aa/']),\n (r'^https?://yandex.ru', ['^http://yandex|ru', '^https://yandex|ru']),\n (r'(^bb|11)$', ['^bb$', '11$']),\n (r'(http|https)', ['http', 'https']),\n (r'1*', ['', '11111']),\n (r'1*?', ['', '11111']),\n (r'1[0]?2', ['102', '12']),\n (r'1[0]2', ['102']),\n (r'1+', ['11111']),\n (r'[^/]?', ['', '|']),\n (r'^http://(foo|bar)|baz', ['^http://foo', '^http://bar', 'baz']),\n (r'[^\\x00-\\x7b|\\x7e-\\xff]', ['\\x7d']),\n (r'(a|b|c)', ['a', 'b', 'c']),\n (r'[xyz]', ['x', 'y', 'z'])\n )\n for case in cases:\n regexp, values = case\n yield check_generate, regexp, values\n\n\ndef test_strict_generate():\n reg = Regexp('^foo|bar', strict=True)\n assert_equals(sorted(reg.generate('|', anchored=True)), sorted(['^foo', '^bar']))\n\n\ndef test_gen_anchor():\n\n reg = Regexp('^some$')\n val = next(reg.generate('', anchored=False))\n assert_equals(val, 'some')\n\n reg = Regexp('^some$')\n val = next(reg.generate('', anchored=True))\n assert_equals(val, '^some$')\n\n reg = Regexp('^some$', strict=True)\n val = next(reg.generate('', anchored=False))\n assert_equals(val, 'some')\n\n reg = Regexp('^some$', strict=True)\n val = next(reg.generate('', anchored=True))\n assert_equals(val, '^some$')\n\n\ndef test_group_can_contains():\n source = '/some/(?P[^/:.]+)/'\n reg = Regexp(source)\n assert_true(reg.can_contain('\\n'),\n 'Whole regex \"{src}\" can contains {sym!r}'.format(src=source, sym='\\\\n'))\n\n assert_true(reg.group(0).can_contain('\\n'),\n 'Group 0 from regex \"{src}\" can contains {sym!r}'.format(src=source, sym='\\\\n'))\n\n assert_true(reg.group('action').can_contain('\\n'),\n 'Group \"action\" from regex \"{src}\" can contains {sym!r}'.format(src=source, sym='\\\\n'))\n\n assert_true(reg.group(1).can_contain('\\n'),\n 'Group 1 from regex \"{src}\" can contains {sym!r}'.format(src=source, sym='\\\\n'))\n\n assert_false(reg.group('action').can_contain('/'),\n 'Group \"action\" from regex \"{src}\" CAN\\'T (!) contain {sym!r}'.format(src=source, sym='/'))\n\n\ndef check_positive_contain(regexp, char):\n reg = Regexp(regexp, case_sensitive=True)\n assert_true(reg.can_contain(char),\n '{reg!r} should contain {chr!r}'.format(reg=regexp, chr=char))\n\n reg = Regexp(regexp, case_sensitive=False)\n char = char.upper()\n assert_true(reg.can_contain(char),\n '{reg!r} (case insensitive) should contain {chr!r}'.format(reg=regexp, chr=char))\n\n\ndef check_negative_contain(regexp, char):\n reg = Regexp(regexp, case_sensitive=True)\n assert_false(reg.can_contain(char),\n '{reg!r} should not contain {chr!r}'.format(reg=regexp, chr=char))\n\n reg = Regexp(regexp, case_sensitive=False)\n char = char.upper()\n assert_false(reg.can_contain(char),\n '{reg!r} (case insensitive) should not contain {chr!r}'.format(reg=regexp, chr=char))\n\n\ndef check_positive_startswith(regexp, char, strict):\n reg = Regexp(regexp, case_sensitive=True, strict=strict)\n assert_true(reg.can_startswith(char),\n '{reg!r} can start\\'s with {chr!r}'.format(reg=regexp, chr=char))\n\n reg = Regexp(regexp, case_sensitive=False, strict=strict)\n char = char.upper()\n assert_true(reg.can_startswith(char),\n '{reg!r} (case insensitive) can start\\'s with {chr!r}'.format(reg=regexp, chr=char))\n\n\ndef check_negative_startswith(regexp, char, strict):\n reg = Regexp(regexp, case_sensitive=True, strict=strict)\n assert_false(reg.can_startswith(char),\n '{reg!r} can\\'t start\\'s with {chr!r}'.format(reg=regexp, chr=char))\n\n reg = Regexp(regexp, case_sensitive=False, strict=strict)\n char = char.upper()\n assert_false(reg.can_startswith(char),\n '{reg!r} (case insensitive) can\\'t start\\'s with {chr!r}'.format(reg=regexp, chr=char))\n\n\ndef check_groups_names(regexp, groups):\n reg = Regexp(regexp)\n assert_equals(set(reg.groups.keys()), set(groups))\n\n\ndef check_to_string(regexp, string):\n reg = Regexp(regexp)\n assert_equals(str(reg), string)\n\n\ndef check_positive_must_contain(regexp, char):\n reg = Regexp(regexp, case_sensitive=True)\n assert_true(reg.must_contain(char),\n '{reg!r} must contain with {chr!r}'.format(reg=regexp, chr=char))\n\n reg = Regexp(regexp, case_sensitive=False)\n char = char.upper()\n assert_true(reg.must_contain(char),\n '{reg!r} (case insensitive) must contain with {chr!r}'.format(reg=regexp, chr=char))\n\n\ndef check_negative_must_contain(regexp, char):\n reg = Regexp(regexp, case_sensitive=True)\n assert_false(reg.must_contain(char),\n '{reg!r} must NOT contain with {chr!r}'.format(reg=regexp, chr=char))\n\n reg = Regexp(regexp, case_sensitive=False)\n char = char.upper()\n assert_false(reg.must_contain(char),\n '{reg!r} (case insensitive) must NOT contain with {chr!r}'.format(reg=regexp, chr=char))\n\n\ndef check_positive_must_startswith(regexp, char, strict):\n reg = Regexp(regexp, case_sensitive=True, strict=strict)\n assert_true(reg.must_startswith(char),\n '{reg!r} MUST start\\'s with {chr!r}'.format(reg=regexp, chr=char))\n\n reg = Regexp(regexp, case_sensitive=False, strict=strict)\n char = char.upper()\n assert_true(reg.must_startswith(char),\n '{reg!r} (case insensitive) MUST start\\'s with {chr!r}'.format(reg=regexp, chr=char))\n\n\ndef check_negative_must_startswith(regexp, char, strict):\n reg = Regexp(regexp, case_sensitive=True, strict=strict)\n assert_false(reg.must_startswith(char),\n '{reg!r} MUST NOT start\\'s with {chr!r}'.format(reg=regexp, chr=char))\n\n reg = Regexp(regexp, case_sensitive=False, strict=strict)\n char = char.upper()\n assert_false(reg.must_startswith(char),\n '{reg!r} (case insensitive) MUST NOT start\\'s with {chr!r}'.format(reg=regexp, chr=char))\n\n\ndef check_generate(regexp, values):\n reg = Regexp(regexp)\n assert_equals(sorted(reg.generate('|', anchored=True)), sorted(values))\n" }, { "path": "tests/core/test_variable.py", "content": "from nose.tools import assert_true, assert_false, assert_equals, with_setup\nfrom gixy.core.context import get_context, push_context, purge_context\nfrom gixy.directives.block import Root\nfrom gixy.core.regexp import Regexp\nfrom gixy.core.variable import Variable\n\ndef setup():\n push_context(Root())\n\n\ndef tear_down():\n purge_context()\n\n\n@with_setup(setup, tear_down)\ndef test_literal():\n var = Variable(name='simple', value='$uri', have_script=False)\n assert_false(var.depends)\n assert_false(var.regexp)\n assert_equals(var.value, '$uri')\n\n assert_false(var.can_startswith('$'))\n assert_false(var.can_contain('i'))\n assert_true(var.must_contain('$'))\n assert_true(var.must_contain('u'))\n assert_false(var.must_contain('a'))\n assert_true(var.must_startswith('$'))\n assert_false(var.must_startswith('u'))\n\n\n@with_setup(setup, tear_down)\ndef test_regexp():\n var = Variable(name='simple', value=Regexp('^/.*'))\n assert_false(var.depends)\n assert_true(var.regexp)\n\n assert_true(var.can_startswith('/'))\n assert_false(var.can_startswith('a'))\n assert_true(var.can_contain('a'))\n assert_false(var.can_contain('\\n'))\n assert_true(var.must_contain('/'))\n assert_false(var.must_contain('a'))\n assert_true(var.must_startswith('/'))\n assert_false(var.must_startswith('a'))\n\n\n@with_setup(setup, tear_down)\ndef test_script():\n get_context().add_var('foo', Variable(name='foo', value=Regexp('.*')))\n var = Variable(name='simple', value='/$foo')\n assert_true(var.depends)\n assert_false(var.regexp)\n\n assert_false(var.can_startswith('/'))\n assert_false(var.can_startswith('a'))\n assert_true(var.can_contain('/'))\n assert_true(var.can_contain('a'))\n assert_false(var.can_contain('\\n'))\n assert_true(var.must_contain('/'))\n assert_false(var.must_contain('a'))\n assert_true(var.must_startswith('/'))\n assert_false(var.must_startswith('a'))\n\n\n@with_setup(setup, tear_down)\ndef test_regexp_boundary():\n var = Variable(name='simple', value=Regexp('.*'), boundary=Regexp('/[a-z]', strict=True))\n assert_false(var.depends)\n assert_true(var.regexp)\n\n assert_true(var.can_startswith('/'))\n assert_false(var.can_startswith('a'))\n assert_false(var.can_contain('/'))\n assert_true(var.can_contain('a'))\n assert_false(var.can_contain('0'))\n assert_false(var.can_contain('\\n'))\n assert_true(var.must_contain('/'))\n assert_false(var.must_contain('a'))\n assert_true(var.must_startswith('/'))\n assert_false(var.must_startswith('a'))\n\n\n@with_setup(setup, tear_down)\ndef test_script_boundary():\n get_context().add_var('foo', Variable(name='foo', value=Regexp('.*'), boundary=Regexp('[a-z]', strict=True)))\n var = Variable(name='simple', value='/$foo', boundary=Regexp('[/a-z0-9]', strict=True))\n assert_true(var.depends)\n assert_false(var.regexp)\n\n assert_false(var.can_startswith('/'))\n assert_false(var.can_startswith('a'))\n assert_false(var.can_contain('/'))\n assert_true(var.can_contain('a'))\n assert_false(var.can_contain('\\n'))\n assert_false(var.can_contain('0'))\n assert_true(var.must_contain('/'))\n assert_false(var.must_contain('a'))\n assert_true(var.must_startswith('/'))\n assert_false(var.must_startswith('a'))\n" }, { "path": "tests/directives/__init__.py", "content": "" }, { "path": "tests/directives/test_block.py", "content": "from nose.tools import assert_equals, assert_true, assert_false\nfrom tests.asserts import assert_is_instance, assert_is_none, assert_is_not_none\nfrom gixy.parser.nginx_parser import NginxParser\nfrom gixy.directives.block import *\n\n# TODO(buglloc): what about include block?\n\n\ndef _get_parsed(config):\n root = NginxParser(cwd='', allow_includes=False).parse(config)\n return root.children[0]\n\n\ndef test_block():\n config = 'some {some;}'\n\n directive = _get_parsed(config)\n assert_is_instance(directive, Block)\n assert_true(directive.is_block)\n assert_true(directive.self_context)\n assert_false(directive.provide_variables)\n\n\ndef test_http():\n config = '''\nhttp {\n default_type application/octet-stream;\n sendfile on;\n keepalive_timeout 65;\n}\n '''\n\n directive = _get_parsed(config)\n assert_is_instance(directive, HttpBlock)\n assert_true(directive.is_block)\n assert_true(directive.self_context)\n assert_false(directive.provide_variables)\n\n\ndef test_server():\n config = '''\nserver {\n listen 80;\n server_name _;\n server_name cool.io;\n}\n\n '''\n\n directive = _get_parsed(config)\n assert_is_instance(directive, ServerBlock)\n assert_true(directive.is_block)\n assert_true(directive.self_context)\n assert_equals([d.args[0] for d in directive.get_names()], ['_', 'cool.io'])\n assert_false(directive.provide_variables)\n\n\ndef test_location():\n config = '''\nlocation / {\n}\n '''\n\n directive = _get_parsed(config)\n assert_is_instance(directive, LocationBlock)\n assert_true(directive.is_block)\n assert_true(directive.self_context)\n assert_true(directive.provide_variables)\n assert_is_none(directive.modifier)\n assert_equals(directive.path, '/')\n assert_false(directive.is_internal)\n\n\ndef test_location_internal():\n config = '''\nlocation / {\n internal;\n}\n '''\n\n directive = _get_parsed(config)\n assert_is_instance(directive, LocationBlock)\n assert_true(directive.is_internal)\n\n\ndef test_location_modifier():\n config = '''\nlocation = / {\n}\n '''\n\n directive = _get_parsed(config)\n assert_is_instance(directive, LocationBlock)\n assert_equals(directive.modifier, '=')\n assert_equals(directive.path, '/')\n\n\ndef test_if():\n config = '''\nif ($some) {\n}\n '''\n\n directive = _get_parsed(config)\n assert_is_instance(directive, IfBlock)\n assert_true(directive.is_block)\n assert_false(directive.self_context)\n assert_false(directive.provide_variables)\n assert_equals(directive.variable, '$some')\n assert_is_none(directive.operand)\n assert_is_none(directive.value)\n\n\ndef test_if_modifier():\n config = '''\nif (-f /some) {\n}\n '''\n\n directive = _get_parsed(config)\n assert_is_instance(directive, IfBlock)\n assert_equals(directive.operand, '-f')\n assert_equals(directive.value, '/some')\n assert_is_none(directive.variable)\n\n\ndef test_if_variable():\n config = '''\nif ($http_some = '/some') {\n}\n '''\n\n directive = _get_parsed(config)\n assert_is_instance(directive, IfBlock)\n assert_equals(directive.variable, '$http_some')\n assert_equals(directive.operand, '=')\n assert_equals(directive.value, '/some')\n\n\ndef test_block_some_flat():\n config = '''\n some {\n default_type application/octet-stream;\n sendfile on;\n if (-f /some/) {\n keepalive_timeout 65;\n }\n }\n '''\n\n directive = _get_parsed(config)\n for d in ['default_type', 'sendfile', 'keepalive_timeout']:\n c = directive.some(d, flat=True)\n assert_is_not_none(c)\n assert_equals(c.name, d)\n\n\ndef test_block_some_not_flat():\n config = '''\n some {\n default_type application/octet-stream;\n sendfile on;\n if (-f /some/) {\n keepalive_timeout 65;\n }\n }\n '''\n\n directive = _get_parsed(config)\n c = directive.some('keepalive_timeout', flat=False)\n assert_is_none(c)\n\n\ndef test_block_find_flat():\n config = '''\n some {\n directive 1;\n if (-f /some/) {\n directive 2;\n }\n }\n '''\n\n directive = _get_parsed(config)\n finds = directive.find('directive', flat=True)\n assert_equals(len(finds), 2)\n assert_equals([x.name for x in finds], ['directive', 'directive'])\n assert_equals([x.args[0] for x in finds], ['1', '2'])\n\n\ndef test_block_find_not_flat():\n config = '''\n some {\n directive 1;\n if (-f /some/) {\n directive 2;\n }\n }\n '''\n\n directive = _get_parsed(config)\n finds = directive.find('directive', flat=False)\n assert_equals(len(finds), 1)\n assert_equals([x.name for x in finds], ['directive'])\n assert_equals([x.args[0] for x in finds], ['1'])\n\n\ndef test_block_map():\n config = '''\nmap $some_var $some_other_var {\n a b;\n default c;\n}\n '''\n\n directive = _get_parsed(config)\n assert_is_instance(directive, MapBlock)\n assert_true(directive.is_block)\n assert_false(directive.self_context)\n assert_true(directive.provide_variables)\n assert_equals(directive.variable, 'some_other_var')\n\n\ndef test_block_geo_two_vars():\n config = '''\ngeo $some_var $some_other_var {\n 1.2.3.4 b;\n default c;\n}\n '''\n\n directive = _get_parsed(config)\n assert_is_instance(directive, GeoBlock)\n assert_true(directive.is_block)\n assert_false(directive.self_context)\n assert_true(directive.provide_variables)\n assert_equals(directive.variable, 'some_other_var')\n\n\ndef test_block_geo_one_var():\n config = '''\ngeo $some_var {\n 5.6.7.8 d;\n default e;\n}\n '''\n\n directive = _get_parsed(config)\n assert_is_instance(directive, GeoBlock)\n assert_true(directive.is_block)\n assert_false(directive.self_context)\n assert_true(directive.provide_variables)\n assert_equals(directive.variable, 'some_var')\n" }, { "path": "tests/directives/test_directive.py", "content": "from nose.tools import assert_equals, assert_false, assert_true\nfrom tests.asserts import assert_is_instance\nfrom gixy.parser.nginx_parser import NginxParser\nfrom gixy.directives.directive import *\n\n\ndef _get_parsed(config):\n root = NginxParser(cwd='', allow_includes=False).parse(config)\n return root.children[0]\n\n\ndef test_directive():\n config = 'some \"foo\" \"bar\";'\n\n directive = _get_parsed(config)\n assert_is_instance(directive, Directive)\n assert_equals(directive.name, 'some')\n assert_equals(directive.args, ['foo', 'bar'])\n assert_equals(str(directive), 'some foo bar;')\n\n\ndef test_add_header():\n config = 'add_header \"X-Foo\" \"bar\";'\n\n directive = _get_parsed(config)\n assert_is_instance(directive, AddHeaderDirective)\n assert_equals(directive.name, 'add_header')\n assert_equals(directive.args, ['X-Foo', 'bar'])\n assert_equals(directive.header, 'x-foo')\n assert_equals(directive.value, 'bar')\n assert_false(directive.always)\n assert_equals(str(directive), 'add_header X-Foo bar;')\n\n\ndef test_add_header_always():\n config = 'add_header \"X-Foo\" \"bar\" always;'\n\n directive = _get_parsed(config)\n assert_is_instance(directive, AddHeaderDirective)\n assert_equals(directive.name, 'add_header')\n assert_equals(directive.args, ['X-Foo', 'bar', 'always'])\n assert_equals(directive.header, 'x-foo')\n assert_equals(directive.value, 'bar')\n assert_true(directive.always)\n assert_equals(str(directive), 'add_header X-Foo bar always;')\n\n\ndef test_set():\n config = 'set $foo bar;'\n\n directive = _get_parsed(config)\n assert_is_instance(directive, SetDirective)\n assert_equals(directive.name, 'set')\n assert_equals(directive.args, ['$foo', 'bar'])\n assert_equals(directive.variable, 'foo')\n assert_equals(directive.value, 'bar')\n assert_equals(str(directive), 'set $foo bar;')\n assert_true(directive.provide_variables)\n\n\ndef test_rewrite():\n config = 'rewrite ^ http://some;'\n\n directive = _get_parsed(config)\n assert_is_instance(directive, RewriteDirective)\n assert_equals(directive.name, 'rewrite')\n assert_equals(directive.args, ['^', 'http://some'])\n assert_equals(str(directive), 'rewrite ^ http://some;')\n assert_true(directive.provide_variables)\n\n assert_equals(directive.pattern, '^')\n assert_equals(directive.replace, 'http://some')\n assert_equals(directive.flag, None)\n\n\ndef test_rewrite_flags():\n config = 'rewrite ^/(.*)$ http://some/$1 redirect;'\n\n directive = _get_parsed(config)\n assert_is_instance(directive, RewriteDirective)\n assert_equals(directive.name, 'rewrite')\n assert_equals(directive.args, ['^/(.*)$', 'http://some/$1', 'redirect'])\n assert_equals(str(directive), 'rewrite ^/(.*)$ http://some/$1 redirect;')\n assert_true(directive.provide_variables)\n\n assert_equals(directive.pattern, '^/(.*)$')\n assert_equals(directive.replace, 'http://some/$1')\n assert_equals(directive.flag, 'redirect')\n\n\ndef test_root():\n config = 'root /var/www/html;'\n\n directive = _get_parsed(config)\n assert_is_instance(directive, RootDirective)\n assert_equals(directive.name, 'root')\n assert_equals(directive.args, ['/var/www/html'])\n assert_equals(str(directive), 'root /var/www/html;')\n assert_true(directive.provide_variables)\n\n assert_equals(directive.path, '/var/www/html')\n" }, { "path": "tests/parser/__init__.py", "content": "" }, { "path": "tests/parser/test_nginx_parser.py", "content": "from nose.tools import assert_equal\nfrom tests.asserts import assert_is_instance\nfrom gixy.parser.nginx_parser import NginxParser\nfrom gixy.directives.directive import *\nfrom gixy.directives.block import *\n\n\ndef _parse(config):\n return NginxParser(cwd='', allow_includes=False).parse(config)\n\n\ndef test_directive():\n configs = [\n 'access_log syslog:server=127.0.0.1,tag=nginx_sentry toolsformat;',\n 'user http;',\n 'internal;',\n 'set $foo \"bar\";',\n \"set $foo 'bar';\",\n 'proxy_pass http://unix:/run/sock.socket;',\n 'rewrite ^/([a-zA-Z0-9]+)$ /$1/${arg_v}.pb break;'\n ]\n\n expected = [\n [Directive],\n [Directive],\n [Directive],\n [Directive, SetDirective],\n [Directive],\n [Directive, RewriteDirective]\n ]\n\n for i, config in enumerate(configs):\n return assert_config, config, expected[i]\n\n\ndef test_blocks():\n configs = [\n 'if (-f /some) {}',\n 'location / {}'\n ]\n\n expected = [\n [Directive, Block, IfBlock],\n [Directive, Block, LocationBlock],\n ]\n\n for i, config in enumerate(configs):\n yield assert_config, config, expected[i]\n\n\ndef test_dump_simple():\n config = '''\n# configuration file /etc/nginx/nginx.conf:\nhttp {\n include sites/*.conf;\n}\n\n# configuration file /etc/nginx/conf.d/listen:\nlisten 80;\n\n# configuration file /etc/nginx/sites/default.conf:\nserver {\n include conf.d/listen;\n}\n '''\n\n tree = _parse(config)\n assert_is_instance(tree, Directive)\n assert_is_instance(tree, Block)\n assert_is_instance(tree, Root)\n\n assert_equal(len(tree.children), 1)\n http = tree.children[0]\n assert_is_instance(http, Directive)\n assert_is_instance(http, Block)\n assert_is_instance(http, HttpBlock)\n\n assert_equal(len(http.children), 1)\n include_server = http.children[0]\n assert_is_instance(include_server, Directive)\n assert_is_instance(include_server, IncludeBlock)\n assert_equal(include_server.file_path, '/etc/nginx/sites/default.conf')\n\n assert_equal(len(include_server.children), 1)\n server = include_server.children[0]\n assert_is_instance(server, Directive)\n assert_is_instance(server, Block)\n assert_is_instance(server, ServerBlock)\n\n assert_equal(len(server.children), 1)\n include_listen = server.children[0]\n assert_is_instance(include_listen, Directive)\n assert_is_instance(include_listen, IncludeBlock)\n assert_equal(include_listen.file_path, '/etc/nginx/conf.d/listen')\n\n assert_equal(len(include_listen.children), 1)\n listen = include_listen.children[0]\n assert_is_instance(listen, Directive)\n assert_equal(listen.args, ['80'])\n\n\ndef test_encoding():\n configs = [\n 'bar \"\\xD1\\x82\\xD0\\xB5\\xD1\\x81\\xD1\\x82\";'\n ]\n\n for i, config in enumerate(configs):\n _parse(config)\n\n\ndef assert_config(config, expected):\n tree = _parse(config)\n assert_is_instance(tree, Directive)\n assert_is_instance(tree, Block)\n assert_is_instance(tree, Root)\n\n child = tree.children[0]\n for ex in expected:\n assert_is_instance(child, ex)\n" }, { "path": "tests/parser/test_raw_parser.py", "content": "from nose.tools import assert_equals\nfrom gixy.parser.raw_parser import *\n\n\ndef test_directive():\n config = '''\naccess_log syslog:server=127.0.0.1,tag=nginx_sentry toolsformat;\nuser http;\ninternal;\nset $foo \"bar\";\nset $foo 'bar';\nproxy_pass http://unix:/run/sock.socket;\nrewrite ^/([a-zA-Z0-9]+)$ /$1/${arg_v}.pb break;\n server_name some.tld ~^(www\\.)?podberi.(?:ru|com|ua)$\n ~^(www\\.)?guru.yandex.ru$;\n '''\n\n expected = [\n ['access_log', 'syslog:server=127.0.0.1,tag=nginx_sentry', 'toolsformat'],\n ['user', 'http'],\n ['internal'],\n ['set', '$foo', 'bar'],\n ['set', '$foo', 'bar'],\n ['proxy_pass', 'http://unix:/run/sock.socket'],\n ['rewrite', '^/([a-zA-Z0-9]+)$', '/$1/${arg_v}.pb', 'break'],\n ['server_name', 'some.tld', '~^(www\\.)?podberi.(?:ru|com|ua)$', '~^(www\\.)?guru.yandex.ru$']\n ]\n\n assert_config(config, expected)\n\n\ndef test_block():\n config = '''\nhttp {\n}\n '''\n\n expected = [['http', [], []]]\n assert_config(config, expected)\n\n\ndef test_block_with_child():\n config = '''\nhttp {\n gzip on;\n}\n '''\n\n expected = [['http', [], [['gzip', 'on']]]]\n assert_config(config, expected)\n\n\ndef test_location_simple():\n config = '''\nlocation / {\n}\nlocation = /foo {\n}\nlocation ~ ^/bar {\n}\nlocation ~* ^/baz$ {\n}\nlocation ^~ ^/bazz {\n}\n# Whitespace may be omitted:((\nlocation ~\\.(js|css)$ {\n}\n '''\n\n expected = [['location', ['/'], []],\n ['location', ['=', '/foo'], []],\n ['location', ['~', '^/bar'], []],\n ['location', ['~*', '^/baz$'], []],\n ['location', ['^~', '^/bazz'], []],\n ['Whitespace may be omitted:(('],\n ['location', ['~', '\\.(js|css)$'], []]]\n\n assert_config(config, expected)\n\n\ndef test_quoted_strings():\n config = '''\nsome_sq '\\\\'la\\\\.\\\\/\\\\\"';\nsome_dq '\\\\'la\\\\.\\\\/\\\\\"';\n '''\n\n expected = [['some_sq', '\\'la\\\\.\\\\/\\\"'],\n ['some_dq', '\\'la\\\\.\\\\/\\\"']]\n\n assert_config(config, expected)\n\n\ndef test_location_child():\n config = '''\nlocation = /foo {\n proxy_pass http://unix:/run/sock.socket;\n}\n '''\n\n expected = [['location', ['=', '/foo'], [\n ['proxy_pass', 'http://unix:/run/sock.socket']\n ]]]\n assert_config(config, expected)\n\n\ndef test_nested_location():\n config = '''\nlocation ~* ^/foo {\n location = /foo/bar {\n internal;\n proxy_pass http://any.yandex.ru;\n }\n\n location = /foo/baz {\n proxy_pass upstream;\n }\n}\n '''\n\n expected = [['location', ['~*', '^/foo'], [\n ['location', ['=', '/foo/bar'], [\n ['internal'],\n ['proxy_pass', 'http://any.yandex.ru']\n ]],\n ['location', ['=', '/foo/baz'], [\n ['proxy_pass', 'upstream']\n ]],\n ]]]\n\n assert_config(config, expected)\n\n\ndef test_hash_block():\n config = '''\ngeo $geo {\n default 0;\n\n 127.0.0.1 2;\n 192.168.1.0/24 1;\n 10.1.0.0/16 1;\n\n ::1 2;\n 2001:0db8::/32 1;\n}\n '''\n\n expected = [['geo', ['$geo'], [\n ['default', '0'],\n ['127.0.0.1', '2'],\n ['192.168.1.0/24', '1'],\n ['10.1.0.0/16', '1'],\n ['::1', '2'],\n ['2001:0db8::/32', '1']\n ]]]\n\n assert_config(config, expected)\n\n\ndef test_hash_block_in_location():\n config = '''\nlocation /iphone/ {\n types {\n text/html html htm shtml;\n application/json json;\n application/rss+xml rss;\n text/vnd.sun.j2me.app-descriptor jad;\n }\n}\n '''\n\n expected = [['location', ['/iphone/'], [\n ['types', [], [\n ['text/html', 'html', 'htm', 'shtml'],\n ['application/json', 'json'],\n ['application/rss+xml', 'rss'],\n ['text/vnd.sun.j2me.app-descriptor', 'jad']\n ]],\n ]]]\n\n assert_config(config, expected)\n\n\ndef test_named_location():\n config = '''\nlocation @foo {\n proxy_pass http://any.yandex.ru;\n}\n '''\n\n expected = [['location', ['@foo'], [\n ['proxy_pass', 'http://any.yandex.ru']\n ]]]\n\n assert_config(config, expected)\n\n\ndef test_if():\n config = '''\n# http://nginx.org/ru/docs/http/ngx_http_rewrite_module.html#if\n\nif ($http_user_agent ~ MSIE) {\n rewrite ^(.*)$ /msie/$1 break;\n}\n\nif ($http_cookie ~* \"id=([^;]+)(?:;|$)\") {\n set $id $1;\n}\n\nif ($request_method = POST) {\n return 405;\n}\n\nif ($slow) {\n limit_rate 10k;\n}\n\nif ($invalid_referer) {\n return 403;\n}\n\nif (!-e \"/var/data/$dataset\") {\n return 503;\n}\n\nif ($https_or_slb = (by_\\(sl\\)b|https)) {\n}\n\nif ($host ~* (lori|rage2)\\.yandex\\.(ru|ua|com|com\\.tr)) {\n set $x_frame_options ALLOW;\n}\n\nif ($request_filename ~* ^.*?/(\\d+_)([^/]+)$) {\n}\n\nif ($http_user_agent ~* \"^WordPress.*; verifying pingback\") {\n}\n\nif ($foo = \"BAR\") { rewrite ^(.*)$ /bar; }\n '''\n\n expected = [\n ['http://nginx.org/ru/docs/http/ngx_http_rewrite_module.html#if'],\n ['if', ['$http_user_agent', '~', 'MSIE'], [\n ['rewrite', '^(.*)$', '/msie/$1', 'break']\n ]],\n ['if', ['$http_cookie', '~*', 'id=([^;]+)(?:;|$)'], [\n ['set', '$id', '$1']\n ]],\n ['if', ['$request_method', '=', 'POST'], [\n ['return', '405']\n ]],\n ['if', ['$slow'], [\n ['limit_rate', '10k']\n ]],\n ['if', ['$invalid_referer'], [\n ['return', '403']\n ]],\n ['if', ['!-e', '/var/data/$dataset'], [\n ['return', '503']\n ]],\n ['if', ['$https_or_slb', '=', '(by_\\(sl\\)b|https)'], [\n ]],\n ['if', ['$host', '~*', '(lori|rage2)\\.yandex\\.(ru|ua|com|com\\.tr)'], [\n ['set', '$x_frame_options', 'ALLOW']\n ]],\n ['if', ['$request_filename', '~*', '^.*?/(\\d+_)([^/]+)$'], [\n ]],\n ['if', ['$http_user_agent', '~*', '^WordPress.*; verifying pingback'], [\n ]],\n ['if', ['$foo', '=', 'BAR'], [\n ['rewrite', '^(.*)$', '/bar']\n ]]\n ]\n\n assert_config(config, expected)\n\n\ndef test_hash_block_map():\n config = '''\n# http://nginx.org/ru/docs/http/ngx_http_map_module.html\n\nmap $http_host $name {\n hostnames;\n\n default 0;\n\n example.com 1;\n *.example.com 1;\n example.org 2;\n *.example.org 2;\n .example.net 3;\n wap.* 4;\n}\n\nmap $http_user_agent $mobile {\n default 0;\n \"~Opera Mini\" 1;\n}\n '''\n\n expected = [\n ['http://nginx.org/ru/docs/http/ngx_http_map_module.html'],\n ['map', ['$http_host', '$name'], [\n ['hostnames'],\n ['default', '0'],\n ['example.com', '1'],\n ['*.example.com', '1'],\n ['example.org', '2'],\n ['*.example.org', '2'],\n ['.example.net', '3'],\n ['wap.*', '4'],\n ]],\n ['map', ['$http_user_agent', '$mobile'], [\n ['default', '0'],\n ['~Opera Mini', '1'],\n ]]\n ]\n\n assert_config(config, expected)\n\n\ndef test_upstream():\n config = '''\n# http://nginx.org/ru/docs/http/ngx_http_upstream_module.html\n\nupstream backend {\n server backend1.example.com weight=5;\n server backend2.example.com:8080;\n server unix:/tmp/backend3;\n\n server backup1.example.com:8080 backup;\n server backup2.example.com:8080 backup;\n}\n\nserver {\n location / {\n proxy_pass http://backend;\n }\n}\n '''\n\n expected = [\n ['http://nginx.org/ru/docs/http/ngx_http_upstream_module.html'],\n ['upstream', ['backend'], [\n ['server', 'backend1.example.com', 'weight=5'],\n ['server', 'backend2.example.com:8080'],\n ['server', 'unix:/tmp/backend3'],\n ['server', 'backup1.example.com:8080', 'backup'],\n ['server', 'backup2.example.com:8080', 'backup'],\n ]],\n ['server', [], [\n ['location', ['/'], [\n ['proxy_pass', 'http://backend']\n ]]\n ]]]\n\n assert_config(config, expected)\n\n\ndef test_issue_8():\n config = '''\n# http://nginx.org/ru/docs/http/ngx_http_upstream_module.html\nif ($http_referer ~* (\\.(ru|ua|by|kz)/(pages/music|partners/|page-no-rights\\.xml)) ) {\n set $temp A;\n}\n '''\n\n expected = [\n ['http://nginx.org/ru/docs/http/ngx_http_upstream_module.html'],\n ['if', ['$http_referer', '~*', '(\\.(ru|ua|by|kz)/(pages/music|partners/|page-no-rights\\.xml))'], [\n ['set', '$temp', 'A']\n ]]\n ]\n\n assert_config(config, expected)\n\n\ndef test_issue_11():\n config = '''\ninit_by_lua_block {\n tvm = require \"nginx.tvm\"\n}\n '''\n\n expected = [\n ['init_by_lua_block', [], ['tvm', '=', 'require', '\"nginx.tvm\"']]\n ]\n\n assert_config(config, expected)\n\n\ndef test_lua_block():\n config = '''\n# https://github.com/openresty/lua-nginx-module#typical-uses\nlocation = /lua {\n # MIME type determined by default_type:\n default_type 'text/plain';\n\n content_by_lua_block {\n local res = ngx.location.capture(\"/some_other_location\")\n if res then\n ngx.say(\"status: \", res.status)\n ngx.say(\"body:\")\n ngx.print(res.body)\n end\n }\n}\n '''\n\n expected = [\n ['https://github.com/openresty/lua-nginx-module#typical-uses'],\n ['location', ['=', '/lua'], [\n ['MIME type determined by default_type:'],\n ['default_type', 'text/plain'],\n ['content_by_lua_block', [], [\n 'local', 'res', '=', 'ngx.location.capture(', '\"/some_other_location\"', ')',\n 'if', 'res', 'then',\n 'ngx.say(', '\"status: \"', ',', 'res.status)',\n 'ngx.say(', '\"body:\"', ')',\n 'ngx.print(res.body)',\n 'end']]\n ]]\n ]\n\n assert_config(config, expected)\n\n\ndef test_lua_block_brackets():\n config = '''\nlocation = /foo {\n rewrite_by_lua_block {\n res = ngx.location.capture(\"/memc\",\n { args = { cmd = \"incr\", key = ngx.var.uri } }\n )\n }\n\n proxy_pass http://blah.blah.com;\n}\n '''\n\n expected = [\n ['location', ['=', '/foo'], [\n ['rewrite_by_lua_block', [], [\n 'res', '=', 'ngx.location.capture(', '\"/memc\"', ',',\n ['args', '=', ['cmd', '=', '\"incr\"', ',', 'key', '=', 'ngx.var.uri']],\n ')']],\n ['proxy_pass', 'http://blah.blah.com']\n ]]\n ]\n\n assert_config(config, expected)\n\n\ndef test_file_delims():\n config = '''\n# configuration file /etc/nginx/nginx.conf:\nhttp {\n include sites/*.conf;\n}\n\n# configuration file /etc/nginx/sites/default.conf:\nserver {\n\n}\n '''\n\n expected = [\n ['/etc/nginx/nginx.conf'],\n ['http', [], [\n ['include', 'sites/*.conf']\n ]],\n ['/etc/nginx/sites/default.conf'],\n ['server', [], []]\n ]\n\n assert_config(config, expected)\n\n\ndef test_comments():\n config = '''\n# Some comment\nadd_header X-Some-Comment some;\n\n# \n# Comment with padding\n# \nadd_header X-Padding-Comment padding;\n\n#\nadd_header X-Blank-Comment blank;\n\nif (1) # Comment \n{\n add_header X-Inline blank;\n}\n '''\n\n expected = [\n ['Some comment'],\n ['add_header', 'X-Some-Comment', 'some'],\n [''],\n ['Comment with padding'],\n [''],\n ['add_header', 'X-Padding-Comment', 'padding'],\n [''],\n ['add_header', 'X-Blank-Comment', 'blank'],\n ['if', ['1'], [\n ['add_header', 'X-Inline', 'blank'],\n ]],\n ]\n\n assert_config(config, expected)\n\n\ndef test_upstream_dot():\n config = '''\nupstream test.mysite.com {\n server 127.0.0.1:9009;\n}\n '''\n\n expected = [\n ['upstream', ['test.mysite.com'], [\n ['server', '127.0.0.1:9009']\n ]],\n ]\n\n assert_config(config, expected)\n\n\ndef test_empty_config():\n config = '''\n '''\n\n expected = []\n\n assert_config(config, expected)\n\n\ndef test_utfbom_decoding():\n config = b'''\\xef\\xbb\\xbf\nadd_header X-Test \"Windows-1251\";\n '''\n\n expected = [\n ['add_header', 'X-Test', 'Windows-1251']\n ]\n\n assert_config(config, expected)\n\n\ndef test_national_comment_decoding():\n config = b'''\n# \\xeb\\xff-\\xeb\\xff-\\xeb\\xff = Lya-lya-lya\nadd_header X-Test \"Windows-1251\";\n '''\n\n actual = RawParser().parse(config)\n assert_equals(len(actual.asList()), 2)\n\n\ndef assert_config(config, expected):\n actual = RawParser().parse(config)\n assert_equals(actual.asList(), expected)\n" }, { "path": "tests/plugins/__init__.py", "content": "" }, { "path": "tests/plugins/simply/add_header_multiline/add_header.conf", "content": "add_header Content-Security-Policy \"\n default-src: 'none';\n font-src data: https://yastatic.net;\";" }, { "path": "tests/plugins/simply/add_header_multiline/add_header_fp.conf", "content": "add_header X-Foo foo;" }, { "path": "tests/plugins/simply/add_header_multiline/config.json", "content": "{\n \"severity\": \"LOW\"\n}" }, { "path": "tests/plugins/simply/add_header_multiline/more_set_headers.conf", "content": "more_set_headers -t 'text/html text/plain'\n 'X-Foo: Bar\n multiline';" }, { "path": "tests/plugins/simply/add_header_multiline/more_set_headers_fp.conf", "content": "more_set_headers -t 'text/html text/plain'\n 'X-Foo: Bar multiline';" }, { "path": "tests/plugins/simply/add_header_multiline/more_set_headers_multiple.conf", "content": "more_set_headers -t 'text/html text/plain'\n 'X-Foo: some\n multiline'\n 'X-Bar: some\n multiline'\n 'X-Baz: some\n multiline';" }, { "path": "tests/plugins/simply/add_header_multiline/more_set_headers_replace.conf", "content": "more_set_headers -r 'Foo:\n multiline';" }, { "path": "tests/plugins/simply/add_header_multiline/more_set_headers_replace_fp.conf", "content": "more_set_headers -r 'Foo: multiline';" }, { "path": "tests/plugins/simply/add_header_multiline/more_set_headers_status_fp.conf", "content": "more_set_headers -s 404 -s '500 503' 'Foo: bar';" }, { "path": "tests/plugins/simply/add_header_multiline/more_set_headers_type_fp.conf", "content": "more_set_headers -t 'text/html\n text/plain' 'X-Foo: some';" }, { "path": "tests/plugins/simply/add_header_redefinition/config.json", "content": "{\n \"severity\": \"MEDIUM\"\n}" }, { "path": "tests/plugins/simply/add_header_redefinition/duplicate_fp.conf", "content": "http {\nadd_header X-Frame-Options \"DENY\" always;\n server {\n location /new-headers {\n add_header X-Frame-Options \"DENY\" always;\n add_header X-Foo foo;\n }\n }\n}" }, { "path": "tests/plugins/simply/add_header_redefinition/if_replaces.conf", "content": "add_header X-Frame-Options \"DENY\" always;\n\nif (1) {\n add_header X-Foo foo;\n}" }, { "path": "tests/plugins/simply/add_header_redefinition/location_replaces.conf", "content": "add_header X-Frame-Options \"DENY\" always;\n\nlocation /new-headers {\n add_header X-Foo foo;\n}" }, { "path": "tests/plugins/simply/add_header_redefinition/nested_block.conf", "content": "server {\n add_header X-Frame-Options \"DENY\" always;\n location / {\n location /some {\n add_header X-Frame-Options \"DENY\" always;\n }\n\n location /another {\n add_header X-Foo foo;\n }\n }\n}" }, { "path": "tests/plugins/simply/add_header_redefinition/non_block_fp.conf", "content": "add_header X-Frame-Options \"DENY\" always;\nserver \"some\";\nadd_header X-Foo foo;\n" }, { "path": "tests/plugins/simply/add_header_redefinition/not_secure_both_fp.conf", "content": "add_header X-Bar bar;\n\nlocation /new-headers {\n add_header X-Foo foo;\n}" }, { "path": "tests/plugins/simply/add_header_redefinition/not_secure_outer_fp.conf", "content": "add_header X-Bar bar;\n\nlocation /new-headers {\n add_header X-Frame-Options \"DENY\" always;\n}" }, { "path": "tests/plugins/simply/add_header_redefinition/step_replaces.conf", "content": "http {\nadd_header X-Frame-Options \"DENY\" always;\n server {\n location /new-headers {\n add_header X-Foo foo;\n }\n }\n}" }, { "path": "tests/plugins/simply/alias_traversal/config.json", "content": "{\n \"severity\": [\"MEDIUM\", \"HIGH\"]\n}\n" }, { "path": "tests/plugins/simply/alias_traversal/nested.conf", "content": "location /files/ {\n location /files/images {\n alias /home/;\n }\n}\n" }, { "path": "tests/plugins/simply/alias_traversal/nested_fp.conf", "content": "location /files/ {\n location /files/images {\n }\n alias /home/;\n}\n" }, { "path": "tests/plugins/simply/alias_traversal/not_slashed_alias.conf", "content": "location /files {\n alias /home;\n}\n" }, { "path": "tests/plugins/simply/alias_traversal/not_slashed_alias_fp.conf", "content": "location /files/ {\n alias /home;\n}\n" }, { "path": "tests/plugins/simply/alias_traversal/simple.conf", "content": "location /files {\n alias /home/;\n}\n" }, { "path": "tests/plugins/simply/alias_traversal/simple_fp.conf", "content": "location /files/ {\n alias /home/;\n}\n" }, { "path": "tests/plugins/simply/alias_traversal/slashed_alias.conf", "content": "location /files {\n alias /home/;\n}\n" }, { "path": "tests/plugins/simply/alias_traversal/slashed_alias_fp.conf", "content": "location /files/ {\n alias /home/;\n}\n" }, { "path": "tests/plugins/simply/host_spoofing/config.json", "content": "{\n \"severity\": \"MEDIUM\"\n}" }, { "path": "tests/plugins/simply/host_spoofing/http_fp.conf", "content": "proxy_set_header Host $host;" }, { "path": "tests/plugins/simply/host_spoofing/http_host.conf", "content": "proxy_set_header Host $http_host;" }, { "path": "tests/plugins/simply/host_spoofing/http_host_diff_case.conf", "content": "proxy_set_header HoSt $http_host;" }, { "path": "tests/plugins/simply/host_spoofing/some_arg.conf", "content": "proxy_set_header host $arg_host;" }, { "path": "tests/plugins/simply/http_splitting/add_header_uri.conf", "content": "add_header X-Uri $uri;" }, { "path": "tests/plugins/simply/http_splitting/config.json", "content": "{\n \"severity\": \"HIGH\"\n}" }, { "path": "tests/plugins/simply/http_splitting/dont_report_not_resolved_var_fp.conf", "content": "location ~ /proxy/(a|b)/(\\W*)$ {\n proxy_pass http://storage/$some;\n}" }, { "path": "tests/plugins/simply/http_splitting/proxy_from_location_var.conf", "content": "location ~ /proxy/(a|b)/(\\W*)$ {\n proxy_pass http://storage/$2;\n}" }, { "path": "tests/plugins/simply/http_splitting/proxy_from_location_var_var.conf", "content": "location ~ /proxy/(a|b)/(\\W*)$ {\n set $p $2;\n proxy_pass http://storage/$p;\n}" }, { "path": "tests/plugins/simply/http_splitting/proxy_from_location_var_var_fp.conf", "content": "location ~ /proxy/(a|b)/(\\W*)$ {\n set $p $1;\n proxy_pass http://storage/$p;\n}" }, { "path": "tests/plugins/simply/http_splitting/proxy_from_location_var_var_var.conf", "content": "location ~ /proxy/(a|b)/(?

\\W*)$ {\n set $upstream \"http://$1/$p?\";\n proxy_pass $upstream;\n}" }, { "path": "tests/plugins/simply/http_splitting/proxy_pass_cr_fp.conf", "content": "location ~* ^/test/(.*) {\n proxy_pass http://10.10.10.10/$1;\n}\n" }, { "path": "tests/plugins/simply/http_splitting/proxy_pass_ducument_uri.conf", "content": "proxy_pass http://upstream$document_uri;" }, { "path": "tests/plugins/simply/http_splitting/proxy_pass_lf.conf", "content": "location ~* ^/test/([^/]+)/ {\n proxy_pass http://10.10.10.10/$1;\n}\n" }, { "path": "tests/plugins/simply/http_splitting/proxy_set_header_ducument_uri.conf", "content": "proxy_set_header \"X-Original-Uri\" $document_uri;" }, { "path": "tests/plugins/simply/http_splitting/return_403_fp.conf", "content": "return 403;" }, { "path": "tests/plugins/simply/http_splitting/return_request_uri_fp.conf", "content": "return 301 https://some$request_uri;" }, { "path": "tests/plugins/simply/http_splitting/rewrite_extract_fp.conf", "content": "rewrite ^/proxy/(a|b)/(?\\W*)$ http://storage/$path redirect;" }, { "path": "tests/plugins/simply/http_splitting/rewrite_uri.conf", "content": "rewrite ^ http://some$uri;" }, { "path": "tests/plugins/simply/http_splitting/rewrite_uri_after_var.conf", "content": "return 301 https://$host$uri;" }, { "path": "tests/plugins/simply/origins/config.json", "content": "{\n \"severity\": [\"MEDIUM\", \"HIGH\"]\n}" }, { "path": "tests/plugins/simply/origins/metrika.conf", "content": "if ($http_referer !~ \"^https?://([^/]+metrika.*yandex\\.(ru|ua|com|com\\.tr|by|kz)|([^/]+\\.)?webvisor\\.com)/\"){\n add_header X-Frame-Options SAMEORIGIN;\n}" }, { "path": "tests/plugins/simply/origins/origin.conf", "content": "if ($http_origin !~ '^https?:\\/\\/yandex.ru\\/') {\n\n}" }, { "path": "tests/plugins/simply/origins/origin_fp.conf", "content": "if ($http_origin !~ '^https?:\\/\\/yandex\\.ru\\/') {\n\n}" }, { "path": "tests/plugins/simply/origins/origin_https.conf", "content": "# Options: {\"domains\": [\"yandex.ru\"], \"https_only\": true}\n\nif ($http_origin !~ '^https?:\\/\\/yandex\\.ru\\/') {\n\n}" }, { "path": "tests/plugins/simply/origins/origin_https_fp.conf", "content": "# Options: {\"domains\": [\"yandex.ru\"], \"https_only\": true}\n\nif ($http_origin !~ '^https:\\/\\/yandex\\.ru\\/') {\n\n}" }, { "path": "tests/plugins/simply/origins/origin_w_slash_anchored_fp.conf", "content": "if ($http_origin !~ '^https?:\\/\\/yandex\\.ru/$') {\n\n}" }, { "path": "tests/plugins/simply/origins/origin_w_slash_fp.conf", "content": "if ($http_origin !~ '^https?:\\/\\/yandex\\.ru/') {\n\n}" }, { "path": "tests/plugins/simply/origins/origin_wo_slash.conf", "content": "# Options: {\"domains\": [\"yandex.ru\"]}\n\nhttp {\nif ($http_origin !~ '^https?:\\/\\/yandex\\.ru') {\n\n}\n}" }, { "path": "tests/plugins/simply/origins/referer.conf", "content": "if ($http_referer !~ '^https?:\\/\\/yandex.ru\\/') {\n\n}" }, { "path": "tests/plugins/simply/origins/referer_fp.conf", "content": "if ($http_referer !~ '^https?:\\/\\/yandex\\.ru\\/') {\n\n}" }, { "path": "tests/plugins/simply/origins/referer_subdomain.conf", "content": "if ($http_referer !~ '^https?:\\/\\/some.yandex\\.ru\\/') {\n\n}" }, { "path": "tests/plugins/simply/origins/referer_subdomain_fp.conf", "content": "if ($http_referer !~ '^https?:\\/\\/some\\.yandex\\.ru\\/') {\n\n}" }, { "path": "tests/plugins/simply/origins/structure_dot.conf", "content": "if ($http_referer !~ \"^https://example.com/\"){\n add_header X-Frame-Options SAMEORIGIN;\n}" }, { "path": "tests/plugins/simply/origins/structure_fp.conf", "content": "if ($http_referer !~ \"^https://example\\.com/\"){\n add_header X-Frame-Options SAMEORIGIN;\n}" }, { "path": "tests/plugins/simply/origins/structure_prefix.conf", "content": "if ($http_referer !~ \"https://example\\.com/\"){\n add_header X-Frame-Options SAMEORIGIN;\n}" }, { "path": "tests/plugins/simply/origins/structure_suffix.conf", "content": "if ($http_referer !~ \"^https://example\\.com\"){\n add_header X-Frame-Options SAMEORIGIN;\n}" }, { "path": "tests/plugins/simply/origins/webvisor.conf", "content": "# Options: {\"domains\": [\"webvisor.com\", \"yandex.com\"]}\n\nif ($http_referer !~ \"^https?://([^/]+\\.)?yandex\\.com/|([^/]+\\.)?webvisor\\.com/\"){\n add_header X-Frame-Options SAMEORIGIN;\n}" }, { "path": "tests/plugins/simply/ssrf/config.json", "content": "{\n \"severity\": \"HIGH\"\n}" }, { "path": "tests/plugins/simply/ssrf/have_internal_fp.conf", "content": "location /proxy/ {\n internal;\n proxy_pass $arg_some;\n}" }, { "path": "tests/plugins/simply/ssrf/host_w_const_start.conf", "content": "location ~* ^/backend/(?.*) {\n proxy_pass http://some$path;\n}" }, { "path": "tests/plugins/simply/ssrf/host_w_const_start_arg.conf", "content": "location /backend/ {\n proxy_pass http://some${arg_la}.shit;\n}" }, { "path": "tests/plugins/simply/ssrf/not_host_var_fp.conf", "content": "location ~ /proxy/(.*)$ {\n proxy_pass http://yastatic.net/$1;\n}" }, { "path": "tests/plugins/simply/ssrf/request_uri_fp.conf", "content": "location /backend/ {\n proxy_pass http://some$request_uri;\n}" }, { "path": "tests/plugins/simply/ssrf/request_uri_var_fp.conf", "content": "location / {\n set $upstream \"http://some$request_uri\";\n proxy_pass $upstream;\n}" }, { "path": "tests/plugins/simply/ssrf/scheme_var.conf", "content": "location ~ /proxy/$ {\n proxy_pass $http_proxy_scheme://some/file.conf;\n}" }, { "path": "tests/plugins/simply/ssrf/single_var.conf", "content": "location ~ /proxy/(?P.*)$ {\n proxy_pass $proxy;\n}" }, { "path": "tests/plugins/simply/ssrf/used_arg.conf", "content": "location /proxy/ {\n proxy_pass $arg_some;\n}" }, { "path": "tests/plugins/simply/ssrf/vars_from_loc.conf", "content": "location ~ /proxy/(.*)/(.*)/(.*)$ {\n set $scheme $1;\n set $host $2;\n set $path $3;\n proxy_pass $scheme://$host/$path;\n}" }, { "path": "tests/plugins/simply/ssrf/with_const_scheme.conf", "content": "location ~* ^/internal-proxy/(https?)/(.*?)/(.*) {\n resolver 127.0.0.1;\n\n set $proxy_protocol $1;\n set $proxy_host $2;\n set $proxy_path $3;\n\n proxy_pass $proxy_protocol://$proxy_host/$proxy_path ;\n proxy_set_header Host $proxy_host;\n}" }, { "path": "tests/plugins/simply/valid_referers/config.json", "content": "{\n \"severity\": \"HIGH\"\n}" }, { "path": "tests/plugins/simply/valid_referers/none_first.conf", "content": "valid_referers none server_names *.webvisor.com;" }, { "path": "tests/plugins/simply/valid_referers/none_last.conf", "content": "valid_referers server_names\n foo.com\n none;" }, { "path": "tests/plugins/simply/valid_referers/none_middle.conf", "content": "valid_referers server_names foo.com\n none bar.com;" }, { "path": "tests/plugins/simply/valid_referers/wo_none_fp.conf", "content": "valid_referers server_names foo.com bar.com *.none.com none.ru;" }, { "path": "tests/plugins/test_simply.py", "content": "from nose.tools import assert_equals, assert_true\nfrom tests.asserts import assert_in\nimport os\nfrom os import path\nimport json\n\nfrom ..utils import *\nfrom gixy.core.manager import Manager as Gixy\nfrom gixy.core.plugins_manager import PluginsManager\nfrom gixy.core.config import Config\n\n\ndef setup_module():\n pass\n\n\ndef teardown_module():\n pass\n\n\ndef test_from_config():\n tested_plugins = set()\n tested_fp_plugins = set()\n\n conf_dir = path.join(path.dirname(__file__), 'simply')\n for plugin in os.listdir(conf_dir):\n if plugin in ('.', '..'):\n continue\n\n plugin_path = path.join(conf_dir, plugin)\n if not path.isdir(plugin_path):\n continue\n\n config = {}\n if path.exists(path.join(plugin_path, 'config.json')):\n with open(path.join(plugin_path, 'config.json'), 'r') as file:\n config = json.loads(file.read())\n\n for test_case in os.listdir(plugin_path):\n if not test_case.endswith('.conf'):\n continue\n\n config_path = path.join(plugin_path, test_case)\n if not test_case.endswith('_fp.conf'):\n # Not False Positive test\n tested_plugins.add(plugin)\n test_func = check_configuration\n else:\n tested_fp_plugins.add(plugin)\n test_func = check_configuration_fp\n\n yield test_func, plugin, config_path, config\n\n manager = PluginsManager()\n for plugin in manager.plugins:\n plugin = plugin.name\n assert_true(plugin in tested_plugins,\n 'Plugin {name!r} should have at least one simple test config'.format(name=plugin))\n assert_true(plugin in tested_fp_plugins,\n 'Plugin {name!r} should have at least one simple test config with false positive'.format(name=plugin))\n\n\ndef parse_plugin_options(config_path):\n with open(config_path, 'r') as f:\n config_line = f.readline()\n if config_line.startswith('# Options: '):\n return json.loads(config_line[10:])\n return None\n\n\ndef yoda_provider(plugin, plugin_options=None):\n config = Config(\n allow_includes=False,\n plugins=[plugin]\n )\n if plugin_options:\n config.set_for(plugin, plugin_options)\n return Gixy(config=config)\n\n\ndef check_configuration(plugin, config_path, test_config):\n plugin_options = parse_plugin_options(config_path)\n with yoda_provider(plugin, plugin_options) as yoda:\n yoda.audit(config_path, open(config_path, mode='r'))\n formatter = BaseFormatter()\n formatter.feed(config_path, yoda)\n _, results = formatter.reports.popitem()\n\n assert_equals(len(results), 1, 'Should have one report')\n result = results[0]\n\n if 'severity' in test_config:\n if not hasattr(test_config['severity'], '__iter__'):\n assert_equals(result['severity'], test_config['severity'])\n else:\n assert_in(result['severity'], test_config['severity'])\n assert_equals(result['plugin'], plugin)\n assert_true(result['summary'])\n assert_true(result['description'])\n assert_true(result['config'])\n assert_true(result['help_url'].startswith('https://'),\n 'help_url must starts with https://. It\\'is URL!')\n\n\ndef check_configuration_fp(plugin, config_path, test_config):\n with yoda_provider(plugin) as yoda:\n yoda.audit(config_path, open(config_path, mode='r'))\n assert_equals(len([x for x in yoda.results]), 0,\n 'False positive configuration must not trigger any plugins')\n" }, { "path": "tests/utils.py", "content": "from logging.handlers import BufferingHandler\n\n\nclass LogHandler(BufferingHandler):\n def __init__(self, matcher):\n # BufferingHandler takes a \"capacity\" argument\n # so as to know when to flush. As we're overriding\n # shouldFlush anyway, we can set a capacity of zero.\n # You can call flush() manually to clear out the\n # buffer.\n super(LogHandler, self).__init__(0)\n self.matcher = matcher\n\n def shouldFlush(self, **kwargs):\n return False\n\n def emit(self, record):\n self.buffer.append(record.__dict__)\n\n def matches(self, **kwargs):\n \"\"\"\n Look for a saved dict whose keys/values match the supplied arguments.\n \"\"\"\n result = False\n for d in self.buffer:\n if self.matcher.matches(d, **kwargs):\n result = True\n break\n return result\n\n\nclass Matcher(object):\n\n _partial_matches = ('msg', 'message')\n\n def matches(self, d, **kwargs):\n \"\"\"\n Try to match a single dict with the supplied arguments.\n\n Keys whose values are strings and which are in self._partial_matches\n will be checked for partial (i.e. substring) matches. You can extend\n this scheme to (for example) do regular expression matching, etc.\n \"\"\"\n result = True\n for k in kwargs:\n v = kwargs[k]\n dv = d.get(k)\n if not self.match_value(k, dv, v):\n result = False\n break\n return result\n\n def match_value(self, k, dv, v):\n \"\"\"\n Try to match a single stored value (dv) with a supplied value (v).\n \"\"\"\n if type(v) != type(dv):\n result = False\n elif type(dv) is not str or k not in self._partial_matches:\n result = (v == dv)\n else:\n result = dv.find(v) >= 0\n return result\n" }, { "path": "tox.ini", "content": "[tox]\nenvlist = py26, py27, py34, py35, py36, py37, flake8\nskip_missing_interpreters = True\n\n[testenv]\ndeps =\n -rrequirements.txt\n -rrequirements.dev.txt\ncommands = nosetests -v\n\n[testenv:flake8]\ndeps =\n flake8\nbasepython = python3\ncommands =\n flake8 setup.py gixy\n\n[flake8]\nmax_line_length = 120\n" } ]