[ { "path": ".gitignore", "content": "# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\n\n# C extensions\n*.so\n\n# Distribution / packaging\n.Python\nenv/\nbuild/\ndevelop-eggs/\ndist/\ndownloads/\neggs/\n.eggs/\nlib/\nlib64/\nparts/\nsdist/\nvar/\n*.egg-info/\n.installed.cfg\n*.egg\n\n# PyInstaller\n# Usually these files are written by a python script from a template\n# before PyInstaller builds the exe, so as to inject date/other infos into it.\n*.manifest\n*.spec\n\n# Installer logs\npip-log.txt\npip-delete-this-directory.txt\n\n# Unit test / coverage reports\nhtmlcov/\n.tox/\n.coverage\n.coverage.*\n.cache\nnosetests.xml\ncoverage.xml\n*,cover\n.hypothesis/\n\n# Translations\n*.mo\n*.pot\n\n# Django stuff:\n*.log\n\n# Sphinx documentation\ndocs/_build/\n\n# PyBuilder\ntarget/\n\n#Ipython Notebook\n.ipynb_checkpoints\n\n# temporary files from vim and emacs\n*~\n*#\n.#*\n*.swp\n*.swo\n\n# Virtualenv\npip-selfcheck.json\nbin/\ninclude/\n\n# Mac\n.DS_Store\n\n* Misc\nscratch/\n*.egg-info/\n\n*.pyc\n*.out\n*.xcf\n\nrecipe/\n.binstar.yml\nbinstar.yml\n.landscape.yaml\n.cache/\nbuild/\ndist/\ngitsome.egg-info/\ndocs/_build/\n.tox/\n\nlexer_table.py\nparser_table.py\nparser_test_table.py\ngitsome/lexer_table.py\ngitsome/parser_table.py\ntests/lexer_table.py\ntests/parser_table.py\ntests/lexer_test_table.py\ntests/parser_test_table.py\n" }, { "path": ".gitsomeconfig", "content": "[github]\nuser_login = None\nenable_avatar = True\nverify_ssl = True\nclr_primary = None\nclr_secondary = green\nclr_tertiary = cyan\nclr_quaternary = yellow\nclr_bold = cyan\nclr_code = red\nclr_error = red\nclr_header = yellow\nclr_link = green\nclr_list = cyan\nclr_message = None\nclr_num_comments = green\nclr_num_points = green\nclr_tag = cyan\nclr_time = yellow\nclr_title = None\nclr_tooltip = None\nclr_user = cyan\nclr_view_link = magenta\nclr_view_index = magenta\n\n" }, { "path": ".gitsomeconfigurl", "content": "[url]\nurl_list = ['https://github.com/octocat/spoon-knife']\n\n" }, { "path": ".travis.yml", "content": "language: python\nmatrix:\n include:\n - python: 3.5\n env: TOXENV=py35\n - python: 3.6\n env: TOXENV=py36\n - python: 3.7\n dist: xenial # required for Python >= 3.7\n env: TOXENV=py37\ninstall:\n - travis_retry pip install codecov tox\nscript:\n - tox\nafter_success:\n - codecov\n" }, { "path": "CHANGELOG.md", "content": "![](http://i.imgur.com/0SXZ90y.gif)\n\ngitsome\n=======\n\n[![Build Status](https://travis-ci.org/donnemartin/gitsome.svg?branch=master)](https://travis-ci.org/donnemartin/gitsome) [![Codecov](https://img.shields.io/codecov/c/github/donnemartin/gitsome.svg)](https://codecov.io/github/donnemartin/gitsome)\n\n[![PyPI version](https://badge.fury.io/py/gitsome.svg)](http://badge.fury.io/py/gitsome) [![PyPI](https://img.shields.io/pypi/pyversions/gitsome.svg)](https://pypi.python.org/pypi/gitsome/) [![License](https://img.shields.io/:license-apache-blue.svg)](http://www.apache.org/licenses/LICENSE-2.0.html)\n\nTo view the latest `README`, `docs`, and `code`, visit the GitHub repo:\n\nhttps://github.com/donnemartin/gitsome\n\nTo submit bugs or feature requests, visit the issue tracker:\n\nhttps://github.com/donnemartin/gitsome/issues\n\nChangelog\n=========\n\n0.8.0 (2019-04-07)\n------------------\n\nThis version adds support for Python 3.7.\n\n### Updates\n\n* [#160](https://github.com/donnemartin/gitsome/pull/160) - Add Python 3.7 support. Fixes [#152](https://github.com/donnemartin/gitsome/pull/152), [#144](https://github.com/donnemartin/gitsome/pull/144), [#126](https://github.com/donnemartin/gitsome/pull/126), [#105](https://github.com/donnemartin/gitsome/pull/105) and several other related bugs.\n* [#147](https://github.com/donnemartin/gitsome/pull/148) - Gracefully ignore missing avatar image, by [kBite](https://github.com/kBite).\n* [#142](https://github.com/donnemartin/gitsome/pull/142) - Update release checklist.\n* [#134](https://github.com/donnemartin/gitsome/pull/134) - Update GitHub integrations link.\n* [#120](https://github.com/donnemartin/gitsome/pull/120) - Add license disclaimer.\n\n### Bug Fixes\n\n* [#151](https://github.com/donnemartin/gitsome/pull/151) - Fix gh command typos in docs, by [cyblue9](https://github.com/cyblue9).\n* [#137](https://github.com/donnemartin/gitsome/pull/137) - Fix Running as a Docker Container anchor in README, by [kamontat](https://github.com/kamontat).\n* [#129](https://github.com/donnemartin/gitsome/pull/129) - Fix trending command to handle empty summaries, by [emres](https://github.com/emres).\n* [#123](https://github.com/donnemartin/gitsome/pull/123) - Remove buggy codecov badge.\n* [#117](https://github.com/donnemartin/gitsome/pull/117) - Fix 0.7.0 CHANGELOG date, by [dbaio](https://github.com/dbaio).\n\n0.7.0 (2017-03-26)\n------------------\n\n### Features\n\n* [#99](https://github.com/donnemartin/gitsome/pull/99) - Add Dockerfile to run gitsome in a Docker container, by [l0rd](https://github.com/l0rd) and [larson004](https://github.com/larson004).\n\n### Bug Fixes\n\n* [#67](https://github.com/donnemartin/gitsome/pull/67) - Fix `gh_issues` typo in the `README`, by [srisankethu](https://github.com/srisankethu).\n* [#69](https://github.com/donnemartin/gitsome/pull/69) - Fix `--issue_filter` typo for `gh_issues` command in `COMMANDS.md`.\n* [#80](https://github.com/donnemartin/gitsome/pull/80) - Fix path for auto completions in `README`.\n* [#92](https://github.com/donnemartin/gitsome/pull/92) - Fix viewing HTML contents in the terminal for GitHub Enterprise users, by [dongweiming](https://github.com/dongweiming).\n* [#97](https://github.com/donnemartin/gitsome/pull/97) - Fix error hint from `gh gitignores` to `gh gitignore-templates`, by [zYeoman](https://github.com/zYeoman).\n* [#116](https://github.com/donnemartin/gitsome/pull/116) - Fix gh trending command resulting in an error.\n\n### Updates\n\n* [#58](https://github.com/donnemartin/gitsome/pull/58) - Tweak `README` intro, add logo.\n* [#74](https://github.com/donnemartin/gitsome/pull/74) - Add link to official GitHub integration page in `README`.\n* [#79](https://github.com/donnemartin/gitsome/pull/79) - Only store password in config for GitHub Enterprise (due to Enterprise limitations), by [nttibbetts](https://github.com/nttibbetts).\n* [#86](https://github.com/donnemartin/gitsome/pull/86) - Update dependency info for `uritemplate`.\n* [#89](https://github.com/donnemartin/gitsome/pull/89) - Fix a bug listing info on repos without a desc field, by [SanketDG](https://github.com/SanketDG).\n* [#98](https://github.com/donnemartin/gitsome/pull/98) - Prefer GitHub Enterprise token before password.\n* [#104](https://github.com/donnemartin/gitsome/pull/104) - Update install instructions to use pip3.\n* [#111](https://github.com/donnemartin/gitsome/pull/111) - Add note about current Python 3.6 incompatibility.\n* [#115](https://github.com/donnemartin/gitsome/pull/115) - Set current Python support to 3.4 and 3.5.\n\n0.6.0 (2016-05-29)\n------------------\n\n### Features\n\n* [#3](https://github.com/donnemartin/gitsome/issues/3) - Add GitHub Enterprise support.\n* [#33](https://github.com/donnemartin/gitsome/issues/33) - Revamp the info shown with the `gh feed` command.\n\n### Bug Fixes\n\n* [#30](https://github.com/donnemartin/gitsome/issues/30) - Fix a typo in the `pip3` install instructions.\n* [#39](https://github.com/donnemartin/gitsome/issues/39) - Fix `gh feed` `-pr/--private` flag in docs.\n* [#40](https://github.com/donnemartin/gitsome/issues/40) - Fix `create-issue` `NoneType` error if no `-b/--body` is specified.\n* [#46](https://github.com/donnemartin/gitsome/issues/46) - Fix `gh view` with the -b/--browser option only working for repos, not for issues or PRs.\n* [#48](https://github.com/donnemartin/gitsome/issues/48) - Fix `create-repo` `NoneType` error if no `-d/--description` is specified.\n* [#54](https://github.com/donnemartin/gitsome/pull/54) - Update to `prompt-toolkit` 1.0.0, which includes performance improvements (especially noticeable on Windows) and bug fixes.\n* Fix `Config` docstrings.\n\n### Updates\n\n* [#26](https://github.com/donnemartin/gitsome/issues/26), [#32](https://github.com/donnemartin/gitsome/issues/32) - Add copyright notices for third\nparty libraries.\n* [#44](https://github.com/donnemartin/gitsome/pull/44), [#53](https://github.com/donnemartin/gitsome/pull/53) - Update packaging dependencies based on semantic versioning.\n* Tweak `README` intro.\n\n0.5.0 (2016-05-15)\n------------------\n\n### Features\n\n* [#12](https://github.com/donnemartin/gitsome/issues/12) - Allow 2FA-enabled users to log in with a password + 2FA code. Previously 2FA-enabled users could only log in with a [personal access token](https://github.com/settings/tokens). Also includes an update of login prompts to improve clarity.\n\n### Bug Fixes\n\n* [#16](https://github.com/donnemartin/gitsome/pull/16), [#28](https://github.com/donnemartin/gitsome/pull/28) - Fix typos in README.\n* [#18](https://github.com/donnemartin/gitsome/pull/18) - Fix dev install instructions in README.\n* [#24](https://github.com/donnemartin/gitsome/pull/24) - Fix style guide broken link in CONTRIBUTING.\n\n### Updates\n\n* [#1](https://github.com/donnemartin/gitsome/issues/1) - Add Codecov coverage testing status to README.\n* [#2](https://github.com/donnemartin/gitsome/issues/2) - Add note about enabling Zsh completions to README.\n* [#4](https://github.com/donnemartin/gitsome/issues/4) - Add note about using `pip3` to README.\n* [#5](https://github.com/donnemartin/gitsome/issues/5) - Decrease speed of README gif.\n* [#6](https://github.com/donnemartin/gitsome/pull/6) - Update url for `click`.\n* [#20](https://github.com/donnemartin/gitsome/issues/20) - Add note about enabling more completions to README.\n* [#21](https://github.com/donnemartin/gitsome/issues/21) - Bump up `prompt-toolkit` version from `0.51` to `0.52`.\n* [#26](https://github.com/donnemartin/gitsome/issues/26) - Add `xonsh` copyright notice to LICENSE.\n* [#32](https://github.com/donnemartin/gitsome/pull/32) - Add `github3.py`, `html2text`, and `img2txt` copyright notices to LICENSE.\n* Update installation instructions in README.\n* Update color customization discussion in README.\n\n0.4.0 (2016-05-09)\n------------------\n\n* Initial release.\n" }, { "path": "CHANGELOG.rst", "content": ".. figure:: http://i.imgur.com/0SXZ90y.gif\n :alt:\n\ngitsome\n=======\n\n|Build Status| |Codecov|\n\n|PyPI version| |PyPI| |License|\n\nTo view the latest ``README``, ``docs``, and ``code``, visit the GitHub\nrepo:\n\nhttps://github.com/donnemartin/gitsome\n\nTo submit bugs or feature requests, visit the issue tracker:\n\nhttps://github.com/donnemartin/gitsome/issues\n\nChangelog\n=========\n\n0.8.0 (2019-04-07)\n------------------\n\nThis version adds support for Python 3.7.\n\nUpdates\n~~~~~~~\n\n- `#160 `__ - Add\n Python 3.7 support. Fixes\n `#152 `__,\n `#144 `__,\n `#126 `__,\n `#105 `__ and\n several other related bugs.\n- `#147 `__ -\n Gracefully ignore missing avatar image, by\n `kBite `__.\n- `#142 `__ - Update\n release checklist.\n- `#134 `__ - Update\n GitHub integrations link.\n- `#120 `__ - Add\n license disclaimer.\n\nBug Fixes\n~~~~~~~~~\n\n- `#151 `__ - Fix gh\n command typos in docs, by `cyblue9 `__.\n- `#137 `__ - Fix\n Running as a Docker Container anchor in README, by\n `kamontat `__.\n- `#129 `__ - Fix\n trending command to handle empty summaries, by\n `emres `__.\n- `#123 `__ - Remove\n buggy codecov badge.\n- `#117 `__ - Fix\n 0.7.0 CHANGELOG date, by `dbaio `__.\n\n0.7.0 (2017-03-26)\n------------------\n\nFeatures\n~~~~~~~~\n\n- `#99 `__ - Add\n Dockerfile to run gitsome in a Docker container, by\n `l0rd `__ and\n `larson004 `__.\n\nBug Fixes\n~~~~~~~~~\n\n- `#67 `__ - Fix\n ``gh_issues`` typo in the ``README``, by\n `srisankethu `__.\n- `#69 `__ - Fix\n ``--issue_filter`` typo for ``gh_issues`` command in ``COMMANDS.md``.\n- `#80 `__ - Fix path\n for auto completions in ``README``.\n- `#92 `__ - Fix\n viewing HTML contents in the terminal for GitHub Enterprise users, by\n `dongweiming `__.\n- `#97 `__ - Fix error\n hint from ``gh gitignores`` to ``gh gitignore-templates``, by\n `zYeoman `__.\n- `#116 `__ - Fix gh\n trending command resulting in an error.\n\nUpdates\n~~~~~~~\n\n- `#58 `__ - Tweak\n ``README`` intro, add logo.\n- `#74 `__ - Add link\n to official GitHub integration page in ``README``.\n- `#79 `__ - Only store\n password in config for GitHub Enterprise (due to Enterprise\n limitations), by `nttibbetts `__.\n- `#86 `__ - Update\n dependency info for ``uritemplate``.\n- `#89 `__ - Fix a bug\n listing info on repos without a desc field, by\n `SanketDG `__.\n- `#98 `__ - Prefer\n GitHub Enterprise token before password.\n- `#104 `__ - Update\n install instructions to use pip3.\n- `#111 `__ - Add note\n about current Python 3.6 incompatibility.\n- `#115 `__ - Set\n current Python support to 3.4 and 3.5.\n\n0.6.0 (2016-05-29)\n------------------\n\nFeatures\n~~~~~~~~\n\n- `#3 `__ - Add GitHub\n Enterprise support.\n- `#33 `__ - Revamp\n the info shown with the ``gh feed`` command.\n\nBug Fixes\n~~~~~~~~~\n\n- `#30 `__ - Fix a\n typo in the ``pip3`` install instructions.\n- `#39 `__ - Fix\n ``gh feed`` ``-pr/--private`` flag in docs.\n- `#40 `__ - Fix\n ``create-issue`` ``NoneType`` error if no ``-b/--body`` is specified.\n- `#46 `__ - Fix\n ``gh view`` with the -b/--browser option only working for repos, not\n for issues or PRs.\n- `#48 `__ - Fix\n ``create-repo`` ``NoneType`` error if no ``-d/--description`` is\n specified.\n- `#54 `__ - Update to\n ``prompt-toolkit`` 1.0.0, which includes performance improvements\n (especially noticeable on Windows) and bug fixes.\n- Fix ``Config`` docstrings.\n\nUpdates\n~~~~~~~\n\n- `#26 `__,\n `#32 `__ - Add\n copyright notices for third party libraries.\n- `#44 `__,\n `#53 `__ - Update\n packaging dependencies based on semantic versioning.\n- Tweak ``README`` intro.\n\n0.5.0 (2016-05-15)\n------------------\n\nFeatures\n~~~~~~~~\n\n- `#12 `__ - Allow\n 2FA-enabled users to log in with a password + 2FA code. Previously\n 2FA-enabled users could only log in with a `personal access\n token `__. Also includes an\n update of login prompts to improve clarity.\n\nBug Fixes\n~~~~~~~~~\n\n- `#16 `__,\n `#28 `__ - Fix typos\n in README.\n- `#18 `__ - Fix dev\n install instructions in README.\n- `#24 `__ - Fix style\n guide broken link in CONTRIBUTING.\n\nUpdates\n~~~~~~~\n\n- `#1 `__ - Add\n Codecov coverage testing status to README.\n- `#2 `__ - Add note\n about enabling Zsh completions to README.\n- `#4 `__ - Add note\n about using ``pip3`` to README.\n- `#5 `__ - Decrease\n speed of README gif.\n- `#6 `__ - Update url\n for ``click``.\n- `#20 `__ - Add note\n about enabling more completions to README.\n- `#21 `__ - Bump up\n ``prompt-toolkit`` version from ``0.51`` to ``0.52``.\n- `#26 `__ - Add\n ``xonsh`` copyright notice to LICENSE.\n- `#32 `__ - Add\n ``github3.py``, ``html2text``, and ``img2txt`` copyright notices to\n LICENSE.\n- Update installation instructions in README.\n- Update color customization discussion in README.\n\n0.4.0 (2016-05-09)\n------------------\n\n- Initial release.\n\n.. |Build Status| image:: https://travis-ci.org/donnemartin/gitsome.svg?branch=master\n :target: https://travis-ci.org/donnemartin/gitsome\n.. |Codecov| image:: https://img.shields.io/codecov/c/github/donnemartin/gitsome.svg\n :target: https://codecov.io/github/donnemartin/gitsome\n.. |PyPI version| image:: https://badge.fury.io/py/gitsome.svg\n :target: http://badge.fury.io/py/gitsome\n.. |PyPI| image:: https://img.shields.io/pypi/pyversions/gitsome.svg\n :target: https://pypi.python.org/pypi/gitsome/\n.. |License| image:: https://img.shields.io/:license-apache-blue.svg\n :target: http://www.apache.org/licenses/LICENSE-2.0.html\n" }, { "path": "CHECKLIST.md", "content": "Release Checklist\n=================\n\nA. Install in a new venv and run unit tests\n\nNote, you can't seem to script the virtualenv calls, see:\nhttps://bitbucket.org/dhellmann/virtualenvwrapper/issues/219/cant-deactivate-active-virtualenv-from\n\n $ deactivate\n $ rmvirtualenv gitsome\n $ mkvirtualenv gitsome\n $ pip install -e .\n $ pip install -r requirements-dev.txt\n $ rm -rf .tox && tox\n\nB. Run code checks\n\n $ scripts/run_code_checks.sh\n\nC. Run manual [smoke tests](#smoke-tests) on Mac, Ubuntu, Windows\n\nD. Update and review `README.rst` and `Sphinx` docs, then check gitsome/docs/build/html/index.html\n\n $ scripts/update_docs.sh\n\nE. Push changes\n\nF. Review Travis, Codecov, and Gemnasium\n\nG. Start a new release branch\n\n $ git flow release start x.y.z\n\nH. Increment the version number in `gitsome/__init__.py`\n\nI. Update and review `CHANGELOG`\n\n $ scripts/create_changelog.sh\n\nJ. Commit the changes\n\nK. Finish the release branch\n\n $ git flow release finish 'x.y.z'\n\nL. Input a tag\n\n $ vx.y.z\n\nM. Push tagged release to develop and master\n\n $ git checkout master\n $ git push\n\n Might need to recreate develop branch.\n\nN. Set CHANGELOG.rst as `README.rst`\n\n $ scripts/set_changelog_as_readme.sh\n\nO. Register package with PyPi\n\n $ python setup.py register -r pypi\n\nP. Upload to PyPi\n\n $ python setup.py sdist upload -r pypi\n\nQ. Upload Sphinx docs to PyPi\n\n $ python setup.py upload_sphinx\n\nR. Review newly released package from PyPi\n\nS. Release on GitHub: https://github.com/donnemartin/gitsome/tags\n\n 1. Click \"Add release notes\" for latest release\n 2. Copy release notes from `CHANGELOG.md`\n 3. Click \"Publish release\"\n\nT. Install in a new venv and run manual [smoke tests](#smoke-tests) on Mac, Ubuntu, Windows\n\n## Smoke Tests\n\nRun the following on Python 3.4:\n\n* Craete a new `virtualenv`\n* Pip install `gitsome` into new `virtualenv`\n* Run `gitsome`\n* Run targeted tests based on recent code changes\n" }, { "path": "COMMANDS.md", "content": "# Commands\n\n## GitHub Integration Commands Reference\n\nCheck out the handy [autocompleter with interactive help](https://github.com/donnemartin/gitsome/blob/master/README.md#git-and-github-autocompleter-with-interactive-help) to guide you through each command.\n\n### gh configure\n\nConfigure `gitsome`.\n\nAttempts to authenticate the user and to set up the user's news feed.\n\nIf `gitsome` has not yet been configured, calling a `gh` command that requires authentication will automatically invoke the `configure` command.\n\nUsage/Example(s):\n\n $ gh configure\n\nFor GitHub Enterprise users, run with the `-e/--enterprise` flag:\n\n $ gh configure -e\n\n#### Authentication\n\nTo properly integrate with GitHub, you will be asked to enter a user name and either a password or a [personal access token](https://github.com/settings/tokens). If you use two-factor authentication, you will also need to enter your 2FA code, or you can log in with a personal access token.\n\nVisit the following page to generate a token:\n\n[https://github.com/settings/tokens](https://github.com/settings/tokens)\n\n`gitsome` will need the 'repo' and 'user' permissions.\n\n![Imgur](http://i.imgur.com/1C7gBHz.png)\n\n#### GitHub Enterprise\n\nGitHub Enterprise users will be asked to enter the GitHub Enterprise url and whether they want to verify SSL certificates.\n\n#### Authentication Source Code\n\nCurious what's going on behind the scenes with authentication? Check out the [authentication source code](https://github.com/donnemartin/gitsome/blob/master/gitsome/config.py#L177-L328).\n\n#### User Feed\n\n`gitsome` will need your news feed url to run the `gh feed` command with no arguments.\n\n![Imgur](http://i.imgur.com/2LWcyS6.png)\n\nTo integrate `gitsome` with your news feed, visit the following url while logged into GitHub:\n\n[https://github.com](https://github.com)\n\nYou will be asked to enter the url found when clicking 'Subscribe to your news feed', which will look something like this:\n\n https://github.com/donnemartin.private.atom?token=TOKEN\n\n![Imgur](http://i.imgur.com/f2zvdIm.png)\n\n### gh create-comment\n\nCreate a comment on the given issue.\n\nUsage:\n\n $ gh create-comment [user_repo_number] [-t/--text]\n\nParam(s):\n\n```\n:type user_repo_number: str\n:param user_repo_number: The user/repo/issue_number.\n```\n\nOption(s):\n\n```\n:type text: str\n:param text: The comment text.\n```\n\nExample(s):\n\n $ gh create-comment donnemartin/saws/1 -t \"hello world\"\n $ gh create-comment donnemartin/saws/1 --text \"hello world\"\n\n### gh create-issue\n\nCreate an issue.\n\nUsage:\n\n $ gh create-issue [user_repo] [-t/--issue_title] [-d/--issue_desc]\n\nParam(s):\n\n```\n:type user_repo: str\n:param user_repo: The user/repo.\n```\n\nOption(s):\n\n```\n:type issue_title: str\n:param issue_title: The issue title.\n\n:type issue_desc: str\n:param issue_desc: The issue body (optional).\n```\n\nExample(s):\n\n $ gh create-issue donnemartin/gitsome -t \"title\"\n $ gh create-issue donnemartin/gitsome -t \"title\" -d \"desc\"\n $ gh create-issue donnemartin/gitsome --issue_title \"title\" --issue_desc \"desc\"\n\n### gh create-repo\n\nCreate a repo.\n\nUsage:\n\n $ gh create-repo [repo_name] [-d/--repo_desc] [-pr/--private]\n\nParam(s):\n\n```\n:type repo_name: str\n:param repo_name: The repo name.\n```\n\nOption(s):\n\n```\n:type repo_desc: str\n:param repo_desc: The repo description (optional).\n\n:type private: bool\n:param private: Determines whether the repo is private.\n Default: False.\n```\n\nExample(s):\n\n $ gh create-repo repo_name\n $ gh create-repo repo_name -d \"desc\"\n $ gh create-repo repo_name --repo_desc \"desc\"\n $ gh create-repo repo_name -pr\n $ gh create-repo repo_name --repo_desc \"desc\" --private\n\n### gh emails\n\nList all the user's registered emails.\n\nUsage/Example(s):\n\n $ gh emails\n\n### gh emojis\n\nList all GitHub supported emojis.\n\nUsage:\n\n $ gh emojis [-p/--pager]\n\nOption(s):\n\n```\n:type pager: bool\n:param pager: Determines whether to show the output in a pager,\n if available.\n```\n\nExample(s):\n\n $ gh emojis\n $ gh emojis -p\n $ gh emojis --pager\n\n### gh feed\n\nList all activity for the given user or repo.\n\nIf `user_or_repo` is not provided, uses the logged in user's news feed seen while visiting https://github.com. If `user_or_repo` is provided, shows either the public or `[-pr/--private]` feed activity of the user or repo.\n\nUsage:\n\n $ gh feed [user_or_repo] [-pr/--private] [-p/--pager]\n\nParam(s):\n\n```\n:type user_or_repo: str\n:param user_or_repo: The user or repo to list events for (optional).\n If no entry, defaults to the logged in user's feed.\n```\n\nOption(s):\n\n```\n:type private: bool\n:param private: Determines whether to show the private events (True)\n or public events (False).\n\n:type pager: bool\n:param pager: Determines whether to show the output in a pager,\n if available.\n```\n\nExample(s):\n\n $ gh feed\n $ gh feed | grep foo\n $ gh feed donnemartin\n $ gh feed donnemartin -pr -p\n $ gh feed donnemartin --private --pager\n $ gh feed donnemartin/haxor-news -p\n\n#### News Feed\n\n![Imgur](http://i.imgur.com/2LWcyS6.png)\n\n#### User Activity Feed\n\n![Imgur](http://i.imgur.com/kryGLXz.png)\n\n#### Repo Activity Feed\n\n![Imgur](http://i.imgur.com/d2kxDg9.png)\n\n### gh following\n\nList all followed users and the total followed count.\n\nUsage:\n\n $ gh following [user] [-p/--pager]\n\nParam(s):\n\n```\n:type user: str\n:param user: The user login.\n If None, returns the followed users of the logged in user.\n```\n\nOption(s):\n\n```\n:type pager: bool\n:param pager: Determines whether to show the output in a pager,\n if available.\n```\n\nExample(s):\n\n $ gh following\n $ gh following -p\n $ gh following octocat --pager\n\n![Imgur](http://i.imgur.com/bjUmbf3.png)\n\nAlso check out the [`gh user`](#gh-user) command.\n\n### gh followers\n\nList all followers and the total follower count.\n\nUsage:\n\n $ gh followers [user] [-p/--pager]\n\nParam(s):\n\n```\n:type user: str\n:param user: The user login (optional).\n If None, returns the followers of the logged in user.\n```\n\nOption(s):\n\n```\n:type pager: bool\n:param pager: Determines whether to show the output in a pager,\n if available.\n```\n\nExample(s):\n\n $ gh followers\n $ gh followers -p\n $ gh followers octocat --pager\n\n### gh gitignore-template\n\nOutput the gitignore template for the given language.\n\nUsage:\n\n $ gh gitignore-template [language]\n\nParam(s):\n\n```\n:type language: str\n:param language: The language.\n```\n\nExample(s):\n\n $ gh gitignore-template Python\n $ gh gitignore-template Python > .gitignore\n\n![Imgur](http://i.imgur.com/S5m5ZcO.png)\n\n### gh gitignore-templates\n\nOutput all supported gitignore templates.\n\nUsage:\n\n $ gh gitignore-templates [-p/--pager]\n\nOption(s):\n\n```\n:type pager: bool\n:param pager: Determines whether to show the output in a pager,\n if available.\n```\n\nExample(s):\n\n $ gh gitignore-templates\n $ gh gitignore-templates -p\n $ gh gitignore-templates --pager\n\n![Imgur](http://i.imgur.com/u8qYx1s.png)\n\n### gh issue\n\nOutput detailed information about the given issue.\n\nUsage:\n\n $ gh issue [user_repo_number]\n\nParam(s):\n\n```\n:type user_repo_number: str\n:param user_repo_number: The user/repo/issue_number.\n```\n\nExample(s):\n\n $ gh issue donnemartin/saws/1\n\n![Imgur](http://i.imgur.com/ZFv9MuV.png)\n\n### gh issues\n\nList all issues matching the filter.\n\nUsage:\n\n $ gh issues [-f/--issue_filter] [-s/--issue_state] [-l/--limit] [-p/--pager]\n\nOption(s):\n\n```\n:type issue_filter: str\n:param issue_filter: assigned, created, mentioned, subscribed (default).\n\n:type issue_state: str\n:param issue_state: all, open (default), closed.\n\n:type limit: int\n:param limit: The number of items to display.\n\n:type pager: bool\n:param pager: Determines whether to show the output in a pager,\n if available.\n```\n\nExample(s):\n\n $ gh issues\n $ gh issues -f assigned\n $ gh issues --issue_filter created\n $ gh issues -s all -l 20 -p\n $ gh issues --issue_state closed --limit 20 --pager\n $ gh issues -f created -s all -p\n\n![Imgur](http://i.imgur.com/AB5zxxo.png)\n\n### gh license\n\nOutput the license template for the given license.\n\nUsage:\n\n $ gh license [license_name]\n\nParam(s):\n\n```\n:type license_name: str\n:param license_name: The license name.\n```\n\nExample(s):\n\n $ gh license apache-2.0\n $ gh license mit > LICENSE\n\n![Imgur](http://i.imgur.com/zJHVxaA.png)\n\n### gh licenses\n\nOutput all supported license templates.\n\nUsage/Licenses:\n\n $ gh licenses\n\n![Imgur](http://i.imgur.com/S9SbMLJ.png)\n\n### gh me\n\nList information about the logged in user.\n\nDisplaying the avatar will require [installing the optional `PIL` dependency](#installing-pil).\n\nUsage:\n\n $ gh me [-b/--browser] [-t/--text_avatar] [-l/--limit] [-p/--pager]\n\nOption(s):\n\n```\n:type browser: bool\n:param browser: Determines whether to view the profile\n in a browser, or in the terminal.\n\n:type text_avatar: bool\n:param text_avatar: Determines whether to view the profile\n avatar in plain text.\n On Windows this value is always set to True due to lack of\n support of `img2txt` on Windows.\n\n:type limit: int\n:param limit: The number of user repos to display.\n\n:type pager: bool\n:param pager: Determines whether to show the output in a pager,\n if available.\n```\n\nExample(s):\n\n $ gh me\n $ gh me -b\n $ gh me --browser\n $ gh me -t -l 20 -p\n $ gh me --text_avatar --limit 20 --pager\n\n![Imgur](http://i.imgur.com/csk5j0S.png)\n\n### gh notifications\n\nList all notifications.\n\nUsage:\n\n $ gh notifications [-l/--limit] [-p/--pager]\n\nOption(s):\n\n```\n:type limit: int\n:param limit: The number of items to display.\n\n:type pager: bool\n:param pager: Determines whether to show the output in a pager,\n if available.\n```\n\nExample(s):\n\n $ gh notifications\n $ gh notifications -l 20 -p\n $ gh notifications --limit 20 --pager\n\n![Imgur](http://i.imgur.com/uwmwxsW.png)\n\n### gh octo\n\nOutput an Easter egg or the given message from Octocat.\n\nUsage:\n\n $ gh octo [say]\n\nParam(s):\n\n```\n:type say: str\n:param say: What Octocat should say.\n If say is None, octocat speaks an Easter egg.\n```\n\nExample(s):\n\n $ gh octo\n $ gh octo \"foo bar\"\n\n![Imgur](http://i.imgur.com/bNzCa5p.png)\n\n### gh pull-request\n\nOutput detailed information about the given pull request.\n\nUsage:\n\n $ gh pull-request [user_repo_number]\n\nParam(s):\n\n```\n:type user_repo_number: str\n:param user_repo_number: The user/repo/pull_number.\n```\n\nExample(s):\n\n $ gh pull-request donnemartin/saws/80\n\n![Imgur](http://i.imgur.com/3MtKjKy.png)\n\n### gh pull-requests\n\nList all pull requests.\n\nUsage:\n\n $ gh pull-requests [-l/--limit] [-p/--pager]\n\nOption(s):\n\n```\n:type limit: int\n:param limit: The number of items to display.\n\n:type pager: bool\n:param pager: Determines whether to show the output in a pager,\n if available.\n```\n\nExample(s):\n\n $ gh pull-requests\n $ gh pull-requests -l 20 -p\n $ gh pull-requests --limit 20 --pager\n\n![Imgur](http://i.imgur.com/4A2eYM9.png)\n\n### gh rate-limit\n\nOutput the rate limit. Not available for GitHub Enterprise.\n\nUsage/Example(s):\n\n $ gh rate-limit\n\n### gh repo\n\nOutput detailed information about the given repo.\n\nUsage:\n\n $ gh repo [user_repo]\n\nParam(s):\n\n```\n:type user_repo: str\n:param user_repo: The user/repo.\n```\n\nExample(s):\n\n $ gh repo donnemartin/haxor-news\n\n![Imgur](http://i.imgur.com/XFMpWCI.png)\n\n### gh repos\n\nList all repos matching the given filter.\n\nUsage:\n\n $ gh repos [repo_filter] [-l/--limit] [-p/--pager]\n\nParam(s):\n\n```\n:type repo_filter: str\n:param repo_filter: The filter for repo names.\n Only repos matching the filter will be returned.\n If None, outputs all the logged in user's repos.\n```\n\nOption(s):\n\n```\n:type limit: int\n:param limit: The number of items to display.\n\n:type pager: bool\n:param pager: Determines whether to show the output in a pager,\n if available.\n```\n\nExample(s):\n\n $ gh repos\n $ gh repos aws\n $ gh repos aws -l 20 -p\n $ gh repos aws --limit 20 --pager\n\n![Imgur](http://i.imgur.com/YXWPWma.png)\n\n### gh search-issues\n\nSearch for all issues matching the given query.\n\nFor more information about the query qualifiers, visit the [searching issues reference](https://help.github.com/articles/searching-issues/).\n\nUsage:\n\n $ gh search-issues [query] [-l/--limit] [-p/--pager]\n\nParam(s):\n\n```\n:type query: str\n:param query: The search query.\n\nThe query can contain any combination of the following supported\nqualifers:\n\n- `type` With this qualifier you can restrict the search to issues\n or pull request only.\n- `in` Qualifies which fields are searched. With this qualifier you\n can restrict the search to just the title, body, comments, or any\n combination of these.\n- `author` Finds issues created by a certain user.\n- `assignee` Finds issues that are assigned to a certain user.\n- `mentions` Finds issues that mention a certain user.\n- `commenter` Finds issues that a certain user commented on.\n- `involves` Finds issues that were either created by a certain user,\n assigned to that user, mention that user, or were commented on by\n that user.\n- `state` Filter issues based on whether they’re open or closed.\n- `labels` Filters issues based on their labels.\n- `language` Searches for issues within repositories that match a\n certain language.\n- `created` or `updated` Filters issues based on times of creation,\n or when they were last updated.\n- `comments` Filters issues based on the quantity of comments.\n- `user` or `repo` Limits searches to a specific user or\n repository.\n\nFor more information about these qualifiers, see: http://git.io/d1oELA\n```\n\nOption(s):\n\n```\n:type limit: int\n:param limit: The number of items to display.\n\n:type pager: bool\n:param pager: Determines whether to show the output in a pager,\n if available.\n```\n\nExample(s):\n\n $ gh search-issues \"foo type:pr author:donnemartin\" -l 20 -p\n $ gh search-issues \"foobarbaz in:title created:>=2015-01-01\" --limit 20 --pager\n\n\nAdditional Example(s):\n\n```\nSearch issues that have your user name tagged @donnemartin:\n $ gh search-issues \"is:issue donnemartin is:open\" -p\n\nSearch issues that have the most +1s:\n $ gh search-issues \"is:open is:issue sort:reactions-+1-desc\" -p\n\nSearch issues that have the most comments:\n $ gh search-issues \"is:open is:issue sort:comments-desc\" -p\n\nSearch issues with the \"help wanted\" tag:\n $ gh search-issues \"is:open is:issue label:\\\"help wanted\\\"\" -p\n\nSearch all your open private issues:\n $ gh search-issues \"is:open is:issue is:private\" -p\n```\n\n![Imgur](http://i.imgur.com/DXXxkBD.png)\n\n### gh search-repos\n\nSearch for all repos matching the given query.\n\nFor more information about the query qualifiers, visit the [searching repos reference](https://help.github.com/articles/searching-repositories/).\n\nUsage:\n\n $ gh search-repos [query] [-s/--sort] [-l/--limit] [-p/--pager]\n\nParam(s):\n\n```\n:type query: str\n:param query: The search query.\n\nThe query can contain any combination of the following supported\nqualifers:\n\n- `in` Qualifies which fields are searched. With this qualifier you\n can restrict the search to just the repository name, description,\n readme, or any combination of these.\n- `size` Finds repositories that match a certain size (in\n kilobytes).\n- `forks` Filters repositories based on the number of forks, and/or\n whether forked repositories should be included in the results at\n all.\n- `created` or `pushed` Filters repositories based on times of\n creation, or when they were last updated. Format: `YYYY-MM-DD`.\n Examples: `created:<2011`, `pushed:<2013-02`,\n `pushed:>=2013-03-06`\n- `user` or `repo` Limits searches to a specific user or\n repository.\n- `language` Searches repositories based on the language they're\n written in.\n- `stars` Searches repositories based on the number of stars.\n\nFor more information about these qualifiers, see: http://git.io/4Z8AkA\n```\n\nOption(s):\n\n```\n:type sort: str\n:param sort: Optional: 'stars', 'forks', 'updated'.\n If not specified, sorting is done by query best match.\n\n:type limit: int\n:param limit: The number of items to display.\n\n:type pager: bool\n:param pager: Determines whether to show the output in a pager,\n if available.\n```\n\nExample(s):\n\n $ gh search-repos \"maps language:python\" -s stars -l 20 -p\n $ gh search-repos \"created:>=2015-01-01 stars:>=1000 language:python\" --sort stars --limit 20 --pager\n\n![Imgur](http://i.imgur.com/kazXWWY.png)\n\n### gh starred\n\nOutput starred repos.\n\nUsage:\n\n $ gh starred [repo_filter] [-l/--limit] [-p/--pager]\n\nParam(s):\n\n```\n:type repo_filter: str\n:param repo_filter: The filter for repo names.\n Only repos matching the filter will be returned.\n If None, outputs all starred repos.\n```\n\nOption(s):\n\n```\n:type limit: int\n:param limit: The number of items to display.\n\n:type pager: bool\n:param pager: Determines whether to show the output in a pager,\n if available.\n```\n\nExample(s):\n\n $ gh starred\n $ gh starred foo -l 20 -p\n $ gh starred foo --limit 20 --pager\n\n![Imgur](http://i.imgur.com/JB88Kw8.png)\n\n### gh trending\n\nList trending repos for the given language.\n\nUsage:\n\n $ gh trending [language] [-w/--weekly] [-m/--monthly] [-D/--devs] [-b/--browser] [-p/--pager]\n\nParam(s):\n\n```\n:type language: str\n:param language: The language (optional).\n If blank, shows 'Overall'.\n```\n\nOption(s):\n\n```\n:type weekly: bool\n:param weekly: Determines whether to show the weekly rankings.\n Daily is the default.\n\n:type monthly: bool\n:param monthly: Determines whether to show the monthly rankings.\n Daily is the default.\n If both `monthly` and `weekly` are set, `monthly` takes precedence.\n\n:type devs: bool\n:param devs: determines whether to display the trending\n devs or repos. Only valid with the -b/--browser option.\n\n:type browser: bool\n:param browser: Determines whether to view the profile\n in a browser, or in the terminal.\n\n:type pager: bool\n:param pager: Determines whether to show the output in a pager,\n if available.\n```\n\nExample(s):\n\n $ gh trending\n $ gh trending Python -w -p\n $ gh trending Python --weekly --devs --browser\n $ gh trending --browser\n\n![Imgur](http://i.imgur.com/Dx77HFW.png)\n\n### gh user\n\nList information about the given user.\n\nDisplaying the avatar will require [installing the optional `PIL` dependency](#installing-pil).\n\nUsage:\n\n $ gh user [user_id] [-b/--browser] [-t/--text_avatar] [-l/--limit] [-p/--pager]\n\nParam(s):\n\n```\n:type user_id: str\n:param user_id: The user id/login.\n If None, returns followers of the logged in user.\n```\n\nOption(s):\n\n```\n:type browser: bool\n:param browser: Determines whether to view the profile\n in a browser, or in the terminal.\n\n:type text_avatar: bool\n:param text_avatar: Determines whether to view the profile\n avatar in plain text instead of ansi (default).\n On Windows this value is always set to True due to lack of\n support of `img2txt` on Windows.\n\n:type limit: int\n:param limit: The number of items to display.\n\n:type pager: bool\n:param pager: Determines whether to show the output in a pager,\n if available.\n```\n\nExample(s):\n\n $ gh user octocat\n $ gh user octocat -b\n $ gh user octocat --browser\n $ gh user octocat -t -l 10 -p\n $ gh user octocat --text_avatar --limit 10 --pager\n\n![Imgur](http://i.imgur.com/xVoVPVe.png)\n\n### gh view\n\nView the given notification/repo/issue/pull_request/user index in the terminal or a browser.\n\nThis method is meant to be called after one of the following commands\nwhich outputs a table of notifications/repos/issues/pull_requests/users:\n\n $ gh repos\n $ gh search-repos\n $ gh starred\n\n $ gh issues\n $ gh pull-requests\n $ gh search-issues\n\n $ gh notifications\n $ gh trending\n\n $ gh user\n $ gh me\n\nUsage:\n\n $ gh view [index] [-b/--browser]\n\nParam(s):\n\n```\n:type index: str\n:param index: Determines the index to view.\n```\n\nOption(s):\n\n```\n:type browser: bool\n:param browser: Determines whether to view the profile\n in a browser, or in the terminal.\n```\n\nExample(s):\n\n $ gh repos\n $ gh view 1\n\n $ gh starred\n $ gh view 1 -b\n $ gh view 1 --browser\n\n![Imgur](http://i.imgur.com/NVEwGbV.png)\n\n### Option: View in a Pager\n\nMany `gh` commands support a `-p/--pager` option that displays results in a pager, where available.\n\nUsage:\n\n $ gh [param] [options] -p\n $ gh [param] [options] --pager\n\n### Option: View in a Browser\n\nMany `gh` commands support a `-b/--browser` option that displays results in your default browser instead of your terminal.\n\nUsage:\n\n $ gh [param] [options] -b\n $ gh [param] [options] --browser\n" }, { "path": "CONTRIBUTING.md", "content": "Contributing\n============\n\nContributions are welcome!\n\n**Please carefully read this page to make the code review process go as smoothly as possible and to maximize the likelihood of your contribution being merged.**\n\n## Bug Reports\n\nFor bug reports or requests [submit an issue](https://github.com/donnemartin/gitsome/issues).\n\n## Pull Requests\n\nThe preferred way to contribute is to fork the\n[main repository](https://github.com/donnemartin/gitsome) on GitHub.\n\n1. Fork the [main repository](https://github.com/donnemartin/gitsome). Click on the 'Fork' button near the top of the page. This creates a copy of the code under your account on the GitHub server.\n\n2. Clone this copy to your local disk:\n\n $ git clone git@github.com:YourLogin/gitsome.git\n $ cd gitsome\n\n3. Create a branch to hold your changes and start making changes. Don't work in the `master` branch!\n\n $ git checkout -b my-feature\n\n4. Work on this copy on your computer using Git to do the version control. When you're done editing, run the following to record your changes in Git:\n\n $ git add modified_files\n $ git commit\n\n5. Push your changes to GitHub with:\n\n $ git push -u origin my-feature\n\n6. Finally, go to the web page of your fork of the `gitsome` repo and click 'Pull Request' to send your changes for review.\n\n### GitHub Pull Requests Docs\n\nIf you are not familiar with pull requests, review the [pull request docs](https://help.github.com/articles/using-pull-requests/).\n\n### Code Quality\n\nEnsure your pull request satisfies all of the following, where applicable:\n\n* Is covered by [unit tests](https://github.com/donnemartin/gitsome#unit-tests-and-code-coverage)\n* Passes [continuous integration](https://github.com/donnemartin/gitsome#continuous-integration)\n* Is covered by [documentation](https://github.com/donnemartin/gitsome#documentation)\n\nReview the following [style guide](https://google.github.io/styleguide/pyguide.html).\n\nRun code checks and fix any issues:\n\n $ scripts/run_code_checks.sh\n\n### Installation\n\nRefer to the [Installation](https://github.com/donnemartin/gitsome#installation) and [Developer Installation](https://github.com/donnemartin/gitsome#developer-installation) sections.\n" }, { "path": "Dockerfile", "content": "##########################################################\n# #\n# Build the image: #\n# docker build -t gitsome . #\n# #\n# Run the container: #\n# docker run -ti --rm -v $(pwd):/src/ \\ #\n# -v ${HOME}/.gitsomeconfig:/root/.gitsomeconfig \\ #\n# -v ${HOME}/.gitconfig:/root/.gitconfig \\ #\n# gitsome #\n# #\n##########################################################\nFROM python:3.5\n\nRUN pip install Pillow\n\nCOPY /requirements-dev.txt /gitsome/\nWORKDIR /gitsome/\nRUN pip install -r requirements-dev.txt\n\nCOPY / /gitsome/\nRUN pip install -e .\n\nRUN mkdir /src/\nWORKDIR /src/\nENTRYPOINT [\"gitsome\"]\n" }, { "path": "LICENSE.txt", "content": "gitsome is licensed under Apache 2.0:\n\nI am providing code and resources in this repository to you under an open source\nlicense. Because this is my personal repository, the license you receive to my\ncode and resources is from me and not my employer (Facebook).\n\n Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"{}\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright 2016 Donne Martin. All Rights Reserved.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n\nxonsh is licensed under BSD:\n\nCopyright 2015, the xonsh developers. All rights reserved.\n\nRedistribution and use in source and binary forms, with or without modification, are\npermitted provided that the following conditions are met:\n\n 1. Redistributions of source code must retain the above copyright notice, this list of\n conditions and the following disclaimer.\n\n 2. Redistributions in binary form must reproduce the above copyright notice, this list\n of conditions and the following disclaimer in the documentation and/or other materials\n provided with the distribution.\n\nTHIS SOFTWARE IS PROVIDED BY THE XONSH DEVELOPERS ``AS IS'' AND ANY EXPRESS OR IMPLIED\nWARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND\nFITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE XONSH DEVELOPERS OR\nCONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR\nCONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\nSERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON\nANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING\nNEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF\nADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nThe views and conclusions contained in the software and documentation are those of the\nauthors and should not be interpreted as representing official policies, either expressed\nor implied, of the stakeholders of the xonsh project or the employers of xonsh developers.\n\n\ngithub3.py is licensed under BSD:\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are\nmet:\n\n1. Redistributions of source code must retain the above copyright\nnotice, this list of conditions and the following disclaimer.\n2. Redistributions in binary form must reproduce the above copyright\nnotice, this list of conditions and the following disclaimer in the\ndocumentation and/or other materials provided with the distribution.\n3. The name of the author may not be used to endorse or promote products\nderived from this software without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR\nIMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\nDISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,\nINDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\nSERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)\nHOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,\nSTRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN\nANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE\nPOSSIBILITY OF SUCH DAMAGE.\n\n\nimg2txt is licensed under BSD:\n\nCopyright (c) 2013 - 2015, hit9\n\nAll rights reserved.\n\nRedistribution and use in source and binary forms, with or without modification,\nare permitted provided that the following conditions are met:\n\n * Redistributions of source code must retain the above copyright notice,\n this list of conditions and the following disclaimer.\n * Redistributions in binary form must reproduce the above copyright notice,\n this list of conditions and the following disclaimer in the documentation\n and/or other materials provided with the distribution.\n * Neither the name of img2txt nor the names of its contributors\n may be used to endorse or promote products derived from this software\n without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n\"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\nLIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\nA PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\nCONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\nEXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\nPROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\nPROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF\nLIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING\nNEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\n\nhtml2text is licensed under GNU:\n\n GNU GENERAL PUBLIC LICENSE\n Version 3, 29 June 2007\n\n Copyright (C) 2007 Free Software Foundation, Inc. \n Everyone is permitted to copy and distribute verbatim copies\n of this license document, but changing it is not allowed.\n\n Preamble\n\n The GNU General Public License is a free, copyleft license for\nsoftware and other kinds of works.\n\n The licenses for most software and other practical works are designed\nto take away your freedom to share and change the works. By contrast,\nthe GNU General Public License is intended to guarantee your freedom to\nshare and change all versions of a program--to make sure it remains free\nsoftware for all its users. We, the Free Software Foundation, use the\nGNU General Public License for most of our software; it applies also to\nany other work released this way by its authors. You can apply it to\nyour programs, too.\n\n When we speak of free software, we are referring to freedom, not\nprice. Our General Public Licenses are designed to make sure that you\nhave the freedom to distribute copies of free software (and charge for\nthem if you wish), that you receive source code or can get it if you\nwant it, that you can change the software or use pieces of it in new\nfree programs, and that you know you can do these things.\n\n To protect your rights, we need to prevent others from denying you\nthese rights or asking you to surrender the rights. Therefore, you have\ncertain responsibilities if you distribute copies of the software, or if\nyou modify it: responsibilities to respect the freedom of others.\n\n For example, if you distribute copies of such a program, whether\ngratis or for a fee, you must pass on to the recipients the same\nfreedoms that you received. You must make sure that they, too, receive\nor can get the source code. And you must show them these terms so they\nknow their rights.\n\n Developers that use the GNU GPL protect your rights with two steps:\n(1) assert copyright on the software, and (2) offer you this License\ngiving you legal permission to copy, distribute and/or modify it.\n\n For the developers' and authors' protection, the GPL clearly explains\nthat there is no warranty for this free software. For both users' and\nauthors' sake, the GPL requires that modified versions be marked as\nchanged, so that their problems will not be attributed erroneously to\nauthors of previous versions.\n\n Some devices are designed to deny users access to install or run\nmodified versions of the software inside them, although the manufacturer\ncan do so. This is fundamentally incompatible with the aim of\nprotecting users' freedom to change the software. The systematic\npattern of such abuse occurs in the area of products for individuals to\nuse, which is precisely where it is most unacceptable. Therefore, we\nhave designed this version of the GPL to prohibit the practice for those\nproducts. If such problems arise substantially in other domains, we\nstand ready to extend this provision to those domains in future versions\nof the GPL, as needed to protect the freedom of users.\n\n Finally, every program is threatened constantly by software patents.\nStates should not allow patents to restrict development and use of\nsoftware on general-purpose computers, but in those that do, we wish to\navoid the special danger that patents applied to a free program could\nmake it effectively proprietary. To prevent this, the GPL assures that\npatents cannot be used to render the program non-free.\n\n The precise terms and conditions for copying, distribution and\nmodification follow.\n\n TERMS AND CONDITIONS\n\n 0. Definitions.\n\n \"This License\" refers to version 3 of the GNU General Public License.\n\n \"Copyright\" also means copyright-like laws that apply to other kinds of\nworks, such as semiconductor masks.\n\n \"The Program\" refers to any copyrightable work licensed under this\nLicense. Each licensee is addressed as \"you\". \"Licensees\" and\n\"recipients\" may be individuals or organizations.\n\n To \"modify\" a work means to copy from or adapt all or part of the work\nin a fashion requiring copyright permission, other than the making of an\nexact copy. The resulting work is called a \"modified version\" of the\nearlier work or a work \"based on\" the earlier work.\n\n A \"covered work\" means either the unmodified Program or a work based\non the Program.\n\n To \"propagate\" a work means to do anything with it that, without\npermission, would make you directly or secondarily liable for\ninfringement under applicable copyright law, except executing it on a\ncomputer or modifying a private copy. Propagation includes copying,\ndistribution (with or without modification), making available to the\npublic, and in some countries other activities as well.\n\n To \"convey\" a work means any kind of propagation that enables other\nparties to make or receive copies. Mere interaction with a user through\na computer network, with no transfer of a copy, is not conveying.\n\n An interactive user interface displays \"Appropriate Legal Notices\"\nto the extent that it includes a convenient and prominently visible\nfeature that (1) displays an appropriate copyright notice, and (2)\ntells the user that there is no warranty for the work (except to the\nextent that warranties are provided), that licensees may convey the\nwork under this License, and how to view a copy of this License. If\nthe interface presents a list of user commands or options, such as a\nmenu, a prominent item in the list meets this criterion.\n\n 1. Source Code.\n\n The \"source code\" for a work means the preferred form of the work\nfor making modifications to it. \"Object code\" means any non-source\nform of a work.\n\n A \"Standard Interface\" means an interface that either is an official\nstandard defined by a recognized standards body, or, in the case of\ninterfaces specified for a particular programming language, one that\nis widely used among developers working in that language.\n\n The \"System Libraries\" of an executable work include anything, other\nthan the work as a whole, that (a) is included in the normal form of\npackaging a Major Component, but which is not part of that Major\nComponent, and (b) serves only to enable use of the work with that\nMajor Component, or to implement a Standard Interface for which an\nimplementation is available to the public in source code form. A\n\"Major Component\", in this context, means a major essential component\n(kernel, window system, and so on) of the specific operating system\n(if any) on which the executable work runs, or a compiler used to\nproduce the work, or an object code interpreter used to run it.\n\n The \"Corresponding Source\" for a work in object code form means all\nthe source code needed to generate, install, and (for an executable\nwork) run the object code and to modify the work, including scripts to\ncontrol those activities. However, it does not include the work's\nSystem Libraries, or general-purpose tools or generally available free\nprograms which are used unmodified in performing those activities but\nwhich are not part of the work. For example, Corresponding Source\nincludes interface definition files associated with source files for\nthe work, and the source code for shared libraries and dynamically\nlinked subprograms that the work is specifically designed to require,\nsuch as by intimate data communication or control flow between those\nsubprograms and other parts of the work.\n\n The Corresponding Source need not include anything that users\ncan regenerate automatically from other parts of the Corresponding\nSource.\n\n The Corresponding Source for a work in source code form is that\nsame work.\n\n 2. Basic Permissions.\n\n All rights granted under this License are granted for the term of\ncopyright on the Program, and are irrevocable provided the stated\nconditions are met. This License explicitly affirms your unlimited\npermission to run the unmodified Program. The output from running a\ncovered work is covered by this License only if the output, given its\ncontent, constitutes a covered work. This License acknowledges your\nrights of fair use or other equivalent, as provided by copyright law.\n\n You may make, run and propagate covered works that you do not\nconvey, without conditions so long as your license otherwise remains\nin force. You may convey covered works to others for the sole purpose\nof having them make modifications exclusively for you, or provide you\nwith facilities for running those works, provided that you comply with\nthe terms of this License in conveying all material for which you do\nnot control copyright. Those thus making or running the covered works\nfor you must do so exclusively on your behalf, under your direction\nand control, on terms that prohibit them from making any copies of\nyour copyrighted material outside their relationship with you.\n\n Conveying under any other circumstances is permitted solely under\nthe conditions stated below. Sublicensing is not allowed; section 10\nmakes it unnecessary.\n\n 3. Protecting Users' Legal Rights From Anti-Circumvention Law.\n\n No covered work shall be deemed part of an effective technological\nmeasure under any applicable law fulfilling obligations under article\n11 of the WIPO copyright treaty adopted on 20 December 1996, or\nsimilar laws prohibiting or restricting circumvention of such\nmeasures.\n\n When you convey a covered work, you waive any legal power to forbid\ncircumvention of technological measures to the extent such circumvention\nis effected by exercising rights under this License with respect to\nthe covered work, and you disclaim any intention to limit operation or\nmodification of the work as a means of enforcing, against the work's\nusers, your or third parties' legal rights to forbid circumvention of\ntechnological measures.\n\n 4. Conveying Verbatim Copies.\n\n You may convey verbatim copies of the Program's source code as you\nreceive it, in any medium, provided that you conspicuously and\nappropriately publish on each copy an appropriate copyright notice;\nkeep intact all notices stating that this License and any\nnon-permissive terms added in accord with section 7 apply to the code;\nkeep intact all notices of the absence of any warranty; and give all\nrecipients a copy of this License along with the Program.\n\n You may charge any price or no price for each copy that you convey,\nand you may offer support or warranty protection for a fee.\n\n 5. Conveying Modified Source Versions.\n\n You may convey a work based on the Program, or the modifications to\nproduce it from the Program, in the form of source code under the\nterms of section 4, provided that you also meet all of these conditions:\n\n a) The work must carry prominent notices stating that you modified\n it, and giving a relevant date.\n\n b) The work must carry prominent notices stating that it is\n released under this License and any conditions added under section\n 7. This requirement modifies the requirement in section 4 to\n \"keep intact all notices\".\n\n c) You must license the entire work, as a whole, under this\n License to anyone who comes into possession of a copy. This\n License will therefore apply, along with any applicable section 7\n additional terms, to the whole of the work, and all its parts,\n regardless of how they are packaged. This License gives no\n permission to license the work in any other way, but it does not\n invalidate such permission if you have separately received it.\n\n d) If the work has interactive user interfaces, each must display\n Appropriate Legal Notices; however, if the Program has interactive\n interfaces that do not display Appropriate Legal Notices, your\n work need not make them do so.\n\n A compilation of a covered work with other separate and independent\nworks, which are not by their nature extensions of the covered work,\nand which are not combined with it such as to form a larger program,\nin or on a volume of a storage or distribution medium, is called an\n\"aggregate\" if the compilation and its resulting copyright are not\nused to limit the access or legal rights of the compilation's users\nbeyond what the individual works permit. Inclusion of a covered work\nin an aggregate does not cause this License to apply to the other\nparts of the aggregate.\n\n 6. Conveying Non-Source Forms.\n\n You may convey a covered work in object code form under the terms\nof sections 4 and 5, provided that you also convey the\nmachine-readable Corresponding Source under the terms of this License,\nin one of these ways:\n\n a) Convey the object code in, or embodied in, a physical product\n (including a physical distribution medium), accompanied by the\n Corresponding Source fixed on a durable physical medium\n customarily used for software interchange.\n\n b) Convey the object code in, or embodied in, a physical product\n (including a physical distribution medium), accompanied by a\n written offer, valid for at least three years and valid for as\n long as you offer spare parts or customer support for that product\n model, to give anyone who possesses the object code either (1) a\n copy of the Corresponding Source for all the software in the\n product that is covered by this License, on a durable physical\n medium customarily used for software interchange, for a price no\n more than your reasonable cost of physically performing this\n conveying of source, or (2) access to copy the\n Corresponding Source from a network server at no charge.\n\n c) Convey individual copies of the object code with a copy of the\n written offer to provide the Corresponding Source. This\n alternative is allowed only occasionally and noncommercially, and\n only if you received the object code with such an offer, in accord\n with subsection 6b.\n\n d) Convey the object code by offering access from a designated\n place (gratis or for a charge), and offer equivalent access to the\n Corresponding Source in the same way through the same place at no\n further charge. You need not require recipients to copy the\n Corresponding Source along with the object code. If the place to\n copy the object code is a network server, the Corresponding Source\n may be on a different server (operated by you or a third party)\n that supports equivalent copying facilities, provided you maintain\n clear directions next to the object code saying where to find the\n Corresponding Source. Regardless of what server hosts the\n Corresponding Source, you remain obligated to ensure that it is\n available for as long as needed to satisfy these requirements.\n\n e) Convey the object code using peer-to-peer transmission, provided\n you inform other peers where the object code and Corresponding\n Source of the work are being offered to the general public at no\n charge under subsection 6d.\n\n A separable portion of the object code, whose source code is excluded\nfrom the Corresponding Source as a System Library, need not be\nincluded in conveying the object code work.\n\n A \"User Product\" is either (1) a \"consumer product\", which means any\ntangible personal property which is normally used for personal, family,\nor household purposes, or (2) anything designed or sold for incorporation\ninto a dwelling. In determining whether a product is a consumer product,\ndoubtful cases shall be resolved in favor of coverage. For a particular\nproduct received by a particular user, \"normally used\" refers to a\ntypical or common use of that class of product, regardless of the status\nof the particular user or of the way in which the particular user\nactually uses, or expects or is expected to use, the product. A product\nis a consumer product regardless of whether the product has substantial\ncommercial, industrial or non-consumer uses, unless such uses represent\nthe only significant mode of use of the product.\n\n \"Installation Information\" for a User Product means any methods,\nprocedures, authorization keys, or other information required to install\nand execute modified versions of a covered work in that User Product from\na modified version of its Corresponding Source. The information must\nsuffice to ensure that the continued functioning of the modified object\ncode is in no case prevented or interfered with solely because\nmodification has been made.\n\n If you convey an object code work under this section in, or with, or\nspecifically for use in, a User Product, and the conveying occurs as\npart of a transaction in which the right of possession and use of the\nUser Product is transferred to the recipient in perpetuity or for a\nfixed term (regardless of how the transaction is characterized), the\nCorresponding Source conveyed under this section must be accompanied\nby the Installation Information. But this requirement does not apply\nif neither you nor any third party retains the ability to install\nmodified object code on the User Product (for example, the work has\nbeen installed in ROM).\n\n The requirement to provide Installation Information does not include a\nrequirement to continue to provide support service, warranty, or updates\nfor a work that has been modified or installed by the recipient, or for\nthe User Product in which it has been modified or installed. Access to a\nnetwork may be denied when the modification itself materially and\nadversely affects the operation of the network or violates the rules and\nprotocols for communication across the network.\n\n Corresponding Source conveyed, and Installation Information provided,\nin accord with this section must be in a format that is publicly\ndocumented (and with an implementation available to the public in\nsource code form), and must require no special password or key for\nunpacking, reading or copying.\n\n 7. Additional Terms.\n\n \"Additional permissions\" are terms that supplement the terms of this\nLicense by making exceptions from one or more of its conditions.\nAdditional permissions that are applicable to the entire Program shall\nbe treated as though they were included in this License, to the extent\nthat they are valid under applicable law. If additional permissions\napply only to part of the Program, that part may be used separately\nunder those permissions, but the entire Program remains governed by\nthis License without regard to the additional permissions.\n\n When you convey a copy of a covered work, you may at your option\nremove any additional permissions from that copy, or from any part of\nit. (Additional permissions may be written to require their own\nremoval in certain cases when you modify the work.) You may place\nadditional permissions on material, added by you to a covered work,\nfor which you have or can give appropriate copyright permission.\n\n Notwithstanding any other provision of this License, for material you\nadd to a covered work, you may (if authorized by the copyright holders of\nthat material) supplement the terms of this License with terms:\n\n a) Disclaiming warranty or limiting liability differently from the\n terms of sections 15 and 16 of this License; or\n\n b) Requiring preservation of specified reasonable legal notices or\n author attributions in that material or in the Appropriate Legal\n Notices displayed by works containing it; or\n\n c) Prohibiting misrepresentation of the origin of that material, or\n requiring that modified versions of such material be marked in\n reasonable ways as different from the original version; or\n\n d) Limiting the use for publicity purposes of names of licensors or\n authors of the material; or\n\n e) Declining to grant rights under trademark law for use of some\n trade names, trademarks, or service marks; or\n\n f) Requiring indemnification of licensors and authors of that\n material by anyone who conveys the material (or modified versions of\n it) with contractual assumptions of liability to the recipient, for\n any liability that these contractual assumptions directly impose on\n those licensors and authors.\n\n All other non-permissive additional terms are considered \"further\nrestrictions\" within the meaning of section 10. If the Program as you\nreceived it, or any part of it, contains a notice stating that it is\ngoverned by this License along with a term that is a further\nrestriction, you may remove that term. If a license document contains\na further restriction but permits relicensing or conveying under this\nLicense, you may add to a covered work material governed by the terms\nof that license document, provided that the further restriction does\nnot survive such relicensing or conveying.\n\n If you add terms to a covered work in accord with this section, you\nmust place, in the relevant source files, a statement of the\nadditional terms that apply to those files, or a notice indicating\nwhere to find the applicable terms.\n\n Additional terms, permissive or non-permissive, may be stated in the\nform of a separately written license, or stated as exceptions;\nthe above requirements apply either way.\n\n 8. Termination.\n\n You may not propagate or modify a covered work except as expressly\nprovided under this License. Any attempt otherwise to propagate or\nmodify it is void, and will automatically terminate your rights under\nthis License (including any patent licenses granted under the third\nparagraph of section 11).\n\n However, if you cease all violation of this License, then your\nlicense from a particular copyright holder is reinstated (a)\nprovisionally, unless and until the copyright holder explicitly and\nfinally terminates your license, and (b) permanently, if the copyright\nholder fails to notify you of the violation by some reasonable means\nprior to 60 days after the cessation.\n\n Moreover, your license from a particular copyright holder is\nreinstated permanently if the copyright holder notifies you of the\nviolation by some reasonable means, this is the first time you have\nreceived notice of violation of this License (for any work) from that\ncopyright holder, and you cure the violation prior to 30 days after\nyour receipt of the notice.\n\n Termination of your rights under this section does not terminate the\nlicenses of parties who have received copies or rights from you under\nthis License. If your rights have been terminated and not permanently\nreinstated, you do not qualify to receive new licenses for the same\nmaterial under section 10.\n\n 9. Acceptance Not Required for Having Copies.\n\n You are not required to accept this License in order to receive or\nrun a copy of the Program. Ancillary propagation of a covered work\noccurring solely as a consequence of using peer-to-peer transmission\nto receive a copy likewise does not require acceptance. However,\nnothing other than this License grants you permission to propagate or\nmodify any covered work. These actions infringe copyright if you do\nnot accept this License. Therefore, by modifying or propagating a\ncovered work, you indicate your acceptance of this License to do so.\n\n 10. Automatic Licensing of Downstream Recipients.\n\n Each time you convey a covered work, the recipient automatically\nreceives a license from the original licensors, to run, modify and\npropagate that work, subject to this License. You are not responsible\nfor enforcing compliance by third parties with this License.\n\n An \"entity transaction\" is a transaction transferring control of an\norganization, or substantially all assets of one, or subdividing an\norganization, or merging organizations. If propagation of a covered\nwork results from an entity transaction, each party to that\ntransaction who receives a copy of the work also receives whatever\nlicenses to the work the party's predecessor in interest had or could\ngive under the previous paragraph, plus a right to possession of the\nCorresponding Source of the work from the predecessor in interest, if\nthe predecessor has it or can get it with reasonable efforts.\n\n You may not impose any further restrictions on the exercise of the\nrights granted or affirmed under this License. For example, you may\nnot impose a license fee, royalty, or other charge for exercise of\nrights granted under this License, and you may not initiate litigation\n(including a cross-claim or counterclaim in a lawsuit) alleging that\nany patent claim is infringed by making, using, selling, offering for\nsale, or importing the Program or any portion of it.\n\n 11. Patents.\n\n A \"contributor\" is a copyright holder who authorizes use under this\nLicense of the Program or a work on which the Program is based. The\nwork thus licensed is called the contributor's \"contributor version\".\n\n A contributor's \"essential patent claims\" are all patent claims\nowned or controlled by the contributor, whether already acquired or\nhereafter acquired, that would be infringed by some manner, permitted\nby this License, of making, using, or selling its contributor version,\nbut do not include claims that would be infringed only as a\nconsequence of further modification of the contributor version. For\npurposes of this definition, \"control\" includes the right to grant\npatent sublicenses in a manner consistent with the requirements of\nthis License.\n\n Each contributor grants you a non-exclusive, worldwide, royalty-free\npatent license under the contributor's essential patent claims, to\nmake, use, sell, offer for sale, import and otherwise run, modify and\npropagate the contents of its contributor version.\n\n In the following three paragraphs, a \"patent license\" is any express\nagreement or commitment, however denominated, not to enforce a patent\n(such as an express permission to practice a patent or covenant not to\nsue for patent infringement). To \"grant\" such a patent license to a\nparty means to make such an agreement or commitment not to enforce a\npatent against the party.\n\n If you convey a covered work, knowingly relying on a patent license,\nand the Corresponding Source of the work is not available for anyone\nto copy, free of charge and under the terms of this License, through a\npublicly available network server or other readily accessible means,\nthen you must either (1) cause the Corresponding Source to be so\navailable, or (2) arrange to deprive yourself of the benefit of the\npatent license for this particular work, or (3) arrange, in a manner\nconsistent with the requirements of this License, to extend the patent\nlicense to downstream recipients. \"Knowingly relying\" means you have\nactual knowledge that, but for the patent license, your conveying the\ncovered work in a country, or your recipient's use of the covered work\nin a country, would infringe one or more identifiable patents in that\ncountry that you have reason to believe are valid.\n\n If, pursuant to or in connection with a single transaction or\narrangement, you convey, or propagate by procuring conveyance of, a\ncovered work, and grant a patent license to some of the parties\nreceiving the covered work authorizing them to use, propagate, modify\nor convey a specific copy of the covered work, then the patent license\nyou grant is automatically extended to all recipients of the covered\nwork and works based on it.\n\n A patent license is \"discriminatory\" if it does not include within\nthe scope of its coverage, prohibits the exercise of, or is\nconditioned on the non-exercise of one or more of the rights that are\nspecifically granted under this License. You may not convey a covered\nwork if you are a party to an arrangement with a third party that is\nin the business of distributing software, under which you make payment\nto the third party based on the extent of your activity of conveying\nthe work, and under which the third party grants, to any of the\nparties who would receive the covered work from you, a discriminatory\npatent license (a) in connection with copies of the covered work\nconveyed by you (or copies made from those copies), or (b) primarily\nfor and in connection with specific products or compilations that\ncontain the covered work, unless you entered into that arrangement,\nor that patent license was granted, prior to 28 March 2007.\n\n Nothing in this License shall be construed as excluding or limiting\nany implied license or other defenses to infringement that may\notherwise be available to you under applicable patent law.\n\n 12. No Surrender of Others' Freedom.\n\n If conditions are imposed on you (whether by court order, agreement or\notherwise) that contradict the conditions of this License, they do not\nexcuse you from the conditions of this License. If you cannot convey a\ncovered work so as to satisfy simultaneously your obligations under this\nLicense and any other pertinent obligations, then as a consequence you may\nnot convey it at all. For example, if you agree to terms that obligate you\nto collect a royalty for further conveying from those to whom you convey\nthe Program, the only way you could satisfy both those terms and this\nLicense would be to refrain entirely from conveying the Program.\n\n 13. Use with the GNU Affero General Public License.\n\n Notwithstanding any other provision of this License, you have\npermission to link or combine any covered work with a work licensed\nunder version 3 of the GNU Affero General Public License into a single\ncombined work, and to convey the resulting work. The terms of this\nLicense will continue to apply to the part which is the covered work,\nbut the special requirements of the GNU Affero General Public License,\nsection 13, concerning interaction through a network will apply to the\ncombination as such.\n\n 14. Revised Versions of this License.\n\n The Free Software Foundation may publish revised and/or new versions of\nthe GNU General Public License from time to time. Such new versions will\nbe similar in spirit to the present version, but may differ in detail to\naddress new problems or concerns.\n\n Each version is given a distinguishing version number. If the\nProgram specifies that a certain numbered version of the GNU General\nPublic License \"or any later version\" applies to it, you have the\noption of following the terms and conditions either of that numbered\nversion or of any later version published by the Free Software\nFoundation. If the Program does not specify a version number of the\nGNU General Public License, you may choose any version ever published\nby the Free Software Foundation.\n\n If the Program specifies that a proxy can decide which future\nversions of the GNU General Public License can be used, that proxy's\npublic statement of acceptance of a version permanently authorizes you\nto choose that version for the Program.\n\n Later license versions may give you additional or different\npermissions. However, no additional obligations are imposed on any\nauthor or copyright holder as a result of your choosing to follow a\nlater version.\n\n 15. Disclaimer of Warranty.\n\n THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY\nAPPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT\nHOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM \"AS IS\" WITHOUT WARRANTY\nOF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM\nIS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF\nALL NECESSARY SERVICING, REPAIR OR CORRECTION.\n\n 16. Limitation of Liability.\n\n IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING\nWILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS\nTHE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY\nGENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE\nUSE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF\nDATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD\nPARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),\nEVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF\nSUCH DAMAGES.\n\n 17. Interpretation of Sections 15 and 16.\n\n If the disclaimer of warranty and limitation of liability provided\nabove cannot be given local legal effect according to their terms,\nreviewing courts shall apply local law that most closely approximates\nan absolute waiver of all civil liability in connection with the\nProgram, unless a warranty or assumption of liability accompanies a\ncopy of the Program in return for a fee.\n\n END OF TERMS AND CONDITIONS\n\n How to Apply These Terms to Your New Programs\n\n If you develop a new program, and you want it to be of the greatest\npossible use to the public, the best way to achieve this is to make it\nfree software which everyone can redistribute and change under these terms.\n\n To do so, attach the following notices to the program. It is safest\nto attach them to the start of each source file to most effectively\nstate the exclusion of warranty; and each file should have at least\nthe \"copyright\" line and a pointer to where the full notice is found.\n\n \n Copyright (C) \n\n This program is free software: you can redistribute it and/or modify\n it under the terms of the GNU General Public License as published by\n the Free Software Foundation, either version 3 of the License, or\n (at your option) any later version.\n\n This program is distributed in the hope that it will be useful,\n but WITHOUT ANY WARRANTY; without even the implied warranty of\n MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n GNU General Public License for more details.\n\n You should have received a copy of the GNU General Public License\n along with this program. If not, see .\n\nAlso add information on how to contact you by electronic and paper mail.\n\n If the program does terminal interaction, make it output a short\nnotice like this when it starts in an interactive mode:\n\n Copyright (C) \n This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.\n This is free software, and you are welcome to redistribute it\n under certain conditions; type `show c' for details.\n\nThe hypothetical commands `show w' and `show c' should show the appropriate\nparts of the General Public License. Of course, your program's commands\nmight be different; for a GUI interface, you would use an \"about box\".\n\n You should also get your employer (if you work as a programmer) or school,\nif any, to sign a \"copyright disclaimer\" for the program, if necessary.\nFor more information on this, and how to apply and follow the GNU GPL, see\n.\n\n The GNU General Public License does not permit incorporating your program\ninto proprietary programs. If your program is a subroutine library, you\nmay consider it more useful to permit linking proprietary applications with\nthe library. If this is what you want to do, use the GNU Lesser General\nPublic License instead of this License. But first, please read\n.\n" }, { "path": "PRIVACY.md", "content": "# Privacy Policy\n\n## What information do we collect?\n\nTo properly integrate with GitHub, `gitsome` must authenticate with GitHub and/or GitHub Enterprise.\n\nWhen using `gitsome`, you may be asked to enter your: user name, password, two factor authentication token, user access token, news feed url, or GitHub Enterprise url. See the [configure](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-configure) section for more details.\n\nYou may, however, use features of `gitsome` that do not require GitHub authentication anonymously.\n\n## What do we use your information for?\n\nAny of the information we collect from you is used solely to interact with GitHub and/or GitHub Enterprise. Other than to provide integration with GitHub, we do not disclose or transfer your information to outside parties.\n\n## How do we protect your information?\n\nAll supplied sensitive information is encrypted during transmission to GitHub via Secure Socket Layer (SSL) technology.\n\n## Terms and Conditions\n\nPlease also visit our Terms and Conditions section establishing the use, disclaimers, and limitations of liability governing the use of `gitsome`.\n\n## Your Consent\n\nBy using `gitsome`, you consent to our privacy policy.\n\n## Changes to our Privacy Policy\n\nIf we decide to change our privacy policy, we will post those changes on this page.\n\n## Contacting Us\n\nAny questions about this document should be addressed to [Donne Martin](donne.martin@gmail.com).\n" }, { "path": "README.md", "content": "

\n \n

\n

\n An Official Integration for GitHub and GitHub Enterprise.\n

\n\ngitsome\n=======\n\n[![Build Status](https://travis-ci.org/donnemartin/gitsome.svg?branch=master)](https://travis-ci.org/donnemartin/gitsome) [![PyPI version](https://badge.fury.io/py/gitsome.svg)](http://badge.fury.io/py/gitsome) [![PyPI](https://img.shields.io/pypi/pyversions/gitsome.svg)](https://pypi.python.org/pypi/gitsome/) [![License](https://img.shields.io/:license-apache-blue.svg)](http://www.apache.org/licenses/LICENSE-2.0.html)\n\n## Why `gitsome`?\n\n### The Git Command Line\n\nAlthough the standard Git command line is a great tool to manage your Git-powered repos, it can be **tough to remember the usage** of:\n\n* 150+ porcelain and plumbing commands\n* Countless command-specific options\n* Resources such as tags and branches\n\nThe Git command line **does not integrate with GitHub**, forcing you to toggle between command line and browser.\n\n## `gitsome` - A Supercharged Git/GitHub CLI With Autocomplete\n\n

\n \n

\n\n`gitsome` aims to supercharge your standard git/shell interface by focusing on:\n\n* **Improving ease-of-use**\n* **Increasing productivity**\n\n### Deep GitHub Integration\n\nNot all GitHub workflows work well in a terminal; `gitsome` attempts to target those that do.\n\n`gitsome` includes 29 GitHub integrated commands that work with **[ALL](#enabling-gh-tab-completions-outside-of-gitsome)** shells:\n\n $ gh [param] [options]\n\n* [Quick reference](#github-integration-commands-quick-reference)\n* [General reference](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md)\n\nRun `gh` commands along with [Git-Extras](https://github.com/tj/git-extras/blob/master/Commands.md) and [hub](https://hub.github.com/) commands to unlock even more GitHub integrations!\n\n![Imgur](http://i.imgur.com/sG09AJH.png)\n\n### Git and GitHub Autocompleter With Interactive Help\n\nYou can run the **optional** shell:\n\n $ gitsome\n\nto enable **autocompletion** and **interactive help** for the following:\n\n* Git commands\n* Git options\n* Git branches, tags, etc\n* [Git-Extras commands](https://github.com/tj/git-extras/blob/master/Commands.md)\n* [GitHub integration commands](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md)\n\n![Imgur](http://i.imgur.com/08OMNjz.png)\n\n![Imgur](http://i.imgur.com/fHjMwlh.png)\n\n### General Autocompleter\n\n`gitsome` autocompletes the following:\n\n* Shell commands\n* Files and directories\n* Environment variables\n* Man pages\n* Python\n\nTo enable additional autocompletions, check out the [Enabling Bash Completions](#enabling-bash-completions) section.\n\n![Imgur](http://i.imgur.com/hg1dpk6.png)\n\n## Fish-Style Auto-Suggestions\n\n`gitsome` supports Fish-style auto-suggestions. Use the `right arrow` key to complete a suggestion.\n\n![Imgur](http://i.imgur.com/ZRaFGpY.png)\n\n## Python REPL\n\n`gitsome` is powered by [`xonsh`](https://github.com/scopatz/xonsh), which supports a Python REPL.\n\nRun Python commands alongside shell commands:\n\n![Imgur](http://i.imgur.com/NYk7WYO.png)\n\nAdditional `xonsh` features can be found in the [`xonsh tutorial`](http://xon.sh/tutorial.html).\n\n## Command History\n\n`gitsome` keeps track of commands you enter and stores them in `~/.xonsh_history.json`. Use the up and down arrow keys to cycle through the command history.\n\n![Imgur](http://i.imgur.com/wq0caZu.png)\n\n## Customizable Highlighting\n\nYou can control the ansi colors used for highlighting by updating your `~/.gitsomeconfig` file.\n\nColor options include:\n\n```\n'black', 'red', 'green', 'yellow',\n'blue', 'magenta', 'cyan', 'white'\n```\n\nFor no color, set the value(s) to `None`. `white` can appear as light gray on some terminals.\n\n![Imgur](http://i.imgur.com/BN1lfEf.png)\n\n## Available Platforms\n\n`gitsome` is available for Mac, Linux, Unix, [Windows](#windows-support), and [Docker](#running-as-docker-container).\n\n## TODO\n\n>Not all GitHub workflows work well in a terminal; `gitsome` attempts to target those that do.\n\n* Add additional GitHub API integrations\n\n`gitsome` is just getting started. Feel free to [contribute!](#contributing)\n\n## Index\n\n### GitHub Integration Commands\n\n* [GitHub Integration Commands Syntax](#github-integration-commands-syntax)\n* [GitHub Integration Commands Listing](#github-integration-commands-listing)\n* [GitHub Integration Commands Quick Reference](#github-integration-commands-quick-reference)\n* [GitHub Integration Commands Reference in COMMANDS.md](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md)\n * [`gh configure`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-configure)\n * [`gh create-comment`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-create-comment)\n * [`gh create-issue`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-create-issue)\n * [`gh create-repo`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-create-repo)\n * [`gh emails`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-emails)\n * [`gh emojis`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-emojis)\n * [`gh feed`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-feed)\n * [`gh followers`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-followers)\n * [`gh following`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-following)\n * [`gh gitignore-template`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-gitignore-template)\n * [`gh gitignore-templates`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-gitignore-templates)\n * [`gh issue`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-issue)\n * [`gh issues`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-issues)\n * [`gh license`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-license)\n * [`gh licenses`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-licenses)\n * [`gh me`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-me)\n * [`gh notifications`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-notifications)\n * [`gh octo`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-octo)\n * [`gh pull-request`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-pull-request)\n * [`gh pull-requests`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-pull-requests)\n * [`gh rate-limit`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-rate-limit)\n * [`gh repo`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-repo)\n * [`gh repos`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-repos)\n * [`gh search-issues`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-search-issues)\n * [`gh search-repos`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-search-repos)\n * [`gh starred`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-starred)\n * [`gh trending`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-trending)\n * [`gh user`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-user)\n * [`gh view`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-view)\n* [Option: View in a Pager](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#option-view-in-a-pager)\n* [Option: View in a Browser](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#option-view-in-a-browser)\n\n### Installation and Tests\n\n* [Installation](#installation)\n * [Pip Installation](#pip-installation)\n * [Virtual Environment Installation](#virtual-environment-installation)\n * [Running as a Docker Container](#running-as-a-docker-container)\n * [Running the `gh configure` Command](#running-the-gh-configure-command)\n * [For GitHub Enterprise Users](#for-github-enterprise-users)\n * [Enabling Bash Completions](#enabling-bash-completions)\n * [Enabling `gh` Tab Completions Outside of `gitsome`](#enabling-gh-tab-completions-outside-of-gitsome)\n * [For Zsh Users](#for-zsh-users)\n * [Optional: Installing `PIL` or `Pillow`](#optional-installing-pil-or-pillow)\n * [Supported Python Versions](#supported-python-versions)\n * [Supported Platforms](#supported-platforms)\n * [Windows Support](#windows-support)\n* [Developer Installation](#developer-installation)\n * [Continuous Integration](#continuous-integration)\n * [Unit Tests and Code Coverage](#unit-tests-and-code-coverage)\n * [Documentation](#documentation)\n\n### Misc\n\n* [Contributing](#contributing)\n* [Credits](#credits)\n* [Contact Info](#contact-info)\n* [License](#license)\n\n## GitHub Integration Commands Syntax\n\nUsage:\n\n $ gh [param] [options]\n\n## GitHub Integration Commands Listing\n\n```\n configure Configure gitsome.\n create-comment Create a comment on the given issue.\n create-issue Create an issue.\n create-repo Create a repo.\n emails List all the user's registered emails.\n emojis List all GitHub supported emojis.\n feed List all activity for the given user or repo.\n followers List all followers and the total follower count.\n following List all followed users and the total followed count.\n gitignore-template Output the gitignore template for the given language.\n gitignore-templates Output all supported gitignore templates.\n issue Output detailed information about the given issue.\n issues List all issues matching the filter.\n license Output the license template for the given license.\n licenses Output all supported license templates.\n me List information about the logged in user.\n notifications List all notifications.\n octo Output an Easter egg or the given message from Octocat.\n pull-request Output detailed information about the given pull request.\n pull-requests List all pull requests.\n rate-limit Output the rate limit. Not available for Enterprise.\n repo Output detailed information about the given filter.\n repos List all repos matching the given filter.\n search-issues Search for all issues matching the given query.\n search-repos Search for all repos matching the given query.\n starred Output starred repos.\n trending List trending repos for the given language.\n user List information about the given user.\n view View the given index in the terminal or a browser.\n```\n\n## GitHub Integration Commands Reference: COMMANDS.md\n\nSee the [GitHub Integration Commands Reference in COMMANDS.md](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md) for a **detailed discussion** of all GitHub integration commands, parameters, options, and examples.\n\nCheck out the next section for a **quick reference**.\n\n## GitHub Integration Commands Quick Reference\n\n### Configuring `gitsome`\n\nTo properly integrate with GitHub, you must first configure `gitsome`:\n\n $ gh configure\n\nFor GitHub Enterprise users, run with the `-e/--enterprise` flag:\n\n $ gh configure -e\n\n### Listing Feeds\n\n#### Listing Your News Feed\n\n $ gh feed\n\n![Imgur](http://i.imgur.com/2LWcyS6.png)\n\n#### Listing A User's Activity Feed\n\nView your activity feed or another user's activity feed, optionally through a pager with `-p/--pager`. The [pager option](#option-view-in-a-pager) is available for many commands.\n\n $ gh feed donnemartin -p\n\n![Imgur](http://i.imgur.com/kryGLXz.png)\n\n#### Listing A Repo's Activity Feed\n\n $ gh feed donnemartin/gitsome -p\n\n![Imgur](http://i.imgur.com/d2kxDg9.png)\n\n### Listing Notifications\n\n $ gh notifications\n\n![Imgur](http://i.imgur.com/uwmwxsW.png)\n\n### Listing Pull Requests\n\nView all pull requests for your repos:\n\n $ gh pull-requests\n\n![Imgur](http://i.imgur.com/4A2eYM9.png)\n\n### Filtering Issues\n\nView all open issues where you have been mentioned:\n\n $ gh issues --issue_state open --issue_filter mentioned\n\n![Imgur](http://i.imgur.com/AB5zxxo.png)\n\nView all issues, filtering for only those assigned to you, regardless of state (open, closed):\n\n $ gh issues --issue_state all --issue_filter assigned\n\nFor more information about the filter and state qualifiers, visit the [`gh issues`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-issues) reference in [COMMANDS.md](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md).\n\n### Filtering Starred Repos\n\n $ gh starred \"repo filter\"\n\n![Imgur](http://i.imgur.com/JB88Kw8.png)\n\n### Searching Issues and Repos\n\n#### Searching Issues\n\nSearch issues that have the most +1s:\n\n $ gh search-issues \"is:open is:issue sort:reactions-+1-desc\" -p\n\n![Imgur](http://i.imgur.com/DXXxkBD.png)\n\nSearch issues that have the most comments:\n\n $ gh search-issues \"is:open is:issue sort:comments-desc\" -p\n\nSearch issues with the \"help wanted\" tag:\n\n $ gh search-issues \"is:open is:issue label:\\\"help wanted\\\"\" -p\n\nSearch issues that have your user name tagged **@donnemartin**:\n\n $ gh search-issues \"is:issue donnemartin is:open\" -p\n\nSearch all your open private issues:\n\n $ gh search-issues \"is:open is:issue is:private\" -p\n\nFor more information about the query qualifiers, visit the [searching issues reference](https://help.github.com/articles/searching-issues/).\n\n#### Searching Repos\n\nSearch all Python repos created on or after 2015, with >= 1000 stars:\n\n $ gh search-repos \"created:>=2015-01-01 stars:>=1000 language:python\" --sort stars -p\n\n![Imgur](http://i.imgur.com/kazXWWY.png)\n\nFor more information about the query qualifiers, visit the [searching repos reference](https://help.github.com/articles/searching-repositories/).\n\n### Listing Trending Repos and Devs\n\nView trending repos:\n\n $ gh trending [language] [-w/--weekly] [-m/--monthly] [-d/--devs] [-b/--browser]\n\n![Imgur](http://i.imgur.com/aa1gOg7.png)\n\nView trending devs (devs are currently only supported in browser):\n\n $ gh trending [language] --devs --browser\n\n### Viewing Content\n\n#### The `view` command\n\nView the previously listed notifications, pull requests, issues, repos, users etc, with HTML nicely formatted for your terminal, or optionally in your browser:\n\n $ gh view [#] [-b/--browser]\n\n![Imgur](http://i.imgur.com/NVEwGbV.png)\n\n#### The `issue` command\n\nView an issue:\n\n $ gh issue donnemartin/saws/1\n\n![Imgur](http://i.imgur.com/ZFv9MuV.png)\n\n#### The `pull-request` command\n\nView a pull request:\n\n $ gh pull-request donnemartin/awesome-aws/2\n\n![Imgur](http://i.imgur.com/3MtKjKy.png)\n\n### Setting Up `.gitignore`\n\nList all available `.gitignore` templates:\n\n $ gh gitignore-templates\n\n![Imgur](http://i.imgur.com/u8qYx1s.png)\n\nSet up your `.gitignore`:\n\n $ gh gitignore-template Python > .gitignore\n\n![Imgur](http://i.imgur.com/S5m5ZcO.png)\n\n### Setting Up `LICENSE`\n\nList all available `LICENSE` templates:\n\n $ gh licenses\n\n![Imgur](http://i.imgur.com/S9SbMLJ.png)\n\nSet up your or `LICENSE`:\n\n $ gh license MIT > LICENSE\n\n![Imgur](http://i.imgur.com/zJHVxaA.png)\n\n### Summoning Octocat\n\nCall on Octocat to say the given message or an Easter egg:\n\n $ gh octo [say]\n\n![Imgur](http://i.imgur.com/bNzCa5p.png)\n\n### Viewing Profiles\n\n#### Viewing A User's Profile\n\n $ gh user octocat\n\n![Imgur](http://i.imgur.com/xVoVPVe.png)\n\n#### Viewing Your Profile\n\nView your profile with the `gh user [YOUR_USER_ID]` command or with the following shortcut:\n\n $ gh me\n\n![Imgur](http://i.imgur.com/csk5j0S.png)\n\n### Creating Comments, Issues, and Repos\n\nCreate a comment:\n\n $ gh create-comment donnemartin/gitsome/1 -t \"hello world\"\n\nCreate an issue:\n\n $ gh create-issue donnemartin/gitsome -t \"title\" -b \"body\"\n\nCreate a repo:\n\n $ gh create-repo gitsome\n\n### Option: View in a Pager\n\nMany `gh` commands support a `-p/--pager` option that displays results in a pager, where available.\n\nUsage:\n\n $ gh [param] [options] -p\n $ gh [param] [options] --pager\n\n### Option: View in a Browser\n\nMany `gh` commands support a `-b/--browser` option that displays results in your default browser instead of your terminal.\n\nUsage:\n\n $ gh [param] [options] -b\n $ gh [param] [options] --browser\n\nSee the [COMMANDS.md](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md) for a detailed listing of all GitHub integration commands, parameters, options, and examples.\n\nHaving trouble remembering these commands? Check out the handy [autocompleter with interactive help](#git-and-github-autocompleter-with-interactive-help) to guide you through each command.\n\n*Note, you can combine `gitsome` with other utilities such as [Git-Extras](https://github.com/tj/git-extras/blob/master/Commands.md).*\n\n## Installation\n\n### Pip Installation\n\n[![PyPI version](https://badge.fury.io/py/gitsome.svg)](http://badge.fury.io/py/gitsome) [![PyPI](https://img.shields.io/pypi/pyversions/gitsome.svg)](https://pypi.python.org/pypi/gitsome/)\n\n`gitsome` is hosted on [PyPI](https://pypi.python.org/pypi/gitsome). The following command will install `gitsome`:\n\n $ pip3 install gitsome\n\nYou can also install the latest `gitsome` from GitHub source which can contain changes not yet pushed to PyPI:\n\n $ pip3 install git+https://github.com/donnemartin/gitsome.git\n\nIf you are not installing in a `virtualenv`, you might need to run with `sudo`:\n\n $ sudo pip3 install gitsome\n\n#### `pip3`\n\nDepending on your setup, you might also want to run `pip3` with the [`-H flag`](http://stackoverflow.com/a/28619739):\n\n $ sudo -H pip3 install gitsome\n\nFor most linux users, `pip3` can be installed on your system using the `python3-pip` package.\n\nFor example, Ubuntu users can run:\n\n $ sudo apt-get install python3-pip\n\nSee this [ticket](https://github.com/donnemartin/gitsome/issues/4) for more details.\n\n### Virtual Environment Installation\n\nYou can install Python packages in a [`virtualenv`](http://docs.python-guide.org/en/latest/dev/virtualenvs/) to avoid potential issues with dependencies or permissions.\n\nIf you are a Windows user or if you would like more details on `virtualenv`, check out this [guide](http://docs.python-guide.org/en/latest/dev/virtualenvs/).\n\nInstall `virtualenv` and `virtualenvwrapper`:\n\n $ pip3 install virtualenv\n $ pip3 install virtualenvwrapper\n $ export WORKON_HOME=~/.virtualenvs\n $ source /usr/local/bin/virtualenvwrapper.sh\n\nCreate a `gitsome` `virtualenv` and install `gitsome`:\n\n $ mkvirtualenv gitsome\n $ pip3 install gitsome\n\nIf the `pip` install does not work, you might be running Python 2 by default. Check what version of Python you are running:\n\n $ python --version\n\nIf the call above results in Python 2, find the path for Python 3:\n\n $ which python3 # Python 3 path for mkvirtualenv's --python option\n\nInstall Python 3 if needed. Set the Python version when calling `mkvirtualenv`:\n\n $ mkvirtualenv --python [Python 3 path from above] gitsome\n $ pip3 install gitsome\n\nIf you want to activate the `gitsome` `virtualenv` again later, run:\n\n $ workon gitsome\n\nTo deactivate the `gitsome` `virtualenv`, run:\n\n $ deactivate\n\n### Running as a Docker Container\n\nYou can run gitsome in a Docker container to avoid installing Python and `pip3` locally. To install Docker check out the [official Docker documentation](https://docs.docker.com/engine/getstarted/step_one/#step-1-get-docker).\n\nOnce you have docker installed you can run gitsome:\n\n $ docker run -ti --rm mariolet/gitsome\n\nYou can use Docker volumes to let gitsome access your working directory, your local .gitsomeconfig and .gitconfig:\n\n $ docker run -ti --rm -v $(pwd):/src/ \\\n -v ${HOME}/.gitsomeconfig:/root/.gitsomeconfig \\\n -v ${HOME}/.gitconfig:/root/.gitconfig \\\n mariolet/gitsome\n\nIf you are running this command often you will probably want to define an alias:\n\n $ alias gitsome=\"docker run -ti --rm -v $(pwd):/src/ \\\n -v ${HOME}/.gitsomeconfig:/root/.gitsomeconfig \\\n -v ${HOME}/.gitconfig:/root/.gitconfig \\\n mariolet/gitsome\"\n\nTo build the Docker image from sources:\n\n $ git clone https://github.com/donnemartin/gitsome.git\n $ cd gitsome\n $ docker build -t gitsome .\n\n### Starting the `gitsome` Shell\n\nOnce installed, run the optional `gitsome` autocompleter with interactive help:\n\n $ gitsome\n\nRunning the optional `gitsome` shell will provide you with autocompletion, interactive help, fish-style suggestions, a Python REPL, etc.\n\n### Running `gh` Commands\n\nRun GitHub-integrated commands:\n\n $ gh [param] [options]\n\nNote: Running the `gitsome` shell is not required to execute `gh` commands. After [installing](#installation) `gitsome` you can run `gh` commands from any shell.\n\n### Running the `gh configure` Command\n\nTo properly integrate with GitHub, `gitsome` must be properly configured:\n\n $ gh configure\n\n#### For GitHub Enterprise Users\n\nRun with the `-e/--enterprise` flag:\n\n $ gh configure -e\n\nView more details in the [gh configure](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-configure) section.\n\n### Enabling Bash Completions\n\nBy default, `gitsome` looks at the following [locations to enable bash completions](https://github.com/donnemartin/gitsome/blob/master/xonsh/environ.py#L123-L131).\n\nTo add additional bash completions, update the `~/.xonshrc` file with the location of your bash completions.\n\nIf `~/.xonshrc` does not exist, create it:\n\n $ touch ~/.xonshrc\n\nFor example, if additional completions are found in `/usr/local/etc/my_bash_completion.d/completion.bash`, add the following line in `~/.xonshrc`:\n\n```\n$BASH_COMPLETIONS.append('/usr/local/etc/my_bash_completion.d/completion.bash')\n```\n\nYou will need to restart `gitsome` for the changes to take effect.\n\n### Enabling `gh` Tab Completions Outside of `gitsome`\n\nYou can run `gh` commands outside of the `gitsome` shell completer. To enable `gh` tab completions for this workflow, copy the [`gh_complete.sh`](https://github.com/donnemartin/gitsome/blob/master/scripts/gh_complete.sh) file locally.\n\nLet bash know completion is available for the `gh` command within your current session:\n\n $ source /path/to/gh_complete.sh\n\nTo enable tab completion for all terminal sessions, add the following to your `bashrc` file:\n\n source /path/to/gh_complete.sh\n\nReload your `bashrc`:\n\n $ source ~/.bashrc\n\nTip: `.` is the short form of `source`, so you can run this instead:\n\n $ . ~/.bashrc\n\n#### For Zsh Users\n\n`zsh` includes a module which is compatible with bash completions.\n\nDownload the [`gh_complete.sh`](https://github.com/donnemartin/gitsome/blob/master/scripts/gh_complete.sh) file as above and append the following to your `.zshrc`:\n\n autoload bashcompinit\n bashcompinit\n source /path/to/gh_complete.sh\n\nReload your `zshrc`:\n\n $ source ~/.zshrc\n\n### Optional: Installing `PIL` or `Pillow`\n\nDisplaying the avatar for the `gh me` and `gh user` commands will require installing the optional `PIL` or `Pillow` dependency.\n\nWindows* and Mac:\n\n $ pip3 install Pillow\n\n*See the [Windows Support](#windows-support) section for limitations on the avatar.\n\nUbuntu users, check out these [instructions on askubuntu](http://askubuntu.com/a/272095)\n\n### Supported Python Versions\n\n* Python 3.4\n* Python 3.5\n* Python 3.6\n* Python 3.7\n\n`gitsome` is powered by `xonsh` which does not currently support Python 2.x, as discussed in this [ticket](https://github.com/scopatz/xonsh/issues/66).\n\n### Supported Platforms\n\n* Mac OS X\n * Tested on OS X 10.10\n* Linux, Unix\n * Tested on Ubuntu 14.04 LTS\n* Windows\n * Tested on Windows 10\n\n### Windows Support\n\n`gitsome` has been tested on Windows 10 with `cmd` and `cmder`.\n\nAlthough you can use the standard Windows command prompt, you'll probably have a better experience with either [cmder](https://github.com/cmderdev/cmder) or [conemu](https://github.com/Maximus5/ConEmu).\n\n![Imgur](http://i.imgur.com/A1VCsjV.png)\n\n#### Text Only Avatar\n\nThe commands [`gh user`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-user) and [`gh me`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-me) will always have the `-t/--text_avatar` flag enabled, since [`img2txt`](#credits) does not support the ansi avatar on Windows.\n\n#### Config File\n\nOn Windows, the `.gitsomeconfig ` file can be found in `%userprofile%`. For example:\n\n C:\\Users\\dmartin\\.gitsomeconfig\n\n## Developer Installation\n\nIf you're interested in contributing to `gitsome`, run the following commands:\n\n $ git clone https://github.com/donnemartin/gitsome.git\n $ cd gitsome\n $ pip3 install -e .\n $ pip3 install -r requirements-dev.txt\n $ gitsome\n $ gh [param] [options]\n\n#### `pip3`\n\nIf you get an error while installing saying that you need Python 3.4+, it could be because your `pip` command is configured for an older version of Python. To fix this issue, it is recommended to install `pip3`:\n\n $ sudo apt-get install python3-pip\n\nSee this [ticket](https://github.com/donnemartin/gitsome/issues/4) for more details.\n\n### Continuous Integration\n\n[![Build Status](https://travis-ci.org/donnemartin/gitsome.svg?branch=master)](https://travis-ci.org/donnemartin/gitsome)\n\nContinuous integration details are available on [Travis CI](https://travis-ci.org/donnemartin/gitsome).\n\n### Unit Tests and Code Coverage\n\nRun unit tests in your active Python environment:\n\n $ python tests/run_tests.py\n\nRun unit tests with [tox](https://pypi.python.org/pypi/tox) on multiple Python environments:\n\n $ tox\n\n### Documentation\n\nSource code documentation will soon be available on [Readthedocs.org](https://readthedocs.org/). Check out the [source docstrings](https://github.com/donnemartin/gitsome/blob/master/gitsome/githubcli.py).\n\nRun the following to build the docs:\n\n $ scripts/update_docs.sh\n\n## Contributing\n\nContributions are welcome!\n\nReview the [Contributing Guidelines](https://github.com/donnemartin/gitsome/blob/master/CONTRIBUTING.md) for details on how to:\n\n* Submit issues\n* Submit pull requests\n\n## Credits\n\n* [click](https://github.com/pallets/click) by [mitsuhiko](https://github.com/mitsuhiko)\n* [github_trends_rss](https://github.com/ryotarai/github_trends_rss) by [ryotarai](https://github.com/ryotarai)\n* [github3.py](https://github.com/sigmavirus24/github3.py) by [sigmavirus24](https://github.com/sigmavirus24)\n* [html2text](https://github.com/aaronsw/html2text) by [aaronsw](https://github.com/aaronsw)\n* [img2txt](https://github.com/hit9/img2txt) by [hit9](https://github.com/hit9)\n* [python-prompt-toolkit](https://github.com/jonathanslenders/python-prompt-toolkit) by [jonathanslenders](https://github.com/jonathanslenders)\n* [requests](https://github.com/kennethreitz/requests) by [kennethreitz](https://github.com/kennethreitz)\n* [xonsh](https://github.com/scopatz/xonsh) by [scopatz](https://github.com/scopatz)\n\n## Contact Info\n\nFeel free to contact me to discuss any issues, questions, or comments.\n\nMy contact info can be found on my [GitHub page](https://github.com/donnemartin).\n\n## License\n\n*I am providing code and resources in this repository to you under an open source license. Because this is my personal repository, the license you receive to my code and resources is from me and not my employer (Facebook).*\n\n[![License](http://img.shields.io/:license-apache-blue.svg)](http://www.apache.org/licenses/LICENSE-2.0.html)\n\n Copyright 2016 Donne Martin\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" }, { "path": "TOS.md", "content": "# Terms of Service\n\nBy using `gitsome`, you are agreeing to be bound by the following terms and conditions (\"Terms of Service\"). If you are entering into this agreement on behalf of a company or other legal entity, you represent that you have the authority to bind such entity, its affiliates and all users who access `gitsome` through your account to these terms and conditions, in which case the terms \"you\" or \"your\" shall refer to such entity, its affiliates and users associated with it. If you do not have such authority, or if you do not agree with these terms and conditions, you must not accept this agreement and may not use `gitsome`.\n\n`gitsome` is maintained by [Donne Martin](donne.martin@gmail.com).\n\nDonne Martin reserves the right to update and change the Terms of Service from time to time without notice. Any new features that augment or enhance `gitsome`, including the release of new tools and resources, shall be subject to the Terms of Service. Continued use of `gitsome` after any such changes shall constitute your consent to such changes. \n\n## Account Terms\n\n* You are responsible for maintaining the security of your GitHub account.\n* You are responsible for all activity that occurs by using `gitsome`.\n* You may not use the `gitsome` for any illegal or unauthorized purpose. \n* You must not, in the use of `gitsome`, violate any laws in your jurisdiction.\n* Your use of `gitsome` is at your sole risk.\n\n## Copyright\n\n`gitsome` is copyright ©2016 Donne Martin.\n\n## Questions\n\nAny questions about this document should be addressed to [Donne Martin](donne.martin@gmail.com).\n" }, { "path": "appveyor.yml", "content": "# What Python version is installed where:\n# http://www.appveyor.com/docs/installed-software#python\n\nenvironment:\n matrix:\n - PYTHON: \"C:\\\\Python27\"\n TOX_ENV: \"py27\"\n\n - PYTHON: \"C:\\\\Python33\"\n TOX_ENV: \"py33\"\n\n - PYTHON: \"C:\\\\Python34\"\n TOX_ENV: \"py34\"\n\n - PYTHON: \"C:\\\\Python35\"\n TOX_ENV: \"py35\"\n\n\ninit:\n - \"%PYTHON%/python -V\"\n - \"%PYTHON%/python -c \\\"import struct;print( 8 * struct.calcsize(\\'P\\'))\\\"\"\n\ninstall:\n - \"%PYTHON%/Scripts/easy_install -U pip\"\n - \"%PYTHON%/Scripts/pip install tox\"\n - \"%PYTHON%/Scripts/pip install wheel\"\n\nbuild: false # Not a C# project, build stuff at the test step instead.\n\ntest_script:\n - \"%PYTHON%/Scripts/tox -e %TOX_ENV%\"\n\nafter_test:\n - \"%PYTHON%/python setup.py bdist_wheel\"\n - ps: \"ls dist\"\n\nartifacts:\n - path: dist\\*\n\n#on_success:\n# - TODO: upload the content of dist/*.whl to a public wheelhouse" }, { "path": "codecov.yml", "content": "comment:\n layout: header, changes, diff\n\ncoverage:\n ignore:\n - gitsome/lib/*\n - gitsome/compat.py\n" }, { "path": "gitsome/__init__.py", "content": "# -*- coding: utf-8 -*-\n\n# Copyright 2015 Donne Martin. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"). You\n# may not use this file except in compliance with the License. A copy of\n# the License is located at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# or in the \"license\" file accompanying this file. This file is\n# distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF\n# ANY KIND, either express or implied. See the License for the specific\n# language governing permissions and limitations under the License.\n\n__version__ = '0.8.4'\n" }, { "path": "gitsome/compat.py", "content": "# -*- coding: utf-8 -*-\n\n# Copyright 2015 Donne Martin. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"). You\n# may not use this file except in compliance with the License. A copy of\n# the License is located at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# or in the \"license\" file accompanying this file. This file is\n# distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF\n# ANY KIND, either express or implied. See the License for the specific\n# language governing permissions and limitations under the License.\n\nimport sys\nimport urllib\ntry:\n # Python 3\n import configparser\n from urllib.parse import urlparse\n from urllib.request import urlretrieve\n from urllib.error import URLError\nexcept ImportError:\n # Python 2\n import ConfigParser as configparser\n from urlparse import urlparse\n from urllib import urlretrieve\n from urllib2 import URLError\nif sys.version_info < (3, 3):\n import HTMLParser\nelse:\n import html as HTMLParser\n" }, { "path": "gitsome/completer.py", "content": "# -*- coding: utf-8 -*-\n\n# Copyright 2015 Donne Martin. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"). You\n# may not use this file except in compliance with the License. A copy of\n# the License is located at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# or in the \"license\" file accompanying this file. This file is\n# distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF\n# ANY KIND, either express or implied. See the License for the specific\n# language governing permissions and limitations under the License.\n\nfrom __future__ import unicode_literals\nfrom __future__ import print_function\n\nfrom prompt_toolkit.completion import Completer, Completion\n\nfrom .completions import COMPLETIONS_GH, SUBCOMMANDS\nfrom .completions_git import META_LOOKUP_GIT, META_LOOKUP_GIT_EXTRAS\nfrom .utils import TextUtils\n\n\nclass CompleterGitsome(Completer):\n \"\"\"Gitsome prompt toolkit completer.\n\n :type text_utils: :class:`utils.TextUtils`\n :param text_utils: An instance of `utils.TextUtils`.\n\n :type fuzzy_match: bool\n :param fuzzy_match: Determines whether to use fuzzy matching.\n Currently, this is always set to False but can be enabled.\n \"\"\"\n\n def __init__(self):\n self.fuzzy_match = False\n self.text_utils = TextUtils()\n\n def build_completions_with_meta(self, line, prefix, completions):\n \"\"\"Build prompt_toolkit Completions with meta info.\n\n :type line: list\n :param line: The input text as a list of words.\n\n :type prefix: string\n :param prefix: The current word.\n\n :type completions: list\n :param completions: Completions to build meta info for.\n\n :rtype: list\n :return: Completions with meta info.\n \"\"\"\n completions_with_meta = []\n tokens = line.split(' ')\n if tokens[0] != 'gh':\n for comp in completions:\n display = None\n display_meta = None\n if 'git' in line and comp.strip() in META_LOOKUP_GIT:\n display_meta = META_LOOKUP_GIT[comp.strip()]\n elif 'git' in line and comp.strip() in META_LOOKUP_GIT_EXTRAS:\n display_meta = META_LOOKUP_GIT_EXTRAS[comp.strip()]\n completions_with_meta.append(\n Completion(comp,\n -len(prefix),\n display=display,\n display_meta=display_meta))\n return completions_with_meta\n\n def completing_command(self, words, word_before_cursor):\n \"\"\"Determine if we are currently completing the gh command.\n\n :type words: list\n :param words: The input text broken into word tokens.\n\n :type word_before_cursor: str\n :param word_before_cursor: The current word before the cursor,\n which might be one or more blank spaces.\n\n :rtype: bool\n :return: Specifies whether we are currently completing the gh command.\n \"\"\"\n if len(words) == 1 and word_before_cursor != '':\n return True\n else:\n return False\n\n def completing_subcommand(self, words, word_before_cursor):\n \"\"\"Determine if we are currently completing a subcommand.\n\n :type words: list\n :param words: The input text broken into word tokens.\n\n :type word_before_cursor: str\n :param word_before_cursor: The current word before the cursor,\n which might be one or more blank spaces.\n\n :rtype: bool\n :return: Specifies whether we are currently completing a subcommand.\n \"\"\"\n if (len(words) == 1 and word_before_cursor == '') \\\n or (len(words) == 2 and word_before_cursor != ''):\n return True\n else:\n return False\n\n def completing_arg(self, words, word_before_cursor):\n \"\"\"Determine if we are currently completing an arg.\n\n :type words: list\n :param words: The input text broken into word tokens.\n\n :type word_before_cursor: str\n :param word_before_cursor: The current word before the cursor,\n which might be one or more blank spaces.\n\n :rtype: bool\n :return: Specifies whether we are currently completing an arg.\n \"\"\"\n if (len(words) == 2 and word_before_cursor == '') \\\n or (len(words) == 3 and word_before_cursor != ''):\n return True\n else:\n return False\n\n def completing_subcommand_option(self, words, word_before_cursor):\n \"\"\"Determine if we are currently completing an option.\n\n :type words: list\n :param words: The input text broken into word tokens.\n\n :type word_before_cursor: str\n :param word_before_cursor: The current word before the cursor,\n which might be one or more blank spaces.\n\n :rtype: list\n :return: A list of options.\n \"\"\"\n options = []\n for subcommand, args_opts in COMPLETIONS_GH.items():\n if subcommand in words and \\\n (words[-2] == subcommand or\n self.completing_subcommand_option_util(subcommand, words)):\n options.extend(COMPLETIONS_GH[subcommand]['opts'])\n return options\n\n def completing_subcommand_option_util(self, option, words):\n \"\"\"Determine if we are currently completing an option.\n\n Called by completing_subcommand_option as a utility method.\n\n :type words: list\n :param words: The input text broken into word tokens.\n\n :type word_before_cursor: str\n :param word_before_cursor: The current word before the cursor,\n which might be one or more blank spaces.\n\n :rtype: bool\n :return: Specifies whether we are currently completing an option.\n \"\"\"\n # Example: Return True for: gh view 1 --pag\n if len(words) > 3:\n if option in words:\n return True\n return False\n\n def arg_completions(self, words, word_before_cursor):\n \"\"\"Generates arguments completions based on the input.\n\n :type words: list\n :param words: The input text broken into word tokens.\n\n :type word_before_cursor: str\n :param word_before_cursor: The current word before the cursor,\n which might be one or more blank spaces.\n\n :rtype: list\n :return: A list of completions.\n \"\"\"\n if 'gh' not in words:\n return []\n for subcommand, args_opts in COMPLETIONS_GH.items():\n if subcommand in words:\n args = list(COMPLETIONS_GH[subcommand]['args'].keys())\n if not args:\n # Some commands don't have args, complete options instead.\n args = list(COMPLETIONS_GH[subcommand]['opts'].keys())\n return args if args else []\n return []\n\n def get_completions(self, document, _):\n \"\"\"Get completions for the current scope.\n\n :type document: :class:`prompt_toolkit.Document`\n :param document: An instance of `prompt_toolkit.Document`.\n\n :type _: :class:`prompt_toolkit.completion.Completion`\n :param _: (Unused).\n\n :rtype: generator\n :return: Yields an instance of `prompt_toolkit.completion.Completion`.\n \"\"\"\n word_before_cursor = document.get_word_before_cursor(WORD=True)\n words = self.text_utils.get_tokens(document.text)\n commands = []\n if len(words) == 0:\n return commands\n if self.completing_command(words, word_before_cursor):\n commands = ['gh']\n else:\n if 'gh' not in words:\n return commands\n if self.completing_subcommand(words, word_before_cursor):\n commands = list(SUBCOMMANDS.keys())\n else:\n if self.completing_arg(words, word_before_cursor):\n commands = self.arg_completions(words, word_before_cursor)\n else:\n commands = self.completing_subcommand_option(\n words,\n word_before_cursor)\n completions = self.text_utils.find_matches(\n word_before_cursor, commands, fuzzy=self.fuzzy_match)\n return completions\n" }, { "path": "gitsome/completions.py", "content": "# -*- coding: utf-8 -*-\n\n# Copyright 2015 Donne Martin. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"). You\n# may not use this file except in compliance with the License. A copy of\n# the License is located at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# or in the \"license\" file accompanying this file. This file is\n# distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF\n# ANY KIND, either express or implied. See the License for the specific\n# language governing permissions and limitations under the License.\n\nCOMPLETIONS_GH = {\n 'configure': {\n 'desc': \"Configures gitsome.\",\n 'args': {},\n 'opts': {\n '-e': 'flag (opt) configure GitHub Enterprise.',\n '--enterprise': 'flag (opt) configure GitHub Enterprise.',\n },\n },\n 'create-comment': {\n 'desc': 'Creates a comment on the given issue.',\n 'args': {\n 'octocat/Spoon-Knife/1': 'str (req) user/repo/issue_number combo.',\n },\n 'opts': {\n '-t': 'see associated -- option for details.',\n '--text': 'str (req) comment text.',\n },\n },\n 'create-issue': {\n 'desc': 'Creates an issue.',\n 'args': {\n 'octocat/Spoon-Knife': 'str (req) user/repository combo.',\n },\n 'opts': {\n '-t': 'see associated -- option for details.',\n '--issue_title': 'str (req) issue title.',\n '-d': 'str (opt) issue description.',\n '--issue_desc': 'str (opt) issue description.',\n },\n },\n 'create-repo': {\n 'desc': 'Creates a repository.',\n 'args': {\n 'Spoon-Knife': 'str (req) repository name.',\n },\n 'opts': {\n '-d': 'str (opt) repo description',\n '--repo_desc': 'str (opt) repo description.',\n '-pr': 'flag (opt) create a private repo',\n '--private': 'flag (opt) create a private repo',\n },\n },\n 'emails': {\n 'desc': \"Lists all the user's registered emails.\",\n 'args': {},\n 'opts': {},\n },\n 'emojis': {\n 'desc': 'Lists all GitHub supported emojis.',\n 'args': {},\n 'opts': {\n '-p': 'flag (req) show results in a pager.',\n '--pager': 'flag (req) show results in a pager.',\n },\n },\n 'feed': {\n 'desc': \"Lists all activity for the given user or repo, if called with no arg, shows the logged in user's feed.\",\n 'args': {\n 'octocat/Hello-World --pager': \"str (opt) user or user/repository combo, if blank, shows the logged in user's feed.\",\n },\n 'opts': {\n '-pr': 'flag (req) also show private events.',\n '--private': 'flag (req) also show private events.',\n '-p': 'flag (req) show results in a pager.',\n '--pager': 'flag (req) show results in a pager.',\n },\n },\n 'followers': {\n 'desc': 'Lists all followers and the total follower count.',\n 'args': {\n 'octocat': \"str (req) the user's login id, if blank, shows logged in user's info.\",\n },\n 'opts': {\n '-p': 'flag (opt) show results in a pager.',\n '--pager': 'flag (opt) show results in a pager.',\n },\n },\n 'following': {\n 'desc': 'Lists all followed users and the total followed count.',\n 'args': {\n 'octocat': \"str (req) the user's login id, if blank, shows logged in user's info.\",\n },\n 'opts': {\n '-p': 'flag (opt) show results in a pager.',\n '--pager': 'flag (opt) show results in a pager.',\n },\n },\n 'gitignore-template': {\n 'desc': 'Outputs the gitignore template for the given language.',\n 'args': {\n 'Python': 'str (req) the language-specific .gitignore.',\n },\n 'opts': {},\n },\n 'gitignore-templates': {\n 'desc': 'Outputs all supported gitignore templates.',\n 'args': {},\n 'opts': {\n '-p': 'flag (opt) show results in a pager.',\n '--pager': 'flag (opt) show results in a pager.',\n },\n },\n 'issue': {\n 'desc': 'Outputs detailed information about the given issue.',\n 'args': {\n 'octocat/Spoon-Knife/1': 'str (req) user/repo/issue_number combo.',\n },\n 'opts': {},\n },\n 'issues': {\n 'desc': 'Lists all issues matching the filter.',\n 'args': {},\n 'opts': {\n '-f': 'str (opt) \"assigned\", \"created\", \"mentioned\", \"subscribed\" (default).',\n '--issue_filter': 'str (opt) \"assigned\", \"created\", \"mentioned\", \"subscribed\" (default).',\n '-s': 'str (opt) \"all\", \"open\" (default), \"closed\".',\n '--issue_state': 'str (opt) \"all\", \"open\" (default), \"closed\".',\n '-l': 'flag (opt) num items to show, defaults to 1000.',\n '--limit': 'flag (opt) num items to show, defaults to 1000.',\n '-p': 'flag (opt) show results in a pager.',\n '--pager': 'flag (opt) show results in a pager.',\n },\n },\n 'license': {\n 'desc': 'Outputs the license template for the given license.',\n 'args': {\n 'apache-2.0': 'str (req) the license name.',\n },\n 'opts': {},\n },\n 'licenses': {\n 'desc': 'Outputs all supported license templates.',\n 'args': {},\n 'opts': {},\n },\n 'me': {\n 'desc': 'Lists information about the logged in user.',\n 'args': {},\n 'opts': {\n '-b': 'flag (opt) view profile in a browser instead of the terminal.',\n '--browser': 'flag (opt) view profile in a browser instead of the terminal.',\n '-t': 'see associated -- option for details.',\n '--text_avatar': 'flag (opt) view profile pic in plain text.',\n '-l': 'flag (opt) num items to show, defaults to 1000.',\n '--limit': 'flag (opt) num items to show, defaults to 1000.',\n '-p': 'flag (opt) show results in a pager.',\n '--pager': 'flag (opt) show results in a pager.',\n },\n },\n 'notifications': {\n 'desc': 'Lists all notifications.',\n 'args': {},\n 'opts': {\n '-l': 'flag (opt) num items to show, defaults to 1000.',\n '--limit': 'flag (opt) num items to show, defaults to 1000.',\n '-p': 'flag (opt) show results in a pager.',\n '--pager': 'flag (opt) show results in a pager.',\n },\n },\n 'octo': {\n 'desc': 'Outputs an Easter egg or the given message from Octocat.',\n 'args': {\n '\"Keep it logically awesome\"': 'str (req) a message from Octocat, if empty, Octocat speaks an Easter egg.',\n },\n 'opts': {},\n },\n 'pull-request': {\n 'desc': 'Outputs detailed information about the given pull request.',\n 'args': {\n 'octocat/Spoon-Knife/3': 'str (req) user/repo/pull_number combo.',\n },\n 'opts': {},\n },\n 'pull-requests': {\n 'desc': 'Lists all pull requests.',\n 'args': {},\n 'opts': {\n '-l': 'flag (opt) num items to show, defaults to 1000.',\n '--limit': 'flag (opt) num items to show, defaults to 1000.',\n '-p': 'flag (opt) show results in a pager.',\n '--pager': 'flag (opt) show results in a pager.',\n },\n },\n 'rate-limit': {\n 'desc': 'Outputs the rate limit.',\n 'args': {},\n 'opts': {},\n },\n 'repo': {\n 'desc': 'Outputs detailed information about the given repo.',\n 'args': {\n 'octocat/Spoon-Knife': 'str (req) user/repository combo.',\n },\n 'opts': {},\n },\n 'repos': {\n 'desc': 'Lists all repos matching the given filter.',\n 'args': {\n '\"foo bar optional filter\"': 'str (opt) filters repos by name.',\n },\n 'opts': {\n '-l': 'flag (opt) num items to show, defaults to 1000.',\n '--limit': 'flag (opt) num items to show, defaults to 1000.',\n '-p': 'flag (opt) show results in a pager.',\n '--pager': 'flag (opt) show results in a pager.',\n },\n },\n 'search-issues': {\n 'desc': 'Searches for all issues matching the given query.',\n 'args': {\n '\"foobarbaz in:title created:>=2015-01-01\"': 'str (req) the search query.',\n },\n 'opts': {\n '-s': 'str (opt) \"stars\", \"forks\", \"updated\", if blank, sorts by query best match.',\n '--sort': 'str (opt) \"stars\", \"forks\", \"updated\", if blank, sorts by query best match.',\n '-l': 'flag (opt) num items to show, defaults to 1000.',\n '--limit': 'flag (opt) num items to show, defaults to 1000.',\n '-p': 'flag (opt) show results in a pager.',\n '--pager': 'flag (opt) show results in a pager.',\n },\n },\n 'search-repos': {\n 'desc': 'Searches for all repos matching the given query.',\n 'args': {\n '\"created:>=2015-01-01 stars:>=1500 language:python\"': 'str (req) the search query.',\n },\n 'opts': {\n '-s': 'str (opt) \"stars\", \"forks\", \"updated\", if blank, sorts by query best match.',\n '--sort': 'str (opt) \"stars\", \"forks\", \"updated\", if blank, sorts by query best match.',\n '-l': 'flag (opt) num items to show, defaults to 1000.',\n '--limit': 'flag (opt) num items to show, defaults to 1000.',\n '-p': 'flag (opt) show results in a pager.',\n '--pager': 'flag (opt) show results in a pager.',\n },\n },\n 'starred': {\n 'desc': 'Outputs starred repos.',\n 'args': {\n '\"foo bar optional filter\"': 'str (opt) filters repos by name.',\n },\n 'opts': {\n '-l': 'flag (opt) num items to show, defaults to 1000.',\n '--limit': 'flag (opt) num items to show, defaults to 1000.',\n '-p': 'flag (opt) show results in a pager.',\n '--pager': 'flag (opt) show results in a pager.',\n },\n },\n 'trending': {\n 'desc': 'Lists trending repos for the given language.',\n 'args': {\n 'Overall': 'str (opt) the language filter for trending repos, if blank, the overall rankings are shown.',\n },\n 'opts': {\n '-w': 'flag (opt) show the weekly trending repos.',\n '--weekly': 'flag (opt) show the weekly trending repos.',\n '-m': 'flag (opt) show the monthly trending repos.',\n '--monthly': 'flag (opt) show the monthly trending repos.',\n '-D': 'flag (opt) view trending devs. Only valid with -b/--browser.',\n '--devs': 'flag (opt) view trending devs. Only valid with -b/--browser.',\n '-b': 'flag (opt) view profile in a browser instead of the terminal.',\n '--browser': 'flag (opt) view profile in a browser instead of the terminal.',\n '-p': 'flag (opt) show results in a pager.',\n '--pager': 'flag (opt) show results in a pager.',\n },\n },\n 'user': {\n 'desc': 'Lists information about the given user.',\n 'args': {\n 'github --pager': \"str (req) the user's login id.\",\n },\n 'opts': {\n '-b': 'flag (opt) view profile in a browser instead of the terminal.',\n '--browser': 'flag (opt) view profile in a browser instead of the terminal.',\n '-t': 'see associated -- option for details.',\n '--text_avatar': 'flag (opt) view profile pic in plain text.',\n '-l': 'flag (opt) num items to show, defaults to 1000.',\n '--limit': 'flag (opt) num items to show, defaults to 1000.',\n '-p': 'flag (opt) show results in a pager.',\n '--pager': 'flag (opt) show results in a pager.',\n },\n },\n 'view': {\n 'desc': 'Views the given repo or issue index in the terminal or a browser.',\n 'args': {\n '1': 'int (req) the 1-based index to view.',\n },\n 'opts': {\n '-b': 'flag (opt) view profile in a browser instead of the terminal.',\n '--browser': 'flag (opt) view profile in a browser instead of the terminal.',\n },\n },\n}\nMETA_LOOKUP_GH = {\n '10': 'limit: int (opt) limits the posts displayed',\n '\"(?i)(Python|Django)\"': ('regex_query: string (opt) applies a regular '\n 'expression comment filter'),\n '1': 'index: int (req) views the post index',\n '\"user\"': 'user:string (req) shows info on the specified user',\n 'gh': 'Git auto-completer with GitHub integration.',\n}\nSUBCOMMANDS = {}\n\n\ndef build_meta_lookups():\n for subcommand, args_opts in COMPLETIONS_GH.items():\n META_LOOKUP_GH.update({subcommand: COMPLETIONS_GH[subcommand]['desc']})\n SUBCOMMANDS.update({subcommand: COMPLETIONS_GH[subcommand]['desc']})\n for opt, meta in args_opts['opts'].items():\n META_LOOKUP_GH.update({opt: meta})\n for arg, meta in args_opts['args'].items():\n META_LOOKUP_GH.update({arg: meta})\n\n\nbuild_meta_lookups()\n" }, { "path": "gitsome/completions_git.py", "content": "# -*- coding: utf-8 -*-\n\n# Copyright 2015 Donne Martin. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"). You\n# may not use this file except in compliance with the License. A copy of\n# the License is located at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# or in the \"license\" file accompanying this file. This file is\n# distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF\n# ANY KIND, either express or implied. See the License for the specific\n# language governing permissions and limitations under the License.\n\nMETA_LOOKUP_GIT = {\n 'git': 'The stupid content tracker.',\n '--version': 'Prints the Git suite version that the git program came from.',\n '--help': 'Prints the synopsis and a list of the most commonly used commands.',\n '-C': 'Run as if git was started in instead of the current working directory.',\n '-c': 'Pass a configuration parameter to the command.',\n '--exec-path': 'Path to wherever your core Git programs are installed.',\n '--html-path': \"Print the path, without trailing slash, where Git's HTML documentation is installed and exit.\",\n '--man-path': 'Print the manpath (see man(1)) for the man pages for this version of Git and exit.',\n '--info-path': 'Print the path where the Info files documenting this version of Git are installed and exit.',\n '-p': 'Pipe all output into less (or if set, $PAGER) if standard output is a terminal.',\n '--paginate': 'Pipe all output into less (or if set, $PAGER) if standard output is a terminal.',\n '--no-pager': 'Do not pipe Git output into a pager.',\n '--git-dir=': 'Set the path to the repository. This can also be controlled by setting the GIT_DIR environment variable.',\n '--work-tree=': 'Set the path to the working tree. It can be an absolute path or a path relative to the current working directory.',\n '--namespace=': 'Set the Git namespace. See gitnamespaces(7) for more details.',\n '--bare': 'Treat the repository as a bare repository. If GIT_DIR environment is not set, it is set to the current working directory.',\n '--no-replace-objects': 'Do not use replacement refs to replace Git objects.',\n '--literal-pathspecs': 'Treat pathspecs literally (i.e. no globbing, no pathspec magic).',\n '--glob-pathspecs': 'Add \"glob\" magic to all pathspec.',\n '--noglob-pathspecs': 'Add \"literal\" magic to all pathspec.',\n '--icase-pathspecs': 'Add \"icase\" magic to all pathspec.',\n 'add': 'Add file contents to the index.',\n 'am': 'Apply a series of patches from a mailbox.',\n 'amend': 'Amend the previous commit.',\n 'archive': 'Create an archive of files from a named tree.',\n 'bisect': 'Use binary search to find the commit that introduced a bug.',\n 'branch': 'List, create, or delete branches.',\n 'branches': 'List all branches, including remotes.',\n 'bundle': 'Move objects and refs by archive.',\n 'c': 'Alias for git clone.',\n 'ca': 'Alias for git add -A && git commit -av.',\n 'checkout': 'Switch branches or restore working tree files.',\n 'cherry-pick': 'Apply the changes introduced by some existing commits.',\n 'citool': 'Graphical alternative to git-commit.',\n 'clean': 'Remove untracked files from the working tree.',\n 'clone': 'Clone a repository into a new directory.',\n 'commit': 'Record changes to the repository.',\n 'contributors': 'List repo contributors.',\n 'describe': 'Describe a commit using the most recent tag reachable from it.',\n 'diff': 'Show changes between commits, commit and working tree, etc.',\n 'fetch': 'Download objects and refs from another repository.',\n 'flow': 'Git branching model extensions.',\n 'format-patch': 'Prepare patches for e-mail submission.',\n 'gc': 'Cleanup unnecessary files and optimize the local repository.',\n 'grep': 'Print lines matching a pattern.',\n 'gui': 'A portable graphical interface to Git.',\n 'init': 'Create an empty Git repository or reinitialize an existing one.',\n 'log': 'Show commit logs.',\n 'merge': 'Join two or more development histories together.',\n 'mv': 'Move or rename a file, a directory, or a symlink.',\n 'notes': 'Add or inspect object notes.',\n 'pull': 'Fetch from and integrate with another repository or a local branch.',\n 'push': 'Update remote refs along with associated objects.',\n 'rebase': 'Forward-port local commits to the updated upstream head.',\n 'reset': 'Reset current HEAD to the specified state.',\n 'revert': 'Revert some existing commits.',\n 'rm': 'Remove files from the working tree and from the index.',\n 'shortlog': 'Summarize git log output.',\n 'show': 'Show various types of objects.',\n 'stage': 'Add file contents to the staging area.',\n 'stash': 'Stash the changes in a dirty working directory away.',\n 'status': 'Show the working tree status.',\n 'submodule': 'Initialize, update or inspect submodules.',\n 'tag': 'Create, list, delete or verify a tag object signed with GPG.',\n 'tags': 'List all tags.',\n 'worktree': 'Manage multiple working trees.',\n 'gitk': 'The Git repository browser.',\n 'config': 'Get and set repository or global options.',\n 'fast-export': 'Git data exporter.',\n 'fast-import': 'Backend for fast Git data importers.',\n 'filter-branch': 'Rewrite branches.',\n 'mergetool': 'Run merge conflict resolution tools to resolve merge conflicts.',\n 'pack-refs': 'Pack heads and tags for efficient repository access.',\n 'prune': 'Prune all unreachable objects from the object database.',\n 'reflog': 'Manage reflog information.',\n 'relink': 'Hardlink common objects in local repositories.',\n 'remote': 'Manage set of tracked repositories.',\n 'remotes': 'List set of tracked repositories.',\n 'repack': 'Pack unpacked objects in a repository.',\n 'replace': 'Create, list, delete refs to replace objects.',\n 'annotate': 'Annotate file lines with commit information.',\n 'blame': 'Show what revision and author last modified each line of a file.',\n 'cherry': 'Find commits yet to be applied to upstream.',\n 'count-objects': 'Count unpacked number of objects and their disk consumption.',\n 'difftool': 'Show changes using common diff tools.',\n 'fsck': 'Verifies the connectivity and validity of the objects in the database.',\n 'get-tar-commit-id': 'Extract commit ID from an archive created using git-archive.',\n 'help': 'Display help information about Git.',\n 'instaweb': 'Instantly browse your working repository in gitweb.',\n 'merge-tree': 'Show three-way merge without touching index.',\n 'rerere': 'Reuse recorded resolution of conflicted merges.',\n 'rev-parse': 'Pick out and massage parameters.',\n 'show-branch': 'Show branches and their commits.',\n 'verify-commit': 'Check the GPG signature of commits.',\n 'verify-tag': 'Check the GPG signature of tags.',\n 'whatchanged': 'Show logs with difference each commit introduces.',\n 'gitweb': 'Git web interface (web frontend to Git repositories).',\n 'archimport': 'Import an Arch repository into Git.',\n 'cvsexportcommit': 'Export a single commit to a CVS checkout.',\n 'cvsimport': 'Salvage your data out of another SCM people love to hate.',\n 'cvsserver': 'A CVS server emulator for Git.',\n 'imap-send': 'Send a collection of patches from stdin to an IMAP folder.',\n 'p4': 'Import from and submit to Perforce repositories.',\n 'quiltimport': 'Applies a quilt patchset onto the current branch.',\n 'request-pull': 'Generates a summary of pending changes.',\n 'send-email': 'Send a collection of patches as emails.',\n 'svn': 'Bidirectional operation between a Subversion repository and Git.',\n 'apply': 'Apply a patch to files and/or to the index.',\n 'checkout-index': 'Copy files from the index to the working tree.',\n 'commit-tree': 'Create a new commit object.',\n 'hash-object': 'Compute object ID and optionally creates a blob from a file.',\n 'index-pack': 'Build pack index file for an existing packed archive.',\n 'lfs': 'Git extension for versioning large files.',\n 'merge-file': 'Run a three-way file merge.',\n 'merge-index': 'Run a merge for files needing merging.',\n 'mktag': 'Creates a tag object.',\n 'mktree': 'Build a tree-object from ls-tree formatted text.',\n 'pack-objects': 'Create a packed archive of objects.',\n 'prune-packed': 'Remove extra objects that are already in pack files.',\n 'read-tree': 'Reads tree information into the index.',\n 'symbolic-ref': 'Read, modify and delete symbolic refs.',\n 'unpack-objects': 'Unpack objects from a packed archive.',\n 'update-index': 'Register file contents in the working tree to the index.',\n 'update-ref': 'Update the object name stored in a ref safely.',\n 'write-tree': 'Create a tree object from the current index.',\n 'cat-file': 'Provide content or type and size information for repository objects.',\n 'diff-files': 'Compares files in the working tree and the index.',\n 'diff-index': 'Compare a tree to the working tree or index.',\n 'diff-tree': 'Compares the content and mode of blobs found via two tree objects.',\n 'for-each-ref': 'Output information on each ref.',\n 'ls-files': 'Show information about files in the index and the working tree.',\n 'ls-remote': 'List references in a remote repository.',\n 'ls-tree': 'List the contents of a tree object.',\n 'merge-base': 'Find as good common ancestors as possible for a merge.',\n 'name-rev': 'Find symbolic names for given revs.',\n 'pack-redundant': 'Find redundant pack files.',\n 'rev-list': 'Lists commit objects in reverse chronological order.',\n 'show-index': 'Show packed archive index.',\n 'show-ref': 'List references in a local repository.',\n 'unpack-file': \"Creates a temporary file with a blob's contents.\",\n 'var': 'Show a Git logical variable.',\n 'verify-pack': 'Validate packed Git archive files.',\n 'daemon': 'A really simple server for Git repositories.',\n 'fetch-pack': 'Receive missing objects from another repository.',\n 'http-backend': 'Server side implementation of Git over HTTP.',\n 'send-pack': 'Push objects over Git protocol to another repository.',\n 'update-server-info': 'Update auxiliary info file to help dumb servers.',\n 'http-fetch': 'Download from a remote Git repository via HTTP.',\n 'http-push': 'Push objects over HTTP/DAV to another repository.',\n 'parse-remote': 'Routines to help parsing remote repository access parameters.',\n 'receive-pack': 'Receive what is pushed into the repository.',\n 'shell': 'Restricted login shell for Git-only SSH access.',\n 'upload-archive': 'Send archive back to git-archive.',\n 'upload-pack': 'Send objects packed back to git-fetch-pack.',\n 'check-attr': 'Display gitattributes information.',\n 'check-ignore': 'Debug gitignore / exclude files.',\n 'check-mailmap': 'Show canonical names and email addresses of contacts.',\n 'check-ref-format': 'Ensures that a reference name is well formed.',\n 'column': 'Display data in columns.',\n 'credential': 'Retrieve and store user credentials.',\n 'credential-cache': 'Helper to temporarily store passwords in memory.',\n 'credential-store': 'Helper to store credentials on disk.',\n 'fmt-merge-msg': 'Produce a merge commit message.',\n 'interpret-trailers': 'help add structured information into commit messages.',\n 'mailinfo': 'Extracts patch and authorship from a single e-mail message.',\n 'mailsplit': 'Simple UNIX mbox splitter program.',\n 'merge-one-file': 'The standard helper program to use with git-merge-index.',\n 'patch-id': 'Compute unique ID for a patch.',\n 'sh-i18n': \"Git's i18n setup code for shell scripts.\",\n 'sh-setup': 'Common Git shell script setup code.',\n 'stripspace': 'Remove unnecessary whitespace.',\n 'subtree': 'Merge subtrees together and split repository into subtrees.',\n 'git-cvsserver': 'A CVS server emulator for Git.',\n 'git-credential-osxkeychain': 'Cache GitHub credentials in the OS X Keychain.',\n 'git-lfs': 'Git extension for versioning large files.',\n 'git-receive-pack': 'Receive what is pushed into the repository',\n 'git-shell': 'Restricted login shell for Git-only SSH access.',\n 'git-subtree': 'Merge subtrees together and split repository into subtrees.',\n 'git-upload-archive': 'Send archive back to git-archive.',\n 'git-upload-pack': 'Send objects packed back to git-fetch-pack.',\n 'git-flow': 'Git branching model extensions.',\n 'gitchangelog': 'Generates a changelog from git tags and commit messages.',\n}\nMETA_LOOKUP_GIT_EXTRAS = {\n 'git-extras': 'Various Git utilities.',\n 'git-alias': 'Define, search and show aliases.',\n 'git-archive-file': 'Export the current HEAD of the git repository to a archive.',\n 'git-authors': 'Generate authors report.',\n 'git-back': 'Undo and Stage latest commits.',\n 'git-bug': 'Create bug branch.',\n 'git-changelog': 'Generate a changelog report.',\n 'git-chore': 'Create chore branch.',\n 'git-clear': 'Rigorously clean up a repository.',\n 'git-commits-since': 'Show commit logs since some date.',\n 'git-contrib': \"Show user's contributions\",\n 'git-count': 'Show commit count.',\n 'git-create-branch': 'Create branches.',\n 'git-delete-branch': 'Delete branches.',\n 'git-delete-merged-branches': 'Delete merged branches.',\n 'git-delete-submodule': 'Delete submodules.',\n 'git-delete-tag': 'Delete tags.',\n 'git-delta': 'Lists changed files.',\n 'git-effort': 'Show effort statistics on file(s).',\n 'git-feature': 'Create/Merge feature branch.',\n 'git-fork': 'Fork a repo on github.',\n 'git-fresh-branch': 'Create fresh branches.',\n 'git-gh-pages': 'Create the GitHub Pages branch.',\n 'git-graft': 'Merge and destroy a given branch.',\n 'git-guilt': 'calculate change between two revisions.',\n 'git-ignore-io': 'Get sample gitignore file.',\n 'git-ignore': 'Add .gitignore patterns.',\n 'git-info': 'Returns information on current repository.',\n 'git-line-summary': 'Show repository summary by line.',\n 'git-local-commits': 'List local commits.',\n 'git-lock': 'Lock a file excluded from version control.',\n 'git-locked': 'ls files that have been locked.',\n 'git-merge-into': 'Merge one branch into another.',\n 'git-merge-repo': 'Merge two repo histories.',\n 'git-missing': 'Show commits missing from another branch.',\n 'git-obliterate': 'Completely remove a file from the repository, including past commits and tags.',\n 'git-pr': 'Checks out a pull request locally.',\n 'git-psykorebase': 'Rebase a branch with a merge commit.',\n 'git-pull-request': 'Checks out a pull request from GitHub.',\n 'git-rebase-patch': 'Rebases a patch.',\n 'git-refactor': 'Create refactor branch.',\n 'git-release': 'Commit, tag and push changes to the repository.',\n 'git-rename-tag': 'Rename a tag.',\n 'git-repl': 'git read-eval-print-loop.',\n 'git-reset-file': 'Reset one file.',\n 'git-root': 'show path of root.',\n 'git-rscp': 'Copies specific files from the working directory of a remote repository to the current working directory.',\n 'git-scp': 'Copy files to SSH compatible git-remote.',\n 'git-sed': 'replace patterns in git-controlled files.',\n 'git-setup': 'Set up a git repository.',\n 'git-show-merged-branches': 'Show merged branches.',\n 'git-show-tree': 'show branch tree of commit history.',\n 'git-show-unmerged-branches': 'Show unmerged branches.',\n 'git-squash': 'Import changes from a branch.',\n 'git-summary': 'Show repository summary.',\n 'git-touch': 'Touch and add file to the index.',\n 'git-undo': 'Remove latest commits.',\n 'git-unlock': 'Unlock a file excluded from version control.',\n\n 'extras': 'Various Git utilities.',\n 'alias': 'Define, search and show aliases.',\n 'archive-file': 'Export the current HEAD of the git repository to a archive.',\n 'authors': 'Generate authors report.',\n 'back': 'Undo and Stage latest commits.',\n 'bug': 'Create bug branch.',\n 'changelog': 'Generate a changelog report.',\n 'chore': 'Create chore branch.',\n 'clear': 'Rigorously clean up a repository.',\n 'commits-since': 'Show commit logs since some date.',\n 'contrib': \"Show user's contributions\",\n 'count': 'Show commit count.',\n 'create-branch': 'Create branches.',\n 'delete-branch': 'Delete branches.',\n 'delete-merged-branches': 'Delete merged branches.',\n 'delete-submodule': 'Delete submodules.',\n 'delete-tag': 'Delete tags.',\n 'delta': 'Lists changed files.',\n 'effort': 'Show effort statistics on file(s).',\n 'feature': 'Create/Merge feature branch.',\n 'fork': 'Fork a repo on github.',\n 'fresh-branch': 'Create fresh branches.',\n 'gh-pages': 'Create the GitHub Pages branch.',\n 'graft': 'Merge and destroy a given branch.',\n 'guilt': 'calculate change between two revisions.',\n 'ignore-io': 'Get sample gitignore file.',\n 'ignore': 'Add .gitignore patterns.',\n 'info': 'Returns information on current repository.',\n 'line-summary': 'Show repository summary by line.',\n 'local-commits': 'List local commits.',\n 'lock': 'Lock a file excluded from version control.',\n 'locked': 'ls files that have been locked.',\n 'merge-into': 'Merge one branch into another.',\n 'merge-repo': 'Merge two repo histories.',\n 'missing': 'Show commits missing from another branch.',\n 'obliterate': 'Completely remove a file from the repository, including past commits and tags.',\n 'pr': 'Checks out a pull request locally.',\n 'psykorebase': 'Rebase a branch with a merge commit.',\n 'pull-request': 'Checks out a pull request from GitHub.',\n 'rebase-patch': 'Rebases a patch.',\n 'refactor': 'Create refactor branch.',\n 'release': 'Commit, tag and push changes to the repository.',\n 'rename-tag': 'Rename a tag.',\n 'repl': 'git read-eval-print-loop.',\n 'reset-file': 'Reset one file.',\n 'root': 'show path of root.',\n 'rscp': 'Copies specific files from the working directory of a remote repository to the current working directory.',\n 'scp': 'Copy files to SSH compatible git-remote.',\n 'sed': 'replace patterns in git-controlled files.',\n 'setup': 'Set up a git repository.',\n 'show-merged-branches': 'Show merged branches.',\n 'show-tree': 'show branch tree of commit history.',\n 'show-unmerged-branches': 'Show unmerged branches.',\n 'squash': 'Import changes from a branch.',\n 'summary': 'Show repository summary.',\n 'touch': 'Touch and add file to the index.',\n 'undo': 'Remove latest commits.',\n 'unlock': 'Unlock a file excluded from version control.',\n}\n" }, { "path": "gitsome/config.py", "content": "# -*- coding: utf-8 -*-\n\n# Copyright 2015 Donne Martin. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"). You\n# may not use this file except in compliance with the License. A copy of\n# the License is located at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# or in the \"license\" file accompanying this file. This file is\n# distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF\n# ANY KIND, either express or implied. See the License for the specific\n# language governing permissions and limitations under the License.\n\nfrom __future__ import unicode_literals\nfrom __future__ import print_function\n\nimport click\nfrom getpass import getpass\nimport os\nimport requests\nfrom requests.packages.urllib3.exceptions import InsecureRequestWarning\n\nfrom .compat import configparser\nfrom .lib.github3 import authorize, enterprise_login, login\nfrom .lib.github3.exceptions import AuthenticationFailed, UnprocessableEntity\n\n\nclass Config(object):\n \"\"\"Gitsome config.\n\n :type api: :class:`github3.github.Github`\n :param api: An instance of github3.github.Github.\n\n :type clr_x: str\n :param clr_x: Various ansi color config colors to use for highlights.\n\n :type CONFIG: str\n :param CONFIG: The config file name.\n\n :type CONFIG_SECTION: str\n :param CONFIG_SECTION: The main config file section label.\n\n :type CONFIG_CLR_X: str\n :param CONFIG_CLR_X: Various ansi color config labels to use for highlights.\n\n :type CONFIG_ENTERPRISE_URL: str\n :param CONFIG_ENTERPRISE_URL: The GitHub Enterprise url.\n\n :type CONFIG_USER_LOGIN: str\n :param CONFIG_USER_LOGIN: The user login.\n\n :type CONFIG_USER_PASS: str\n :param CONFIG_USER_PASS: The user password.\n\n :type CONFIG_USER_TOKEN: str\n :param CONFIG_USER_TOKEN: The user token.\n\n :type CONFIG_USER_FEED: str\n :param CONFIG_USER_FEED: The user feed config. This is the feed on\n https://github.com/ when logged in and requires the basic auth model,\n which doesn't work when logging in with tokens or 2FA. This config\n listed the pre-signed url to access the feed.\n\n :type CONFIG_URL: str\n :param CONFIG_URL: The config file name that contains urls used in the\n `gh view` command.\n\n :type CONFIG_URL_SECTION: str\n :param CONFIG_URL_SECTION: The config file section that contains urls used\n in the `gh view [url_index]` command.\n\n :type CONFIG_URL_LIST: str\n :param CONFIG_URL_LIST: The config containing a list of the last set of\n urls the user has seen, which allows the user to quickly access a repo\n url with the `gh view [url_index]` command.\n\n :type CONFIG_VERIFY_SSL: str\n :param CONFIG_VERIFY_SSL: Determines whether to verify SSL certs.\n\n :type enterprise_url: str\n :param enterprise_url: The GitHub Enterprise url.\n\n :type urls: list\n :param urls: The last set of urls the user has seen, which allows the user\n to quickly access a repo url with the gh view [url_index] command.\n\n :type user_login: str\n :param user_login: The user's login in ~/.gitsomeconfig.\n\n :type user_pass: str\n :param user_pass: The user's pass in ~/.gitsomeconfig.\n This is only stored for GitHub Enterprise users since using only a\n personal access token does not seem to be supported.\n\n :type user_token: str\n :param user_token: The user's token in ~/.gitsomeconfig.\n\n :type verify_ssl: bool\n :param verify_ssl: Determines whether to verify SSL certs.\n\n :type enable_avatar: bool\n :param enable_avatar: Determines whether to request avatar image.\n \"\"\"\n\n CONFIG = '.gitsomeconfig'\n CONFIG_CLR_PRIMARY = 'clr_primary'\n CONFIG_CLR_SECONDARY = 'clr_secondary'\n CONFIG_CLR_TERTIARY = 'clr_tertiary'\n CONFIG_CLR_QUATERNARY = 'clr_quaternary'\n CONFIG_CLR_BOLD = 'clr_bold'\n CONFIG_CLR_CODE = 'clr_code'\n CONFIG_CLR_ERROR = 'clr_error'\n CONFIG_CLR_HEADER = 'clr_header'\n CONFIG_CLR_LINK = 'clr_link'\n CONFIG_CLR_LIST = 'clr_list'\n CONFIG_CLR_MESSAGE = 'clr_message'\n CONFIG_CLR_NUM_COMMENTS = 'clr_num_comments'\n CONFIG_CLR_NUM_POINTS = 'clr_num_points'\n CONFIG_CLR_TAG = 'clr_tag'\n CONFIG_CLR_TIME = 'clr_time'\n CONFIG_CLR_TITLE = 'clr_title'\n CONFIG_CLR_TOOLTIP = 'clr_tooltip'\n CONFIG_CLR_USER = 'clr_user'\n CONFIG_CLR_VIEW_LINK = 'clr_view_link'\n CONFIG_CLR_VIEW_INDEX = 'clr_view_index'\n CONFIG_SECTION = 'github'\n CONFIG_USER_LOGIN = 'user_login'\n CONFIG_USER_PASS = 'user_pass'\n CONFIG_USER_TOKEN = 'user_token'\n CONFIG_USER_FEED = 'user_feed'\n CONFIG_ENTERPRISE_URL = 'enterprise_url'\n CONFIG_VERIFY_SSL = 'verify_ssl'\n CONFIG_URL = '.gitsomeconfigurl'\n CONFIG_URL_SECTION = 'url'\n CONFIG_URL_LIST = 'url_list'\n CONFIG_AVATAR = '.gitsomeconfigavatar.png'\n CONFIG_ENABLE_AVATAR = 'enable_avatar'\n\n def __init__(self):\n self.api = None\n self.user_login = None\n self.user_pass = None\n self.user_token = None\n self.user_feed = None\n self.enterprise_url = None\n self.verify_ssl = True\n self.urls = []\n self.enable_avatar = True\n self._init_colors()\n self.load_configs([\n self.load_config_colors,\n ])\n self.login = login\n self.authorize = authorize\n self.getpass = getpass\n\n def _init_colors(self):\n \"\"\"Initialize colors to their defaults.\"\"\"\n self.clr_primary = None\n self.clr_secondary = 'green'\n self.clr_tertiary = 'cyan'\n self.clr_quaternary = 'yellow'\n self.clr_bold = 'cyan'\n self.clr_code = 'cyan'\n self.clr_error = 'red'\n self.clr_header = 'yellow'\n self.clr_link = 'green'\n self.clr_list = 'cyan'\n self.clr_message = None\n self.clr_num_comments = 'green'\n self.clr_num_points = 'green'\n self.clr_tag = 'cyan'\n self.clr_time = 'yellow'\n self.clr_title = None\n self.clr_tooltip = None\n self.clr_user = 'cyan'\n self.clr_view_link = 'magenta'\n self.clr_view_index = 'magenta'\n\n def authenticate_cached_credentials(self, config, parser,\n enterprise_auth=enterprise_login):\n \"\"\"Authenticate with the user's credentials in ~/.gitsomeconfig.\n\n :type config: str\n :param config: The config path.\n\n :type parser: :class:`ConfigParser.RawConfigParser`\n :param parser: An instance of `ConfigParser.RawConfigParser.\n \"\"\"\n with open(config) as config_file:\n try:\n parser.read_file(config_file)\n except AttributeError:\n parser.readfp(config_file)\n self.user_login = self.load_config(\n parser=parser,\n cfg_label=self.CONFIG_USER_LOGIN)\n self.user_pass = self.load_config(\n parser=parser,\n cfg_label=self.CONFIG_USER_PASS)\n self.user_token = self.load_config(\n parser=parser,\n cfg_label=self.CONFIG_USER_TOKEN)\n self.enterprise_url = self.load_config(\n parser=parser,\n cfg_label=self.CONFIG_ENTERPRISE_URL)\n self.verify_ssl = self.load_config(\n parser=parser,\n cfg_label=self.CONFIG_VERIFY_SSL,\n boolean_config=True)\n self.enable_avatar = self.load_config(\n parser=parser,\n cfg_label=self.CONFIG_ENABLE_AVATAR,\n boolean_config=True)\n self.user_feed = self.load_config(\n parser=parser,\n cfg_label=self.CONFIG_USER_FEED)\n if not self.verify_ssl:\n # The user has chosen not to verify SSL certs.\n # Disable warnings related to this option.\n requests.packages.urllib3.disable_warnings(\n InsecureRequestWarning)\n login_kwargs = {\n 'username': self.user_login,\n 'two_factor_callback': self.request_two_factor_code,\n }\n if self.enterprise_url is not None:\n self.login = enterprise_auth\n login_kwargs.update({\n 'url': self.enterprise_url,\n 'verify': self.verify_ssl,\n })\n if self.user_token is not None:\n login_kwargs.update({'token': self.user_token})\n elif self.user_pass is not None:\n login_kwargs.update({'password': self.user_pass})\n else:\n self.print_auth_error()\n return\n else:\n login_kwargs.update({'token': self.user_token})\n self.api = self.login(**login_kwargs)\n\n def authenticate(self, enterprise=False,\n enterprise_auth=enterprise_login, overwrite=False):\n \"\"\"Log into GitHub.\n\n Adapted from https://github.com/sigmavirus24/github-cli.\n\n :type enterprise: bool\n :param enterprise: Determines whether to configure GitHub Enterprise.\n Default: False.\n\n :type overwrite: bool\n :param overwrite: indicates whether we cant to overwrite the current\n set of credentials. Default: False.\n \"\"\"\n if self.api is not None and not overwrite:\n return\n # Get the full path to the configuration file.\n config = self.get_github_config_path(self.CONFIG)\n parser = configparser.RawConfigParser()\n # Check to make sure the file exists and we are allowed to read it.\n # Skip if we want to overwrite the auth settings.\n if os.path.isfile(config) and os.access(config, os.R_OK | os.W_OK) and \\\n not overwrite:\n with open(config) as config_file:\n try:\n parser.read_file(config_file)\n except AttributeError:\n parser.readfp(config_file)\n self.authenticate_cached_credentials(config, parser)\n else:\n # The file didn't exist or we don't have the correct permissions.\n login_kwargs = {\n 'two_factor_callback': self.request_two_factor_code,\n }\n if enterprise:\n self.login = enterprise_auth\n while not self.enterprise_url:\n self.enterprise_url = input('Enterprise URL: ')\n if click.confirm('Do you want to verify SSL certs?',\n default=True):\n self.verify_ssl = True\n else:\n self.verify_ssl = False\n login_kwargs.update({\n 'url': self.enterprise_url,\n 'verify': self.verify_ssl,\n })\n while not self.user_login:\n self.user_login = input('User Login: ')\n login_kwargs.update({'username': self.user_login})\n if click.confirm(('Do you want to log in with a password [Y] or '\n 'a personal access token [n]?'),\n default=True):\n user_pass = None\n while not user_pass:\n user_pass = self.getpass('Password: ')\n login_kwargs.update({'password': user_pass})\n try:\n if not enterprise:\n # Trade the user password for a personal access token.\n # This does not seem to be available for Enterprise.\n auth = self.authorize(\n self.user_login,\n user_pass,\n scopes=['user', 'repo'],\n note='gitsome',\n note_url='https://github.com/donnemartin/gitsome',\n two_factor_callback=self.request_two_factor_code\n )\n self.user_token = auth.token\n else:\n self.user_pass = user_pass\n except (UnprocessableEntity, AuthenticationFailed):\n click.secho('Error creating token.',\n fg=self.clr_error)\n click.secho(('Visit the following page and verify you do '\n 'not have an existing token named \"gitsome\":\\n'\n ' https://github.com/settings/tokens\\n'\n 'If a token already exists, update your '\n '~/.gitsomeconfig file with your token:\\n'\n ' user_token = TOKEN\\n'\n 'You can also generate a new token.'),\n fg=self.clr_message)\n self.print_auth_error()\n return\n else:\n # The user has chosen to authenticate with a token.\n while not self.user_token:\n self.user_token = input('Token: ')\n login_kwargs.update({'token': self.user_token})\n self.api = self.login(**login_kwargs)\n if self.user_feed:\n parser.set(self.CONFIG_SECTION,\n self.CONFIG_USER_FEED,\n self.user_feed)\n\n def check_auth(self):\n \"\"\"Check if the current authorization is valid.\n\n This method uses the ratelimit_remaining api to check whether\n the currently authenticated user's credentials are valid without\n deducting from the rate limit. The ratelimit_remaining api does not\n seem to be available for GitHub Enterprise.\n\n github3.py's method check_authorization seems to only work given\n an authorization created by a registered application.\n\n TODO: Determine a better way to check the authorization for\n GitHub Enterprise.\n\n :type enterprise: bool\n :param enterprise: Determines whether we are authenticating with\n GitHub Enterprise.\n \"\"\"\n if self.enterprise_url is not None:\n return True\n try:\n if self.api is not None:\n # Throws AuthenticationFailed if invalid credentials but\n # does not deduct from the rate limit.\n self.api.ratelimit_remaining\n return True\n else:\n self.print_auth_error()\n except AuthenticationFailed:\n self.print_auth_error()\n return False\n\n def get_github_config_path(self, config_file_name):\n \"\"\"Attempt to find the github config file.\n\n Adapted from https://github.com/sigmavirus24/github-cli.\n\n :type config_file_name: str\n :param config_file_name: The config file name.\n\n :rtype: str\n :return: The github config file path.\n \"\"\"\n home = os.path.abspath(os.environ.get('HOME', ''))\n config_file_path = os.path.join(home, config_file_name)\n return config_file_path\n\n def load_config(self, parser, cfg_label, default=None,\n color_config=False, boolean_config=False):\n \"\"\"Load the specified config from ~/.gitsomeconfig.\n\n :type parser: :class:`ConfigParser.RawConfigParser`\n :param parser: An instance of `ConfigParser.RawConfigParser`.\n\n :type cfg_label: str\n :param cfg_label: The config label to load.\n\n :type default: str\n :param default: The default color if no color config exists.\n Default: None.\n\n :type color_config: bool\n :param color_config: Determines whether this is a color config.\n Default: False.\n\n :type boolean_config: bool\n :param boolean_config: Determines whether to load a boolean config.\n Default: False.\n \"\"\"\n try:\n if boolean_config:\n cfg = parser.getboolean(self.CONFIG_SECTION, cfg_label)\n else:\n cfg = parser.get(self.CONFIG_SECTION, cfg_label)\n if color_config:\n if cfg == 'none':\n cfg = None\n # Check if the user input a valid color.\n # If invalid, this will throw a TypeError\n click.style('', fg=cfg)\n except (TypeError, configparser.NoOptionError):\n return default\n return cfg\n\n def load_configs(self, config_funcs):\n \"\"\"Load the specified config from ~/.gitsomeconfig.\n\n :type foo: list\n :param foo: The config methods to run.\n \"\"\"\n config_file_path = self.get_github_config_path(self.CONFIG)\n parser = configparser.RawConfigParser()\n try:\n with open(config_file_path) as config_file:\n try:\n parser.read_file(config_file)\n except AttributeError:\n parser.readfp(config_file)\n for config_func in config_funcs:\n config_func(parser)\n except IOError:\n # There might not be a cache yet, just silently return.\n return None\n\n def load_config_colors(self, parser):\n \"\"\"Load the color config from ~/.gitsomeconfig.\n\n :type parser: :class:`ConfigParser.RawConfigParser`\n :param parser: An instance of `ConfigParser.RawConfigParser`.\n \"\"\"\n self.load_colors(parser)\n\n def load_colors(self, parser):\n \"\"\"Load all colors from ~/.gitsomeconfig.\n\n :type parser: :class:`ConfigParser.RawConfigParser`\n :param parser: An instance of `ConfigParser.RawConfigParser`.\n \"\"\"\n self.clr_primary = self.load_config(\n parser=parser,\n cfg_label=self.CONFIG_CLR_PRIMARY,\n default=self.clr_primary,\n color_config=True)\n self.clr_secondary = self.load_config(\n parser=parser,\n cfg_label=self.CONFIG_CLR_SECONDARY,\n default=self.clr_secondary,\n color_config=True)\n self.clr_tertiary = self.load_config(\n parser=parser,\n cfg_label=self.CONFIG_CLR_TERTIARY,\n default=self.clr_tertiary,\n color_config=True)\n self.clr_quaternary = self.load_config(\n parser=parser,\n cfg_label=self.CONFIG_CLR_QUATERNARY,\n default=self.clr_quaternary,\n color_config=True)\n self.clr_bold = self.load_config(\n parser=parser,\n cfg_label=self.CONFIG_CLR_BOLD,\n default=self.clr_bold,\n color_config=True)\n self.clr_code = self.load_config(\n parser=parser,\n cfg_label=self.CONFIG_CLR_CODE,\n default=self.clr_code,\n color_config=True)\n self.clr_code = self.load_config(\n parser=parser,\n cfg_label=self.CONFIG_CLR_ERROR,\n default=self.clr_code,\n color_config=True)\n self.clr_header = self.load_config(\n parser=parser,\n cfg_label=self.CONFIG_CLR_HEADER,\n default=self.clr_header,\n color_config=True)\n self.clr_link = self.load_config(\n parser=parser,\n cfg_label=self.CONFIG_CLR_LINK,\n default=self.clr_link,\n color_config=True)\n self.clr_list = self.load_config(\n parser=parser,\n cfg_label=self.CONFIG_CLR_LIST,\n default=self.clr_list,\n color_config=True)\n self.clr_message = self.load_config(\n parser=parser,\n cfg_label=self.CONFIG_CLR_MESSAGE,\n default=self.clr_message,\n color_config=True)\n self.clr_num_comments = self.load_config(\n parser=parser,\n cfg_label=self.CONFIG_CLR_NUM_COMMENTS,\n default=self.clr_num_comments,\n color_config=True)\n self.clr_num_points = self.load_config(\n parser=parser,\n cfg_label=self.CONFIG_CLR_NUM_POINTS,\n default=self.clr_num_points,\n color_config=True)\n self.clr_tag = self.load_config(\n parser=parser,\n cfg_label=self.CONFIG_CLR_TAG,\n default=self.clr_tag,\n color_config=True)\n self.clr_time = self.load_config(\n parser=parser,\n cfg_label=self.CONFIG_CLR_TIME,\n default=self.clr_time,\n color_config=True)\n self.clr_title = self.load_config(\n parser=parser,\n cfg_label=self.CONFIG_CLR_TITLE,\n default=self.clr_title,\n color_config=True)\n self.clr_tooltip = self.load_config(\n parser=parser,\n cfg_label=self.CONFIG_CLR_TOOLTIP,\n default=self.clr_tooltip,\n color_config=True)\n self.clr_user = self.load_config(\n parser=parser,\n cfg_label=self.CONFIG_CLR_USER,\n default=self.clr_user,\n color_config=True)\n self.clr_view_link = self.load_config(\n parser=parser,\n cfg_label=self.CONFIG_CLR_VIEW_LINK,\n default=self.clr_view_link,\n color_config=True)\n self.clr_view_index = self.load_config(\n parser=parser,\n cfg_label=self.CONFIG_CLR_VIEW_INDEX,\n default=self.clr_view_index,\n color_config=True)\n\n def load_urls(self, view_in_browser):\n \"\"\"Load the current set of urls from ~/.gitsomeconfigurl.\n\n :type view_in_browser: bool\n :param view_in_browser: Determines whether to view the urls in a\n browser.\n\n :rtype: list\n :return: Collection of urls.\n \"\"\"\n config = self.get_github_config_path(self.CONFIG_URL)\n parser = configparser.RawConfigParser()\n with open(config) as config_file:\n try:\n parser.read_file(config_file)\n except AttributeError:\n parser.readfp(config_file)\n urls = parser.get(self.CONFIG_URL_SECTION,\n self.CONFIG_URL_LIST)\n urls = urls.strip()\n excludes = ['[', ']', \"'\"]\n for exclude in excludes:\n urls = urls.replace(exclude, '')\n if not view_in_browser:\n urls = urls.replace('https://github.com/', '')\n return urls.split(', ')\n\n def print_auth_error(self):\n \"\"\"Print a message the authorization has failed.\"\"\"\n click.secho('Authentication error.', fg=self.clr_error)\n click.secho(('Update your credentials in ~/.gitsomeconfig '\n 'or run:\\n gh configure'),\n fg=self.clr_message)\n\n def prompt_news_feed(self):\n \"\"\"Prompt the user to enter a news feed url.\"\"\"\n if click.confirm(('No feed url detected.\\n Calling gh events without '\n \"an argument\\n displays the logged in user's \"\n 'news feed.\\nDo you want gitsome to track your '\n 'news feed?'),\n default=True):\n click.secho(('Visit the following url while logged into GitHub:\\n'\n ' https://github.com\\n'\n 'Enter the url found under \"Subscribe to your '\n 'news feed\".'),\n fg=self.clr_message)\n self.user_feed = ''\n while not self.user_feed:\n self.user_feed = input('URL: ')\n\n def request_two_factor_code(self):\n \"\"\"Request two factor authentication code.\n\n Callback if two factor authentication is requested.\n\n :rtype: str\n :return: The user input two factor authentication code.\n \"\"\"\n code = ''\n while not code:\n code = input('Enter 2FA code: ')\n return code\n\n def save_config(self):\n \"\"\"Saves the config to ~/.gitsomeconfig.\"\"\"\n if self.check_auth():\n config = self.get_github_config_path(self.CONFIG)\n parser = configparser.RawConfigParser()\n parser.add_section(self.CONFIG_SECTION)\n parser.set(self.CONFIG_SECTION,\n self.CONFIG_USER_LOGIN,\n self.user_login)\n if self.user_token is not None:\n parser.set(self.CONFIG_SECTION,\n self.CONFIG_USER_TOKEN,\n self.user_token)\n if self.user_feed is not None:\n parser.set(self.CONFIG_SECTION,\n self.CONFIG_USER_FEED,\n self.user_feed)\n if self.enable_avatar is not None:\n parser.set(self.CONFIG_SECTION,\n self.CONFIG_ENABLE_AVATAR,\n self.enable_avatar)\n if self.enterprise_url is not None:\n parser.set(self.CONFIG_SECTION,\n self.CONFIG_ENTERPRISE_URL,\n self.enterprise_url)\n if self.user_pass is not None:\n parser.set(self.CONFIG_SECTION,\n self.CONFIG_USER_PASS,\n self.user_pass)\n else:\n parser.remove_option(self.CONFIG_SECTION,\n self.CONFIG_USER_PASS)\n parser.set(self.CONFIG_SECTION,\n self.CONFIG_VERIFY_SSL,\n self.verify_ssl)\n parser.set(self.CONFIG_SECTION,\n self.CONFIG_CLR_PRIMARY,\n self.clr_primary)\n parser.set(self.CONFIG_SECTION,\n self.CONFIG_CLR_SECONDARY,\n self.clr_secondary)\n parser.set(self.CONFIG_SECTION,\n self.CONFIG_CLR_TERTIARY,\n self.clr_tertiary)\n parser.set(self.CONFIG_SECTION,\n self.CONFIG_CLR_QUATERNARY,\n self.clr_quaternary)\n parser.set(self.CONFIG_SECTION,\n self.CONFIG_CLR_BOLD,\n self.clr_bold)\n parser.set(self.CONFIG_SECTION,\n self.CONFIG_CLR_CODE,\n self.clr_code)\n parser.set(self.CONFIG_SECTION,\n self.CONFIG_CLR_ERROR,\n self.clr_error)\n parser.set(self.CONFIG_SECTION,\n self.CONFIG_CLR_HEADER,\n self.clr_header)\n parser.set(self.CONFIG_SECTION,\n self.CONFIG_CLR_LINK,\n self.clr_link)\n parser.set(self.CONFIG_SECTION,\n self.CONFIG_CLR_LIST,\n self.clr_list)\n parser.set(self.CONFIG_SECTION,\n self.CONFIG_CLR_MESSAGE,\n self.clr_message)\n parser.set(self.CONFIG_SECTION,\n self.CONFIG_CLR_NUM_COMMENTS,\n self.clr_num_comments)\n parser.set(self.CONFIG_SECTION,\n self.CONFIG_CLR_NUM_POINTS,\n self.clr_num_points)\n parser.set(self.CONFIG_SECTION,\n self.CONFIG_CLR_TAG,\n self.clr_tag)\n parser.set(self.CONFIG_SECTION,\n self.CONFIG_CLR_TIME,\n self.clr_time)\n parser.set(self.CONFIG_SECTION,\n self.CONFIG_CLR_TITLE,\n self.clr_title)\n parser.set(self.CONFIG_SECTION,\n self.CONFIG_CLR_TOOLTIP,\n self.clr_tooltip)\n parser.set(self.CONFIG_SECTION,\n self.CONFIG_CLR_USER,\n self.clr_user)\n parser.set(self.CONFIG_SECTION,\n self.CONFIG_CLR_VIEW_LINK,\n self.clr_view_link)\n parser.set(self.CONFIG_SECTION,\n self.CONFIG_CLR_VIEW_INDEX,\n self.clr_view_index)\n with open(config, 'w+') as config_file:\n parser.write(config_file)\n\n def save_urls(self):\n \"\"\"Save the current set of urls to ~/.gitsomeconfigurl.\"\"\"\n config = self.get_github_config_path(self.CONFIG_URL)\n parser = configparser.RawConfigParser()\n try:\n parser.add_section(self.CONFIG_URL_SECTION)\n except configparser.DuplicateSectionError:\n pass\n parser.set(self.CONFIG_URL_SECTION, self.CONFIG_URL_LIST, self.urls)\n with open(config, 'w+') as config_file:\n parser.write(config_file)\n\n def show_bash_completions_info(self):\n \"\"\"Show info on how to enable bash completions\"\"\"\n click.secho(('By default, gitsome looks at the following locations '\n 'to enable bash completions:\\n'\n ' https://github.com/donnemartin/gitsome/blob/master/xonsh/environ.py#L123-L130\\n' # NOQA\n 'If bash completions are not working for you, check out '\n 'the following link:\\n'\n ' https://github.com/donnemartin/gitsome#enabling-bash-completions'), # NOQA\n fg=self.clr_message)\n" }, { "path": "gitsome/formatter.py", "content": "# -*- coding: utf-8 -*-\n\n# Copyright 2015 Donne Martin. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"). You\n# may not use this file except in compliance with the License. A copy of\n# the License is located at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# or in the \"license\" file accompanying this file. This file is\n# distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF\n# ANY KIND, either express or implied. See the License for the specific\n# language governing permissions and limitations under the License.\n\nfrom __future__ import unicode_literals\nfrom __future__ import print_function\n\nimport re\n\nfrom .lib.pretty_date_time import pretty_date_time\nimport click\n\n\nclass Formatter(object):\n \"\"\"Handle formatting of isssues, repos, threads, etc.\n\n :type config: :class:`config.Config`\n :param config: An instance of config.Config.\n\n :type event_handlers: dict\n :param event_handlers: A mapping of raw event types to format methods.\n\n :type event_type_mapping: dict\n :param event_type_mapping: A mapping of raw event types to more\n human readable text.\n\n :type pretty_dt: :class:`pretty_date_time`\n :param pretty_dt: An instance of pretty_date_time.\n \"\"\"\n\n def __init__(self, config):\n self.config = config\n self.event_type_mapping = {\n 'CommitCommentEvent': 'commented on commit',\n 'CreateEvent': 'created',\n 'DeleteEvent': 'deleted',\n 'FollowEvent': 'followed',\n 'ForkEvent': 'forked',\n 'GistEvent': 'created/updated gist',\n 'GollumEvent': 'created/updated wiki',\n 'IssueCommentEvent': 'commented on',\n 'IssuesEvent': '',\n 'MemberEvent': 'added collaborator',\n 'MembershipEvent': 'added/removed user',\n 'PublicEvent': 'open sourced',\n 'PullRequestEvent': '',\n 'PullRequestReviewCommentEvent': 'commented on pull request',\n 'PushEvent': 'pushed to',\n 'ReleaseEvent': 'released',\n 'RepositoryEvent': 'created repository',\n 'WatchEvent': 'starred',\n }\n self.event_handlers = {\n 'CommitCommentEvent': self._format_commit_comment_event,\n 'CreateEvent': self._format_create_delete_event,\n 'DeleteEvent': self._format_create_delete_event,\n 'FollowEvent': self._format_general_event,\n 'ForkEvent': self._format_fork_event,\n 'ForkApplyEvent': self._format_general_event,\n 'GistEvent': self._format_general_event,\n 'GollumEvent': self._format_general_event,\n 'IssueCommentEvent': self._format_issue_commment_event,\n 'IssuesEvent': self._format_issues_event,\n 'MemberEvent': self._format_general_event,\n 'MembershipEvent': self._format_general_event,\n 'PublicEvent': self._format_general_event,\n 'PullRequestEvent': self._format_pull_request_event,\n 'PullRequestReviewCommentEvent': self._format_commit_comment_event,\n 'PushEvent': self._format_push_event,\n 'ReleaseEvent': self._format_release_event,\n 'StatusEvent': self._format_general_event,\n 'TeamAddEvent': self._format_general_event,\n 'RepositoryEvent': self._format_general_event,\n 'WatchEvent': self._format_general_event,\n }\n self.pretty_dt = pretty_date_time\n\n def _format_time(self, event):\n \"\"\"Format time.\n\n :type event: :class:`github3` Event.\n :param event: An instance of `github3` Event.\n \"\"\"\n item = click.style(\n ' (' + str(self.pretty_dt(event.created_at)) + ')',\n fg=self.config.clr_time)\n return item\n\n def _format_issue_comment(self, event, key):\n \"\"\"Format an issue comment.\n\n :type event: :class:`github3` Event.\n :param event: An instance of `github3` Event.\n \"\"\"\n issue = '{repo[0]}/{repo[1]}#{num}'.format(\n repo=event.payload[key].repository,\n num=event.payload[key].number)\n return click.style(issue, fg=self.config.clr_tertiary)\n\n def _format_indented_message(self, message, newline=True,\n indent=' ', sha=''):\n \"\"\"Format an indented message.\n\n :type message: str\n :param message: The commit comment.\n\n :type newline: bool\n :param newline: Determines whether to prepend a newline.\n\n :type indent: str\n :param indent: The indent, consisting of blank chars.\n TODO: Consider passing an int denoting # blank chars, or try to\n calculate the indent dynamically.\n\n :type sha: str\n :param sha: The commit hash.\n\n :rtype: str\n :return: The formattted commit comment.\n \"\"\"\n subsequent_indent = indent\n if sha != '':\n subsequent_indent += ' '\n message = self.strip_line_breaks(message)\n formatted_message = click.wrap_text(\n text=click.style(sha, fg=self.config.clr_tertiary)+message,\n initial_indent=indent,\n subsequent_indent=subsequent_indent)\n if newline:\n formatted_message = click.style('\\n' + formatted_message)\n return formatted_message\n\n def _format_sha(self, sha):\n \"\"\"Format commit hash.\n\n :type sha: str\n :param sha: The commit hash.\n \"\"\"\n return sha[:7]\n\n def _format_commit_comment_event(self, event):\n \"\"\"Format commit comment and commit hash.\n\n :type event: :class:`github3` Event.\n :param event: An instance of `github3` Event.\n \"\"\"\n item = click.style(self.event_type_mapping[event.type] + ' ',\n fg=self.config.clr_secondary)\n item += click.style(\n self._format_sha(event.payload['comment'].commit_id),\n fg=self.config.clr_tertiary)\n item += click.style(' at ', fg=self.config.clr_secondary)\n item += click.style(self.format_user_repo(event.repo),\n fg=self.config.clr_tertiary)\n try:\n item += click.style(\n '#' + str(event.payload['pull_request'].number) + ' ',\n fg=self.config.clr_tertiary)\n except KeyError:\n pass\n item += self._format_time(event)\n try:\n item += self._format_indented_message(\n event.payload['pull_request'].title)\n item += self._format_indented_message(\n event.payload['comment'].body, indent=' ')\n except KeyError:\n item += self._format_indented_message(\n event.payload['comment'].body)\n return item\n\n def _format_create_delete_event(self, event):\n \"\"\"Format a create or delete event.\n\n :type event: :class:`github3` Event.\n :param event: An instance of `github3` Event.\n \"\"\"\n item = click.style(self.event_type_mapping[event.type],\n fg=self.config.clr_secondary)\n item += click.style(' ' + event.payload['ref_type'],\n fg=self.config.clr_secondary)\n if event.payload['ref']:\n item += click.style(' ' + event.payload['ref'],\n fg=self.config.clr_tertiary)\n item += click.style(' at ', fg=self.config.clr_secondary)\n item += click.style(self.format_user_repo(event.repo),\n fg=self.config.clr_tertiary)\n item += self._format_time(event)\n try:\n item += self._format_indented_message(\n ('' if event.payload['description'] is None\n else event.payload['description']))\n except KeyError:\n pass\n return item\n\n def _format_fork_event(self, event):\n \"\"\"Format a repo fork event.\n\n :type event: :class:`github3` Event.\n :param event: An instance of `github3` Event.\n \"\"\"\n item = click.style(self.event_type_mapping[event.type],\n fg=self.config.clr_secondary)\n item += click.style(' ' + self.format_user_repo(event.repo),\n fg=self.config.clr_tertiary)\n item += self._format_time(event)\n return item\n\n def _format_issue_commment_event(self, event):\n \"\"\"Format a repo fork event.\n\n :type event: :class:`github3` Event.\n :param event: An instance of `github3` Event.\n \"\"\"\n item = click.style(self.event_type_mapping[event.type] + ' ',\n fg=self.config.clr_secondary)\n item += self._format_issue_comment(event, key='issue')\n item += self._format_time(event)\n item += self._format_indented_message(\n event.payload['issue'].title)\n item += self._format_indented_message(\n event.payload['comment'].body, indent=' ')\n return item\n\n def _format_issues_event(self, event):\n \"\"\"Format an issue event.\n\n :type event: :class:`github3` Event.\n :param event: An instance of `github3` Event.\n \"\"\"\n item = click.style(event.payload['action'] + ' issue ',\n fg=self.config.clr_secondary)\n item += self._format_issue_comment(event, key='issue')\n item += self._format_time(event)\n item += self._format_indented_message(\n event.payload['issue'].title)\n return item\n\n def _format_pull_request_event(self, event):\n \"\"\"Format a pull request event.\n\n :type event: :class:`github3` Event.\n :param event: An instance of `github3` Event.\n \"\"\"\n item = click.style(event.payload['action'] + ' pull request ',\n fg=self.config.clr_secondary)\n item += self._format_issue_comment(event, key='pull_request')\n item += self._format_time(event)\n item += self._format_indented_message(\n event.payload['pull_request'].title)\n return item\n\n def _format_push_event(self, event):\n \"\"\"Format a push event.\n\n :type event: :class:`github3` Event.\n :param event: An instance of `github3` Event.\n \"\"\"\n item = click.style(self.event_type_mapping[event.type],\n fg=self.config.clr_secondary)\n branch = event.payload['ref'].split('/')[-1]\n item += click.style(' ' + branch, fg=self.config.clr_tertiary)\n item += click.style(' at ', fg=self.config.clr_secondary)\n item += click.style(self.format_user_repo(event.repo),\n fg=self.config.clr_tertiary)\n item += self._format_time(event)\n for commit in event.payload['commits']:\n sha = click.style(self._format_sha(commit['sha']) + ': ',\n fg=self.config.clr_message)\n item += self._format_indented_message(\n commit['message'], sha=sha)\n return item\n\n def _format_release_event(self, event):\n \"\"\"Format a release event.\n\n :type event: :class:`github3` Event.\n :param event: An instance of `github3` Event.\n \"\"\"\n item = click.style(self.event_type_mapping[event.type] + ' ',\n fg=self.config.clr_secondary)\n item += click.style(event.payload['release'].tag_name + ' ',\n fg=self.config.clr_tertiary)\n item += click.style('at ', fg=self.config.clr_secondary)\n item += click.style(self.format_user_repo(event.repo),\n fg=self.config.clr_tertiary)\n item += self._format_time(event)\n return item\n\n def _format_general_event(self, event):\n \"\"\"Format an event, general case used by various event types.\n\n :type event: :class:`github3` Event.\n :param event: An instance of `github3` Event.\n \"\"\"\n item = click.style(self.event_type_mapping[event.type] + ' ',\n fg=self.config.clr_secondary)\n item += click.style(self.format_user_repo(event.repo),\n fg=self.config.clr_tertiary)\n item += self._format_time(event)\n return item\n\n def format_email(self, view_entry):\n \"\"\"Format an email.\n\n :type view_entry: :class:`github3` Email\n :param view_entry: An instance of `github3` Email.\n\n :rtype: str\n :return: The formattted email.\n \"\"\"\n email = view_entry.item\n item = self.format_index_title(view_entry.index, email.email)\n item += '\\n'\n item += click.style((' ' + 'Primary: ' +\n str(email.primary).ljust(7) + ' '),\n fg=self.config.clr_secondary)\n item += click.style(('Verified: ' +\n str(email.verified).ljust(5) + ' '),\n fg=self.config.clr_tertiary)\n return item\n\n def format_emoji(self, view_entry):\n \"\"\"Format an emoji.\n\n :type view_entry: str\n :param view_entry: The emoji name.\n\n :rtype: str\n :return: The formattted emoji.\n \"\"\"\n emoji = view_entry.item\n item = self.format_index_title(view_entry.index, emoji)\n return item\n\n def format_event(self, view_entry):\n \"\"\"Format an event.\n\n :type view_entry: :class:`github3` Event\n :param view_entry: An instance of `github3` Event.\n\n :rtype: str\n :return: The formattted event.\n \"\"\"\n event = view_entry.item\n item = self.format_index_title(view_entry.index, str(event.actor))\n item += self.event_handlers[event.type](event)\n return item\n\n def format_gitignore_template_name(self, view_entry):\n \"\"\"Format a gitignore template name.\n\n :type view_entry: str\n :param view_entry: The gitignore template name.\n\n :rtype: str\n :return: The formattted gitignore template name.\n \"\"\"\n gitignore_template_name = view_entry.item\n item = self.format_index_title(view_entry.index,\n gitignore_template_name)\n return item\n\n def format_feed_entry(self, view_entry):\n \"\"\"Format a feed entry.\n\n :type view_entry: dict\n :param view_entry: The URITemplates feed.\n\n :rtype: str\n :return: The formattted feed entry.\n \"\"\"\n feed_entry = view_entry.item\n item_parts = feed_entry.title.split(' ')\n title = item_parts[0]\n action = item_parts[1:-1]\n repo = item_parts[-1]\n item = self.format_index_title(view_entry.index, title)\n if action[0] == 'forked':\n item += click.style(action[0] + ' ', fg=self.config.clr_secondary)\n item += click.style(action[1] + ' ', fg=self.config.clr_tertiary)\n else:\n item += click.style(' '.join(action), fg=self.config.clr_secondary)\n item += click.style(' ' + repo + ' ', fg=self.config.clr_tertiary)\n item += click.style(\n '(' + str(self.pretty_dt(feed_entry.updated_parsed)) + ')',\n fg=self.config.clr_time)\n if action[0] == 'commented':\n comment_parts = feed_entry['summary'].split('blockquote')\n if len(comment_parts) > 2:\n comment = comment_parts[-2]\n parts_mention = comment.split('class=\"user-mention\">')\n if len(parts_mention) > 1:\n comment = parts_mention[1]\n comment = self._format_indented_message(comment, newline=False)\n comment = re.sub(r'(*)', r'', comment)\n comment = re.sub(r'(

*)', r'', comment)\n comment = re.sub(r'(

*)', r'', comment)\n comment = re.sub(r'( *)', r'', comment)\n item += click.style('\\n' + comment, fg=self.config.clr_message)\n return item\n\n def format_license_name(self, view_entry):\n \"\"\"Format a license template name.\n\n :type view_entry: :class:`github3` License\n :param view_entry: An instance of `github3` License.\n\n :rtype: str\n :return: The formattted license template name.\n \"\"\"\n license_template_name = view_entry.item\n item = self.format_index_title(view_entry.index,\n license_template_name.key)\n item += click.style('(' + license_template_name.name + ')',\n fg=self.config.clr_secondary)\n return item\n\n def format_user(self, view_entry):\n \"\"\"Format a user.\n\n :type view_entry: :class:`github3` User\n :param view_entry: An instance of `github3` User.\n\n :rtype: str\n :return: The formattted user.\n \"\"\"\n user = view_entry.item\n item = self.format_index_title(view_entry.index, user.login)\n return item\n\n def format_issues_url_from_issue(self, issue):\n \"\"\"Format the issue url based on the given issue.\n\n :type issue: :class:`github3` Issue\n :param issue: An instance of `github3` Issue.\n\n :rtype: str\n :return: The formattted issues url.\n \"\"\"\n return self.format_user_repo(issue.repository) + '/' + \\\n 'issues/' + str(issue.number)\n\n def format_issues_url_from_thread(self, thread):\n \"\"\"Format the issue url based on the given thread.\n\n :type issue: :class:`github3` Thread\n :param issue: An instance of `github3` Thread.\n\n :rtype: str\n :return: The formattted issues url.\n \"\"\"\n url_parts = thread.subject['url'].split('/')\n user = url_parts[4]\n repo = url_parts[5]\n issues_uri = 'issues'\n issue_id = url_parts[7]\n return '/'.join([user, repo, issues_uri, issue_id])\n\n def format_index_title(self, index, title):\n \"\"\"Format an item's index and title.\n\n :type index: str\n :param index: The index for the given item.\n\n :type title: str\n :param title: The item's title.\n\n :rtype: str\n :return: The formatted index and title.\n \"\"\"\n formatted_index_title = click.style(' ' + (str(index) + '.').ljust(5),\n fg=self.config.clr_view_index)\n formatted_index_title += click.style(title + ' ',\n fg=self.config.clr_primary)\n return formatted_index_title\n\n def format_issue(self, view_entry):\n \"\"\"Format an issue.\n\n :type view_entry: :class:`github3` Issue\n :param view_entry: An instance of `github3` Issue.\n\n :rtype: str\n :return: The formatted issue.\n \"\"\"\n issue = view_entry.item\n item = self.format_index_title(view_entry.index, issue.title)\n item += click.style('@' + str(issue.user) + ' ',\n fg=self.config.clr_primary)\n item += click.style(('(' +\n self.format_issues_url_from_issue(issue) +\n ')'),\n fg=self.config.clr_view_link)\n item += '\\n'\n indent = ' '\n if len(item) == 8:\n item += click.style((' Score: ' +\n str(item[7]).ljust(10) + ' '),\n fg=self.config.clr_quaternary)\n indent = ' '\n item += click.style((indent + 'State: ' +\n str(issue.state).ljust(10) + ' '),\n fg=self.config.clr_secondary)\n item += click.style(('Comments: ' +\n str(issue.comments_count).ljust(5) + ' '),\n fg=self.config.clr_tertiary)\n item += click.style(('Assignee: ' +\n str(issue.assignee).ljust(10) + ' '),\n fg=self.config.clr_quaternary)\n return item\n\n def format_repo(self, view_entry):\n \"\"\"Format a repo.\n\n :type view_entry: :class:`github3` Repository\n :param view_entry: An instance of `github3` Repository.\n\n :rtype: str\n :return: The formatted repo.\n \"\"\"\n repo = view_entry.item\n item = self.format_index_title(view_entry.index, repo.full_name)\n language = repo.language if repo.language is not None else 'Unknown'\n item += click.style('(' + language + ')',\n fg=self.config.clr_message)\n item += '\\n'\n item += click.style((' ' + 'Stars: ' +\n str(repo.stargazers_count).ljust(6) + ' '),\n fg=self.config.clr_secondary)\n item += click.style('Forks: ' + str(repo.forks_count).ljust(6) + ' ',\n fg=self.config.clr_tertiary)\n item += click.style(('Updated: ' +\n str(self.pretty_dt(repo.updated_at)) + ' '),\n fg=self.config.clr_time)\n return item\n\n def format_thread(self, view_entry):\n \"\"\"Format a thread.\n\n :type view_entry: :class:`github3` Thread\n :param view_entry: An instance of `github3` Thread.\n\n :rtype: str\n :return: The formatted thread.\n \"\"\"\n thread = view_entry.item\n item = self.format_index_title(view_entry.index,\n thread.subject['title'])\n item += click.style('(' + view_entry.url + ')',\n fg=self.config.clr_view_link)\n item += '\\n'\n item += click.style((' ' + 'Seen: ' +\n str(not thread.unread).ljust(7) + ' '),\n fg=self.config.clr_secondary)\n item += click.style(('Type: ' +\n str(thread.subject['type']).ljust(12) + ' '),\n fg=self.config.clr_tertiary)\n item += click.style(('Updated: ' +\n str(self.pretty_dt(thread.updated_at)) + ' '),\n fg=self.config.clr_time)\n return item\n\n def format_trending_entry(self, view_entry):\n \"\"\"Formats a trending repo entry.\n\n :type view_entry: dict\n :param view_entry: The URITemplates feed.\n\n :rtype: str\n :return: The formattted trending entry.\n \"\"\"\n trending_entry = view_entry.item\n item_parts = trending_entry.title.split(' ')\n title = item_parts[0]\n item = self.format_index_title(view_entry.index, title)\n\n try:\n summary_parts = trending_entry.summary.split('\\n')\n summary = summary_parts[0] if len(summary_parts) > 1 else ''\n summary = self.strip_line_breaks(summary)\n language = summary_parts[-1]\n except AttributeError:\n summary = ''\n language = ''\n\n if language == '()':\n language = '(Unknown)'\n language = re.sub(r'(\\()', r'', language)\n language = re.sub(r'(\\))', r'', language)\n if summary:\n item += '\\n'\n summary = click.wrap_text(\n text=summary,\n initial_indent=' ',\n subsequent_indent=' ')\n item += click.style(summary, self.config.clr_message)\n item += '\\n'\n item += click.style(' ' + language,\n fg=self.config.clr_message)\n return item\n\n def format_user_repo(self, user_repo_tuple):\n \"\"\"Format a repo tuple for pretty print.\n\n Example:\n Input: ('donnemartin', 'gitsome')\n Output: donnemartin/gitsome\n Input: ('repos/donnemartin', 'gitsome')\n Output: donnemartin/gitsome\n\n :type user_repo_tuple: tuple\n :param user_repo_tuple: The user and repo.\n\n :rtype: str\n :return: A string of the form user/repo.\n \"\"\"\n result = '/'.join(user_repo_tuple)\n if result.startswith('repos/'):\n return result[len('repos/'):]\n return result\n\n def strip_line_breaks(self, text):\n \"\"\"Strips \\r and \\n characters.\n\n These characters seem to cause issues with `click.wrap_text`.\n\n :type text: str\n :param text: The text to strip of line breaks.\n\n :rtype: str\n :return: The input text without line breaks.\n \"\"\"\n text = re.sub(r'\\r', '', text)\n text = re.sub(r'\\n', ' ', text)\n return text\n" }, { "path": "gitsome/github.py", "content": "# -*- coding: utf-8 -*-\n\n# Copyright 2015 Donne Martin. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"). You\n# may not use this file except in compliance with the License. A copy of\n# the License is located at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# or in the \"license\" file accompanying this file. This file is\n# distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF\n# ANY KIND, either express or implied. See the License for the specific\n# language governing permissions and limitations under the License.\n\nfrom __future__ import unicode_literals\nfrom __future__ import print_function\n\nimport os\nimport platform\nimport sys\nimport urllib\nimport webbrowser\n\nfrom .lib.github3 import null\nfrom .lib.github3.exceptions import AuthenticationFailed, UnprocessableEntity\nfrom .lib.img2txt import img2txt\nimport click\nimport feedparser\nfrom requests.exceptions import MissingSchema, SSLError\n\nfrom .config import Config\nfrom .formatter import Formatter\nfrom .rss_feed import language_rss_map\nfrom .table import Table\nfrom .view_entry import ViewEntry\nfrom .web_viewer import WebViewer\nfrom .utils import TextUtils\n\n\nclass GitHub(object):\n \"\"\"Provide integration with the GitHub API.\n\n :type config: :class:`config.Config`\n :param config: An instance of `config.Config`.\n\n :type formatter: :class:`formatter.Formatter`\n :param formatter: An instance of `formatter.Formatter`.\n\n :type img2txt: callable\n :param img2txt: A callable fom img2txt.\n\n :type table: :class:`table.Table`\n :param table: An instance of `table.Table`.\n\n :type trend_parser: :class:`feedparser`\n :param trend_parser: An instance of `feedparser`.\n\n :type web_viewer: :class:`web_viewer.WebViewer`\n :param web_viewer: An instance of `web_viewer.WebViewer`.\n\n :type _base_url: str\n :param _base_url: The base GitHub or GitHub Enterprise url.\n \"\"\"\n\n def __init__(self):\n self.config = Config()\n self.formatter = Formatter(self.config)\n self.img2txt = img2txt.img2txt\n self.table = Table(self.config)\n self.web_viewer = WebViewer(self.config)\n self.trend_parser = feedparser\n self.text_utils = TextUtils()\n self._base_url = 'https://github.com/'\n\n @property\n def base_url(self):\n return self.config.enterprise_url or self._base_url\n\n def add_base_url(self, url):\n \"\"\"Add the base url if it is not already part of the given url.\n\n :type url: str\n :param url: The url.\n\n :return: The url including the base url.\n \"\"\"\n return self.base_url + url if self.base_url not in url else url\n\n def authenticate(func):\n \"\"\"Decorator that authenticates credentials.\n\n :type func: callable\n :param func: A method to execute if authorization passes.\n\n :return: The return value of `func` if authorization passes, or\n None if authorization fails.\n \"\"\"\n def auth_wrapper(self, *args, **kwargs):\n self.config.authenticate()\n self.config.save_config()\n if self.config.check_auth():\n try:\n return func(self, *args, **kwargs)\n except SSLError:\n click.secho(('SSL cert verification failed.\\n Try running '\n 'gh configure --enterprise\\n and type '\n \"'n' when asked whether to verify SSL certs.\"),\n fg=self.config.clr_error)\n except MissingSchema:\n click.secho('Invalid GitHub Enterprise url',\n fg=self.config.clr_error)\n except AuthenticationFailed:\n self.config.print_auth_error()\n return auth_wrapper\n\n def avatar(self, url, text_avatar):\n \"\"\"Display the user's avatar from the specified url.\n\n This method requires PIL.\n\n :type url: str\n :param url: The user's avatar image.\n\n :type text_avatar: bool\n :param text_avatar: Determines whether to view the profile avatar\n in plain text (True) or in ansi (False).\n On Windows this value is always set to True due to lack of\n support of `img2txt` on Windows.\n\n :rtype: str\n :return: The avatar.\n \"\"\"\n if platform.system() == 'Windows':\n text_avatar = True\n avatar_enabled = self.config.enable_avatar\n avatar_text = ''\n if avatar_enabled:\n avatar = self.config.get_github_config_path(\n self.config.CONFIG_AVATAR)\n try:\n urllib.request.urlretrieve(url, avatar)\n except urllib.error.URLError:\n pass\n if os.path.exists(avatar):\n avatar_text = self.img2txt(avatar, ansi=(not text_avatar))\n avatar_text += '\\n'\n os.remove(avatar)\n return avatar_text\n\n def avatar_setup(self, url, text_avatar):\n \"\"\"Prepare to display the user's avatar from the specified url.\n\n This method requires PIL.\n\n :type url: str\n :param url: The user's avatar image.\n\n :type text_avatar: bool\n :param text_avatar: Determines whether to view the profile avatar\n in plain text (True) or in ansi (False).\n On Windows this value is always set to True due to lack of\n support of `img2txt` on Windows.\n\n :rtype: str\n :return: The avatar.\n \"\"\"\n try:\n import PIL # NOQA\n return self.avatar(url, text_avatar)\n except ImportError:\n avatar_text = click.style(('To view the avatar in your terminal, '\n 'install the Python Image Library.\\n'),\n fg=self.config.clr_message)\n return avatar_text\n\n def configure(self, enterprise):\n \"\"\"Configure gitsome.\n\n Attempts to authenticate the user and to set up the user's news feed.\n\n If `gitsome` has not yet been configured, calling a `gh` command that\n requires authentication will automatically invoke the `configure`\n command.\n\n :type enterprise: bool\n :param enterprise: Determines whether to configure GitHub Enterprise.\n \"\"\"\n self.config.authenticate(enterprise=enterprise, overwrite=True)\n self.config.prompt_news_feed()\n self.config.show_bash_completions_info()\n self.config.save_config()\n\n @authenticate\n def create_comment(self, user_repo_number, text):\n \"\"\"Create a comment on the given issue.\n\n :type user_repo_number: str\n :param user_repo_number: The user/repo/issue_number.\n\n :type text: str\n :param text: The comment text.\n \"\"\"\n try:\n user, repo, number = user_repo_number.split('/')\n int(number) # Check for int\n except ValueError:\n click.secho(('Expected argument: user/repo/# and option -t '\n '\"comment\".'),\n fg=self.config.clr_error)\n return\n issue = self.config.api.issue(user, repo, number)\n issue_comment = issue.create_comment(text)\n if type(issue_comment) is not null.NullObject:\n click.secho('Created comment: ' + issue_comment.body,\n fg=self.config.clr_message)\n else:\n click.secho('Error creating comment',\n fg=self.config.clr_error)\n\n @authenticate\n def create_issue(self, user_repo, issue_title, issue_desc=''):\n \"\"\"Create an issue.\n\n :type user_repo: str\n :param user_repo: The user/repo.\n\n :type issue_title: str\n :param issue_title: The issue title.\n\n :type issue_desc: str\n :param issue_desc: The issue body (optional).\n \"\"\"\n try:\n user, repo_name = user_repo.split('/')\n except ValueError:\n click.secho('Expected argument: user/repo and option -t \"title\".',\n fg=self.config.clr_error)\n return\n issue = self.config.api.create_issue(user,\n repo_name,\n issue_title,\n issue_desc)\n if type(issue) is not null.NullObject:\n body = self.text_utils.sanitize_if_none(issue.body)\n click.secho('Created issue: ' + issue.title + '\\n' + body,\n fg=self.config.clr_message)\n else:\n click.secho('Error creating issue.', fg=self.config.clr_error)\n\n @authenticate\n def create_repo(self, repo_name, repo_desc='', private=False):\n \"\"\"Create a repo.\n\n :type repo_name: str\n :param repo_name: The repo name.\n\n :type repo_desc: str\n :param repo_desc: The repo description (optional).\n\n :type private: bool\n :param private: Determines whether the repo is private. Default: False.\n \"\"\"\n try:\n repo = self.config.api.create_repository(repo_name,\n repo_desc,\n private=private)\n desc = self.text_utils.sanitize_if_none(repo.description)\n click.secho(('Created repo: ' + repo.full_name + '\\n' + desc),\n fg=self.config.clr_message)\n except UnprocessableEntity as e:\n click.secho('Error creating repo: ' + str(e.msg),\n fg=self.config.clr_error)\n\n @authenticate\n def emails(self):\n \"\"\"List all the user's registered emails.\"\"\"\n self.table.build_table_setup(self.config.api.emails(),\n self.formatter.format_email,\n limit=sys.maxsize,\n pager=False,\n build_urls=False)\n\n @authenticate\n def emojis(self, pager=False):\n \"\"\"List all GitHub supported emojis.\n\n :type pager: bool\n :param pager: Determines whether to show the output in a pager,\n if available.\n \"\"\"\n self.table.build_table_setup(self.config.api.emojis(),\n self.formatter.format_emoji,\n limit=sys.maxsize,\n pager=pager,\n build_urls=False)\n\n @authenticate\n def feed(self, user_or_repo='', private=False, pager=False):\n \"\"\"List all activity for the given user or repo.\n\n If `user_or_repo` is not provided, uses the logged in user's news feed\n seen while visiting https://github.com. If `user_or_repo` is provided,\n shows either the public or `[-p/--private]` feed activity of the user\n or repo.\n\n :type user_or_repo: str\n :param user_or_repo: The user or repo to list events for (optional).\n If no entry, defaults to the logged in user's feed.\n\n :type private: bool\n :param private: Determines whether to show the private events (True)\n or public events (False).\n\n :type pager: bool\n :param pager: Determines whether to show the output in a pager,\n if available.\n \"\"\"\n click.secho('Listing events...', fg=self.config.clr_message)\n if user_or_repo == '':\n if self.config.user_feed is None:\n self.config.prompt_news_feed()\n self.config.save_config()\n if self.config.user_feed:\n items = self.trend_parser.parse(self.config.user_feed)\n self.table.build_table_setup_feed(\n items,\n self.formatter.format_feed_entry,\n pager)\n else:\n if '/' in user_or_repo:\n user, repo = user_or_repo.split('/')\n repo = self.config.api.repository(user, repo)\n items = repo.events()\n else:\n public = False if private else True\n items = self.config.api.user(user_or_repo).events(public=public)\n self.table.build_table_setup(\n items,\n self.formatter.format_event,\n limit=sys.maxsize,\n pager=pager,\n build_urls=False)\n\n @authenticate\n def followers(self, user, pager=False):\n \"\"\"List all followers and the total follower count.\n\n :type user: str\n :param user: The user login (optional).\n If None, returns the followers of the logged in user.\n\n :type pager: bool\n :param pager: Determines whether to show the output in a pager,\n if available.\n \"\"\"\n if user is None:\n user = self.config.user_login\n self.table.build_table_setup_user(\n self.config.api.followers_of(user),\n self.formatter.format_user,\n limit=sys.maxsize,\n pager=pager)\n\n @authenticate\n def following(self, user, pager=False):\n \"\"\"List all followed users and the total followed count.\n\n :type user: str\n :param user: The user login.\n If None, returns the followed users of the logged in user.\n\n :type pager: bool\n :param pager: Determines whether to show the output in a pager,\n if available.\n \"\"\"\n if user is None:\n user = self.config.user_login\n self.table.build_table_setup_user(\n self.config.api.followed_by(user),\n self.formatter.format_user,\n limit=sys.maxsize,\n pager=pager)\n\n @authenticate\n def gitignore_template(self, language):\n \"\"\"Output the gitignore template for the given language.\n\n :type language: str\n :param language: The language.\n \"\"\"\n template = self.config.api.gitignore_template(language)\n if template:\n click.secho(template, fg=self.config.clr_message)\n else:\n click.secho(('Invalid case-sensitive template requested, run the '\n 'following command to see available templates:\\n'\n ' gh gitignore-templates'),\n fg=self.config.clr_error)\n\n @authenticate\n def gitignore_templates(self, pager=False):\n \"\"\"Output all supported gitignore templates.\n\n :type pager: bool\n :param pager: Determines whether to show the output in a pager,\n if available.\n \"\"\"\n self.table.build_table_setup(\n self.config.api.gitignore_templates(),\n self.formatter.format_gitignore_template_name,\n limit=sys.maxsize,\n pager=pager,\n build_urls=False)\n click.secho((' Run the following command to view or download a '\n '.gitignore file:\\n'\n ' View: gh gitignore Python\\n'\n ' Download: gh gitignore Python > .gitignore\\n'),\n fg=self.config.clr_message)\n\n @authenticate\n def issue(self, user_repo_number):\n \"\"\"Output detailed information about the given issue.\n\n :type user_repo_number: str\n :param user_repo_number: The user/repo/issue_number.\n \"\"\"\n try:\n user, repo, number = user_repo_number.split('/')\n int(number) # Check for int\n except ValueError:\n click.secho('Expected argument: user/repo/#.',\n fg=self.config.clr_error)\n return\n url = (self.base_url + user + '/' + repo + '/' +\n 'issues/' + number)\n self.web_viewer.view_url(url)\n\n @authenticate\n def issues(self, issues_list, limit=1000, pager=False, sort=True):\n \"\"\"List all issues.\n\n :type issues_list: list\n :param issues_list: A list of `github3` Issues.\n\n :type limit: int\n :param limit: The number of items to display.\n\n :type pager: bool\n :param pager: Determines whether to show the output in a pager,\n if available.\n\n :type sort: bool\n :param sort: Determines whether to sort the issues by:\n state, repo, created_at.\n \"\"\"\n view_entries = []\n for current_issue in issues_list:\n url = self.formatter.format_issues_url_from_issue(current_issue)\n view_entries.append(\n ViewEntry(\n current_issue,\n url=url,\n sort_key_primary=current_issue.state,\n sort_key_secondary=current_issue.repository,\n sort_key_tertiary=current_issue.created_at))\n if sort:\n view_entries = sorted(view_entries, reverse=False)\n self.table.build_table(view_entries,\n limit,\n pager,\n self.formatter.format_issue)\n\n @authenticate\n def issues_setup(self, issue_filter='subscribed', issue_state='open',\n limit=1000, pager=False):\n \"\"\"Prepare to list all issues matching the filter.\n\n :type issue_filter: str\n :param issue_filter: 'assigned', 'created', 'mentioned',\n 'subscribed' (default).\n\n :type issue_state: str\n :param issue_state: 'all', 'open' (default), 'closed'.\n\n :type limit: int\n :param limit: The number of items to display.\n\n :type pager: bool\n :param pager: Determines whether to show the output in a pager,\n if available.\n \"\"\"\n self.issues(self.config.api.issues(issue_filter, issue_state),\n limit,\n pager)\n\n @authenticate\n def license(self, license_name):\n \"\"\"Output the gitignore template for the given language.\n\n :type license_name: str\n :param license_name: The license name.\n \"\"\"\n result = self.config.api.license(license_name)\n if type(result) is not null.NullObject:\n click.secho(result.body, fg=self.config.clr_message)\n else:\n click.secho((' Invalid case-sensitive license requested, run the '\n 'following command to see available licenses:\\n'\n ' gh licenses'),\n fg=self.config.clr_error)\n\n @authenticate\n def licenses(self):\n \"\"\"Output the gitignore template for the given language.\"\"\"\n self.table.build_table_setup(\n self.config.api.licenses(),\n self.formatter.format_license_name,\n limit=sys.maxsize,\n pager=False,\n build_urls=False)\n click.secho((' Run the following command to view or download a '\n 'LICENSE file:\\n'\n ' gh license apache-2.0\\n'\n ' gh license apache-2.0 > LICENSE\\n'),\n fg=self.config.clr_message)\n\n @authenticate\n def notifications(self, limit=1000, pager=False):\n \"\"\"List all notifications.\n\n :type limit: int\n :param limit: The number of items to display.\n\n :type pager: bool\n :param pager: Determines whether to show the output in a pager,\n if available.\n \"\"\"\n view_entries = []\n for thread in self.config.api.notifications(all=True,\n participating=False):\n url = self.formatter.format_issues_url_from_thread(thread)\n view_entries.append(ViewEntry(thread, url=url))\n self.table.build_table(view_entries,\n limit,\n pager,\n self.formatter.format_thread)\n\n @authenticate\n def octocat(self, say):\n \"\"\"Output an Easter egg or the given message from Octocat.\n\n :type say: str\n :param say: What Octocat should say.\n If say is None, octocat speaks an Easter egg.\n \"\"\"\n output = str(self.config.api.octocat(say))\n output = output.replace('\\\\n', '\\n')\n click.secho(output, fg=self.config.clr_message)\n\n @authenticate\n def pull_requests(self, limit=1000, pager=False):\n \"\"\"List all pull requests.\n\n :type limit: int\n :param limit: The number of items to display.\n\n :type pager: bool\n :param pager: Determines whether to show the output in a pager,\n if available.\n \"\"\"\n issues_list = []\n repositories = self.config.api.repositories()\n for repository in repositories:\n repo_pulls = repository.pull_requests()\n for repo_pull in repo_pulls:\n url = self.formatter.format_issues_url_from_issue(repo_pull)\n user, repo, issues, number = url.split('/')\n repo_pull = self.config.api.pull_request(user, repo, number)\n issues_list.append(repo_pull)\n self.issues(issues_list, limit, pager)\n\n @authenticate\n def rate_limit(self):\n \"\"\"Output the rate limit. Not available for GitHub Enterprise.\n\n Logged in users can make 5000 requests per hour.\n See: https://developer.github.com/v3/#rate-limiting\n \"\"\"\n click.secho('Rate limit: ' + str(self.config.api.ratelimit_remaining),\n fg=self.config.clr_message)\n\n @authenticate\n def repositories(self, repos, limit=1000, pager=False,\n repo_filter='', print_output=True, sort=True):\n \"\"\"List all repos matching the given filter.\n\n :type repos: list\n :param repos: A list of `github3` Repository.\n\n :type limit: int\n :param limit: The number of items to display.\n\n :type pager: bool\n :param pager: Determines whether to show the output in a pager,\n if available.\n\n :type repo_filter: str\n :param repo_filter: The filter for repo names.\n Only repos matching the filter will be returned.\n If None, outputs all repos.\n\n :type print_output: bool\n :param print_output: Determines whether to print the output\n (True) or return the output as a string (False).\n\n :type sort: bool\n :param sort: Determines whether to sort the issues by:\n state, repo, created_at.\n\n :rtype: str\n :return: The output if print_output is True\n else, returns None.\n \"\"\"\n view_entries = []\n for repo in repos:\n url = repo.clone_url\n if (repo.full_name is not None and\n repo_filter in repo.full_name.lower()) or \\\n (repo.description is not None and\n repo_filter in repo.description.lower()):\n view_entries.append(\n ViewEntry(repo,\n url=url,\n sort_key_primary=repo.stargazers_count))\n if sort:\n view_entries = sorted(view_entries, reverse=True)\n return self.table.build_table(view_entries,\n limit,\n pager,\n self.formatter.format_repo,\n print_output=print_output)\n\n @authenticate\n def repositories_setup(self, repo_filter, limit=1000, pager=False):\n \"\"\"Prepare to list all repos matching the given filter.\n\n :type repo_filter: str\n :param repo_filter: The filter for repo names.\n Only repos matching the filter will be returned.\n If None, outputs all repos retrieved by the GitHub API.\n\n :type limit: int\n :param limit: The number of items to display.\n\n :type pager: bool\n :param pager: Determines whether to show the output in a pager,\n if available.\n \"\"\"\n self.repositories(self.config.api.repositories(),\n limit,\n pager,\n repo_filter)\n\n @authenticate\n def repository(self, user_repo):\n \"\"\"Output detailed information about the given repo.\n\n :type user_repo: str\n :param user_repo: The user/repo.\n \"\"\"\n try:\n user, repo = user_repo.split('/')\n except ValueError:\n click.secho('Expected argument: user/repo.',\n fg=self.config.clr_error)\n return\n self.web_viewer.view_url(self.base_url + user_repo)\n\n @authenticate\n def search_issues(self, query, limit=1000, pager=False):\n \"\"\"Search for all issues matching the given query.\n\n :type query: str\n :param query: The search query.\n\n :type limit: int\n :param limit: The number of items to display.\n\n :type pager: bool\n :param pager: Determines whether to show the output in a pager,\n if available.\n \"\"\"\n click.secho('Searching for all matching issues on GitHub...',\n fg=self.config.clr_message)\n results = self.config.api.search_issues(query)\n issues_list = []\n for result in results:\n issues_list.append(result.issue)\n self.issues(issues_list, limit, pager, sort=False)\n\n @authenticate\n def search_repositories(self, query, sort, limit=1000, pager=False):\n \"\"\"Search for all repos matching the given query.\n\n :type query: str\n :param query: The search query.\n\n :type sort: str\n :param sort: Optional: 'stars', 'forks', 'updated'.\n If not specified, sorting is done by query best match.\n\n :type limit: int\n :param limit: The number of items to display.\n\n :type pager: bool\n :param pager: Determines whether to show the output in a pager,\n if available.\n \"\"\"\n click.secho('Searching for all matching repos on GitHub...',\n fg=self.config.clr_message)\n results = self.config.api.search_repositories(query, sort)\n repos = []\n for result in results:\n repos.append(result.repository)\n self.repositories(repos, limit, pager, sort=False)\n\n @authenticate\n def starred(self, repo_filter, limit=1000, pager=False):\n \"\"\"Output starred repos.\n\n :type repo_filter: str\n :param repo_filter: The filter for repo names.\n Only repos matching the filter will be returned.\n If None, outputs all starred repos.\n\n :type limit: int\n :param limit: The number of items to display.\n\n :type pager: bool\n :param pager: Determines whether to show the output in a pager,\n if available.\n \"\"\"\n self.repositories(self.config.api.starred(),\n limit,\n pager,\n repo_filter.lower())\n\n def trending(self, language, weekly, monthly,\n devs=False, browser=False, pager=False):\n \"\"\"List trending repos for the given language.\n\n :type language: str\n :param language: The language (optional).\n If blank, shows 'Overall'.\n\n :type weekly: bool\n :param weekly: Determines whether to show the weekly rankings.\n Daily is the default.\n\n :type monthly: bool\n :param monthly: Determines whether to show the monthly rankings.\n Daily is the default.\n If both `monthly` and `weekly` are set, `monthly` takes precedence.\n\n :type devs: bool\n :param devs: determines whether to display the trending\n devs or repos. Only valid with the -b/--browser option.\n\n :type browser: bool\n :param browser: Determines whether to view the profile\n in a browser, or in the terminal.\n\n :type pager: bool\n :param pager: Determines whether to show the output in a pager,\n if available.\n \"\"\"\n language = language.lower()\n if language in language_rss_map:\n language = language_rss_map[language]\n if monthly:\n period = 'monthly'\n url_param = '?since=monthly'\n elif weekly:\n period = 'weekly'\n url_param = '?since=weekly'\n else:\n period = 'daily'\n url_param = ''\n if browser:\n webbrowser.open(\n ('https://github.com/trending' +\n ('/developers' if devs else '') +\n ('/' + language if language is not 'overall' else '') +\n url_param))\n else:\n click.secho(\n 'Listing {p} trending {l} repos...'.format(l=language,\n p=period),\n fg=self.config.clr_message)\n url = ('http://github-trends.ryotarai.info/rss/github_trends_' +\n language + '_')\n url += period + '.rss'\n items = self.trend_parser.parse(url)\n self.table.build_table_setup_trending(\n items.entries,\n self.formatter.format_trending_entry,\n limit=sys.maxsize,\n pager=pager)\n\n @authenticate\n def user(self, user_id, browser=False, text_avatar=False,\n limit=1000, pager=False):\n \"\"\"List information about the logged in user.\n\n :type user_id: str\n :param user_id: The user id/login.\n If None, returns followers of the logged in user.\n\n :type browser: bool\n :param browser: Determines whether to view the profile\n in a browser, or in the terminal.\n\n :type text_avatar: bool\n :param text_avatar: Determines whether to view the profile\n avatar in plain text instead of ansi (default).\n On Windows this value is always set to True due to lack of\n support of `img2txt` on Windows.\n\n :type limit: int\n :param limit: The number of items to display.\n\n :type pager: bool\n :param pager: Determines whether to show the output in a pager,\n if available.\n \"\"\"\n if browser:\n webbrowser.open(self.base_url + user_id)\n else:\n user = self.config.api.user(user_id)\n if type(user) is null.NullObject:\n click.secho('Invalid user.', fg=self.config.clr_error)\n return\n output = ''\n output += click.style(self.avatar_setup(user.avatar_url,\n text_avatar))\n output += click.style(user.login + '\\n', fg=self.config.clr_primary)\n if user.company is not None:\n output += click.style(user.company + '\\n',\n fg=self.config.clr_secondary)\n if user.location is not None:\n output += click.style(user.location + '\\n',\n fg=self.config.clr_secondary)\n if user.email is not None:\n output += click.style(user.email + '\\n',\n fg=self.config.clr_secondary)\n if user.type == 'Organization':\n output += click.style('Organization\\n\\n',\n fg=self.config.clr_tertiary)\n else:\n output += click.style(\n 'Followers: ' + str(user.followers_count) + ' | ',\n fg=self.config.clr_tertiary)\n output += click.style(\n 'Following: ' + str(user.following_count) + '\\n\\n',\n fg=self.config.clr_tertiary)\n output += self.repositories(self.config.api.repositories(user_id),\n limit,\n pager,\n print_output=False)\n if pager:\n color = None\n if platform.system() == 'Windows':\n color = True\n click.echo_via_pager(output, color)\n else:\n click.secho(output)\n\n @authenticate\n def user_me(self, browser, text_avatar, limit=1000, pager=False):\n \"\"\"List information about the logged in user.\n\n :type browser: bool\n :param browser: Determines whether to view the profile\n in a browser, or in the terminal.\n\n :type text_avatar: bool\n :param text_avatar: Determines whether to view the profile\n avatar in plain text.\n On Windows this value is always set to True due to lack of\n support of `img2txt` on Windows.\n\n :type limit: int\n :param limit: The number of items to display.\n\n :type pager: bool\n :param pager: Determines whether to show the output in a pager,\n if available.\n \"\"\"\n self.user(self.config.user_login, browser, text_avatar, limit, pager)\n\n @authenticate\n def view(self, index, view_in_browser=False):\n \"\"\"View the given index in a browser.\n\n Load urls from ~/.gitsomeconfigurl and stores them in self.config.urls.\n Open a browser with the url based on the given index.\n\n :type index: int\n :param index: Determines the index to view.\n\n :type browser: bool\n :param browser: Determines whether to view the profile\n in a browser, or in the terminal.\n \"\"\"\n self.config.urls = self.config.load_urls(view_in_browser)\n url = self.config.urls[index-1]\n click.secho('Viewing ' + url + '...', fg=self.config.clr_message)\n if view_in_browser:\n webbrowser.open(self.add_base_url(url))\n else:\n if 'issues/' in url:\n url = url.replace('issues/', '')\n self.issue(url)\n elif len(url.split('/')) == 2:\n self.repository(url)\n else:\n self.web_viewer.view_url(self.add_base_url(url))\n" }, { "path": "gitsome/githubcli.py", "content": "# -*- coding: utf-8 -*-\n\n# Copyright 2015 Donne Martin. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"). You\n# may not use this file except in compliance with the License. A copy of\n# the License is located at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# or in the \"license\" file accompanying this file. This file is\n# distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF\n# ANY KIND, either express or implied. See the License for the specific\n# language governing permissions and limitations under the License.\n\nfrom __future__ import unicode_literals\nfrom __future__ import print_function\n\nimport click\n\nfrom .github import GitHub\n\n\nclick.disable_unicode_literals_warning = True\npass_github = click.make_pass_decorator(GitHub)\n\n\nclass GitHubCli(object):\n \"\"\"The GitHubCli, builds `click` commands and runs `GitHub` methods.\"\"\"\n\n @click.group()\n @click.pass_context\n def cli(ctx):\n \"\"\"Main entry point for GitHubCli.\n\n :type ctx: :class:`click.core.Context`\n :param ctx: An instance of click.core.Context that stores an instance\n of `github.GitHub`.\n \"\"\"\n # Create a GitHub object and remember it as the context object.\n # From this point onwards other commands can refer to it by using the\n # @pass_github decorator.\n ctx.obj = GitHub()\n\n @cli.command()\n @click.option('-e', '--enterprise', is_flag=True)\n @pass_github\n def configure(github, enterprise):\n \"\"\"Configure gitsome.\n\n Attempts to authenticate the user and to set up the user's news feed.\n\n Usage/Example(s):\n gh configure\n gh configure -e\n gh configure --enterprise\n\n :type github: :class:`github.GitHub`\n :param github: An instance of `github.GitHub`.\n :type enterprise: bool\n :param enterprise: Determines whether to configure GitHub Enterprise.\n Default: False.\n \"\"\"\n github.configure(enterprise)\n\n @cli.command('create-comment')\n @click.argument('user_repo_number')\n @click.option('-t', '--text')\n @pass_github\n def create_comment(github, user_repo_number, text):\n \"\"\"Create a comment on the given issue.\n\n Usage:\n gh create-comment [user_repo_number] [-t/--text]\n\n Example(s):\n gh create-comment donnemartin/saws/1 -t \"hello world\"\n gh create-comment donnemartin/saws/1 --text \"hello world\"\n\n :type github: :class:`github.GitHub`\n :param github: An instance of `github.GitHub`.\n\n :type user_repo_number: str\n :param user_repo_number: The user/repo/issue_number.\n\n :type text: str\n :param text: The comment text.\n \"\"\"\n github.create_comment(user_repo_number, text)\n\n @cli.command('create-issue')\n @click.argument('user_repo')\n @click.option('-t', '--issue_title')\n @click.option('-d', '--issue_desc', required=False)\n @pass_github\n def create_issue(github, user_repo, issue_title, issue_desc):\n \"\"\"Create an issue.\n\n Usage:\n gh create-issue [user_repo] [-t/--issue_title] [-d/--issue_desc]\n\n Example(s):\n gh create-issue donnemartin/gitsome -t \"title\"\n gh create-issue donnemartin/gitsome -t \"title\" -d \"desc\"\n gh create-issue donnemartin/gitsome --issue_title \"title\" --issue_desc \"desc\" # NOQA\n\n :type github: :class:`github.GitHub`\n :param github: An instance of `github.GitHub`.\n\n :type user_repo: str\n :param user_repo: The user/repo.\n\n :type issue_title: str\n :param issue_title: The issue title.\n\n :type issue_desc: str\n :param issue_desc: The issue body (optional).\n \"\"\"\n github.create_issue(user_repo, issue_title, issue_desc)\n\n @cli.command('create-repo')\n @click.argument('repo_name')\n @click.option('-d', '--repo_desc', required=False)\n @click.option('-pr', '--private', is_flag=True)\n @pass_github\n def create_repo(github, repo_name, repo_desc, private):\n \"\"\"Create a repo.\n\n Usage:\n gh create-repo [repo_name] [-d/--repo_desc] [-pr/--private]\n\n Example(s):\n gh create-repo repo_name\n gh create-repo repo_name -d \"desc\"\n gh create-repo repo_name --repo_desc \"desc\"\n gh create-repo repo_name -pr\n gh create-repo repo_name --repo_desc \"desc\" --private\n\n :type github: :class:`github.GitHub`\n :param github: An instance of `github.GitHub`.\n\n :type repo_name: str\n :param repo_name: The repo name.\n\n :type repo_desc: str\n :param repo_desc: The repo description (optional).\n\n :type private: bool\n :param private: Determines whether the repo is private. Default: False.\n \"\"\"\n github.create_repo(repo_name, repo_desc, private)\n\n @cli.command()\n @pass_github\n def emails(github):\n \"\"\"List all the user's registered emails.\n\n Usage/Example(s):\n gh emails\n\n :type github: :class:`github.GitHub`\n :param github: An instance of `github.GitHub`.\n \"\"\"\n github.emails()\n\n @cli.command()\n @click.option('-p', '--pager', is_flag=True)\n @pass_github\n def emojis(github, pager):\n \"\"\"List all GitHub supported emojis.\n\n Usage:\n gh emojis [-p/--pager]\n\n Example(s):\n gh emojis | grep octo\n\n :type github: :class:`github.GitHub`\n :param github: An instance of `github.GitHub`.\n\n :type pager: bool\n :param pager: Determines whether to show the output in a pager,\n if available.\n \"\"\"\n github.emojis(pager)\n\n @cli.command()\n @click.argument('user_or_repo', required=False, default='')\n @click.option('-pr', '--private', is_flag=True, default=False)\n @click.option('-p', '--pager', is_flag=True)\n @pass_github\n def feed(github, user_or_repo, private, pager):\n \"\"\"List all activity for the given user or repo.\n\n If `user_or_repo` is not provided, uses the logged in user's news feed\n seen while visiting https://github.com. If `user_or_repo` is provided,\n shows either the public or `[-pr/--private]` feed activity of the user\n or repo.\n\n Usage:\n gh feed [user_or_repo] [-pr/--private] [-p/--pager]\n\n Examples:\n gh feed\n gh feed | grep foo\n gh feed donnemartin\n gh feed donnemartin -pr -p\n gh feed donnemartin --private --pager\n gh feed donnemartin/haxor-news -p\n\n :type github: :class:`github.GitHub`\n :param github: An instance of `github.GitHub`.\n\n :type user_or_repo: str\n :param user_or_repo: The user or repo to list events for (optional).\n If no entry, defaults to the logged in user's feed.\n\n :type private: bool\n :param private: Determines whether to show the private events (True)\n or public events (False).\n\n :type pager: bool\n :param pager: Determines whether to show the output in a pager,\n if available.\n \"\"\"\n github.feed(user_or_repo, private, pager)\n\n @cli.command()\n @click.argument('user', required=False)\n @click.option('-p', '--pager', is_flag=True)\n @pass_github\n def followers(github, user, pager):\n \"\"\"List all followers and the total follower count.\n\n Usage:\n gh followers [user] [-p/--pager]\n\n Example(s):\n gh followers\n gh followers -p\n gh followers octocat --pager\n\n :type github: :class:`github.GitHub`\n :param github: An instance of `github.GitHub`.\n\n :type user: str\n :param user: The user login (optional).\n If None, returns the followers of the logged in user.\n\n :type pager: bool\n :param pager: Determines whether to show the output in a pager,\n if available.\n \"\"\"\n github.followers(user, pager)\n\n @cli.command()\n @click.argument('user', required=False)\n @click.option('-p', '--pager', is_flag=True)\n @pass_github\n def following(github, user, pager):\n \"\"\"List all followed users and the total followed count.\n\n Usage:\n gh following [user] [-p/--pager]\n\n Example(s):\n gh following\n gh following -p\n gh following octocat --pager\n\n :type github: :class:`github.GitHub`\n :param github: An instance of `github.GitHub`.\n\n :type user: str\n :param user: The user login.\n If None, returns the followed users of the logged in user.\n\n :type pager: bool\n :param pager: Determines whether to show the output in a pager,\n if available.\n \"\"\"\n github.following(user, pager)\n\n @cli.command('gitignore-template')\n @click.argument('language')\n @pass_github\n def gitignore_template(github, language):\n \"\"\"Output the gitignore template for the given language.\n\n Usage:\n gh gitignore-template [language]\n\n Example(s):\n gh gitignore-template Python\n gh gitignore-template Python > .gitignore\n\n :type github: :class:`github.GitHub`\n :param github: An instance of `github.GitHub`.\n\n :type language: str\n :param language: The language.\n \"\"\"\n github.gitignore_template(language)\n\n @cli.command('gitignore-templates')\n @click.option('-p', '--pager', is_flag=True)\n @pass_github\n def gitignore_templates(github, pager):\n \"\"\"Output all supported gitignore templates.\n\n Usage:\n gh gitignore-templates\n\n Example(s):\n gh gitignore-templates\n gh gitignore-templates -p\n gh gitignore-templates --pager\n\n :type github: :class:`github.GitHub`\n :param github: An instance of `github.GitHub`.\n\n :type pager: bool\n :param pager: Determines whether to show the output in a pager,\n if available.\n \"\"\"\n github.gitignore_templates(pager)\n\n @cli.command()\n @click.argument('user_repo_number')\n @pass_github\n def issue(github, user_repo_number):\n \"\"\"Output detailed information about the given issue.\n\n Usage:\n gh issue [user_repo_number]\n\n Example(s):\n gh issue donnemartin/saws/1\n\n :type github: :class:`github.GitHub`\n :param github: An instance of `github.GitHub`.\n\n :type user_repo_number: str\n :param user_repo_number: The user/repo/issue_number.\n \"\"\"\n github.issue(user_repo_number)\n\n @cli.command()\n @click.option('-f', '--issue_filter', required=False, default='subscribed')\n @click.option('-s', '--issue_state', required=False, default='open')\n @click.option('-l', '--limit', required=False, default=1000)\n @click.option('-p', '--pager', is_flag=True)\n @pass_github\n def issues(github, issue_filter, issue_state, limit, pager):\n \"\"\"List all issues matching the filter.\n\n Usage:\n gh issues [-f/--issue_filter] [-s/--issue_state] [-l/--limit] [-p/--pager] # NOQA\n\n Example(s):\n gh issues\n gh issues -f assigned\n gh issues ---issue_filter created\n gh issues -s all -l 20 -p\n gh issues --issue_state closed --limit 20 --pager\n gh issues -f created -s all -p\n\n :type github: :class:`github.GitHub`\n :param github: An instance of `github.GitHub`.\n\n :type issue_filter: str\n :param issue_filter: assigned, created, mentioned, subscribed (default).\n\n :type issue_state: str\n :param issue_state: all, open (default), closed.\n\n :type limit: int\n :param limit: The number of items to display.\n\n :type pager: bool\n :param pager: Determines whether to show the output in a pager,\n if available.\n \"\"\"\n github.issues_setup(issue_filter, issue_state, limit, pager)\n\n @cli.command()\n @click.argument('license_name')\n @pass_github\n def license(github, license_name):\n \"\"\"Output the license template for the given license.\n\n Usage:\n gh license [license_name]\n\n Example(s):\n gh license apache-2.0\n gh license mit > LICENSE\n\n :type github: :class:`github.GitHub`\n :param github: An instance of `github.GitHub`.\n\n :type license_name: str\n :param license_name: The license name.\n \"\"\"\n github.license(license_name)\n\n @cli.command()\n @pass_github\n def licenses(github):\n \"\"\"Output all supported license templates.\n\n Usage/Example(s):\n gh licenses\n\n :type github: :class:`github.GitHub`\n :param github: An instance of `github.GitHub`.\n \"\"\"\n github.licenses()\n\n @cli.command()\n @click.option('-b', '--browser', is_flag=True)\n @click.option('-t', '--text_avatar', is_flag=True)\n @click.option('-l', '--limit', required=False, default=1000)\n @click.option('-p', '--pager', is_flag=True)\n @pass_github\n def me(github, browser, text_avatar, limit, pager):\n \"\"\"List information about the logged in user.\n\n Usage:\n gh me [-b/--browser] [-t/--text_avatar] [-l/--limit] [-p/--pager]\n\n Example(s):\n gh me\n gh me -b\n gh me --browser\n gh me -t -l 20 -p\n gh me --text_avatar --limit 20 --pager\n\n :type github: :class:`github.GitHub`\n :param github: An instance of `github.GitHub`.\n\n :type browser: bool\n :param browser: Determines whether to view the profile\n in a browser, or in the terminal.\n\n :type text_avatar: bool\n :param text_avatar: Determines whether to view the profile\n avatar in plain text instead of ansi (default).\n On Windows this value is always set to True due to lack of\n support of `img2txt` on Windows.\n\n :type limit: int\n :param limit: The number of user repos to display.\n\n :type pager: bool\n :param pager: Determines whether to show the output in a pager,\n if available.\n \"\"\"\n github.user_me(browser, text_avatar, limit, pager)\n\n @cli.command()\n @click.option('-l', '--limit', required=False, default=1000)\n @click.option('-p', '--pager', is_flag=True)\n @pass_github\n def notifications(github, limit, pager):\n \"\"\"List all notifications.\n\n Usage:\n gh notifications [-l/--limit] [-p/--pager]\n\n Example(s):\n gh notifications\n gh notifications -l 20 -p\n gh notifications --limit 20 --pager\n\n :type github: :class:`github.GitHub`\n :param github: An instance of `github.GitHub`.\n\n :type limit: int\n :param limit: The number of items to display.\n\n :type pager: bool\n :param pager: Determines whether to show the output in a pager,\n if available.\n \"\"\"\n github.notifications(limit, pager)\n\n @cli.command('octo')\n @click.argument('say', required=False)\n @pass_github\n def octocat(github, say):\n \"\"\"Output an Easter egg or the given message from Octocat.\n\n Usage:\n gh octo [say]\n\n Example(s):\n gh octo\n gh octo \"foo bar\"\n\n :type github: :class:`github.GitHub`\n :param github: An instance of `github.GitHub`.\n\n :type say: str\n :param say: What Octocat should say.\n If say is None, octocat speaks an Easter egg.\n \"\"\"\n github.octocat(say)\n\n @cli.command('pull-request')\n @click.argument('user_repo_number')\n @pass_github\n def pull_request(github, user_repo_number):\n \"\"\"Output detailed information about the given pull request.\n\n Usage:\n gh pull-request [user_repo_number]\n\n Example(s):\n gh pull-request donnemartin/saws/80\n\n :type github: :class:`github.GitHub`\n :param github: An instance of `github.GitHub`.\n\n :type user_repo_number: str\n :param user_repo_number: The user/repo/pull_number.\n \"\"\"\n github.issue(user_repo_number)\n\n @cli.command('pull-requests')\n @click.option('-l', '--limit', required=False, default=1000)\n @click.option('-p', '--pager', is_flag=True)\n @pass_github\n def pull_requests(github, limit, pager):\n \"\"\"List all pull requests.\n\n Usage:\n gh pull-requests [-l/--limit] [-p/--pager]\n\n Example(s):\n gh pull-requests\n gh pull-requests -l 20 -p\n gh pull-requests --limit 20 --pager\n\n :type github: :class:`github.GitHub`\n :param github: An instance of `github.GitHub`.\n\n :type limit: int\n :param limit: The number of items to display.\n\n :type pager: bool\n :param pager: Determines whether to show the output in a pager,\n if available.\n \"\"\"\n github.pull_requests(limit, pager)\n\n @cli.command('rate-limit')\n @pass_github\n def rate_limit(github):\n \"\"\"Output the rate limit. Not available for GitHub Enterprise.\n\n Logged in users can make 5000 requests per hour.\n See: https://developer.github.com/v3/#rate-limiting\n\n Usage/Example(s):\n gh rate-limit\n\n :type github: :class:`github.GitHub`\n :param github: An instance of `github.GitHub`.\n \"\"\"\n github.rate_limit()\n\n @cli.command('repos')\n @click.argument('repo_filter', required=False, default='')\n @click.option('-l', '--limit', required=False, default=1000)\n @click.option('-p', '--pager', is_flag=True)\n @pass_github\n def repositories(github, repo_filter, limit, pager):\n \"\"\"List all repos matching the given filter.\n\n Usage:\n gh repos [repo_filter] [-l/--limit] [-p/--pager]\n\n Example(s):\n gh repos\n gh repos \"data-science\"\n gh repos \"data-science\" -l 20 -p\n gh repos \"data-science\" --limit 20 --pager\n\n :type github: :class:`github.GitHub`\n :param github: An instance of `github.GitHub`.\n\n :type repo_filter: str\n :param repo_filter: The filter for repo names.\n Only repos matching the filter will be returned.\n If None, outputs all the logged in user's repos.\n\n :type limit: int\n :param limit: The number of items to display.\n\n :type pager: bool\n :param pager: Determines whether to show the output in a pager,\n if available.\n \"\"\"\n github.repositories_setup(repo_filter, limit, pager)\n\n @cli.command('repo')\n @click.argument('user_repo')\n @pass_github\n def repository(github, user_repo):\n \"\"\"Output detailed information about the given repo.\n\n Usage:\n gh repo [user_repo]\n\n Example(s):\n gh repo donnemartin/gitsome\n\n :type github: :class:`github.GitHub`\n :param github: An instance of `github.GitHub`.\n\n :type user_repo: str\n :param user_repo: The user/repo.\n \"\"\"\n github.repository(user_repo)\n\n @cli.command('search-issues')\n @click.argument('query')\n @click.option('-l', '--limit', required=False, default=1000)\n @click.option('-p', '--pager', is_flag=True)\n @pass_github\n def search_issues(github, query, limit, pager):\n \"\"\"Search for all issues matching the given query.\n\n Usage:\n gh search-issues [query] [-l/--limit] [-p/--pager]\n\n Example(s):\n gh search-issues \"foo type:pr author:donnemartin\" -l 20 -p\n gh search-issues \"foobarbaz in:title created:>=2015-01-01\" --limit 20 --pager # NOQA\n\n Additional Example(s):\n Search issues that have your user name tagged @donnemartin:\n gh search-issues \"is:issue donnemartin is:open\" -p\n\n Search issues that have the most +1s:\n gh search-issues \"is:open is:issue sort:reactions-+1-desc\" -p\n\n Search issues that have the most comments:\n gh search-issues \"is:open is:issue sort:comments-desc\" -p\n\n Search issues with the \"help wanted\" tag:\n gh search-issues \"is:open is:issue label:\\\"help wanted\\\"\" -p\n\n Search all your open private issues:\n gh search-issues \"is:open is:issue is:private\" -p\n\n The query can contain any combination of the following supported\n qualifers:\n\n - `type` With this qualifier you can restrict the search to issues\n or pull request only.\n - `in` Qualifies which fields are searched. With this qualifier you\n can restrict the search to just the title, body, comments, or any\n combination of these.\n - `author` Finds issues created by a certain user.\n - `assignee` Finds issues that are assigned to a certain user.\n - `mentions` Finds issues that mention a certain user.\n - `commenter` Finds issues that a certain user commented on.\n - `involves` Finds issues that were either created by a certain user,\n assigned to that user, mention that user, or were commented on by\n that user.\n - `state` Filter issues based on whether they’re open or closed.\n - `labels` Filters issues based on their labels.\n - `language` Searches for issues within repositories that match a\n certain language.\n - `created` or `updated` Filters issues based on times of creation,\n or when they were last updated.\n - `comments` Filters issues based on the quantity of comments.\n - `user` or `repo` Limits searches to a specific user or\n repository.\n\n For more information about these qualifiers, see: http://git.io/d1oELA\n\n :type github: :class:`github.GitHub`\n :param github: An instance of `github.GitHub`.\n\n :type query: str\n :param query: The search query.\n\n :type limit: int\n :param limit: The number of items to display.\n\n :type pager: bool\n :param pager: Determines whether to show the output in a pager,\n if available.\n \"\"\"\n github.search_issues(query, limit, pager)\n\n @cli.command('search-repos')\n @click.argument('query')\n @click.option('-s', '--sort', required=False, default='')\n @click.option('-l', '--limit', required=False, default=1000)\n @click.option('-p', '--pager', is_flag=True)\n @pass_github\n def search_repositories(github, query, sort, limit, pager):\n \"\"\"Search for all repos matching the given query.\n\n Usage:\n gh search-repos [query] [-s/--sort] [-l/--limit] [-p/--pager]\n\n Example(s):\n gh search-repos \"maps language:python\" -s stars -l 20 -p\n gh search-repos \"created:>=2015-01-01 stars:>=1000 language:python\" --sort stars --limit 20 --pager # NOQA\n\n The query can contain any combination of the following supported\n qualifers:\n\n - `in` Qualifies which fields are searched. With this qualifier you\n can restrict the search to just the repository name, description,\n readme, or any combination of these.\n - `size` Finds repositories that match a certain size (in\n kilobytes).\n - `forks` Filters repositories based on the number of forks, and/or\n whether forked repositories should be included in the results at\n all.\n - `created` or `pushed` Filters repositories based on times of\n creation, or when they were last updated. Format: `YYYY-MM-DD`.\n Examples: `created:<2011`, `pushed:<2013-02`,\n `pushed:>=2013-03-06`\n - `user` or `repo` Limits searches to a specific user or\n repository.\n - `language` Searches repositories based on the language they're\n written in.\n - `stars` Searches repositories based on the number of stars.\n\n For more information about these qualifiers, see: http://git.io/4Z8AkA\n\n :type github: :class:`github.GitHub`\n :param github: An instance of `github.GitHub`.\n\n :type query: str\n :param query: The search query.\n\n :type sort: str\n :param sort: Optional: 'stars', 'forks', 'updated'.\n If not specified, sorting is done by query best match.\n\n :type limit: int\n :param limit: The number of items to display.\n\n :type pager: bool\n :param pager: Determines whether to show the output in a pager,\n if available.\n \"\"\"\n github.search_repositories(query, sort, limit, pager)\n\n @cli.command()\n @click.argument('repo_filter', required=False, default='')\n @click.option('-l', '--limit', required=False, default=1000)\n @click.option('-p', '--pager', is_flag=True)\n @pass_github\n def starred(github, repo_filter, limit, pager):\n \"\"\"Output starred repos.\n\n Usage:\n gh starred [repo_filter] [-l/--limit] [-p/--pager]\n\n Example(s):\n gh starred\n gh starred foo -l 20 -p\n gh starred foo --limit 20 --pager\n\n :type github: :class:`github.GitHub`\n :param github: An instance of `github.GitHub`.\n\n :type repo_filter: str\n :param repo_filter: The filter for repo names.\n Only repos matching the filter will be returned.\n If None, outputs all starred repos.\n\n :type limit: int\n :param limit: The number of items to display.\n\n :type pager: bool\n :param pager: Determines whether to show the output in a pager,\n if available.\n \"\"\"\n github.starred(repo_filter, limit, pager)\n\n @cli.command()\n @click.argument('language', required=False, default='Overall')\n @click.option('-w', '--weekly', is_flag=True)\n @click.option('-m', '--monthly', is_flag=True)\n @click.option('-D', '--devs', is_flag=True)\n @click.option('-b', '--browser', is_flag=True)\n @click.option('-p', '--pager', is_flag=True)\n @pass_github\n def trending(github, language, weekly, monthly, devs, browser, pager):\n \"\"\"List trending repos for the given language.\n\n Usage:\n gh trending [language] [-w/--weekly] [-m/--monthly] [-D/--devs] [-b/--browser] [-p/--pager] # NOQA\n\n Example(s):\n gh trending\n gh trending Python -w -p\n gh trending Python --weekly --devs --browser\n gh trending --browser\n\n :type github: :class:`github.GitHub`\n :param github: An instance of `github.GitHub`.\n\n :type language: str\n :param language: The language (optional).\n If blank, shows 'Overall'.\n\n :type weekly: bool\n :param weekly: Determines whether to show the weekly rankings.\n Daily is the default.\n\n :type monthly: bool\n :param monthly: Determines whether to show the monthly rankings.\n Daily is the default.\n If both `monthly` and `weekly` are set, `monthly` takes precedence.\n\n :type devs: bool\n :param devs: determines whether to display the trending\n devs or repos. Only valid with the -b/--browser option.\n\n :type browser: bool\n :param browser: Determines whether to view the profile\n in a browser, or in the terminal.\n\n :type pager: bool\n :param pager: Determines whether to show the output in a pager,\n if available.\n \"\"\"\n github.trending(language, weekly, monthly, devs, browser, pager)\n\n @cli.command()\n @click.argument('user_id', required=True)\n @click.option('-b', '--browser', is_flag=True)\n @click.option('-t', '--text_avatar', is_flag=True)\n @click.option('-l', '--limit', required=False, default=1000)\n @click.option('-p', '--pager', is_flag=True)\n @pass_github\n def user(github, user_id, browser, text_avatar, limit, pager):\n \"\"\"List information about the given user.\n\n Usage:\n gh user [user_id] [-b/--browser] [-t/--text_avatar] [-l/--limit] [-p/--pager] # NOQA\n\n Example(s):\n gh user octocat\n gh user octocat -b\n gh user octocat --browser\n gh user octocat -t -l 10 -p\n gh user octocat --text_avatar --limit 10 --pager\n\n :type github: :class:`github.GitHub`\n :param github: An instance of `github.GitHub`.\n\n :type user_id: str\n :param user_id: The user id/login.\n If None, returns followers of the logged in user.\n\n :type browser: bool\n :param browser: Determines whether to view the profile\n in a browser, or in the terminal.\n\n :type text_avatar: bool\n :param text_avatar: Determines whether to view the profile\n avatar in plain text instead of ansi (default).\n On Windows this value is always set to True due to lack of\n support of `img2txt` on Windows.\n\n :type limit: int\n :param limit: The number of items to display.\n\n :type pager: bool\n :param pager: Determines whether to show the output in a pager,\n if available.\n \"\"\"\n github.user(user_id, browser, text_avatar, limit, pager)\n\n @cli.command()\n @click.argument('index')\n @click.option('-b', '--browser', is_flag=True)\n @pass_github\n def view(github, index, browser):\n \"\"\"View the given notification/repo/issue/pull_request/user index.\n\n This method is meant to be called after one of the following commands\n which outputs a table of notifications/repos/issues/pull_requests/users:\n\n gh repos\n gh search_repos\n gh starred\n\n gh issues\n gh pull_requests\n gh search_issues\n\n gh notifications\n gh trending\n\n gh user\n gh me\n\n Usage:\n gh view [index] [-b/--browser]\n\n Example(s):\n gh repos\n gh view 1\n\n gh starred\n gh view 1 -b\n gh view 1 --browser\n\n :type github: :class:`github.GitHub`\n :param github: An instance of `github.GitHub`.\n\n :type index: str\n :param index: Determines the index to view.\n\n :type browser: bool\n :param browser: Determines whether to view the profile\n in a browser, or in the terminal.\n \"\"\"\n github.view(int(index), browser)\n" }, { "path": "gitsome/main.py", "content": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n\n# Copyright 2015 Donne Martin. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"). You\n# may not use this file except in compliance with the License. A copy of\n# the License is located at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# or in the \"license\" file accompanying this file. This file is\n# distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF\n# ANY KIND, either express or implied. See the License for the specific\n# language governing permissions and limitations under the License.\n\nfrom __future__ import unicode_literals\nfrom __future__ import print_function\n\nfrom .gitsome import GitSome\n\n\ndef cli():\n \"\"\"Create and call Gitsome.\"\"\"\n try:\n gitsome = GitSome()\n gitsome.run_cli()\n except (EOFError, KeyboardInterrupt):\n gitsome.cli.set_return_value(None)\n\n\nif __name__ == \"__main__\":\n cli()\n" }, { "path": "gitsome/main_cli.py", "content": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n\n# Copyright 2015 Donne Martin. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"). You\n# may not use this file except in compliance with the License. A copy of\n# the License is located at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# or in the \"license\" file accompanying this file. This file is\n# distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF\n# ANY KIND, either express or implied. See the License for the specific\n# language governing permissions and limitations under the License.\n\nfrom __future__ import unicode_literals\nfrom __future__ import print_function\n\nfrom .githubcli import GitHubCli\n\n\ndef cli():\n \"\"\"Create and call GitHubCli.\"\"\"\n github = GitHubCli()\n github.cli()\n\n\nif __name__ == \"__main__\":\n cli()\n" }, { "path": "gitsome/rss_feed.py", "content": "# -*- coding: utf-8 -*-\n\n# Copyright 2015 Donne Martin. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"). You\n# may not use this file except in compliance with the License. A copy of\n# the License is located at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# or in the \"license\" file accompanying this file. This file is\n# distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF\n# ANY KIND, either express or implied. See the License for the specific\n# language governing permissions and limitations under the License.\n\nlanguage_rss_map = {\n 'c#': 'csharp',\n 'c++': 'cpp',\n 'overall': 'all',\n}\n" }, { "path": "gitsome/table.py", "content": "# -*- coding: utf-8 -*-\n\n# Copyright 2015 Donne Martin. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"). You\n# may not use this file except in compliance with the License. A copy of\n# the License is located at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# or in the \"license\" file accompanying this file. This file is\n# distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF\n# ANY KIND, either express or implied. See the License for the specific\n# language governing permissions and limitations under the License.\n\nfrom __future__ import unicode_literals\nfrom __future__ import print_function\n\nimport platform\nimport re\nimport sys\n\nimport click\n\nfrom .view_entry import ViewEntry\n\n\nclass Table(object):\n \"\"\"Display table information for repos, issues, prs, etc.\n\n :type config: :class:`config.Config`\n :param config: An instance of `config.Config`.\n \"\"\"\n\n def __init__(self, config):\n self.config = config\n\n def build_table(self, view_entries, limit, pager, format_method,\n build_urls=True, print_output=True):\n \"\"\"Build the table used for the gh view command.\n\n :type view_entries: list\n :param view_entries: A list of `github3` items.\n\n :type limit: int\n :param limit: Determines the number of items to show.\n\n :type format_method: callable\n :param format_method: A method called to format each item in the table.\n\n :type build_urls: bool\n :param build_urls: Determines whether to build urls for the\n gh view # command.\n\n :type print_output: bool\n :param print_output: determines whether to print the output\n (True) or return the output as a string (False).\n\n :rtype: str\n :return: the output if print_output is True, else, return None.\n \"\"\"\n if build_urls:\n self.build_table_urls(view_entries)\n index = 0\n output = ''\n for view_entry in view_entries:\n index += 1\n view_entry.index = index\n output += format_method(view_entry) + '\\n'\n if index >= limit:\n break\n if build_urls:\n if len(view_entries) > limit:\n output += click.style((' \\n'),\n fg=self.config.clr_message)\n if index == 0:\n output += click.style('No results found',\n fg=self.config.clr_message)\n elif build_urls:\n output += click.style(self.create_tip(index))\n else:\n output += click.style('')\n if print_output:\n if pager:\n color = None\n if platform.system() == 'Windows':\n color = True\n # Strip out Unicode, which seems to have issues on\n # Windows with click.echo_via_pager.\n output = re.sub(r'[^\\x00-\\x7F]+', '', output)\n click.echo_via_pager(output, color)\n else:\n click.secho(output)\n return None\n else:\n return output\n\n def build_table_setup(self, items, format_method,\n limit, pager, build_urls=True):\n \"\"\"Convert items to a list of ViewEntry before calling `build_table`.\n\n :type items: list\n :param items: A list of `github3` items.\n\n :type format_method: callable\n :param format_method: A method called to format each item in the table.\n\n :type limit: int\n :param limit: Determines the number of items to show.\n\n :type pager: bool\n :param pager: Determines whether to show results in a pager,\n if available.\n\n :type build_urls: bool\n :param build_urls: determines whether to build urls for the\n `gh view` [#] command.\n \"\"\"\n view_entries = []\n for item in items:\n view_entries.append(ViewEntry(item))\n self.build_table(view_entries,\n limit,\n pager,\n format_method,\n build_urls)\n\n def build_table_setup_feed(self, items, format_method, pager):\n \"\"\"Perform feed-specific processing before calling `build_table`.\n\n :type items: list\n :param items: A list of `github3` items.\n\n :type format_method: callable\n :param format_method: A method called to format each item in the table.\n\n :type pager: bool\n :param pager: Determines whether to show results in a pager,\n if available.\n \"\"\"\n self.build_table_setup(items.entries,\n format_method,\n limit=sys.maxsize,\n pager=pager,\n build_urls=False)\n\n def build_table_setup_user(self, items, format_method,\n limit, pager, build_urls=True):\n \"\"\"Convert items to a list of ViewEntry before calling `build_table`.\n\n Specific to GitHub3.User.users.\n\n :type items: list\n :param items: A list of `github3` items.\n\n :type format_method: callable\n :param format_method: A method called to format each item in the table.\n\n :type limit: int\n :param limit: Determines the number of items to show.\n\n :type pager: bool\n :param pager: Determines whether to show results in a pager,\n if available.\n\n :type build_urls: bool\n :param build_urls: determines whether to build urls for the\n `gh view` [#] command.\n \"\"\"\n view_entries = []\n for item in items:\n view_entries.append(ViewEntry(item=item, url=item.login))\n self.build_table(view_entries,\n limit=sys.maxsize,\n pager=pager,\n format_method=format_method)\n\n def build_table_setup_trending(self, items, format_method,\n limit, pager, build_urls=True):\n \"\"\"Convert items to a list of ViewEntry before calling `build_table`.\n\n Specific to feedparser entries.\n\n :type items: list\n :param items: A list of `github3` items.\n\n :type format_method: callable\n :param format_method: A method called to format each item in the table.\n\n :type limit: int\n :param limit: Determines the number of items to show.\n\n :type pager: bool\n :param pager: Determines whether to show results in a pager,\n if available.\n\n :type build_urls: bool\n :param build_urls: determines whether to build urls for the\n `gh view` [#] command.\n \"\"\"\n view_entries = []\n for item in items:\n url = 'https://github.com/' + '/'.join(item['link'].split('/')[-2:])\n view_entries.append(ViewEntry(item=item, url=url))\n self.build_table(view_entries,\n limit=sys.maxsize,\n pager=pager,\n format_method=format_method)\n\n def build_table_urls(self, view_entries):\n \"\"\"Build the GitHub urls for the specified view_entries.\n\n :type view_entries: list\n :param view_entries: A list of ViewEntry items.\n \"\"\"\n for view_entry in view_entries:\n self.config.urls.append(view_entry.url)\n self.config.save_urls()\n\n def create_tip(self, max_index):\n \"\"\"Create the tip about the view command after showing a table.\n\n :type max_index: int\n :param max_index: The index upper bound.\n\n :rtype: str\n :return: The formatted tip.\n \"\"\"\n tip = click.style(' View the page for ',\n fg=self.config.clr_message)\n tip += click.style('1 through ', fg=self.config.clr_view_index)\n tip += click.style(str(max_index), fg=self.config.clr_view_index)\n tip += click.style(' with the following command:\\n',\n fg=self.config.clr_message)\n tip += click.style(' gh view [#] ', fg=self.config.clr_view_index)\n tip += click.style('optional: [-b/--browser] [--help]\\n',\n fg=self.config.clr_message)\n return tip\n" }, { "path": "gitsome/utils.py", "content": "# -*- coding: utf-8 -*-\n\n# Copyright 2015 Donne Martin. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"). You\n# may not use this file except in compliance with the License. A copy of\n# the License is located at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# or in the \"license\" file accompanying this file. This file is\n# distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF\n# ANY KIND, either express or implied. See the License for the specific\n# language governing permissions and limitations under the License.\n\nfrom __future__ import unicode_literals\nfrom __future__ import print_function\n\nimport re\n\nimport six\nimport shlex\nfrom prompt_toolkit.completion import Completion\n\nfrom .completions import META_LOOKUP_GH\n\n\nclass TextUtils(object):\n \"\"\"Utilities for parsing and matching text.\"\"\"\n\n def find_matches(self, word, collection, fuzzy):\n \"\"\"Find all matches in collection for word.\n\n :type word: str\n :param word: The word before the cursor.\n\n :type collection: iterable\n :param collection: A collection of words to match.\n\n :type fuzzy: bool\n :param fuzzy: Determines whether to use fuzzy matching.\n\n :rtype: generator\n :return: Yields an instance of `prompt_toolkit.completion.Completion`.\n \"\"\"\n word = self._last_token(word).lower()\n matches = []\n for suggestion in self._find_collection_matches(word,\n collection,\n fuzzy):\n matches.append(suggestion)\n return matches\n\n def get_tokens(self, text):\n \"\"\"Parse out all tokens.\n\n :type text: str\n :param text: A string to be split into tokens.\n\n :rtype: list\n :return: A list of strings for each word in the text.\n \"\"\"\n if text is not None:\n text = text.strip()\n words = self._safe_split(text)\n return words\n return []\n\n def _last_token(self, text):\n \"\"\"Find the last word in text.\n\n :type text: str\n :param text: A string to parse and obtain the last word.\n\n :rtype: str\n :return: The last word in the text.\n \"\"\"\n if text is not None:\n text = text.strip()\n if len(text) > 0:\n word = self._safe_split(text)[-1]\n word = word.strip()\n return word\n return ''\n\n def _fuzzy_finder(self, text, collection, case_sensitive=True):\n \"\"\"Customized fuzzy finder with optional case-insensitive matching.\n\n Adapted from: https://github.com/amjith/fuzzyfinder.\n\n :type text: str\n :param text: Input string entered by user.\n\n :type collection: iterable\n :param collection: collection of strings which will be filtered based\n on the input `text`.\n\n :type case_sensitive: bool\n :param case_sensitive: Determines whether the find will be case\n sensitive.\n\n :rtype: generator\n :return: Yields a list of suggestions narrowed down from `collections`\n using the `text` input.\n \"\"\"\n suggestions = []\n if case_sensitive:\n pat = '.*?'.join(map(re.escape, text))\n else:\n pat = '.*?'.join(map(re.escape, text.lower()))\n regex = re.compile(pat)\n for item in collection:\n if case_sensitive:\n r = regex.search(item)\n else:\n r = regex.search(item.lower())\n if r:\n suggestions.append((len(r.group()), r.start(), item))\n\n return (z for _, _, z in sorted(suggestions))\n\n def _find_collection_matches(self, word, collection, fuzzy):\n \"\"\"Yield all matching names in list.\n\n :type word: str\n :param word: The word before the cursor.\n\n :type collection: iterable\n :param collection: A collection of words to match.\n\n :type fuzzy: bool\n :param fuzzy: Determines whether to use fuzzy matching.\n\n :rtype: generator\n :return: Yields an instance of `prompt_toolkit.completion.Completion`.\n \"\"\"\n word = word.lower()\n if fuzzy:\n for suggestion in self._fuzzy_finder(word,\n collection,\n case_sensitive=False):\n yield Completion(suggestion,\n -len(word),\n display_meta='display_meta')\n else:\n for name in sorted(collection):\n if name.lower().startswith(word) or not word:\n display = None\n display_meta = None\n # print(len(words))\n if name in META_LOOKUP_GH:\n display_meta = META_LOOKUP_GH[name]\n yield Completion(name,\n -len(word),\n display=display,\n display_meta=display_meta)\n\n def _shlex_split(self, text):\n \"\"\"Wrapper for shlex, because it does not seem to handle unicode in 2.6.\n\n :type text: str\n :param text: A string to split.\n\n :rtype: list\n :return: A list that contains words for each split element of text.\n \"\"\"\n if six.PY2:\n text = text.encode('utf-8')\n return shlex.split(text)\n\n def _safe_split(self, text):\n \"\"\"Safely splits the input text.\n\n Shlex can't always split. For example, \"\\\" crashes the completer.\n\n :type text: str\n :param text: A string to split.\n\n :rtype: list\n :return: A list that contains words for each split element of text.\n \"\"\"\n try:\n words = self._shlex_split(text)\n return words\n except:\n return text\n\n def sanitize_if_none(self, text):\n \"\"\"Sanitize text to ensure it is not None\n\n :type text:str\n :param text: String to sanitize.\n\n :rtype: str\n :return: String which is empty if None.\n \"\"\"\n return text if text is not None else ''\n" }, { "path": "gitsome/view_entry.py", "content": "# -*- coding: utf-8 -*-\n\n# Copyright 2015 Donne Martin. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"). You\n# may not use this file except in compliance with the License. A copy of\n# the License is located at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# or in the \"license\" file accompanying this file. This file is\n# distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF\n# ANY KIND, either express or implied. See the License for the specific\n# language governing permissions and limitations under the License.\n\nfrom __future__ import unicode_literals\nfrom __future__ import print_function\n\n\nclass ViewEntry(object):\n \"\"\"A table entry used with the `gh view` command.\n\n :type index: int\n :param index: The row index.\n\n :type item: :class:`github3` repo, issue, thread, etc.\n :param item: An instance of a `github3` repo, issue, thread, etc.\n\n :param sort_key_primary: A class member representing the primary\n sort key.\n\n :param sort_key_secondary: A class member representing the secondary\n sort key.\n\n :param sort_key_tertiary: A class member representing the tertiary\n sort key.\n\n :type url: str\n :param url: The item's url.\n \"\"\"\n\n def __init__(self, item, url=None, index=-1,\n sort_key_primary=None,\n sort_key_secondary=None,\n sort_key_tertiary=None):\n self.item = item\n self.url = url\n self.index = index\n self.sort_key_primary = sort_key_primary\n self.sort_key_secondary = sort_key_secondary\n self.sort_key_tertiary = sort_key_tertiary\n\n def __lt__(self, other):\n \"\"\"Implement 'less than' used for sorting.\n\n :type other: :class:`ViewEntry`\n :param other: An instance of `ViewEntry` used for comparison.\n\n :rtype: bool\n :return: Determines whether the current ViewEntry is less than the\n `other` view entry.\n \"\"\"\n if self.sort_key_primary != other.sort_key_primary or \\\n not self.sort_key_secondary:\n return self.sort_key_primary < other.sort_key_primary\n else:\n if self.sort_key_secondary != other.sort_key_secondary or \\\n not self.sort_key_tertiary:\n return self.sort_key_secondary < other.sort_key_secondary\n else:\n return self.sort_key_tertiary < other.sort_key_tertiary\n" }, { "path": "gitsome/web_viewer.py", "content": "# -*- coding: utf-8 -*-\n\n# Copyright 2015 Donne Martin. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"). You\n# may not use this file except in compliance with the License. A copy of\n# the License is located at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# or in the \"license\" file accompanying this file. This file is\n# distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF\n# ANY KIND, either express or implied. See the License for the specific\n# language governing permissions and limitations under the License.\n\nfrom __future__ import unicode_literals\nfrom __future__ import print_function\n\nimport platform\nimport re\n\nfrom .compat import HTMLParser\nimport click\nfrom .lib.html2text.html2text import HTML2Text\nimport requests\n\n\nclass WebViewer(object):\n \"\"\"Handle viewing of web content within the terminal.\n\n :type config: :class:`config.Config`\n :param config: An instance of `config.Config`.\n\n :type html: :class:`HTMLParser.HTMLParser`\n :param html: An instance of `HTMLParser.HTMLParser`.\n\n :type html_to_text: :class:`html2text.HTML2Text`\n :param html_to_text: An instance of `html2text.HTML2Text`.\n \"\"\"\n\n def __init__(self, config):\n self.config = config\n try:\n self.html = HTMLParser.HTMLParser()\n except:\n self.html = HTMLParser\n self.html_to_text = None\n self._init_html_to_text()\n\n def _init_html_to_text(self):\n \"\"\"Initialize HTML2Text.\"\"\"\n self.html_to_text = HTML2Text()\n self.html_to_text.body_width = 0\n self.html_to_text.ignore_images = False\n self.html_to_text.ignore_emphasis = False\n self.html_to_text.ignore_links = False\n self.html_to_text.skip_internal_links = False\n self.html_to_text.inline_links = False\n self.html_to_text.links_each_paragraph = False\n\n def format_markdown(self, text):\n \"\"\"Add color to the input markdown using click.style.\n\n :type text: str\n :param text: The markdown text.\n\n :rtype: str\n :return: The input `text`, formatted.\n \"\"\"\n pattern_url_name = r'[^]]*'\n pattern_url_link = r'[^)]+'\n pattern_url = r'([!]*\\[{0}]\\(\\s*{1}\\s*\\))'.format(\n pattern_url_name,\n pattern_url_link)\n regex_url = re.compile(pattern_url)\n text = regex_url.sub(click.style(r'\\1', fg='green'), text)\n pattern_url_ref_name = r'[^]]*'\n pattern_url_ref_link = r'[^]]+'\n pattern_url_ref = r'([!]*\\[{0}]\\[\\s*{1}\\s*\\])'.format(\n pattern_url_ref_name,\n pattern_url_ref_link)\n regex_url_ref = re.compile(pattern_url_ref)\n text = regex_url_ref.sub(click.style(r'\\1', fg='green'),\n text)\n regex_list = re.compile(r'( \\*.*)')\n text = regex_list.sub(click.style(r'\\1', fg='cyan'),\n text)\n regex_header = re.compile(r'(#+) (.*)')\n text = regex_header.sub(click.style(r'\\2', fg='yellow'),\n text)\n regex_bold = re.compile(r'(\\*\\*|__)(.*?)\\1')\n text = regex_bold.sub(click.style(r'\\2', fg='cyan'),\n text)\n regex_code = re.compile(r'(`)(.*?)\\1')\n text = regex_code.sub(click.style(r'\\1\\2\\1', fg='cyan'),\n text)\n text = re.sub(r'(\\s*\\r?\\n\\s*){2,}', r'\\n\\n', text)\n return text\n\n def generate_url_contents(self, url):\n \"\"\"Generate the formatted contents of the given item's url.\n\n Converts the HTML to text using HTML2Text, colors it, then displays\n the output in a pager.\n\n :type url: str\n :param url: The url whose contents to fetch.\n\n :rtype: str\n :return: The string representation of the formatted url contents.\n \"\"\"\n try:\n headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'} # NOQA\n raw_response = requests.get(url, headers=headers,\n verify=self.config.verify_ssl)\n except (requests.exceptions.SSLError,\n requests.exceptions.ConnectionError) as e:\n contents = 'Error: ' + str(e) + '\\n'\n contents += 'Try running gh view # with the --browser/-b flag\\n'\n return contents\n contents = self.html_to_text.handle(raw_response.text)\n # Strip out Unicode, which seems to have issues when html2txt is\n # coupled with click.echo_via_pager.\n contents = re.sub(r'[^\\x00-\\x7F]+', '', contents)\n contents = self.format_markdown(contents)\n return contents\n\n def view_url(self, url):\n \"\"\"View the given url.\n\n :type index: int\n :param index: The index for the given item, used with the\n gh view [index] commend.\n\n :type url: str\n :param url: The url to view\n \"\"\"\n contents = self.generate_url_contents(url)\n header = click.style('Viewing ' + url + '\\n\\n',\n fg=self.config.clr_message)\n contents = header + contents\n contents += click.style(('\\nView this article in a browser with'\n ' the -b/--browser flag.\\n'),\n fg=self.config.clr_message)\n contents += click.style(('\\nPress q to quit viewing this '\n 'article.\\n'),\n fg=self.config.clr_message)\n if contents == '{\"error\":\"Not Found\"}\\n':\n click.secho('Invalid user/repo combination.')\n return\n color = None\n if platform.system() == 'Windows':\n color = True\n click.echo_via_pager(contents, color)\n" }, { "path": "requirements-dev.txt", "content": "codecov>=1.3.3\nflake8>=2.4.1\ngitchangelog>=2.2.1\nmock>=1.0.1\npexpect>=3.3\npytest>=2.7.0\nsphinx>=1.3.1\nSphinx-PyPI-upload>=0.2.1\ntox>=1.9.2\n" }, { "path": "scripts/aws_complete.sh", "content": "complete -C aws_completer aws\n#complete -C /Users/donnemartin/.virtualenvs/gitsome/bin/aws_completer aws" }, { "path": "scripts/create_changelog.sh", "content": "#!/usr/bin/env bash\n\ngitchangelog > CHANGELOG_DRAFT\npandoc --from=markdown --to=rst --output=CHANGELOG.rst CHANGELOG.md\n" }, { "path": "scripts/create_readme_rst.sh", "content": "#!/usr/bin/env bash\n\npandoc --from=markdown --to=rst --output=docs/source/README.rst README.md" }, { "path": "scripts/gh_complete.sh", "content": "_gh_completion() {\n COMPREPLY=( $( env COMP_WORDS=\"${COMP_WORDS[*]}\" \\\n COMP_CWORD=$COMP_CWORD \\\n _GH_COMPLETE=complete $1 ) )\n return 0\n}\n\ncomplete -F _gh_completion -o default gh;\n" }, { "path": "scripts/gitsome", "content": "#!/usr/bin/env python -u\nfrom xonsh.main import main\nmain()\n" }, { "path": "scripts/gitsome.bat", "content": "@echo off\r\npython -m xonsh.main\r\n" }, { "path": "scripts/run_code_checks.sh", "content": "#!/usr/bin/env bash\n\nflake8 --max-line-length=80 --exclude=build,docs,scratch,docs,html2text.py,compat.py,xonsh,lib,parser.table.py,parser.out,parser_test_table.py,parser_table.py,completions.py,completions_git.py,tests/data/*,.tox/,.eggs/ .\n" }, { "path": "scripts/set_changelog_as_readme.sh", "content": "mv README.md README_temp.md\ncp CHANGELOG.rst README.md" }, { "path": "scripts/set_changelog_as_readme_undo.sh", "content": "mv README_temp.md README.md" }, { "path": "scripts/update_docs.sh", "content": "#!/usr/bin/env bash\n\nscripts/create_readme_rst.sh\npython setup.py build_sphinx" }, { "path": "scripts/upload_pypi.sh", "content": "#!/usr/bin/env bash\n\n# N. Set CHANGELOG as `README.md`\nscripts/set_changelog_as_readme.sh\n# O. Register package with PyPi\npython setup.py register -r pypi\n# P. Upload to PyPi\npython setup.py sdist upload -r pypi\n# Q. Upload Sphinx docs to PyPi\npython setup.py upload_sphinx\n# R. Restore `README.md`\nscripts/set_changelog_as_readme_undo.sh\n" }, { "path": "scripts/xonsh", "content": "#!/usr/bin/env python -u\nfrom xonsh.main import main\nmain()\n" }, { "path": "scripts/xonsh.bat", "content": "@echo off\r\npython -m xonsh.main\r\n" }, { "path": "setup.py", "content": "#!/usr/bin/env python\n# coding=utf-8\n\n\"\"\"The gitsome installer.\"\"\"\nfrom __future__ import print_function, unicode_literals\nimport os\nimport sys\ntry:\n from setuptools import setup, find_packages\n from setuptools.command.sdist import sdist\n from setuptools.command.install import install\n from setuptools.command.develop import develop\n HAVE_SETUPTOOLS = True\nexcept ImportError:\n from distutils.core import setup\n from distutils.command.sdist import sdist as sdist\n from distutils.command.install import install as install\n HAVE_SETUPTOOLS = False\nfrom gitsome.__init__ import __version__ as VERSION\n\n\nTABLES = ['xonsh/lexer_table.py', 'xonsh/parser_table.py']\n\n\ndef clean_tables():\n for f in TABLES:\n if os.path.isfile(f):\n os.remove(f)\n print('Remove ' + f)\n\n\ndef build_tables():\n print('Building lexer and parser tables.')\n sys.path.insert(0, os.path.dirname(__file__))\n from xonsh.parser import Parser\n Parser(lexer_table='lexer_table', yacc_table='parser_table',\n outputdir='xonsh')\n sys.path.pop(0)\n\n\nclass xinstall(install):\n def run(self):\n clean_tables()\n build_tables()\n install.run(self)\n\n\nclass xsdist(sdist):\n def make_release_tree(self, basedir, files):\n clean_tables()\n build_tables()\n sdist.make_release_tree(self, basedir, files)\n\n\nif HAVE_SETUPTOOLS:\n class xdevelop(develop):\n def run(self):\n clean_tables()\n build_tables()\n develop.run(self)\n\n\ndef main():\n if sys.version_info < (3, 5):\n print('gitsome requires at least Python 3.5.')\n sys.exit(1)\n try:\n if '--name' not in sys.argv:\n print(logo)\n except UnicodeEncodeError:\n pass\n skw = dict(\n name='gitsome',\n description='A Supercharged Git/Shell Autocompleter with GitHub Integration.', # NOQA\n license='Apache License 2.0',\n version=VERSION,\n author='Donne Martin',\n maintainer='Donne Martin',\n author_email='donne.martin@gmail.com',\n url='https://github.com/donnemartin/gitsome',\n platforms='Cross Platform',\n classifiers=[\n 'Intended Audience :: Developers',\n 'Intended Audience :: System Administrators',\n 'License :: OSI Approved :: Apache Software License',\n 'Natural Language :: English',\n 'Programming Language :: Python :: 3',\n 'Programming Language :: Python :: 3.5',\n 'Programming Language :: Python :: 3.6',\n 'Programming Language :: Python :: 3.7',\n 'Topic :: Software Development',\n 'Topic :: Software Development :: Libraries :: Python Modules',\n ],\n packages=find_packages(),\n scripts=['scripts/xonsh',\n 'scripts/xonsh.bat',\n 'scripts/gitsome',\n 'scripts/gitsome.bat'],\n cmdclass={'install': xinstall, 'sdist': xsdist},\n )\n if HAVE_SETUPTOOLS:\n skw['setup_requires'] = ['ply']\n skw['install_requires'] = [\n 'ply>=3.4,<4.0',\n 'prompt_toolkit>=2.0.0,<2.1.0',\n 'requests>=2.8.1,<3.0.0',\n 'colorama>=0.3.3,<1.0.0',\n 'click>=5.1,<7.0',\n 'pygments>=2.0.2,<3.0.0',\n 'feedparser>=5.2.1,<6.0.0',\n 'pytz>=2016.3,<2017.0',\n 'docopt>=0.6.2,<1.0.0',\n 'uritemplate.py>=1.0.0,<4.0.0',\n ],\n skw['entry_points'] = {\n 'pygments.lexers': ['gitsome = xonsh.pyghooks:XonshLexer',\n 'gitsomecon = xonsh.pyghooks:XonshConsoleLexer',\n ],\n 'console_scripts': ['gh = gitsome.main_cli:cli',\n ],\n }\n skw['cmdclass']['develop'] = xdevelop\n setup(**skw)\n\n\nlogo = ''\n\n\nif __name__ == '__main__':\n main()\n" }, { "path": "tests/__init__.py", "content": "# -*- coding: utf-8 -*-\n\n# Copyright 2015 Donne Martin. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"). You\n# may not use this file except in compliance with the License. A copy of\n# the License is located at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# or in the \"license\" file accompanying this file. This file is\n# distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF\n# ANY KIND, either express or implied. See the License for the specific\n# language governing permissions and limitations under the License.\n" }, { "path": "tests/compat.py", "content": "# -*- coding: utf-8 -*-\n\n# Copyright 2015 Donne Martin. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"). You\n# may not use this file except in compliance with the License. A copy of\n# the License is located at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# or in the \"license\" file accompanying this file. This file is\n# distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF\n# ANY KIND, either express or implied. See the License for the specific\n# language governing permissions and limitations under the License.\n\nimport sys\nif sys.version_info < (2, 7):\n import unittest2 as unittest\nelse:\n import unittest\n" }, { "path": "tests/data/__init__.py", "content": "# -*- coding: utf-8 -*-\n\n# Copyright 2015 Donne Martin. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"). You\n# may not use this file except in compliance with the License. A copy of\n# the License is located at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# or in the \"license\" file accompanying this file. This file is\n# distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF\n# ANY KIND, either express or implied. See the License for the specific\n# language governing permissions and limitations under the License.\n" }, { "path": "tests/data/email.py", "content": "# -*- coding: utf-8 -*-\n\n# Copyright 2015 Donne Martin. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"). You\n# may not use this file except in compliance with the License. A copy of\n# the License is located at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# or in the \"license\" file accompanying this file. This file is\n# distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF\n# ANY KIND, either express or implied. See the License for the specific\n# language governing permissions and limitations under the License.\n\nformatted_emails = u'\\x1b[35m 1. \\x1b[0mfoo@baz.com \\x1b[0m\\n\\x1b[32m Primary: True \\x1b[0m\\x1b[36mVerified: False \\x1b[0m\\n\\x1b[35m 2. \\x1b[0mbar@baz.com \\x1b[0m\\n\\x1b[32m Primary: False \\x1b[0m\\x1b[36mVerified: True \\x1b[0m\\n\\x1b[0m' # NOQA\n" }, { "path": "tests/data/emoji.py", "content": "# -*- coding: utf-8 -*-\n\n# Copyright 2015 Donne Martin. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"). You\n# may not use this file except in compliance with the License. A copy of\n# the License is located at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# or in the \"license\" file accompanying this file. This file is\n# distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF\n# ANY KIND, either express or implied. See the License for the specific\n# language governing permissions and limitations under the License.\n\nformatted_emojis = u'\\x1b[35m 1. \\x1b[0mdolls \\x1b[0m\\n\\x1b[35m 2. \\x1b[0mpalm_tree \\x1b[0m\\n\\x1b[35m 3. \\x1b[0muk \\x1b[0m\\n\\x1b[35m 4. \\x1b[0m100 \\x1b[0m\\n\\x1b[35m 5. \\x1b[0mbaby_chick \\x1b[0m\\n\\x1b[0m' # NOQA\n" }, { "path": "tests/data/events.py", "content": "# -*- coding: utf-8 -*-\n\n# Copyright 2015 Donne Martin. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"). You\n# may not use this file except in compliance with the License. A copy of\n# the License is located at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# or in the \"license\" file accompanying this file. This file is\n# distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF\n# ANY KIND, either express or implied. See the License for the specific\n# language governing permissions and limitations under the License.\n\nformatted_events = u'\\x1b[35m 1. \\x1b[0mdonnemartin \\x1b[0m\\x1b[32mcommented on commit \\x1b[0m\\x1b[36mAAA23e2\\x1b[0m\\x1b[32m at \\x1b[0m\\x1b[36muser1/repo1\\x1b[0m\\x1b[33m (just now)\\x1b[0m\\n \\x1b[36m\\x1b[0mfoo\\x1b[0m\\n\\x1b[35m 2. \\x1b[0mdonnemartin \\x1b[0m\\x1b[32mcreated\\x1b[0m\\x1b[32m branch\\x1b[0m\\x1b[36m master\\x1b[0m\\x1b[32m at \\x1b[0m\\x1b[36muser1/repo1\\x1b[0m\\x1b[33m (just now)\\x1b[0m\\n \\x1b[36m\\x1b[0m\\x1b[0m\\n\\x1b[35m 3. \\x1b[0mdonnemartin \\x1b[0m\\x1b[32mfollowed \\x1b[0m\\x1b[36muser1/repo1\\x1b[0m\\x1b[33m (just now)\\x1b[0m\\n\\x1b[35m 4. \\x1b[0mdonnemartin \\x1b[0m\\x1b[32mforked\\x1b[0m\\x1b[36m user1/repo1\\x1b[0m\\x1b[33m (just now)\\x1b[0m\\n\\x1b[35m 5. \\x1b[0mdonnemartin \\x1b[0m\\x1b[32mcommented on \\x1b[0m\\x1b[36muser1/repo1#1\\x1b[0m\\x1b[33m (just now)\\x1b[0m\\n \\x1b[36m\\x1b[0mfoo\\x1b[0m\\n \\x1b[36m\\x1b[0mfoo\\x1b[0m\\n\\x1b[35m 6. \\x1b[0mdonnemartin \\x1b[0m\\x1b[32mclosed issue \\x1b[0m\\x1b[36muser1/repo1#1\\x1b[0m\\x1b[33m (just now)\\x1b[0m\\n \\x1b[36m\\x1b[0mfoo\\x1b[0m\\n\\x1b[35m 7. \\x1b[0mdonnemartin \\x1b[0m\\x1b[32mclosed pull request \\x1b[0m\\x1b[36muser1/repo1#1\\x1b[0m\\x1b[33m (just now)\\x1b[0m\\n \\x1b[36m\\x1b[0mfoo\\x1b[0m\\n\\x1b[35m 8. \\x1b[0mdonnemartin \\x1b[0m\\x1b[32mpushed to\\x1b[0m\\x1b[36m master\\x1b[0m\\x1b[32m at \\x1b[0m\\x1b[36muser1/repo1\\x1b[0m\\x1b[33m (just now)\\x1b[0m\\n \\x1b[36m5ee4d1b: \\x1b[0m\\x1b[0mFix GitHubCli class docstring\\x1b[0m\\n \\x1b[36mfc2309b: \\x1b[0m\\x1b[0mUpdate gh configure docstring\\x1b[0m\\n \\x1b[36mdde19b7: \\x1b[0m\\x1b[0mUpdate gh create-comment docstring\\x1b[0m\\n\\x1b[35m 9. \\x1b[0mdonnemartin \\x1b[0m\\x1b[32mreleased \\x1b[0m\\x1b[36m0.5.0 \\x1b[0m\\x1b[32mat \\x1b[0m\\x1b[36muser1/repo1\\x1b[0m\\x1b[33m (just now)\\x1b[0m\\n\\x1b[0m'\n" }, { "path": "tests/data/gitignores.py", "content": "# -*- coding: utf-8 -*-\n\n# Copyright 2015 Donne Martin. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"). You\n# may not use this file except in compliance with the License. A copy of\n# the License is located at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# or in the \"license\" file accompanying this file. This file is\n# distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF\n# ANY KIND, either express or implied. See the License for the specific\n# language governing permissions and limitations under the License.\n\nformatted_gitignores = u'\\x1b[35m 1. \\x1b[0mActionscript \\x1b[0m\\n\\x1b[35m 2. \\x1b[0mAda \\x1b[0m\\n\\x1b[35m 3. \\x1b[0mAgda \\x1b[0m\\n\\x1b[35m 4. \\x1b[0mAndroid \\x1b[0m\\n\\x1b[35m 5. \\x1b[0mAppEngine \\x1b[0m\\n\\x1b[0m'\nformatted_gitignores_tip = u' Run the following command to view or download a .gitignore file:\\n View: gh gitignore Python\\n Download: gh gitignore Python > .gitignore\\n'\n" }, { "path": "tests/data/issue.py", "content": "# -*- coding: utf-8 -*-\n\n# Copyright 2015 Donne Martin. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"). You\n# may not use this file except in compliance with the License. A copy of\n# the License is located at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# or in the \"license\" file accompanying this file. This file is\n# distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF\n# ANY KIND, either express or implied. See the License for the specific\n# language governing permissions and limitations under the License.\n\nformatted_issues = u'\\x1b[35m 1. \\x1b[0mtitle1 \\x1b[0m@user2 \\x1b[0m\\x1b[35m(user1/repo1/issues/1)\\x1b[0m\\n\\x1b[32m State: open \\x1b[0m\\x1b[36mComments: 1 \\x1b[0m\\x1b[33mAssignee: user1 \\x1b[0m\\n\\x1b[35m 2. \\x1b[0mtitle2 \\x1b[0m@user2 \\x1b[0m\\x1b[35m(user1/repo1/issues/2)\\x1b[0m\\n\\x1b[32m State: open \\x1b[0m\\x1b[36mComments: 1 \\x1b[0m\\x1b[33mAssignee: user1 \\x1b[0m\\n\\x1b[35m 3. \\x1b[0mtitle3 \\x1b[0m@user2 \\x1b[0m\\x1b[35m(user1/repo1/issues/3)\\x1b[0m\\n\\x1b[32m State: open \\x1b[0m\\x1b[36mComments: 1 \\x1b[0m\\x1b[33mAssignee: user1 \\x1b[0m\\n View the page for \\x1b[0m\\x1b[35m1 through \\x1b[0m\\x1b[35m3\\x1b[0m with the following command:\\n\\x1b[0m\\x1b[35m gh view [#] \\x1b[0moptional: [-b/--browser] [--help]\\n\\x1b[0m\\x1b[0m'\nformatted_pull_requests = u'\\x1b[35m 1. \\x1b[0mtitle1 \\x1b[0m@user2 \\x1b[0m\\x1b[35m(user1/repo1/issues/1)\\x1b[0m\\n\\x1b[32m State: open \\x1b[0m\\x1b[36mComments: 1 \\x1b[0m\\x1b[33mAssignee: user1 \\x1b[0m\\n\\x1b[35m 2. \\x1b[0mtitle1 \\x1b[0m@user2 \\x1b[0m\\x1b[35m(user1/repo1/issues/1)\\x1b[0m\\n\\x1b[32m State: open \\x1b[0m\\x1b[36mComments: 1 \\x1b[0m\\x1b[33mAssignee: user1 \\x1b[0m\\n\\x1b[35m 3. \\x1b[0mtitle1 \\x1b[0m@user2 \\x1b[0m\\x1b[35m(user1/repo1/issues/1)\\x1b[0m\\n\\x1b[32m State: open \\x1b[0m\\x1b[36mComments: 1 \\x1b[0m\\x1b[33mAssignee: user1 \\x1b[0m\\n View the page for \\x1b[0m\\x1b[35m1 through \\x1b[0m\\x1b[35m3\\x1b[0m with the following command:\\n\\x1b[0m\\x1b[35m gh view [#] \\x1b[0moptional: [-b/--browser] [--help]\\n\\x1b[0m\\x1b[0m'\n" }, { "path": "tests/data/license.py", "content": "# -*- coding: utf-8 -*-\n\n# Copyright 2015 Donne Martin. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"). You\n# may not use this file except in compliance with the License. A copy of\n# the License is located at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# or in the \"license\" file accompanying this file. This file is\n# distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF\n# ANY KIND, either express or implied. See the License for the specific\n# language governing permissions and limitations under the License.\n\nformatted_licenses = u'\\x1b[35m 1. \\x1b[0mmit \\x1b[0m\\x1b[32m((MIT License))\\x1b[0m\\n\\x1b[35m 2. \\x1b[0mgpl-2.0 \\x1b[0m\\x1b[32m((GNU General Public License v2.0))\\x1b[0m\\n\\x1b[35m 3. \\x1b[0mbsd-2-clause \\x1b[0m\\x1b[32m((BSD 2-clause \"Simplified\" License))\\x1b[0m\\n\\x1b[35m 4. \\x1b[0misc \\x1b[0m\\x1b[32m((ISC License))\\x1b[0m\\n\\x1b[35m 5. \\x1b[0mepl-1.0 \\x1b[0m\\x1b[32m((Eclipse Public License 1.0))\\x1b[0m\\n\\x1b[0m'\nformatted_licenses_tip = u' Run the following command to view or download a LICENSE file:\\n gh license apache-2.0\\n gh license apache-2.0 > LICENSE\\n'\n" }, { "path": "tests/data/markdown.py", "content": "# -*- coding: utf-8 -*-\n\n# Copyright 2015 Donne Martin. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"). You\n# may not use this file except in compliance with the License. A copy of\n# the License is located at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# or in the \"license\" file accompanying this file. This file is\n# distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF\n# ANY KIND, either express or implied. See the License for the specific\n# language governing permissions and limitations under the License.\n\nraw_markdown = u\"\"\"\n[Skip to content](#start-of-content)\\n\\n[ ](https://github.com/)\\n\\n[Sign up](/join) [Sign in](/login?return_to=%2Fdonnemartin%2Fsaws)\\n\\nThis repository\\n\\n * [Explore](/explore)\\n * [Features](/features)\\n * [Enterprise](https://enterprise.github.com/)\\n * [Pricing](/pricing)\\n\\n * [ Watch ](/login?return_to=%2Fdonnemartin%2Fsaws) [ 80 ](/donnemartin/saws/watchers)\\n * [ Star ](/login?return_to=%2Fdonnemartin%2Fsaws) [ 2,428 ](/donnemartin/saws/stargazers)\\n * [ Fork ](/login?return_to=%2Fdonnemartin%2Fsaws) [ 77 ](/donnemartin/saws/network)\\n\\n# [donnemartin](/donnemartin)/**[saws](/donnemartin/saws)** ![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\\n\\n * [ Code ![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) ](/donnemartin/saws)\\n * [ Issues ![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) ](/donnemartin/saws/issues)\\n * [ Pull requests ![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) ](/donnemartin/saws/pulls)\\n\\n * [ Pulse ![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) ](/donnemartin/saws/pulse)\\n * [ Graphs ![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) ](/donnemartin/saws/graphs)\\n\\n### HTTPS clone URL\\n\\n### Subversion checkout URL\\n\\nYou can clone with \\n\\nHTTPS or \\n\\nSubversion. [ ](https://help.github.com/articles/which-remote-url-should-i-use)\\n\\n[ Download ZIP ](/donnemartin/saws/archive/master.zip \"Download the contents of donnemartin/saws as a zip file\" )\\n\\nA Supercharged AWS Command Line Interface (CLI). \\n\\n * [ 544 commits ](/donnemartin/saws/commits/master)\\n * [ 2 branches ](/donnemartin/saws/branches)\\n * [ 7 releases ](/donnemartin/saws/releases)\\n * [ 5 contributors ](/donnemartin/saws/graphs/contributors)\\n\\n 1. [ Python 99.4% ](/donnemartin/saws/search?l=python)\\n 2. [ Shell 0.6% ](/donnemartin/saws/search?l=bash)\\n\\nPython Shell\\n\\n[ ](/donnemartin/saws/find/master) [ ](/donnemartin/saws/compare)\\n\\n_Branch:_ master\\n\\nSwitch branches/tags\\n\\n * [Branches](#)\\n * [Tags](#)\\n\\n[ develop ](/donnemartin/saws/tree/develop) [ master ](/donnemartin/saws/tree/master)\\n\\nNothing to show\\n\\n[0.3.2](/donnemartin/saws/tree/0.3.2 \"0.3.2\" )\\n\\n[0.3.1](/donnemartin/saws/tree/0.3.1 \"0.3.1\" )\\n\\n[0.3.0](/donnemartin/saws/tree/0.3.0 \"0.3.0\" )\\n\\n[0.2.1](/donnemartin/saws/tree/0.2.1 \"0.2.1\" )\\n\\n[0.2.0](/donnemartin/saws/tree/0.2.0 \"0.2.0\" )\\n\\n[0.1.1](/donnemartin/saws/tree/0.1.1 \"0.1.1\" )\\n\\n[0.1.0](/donnemartin/saws/tree/0.1.0 \"0.1.0\" )\\n\\nNothing to show\\n\\n[saws](/donnemartin/saws)/\\n\\nLatest commit [ 07c1d7d ](/donnemartin/saws/commit/07c1d7df22c9032b993d18f22162b91fb16e86d6) Nov 30, 2015 ![@donnemartin](https://avatars3.githubusercontent.com/u/5458997?v=3&s=40) [donnemartin](/donnemartin) [Fix](/donnemartin/saws/commit/07c1d7df22c9032b993d18f22162b91fb16e86d6 \"Fix #66: Remove docs/build from the source repository.\" ) [#66](https://github.com/donnemartin/saws/issues/66 \"Remove docs/build from the source repository.\" )[: Remove docs/build from the source repository.](/donnemartin/saws/commit/07c1d7df22c9032b993d18f22162b91fb16e86d6 \"Fix #66: Remove docs/build from the source repository.\" )\\n\\n[Permalink](/donnemartin/saws/tree/07c1d7df22c9032b993d18f22162b91fb16e86d6) | Failed to load latest commit information. \\n---|--- \\n![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) | [docs](/donnemartin/saws/tree/master/docs \"docs\" ) | [Fix](/donnemartin/saws/commit/07c1d7df22c9032b993d18f22162b91fb16e86d6 \"Fix #66: Remove docs/build from the source repository.\" ) [#66](https://github.com/donnemartin/saws/issues/66 \"Remove docs/build from the source repository.\" )[: Remove docs/build from the source repository.](/donnemartin/saws/commit/07c1d7df22c9032b993d18f22162b91fb16e86d6 \"Fix #66: Remove docs/build from the source repository.\" ) | Nov 30, 2015 \\n![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) | [saws](/donnemartin/saws/tree/master/saws \"saws\" ) | [Merge branch \\'master\\' into develop](/donnemartin/saws/commit/4d174fe79f61eeeda363c0b03e9e53392da671ef \"Merge branch \\'master\\' into develop\\n\\n* master:\\n Removed mention of scripts/release_pypi.sh, which is no longer used.\\n Updated commands.\" ) | Nov 30, 2015 \\n![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) | [scripts](/donnemartin/saws/tree/master/scripts \"scripts\" ) | [Updated CHANGELOG to README script to use the new CHANGELOG.rst.](/donnemartin/saws/commit/5c03b2cc6882590aa567a98cc8dceae780d1428c \"Updated CHANGELOG to README script to use the new CHANGELOG.rst.\" ) | Oct 16, 2015 \\n![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) | [tests](/donnemartin/saws/tree/master/tests \"tests\" ) | [Fix linter line length issues.](/donnemartin/saws/commit/bf1afa4c73bf1a7dd462cb0c92f945d2b7d7cf5a \"Fix linter line length issues.\" ) | Oct 15, 2015 \\n![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) | [.gitchangelog.rc](/donnemartin/saws/blob/master/.gitchangelog.rc \".gitchangelog.rc\" ) | [Added gitchangelog.](/donnemartin/saws/commit/7347e0cc55a9ac75368152b9683881d41982a943 \"Added gitchangelog.\" ) | Sep 15, 2015 \\n![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) | [.gitignore](/donnemartin/saws/blob/master/.gitignore \".gitignore\" ) | [Fix](/donnemartin/saws/commit/07c1d7df22c9032b993d18f22162b91fb16e86d6 \"Fix #66: Remove docs/build from the source repository.\" ) [#66](https://github.com/donnemartin/saws/issues/66 \"Remove docs/build from the source repository.\" )[: Remove docs/build from the source repository.](/donnemartin/saws/commit/07c1d7df22c9032b993d18f22162b91fb16e86d6 \"Fix #66: Remove docs/build from the source repository.\" ) | Nov 30, 2015 \\n![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) | [.travis.yml](/donnemartin/saws/blob/master/.travis.yml \".travis.yml\" ) | [Revert Python 3.5 build and test support due to:](/donnemartin/saws/commit/36e073de59b0fbe872fb30c80f3521c3b558b37f \"Revert Python 3.5 build and test support due to: https://github.com/travis-ci/travis-ci/issues/4794.\" ) [travis-ci/travis-ci#\\u2026](https://github.com/travis-ci/travis-ci/issues/4794 \"Tox builds for Python 3.5 fail\" ) | Sep 23, 2015 \\n![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) | [CHANGELOG.md](/donnemartin/saws/blob/master/CHANGELOG.md \"CHANGELOG.md\" ) | [Prepped version 0.3.2.](/donnemartin/saws/commit/0408df93b135542c2388fe60444ba4651feec481 \"Prepped version 0.3.2.\" ) | Oct 16, 2015 \\n![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) | [CHANGELOG.rst](/donnemartin/saws/blob/master/CHANGELOG.rst \"CHANGELOG.rst\" ) | [Prepped version 0.3.2.](/donnemartin/saws/commit/0408df93b135542c2388fe60444ba4651feec481 \"Prepped version 0.3.2.\" ) | Oct 16, 2015 \\n![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) | [CHECKLIST.md](/donnemartin/saws/blob/master/CHECKLIST.md \"CHECKLIST.md\" ) | [Removed mention of scripts/release_pypi.sh, which is no longer used.](/donnemartin/saws/commit/ba73311f8dd04e9a1eb159ff4d8c741a22027e50 \"Removed mention of scripts/release_pypi.sh, which is no longer used.\" ) | Nov 16, 2015 \\n![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) | [CONTRIBUTING.md](/donnemartin/saws/blob/master/CONTRIBUTING.md \"CONTRIBUTING.md\" ) | [Add note about AWS collaboration.](/donnemartin/saws/commit/9fb37599835024ef7c99a7f00822aab54904fa5b \"Add note about AWS collaboration.\" ) | Nov 30, 2015 \\n![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) | [Dockerfile](/donnemartin/saws/blob/master/Dockerfile \"Dockerfile\" ) | [Add Dockerfile](/donnemartin/saws/commit/7e185a10918059f0c763b1437d540b78432ea1a9 \"Add Dockerfile\" ) | Sep 26, 2015 \\n![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) | [INSTALLATION.md](/donnemartin/saws/blob/master/INSTALLATION.md \"INSTALLATION.md\" ) | [Added INSTALLATION.md.](/donnemartin/saws/commit/8317c4c92441f40ce20dad5ced97fe090f77e93e \"Added INSTALLATION.md.\" ) | Oct 3, 2015 \\n![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) | [LICENSE.txt](/donnemartin/saws/blob/master/LICENSE.txt \"LICENSE.txt\" ) | [Added LICENSE.](/donnemartin/saws/commit/1166ffddffffbd981868d700df1f8ecc2061847f \"Added LICENSE.\" ) | Sep 19, 2015 \\n![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) | [MANIFEST.in](/donnemartin/saws/blob/master/MANIFEST.in \"MANIFEST.in\" ) | [Updated MANIFEST.in.](/donnemartin/saws/commit/cba64a756e8dd193bb93083ee7aecd4316c2c9b4 \"Updated MANIFEST.in.\" ) | Sep 19, 2015 \\n![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) | [README.md](/donnemartin/saws/blob/master/README.md \"README.md\" ) | [Fix #todo-add-more-resources anchor link.](/donnemartin/saws/commit/8427a6432c58aa9f95bf4fe26ab76a13645a8261 \"Fix #todo-add-more-resources anchor link.\" ) | Nov 30, 2015 \\n![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) | [appveyor.yml](/donnemartin/saws/blob/master/appveyor.yml \"appveyor.yml\" ) | [Added Appveyor support, inspired by](/donnemartin/saws/commit/f756340ad4f9f4d47860644b0e677f49dac575d4 \"Added Appveyor support, inspired by https://github.com/audreyr/cookiecutter/blob/master/appveyor.yml.\" ) [https://github.com/audreyr/cookie\\u2026](https://github.com/audreyr/cookiecutter/blob/master/appveyor.yml) | Sep 28, 2015 \\n![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) | [requirements-dev.txt](/donnemartin/saws/blob/master/requirements-dev.txt \"requirements-dev.txt\" ) | [Added Sphinx-PyPI-upload support.](/donnemartin/saws/commit/7068598aa117cc16d6e32000afa05963d3d76093 \"Added Sphinx-PyPI-upload support.\" ) | Sep 19, 2015 \\n![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) | [setup.cfg](/donnemartin/saws/blob/master/setup.cfg \"setup.cfg\" ) | [Added Sphinx-PyPI-upload support.](/donnemartin/saws/commit/7068598aa117cc16d6e32000afa05963d3d76093 \"Added Sphinx-PyPI-upload support.\" ) | Sep 19, 2015 \\n![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) | [setup.py](/donnemartin/saws/blob/master/setup.py \"setup.py\" ) | [Fixed](/donnemartin/saws/commit/1628d3efda27ba638833b54bdc3124db68a7fb30 \"Fixed #53: Shortcuts should be placed separately in a .saws.shortcuts file.\" ) [#53](https://github.com/donnemartin/saws/issues/53 \"Shortcuts should be placed separately in a .saws.shortcuts file\" )[: Shortcuts should be placed separately in a .saws.shortcuts\\u2026](/donnemartin/saws/commit/1628d3efda27ba638833b54bdc3124db68a7fb30 \"Fixed #53: Shortcuts should be placed separately in a .saws.shortcuts file.\" ) | Oct 14, 2015 \\n![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) | [tox.ini](/donnemartin/saws/blob/master/tox.ini \"tox.ini\" ) | [Revert Python 3.5 build and test support due to:](/donnemartin/saws/commit/36e073de59b0fbe872fb30c80f3521c3b558b37f \"Revert Python 3.5 build and test support due to: https://github.com/travis-ci/travis-ci/issues/4794.\" ) [travis-ci/travis-ci#\\u2026](https://github.com/travis-ci/travis-ci/issues/4794 \"Tox builds for Python 3.5 fail\" ) | Sep 23, 2015 \\n \\n### README.md \\n\\n[![](https://camo.githubusercontent.com/2af72023269b0b320adf4ec55576435f5c8b79c4/687474703a2f2f692e696d6775722e636f6d2f767a43357a6d412e676966)](https://camo.githubusercontent.com/2af72023269b0b320adf4ec55576435f5c8b79c4/687474703a2f2f692e696d6775722e636f6d2f767a43357a6d412e676966)\\n\\n[![Build Status](https://camo.githubusercontent.com/c1d6d1e8f7cdac1e6a82bf900832bbbe52d61e95/68747470733a2f2f7472617669732d63692e6f72672f646f6e6e656d617274696e2f736177732e7376673f6272616e63683d6d6173746572)](https://travis-ci.org/donnemartin/saws) [![Documentation Status](https://camo.githubusercontent.com/41afe5f96e9f915ecd9706bdc7dfd19518eaf05a/68747470733a2f2f72656164746865646f63732e6f72672f70726f6a656374732f736177732f62616467652f3f76657273696f6e3d6c6174657374)](http://saws.readthedocs.org/en/latest/?badge=latest) [![Dependency Status](https://camo.githubusercontent.com/11e16ba1560f0232928945436bf86340615d85ae/68747470733a2f2f67656d6e617369756d2e636f6d2f646f6e6e656d617274696e2f736177732e737667)](https://gemnasium.com/donnemartin/saws) [![Codecov](https://camo.githubusercontent.com/e24a50e9dd63768d90d26a9f6be1cf3d0c3b0230/68747470733a2f2f696d672e736869656c64732e696f2f636f6465636f762f632f6769746875622f646f6e6e656d617274696e2f736177732e737667)](https://codecov.io/github/donnemartin/saws/saws)\\n\\n[![PyPI version](https://camo.githubusercontent.com/818edda38216a62a4bb36557372877febec35086/68747470733a2f2f62616467652e667572792e696f2f70792f736177732e737667)](http://badge.fury.io/py/saws) [![PyPI](https://camo.githubusercontent.com/04574f61c5dff98734bd6a013cc9217b9a428e47/68747470733a2f2f696d672e736869656c64732e696f2f707970692f707976657273696f6e732f736177732e737667)](https://pypi.python.org/pypi/saws/) [![License](https://camo.githubusercontent.com/8bf81d4b6a63eaf6f712e3092fe7238f8716e615/687474703a2f2f696d672e736869656c64732e696f2f3a6c6963656e73652d6170616368652d626c75652e737667)](http://www.apache.org/licenses/LICENSE-2.0.html)\\n\\n# [](#saws)SAWS\\n\\n## [](#motivation)Motivation\\n\\n### [](#aws-cli)AWS CLI\\n\\nAlthough the [AWS CLI](https://github.com/aws/aws-cli) is a great resource to manage your AWS-powered services, it\\'s **tough to remember usage** of:\\n\\n * 50+ top-level commands\\n * 1400+ subcommands\\n * Countless command-specific options\\n * Resources such as instance tags and buckets\\n\\n### [](#saws-a-supercharged-aws-cli)SAWS: A Supercharged AWS CLI\\n\\n`SAWS` aims to **supercharge** the AWS CLI with features focusing on:\\n\\n * **Improving ease-of-use**\\n * **Increasing productivity**\\n\\nUnder the hood, `SAWS` is **powered by the AWS CLI** and supports the **same commands** and **command structure**.\\n\\n`SAWS` and `AWS CLI` Usage:\\n \\n \\n aws [parameters] [options]\\n \\n\\n`SAWS` features:\\n\\n * Auto-completion of: \\n * Commands\\n * Subcommands\\n * Options\\n * Auto-completion of resources: \\n * Bucket names\\n * Instance ids\\n * Instance tags\\n * [More coming soon!](#todo-add-more-resources)\\n * Customizable shortcuts\\n * Fuzzy completion of resources and shortcuts\\n * Fish-style auto-suggestions\\n * Syntax and output highlighting\\n * Execution of shell commands\\n * Command history\\n * Contextual help\\n * Toolbar options\\n\\n`SAWS` is available for Mac, Linux, Unix, and [Windows](#windows-support).\\n\\n[![](https://camo.githubusercontent.com/571a2d19a969447b477713d54d1d3ba46d7418ee/687474703a2f2f692e696d6775722e636f6d2f456f31327139542e706e67)](https://camo.githubusercontent.com/571a2d19a969447b477713d54d1d3ba46d7418ee/687474703a2f2f692e696d6775722e636f6d2f456f31327139542e706e67)\\n\\n## [](#index)Index\\n\\n### [](#features)Features\\n\\n * [Syntax and Output Highlighting](#syntax-and-output-highlighting)\\n * [Auto-Completion of Commands, Subcommands, and Options](#auto-completion-of-commands-subcommands-and-options)\\n * [Auto-Completion of AWS Resources](#auto-completion-of-aws-resources)\\n * [S3 Buckets](#s3-buckets)\\n * [EC2 Instance Ids](#ec2-instance-ids)\\n * [EC2 Instance Tags](#ec2-instance-tags)\\n * [TODO: Add More Resources](#todo-add-more-resources)\\n * [Customizable Shortcuts](#customizable-shortcuts)\\n * [Fuzzy Resource and Shortcut Completion](#fuzzy-resource-and-shortcut-completion)\\n * [Fish-Style Auto-Suggestions](#fish-style-auto-suggestions)\\n * [Executing Shell Commands](#executing-shell-commands)\\n * [Command History](#command-history)\\n * [Contextual Help](#contextual-help)\\n * [Contextual Command Line Help](#contextual-command-line-help)\\n * [Contextual Web Docs](#contextual-web-docs)\\n * [Toolbar Options](#toolbar-options)\\n * [Windows Support](#windows-support)\\n\\n### [](#installation-and-tests)Installation and Tests\\n\\n * [Installation](#installation)\\n * [Pip Installation](#pip-installation)\\n * [Virtual Environment and Docker Installation](#virtual-environment-and-docker-installation)\\n * [AWS Credentials and Named Profiles](#aws-credentials-and-named-profiles)\\n * [Supported Python Versions](#supported-python-versions)\\n * [Supported Platforms](#supported-platforms)\\n * [Developer Installation](#developer-installation)\\n * [Continuous Integration](#continuous-integration)\\n * [Dependencies Management](#dependencies-management)\\n * [Unit Tests and Code Coverage](#unit-tests-and-code-coverage)\\n * [Documentation](#documentation)\\n\\n### [](#misc)Misc\\n\\n * [Contributing](#contributing)\\n * [Credits](#credits)\\n * [Contact Info](#contact-info)\\n * [License](#license)\\n\\n## [](#syntax-and-output-highlighting)Syntax and Output Highlighting\\n\\n[![](https://camo.githubusercontent.com/c527a4c00862bb1fe47c272a3c3563418f06866f/687474703a2f2f692e696d6775722e636f6d2f785144707737302e706e67)](https://camo.githubusercontent.com/c527a4c00862bb1fe47c272a3c3563418f06866f/687474703a2f2f692e696d6775722e636f6d2f785144707737302e706e67)\\n\\nYou can control which theme to load for syntax highlighting by updating your [~/.sawsrc](https://github.com/donnemartin/saws/blob/master/saws/sawsrc) file:\\n \\n \\n # Visual theme. Possible values: manni, igor, xcode, vim, autumn, vs, rrt,\\n # native, perldoc, borland, tango, emacs, friendly, monokai, paraiso-dark,\\n # colorful, murphy, bw, pastie, paraiso-light, trac, default, fruity\\n theme = vim\\n \\n\\n## [](#auto-completion-of-commands-subcommands-and-options)Auto-Completion of Commands, Subcommands, and Options\\n\\n`SAWS` provides smart autocompletion as you type. Entering the following command will interactively list and auto-complete all subcommands **specific only** to `ec2`:\\n \\n \\n aws ec2\\n \\n\\n[![](https://camo.githubusercontent.com/df93a08e1371694a8086f3b45ba6e5ec955212e2/687474703a2f2f692e696d6775722e636f6d2f5032744c3976572e706e67)](https://camo.githubusercontent.com/df93a08e1371694a8086f3b45ba6e5ec955212e2/687474703a2f2f692e696d6775722e636f6d2f5032744c3976572e706e67)\\n\\n## [](#auto-completion-of-aws-resources)Auto-Completion of AWS Resources\\n\\nIn addition to the default commands, subcommands, and options the AWS CLI provides, `SAWS` supports auto-completion of your AWS resources. Currently, bucket names, instance ids, and instance tags are included, with additional support for more resources [under development](#todo-add-more-resources).\\n\\n### [](#s3-buckets)S3 Buckets\\n\\nOption for `s3api`:\\n \\n \\n --bucket\\n \\n\\nSample Usage:\\n \\n \\n aws s3api get-bucket-acl --bucket\\n \\n\\nSyntax for `s3`:\\n \\n \\n s3://\\n \\n\\nSample Usage:\\n \\n \\n aws s3 ls s3://\\n \\n\\nNote: The example below demonstrates the use of [fuzzy resource completion](/donnemartin/saws/blob/master/fuzzy-resource-and-shortcutcompletion):\\n\\n[![](https://camo.githubusercontent.com/756f14d2ed77a7a9a3da9b0d9501cab9b7433e36/687474703a2f2f692e696d6775722e636f6d2f333943415335542e706e67)](https://camo.githubusercontent.com/756f14d2ed77a7a9a3da9b0d9501cab9b7433e36/687474703a2f2f692e696d6775722e636f6d2f333943415335542e706e67)\\n\\n### [](#ec2-instance-ids)EC2 Instance Ids\\n\\nOption for `ec2`:\\n \\n \\n --instance-ids\\n \\n\\nSample Usage:\\n \\n \\n aws ec2 describe-instances --instance-ids\\n aws ec2 ls --instance-ids\\n \\n\\nNote: The `ls` command demonstrates the use of [customizable shortcuts](#customizable-shortcuts):\\n\\n[![](https://camo.githubusercontent.com/d4b0298e0ecd0afc1dac0d62617a4be53a44f598/687474703a2f2f692e696d6775722e636f6d2f6a46794353586c2e706e67)](https://camo.githubusercontent.com/d4b0298e0ecd0afc1dac0d62617a4be53a44f598/687474703a2f2f692e696d6775722e636f6d2f6a46794353586c2e706e67)\\n\\n### [](#ec2-instance-tags)EC2 Instance Tags\\n\\nOption for `ec2`:\\n \\n \\n --ec2-tag-key\\n --ec2-tag-value\\n \\n\\nSample Usage:\\n \\n \\n aws ec2 ls --ec2-tag-key\\n aws ec2 ls --ec2-tag-value\\n \\n\\n**Tags support wildcards** with the `*` character.\\n\\nNote: `ls`, `--ec2-tag-value`, and `--ec2-tag-key` demonstrate the use of [customizable shortcuts](#customizable-shortcuts):\\n\\n[![](https://camo.githubusercontent.com/593e9fc80cc6e1fb8e0fd7324cb4cb3a1b51bf2f/687474703a2f2f692e696d6775722e636f6d2f56494b7747335a2e706e67)](https://camo.githubusercontent.com/593e9fc80cc6e1fb8e0fd7324cb4cb3a1b51bf2f/687474703a2f2f692e696d6775722e636f6d2f56494b7747335a2e706e67)\\n\\n### [](#todo-add-more-resources)TODO: Add More Resources\\n\\nFeel free to [submit an issue or a pull request](#contributions) if you\\'d like support for additional resources.\\n\\n## [](#customizable-shortcuts)Customizable Shortcuts\\n\\nThe [~/.saws.shortcuts](https://github.com/donnemartin/saws/blob/master/saws/saws.shortcuts) file contains shortcuts that you can modify. It comes pre-populated with several [handy shortcuts](https://github.com/donnemartin/saws/blob/master/saws/saws.shortcuts) out of the box. You can combine shortcuts with [fuzzy completion](#fuzzy-resource-and-shortcut-completion) for even less keystrokes. Below are a few examples.\\n\\nList all EC2 instances:\\n \\n \\n aws ec2 ls\\n \\n\\nList all running EC2 instances:\\n \\n \\n aws ec2 ls --ec2-state running # fuzzy shortcut: aws ecstate\\n \\n\\n[![](https://camo.githubusercontent.com/7b56fd124f76cf2a9573de7b963bbb791f4cae16/687474703a2f2f692e696d6775722e636f6d2f6a594645736f4d2e706e67)](https://camo.githubusercontent.com/7b56fd124f76cf2a9573de7b963bbb791f4cae16/687474703a2f2f692e696d6775722e636f6d2f6a594645736f4d2e706e67)\\n\\nList all EC2 instances with a matching tag (supports wildcards `*`):\\n \\n \\n aws ec2 ls --ec2-tag-key # fuzzy shortcut: aws ectagk\\n aws ec2 ls --ec2-tag-value # fuzzy shortcut: aws ectagv\\n \\n\\n[![](https://camo.githubusercontent.com/c45874a55d4ac6485ba78c6d4a91cc1d928703a4/687474703a2f2f692e696d6775722e636f6d2f505375775549772e706e67)](https://camo.githubusercontent.com/c45874a55d4ac6485ba78c6d4a91cc1d928703a4/687474703a2f2f692e696d6775722e636f6d2f505375775549772e706e67)\\n\\nList EC2 instance with matching id:\\n \\n \\n aws ec2 ls --instance-ids # fuzzy shortcut: aws eclsi\\n \\n\\n[![](https://camo.githubusercontent.com/48136027017398a300b9fe968e50e2e7c2a8b18b/687474703a2f2f692e696d6775722e636f6d2f774763554373612e706e67)](https://camo.githubusercontent.com/48136027017398a300b9fe968e50e2e7c2a8b18b/687474703a2f2f692e696d6775722e636f6d2f774763554373612e706e67)\\n\\nList all DynamoDB tables:\\n \\n \\n aws dynamodb ls # fuzzy shortcut: aws dls\\n \\n\\nList all EMR clusters:\\n \\n \\n aws emr ls # fuzzy shortcut: aws emls\\n \\n\\nAdd/remove/modify shortcuts in your [~/.saws.shortcuts](https://github.com/donnemartin/saws/blob/master/saws/shortcuts) file to suit your needs.\\n\\nFeel free to submit:\\n\\n * An issue to request additional shortcuts\\n * A pull request if you\\'d like to share your shortcuts (see [contributing guidelines](#contributions))\\n\\n### [](#fuzzy-resource-and-shortcut-completion)Fuzzy Resource and Shortcut Completion\\n\\nTo toggle fuzzy completion of AWS resources and shortcuts, use `F3` key.\\n\\nSample fuzzy shortcuts to start and stop EC2 instances:\\n \\n \\n aws ecstop\\n aws ecstart\\n \\n\\nNote: Fuzzy completion currently only works with AWS [resources](#auto-completion-of-aws-resources) and [shortcuts](/donnemartin/saws/blob/master/customizable-shortcuts).\\n\\n[![](https://camo.githubusercontent.com/a918a4d70ca61189149746e052f5bb2ffec9ae65/687474703a2f2f692e696d6775722e636f6d2f374f76464843772e706e67)](https://camo.githubusercontent.com/a918a4d70ca61189149746e052f5bb2ffec9ae65/687474703a2f2f692e696d6775722e636f6d2f374f76464843772e706e67)\\n\\n### [](#fish-style-auto-suggestions)Fish-Style Auto-Suggestions\\n\\n`SAWS` supports Fish-style auto-suggestions. Use the `right arrow` key to complete a suggestion.\\n\\n[![](https://camo.githubusercontent.com/6671d93da542b2f2cdec589fcb2c174e3bbc25f6/687474703a2f2f692e696d6775722e636f6d2f743532303071312e706e67)](https://camo.githubusercontent.com/6671d93da542b2f2cdec589fcb2c174e3bbc25f6/687474703a2f2f692e696d6775722e636f6d2f743532303071312e706e67)\\n\\n### [](#executing-shell-commands)Executing Shell Commands\\n\\n`SAWS` allows you to execute shell commands from the `saws>` prompt.\\n\\n[![](https://camo.githubusercontent.com/4d26d484e5b94b7ca62607ec1557d847f5db96f3/687474703a2f2f692e696d6775722e636f6d2f4669536e3662322e706e67)](https://camo.githubusercontent.com/4d26d484e5b94b7ca62607ec1557d847f5db96f3/687474703a2f2f692e696d6775722e636f6d2f4669536e3662322e706e67)\\n\\n### [](#command-history)Command History\\n\\n`SAWS` keeps track of commands you enter and stores them in `~/.saws-history`. Use the up and down arrow keys to cycle through the command history.\\n\\n[![](https://camo.githubusercontent.com/a5ded83f813be928236c804a90d85fc0fd5e517d/687474703a2f2f692e696d6775722e636f6d2f7a3852724451422e706e67)](https://camo.githubusercontent.com/a5ded83f813be928236c804a90d85fc0fd5e517d/687474703a2f2f692e696d6775722e636f6d2f7a3852724451422e706e67)\\n\\n## [](#contextual-help)Contextual Help\\n\\n`SAWS` supports contextual command line `help` and contextual web `docs`.\\n\\n### [](#contextual-command-line-help)Contextual Command Line Help\\n\\nThe `help` command is powered by the AWS CLI and outputs help within the command line.\\n\\nUsage:\\n \\n \\n aws help\\n \\n\\n[![](https://camo.githubusercontent.com/e10dae70d7f9303bc5fae0e3f5dba453fa905df7/687474703a2f2f692e696d6775722e636f6d2f7a536b7a7436792e706e67)](https://camo.githubusercontent.com/e10dae70d7f9303bc5fae0e3f5dba453fa905df7/687474703a2f2f692e696d6775722e636f6d2f7a536b7a7436792e706e67)\\n\\n### [](#contextual-web-docs)Contextual Web Docs\\n\\nSometimes you\\'re not quite sure what specific command/subcommand/option combination you need to use. In such cases, browsing through several combinations with the `help` command line is cumbersome versus browsing the online AWS CLI docs through a web browser.\\n\\n`SAWS` supports contextual web docs with the `docs` command or the `F9` key. `SAWS` will display the web docs specific to the currently entered command and subcommand.\\n\\nUsage:\\n \\n \\n aws docs\\n \\n\\n[![](https://camo.githubusercontent.com/e0a9e9bd21f2ff174f2e257f0c9966c6620d97d7/687474703a2f2f692e696d6775722e636f6d2f7a4b34494a59702e706e67)](https://camo.githubusercontent.com/e0a9e9bd21f2ff174f2e257f0c9966c6620d97d7/687474703a2f2f692e696d6775722e636f6d2f7a4b34494a59702e706e67)\\n\\n## [](#toolbar-options)Toolbar Options\\n\\n`SAWS` supports a number of toolbar options:\\n\\n * `F2` toggles [output syntax highlighting](#syntax-and-output-highlighting)\\n * `F3` toggles [fuzzy completion of AWS resources and shortcuts](#fuzzy-resource-and-shortcut-completion)\\n * `F4` toggles [completion of shortcuts](#customizable-shortcuts)\\n * `F5` refreshes [resources for auto-completion](#auto-completion-of-aws-resources)\\n * `F9` displays the [contextual web docs](#contextual-web-docs)\\n * `F10` or `control d` exits `SAWS`\\n\\n[![](https://camo.githubusercontent.com/f1bb53eda629178af0d849a112193a91daca5628/687474703a2f2f692e696d6775722e636f6d2f37767a384f53632e706e67)](https://camo.githubusercontent.com/f1bb53eda629178af0d849a112193a91daca5628/687474703a2f2f692e696d6775722e636f6d2f37767a384f53632e706e67)\\n\\n### [](#windows-support)Windows Support\\n\\n`SAWS` has been tested on Windows 7 and Windows 10.\\n\\nOn Windows, the [.sawsrc](https://github.com/donnemartin/saws/blob/master/saws/sawsrc) file can be found in `%userprofile%`. For example:\\n \\n \\n C:\\\\Users\\\\dmartin\\\\.sawsrc\\n \\n\\nAlthough you can use the standard Windows command prompt, you\\'ll probably have a better experience with either [cmder](https://github.com/cmderdev/cmder) or [conemu](https://github.com/Maximus5/ConEmu).\\n\\n[![](https://camo.githubusercontent.com/cfd9f911b9960308e0346d90ba4cf2338145fb52/687474703a2f2f692e696d6775722e636f6d2f7055774a57636b2e706e67)](https://camo.githubusercontent.com/cfd9f911b9960308e0346d90ba4cf2338145fb52/687474703a2f2f692e696d6775722e636f6d2f7055774a57636b2e706e67)\\n\\n## [](#installation)Installation\\n\\n### [](#pip-installation)Pip Installation\\n\\n[![PyPI version](https://camo.githubusercontent.com/818edda38216a62a4bb36557372877febec35086/68747470733a2f2f62616467652e667572792e696f2f70792f736177732e737667)](http://badge.fury.io/py/saws) [![PyPI](https://camo.githubusercontent.com/04574f61c5dff98734bd6a013cc9217b9a428e47/68747470733a2f2f696d672e736869656c64732e696f2f707970692f707976657273696f6e732f736177732e737667)](https://pypi.python.org/pypi/saws/)\\n\\n`SAWS` is hosted on [PyPI](https://pypi.python.org/pypi/saws). The following command will install `SAWS` along with dependencies such as the [AWS CLI](https://github.com/aws/aws-cli):\\n \\n \\n $ pip install saws\\n \\n\\nYou can also install the latest `SAWS` from GitHub source which can contain changes not yet pushed to PyPI:\\n \\n \\n $ pip install git+https://github.com/donnemartin/saws.git\\n \\n\\nIf you are not installing in a [virtualenv](#virtual-environment-and-docker-installation), run with `sudo`:\\n \\n \\n $ sudo pip install saws\\n \\n\\nOnce installed, start `SAWS`:\\n \\n \\n $ saws\\n \\n\\n### [](#virtual-environment-and-docker-installation)Virtual Environment and Docker Installation\\n\\nIt is recommended that you install Python packages in a [virtualenv](http://docs.python-guide.org/en/latest/dev/virtualenvs/) to avoid potential [issues with dependencies or permissions](https://github.com/donnemartin/saws/issues/15).\\n\\nTo view `SAWS` `virtualenv` and [Docker](https://www.docker.com/) installation instructions, click [here](https://github.com/donnemartin/saws/blob/master/INSTALLATION.md).\\n\\n### [](#aws-credentials-and-named-profiles)AWS Credentials and Named Profiles\\n\\n[Configure your credentials](https://github.com/aws/aws-cli#getting-started) with the AWS CLI:\\n \\n \\n $ aws configure\\n \\n\\nIf you\\'d like to use a specific named profile with `SAWS`, run the following commands on OS X, Linux, or Unix:\\n \\n \\n $ export AWS_DEFAULT_PROFILE=user1\\n $ saws\\n \\n\\nOr as a one-liner:\\n \\n \\n $ AWS_DEFAULT_PROFILE=user1 saws\\n \\n\\nWindows users can run the following commands:\\n \\n \\n > set AWS_DEFAULT_PROFILE=user1\\n > saws\\n \\n\\nCommand line options for starting `SAWS` with a specific profile are [under development](https://github.com/donnemartin/saws/issues/16). For more details on how to install and configure the AWS CLI, refer to the following [documentation](http://docs.aws.amazon.com/cli/latest/userguide/installing.html).\\n\\n### [](#supported-python-versions)Supported Python Versions\\n\\n * Python 2.6\\n * Python 2.7\\n * Python 3.3\\n * Python 3.4\\n * Pypy\\n\\nLight testing indicates that `SAWS` also seems to be compatible with Python 3.5.\\n\\nPypy3 is not supported due to [lack of support](https://github.com/boto/botocore/issues/622) from [boto](https://github.com/boto/boto).\\n\\n### [](#supported-platforms)Supported Platforms\\n\\n * Mac OS X \\n * Tested on OS X 10.10\\n * Linux, Unix \\n * Tested on Ubuntu 14.04 LTS\\n * Windows \\n * Tested on Windows 7 and 10\\n\\n## [](#developer-installation)Developer Installation\\n\\nIf you\\'re interested in contributing to `SAWS`, run the following commands:\\n \\n \\n $ git clone https://github.com/donnemartin/saws.git\\n $ pip install -e .\\n $ pip install -r requirements-dev.txt\\n $ saws\\n \\n\\n### [](#continuous-integration)Continuous Integration\\n\\n[![Build Status](https://camo.githubusercontent.com/c1d6d1e8f7cdac1e6a82bf900832bbbe52d61e95/68747470733a2f2f7472617669732d63692e6f72672f646f6e6e656d617274696e2f736177732e7376673f6272616e63683d6d6173746572)](https://travis-ci.org/donnemartin/saws)\\n\\nContinuous integration details are available on [Travis CI](https://travis-ci.org/donnemartin/saws).\\n\\n### [](#dependencies-management)Dependencies Management\\n\\n[![Dependency Status](https://camo.githubusercontent.com/11e16ba1560f0232928945436bf86340615d85ae/68747470733a2f2f67656d6e617369756d2e636f6d2f646f6e6e656d617274696e2f736177732e737667)](https://gemnasium.com/donnemartin/saws)\\n\\nDependencies management details are available on [Gemnasium](https://gemnasium.com/donnemartin/saws).\\n\\n### [](#unit-tests-and-code-coverage)Unit Tests and Code Coverage\\n\\n[![Codecov](https://camo.githubusercontent.com/e24a50e9dd63768d90d26a9f6be1cf3d0c3b0230/68747470733a2f2f696d672e736869656c64732e696f2f636f6465636f762f632f6769746875622f646f6e6e656d617274696e2f736177732e737667)](https://codecov.io/github/donnemartin/saws/saws)\\n\\n[![](https://camo.githubusercontent.com/a3588f226c5b2c9007fd9e49212ae8294c5bc023/687474703a2f2f636f6465636f762e696f2f6769746875622f646f6e6e656d617274696e2f736177732f6272616e63682e7376673f6272616e63683d6d6173746572)](https://camo.githubusercontent.com/a3588f226c5b2c9007fd9e49212ae8294c5bc023/687474703a2f2f636f6465636f762e696f2f6769746875622f646f6e6e656d617274696e2f736177732f6272616e63682e7376673f6272616e63683d6d6173746572)\\n\\nCode coverage details are available on [Codecov](https://codecov.io/github/donnemartin/saws/saws).\\n\\nRun unit tests in your active Python environment:\\n \\n \\n $ python tests/run_tests.py\\n \\n\\nRun unit tests with [tox](https://pypi.python.org/pypi/tox) on multiple Python environments:\\n \\n \\n $ tox\\n \\n\\n### [](#documentation)Documentation\\n\\n[![Documentation Status](https://camo.githubusercontent.com/41afe5f96e9f915ecd9706bdc7dfd19518eaf05a/68747470733a2f2f72656164746865646f63732e6f72672f70726f6a656374732f736177732f62616467652f3f76657273696f6e3d6c6174657374)](http://saws.readthedocs.org/en/latest/?badge=latest)\\n\\nSource code documentation is available on [Readthedocs.org](http://saws.readthedocs.org/en/latest/?badge=latest).\\n\\nRun the following to build the docs:\\n \\n \\n $ scripts/update_docs.sh\\n \\n\\n## [](#contributing)Contributing\\n\\nContributions are welcome!\\n\\nReview the [Contributing Guidelines](https://github.com/donnemartin/saws/blob/master/CONTRIBUTING.md) for details on how to:\\n\\n * Submit issues\\n * Submit pull requests\\n\\n## [](#credits)Credits\\n\\n * [AWS CLI](https://github.com/aws/aws-cli) by [AWS](https://github.com/aws) for powering `SAWS` under the hood\\n * [Python Prompt Toolkit](https://github.com/jonathanslenders/python-prompt-toolkit) by [jonathanslenders](https://github.com/jonathanslenders) for simplifying the creation of `SAWS`\\n * [Wharfee](https://github.com/j-bennet/wharfee) by [j-bennet](https://github.com/j-bennet) for inspiring the creation of `SAWS` and for some handy utility functions\\n\\n## [](#contact-info)Contact Info\\n\\nFeel free to contact me to discuss any issues, questions, or comments.\\n\\n * Email: [donne.martin@gmail.com](mailto:donne.martin@gmail.com)\\n * Twitter: [donne_martin](https://twitter.com/donne_martin)\\n * GitHub: [donnemartin](https://github.com/donnemartin)\\n * LinkedIn: [donnemartin](https://www.linkedin.com/in/donnemartin)\\n * Website: [donnemartin.com](http://donnemartin.com)\\n\\n## [](#license)License\\n \\n \\n Copyright 2015 Donne Martin\\n \\n Licensed under the Apache License, Version 2.0 (the \"License\");\\n you may not use this file except in compliance with the License.\\n You may obtain a copy of the License at\\n \\n http://www.apache.org/licenses/LICENSE-2.0\\n \\n Unless required by applicable law or agreed to in writing, software\\n distributed under the License is distributed on an \"AS IS\" BASIS,\\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\\n See the License for the specific language governing permissions and\\n limitations under the License.\\n \\n\\n * [Status](https://status.github.com/)\\n * [API](https://developer.github.com)\\n * [Training](https://training.github.com)\\n * [Shop](https://shop.github.com)\\n * [Blog](https://github.com/blog)\\n * [About](https://github.com/about)\\n * [Pricing](https://github.com/pricing)\\n[ ](https://github.com)\\n\\n * (C) 2015 GitHub, Inc.\\n * [Terms](https://github.com/site/terms)\\n * [Privacy](https://github.com/site/privacy)\\n * [Security](https://github.com/security)\\n * [Contact](https://github.com/contact)\\n * [Help](https://help.github.com)\\n\\nSomething went wrong with that request. Please try again. \\n\\nYou signed in with another tab or window. [Reload]() to refresh your session. You signed out in another tab or window. [Reload]() to refresh your session.\\n\n\"\"\" # NOQA\nformatted_markdown = u\"\"\"\n\\x1b[32m[Skip to content](#start-of-content)\\x1b[0m\\n\\n\\x1b[32m[ ](https://github.com/)\\x1b[0m\\n\\n\\x1b[32m[Sign up](/join)\\x1b[0m \\x1b[32m[Sign in](/login?return_to=%2Fdonnemartin%2Fsaws)\\x1b[0m\\n\\nThis repository\\n\\n\\x1b[36m * \\x1b[32m[Explore](/explore)\\x1b[0m\\x1b[0m\\n\\x1b[36m * \\x1b[32m[Features](/features)\\x1b[0m\\x1b[0m\\n\\x1b[36m * \\x1b[32m[Enterprise](https://enterprise.github.com/)\\x1b[0m\\x1b[0m\\n\\x1b[36m * \\x1b[32m[Pricing](/pricing)\\x1b[0m\\x1b[0m\\n\\n\\x1b[36m * \\x1b[32m[ Watch ](/login?return_to=%2Fdonnemartin%2Fsaws)\\x1b[0m \\x1b[32m[ 80 ](/donnemartin/saws/watchers)\\x1b[0m\\x1b[0m\\n\\x1b[36m * \\x1b[32m[ Star ](/login?return_to=%2Fdonnemartin%2Fsaws)\\x1b[0m \\x1b[32m[ 2,428 ](/donnemartin/saws/stargazers)\\x1b[0m\\x1b[0m\\n\\x1b[36m * \\x1b[32m[ Fork ](/login?return_to=%2Fdonnemartin%2Fsaws)\\x1b[0m \\x1b[32m[ 77 ](/donnemartin/saws/network)\\x1b[0m\\x1b[0m\\n\\n\\x1b[33m \\x1b[32m[donnemartin](/donnemartin)\\x1b[0m/\\x1b[36m\\x1b[32m[saws](/donnemartin/saws)\\x1b[0m\\x1b[0m \\x1b[32m![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\\x1b[0m\\x1b[0m\\n\\n\\x1b[36m * \\x1b[32m[ Code ![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\\x1b[0m ](/donnemartin/saws)\\x1b[0m\\n\\x1b[36m * \\x1b[32m[ Issues ![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\\x1b[0m ](/donnemartin/saws/issues)\\x1b[0m\\n\\x1b[36m * \\x1b[32m[ Pull requests ![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\\x1b[0m ](/donnemartin/saws/pulls)\\x1b[0m\\n\\n\\x1b[36m * \\x1b[32m[ Pulse ![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\\x1b[0m ](/donnemartin/saws/pulse)\\x1b[0m\\n\\x1b[36m * \\x1b[32m[ Graphs ![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\\x1b[0m ](/donnemartin/saws/graphs)\\x1b[0m\\n\\n\\x1b[33mHTTPS clone URL\\x1b[0m\\n\\n\\x1b[33mSubversion checkout URL\\x1b[0m\\n\\nYou can clone with\\n\\nHTTPS or\\n\\nSubversion. \\x1b[32m[ ](https://help.github.com/articles/which-remote-url-should-i-use)\\x1b[0m\\n\\n\\x1b[32m[ Download ZIP ](/donnemartin/saws/archive/master.zip \"Download the contents of donnemartin/saws as a zip file\" )\\x1b[0m\\n\\nA Supercharged AWS Command Line Interface (CLI). \\n\\n\\x1b[36m * \\x1b[32m[ 544 commits ](/donnemartin/saws/commits/master)\\x1b[0m\\x1b[0m\\n\\x1b[36m * \\x1b[32m[ 2 branches ](/donnemartin/saws/branches)\\x1b[0m\\x1b[0m\\n\\x1b[36m * \\x1b[32m[ 7 releases ](/donnemartin/saws/releases)\\x1b[0m\\x1b[0m\\n\\x1b[36m * \\x1b[32m[ 5 contributors ](/donnemartin/saws/graphs/contributors)\\x1b[0m\\x1b[0m\\n\\n1. \\x1b[32m[ Python 99.4% ](/donnemartin/saws/search?l=python)\\x1b[0m\\n 2. \\x1b[32m[ Shell 0.6% ](/donnemartin/saws/search?l=bash)\\x1b[0m\\n\\nPython Shell\\n\\n\\x1b[32m[ ](/donnemartin/saws/find/master)\\x1b[0m \\x1b[32m[ ](/donnemartin/saws/compare)\\x1b[0m\\n\\n_Branch:_ master\\n\\nSwitch branches/tags\\n\\n\\x1b[36m * \\x1b[32m[Branches](#)\\x1b[0m\\x1b[0m\\n\\x1b[36m * \\x1b[32m[Tags](#)\\x1b[0m\\x1b[0m\\n\\n\\x1b[32m[ develop ](/donnemartin/saws/tree/develop)\\x1b[0m \\x1b[32m[ master ](/donnemartin/saws/tree/master)\\x1b[0m\\n\\nNothing to show\\n\\n\\x1b[32m[0.3.2](/donnemartin/saws/tree/0.3.2 \"0.3.2\" )\\x1b[0m\\n\\n\\x1b[32m[0.3.1](/donnemartin/saws/tree/0.3.1 \"0.3.1\" )\\x1b[0m\\n\\n\\x1b[32m[0.3.0](/donnemartin/saws/tree/0.3.0 \"0.3.0\" )\\x1b[0m\\n\\n\\x1b[32m[0.2.1](/donnemartin/saws/tree/0.2.1 \"0.2.1\" )\\x1b[0m\\n\\n\\x1b[32m[0.2.0](/donnemartin/saws/tree/0.2.0 \"0.2.0\" )\\x1b[0m\\n\\n\\x1b[32m[0.1.1](/donnemartin/saws/tree/0.1.1 \"0.1.1\" )\\x1b[0m\\n\\n\\x1b[32m[0.1.0](/donnemartin/saws/tree/0.1.0 \"0.1.0\" )\\x1b[0m\\n\\nNothing to show\\n\\n\\x1b[32m[saws](/donnemartin/saws)\\x1b[0m/\\n\\nLatest commit \\x1b[32m[ 07c1d7d ](/donnemartin/saws/commit/07c1d7df22c9032b993d18f22162b91fb16e86d6)\\x1b[0m Nov 30, 2015 \\x1b[32m![@donnemartin](https://avatars3.githubusercontent.com/u/5458997?v=3&s=40)\\x1b[0m \\x1b[32m[donnemartin](/donnemartin)\\x1b[0m \\x1b[32m[Fix](/donnemartin/saws/commit/07c1d7df22c9032b993d18f22162b91fb16e86d6 \"Fix #66: Remove docs/build from the source repository.\" )\\x1b[0m \\x1b[32m[#66](https://github.com/donnemartin/saws/issues/66 \"Remove docs/build from the source repository.\" )\\x1b[0m\\x1b[32m[: Remove docs/build from the source repository.](/donnemartin/saws/commit/07c1d7df22c9032b993d18f22162b91fb16e86d6 \"Fix #66: Remove docs/build from the source repository.\" )\\x1b[0m\\n\\n\\x1b[32m[Permalink](/donnemartin/saws/tree/07c1d7df22c9032b993d18f22162b91fb16e86d6)\\x1b[0m | Failed to load latest commit information. \\n---|--- \\n\\x1b[32m![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\\x1b[0m | \\x1b[32m[docs](/donnemartin/saws/tree/master/docs \"docs\" )\\x1b[0m | \\x1b[32m[Fix](/donnemartin/saws/commit/07c1d7df22c9032b993d18f22162b91fb16e86d6 \"Fix #66: Remove docs/build from the source repository.\" )\\x1b[0m \\x1b[32m[#66](https://github.com/donnemartin/saws/issues/66 \"Remove docs/build from the source repository.\" )\\x1b[0m\\x1b[32m[: Remove docs/build from the source repository.](/donnemartin/saws/commit/07c1d7df22c9032b993d18f22162b91fb16e86d6 \"Fix #66: Remove docs/build from the source repository.\" )\\x1b[0m | Nov 30, 2015 \\n\\x1b[32m![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\\x1b[0m | \\x1b[32m[saws](/donnemartin/saws/tree/master/saws \"saws\" )\\x1b[0m | \\x1b[32m[Merge branch \\'master\\' into develop](/donnemartin/saws/commit/4d174fe79f61eeeda363c0b03e9e53392da671ef \"Merge branch \\'master\\' into develop\\n\\n* master:\\n Removed mention of scripts/release_pypi.sh, which is no longer used.\\n Updated commands.\" )\\x1b[0m | Nov 30, 2015 \\n\\x1b[32m![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\\x1b[0m | \\x1b[32m[scripts](/donnemartin/saws/tree/master/scripts \"scripts\" )\\x1b[0m | \\x1b[32m[Updated CHANGELOG to README script to use the new CHANGELOG.rst.](/donnemartin/saws/commit/5c03b2cc6882590aa567a98cc8dceae780d1428c \"Updated CHANGELOG to README script to use the new CHANGELOG.rst.\" )\\x1b[0m | Oct 16, 2015 \\n\\x1b[32m![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\\x1b[0m | \\x1b[32m[tests](/donnemartin/saws/tree/master/tests \"tests\" )\\x1b[0m | \\x1b[32m[Fix linter line length issues.](/donnemartin/saws/commit/bf1afa4c73bf1a7dd462cb0c92f945d2b7d7cf5a \"Fix linter line length issues.\" )\\x1b[0m | Oct 15, 2015 \\n\\x1b[32m![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\\x1b[0m | \\x1b[32m[.gitchangelog.rc](/donnemartin/saws/blob/master/.gitchangelog.rc \".gitchangelog.rc\" )\\x1b[0m | \\x1b[32m[Added gitchangelog.](/donnemartin/saws/commit/7347e0cc55a9ac75368152b9683881d41982a943 \"Added gitchangelog.\" )\\x1b[0m | Sep 15, 2015 \\n\\x1b[32m![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\\x1b[0m | \\x1b[32m[.gitignore](/donnemartin/saws/blob/master/.gitignore \".gitignore\" )\\x1b[0m | \\x1b[32m[Fix](/donnemartin/saws/commit/07c1d7df22c9032b993d18f22162b91fb16e86d6 \"Fix #66: Remove docs/build from the source repository.\" )\\x1b[0m \\x1b[32m[#66](https://github.com/donnemartin/saws/issues/66 \"Remove docs/build from the source repository.\" )\\x1b[0m\\x1b[32m[: Remove docs/build from the source repository.](/donnemartin/saws/commit/07c1d7df22c9032b993d18f22162b91fb16e86d6 \"Fix #66: Remove docs/build from the source repository.\" )\\x1b[0m | Nov 30, 2015 \\n\\x1b[32m![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\\x1b[0m | \\x1b[32m[.travis.yml](/donnemartin/saws/blob/master/.travis.yml \".travis.yml\" )\\x1b[0m | \\x1b[32m[Revert Python 3.5 build and test support due to:](/donnemartin/saws/commit/36e073de59b0fbe872fb30c80f3521c3b558b37f \"Revert Python 3.5 build and test support due to: https://github.com/travis-ci/travis-ci/issues/4794.\" )\\x1b[0m \\x1b[32m[travis-ci/travis-ci#…](https://github.com/travis-ci/travis-ci/issues/4794 \"Tox builds for Python 3.5 fail\" )\\x1b[0m | Sep 23, 2015 \\n\\x1b[32m![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\\x1b[0m | \\x1b[32m[CHANGELOG.md](/donnemartin/saws/blob/master/CHANGELOG.md \"CHANGELOG.md\" )\\x1b[0m | \\x1b[32m[Prepped version 0.3.2.](/donnemartin/saws/commit/0408df93b135542c2388fe60444ba4651feec481 \"Prepped version 0.3.2.\" )\\x1b[0m | Oct 16, 2015 \\n\\x1b[32m![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\\x1b[0m | \\x1b[32m[CHANGELOG.rst](/donnemartin/saws/blob/master/CHANGELOG.rst \"CHANGELOG.rst\" )\\x1b[0m | \\x1b[32m[Prepped version 0.3.2.](/donnemartin/saws/commit/0408df93b135542c2388fe60444ba4651feec481 \"Prepped version 0.3.2.\" )\\x1b[0m | Oct 16, 2015 \\n\\x1b[32m![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\\x1b[0m | \\x1b[32m[CHECKLIST.md](/donnemartin/saws/blob/master/CHECKLIST.md \"CHECKLIST.md\" )\\x1b[0m | \\x1b[32m[Removed mention of scripts/release_pypi.sh, which is no longer used.](/donnemartin/saws/commit/ba73311f8dd04e9a1eb159ff4d8c741a22027e50 \"Removed mention of scripts/release_pypi.sh, which is no longer used.\" )\\x1b[0m | Nov 16, 2015 \\n\\x1b[32m![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\\x1b[0m | \\x1b[32m[CONTRIBUTING.md](/donnemartin/saws/blob/master/CONTRIBUTING.md \"CONTRIBUTING.md\" )\\x1b[0m | \\x1b[32m[Add note about AWS collaboration.](/donnemartin/saws/commit/9fb37599835024ef7c99a7f00822aab54904fa5b \"Add note about AWS collaboration.\" )\\x1b[0m | Nov 30, 2015 \\n\\x1b[32m![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\\x1b[0m | \\x1b[32m[Dockerfile](/donnemartin/saws/blob/master/Dockerfile \"Dockerfile\" )\\x1b[0m | \\x1b[32m[Add Dockerfile](/donnemartin/saws/commit/7e185a10918059f0c763b1437d540b78432ea1a9 \"Add Dockerfile\" )\\x1b[0m | Sep 26, 2015 \\n\\x1b[32m![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\\x1b[0m | \\x1b[32m[INSTALLATION.md](/donnemartin/saws/blob/master/INSTALLATION.md \"INSTALLATION.md\" )\\x1b[0m | \\x1b[32m[Added INSTALLATION.md.](/donnemartin/saws/commit/8317c4c92441f40ce20dad5ced97fe090f77e93e \"Added INSTALLATION.md.\" )\\x1b[0m | Oct 3, 2015 \\n\\x1b[32m![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\\x1b[0m | \\x1b[32m[LICENSE.txt](/donnemartin/saws/blob/master/LICENSE.txt \"LICENSE.txt\" )\\x1b[0m | \\x1b[32m[Added LICENSE.](/donnemartin/saws/commit/1166ffddffffbd981868d700df1f8ecc2061847f \"Added LICENSE.\" )\\x1b[0m | Sep 19, 2015 \\n\\x1b[32m![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\\x1b[0m | \\x1b[32m[MANIFEST.in](/donnemartin/saws/blob/master/MANIFEST.in \"MANIFEST.in\" )\\x1b[0m | \\x1b[32m[Updated MANIFEST.in.](/donnemartin/saws/commit/cba64a756e8dd193bb93083ee7aecd4316c2c9b4 \"Updated MANIFEST.in.\" )\\x1b[0m | Sep 19, 2015 \\n\\x1b[32m![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\\x1b[0m | \\x1b[32m[README.md](/donnemartin/saws/blob/master/README.md \"README.md\" )\\x1b[0m | \\x1b[32m[Fix #todo-add-more-resources anchor link.](/donnemartin/saws/commit/8427a6432c58aa9f95bf4fe26ab76a13645a8261 \"Fix #todo-add-more-resources anchor link.\" )\\x1b[0m | Nov 30, 2015 \\n\\x1b[32m![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\\x1b[0m | \\x1b[32m[appveyor.yml](/donnemartin/saws/blob/master/appveyor.yml \"appveyor.yml\" )\\x1b[0m | \\x1b[32m[Added Appveyor support, inspired by](/donnemartin/saws/commit/f756340ad4f9f4d47860644b0e677f49dac575d4 \"Added Appveyor support, inspired by https://github.com/audreyr/cookiecutter/blob/master/appveyor.yml.\" )\\x1b[0m \\x1b[32m[https://github.com/audreyr/cookie…](https://github.com/audreyr/cookiecutter/blob/master/appveyor.yml)\\x1b[0m | Sep 28, 2015 \\n\\x1b[32m![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\\x1b[0m | \\x1b[32m[requirements-dev.txt](/donnemartin/saws/blob/master/requirements-dev.txt \"requirements-dev.txt\" )\\x1b[0m | \\x1b[32m[Added Sphinx-PyPI-upload support.](/donnemartin/saws/commit/7068598aa117cc16d6e32000afa05963d3d76093 \"Added Sphinx-PyPI-upload support.\" )\\x1b[0m | Sep 19, 2015 \\n\\x1b[32m![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\\x1b[0m | \\x1b[32m[setup.cfg](/donnemartin/saws/blob/master/setup.cfg \"setup.cfg\" )\\x1b[0m | \\x1b[32m[Added Sphinx-PyPI-upload support.](/donnemartin/saws/commit/7068598aa117cc16d6e32000afa05963d3d76093 \"Added Sphinx-PyPI-upload support.\" )\\x1b[0m | Sep 19, 2015 \\n\\x1b[32m![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\\x1b[0m | \\x1b[32m[setup.py](/donnemartin/saws/blob/master/setup.py \"setup.py\" )\\x1b[0m | \\x1b[32m[Fixed](/donnemartin/saws/commit/1628d3efda27ba638833b54bdc3124db68a7fb30 \"Fixed #53: Shortcuts should be placed separately in a .saws.shortcuts file.\" )\\x1b[0m \\x1b[32m[#53](https://github.com/donnemartin/saws/issues/53 \"Shortcuts should be placed separately in a .saws.shortcuts file\" )\\x1b[0m\\x1b[32m[: Shortcuts should be placed separately in a .saws.shortcuts…](/donnemartin/saws/commit/1628d3efda27ba638833b54bdc3124db68a7fb30 \"Fixed #53: Shortcuts should be placed separately in a .saws.shortcuts file.\" )\\x1b[0m | Oct 14, 2015 \\n\\x1b[32m![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\\x1b[0m | \\x1b[32m[tox.ini](/donnemartin/saws/blob/master/tox.ini \"tox.ini\" )\\x1b[0m | \\x1b[32m[Revert Python 3.5 build and test support due to:](/donnemartin/saws/commit/36e073de59b0fbe872fb30c80f3521c3b558b37f \"Revert Python 3.5 build and test support due to: https://github.com/travis-ci/travis-ci/issues/4794.\" )\\x1b[0m \\x1b[32m[travis-ci/travis-ci#…](https://github.com/travis-ci/travis-ci/issues/4794 \"Tox builds for Python 3.5 fail\" )\\x1b[0m | Sep 23, 2015\\n\\n\\x1b[33m README.md \\x1b[0m\\n\\n\\x1b[32m[![](https://camo.githubusercontent.com/2af72023269b0b320adf4ec55576435f5c8b79c4/687474703a2f2f692e696d6775722e636f6d2f767a43357a6d412e676966)\\x1b[0m](https://camo.githubusercontent.com/2af72023269b0b320adf4ec55576435f5c8b79c4/687474703a2f2f692e696d6775722e636f6d2f767a43357a6d412e676966)\\n\\n\\x1b[32m[![Build Status](https://camo.githubusercontent.com/c1d6d1e8f7cdac1e6a82bf900832bbbe52d61e95/68747470733a2f2f7472617669732d63692e6f72672f646f6e6e656d617274696e2f736177732e7376673f6272616e63683d6d6173746572)\\x1b[0m](https://travis-ci.org/donnemartin/saws) \\x1b[32m[![Documentation Status](https://camo.githubusercontent.com/41afe5f96e9f915ecd9706bdc7dfd19518eaf05a/68747470733a2f2f72656164746865646f63732e6f72672f70726f6a656374732f736177732f62616467652f3f76657273696f6e3d6c6174657374)\\x1b[0m](http://saws.readthedocs.org/en/latest/?badge=latest) \\x1b[32m[![Dependency Status](https://camo.githubusercontent.com/11e16ba1560f0232928945436bf86340615d85ae/68747470733a2f2f67656d6e617369756d2e636f6d2f646f6e6e656d617274696e2f736177732e737667)\\x1b[0m](https://gemnasium.com/donnemartin/saws) \\x1b[32m[![Codecov](https://camo.githubusercontent.com/e24a50e9dd63768d90d26a9f6be1cf3d0c3b0230/68747470733a2f2f696d672e736869656c64732e696f2f636f6465636f762f632f6769746875622f646f6e6e656d617274696e2f736177732e737667)\\x1b[0m](https://codecov.io/github/donnemartin/saws/saws)\\n\\n\\x1b[32m[![PyPI version](https://camo.githubusercontent.com/818edda38216a62a4bb36557372877febec35086/68747470733a2f2f62616467652e667572792e696f2f70792f736177732e737667)\\x1b[0m](http://badge.fury.io/py/saws) \\x1b[32m[![PyPI](https://camo.githubusercontent.com/04574f61c5dff98734bd6a013cc9217b9a428e47/68747470733a2f2f696d672e736869656c64732e696f2f707970692f707976657273696f6e732f736177732e737667)\\x1b[0m](https://pypi.python.org/pypi/saws/) \\x1b[32m[![License](https://camo.githubusercontent.com/8bf81d4b6a63eaf6f712e3092fe7238f8716e615/687474703a2f2f696d672e736869656c64732e696f2f3a6c6963656e73652d6170616368652d626c75652e737667)\\x1b[0m](http://www.apache.org/licenses/LICENSE-2.0.html)\\n\\n\\x1b[33m\\x1b[32m[](#saws)\\x1b[0mSAWS\\x1b[0m\\n\\n\\x1b[33m\\x1b[32m[](#motivation)\\x1b[0mMotivation\\x1b[0m\\n\\n\\x1b[33m\\x1b[32m[](#aws-cli)\\x1b[0mAWS CLI\\x1b[0m\\n\\nAlthough the \\x1b[32m[AWS CLI](https://github.com/aws/aws-cli)\\x1b[0m is a great resource to manage your AWS-powered services, it\\'s \\x1b[36mtough to remember usage\\x1b[0m of:\\n\\n\\x1b[36m * 50+ top-level commands\\x1b[0m\\n\\x1b[36m * 1400+ subcommands\\x1b[0m\\n\\x1b[36m * Countless command-specific options\\x1b[0m\\n\\x1b[36m * Resources such as instance tags and buckets\\x1b[0m\\n\\n\\x1b[33m\\x1b[32m[](#saws-a-supercharged-aws-cli)\\x1b[0mSAWS: A Supercharged AWS CLI\\x1b[0m\\n\\n\\x1b[36m`SAWS`\\x1b[0m aims to \\x1b[36msupercharge\\x1b[0m the AWS CLI with features focusing on:\\n\\n\\x1b[36m * \\x1b[36mImproving ease-of-use\\x1b[0m\\x1b[0m\\n\\x1b[36m * \\x1b[36mIncreasing productivity\\x1b[0m\\x1b[0m\\n\\nUnder the hood, \\x1b[36m`SAWS`\\x1b[0m is \\x1b[36mpowered by the AWS CLI\\x1b[0m and supports the \\x1b[36msame commands\\x1b[0m and \\x1b[36mcommand structure\\x1b[0m.\\n\\n\\x1b[36m`SAWS`\\x1b[0m and \\x1b[36m`AWS CLI`\\x1b[0m Usage:\\n\\naws [parameters] [options]\\n\\n\\x1b[36m`SAWS`\\x1b[0m features:\\n\\n\\x1b[36m * Auto-completion of: \\x1b[0m\\n \\x1b[36m * Commands\\x1b[0m\\n \\x1b[36m * Subcommands\\x1b[0m\\n \\x1b[36m * Options\\x1b[0m\\n\\x1b[36m * Auto-completion of resources: \\x1b[0m\\n \\x1b[36m * Bucket names\\x1b[0m\\n \\x1b[36m * Instance ids\\x1b[0m\\n \\x1b[36m * Instance tags\\x1b[0m\\n \\x1b[36m * \\x1b[32m[More coming soon!](#todo-add-more-resources)\\x1b[0m\\x1b[0m\\n\\x1b[36m * Customizable shortcuts\\x1b[0m\\n\\x1b[36m * Fuzzy completion of resources and shortcuts\\x1b[0m\\n\\x1b[36m * Fish-style auto-suggestions\\x1b[0m\\n\\x1b[36m * Syntax and output highlighting\\x1b[0m\\n\\x1b[36m * Execution of shell commands\\x1b[0m\\n\\x1b[36m * Command history\\x1b[0m\\n\\x1b[36m * Contextual help\\x1b[0m\\n\\x1b[36m * Toolbar options\\x1b[0m\\n\\n\\x1b[36m`SAWS`\\x1b[0m is available for Mac, Linux, Unix, and \\x1b[32m[Windows](#windows-support)\\x1b[0m.\\n\\n\\x1b[32m[![](https://camo.githubusercontent.com/571a2d19a969447b477713d54d1d3ba46d7418ee/687474703a2f2f692e696d6775722e636f6d2f456f31327139542e706e67)\\x1b[0m](https://camo.githubusercontent.com/571a2d19a969447b477713d54d1d3ba46d7418ee/687474703a2f2f692e696d6775722e636f6d2f456f31327139542e706e67)\\n\\n\\x1b[33m\\x1b[32m[](#index)\\x1b[0mIndex\\x1b[0m\\n\\n\\x1b[33m\\x1b[32m[](#features)\\x1b[0mFeatures\\x1b[0m\\n\\n\\x1b[36m * \\x1b[32m[Syntax and Output Highlighting](#syntax-and-output-highlighting)\\x1b[0m\\x1b[0m\\n\\x1b[36m * \\x1b[32m[Auto-Completion of Commands, Subcommands, and Options](#auto-completion-of-commands-subcommands-and-options)\\x1b[0m\\x1b[0m\\n\\x1b[36m * \\x1b[32m[Auto-Completion of AWS Resources](#auto-completion-of-aws-resources)\\x1b[0m\\x1b[0m\\n \\x1b[36m * \\x1b[32m[S3 Buckets](#s3-buckets)\\x1b[0m\\x1b[0m\\n \\x1b[36m * \\x1b[32m[EC2 Instance Ids](#ec2-instance-ids)\\x1b[0m\\x1b[0m\\n \\x1b[36m * \\x1b[32m[EC2 Instance Tags](#ec2-instance-tags)\\x1b[0m\\x1b[0m\\n \\x1b[36m * \\x1b[32m[TODO: Add More Resources](#todo-add-more-resources)\\x1b[0m\\x1b[0m\\n\\x1b[36m * \\x1b[32m[Customizable Shortcuts](#customizable-shortcuts)\\x1b[0m\\x1b[0m\\n\\x1b[36m * \\x1b[32m[Fuzzy Resource and Shortcut Completion](#fuzzy-resource-and-shortcut-completion)\\x1b[0m\\x1b[0m\\n\\x1b[36m * \\x1b[32m[Fish-Style Auto-Suggestions](#fish-style-auto-suggestions)\\x1b[0m\\x1b[0m\\n\\x1b[36m * \\x1b[32m[Executing Shell Commands](#executing-shell-commands)\\x1b[0m\\x1b[0m\\n\\x1b[36m * \\x1b[32m[Command History](#command-history)\\x1b[0m\\x1b[0m\\n\\x1b[36m * \\x1b[32m[Contextual Help](#contextual-help)\\x1b[0m\\x1b[0m\\n \\x1b[36m * \\x1b[32m[Contextual Command Line Help](#contextual-command-line-help)\\x1b[0m\\x1b[0m\\n \\x1b[36m * \\x1b[32m[Contextual Web Docs](#contextual-web-docs)\\x1b[0m\\x1b[0m\\n\\x1b[36m * \\x1b[32m[Toolbar Options](#toolbar-options)\\x1b[0m\\x1b[0m\\n\\x1b[36m * \\x1b[32m[Windows Support](#windows-support)\\x1b[0m\\x1b[0m\\n\\n\\x1b[33m\\x1b[32m[](#installation-and-tests)\\x1b[0mInstallation and Tests\\x1b[0m\\n\\n\\x1b[36m * \\x1b[32m[Installation](#installation)\\x1b[0m\\x1b[0m\\n \\x1b[36m * \\x1b[32m[Pip Installation](#pip-installation)\\x1b[0m\\x1b[0m\\n \\x1b[36m * \\x1b[32m[Virtual Environment and Docker Installation](#virtual-environment-and-docker-installation)\\x1b[0m\\x1b[0m\\n \\x1b[36m * \\x1b[32m[AWS Credentials and Named Profiles](#aws-credentials-and-named-profiles)\\x1b[0m\\x1b[0m\\n \\x1b[36m * \\x1b[32m[Supported Python Versions](#supported-python-versions)\\x1b[0m\\x1b[0m\\n \\x1b[36m * \\x1b[32m[Supported Platforms](#supported-platforms)\\x1b[0m\\x1b[0m\\n\\x1b[36m * \\x1b[32m[Developer Installation](#developer-installation)\\x1b[0m\\x1b[0m\\n \\x1b[36m * \\x1b[32m[Continuous Integration](#continuous-integration)\\x1b[0m\\x1b[0m\\n \\x1b[36m * \\x1b[32m[Dependencies Management](#dependencies-management)\\x1b[0m\\x1b[0m\\n \\x1b[36m * \\x1b[32m[Unit Tests and Code Coverage](#unit-tests-and-code-coverage)\\x1b[0m\\x1b[0m\\n \\x1b[36m * \\x1b[32m[Documentation](#documentation)\\x1b[0m\\x1b[0m\\n\\n\\x1b[33m\\x1b[32m[](#misc)\\x1b[0mMisc\\x1b[0m\\n\\n\\x1b[36m * \\x1b[32m[Contributing](#contributing)\\x1b[0m\\x1b[0m\\n\\x1b[36m * \\x1b[32m[Credits](#credits)\\x1b[0m\\x1b[0m\\n\\x1b[36m * \\x1b[32m[Contact Info](#contact-info)\\x1b[0m\\x1b[0m\\n\\x1b[36m * \\x1b[32m[License](#license)\\x1b[0m\\x1b[0m\\n\\n\\x1b[33m\\x1b[32m[](#syntax-and-output-highlighting)\\x1b[0mSyntax and Output Highlighting\\x1b[0m\\n\\n\\x1b[32m[![](https://camo.githubusercontent.com/c527a4c00862bb1fe47c272a3c3563418f06866f/687474703a2f2f692e696d6775722e636f6d2f785144707737302e706e67)\\x1b[0m](https://camo.githubusercontent.com/c527a4c00862bb1fe47c272a3c3563418f06866f/687474703a2f2f692e696d6775722e636f6d2f785144707737302e706e67)\\n\\nYou can control which theme to load for syntax highlighting by updating your \\x1b[32m[~/.sawsrc](https://github.com/donnemartin/saws/blob/master/saws/sawsrc)\\x1b[0m file:\\n\\n\\x1b[33mVisual theme. Possible values: manni, igor, xcode, vim, autumn, vs, rrt,\\x1b[0m\\n \\x1b[33mnative, perldoc, borland, tango, emacs, friendly, monokai, paraiso-dark,\\x1b[0m\\n \\x1b[33mcolorful, murphy, bw, pastie, paraiso-light, trac, default, fruity\\x1b[0m\\n theme = vim\\n\\n\\x1b[33m\\x1b[32m[](#auto-completion-of-commands-subcommands-and-options)\\x1b[0mAuto-Completion of Commands, Subcommands, and Options\\x1b[0m\\n\\n\\x1b[36m`SAWS`\\x1b[0m provides smart autocompletion as you type. Entering the following command will interactively list and auto-complete all subcommands \\x1b[36mspecific only\\x1b[0m to \\x1b[36m`ec2`\\x1b[0m:\\n\\naws ec2\\n\\n\\x1b[32m[![](https://camo.githubusercontent.com/df93a08e1371694a8086f3b45ba6e5ec955212e2/687474703a2f2f692e696d6775722e636f6d2f5032744c3976572e706e67)\\x1b[0m](https://camo.githubusercontent.com/df93a08e1371694a8086f3b45ba6e5ec955212e2/687474703a2f2f692e696d6775722e636f6d2f5032744c3976572e706e67)\\n\\n\\x1b[33m\\x1b[32m[](#auto-completion-of-aws-resources)\\x1b[0mAuto-Completion of AWS Resources\\x1b[0m\\n\\nIn addition to the default commands, subcommands, and options the AWS CLI provides, \\x1b[36m`SAWS`\\x1b[0m supports auto-completion of your AWS resources. Currently, bucket names, instance ids, and instance tags are included, with additional support for more resources \\x1b[32m[under development](#todo-add-more-resources)\\x1b[0m.\\n\\n\\x1b[33m\\x1b[32m[](#s3-buckets)\\x1b[0mS3 Buckets\\x1b[0m\\n\\nOption for \\x1b[36m`s3api`\\x1b[0m:\\n\\n--bucket\\n\\nSample Usage:\\n\\naws s3api get-bucket-acl --bucket\\n\\nSyntax for \\x1b[36m`s3`\\x1b[0m:\\n\\ns3://\\n\\nSample Usage:\\n\\naws s3 ls s3://\\n\\nNote: The example below demonstrates the use of \\x1b[32m[fuzzy resource completion](/donnemartin/saws/blob/master/fuzzy-resource-and-shortcutcompletion)\\x1b[0m:\\n\\n\\x1b[32m[![](https://camo.githubusercontent.com/756f14d2ed77a7a9a3da9b0d9501cab9b7433e36/687474703a2f2f692e696d6775722e636f6d2f333943415335542e706e67)\\x1b[0m](https://camo.githubusercontent.com/756f14d2ed77a7a9a3da9b0d9501cab9b7433e36/687474703a2f2f692e696d6775722e636f6d2f333943415335542e706e67)\\n\\n\\x1b[33m\\x1b[32m[](#ec2-instance-ids)\\x1b[0mEC2 Instance Ids\\x1b[0m\\n\\nOption for \\x1b[36m`ec2`\\x1b[0m:\\n\\n--instance-ids\\n\\nSample Usage:\\n\\naws ec2 describe-instances --instance-ids\\n aws ec2 ls --instance-ids\\n\\nNote: The \\x1b[36m`ls`\\x1b[0m command demonstrates the use of \\x1b[32m[customizable shortcuts](#customizable-shortcuts)\\x1b[0m:\\n\\n\\x1b[32m[![](https://camo.githubusercontent.com/d4b0298e0ecd0afc1dac0d62617a4be53a44f598/687474703a2f2f692e696d6775722e636f6d2f6a46794353586c2e706e67)\\x1b[0m](https://camo.githubusercontent.com/d4b0298e0ecd0afc1dac0d62617a4be53a44f598/687474703a2f2f692e696d6775722e636f6d2f6a46794353586c2e706e67)\\n\\n\\x1b[33m\\x1b[32m[](#ec2-instance-tags)\\x1b[0mEC2 Instance Tags\\x1b[0m\\n\\nOption for \\x1b[36m`ec2`\\x1b[0m:\\n\\n--ec2-tag-key\\n --ec2-tag-value\\n\\nSample Usage:\\n\\naws ec2 ls --ec2-tag-key\\n aws ec2 ls --ec2-tag-value\\n\\n\\x1b[36mTags support wildcards\\x1b[0m with the \\x1b[36m`*`\\x1b[0m character.\\n\\nNote: \\x1b[36m`ls`\\x1b[0m, \\x1b[36m`--ec2-tag-value`\\x1b[0m, and \\x1b[36m`--ec2-tag-key`\\x1b[0m demonstrate the use of \\x1b[32m[customizable shortcuts](#customizable-shortcuts)\\x1b[0m:\\n\\n\\x1b[32m[![](https://camo.githubusercontent.com/593e9fc80cc6e1fb8e0fd7324cb4cb3a1b51bf2f/687474703a2f2f692e696d6775722e636f6d2f56494b7747335a2e706e67)\\x1b[0m](https://camo.githubusercontent.com/593e9fc80cc6e1fb8e0fd7324cb4cb3a1b51bf2f/687474703a2f2f692e696d6775722e636f6d2f56494b7747335a2e706e67)\\n\\n\\x1b[33m\\x1b[32m[](#todo-add-more-resources)\\x1b[0mTODO: Add More Resources\\x1b[0m\\n\\nFeel free to \\x1b[32m[submit an issue or a pull request](#contributions)\\x1b[0m if you\\'d like support for additional resources.\\n\\n\\x1b[33m\\x1b[32m[](#customizable-shortcuts)\\x1b[0mCustomizable Shortcuts\\x1b[0m\\n\\nThe \\x1b[32m[~/.saws.shortcuts](https://github.com/donnemartin/saws/blob/master/saws/saws.shortcuts)\\x1b[0m file contains shortcuts that you can modify. It comes pre-populated with several \\x1b[32m[handy shortcuts](https://github.com/donnemartin/saws/blob/master/saws/saws.shortcuts)\\x1b[0m out of the box. You can combine shortcuts with \\x1b[32m[fuzzy completion](#fuzzy-resource-and-shortcut-completion)\\x1b[0m for even less keystrokes. Below are a few examples.\\n\\nList all EC2 instances:\\n\\naws ec2 ls\\n\\nList all running EC2 instances:\\n\\naws ec2 ls --ec2-state running \\x1b[33mfuzzy shortcut: aws ecstate\\x1b[0m\\n\\n\\x1b[32m[![](https://camo.githubusercontent.com/7b56fd124f76cf2a9573de7b963bbb791f4cae16/687474703a2f2f692e696d6775722e636f6d2f6a594645736f4d2e706e67)\\x1b[0m](https://camo.githubusercontent.com/7b56fd124f76cf2a9573de7b963bbb791f4cae16/687474703a2f2f692e696d6775722e636f6d2f6a594645736f4d2e706e67)\\n\\nList all EC2 instances with a matching tag (supports wildcards \\x1b[36m`*`\\x1b[0m):\\n\\naws ec2 ls --ec2-tag-key \\x1b[33mfuzzy shortcut: aws ectagk\\x1b[0m\\n aws ec2 ls --ec2-tag-value \\x1b[33mfuzzy shortcut: aws ectagv\\x1b[0m\\n\\n\\x1b[32m[![](https://camo.githubusercontent.com/c45874a55d4ac6485ba78c6d4a91cc1d928703a4/687474703a2f2f692e696d6775722e636f6d2f505375775549772e706e67)\\x1b[0m](https://camo.githubusercontent.com/c45874a55d4ac6485ba78c6d4a91cc1d928703a4/687474703a2f2f692e696d6775722e636f6d2f505375775549772e706e67)\\n\\nList EC2 instance with matching id:\\n\\naws ec2 ls --instance-ids \\x1b[33mfuzzy shortcut: aws eclsi\\x1b[0m\\n\\n\\x1b[32m[![](https://camo.githubusercontent.com/48136027017398a300b9fe968e50e2e7c2a8b18b/687474703a2f2f692e696d6775722e636f6d2f774763554373612e706e67)\\x1b[0m](https://camo.githubusercontent.com/48136027017398a300b9fe968e50e2e7c2a8b18b/687474703a2f2f692e696d6775722e636f6d2f774763554373612e706e67)\\n\\nList all DynamoDB tables:\\n\\naws dynamodb ls \\x1b[33mfuzzy shortcut: aws dls\\x1b[0m\\n\\nList all EMR clusters:\\n\\naws emr ls \\x1b[33mfuzzy shortcut: aws emls\\x1b[0m\\n\\nAdd/remove/modify shortcuts in your \\x1b[32m[~/.saws.shortcuts](https://github.com/donnemartin/saws/blob/master/saws/shortcuts)\\x1b[0m file to suit your needs.\\n\\nFeel free to submit:\\n\\n\\x1b[36m * An issue to request additional shortcuts\\x1b[0m\\n\\x1b[36m * A pull request if you\\'d like to share your shortcuts (see \\x1b[32m[contributing guidelines](#contributions)\\x1b[0m)\\x1b[0m\\n\\n\\x1b[33m\\x1b[32m[](#fuzzy-resource-and-shortcut-completion)\\x1b[0mFuzzy Resource and Shortcut Completion\\x1b[0m\\n\\nTo toggle fuzzy completion of AWS resources and shortcuts, use \\x1b[36m`F3`\\x1b[0m key.\\n\\nSample fuzzy shortcuts to start and stop EC2 instances:\\n\\naws ecstop\\n aws ecstart\\n\\nNote: Fuzzy completion currently only works with AWS \\x1b[32m[resources](#auto-completion-of-aws-resources)\\x1b[0m and \\x1b[32m[shortcuts](/donnemartin/saws/blob/master/customizable-shortcuts)\\x1b[0m.\\n\\n\\x1b[32m[![](https://camo.githubusercontent.com/a918a4d70ca61189149746e052f5bb2ffec9ae65/687474703a2f2f692e696d6775722e636f6d2f374f76464843772e706e67)\\x1b[0m](https://camo.githubusercontent.com/a918a4d70ca61189149746e052f5bb2ffec9ae65/687474703a2f2f692e696d6775722e636f6d2f374f76464843772e706e67)\\n\\n\\x1b[33m\\x1b[32m[](#fish-style-auto-suggestions)\\x1b[0mFish-Style Auto-Suggestions\\x1b[0m\\n\\n\\x1b[36m`SAWS`\\x1b[0m supports Fish-style auto-suggestions. Use the \\x1b[36m`right arrow`\\x1b[0m key to complete a suggestion.\\n\\n\\x1b[32m[![](https://camo.githubusercontent.com/6671d93da542b2f2cdec589fcb2c174e3bbc25f6/687474703a2f2f692e696d6775722e636f6d2f743532303071312e706e67)\\x1b[0m](https://camo.githubusercontent.com/6671d93da542b2f2cdec589fcb2c174e3bbc25f6/687474703a2f2f692e696d6775722e636f6d2f743532303071312e706e67)\\n\\n\\x1b[33m\\x1b[32m[](#executing-shell-commands)\\x1b[0mExecuting Shell Commands\\x1b[0m\\n\\n\\x1b[36m`SAWS`\\x1b[0m allows you to execute shell commands from the \\x1b[36m`saws>`\\x1b[0m prompt.\\n\\n\\x1b[32m[![](https://camo.githubusercontent.com/4d26d484e5b94b7ca62607ec1557d847f5db96f3/687474703a2f2f692e696d6775722e636f6d2f4669536e3662322e706e67)\\x1b[0m](https://camo.githubusercontent.com/4d26d484e5b94b7ca62607ec1557d847f5db96f3/687474703a2f2f692e696d6775722e636f6d2f4669536e3662322e706e67)\\n\\n\\x1b[33m\\x1b[32m[](#command-history)\\x1b[0mCommand History\\x1b[0m\\n\\n\\x1b[36m`SAWS`\\x1b[0m keeps track of commands you enter and stores them in \\x1b[36m`~/.saws-history`\\x1b[0m. Use the up and down arrow keys to cycle through the command history.\\n\\n\\x1b[32m[![](https://camo.githubusercontent.com/a5ded83f813be928236c804a90d85fc0fd5e517d/687474703a2f2f692e696d6775722e636f6d2f7a3852724451422e706e67)\\x1b[0m](https://camo.githubusercontent.com/a5ded83f813be928236c804a90d85fc0fd5e517d/687474703a2f2f692e696d6775722e636f6d2f7a3852724451422e706e67)\\n\\n\\x1b[33m\\x1b[32m[](#contextual-help)\\x1b[0mContextual Help\\x1b[0m\\n\\n\\x1b[36m`SAWS`\\x1b[0m supports contextual command line \\x1b[36m`help`\\x1b[0m and contextual web \\x1b[36m`docs`\\x1b[0m.\\n\\n\\x1b[33m\\x1b[32m[](#contextual-command-line-help)\\x1b[0mContextual Command Line Help\\x1b[0m\\n\\nThe \\x1b[36m`help`\\x1b[0m command is powered by the AWS CLI and outputs help within the command line.\\n\\nUsage:\\n\\naws help\\n\\n\\x1b[32m[![](https://camo.githubusercontent.com/e10dae70d7f9303bc5fae0e3f5dba453fa905df7/687474703a2f2f692e696d6775722e636f6d2f7a536b7a7436792e706e67)\\x1b[0m](https://camo.githubusercontent.com/e10dae70d7f9303bc5fae0e3f5dba453fa905df7/687474703a2f2f692e696d6775722e636f6d2f7a536b7a7436792e706e67)\\n\\n\\x1b[33m\\x1b[32m[](#contextual-web-docs)\\x1b[0mContextual Web Docs\\x1b[0m\\n\\nSometimes you\\'re not quite sure what specific command/subcommand/option combination you need to use. In such cases, browsing through several combinations with the \\x1b[36m`help`\\x1b[0m command line is cumbersome versus browsing the online AWS CLI docs through a web browser.\\n\\n\\x1b[36m`SAWS`\\x1b[0m supports contextual web docs with the \\x1b[36m`docs`\\x1b[0m command or the \\x1b[36m`F9`\\x1b[0m key. \\x1b[36m`SAWS`\\x1b[0m will display the web docs specific to the currently entered command and subcommand.\\n\\nUsage:\\n\\naws docs\\n\\n\\x1b[32m[![](https://camo.githubusercontent.com/e0a9e9bd21f2ff174f2e257f0c9966c6620d97d7/687474703a2f2f692e696d6775722e636f6d2f7a4b34494a59702e706e67)\\x1b[0m](https://camo.githubusercontent.com/e0a9e9bd21f2ff174f2e257f0c9966c6620d97d7/687474703a2f2f692e696d6775722e636f6d2f7a4b34494a59702e706e67)\\n\\n\\x1b[33m\\x1b[32m[](#toolbar-options)\\x1b[0mToolbar Options\\x1b[0m\\n\\n\\x1b[36m`SAWS`\\x1b[0m supports a number of toolbar options:\\n\\n\\x1b[36m * \\x1b[36m`F2`\\x1b[0m toggles \\x1b[32m[output syntax highlighting](#syntax-and-output-highlighting)\\x1b[0m\\x1b[0m\\n\\x1b[36m * \\x1b[36m`F3`\\x1b[0m toggles \\x1b[32m[fuzzy completion of AWS resources and shortcuts](#fuzzy-resource-and-shortcut-completion)\\x1b[0m\\x1b[0m\\n\\x1b[36m * \\x1b[36m`F4`\\x1b[0m toggles \\x1b[32m[completion of shortcuts](#customizable-shortcuts)\\x1b[0m\\x1b[0m\\n\\x1b[36m * \\x1b[36m`F5`\\x1b[0m refreshes \\x1b[32m[resources for auto-completion](#auto-completion-of-aws-resources)\\x1b[0m\\x1b[0m\\n\\x1b[36m * \\x1b[36m`F9`\\x1b[0m displays the \\x1b[32m[contextual web docs](#contextual-web-docs)\\x1b[0m\\x1b[0m\\n\\x1b[36m * \\x1b[36m`F10`\\x1b[0m or \\x1b[36m`control d`\\x1b[0m exits \\x1b[36m`SAWS`\\x1b[0m\\x1b[0m\\n\\n\\x1b[32m[![](https://camo.githubusercontent.com/f1bb53eda629178af0d849a112193a91daca5628/687474703a2f2f692e696d6775722e636f6d2f37767a384f53632e706e67)\\x1b[0m](https://camo.githubusercontent.com/f1bb53eda629178af0d849a112193a91daca5628/687474703a2f2f692e696d6775722e636f6d2f37767a384f53632e706e67)\\n\\n\\x1b[33m\\x1b[32m[](#windows-support)\\x1b[0mWindows Support\\x1b[0m\\n\\n\\x1b[36m`SAWS`\\x1b[0m has been tested on Windows 7 and Windows 10.\\n\\nOn Windows, the \\x1b[32m[.sawsrc](https://github.com/donnemartin/saws/blob/master/saws/sawsrc)\\x1b[0m file can be found in \\x1b[36m`%userprofile%`\\x1b[0m. For example:\\n\\nC:\\\\Users\\\\dmartin\\\\.sawsrc\\n\\nAlthough you can use the standard Windows command prompt, you\\'ll probably have a better experience with either \\x1b[32m[cmder](https://github.com/cmderdev/cmder)\\x1b[0m or \\x1b[32m[conemu](https://github.com/Maximus5/ConEmu)\\x1b[0m.\\n\\n\\x1b[32m[![](https://camo.githubusercontent.com/cfd9f911b9960308e0346d90ba4cf2338145fb52/687474703a2f2f692e696d6775722e636f6d2f7055774a57636b2e706e67)\\x1b[0m](https://camo.githubusercontent.com/cfd9f911b9960308e0346d90ba4cf2338145fb52/687474703a2f2f692e696d6775722e636f6d2f7055774a57636b2e706e67)\\n\\n\\x1b[33m\\x1b[32m[](#installation)\\x1b[0mInstallation\\x1b[0m\\n\\n\\x1b[33m\\x1b[32m[](#pip-installation)\\x1b[0mPip Installation\\x1b[0m\\n\\n\\x1b[32m[![PyPI version](https://camo.githubusercontent.com/818edda38216a62a4bb36557372877febec35086/68747470733a2f2f62616467652e667572792e696f2f70792f736177732e737667)\\x1b[0m](http://badge.fury.io/py/saws) \\x1b[32m[![PyPI](https://camo.githubusercontent.com/04574f61c5dff98734bd6a013cc9217b9a428e47/68747470733a2f2f696d672e736869656c64732e696f2f707970692f707976657273696f6e732f736177732e737667)\\x1b[0m](https://pypi.python.org/pypi/saws/)\\n\\n\\x1b[36m`SAWS`\\x1b[0m is hosted on \\x1b[32m[PyPI](https://pypi.python.org/pypi/saws)\\x1b[0m. The following command will install \\x1b[36m`SAWS`\\x1b[0m along with dependencies such as the \\x1b[32m[AWS CLI](https://github.com/aws/aws-cli)\\x1b[0m:\\n\\n$ pip install saws\\n\\nYou can also install the latest \\x1b[36m`SAWS`\\x1b[0m from GitHub source which can contain changes not yet pushed to PyPI:\\n\\n$ pip install git+https://github.com/donnemartin/saws.git\\n\\nIf you are not installing in a \\x1b[32m[virtualenv](#virtual-environment-and-docker-installation)\\x1b[0m, run with \\x1b[36m`sudo`\\x1b[0m:\\n\\n$ sudo pip install saws\\n\\nOnce installed, start \\x1b[36m`SAWS`\\x1b[0m:\\n\\n$ saws\\n\\n\\x1b[33m\\x1b[32m[](#virtual-environment-and-docker-installation)\\x1b[0mVirtual Environment and Docker Installation\\x1b[0m\\n\\nIt is recommended that you install Python packages in a \\x1b[32m[virtualenv](http://docs.python-guide.org/en/latest/dev/virtualenvs/)\\x1b[0m to avoid potential \\x1b[32m[issues with dependencies or permissions](https://github.com/donnemartin/saws/issues/15)\\x1b[0m.\\n\\nTo view \\x1b[36m`SAWS`\\x1b[0m \\x1b[36m`virtualenv`\\x1b[0m and \\x1b[32m[Docker](https://www.docker.com/)\\x1b[0m installation instructions, click \\x1b[32m[here](https://github.com/donnemartin/saws/blob/master/INSTALLATION.md)\\x1b[0m.\\n\\n\\x1b[33m\\x1b[32m[](#aws-credentials-and-named-profiles)\\x1b[0mAWS Credentials and Named Profiles\\x1b[0m\\n\\n\\x1b[32m[Configure your credentials](https://github.com/aws/aws-cli#getting-started)\\x1b[0m with the AWS CLI:\\n\\n$ aws configure\\n\\nIf you\\'d like to use a specific named profile with \\x1b[36m`SAWS`\\x1b[0m, run the following commands on OS X, Linux, or Unix:\\n\\n$ export AWS_DEFAULT_PROFILE=user1\\n $ saws\\n\\nOr as a one-liner:\\n\\n$ AWS_DEFAULT_PROFILE=user1 saws\\n\\nWindows users can run the following commands:\\n\\n> set AWS_DEFAULT_PROFILE=user1\\n > saws\\n\\nCommand line options for starting \\x1b[36m`SAWS`\\x1b[0m with a specific profile are \\x1b[32m[under development](https://github.com/donnemartin/saws/issues/16)\\x1b[0m. For more details on how to install and configure the AWS CLI, refer to the following \\x1b[32m[documentation](http://docs.aws.amazon.com/cli/latest/userguide/installing.html)\\x1b[0m.\\n\\n\\x1b[33m\\x1b[32m[](#supported-python-versions)\\x1b[0mSupported Python Versions\\x1b[0m\\n\\n\\x1b[36m * Python 2.6\\x1b[0m\\n\\x1b[36m * Python 2.7\\x1b[0m\\n\\x1b[36m * Python 3.3\\x1b[0m\\n\\x1b[36m * Python 3.4\\x1b[0m\\n\\x1b[36m * Pypy\\x1b[0m\\n\\nLight testing indicates that \\x1b[36m`SAWS`\\x1b[0m also seems to be compatible with Python 3.5.\\n\\nPypy3 is not supported due to \\x1b[32m[lack of support](https://github.com/boto/botocore/issues/622)\\x1b[0m from \\x1b[32m[boto](https://github.com/boto/boto)\\x1b[0m.\\n\\n\\x1b[33m\\x1b[32m[](#supported-platforms)\\x1b[0mSupported Platforms\\x1b[0m\\n\\n\\x1b[36m * Mac OS X \\x1b[0m\\n \\x1b[36m * Tested on OS X 10.10\\x1b[0m\\n\\x1b[36m * Linux, Unix \\x1b[0m\\n \\x1b[36m * Tested on Ubuntu 14.04 LTS\\x1b[0m\\n\\x1b[36m * Windows \\x1b[0m\\n \\x1b[36m * Tested on Windows 7 and 10\\x1b[0m\\n\\n\\x1b[33m\\x1b[32m[](#developer-installation)\\x1b[0mDeveloper Installation\\x1b[0m\\n\\nIf you\\'re interested in contributing to \\x1b[36m`SAWS`\\x1b[0m, run the following commands:\\n\\n$ git clone https://github.com/donnemartin/saws.git\\n $ pip install -e .\\n $ pip install -r requirements-dev.txt\\n $ saws\\n\\n\\x1b[33m\\x1b[32m[](#continuous-integration)\\x1b[0mContinuous Integration\\x1b[0m\\n\\n\\x1b[32m[![Build Status](https://camo.githubusercontent.com/c1d6d1e8f7cdac1e6a82bf900832bbbe52d61e95/68747470733a2f2f7472617669732d63692e6f72672f646f6e6e656d617274696e2f736177732e7376673f6272616e63683d6d6173746572)\\x1b[0m](https://travis-ci.org/donnemartin/saws)\\n\\nContinuous integration details are available on \\x1b[32m[Travis CI](https://travis-ci.org/donnemartin/saws)\\x1b[0m.\\n\\n\\x1b[33m\\x1b[32m[](#dependencies-management)\\x1b[0mDependencies Management\\x1b[0m\\n\\n\\x1b[32m[![Dependency Status](https://camo.githubusercontent.com/11e16ba1560f0232928945436bf86340615d85ae/68747470733a2f2f67656d6e617369756d2e636f6d2f646f6e6e656d617274696e2f736177732e737667)\\x1b[0m](https://gemnasium.com/donnemartin/saws)\\n\\nDependencies management details are available on \\x1b[32m[Gemnasium](https://gemnasium.com/donnemartin/saws)\\x1b[0m.\\n\\n\\x1b[33m\\x1b[32m[](#unit-tests-and-code-coverage)\\x1b[0mUnit Tests and Code Coverage\\x1b[0m\\n\\n\\x1b[32m[![Codecov](https://camo.githubusercontent.com/e24a50e9dd63768d90d26a9f6be1cf3d0c3b0230/68747470733a2f2f696d672e736869656c64732e696f2f636f6465636f762f632f6769746875622f646f6e6e656d617274696e2f736177732e737667)\\x1b[0m](https://codecov.io/github/donnemartin/saws/saws)\\n\\n\\x1b[32m[![](https://camo.githubusercontent.com/a3588f226c5b2c9007fd9e49212ae8294c5bc023/687474703a2f2f636f6465636f762e696f2f6769746875622f646f6e6e656d617274696e2f736177732f6272616e63682e7376673f6272616e63683d6d6173746572)\\x1b[0m](https://camo.githubusercontent.com/a3588f226c5b2c9007fd9e49212ae8294c5bc023/687474703a2f2f636f6465636f762e696f2f6769746875622f646f6e6e656d617274696e2f736177732f6272616e63682e7376673f6272616e63683d6d6173746572)\\n\\nCode coverage details are available on \\x1b[32m[Codecov](https://codecov.io/github/donnemartin/saws/saws)\\x1b[0m.\\n\\nRun unit tests in your active Python environment:\\n\\n$ python tests/run_tests.py\\n\\nRun unit tests with \\x1b[32m[tox](https://pypi.python.org/pypi/tox)\\x1b[0m on multiple Python environments:\\n\\n$ tox\\n\\n\\x1b[33m\\x1b[32m[](#documentation)\\x1b[0mDocumentation\\x1b[0m\\n\\n\\x1b[32m[![Documentation Status](https://camo.githubusercontent.com/41afe5f96e9f915ecd9706bdc7dfd19518eaf05a/68747470733a2f2f72656164746865646f63732e6f72672f70726f6a656374732f736177732f62616467652f3f76657273696f6e3d6c6174657374)\\x1b[0m](http://saws.readthedocs.org/en/latest/?badge=latest)\\n\\nSource code documentation is available on \\x1b[32m[Readthedocs.org](http://saws.readthedocs.org/en/latest/?badge=latest)\\x1b[0m.\\n\\nRun the following to build the docs:\\n\\n$ scripts/update_docs.sh\\n\\n\\x1b[33m\\x1b[32m[](#contributing)\\x1b[0mContributing\\x1b[0m\\n\\nContributions are welcome!\\n\\nReview the \\x1b[32m[Contributing Guidelines](https://github.com/donnemartin/saws/blob/master/CONTRIBUTING.md)\\x1b[0m for details on how to:\\n\\n\\x1b[36m * Submit issues\\x1b[0m\\n\\x1b[36m * Submit pull requests\\x1b[0m\\n\\n\\x1b[33m\\x1b[32m[](#credits)\\x1b[0mCredits\\x1b[0m\\n\\n\\x1b[36m * \\x1b[32m[AWS CLI](https://github.com/aws/aws-cli)\\x1b[0m by \\x1b[32m[AWS](https://github.com/aws)\\x1b[0m for powering \\x1b[36m`SAWS`\\x1b[0m under the hood\\x1b[0m\\n\\x1b[36m * \\x1b[32m[Python Prompt Toolkit](https://github.com/jonathanslenders/python-prompt-toolkit)\\x1b[0m by \\x1b[32m[jonathanslenders](https://github.com/jonathanslenders)\\x1b[0m for simplifying the creation of \\x1b[36m`SAWS`\\x1b[0m\\x1b[0m\\n\\x1b[36m * \\x1b[32m[Wharfee](https://github.com/j-bennet/wharfee)\\x1b[0m by \\x1b[32m[j-bennet](https://github.com/j-bennet)\\x1b[0m for inspiring the creation of \\x1b[36m`SAWS`\\x1b[0m and for some handy utility functions\\x1b[0m\\n\\n\\x1b[33m\\x1b[32m[](#contact-info)\\x1b[0mContact Info\\x1b[0m\\n\\nFeel free to contact me to discuss any issues, questions, or comments.\\n\\n\\x1b[36m * Email: \\x1b[32m[donne.martin@gmail.com](mailto:donne.martin@gmail.com)\\x1b[0m\\x1b[0m\\n\\x1b[36m * Twitter: \\x1b[32m[donne_martin](https://twitter.com/donne_martin)\\x1b[0m\\x1b[0m\\n\\x1b[36m * GitHub: \\x1b[32m[donnemartin](https://github.com/donnemartin)\\x1b[0m\\x1b[0m\\n\\x1b[36m * LinkedIn: \\x1b[32m[donnemartin](https://www.linkedin.com/in/donnemartin)\\x1b[0m\\x1b[0m\\n\\x1b[36m * Website: \\x1b[32m[donnemartin.com](http://donnemartin.com)\\x1b[0m\\x1b[0m\\n\\n\\x1b[33m\\x1b[32m[](#license)\\x1b[0mLicense\\x1b[0m\\n\\nCopyright 2015 Donne Martin\\n\\nLicensed under the Apache License, Version 2.0 (the \"License\");\\n you may not use this file except in compliance with the License.\\n You may obtain a copy of the License at\\n\\nhttp://www.apache.org/licenses/LICENSE-2.0\\n\\nUnless required by applicable law or agreed to in writing, software\\n distributed under the License is distributed on an \"AS IS\" BASIS,\\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\\n See the License for the specific language governing permissions and\\n limitations under the License.\\n\\n\\x1b[36m * \\x1b[32m[Status](https://status.github.com/)\\x1b[0m\\x1b[0m\\n\\x1b[36m * \\x1b[32m[API](https://developer.github.com)\\x1b[0m\\x1b[0m\\n\\x1b[36m * \\x1b[32m[Training](https://training.github.com)\\x1b[0m\\x1b[0m\\n\\x1b[36m * \\x1b[32m[Shop](https://shop.github.com)\\x1b[0m\\x1b[0m\\n\\x1b[36m * \\x1b[32m[Blog](https://github.com/blog)\\x1b[0m\\x1b[0m\\n\\x1b[36m * \\x1b[32m[About](https://github.com/about)\\x1b[0m\\x1b[0m\\n\\x1b[36m * \\x1b[32m[Pricing](https://github.com/pricing)\\x1b[0m\\x1b[0m\\n\\x1b[32m[ ](https://github.com)\\x1b[0m\\n\\n\\x1b[36m * (C) 2015 GitHub, Inc.\\x1b[0m\\n\\x1b[36m * \\x1b[32m[Terms](https://github.com/site/terms)\\x1b[0m\\x1b[0m\\n\\x1b[36m * \\x1b[32m[Privacy](https://github.com/site/privacy)\\x1b[0m\\x1b[0m\\n\\x1b[36m * \\x1b[32m[Security](https://github.com/security)\\x1b[0m\\x1b[0m\\n\\x1b[36m * \\x1b[32m[Contact](https://github.com/contact)\\x1b[0m\\x1b[0m\\n\\x1b[36m * \\x1b[32m[Help](https://help.github.com)\\x1b[0m\\x1b[0m\\n\\nSomething went wrong with that request. Please try again.\\n\\nYou signed in with another tab or window. [Reload]() to refresh your session. You signed out in another tab or window. [Reload]() to refresh your session.\\n\n\"\"\" # NOQA\nssl_error = u\"Viewing https://requestb.in\\n\\n\\x1b[0mError: hostname 'requestb.in' doesn't match either of '*.herokuapp.com', 'herokuapp.com'\\nTry running gh view # with the --browser/-b flag\\n\\nView this article in a browser with the -b/--browser flag.\\n\\x1b[0m\\nPress q to quit viewing this article.\\n\\x1b[0m\" # NOQA\n" }, { "path": "tests/data/thread.py", "content": "# -*- coding: utf-8 -*-\n\n# Copyright 2015 Donne Martin. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"). You\n# may not use this file except in compliance with the License. A copy of\n# the License is located at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# or in the \"license\" file accompanying this file. This file is\n# distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF\n# ANY KIND, either express or implied. See the License for the specific\n# language governing permissions and limitations under the License.\n\nformatted_threads = u'\\x1b[35m 1. \\x1b[0mtitle1 \\x1b[0m\\x1b[35m(foo/bar/issues/1)\\x1b[0m\\n\\x1b[32m Seen: False \\x1b[0m\\x1b[36mType: type1 \\x1b[0m\\x1b[33mUpdated: just now \\x1b[0m\\n\\x1b[35m 2. \\x1b[0mtitle2 \\x1b[0m\\x1b[35m(foo/bar/issues/1)\\x1b[0m\\n\\x1b[32m Seen: True \\x1b[0m\\x1b[36mType: type2 \\x1b[0m\\x1b[33mUpdated: just now \\x1b[0m\\n\\x1b[35m 3. \\x1b[0mtitle3 \\x1b[0m\\x1b[35m(foo/bar/issues/1)\\x1b[0m\\n\\x1b[32m Seen: False \\x1b[0m\\x1b[36mType: type3 \\x1b[0m\\x1b[33mUpdated: just now \\x1b[0m\\n View the page for \\x1b[0m\\x1b[35m1 through \\x1b[0m\\x1b[35m3\\x1b[0m with the following command:\\n\\x1b[0m\\x1b[35m gh view [#] \\x1b[0moptional: [-b/--browser] [--help]\\n\\x1b[0m\\x1b[0m'\n" }, { "path": "tests/data/trends.py", "content": "# -*- coding: utf-8 -*-\n\n# Copyright 2015 Donne Martin. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"). You\n# may not use this file except in compliance with the License. A copy of\n# the License is located at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# or in the \"license\" file accompanying this file. This file is\n# distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF\n# ANY KIND, either express or implied. See the License for the specific\n# language governing permissions and limitations under the License.\n\nformatted_trends = u'\\x1b[35m 1. \\x1b[0mpuikinsh/gentelella \\x1b[0m\\n Free Bootstrap 3 Admin Template\\x1b[0m\\n HTML\\x1b[0m\\n\\x1b[35m 2. \\x1b[0mrg3/youtube-dl \\x1b[0m\\n Small command-line program to download videos from YouTube.com and\\n other video sites\\x1b[0m\\n Python\\x1b[0m\\n\\x1b[35m 3. \\x1b[0mFreeCodeCamp/FreeCodeCamp \\x1b[0m\\n The http://FreeCodeCamp.com open source codebase and curriculum.\\n Learn to code and help nonprofits.\\x1b[0m\\n JavaScript\\x1b[0m\\n\\x1b[35m 4. \\x1b[0mmartanne/vis \\x1b[0m\\n a vim like text editor\\x1b[0m\\n C\\x1b[0m\\n\\x1b[35m 5. \\x1b[0mFezVrasta/popper.js \\x1b[0m\\n A ~3.5KB kickass library to manage your poppers\\x1b[0m\\n JavaScript\\x1b[0m\\n\\x1b[35m 6. \\x1b[0mfirehol/netdata \\x1b[0m\\n Real-time performance monitoring, done right!\\x1b[0m\\n C\\x1b[0m\\n\\x1b[35m 7. \\x1b[0marachnys/athenapdf \\x1b[0m\\n Drop-in replacement for wkhtmltopdf built on Go, Electron and Docker\\x1b[0m\\n Go\\x1b[0m\\n\\x1b[35m 8. \\x1b[0mmortenjust/cleartext-mac \\x1b[0m\\n A text editor that only allows the top 1,000 most common words in\\n English\\x1b[0m\\n Swift\\x1b[0m\\n\\x1b[35m 9. \\x1b[0mfeross/webtorrent-desktop \\x1b[0m\\n WebTorrent, the streaming torrent client. For OS X, Windows, and\\n Linux.\\x1b[0m\\n JavaScript\\x1b[0m\\n\\x1b[35m 10. \\x1b[0mfuture-architect/vuls \\x1b[0m\\n Vulnerability scanner for Linux, agentless, written in golang.\\x1b[0m\\n Go\\x1b[0m\\n\\x1b[35m 11. \\x1b[0mDevLight-Mobile-Agency/NavigationTabBar \\x1b[0m\\n Navigation tab bar with colorful interactions.\\x1b[0m\\n Java\\x1b[0m\\n\\x1b[35m 12. \\x1b[0mfeross/webtorrent \\x1b[0m\\n Streaming torrent client for the web\\x1b[0m\\n JavaScript\\x1b[0m\\n\\x1b[35m 13. \\x1b[0mfacebook/react \\x1b[0m\\n A declarative, efficient, and flexible JavaScript library for\\n building user interfaces.\\x1b[0m\\n JavaScript\\x1b[0m\\n\\x1b[35m 14. \\x1b[0mtatsuhiro-t/aria2 \\x1b[0m\\n aria2 is a lightweight multi-protocol & multi-source, cross platform\\n download utility operated in command-line. It supports HTTP/HTTPS,\\n FTP, SFTP, BitTorrent and Metalink.\\x1b[0m\\n C++\\x1b[0m\\n\\x1b[35m 15. \\x1b[0mjessepollak/command \\x1b[0m\\n Making the web better with Slack-like slash commands.\\x1b[0m\\n JavaScript\\x1b[0m\\n\\x1b[35m 16. \\x1b[0msoimort/you-get \\x1b[0m\\n Dumb downloader that scrapes the web\\x1b[0m\\n Python\\x1b[0m\\n\\x1b[35m 17. \\x1b[0mpacketzoom/logzoom \\x1b[0m\\n A lightweight replacement for logstash indexer in Go\\x1b[0m\\n Go\\x1b[0m\\n\\x1b[35m 18. \\x1b[0mairbnb/caravel \\x1b[0m\\n Caravel is a data exploration platform designed to be visual,\\n intuitive, and interactive\\x1b[0m\\n Python\\x1b[0m\\n\\x1b[35m 19. \\x1b[0mdylang/npm-check \\x1b[0m\\n Check for outdated, incorrect, and unused dependencies.\\x1b[0m\\n JavaScript\\x1b[0m\\n\\x1b[35m 20. \\x1b[0mkadirahq/react-storybook \\x1b[0m\\n Isolate your React UI Component development from the main app\\x1b[0m\\n JavaScript\\x1b[0m\\n\\x1b[35m 21. \\x1b[0mnnupoor/js_designpatterns \\x1b[0m\\x1b[0m\\n JavaScript\\x1b[0m\\n\\x1b[35m 22. \\x1b[0mgooglesamples/android-architecture \\x1b[0m\\n A collection of samples to discuss and showcase different\\n architectural tools and patterns for Android apps.\\x1b[0m\\n Unknown\\x1b[0m\\n\\x1b[35m 23. \\x1b[0mMrS0m30n3/youtube-dl-gui \\x1b[0m\\n A cross platform front-end GUI of the popular youtube-dl written in\\n wxPython.\\x1b[0m\\n Python\\x1b[0m\\n\\x1b[35m 24. \\x1b[0mSkyscanner/SkyFloatingLabelTextField \\x1b[0m\\n A beautiful and flexible text field control implementation of \"Float\\n Label Pattern\". Written in Swift. As used in the Skyscanner TravelPro\\n iOS app.\\x1b[0m\\n Swift\\x1b[0m\\n\\x1b[35m 25. \\x1b[0mshutterstock/List-of-Dirty-Naughty-Obscene-and-Otherwise-Bad-Words \\x1b[0m\\n List of Dirty, Naughty, Obscene, and Otherwise Bad Words\\x1b[0m\\n Unknown\\x1b[0m\\n View the page for \\x1b[0m\\x1b[35m1 through \\x1b[0m\\x1b[35m25\\x1b[0m with the following command:\\n\\x1b[0m\\x1b[35m gh view [#] \\x1b[0moptional: [-b/--browser] [--help]\\n\\x1b[0m\\x1b[0m'\n" }, { "path": "tests/data/user.py", "content": "# -*- coding: utf-8 -*-\n\n# Copyright 2015 Donne Martin. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"). You\n# may not use this file except in compliance with the License. A copy of\n# the License is located at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# or in the \"license\" file accompanying this file. This file is\n# distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF\n# ANY KIND, either express or implied. See the License for the specific\n# language governing permissions and limitations under the License.\n\nformatted_org = u'To view the avatar in your terminal, install the Python Image Library.\\n\\x1b[0m\\x1b[0muser2\\n\\x1b[0m\\x1b[32mbigcorp\\n\\x1b[0m\\x1b[32minterwebz\\n\\x1b[0m\\x1b[32muser2@foo.com\\n\\x1b[0m\\x1b[36mOrganization\\n\\n\\x1b[0mNo results found\\x1b[0m'\nformatted_user = u'To view the avatar in your terminal, install the Python Image Library.\\n\\x1b[0m\\x1b[0muser1\\n\\x1b[0m\\x1b[32mbigcorp\\n\\x1b[0m\\x1b[32minterwebz\\n\\x1b[0m\\x1b[32muser1@foo.com\\n\\x1b[0m\\x1b[36mFollowers: 0 | \\x1b[0m\\x1b[36mFollowing: 0\\n\\n\\x1b[0m\\x1b[35m 1. \\x1b[0mrepo1 \\x1b[0m()\\x1b[0m\\n\\x1b[32m Stars: 1 \\x1b[0m\\x1b[36mForks: 1 \\x1b[0m\\x1b[33mUpdated: just now \\x1b[0m\\n View the page for \\x1b[0m\\x1b[35m1 through \\x1b[0m\\x1b[35m1\\x1b[0m with the following command:\\n\\x1b[0m\\x1b[35m gh view [#] \\x1b[0moptional: [-b/--browser] [--help]\\n\\x1b[0m\\x1b[0m'\nformatted_users = u'\\x1b[35m 1. \\x1b[0mfoo1 \\x1b[0m\\n\\x1b[35m 2. \\x1b[0mfoo2 \\x1b[0m\\n\\x1b[35m 3. \\x1b[0mfoo3 \\x1b[0m\\n View the page for \\x1b[0m\\x1b[35m1 through \\x1b[0m\\x1b[35m3\\x1b[0m with the following command:\\n\\x1b[0m\\x1b[35m gh view [#] \\x1b[0moptional: [-b/--browser] [--help]\\n\\x1b[0m\\x1b[0m'\n" }, { "path": "tests/data/user_feed.py", "content": "# -*- coding: utf-8 -*-\n\n# Copyright 2015 Donne Martin. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"). You\n# may not use this file except in compliance with the License. A copy of\n# the License is located at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# or in the \"license\" file accompanying this file. This file is\n# distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF\n# ANY KIND, either express or implied. See the License for the specific\n# language governing permissions and limitations under the License.\n\nformatted_user_feed = u'\\x1b[35m 1. \\x1b[0mLeonardoGitHub \\x1b[0m\\x1b[32mforked \\x1b[0m\\x1b[36mdonnemartin/data-science-ipython-notebooks \\x1b[0m\\x1b[33m(just now)\\x1b[0m\\n\\x1b[35m 2. \\x1b[0mnavroopsingh \\x1b[0m\\x1b[32mstarred\\x1b[0m\\x1b[36m donnemartin/dev-setup \\x1b[0m\\x1b[33m(just now)\\x1b[0m\\n\\x1b[35m 3. \\x1b[0mprabaharanr \\x1b[0m\\x1b[32mforked \\x1b[0m\\x1b[36mdonnemartin/data-science-ipython-notebooks \\x1b[0m\\x1b[33m(just now)\\x1b[0m\\n\\x1b[35m 4. \\x1b[0mwdavis12 \\x1b[0m\\x1b[32mstarred\\x1b[0m\\x1b[36m donnemartin/data-science-ipython-notebooks \\x1b[0m\\x1b[33m(just now)\\x1b[0m\\n\\x1b[35m 5. \\x1b[0mvonec \\x1b[0m\\x1b[32mcommented on issue\\x1b[0m\\x1b[36m awslabs/aws-shell#115 \\x1b[0m\\x1b[33m(just now)\\x1b[0m\\n \\x1b[36m\\x1b[0mFixed \\x1b[0m\\n\\x1b[35m 6. \\x1b[0mbenc \\x1b[0m\\x1b[32mstarred\\x1b[0m\\x1b[36m donnemartin/dev-setup \\x1b[0m\\x1b[33m(just now)\\x1b[0m\\n\\x1b[35m 7. \\x1b[0mryusun \\x1b[0m\\x1b[32mstarred\\x1b[0m\\x1b[36m donnemartin/data-science-ipython-notebooks \\x1b[0m\\x1b[33m(just now)\\x1b[0m\\n\\x1b[35m 8. \\x1b[0mchirag200666 \\x1b[0m\\x1b[32mstarred\\x1b[0m\\x1b[36m donnemartin/data-science-ipython-notebooks \\x1b[0m\\x1b[33m(just now)\\x1b[0m\\n\\x1b[35m 9. \\x1b[0mravi-kothari \\x1b[0m\\x1b[32mstarred\\x1b[0m\\x1b[36m donnemartin/interactive-coding-challenges \\x1b[0m\\x1b[33m(just now)\\x1b[0m\\n\\x1b[35m 10. \\x1b[0mravi-kothari \\x1b[0m\\x1b[32mstarred\\x1b[0m\\x1b[36m donnemartin/data-science-ipython-notebooks \\x1b[0m\\x1b[33m(just now)\\x1b[0m\\n\\x1b[35m 11. \\x1b[0mmarnovo \\x1b[0m\\x1b[32mstarred\\x1b[0m\\x1b[36m donnemartin/data-science-ipython-notebooks \\x1b[0m\\x1b[33m(just now)\\x1b[0m\\n\\x1b[35m 12. \\x1b[0mandersy005 \\x1b[0m\\x1b[32mstarred\\x1b[0m\\x1b[36m donnemartin/interactive-coding-challenges \\x1b[0m\\x1b[33m(just now)\\x1b[0m\\n\\x1b[35m 13. \\x1b[0mandersy005 \\x1b[0m\\x1b[32mstarred\\x1b[0m\\x1b[36m donnemartin/viz \\x1b[0m\\x1b[33m(just now)\\x1b[0m\\n\\x1b[35m 14. \\x1b[0mt9md \\x1b[0m\\x1b[32mstarred\\x1b[0m\\x1b[36m donnemartin/interactive-coding-challenges \\x1b[0m\\x1b[33m(just now)\\x1b[0m\\n\\x1b[35m 15. \\x1b[0msilivizzle \\x1b[0m\\x1b[32mstarred\\x1b[0m\\x1b[36m donnemartin/dev-setup \\x1b[0m\\x1b[33m(just now)\\x1b[0m\\n\\x1b[35m 16. \\x1b[0mahna \\x1b[0m\\x1b[32mstarred\\x1b[0m\\x1b[36m donnemartin/data-science-ipython-notebooks \\x1b[0m\\x1b[33m(just now)\\x1b[0m\\n\\x1b[35m 17. \\x1b[0mziyasal \\x1b[0m\\x1b[32mstarred\\x1b[0m\\x1b[36m donnemartin/saws \\x1b[0m\\x1b[33m(just now)\\x1b[0m\\n\\x1b[35m 18. \\x1b[0mchefablz \\x1b[0m\\x1b[32mstarred\\x1b[0m\\x1b[36m donnemartin/data-science-ipython-notebooks \\x1b[0m\\x1b[33m(just now)\\x1b[0m\\n\\x1b[35m 19. \\x1b[0mkevinkn \\x1b[0m\\x1b[32mstarred\\x1b[0m\\x1b[36m donnemartin/dev-setup \\x1b[0m\\x1b[33m(just now)\\x1b[0m\\n\\x1b[35m 20. \\x1b[0mfrrp \\x1b[0m\\x1b[32mstarred\\x1b[0m\\x1b[36m donnemartin/interactive-coding-challenges \\x1b[0m\\x1b[33m(just now)\\x1b[0m\\n\\x1b[35m 21. \\x1b[0mfrrp \\x1b[0m\\x1b[32mstarred\\x1b[0m\\x1b[36m donnemartin/data-science-ipython-notebooks \\x1b[0m\\x1b[33m(just now)\\x1b[0m\\n\\x1b[35m 22. \\x1b[0mLiberTang0 \\x1b[0m\\x1b[32mforked \\x1b[0m\\x1b[36mdonnemartin/data-science-ipython-notebooks \\x1b[0m\\x1b[33m(just now)\\x1b[0m\\n\\x1b[35m 23. \\x1b[0mPhilippe-Caparroy \\x1b[0m\\x1b[32mstarred\\x1b[0m\\x1b[36m donnemartin/data-science-ipython-notebooks \\x1b[0m\\x1b[33m(just now)\\x1b[0m\\n\\x1b[35m 24. \\x1b[0mPhilippe-Caparroy \\x1b[0m\\x1b[32mforked \\x1b[0m\\x1b[36mdonnemartin/data-science-ipython-notebooks \\x1b[0m\\x1b[33m(just now)\\x1b[0m\\n\\x1b[35m 25. \\x1b[0mtiendan \\x1b[0m\\x1b[32mstarred\\x1b[0m\\x1b[36m donnemartin/data-science-ipython-notebooks \\x1b[0m\\x1b[33m(just now)\\x1b[0m\\n\\x1b[35m 26. \\x1b[0mSkyfold \\x1b[0m\\x1b[32mclosed issue\\x1b[0m\\x1b[36m awslabs/aws-shell#115 \\x1b[0m\\x1b[33m(just now)\\x1b[0m\\n\\x1b[35m 27. \\x1b[0mSkyfold \\x1b[0m\\x1b[32mcommented on issue\\x1b[0m\\x1b[36m awslabs/aws-shell#115 \\x1b[0m\\x1b[33m(just now)\\x1b[0m\\n \\x1b[36m\\x1b[0mFixed \\x1b[0m\\n\\x1b[35m 28. \\x1b[0msouadah \\x1b[0m\\x1b[32mforked \\x1b[0m\\x1b[36mdonnemartin/data-science-ipython-notebooks \\x1b[0m\\x1b[33m(just now)\\x1b[0m\\n\\x1b[35m 29. \\x1b[0mjoshuadsilva \\x1b[0m\\x1b[32mcommented on issue\\x1b[0m\\x1b[36m awslabs/aws-shell#115 \\x1b[0m\\x1b[33m(just now)\\x1b[0m\\n \\x1b[36m\\x1b[0mWorks for me too img class=\"emoji\" title=\":+1:\"\\n alt=\":+1:\" src=\"https://assets-\\n cdn.github.com/images/icons/emoji/unicode/1f44d.png\" height=\"20\"\\n width=\"20\" align=\"absmiddle\"\\x1b[0m\\n\\x1b[35m 30. \\x1b[0mwilsantos99 \\x1b[0m\\x1b[32mstarred\\x1b[0m\\x1b[36m donnemartin/awesome-aws \\x1b[0m\\x1b[33m(just now)\\x1b[0m\\n\\x1b[0m'\n" }, { "path": "tests/mock_feed_parser.py", "content": "# -*- coding: utf-8 -*-\n\n# Copyright 2015 Donne Martin. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"). You\n# may not use this file except in compliance with the License. A copy of\n# the License is located at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# or in the \"license\" file accompanying this file. This file is\n# distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF\n# ANY KIND, either express or implied. See the License for the specific\n# language governing permissions and limitations under the License.\n\nfrom __future__ import unicode_literals\nfrom __future__ import print_function\n\nimport pickle\n\n\nclass MockFeedParser(object):\n\n def parse(self, url):\n if url == 'user_feed':\n with open('tests/data/user_feed.p', 'rb') as fp:\n return pickle.load(fp)\n else:\n with open('tests/data/trends.p', 'rb') as fp:\n return pickle.load(fp)\n" }, { "path": "tests/mock_github_api.py", "content": "# -*- coding: utf-8 -*-\n\n# Copyright 2015 Donne Martin. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"). You\n# may not use this file except in compliance with the License. A copy of\n# the License is located at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# or in the \"license\" file accompanying this file. This file is\n# distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF\n# ANY KIND, either express or implied. See the License for the specific\n# language governing permissions and limitations under the License.\n\nfrom __future__ import unicode_literals\nfrom __future__ import print_function\n\nfrom datetime import datetime\nimport mock\nimport pytz\n\nfrom gitsome.lib.github3 import null\nfrom gitsome.lib.github3.exceptions import UnprocessableEntity\n\n\nclass MockEmail(object):\n\n def __init__(self, email, primary=False, verified=False):\n self.email = email\n self.primary = primary\n self.verified = verified\n\n\nclass MockUser(object):\n\n def __init__(self, login, user_type='User'):\n self.login = login\n self.repositories = {}\n self.emails = []\n self.avatar_url = 'https://www.github.com'\n self.company = 'bigcorp'\n self.location = 'interwebz'\n self.email = login + '@foo.com'\n self.type = user_type\n self.followers_count = 0\n self.following_count = 0\n self.feed_events = []\n\n def events(self, public):\n feed_events = []\n feed_events.append(MockEvent(\n event_type='CommitCommentEvent',\n payload={\n 'comment': MockRepoComment(),\n }))\n feed_events.append(MockEvent(\n event_type='CreateEvent',\n payload={\n 'ref_type': 'branch',\n 'ref': 'master',\n 'description': None,\n }))\n feed_events.append(MockEvent(\n event_type='FollowEvent'))\n feed_events.append(MockEvent(\n event_type='ForkEvent'))\n feed_events.append(MockEvent(\n event_type='IssueCommentEvent',\n payload={\n 'comment': MockIssueComment('foo'),\n 'issue': MockIssue('1', MockRepo(self, 'repo1'), 'foo'),\n }))\n feed_events.append(MockEvent(\n event_type='IssuesEvent',\n payload={\n 'action': 'closed',\n 'issue': MockIssue('1', MockRepo(self, 'repo1'), 'foo'),\n }))\n feed_events.append(MockEvent(\n event_type='PullRequestEvent',\n payload={\n 'action': 'closed',\n 'pull_request': MockIssue('1', MockRepo(self, 'repo1'), 'foo'),\n }))\n feed_events.append(MockEvent(\n event_type='PushEvent',\n payload={\n 'ref': 'refs/heads/master',\n 'commits': [{'url': 'https://api.github.com/repos/donnemartin/gitsome/commits/5ee4d1b20ee7cb16cd5be19b103301541a41003f', 'message': 'Fix GitHubCli class docstring', 'distinct': True, 'author': {'email': 'donne.martin@gmail.com', 'name': 'Donne Martin'}, 'sha': '5ee4d1b20ee7cb16cd5be19b103301541a41003f'}, {'url': 'https://api.github.com/repos/donnemartin/gitsome/commits/fc2309b645313646a3792eca9e0e9168cf25b267', 'message': 'Update gh configure docstring', 'distinct': True, 'author': {'email': 'donne.martin@gmail.com', 'name': 'Donne Martin'}, 'sha': 'fc2309b645313646a3792eca9e0e9168cf25b267'}, {'url': 'https://api.github.com/repos/donnemartin/gitsome/commits/dde19b7685ad7a07872fea1b4dc8019585322fdb', 'message': 'Update gh create-comment docstring', 'distinct': True, 'author': {'email': 'donne.martin@gmail.com', 'name': 'Donne Martin'}, 'sha': 'dde19b7685ad7a07872fea1b4dc8019585322fdb'}] # NOQA\n }))\n mock_release_tag = mock.Mock()\n mock_release_tag.tag_name = '0.5.0'\n feed_events.append(MockEvent(\n event_type='ReleaseEvent',\n payload={\n 'release': mock_release_tag,\n }))\n return feed_events\n\n def raise_mock_unprocessableentity(self):\n response = mock.Mock()\n response.json = lambda: exec('raise(Exception())')\n response.content = 'foobar'\n raise UnprocessableEntity(response)\n\n def create_repo(self, name, desc='', private=False):\n if name in self.repositories:\n self.raise_mock_unprocessableentity()\n repo = MockRepo(self, name, desc, private)\n self.repositories.update({repo.full_name: repo})\n return repo\n\n\nclass MockRepo(object):\n\n def __init__(self, user, full_name, description='', private=False):\n self.user = user\n self.full_name = full_name\n self.description = description\n self.private = private\n self.issues = {}\n self.clone_url = 'https://github.com/octocat/spoon-knife'\n self.stargazers_count = 1\n self.forks_count = 1\n self.language = ''\n self.updated_at = ''\n self.repository = 'foobar'\n\n def __lt__(self, other):\n return self.full_name < other.full_name\n\n def gen_key(self):\n return len(self.issues) + 1\n\n def create_issue(self, issue_title, issue_desc=''):\n number = self.gen_key()\n issue = MockIssue(number, self, issue_title, issue_desc)\n self.issues.update({number: issue})\n return issue\n\n def pull_requests(self):\n return list(self.issues.values())\n\n\nclass MockIssue(object):\n\n def __init__(self, number, repository, title, body=''):\n self.number = number\n self.repository = (repository.user.login, repository.full_name)\n self.title = title\n self.body = body\n self.state = 'open'\n self.comments_count = 1\n self.assignee = 'user1'\n self.user = 'user2'\n self.created_at = ''\n self.comments = []\n self.issue = 'foobar'\n\n def create_comment(self, body):\n issue_comment = MockIssueComment(body)\n self.comments.append(issue_comment)\n return issue_comment\n\n\nclass MockIssueComment(object):\n\n def __init__(self, body):\n self.body = body\n\n\nclass MockRepoComment(object):\n\n def __init__(self):\n self.commit_id = 'AAA23e2c6cb6997d25cfe61673aea6d701e9bZZZ'\n self.body = 'foo'\n\n\nclass MockLicense(object):\n\n def __init__(self, key, name):\n self.key = key\n self.name = name\n\n\nclass MockThread(object):\n\n def __init__(self, thread_type, title, unread):\n self.subject = {\n 'title': title,\n 'type': thread_type,\n 'url': 'https://api.github.com/repos/foo/bar/pulls/1',\n }\n self.unread = unread\n self.updated_at = ''\n\n\nclass MockEvent(object):\n\n def __init__(self, event_type, payload=''):\n self.id = 1\n self.created_at = datetime.now(pytz.utc)\n self.actor = 'donnemartin'\n self.org = 'org'\n self.type = event_type\n self.payload = payload\n self.repo = ('user1', 'repo1')\n self.public = True\n\n\nclass MockGitHubApi(object):\n\n def __init__(self):\n self.users = {}\n self.current_user = 'user1'\n self.ratelimit_remaining = 5000\n self._generate_mock_data()\n\n def _generate_mock_data(self):\n user1 = MockUser(self.current_user, 'User')\n user1_repo1 = user1.create_repo('repo1')\n user1_repo1.create_issue('title1', 'body1')\n user1_repo1.create_issue('title2', 'body2')\n user1_repo1.create_issue('title3', 'body3')\n user1.emails.extend([\n MockEmail('foo@baz.com', True, False),\n MockEmail('bar@baz.com', False, True),\n ])\n user2 = MockUser('user2', 'Organization')\n self.users.update({\n user1.login: user1,\n user2.login: user2,\n })\n\n def create_issue(self, user_login, repo_name, issue_title, issue_desc):\n try:\n user = self.users[user_login]\n repo = user.repositories[repo_name]\n issue = repo.create_issue(issue_title, issue_desc)\n return issue\n except KeyError:\n return null.NullObject('Issue')\n\n def create_repository(self, repo_name, repo_desc='', private=False):\n user = self.users[self.current_user]\n return user.create_repo(repo_name, repo_desc, private)\n\n def emails(self):\n user = self.users[self.current_user]\n return user.emails\n\n def emojis(self, pager=False):\n return [\n 'dolls',\n 'palm_tree',\n 'uk',\n '100',\n 'baby_chick',\n ]\n\n def followers_of(self, user_login):\n return [\n MockUser('foo1'),\n MockUser('foo2'),\n MockUser('foo3'),\n ]\n\n def followed_by(self, user_login):\n return self.followers_of(user_login)\n\n def gitignore_template(self, language):\n if language == 'valid_language':\n return 'template'\n else:\n return ''\n\n def gitignore_templates(self):\n return [\n 'Actionscript',\n 'Ada',\n 'Agda',\n 'Android',\n 'AppEngine',\n ]\n\n def issue(self, user_login, repo_name, number):\n try:\n user = self.users[user_login]\n repo = user.repositories[repo_name]\n return repo.issues[int(number)]\n except KeyError:\n return null.NullObject('Issue')\n\n def issues(self, issue_filter='subscribed', issue_state='open'):\n user = self.users[self.current_user]\n repo = user.repositories['repo1']\n issues_dict = repo.issues\n issues = list(issues_dict.values())\n return issues\n\n def license(self, license):\n if license == 'valid_license':\n template = mock.Mock()\n template.body = 'template'\n return template\n else:\n return null.NullObject('License')\n\n def licenses(self):\n return [\n MockLicense('mit', '(MIT License)'),\n MockLicense('gpl-2.0', '(GNU General Public License v2.0)'),\n MockLicense('bsd-2-clause', '(BSD 2-clause \"Simplified\" License)'),\n MockLicense('isc', '(ISC License)'),\n MockLicense('epl-1.0', '(Eclipse Public License 1.0)'),\n ]\n\n def notifications(self, all=True, participating=False):\n return [\n MockThread('type1', 'title1', True),\n MockThread('type2', 'title2', False),\n MockThread('type3', 'title3', True),\n ]\n\n def octocat(self, say):\n return say\n\n def pull_request(self, owner, repository, number):\n pull_requests = self.issues()\n return pull_requests[0]\n\n def search_issues(self, query):\n return self.issues()\n\n def search_repositories(self, query, sort):\n return self.repositories()\n\n def repositories(self, user_id=None):\n if user_id is None:\n user_id = self.current_user\n user = self.users[user_id]\n repos = list(user.repositories.values())\n repos_sorted = sorted(repos)\n return repos_sorted\n\n def user(self, user_id):\n try:\n return self.users[user_id]\n except KeyError:\n return null.NullObject('User')\n" }, { "path": "tests/mock_pretty_date_time.py", "content": "# -*- coding: utf-8 -*-\n\n# Copyright 2015 Donne Martin. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"). You\n# may not use this file except in compliance with the License. A copy of\n# the License is located at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# or in the \"license\" file accompanying this file. This file is\n# distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF\n# ANY KIND, either express or implied. See the License for the specific\n# language governing permissions and limitations under the License.\n\nfrom __future__ import unicode_literals\nfrom __future__ import print_function\n\n\ndef pretty_date_time(date_time):\n return \"just now\"\n" }, { "path": "tests/run_tests.py", "content": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n\n# Copyright 2015 Donne Martin. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"). You\n# may not use this file except in compliance with the License. A copy of\n# the License is located at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# or in the \"license\" file accompanying this file. This file is\n# distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF\n# ANY KIND, either express or implied. See the License for the specific\n# language governing permissions and limitations under the License.\n\nfrom __future__ import unicode_literals\nfrom __future__ import print_function\n\nfrom compat import unittest\n\nfrom test_completer import CompleterTest # NOQA\nfrom test_config import ConfigTest # NOQA\nfrom test_github import GitHubTest # NOQA\nfrom test_github_cli import GitHubCliTest # NOQA\nfrom test_web_viewer import WebViewerTest # NOQA\n\n\nif __name__ == '__main__':\n unittest.main()\n" }, { "path": "tests/test_completer.py", "content": "# -*- coding: utf-8 -*-\n\n# Copyright 2015 Donne Martin. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"). You\n# may not use this file except in compliance with the License. A copy of\n# the License is located at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# or in the \"license\" file accompanying this file. This file is\n# distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF\n# ANY KIND, either express or implied. See the License for the specific\n# language governing permissions and limitations under the License.\n\nfrom __future__ import unicode_literals\nfrom __future__ import print_function\n\nimport mock\nfrom tests.compat import unittest\n\nfrom prompt_toolkit.document import Document\n\nfrom gitsome.completer import CompleterGitsome\n\n\nclass CompleterTest(unittest.TestCase):\n\n def setUp(self):\n self.completer = CompleterGitsome()\n self.completer_event = self.create_completer_event()\n\n def create_completer_event(self):\n return mock.Mock()\n\n def _get_completions(self, command):\n position = len(command)\n result = self.completer.get_completions(\n Document(text=command, cursor_position=position),\n self.completer_event)\n return result\n\n def verify_completions(self, commands, expected):\n result = []\n for command in commands:\n # Call the AWS CLI autocompleter\n result.extend(self._get_completions(command))\n result_texts = []\n for item in result:\n # Each result item is a Completion object,\n # we are only interested in the text portion\n result_texts.append(item.text)\n assert result_texts\n if len(expected) == 1:\n assert expected[0] in result_texts\n else:\n for item in expected:\n assert item in result_texts\n\n def test_blank(self):\n text = ''\n expected = []\n result = self._get_completions(text)\n assert result == expected\n\n def test_no_completions(self):\n text = 'foo'\n expected = []\n result = self._get_completions(text)\n assert result == expected\n\n def test_command(self):\n text = ['g']\n expected = ['gh']\n self.verify_completions(text, expected)\n\n def test_subcommand(self):\n self.verify_completions(['gh c'], ['configure', 'create-comment',\n 'create-issue', 'create-repo'])\n self.verify_completions(['gh e'], ['emails', 'emojis'])\n self.verify_completions(['gh f'], ['feed', 'followers', 'following'])\n self.verify_completions(['gh g'], ['gitignore-template',\n 'gitignore-templates'])\n self.verify_completions(['gh i'], ['issue', 'issues'])\n self.verify_completions(['gh l'], ['license', 'licenses'])\n self.verify_completions(['gh m'], ['me'])\n self.verify_completions(['gh n'], ['notifications'])\n self.verify_completions(['gh o'], ['octo'])\n self.verify_completions(['gh p'], ['pull-request', 'pull-requests'])\n self.verify_completions(['gh r'], ['rate-limit', 'repo', 'repos'])\n self.verify_completions(['gh s'], ['search-issues', 'search-repos',\n 'starred'])\n self.verify_completions(['gh t'], ['trending'])\n self.verify_completions(['gh u'], ['user'])\n self.verify_completions(['gh v'], ['view'])\n\n def test_args(self):\n self.verify_completions(['gh octo '], ['\"Keep it logically awesome\"'])\n\n def test_no_args_with_options(self):\n self.verify_completions(['gh octo '], ['\"Keep it logically awesome\"'])\n\n def test_options(self):\n self.verify_completions(['gh emojis '],\n ['-p', '--pager'])\n\n def test_multiple_options(self):\n self.verify_completions(['gh feed -pr --pa'], ['--pager'])\n self.verify_completions(['gh feed foo -pr --pa'], ['--pager'])\n\n def test_fuzzy(self):\n self.completer.fuzzy_match = True\n self.verify_completions(['gh ot'], ['octo'])\n\n def test_build_completions_with_meta(self):\n result = self.completer.build_completions_with_meta('git ad',\n 'ad',\n ['add'])\n assert result[0].display_meta_text == 'Add file contents to the index.'\n result = self.completer.build_completions_with_meta('git-alia',\n 'git-alia',\n ['git-alias'])\n assert result[0].display_meta_text == 'Define, search and show aliases.'\n" }, { "path": "tests/test_config.py", "content": "# -*- coding: utf-8 -*-\n\n# Copyright 2015 Donne Martin. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"). You\n# may not use this file except in compliance with the License. A copy of\n# the License is located at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# or in the \"license\" file accompanying this file. This file is\n# distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF\n# ANY KIND, either express or implied. See the License for the specific\n# language governing permissions and limitations under the License.\n\nfrom __future__ import unicode_literals\nfrom __future__ import print_function\n\nimport mock\nimport os\nfrom gitsome.compat import configparser\nfrom tests.compat import unittest\n\nfrom gitsome.github import GitHub\nfrom tests.mock_github_api import MockGitHubApi\n\n\nclass ConfigTest(unittest.TestCase):\n\n def setUp(self):\n self.github = GitHub()\n self.github.config.api = MockGitHubApi()\n self.github.config.login = mock.Mock()\n self.github.config.authorize = mock.Mock()\n self.github.config.getpass = mock.Mock()\n\n def verify_login_token(self, username=None, password=None,\n token=None, url=None,\n two_factor_callback=None,\n verify=True):\n assert username is not None\n assert token is not None\n assert two_factor_callback is not None\n assert verify\n\n def verify_login_pass(self, username=None, password=None,\n token=None, url=None,\n two_factor_callback=None,\n verify=True):\n assert username is not None\n assert password is not None\n assert password\n assert two_factor_callback is not None\n assert verify\n\n def verify_login_token_url_enterprise(self, username=None, password=None,\n token=None, url=None,\n two_factor_callback=None,\n verify=True):\n assert username is not None\n assert token is not None\n assert url is not None\n assert two_factor_callback is not None\n assert verify\n\n def verify_login_token_url_no_verify_enterprise(self, username=None,\n password=None, token=None,\n url=None,\n two_factor_callback=None,\n verify=True):\n assert username is not None\n assert token is not None\n assert url is not None\n assert two_factor_callback is not None\n assert not verify\n\n def verify_login_pass_url_enterprise(self, username=None, password=None,\n token=None, url=None,\n two_factor_callback=None,\n verify=True):\n assert username is not None\n assert password is not None\n assert url is not None\n assert two_factor_callback is not None\n assert verify\n\n def test_config(self):\n expected = os.path.join(os.path.abspath(os.environ.get('HOME', '')),\n self.github.config.CONFIG)\n assert self.github.config \\\n .get_github_config_path(self.github.config.CONFIG) == expected\n\n def test_authenticate_cached_credentials_token(self):\n self.github.config.user_login = 'foo'\n self.github.config.user_token = 'bar'\n self.github.config.save_config()\n self.github.config.user_login = ''\n self.github.config.user_token = ''\n self.github.config.api = None\n config = self.github.config.get_github_config_path(\n self.github.config.CONFIG)\n parser = configparser.RawConfigParser()\n self.github.config.login = self.verify_login_token\n self.github.config.authenticate_cached_credentials(config, parser)\n assert self.github.config.user_login == 'foo'\n assert self.github.config.user_token == 'bar'\n\n def test_authenticate_cached_credentials_pass(self):\n self.github.config.user_login = 'foo'\n self.github.config.user_pass = 'bar'\n self.github.config.save_config()\n self.github.config.user_login = ''\n self.github.config.user_pass = ''\n self.github.config.api = None\n config = self.github.config.get_github_config_path(\n self.github.config.CONFIG)\n parser = configparser.RawConfigParser()\n self.github.config.authenticate_cached_credentials(config, parser)\n assert self.github.config.user_login == 'foo'\n assert self.github.config.user_pass is None\n\n def test_authenticate_cached_credentials_token_enterprise(self):\n self.github.config.user_login = 'foo'\n self.github.config.user_token = 'bar'\n self.github.config.enterprise_url = 'baz'\n self.github.config.save_config()\n self.github.config.user_login = ''\n self.github.config.user_token = ''\n self.github.config.enterprise_url = ''\n self.github.config.api = None\n config = self.github.config.get_github_config_path(\n self.github.config.CONFIG)\n parser = configparser.RawConfigParser()\n self.github.config.authenticate_cached_credentials(\n config,\n parser,\n enterprise_auth=self.verify_login_token_url_enterprise)\n assert self.github.config.user_login == 'foo'\n assert self.github.config.user_token == 'bar'\n assert self.github.config.enterprise_url == 'baz'\n\n def test_authenticate_cached_credentials_pass_enterprise(self):\n self.github.config.user_login = 'foo'\n self.github.config.user_pass = 'bar'\n self.github.config.enterprise_url = 'baz'\n self.github.config.save_config()\n self.github.config.user_login = ''\n self.github.config.user_pass = ''\n self.github.config.enterprise_url = ''\n self.github.config.api = None\n config = self.github.config.get_github_config_path(\n self.github.config.CONFIG)\n parser = configparser.RawConfigParser()\n self.github.config.authenticate_cached_credentials(\n config,\n parser,\n enterprise_auth=self.verify_login_pass_url_enterprise)\n assert self.github.config.user_login == 'foo'\n assert self.github.config.user_pass == 'bar'\n assert self.github.config.enterprise_url == 'baz'\n\n @mock.patch('gitsome.github.click.secho')\n @mock.patch('gitsome.config.Config.authenticate_cached_credentials')\n def test_authenticate_token(self, mock_auth, mock_click_secho):\n with mock.patch('click.confirm', return_value=False):\n with mock.patch('builtins.input', return_value='foo'):\n self.github.config.login = self.verify_login_token\n self.github.config.user_login = 'foo'\n self.github.config.user_token = 'bar'\n self.github.config.authenticate(\n enterprise=False,\n overwrite=True)\n\n @mock.patch('gitsome.github.click.secho')\n @mock.patch('gitsome.config.Config.authenticate_cached_credentials')\n def test_authenticate_pass(self, mock_auth, mock_click_secho):\n self.github.config.getpass.return_value = 'bar'\n with mock.patch('click.confirm', return_value=True):\n with mock.patch('builtins.input', return_value='foo'):\n self.github.config.login = self.verify_login_pass\n self.github.config.user_login = 'foo'\n self.github.config.authenticate(\n enterprise=False,\n overwrite=True)\n assert self.github.config.user_pass is None\n\n @mock.patch('gitsome.github.click.secho')\n @mock.patch('gitsome.config.Config.authenticate_cached_credentials')\n def test_authenticate_enterprise_token(self, mock_auth, mock_click_secho):\n with mock.patch('click.confirm', return_value=False):\n with mock.patch('builtins.input', return_value='foo'):\n self.github.config.user_login = 'foo'\n self.github.config.user_token = 'bar'\n enterprise_auth = \\\n self.verify_login_token_url_no_verify_enterprise\n self.github.config.authenticate(\n enterprise=True,\n enterprise_auth=enterprise_auth,\n overwrite=True)\n\n @mock.patch('gitsome.github.click.secho')\n @mock.patch('gitsome.config.Config.authenticate_cached_credentials')\n def test_authenticate_enterprise_pass(self, mock_auth, mock_click_secho):\n self.github.config.getpass.return_value = 'bar'\n with mock.patch('click.confirm', return_value=True):\n with mock.patch('builtins.input', return_value='foo'):\n self.github.config.user_login = 'foo'\n self.github.config.authenticate(\n enterprise=True,\n enterprise_auth=self.verify_login_pass_url_enterprise,\n overwrite=True)\n assert self.github.config.user_pass is not None\n\n @mock.patch('gitsome.github.click.secho')\n def test_check_auth_error(self, mock_click_secho):\n self.github.config.api = None\n self.github.config.check_auth()\n mock_click_secho.assert_any_call('Authentication error.', fg='red')\n mock_click_secho.assert_any_call('Update your credentials in ~/.gitsomeconfig or run:\\n gh configure', fg=None) # NOQA\n\n def test_load_urls(self):\n urls = self.github.config.load_urls(view_in_browser=False)\n assert urls == ['octocat/spoon-knife']\n\n def test_request_two_factor_code(self):\n with mock.patch('builtins.input', return_value='code'):\n assert self.github.config.request_two_factor_code() == 'code'\n\n @mock.patch('gitsome.github.click.secho')\n def test_prompt_news_feed(self, mock_click_secho):\n with mock.patch('click.confirm', return_value='y'):\n with mock.patch('builtins.input', return_value='feed'):\n self.github.config.prompt_news_feed()\n assert self.github.config.user_feed == 'feed'\n" }, { "path": "tests/test_github.py", "content": "# -*- coding: utf-8 -*-\n\n# Copyright 2015 Donne Martin. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"). You\n# may not use this file except in compliance with the License. A copy of\n# the License is located at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# or in the \"license\" file accompanying this file. This file is\n# distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF\n# ANY KIND, either express or implied. See the License for the specific\n# language governing permissions and limitations under the License.\n\nfrom __future__ import unicode_literals\nfrom __future__ import print_function\n\nimport mock\n\nfrom compat import unittest\n\nfrom gitsome.github import GitHub\nfrom tests.mock_feed_parser import MockFeedParser\nfrom tests.mock_github_api import MockGitHubApi\nfrom tests.mock_pretty_date_time import pretty_date_time\nfrom tests.data.email import formatted_emails\nfrom tests.data.emoji import formatted_emojis\nfrom tests.data.events import formatted_events\nfrom tests.data.user import formatted_org, formatted_user, formatted_users\nfrom tests.data.gitignores import formatted_gitignores, formatted_gitignores_tip\nfrom tests.data.issue import formatted_issues, formatted_pull_requests\nfrom tests.data.license import formatted_licenses, formatted_licenses_tip\nfrom tests.data.thread import formatted_threads\nfrom tests.data.trends import formatted_trends\nfrom tests.data.user_feed import formatted_user_feed\n\n\nclass GitHubTest(unittest.TestCase):\n\n def setUp(self):\n self.github = GitHub()\n self.github.config.api = MockGitHubApi()\n self.github.formatter.pretty_dt = pretty_date_time\n self.github.trend_parser = MockFeedParser()\n\n def test_avatar_no_pil(self):\n avatar_text = self.github.avatar(\n 'https://avatars.githubusercontent.com/u/583231?v=3', False)\n assert avatar_text == 'PIL not found.\\n'\n\n @mock.patch('gitsome.github.click.secho')\n def test_create_comment(self, mock_click_secho):\n self.github.create_comment('user1/repo1/1', 'text')\n mock_click_secho.assert_called_with(\n 'Created comment: text',\n fg=self.github.config.clr_message)\n\n @mock.patch('gitsome.github.click.secho')\n def test_create_comment_invalid_args(self, mock_click_secho):\n self.github.create_comment('invalid/repo1/1', 'text')\n mock_click_secho.assert_called_with(\n 'Error creating comment',\n fg=self.github.config.clr_error)\n self.github.create_comment('user1/repo1/foo', 'text')\n mock_click_secho.assert_called_with(\n 'Expected argument: user/repo/# and option -t \"comment\".',\n fg=self.github.config.clr_error)\n\n @mock.patch('gitsome.github.click.secho')\n def test_create_issue(self, mock_click_secho):\n self.github.create_issue('user1/repo1', 'title', 'desc')\n mock_click_secho.assert_called_with(\n 'Created issue: title\\ndesc',\n fg=self.github.config.clr_message)\n\n @mock.patch('gitsome.github.click.secho')\n def test_create_issue_no_desc(self, mock_click_secho):\n self.github.create_issue('user1/repo1', 'title', issue_desc=None)\n mock_click_secho.assert_called_with(\n 'Created issue: title\\n',\n fg=self.github.config.clr_message)\n\n @mock.patch('gitsome.github.click.secho')\n def test_create_issue_invalid_args(self, mock_click_secho):\n self.github.create_issue('invalid/repo1', 'title', 'desc')\n mock_click_secho.assert_called_with(\n 'Error creating issue.',\n fg=self.github.config.clr_error)\n self.github.create_issue('u', 'title', 'desc')\n mock_click_secho.assert_called_with(\n 'Expected argument: user/repo and option -t \"title\".',\n fg=self.github.config.clr_error)\n\n @mock.patch('gitsome.github.click.secho')\n def test_create_repo(self, mock_click_secho):\n self.github.create_repo('name', 'desc', True)\n mock_click_secho.assert_called_with(\n 'Created repo: name\\ndesc',\n fg=self.github.config.clr_message)\n\n @mock.patch('gitsome.github.click.secho')\n def test_create_repo_no_desc(self, mock_click_secho):\n self.github.create_repo('name', repo_desc=None)\n mock_click_secho.assert_called_with(\n 'Created repo: name\\n',\n fg=self.github.config.clr_message)\n\n @mock.patch('gitsome.github.click.secho')\n def test_create_repo_invalid_args(self, mock_click_secho):\n self.github.create_repo('repo1', 'desc', True)\n mock_click_secho.assert_called_with(\n 'Error creating repo: foobar',\n fg=self.github.config.clr_error)\n\n @mock.patch('gitsome.github.click.secho')\n def test_emails(self, mock_click_secho):\n self.github.emails()\n mock_click_secho.assert_called_with(formatted_emails)\n\n @mock.patch('gitsome.github.click.secho')\n @mock.patch('gitsome.config.Config.prompt_news_feed')\n def test_feed_config(self, mock_config_prompt_news_feed, mock_click_secho):\n self.github.feed()\n mock_config_prompt_news_feed.assert_called_with()\n\n @mock.patch('gitsome.github.click.secho')\n def test_feed(self, mock_click_secho):\n self.github.config.user_feed = 'user_feed'\n self.github.feed()\n mock_click_secho.assert_called_with(formatted_user_feed)\n\n @mock.patch('gitsome.github.click.secho')\n @mock.patch('gitsome.config.Config')\n def test_feed_user(self, mock_config, mock_click_secho):\n self.github.feed('user1')\n mock_click_secho.assert_called_with(formatted_events)\n\n @mock.patch('gitsome.github.click.secho')\n def test_emojis(self, mock_click_secho):\n self.github.emojis()\n mock_click_secho.assert_called_with(formatted_emojis)\n\n @mock.patch('gitsome.github.click.secho')\n def test_followers(self, mock_click_secho):\n self.github.followers('foo')\n mock_click_secho.assert_called_with(formatted_users)\n\n @mock.patch('gitsome.github.click.secho')\n def test_following(self, mock_click_secho):\n self.github.following('foo')\n mock_click_secho.assert_called_with(formatted_users)\n\n @mock.patch('gitsome.github.click.secho')\n def test_gitignore_template(self, mock_click_secho):\n self.github.gitignore_template('valid_language')\n mock_click_secho.assert_called_with(\n 'template',\n fg=self.github.config.clr_message)\n\n @mock.patch('gitsome.github.click.secho')\n def test_gitignore_template_invalid(self, mock_click_secho):\n self.github.gitignore_template('invalid_language')\n mock_click_secho.assert_called_with(\n ('Invalid case-sensitive template requested, run the '\n 'following command to see available templates:\\n'\n ' gh gitignore-templates'),\n fg=self.github.config.clr_error)\n\n @mock.patch('gitsome.github.click.secho')\n def test_gitignore_templates(self, mock_click_secho):\n self.github.gitignore_templates()\n mock_click_secho.assert_any_call(formatted_gitignores)\n mock_click_secho.assert_any_call(formatted_gitignores_tip,\n fg=self.github.config.clr_message)\n\n @mock.patch('gitsome.web_viewer.WebViewer.view_url')\n def test_issue(self, mock_view_url):\n self.github.issue('user1/repo1/1')\n mock_view_url.assert_called_with(\n 'https://github.com/user1/repo1/issues/1')\n\n @mock.patch('gitsome.github.click.secho')\n def test_issue_invalid_args(self, mock_click_secho):\n self.github.issue('user1/repo1/foo')\n mock_click_secho.assert_called_with(\n 'Expected argument: user/repo/#.',\n fg=self.github.config.clr_error)\n\n @mock.patch('gitsome.github.click.secho')\n def test_issues_setup(self, mock_click_secho):\n self.github.issues_setup()\n mock_click_secho.assert_called_with(formatted_issues)\n\n @mock.patch('gitsome.github.click.secho')\n def test_license(self, mock_click_secho):\n self.github.license('valid_license')\n mock_click_secho.assert_called_with(\n 'template',\n fg=self.github.config.clr_message)\n\n @mock.patch('gitsome.github.click.secho')\n def test_license_invalid(self, mock_click_secho):\n self.github.license('invalid_license')\n mock_click_secho.assert_called_with(\n (' Invalid case-sensitive license requested, run the '\n 'following command to see available licenses:\\n'\n ' gh licenses'),\n fg=self.github.config.clr_error)\n\n @mock.patch('gitsome.github.click.secho')\n def test_licenses(self, mock_click_secho):\n self.github.licenses()\n mock_click_secho.assert_any_call(formatted_licenses)\n mock_click_secho.assert_any_call(formatted_licenses_tip,\n fg=self.github.config.clr_message)\n\n @mock.patch('gitsome.github.click.secho')\n def test_notifications(self, mock_click_secho):\n self.github.notifications()\n mock_click_secho.assert_called_with(formatted_threads)\n\n @mock.patch('gitsome.github.click.secho')\n def test_octocat(self, mock_click_secho):\n self.github.octocat('foo\\\\nbar')\n mock_click_secho.assert_called_with(\n 'foo\\nbar',\n fg=self.github.config.clr_message)\n\n @mock.patch('gitsome.github.click.secho')\n def test_pull_requests(self, mock_click_secho):\n self.github.pull_requests()\n mock_click_secho.assert_called_with(formatted_pull_requests)\n\n @mock.patch('gitsome.github.click.secho')\n def test_rate_limit(self, mock_click_secho):\n self.github.rate_limit()\n mock_click_secho.assert_called_with(\n 'Rate limit: 5000',\n fg=self.github.config.clr_message)\n\n @mock.patch('gitsome.web_viewer.WebViewer.view_url')\n def test_repository(self, mock_view_url):\n self.github.repository('user1/repo1')\n mock_view_url.assert_called_with(\n 'https://github.com/user1/repo1')\n\n @mock.patch('gitsome.github.click.secho')\n def test_repository_invalid(self, mock_click_secho):\n self.github.repository('user1/repo1/1')\n mock_click_secho.assert_called_with(\n 'Expected argument: user/repo.',\n fg=self.github.config.clr_error)\n\n @mock.patch('gitsome.github.click.secho')\n @mock.patch('gitsome.github.GitHub.issues')\n def test_search_issues(self, mock_github_issues, mock_click_secho):\n self.github.search_issues('foo')\n mock_github_issues.assert_called_with(\n ['foobar', 'foobar', 'foobar'], 1000, False, sort=False)\n\n @mock.patch('gitsome.github.click.secho')\n @mock.patch('gitsome.github.GitHub.repositories')\n def test_search_repos(self, mock_github_repositories, mock_click_secho):\n self.github.search_repositories('foo', 'stars')\n mock_github_repositories.assert_called_with(\n ['foobar'], 1000, False, sort=False)\n\n @mock.patch('gitsome.github.click.secho')\n def test_trending(self, mock_click_secho):\n self.github.trending('Python', False, False, False)\n mock_click_secho.assert_called_with(formatted_trends)\n\n @mock.patch('gitsome.github.click.secho')\n def test_user(self, mock_click_secho):\n self.github.user('user1')\n mock_click_secho.assert_called_with(formatted_user)\n self.github.user('user2')\n mock_click_secho.assert_called_with(formatted_org)\n\n @mock.patch('gitsome.github.click.secho')\n def test_user_invalid(self, mock_click_secho):\n self.github.user('invalid_user')\n mock_click_secho.assert_called_with(\n 'Invalid user.',\n fg=self.github.config.clr_error)\n\n @mock.patch('gitsome.github.click.secho')\n @mock.patch('gitsome.github.webbrowser.open')\n def test_user_browser(self, mock_webbrowser_open, mock_click_secho):\n self.github.user('invalid_user', browser=True)\n mock_webbrowser_open.assert_called_with(\n 'https://github.com/invalid_user')\n\n @mock.patch('gitsome.github.click.secho')\n @mock.patch('gitsome.github.webbrowser.open')\n def test_view_browser(self, mock_webbrowser_open, mock_click_secho):\n self.github.config.load_urls = lambda x: ['user1/foo']\n self.github.view(1, view_in_browser=True)\n mock_webbrowser_open.assert_called_with(\n 'https://github.com/user1/foo')\n\n @mock.patch('gitsome.github.click.secho')\n @mock.patch('gitsome.github.GitHub.issue')\n def test_view_issue(self, mock_github_issue, mock_click_secho):\n self.github.config.load_urls = lambda x: ['user1/foo/issues/1']\n self.github.view(0)\n mock_github_issue.assert_called_with('user1/foo/1')\n\n @mock.patch('gitsome.github.click.secho')\n @mock.patch('gitsome.github.GitHub.repository')\n def test_view_repo(self, mock_github_repository, mock_click_secho):\n self.github.config.load_urls = lambda x: ['user1/foo']\n self.github.view(0)\n mock_github_repository.assert_called_with('user1/foo')\n\n @mock.patch('gitsome.github.click.secho')\n @mock.patch('gitsome.web_viewer.WebViewer.view_url')\n def test_view_user(self, mock_view_url, mock_click_secho):\n self.github.config.load_urls = lambda x: ['user1']\n self.github.view(0)\n mock_view_url.assert_called_with('https://github.com/user1')\n\n def test_base_url(self):\n self.github.config.enterprise_url = 'https://github.intra.example.com'\n assert self.github.base_url == 'https://github.intra.example.com'\n self.github.config.enterprise_url = None\n assert self.github.base_url == self.github._base_url\n\n def test_add_base_url(self):\n expected = self.github.base_url + 'foo.html'\n assert self.github.add_base_url('foo.html') == expected\n assert self.github.add_base_url(expected) == expected\n" }, { "path": "tests/test_github_cli.py", "content": "# -*- coding: utf-8 -*-\n\n# Copyright 2015 Donne Martin. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"). You\n# may not use this file except in compliance with the License. A copy of\n# the License is located at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# or in the \"license\" file accompanying this file. This file is\n# distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF\n# ANY KIND, either express or implied. See the License for the specific\n# language governing permissions and limitations under the License.\n\nfrom __future__ import unicode_literals\nfrom __future__ import print_function\n\nimport mock\nfrom compat import unittest\n\nfrom click.testing import CliRunner\n\nfrom gitsome.githubcli import GitHubCli\n\n\nclass GitHubCliTest(unittest.TestCase):\n\n def setUp(self):\n self.runner = CliRunner()\n self.github_cli = GitHubCli()\n self.limit = 1000\n\n def test_cli(self):\n result = self.runner.invoke(self.github_cli.cli)\n assert result.exit_code == 0\n\n @mock.patch('gitsome.githubcli.GitHub.configure')\n def test_configure(self, mock_gh_call):\n result = self.runner.invoke(self.github_cli.cli, ['configure'])\n mock_gh_call.assert_called_with(False)\n assert result.exit_code == 0\n\n @mock.patch('gitsome.githubcli.GitHub.create_comment')\n def test_create_comment(self, mock_gh_call):\n result = self.runner.invoke(self.github_cli.cli,\n ['create-comment', 'u/r/n',\n '--text', 'foo'])\n mock_gh_call.assert_called_with('u/r/n', 'foo')\n assert result.exit_code == 0\n\n @mock.patch('gitsome.githubcli.GitHub.create_issue')\n def test_create_issue(self, mock_gh_call):\n result = self.runner.invoke(self.github_cli.cli,\n ['create-issue', 'u/r',\n '--issue_title', 'foo',\n '--issue_desc', 'bar'])\n mock_gh_call.assert_called_with('u/r', 'foo', 'bar')\n assert result.exit_code == 0\n\n @mock.patch('gitsome.githubcli.GitHub.create_repo')\n def test_create_repo(self, mock_gh_call):\n result = self.runner.invoke(self.github_cli.cli,\n ['create-repo', 'r',\n '--repo_desc', 'foo',\n '--private'])\n mock_gh_call.assert_called_with('r', 'foo', True)\n assert result.exit_code == 0\n\n @mock.patch('gitsome.githubcli.GitHub.emails')\n def test_emails(self, mock_gh_call):\n result = self.runner.invoke(self.github_cli.cli,\n ['emails'])\n mock_gh_call.assert_called_with()\n assert result.exit_code == 0\n\n @mock.patch('gitsome.githubcli.GitHub.emojis')\n def test_emojis(self, mock_gh_call):\n result = self.runner.invoke(self.github_cli.cli,\n ['emojis',\n '--pager'])\n mock_gh_call.assert_called_with(True)\n assert result.exit_code == 0\n\n @mock.patch('gitsome.githubcli.GitHub.feed')\n def test_feed(self, mock_gh_call):\n result = self.runner.invoke(self.github_cli.cli,\n ['feed', 'u',\n '--private', '--pager'])\n mock_gh_call.assert_called_with('u', True, True)\n assert result.exit_code == 0\n\n @mock.patch('gitsome.githubcli.GitHub.followers')\n def test_followers(self, mock_gh_call):\n result = self.runner.invoke(self.github_cli.cli,\n ['followers', 'u',\n '--pager'])\n mock_gh_call.assert_called_with('u', True)\n assert result.exit_code == 0\n\n @mock.patch('gitsome.githubcli.GitHub.following')\n def test_following(self, mock_gh_call):\n result = self.runner.invoke(self.github_cli.cli,\n ['following', 'u',\n '--pager'])\n mock_gh_call.assert_called_with('u', True)\n assert result.exit_code == 0\n\n @mock.patch('gitsome.githubcli.GitHub.gitignore_template')\n def test_gitignore_template(self, mock_gh_call):\n result = self.runner.invoke(self.github_cli.cli,\n ['gitignore-template', 'l'])\n mock_gh_call.assert_called_with('l')\n assert result.exit_code == 0\n\n @mock.patch('gitsome.githubcli.GitHub.gitignore_templates')\n def test_gitignore_templates(self, mock_gh_call):\n result = self.runner.invoke(self.github_cli.cli,\n ['gitignore-templates',\n '--pager'])\n mock_gh_call.assert_called_with(True)\n assert result.exit_code == 0\n\n @mock.patch('gitsome.githubcli.GitHub.issue')\n def test_issue(self, mock_gh_call):\n result = self.runner.invoke(self.github_cli.cli,\n ['issue', 'u/r/n'])\n mock_gh_call.assert_called_with('u/r/n')\n assert result.exit_code == 0\n\n @mock.patch('gitsome.githubcli.GitHub.issues_setup')\n def test_issues(self, mock_gh_call):\n result = self.runner.invoke(self.github_cli.cli,\n ['issues',\n '--issue_filter', 'mentioned',\n '--issue_state', 'closed',\n '--limit', '10',\n '--pager'])\n mock_gh_call.assert_called_with('mentioned', 'closed', 10, True)\n assert result.exit_code == 0\n\n @mock.patch('gitsome.githubcli.GitHub.license')\n def test_license(self, mock_gh_call):\n result = self.runner.invoke(self.github_cli.cli,\n ['license', 'l'])\n mock_gh_call.assert_called_with('l')\n assert result.exit_code == 0\n\n @mock.patch('gitsome.githubcli.GitHub.licenses')\n def test_licenses(self, mock_gh_call):\n result = self.runner.invoke(self.github_cli.cli,\n ['licenses'])\n mock_gh_call.assert_called_with()\n assert result.exit_code == 0\n\n @mock.patch('gitsome.githubcli.GitHub.user_me')\n def test_me(self, mock_gh_call):\n result = self.runner.invoke(self.github_cli.cli,\n ['me',\n '--browser', '--text_avatar',\n '--limit', 10,\n '--pager'])\n mock_gh_call.assert_called_with(True, True, 10, True)\n assert result.exit_code == 0\n\n @mock.patch('gitsome.githubcli.GitHub.notifications')\n def test_notifications(self, mock_gh_call):\n result = self.runner.invoke(self.github_cli.cli,\n ['notifications',\n '--limit', 10,\n '--pager'])\n mock_gh_call.assert_called_with(10, True)\n assert result.exit_code == 0\n\n @mock.patch('gitsome.githubcli.GitHub.octocat')\n def test_octocat(self, mock_gh_call):\n result = self.runner.invoke(self.github_cli.cli,\n ['octo', 'foo'])\n mock_gh_call.assert_called_with('foo')\n assert result.exit_code == 0\n\n @mock.patch('gitsome.githubcli.GitHub.issue')\n def test_pull_request(self, mock_gh_call):\n result = self.runner.invoke(self.github_cli.cli,\n ['pull-request', 'u/r/n'])\n mock_gh_call.assert_called_with('u/r/n')\n assert result.exit_code == 0\n\n @mock.patch('gitsome.githubcli.GitHub.pull_requests')\n def test_pull_requests(self, mock_gh_call):\n result = self.runner.invoke(self.github_cli.cli,\n ['pull-requests',\n '--limit', 10,\n '--pager'])\n mock_gh_call.assert_called_with(10, True)\n assert result.exit_code == 0\n\n @mock.patch('gitsome.githubcli.GitHub.rate_limit')\n def test_rate_limit(self, mock_gh_call):\n result = self.runner.invoke(self.github_cli.cli,\n ['rate-limit'])\n mock_gh_call.assert_called_with()\n assert result.exit_code == 0\n\n @mock.patch('gitsome.githubcli.GitHub.repository')\n def test_repository(self, mock_gh_call):\n result = self.runner.invoke(self.github_cli.cli,\n ['repo', 'u/r'])\n mock_gh_call.assert_called_with('u/r')\n assert result.exit_code == 0\n\n @mock.patch('gitsome.githubcli.GitHub.repositories_setup')\n def test_repositories(self, mock_gh_call):\n result = self.runner.invoke(self.github_cli.cli,\n ['repos', 'foo',\n '--limit', 10,\n '--pager'])\n mock_gh_call.assert_called_with('foo', 10, True)\n assert result.exit_code == 0\n\n @mock.patch('gitsome.githubcli.GitHub.search_issues')\n def test_search_issues(self, mock_gh_call):\n result = self.runner.invoke(self.github_cli.cli,\n ['search-issues', 'foo',\n '--limit', 10,\n '--pager'])\n mock_gh_call.assert_called_with('foo', 10, True)\n assert result.exit_code == 0\n\n @mock.patch('gitsome.githubcli.GitHub.search_repositories')\n def test_search_repositories(self, mock_gh_call):\n result = self.runner.invoke(self.github_cli.cli,\n ['search-repos', 'foo',\n '--sort', 'stars',\n '--limit', 10,\n '--pager'])\n mock_gh_call.assert_called_with('foo', 'stars', 10, True)\n assert result.exit_code == 0\n\n @mock.patch('gitsome.githubcli.GitHub.starred')\n def test_starred(self, mock_gh_call):\n result = self.runner.invoke(self.github_cli.cli,\n ['starred', 'foo',\n '--limit', 10,\n '--pager'])\n mock_gh_call.assert_called_with('foo', 10, True)\n assert result.exit_code == 0\n\n @mock.patch('gitsome.githubcli.GitHub.trending')\n def test_trending(self, mock_gh_call):\n result = self.runner.invoke(self.github_cli.cli,\n ['trending', 'l',\n '--pager'])\n mock_gh_call.assert_called_with('l', False, False, False, False, True)\n result = self.runner.invoke(self.github_cli.cli,\n ['trending', 'l',\n '--weekly',\n '--pager'])\n mock_gh_call.assert_called_with('l', True, False, False, False, True)\n result = self.runner.invoke(self.github_cli.cli,\n ['trending', 'l',\n '--monthly',\n '--devs',\n '--browser',\n '--pager'])\n mock_gh_call.assert_called_with('l', False, True, True, True, True)\n assert result.exit_code == 0\n\n @mock.patch('gitsome.githubcli.GitHub.user')\n def test_user(self, mock_gh_call):\n result = self.runner.invoke(self.github_cli.cli,\n ['user', 'foo',\n '--browser', '--text_avatar',\n '--limit', 10,\n '--pager'])\n mock_gh_call.assert_called_with('foo', True, True, 10, True)\n assert result.exit_code == 0\n\n @mock.patch('gitsome.githubcli.GitHub.view')\n def test_view(self, mock_gh_call):\n result = self.runner.invoke(self.github_cli.cli,\n ['view', '1',\n '--browser'])\n mock_gh_call.assert_called_with(1, True)\n assert result.exit_code == 0\n" }, { "path": "tests/test_web_viewer.py", "content": "# -*- coding: utf-8 -*-\n\n# Copyright 2015 Donne Martin. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"). You\n# may not use this file except in compliance with the License. A copy of\n# the License is located at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# or in the \"license\" file accompanying this file. This file is\n# distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF\n# ANY KIND, either express or implied. See the License for the specific\n# language governing permissions and limitations under the License.\n\nfrom __future__ import unicode_literals\nfrom __future__ import print_function\n\nimport mock\nfrom tests.compat import unittest\n\nfrom gitsome.github import GitHub\nfrom tests.data.markdown import formatted_markdown, raw_markdown, ssl_error\n\n\nclass WebViewerTest(unittest.TestCase):\n\n def setUp(self):\n self.github = GitHub()\n\n def test_format_markdown(self):\n result = self.github.web_viewer.format_markdown(raw_markdown)\n assert result == formatted_markdown\n\n @mock.patch('gitsome.github.click.echo_via_pager')\n def test_view_url(self, mock_click_echo_via_pager):\n url = 'https://www.github.com/donnemartin/gitsome'\n self.github.web_viewer.view_url(url)\n assert mock_click_echo_via_pager.mock_calls\n\n @unittest.skip('Skipping test_view_url_ssl_error')\n @mock.patch('gitsome.github.click.echo_via_pager')\n def test_view_url_ssl_error(self, mock_click_echo_via_pager):\n \"\"\"Temp skipping this test due to a change [undocumented?] in the way\n the requests ssl error sample website is handled:\n http://docs.python-requests.org/en/master/user/advanced/#ssl-cert-verification # NOQA\n See https://github.com/donnemartin/gitsome/pull/64 for more details.\n \"\"\"\n url = 'https://requestb.in'\n self.github.web_viewer.view_url(url)\n mock_click_echo_via_pager.assert_called_with(ssl_error, None)\n" }, { "path": "tox.ini", "content": "# Tox (http://tox.testrun.org/) is a tool for running tests\n# in multiple virtualenvs. This configuration file will run the\n# test suite on all supported python versions. To use it, \"pip install tox\"\n# and then run \"tox\" from this directory.\n\n[tox]\nenvlist = py35, py36, py37\n\n[testenv]\npassenv = TRAVIS TRAVIS_JOB_ID TRAVIS_BRANCH\ndeps =\n coverage\n mock\n unittest2\ncommands =\n coverage erase\n coverage run {toxinidir}/tests/run_tests.py\n coverage report --include={toxinidir}/*gitsome/* --omit={toxinidir}/*gitsome/lib/*,*compat.py -m\n" }, { "path": "xonsh/__init__.py", "content": "__version__ = \"0.8.10\"\n\n\n# amalgamate exclude jupyter_kernel parser_table parser_test_table pyghooks\n# amalgamate exclude winutils wizard pytest_plugin fs macutils pygments_cache\n# amalgamate exclude jupyter_shell\nimport os as _os\n\nif _os.getenv(\"XONSH_DEBUG\", \"\"):\n pass\nelse:\n import sys as _sys\n\n try:\n from xonsh import __amalgam__\n\n completer = __amalgam__\n _sys.modules[\"xonsh.completer\"] = __amalgam__\n contexts = __amalgam__\n _sys.modules[\"xonsh.contexts\"] = __amalgam__\n lazyasd = __amalgam__\n _sys.modules[\"xonsh.lazyasd\"] = __amalgam__\n lazyjson = __amalgam__\n _sys.modules[\"xonsh.lazyjson\"] = __amalgam__\n color_tools = __amalgam__\n _sys.modules[\"xonsh.color_tools\"] = __amalgam__\n platform = __amalgam__\n _sys.modules[\"xonsh.platform\"] = __amalgam__\n pretty = __amalgam__\n _sys.modules[\"xonsh.pretty\"] = __amalgam__\n codecache = __amalgam__\n _sys.modules[\"xonsh.codecache\"] = __amalgam__\n lazyimps = __amalgam__\n _sys.modules[\"xonsh.lazyimps\"] = __amalgam__\n parser = __amalgam__\n _sys.modules[\"xonsh.parser\"] = __amalgam__\n tokenize = __amalgam__\n _sys.modules[\"xonsh.tokenize\"] = __amalgam__\n tools = __amalgam__\n _sys.modules[\"xonsh.tools\"] = __amalgam__\n ansi_colors = __amalgam__\n _sys.modules[\"xonsh.ansi_colors\"] = __amalgam__\n ast = __amalgam__\n _sys.modules[\"xonsh.ast\"] = __amalgam__\n commands_cache = __amalgam__\n _sys.modules[\"xonsh.commands_cache\"] = __amalgam__\n diff_history = __amalgam__\n _sys.modules[\"xonsh.diff_history\"] = __amalgam__\n events = __amalgam__\n _sys.modules[\"xonsh.events\"] = __amalgam__\n foreign_shells = __amalgam__\n _sys.modules[\"xonsh.foreign_shells\"] = __amalgam__\n jobs = __amalgam__\n _sys.modules[\"xonsh.jobs\"] = __amalgam__\n jsonutils = __amalgam__\n _sys.modules[\"xonsh.jsonutils\"] = __amalgam__\n lexer = __amalgam__\n _sys.modules[\"xonsh.lexer\"] = __amalgam__\n openpy = __amalgam__\n _sys.modules[\"xonsh.openpy\"] = __amalgam__\n style_tools = __amalgam__\n _sys.modules[\"xonsh.style_tools\"] = __amalgam__\n xontribs = __amalgam__\n _sys.modules[\"xonsh.xontribs\"] = __amalgam__\n dirstack = __amalgam__\n _sys.modules[\"xonsh.dirstack\"] = __amalgam__\n inspectors = __amalgam__\n _sys.modules[\"xonsh.inspectors\"] = __amalgam__\n proc = __amalgam__\n _sys.modules[\"xonsh.proc\"] = __amalgam__\n shell = __amalgam__\n _sys.modules[\"xonsh.shell\"] = __amalgam__\n timings = __amalgam__\n _sys.modules[\"xonsh.timings\"] = __amalgam__\n xonfig = __amalgam__\n _sys.modules[\"xonsh.xonfig\"] = __amalgam__\n base_shell = __amalgam__\n _sys.modules[\"xonsh.base_shell\"] = __amalgam__\n environ = __amalgam__\n _sys.modules[\"xonsh.environ\"] = __amalgam__\n tracer = __amalgam__\n _sys.modules[\"xonsh.tracer\"] = __amalgam__\n readline_shell = __amalgam__\n _sys.modules[\"xonsh.readline_shell\"] = __amalgam__\n replay = __amalgam__\n _sys.modules[\"xonsh.replay\"] = __amalgam__\n aliases = __amalgam__\n _sys.modules[\"xonsh.aliases\"] = __amalgam__\n dumb_shell = __amalgam__\n _sys.modules[\"xonsh.dumb_shell\"] = __amalgam__\n built_ins = __amalgam__\n _sys.modules[\"xonsh.built_ins\"] = __amalgam__\n execer = __amalgam__\n _sys.modules[\"xonsh.execer\"] = __amalgam__\n imphooks = __amalgam__\n _sys.modules[\"xonsh.imphooks\"] = __amalgam__\n main = __amalgam__\n _sys.modules[\"xonsh.main\"] = __amalgam__\n del __amalgam__\n except ImportError:\n pass\n del _sys\ndel _os\n# amalgamate end\n" }, { "path": "xonsh/__main__.py", "content": "from xonsh.main import main\n\nmain()\n" }, { "path": "xonsh/aliases.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"Aliases for the xonsh shell.\"\"\"\nimport os\nimport re\nimport sys\nimport inspect\nimport argparse\nimport builtins\nimport collections.abc as cabc\n\nfrom xonsh.lazyasd import lazyobject\nfrom xonsh.dirstack import cd, pushd, popd, dirs, _get_cwd\nfrom xonsh.environ import locate_binary, make_args_env\nfrom xonsh.foreign_shells import foreign_shell_data\nfrom xonsh.jobs import jobs, fg, bg, clean_jobs\nfrom xonsh.platform import (\n ON_ANACONDA,\n ON_DARWIN,\n ON_WINDOWS,\n ON_FREEBSD,\n ON_NETBSD,\n ON_DRAGONFLY,\n)\nfrom xonsh.tools import (\n XonshError,\n argvquote,\n escape_windows_cmd_string,\n to_bool,\n swap_values,\n strip_simple_quotes,\n ALIAS_KWARG_NAMES,\n unthreadable,\n print_color,\n)\nfrom xonsh.replay import replay_main\nfrom xonsh.timings import timeit_alias\nfrom xonsh.xontribs import xontribs_main\nfrom xonsh.ast import isexpression\n\nimport xonsh.completers._aliases as xca\nimport xonsh.history.main as xhm\nimport xonsh.xoreutils.which as xxw\n\n\n@lazyobject\ndef SUB_EXEC_ALIAS_RE():\n return re.compile(r\"@\\(|\\$\\(|!\\(|\\$\\[|!\\[\")\n\n\nclass Aliases(cabc.MutableMapping):\n \"\"\"Represents a location to hold and look up aliases.\"\"\"\n\n def __init__(self, *args, **kwargs):\n self._raw = {}\n self.update(*args, **kwargs)\n\n def get(self, key, default=None):\n \"\"\"Returns the (possibly modified) value. If the key is not present,\n then `default` is returned.\n If the value is callable, it is returned without modification. If it\n is an iterable of strings it will be evaluated recursively to expand\n other aliases, resulting in a new list or a \"partially applied\"\n callable.\n \"\"\"\n val = self._raw.get(key)\n if val is None:\n return default\n elif isinstance(val, cabc.Iterable) or callable(val):\n return self.eval_alias(val, seen_tokens={key})\n else:\n msg = \"alias of {!r} has an inappropriate type: {!r}\"\n raise TypeError(msg.format(key, val))\n\n def eval_alias(self, value, seen_tokens=frozenset(), acc_args=()):\n \"\"\"\n \"Evaluates\" the alias ``value``, by recursively looking up the leftmost\n token and \"expanding\" if it's also an alias.\n\n A value like ``[\"cmd\", \"arg\"]`` might transform like this:\n ``> [\"cmd\", \"arg\"] -> [\"ls\", \"-al\", \"arg\"] -> callable()``\n where ``cmd=ls -al`` and ``ls`` is an alias with its value being a\n callable. The resulting callable will be \"partially applied\" with\n ``[\"-al\", \"arg\"]``.\n \"\"\"\n # Beware of mutability: default values for keyword args are evaluated\n # only once.\n if callable(value):\n return partial_eval_alias(value, acc_args=acc_args)\n else:\n expand_path = builtins.__xonsh__.expand_path\n token, *rest = map(expand_path, value)\n if token in seen_tokens or token not in self._raw:\n # ^ Making sure things like `egrep=egrep --color=auto` works,\n # and that `l` evals to `ls --color=auto -CF` if `l=ls -CF`\n # and `ls=ls --color=auto`\n rtn = [token]\n rtn.extend(rest)\n rtn.extend(acc_args)\n return rtn\n else:\n seen_tokens = seen_tokens | {token}\n acc_args = rest + list(acc_args)\n return self.eval_alias(self._raw[token], seen_tokens, acc_args)\n\n def expand_alias(self, line):\n \"\"\"Expands any aliases present in line if alias does not point to a\n builtin function and if alias is only a single command.\n \"\"\"\n word = line.split(\" \", 1)[0]\n if word in builtins.aliases and isinstance(self.get(word), cabc.Sequence):\n word_idx = line.find(word)\n expansion = \" \".join(self.get(word))\n line = line[:word_idx] + expansion + line[word_idx + len(word) :]\n return line\n\n #\n # Mutable mapping interface\n #\n\n def __getitem__(self, key):\n return self._raw[key]\n\n def __setitem__(self, key, val):\n if isinstance(val, str):\n f = \"\"\n if SUB_EXEC_ALIAS_RE.search(val) is not None:\n # We have a sub-command, e.g. $(cmd), to evaluate\n self._raw[key] = ExecAlias(val, filename=f)\n elif isexpression(val):\n # expansion substitution\n lexer = builtins.__xonsh__.execer.parser.lexer\n self._raw[key] = list(map(strip_simple_quotes, lexer.split(val)))\n else:\n # need to exec alias\n self._raw[key] = ExecAlias(val, filename=f)\n else:\n self._raw[key] = val\n\n def __delitem__(self, key):\n del self._raw[key]\n\n def update(self, *args, **kwargs):\n for key, val in dict(*args, **kwargs).items():\n self[key] = val\n\n def __iter__(self):\n yield from self._raw\n\n def __len__(self):\n return len(self._raw)\n\n def __str__(self):\n return str(self._raw)\n\n def __repr__(self):\n return \"{0}.{1}({2})\".format(\n self.__class__.__module__, self.__class__.__name__, self._raw\n )\n\n def _repr_pretty_(self, p, cycle):\n name = \"{0}.{1}\".format(self.__class__.__module__, self.__class__.__name__)\n with p.group(0, name + \"(\", \")\"):\n if cycle:\n p.text(\"...\")\n elif len(self):\n p.break_()\n p.pretty(dict(self))\n\n\nclass ExecAlias:\n \"\"\"Provides a callable alias for xonsh source code.\"\"\"\n\n def __init__(self, src, filename=\"\"):\n \"\"\"\n Parameters\n ----------\n src : str\n Source code that will be\n \"\"\"\n self.src = src if src.endswith(\"\\n\") else src + \"\\n\"\n self.filename = filename\n\n def __call__(\n self, args, stdin=None, stdout=None, stderr=None, spec=None, stack=None\n ):\n execer = builtins.__xonsh__.execer\n frame = stack[0][0] # execute as though we are at the call site\n execer.exec(\n self.src, glbs=frame.f_globals, locs=frame.f_locals, filename=self.filename\n )\n\n def __repr__(self):\n return \"ExecAlias({0!r}, filename={1!r})\".format(self.src, self.filename)\n\n\nclass PartialEvalAliasBase:\n \"\"\"Partially evaluated alias.\"\"\"\n\n def __init__(self, f, acc_args=()):\n \"\"\"\n Parameters\n ----------\n f : callable\n A function to dispatch to.\n acc_args : sequence of strings, optional\n Additional arguments to prepent to the argument list passed in\n when the alias is called.\n \"\"\"\n self.f = f\n self.acc_args = acc_args\n self.__name__ = getattr(f, \"__name__\", self.__class__.__name__)\n\n def __call__(\n self, args, stdin=None, stdout=None, stderr=None, spec=None, stack=None\n ):\n args = list(self.acc_args) + args\n return self.f(args, stdin, stdout, stderr, spec, stack)\n\n def __repr__(self):\n return \"{name}({f!r}, acc_args={acc_args!r})\".format(\n name=self.__class__.__name__, f=self.f, acc_args=self.acc_args\n )\n\n\nclass PartialEvalAlias0(PartialEvalAliasBase):\n def __call__(\n self, args, stdin=None, stdout=None, stderr=None, spec=None, stack=None\n ):\n args = list(self.acc_args) + args\n if args:\n msg = \"callable alias {f!r} takes no arguments, but {args!f} provided. \"\n msg += \"Of these {acc_args!r} were partially applied.\"\n raise XonshError(msg.format(f=self.f, args=args, acc_args=self.acc_args))\n return self.f()\n\n\nclass PartialEvalAlias1(PartialEvalAliasBase):\n def __call__(\n self, args, stdin=None, stdout=None, stderr=None, spec=None, stack=None\n ):\n args = list(self.acc_args) + args\n return self.f(args)\n\n\nclass PartialEvalAlias2(PartialEvalAliasBase):\n def __call__(\n self, args, stdin=None, stdout=None, stderr=None, spec=None, stack=None\n ):\n args = list(self.acc_args) + args\n return self.f(args, stdin)\n\n\nclass PartialEvalAlias3(PartialEvalAliasBase):\n def __call__(\n self, args, stdin=None, stdout=None, stderr=None, spec=None, stack=None\n ):\n args = list(self.acc_args) + args\n return self.f(args, stdin, stdout)\n\n\nclass PartialEvalAlias4(PartialEvalAliasBase):\n def __call__(\n self, args, stdin=None, stdout=None, stderr=None, spec=None, stack=None\n ):\n args = list(self.acc_args) + args\n return self.f(args, stdin, stdout, stderr)\n\n\nclass PartialEvalAlias5(PartialEvalAliasBase):\n def __call__(\n self, args, stdin=None, stdout=None, stderr=None, spec=None, stack=None\n ):\n args = list(self.acc_args) + args\n return self.f(args, stdin, stdout, stderr, spec)\n\n\nclass PartialEvalAlias6(PartialEvalAliasBase):\n def __call__(\n self, args, stdin=None, stdout=None, stderr=None, spec=None, stack=None\n ):\n args = list(self.acc_args) + args\n return self.f(args, stdin, stdout, stderr, spec, stack)\n\n\nPARTIAL_EVAL_ALIASES = (\n PartialEvalAlias0,\n PartialEvalAlias1,\n PartialEvalAlias2,\n PartialEvalAlias3,\n PartialEvalAlias4,\n PartialEvalAlias5,\n PartialEvalAlias6,\n)\n\n\ndef partial_eval_alias(f, acc_args=()):\n \"\"\"Dispatches the appropriate eval alias based on the number of args to the original callable alias\n and how many arguments to apply.\n \"\"\"\n # no partial needed if no extra args\n if not acc_args:\n return f\n # need to dispatch\n numargs = 0\n for name, param in inspect.signature(f).parameters.items():\n if (\n param.kind == param.POSITIONAL_ONLY\n or param.kind == param.POSITIONAL_OR_KEYWORD\n ):\n numargs += 1\n elif name in ALIAS_KWARG_NAMES and param.kind == param.KEYWORD_ONLY:\n numargs += 1\n if numargs < 7:\n return PARTIAL_EVAL_ALIASES[numargs](f, acc_args=acc_args)\n else:\n e = \"Expected proxy with 6 or fewer arguments for {}, not {}\"\n raise XonshError(e.format(\", \".join(ALIAS_KWARG_NAMES), numargs))\n\n\n#\n# Actual aliases below\n#\n\n\ndef xonsh_exit(args, stdin=None):\n \"\"\"Sends signal to exit shell.\"\"\"\n if not clean_jobs():\n # Do not exit if jobs not cleaned up\n return None, None\n builtins.__xonsh__.exit = True\n print() # gimme a newline\n return None, None\n\n\ndef xonsh_reset(args, stdin=None):\n \"\"\" Clears __xonsh__.ctx\"\"\"\n builtins.__xonsh__.ctx.clear()\n\n\n@lazyobject\ndef _SOURCE_FOREIGN_PARSER():\n desc = \"Sources a file written in a foreign shell language.\"\n parser = argparse.ArgumentParser(\"source-foreign\", description=desc)\n parser.add_argument(\"shell\", help=\"Name or path to the foreign shell\")\n parser.add_argument(\n \"files_or_code\",\n nargs=\"+\",\n help=\"file paths to source or code in the target \" \"language.\",\n )\n parser.add_argument(\n \"-i\",\n \"--interactive\",\n type=to_bool,\n default=True,\n help=\"whether the sourced shell should be interactive\",\n dest=\"interactive\",\n )\n parser.add_argument(\n \"-l\",\n \"--login\",\n type=to_bool,\n default=False,\n help=\"whether the sourced shell should be login\",\n dest=\"login\",\n )\n parser.add_argument(\n \"--envcmd\", default=None, dest=\"envcmd\", help=\"command to print environment\"\n )\n parser.add_argument(\n \"--aliascmd\", default=None, dest=\"aliascmd\", help=\"command to print aliases\"\n )\n parser.add_argument(\n \"--extra-args\",\n default=(),\n dest=\"extra_args\",\n type=(lambda s: tuple(s.split())),\n help=\"extra arguments needed to run the shell\",\n )\n parser.add_argument(\n \"-s\",\n \"--safe\",\n type=to_bool,\n default=True,\n help=\"whether the source shell should be run safely, \"\n \"and not raise any errors, even if they occur.\",\n dest=\"safe\",\n )\n parser.add_argument(\n \"-p\",\n \"--prevcmd\",\n default=None,\n dest=\"prevcmd\",\n help=\"command(s) to run before any other commands, \"\n \"replaces traditional source.\",\n )\n parser.add_argument(\n \"--postcmd\",\n default=\"\",\n dest=\"postcmd\",\n help=\"command(s) to run after all other commands\",\n )\n parser.add_argument(\n \"--funcscmd\",\n default=None,\n dest=\"funcscmd\",\n help=\"code to find locations of all native functions \" \"in the shell language.\",\n )\n parser.add_argument(\n \"--sourcer\",\n default=None,\n dest=\"sourcer\",\n help=\"the source command in the target shell \" \"language, default: source.\",\n )\n parser.add_argument(\n \"--use-tmpfile\",\n type=to_bool,\n default=False,\n help=\"whether the commands for source shell should be \"\n \"written to a temporary file.\",\n dest=\"use_tmpfile\",\n )\n parser.add_argument(\n \"--seterrprevcmd\",\n default=None,\n dest=\"seterrprevcmd\",\n help=\"command(s) to set exit-on-error before any\" \"other commands.\",\n )\n parser.add_argument(\n \"--seterrpostcmd\",\n default=None,\n dest=\"seterrpostcmd\",\n help=\"command(s) to set exit-on-error after all\" \"other commands.\",\n )\n parser.add_argument(\n \"--overwrite-aliases\",\n default=False,\n action=\"store_true\",\n dest=\"overwrite_aliases\",\n help=\"flag for whether or not sourced aliases should \"\n \"replace the current xonsh aliases.\",\n )\n parser.add_argument(\n \"--suppress-skip-message\",\n default=None,\n action=\"store_true\",\n dest=\"suppress_skip_message\",\n help=\"flag for whether or not skip messages should be suppressed.\",\n )\n parser.add_argument(\n \"--show\",\n default=False,\n action=\"store_true\",\n dest=\"show\",\n help=\"Will show the script output.\",\n )\n parser.add_argument(\n \"-d\",\n \"--dry-run\",\n default=False,\n action=\"store_true\",\n dest=\"dryrun\",\n help=\"Will not actually source the file.\",\n )\n return parser\n\n\ndef source_foreign(args, stdin=None, stdout=None, stderr=None):\n \"\"\"Sources a file written in a foreign shell language.\"\"\"\n env = builtins.__xonsh__.env\n ns = _SOURCE_FOREIGN_PARSER.parse_args(args)\n ns.suppress_skip_message = (\n env.get(\"FOREIGN_ALIASES_SUPPRESS_SKIP_MESSAGE\")\n if ns.suppress_skip_message is None\n else ns.suppress_skip_message\n )\n if ns.prevcmd is not None:\n pass # don't change prevcmd if given explicitly\n elif os.path.isfile(ns.files_or_code[0]):\n # we have filename to source\n ns.prevcmd = '{} \"{}\"'.format(ns.sourcer, '\" \"'.join(ns.files_or_code))\n elif ns.prevcmd is None:\n ns.prevcmd = \" \".join(ns.files_or_code) # code to run, no files\n foreign_shell_data.cache_clear() # make sure that we don't get prev src\n fsenv, fsaliases = foreign_shell_data(\n shell=ns.shell,\n login=ns.login,\n interactive=ns.interactive,\n envcmd=ns.envcmd,\n aliascmd=ns.aliascmd,\n extra_args=ns.extra_args,\n safe=ns.safe,\n prevcmd=ns.prevcmd,\n postcmd=ns.postcmd,\n funcscmd=ns.funcscmd,\n sourcer=ns.sourcer,\n use_tmpfile=ns.use_tmpfile,\n seterrprevcmd=ns.seterrprevcmd,\n seterrpostcmd=ns.seterrpostcmd,\n show=ns.show,\n dryrun=ns.dryrun,\n )\n if fsenv is None:\n if ns.dryrun:\n return\n else:\n msg = \"xonsh: error: Source failed: {0!r}\\n\".format(ns.prevcmd)\n msg += \"xonsh: error: Possible reasons: File not found or syntax error\\n\"\n return (None, msg, 1)\n # apply results\n denv = env.detype()\n for k, v in fsenv.items():\n if k in denv and v == denv[k]:\n continue # no change from original\n env[k] = v\n # Remove any env-vars that were unset by the script.\n for k in denv:\n if k not in fsenv:\n env.pop(k, None)\n # Update aliases\n baliases = builtins.aliases\n for k, v in fsaliases.items():\n if k in baliases and v == baliases[k]:\n continue # no change from original\n elif ns.overwrite_aliases or k not in baliases:\n baliases[k] = v\n elif ns.suppress_skip_message:\n pass\n else:\n msg = (\n \"Skipping application of {0!r} alias from {1!r} \"\n \"since it shares a name with an existing xonsh alias. \"\n 'Use \"--overwrite-alias\" option to apply it anyway.'\n 'You may prevent this message with \"--suppress-skip-message\" or '\n '\"$FOREIGN_ALIASES_SUPPRESS_SKIP_MESSAGE = True\".'\n )\n print(msg.format(k, ns.shell), file=stderr)\n\n\ndef source_alias(args, stdin=None):\n \"\"\"Executes the contents of the provided files in the current context.\n If sourced file isn't found in cwd, search for file along $PATH to source\n instead.\n \"\"\"\n env = builtins.__xonsh__.env\n encoding = env.get(\"XONSH_ENCODING\")\n errors = env.get(\"XONSH_ENCODING_ERRORS\")\n for i, fname in enumerate(args):\n fpath = fname\n if not os.path.isfile(fpath):\n fpath = locate_binary(fname)\n if fpath is None:\n if env.get(\"XONSH_DEBUG\"):\n print(\"source: {}: No such file\".format(fname), file=sys.stderr)\n if i == 0:\n raise RuntimeError(\n \"must source at least one file, \" + fname + \"does not exist.\"\n )\n break\n _, fext = os.path.splitext(fpath)\n if fext and fext != \".xsh\" and fext != \".py\":\n raise RuntimeError(\n \"attempting to source non-xonsh file! If you are \"\n \"trying to source a file in another language, \"\n \"then please use the appropriate source command. \"\n \"For example, source-bash script.sh\"\n )\n with open(fpath, \"r\", encoding=encoding, errors=errors) as fp:\n src = fp.read()\n if not src.endswith(\"\\n\"):\n src += \"\\n\"\n ctx = builtins.__xonsh__.ctx\n updates = {\"__file__\": fpath, \"__name__\": os.path.abspath(fpath)}\n with env.swap(**make_args_env(args[i + 1 :])), swap_values(ctx, updates):\n try:\n builtins.execx(src, \"exec\", ctx, filename=fpath)\n except Exception:\n print_color(\n \"{RED}You may be attempting to source non-xonsh file! \"\n \"{NO_COLOR}If you are trying to source a file in \"\n \"another language, then please use the appropriate \"\n \"source command. For example, {GREEN}source-bash \"\n \"script.sh{NO_COLOR}\",\n file=sys.stderr,\n )\n raise\n\n\ndef source_cmd(args, stdin=None):\n \"\"\"Simple cmd.exe-specific wrapper around source-foreign.\"\"\"\n args = list(args)\n fpath = locate_binary(args[0])\n args[0] = fpath if fpath else args[0]\n if not os.path.isfile(args[0]):\n return (None, \"xonsh: error: File not found: {}\\n\".format(args[0]), 1)\n prevcmd = \"call \"\n prevcmd += \" \".join([argvquote(arg, force=True) for arg in args])\n prevcmd = escape_windows_cmd_string(prevcmd)\n args.append(\"--prevcmd={}\".format(prevcmd))\n args.insert(0, \"cmd\")\n args.append(\"--interactive=0\")\n args.append(\"--sourcer=call\")\n args.append(\"--envcmd=set\")\n args.append(\"--seterrpostcmd=if errorlevel 1 exit 1\")\n args.append(\"--use-tmpfile=1\")\n with builtins.__xonsh__.env.swap(PROMPT=\"$P$G\"):\n return source_foreign(args, stdin=stdin)\n\n\ndef xexec(args, stdin=None):\n \"\"\"exec [-h|--help] command [args...]\n\n exec (also aliased as xexec) uses the os.execvpe() function to\n replace the xonsh process with the specified program. This provides\n the functionality of the bash 'exec' builtin::\n\n >>> exec bash -l -i\n bash $\n\n The '-h' and '--help' options print this message and exit.\n\n Notes\n -----\n This command **is not** the same as the Python builtin function\n exec(). That function is for running Python code. This command,\n which shares the same name as the sh-lang statement, is for launching\n a command directly in the same process. In the event of a name conflict,\n please use the xexec command directly or dive into subprocess mode\n explicitly with ![exec command]. For more details, please see\n http://xon.sh/faq.html#exec.\n \"\"\"\n if len(args) == 0:\n return (None, \"xonsh: exec: no args specified\\n\", 1)\n elif args[0] == \"-h\" or args[0] == \"--help\":\n return inspect.getdoc(xexec)\n else:\n denv = builtins.__xonsh__.env.detype()\n try:\n os.execvpe(args[0], args, denv)\n except FileNotFoundError as e:\n return (\n None,\n \"xonsh: exec: file not found: {}: {}\" \"\\n\".format(e.args[1], args[0]),\n 1,\n )\n\n\nclass AWitchAWitch(argparse.Action):\n SUPPRESS = \"==SUPPRESS==\"\n\n def __init__(\n self, option_strings, version=None, dest=SUPPRESS, default=SUPPRESS, **kwargs\n ):\n super().__init__(\n option_strings=option_strings, dest=dest, default=default, nargs=0, **kwargs\n )\n\n def __call__(self, parser, namespace, values, option_string=None):\n import webbrowser\n\n webbrowser.open(\"https://github.com/xonsh/xonsh/commit/f49b400\")\n parser.exit()\n\n\ndef xonfig(args, stdin=None):\n \"\"\"Runs the xonsh configuration utility.\"\"\"\n from xonsh.xonfig import xonfig_main # lazy import\n\n return xonfig_main(args)\n\n\n@unthreadable\ndef trace(args, stdin=None, stdout=None, stderr=None, spec=None):\n \"\"\"Runs the xonsh tracer utility.\"\"\"\n from xonsh.tracer import tracermain # lazy import\n\n try:\n return tracermain(args, stdin=stdin, stdout=stdout, stderr=stderr, spec=spec)\n except SystemExit:\n pass\n\n\ndef showcmd(args, stdin=None):\n \"\"\"usage: showcmd [-h|--help|cmd args]\n\n Displays the command and arguments as a list of strings that xonsh would\n run in subprocess mode. This is useful for determining how xonsh evaluates\n your commands and arguments prior to running these commands.\n\n optional arguments:\n -h, --help show this help message and exit\n\n example:\n >>> showcmd echo $USER can't hear \"the sea\"\n ['echo', 'I', \"can't\", 'hear', 'the sea']\n \"\"\"\n if len(args) == 0 or (len(args) == 1 and args[0] in {\"-h\", \"--help\"}):\n print(showcmd.__doc__.rstrip().replace(\"\\n \", \"\\n\"))\n else:\n sys.displayhook(args)\n\n\ndef detect_xpip_alias():\n \"\"\"\n Determines the correct invocation to get xonsh's pip\n \"\"\"\n if not getattr(sys, \"executable\", None):\n return lambda args, stdin=None: (\n \"\",\n \"Sorry, unable to run pip on your system (missing sys.executable)\",\n 1,\n )\n\n basecmd = [sys.executable, \"-m\", \"pip\"]\n try:\n if ON_WINDOWS:\n # XXX: Does windows have an installation mode that requires UAC?\n return basecmd\n elif not os.access(os.path.dirname(sys.executable), os.W_OK):\n return [\"sudo\"] + basecmd\n else:\n return basecmd\n except Exception:\n # Something freaky happened, return something that'll probably work\n return basecmd\n\n\ndef make_default_aliases():\n \"\"\"Creates a new default aliases dictionary.\"\"\"\n default_aliases = {\n \"cd\": cd,\n \"pushd\": pushd,\n \"popd\": popd,\n \"dirs\": dirs,\n \"jobs\": jobs,\n \"fg\": fg,\n \"bg\": bg,\n \"EOF\": xonsh_exit,\n \"exit\": xonsh_exit,\n \"quit\": xonsh_exit,\n \"exec\": xexec,\n \"xexec\": xexec,\n \"source\": source_alias,\n \"source-zsh\": [\"source-foreign\", \"zsh\", \"--sourcer=source\"],\n \"source-bash\": [\"source-foreign\", \"bash\", \"--sourcer=source\"],\n \"source-cmd\": source_cmd,\n \"source-foreign\": source_foreign,\n \"history\": xhm.history_main,\n \"replay\": replay_main,\n \"trace\": trace,\n \"timeit\": timeit_alias,\n \"xonfig\": xonfig,\n \"scp-resume\": [\"rsync\", \"--partial\", \"-h\", \"--progress\", \"--rsh=ssh\"],\n \"showcmd\": showcmd,\n \"ipynb\": [\"jupyter\", \"notebook\", \"--no-browser\"],\n \"which\": xxw.which,\n \"xontrib\": xontribs_main,\n \"completer\": xca.completer_alias,\n \"xpip\": detect_xpip_alias(),\n \"xonsh-reset\": xonsh_reset,\n }\n if ON_WINDOWS:\n # Borrow builtin commands from cmd.exe.\n windows_cmd_aliases = {\n \"cls\",\n \"copy\",\n \"del\",\n \"dir\",\n \"echo\",\n \"erase\",\n \"md\",\n \"mkdir\",\n \"mklink\",\n \"move\",\n \"rd\",\n \"ren\",\n \"rename\",\n \"rmdir\",\n \"time\",\n \"type\",\n \"vol\",\n }\n for alias in windows_cmd_aliases:\n default_aliases[alias] = [\"cmd\", \"/c\", alias]\n default_aliases[\"call\"] = [\"source-cmd\"]\n default_aliases[\"source-bat\"] = [\"source-cmd\"]\n default_aliases[\"clear\"] = \"cls\"\n if ON_ANACONDA:\n # Add aliases specific to the Anaconda python distribution.\n default_aliases[\"activate\"] = [\"source-cmd\", \"activate.bat\"]\n default_aliases[\"deactivate\"] = [\"source-cmd\", \"deactivate.bat\"]\n if not locate_binary(\"sudo\"):\n import xonsh.winutils as winutils\n\n def sudo(args):\n if len(args) < 1:\n print(\n \"You need to provide an executable to run as \" \"Administrator.\"\n )\n return\n cmd = args[0]\n if locate_binary(cmd):\n return winutils.sudo(cmd, args[1:])\n elif cmd.lower() in windows_cmd_aliases:\n args = [\"/D\", \"/C\", \"CD\", _get_cwd(), \"&&\"] + args\n return winutils.sudo(\"cmd\", args)\n else:\n msg = 'Cannot find the path for executable \"{0}\".'\n print(msg.format(cmd))\n\n default_aliases[\"sudo\"] = sudo\n elif ON_DARWIN:\n default_aliases[\"ls\"] = [\"ls\", \"-G\"]\n elif ON_FREEBSD or ON_DRAGONFLY:\n default_aliases[\"grep\"] = [\"grep\", \"--color=auto\"]\n default_aliases[\"egrep\"] = [\"egrep\", \"--color=auto\"]\n default_aliases[\"fgrep\"] = [\"fgrep\", \"--color=auto\"]\n default_aliases[\"ls\"] = [\"ls\", \"-G\"]\n elif ON_NETBSD:\n default_aliases[\"grep\"] = [\"grep\", \"--color=auto\"]\n default_aliases[\"egrep\"] = [\"egrep\", \"--color=auto\"]\n default_aliases[\"fgrep\"] = [\"fgrep\", \"--color=auto\"]\n else:\n default_aliases[\"grep\"] = [\"grep\", \"--color=auto\"]\n default_aliases[\"egrep\"] = [\"egrep\", \"--color=auto\"]\n default_aliases[\"fgrep\"] = [\"fgrep\", \"--color=auto\"]\n default_aliases[\"ls\"] = [\"ls\", \"--color=auto\", \"-v\"]\n return default_aliases\n" }, { "path": "xonsh/ansi_colors.py", "content": "\"\"\"Tools for helping with ANSI color codes.\"\"\"\nimport re\nimport sys\nimport warnings\nimport builtins\n\nfrom xonsh.platform import HAS_PYGMENTS\nfrom xonsh.lazyasd import LazyDict, lazyobject\nfrom xonsh.color_tools import (\n RE_BACKGROUND,\n BASE_XONSH_COLORS,\n make_palette,\n find_closest_color,\n rgb2short,\n rgb_to_256,\n short_to_ints,\n)\nfrom xonsh.tools import FORMATTER\n\n\ndef ansi_partial_color_format(template, style=\"default\", cmap=None, hide=False):\n \"\"\"Formats a template string but only with respect to the colors.\n Another template string is returned, with the color values filled in.\n\n Parameters\n ----------\n template : str\n The template string, potentially with color names.\n style : str, optional\n Style name to look up color map from.\n cmap : dict, optional\n A color map to use, this will prevent the color map from being\n looked up via the style name.\n hide : bool, optional\n Whether to wrap the color codes in the \\\\001 and \\\\002 escape\n codes, so that the color codes are not counted against line\n length.\n\n Returns\n -------\n A template string with the color values filled in.\n \"\"\"\n try:\n return _ansi_partial_color_format_main(\n template, style=style, cmap=cmap, hide=hide\n )\n except Exception:\n return template\n\n\ndef _ansi_partial_color_format_main(template, style=\"default\", cmap=None, hide=False):\n if cmap is not None:\n pass\n elif style in ANSI_STYLES:\n cmap = ANSI_STYLES[style]\n else:\n try: # dynamically loading the style\n cmap = ansi_style_by_name(style)\n except Exception:\n msg = \"Could not find color style {0!r}, using default.\"\n print(msg.format(style), file=sys.stderr)\n builtins.__xonsh__.env[\"XONSH_COLOR_STYLE\"] = \"default\"\n cmap = ANSI_STYLES[\"default\"]\n esc = (\"\\001\" if hide else \"\") + \"\\033[\"\n m = \"m\" + (\"\\002\" if hide else \"\")\n bopen = \"{\"\n bclose = \"}\"\n colon = \":\"\n expl = \"!\"\n toks = []\n for literal, field, spec, conv in FORMATTER.parse(template):\n toks.append(literal)\n if field is None:\n pass\n elif field in cmap:\n toks.extend([esc, cmap[field], m])\n elif \"#\" in field:\n field = field.lower()\n pre, _, post = field.partition(\"#\")\n f_or_b = \"38\" if RE_BACKGROUND.search(pre) is None else \"48\"\n rgb, _, post = post.partition(\"_\")\n c256, _ = rgb_to_256(rgb)\n color = f_or_b + \";5;\" + c256\n mods = pre + \"_\" + post\n if \"underline\" in mods:\n color = \"4;\" + color\n if \"bold\" in mods:\n color = \"1;\" + color\n toks.extend([esc, color, m])\n elif field is not None:\n toks.append(bopen)\n toks.append(field)\n if conv is not None and len(conv) > 0:\n toks.append(expl)\n toks.append(conv)\n if spec is not None and len(spec) > 0:\n toks.append(colon)\n toks.append(spec)\n toks.append(bclose)\n return \"\".join(toks)\n\n\ndef ansi_color_style_names():\n \"\"\"Returns an iterable of all ANSI color style names.\"\"\"\n return ANSI_STYLES.keys()\n\n\ndef ansi_color_style(style=\"default\"):\n \"\"\"Returns the current color map.\"\"\"\n if style in ANSI_STYLES:\n cmap = ANSI_STYLES[style]\n else:\n msg = \"Could not find color style {0!r}, using default.\".format(style)\n warnings.warn(msg, RuntimeWarning)\n cmap = ANSI_STYLES[\"default\"]\n return cmap\n\n\ndef ansi_reverse_style(style=\"default\", return_style=False):\n \"\"\"Reverses an ANSI color style mapping so that escape codes map to\n colors. Style may either be string or mapping. May also return\n the style it looked up.\n \"\"\"\n style = ansi_style_by_name(style) if isinstance(style, str) else style\n reversed_style = {v: k for k, v in style.items()}\n # add keys to make this more useful\n updates = {\n \"1\": \"BOLD_\",\n \"2\": \"FAINT_\",\n \"4\": \"UNDERLINE_\",\n \"5\": \"SLOWBLINK_\",\n \"1;4\": \"BOLD_UNDERLINE_\",\n \"4;1\": \"BOLD_UNDERLINE_\",\n \"38\": \"SET_FOREGROUND_\",\n \"48\": \"SET_BACKGROUND_\",\n \"38;2\": \"SET_FOREGROUND_3INTS_\",\n \"48;2\": \"SET_BACKGROUND_3INTS_\",\n \"38;5\": \"SET_FOREGROUND_SHORT_\",\n \"48;5\": \"SET_BACKGROUND_SHORT_\",\n }\n for ec, name in reversed_style.items():\n no_left_zero = ec.lstrip(\"0\")\n if no_left_zero.startswith(\";\"):\n updates[no_left_zero[1:]] = name\n elif no_left_zero != ec:\n updates[no_left_zero] = name\n reversed_style.update(updates)\n # return results\n if return_style:\n return style, reversed_style\n else:\n return reversed_style\n\n\n@lazyobject\ndef ANSI_ESCAPE_CODE_RE():\n return re.compile(r\"\\001?(\\033\\[)?([0-9;]+)m?\\002?\")\n\n\n@lazyobject\ndef ANSI_REVERSE_COLOR_NAME_TRANSLATIONS():\n base = {\n \"SET_FOREGROUND_FAINT_\": \"SET_FOREGROUND_3INTS_\",\n \"SET_BACKGROUND_FAINT_\": \"SET_BACKGROUND_3INTS_\",\n \"SET_FOREGROUND_SLOWBLINK_\": \"SET_FOREGROUND_SHORT_\",\n \"SET_BACKGROUND_SLOWBLINK_\": \"SET_BACKGROUND_SHORT_\",\n }\n data = {\"UNDERLINE_BOLD_\": \"BOLD_UNDERLINE_\"}\n data.update(base)\n data.update({\"BOLD_\" + k: \"BOLD_\" + v for k, v in base.items()})\n data.update({\"UNDERLINE_\" + k: \"UNDERLINE_\" + v for k, v in base.items()})\n data.update({\"BOLD_UNDERLINE_\" + k: \"BOLD_UNDERLINE_\" + v for k, v in base.items()})\n data.update({\"UNDERLINE_BOLD_\" + k: \"BOLD_UNDERLINE_\" + v for k, v in base.items()})\n return data\n\n\n@lazyobject\ndef ANSI_COLOR_NAME_SET_3INTS_RE():\n return re.compile(r\"(\\w+_)?SET_(FORE|BACK)GROUND_3INTS_(\\d+)_(\\d+)_(\\d+)\")\n\n\n@lazyobject\ndef ANSI_COLOR_NAME_SET_SHORT_RE():\n return re.compile(r\"(\\w+_)?SET_(FORE|BACK)GROUND_SHORT_(\\d+)\")\n\n\ndef _color_name_from_ints(ints, background=False, prefix=None):\n name = find_closest_color(ints, BASE_XONSH_COLORS)\n if background:\n name = \"BACKGROUND_\" + name\n name = name if prefix is None else prefix + name\n return name\n\n\n_ANSI_COLOR_ESCAPE_CODE_TO_NAME_CACHE = {}\n\n\ndef ansi_color_escape_code_to_name(escape_code, style, reversed_style=None):\n \"\"\"Converts an ASNI color code escape sequence to a tuple of color names\n in the provided style ('default' should almost be the style). For example,\n '0' becomes ('NO_COLOR',) and '32;41' becomes ('GREEN', 'BACKGROUND_RED').\n The style keyword may either be a string, in which the style is looked up,\n or an actual style dict. You can also provide a reversed style mapping,\n too, which is just the keys/values of the style dict swapped. If reversed\n style is not provided, it is computed.\n \"\"\"\n key = (escape_code, style)\n if key in _ANSI_COLOR_ESCAPE_CODE_TO_NAME_CACHE:\n return _ANSI_COLOR_ESCAPE_CODE_TO_NAME_CACHE[key]\n if reversed_style is None:\n style, reversed_style = ansi_reverse_style(style, return_style=True)\n # strip some actual escape codes, if needed.\n ec = ANSI_ESCAPE_CODE_RE.match(escape_code).group(2)\n names = []\n n_ints = 0\n seen_set_foreback = False\n for e in ec.split(\";\"):\n no_left_zero = e.lstrip(\"0\") if len(e) > 1 else e\n if seen_set_foreback and n_ints > 0:\n names.append(e)\n n_ints -= 1\n if n_ints == 0:\n seen_set_foreback = False\n continue\n else:\n names.append(reversed_style.get(no_left_zero, no_left_zero))\n # set the flags for next time\n if \"38\" == e or \"48\" == e:\n seen_set_foreback = True\n elif \"2\" == e:\n n_ints = 3\n elif \"5\" == e:\n n_ints = 1\n # normalize names\n n = \"\"\n norm_names = []\n colors = set(reversed_style.values())\n for name in names:\n if name == \"NO_COLOR\":\n # skip most '0' entries\n continue\n n = n + name if n else name\n n = ANSI_REVERSE_COLOR_NAME_TRANSLATIONS.get(n, n)\n if n.endswith(\"_\"):\n continue\n elif ANSI_COLOR_NAME_SET_SHORT_RE.match(n) is not None:\n pre, fore_back, short = ANSI_COLOR_NAME_SET_SHORT_RE.match(n).groups()\n n = _color_name_from_ints(\n short_to_ints(short), background=(fore_back == \"BACK\"), prefix=pre\n )\n elif ANSI_COLOR_NAME_SET_3INTS_RE.match(n) is not None:\n pre, fore_back, r, g, b = ANSI_COLOR_NAME_SET_3INTS_RE.match(n).groups()\n n = _color_name_from_ints(\n (int(r), int(g), int(b)), background=(fore_back == \"BACK\"), prefix=pre\n )\n elif \"GROUND_3INTS_\" in n:\n # have 1 or 2, but not 3 ints\n n += \"_\"\n continue\n # error check\n if n not in colors:\n msg = (\n \"Could not translate ANSI color code {escape_code!r} \"\n \"into a known color in the palette. Specifically, the {n!r} \"\n \"portion of {name!r} in {names!r} seems to missing.\"\n )\n raise ValueError(\n msg.format(escape_code=escape_code, names=names, name=name, n=n)\n )\n norm_names.append(n)\n n = \"\"\n # return\n if len(norm_names) == 0:\n return (\"NO_COLOR\",)\n else:\n return tuple(norm_names)\n\n\ndef _ansi_expand_style(cmap):\n \"\"\"Expands a style in order to more quickly make color map changes.\"\"\"\n for key, val in list(cmap.items()):\n if key == \"NO_COLOR\":\n continue\n elif len(val) == 0:\n cmap[\"BOLD_\" + key] = \"1\"\n cmap[\"UNDERLINE_\" + key] = \"4\"\n cmap[\"BOLD_UNDERLINE_\" + key] = \"1;4\"\n cmap[\"BACKGROUND_\" + key] = val\n else:\n cmap[\"BOLD_\" + key] = \"1;\" + val\n cmap[\"UNDERLINE_\" + key] = \"4;\" + val\n cmap[\"BOLD_UNDERLINE_\" + key] = \"1;4;\" + val\n cmap[\"BACKGROUND_\" + key] = val.replace(\"38\", \"48\", 1)\n\n\ndef _bw_style():\n style = {\n \"BLACK\": \"0;30\",\n \"BLUE\": \"0;37\",\n \"CYAN\": \"0;37\",\n \"GREEN\": \"0;37\",\n \"INTENSE_BLACK\": \"0;90\",\n \"INTENSE_BLUE\": \"0;97\",\n \"INTENSE_CYAN\": \"0;97\",\n \"INTENSE_GREEN\": \"0;97\",\n \"INTENSE_PURPLE\": \"0;97\",\n \"INTENSE_RED\": \"0;97\",\n \"INTENSE_WHITE\": \"0;97\",\n \"INTENSE_YELLOW\": \"0;97\",\n \"NO_COLOR\": \"0\",\n \"PURPLE\": \"0;37\",\n \"RED\": \"0;37\",\n \"WHITE\": \"0;37\",\n \"YELLOW\": \"0;37\",\n }\n _ansi_expand_style(style)\n return style\n\n\ndef _default_style():\n style = {\n # Reset\n \"NO_COLOR\": \"0\", # Text Reset\n # Regular Colors\n \"BLACK\": \"0;30\", # BLACK\n \"RED\": \"0;31\", # RED\n \"GREEN\": \"0;32\", # GREEN\n \"YELLOW\": \"0;33\", # YELLOW\n \"BLUE\": \"0;34\", # BLUE\n \"PURPLE\": \"0;35\", # PURPLE\n \"CYAN\": \"0;36\", # CYAN\n \"WHITE\": \"0;37\", # WHITE\n # Bold\n \"BOLD_BLACK\": \"1;30\", # BLACK\n \"BOLD_RED\": \"1;31\", # RED\n \"BOLD_GREEN\": \"1;32\", # GREEN\n \"BOLD_YELLOW\": \"1;33\", # YELLOW\n \"BOLD_BLUE\": \"1;34\", # BLUE\n \"BOLD_PURPLE\": \"1;35\", # PURPLE\n \"BOLD_CYAN\": \"1;36\", # CYAN\n \"BOLD_WHITE\": \"1;37\", # WHITE\n # Underline\n \"UNDERLINE_BLACK\": \"4;30\", # BLACK\n \"UNDERLINE_RED\": \"4;31\", # RED\n \"UNDERLINE_GREEN\": \"4;32\", # GREEN\n \"UNDERLINE_YELLOW\": \"4;33\", # YELLOW\n \"UNDERLINE_BLUE\": \"4;34\", # BLUE\n \"UNDERLINE_PURPLE\": \"4;35\", # PURPLE\n \"UNDERLINE_CYAN\": \"4;36\", # CYAN\n \"UNDERLINE_WHITE\": \"4;37\", # WHITE\n # Bold, Underline\n \"BOLD_UNDERLINE_BLACK\": \"1;4;30\", # BLACK\n \"BOLD_UNDERLINE_RED\": \"1;4;31\", # RED\n \"BOLD_UNDERLINE_GREEN\": \"1;4;32\", # GREEN\n \"BOLD_UNDERLINE_YELLOW\": \"1;4;33\", # YELLOW\n \"BOLD_UNDERLINE_BLUE\": \"1;4;34\", # BLUE\n \"BOLD_UNDERLINE_PURPLE\": \"1;4;35\", # PURPLE\n \"BOLD_UNDERLINE_CYAN\": \"1;4;36\", # CYAN\n \"BOLD_UNDERLINE_WHITE\": \"1;4;37\", # WHITE\n # Background\n \"BACKGROUND_BLACK\": \"40\", # BLACK\n \"BACKGROUND_RED\": \"41\", # RED\n \"BACKGROUND_GREEN\": \"42\", # GREEN\n \"BACKGROUND_YELLOW\": \"43\", # YELLOW\n \"BACKGROUND_BLUE\": \"44\", # BLUE\n \"BACKGROUND_PURPLE\": \"45\", # PURPLE\n \"BACKGROUND_CYAN\": \"46\", # CYAN\n \"BACKGROUND_WHITE\": \"47\", # WHITE\n # High Intensity\n \"INTENSE_BLACK\": \"0;90\", # BLACK\n \"INTENSE_RED\": \"0;91\", # RED\n \"INTENSE_GREEN\": \"0;92\", # GREEN\n \"INTENSE_YELLOW\": \"0;93\", # YELLOW\n \"INTENSE_BLUE\": \"0;94\", # BLUE\n \"INTENSE_PURPLE\": \"0;95\", # PURPLE\n \"INTENSE_CYAN\": \"0;96\", # CYAN\n \"INTENSE_WHITE\": \"0;97\", # WHITE\n # Bold High Intensity\n \"BOLD_INTENSE_BLACK\": \"1;90\", # BLACK\n \"BOLD_INTENSE_RED\": \"1;91\", # RED\n \"BOLD_INTENSE_GREEN\": \"1;92\", # GREEN\n \"BOLD_INTENSE_YELLOW\": \"1;93\", # YELLOW\n \"BOLD_INTENSE_BLUE\": \"1;94\", # BLUE\n \"BOLD_INTENSE_PURPLE\": \"1;95\", # PURPLE\n \"BOLD_INTENSE_CYAN\": \"1;96\", # CYAN\n \"BOLD_INTENSE_WHITE\": \"1;97\", # WHITE\n # Underline High Intensity\n \"UNDERLINE_INTENSE_BLACK\": \"4;90\", # BLACK\n \"UNDERLINE_INTENSE_RED\": \"4;91\", # RED\n \"UNDERLINE_INTENSE_GREEN\": \"4;92\", # GREEN\n \"UNDERLINE_INTENSE_YELLOW\": \"4;93\", # YELLOW\n \"UNDERLINE_INTENSE_BLUE\": \"4;94\", # BLUE\n \"UNDERLINE_INTENSE_PURPLE\": \"4;95\", # PURPLE\n \"UNDERLINE_INTENSE_CYAN\": \"4;96\", # CYAN\n \"UNDERLINE_INTENSE_WHITE\": \"4;97\", # WHITE\n # Bold Underline High Intensity\n \"BOLD_UNDERLINE_INTENSE_BLACK\": \"1;4;90\", # BLACK\n \"BOLD_UNDERLINE_INTENSE_RED\": \"1;4;91\", # RED\n \"BOLD_UNDERLINE_INTENSE_GREEN\": \"1;4;92\", # GREEN\n \"BOLD_UNDERLINE_INTENSE_YELLOW\": \"1;4;93\", # YELLOW\n \"BOLD_UNDERLINE_INTENSE_BLUE\": \"1;4;94\", # BLUE\n \"BOLD_UNDERLINE_INTENSE_PURPLE\": \"1;4;95\", # PURPLE\n \"BOLD_UNDERLINE_INTENSE_CYAN\": \"1;4;96\", # CYAN\n \"BOLD_UNDERLINE_INTENSE_WHITE\": \"1;4;97\", # WHITE\n # High Intensity backgrounds\n \"BACKGROUND_INTENSE_BLACK\": \"0;100\", # BLACK\n \"BACKGROUND_INTENSE_RED\": \"0;101\", # RED\n \"BACKGROUND_INTENSE_GREEN\": \"0;102\", # GREEN\n \"BACKGROUND_INTENSE_YELLOW\": \"0;103\", # YELLOW\n \"BACKGROUND_INTENSE_BLUE\": \"0;104\", # BLUE\n \"BACKGROUND_INTENSE_PURPLE\": \"0;105\", # PURPLE\n \"BACKGROUND_INTENSE_CYAN\": \"0;106\", # CYAN\n \"BACKGROUND_INTENSE_WHITE\": \"0;107\", # WHITE\n }\n return style\n\n\ndef _monokai_style():\n style = {\n \"NO_COLOR\": \"0\",\n \"BLACK\": \"38;5;16\",\n \"BLUE\": \"38;5;63\",\n \"CYAN\": \"38;5;81\",\n \"GREEN\": \"38;5;40\",\n \"PURPLE\": \"38;5;89\",\n \"RED\": \"38;5;124\",\n \"WHITE\": \"38;5;188\",\n \"YELLOW\": \"38;5;184\",\n \"INTENSE_BLACK\": \"38;5;59\",\n \"INTENSE_BLUE\": \"38;5;20\",\n \"INTENSE_CYAN\": \"38;5;44\",\n \"INTENSE_GREEN\": \"38;5;148\",\n \"INTENSE_PURPLE\": \"38;5;141\",\n \"INTENSE_RED\": \"38;5;197\",\n \"INTENSE_WHITE\": \"38;5;15\",\n \"INTENSE_YELLOW\": \"38;5;186\",\n }\n _ansi_expand_style(style)\n return style\n\n\n####################################\n# Auto-generated below this line #\n####################################\n\n\ndef _algol_style():\n style = {\n \"BLACK\": \"38;5;59\",\n \"BLUE\": \"38;5;59\",\n \"CYAN\": \"38;5;59\",\n \"GREEN\": \"38;5;59\",\n \"INTENSE_BLACK\": \"38;5;59\",\n \"INTENSE_BLUE\": \"38;5;102\",\n \"INTENSE_CYAN\": \"38;5;102\",\n \"INTENSE_GREEN\": \"38;5;102\",\n \"INTENSE_PURPLE\": \"38;5;102\",\n \"INTENSE_RED\": \"38;5;09\",\n \"INTENSE_WHITE\": \"38;5;102\",\n \"INTENSE_YELLOW\": \"38;5;102\",\n \"NO_COLOR\": \"0\",\n \"PURPLE\": \"38;5;59\",\n \"RED\": \"38;5;09\",\n \"WHITE\": \"38;5;102\",\n \"YELLOW\": \"38;5;09\",\n }\n _ansi_expand_style(style)\n return style\n\n\ndef _algol_nu_style():\n style = {\n \"BLACK\": \"38;5;59\",\n \"BLUE\": \"38;5;59\",\n \"CYAN\": \"38;5;59\",\n \"GREEN\": \"38;5;59\",\n \"INTENSE_BLACK\": \"38;5;59\",\n \"INTENSE_BLUE\": \"38;5;102\",\n \"INTENSE_CYAN\": \"38;5;102\",\n \"INTENSE_GREEN\": \"38;5;102\",\n \"INTENSE_PURPLE\": \"38;5;102\",\n \"INTENSE_RED\": \"38;5;09\",\n \"INTENSE_WHITE\": \"38;5;102\",\n \"INTENSE_YELLOW\": \"38;5;102\",\n \"NO_COLOR\": \"0\",\n \"PURPLE\": \"38;5;59\",\n \"RED\": \"38;5;09\",\n \"WHITE\": \"38;5;102\",\n \"YELLOW\": \"38;5;09\",\n }\n _ansi_expand_style(style)\n return style\n\n\ndef _autumn_style():\n style = {\n \"BLACK\": \"38;5;18\",\n \"BLUE\": \"38;5;19\",\n \"CYAN\": \"38;5;37\",\n \"GREEN\": \"38;5;34\",\n \"INTENSE_BLACK\": \"38;5;59\",\n \"INTENSE_BLUE\": \"38;5;33\",\n \"INTENSE_CYAN\": \"38;5;33\",\n \"INTENSE_GREEN\": \"38;5;64\",\n \"INTENSE_PURPLE\": \"38;5;217\",\n \"INTENSE_RED\": \"38;5;130\",\n \"INTENSE_WHITE\": \"38;5;145\",\n \"INTENSE_YELLOW\": \"38;5;217\",\n \"NO_COLOR\": \"0\",\n \"PURPLE\": \"38;5;90\",\n \"RED\": \"38;5;124\",\n \"WHITE\": \"38;5;145\",\n \"YELLOW\": \"38;5;130\",\n }\n _ansi_expand_style(style)\n return style\n\n\ndef _borland_style():\n style = {\n \"BLACK\": \"38;5;16\",\n \"BLUE\": \"38;5;18\",\n \"CYAN\": \"38;5;30\",\n \"GREEN\": \"38;5;28\",\n \"INTENSE_BLACK\": \"38;5;59\",\n \"INTENSE_BLUE\": \"38;5;21\",\n \"INTENSE_CYAN\": \"38;5;194\",\n \"INTENSE_GREEN\": \"38;5;102\",\n \"INTENSE_PURPLE\": \"38;5;188\",\n \"INTENSE_RED\": \"38;5;09\",\n \"INTENSE_WHITE\": \"38;5;224\",\n \"INTENSE_YELLOW\": \"38;5;188\",\n \"NO_COLOR\": \"0\",\n \"PURPLE\": \"38;5;90\",\n \"RED\": \"38;5;124\",\n \"WHITE\": \"38;5;145\",\n \"YELLOW\": \"38;5;124\",\n }\n _ansi_expand_style(style)\n return style\n\n\ndef _colorful_style():\n style = {\n \"BLACK\": \"38;5;16\",\n \"BLUE\": \"38;5;20\",\n \"CYAN\": \"38;5;31\",\n \"GREEN\": \"38;5;34\",\n \"INTENSE_BLACK\": \"38;5;59\",\n \"INTENSE_BLUE\": \"38;5;61\",\n \"INTENSE_CYAN\": \"38;5;145\",\n \"INTENSE_GREEN\": \"38;5;102\",\n \"INTENSE_PURPLE\": \"38;5;217\",\n \"INTENSE_RED\": \"38;5;166\",\n \"INTENSE_WHITE\": \"38;5;15\",\n \"INTENSE_YELLOW\": \"38;5;217\",\n \"NO_COLOR\": \"0\",\n \"PURPLE\": \"38;5;90\",\n \"RED\": \"38;5;124\",\n \"WHITE\": \"38;5;145\",\n \"YELLOW\": \"38;5;130\",\n }\n _ansi_expand_style(style)\n return style\n\n\ndef _emacs_style():\n style = {\n \"BLACK\": \"38;5;28\",\n \"BLUE\": \"38;5;18\",\n \"CYAN\": \"38;5;26\",\n \"GREEN\": \"38;5;34\",\n \"INTENSE_BLACK\": \"38;5;59\",\n \"INTENSE_BLUE\": \"38;5;26\",\n \"INTENSE_CYAN\": \"38;5;145\",\n \"INTENSE_GREEN\": \"38;5;34\",\n \"INTENSE_PURPLE\": \"38;5;129\",\n \"INTENSE_RED\": \"38;5;167\",\n \"INTENSE_WHITE\": \"38;5;145\",\n \"INTENSE_YELLOW\": \"38;5;145\",\n \"NO_COLOR\": \"0\",\n \"PURPLE\": \"38;5;90\",\n \"RED\": \"38;5;124\",\n \"WHITE\": \"38;5;145\",\n \"YELLOW\": \"38;5;130\",\n }\n _ansi_expand_style(style)\n return style\n\n\ndef _friendly_style():\n style = {\n \"BLACK\": \"38;5;22\",\n \"BLUE\": \"38;5;18\",\n \"CYAN\": \"38;5;31\",\n \"GREEN\": \"38;5;34\",\n \"INTENSE_BLACK\": \"38;5;59\",\n \"INTENSE_BLUE\": \"38;5;74\",\n \"INTENSE_CYAN\": \"38;5;74\",\n \"INTENSE_GREEN\": \"38;5;71\",\n \"INTENSE_PURPLE\": \"38;5;134\",\n \"INTENSE_RED\": \"38;5;167\",\n \"INTENSE_WHITE\": \"38;5;15\",\n \"INTENSE_YELLOW\": \"38;5;145\",\n \"NO_COLOR\": \"0\",\n \"PURPLE\": \"38;5;90\",\n \"RED\": \"38;5;124\",\n \"WHITE\": \"38;5;145\",\n \"YELLOW\": \"38;5;166\",\n }\n _ansi_expand_style(style)\n return style\n\n\ndef _fruity_style():\n style = {\n \"BLACK\": \"38;5;16\",\n \"BLUE\": \"38;5;32\",\n \"CYAN\": \"38;5;32\",\n \"GREEN\": \"38;5;28\",\n \"INTENSE_BLACK\": \"38;5;59\",\n \"INTENSE_BLUE\": \"38;5;33\",\n \"INTENSE_CYAN\": \"38;5;33\",\n \"INTENSE_GREEN\": \"38;5;102\",\n \"INTENSE_PURPLE\": \"38;5;198\",\n \"INTENSE_RED\": \"38;5;202\",\n \"INTENSE_WHITE\": \"38;5;15\",\n \"INTENSE_YELLOW\": \"38;5;187\",\n \"NO_COLOR\": \"0\",\n \"PURPLE\": \"38;5;198\",\n \"RED\": \"38;5;09\",\n \"WHITE\": \"38;5;187\",\n \"YELLOW\": \"38;5;202\",\n }\n _ansi_expand_style(style)\n return style\n\n\ndef _igor_style():\n style = {\n \"BLACK\": \"38;5;34\",\n \"BLUE\": \"38;5;21\",\n \"CYAN\": \"38;5;30\",\n \"GREEN\": \"38;5;34\",\n \"INTENSE_BLACK\": \"38;5;30\",\n \"INTENSE_BLUE\": \"38;5;21\",\n \"INTENSE_CYAN\": \"38;5;30\",\n \"INTENSE_GREEN\": \"38;5;34\",\n \"INTENSE_PURPLE\": \"38;5;163\",\n \"INTENSE_RED\": \"38;5;166\",\n \"INTENSE_WHITE\": \"38;5;163\",\n \"INTENSE_YELLOW\": \"38;5;166\",\n \"NO_COLOR\": \"0\",\n \"PURPLE\": \"38;5;163\",\n \"RED\": \"38;5;166\",\n \"WHITE\": \"38;5;163\",\n \"YELLOW\": \"38;5;166\",\n }\n _ansi_expand_style(style)\n return style\n\n\ndef _lovelace_style():\n style = {\n \"BLACK\": \"38;5;59\",\n \"BLUE\": \"38;5;25\",\n \"CYAN\": \"38;5;29\",\n \"GREEN\": \"38;5;65\",\n \"INTENSE_BLACK\": \"38;5;59\",\n \"INTENSE_BLUE\": \"38;5;25\",\n \"INTENSE_CYAN\": \"38;5;102\",\n \"INTENSE_GREEN\": \"38;5;29\",\n \"INTENSE_PURPLE\": \"38;5;133\",\n \"INTENSE_RED\": \"38;5;131\",\n \"INTENSE_WHITE\": \"38;5;102\",\n \"INTENSE_YELLOW\": \"38;5;136\",\n \"NO_COLOR\": \"0\",\n \"PURPLE\": \"38;5;133\",\n \"RED\": \"38;5;124\",\n \"WHITE\": \"38;5;102\",\n \"YELLOW\": \"38;5;130\",\n }\n _ansi_expand_style(style)\n return style\n\n\ndef _manni_style():\n style = {\n \"BLACK\": \"38;5;16\",\n \"BLUE\": \"38;5;18\",\n \"CYAN\": \"38;5;30\",\n \"GREEN\": \"38;5;40\",\n \"INTENSE_BLACK\": \"38;5;59\",\n \"INTENSE_BLUE\": \"38;5;105\",\n \"INTENSE_CYAN\": \"38;5;45\",\n \"INTENSE_GREEN\": \"38;5;113\",\n \"INTENSE_PURPLE\": \"38;5;165\",\n \"INTENSE_RED\": \"38;5;202\",\n \"INTENSE_WHITE\": \"38;5;224\",\n \"INTENSE_YELLOW\": \"38;5;221\",\n \"NO_COLOR\": \"0\",\n \"PURPLE\": \"38;5;165\",\n \"RED\": \"38;5;124\",\n \"WHITE\": \"38;5;145\",\n \"YELLOW\": \"38;5;166\",\n }\n _ansi_expand_style(style)\n return style\n\n\ndef _murphy_style():\n style = {\n \"BLACK\": \"38;5;16\",\n \"BLUE\": \"38;5;18\",\n \"CYAN\": \"38;5;31\",\n \"GREEN\": \"38;5;34\",\n \"INTENSE_BLACK\": \"38;5;59\",\n \"INTENSE_BLUE\": \"38;5;63\",\n \"INTENSE_CYAN\": \"38;5;86\",\n \"INTENSE_GREEN\": \"38;5;86\",\n \"INTENSE_PURPLE\": \"38;5;213\",\n \"INTENSE_RED\": \"38;5;209\",\n \"INTENSE_WHITE\": \"38;5;15\",\n \"INTENSE_YELLOW\": \"38;5;222\",\n \"NO_COLOR\": \"0\",\n \"PURPLE\": \"38;5;90\",\n \"RED\": \"38;5;124\",\n \"WHITE\": \"38;5;145\",\n \"YELLOW\": \"38;5;166\",\n }\n _ansi_expand_style(style)\n return style\n\n\ndef _native_style():\n style = {\n \"BLACK\": \"38;5;52\",\n \"BLUE\": \"38;5;67\",\n \"CYAN\": \"38;5;31\",\n \"GREEN\": \"38;5;64\",\n \"INTENSE_BLACK\": \"38;5;59\",\n \"INTENSE_BLUE\": \"38;5;68\",\n \"INTENSE_CYAN\": \"38;5;87\",\n \"INTENSE_GREEN\": \"38;5;70\",\n \"INTENSE_PURPLE\": \"38;5;188\",\n \"INTENSE_RED\": \"38;5;160\",\n \"INTENSE_WHITE\": \"38;5;15\",\n \"INTENSE_YELLOW\": \"38;5;214\",\n \"NO_COLOR\": \"0\",\n \"PURPLE\": \"38;5;59\",\n \"RED\": \"38;5;124\",\n \"WHITE\": \"38;5;145\",\n \"YELLOW\": \"38;5;124\",\n }\n _ansi_expand_style(style)\n return style\n\n\ndef _paraiso_dark_style():\n style = {\n \"BLACK\": \"38;5;95\",\n \"BLUE\": \"38;5;97\",\n \"CYAN\": \"38;5;39\",\n \"GREEN\": \"38;5;72\",\n \"INTENSE_BLACK\": \"38;5;95\",\n \"INTENSE_BLUE\": \"38;5;97\",\n \"INTENSE_CYAN\": \"38;5;79\",\n \"INTENSE_GREEN\": \"38;5;72\",\n \"INTENSE_PURPLE\": \"38;5;188\",\n \"INTENSE_RED\": \"38;5;203\",\n \"INTENSE_WHITE\": \"38;5;188\",\n \"INTENSE_YELLOW\": \"38;5;220\",\n \"NO_COLOR\": \"0\",\n \"PURPLE\": \"38;5;97\",\n \"RED\": \"38;5;203\",\n \"WHITE\": \"38;5;79\",\n \"YELLOW\": \"38;5;214\",\n }\n _ansi_expand_style(style)\n return style\n\n\ndef _paraiso_light_style():\n style = {\n \"BLACK\": \"38;5;16\",\n \"BLUE\": \"38;5;16\",\n \"CYAN\": \"38;5;39\",\n \"GREEN\": \"38;5;72\",\n \"INTENSE_BLACK\": \"38;5;16\",\n \"INTENSE_BLUE\": \"38;5;97\",\n \"INTENSE_CYAN\": \"38;5;79\",\n \"INTENSE_GREEN\": \"38;5;72\",\n \"INTENSE_PURPLE\": \"38;5;97\",\n \"INTENSE_RED\": \"38;5;203\",\n \"INTENSE_WHITE\": \"38;5;79\",\n \"INTENSE_YELLOW\": \"38;5;220\",\n \"NO_COLOR\": \"0\",\n \"PURPLE\": \"38;5;97\",\n \"RED\": \"38;5;16\",\n \"WHITE\": \"38;5;102\",\n \"YELLOW\": \"38;5;214\",\n }\n _ansi_expand_style(style)\n return style\n\n\ndef _pastie_style():\n style = {\n \"BLACK\": \"38;5;16\",\n \"BLUE\": \"38;5;20\",\n \"CYAN\": \"38;5;25\",\n \"GREEN\": \"38;5;28\",\n \"INTENSE_BLACK\": \"38;5;59\",\n \"INTENSE_BLUE\": \"38;5;61\",\n \"INTENSE_CYAN\": \"38;5;194\",\n \"INTENSE_GREEN\": \"38;5;34\",\n \"INTENSE_PURPLE\": \"38;5;188\",\n \"INTENSE_RED\": \"38;5;172\",\n \"INTENSE_WHITE\": \"38;5;15\",\n \"INTENSE_YELLOW\": \"38;5;188\",\n \"NO_COLOR\": \"0\",\n \"PURPLE\": \"38;5;125\",\n \"RED\": \"38;5;124\",\n \"WHITE\": \"38;5;145\",\n \"YELLOW\": \"38;5;130\",\n }\n _ansi_expand_style(style)\n return style\n\n\ndef _perldoc_style():\n style = {\n \"BLACK\": \"38;5;18\",\n \"BLUE\": \"38;5;18\",\n \"CYAN\": \"38;5;31\",\n \"GREEN\": \"38;5;34\",\n \"INTENSE_BLACK\": \"38;5;59\",\n \"INTENSE_BLUE\": \"38;5;134\",\n \"INTENSE_CYAN\": \"38;5;145\",\n \"INTENSE_GREEN\": \"38;5;28\",\n \"INTENSE_PURPLE\": \"38;5;134\",\n \"INTENSE_RED\": \"38;5;167\",\n \"INTENSE_WHITE\": \"38;5;188\",\n \"INTENSE_YELLOW\": \"38;5;188\",\n \"NO_COLOR\": \"0\",\n \"PURPLE\": \"38;5;90\",\n \"RED\": \"38;5;124\",\n \"WHITE\": \"38;5;145\",\n \"YELLOW\": \"38;5;166\",\n }\n _ansi_expand_style(style)\n return style\n\n\ndef _rrt_style():\n style = {\n \"BLACK\": \"38;5;09\",\n \"BLUE\": \"38;5;117\",\n \"CYAN\": \"38;5;117\",\n \"GREEN\": \"38;5;46\",\n \"INTENSE_BLACK\": \"38;5;117\",\n \"INTENSE_BLUE\": \"38;5;117\",\n \"INTENSE_CYAN\": \"38;5;122\",\n \"INTENSE_GREEN\": \"38;5;46\",\n \"INTENSE_PURPLE\": \"38;5;213\",\n \"INTENSE_RED\": \"38;5;09\",\n \"INTENSE_WHITE\": \"38;5;188\",\n \"INTENSE_YELLOW\": \"38;5;222\",\n \"NO_COLOR\": \"0\",\n \"PURPLE\": \"38;5;213\",\n \"RED\": \"38;5;09\",\n \"WHITE\": \"38;5;117\",\n \"YELLOW\": \"38;5;09\",\n }\n _ansi_expand_style(style)\n return style\n\n\ndef _tango_style():\n style = {\n \"BLACK\": \"38;5;16\",\n \"BLUE\": \"38;5;20\",\n \"CYAN\": \"38;5;61\",\n \"GREEN\": \"38;5;34\",\n \"INTENSE_BLACK\": \"38;5;24\",\n \"INTENSE_BLUE\": \"38;5;62\",\n \"INTENSE_CYAN\": \"38;5;15\",\n \"INTENSE_GREEN\": \"38;5;64\",\n \"INTENSE_PURPLE\": \"38;5;15\",\n \"INTENSE_RED\": \"38;5;09\",\n \"INTENSE_WHITE\": \"38;5;15\",\n \"INTENSE_YELLOW\": \"38;5;178\",\n \"NO_COLOR\": \"0\",\n \"PURPLE\": \"38;5;90\",\n \"RED\": \"38;5;124\",\n \"WHITE\": \"38;5;15\",\n \"YELLOW\": \"38;5;94\",\n }\n _ansi_expand_style(style)\n return style\n\n\ndef _trac_style():\n style = {\n \"BLACK\": \"38;5;16\",\n \"BLUE\": \"38;5;18\",\n \"CYAN\": \"38;5;30\",\n \"GREEN\": \"38;5;100\",\n \"INTENSE_BLACK\": \"38;5;59\",\n \"INTENSE_BLUE\": \"38;5;60\",\n \"INTENSE_CYAN\": \"38;5;194\",\n \"INTENSE_GREEN\": \"38;5;102\",\n \"INTENSE_PURPLE\": \"38;5;188\",\n \"INTENSE_RED\": \"38;5;137\",\n \"INTENSE_WHITE\": \"38;5;224\",\n \"INTENSE_YELLOW\": \"38;5;188\",\n \"NO_COLOR\": \"0\",\n \"PURPLE\": \"38;5;90\",\n \"RED\": \"38;5;124\",\n \"WHITE\": \"38;5;145\",\n \"YELLOW\": \"38;5;100\",\n }\n _ansi_expand_style(style)\n return style\n\n\ndef _vim_style():\n style = {\n \"BLACK\": \"38;5;18\",\n \"BLUE\": \"38;5;18\",\n \"CYAN\": \"38;5;44\",\n \"GREEN\": \"38;5;40\",\n \"INTENSE_BLACK\": \"38;5;60\",\n \"INTENSE_BLUE\": \"38;5;68\",\n \"INTENSE_CYAN\": \"38;5;44\",\n \"INTENSE_GREEN\": \"38;5;40\",\n \"INTENSE_PURPLE\": \"38;5;164\",\n \"INTENSE_RED\": \"38;5;09\",\n \"INTENSE_WHITE\": \"38;5;188\",\n \"INTENSE_YELLOW\": \"38;5;184\",\n \"NO_COLOR\": \"0\",\n \"PURPLE\": \"38;5;164\",\n \"RED\": \"38;5;160\",\n \"WHITE\": \"38;5;188\",\n \"YELLOW\": \"38;5;160\",\n }\n _ansi_expand_style(style)\n return style\n\n\ndef _vs_style():\n style = {\n \"BLACK\": \"38;5;28\",\n \"BLUE\": \"38;5;21\",\n \"CYAN\": \"38;5;31\",\n \"GREEN\": \"38;5;28\",\n \"INTENSE_BLACK\": \"38;5;31\",\n \"INTENSE_BLUE\": \"38;5;31\",\n \"INTENSE_CYAN\": \"38;5;31\",\n \"INTENSE_GREEN\": \"38;5;31\",\n \"INTENSE_PURPLE\": \"38;5;31\",\n \"INTENSE_RED\": \"38;5;09\",\n \"INTENSE_WHITE\": \"38;5;31\",\n \"INTENSE_YELLOW\": \"38;5;31\",\n \"NO_COLOR\": \"0\",\n \"PURPLE\": \"38;5;124\",\n \"RED\": \"38;5;124\",\n \"WHITE\": \"38;5;31\",\n \"YELLOW\": \"38;5;124\",\n }\n _ansi_expand_style(style)\n return style\n\n\ndef _xcode_style():\n style = {\n \"BLACK\": \"38;5;16\",\n \"BLUE\": \"38;5;20\",\n \"CYAN\": \"38;5;60\",\n \"GREEN\": \"38;5;28\",\n \"INTENSE_BLACK\": \"38;5;60\",\n \"INTENSE_BLUE\": \"38;5;20\",\n \"INTENSE_CYAN\": \"38;5;60\",\n \"INTENSE_GREEN\": \"38;5;60\",\n \"INTENSE_PURPLE\": \"38;5;126\",\n \"INTENSE_RED\": \"38;5;160\",\n \"INTENSE_WHITE\": \"38;5;60\",\n \"INTENSE_YELLOW\": \"38;5;94\",\n \"NO_COLOR\": \"0\",\n \"PURPLE\": \"38;5;126\",\n \"RED\": \"38;5;160\",\n \"WHITE\": \"38;5;60\",\n \"YELLOW\": \"38;5;94\",\n }\n _ansi_expand_style(style)\n return style\n\n\nANSI_STYLES = LazyDict(\n {\n \"algol\": _algol_style,\n \"algol_nu\": _algol_nu_style,\n \"autumn\": _autumn_style,\n \"borland\": _borland_style,\n \"bw\": _bw_style,\n \"colorful\": _colorful_style,\n \"default\": _default_style,\n \"emacs\": _emacs_style,\n \"friendly\": _friendly_style,\n \"fruity\": _fruity_style,\n \"igor\": _igor_style,\n \"lovelace\": _lovelace_style,\n \"manni\": _manni_style,\n \"monokai\": _monokai_style,\n \"murphy\": _murphy_style,\n \"native\": _native_style,\n \"paraiso-dark\": _paraiso_dark_style,\n \"paraiso-light\": _paraiso_light_style,\n \"pastie\": _pastie_style,\n \"perldoc\": _perldoc_style,\n \"rrt\": _rrt_style,\n \"tango\": _tango_style,\n \"trac\": _trac_style,\n \"vim\": _vim_style,\n \"vs\": _vs_style,\n \"xcode\": _xcode_style,\n },\n globals(),\n \"ANSI_STYLES\",\n)\n\ndel (\n _algol_style,\n _algol_nu_style,\n _autumn_style,\n _borland_style,\n _bw_style,\n _colorful_style,\n _default_style,\n _emacs_style,\n _friendly_style,\n _fruity_style,\n _igor_style,\n _lovelace_style,\n _manni_style,\n _monokai_style,\n _murphy_style,\n _native_style,\n _paraiso_dark_style,\n _paraiso_light_style,\n _pastie_style,\n _perldoc_style,\n _rrt_style,\n _tango_style,\n _trac_style,\n _vim_style,\n _vs_style,\n _xcode_style,\n)\n\n\n#\n# Dynamically generated styles\n#\ndef make_ansi_style(palette):\n \"\"\"Makes an ANSI color style from a color palette\"\"\"\n style = {\"NO_COLOR\": \"0\"}\n for name, t in BASE_XONSH_COLORS.items():\n closest = find_closest_color(t, palette)\n if len(closest) == 3:\n closest = \"\".join([a * 2 for a in closest])\n short = rgb2short(closest)[0]\n style[name] = \"38;5;\" + short\n style[\"BOLD_\" + name] = \"1;38;5;\" + short\n style[\"UNDERLINE_\" + name] = \"4;38;5;\" + short\n style[\"BOLD_UNDERLINE_\" + name] = \"1;4;38;5;\" + short\n style[\"BACKGROUND_\" + name] = \"48;5;\" + short\n return style\n\n\ndef ansi_style_by_name(name):\n \"\"\"Gets or makes an ANSI color style by name. If the styles does not\n exist, it will look for a style using the pygments name.\n \"\"\"\n if name in ANSI_STYLES:\n return ANSI_STYLES[name]\n elif not HAS_PYGMENTS:\n raise KeyError(\"could not find style {0!r}\".format(name))\n from xonsh.pygments_cache import get_style_by_name\n\n pstyle = get_style_by_name(name)\n palette = make_palette(pstyle.styles.values())\n astyle = make_ansi_style(palette)\n ANSI_STYLES[name] = astyle\n return astyle\n" }, { "path": "xonsh/ast.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"The xonsh abstract syntax tree node.\"\"\"\n# These are imported into our module namespace for the benefit of parser.py.\n# pylint: disable=unused-import\nimport sys\nimport builtins\nfrom ast import (\n Module,\n Num,\n Expr,\n Str,\n Bytes,\n UnaryOp,\n UAdd,\n USub,\n Invert,\n BinOp,\n Add,\n Sub,\n Mult,\n Div,\n FloorDiv,\n Mod,\n Pow,\n Compare,\n Lt,\n Gt,\n LtE,\n GtE,\n Eq,\n NotEq,\n In,\n NotIn,\n Is,\n IsNot,\n Not,\n BoolOp,\n Or,\n And,\n Subscript,\n Load,\n Slice,\n ExtSlice,\n List,\n Tuple,\n Set,\n Dict,\n AST,\n NameConstant,\n Name,\n GeneratorExp,\n Store,\n comprehension,\n ListComp,\n SetComp,\n DictComp,\n Assign,\n AugAssign,\n BitXor,\n BitAnd,\n BitOr,\n LShift,\n RShift,\n Assert,\n Delete,\n Del,\n Pass,\n Raise,\n Import,\n alias,\n ImportFrom,\n Continue,\n Break,\n Yield,\n YieldFrom,\n Return,\n IfExp,\n Lambda,\n arguments,\n arg,\n Call,\n keyword,\n Attribute,\n Global,\n Nonlocal,\n If,\n While,\n For,\n withitem,\n With,\n Try,\n ExceptHandler,\n FunctionDef,\n ClassDef,\n Starred,\n NodeTransformer,\n Interactive,\n Expression,\n Index,\n literal_eval,\n dump,\n walk,\n increment_lineno,\n)\nfrom ast import Ellipsis as EllipsisNode\n\n# pylint: enable=unused-import\nimport textwrap\nimport itertools\n\nfrom xonsh.tools import subproc_toks, find_next_break, get_logical_line\nfrom xonsh.platform import PYTHON_VERSION_INFO\n\nif PYTHON_VERSION_INFO >= (3, 5, 0):\n # pylint: disable=unused-import\n # pylint: disable=no-name-in-module\n from ast import MatMult, AsyncFunctionDef, AsyncWith, AsyncFor, Await\nelse:\n MatMult = AsyncFunctionDef = AsyncWith = AsyncFor = Await = None\n\nif PYTHON_VERSION_INFO >= (3, 6, 0):\n # pylint: disable=unused-import\n # pylint: disable=no-name-in-module\n from ast import JoinedStr, FormattedValue, AnnAssign\nelse:\n JoinedStr = FormattedValue = AnnAssign = None\n\nSTATEMENTS = (\n FunctionDef,\n ClassDef,\n Return,\n Delete,\n Assign,\n AugAssign,\n For,\n While,\n If,\n With,\n Raise,\n Try,\n Assert,\n Import,\n ImportFrom,\n Global,\n Nonlocal,\n Expr,\n Pass,\n Break,\n Continue,\n)\nif PYTHON_VERSION_INFO >= (3, 6, 0):\n STATEMENTS += (AnnAssign,)\n\n\ndef leftmostname(node):\n \"\"\"Attempts to find the first name in the tree.\"\"\"\n if isinstance(node, Name):\n rtn = node.id\n elif isinstance(node, (BinOp, Compare)):\n rtn = leftmostname(node.left)\n elif isinstance(node, (Attribute, Subscript, Starred, Expr)):\n rtn = leftmostname(node.value)\n elif isinstance(node, Call):\n rtn = leftmostname(node.func)\n elif isinstance(node, UnaryOp):\n rtn = leftmostname(node.operand)\n elif isinstance(node, BoolOp):\n rtn = leftmostname(node.values[0])\n elif isinstance(node, (Assign, AnnAssign)):\n rtn = leftmostname(node.targets[0])\n elif isinstance(node, (Str, Bytes, JoinedStr)):\n # handles case of \"./my executable\"\n rtn = leftmostname(node.s)\n elif isinstance(node, Tuple) and len(node.elts) > 0:\n # handles case of echo ,1,2,3\n rtn = leftmostname(node.elts[0])\n else:\n rtn = None\n return rtn\n\n\ndef get_lineno(node, default=0):\n \"\"\"Gets the lineno of a node or returns the default.\"\"\"\n return getattr(node, \"lineno\", default)\n\n\ndef min_line(node):\n \"\"\"Computes the minimum lineno.\"\"\"\n node_line = get_lineno(node)\n return min(map(get_lineno, walk(node), itertools.repeat(node_line)))\n\n\ndef max_line(node):\n \"\"\"Computes the maximum lineno.\"\"\"\n return max(map(get_lineno, walk(node)))\n\n\ndef get_col(node, default=-1):\n \"\"\"Gets the col_offset of a node, or returns the default\"\"\"\n return getattr(node, \"col_offset\", default)\n\n\ndef min_col(node):\n \"\"\"Computes the minimum col_offset.\"\"\"\n return min(map(get_col, walk(node), itertools.repeat(node.col_offset)))\n\n\ndef max_col(node):\n \"\"\"Returns the maximum col_offset of the node and all sub-nodes.\"\"\"\n col = getattr(node, \"max_col\", None)\n if col is not None:\n return col\n highest = max(walk(node), key=get_col)\n col = highest.col_offset + node_len(highest)\n return col\n\n\ndef node_len(node):\n \"\"\"The length of a node as a string\"\"\"\n val = 0\n for n in walk(node):\n if isinstance(n, Name):\n val += len(n.id)\n elif isinstance(n, Attribute):\n val += 1 + (len(n.attr) if isinstance(n.attr, str) else 0)\n # this may need to be added to for more nodes as more cases are found\n return val\n\n\ndef get_id(node, default=None):\n \"\"\"Gets the id attribute of a node, or returns a default.\"\"\"\n return getattr(node, \"id\", default)\n\n\ndef gather_names(node):\n \"\"\"Returns the set of all names present in the node's tree.\"\"\"\n rtn = set(map(get_id, walk(node)))\n rtn.discard(None)\n return rtn\n\n\ndef get_id_ctx(node):\n \"\"\"Gets the id and attribute of a node, or returns a default.\"\"\"\n nid = getattr(node, \"id\", None)\n if nid is None:\n return (None, None)\n return (nid, node.ctx)\n\n\ndef gather_load_store_names(node):\n \"\"\"Returns the names present in the node's tree in a set of load nodes and\n a set of store nodes.\n \"\"\"\n load = set()\n store = set()\n for nid, ctx in map(get_id_ctx, walk(node)):\n if nid is None:\n continue\n elif isinstance(ctx, Load):\n load.add(nid)\n else:\n store.add(nid)\n return (load, store)\n\n\ndef has_elts(x):\n \"\"\"Tests if x is an AST node with elements.\"\"\"\n return isinstance(x, AST) and hasattr(x, \"elts\")\n\n\ndef load_attribute_chain(name, lineno=None, col=None):\n \"\"\"Creates an AST that loads variable name that may (or may not)\n have attribute chains. For example, \"a.b.c\"\n \"\"\"\n names = name.split(\".\")\n node = Name(id=names.pop(0), ctx=Load(), lineno=lineno, col_offset=col)\n for attr in names:\n node = Attribute(\n value=node, attr=attr, ctx=Load(), lineno=lineno, col_offset=col\n )\n return node\n\n\ndef xonsh_call(name, args, lineno=None, col=None):\n \"\"\"Creates the AST node for calling a function of a given name.\n Functions names may contain attribute access, e.g. __xonsh__.env.\n \"\"\"\n return Call(\n func=load_attribute_chain(name, lineno=lineno, col=col),\n args=args,\n keywords=[],\n starargs=None,\n kwargs=None,\n lineno=lineno,\n col_offset=col,\n )\n\n\ndef isdescendable(node):\n \"\"\"Determines whether or not a node is worth visiting. Currently only\n UnaryOp and BoolOp nodes are visited.\n \"\"\"\n return isinstance(node, (UnaryOp, BoolOp))\n\n\ndef isexpression(node, ctx=None, *args, **kwargs):\n \"\"\"Determines whether a node (or code string) is an expression, and\n does not contain any statements. The execution context (ctx) and\n other args and kwargs are passed down to the parser, as needed.\n \"\"\"\n # parse string to AST\n if isinstance(node, str):\n node = node if node.endswith(\"\\n\") else node + \"\\n\"\n ctx = builtins.__xonsh__.ctx if ctx is None else ctx\n node = builtins.__xonsh__.execer.parse(node, ctx, *args, **kwargs)\n # determin if expresission-like enough\n if isinstance(node, (Expr, Expression)):\n isexpr = True\n elif isinstance(node, Module) and len(node.body) == 1:\n isexpr = isinstance(node.body[0], (Expr, Expression))\n else:\n isexpr = False\n return isexpr\n\n\nclass CtxAwareTransformer(NodeTransformer):\n \"\"\"Transforms a xonsh AST based to use subprocess calls when\n the first name in an expression statement is not known in the context.\n This assumes that the expression statement is instead parseable as\n a subprocess.\n \"\"\"\n\n def __init__(self, parser):\n \"\"\"Parameters\n ----------\n parser : xonsh.Parser\n A parse instance to try to parse subprocess statements with.\n \"\"\"\n super(CtxAwareTransformer, self).__init__()\n self.parser = parser\n self.input = None\n self.contexts = []\n self.lines = None\n self.mode = None\n self._nwith = 0\n self.filename = \"\"\n self.debug_level = 0\n\n def ctxvisit(self, node, inp, ctx, mode=\"exec\", filename=None, debug_level=0):\n \"\"\"Transforms the node in a context-dependent way.\n\n Parameters\n ----------\n node : ast.AST\n A syntax tree to transform.\n input : str\n The input code in string format.\n ctx : dict\n The root context to use.\n filename : str, optional\n File we are to transform.\n debug_level : int, optional\n Debugging level to use in lexing and parsing.\n\n Returns\n -------\n node : ast.AST\n The transformed node.\n \"\"\"\n self.filename = self.filename if filename is None else filename\n self.debug_level = debug_level\n self.lines = inp.splitlines()\n self.contexts = [ctx, set()]\n self.mode = mode\n self._nwith = 0\n node = self.visit(node)\n del self.lines, self.contexts, self.mode\n self._nwith = 0\n return node\n\n def ctxupdate(self, iterable):\n \"\"\"Updated the most recent context.\"\"\"\n self.contexts[-1].update(iterable)\n\n def ctxadd(self, value):\n \"\"\"Adds a value the most recent context.\"\"\"\n self.contexts[-1].add(value)\n\n def ctxremove(self, value):\n \"\"\"Removes a value the most recent context.\"\"\"\n for ctx in reversed(self.contexts):\n if value in ctx:\n ctx.remove(value)\n break\n\n def try_subproc_toks(self, node, strip_expr=False):\n \"\"\"Tries to parse the line of the node as a subprocess.\"\"\"\n line, nlogical, idx = get_logical_line(self.lines, node.lineno - 1)\n if self.mode == \"eval\":\n mincol = len(line) - len(line.lstrip())\n maxcol = None\n else:\n mincol = max(min_col(node) - 1, 0)\n maxcol = max_col(node)\n if mincol == maxcol:\n maxcol = find_next_break(line, mincol=mincol, lexer=self.parser.lexer)\n elif nlogical > 1:\n maxcol = None\n elif maxcol < len(line) and line[maxcol] == \";\":\n pass\n else:\n maxcol += 1\n spline = subproc_toks(\n line,\n mincol=mincol,\n maxcol=maxcol,\n returnline=False,\n lexer=self.parser.lexer,\n )\n if spline is None or spline != \"![{}]\".format(line[mincol:maxcol].strip()):\n # failed to get something consistent, try greedy wrap\n spline = subproc_toks(\n line,\n mincol=mincol,\n maxcol=maxcol,\n returnline=False,\n lexer=self.parser.lexer,\n greedy=True,\n )\n if spline is None:\n return node\n try:\n newnode = self.parser.parse(\n spline,\n mode=self.mode,\n filename=self.filename,\n debug_level=(self.debug_level > 2),\n )\n newnode = newnode.body\n if not isinstance(newnode, AST):\n # take the first (and only) Expr\n newnode = newnode[0]\n increment_lineno(newnode, n=node.lineno - 1)\n newnode.col_offset = node.col_offset\n if self.debug_level > 1:\n msg = \"{0}:{1}:{2}{3} - {4}\\n\" \"{0}:{1}:{2}{3} + {5}\"\n mstr = \"\" if maxcol is None else \":\" + str(maxcol)\n msg = msg.format(self.filename, node.lineno, mincol, mstr, line, spline)\n print(msg, file=sys.stderr)\n except SyntaxError:\n newnode = node\n if strip_expr and isinstance(newnode, Expr):\n newnode = newnode.value\n return newnode\n\n def is_in_scope(self, node):\n \"\"\"Determines whether or not the current node is in scope.\"\"\"\n names, store = gather_load_store_names(node)\n names -= store\n if not names:\n return True\n inscope = False\n for ctx in reversed(self.contexts):\n names -= ctx\n if not names:\n inscope = True\n break\n return inscope\n\n #\n # Replacement visitors\n #\n\n def visit_Expression(self, node):\n \"\"\"Handle visiting an expression body.\"\"\"\n if isdescendable(node.body):\n node.body = self.visit(node.body)\n body = node.body\n inscope = self.is_in_scope(body)\n if not inscope:\n node.body = self.try_subproc_toks(body)\n return node\n\n def visit_Expr(self, node):\n \"\"\"Handle visiting an expression.\"\"\"\n if isdescendable(node.value):\n node.value = self.visit(node.value) # this allows diving into BoolOps\n if self.is_in_scope(node) or isinstance(node.value, Lambda):\n return node\n else:\n newnode = self.try_subproc_toks(node)\n if not isinstance(newnode, Expr):\n newnode = Expr(\n value=newnode, lineno=node.lineno, col_offset=node.col_offset\n )\n if hasattr(node, \"max_lineno\"):\n newnode.max_lineno = node.max_lineno\n newnode.max_col = node.max_col\n return newnode\n\n def visit_UnaryOp(self, node):\n \"\"\"Handle visiting an unary operands, like not.\"\"\"\n if isdescendable(node.operand):\n node.operand = self.visit(node.operand)\n operand = node.operand\n inscope = self.is_in_scope(operand)\n if not inscope:\n node.operand = self.try_subproc_toks(operand, strip_expr=True)\n return node\n\n def visit_BoolOp(self, node):\n \"\"\"Handle visiting an boolean operands, like and/or.\"\"\"\n for i in range(len(node.values)):\n val = node.values[i]\n if isdescendable(val):\n val = node.values[i] = self.visit(val)\n inscope = self.is_in_scope(val)\n if not inscope:\n node.values[i] = self.try_subproc_toks(val, strip_expr=True)\n return node\n\n #\n # Context aggregator visitors\n #\n\n def visit_Assign(self, node):\n \"\"\"Handle visiting an assignment statement.\"\"\"\n ups = set()\n for targ in node.targets:\n if isinstance(targ, (Tuple, List)):\n ups.update(leftmostname(elt) for elt in targ.elts)\n elif isinstance(targ, BinOp):\n newnode = self.try_subproc_toks(node)\n if newnode is node:\n ups.add(leftmostname(targ))\n else:\n return newnode\n else:\n ups.add(leftmostname(targ))\n self.ctxupdate(ups)\n return node\n\n visit_AnnAssign = visit_Assign\n\n def visit_Import(self, node):\n \"\"\"Handle visiting a import statement.\"\"\"\n for name in node.names:\n if name.asname is None:\n self.ctxadd(name.name)\n else:\n self.ctxadd(name.asname)\n return node\n\n def visit_ImportFrom(self, node):\n \"\"\"Handle visiting a \"from ... import ...\" statement.\"\"\"\n for name in node.names:\n if name.asname is None:\n self.ctxadd(name.name)\n else:\n self.ctxadd(name.asname)\n return node\n\n def visit_With(self, node):\n \"\"\"Handle visiting a with statement.\"\"\"\n for item in node.items:\n if item.optional_vars is not None:\n self.ctxupdate(gather_names(item.optional_vars))\n self._nwith += 1\n self.generic_visit(node)\n self._nwith -= 1\n return node\n\n def visit_For(self, node):\n \"\"\"Handle visiting a for statement.\"\"\"\n targ = node.target\n self.ctxupdate(gather_names(targ))\n self.generic_visit(node)\n return node\n\n def visit_FunctionDef(self, node):\n \"\"\"Handle visiting a function definition.\"\"\"\n self.ctxadd(node.name)\n self.contexts.append(set())\n args = node.args\n argchain = [args.args, args.kwonlyargs]\n if args.vararg is not None:\n argchain.append((args.vararg,))\n if args.kwarg is not None:\n argchain.append((args.kwarg,))\n self.ctxupdate(a.arg for a in itertools.chain.from_iterable(argchain))\n self.generic_visit(node)\n self.contexts.pop()\n return node\n\n def visit_ClassDef(self, node):\n \"\"\"Handle visiting a class definition.\"\"\"\n self.ctxadd(node.name)\n self.contexts.append(set())\n self.generic_visit(node)\n self.contexts.pop()\n return node\n\n def visit_Delete(self, node):\n \"\"\"Handle visiting a del statement.\"\"\"\n for targ in node.targets:\n if isinstance(targ, Name):\n self.ctxremove(targ.id)\n self.generic_visit(node)\n return node\n\n def visit_Try(self, node):\n \"\"\"Handle visiting a try statement.\"\"\"\n for handler in node.handlers:\n if handler.name is not None:\n self.ctxadd(handler.name)\n self.generic_visit(node)\n return node\n\n def visit_Global(self, node):\n \"\"\"Handle visiting a global statement.\"\"\"\n self.contexts[1].update(node.names) # contexts[1] is the global ctx\n self.generic_visit(node)\n return node\n\n\ndef pdump(s, **kwargs):\n \"\"\"performs a pretty dump of an AST node.\"\"\"\n if isinstance(s, AST):\n s = dump(s, **kwargs).replace(\",\", \",\\n\")\n openers = \"([{\"\n closers = \")]}\"\n lens = len(s) + 1\n if lens == 1:\n return s\n i = min([s.find(o) % lens for o in openers])\n if i == lens - 1:\n return s\n closer = closers[openers.find(s[i])]\n j = s.rfind(closer)\n if j == -1 or j <= i:\n return s[: i + 1] + \"\\n\" + textwrap.indent(pdump(s[i + 1 :]), \" \")\n pre = s[: i + 1] + \"\\n\"\n mid = s[i + 1 : j]\n post = \"\\n\" + s[j:]\n mid = textwrap.indent(pdump(mid), \" \")\n if \"(\" in post or \"[\" in post or \"{\" in post:\n post = pdump(post)\n return pre + mid + post\n\n\ndef pprint_ast(s, *, sep=None, end=None, file=None, flush=False, **kwargs):\n \"\"\"Performs a pretty print of the AST nodes.\"\"\"\n print(pdump(s, **kwargs), sep=sep, end=end, file=file, flush=flush)\n\n\n#\n# Private helpers\n#\n\n\ndef _getblockattr(name, lineno, col):\n \"\"\"calls getattr(name, '__xonsh_block__', False).\"\"\"\n return xonsh_call(\n \"getattr\",\n args=[\n Name(id=name, ctx=Load(), lineno=lineno, col_offset=col),\n Str(s=\"__xonsh_block__\", lineno=lineno, col_offset=col),\n NameConstant(value=False, lineno=lineno, col_offset=col),\n ],\n lineno=lineno,\n col=col,\n )\n" }, { "path": "xonsh/base_shell.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"The base class for xonsh shell\"\"\"\nimport io\nimport os\nimport sys\nimport time\nimport builtins\n\nfrom xonsh.tools import (\n XonshError,\n print_exception,\n DefaultNotGiven,\n check_for_partial_string,\n format_std_prepost,\n get_line_continuation,\n)\nfrom xonsh.platform import HAS_PYGMENTS, ON_WINDOWS\nfrom xonsh.codecache import (\n should_use_cache,\n code_cache_name,\n code_cache_check,\n get_cache_filename,\n update_cache,\n run_compiled_code,\n)\nfrom xonsh.completer import Completer\nfrom xonsh.prompt.base import multiline_prompt, PromptFormatter\nfrom xonsh.events import events\nfrom xonsh.shell import transform_command\nfrom xonsh.lazyimps import pygments, pyghooks\nfrom xonsh.ansi_colors import ansi_partial_color_format\n\nif ON_WINDOWS:\n import ctypes\n\n kernel32 = ctypes.windll.kernel32\n kernel32.SetConsoleTitleW.argtypes = [ctypes.c_wchar_p]\n\n\nclass _TeeStdBuf(io.RawIOBase):\n \"\"\"A dispatcher for bytes to two buffers, as std stream buffer and an\n in memory buffer.\n \"\"\"\n\n def __init__(\n self, stdbuf, membuf, encoding=None, errors=None, prestd=b\"\", poststd=b\"\"\n ):\n \"\"\"\n Parameters\n ----------\n stdbuf : BytesIO-like or StringIO-like\n The std stream buffer.\n membuf : BytesIO-like\n The in memory stream buffer.\n encoding : str or None, optional\n The encoding of the stream. Only used if stdbuf is a text stream,\n rather than a binary one. Defaults to $XONSH_ENCODING if None.\n errors : str or None, optional\n The error form for the encoding of the stream. Only used if stdbuf\n is a text stream, rather than a binary one. Deafults to\n $XONSH_ENCODING_ERRORS if None.\n prestd : bytes, optional\n The prefix to prepend to the standard buffer.\n poststd : bytes, optional\n The postfix to append to the standard buffer.\n \"\"\"\n self.stdbuf = stdbuf\n self.membuf = membuf\n env = builtins.__xonsh__.env\n self.encoding = env.get(\"XONSH_ENCODING\") if encoding is None else encoding\n self.errors = env.get(\"XONSH_ENCODING_ERRORS\") if errors is None else errors\n self.prestd = prestd\n self.poststd = poststd\n self._std_is_binary = not hasattr(stdbuf, \"encoding\")\n\n def fileno(self):\n \"\"\"Returns the file descriptor of the std buffer.\"\"\"\n return self.stdbuf.fileno()\n\n def seek(self, offset, whence=io.SEEK_SET):\n \"\"\"Sets the location in both the stdbuf and the membuf.\"\"\"\n self.stdbuf.seek(offset, whence)\n self.membuf.seek(offset, whence)\n\n def truncate(self, size=None):\n \"\"\"Truncate both buffers.\"\"\"\n self.stdbuf.truncate(size)\n self.membuf.truncate(size)\n\n def readinto(self, b):\n \"\"\"Read bytes into buffer from both streams.\"\"\"\n if self._std_is_binary:\n self.stdbuf.readinto(b)\n return self.membuf.readinto(b)\n\n def write(self, b):\n \"\"\"Write bytes into both buffers.\"\"\"\n std_b = b\n if self.prestd:\n std_b = self.prestd + b\n if self.poststd:\n std_b += self.poststd\n # write to stdbuf\n if self._std_is_binary:\n self.stdbuf.write(std_b)\n else:\n self.stdbuf.write(std_b.decode(encoding=self.encoding, errors=self.errors))\n return self.membuf.write(b)\n\n\nclass _TeeStd(io.TextIOBase):\n \"\"\"Tees a std stream into an in-memory container and the original stream.\"\"\"\n\n def __init__(self, name, mem, prestd=\"\", poststd=\"\"):\n \"\"\"\n Parameters\n ----------\n name : str\n The name of the buffer in the sys module, e.g. 'stdout'.\n mem : io.TextIOBase-like\n The in-memory text-based representation.\n prestd : str, optional\n The prefix to prepend to the standard stream.\n poststd : str, optional\n The postfix to append to the standard stream.\n \"\"\"\n self._name = name\n self.std = std = getattr(sys, name)\n self.mem = mem\n self.prestd = prestd\n self.poststd = poststd\n preb = prestd.encode(encoding=mem.encoding, errors=mem.errors)\n postb = poststd.encode(encoding=mem.encoding, errors=mem.errors)\n if hasattr(std, \"buffer\"):\n buffer = _TeeStdBuf(std.buffer, mem.buffer, prestd=preb, poststd=postb)\n else:\n # TextIO does not have buffer as part of the API, so std streams\n # may not either.\n buffer = _TeeStdBuf(\n std,\n mem.buffer,\n encoding=mem.encoding,\n errors=mem.errors,\n prestd=preb,\n poststd=postb,\n )\n self.buffer = buffer\n setattr(sys, name, self)\n\n @property\n def encoding(self):\n \"\"\"The encoding of the in-memory buffer.\"\"\"\n return self.mem.encoding\n\n @property\n def errors(self):\n \"\"\"The errors of the in-memory buffer.\"\"\"\n return self.mem.errors\n\n @property\n def newlines(self):\n \"\"\"The newlines of the in-memory buffer.\"\"\"\n return self.mem.newlines\n\n def _replace_std(self):\n std = self.std\n if std is None:\n return\n setattr(sys, self._name, std)\n self.std = self._name = None\n\n def __del__(self):\n self._replace_std()\n\n def close(self):\n \"\"\"Restores the original std stream.\"\"\"\n self._replace_std()\n\n def write(self, s):\n \"\"\"Writes data to the original std stream and the in-memory object.\"\"\"\n self.mem.write(s)\n if self.std is None:\n return\n std_s = s\n if self.prestd:\n std_s = self.prestd + std_s\n if self.poststd:\n std_s += self.poststd\n self.std.write(std_s)\n\n def flush(self):\n \"\"\"Flushes both the original stdout and the buffer.\"\"\"\n self.std.flush()\n self.mem.flush()\n\n def fileno(self):\n \"\"\"Tunnel fileno() calls to the std stream.\"\"\"\n return self.std.fileno()\n\n def seek(self, offset, whence=io.SEEK_SET):\n \"\"\"Seek to a location in both streams.\"\"\"\n self.std.seek(offset, whence)\n self.mem.seek(offset, whence)\n\n def truncate(self, size=None):\n \"\"\"Seek to a location in both streams.\"\"\"\n self.std.truncate(size)\n self.mem.truncate(size)\n\n def detach(self):\n \"\"\"This operation is not supported.\"\"\"\n raise io.UnsupportedOperation\n\n def read(self, size=None):\n \"\"\"Read from the in-memory stream and seek to a new location in the\n std stream.\n \"\"\"\n s = self.mem.read(size)\n loc = self.std.tell()\n self.std.seek(loc + len(s))\n return s\n\n def readline(self, size=-1):\n \"\"\"Read a line from the in-memory stream and seek to a new location\n in the std stream.\n \"\"\"\n s = self.mem.readline(size)\n loc = self.std.tell()\n self.std.seek(loc + len(s))\n return s\n\n\nclass Tee:\n \"\"\"Class that merges tee'd stdout and stderr into a single stream.\n\n This represents what a user would actually see on the command line.\n This class has the same interface as io.TextIOWrapper, except that\n the buffer is optional.\n \"\"\"\n\n # pylint is a stupid about counting public methods when using inheritance.\n # pylint: disable=too-few-public-methods\n\n def __init__(\n self,\n buffer=None,\n encoding=None,\n errors=None,\n newline=None,\n line_buffering=False,\n write_through=False,\n ):\n self.buffer = io.BytesIO() if buffer is None else buffer\n self.memory = io.TextIOWrapper(\n self.buffer,\n encoding=encoding,\n errors=errors,\n newline=newline,\n line_buffering=line_buffering,\n write_through=write_through,\n )\n self.stdout = _TeeStd(\"stdout\", self.memory)\n env = builtins.__xonsh__.env\n prestderr = format_std_prepost(env.get(\"XONSH_STDERR_PREFIX\"))\n poststderr = format_std_prepost(env.get(\"XONSH_STDERR_POSTFIX\"))\n self.stderr = _TeeStd(\n \"stderr\", self.memory, prestd=prestderr, poststd=poststderr\n )\n\n @property\n def line_buffering(self):\n return self.memory.line_buffering\n\n def __del__(self):\n del self.stdout, self.stderr\n self.stdout = self.stderr = None\n\n def close(self):\n \"\"\"Closes the buffer as well as the stdout and stderr tees.\"\"\"\n self.stdout.close()\n self.stderr.close()\n self.memory.close()\n\n def getvalue(self):\n \"\"\"Gets the current contents of the in-memory buffer.\"\"\"\n m = self.memory\n loc = m.tell()\n m.seek(0)\n s = m.read()\n m.seek(loc)\n return s\n\n\nclass BaseShell(object):\n \"\"\"The xonsh shell.\"\"\"\n\n def __init__(self, execer, ctx, **kwargs):\n super().__init__()\n self.execer = execer\n self.ctx = ctx\n self.completer = Completer() if kwargs.get(\"completer\", True) else None\n self.buffer = []\n self.need_more_lines = False\n self.mlprompt = None\n self._styler = DefaultNotGiven\n self.prompt_formatter = PromptFormatter()\n self.accumulated_inputs = \"\"\n\n @property\n def styler(self):\n if self._styler is DefaultNotGiven:\n if HAS_PYGMENTS:\n from xonsh.pyghooks import XonshStyle\n\n env = builtins.__xonsh__.env\n self._styler = XonshStyle(env.get(\"XONSH_COLOR_STYLE\"))\n else:\n self._styler = None\n return self._styler\n\n @styler.setter\n def styler(self, value):\n self._styler = value\n\n @styler.deleter\n def styler(self):\n self._styler = DefaultNotGiven\n\n def emptyline(self):\n \"\"\"Called when an empty line has been entered.\"\"\"\n self.need_more_lines = False\n self.default(\"\")\n\n def singleline(self, **kwargs):\n \"\"\"Reads a single line of input from the shell.\"\"\"\n msg = \"{0} has not implemented singleline().\"\n raise RuntimeError(msg.format(self.__class__.__name__))\n\n def precmd(self, line):\n \"\"\"Called just before execution of line.\"\"\"\n return line if self.need_more_lines else line.lstrip()\n\n def default(self, line):\n \"\"\"Implements code execution.\"\"\"\n line = line if line.endswith(\"\\n\") else line + \"\\n\"\n src, code = self.push(line)\n if code is None:\n return\n\n events.on_precommand.fire(cmd=src)\n\n env = builtins.__xonsh__.env\n hist = builtins.__xonsh__.history # pylint: disable=no-member\n ts1 = None\n enc = env.get(\"XONSH_ENCODING\")\n err = env.get(\"XONSH_ENCODING_ERRORS\")\n tee = Tee(encoding=enc, errors=err)\n try:\n ts0 = time.time()\n run_compiled_code(code, self.ctx, None, \"single\")\n ts1 = time.time()\n if hist is not None and hist.last_cmd_rtn is None:\n hist.last_cmd_rtn = 0 # returncode for success\n except XonshError as e:\n print(e.args[0], file=sys.stderr)\n if hist is not None and hist.last_cmd_rtn is None:\n hist.last_cmd_rtn = 1 # return code for failure\n except Exception: # pylint: disable=broad-except\n print_exception()\n if hist is not None and hist.last_cmd_rtn is None:\n hist.last_cmd_rtn = 1 # return code for failure\n finally:\n ts1 = ts1 or time.time()\n tee_out = tee.getvalue()\n self._append_history(inp=src, ts=[ts0, ts1], tee_out=tee_out)\n self.accumulated_inputs += src\n if (\n tee_out\n and env.get(\"XONSH_APPEND_NEWLINE\")\n and not tee_out.endswith(os.linesep)\n ):\n print(os.linesep, end=\"\")\n tee.close()\n self._fix_cwd()\n if builtins.__xonsh__.exit: # pylint: disable=no-member\n return True\n\n def _append_history(self, tee_out=None, **info):\n \"\"\"Append information about the command to the history.\n\n This also handles on_postcommand because this is the place where all the\n information is available.\n \"\"\"\n hist = builtins.__xonsh__.history # pylint: disable=no-member\n info[\"rtn\"] = hist.last_cmd_rtn if hist is not None else None\n tee_out = tee_out or None\n last_out = hist.last_cmd_out if hist is not None else None\n if last_out is None and tee_out is None:\n pass\n elif last_out is None and tee_out is not None:\n info[\"out\"] = tee_out\n elif last_out is not None and tee_out is None:\n info[\"out\"] = last_out\n else:\n info[\"out\"] = tee_out + \"\\n\" + last_out\n events.on_postcommand.fire(\n cmd=info[\"inp\"], rtn=info[\"rtn\"], out=info.get(\"out\", None), ts=info[\"ts\"]\n )\n if hist is not None:\n hist.append(info)\n hist.last_cmd_rtn = hist.last_cmd_out = None\n\n def _fix_cwd(self):\n \"\"\"Check if the cwd changed out from under us.\"\"\"\n env = builtins.__xonsh__.env\n try:\n cwd = os.getcwd()\n except (FileNotFoundError, OSError):\n cwd = None\n if cwd is None:\n # directory has been deleted out from under us, most likely\n pwd = env.get(\"PWD\", None)\n if pwd is None:\n # we have no idea where we are\n env[\"PWD\"] = \"\"\n elif os.path.isdir(pwd):\n # unclear why os.getcwd() failed. do nothing.\n pass\n else:\n # OK PWD is really gone.\n msg = \"{UNDERLINE_INTENSE_WHITE}{BACKGROUND_INTENSE_BLACK}\"\n msg += \"xonsh: working directory does not exist: \" + pwd\n msg += \"{NO_COLOR}\"\n self.print_color(msg, file=sys.stderr)\n elif \"PWD\" not in env:\n # $PWD is missing from env, recreate it\n env[\"PWD\"] = cwd\n elif os.path.realpath(cwd) != os.path.realpath(env[\"PWD\"]):\n # The working directory has changed without updating $PWD, fix this\n old = env[\"PWD\"]\n env[\"PWD\"] = cwd\n env[\"OLDPWD\"] = old\n events.on_chdir.fire(olddir=old, newdir=cwd)\n\n def push(self, line):\n \"\"\"Pushes a line onto the buffer and compiles the code in a way that\n enables multiline input.\n \"\"\"\n self.buffer.append(line)\n if self.need_more_lines:\n return None, None\n src = \"\".join(self.buffer)\n src = transform_command(src)\n return self.compile(src)\n\n def compile(self, src):\n \"\"\"Compiles source code and returns the (possibly modified) source and\n a valid code object.\n \"\"\"\n _cache = should_use_cache(self.execer, \"single\")\n if _cache:\n codefname = code_cache_name(src)\n cachefname = get_cache_filename(codefname, code=True)\n usecache, code = code_cache_check(cachefname)\n if usecache:\n self.reset_buffer()\n return src, code\n lincont = get_line_continuation()\n if src.endswith(lincont + \"\\n\"):\n self.need_more_lines = True\n return src, None\n try:\n code = self.execer.compile(src, mode=\"single\", glbs=self.ctx, locs=None)\n if _cache:\n update_cache(code, cachefname)\n self.reset_buffer()\n except SyntaxError:\n partial_string_info = check_for_partial_string(src)\n in_partial_string = (\n partial_string_info[0] is not None and partial_string_info[1] is None\n )\n if (src == \"\\n\" or src.endswith(\"\\n\\n\")) and not in_partial_string:\n self.reset_buffer()\n print_exception()\n return src, None\n self.need_more_lines = True\n code = None\n except Exception: # pylint: disable=broad-except\n self.reset_buffer()\n print_exception()\n code = None\n return src, code\n\n def reset_buffer(self):\n \"\"\"Resets the line buffer.\"\"\"\n self.buffer.clear()\n self.need_more_lines = False\n self.mlprompt = None\n\n def settitle(self):\n \"\"\"Sets terminal title.\"\"\"\n env = builtins.__xonsh__.env # pylint: disable=no-member\n term = env.get(\"TERM\", None)\n # Shells running in emacs sets TERM to \"dumb\" or \"eterm-color\".\n # Do not set title for these to avoid garbled prompt.\n if (term is None and not ON_WINDOWS) or term in [\n \"dumb\",\n \"eterm-color\",\n \"linux\",\n ]:\n return\n t = 'gitsome'\n if t is None:\n return\n t = self.prompt_formatter(t)\n if ON_WINDOWS and \"ANSICON\" not in env:\n kernel32.SetConsoleTitleW(t)\n else:\n with open(1, \"wb\", closefd=False) as f:\n # prevent xonsh from answering interactive questions\n # on the next command by writing the title\n f.write(\"\\x1b]0;{0}\\x07\".format(t).encode())\n f.flush()\n\n @property\n def prompt(self):\n \"\"\"Obtains the current prompt string.\"\"\"\n if self.need_more_lines:\n if self.mlprompt is None:\n try:\n self.mlprompt = multiline_prompt()\n except Exception: # pylint: disable=broad-except\n print_exception()\n self.mlprompt = \" \"\n return self.mlprompt\n env = builtins.__xonsh__.env # pylint: disable=no-member\n p = env.get(\"PROMPT\")\n try:\n p = self.prompt_formatter(p)\n except Exception: # pylint: disable=broad-except\n print_exception()\n self.settitle()\n return p\n\n def format_color(self, string, hide=False, force_string=False, **kwargs):\n \"\"\"Formats the colors in a string. ``BaseShell``'s default implementation\n of this method uses colors based on ANSI color codes.\n \"\"\"\n style = builtins.__xonsh__.env.get(\"XONSH_COLOR_STYLE\")\n return ansi_partial_color_format(string, hide=hide, style=style)\n\n def print_color(self, string, hide=False, **kwargs):\n \"\"\"Prints a string in color. This base implementation's colors are based\n on ANSI color codes if a string was given as input. If a list of token\n pairs is given, it will color based on pygments, if available. If\n pygments is not available, it will print a colorless string.\n \"\"\"\n if isinstance(string, str):\n s = self.format_color(string, hide=hide)\n elif HAS_PYGMENTS:\n # assume this is a list of (Token, str) tuples and format it\n env = builtins.__xonsh__.env\n self.styler.style_name = env.get(\"XONSH_COLOR_STYLE\")\n style_proxy = pyghooks.xonsh_style_proxy(self.styler)\n formatter = pyghooks.XonshTerminal256Formatter(style=style_proxy)\n s = pygments.format(string, formatter).rstrip()\n else:\n # assume this is a list of (Token, str) tuples and remove color\n s = \"\".join([x for _, x in string])\n print(s, **kwargs)\n\n def color_style_names(self):\n \"\"\"Returns an iterable of all available style names.\"\"\"\n return ()\n\n def color_style(self):\n \"\"\"Returns the current color map.\"\"\"\n return {}\n\n def restore_tty_sanity(self):\n \"\"\"An interface for resetting the TTY stdin mode. This is highly\n dependent on the shell backend. Also it is mostly optional since\n it only affects ^Z backgrounding behaviour.\n \"\"\"\n pass\n" }, { "path": "xonsh/built_ins.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"The xonsh built-ins.\n\nNote that this module is named 'built_ins' so as not to be confused with the\nspecial Python builtins module.\n\"\"\"\nimport io\nimport os\nimport re\nimport sys\nimport types\nimport shlex\nimport signal\nimport atexit\nimport pathlib\nimport inspect\nimport warnings\nimport builtins\nimport itertools\nimport subprocess\nimport contextlib\nimport collections.abc as cabc\n\nfrom xonsh.ast import AST\nfrom xonsh.lazyasd import LazyObject, lazyobject\nfrom xonsh.inspectors import Inspector\nfrom xonsh.aliases import Aliases, make_default_aliases\nfrom xonsh.environ import Env, default_env, locate_binary\nfrom xonsh.jobs import add_job\nfrom xonsh.platform import ON_POSIX, ON_WINDOWS, ON_WSL\nfrom xonsh.proc import (\n PopenThread,\n ProcProxyThread,\n ProcProxy,\n ConsoleParallelReader,\n pause_call_resume,\n CommandPipeline,\n HiddenCommandPipeline,\n STDOUT_CAPTURE_KINDS,\n)\nfrom xonsh.tools import (\n suggest_commands,\n expand_path,\n globpath,\n XonshError,\n XonshCalledProcessError,\n)\nfrom xonsh.lazyimps import pty, termios\nfrom xonsh.commands_cache import CommandsCache\nfrom xonsh.events import events\n\nimport xonsh.completers.init\n\nBUILTINS_LOADED = False\nINSPECTOR = LazyObject(Inspector, globals(), \"INSPECTOR\")\n\nwarnings.filterwarnings(\"once\", category=DeprecationWarning)\n\n\n@lazyobject\ndef AT_EXIT_SIGNALS():\n sigs = (\n signal.SIGABRT,\n signal.SIGFPE,\n signal.SIGILL,\n signal.SIGSEGV,\n signal.SIGTERM,\n )\n if ON_POSIX:\n sigs += (signal.SIGTSTP, signal.SIGQUIT, signal.SIGHUP)\n return sigs\n\n\ndef resetting_signal_handle(sig, f):\n \"\"\"Sets a new signal handle that will automatically restore the old value\n once the new handle is finished.\n \"\"\"\n oldh = signal.getsignal(sig)\n\n def newh(s=None, frame=None):\n f(s, frame)\n signal.signal(sig, oldh)\n if sig != 0:\n sys.exit(sig)\n\n signal.signal(sig, newh)\n\n\ndef helper(x, name=\"\"):\n \"\"\"Prints help about, and then returns that variable.\"\"\"\n INSPECTOR.pinfo(x, oname=name, detail_level=0)\n return x\n\n\ndef superhelper(x, name=\"\"):\n \"\"\"Prints help about, and then returns that variable.\"\"\"\n INSPECTOR.pinfo(x, oname=name, detail_level=1)\n return x\n\n\ndef reglob(path, parts=None, i=None):\n \"\"\"Regular expression-based globbing.\"\"\"\n if parts is None:\n path = os.path.normpath(path)\n drive, tail = os.path.splitdrive(path)\n parts = tail.split(os.sep)\n d = os.sep if os.path.isabs(path) else \".\"\n d = os.path.join(drive, d)\n return reglob(d, parts, i=0)\n base = subdir = path\n if i == 0:\n if not os.path.isabs(base):\n base = \"\"\n elif len(parts) > 1:\n i += 1\n regex = os.path.join(base, parts[i])\n if ON_WINDOWS:\n # currently unable to access regex backslash sequences\n # on Windows due to paths using \\.\n regex = regex.replace(\"\\\\\", \"\\\\\\\\\")\n regex = re.compile(regex)\n files = os.listdir(subdir)\n files.sort()\n paths = []\n i1 = i + 1\n if i1 == len(parts):\n for f in files:\n p = os.path.join(base, f)\n if regex.fullmatch(p) is not None:\n paths.append(p)\n else:\n for f in files:\n p = os.path.join(base, f)\n if regex.fullmatch(p) is None or not os.path.isdir(p):\n continue\n paths += reglob(p, parts=parts, i=i1)\n return paths\n\n\ndef path_literal(s):\n s = expand_path(s)\n return pathlib.Path(s)\n\n\ndef regexsearch(s):\n s = expand_path(s)\n return reglob(s)\n\n\ndef globsearch(s):\n csc = builtins.__xonsh__.env.get(\"CASE_SENSITIVE_COMPLETIONS\")\n glob_sorted = builtins.__xonsh__.env.get(\"GLOB_SORTED\")\n dotglob = builtins.__xonsh__.env.get(\"DOTGLOB\")\n return globpath(\n s,\n ignore_case=(not csc),\n return_empty=True,\n sort_result=glob_sorted,\n include_dotfiles=dotglob,\n )\n\n\ndef pathsearch(func, s, pymode=False, pathobj=False):\n \"\"\"\n Takes a string and returns a list of file paths that match (regex, glob,\n or arbitrary search function). If pathobj=True, the return is a list of\n pathlib.Path objects instead of strings.\n \"\"\"\n if not callable(func) or len(inspect.signature(func).parameters) != 1:\n error = \"%r is not a known path search function\"\n raise XonshError(error % func)\n o = func(s)\n if pathobj and pymode:\n o = list(map(pathlib.Path, o))\n no_match = [] if pymode else [s]\n return o if len(o) != 0 else no_match\n\n\nRE_SHEBANG = LazyObject(lambda: re.compile(r\"#![ \\t]*(.+?)$\"), globals(), \"RE_SHEBANG\")\n\n\ndef _is_binary(fname, limit=80):\n with open(fname, \"rb\") as f:\n for i in range(limit):\n char = f.read(1)\n if char == b\"\\0\":\n return True\n if char == b\"\\n\":\n return False\n if char == b\"\":\n return False\n return False\n\n\ndef _un_shebang(x):\n if x == \"/usr/bin/env\":\n return []\n elif any(x.startswith(i) for i in [\"/usr/bin\", \"/usr/local/bin\", \"/bin\"]):\n x = os.path.basename(x)\n elif x.endswith(\"python\") or x.endswith(\"python.exe\"):\n x = \"python\"\n if x == \"xonsh\":\n return [\"python\", \"-m\", \"xonsh.main\"]\n return [x]\n\n\ndef get_script_subproc_command(fname, args):\n \"\"\"Given the name of a script outside the path, returns a list representing\n an appropriate subprocess command to execute the script. Raises\n PermissionError if the script is not executable.\n \"\"\"\n # make sure file is executable\n if not os.access(fname, os.X_OK):\n raise PermissionError\n if ON_POSIX and not os.access(fname, os.R_OK):\n # on some systems, some important programs (e.g. sudo) will have\n # execute permissions but not read/write permissions. This enables\n # things with the SUID set to be run. Needs to come before _is_binary()\n # is called, because that function tries to read the file.\n return [fname] + args\n elif _is_binary(fname):\n # if the file is a binary, we should call it directly\n return [fname] + args\n if ON_WINDOWS:\n # Windows can execute various filetypes directly\n # as given in PATHEXT\n _, ext = os.path.splitext(fname)\n if ext.upper() in builtins.__xonsh__.env.get(\"PATHEXT\"):\n return [fname] + args\n # find interpreter\n with open(fname, \"rb\") as f:\n first_line = f.readline().decode().strip()\n m = RE_SHEBANG.match(first_line)\n # xonsh is the default interpreter\n if m is None:\n interp = [\"xonsh\"]\n else:\n interp = m.group(1).strip()\n if len(interp) > 0:\n interp = shlex.split(interp)\n else:\n interp = [\"xonsh\"]\n if ON_WINDOWS:\n o = []\n for i in interp:\n o.extend(_un_shebang(i))\n interp = o\n return interp + [fname] + args\n\n\n@lazyobject\ndef _REDIR_REGEX():\n name = r\"(o(?:ut)?|e(?:rr)?|a(?:ll)?|&?\\d?)\"\n return re.compile(\"{r}(>?>|<){r}$\".format(r=name))\n\n\n_MODES = LazyObject(lambda: {\">>\": \"a\", \">\": \"w\", \"<\": \"r\"}, globals(), \"_MODES\")\n_WRITE_MODES = LazyObject(lambda: frozenset({\"w\", \"a\"}), globals(), \"_WRITE_MODES\")\n_REDIR_ALL = LazyObject(lambda: frozenset({\"&\", \"a\", \"all\"}), globals(), \"_REDIR_ALL\")\n_REDIR_ERR = LazyObject(lambda: frozenset({\"2\", \"e\", \"err\"}), globals(), \"_REDIR_ERR\")\n_REDIR_OUT = LazyObject(\n lambda: frozenset({\"\", \"1\", \"o\", \"out\"}), globals(), \"_REDIR_OUT\"\n)\n_E2O_MAP = LazyObject(\n lambda: frozenset(\n {\"{}>{}\".format(e, o) for e in _REDIR_ERR for o in _REDIR_OUT if o != \"\"}\n ),\n globals(),\n \"_E2O_MAP\",\n)\n_O2E_MAP = LazyObject(\n lambda: frozenset(\n {\"{}>{}\".format(o, e) for e in _REDIR_ERR for o in _REDIR_OUT if o != \"\"}\n ),\n globals(),\n \"_O2E_MAP\",\n)\n\n\ndef _is_redirect(x):\n return isinstance(x, str) and _REDIR_REGEX.match(x)\n\n\ndef safe_open(fname, mode, buffering=-1):\n \"\"\"Safely attempts to open a file in for xonsh subprocs.\"\"\"\n # file descriptors\n try:\n return io.open(fname, mode, buffering=buffering)\n except PermissionError:\n raise XonshError(\"xonsh: {0}: permission denied\".format(fname))\n except FileNotFoundError:\n raise XonshError(\"xonsh: {0}: no such file or directory\".format(fname))\n except Exception:\n raise XonshError(\"xonsh: {0}: unable to open file\".format(fname))\n\n\ndef safe_close(x):\n \"\"\"Safely attempts to close an object.\"\"\"\n if not isinstance(x, io.IOBase):\n return\n if x.closed:\n return\n try:\n x.close()\n except Exception:\n pass\n\n\ndef _parse_redirects(r, loc=None):\n \"\"\"returns origin, mode, destination tuple\"\"\"\n orig, mode, dest = _REDIR_REGEX.match(r).groups()\n # redirect to fd\n if dest.startswith(\"&\"):\n try:\n dest = int(dest[1:])\n if loc is None:\n loc, dest = dest, \"\" # NOQA\n else:\n e = \"Unrecognized redirection command: {}\".format(r)\n raise XonshError(e)\n except (ValueError, XonshError):\n raise\n except Exception:\n pass\n mode = _MODES.get(mode, None)\n if mode == \"r\" and (len(orig) > 0 or len(dest) > 0):\n raise XonshError(\"Unrecognized redirection command: {}\".format(r))\n elif mode in _WRITE_MODES and len(dest) > 0:\n raise XonshError(\"Unrecognized redirection command: {}\".format(r))\n return orig, mode, dest\n\n\ndef _redirect_streams(r, loc=None):\n \"\"\"Returns stdin, stdout, stderr tuple of redirections.\"\"\"\n stdin = stdout = stderr = None\n no_ampersand = r.replace(\"&\", \"\")\n # special case of redirecting stderr to stdout\n if no_ampersand in _E2O_MAP:\n stderr = subprocess.STDOUT\n return stdin, stdout, stderr\n elif no_ampersand in _O2E_MAP:\n stdout = 2 # using 2 as a flag, rather than using a file object\n return stdin, stdout, stderr\n # get streams\n orig, mode, dest = _parse_redirects(r)\n if mode == \"r\":\n stdin = safe_open(loc, mode)\n elif mode in _WRITE_MODES:\n if orig in _REDIR_ALL:\n stdout = stderr = safe_open(loc, mode)\n elif orig in _REDIR_OUT:\n stdout = safe_open(loc, mode)\n elif orig in _REDIR_ERR:\n stderr = safe_open(loc, mode)\n else:\n raise XonshError(\"Unrecognized redirection command: {}\".format(r))\n else:\n raise XonshError(\"Unrecognized redirection command: {}\".format(r))\n return stdin, stdout, stderr\n\n\ndef default_signal_pauser(n, f):\n \"\"\"Pauses a signal, as needed.\"\"\"\n signal.pause()\n\n\ndef no_pg_xonsh_preexec_fn():\n \"\"\"Default subprocess preexec function for when there is no existing\n pipeline group.\n \"\"\"\n os.setpgrp()\n signal.signal(signal.SIGTSTP, default_signal_pauser)\n\n\nclass SubprocSpec:\n \"\"\"A container for specifying how a subprocess command should be\n executed.\n \"\"\"\n\n kwnames = (\"stdin\", \"stdout\", \"stderr\", \"universal_newlines\", \"close_fds\")\n\n def __init__(\n self,\n cmd,\n cls=subprocess.Popen,\n stdin=None,\n stdout=None,\n stderr=None,\n universal_newlines=False,\n close_fds=False,\n captured=False,\n ):\n \"\"\"\n Parameters\n ----------\n cmd : list of str\n Command to be run.\n cls : Popen-like\n Class to run the subprocess with.\n stdin : file-like\n Popen file descriptor or flag for stdin.\n stdout : file-like\n Popen file descriptor or flag for stdout.\n stderr : file-like\n Popen file descriptor or flag for stderr.\n universal_newlines : bool\n Whether or not to use universal newlines.\n close_fds : bool\n Whether or not to close the file descriptiors when the\n process exits.\n captured : bool or str, optional\n The flag for if the subprocess is captured, may be one of:\n False for $[], 'stdout' for $(), 'hiddenobject' for ![], or\n 'object' for !().\n\n Attributes\n ----------\n args : list of str\n Arguments as originally supplied.\n alias : list of str, callable, or None\n The alias that was resolved for this command, if any.\n binary_loc : str or None\n Path to binary to execute.\n is_proxy : bool\n Whether or not the subprocess is or should be run as a proxy.\n background : bool\n Whether or not the subprocess should be started in the background.\n threadable : bool\n Whether or not the subprocess is able to be run in a background\n thread, rather than the main thread.\n pipeline_index : int or None\n The index number of this sepc into the pipeline that is being setup.\n last_in_pipeline : bool\n Whether the subprocess is the last in the execution pipeline.\n captured_stdout : file-like\n Handle to captured stdin\n captured_stderr : file-like\n Handle to captured stderr\n stack : list of FrameInfo namedtuples or None\n The stack of the call-site of alias, if the alias requires it.\n None otherwise.\n \"\"\"\n self._stdin = self._stdout = self._stderr = None\n # args\n self.cmd = list(cmd)\n self.cls = cls\n self.stdin = stdin\n self.stdout = stdout\n self.stderr = stderr\n self.universal_newlines = universal_newlines\n self.close_fds = close_fds\n self.captured = captured\n # pure attrs\n self.args = list(cmd)\n self.alias = None\n self.binary_loc = None\n self.is_proxy = False\n self.background = False\n self.threadable = True\n self.pipeline_index = None\n self.last_in_pipeline = False\n self.captured_stdout = None\n self.captured_stderr = None\n self.stack = None\n\n def __str__(self):\n s = self.__class__.__name__ + \"(\" + str(self.cmd) + \", \"\n s += self.cls.__name__ + \", \"\n kws = [n + \"=\" + str(getattr(self, n)) for n in self.kwnames]\n s += \", \".join(kws) + \")\"\n return s\n\n def __repr__(self):\n s = self.__class__.__name__ + \"(\" + repr(self.cmd) + \", \"\n s += self.cls.__name__ + \", \"\n kws = [n + \"=\" + repr(getattr(self, n)) for n in self.kwnames]\n s += \", \".join(kws) + \")\"\n return s\n\n #\n # Properties\n #\n\n @property\n def stdin(self):\n return self._stdin\n\n @stdin.setter\n def stdin(self, value):\n if self._stdin is None:\n self._stdin = value\n elif value is None:\n pass\n else:\n safe_close(value)\n msg = \"Multiple inputs for stdin for {0!r}\"\n msg = msg.format(\" \".join(self.args))\n raise XonshError(msg)\n\n @property\n def stdout(self):\n return self._stdout\n\n @stdout.setter\n def stdout(self, value):\n if self._stdout is None:\n self._stdout = value\n elif value is None:\n pass\n else:\n safe_close(value)\n msg = \"Multiple redirections for stdout for {0!r}\"\n msg = msg.format(\" \".join(self.args))\n raise XonshError(msg)\n\n @property\n def stderr(self):\n return self._stderr\n\n @stderr.setter\n def stderr(self, value):\n if self._stderr is None:\n self._stderr = value\n elif value is None:\n pass\n else:\n safe_close(value)\n msg = \"Multiple redirections for stderr for {0!r}\"\n msg = msg.format(\" \".join(self.args))\n raise XonshError(msg)\n\n #\n # Execution methods\n #\n\n def run(self, *, pipeline_group=None):\n \"\"\"Launches the subprocess and returns the object.\"\"\"\n event_name = self._cmd_event_name()\n self._pre_run_event_fire(event_name)\n kwargs = {n: getattr(self, n) for n in self.kwnames}\n self.prep_env(kwargs)\n self.prep_preexec_fn(kwargs, pipeline_group=pipeline_group)\n if callable(self.alias):\n if \"preexec_fn\" in kwargs:\n kwargs.pop(\"preexec_fn\")\n p = self.cls(self.alias, self.cmd, **kwargs)\n else:\n self._fix_null_cmd_bytes()\n p = self._run_binary(kwargs)\n p.spec = self\n p.last_in_pipeline = self.last_in_pipeline\n p.captured_stdout = self.captured_stdout\n p.captured_stderr = self.captured_stderr\n self._post_run_event_fire(event_name, p)\n return p\n\n def _run_binary(self, kwargs):\n try:\n bufsize = 1\n p = self.cls(self.cmd, bufsize=bufsize, **kwargs)\n except PermissionError:\n e = \"xonsh: subprocess mode: permission denied: {0}\"\n raise XonshError(e.format(self.cmd[0]))\n except FileNotFoundError:\n cmd0 = self.cmd[0]\n e = \"xonsh: subprocess mode: command not found: {0}\".format(cmd0)\n env = builtins.__xonsh__.env\n sug = suggest_commands(cmd0, env, builtins.aliases)\n if len(sug.strip()) > 0:\n e += \"\\n\" + suggest_commands(cmd0, env, builtins.aliases)\n raise XonshError(e)\n return p\n\n def prep_env(self, kwargs):\n \"\"\"Prepares the environment to use in the subprocess.\"\"\"\n denv = builtins.__xonsh__.env.detype()\n if ON_WINDOWS:\n # Over write prompt variable as xonsh's $PROMPT does\n # not make much sense for other subprocs\n denv[\"PROMPT\"] = \"$P$G\"\n kwargs[\"env\"] = denv\n\n def prep_preexec_fn(self, kwargs, pipeline_group=None):\n \"\"\"Prepares the 'preexec_fn' keyword argument\"\"\"\n if not ON_POSIX:\n return\n if not builtins.__xonsh__.env.get(\"XONSH_INTERACTIVE\"):\n return\n if pipeline_group is None or ON_WSL:\n # If there is no pipeline group\n # or the platform is windows subsystem for linux (WSL)\n xonsh_preexec_fn = no_pg_xonsh_preexec_fn\n else:\n\n def xonsh_preexec_fn():\n \"\"\"Preexec function bound to a pipeline group.\"\"\"\n os.setpgid(0, pipeline_group)\n signal.signal(signal.SIGTSTP, default_signal_pauser)\n\n kwargs[\"preexec_fn\"] = xonsh_preexec_fn\n\n def _fix_null_cmd_bytes(self):\n # Popen does not accept null bytes in its input commands.\n # That doesn't stop some subprocesses from using them. Here we\n # escape them just in case.\n cmd = self.cmd\n for i in range(len(cmd)):\n cmd[i] = cmd[i].replace(\"\\0\", \"\\\\0\")\n\n def _cmd_event_name(self):\n if callable(self.alias):\n return self.alias.__name__\n elif self.binary_loc is None:\n return \"\"\n else:\n return os.path.basename(self.binary_loc)\n\n def _pre_run_event_fire(self, name):\n event_name = \"on_pre_spec_run_\" + name\n if events.exists(event_name):\n event = getattr(events, event_name)\n event.fire(spec=self)\n\n def _post_run_event_fire(self, name, proc):\n event_name = \"on_post_spec_run_\" + name\n if events.exists(event_name):\n event = getattr(events, event_name)\n event.fire(spec=self, proc=proc)\n\n #\n # Building methods\n #\n\n @classmethod\n def build(kls, cmd, *, cls=subprocess.Popen, **kwargs):\n \"\"\"Creates an instance of the subprocess command, with any\n modifications and adjustments based on the actual cmd that\n was received.\n \"\"\"\n # modifications that do not alter cmds may come before creating instance\n spec = kls(cmd, cls=cls, **kwargs)\n # modifications that alter cmds must come after creating instance\n # perform initial redirects\n spec.redirect_leading()\n spec.redirect_trailing()\n # apply aliases\n spec.resolve_alias()\n spec.resolve_binary_loc()\n spec.resolve_auto_cd()\n spec.resolve_executable_commands()\n spec.resolve_alias_cls()\n spec.resolve_stack()\n return spec\n\n def redirect_leading(self):\n \"\"\"Manage leading redirects such as with '< input.txt COMMAND'. \"\"\"\n while len(self.cmd) >= 3 and self.cmd[0] == \"<\":\n self.stdin = safe_open(self.cmd[1], \"r\")\n self.cmd = self.cmd[2:]\n\n def redirect_trailing(self):\n \"\"\"Manages trailing redirects.\"\"\"\n while True:\n cmd = self.cmd\n if len(cmd) >= 3 and _is_redirect(cmd[-2]):\n streams = _redirect_streams(cmd[-2], cmd[-1])\n self.stdin, self.stdout, self.stderr = streams\n self.cmd = cmd[:-2]\n elif len(cmd) >= 2 and _is_redirect(cmd[-1]):\n streams = _redirect_streams(cmd[-1])\n self.stdin, self.stdout, self.stderr = streams\n self.cmd = cmd[:-1]\n else:\n break\n\n def resolve_alias(self):\n \"\"\"Sets alias in command, if applicable.\"\"\"\n cmd0 = self.cmd[0]\n if callable(cmd0):\n alias = cmd0\n else:\n alias = builtins.aliases.get(cmd0, None)\n self.alias = alias\n\n def resolve_binary_loc(self):\n \"\"\"Sets the binary location\"\"\"\n alias = self.alias\n if alias is None:\n binary_loc = locate_binary(self.cmd[0])\n elif callable(alias):\n binary_loc = None\n else:\n binary_loc = locate_binary(alias[0])\n self.binary_loc = binary_loc\n\n def resolve_auto_cd(self):\n \"\"\"Implements AUTO_CD functionality.\"\"\"\n if not (\n self.alias is None\n and self.binary_loc is None\n and len(self.cmd) == 1\n and builtins.__xonsh__.env.get(\"AUTO_CD\")\n and os.path.isdir(self.cmd[0])\n ):\n return\n self.cmd.insert(0, \"cd\")\n self.alias = builtins.aliases.get(\"cd\", None)\n\n def resolve_executable_commands(self):\n \"\"\"Resolve command executables, if applicable.\"\"\"\n alias = self.alias\n if alias is None:\n pass\n elif callable(alias):\n self.cmd.pop(0)\n return\n else:\n self.cmd = alias + self.cmd[1:]\n # resolve any redirects the aliases may have applied\n self.redirect_leading()\n self.redirect_trailing()\n if self.binary_loc is None:\n return\n try:\n self.cmd = get_script_subproc_command(self.binary_loc, self.cmd[1:])\n except PermissionError:\n e = \"xonsh: subprocess mode: permission denied: {0}\"\n raise XonshError(e.format(self.cmd[0]))\n\n def resolve_alias_cls(self):\n \"\"\"Determine which proxy class to run an alias with.\"\"\"\n alias = self.alias\n if not callable(alias):\n return\n self.is_proxy = True\n thable = getattr(alias, \"__xonsh_threadable__\", True)\n cls = ProcProxyThread if thable else ProcProxy\n self.cls = cls\n self.threadable = thable\n # also check capturability, while we are here\n cpable = getattr(alias, \"__xonsh_capturable__\", self.captured)\n self.captured = cpable\n\n def resolve_stack(self):\n \"\"\"Computes the stack for a callable alias's call-site, if needed.\"\"\"\n if not callable(self.alias):\n return\n # check that we actual need the stack\n sig = inspect.signature(self.alias)\n if len(sig.parameters) <= 5 and \"stack\" not in sig.parameters:\n return\n # compute the stack, and filter out these build methods\n # run_subproc() is the 4th command in the stack\n # we want to filter out one up, e.g. subproc_captured_hiddenobject()\n # after that the stack from the call site starts.\n stack = inspect.stack(context=0)\n assert stack[3][3] == \"run_subproc\", \"xonsh stack has changed!\"\n del stack[:5]\n self.stack = stack\n\n\ndef _safe_pipe_properties(fd, use_tty=False):\n \"\"\"Makes sure that a pipe file descriptor properties are sane.\"\"\"\n if not use_tty:\n return\n # due to some weird, long standing issue in Python, PTYs come out\n # replacing newline \\n with \\r\\n. This causes issues for raw unix\n # protocols, like git and ssh, which expect unix line endings.\n # see https://mail.python.org/pipermail/python-list/2013-June/650460.html\n # for more details and the following solution.\n props = termios.tcgetattr(fd)\n props[1] = props[1] & (~termios.ONLCR) | termios.ONLRET\n termios.tcsetattr(fd, termios.TCSANOW, props)\n\n\ndef _update_last_spec(last):\n captured = last.captured\n last.last_in_pipeline = True\n if not captured:\n return\n callable_alias = callable(last.alias)\n if callable_alias:\n pass\n else:\n cmds_cache = builtins.__xonsh__.commands_cache\n thable = cmds_cache.predict_threadable(\n last.args\n ) and cmds_cache.predict_threadable(last.cmd)\n if captured and thable:\n last.cls = PopenThread\n elif not thable:\n # foreground processes should use Popen\n last.threadable = False\n if captured == \"object\" or captured == \"hiddenobject\":\n # CommandPipeline objects should not pipe stdout, stderr\n return\n # cannot used PTY pipes for aliases, for some dark reason,\n # and must use normal pipes instead.\n use_tty = ON_POSIX and not callable_alias\n # Do not set standard in! Popen is not a fan of redirections here\n # set standard out\n if last.stdout is not None:\n last.universal_newlines = True\n elif captured in STDOUT_CAPTURE_KINDS:\n last.universal_newlines = False\n r, w = os.pipe()\n last.stdout = safe_open(w, \"wb\")\n last.captured_stdout = safe_open(r, \"rb\")\n elif builtins.__xonsh__.stdout_uncaptured is not None:\n last.universal_newlines = True\n last.stdout = builtins.__xonsh__.stdout_uncaptured\n last.captured_stdout = last.stdout\n elif ON_WINDOWS and not callable_alias:\n last.universal_newlines = True\n last.stdout = None # must truly stream on windows\n last.captured_stdout = ConsoleParallelReader(1)\n else:\n last.universal_newlines = True\n r, w = pty.openpty() if use_tty else os.pipe()\n _safe_pipe_properties(w, use_tty=use_tty)\n last.stdout = safe_open(w, \"w\")\n _safe_pipe_properties(r, use_tty=use_tty)\n last.captured_stdout = safe_open(r, \"r\")\n # set standard error\n if last.stderr is not None:\n pass\n elif captured == \"object\":\n r, w = os.pipe()\n last.stderr = safe_open(w, \"w\")\n last.captured_stderr = safe_open(r, \"r\")\n elif builtins.__xonsh__.stderr_uncaptured is not None:\n last.stderr = builtins.__xonsh__.stderr_uncaptured\n last.captured_stderr = last.stderr\n elif ON_WINDOWS and not callable_alias:\n last.universal_newlines = True\n last.stderr = None # must truly stream on windows\n else:\n r, w = pty.openpty() if use_tty else os.pipe()\n _safe_pipe_properties(w, use_tty=use_tty)\n last.stderr = safe_open(w, \"w\")\n _safe_pipe_properties(r, use_tty=use_tty)\n last.captured_stderr = safe_open(r, \"r\")\n # redirect stdout to stderr, if we should\n if isinstance(last.stdout, int) and last.stdout == 2:\n # need to use private interface to avoid duplication.\n last._stdout = last.stderr\n # redirect stderr to stdout, if we should\n if callable_alias and last.stderr == subprocess.STDOUT:\n last._stderr = last.stdout\n last.captured_stderr = last.captured_stdout\n\n\ndef cmds_to_specs(cmds, captured=False):\n \"\"\"Converts a list of cmds to a list of SubprocSpec objects that are\n ready to be executed.\n \"\"\"\n # first build the subprocs independently and separate from the redirects\n i = 0\n specs = []\n redirects = []\n for cmd in cmds:\n if isinstance(cmd, str):\n redirects.append(cmd)\n else:\n if cmd[-1] == \"&\":\n cmd = cmd[:-1]\n redirects.append(\"&\")\n spec = SubprocSpec.build(cmd, captured=captured)\n spec.pipeline_index = i\n specs.append(spec)\n i += 1\n # now modify the subprocs based on the redirects.\n for i, redirect in enumerate(redirects):\n if redirect == \"|\":\n # these should remain integer file descriptors, and not Python\n # file objects since they connect processes.\n r, w = os.pipe()\n specs[i].stdout = w\n specs[i + 1].stdin = r\n elif redirect == \"&\" and i == len(redirects) - 1:\n specs[-1].background = True\n else:\n raise XonshError(\"unrecognized redirect {0!r}\".format(redirect))\n # Apply boundary conditions\n _update_last_spec(specs[-1])\n return specs\n\n\ndef _should_set_title(captured=False):\n env = builtins.__xonsh__.env\n return (\n env.get(\"XONSH_INTERACTIVE\")\n and not env.get(\"XONSH_STORE_STDOUT\")\n and captured not in STDOUT_CAPTURE_KINDS\n and builtins.__xonsh__.shell is not None\n )\n\n\ndef run_subproc(cmds, captured=False):\n \"\"\"Runs a subprocess, in its many forms. This takes a list of 'commands,'\n which may be a list of command line arguments or a string, representing\n a special connecting character. For example::\n\n $ ls | grep wakka\n\n is represented by the following cmds::\n\n [['ls'], '|', ['grep', 'wakka']]\n\n Lastly, the captured argument affects only the last real command.\n \"\"\"\n specs = cmds_to_specs(cmds, captured=captured)\n captured = specs[-1].captured\n if captured == \"hiddenobject\":\n command = HiddenCommandPipeline(specs)\n else:\n command = CommandPipeline(specs)\n proc = command.proc\n background = command.spec.background\n if not all(x.is_proxy for x in specs):\n add_job(\n {\n \"cmds\": cmds,\n \"pids\": [i.pid for i in command.procs],\n \"obj\": proc,\n \"bg\": background,\n \"pipeline\": command,\n \"pgrp\": command.term_pgid,\n }\n )\n if _should_set_title(captured=captured):\n # set title here to get currently executing command\n pause_call_resume(proc, builtins.__xonsh__.shell.settitle)\n else:\n # for some reason, some programs are in a stopped state when the flow\n # reaches this point, hence a SIGCONT should be sent to `proc` to make\n # sure that the shell doesn't hang. This `pause_call_resume` invocation\n # does this\n pause_call_resume(proc, int)\n # create command or return if backgrounding.\n if background:\n return\n # now figure out what we should return.\n if captured == \"stdout\":\n command.end()\n return command.output\n elif captured == \"object\":\n return command\n elif captured == \"hiddenobject\":\n command.end()\n return command\n else:\n command.end()\n return\n\n\ndef subproc_captured_stdout(*cmds):\n \"\"\"Runs a subprocess, capturing the output. Returns the stdout\n that was produced as a str.\n \"\"\"\n return run_subproc(cmds, captured=\"stdout\")\n\n\ndef subproc_captured_inject(*cmds):\n \"\"\"Runs a subprocess, capturing the output. Returns a list of\n whitespace-separated strings of the stdout that was produced.\n The string is split using xonsh's lexer, rather than Python's str.split()\n or shlex.split().\n \"\"\"\n s = run_subproc(cmds, captured=\"stdout\")\n toks = builtins.__xonsh__.execer.parser.lexer.split(s.strip())\n return toks\n\n\ndef subproc_captured_object(*cmds):\n \"\"\"\n Runs a subprocess, capturing the output. Returns an instance of\n CommandPipeline representing the completed command.\n \"\"\"\n return run_subproc(cmds, captured=\"object\")\n\n\ndef subproc_captured_hiddenobject(*cmds):\n \"\"\"Runs a subprocess, capturing the output. Returns an instance of\n HiddenCommandPipeline representing the completed command.\n \"\"\"\n return run_subproc(cmds, captured=\"hiddenobject\")\n\n\ndef subproc_uncaptured(*cmds):\n \"\"\"Runs a subprocess, without capturing the output. Returns the stdout\n that was produced as a str.\n \"\"\"\n return run_subproc(cmds, captured=False)\n\n\ndef ensure_list_of_strs(x):\n \"\"\"Ensures that x is a list of strings.\"\"\"\n if isinstance(x, str):\n rtn = [x]\n elif isinstance(x, cabc.Sequence):\n rtn = [i if isinstance(i, str) else str(i) for i in x]\n else:\n rtn = [str(x)]\n return rtn\n\n\ndef list_of_strs_or_callables(x):\n \"\"\"Ensures that x is a list of strings or functions\"\"\"\n if isinstance(x, str) or callable(x):\n rtn = [x]\n elif isinstance(x, cabc.Iterable):\n rtn = [i if isinstance(i, str) or callable(i) else str(i) for i in x]\n else:\n rtn = [str(x)]\n return rtn\n\n\ndef list_of_list_of_strs_outer_product(x):\n \"\"\"Takes an outer product of a list of strings\"\"\"\n lolos = map(ensure_list_of_strs, x)\n rtn = []\n for los in itertools.product(*lolos):\n s = \"\".join(los)\n if \"*\" in s:\n rtn.extend(builtins.__xonsh__.glob(s))\n else:\n rtn.append(builtins.__xonsh__.expand_path(s))\n return rtn\n\n\n@lazyobject\ndef MACRO_FLAG_KINDS():\n return {\n \"s\": str,\n \"str\": str,\n \"string\": str,\n \"a\": AST,\n \"ast\": AST,\n \"c\": types.CodeType,\n \"code\": types.CodeType,\n \"compile\": types.CodeType,\n \"v\": eval,\n \"eval\": eval,\n \"x\": exec,\n \"exec\": exec,\n \"t\": type,\n \"type\": type,\n }\n\n\ndef _convert_kind_flag(x):\n \"\"\"Puts a kind flag (string) a canonical form.\"\"\"\n x = x.lower()\n kind = MACRO_FLAG_KINDS.get(x, None)\n if kind is None:\n raise TypeError(\"{0!r} not a recognized macro type.\".format(x))\n return kind\n\n\ndef convert_macro_arg(raw_arg, kind, glbs, locs, *, name=\"\", macroname=\"\"):\n \"\"\"Converts a string macro argument based on the requested kind.\n\n Parameters\n ----------\n raw_arg : str\n The str representation of the macro argument.\n kind : object\n A flag or type representing how to convert the argument.\n glbs : Mapping\n The globals from the call site.\n locs : Mapping or None\n The locals from the call site.\n name : str, optional\n The macro argument name.\n macroname : str, optional\n The name of the macro itself.\n\n Returns\n -------\n The converted argument.\n \"\"\"\n # munge kind and mode to start\n mode = None\n if isinstance(kind, cabc.Sequence) and not isinstance(kind, str):\n # have (kind, mode) tuple\n kind, mode = kind\n if isinstance(kind, str):\n kind = _convert_kind_flag(kind)\n if kind is str or kind is None:\n return raw_arg # short circuit since there is nothing else to do\n # select from kind and convert\n execer = builtins.__xonsh__.execer\n filename = macroname + \"(\" + name + \")\"\n if kind is AST:\n ctx = set(dir(builtins)) | set(glbs.keys())\n if locs is not None:\n ctx |= set(locs.keys())\n mode = mode or \"eval\"\n if mode != \"eval\" and not raw_arg.endswith(\"\\n\"):\n raw_arg += \"\\n\"\n arg = execer.parse(raw_arg, ctx, mode=mode, filename=filename)\n elif kind is types.CodeType or kind is compile: # NOQA\n mode = mode or \"eval\"\n arg = execer.compile(\n raw_arg, mode=mode, glbs=glbs, locs=locs, filename=filename\n )\n elif kind is eval:\n arg = execer.eval(raw_arg, glbs=glbs, locs=locs, filename=filename)\n elif kind is exec:\n mode = mode or \"exec\"\n if not raw_arg.endswith(\"\\n\"):\n raw_arg += \"\\n\"\n arg = execer.exec(raw_arg, mode=mode, glbs=glbs, locs=locs, filename=filename)\n elif kind is type:\n arg = type(execer.eval(raw_arg, glbs=glbs, locs=locs, filename=filename))\n else:\n msg = \"kind={0!r} and mode={1!r} was not recognized for macro \" \"argument {2!r}\"\n raise TypeError(msg.format(kind, mode, name))\n return arg\n\n\n@contextlib.contextmanager\ndef in_macro_call(f, glbs, locs):\n \"\"\"Attaches macro globals and locals temporarily to function as a\n context manager.\n\n Parameters\n ----------\n f : callable object\n The function that is called as ``f(*args)``.\n glbs : Mapping\n The globals from the call site.\n locs : Mapping or None\n The locals from the call site.\n \"\"\"\n prev_glbs = getattr(f, \"macro_globals\", None)\n prev_locs = getattr(f, \"macro_locals\", None)\n f.macro_globals = glbs\n f.macro_locals = locs\n yield\n if prev_glbs is None:\n del f.macro_globals\n else:\n f.macro_globals = prev_glbs\n if prev_locs is None:\n del f.macro_locals\n else:\n f.macro_locals = prev_locs\n\n\ndef call_macro(f, raw_args, glbs, locs):\n \"\"\"Calls a function as a macro, returning its result.\n\n Parameters\n ----------\n f : callable object\n The function that is called as ``f(*args)``.\n raw_args : tuple of str\n The str representation of arguments of that were passed into the\n macro. These strings will be parsed, compiled, evaled, or left as\n a string depending on the annotations of f.\n glbs : Mapping\n The globals from the call site.\n locs : Mapping or None\n The locals from the call site.\n \"\"\"\n sig = inspect.signature(f)\n empty = inspect.Parameter.empty\n macroname = f.__name__\n i = 0\n args = []\n for (key, param), raw_arg in zip(sig.parameters.items(), raw_args):\n i += 1\n if raw_arg == \"*\":\n break\n kind = param.annotation\n if kind is empty or kind is None:\n kind = str\n arg = convert_macro_arg(\n raw_arg, kind, glbs, locs, name=key, macroname=macroname\n )\n args.append(arg)\n reg_args, kwargs = _eval_regular_args(raw_args[i:], glbs, locs)\n args += reg_args\n with in_macro_call(f, glbs, locs):\n rtn = f(*args, **kwargs)\n return rtn\n\n\n@lazyobject\ndef KWARG_RE():\n return re.compile(r\"([A-Za-z_]\\w*=|\\*\\*)\")\n\n\ndef _starts_as_arg(s):\n \"\"\"Tests if a string starts as a non-kwarg string would.\"\"\"\n return KWARG_RE.match(s) is None\n\n\ndef _eval_regular_args(raw_args, glbs, locs):\n if not raw_args:\n return [], {}\n arglist = list(itertools.takewhile(_starts_as_arg, raw_args))\n kwarglist = raw_args[len(arglist) :]\n execer = builtins.__xonsh__.execer\n if not arglist:\n args = arglist\n kwargstr = \"dict({})\".format(\", \".join(kwarglist))\n kwargs = execer.eval(kwargstr, glbs=glbs, locs=locs)\n elif not kwarglist:\n argstr = \"({},)\".format(\", \".join(arglist))\n args = execer.eval(argstr, glbs=glbs, locs=locs)\n kwargs = {}\n else:\n argstr = \"({},)\".format(\", \".join(arglist))\n kwargstr = \"dict({})\".format(\", \".join(kwarglist))\n both = \"({}, {})\".format(argstr, kwargstr)\n args, kwargs = execer.eval(both, glbs=glbs, locs=locs)\n return args, kwargs\n\n\ndef enter_macro(obj, raw_block, glbs, locs):\n \"\"\"Prepares to enter a context manager macro by attaching the contents\n of the macro block, globals, and locals to the object. These modifications\n are made in-place and the original object is returned.\n\n\n Parameters\n ----------\n obj : context manager\n The object that is about to be entered via a with-statement.\n raw_block : str\n The str of the block that is the context body.\n This string will be parsed, compiled, evaled, or left as\n a string depending on the return annotation of obj.__enter__.\n glbs : Mapping\n The globals from the context site.\n locs : Mapping or None\n The locals from the context site.\n\n Returns\n -------\n obj : context manager\n The same context manager but with the new macro information applied.\n \"\"\"\n # recurse down sequences\n if isinstance(obj, cabc.Sequence):\n for x in obj:\n enter_macro(x, raw_block, glbs, locs)\n return obj\n # convert block as needed\n kind = getattr(obj, \"__xonsh_block__\", str)\n macroname = getattr(obj, \"__name__\", \"\")\n block = convert_macro_arg(\n raw_block, kind, glbs, locs, name=\"\", macroname=macroname\n )\n # attach attrs\n obj.macro_globals = glbs\n obj.macro_locals = locs\n obj.macro_block = block\n return obj\n\n\ndef load_builtins(execer=None, ctx=None):\n \"\"\"Loads the xonsh builtins into the Python builtins. Sets the\n BUILTINS_LOADED variable to True.\n \"\"\"\n global BUILTINS_LOADED\n if not hasattr(builtins, \"__xonsh__\"):\n builtins.__xonsh__ = XonshSession(execer=execer, ctx=ctx)\n builtins.__xonsh__.load(execer=execer, ctx=ctx)\n builtins.__xonsh__.link_builtins(execer=execer)\n BUILTINS_LOADED = True\n\n\ndef _lastflush(s=None, f=None):\n if hasattr(builtins, \"__xonsh__\"):\n if builtins.__xonsh__.history is not None:\n builtins.__xonsh__.history.flush(at_exit=True)\n\n\ndef unload_builtins():\n \"\"\"Removes the xonsh builtins from the Python builtins, if the\n BUILTINS_LOADED is True, sets BUILTINS_LOADED to False, and returns.\n \"\"\"\n global BUILTINS_LOADED\n if not hasattr(builtins, \"__xonsh__\"):\n BUILTINS_LOADED = False\n return\n env = getattr(builtins.__xonsh__, \"env\", None)\n if isinstance(env, Env):\n env.undo_replace_env()\n if hasattr(builtins.__xonsh__, \"pyexit\"):\n builtins.exit = builtins.__xonsh__.pyexit\n if hasattr(builtins.__xonsh__, \"pyquit\"):\n builtins.quit = builtins.__xonsh__.pyquit\n if not BUILTINS_LOADED:\n return\n builtins.__xonsh__.unlink_builtins()\n delattr(builtins, \"__xonsh__\")\n BUILTINS_LOADED = False\n\n\n@contextlib.contextmanager\ndef xonsh_builtins(execer=None):\n \"\"\"A context manager for using the xonsh builtins only in a limited\n scope. Likely useful in testing.\n \"\"\"\n load_builtins(execer=execer)\n # temporary shims for old __xonsh_*__ builtins\n load_proxies()\n yield\n # temporary shims for old __xonsh_*__ builtins\n unload_proxies()\n unload_builtins()\n\n\nclass XonshSession:\n \"\"\"All components defining a xonsh session.\n\n \"\"\"\n\n def __init__(self, execer=None, ctx=None):\n \"\"\"\n Parameters\n ---------\n execer : Execer, optional\n Xonsh execution object, may be None to start\n ctx : Mapping, optional\n Context to start xonsh session with.\n \"\"\"\n self.execer = execer\n self.ctx = {} if ctx is None else ctx\n\n def load(self, execer=None, ctx=None):\n \"\"\"Loads the session with default values.\n\n Parameters\n ---------\n execer : Execer, optional\n Xonsh execution object, may be None to start\n ctx : Mapping, optional\n Context to start xonsh session with.\n \"\"\"\n if ctx is not None:\n self.ctx = ctx\n self.env = Env(default_env())\n self.help = helper\n self.superhelp = superhelper\n self.pathsearch = pathsearch\n self.globsearch = globsearch\n self.regexsearch = regexsearch\n self.glob = globpath\n self.expand_path = expand_path\n self.exit = False\n self.stdout_uncaptured = None\n self.stderr_uncaptured = None\n\n if hasattr(builtins, \"exit\"):\n self.pyexit = builtins.exit\n del builtins.exit\n\n if hasattr(builtins, \"quit\"):\n self.pyquit = builtins.quit\n del builtins.quit\n\n self.subproc_captured_stdout = subproc_captured_stdout\n self.subproc_captured_inject = subproc_captured_inject\n self.subproc_captured_object = subproc_captured_object\n self.subproc_captured_hiddenobject = subproc_captured_hiddenobject\n self.subproc_uncaptured = subproc_uncaptured\n self.execer = execer\n self.commands_cache = CommandsCache()\n self.all_jobs = {}\n self.ensure_list_of_strs = ensure_list_of_strs\n self.list_of_strs_or_callables = list_of_strs_or_callables\n\n self.list_of_list_of_strs_outer_product = list_of_list_of_strs_outer_product\n\n self.completers = xonsh.completers.init.default_completers()\n self.call_macro = call_macro\n self.enter_macro = enter_macro\n self.path_literal = path_literal\n\n self.builtins = _BuiltIns(execer)\n\n self.history = None\n self.shell = None\n\n def link_builtins(self, execer=None):\n # public built-ins\n builtins.XonshError = self.builtins.XonshError\n builtins.XonshCalledProcessError = self.builtins.XonshCalledProcessError\n builtins.evalx = None if execer is None else execer.eval\n builtins.execx = None if execer is None else execer.exec\n builtins.compilex = None if execer is None else execer.compile\n builtins.events = self.builtins.events\n\n # sneak the path search functions into the aliases\n # Need this inline/lazy import here since we use locate_binary that\n # relies on __xonsh__.env in default aliases\n builtins.default_aliases = builtins.aliases = Aliases(make_default_aliases())\n atexit.register(_lastflush)\n for sig in AT_EXIT_SIGNALS:\n resetting_signal_handle(sig, _lastflush)\n\n def unlink_builtins(self):\n names = [\n \"XonshError\",\n \"XonshCalledProcessError\",\n \"evalx\",\n \"execx\",\n \"compilex\",\n \"default_aliases\",\n ]\n\n for name in names:\n if hasattr(builtins, name):\n delattr(builtins, name)\n\n\nclass _BuiltIns:\n def __init__(self, execer=None):\n # public built-ins\n self.XonshError = XonshError\n self.XonshCalledProcessError = XonshCalledProcessError\n self.evalx = None if execer is None else execer.eval\n self.execx = None if execer is None else execer.exec\n self.compilex = None if execer is None else execer.compile\n self.events = events\n\n\nclass DynamicAccessProxy:\n \"\"\"Proxies access dynamically.\"\"\"\n\n def __init__(self, refname, objname):\n \"\"\"\n Parameters\n ----------\n refname : str\n '.'-separated string that represents the new, reference name that\n the user will access.\n objname : str\n '.'-separated string that represents the name where the target\n object actually lives that refname points to.\n \"\"\"\n super().__setattr__(\"refname\", refname)\n super().__setattr__(\"objname\", objname)\n\n @property\n def obj(self):\n \"\"\"Dynamically grabs object\"\"\"\n names = self.objname.split(\".\")\n obj = builtins\n for name in names:\n obj = getattr(obj, name)\n return obj\n\n def __getattr__(self, name):\n return getattr(self.obj, name)\n\n def __setattr__(self, name, value):\n return super().__setattr__(self.obj, name, value)\n\n def __delattr__(self, name):\n return delattr(self.obj, name)\n\n def __getitem__(self, item):\n return self.obj.__getitem__(item)\n\n def __setitem__(self, item, value):\n return self.obj.__setitem__(item, value)\n\n def __delitem__(self, item):\n del self.obj[item]\n\n def __call__(self, *args, **kwargs):\n return self.obj.__call__(*args, **kwargs)\n\n\nclass DeprecationWarningProxy:\n \"\"\"Proxies access, but warns in the process.\"\"\"\n\n def __init__(self, oldname, newname):\n super().__setattr__(\"oldname\", oldname)\n super().__setattr__(\"newname\", newname)\n\n @property\n def obj(self):\n \"\"\"Dynamically grabs object\"\"\"\n names = self.newname.split(\".\")\n obj = builtins\n for name in names:\n obj = getattr(obj, name)\n return obj\n\n def warn(self):\n \"\"\"Issues deprecation warning.\"\"\"\n warnings.warn(\n \"{} has been deprecated, please use {} instead.\".format(\n self.oldname, self.newname\n ),\n DeprecationWarning,\n stacklevel=3,\n )\n\n def __getattr__(self, name):\n self.warn()\n return getattr(self.obj, name)\n\n def __setattr__(self, name, value):\n self.warn()\n return super().__setattr__(self.obj, name, value)\n\n def __delattr__(self, name):\n self.warn()\n return delattr(self.obj, name)\n\n def __getitem__(self, item):\n self.warn()\n return self.obj.__getitem__(item)\n\n def __setitem__(self, item, value):\n self.warn()\n return self.obj.__setitem__(item, value)\n\n def __delitem__(self, item):\n self.warn()\n del self.obj[item]\n\n def __call__(self, *args, **kwargs):\n self.warn()\n return self.obj.__call__(*args, **kwargs)\n\n\ndef load_proxies():\n \"\"\"Loads builtin dynamic access proxies.\n Also puts temporary shims in place for `__xonsh_*__` builtins.\n \"\"\"\n proxy_mapping = {\n \"XonshError\": \"__xonsh__.builtins.XonshError\",\n \"XonshCalledProcessError\": \"__xonsh__.builtins.XonshCalledProcessError\",\n \"evalx\": \"__xonsh__.builtins.evalx\",\n \"execx\": \"__xonsh__.builtins.execx\",\n \"compilex\": \"__xonsh__.builtins.compilex\",\n \"events\": \"__xonsh__.builtins.events\",\n }\n for refname, objname in proxy_mapping.items():\n proxy = DynamicAccessProxy(refname, objname)\n setattr(builtins, refname, proxy)\n\n deprecated_mapping = {\n \"__xonsh_env__\": \"__xonsh__.env\",\n \"__xonsh_history__\": \"__xonsh__.history\",\n \"__xonsh_ctx__\": \"__xonsh__.ctx\",\n \"__xonsh_help__\": \"__xonsh__.help\",\n \"__xonsh_superhelp__\": \"__xonsh__.superhelp\",\n \"__xonsh_pathsearch__\": \"__xonsh__.pathsearch\",\n \"__xonsh_globsearch__\": \"__xonsh__.globsearch\",\n \"__xonsh_regexsearch__\": \"__xonsh__.regexsearch\",\n \"__xonsh_glob__\": \"__xonsh__.glob\",\n \"__xonsh_expand_path__\": \"__xonsh__.expand_path\",\n \"__xonsh_exit__\": \"__xonsh__.exit\",\n \"__xonsh_stdout_uncaptured__\": \"__xonsh__.stdout_uncaptured\",\n \"__xonsh_stderr_uncaptured__\": \"__xonsh__.stderr_uncaptured\",\n \"__xonsh_subproc_captured_stdout__\": \"__xonsh__.subproc_captured_stdout\",\n \"__xonsh_subproc_captured_inject__\": \"__xonsh__.subproc_captured_inject\",\n \"__xonsh_subproc_captured_object__\": \"__xonsh__.subproc_captured_object\",\n \"__xonsh_subproc_captured_hiddenobject__\": \"__xonsh__.subproc_captured_hiddenobject\",\n \"__xonsh_subproc_uncaptured__\": \"__xonsh__.subproc_uncaptured\",\n \"__xonsh_execer__\": \"__xonsh__.execer\",\n \"__xonsh_commands_cache__\": \"__xonsh__.commands_cache\",\n \"__xonsh_all_jobs__\": \"__xonsh__.all_jobs\",\n \"__xonsh_ensure_list_of_strs__\": \"__xonsh__.ensure_list_of_strs\",\n \"__xonsh_list_of_strs_or_callables__\": \"__xonsh__.list_of_strs_or_callables\",\n \"__xonsh_list_of_list_of_strs_outer_product__\": \"__xonsh__.list_of_list_of_strs_outer_product\",\n \"__xonsh_completers__\": \"__xonsh__.completers\",\n \"__xonsh_call_macro__\": \"__xonsh__.call_macro\",\n \"__xonsh_enter_macro__\": \"__xonsh__.enter_macro\",\n \"__xonsh_path_literal__\": \"__xonsh__.path_literal\",\n }\n for badname, goodname in deprecated_mapping.items():\n proxy = DeprecationWarningProxy(badname, goodname)\n setattr(builtins, badname, proxy)\n\n if hasattr(builtins.__xonsh__, \"pyexit\"):\n builtins.__xonsh_pyexit__ = DeprecationWarningProxy(\n \"builtins.__xonsh_pyexit__\", \"builtins.__xonsh__.pyexit\"\n )\n if hasattr(builtins.__xonsh__, \"quit\"):\n builtins.__xonsh_pyquit__ = DeprecationWarningProxy(\n \"builtins.__xonsh_pyquit__\", \"builtins.__xonsh__.pyquit\"\n )\n\n\ndef unload_proxies():\n \"\"\"Removes the xonsh builtins (proxies) from the Python builtins.\n \"\"\"\n if hasattr(builtins, \"__xonsh_pyexit__\"):\n builtins.exit = builtins.__xonsh_pyexit__\n if hasattr(builtins, \"__xonsh_pyquit__\"):\n builtins.quit = builtins.__xonsh_pyquit__\n\n names = [\n \"__xonsh_env__\",\n \"__xonsh_ctx__\",\n \"__xonsh_help__\",\n \"__xonsh_superhelp__\",\n \"__xonsh_pathsearch__\",\n \"__xonsh_globsearch__\",\n \"__xonsh_regexsearch__\",\n \"__xonsh_glob__\",\n \"__xonsh_expand_path__\",\n \"__xonsh_exit__\",\n \"__xonsh_stdout_uncaptured__\",\n \"__xonsh_stderr_uncaptured__\",\n \"__xonsh_pyexit__\",\n \"__xonsh_pyquit__\",\n \"__xonsh_subproc_captured_stdout__\",\n \"__xonsh_subproc_captured_inject__\",\n \"__xonsh_subproc_captured_object__\",\n \"__xonsh_subproc_captured_hiddenobject__\",\n \"__xonsh_subproc_uncaptured__\",\n \"__xonsh_execer__\",\n \"__xonsh_commands_cache__\",\n \"__xonsh_completers__\",\n \"__xonsh_call_macro__\",\n \"__xonsh_enter_macro__\",\n \"__xonsh_path_literal__\",\n \"XonshError\",\n \"XonshCalledProcessError\",\n \"evalx\",\n \"execx\",\n \"compilex\",\n \"default_aliases\",\n \"__xonsh_all_jobs__\",\n \"__xonsh_ensure_list_of_strs__\",\n \"__xonsh_list_of_strs_or_callables__\",\n \"__xonsh_list_of_list_of_strs_outer_product__\",\n \"__xonsh_history__\",\n ]\n for name in names:\n if hasattr(builtins, name):\n delattr(builtins, name)\n" }, { "path": "xonsh/codecache.py", "content": "\"\"\"Tools for caching xonsh code.\"\"\"\nimport os\nimport sys\nimport hashlib\nimport marshal\nimport builtins\n\nfrom xonsh import __version__ as XONSH_VERSION\nfrom xonsh.lazyasd import lazyobject\nfrom xonsh.platform import PYTHON_VERSION_INFO_BYTES\n\n\ndef _splitpath(path, sofar=[]):\n folder, path = os.path.split(path)\n if path == \"\":\n return sofar[::-1]\n elif folder == \"\":\n return (sofar + [path])[::-1]\n else:\n return _splitpath(folder, sofar + [path])\n\n\n@lazyobject\ndef _CHARACTER_MAP():\n cmap = {chr(o): \"_%s\" % chr(o + 32) for o in range(65, 91)}\n cmap.update({\".\": \"_.\", \"_\": \"__\"})\n return cmap\n\n\ndef _cache_renamer(path, code=False):\n if not code:\n path = os.path.realpath(path)\n o = [\"\".join(_CHARACTER_MAP.get(i, i) for i in w) for w in _splitpath(path)]\n o[-1] = \"{}.{}\".format(o[-1], sys.implementation.cache_tag)\n return o\n\n\ndef _make_if_not_exists(dirname):\n if not os.path.isdir(dirname):\n os.makedirs(dirname)\n\n\ndef should_use_cache(execer, mode):\n \"\"\"\n Return ``True`` if caching has been enabled for this mode (through command\n line flags or environment variables)\n \"\"\"\n if mode == \"exec\":\n return (execer.scriptcache or execer.cacheall) and (\n builtins.__xonsh__.env[\"XONSH_CACHE_SCRIPTS\"]\n or builtins.__xonsh__.env[\"XONSH_CACHE_EVERYTHING\"]\n )\n else:\n return execer.cacheall or builtins.__xonsh__.env[\"XONSH_CACHE_EVERYTHING\"]\n\n\ndef run_compiled_code(code, glb, loc, mode):\n \"\"\"\n Helper to run code in a given mode and context\n \"\"\"\n if code is None:\n return\n if mode in {\"exec\", \"single\"}:\n func = exec\n else:\n func = eval\n func(code, glb, loc)\n\n\ndef get_cache_filename(fname, code=True):\n \"\"\"\n Return the filename of the cache for the given filename.\n\n Cache filenames are similar to those used by the Mercurial DVCS for its\n internal store.\n\n The ``code`` switch should be true if we should use the code store rather\n than the script store.\n \"\"\"\n datadir = builtins.__xonsh__.env[\"XONSH_DATA_DIR\"]\n cachedir = os.path.join(\n datadir, \"xonsh_code_cache\" if code else \"xonsh_script_cache\"\n )\n cachefname = os.path.join(cachedir, *_cache_renamer(fname, code=code))\n return cachefname\n\n\ndef update_cache(ccode, cache_file_name):\n \"\"\"\n Update the cache at ``cache_file_name`` to contain the compiled code\n represented by ``ccode``.\n \"\"\"\n if cache_file_name is not None:\n _make_if_not_exists(os.path.dirname(cache_file_name))\n with open(cache_file_name, \"wb\") as cfile:\n cfile.write(XONSH_VERSION.encode() + b\"\\n\")\n cfile.write(bytes(PYTHON_VERSION_INFO_BYTES) + b\"\\n\")\n marshal.dump(ccode, cfile)\n\n\ndef _check_cache_versions(cfile):\n # version data should be < 1 kb\n ver = cfile.readline(1024).strip()\n if ver != XONSH_VERSION.encode():\n return False\n ver = cfile.readline(1024).strip()\n return ver == PYTHON_VERSION_INFO_BYTES\n\n\ndef compile_code(filename, code, execer, glb, loc, mode):\n \"\"\"\n Wrapper for ``execer.compile`` to compile the given code\n \"\"\"\n try:\n if not code.endswith(\"\\n\"):\n code += \"\\n\"\n old_filename = execer.filename\n execer.filename = filename\n ccode = execer.compile(code, glbs=glb, locs=loc, mode=mode, filename=filename)\n except Exception:\n raise\n finally:\n execer.filename = old_filename\n return ccode\n\n\ndef script_cache_check(filename, cachefname):\n \"\"\"\n Check whether the script cache for a particular file is valid.\n\n Returns a tuple containing: a boolean representing whether the cached code\n should be used, and the cached code (or ``None`` if the cache should not be\n used).\n \"\"\"\n ccode = None\n run_cached = False\n if os.path.isfile(cachefname):\n if os.stat(cachefname).st_mtime >= os.stat(filename).st_mtime:\n with open(cachefname, \"rb\") as cfile:\n if not _check_cache_versions(cfile):\n return False, None\n ccode = marshal.load(cfile)\n run_cached = True\n return run_cached, ccode\n\n\ndef run_script_with_cache(filename, execer, glb=None, loc=None, mode=\"exec\"):\n \"\"\"\n Run a script, using a cached version if it exists (and the source has not\n changed), and updating the cache as necessary.\n \"\"\"\n run_cached = False\n use_cache = should_use_cache(execer, mode)\n cachefname = get_cache_filename(filename, code=False)\n if use_cache:\n run_cached, ccode = script_cache_check(filename, cachefname)\n if not run_cached:\n with open(filename, \"r\") as f:\n code = f.read()\n ccode = compile_code(filename, code, execer, glb, loc, mode)\n update_cache(ccode, cachefname)\n run_compiled_code(ccode, glb, loc, mode)\n\n\ndef code_cache_name(code):\n \"\"\"\n Return an appropriate spoofed filename for the given code.\n \"\"\"\n if isinstance(code, str):\n _code = code.encode()\n else:\n _code = code\n return hashlib.md5(_code).hexdigest()\n\n\ndef code_cache_check(cachefname):\n \"\"\"\n Check whether the code cache for a particular piece of code is valid.\n\n Returns a tuple containing: a boolean representing whether the cached code\n should be used, and the cached code (or ``None`` if the cache should not be\n used).\n \"\"\"\n ccode = None\n run_cached = False\n if os.path.isfile(cachefname):\n with open(cachefname, \"rb\") as cfile:\n if not _check_cache_versions(cfile):\n return False, None\n ccode = marshal.load(cfile)\n run_cached = True\n return run_cached, ccode\n\n\ndef run_code_with_cache(code, execer, glb=None, loc=None, mode=\"exec\"):\n \"\"\"\n Run a piece of code, using a cached version if it exists, and updating the\n cache as necessary.\n \"\"\"\n use_cache = should_use_cache(execer, mode)\n filename = code_cache_name(code)\n cachefname = get_cache_filename(filename, code=True)\n run_cached = False\n if use_cache:\n run_cached, ccode = code_cache_check(cachefname)\n if not run_cached:\n ccode = compile_code(filename, code, execer, glb, loc, mode)\n update_cache(ccode, cachefname)\n run_compiled_code(ccode, glb, loc, mode)\n" }, { "path": "xonsh/color_tools.py", "content": "\"\"\"Tools for color handling in xonsh.\n\nThis includes Convert values between RGB hex codes and xterm-256\ncolor codes. Parts of this file were originally forked from Micah Elliott\nhttp://MicahElliott.com Copyright (C) 2011 Micah Elliott. All rights reserved.\nWTFPL http://sam.zoy.org/wtfpl/\n\"\"\"\nimport re\nimport math\n\nfrom xonsh.lazyasd import lazyobject, LazyObject\n\n\nRE_BACKGROUND = LazyObject(\n lambda: re.compile(\"(BG#|BGHEX|BACKGROUND)\"), globals(), \"RE_BACKGROUND\"\n)\n\n\n@lazyobject\ndef BASE_XONSH_COLORS():\n return {\n \"BLACK\": (0, 0, 0),\n \"RED\": (170, 0, 0),\n \"GREEN\": (0, 170, 0),\n \"YELLOW\": (170, 85, 0),\n \"BLUE\": (0, 0, 170),\n \"PURPLE\": (170, 0, 170),\n \"CYAN\": (0, 170, 170),\n \"WHITE\": (170, 170, 170),\n \"INTENSE_BLACK\": (85, 85, 85),\n \"INTENSE_RED\": (255, 85, 85),\n \"INTENSE_GREEN\": (85, 255, 85),\n \"INTENSE_YELLOW\": (255, 255, 85),\n \"INTENSE_BLUE\": (85, 85, 255),\n \"INTENSE_PURPLE\": (255, 85, 255),\n \"INTENSE_CYAN\": (85, 255, 255),\n \"INTENSE_WHITE\": (255, 255, 255),\n }\n\n\n@lazyobject\ndef CLUT():\n \"\"\"color look-up table\"\"\"\n return [\n # 8-bit, RGB hex\n # Primary 3-bit (8 colors). Unique representation!\n (\"00\", \"000000\"),\n (\"01\", \"800000\"),\n (\"02\", \"008000\"),\n (\"03\", \"808000\"),\n (\"04\", \"000080\"),\n (\"05\", \"800080\"),\n (\"06\", \"008080\"),\n (\"07\", \"c0c0c0\"),\n # Equivalent \"bright\" versions of original 8 colors.\n (\"08\", \"808080\"),\n (\"09\", \"ff0000\"),\n (\"10\", \"00ff00\"),\n (\"11\", \"ffff00\"),\n (\"12\", \"0000ff\"),\n (\"13\", \"ff00ff\"),\n (\"14\", \"00ffff\"),\n (\"15\", \"ffffff\"),\n # Strictly ascending.\n (\"16\", \"000000\"),\n (\"17\", \"00005f\"),\n (\"18\", \"000087\"),\n (\"19\", \"0000af\"),\n (\"20\", \"0000d7\"),\n (\"21\", \"0000ff\"),\n (\"22\", \"005f00\"),\n (\"23\", \"005f5f\"),\n (\"24\", \"005f87\"),\n (\"25\", \"005faf\"),\n (\"26\", \"005fd7\"),\n (\"27\", \"005fff\"),\n (\"28\", \"008700\"),\n (\"29\", \"00875f\"),\n (\"30\", \"008787\"),\n (\"31\", \"0087af\"),\n (\"32\", \"0087d7\"),\n (\"33\", \"0087ff\"),\n (\"34\", \"00af00\"),\n (\"35\", \"00af5f\"),\n (\"36\", \"00af87\"),\n (\"37\", \"00afaf\"),\n (\"38\", \"00afd7\"),\n (\"39\", \"00afff\"),\n (\"40\", \"00d700\"),\n (\"41\", \"00d75f\"),\n (\"42\", \"00d787\"),\n (\"43\", \"00d7af\"),\n (\"44\", \"00d7d7\"),\n (\"45\", \"00d7ff\"),\n (\"46\", \"00ff00\"),\n (\"47\", \"00ff5f\"),\n (\"48\", \"00ff87\"),\n (\"49\", \"00ffaf\"),\n (\"50\", \"00ffd7\"),\n (\"51\", \"00ffff\"),\n (\"52\", \"5f0000\"),\n (\"53\", \"5f005f\"),\n (\"54\", \"5f0087\"),\n (\"55\", \"5f00af\"),\n (\"56\", \"5f00d7\"),\n (\"57\", \"5f00ff\"),\n (\"58\", \"5f5f00\"),\n (\"59\", \"5f5f5f\"),\n (\"60\", \"5f5f87\"),\n (\"61\", \"5f5faf\"),\n (\"62\", \"5f5fd7\"),\n (\"63\", \"5f5fff\"),\n (\"64\", \"5f8700\"),\n (\"65\", \"5f875f\"),\n (\"66\", \"5f8787\"),\n (\"67\", \"5f87af\"),\n (\"68\", \"5f87d7\"),\n (\"69\", \"5f87ff\"),\n (\"70\", \"5faf00\"),\n (\"71\", \"5faf5f\"),\n (\"72\", \"5faf87\"),\n (\"73\", \"5fafaf\"),\n (\"74\", \"5fafd7\"),\n (\"75\", \"5fafff\"),\n (\"76\", \"5fd700\"),\n (\"77\", \"5fd75f\"),\n (\"78\", \"5fd787\"),\n (\"79\", \"5fd7af\"),\n (\"80\", \"5fd7d7\"),\n (\"81\", \"5fd7ff\"),\n (\"82\", \"5fff00\"),\n (\"83\", \"5fff5f\"),\n (\"84\", \"5fff87\"),\n (\"85\", \"5fffaf\"),\n (\"86\", \"5fffd7\"),\n (\"87\", \"5fffff\"),\n (\"88\", \"870000\"),\n (\"89\", \"87005f\"),\n (\"90\", \"870087\"),\n (\"91\", \"8700af\"),\n (\"92\", \"8700d7\"),\n (\"93\", \"8700ff\"),\n (\"94\", \"875f00\"),\n (\"95\", \"875f5f\"),\n (\"96\", \"875f87\"),\n (\"97\", \"875faf\"),\n (\"98\", \"875fd7\"),\n (\"99\", \"875fff\"),\n (\"100\", \"878700\"),\n (\"101\", \"87875f\"),\n (\"102\", \"878787\"),\n (\"103\", \"8787af\"),\n (\"104\", \"8787d7\"),\n (\"105\", \"8787ff\"),\n (\"106\", \"87af00\"),\n (\"107\", \"87af5f\"),\n (\"108\", \"87af87\"),\n (\"109\", \"87afaf\"),\n (\"110\", \"87afd7\"),\n (\"111\", \"87afff\"),\n (\"112\", \"87d700\"),\n (\"113\", \"87d75f\"),\n (\"114\", \"87d787\"),\n (\"115\", \"87d7af\"),\n (\"116\", \"87d7d7\"),\n (\"117\", \"87d7ff\"),\n (\"118\", \"87ff00\"),\n (\"119\", \"87ff5f\"),\n (\"120\", \"87ff87\"),\n (\"121\", \"87ffaf\"),\n (\"122\", \"87ffd7\"),\n (\"123\", \"87ffff\"),\n (\"124\", \"af0000\"),\n (\"125\", \"af005f\"),\n (\"126\", \"af0087\"),\n (\"127\", \"af00af\"),\n (\"128\", \"af00d7\"),\n (\"129\", \"af00ff\"),\n (\"130\", \"af5f00\"),\n (\"131\", \"af5f5f\"),\n (\"132\", \"af5f87\"),\n (\"133\", \"af5faf\"),\n (\"134\", \"af5fd7\"),\n (\"135\", \"af5fff\"),\n (\"136\", \"af8700\"),\n (\"137\", \"af875f\"),\n (\"138\", \"af8787\"),\n (\"139\", \"af87af\"),\n (\"140\", \"af87d7\"),\n (\"141\", \"af87ff\"),\n (\"142\", \"afaf00\"),\n (\"143\", \"afaf5f\"),\n (\"144\", \"afaf87\"),\n (\"145\", \"afafaf\"),\n (\"146\", \"afafd7\"),\n (\"147\", \"afafff\"),\n (\"148\", \"afd700\"),\n (\"149\", \"afd75f\"),\n (\"150\", \"afd787\"),\n (\"151\", \"afd7af\"),\n (\"152\", \"afd7d7\"),\n (\"153\", \"afd7ff\"),\n (\"154\", \"afff00\"),\n (\"155\", \"afff5f\"),\n (\"156\", \"afff87\"),\n (\"157\", \"afffaf\"),\n (\"158\", \"afffd7\"),\n (\"159\", \"afffff\"),\n (\"160\", \"d70000\"),\n (\"161\", \"d7005f\"),\n (\"162\", \"d70087\"),\n (\"163\", \"d700af\"),\n (\"164\", \"d700d7\"),\n (\"165\", \"d700ff\"),\n (\"166\", \"d75f00\"),\n (\"167\", \"d75f5f\"),\n (\"168\", \"d75f87\"),\n (\"169\", \"d75faf\"),\n (\"170\", \"d75fd7\"),\n (\"171\", \"d75fff\"),\n (\"172\", \"d78700\"),\n (\"173\", \"d7875f\"),\n (\"174\", \"d78787\"),\n (\"175\", \"d787af\"),\n (\"176\", \"d787d7\"),\n (\"177\", \"d787ff\"),\n (\"178\", \"d7af00\"),\n (\"179\", \"d7af5f\"),\n (\"180\", \"d7af87\"),\n (\"181\", \"d7afaf\"),\n (\"182\", \"d7afd7\"),\n (\"183\", \"d7afff\"),\n (\"184\", \"d7d700\"),\n (\"185\", \"d7d75f\"),\n (\"186\", \"d7d787\"),\n (\"187\", \"d7d7af\"),\n (\"188\", \"d7d7d7\"),\n (\"189\", \"d7d7ff\"),\n (\"190\", \"d7ff00\"),\n (\"191\", \"d7ff5f\"),\n (\"192\", \"d7ff87\"),\n (\"193\", \"d7ffaf\"),\n (\"194\", \"d7ffd7\"),\n (\"195\", \"d7ffff\"),\n (\"196\", \"ff0000\"),\n (\"197\", \"ff005f\"),\n (\"198\", \"ff0087\"),\n (\"199\", \"ff00af\"),\n (\"200\", \"ff00d7\"),\n (\"201\", \"ff00ff\"),\n (\"202\", \"ff5f00\"),\n (\"203\", \"ff5f5f\"),\n (\"204\", \"ff5f87\"),\n (\"205\", \"ff5faf\"),\n (\"206\", \"ff5fd7\"),\n (\"207\", \"ff5fff\"),\n (\"208\", \"ff8700\"),\n (\"209\", \"ff875f\"),\n (\"210\", \"ff8787\"),\n (\"211\", \"ff87af\"),\n (\"212\", \"ff87d7\"),\n (\"213\", \"ff87ff\"),\n (\"214\", \"ffaf00\"),\n (\"215\", \"ffaf5f\"),\n (\"216\", \"ffaf87\"),\n (\"217\", \"ffafaf\"),\n (\"218\", \"ffafd7\"),\n (\"219\", \"ffafff\"),\n (\"220\", \"ffd700\"),\n (\"221\", \"ffd75f\"),\n (\"222\", \"ffd787\"),\n (\"223\", \"ffd7af\"),\n (\"224\", \"ffd7d7\"),\n (\"225\", \"ffd7ff\"),\n (\"226\", \"ffff00\"),\n (\"227\", \"ffff5f\"),\n (\"228\", \"ffff87\"),\n (\"229\", \"ffffaf\"),\n (\"230\", \"ffffd7\"),\n (\"231\", \"ffffff\"),\n # Gray-scale range.\n (\"232\", \"080808\"),\n (\"233\", \"121212\"),\n (\"234\", \"1c1c1c\"),\n (\"235\", \"262626\"),\n (\"236\", \"303030\"),\n (\"237\", \"3a3a3a\"),\n (\"238\", \"444444\"),\n (\"239\", \"4e4e4e\"),\n (\"240\", \"585858\"),\n (\"241\", \"626262\"),\n (\"242\", \"6c6c6c\"),\n (\"243\", \"767676\"),\n (\"244\", \"808080\"),\n (\"245\", \"8a8a8a\"),\n (\"246\", \"949494\"),\n (\"247\", \"9e9e9e\"),\n (\"248\", \"a8a8a8\"),\n (\"249\", \"b2b2b2\"),\n (\"250\", \"bcbcbc\"),\n (\"251\", \"c6c6c6\"),\n (\"252\", \"d0d0d0\"),\n (\"253\", \"dadada\"),\n (\"254\", \"e4e4e4\"),\n (\"255\", \"eeeeee\"),\n ]\n\n\ndef _str2hex(hexstr):\n return int(hexstr, 16)\n\n\ndef _strip_hash(rgb):\n # Strip leading `#` if exists.\n if rgb.startswith(\"#\"):\n rgb = rgb.lstrip(\"#\")\n return rgb\n\n\n@lazyobject\ndef SHORT_TO_RGB():\n return dict(CLUT)\n\n\n@lazyobject\ndef RGB_TO_SHORT():\n return {v: k for k, v in SHORT_TO_RGB.items()}\n\n\ndef short2rgb(short):\n return SHORT_TO_RGB[short]\n\n\ndef rgb_to_256(rgb):\n \"\"\"Find the closest ANSI 256 approximation to the given RGB value.\n\n >>> rgb2short('123456')\n ('23', '005f5f')\n >>> rgb2short('ffffff')\n ('231', 'ffffff')\n >>> rgb2short('0DADD6') # vimeo logo\n ('38', '00afd7')\n\n Parameters\n ----------\n rgb : Hex code representing an RGB value, eg, 'abcdef'\n\n Returns\n -------\n String between 0 and 255, compatible with xterm.\n \"\"\"\n rgb = rgb.lstrip(\"#\")\n if len(rgb) == 0:\n return \"0\", \"000000\"\n incs = (0x00, 0x5F, 0x87, 0xAF, 0xD7, 0xFF)\n # Break 6-char RGB code into 3 integer vals.\n parts = rgb_to_ints(rgb)\n res = []\n for part in parts:\n i = 0\n while i < len(incs) - 1:\n s, b = incs[i], incs[i + 1] # smaller, bigger\n if s <= part <= b:\n s1 = abs(s - part)\n b1 = abs(b - part)\n if s1 < b1:\n closest = s\n else:\n closest = b\n res.append(closest)\n break\n i += 1\n res = \"\".join([(\"%02.x\" % i) for i in res])\n equiv = RGB_TO_SHORT[res]\n return equiv, res\n\n\nrgb2short = rgb_to_256\n\n\n@lazyobject\ndef RE_RGB3():\n return re.compile(r\"(.)(.)(.)\")\n\n\n@lazyobject\ndef RE_RGB6():\n return re.compile(r\"(..)(..)(..)\")\n\n\ndef rgb_to_ints(rgb):\n if len(rgb) == 6:\n return tuple([int(h, 16) for h in RE_RGB6.split(rgb)[1:4]])\n else:\n return tuple([int(h * 2, 16) for h in RE_RGB3.split(rgb)[1:4]])\n\n\ndef short_to_ints(short):\n \"\"\"Coverts a short (256) color to a 3-tuple of ints.\"\"\"\n return rgb_to_ints(short2rgb(short))\n\n\ndef color_dist(x, y):\n return math.sqrt((x[0] - y[0]) ** 2 + (x[1] - y[1]) ** 2 + (x[2] - y[2]) ** 2)\n\n\ndef find_closest_color(x, palette):\n return min(sorted(palette.keys())[::-1], key=lambda k: color_dist(x, palette[k]))\n\n\ndef make_palette(strings):\n \"\"\"Makes a color palette from a collection of strings.\"\"\"\n palette = {}\n for s in strings:\n while \"#\" in s:\n _, t = s.split(\"#\", 1)\n t, _, s = t.partition(\" \")\n palette[t] = rgb_to_ints(t)\n return palette\n" }, { "path": "xonsh/commands_cache.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"Module for caching command & alias names as well as for predicting whether\na command will be able to be run in the background.\n\nA background predictor is a function that accepts a single argument list\nand returns whether or not the process can be run in the background (returns\nTrue) or must be run the foreground (returns False).\n\"\"\"\nimport os\nimport time\nimport builtins\nimport argparse\nimport collections.abc as cabc\n\nfrom xonsh.platform import ON_WINDOWS, ON_POSIX, pathbasename\nfrom xonsh.tools import executables_in\nfrom xonsh.lazyasd import lazyobject\n\n\nclass CommandsCache(cabc.Mapping):\n \"\"\"A lazy cache representing the commands available on the file system.\n The keys are the command names and the values a tuple of (loc, has_alias)\n where loc is either a str pointing to the executable on the file system or\n None (if no executable exists) and has_alias is a boolean flag for whether\n the command has an alias.\n \"\"\"\n\n def __init__(self):\n self._cmds_cache = {}\n self._path_checksum = None\n self._alias_checksum = None\n self._path_mtime = -1\n self.threadable_predictors = default_threadable_predictors()\n\n def __contains__(self, key):\n _ = self.all_commands\n return self.lazyin(key)\n\n def __iter__(self):\n for cmd, (path, is_alias) in self.all_commands.items():\n if ON_WINDOWS and path is not None:\n # All command keys are stored in uppercase on Windows.\n # This ensures the original command name is returned.\n cmd = pathbasename(path)\n yield cmd\n\n def __len__(self):\n return len(self.all_commands)\n\n def __getitem__(self, key):\n _ = self.all_commands\n return self.lazyget(key)\n\n def is_empty(self):\n \"\"\"Returns whether the cache is populated or not.\"\"\"\n return len(self._cmds_cache) == 0\n\n @staticmethod\n def get_possible_names(name):\n \"\"\"Generates the possible `PATHEXT` extension variants of a given executable\n name on Windows as a list, conserving the ordering in `PATHEXT`.\n Returns a list as `name` being the only item in it on other platforms.\"\"\"\n if ON_WINDOWS:\n pathext = builtins.__xonsh__.env.get(\"PATHEXT\", [])\n name = name.upper()\n return [name + ext for ext in ([\"\"] + pathext)]\n else:\n return [name]\n\n @staticmethod\n def remove_dups(p):\n ret = list()\n for e in p:\n if e not in ret:\n ret.append(e)\n return ret\n\n @property\n def all_commands(self):\n paths = builtins.__xonsh__.env.get(\"PATH\", [])\n paths = CommandsCache.remove_dups(paths)\n path_immut = tuple(x for x in paths if os.path.isdir(x))\n # did PATH change?\n path_hash = hash(path_immut)\n cache_valid = path_hash == self._path_checksum\n self._path_checksum = path_hash\n # did aliases change?\n alss = getattr(builtins, \"aliases\", dict())\n al_hash = hash(frozenset(alss))\n cache_valid = cache_valid and al_hash == self._alias_checksum\n self._alias_checksum = al_hash\n # did the contents of any directory in PATH change?\n max_mtime = 0\n for path in path_immut:\n mtime = os.stat(path).st_mtime\n if mtime > max_mtime:\n max_mtime = mtime\n cache_valid = cache_valid and (max_mtime <= self._path_mtime)\n self._path_mtime = max_mtime\n if cache_valid:\n return self._cmds_cache\n allcmds = {}\n for path in reversed(path_immut):\n # iterate backwards so that entries at the front of PATH overwrite\n # entries at the back.\n for cmd in executables_in(path):\n key = cmd.upper() if ON_WINDOWS else cmd\n allcmds[key] = (os.path.join(path, cmd), alss.get(key, None))\n for cmd in alss:\n if cmd not in allcmds:\n key = cmd.upper() if ON_WINDOWS else cmd\n allcmds[key] = (cmd, True)\n self._cmds_cache = allcmds\n return allcmds\n\n def cached_name(self, name):\n \"\"\"Returns the name that would appear in the cache, if it exists.\"\"\"\n if name is None:\n return None\n cached = pathbasename(name)\n if ON_WINDOWS:\n keys = self.get_possible_names(cached)\n cached = next((k for k in keys if k in self._cmds_cache), None)\n return cached\n\n def lazyin(self, key):\n \"\"\"Checks if the value is in the current cache without the potential to\n update the cache. It just says whether the value is known *now*. This\n may not reflect precisely what is on the $PATH.\n \"\"\"\n return self.cached_name(key) in self._cmds_cache\n\n def lazyiter(self):\n \"\"\"Returns an iterator over the current cache contents without the\n potential to update the cache. This may not reflect what is on the\n $PATH.\n \"\"\"\n return iter(self._cmds_cache)\n\n def lazylen(self):\n \"\"\"Returns the length of the current cache contents without the\n potential to update the cache. This may not reflect precisely\n what is on the $PATH.\n \"\"\"\n return len(self._cmds_cache)\n\n def lazyget(self, key, default=None):\n \"\"\"A lazy value getter.\"\"\"\n return self._cmds_cache.get(self.cached_name(key), default)\n\n def locate_binary(self, name, ignore_alias=False):\n \"\"\"Locates an executable on the file system using the cache.\n\n Arguments\n ---------\n name : str\n name of binary to search for\n ignore_alias : bool, optional\n Force return of binary path even if alias of ``name`` exists\n (default ``False``)\n \"\"\"\n # make sure the cache is up to date by accessing the property\n _ = self.all_commands\n return self.lazy_locate_binary(name, ignore_alias)\n\n def lazy_locate_binary(self, name, ignore_alias=False):\n \"\"\"Locates an executable in the cache, without checking its validity.\n\n Arguments\n ---------\n name : str\n name of binary to search for\n ignore_alias : bool, optional\n Force return of binary path even if alias of ``name`` exists\n (default ``False``)\n \"\"\"\n possibilities = self.get_possible_names(name)\n if ON_WINDOWS:\n # Windows users expect to be able to execute files in the same\n # directory without `./`\n local_bin = next((fn for fn in possibilities if os.path.isfile(fn)), None)\n if local_bin:\n return os.path.abspath(local_bin)\n cached = next((cmd for cmd in possibilities if cmd in self._cmds_cache), None)\n if cached:\n (path, alias) = self._cmds_cache[cached]\n ispure = path == pathbasename(path)\n if alias and ignore_alias and ispure:\n # pure alias, which we are ignoring\n return None\n else:\n return path\n elif os.path.isfile(name) and name != pathbasename(name):\n return name\n\n def is_only_functional_alias(self, name):\n \"\"\"Returns whether or not a command is only a functional alias, and has\n no underlying executable. For example, the \"cd\" command is only available\n as a functional alias.\n \"\"\"\n _ = self.all_commands\n return self.lazy_is_only_functional_alias(name)\n\n def lazy_is_only_functional_alias(self, name):\n \"\"\"Returns whether or not a command is only a functional alias, and has\n no underlying executable. For example, the \"cd\" command is only available\n as a functional alias. This search is performed lazily.\n \"\"\"\n val = self._cmds_cache.get(name, None)\n if val is None:\n return False\n return (\n val == (name, True) and self.locate_binary(name, ignore_alias=True) is None\n )\n\n def predict_threadable(self, cmd):\n \"\"\"Predicts whether a command list is able to be run on a background\n thread, rather than the main thread.\n \"\"\"\n name = self.cached_name(cmd[0])\n predictors = self.threadable_predictors\n if ON_WINDOWS:\n # On all names (keys) are stored in upper case so instead\n # we get the original cmd or alias name\n path, _ = self.lazyget(name, (None, None))\n if path is None:\n return True\n else:\n name = pathbasename(path)\n if name not in predictors:\n pre, ext = os.path.splitext(name)\n if pre in predictors:\n predictors[name] = predictors[pre]\n if name not in predictors:\n predictors[name] = self.default_predictor(name, cmd[0])\n predictor = predictors[name]\n return predictor(cmd[1:])\n\n #\n # Background Predictors (as methods)\n #\n\n def default_predictor(self, name, cmd0):\n if ON_POSIX:\n return self.default_predictor_readbin(\n name, cmd0, timeout=0.1, failure=predict_true\n )\n else:\n return predict_true\n\n def default_predictor_readbin(self, name, cmd0, timeout, failure):\n \"\"\"Make a default predictor by\n analyzing the content of the binary. Should only works on POSIX.\n Return failure if the analysis fails.\n \"\"\"\n fname = cmd0 if os.path.isabs(cmd0) else None\n fname = cmd0 if fname is None and os.sep in cmd0 else fname\n fname = self.lazy_locate_binary(name) if fname is None else fname\n\n if fname is None:\n return failure\n if not os.path.isfile(fname):\n return failure\n\n try:\n fd = os.open(fname, os.O_RDONLY | os.O_NONBLOCK)\n except Exception:\n return failure # opening error\n\n search_for = {\n (b\"ncurses\",): [False],\n (b\"libgpm\",): [False],\n (b\"isatty\", b\"tcgetattr\", b\"tcsetattr\"): [False, False, False],\n }\n tstart = time.time()\n block = b\"\"\n while time.time() < tstart + timeout:\n previous_block = block\n try:\n block = os.read(fd, 2048)\n except Exception:\n # should not occur, except e.g. if a file is deleted a a dir is\n # created with the same name between os.path.isfile and os.open\n os.close(fd)\n return failure\n if len(block) == 0:\n os.close(fd)\n return predict_true # no keys of search_for found\n analyzed_block = previous_block + block\n for k, v in search_for.items():\n for i in range(len(k)):\n if v[i]:\n continue\n if k[i] in analyzed_block:\n v[i] = True\n if all(v):\n os.close(fd)\n return predict_false # use one key of search_for\n os.close(fd)\n return failure # timeout\n\n\n#\n# Background Predictors\n#\n\n\ndef predict_true(args):\n \"\"\"Always say the process is threadable.\"\"\"\n return True\n\n\ndef predict_false(args):\n \"\"\"Never say the process is threadable.\"\"\"\n return False\n\n\n@lazyobject\ndef SHELL_PREDICTOR_PARSER():\n p = argparse.ArgumentParser(\"shell\", add_help=False)\n p.add_argument(\"-c\", nargs=\"?\", default=None)\n p.add_argument(\"filename\", nargs=\"?\", default=None)\n return p\n\n\ndef predict_shell(args):\n \"\"\"Predict the backgroundability of the normal shell interface, which\n comes down to whether it is being run in subproc mode.\n \"\"\"\n ns, _ = SHELL_PREDICTOR_PARSER.parse_known_args(args)\n if ns.c is None and ns.filename is None:\n pred = False\n else:\n pred = True\n return pred\n\n\n@lazyobject\ndef HELP_VER_PREDICTOR_PARSER():\n p = argparse.ArgumentParser(\"cmd\", add_help=False)\n p.add_argument(\"-h\", \"--help\", dest=\"help\", action=\"store_true\", default=None)\n p.add_argument(\n \"-v\", \"-V\", \"--version\", dest=\"version\", action=\"store_true\", default=None\n )\n return p\n\n\ndef predict_help_ver(args):\n \"\"\"Predict the backgroundability of commands that have help & version\n switches: -h, --help, -v, -V, --version. If either of these options is\n present, the command is assumed to print to stdout normally and is therefore\n threadable. Otherwise, the command is assumed to not be threadable.\n This is useful for commands, like top, that normally enter alternate mode\n but may not in certain circumstances.\n \"\"\"\n ns, _ = HELP_VER_PREDICTOR_PARSER.parse_known_args(args)\n pred = ns.help is not None or ns.version is not None\n return pred\n\n\n@lazyobject\ndef HG_PREDICTOR_PARSER():\n p = argparse.ArgumentParser(\"hg\", add_help=False)\n p.add_argument(\"command\")\n p.add_argument(\n \"-i\", \"--interactive\", action=\"store_true\", default=False, dest=\"interactive\"\n )\n return p\n\n\ndef predict_hg(args):\n \"\"\"Predict if mercurial is about to be run in interactive mode.\n If it is interactive, predict False. If it isn't, predict True.\n Also predict False for certain commands, such as split.\n \"\"\"\n ns, _ = HG_PREDICTOR_PARSER.parse_known_args(args)\n if ns.command == \"split\":\n return False\n else:\n return not ns.interactive\n\n\ndef default_threadable_predictors():\n \"\"\"Generates a new defaultdict for known threadable predictors.\n The default is to predict true.\n \"\"\"\n # alphabetical, for what it is worth.\n predictors = {\n \"aurman\": predict_false,\n \"bash\": predict_shell,\n \"csh\": predict_shell,\n \"clear\": predict_false,\n \"cls\": predict_false,\n \"cmd\": predict_shell,\n \"cryptop\": predict_false,\n \"curl\": predict_true,\n \"ex\": predict_false,\n \"emacsclient\": predict_false,\n \"fish\": predict_shell,\n \"gvim\": predict_help_ver,\n \"hg\": predict_hg,\n \"htop\": predict_help_ver,\n \"ipython\": predict_shell,\n \"ksh\": predict_shell,\n \"less\": predict_help_ver,\n \"ls\": predict_true,\n \"man\": predict_help_ver,\n \"more\": predict_help_ver,\n \"mvim\": predict_help_ver,\n \"mutt\": predict_help_ver,\n \"nano\": predict_help_ver,\n \"nvim\": predict_false,\n \"ponysay\": predict_help_ver,\n \"psql\": predict_false,\n \"python\": predict_shell,\n \"python2\": predict_shell,\n \"python3\": predict_shell,\n \"repo\": predict_help_ver,\n \"ranger\": predict_help_ver,\n \"rview\": predict_false,\n \"rvim\": predict_false,\n \"scp\": predict_false,\n \"sh\": predict_shell,\n \"ssh\": predict_false,\n \"startx\": predict_false,\n \"sudo\": predict_help_ver,\n \"tcsh\": predict_shell,\n \"telnet\": predict_false,\n \"top\": predict_help_ver,\n \"vi\": predict_false,\n \"view\": predict_false,\n \"vim\": predict_false,\n \"vimpager\": predict_help_ver,\n \"weechat\": predict_help_ver,\n \"xclip\": predict_help_ver,\n \"xo\": predict_help_ver,\n \"xonsh\": predict_shell,\n \"xon.sh\": predict_shell,\n \"zsh\": predict_shell,\n }\n return predictors\n" }, { "path": "xonsh/completer.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"A (tab-)completer for xonsh.\"\"\"\nimport builtins\nimport collections.abc as cabc\n\n\nclass Completer(object):\n \"\"\"This provides a list of optional completions for the xonsh shell.\"\"\"\n\n def complete(self, prefix, line, begidx, endidx, ctx=None):\n \"\"\"Complete the string, given a possible execution context.\n\n Parameters\n ----------\n prefix : str\n The string to match\n line : str\n The line that prefix appears on.\n begidx : int\n The index in line that prefix starts on.\n endidx : int\n The index in line that prefix ends on.\n ctx : Iterable of str (ie dict, set, etc), optional\n Names in the current execution context.\n\n Returns\n -------\n rtn : list of str\n Possible completions of prefix, sorted alphabetically.\n lprefix : int\n Length of the prefix to be replaced in the completion.\n \"\"\"\n ctx = ctx or {}\n for func in builtins.__xonsh__.completers.values():\n try:\n out = func(prefix, line, begidx, endidx, ctx)\n except StopIteration:\n return set(), len(prefix)\n if isinstance(out, cabc.Sequence):\n res, lprefix = out\n else:\n res = out\n lprefix = len(prefix)\n if res is not None and len(res) != 0:\n\n def sortkey(s):\n return s.lstrip(''''\"''').lower()\n\n return tuple(sorted(res, key=sortkey)), lprefix\n return set(), lprefix\n" }, { "path": "xonsh/completers/__init__.py", "content": "# amalgamate exclude\nimport os as _os\n\nif _os.getenv(\"XONSH_DEBUG\", \"\"):\n pass\nelse:\n import sys as _sys\n\n try:\n from xonsh.completers import __amalgam__\n\n bash_completion = __amalgam__\n _sys.modules[\"xonsh.completers.bash_completion\"] = __amalgam__\n completer = __amalgam__\n _sys.modules[\"xonsh.completers.completer\"] = __amalgam__\n pip = __amalgam__\n _sys.modules[\"xonsh.completers.pip\"] = __amalgam__\n tools = __amalgam__\n _sys.modules[\"xonsh.completers.tools\"] = __amalgam__\n xompletions = __amalgam__\n _sys.modules[\"xonsh.completers.xompletions\"] = __amalgam__\n _aliases = __amalgam__\n _sys.modules[\"xonsh.completers._aliases\"] = __amalgam__\n commands = __amalgam__\n _sys.modules[\"xonsh.completers.commands\"] = __amalgam__\n man = __amalgam__\n _sys.modules[\"xonsh.completers.man\"] = __amalgam__\n path = __amalgam__\n _sys.modules[\"xonsh.completers.path\"] = __amalgam__\n python = __amalgam__\n _sys.modules[\"xonsh.completers.python\"] = __amalgam__\n base = __amalgam__\n _sys.modules[\"xonsh.completers.base\"] = __amalgam__\n bash = __amalgam__\n _sys.modules[\"xonsh.completers.bash\"] = __amalgam__\n dirs = __amalgam__\n _sys.modules[\"xonsh.completers.dirs\"] = __amalgam__\n init = __amalgam__\n _sys.modules[\"xonsh.completers.init\"] = __amalgam__\n del __amalgam__\n except ImportError:\n pass\n del _sys\ndel _os\n# amalgamate end\n" }, { "path": "xonsh/completers/_aliases.py", "content": "import builtins\nimport collections\n\nimport xonsh.lazyasd as xl\n\nfrom xonsh.completers.tools import justify\n\n\nVALID_ACTIONS = xl.LazyObject(\n lambda: frozenset({\"add\", \"remove\", \"list\"}), globals(), \"VALID_ACTIONS\"\n)\n\n\ndef _add_one_completer(name, func, loc=\"end\"):\n new = collections.OrderedDict()\n if loc == \"start\":\n new[name] = func\n for (k, v) in builtins.__xonsh__.completers.items():\n new[k] = v\n elif loc == \"end\":\n for (k, v) in builtins.__xonsh__.completers.items():\n new[k] = v\n new[name] = func\n else:\n direction, rel = loc[0], loc[1:]\n found = False\n for (k, v) in builtins.__xonsh__.completers.items():\n if rel == k and direction == \"<\":\n new[name] = func\n found = True\n new[k] = v\n if rel == k and direction == \">\":\n new[name] = func\n found = True\n if not found:\n new[name] = func\n builtins.__xonsh__.completers.clear()\n builtins.__xonsh__.completers.update(new)\n\n\ndef _list_completers(args, stdin=None, stack=None):\n o = \"Registered Completer Functions: \\n\"\n _comp = builtins.__xonsh__.completers\n ml = max((len(i) for i in _comp), default=0)\n _strs = []\n for c in _comp:\n if _comp[c].__doc__ is None:\n doc = \"No description provided\"\n else:\n doc = \" \".join(_comp[c].__doc__.split())\n doc = justify(doc, 80, ml + 3)\n _strs.append(\"{: >{}} : {}\".format(c, ml, doc))\n return o + \"\\n\".join(_strs) + \"\\n\"\n\n\ndef _remove_completer(args, stdin=None, stack=None):\n err = None\n if len(args) != 1:\n err = \"completer remove takes exactly 1 argument.\"\n else:\n name = args[0]\n if name not in builtins.__xonsh__.completers:\n err = (\"The name %s is not a registered \" \"completer function.\") % name\n if err is None:\n del builtins.__xonsh__.completers[name]\n return\n else:\n return None, err + \"\\n\", 1\n\n\ndef _register_completer(args, stdin=None, stack=None):\n err = None\n if len(args) not in {2, 3}:\n err = (\n \"completer add takes either 2 or 3 arguments.\\n\"\n \"For help, run: completer help add\"\n )\n else:\n name = args[0]\n func_name = args[1]\n if name in builtins.__xonsh__.completers:\n err = (\"The name %s is already a registered \" \"completer function.\") % name\n else:\n if func_name in builtins.__xonsh__.ctx:\n func = builtins.__xonsh__.ctx[func_name]\n if not callable(func):\n err = \"%s is not callable\" % func_name\n else:\n for frame_info in stack:\n frame = frame_info[0]\n if func_name in frame.f_locals:\n func = frame.f_locals[func_name]\n break\n elif func_name in frame.f_globals:\n func = frame.f_globals[func_name]\n break\n else:\n err = \"No such function: %s\" % func_name\n if err is None:\n position = \"start\" if len(args) == 2 else args[2]\n _add_one_completer(name, func, position)\n else:\n return None, err + \"\\n\", 1\n\n\ndef completer_alias(args, stdin=None, stdout=None, stderr=None, spec=None, stack=None):\n err = None\n if len(args) == 0 or args[0] not in (VALID_ACTIONS | {\"help\"}):\n err = (\n \"Please specify an action. Valid actions are: \"\n '\"add\", \"remove\", \"list\", or \"help\".'\n )\n elif args[0] == \"help\":\n if len(args) == 1 or args[1] not in VALID_ACTIONS:\n return (\n \"Valid actions are: add, remove, list. For help with a \"\n \"specific action, run: completer help ACTION\\n\"\n )\n elif args[1] == \"add\":\n return COMPLETER_ADD_HELP_STR\n elif args[1] == \"remove\":\n return COMPLETER_REMOVE_HELP_STR\n elif args[1] == \"list\":\n return COMPLETER_LIST_HELP_STR\n\n if err is not None:\n return None, err + \"\\n\", 1\n\n if args[0] == \"add\":\n func = _register_completer\n elif args[0] == \"remove\":\n func = _remove_completer\n elif args[0] == \"list\":\n func = _list_completers\n return func(args[1:], stdin=stdin, stack=stack)\n\n\nCOMPLETER_LIST_HELP_STR = \"\"\"completer list: ordered list the active completers\n\nUsage:\n completer remove\n\"\"\"\n\nCOMPLETER_REMOVE_HELP_STR = \"\"\"completer remove: removes a completer from xonsh\n\nUsage:\n completer remove NAME\n\nNAME is a unique name of a completer (run \"completer list\" to see the current\n completers in order)\n\"\"\"\n\nCOMPLETER_ADD_HELP_STR = \"\"\"completer add: adds a new completer to xonsh\n\nUsage:\n completer add NAME FUNC [POS]\n\nNAME is a unique name to use in the listing (run \"completer list\" to see the\n current completers in order)\n\nFUNC is the name of a completer function to use. This should be a function\n of the following arguments, and should return a set of valid completions\n for the given prefix. If this completer should not be used in a given\n context, it should return an empty set or None.\n\n Arguments to FUNC:\n * prefix: the string to be matched\n * line: a string representing the whole current line, for context\n * begidx: the index at which prefix starts in line\n * endidx: the index at which prefix ends in line\n * ctx: the current Python environment\n\n If the completer expands the prefix in any way, it should return a tuple\n of two elements: the first should be the set of completions, and the\n second should be the length of the modified prefix (for an example, see\n xonsh.completers.path.complete_path).\n\nPOS (optional) is a position into the list of completers at which the new\n completer should be added. It can be one of the following values:\n * \"start\" indicates that the completer should be added to the start of\n the list of completers (it should be run before all others)\n * \"end\" indicates that the completer should be added to the end of the\n list of completers (it should be run after all others)\n * \">KEY\", where KEY is a pre-existing name, indicates that this should\n be added after the completer named KEY\n * \" /dev/null || echo \"-F _minimal\"\n}}\n\n_complete_stmt=$(_get_complete_statement)\nif echo \"$_complete_stmt\" | grep --quiet -e \"_minimal\"\nthen\n declare -f _completion_loader > /dev/null && _completion_loader {cmd}\n _complete_stmt=$(_get_complete_statement)\nfi\n\n_func=$(echo \"$_complete_stmt\" | grep -o -e '-F \\w\\+' | cut -d ' ' -f 2)\ndeclare -f \"$_func\" > /dev/null || exit 1\n\necho \"$_complete_stmt\"\nCOMP_WORDS=({line})\nCOMP_LINE={comp_line}\nCOMP_POINT=${{#COMP_LINE}}\nCOMP_COUNT={end}\nCOMP_CWORD={n}\n$_func {cmd} {prefix} {prev}\n\n# print out completions, right-stripped if they contain no internal spaces\nshopt -s extglob\nfor ((i=0;i<${{#COMPREPLY[*]}};i++))\ndo\n no_spaces=\"${{COMPREPLY[i]//[[:space:]]}}\"\n no_trailing_spaces=\"${{COMPREPLY[i]%%+([[:space:]])}}\"\n if [[ \"$no_spaces\" == \"$no_trailing_spaces\" ]]; then\n echo \"$no_trailing_spaces\"\n else\n echo \"${{COMPREPLY[i]}}\"\n fi\ndone\n\"\"\"\n\n\ndef bash_completions(\n prefix,\n line,\n begidx,\n endidx,\n env=None,\n paths=None,\n command=None,\n quote_paths=_bash_quote_paths,\n **kwargs\n):\n \"\"\"Completes based on results from BASH completion.\n\n Parameters\n ----------\n prefix : str\n The string to match\n line : str\n The line that prefix appears on.\n begidx : int\n The index in line that prefix starts on.\n endidx : int\n The index in line that prefix ends on.\n env : Mapping, optional\n The environment dict to execute the Bash subprocess in.\n paths : list or tuple of str or None, optional\n This is a list (or tuple) of strings that specifies where the\n ``bash_completion`` script may be found. The first valid path will\n be used. For better performance, bash-completion v2.x is recommended\n since it lazy-loads individual completion scripts. For both\n bash-completion v1.x and v2.x, paths of individual completion scripts\n (like ``.../completes/ssh``) do not need to be included here. The\n default values are platform dependent, but sane.\n command : str or None, optional\n The /path/to/bash to use. If None, it will be selected based on the\n from the environment and platform.\n quote_paths : callable, optional\n A functions that quotes file system paths. You shouldn't normally need\n this as the default is acceptable 99+% of the time. This function should\n return a set of the new paths and a boolean for whether the paths were\n quoted.\n\n Returns\n -------\n rtn : set of str\n Possible completions of prefix\n lprefix : int\n Length of the prefix to be replaced in the completion.\n \"\"\"\n source = _get_bash_completions_source(paths) or \"\"\n\n if prefix.startswith(\"$\"): # do not complete env variables\n return set(), 0\n\n splt = line.split()\n cmd = splt[0]\n idx = n = 0\n prev = \"\"\n for n, tok in enumerate(splt):\n if tok == prefix:\n idx = line.find(prefix, idx)\n if idx >= begidx:\n break\n prev = tok\n\n if len(prefix) == 0:\n prefix_quoted = '\"\"'\n n += 1\n else:\n prefix_quoted = shlex.quote(prefix)\n\n script = BASH_COMPLETE_SCRIPT.format(\n source=source,\n line=\" \".join(shlex.quote(p) for p in splt),\n comp_line=shlex.quote(line),\n n=n,\n cmd=shlex.quote(cmd),\n end=endidx + 1,\n prefix=prefix_quoted,\n prev=shlex.quote(prev),\n )\n\n if command is None:\n command = _bash_command(env=env)\n try:\n out = subprocess.check_output(\n [command, \"-c\", script],\n universal_newlines=True,\n stderr=subprocess.PIPE,\n env=env,\n )\n if not out:\n raise ValueError\n except (\n subprocess.CalledProcessError,\n FileNotFoundError,\n UnicodeDecodeError,\n ValueError,\n ):\n return set(), 0\n\n out = out.splitlines()\n complete_stmt = out[0]\n out = set(out[1:])\n\n # From GNU Bash document: The results of the expansion are prefix-matched\n # against the word being completed\n\n # Ensure input to `commonprefix` is a list (now required by Python 3.6)\n commprefix = os.path.commonprefix(list(out))\n strip_len = 0\n strip_prefix = prefix.strip(\"\\\"'\")\n while strip_len < len(strip_prefix) and strip_len < len(commprefix):\n if commprefix[strip_len] == strip_prefix[strip_len]:\n break\n strip_len += 1\n\n if \"-o noquote\" not in complete_stmt:\n out, need_quotes = quote_paths(out, \"\", \"\")\n if \"-o nospace\" in complete_stmt:\n out = set([x.rstrip() for x in out])\n\n return out, max(len(prefix) - strip_len, 0)\n\n\ndef bash_complete_line(line, return_line=True, **kwargs):\n \"\"\"Provides the completion from the end of the line.\n\n Parameters\n ----------\n line : str\n Line to complete\n return_line : bool, optional\n If true (default), will return the entire line, with the completion added.\n If false, this will instead return the strings to append to the original line.\n kwargs : optional\n All other keyword arguments are passed to the bash_completions() function.\n\n Returns\n -------\n rtn : set of str\n Possible completions of prefix\n \"\"\"\n # set up for completing from the end of the line\n split = line.split()\n if len(split) > 1 and not line.endswith(\" \"):\n prefix = split[-1]\n begidx = len(line.rsplit(prefix)[0])\n else:\n prefix = \"\"\n begidx = len(line)\n endidx = len(line)\n # get completions\n out, lprefix = bash_completions(prefix, line, begidx, endidx, **kwargs)\n # reformat output\n if return_line:\n preline = line[:-lprefix]\n rtn = {preline + o for o in out}\n else:\n rtn = {o[lprefix:] for o in out}\n return rtn\n\n\ndef _bc_main(args=None):\n \"\"\"Runs complete_line() and prints the output.\"\"\"\n from argparse import ArgumentParser\n\n p = ArgumentParser(\"bash_completions\")\n p.add_argument(\n \"--return-line\",\n action=\"store_true\",\n dest=\"return_line\",\n default=True,\n help=\"will return the entire line, with the completion added\",\n )\n p.add_argument(\n \"--no-return-line\",\n action=\"store_false\",\n dest=\"return_line\",\n help=\"will instead return the strings to append to the original line\",\n )\n p.add_argument(\"line\", help=\"line to complete\")\n ns = p.parse_args(args=args)\n out = bash_complete_line(ns.line, return_line=ns.return_line)\n for o in sorted(out):\n print(o)\n\n\nif __name__ == \"__main__\":\n _bc_main()\n" }, { "path": "xonsh/completers/commands.py", "content": "import os\nimport builtins\n\nimport xonsh.tools as xt\nimport xonsh.platform as xp\n\nfrom xonsh.completers.tools import get_filter_function\n\nSKIP_TOKENS = {\"sudo\", \"time\", \"timeit\", \"which\", \"showcmd\", \"man\"}\nEND_PROC_TOKENS = {\"|\", \"||\", \"&&\", \"and\", \"or\"}\n\n\ndef complete_command(cmd, line, start, end, ctx):\n \"\"\"\n Returns a list of valid commands starting with the first argument\n \"\"\"\n space = \" \"\n out = {\n s + space\n for s in builtins.__xonsh__.commands_cache\n if get_filter_function()(s, cmd)\n }\n if xp.ON_WINDOWS:\n out |= {i for i in xt.executables_in(\".\") if i.startswith(cmd)}\n base = os.path.basename(cmd)\n if os.path.isdir(base):\n out |= {\n os.path.join(base, i) for i in xt.executables_in(base) if i.startswith(cmd)\n }\n return out\n\n\ndef complete_skipper(cmd, line, start, end, ctx):\n \"\"\"\n Skip over several tokens (e.g., sudo) and complete based on the rest of the\n line.\n \"\"\"\n parts = line.split(\" \")\n skip_part_num = 0\n for i, s in enumerate(parts):\n if s in END_PROC_TOKENS:\n skip_part_num = i + 1\n while len(parts) > skip_part_num:\n if parts[skip_part_num] not in SKIP_TOKENS:\n break\n skip_part_num += 1\n\n if skip_part_num == 0:\n return set()\n\n # If there's no space following an END_PROC_TOKEN, insert one\n if parts[-1] in END_PROC_TOKENS:\n return(set(\" \"), 0)\n\n if len(parts) == skip_part_num + 1:\n comp_func = complete_command\n else:\n comp = builtins.__xonsh__.shell.shell.completer\n comp_func = comp.complete\n\n skip_len = len(\" \".join(line[:skip_part_num])) + 1\n return comp_func(\n cmd, \" \".join(parts[skip_part_num:]), start - skip_len, end - skip_len, ctx\n )\n" }, { "path": "xonsh/completers/completer.py", "content": "import builtins\n\n\ndef complete_completer(prefix, line, start, end, ctx):\n \"\"\"\n Completion for \"completer\"\n \"\"\"\n args = line.split(\" \")\n if len(args) == 0 or args[0] != \"completer\":\n return None\n curix = args.index(prefix)\n compnames = set(builtins.__xonsh__.completers.keys())\n if curix == 1:\n possible = {\"list\", \"help\", \"add\", \"remove\"}\n elif curix == 2:\n if args[1] == \"help\":\n possible = {\"list\", \"add\", \"remove\"}\n elif args[1] == \"remove\":\n possible = compnames\n else:\n raise StopIteration\n else:\n if args[1] != \"add\":\n raise StopIteration\n if curix == 3:\n possible = {i for i, j in builtins.__xonsh__.ctx.items() if callable(j)}\n elif curix == 4:\n possible = (\n {\"start\", \"end\"}\n | {\">\" + n for n in compnames}\n | {\"<\" + n for n in compnames}\n )\n else:\n raise StopIteration\n return {i for i in possible if i.startswith(prefix)}\n" }, { "path": "xonsh/completers/dirs.py", "content": "from xonsh.completers.man import complete_from_man\nfrom xonsh.completers.path import complete_dir\n\n\ndef complete_cd(prefix, line, start, end, ctx):\n \"\"\"\n Completion for \"cd\", includes only valid directory names.\n \"\"\"\n if start != 0 and line.split(\" \")[0] == \"cd\":\n return complete_dir(prefix, line, start, end, ctx, True)\n return set()\n\n\ndef complete_rmdir(prefix, line, start, end, ctx):\n \"\"\"\n Completion for \"rmdir\", includes only valid directory names.\n \"\"\"\n if start != 0 and line.split(\" \")[0] == \"rmdir\":\n opts = {\n i\n for i in complete_from_man(\"-\", \"rmdir -\", 6, 7, ctx)\n if i.startswith(prefix)\n }\n comps, lp = complete_dir(prefix, line, start, end, ctx, True)\n return comps | opts, lp\n return set()\n" }, { "path": "xonsh/completers/init.py", "content": "\"\"\"Constructor for xonsh completer objects.\"\"\"\nimport collections\n\nfrom xonsh.completers.pip import complete_pip\nfrom xonsh.completers.man import complete_from_man\nfrom xonsh.completers.bash import complete_from_bash\nfrom xonsh.completers.base import complete_base\nfrom xonsh.completers.path import complete_path\nfrom xonsh.completers.dirs import complete_cd, complete_rmdir\nfrom xonsh.completers.python import (\n complete_python,\n complete_import,\n complete_python_mode,\n)\nfrom xonsh.completers.commands import complete_skipper\nfrom xonsh.completers.completer import complete_completer\nfrom xonsh.completers.xompletions import complete_xonfig, complete_xontrib\n\n\ndef default_completers():\n \"\"\"Creates a copy of the default completers.\"\"\"\n return collections.OrderedDict(\n [\n (\"python_mode\", complete_python_mode),\n (\"base\", complete_base),\n (\"completer\", complete_completer),\n (\"skip\", complete_skipper),\n (\"pip\", complete_pip),\n (\"cd\", complete_cd),\n (\"rmdir\", complete_rmdir),\n (\"xonfig\", complete_xonfig),\n (\"xontrib\", complete_xontrib),\n (\"bash\", complete_from_bash),\n (\"man\", complete_from_man),\n (\"import\", complete_import),\n (\"python\", complete_python),\n (\"path\", complete_path),\n ]\n )\n" }, { "path": "xonsh/completers/man.py", "content": "import os\nimport re\nimport pickle\nimport builtins\nimport subprocess\n\nimport xonsh.lazyasd as xl\n\nfrom xonsh.completers.tools import get_filter_function\n\nOPTIONS = None\nOPTIONS_PATH = None\n\n\n@xl.lazyobject\ndef SCRAPE_RE():\n return re.compile(r\"^(?:\\s*(?:-\\w|--[a-z0-9-]+)[\\s,])+\", re.M)\n\n\n@xl.lazyobject\ndef INNER_OPTIONS_RE():\n return re.compile(r\"-\\w|--[a-z0-9-]+\")\n\n\ndef complete_from_man(prefix, line, start, end, ctx):\n \"\"\"\n Completes an option name, based on the contents of the associated man\n page.\n \"\"\"\n global OPTIONS, OPTIONS_PATH\n if OPTIONS is None:\n datadir = builtins.__xonsh__.env[\"XONSH_DATA_DIR\"]\n OPTIONS_PATH = os.path.join(datadir, \"man_completions_cache\")\n try:\n with open(OPTIONS_PATH, \"rb\") as f:\n OPTIONS = pickle.load(f)\n except Exception:\n OPTIONS = {}\n if not prefix.startswith(\"-\"):\n return set()\n cmd = line.split()[0]\n if cmd not in OPTIONS:\n try:\n manpage = subprocess.Popen(\n [\"man\", cmd], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL\n )\n # This is a trick to get rid of reverse line feeds\n text = subprocess.check_output([\"col\", \"-b\"], stdin=manpage.stdout)\n text = text.decode(\"utf-8\")\n scraped_text = \" \".join(SCRAPE_RE.findall(text))\n matches = INNER_OPTIONS_RE.findall(scraped_text)\n OPTIONS[cmd] = matches\n with open(OPTIONS_PATH, \"wb\") as f:\n pickle.dump(OPTIONS, f)\n except Exception:\n return set()\n return {s for s in OPTIONS[cmd] if get_filter_function()(s, prefix)}\n" }, { "path": "xonsh/completers/path.py", "content": "import os\nimport re\nimport ast\nimport glob\nimport builtins\n\nimport xonsh.tools as xt\nimport xonsh.platform as xp\nimport xonsh.lazyasd as xl\n\nfrom xonsh.completers.tools import get_filter_function\n\n\n@xl.lazyobject\ndef PATTERN_NEED_QUOTES():\n pattern = r'\\s`\\$\\{\\}\\,\\*\\(\\)\"\\'\\?&#'\n if xp.ON_WINDOWS:\n pattern += \"%\"\n pattern = \"[\" + pattern + \"]\" + r\"|\\band\\b|\\bor\\b\"\n return re.compile(pattern)\n\n\ndef cd_in_command(line):\n \"\"\"Returns True if \"cd\" is a token in the line, False otherwise.\"\"\"\n lexer = builtins.__xonsh__.execer.parser.lexer\n lexer.reset()\n lexer.input(line)\n have_cd = False\n for tok in lexer:\n if tok.type == \"NAME\" and tok.value == \"cd\":\n have_cd = True\n break\n return have_cd\n\n\ndef _path_from_partial_string(inp, pos=None):\n if pos is None:\n pos = len(inp)\n partial = inp[:pos]\n startix, endix, quote = xt.check_for_partial_string(partial)\n _post = \"\"\n if startix is None:\n return None\n elif endix is None:\n string = partial[startix:]\n else:\n if endix != pos:\n _test = partial[endix:pos]\n if not any(i == \" \" for i in _test):\n _post = _test\n else:\n return None\n string = partial[startix:endix]\n end = xt.RE_STRING_START.sub(\"\", quote)\n _string = string\n if not _string.endswith(end):\n _string = _string + end\n try:\n val = ast.literal_eval(_string)\n except (SyntaxError, ValueError):\n return None\n if isinstance(val, bytes):\n env = builtins.__xonsh__.env\n val = val.decode(\n encoding=env.get(\"XONSH_ENCODING\"), errors=env.get(\"XONSH_ENCODING_ERRORS\")\n )\n return string + _post, val + _post, quote, end\n\n\ndef _normpath(p):\n \"\"\"\n Wraps os.normpath() to avoid removing './' at the beginning\n and '/' at the end. On windows it does the same with backslashes\n \"\"\"\n initial_dotslash = p.startswith(os.curdir + os.sep)\n initial_dotslash |= xp.ON_WINDOWS and p.startswith(os.curdir + os.altsep)\n p = p.rstrip()\n trailing_slash = p.endswith(os.sep)\n trailing_slash |= xp.ON_WINDOWS and p.endswith(os.altsep)\n p = os.path.normpath(p)\n if initial_dotslash and p != \".\":\n p = os.path.join(os.curdir, p)\n if trailing_slash:\n p = os.path.join(p, \"\")\n if xp.ON_WINDOWS and builtins.__xonsh__.env.get(\"FORCE_POSIX_PATHS\"):\n p = p.replace(os.sep, os.altsep)\n return p\n\n\ndef _startswithlow(x, start, startlow=None):\n if startlow is None:\n startlow = start.lower()\n return x.startswith(start) or x.lower().startswith(startlow)\n\n\ndef _startswithnorm(x, start, startlow=None):\n return x.startswith(start)\n\n\ndef _env(prefix):\n if prefix.startswith(\"$\"):\n key = prefix[1:]\n return {\n \"$\" + k for k in builtins.__xonsh__.env if get_filter_function()(k, key)\n }\n return ()\n\n\ndef _dots(prefix):\n slash = xt.get_sep()\n if slash == \"\\\\\":\n slash = \"\"\n if prefix in {\"\", \".\"}:\n return (\".\" + slash, \"..\" + slash)\n elif prefix == \"..\":\n return (\"..\" + slash,)\n else:\n return ()\n\n\ndef _add_cdpaths(paths, prefix):\n \"\"\"Completes current prefix using CDPATH\"\"\"\n env = builtins.__xonsh__.env\n csc = env.get(\"CASE_SENSITIVE_COMPLETIONS\")\n glob_sorted = env.get(\"GLOB_SORTED\")\n for cdp in env.get(\"CDPATH\"):\n test_glob = os.path.join(cdp, prefix) + \"*\"\n for s in xt.iglobpath(\n test_glob, ignore_case=(not csc), sort_result=glob_sorted\n ):\n if os.path.isdir(s):\n paths.add(os.path.basename(s))\n\n\ndef _quote_to_use(x):\n single = \"'\"\n double = '\"'\n if single in x and double not in x:\n return double\n else:\n return single\n\n\ndef _quote_paths(paths, start, end, append_end=True):\n expand_path = builtins.__xonsh__.expand_path\n out = set()\n space = \" \"\n backslash = \"\\\\\"\n double_backslash = \"\\\\\\\\\"\n slash = xt.get_sep()\n orig_start = start\n orig_end = end\n # quote on all or none, to make readline completes to max prefix\n need_quotes = any(\n re.search(PATTERN_NEED_QUOTES, x) or (backslash in x and slash != backslash)\n for x in paths\n )\n\n for s in paths:\n start = orig_start\n end = orig_end\n if start == \"\" and need_quotes:\n start = end = _quote_to_use(s)\n if os.path.isdir(expand_path(s)):\n _tail = slash\n elif end == \"\":\n _tail = space\n else:\n _tail = \"\"\n if start != \"\" and \"r\" not in start and backslash in s:\n start = \"r%s\" % start\n s = s + _tail\n if end != \"\":\n if \"r\" not in start.lower():\n s = s.replace(backslash, double_backslash)\n if s.endswith(backslash) and not s.endswith(double_backslash):\n s += backslash\n if end in s:\n s = s.replace(end, \"\".join(\"\\\\%s\" % i for i in end))\n s = start + s + end if append_end else start + s\n out.add(s)\n return out, need_quotes\n\n\ndef _joinpath(path):\n # convert our tuple representation back into a string representing a path\n if path is None:\n return \"\"\n elif len(path) == 0:\n return \"\"\n elif path == (\"\",):\n return xt.get_sep()\n elif path[0] == \"\":\n return xt.get_sep() + _normpath(os.path.join(*path))\n else:\n return _normpath(os.path.join(*path))\n\n\ndef _splitpath(path):\n # convert a path into an intermediate tuple representation\n # if this tuple starts with '', it means that the path was an absolute path\n path = _normpath(path)\n if path.startswith(xt.get_sep()):\n pre = (\"\",)\n else:\n pre = ()\n return pre + _splitpath_helper(path, ())\n\n\ndef _splitpath_helper(path, sofar=()):\n folder, path = os.path.split(path)\n if path:\n sofar = sofar + (path,)\n if not folder or folder == xt.get_sep():\n return sofar[::-1]\n elif xp.ON_WINDOWS and not path:\n return os.path.splitdrive(folder)[:1] + sofar[::-1]\n elif xp.ON_WINDOWS and os.path.splitdrive(path)[0]:\n return sofar[::-1]\n return _splitpath_helper(folder, sofar)\n\n\ndef subsequence_match(ref, typed, csc):\n \"\"\"\n Detects whether typed is a subsequence of ref.\n\n Returns ``True`` if the characters in ``typed`` appear (in order) in\n ``ref``, regardless of exactly where in ``ref`` they occur. If ``csc`` is\n ``False``, ignore the case of ``ref`` and ``typed``.\n\n Used in \"subsequence\" path completion (e.g., ``~/u/ro`` expands to\n ``~/lou/carcohl``)\n \"\"\"\n if csc:\n return _subsequence_match_iter(ref, typed)\n else:\n return _subsequence_match_iter(ref.lower(), typed.lower())\n\n\ndef _subsequence_match_iter(ref, typed):\n if len(typed) == 0:\n return True\n elif len(ref) == 0:\n return False\n elif ref[0] == typed[0]:\n return _subsequence_match_iter(ref[1:], typed[1:])\n else:\n return _subsequence_match_iter(ref[1:], typed)\n\n\ndef _expand_one(sofar, nextone, csc):\n out = set()\n glob_sorted = builtins.__xonsh__.env.get(\"GLOB_SORTED\")\n for i in sofar:\n _glob = os.path.join(_joinpath(i), \"*\") if i is not None else \"*\"\n for j in xt.iglobpath(_glob, sort_result=glob_sorted):\n j = os.path.basename(j)\n if subsequence_match(j, nextone, csc):\n out.add((i or ()) + (j,))\n return out\n\n\ndef complete_path(prefix, line, start, end, ctx, cdpath=True, filtfunc=None):\n \"\"\"Completes based on a path name.\"\"\"\n # string stuff for automatic quoting\n path_str_start = \"\"\n path_str_end = \"\"\n append_end = True\n p = _path_from_partial_string(line, end)\n lprefix = len(prefix)\n if p is not None:\n lprefix = len(p[0])\n prefix = p[1]\n path_str_start = p[2]\n path_str_end = p[3]\n if len(line) >= end + 1 and line[end] == path_str_end:\n append_end = False\n tilde = \"~\"\n paths = set()\n env = builtins.__xonsh__.env\n csc = env.get(\"CASE_SENSITIVE_COMPLETIONS\")\n glob_sorted = env.get(\"GLOB_SORTED\")\n prefix = glob.escape(prefix)\n for s in xt.iglobpath(prefix + \"*\", ignore_case=(not csc), sort_result=glob_sorted):\n paths.add(s)\n if len(paths) == 0 and env.get(\"SUBSEQUENCE_PATH_COMPLETION\"):\n # this block implements 'subsequence' matching, similar to fish and zsh.\n # matches are based on subsequences, not substrings.\n # e.g., ~/u/ro completes to ~/lou/carcolh\n # see above functions for details.\n p = _splitpath(os.path.expanduser(prefix))\n if len(p) != 0:\n if p[0] == \"\":\n basedir = (\"\",)\n p = p[1:]\n else:\n basedir = None\n matches_so_far = {basedir}\n for i in p:\n matches_so_far = _expand_one(matches_so_far, i, csc)\n paths |= {_joinpath(i) for i in matches_so_far}\n if len(paths) == 0 and env.get(\"FUZZY_PATH_COMPLETION\"):\n threshold = env.get(\"SUGGEST_THRESHOLD\")\n for s in xt.iglobpath(\n os.path.dirname(prefix) + \"*\",\n ignore_case=(not csc),\n sort_result=glob_sorted,\n ):\n if xt.levenshtein(prefix, s, threshold) < threshold:\n paths.add(s)\n if tilde in prefix:\n home = os.path.expanduser(tilde)\n paths = {s.replace(home, tilde) for s in paths}\n if cdpath and cd_in_command(line):\n _add_cdpaths(paths, prefix)\n paths = set(filter(filtfunc, paths))\n paths, _ = _quote_paths(\n {_normpath(s) for s in paths}, path_str_start, path_str_end, append_end\n )\n paths.update(filter(filtfunc, _dots(prefix)))\n paths.update(filter(filtfunc, _env(prefix)))\n return paths, lprefix\n\n\ndef complete_dir(prefix, line, start, end, ctx, cdpath=False):\n return complete_path(prefix, line, start, end, cdpath, filtfunc=os.path.isdir)\n" }, { "path": "xonsh/completers/pip.py", "content": "\"\"\"Completers for pip.\"\"\"\n# pylint: disable=invalid-name, missing-docstring, unsupported-membership-test\n# pylint: disable=unused-argument, not-an-iterable\nimport re\nimport subprocess\n\nimport xonsh.lazyasd as xl\n\n\n@xl.lazyobject\ndef PIP_RE():\n return re.compile(r\"pip(?:\\d|\\.)*\")\n\n\n@xl.lazyobject\ndef PIP_LIST_RE():\n return re.compile(r\"pip(?:\\d|\\.)* (?:uninstall|show)\")\n\n\n@xl.lazyobject\ndef ALL_COMMANDS():\n try:\n help_text = str(\n subprocess.check_output([\"pip\", \"--help\"], stderr=subprocess.DEVNULL)\n )\n except FileNotFoundError:\n return []\n commands = re.findall(r\" (\\w+) \", help_text)\n return [c for c in commands if c not in [\"completion\", \"help\"]]\n\n\ndef complete_pip(prefix, line, begidx, endidx, ctx):\n \"\"\"Completes python's package manager pip\"\"\"\n line_len = len(line.split())\n if (\n (line_len > 3)\n or (line_len > 2 and line.endswith(\" \"))\n or (not PIP_RE.search(line))\n ):\n return\n if PIP_LIST_RE.search(line):\n try:\n items = subprocess.check_output([\"pip\", \"list\"], stderr=subprocess.DEVNULL)\n except FileNotFoundError:\n return set()\n items = items.decode(\"utf-8\").splitlines()\n return set(i.split()[0] for i in items if i.split()[0].startswith(prefix))\n\n if (line_len > 1 and line.endswith(\" \")) or line_len > 2:\n # \"pip show \" -> no complete (note space)\n return\n if prefix not in ALL_COMMANDS:\n suggestions = [c for c in ALL_COMMANDS if c.startswith(prefix)]\n if suggestions:\n return suggestions, len(prefix)\n return ALL_COMMANDS, len(prefix)\n" }, { "path": "xonsh/completers/python.py", "content": "\"\"\"Completers for Python code\"\"\"\nimport re\nimport sys\nimport inspect\nimport builtins\nimport importlib\nimport collections.abc as cabc\n\nimport xonsh.tools as xt\nimport xonsh.lazyasd as xl\n\nfrom xonsh.completers.tools import get_filter_function\n\n\n@xl.lazyobject\ndef RE_ATTR():\n return re.compile(r\"([^\\s\\(\\)]+(\\.[^\\s\\(\\)]+)*)\\.(\\w*)$\")\n\n\n@xl.lazyobject\ndef XONSH_EXPR_TOKENS():\n return {\n \"and \",\n \"else\",\n \"for \",\n \"if \",\n \"in \",\n \"is \",\n \"lambda \",\n \"not \",\n \"or \",\n \"+\",\n \"-\",\n \"/\",\n \"//\",\n \"%\",\n \"**\",\n \"|\",\n \"&\",\n \"~\",\n \"^\",\n \">>\",\n \"<<\",\n \"<\",\n \"<=\",\n \">\",\n \">=\",\n \"==\",\n \"!=\",\n \",\",\n \"?\",\n \"??\",\n \"$(\",\n \"${\",\n \"$[\",\n \"...\",\n \"![\",\n \"!(\",\n \"@(\",\n \"@$(\",\n \"@\",\n }\n\n\n@xl.lazyobject\ndef XONSH_STMT_TOKENS():\n return {\n \"as \",\n \"assert \",\n \"break\",\n \"class \",\n \"continue\",\n \"def \",\n \"del \",\n \"elif \",\n \"except \",\n \"finally:\",\n \"from \",\n \"global \",\n \"import \",\n \"nonlocal \",\n \"pass\",\n \"raise \",\n \"return \",\n \"try:\",\n \"while \",\n \"with \",\n \"yield \",\n \"-\",\n \"/\",\n \"//\",\n \"%\",\n \"**\",\n \"|\",\n \"&\",\n \"~\",\n \"^\",\n \">>\",\n \"<<\",\n \"<\",\n \"<=\",\n \"->\",\n \"=\",\n \"+=\",\n \"-=\",\n \"*=\",\n \"/=\",\n \"%=\",\n \"**=\",\n \">>=\",\n \"<<=\",\n \"&=\",\n \"^=\",\n \"|=\",\n \"//=\",\n \";\",\n \":\",\n \"..\",\n }\n\n\n@xl.lazyobject\ndef XONSH_TOKENS():\n return set(XONSH_EXPR_TOKENS) | set(XONSH_STMT_TOKENS)\n\n\ndef complete_python(prefix, line, start, end, ctx):\n \"\"\"\n Completes based on the contents of the current Python environment,\n the Python built-ins, and xonsh operators.\n If there are no matches, split on common delimiters and try again.\n \"\"\"\n rtn = _complete_python(prefix, line, start, end, ctx)\n if not rtn:\n prefix = (\n re.split(r\"\\(|=|{|\\[|,\", prefix)[-1]\n if not prefix.startswith(\",\")\n else prefix\n )\n start = line.find(prefix)\n rtn = _complete_python(prefix, line, start, end, ctx)\n return rtn, len(prefix)\n return rtn\n\n\ndef _complete_python(prefix, line, start, end, ctx):\n \"\"\"\n Completes based on the contents of the current Python environment,\n the Python built-ins, and xonsh operators.\n \"\"\"\n if line != \"\":\n first = line.split()[0]\n if first in builtins.__xonsh__.commands_cache and first not in ctx:\n return set()\n filt = get_filter_function()\n rtn = set()\n if ctx is not None:\n if \".\" in prefix:\n rtn |= attr_complete(prefix, ctx, filt)\n args = python_signature_complete(prefix, line, end, ctx, filt)\n rtn |= args\n rtn |= {s for s in ctx if filt(s, prefix)}\n else:\n args = ()\n if len(args) == 0:\n # not in a function call, so we can add non-expression tokens\n rtn |= {s for s in XONSH_TOKENS if filt(s, prefix)}\n else:\n rtn |= {s for s in XONSH_EXPR_TOKENS if filt(s, prefix)}\n rtn |= {s for s in dir(builtins) if filt(s, prefix)}\n return rtn\n\n\ndef complete_python_mode(prefix, line, start, end, ctx):\n \"\"\"\n Python-mode completions for @( and ${\n \"\"\"\n if not (prefix.startswith(\"@(\") or prefix.startswith(\"${\")):\n return set()\n prefix_start = prefix[:2]\n python_matches = complete_python(prefix[2:], line, start - 2, end - 2, ctx)\n if isinstance(python_matches, cabc.Sequence):\n python_matches = python_matches[0]\n return set(prefix_start + i for i in python_matches)\n\n\ndef _safe_eval(expr, ctx):\n \"\"\"Safely tries to evaluate an expression. If this fails, it will return\n a (None, None) tuple.\n \"\"\"\n _ctx = None\n xonsh_safe_eval = builtins.__xonsh__.execer.eval\n try:\n val = xonsh_safe_eval(expr, ctx, ctx, transform=False)\n _ctx = ctx\n except: # pylint:disable=bare-except\n try:\n val = xonsh_safe_eval(expr, builtins.__dict__, transform=False)\n _ctx = builtins.__dict__\n except: # pylint:disable=bare-except\n val = _ctx = None\n return val, _ctx\n\n\ndef attr_complete(prefix, ctx, filter_func):\n \"\"\"Complete attributes of an object.\"\"\"\n attrs = set()\n m = RE_ATTR.match(prefix)\n if m is None:\n return attrs\n expr, attr = m.group(1, 3)\n expr = xt.subexpr_from_unbalanced(expr, \"(\", \")\")\n expr = xt.subexpr_from_unbalanced(expr, \"[\", \"]\")\n expr = xt.subexpr_from_unbalanced(expr, \"{\", \"}\")\n val, _ctx = _safe_eval(expr, ctx)\n if val is None and _ctx is None:\n return attrs\n if len(attr) == 0:\n opts = [o for o in dir(val) if not o.startswith(\"_\")]\n else:\n opts = [o for o in dir(val) if filter_func(o, attr)]\n prelen = len(prefix)\n for opt in opts:\n # check whether these options actually work (e.g., disallow 7.imag)\n _expr = \"{0}.{1}\".format(expr, opt)\n _val_, _ctx_ = _safe_eval(_expr, _ctx)\n if _val_ is None and _ctx_ is None:\n continue\n a = getattr(val, opt)\n if builtins.__xonsh__.env[\"COMPLETIONS_BRACKETS\"]:\n if callable(a):\n rpl = opt + \"(\"\n elif isinstance(a, (cabc.Sequence, cabc.Mapping)):\n rpl = opt + \"[\"\n else:\n rpl = opt\n else:\n rpl = opt\n # note that prefix[:prelen-len(attr)] != prefix[:-len(attr)]\n # when len(attr) == 0.\n comp = prefix[: prelen - len(attr)] + rpl\n attrs.add(comp)\n return attrs\n\n\ndef python_signature_complete(prefix, line, end, ctx, filter_func):\n \"\"\"Completes a python function (or other callable) call by completing\n argument and keyword argument names.\n \"\"\"\n front = line[:end]\n if xt.is_balanced(front, \"(\", \")\"):\n return set()\n funcname = xt.subexpr_before_unbalanced(front, \"(\", \")\")\n val, _ctx = _safe_eval(funcname, ctx)\n if val is None:\n return set()\n try:\n sig = inspect.signature(val)\n except ValueError:\n return set()\n args = {p + \"=\" for p in sig.parameters if filter_func(p, prefix)}\n return args\n\n\ndef complete_import(prefix, line, start, end, ctx):\n \"\"\"\n Completes module names and contents for \"import ...\" and \"from ... import\n ...\"\n \"\"\"\n ltoks = line.split()\n ntoks = len(ltoks)\n if ntoks == 2 and ltoks[0] == \"from\":\n # completing module to import\n return {\"{} \".format(i) for i in complete_module(prefix)}\n if ntoks > 1 and ltoks[0] == \"import\" and start == len(\"import \"):\n # completing module to import\n return complete_module(prefix)\n if ntoks > 2 and ltoks[0] == \"from\" and ltoks[2] == \"import\":\n # complete thing inside a module\n try:\n mod = importlib.import_module(ltoks[1])\n except ImportError:\n return set()\n out = {i[0] for i in inspect.getmembers(mod) if i[0].startswith(prefix)}\n return out\n return set()\n\n\ndef complete_module(prefix):\n return {s for s in sys.modules if get_filter_function()(s, prefix)}\n" }, { "path": "xonsh/completers/tools.py", "content": "\"\"\"Xonsh completer tools.\"\"\"\nimport builtins\nimport textwrap\n\n\ndef _filter_normal(s, x):\n return s.startswith(x)\n\n\ndef _filter_ignorecase(s, x):\n return s.lower().startswith(x.lower())\n\n\ndef get_filter_function():\n \"\"\"\n Return an appropriate filtering function for completions, given the valid\n of $CASE_SENSITIVE_COMPLETIONS\n \"\"\"\n csc = builtins.__xonsh__.env.get(\"CASE_SENSITIVE_COMPLETIONS\")\n if csc:\n return _filter_normal\n else:\n return _filter_ignorecase\n\n\ndef justify(s, max_length, left_pad=0):\n \"\"\"\n Re-wrap the string s so that each line is no more than max_length\n characters long, padding all lines but the first on the left with the\n string left_pad.\n \"\"\"\n txt = textwrap.wrap(s, width=max_length, subsequent_indent=\" \" * left_pad)\n return \"\\n\".join(txt)\n" }, { "path": "xonsh/completers/xompletions.py", "content": "\"\"\"Provides completions for xonsh internal utilities\"\"\"\n\nimport xonsh.xontribs as xx\nimport xonsh.tools as xt\n\n\ndef complete_xonfig(prefix, line, start, end, ctx):\n \"\"\"Completion for ``xonfig``\"\"\"\n args = line.split(\" \")\n if len(args) == 0 or args[0] != \"xonfig\":\n return None\n curix = args.index(prefix)\n if curix == 1:\n possible = {\"info\", \"wizard\", \"styles\", \"colors\", \"-h\"}\n elif curix == 2 and args[1] == \"colors\":\n possible = set(xt.color_style_names())\n else:\n raise StopIteration\n return {i for i in possible if i.startswith(prefix)}\n\n\ndef _list_installed_xontribs():\n meta = xx.xontrib_metadata()\n installed = []\n for md in meta[\"xontribs\"]:\n name = md[\"name\"]\n spec = xx.find_xontrib(name)\n if spec is not None:\n installed.append(spec.name.rsplit(\".\")[-1])\n\n return installed\n\n\ndef complete_xontrib(prefix, line, start, end, ctx):\n \"\"\"Completion for ``xontrib``\"\"\"\n args = line.split(\" \")\n if len(args) == 0 or args[0] != \"xontrib\":\n return None\n curix = args.index(prefix)\n if curix == 1:\n possible = {\"list\", \"load\"}\n elif curix == 2:\n if args[1] == \"load\":\n possible = _list_installed_xontribs()\n else:\n raise StopIteration\n\n return {i for i in possible if i.startswith(prefix)}\n" }, { "path": "xonsh/contexts.py", "content": "\"\"\"Context management tools for xonsh.\"\"\"\nimport sys\nimport textwrap\nimport builtins\nfrom collections.abc import Mapping\n\n\nclass Block(object):\n \"\"\"This is a context manager for obtaining a block of lines without actually\n executing the block. The lines are accessible as the 'lines' attribute.\n This must be used as a macro.\n \"\"\"\n\n __xonsh_block__ = str\n\n def __init__(self):\n \"\"\"\n Attributes\n ----------\n lines : list of str or None\n Block lines as if split by str.splitlines(), if available.\n glbs : Mapping or None\n Global execution context, ie globals().\n locs : Mapping or None\n Local execution context, ie locals().\n \"\"\"\n self.lines = self.glbs = self.locs = None\n\n def __enter__(self):\n if not hasattr(self, \"macro_block\"):\n raise XonshError(self.__class__.__name__ + \" must be entered as a macro!\")\n self.lines = self.macro_block.splitlines()\n self.glbs = self.macro_globals\n if self.macro_locals is not self.macro_globals:\n # leave locals as None when it is the same as globals\n self.locs = self.macro_locals\n return self\n\n def __exit__(self, exc_type, exc_value, traceback):\n pass\n\n\nclass Functor(Block):\n \"\"\"This is a context manager that turns the block into a callable\n object, bound to the execution context it was created in.\n \"\"\"\n\n def __init__(self, args=(), kwargs=None, rtn=\"\"):\n \"\"\"\n Parameters\n ----------\n args : Sequence of str, optional\n A tuple of argument names for the functor.\n kwargs : Mapping of str to values or list of item tuples, optional\n Keyword argument names and values, if available.\n rtn : str, optional\n Name of object to return, if available.\n\n Attributes\n ----------\n func : function\n The underlying function object. This defaults to none and is set\n after the the block is exited.\n \"\"\"\n super().__init__()\n self.func = None\n self.args = args\n if kwargs is None:\n self.kwargs = []\n elif isinstance(kwargs, Mapping):\n self.kwargs = sorted(kwargs.items())\n else:\n self.kwargs = kwargs\n self.rtn = rtn\n\n def __enter__(self):\n super().__enter__()\n body = textwrap.indent(self.macro_block, \" \")\n uid = hash(body) + sys.maxsize # should always be a positive int\n name = \"__xonsh_functor_{uid}__\".format(uid=uid)\n # construct signature string\n sig = rtn = \"\"\n sig = \", \".join(self.args)\n kwstr = \", \".join([k + \"=None\" for k, _ in self.kwargs])\n if len(kwstr) > 0:\n sig = kwstr if len(sig) == 0 else sig + \", \" + kwstr\n # construct return string\n rtn = str(self.rtn)\n if len(rtn) > 0:\n rtn = \" return \" + rtn + \"\\n\"\n # construct function string\n fstr = \"def {name}({sig}):\\n{body}\\n{rtn}\"\n fstr = fstr.format(name=name, sig=sig, body=body, rtn=rtn)\n glbs = self.glbs\n locs = self.locs\n execer = builtins.__xonsh__.execer\n execer.exec(fstr, glbs=glbs, locs=locs)\n if locs is not None and name in locs:\n func = locs[name]\n elif name in glbs:\n func = glbs[name]\n else:\n raise ValueError(\"Functor block could not be found in context.\")\n if len(self.kwargs) > 0:\n func.__defaults__ = tuple(v for _, v in self.kwargs)\n self.func = func\n return self\n\n def __exit__(self, exc_type, exc_value, traceback):\n pass\n\n def __call__(self, *args, **kwargs):\n \"\"\"Dispatches to func.\"\"\"\n if self.func is None:\n msg = \"{} block with 'None' func not callable\"\n raise AttributeError(msg.formst(self.__class__.__name__))\n return self.func(*args, **kwargs)\n" }, { "path": "xonsh/data/data.txt", "content": "(u'vnpy', 363)\n(u'thoppe', 104)\n(u'edc', 114)\n(u'projectatomic', 110)\n(u'jrosebr1', 289)\n(u'compose', 305)\n(u'4148', 163)\n(u'Dobiasd', 179)\n(u'blaze', 374)\n(u'Bekt', 133)\n(u'josauder', 251)\n(u'nylas', 101)\n(u'facebook', 2664)\n(u'izacus', 139)\n(u'HelloZeroNet', 1117)\n(u'michael-lazar', 527)\n(u'simonschellaert', 132)\n(u'black-perl', 272)\n(u'andersbll', 351)\n(u'wecite', 274)\n(u'stasinopoulos', 463)\n(u'vmware', 669)\n(u'waditu', 607)\n(u'Leviathan1995', 372)\n(u'TrackMaven', 146)\n(u'tokland', 114)\n(u'dellis23', 144)\n(u'amueller', 143)\n(u'NVIDIA', 733)\n(u'trevorstephens', 207)\n(u'rantonels', 312)\n(u'klen', 293)\n(u'vsergeev', 125)\n(u'madisonmay', 911)\n(u'DanMcInerney', 456)\n(u'realpython', 181)\n(u'graphific', 1418)\n(u'shoopio', 163)\n(u'conorpp', 123)\n(u'sdiehl', 139)\n(u'ajinabraham', 338)\n(u'prashanthellina', 170)\n(u'laike9m', 110)\n(u'spyder-ide', 570)\n(u'GedRap', 156)\n(u'diafygi', 161)\n(u'maxpumperla', 105)\n(u'ClusterHQ', 282)\n(u'dhilipsiva', 110)\n(u'entropy1337', 133)\n(u'dbpedia', 225)\n(u'kelvinxu', 127)\n(u'jsvine', 172)\n(u'python', 147)\n(u'OffensivePython', 305)\n(u'jkbrzt', 476)\n(u'quodlibet', 116)\n(u'ulope', 200)\n(u'andrea-cuttone', 157)\n(u'alehander42', 193)\n(u'KirkHadley', 139)\n(u'revsys', 112)\n(u'aigamedev', 583)\n(u'sphinx-doc', 389)\n(u'rietveld-codereview', 123)\n(u'quantopian', 319)\n(u'Deimos', 285)\n(u'dgilland', 107)\n(u'nccgroup', 146)\n(u'nickjj', 138)\n(u'nbs-system', 107)\n(u'elceef', 272)\n(u'compjour', 630)\n(u'jrnewell', 207)\n(u'bndr', 510)\n(u'EricssonResearch', 188)\n(u'raelgc', 701)\n(u'avinassh', 2659)\n(u'vitruvianscience', 152)\n(u'achiku', 811)\n(u'eugene-eeo', 144)\n(u'paul-nechifor', 465)\n(u'antiboredom', 582)\n(u'datalib', 301)\n(u'brandon-rhodes', 110)\n(u'philpep', 101)\n(u'python-cn', 151)\n(u'jakevdp', 275)\n(u'CiscoCloud', 1635)\n(u'yandex', 174)\n(u'jbornschein', 201)\n(u'j-bennet', 307)\n(u'Skycrab', 124)\n(u'spotify', 376)\n(u'pyparallel', 345)\n(u'JakeCooper', 130)\n(u'jorgebastida', 201)\n(u'rouseguy', 253)\n(u'pfalcon', 108)\n(u'geerlingguy', 155)\n(u'aliyun', 113)\n(u'dorneanu', 124)\n(u'cloudera', 500)\n(u'drzoidberg33', 127)\n(u'brennerm', 1154)\n(u'TomAnthony', 336)\n(u'clarkduvall', 310)\n(u'matthewearl', 322)\n(u'guidepointsecurity', 167)\n(u'samim23', 260)\n(u'rhiever', 1848)\n(u'warner', 345)\n(u'gojhonny', 304)\n(u'joshenders', 210)\n(u'itdxer', 108)\n(u'fraser-hemp', 1366)\n(u'omriher', 269)\n(u'motet', 265)\n(u'waf-project', 158)\n(u'AlexEne', 135)\n(u'jmcarp', 159)\n(u'scrapinghub', 141)\n(u'puniaze', 114)\n(u'orf', 207)\n(u'rasguanabana', 892)\n(u'yasoob', 899)\n(u'biicode', 205)\n(u'pybee', 237)\n(u'apprenticeharper', 625)\n(u'angr', 279)\n(u'Hypsurus', 214)\n(u'PaulSec', 228)\n(u'xybu', 111)\n(u'ryankiros', 343)\n(u'wapiflapi', 171)\n(u'trustedsec', 337)\n(u'jfalken', 135)\n(u'ncrocfer', 337)\n(u'mila-udem', 398)\n(u'mardix', 119)\n(u'automl', 227)\n(u'ylovern', 171)\n(u'jacebrowning', 135)\n(u'Eyepea', 328)\n(u'csvoss', 468)\n(u'dittos', 135)\n(u'icgood', 292)\n(u'dciabrin', 114)\n(u'minimaxir', 8920)\n(u'brandonshin', 341)\n(u'jonathanslenders', 1531)\n(u'donnemartin', 7955)\n(u'KeyboardFire', 3324)\n(u'offapi', 142)\n(u'no13bus', 367)\n(u'tatanus', 108)\n(u'Vector35', 281)\n(u'micheloosterhof', 189)\n(u'gamechanger', 220)\n(u'isislab', 233)\n(u'joxeankoret', 263)\n(u'certsocietegenerale', 189)\n(u'inconvergent', 283)\n(u'rpp0', 235)\n(u'yukuku', 225)\n(u'lijiejie', 174)\n(u'stitchfix', 928)\n(u'LionSec', 108)\n(u'n0tr00t', 108)\n(u'ssteuteville', 150)\n(u'getsentry', 661)\n(u'agermanidis', 731)\n(u'78', 354)\n(u'piskvorky', 308)\n(u'sogisha', 172)\n(u'mre', 113)\n(u'rbgirshick', 435)\n(u'rtluckie', 112)\n(u'dbcli', 2072)\n(u'eternnoir', 170)\n(u'recipy', 164)\n(u'rasbt', 350)\n(u'ant4g0nist', 125)\n(u'm57', 598)\n(u'TailorDev', 119)\n(u'cosven', 169)\n(u'zachriggle', 130)\n(u'knownsec', 151)\n(u'awslabs', 200)\n(u'n1nj4sec', 827)\n(u'armadaplatform', 158)\n(u'DoctorTeeth', 129)\n(u'atmb4u', 155)\n(u'mmin18', 1063)\n(u'divmain', 1206)\n(u'fuzzing', 101)\n(u'urbenlegend', 181)\n(u'regebro', 245)\n(u'QuantumFractal', 222)\n(u'kootenpv', 243)\n(u'Alexis-benoist', 120)\n(u'sontek', 108)\n(u'harshasrinivas', 147)\n(u'rabidgremlin', 187)\n(u'Slava', 171)\n(u'10se1ucgo', 1884)\n(u'fugue', 266)\n(u'AlessandroZ', 1423)\n(u'unixy', 107)\n(u'pindexis', 582)\n(u'Yinzo', 115)\n(u'joelpx', 351)\n(u'ryanss', 864)\n(u'Dynetics', 110)\n(u'ExPHAT', 385)\n(u'hephaest0s', 1246)\n(u'armbues', 112)\n(u'pytest-dev', 360)\n(u'moha99sa', 105)\n(u'nvbn', 14856)\n(u'scopatz', 511)\n(u'andrewgodwin', 205)\n(u'architv', 422)\n(u'owais', 122)\n(u'lorin', 107)\n(u'takluyver', 170)\n(u'garabik', 104)\n(u'kevinlawler', 183)\n(u'krockode', 455)\n(u'IndicoDataSolutions', 384)\n(u'x43x61x69', 107)\n(u'semirook', 103)\n(u'DIYgod', 195)\n(u'fffonion', 101)\n(u'snoack', 429)\n(u'rodricios', 197)\n(u'RealHacker', 132)\n(u'krisfields', 246)\n(u'nils-werner', 202)\n(u'mementum', 124)\n(u'uaca', 220)\n(u'tburmeister', 226)\n(u'CoreSecurity', 294)\n(u'lukasschwab', 298)\n(u'jcjohnson', 275)\n(u'timothycrosley', 903)\n(u'linkedin', 142)\n(u'rushter', 2155)\n(u'WatchPeopleCode', 209)\n(u'hellerve', 426)\n(u'JacobPlaster', 198)\n(u'thampiman', 1101)\n(u'timlib', 116)\n(u'pfnet', 832)\n(u'mitsuhiko', 231)\n(u'ziggear', 153)\n(u'datawire', 152)\n(u'jrfonseca', 199)\n(u'Lasagne', 151)\n(u'ssut', 130)\n(u'ChrisTruncer', 199)\n(u'projectcalico', 167)\n(u'lehui99', 219)\n(u'ring04h', 390)\n(u'amoffat', 1225)\n(u'317070', 134)\n(u'p-e-w', 734)\n(u'srsudar', 735)\n(u'aromanovich', 102)\n(u'twoscoops', 244)\n(u'fchollet', 2958)\n(u'minrk', 113)\n(u'chrissimpkins', 2211)\n(u'qiwsir', 673)\n(u'smarr', 612)\n(u'anishathalye', 796)\n(u'onyxfish', 120)\n(u'hexahedria', 602)\n(u'deepmind', 344)\n(u'wooey', 484)\n(u'manugarri', 129)\n(u'joschu', 387)\n(u'parkouss', 134)\n(u'ShawnDEvans', 165)\n(u'Max00355', 507)\n(u'uptimejp', 156)\n(u'google', 5720)\n(u'yosinski', 352)\n(u'buckyroberts', 211)\n(u'borgbackup', 142)\n(u'chriscannon', 233)\n(u'benanne', 175)\n(u'jandre', 257)\n(u'zeruniverse', 232)\n(u'asciimoo', 167)\n(u'fengsp', 208)\n(u'Koed00', 201)\n(u'duerrp', 193)\n(u'byt3bl33d3r', 464)\n(u'amjith', 154)\n(u'DrkSephy', 404)\n(u'andelf', 832)\n(u'vertical-knowledge', 109)\n(u'Ivaylo-Popov', 248)\n(u'lhartikk', 422)\n(u'lmco', 313)\n(u'paylogic', 161)\n(u'karan', 2672)\n(u'zulip', 3108)\n(u'billpmurphy', 375)\n(u'saelo', 163)\n(u'rdnetto', 173)\n(u'unlimitedlabs', 191)\n(u'lonetwin', 129)\n(u'closeio', 304)\n(u'billryan', 178)\n(u'larsenwork', 2720)\n(u'Dionach', 150)\n(u'ottogroup', 308)\n(u'paypal', 221)\n(u'install-logos', 116)\n(u'Maratyszcza', 610)\n(u'avehtari', 292)\n(u'AtnNn', 125)\n(u'Neo23x0', 208)\n(u'justmarkham', 135)\n(u'vladimarius', 168)\n(u'graphql-python', 128)\n(u'mattya', 115)\n(u'coodict', 537)\n(u'sourcelair', 268)\n(u'walkr', 222)\n(u'bittorrent', 363)\n(u'hugsy', 201)\n(u'ayoungprogrammer', 292)\n(u'MorganZhang100', 249)\n(u'ChrisBeaumont', 245)\n(u'Netflix', 346)\n(u'yahoo', 302)\n(u'mhallsmoore', 136)\n(u'XX-net', 2731)\n(u'leandrotoledo', 175)\n(u'Brobin', 154)\n(u'pyskell', 732)\n(u'NathanEpstein', 119)\n(u'yadayada', 245)\n(u'dxa4481', 216)\n(u'lyst', 408)" }, { "path": "xonsh/diff_history.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"Tools for diff'ing two xonsh history files in a meaningful fashion.\"\"\"\nimport difflib\nimport datetime\nimport itertools\nimport argparse\n\nfrom xonsh.lazyjson import LazyJSON\nfrom xonsh.tools import print_color\n\nNO_COLOR_S = \"{NO_COLOR}\"\nRED_S = \"{RED}\"\nGREEN_S = \"{GREEN}\"\nBOLD_RED_S = \"{BOLD_RED}\"\nBOLD_GREEN_S = \"{BOLD_GREEN}\"\n\n# intern some strings\nREPLACE_S = \"replace\"\nDELETE_S = \"delete\"\nINSERT_S = \"insert\"\nEQUAL_S = \"equal\"\n\n\ndef bold_str_diff(a, b, sm=None):\n if sm is None:\n sm = difflib.SequenceMatcher()\n aline = RED_S + \"- \"\n bline = GREEN_S + \"+ \"\n sm.set_seqs(a, b)\n for tag, i1, i2, j1, j2 in sm.get_opcodes():\n if tag == REPLACE_S:\n aline += BOLD_RED_S + a[i1:i2] + RED_S\n bline += BOLD_GREEN_S + b[j1:j2] + GREEN_S\n elif tag == DELETE_S:\n aline += BOLD_RED_S + a[i1:i2] + RED_S\n elif tag == INSERT_S:\n bline += BOLD_GREEN_S + b[j1:j2] + GREEN_S\n elif tag == EQUAL_S:\n aline += a[i1:i2]\n bline += b[j1:j2]\n else:\n raise RuntimeError(\"tag not understood\")\n return aline + NO_COLOR_S + \"\\n\" + bline + NO_COLOR_S + \"\\n\"\n\n\ndef redline(line):\n return \"{red}- {line}{no_color}\\n\".format(red=RED_S, line=line, no_color=NO_COLOR_S)\n\n\ndef greenline(line):\n return \"{green}+ {line}{no_color}\\n\".format(\n green=GREEN_S, line=line, no_color=NO_COLOR_S\n )\n\n\ndef highlighted_ndiff(a, b):\n \"\"\"Returns a highlighted string, with bold characters where different.\"\"\"\n s = \"\"\n sm = difflib.SequenceMatcher()\n sm.set_seqs(a, b)\n linesm = difflib.SequenceMatcher()\n for tag, i1, i2, j1, j2 in sm.get_opcodes():\n if tag == REPLACE_S:\n for aline, bline in itertools.zip_longest(a[i1:i2], b[j1:j2]):\n if bline is None:\n s += redline(aline)\n elif aline is None:\n s += greenline(bline)\n else:\n s += bold_str_diff(aline, bline, sm=linesm)\n elif tag == DELETE_S:\n for aline in a[i1:i2]:\n s += redline(aline)\n elif tag == INSERT_S:\n for bline in b[j1:j2]:\n s += greenline(bline)\n elif tag == EQUAL_S:\n for aline in a[i1:i2]:\n s += \" \" + aline + \"\\n\"\n else:\n raise RuntimeError(\"tag not understood\")\n return s\n\n\nclass HistoryDiffer(object):\n \"\"\"This class helps diff two xonsh history files.\"\"\"\n\n def __init__(self, afile, bfile, reopen=False, verbose=False):\n \"\"\"\n Parameters\n ----------\n afile : file handle or str\n The first file to diff\n bfile : file handle or str\n The second file to diff\n reopen : bool, optional\n Whether or not to reopen the file handles each time. The default here is\n opposite from the LazyJSON default because we know that we will be doing\n a lot of reading so it is best to keep the handles open.\n verbose : bool, optional\n Whether to print a verbose amount of information.\n \"\"\"\n self.a = LazyJSON(afile, reopen=reopen)\n self.b = LazyJSON(bfile, reopen=reopen)\n self.verbose = verbose\n self.sm = difflib.SequenceMatcher(autojunk=False)\n\n def __del__(self):\n self.a.close()\n self.b.close()\n\n def __str__(self):\n return self.format()\n\n def _header_line(self, lj):\n s = lj._f.name if hasattr(lj._f, \"name\") else \"\"\n s += \" (\" + lj[\"sessionid\"] + \")\"\n s += \" [locked]\" if lj[\"locked\"] else \" [unlocked]\"\n ts = lj[\"ts\"].load()\n ts0 = datetime.datetime.fromtimestamp(ts[0])\n s += \" started: \" + ts0.isoformat(\" \")\n if ts[1] is not None:\n ts1 = datetime.datetime.fromtimestamp(ts[1])\n s += \" stopped: \" + ts1.isoformat(\" \") + \" runtime: \" + str(ts1 - ts0)\n return s\n\n def header(self):\n \"\"\"Computes a header string difference.\"\"\"\n s = \"{red}--- {aline}{no_color}\\n\" \"{green}+++ {bline}{no_color}\"\n s = s.format(\n aline=self._header_line(self.a),\n bline=self._header_line(self.b),\n red=RED_S,\n green=GREEN_S,\n no_color=NO_COLOR_S,\n )\n return s\n\n def _env_both_diff(self, in_both, aenv, benv):\n sm = self.sm\n s = \"\"\n for key in sorted(in_both):\n aval = aenv[key]\n bval = benv[key]\n if aval == bval:\n continue\n s += \"{0!r} is in both, but differs\\n\".format(key)\n s += bold_str_diff(aval, bval, sm=sm) + \"\\n\"\n return s\n\n def _env_in_one_diff(self, x, y, color, xid, xenv):\n only_x = sorted(x - y)\n if len(only_x) == 0:\n return \"\"\n if self.verbose:\n xstr = \",\\n\".join(\n [\" {0!r}: {1!r}\".format(key, xenv[key]) for key in only_x]\n )\n xstr = \"\\n\" + xstr\n else:\n xstr = \", \".join([\"{0!r}\".format(key) for key in only_x])\n in_x = \"These vars are only in {color}{xid}{no_color}: {{{xstr}}}\\n\\n\"\n return in_x.format(xid=xid, color=color, no_color=NO_COLOR_S, xstr=xstr)\n\n def envdiff(self):\n \"\"\"Computes the difference between the environments.\"\"\"\n aenv = self.a[\"env\"].load()\n benv = self.b[\"env\"].load()\n akeys = frozenset(aenv)\n bkeys = frozenset(benv)\n in_both = akeys & bkeys\n if len(in_both) == len(akeys) == len(bkeys):\n keydiff = self._env_both_diff(in_both, aenv, benv)\n if len(keydiff) == 0:\n return \"\"\n in_a = in_b = \"\"\n else:\n keydiff = self._env_both_diff(in_both, aenv, benv)\n in_a = self._env_in_one_diff(akeys, bkeys, RED_S, self.a[\"sessionid\"], aenv)\n in_b = self._env_in_one_diff(\n bkeys, akeys, GREEN_S, self.b[\"sessionid\"], benv\n )\n s = \"Environment\\n-----------\\n\" + in_a + keydiff + in_b\n return s\n\n def _cmd_in_one_diff(self, inp, i, xlj, xid, color):\n s = \"cmd #{i} only in {color}{xid}{no_color}:\\n\"\n s = s.format(i=i, color=color, xid=xid, no_color=NO_COLOR_S)\n lines = inp.splitlines()\n lt = \"{color}{pre}{no_color} {line}\\n\"\n s += lt.format(color=color, no_color=NO_COLOR_S, line=lines[0], pre=\">>>\")\n for line in lines[1:]:\n s += lt.format(color=color, no_color=NO_COLOR_S, line=line, pre=\"...\")\n if not self.verbose:\n return s + \"\\n\"\n out = xlj[\"cmds\"][0].get(\"out\", \"Note: no output stored\")\n s += out.rstrip() + \"\\n\\n\"\n return s\n\n def _cmd_out_and_rtn_diff(self, i, j):\n s = \"\"\n aout = self.a[\"cmds\"][i].get(\"out\", None)\n bout = self.b[\"cmds\"][j].get(\"out\", None)\n if aout is None and bout is None:\n # s += 'Note: neither output stored\\n'\n pass\n elif bout is None:\n aid = self.a[\"sessionid\"]\n s += \"Note: only {red}{aid}{no_color} output stored\\n\".format(\n red=RED_S, aid=aid, no_color=NO_COLOR_S\n )\n elif aout is None:\n bid = self.b[\"sessionid\"]\n s += \"Note: only {green}{bid}{no_color} output stored\\n\".format(\n green=GREEN_S, bid=bid, no_color=NO_COLOR_S\n )\n elif aout != bout:\n s += \"Outputs differ\\n\"\n s += highlighted_ndiff(aout.splitlines(), bout.splitlines())\n else:\n pass\n artn = self.a[\"cmds\"][i][\"rtn\"]\n brtn = self.b[\"cmds\"][j][\"rtn\"]\n if artn != brtn:\n s += (\n \"Return vals {red}{artn}{no_color} & {green}{brtn}{no_color} differ\\n\"\n ).format(\n red=RED_S, green=GREEN_S, no_color=NO_COLOR_S, artn=artn, brtn=brtn\n )\n return s\n\n def _cmd_replace_diff(self, i, ainp, aid, j, binp, bid):\n s = (\n \"cmd #{i} in {red}{aid}{no_color} is replaced by \\n\"\n \"cmd #{j} in {green}{bid}{no_color}:\\n\"\n )\n s = s.format(\n i=i, aid=aid, j=j, bid=bid, red=RED_S, green=GREEN_S, no_color=NO_COLOR_S\n )\n s += highlighted_ndiff(ainp.splitlines(), binp.splitlines())\n if not self.verbose:\n return s + \"\\n\"\n s += self._cmd_out_and_rtn_diff(i, j)\n return s + \"\\n\"\n\n def cmdsdiff(self):\n \"\"\"Computes the difference of the commands themselves.\"\"\"\n aid = self.a[\"sessionid\"]\n bid = self.b[\"sessionid\"]\n ainps = [c[\"inp\"] for c in self.a[\"cmds\"]]\n binps = [c[\"inp\"] for c in self.b[\"cmds\"]]\n sm = self.sm\n sm.set_seqs(ainps, binps)\n s = \"\"\n for tag, i1, i2, j1, j2 in sm.get_opcodes():\n if tag == REPLACE_S:\n zipper = itertools.zip_longest\n for i, ainp, j, binp in zipper(\n range(i1, i2), ainps[i1:i2], range(j1, j2), binps[j1:j2]\n ):\n if j is None:\n s += self._cmd_in_one_diff(ainp, i, self.a, aid, RED_S)\n elif i is None:\n s += self._cmd_in_one_diff(binp, j, self.b, bid, GREEN_S)\n else:\n self._cmd_replace_diff(i, ainp, aid, j, binp, bid)\n elif tag == DELETE_S:\n for i, inp in enumerate(ainps[i1:i2], i1):\n s += self._cmd_in_one_diff(inp, i, self.a, aid, RED_S)\n elif tag == INSERT_S:\n for j, inp in enumerate(binps[j1:j2], j1):\n s += self._cmd_in_one_diff(inp, j, self.b, bid, GREEN_S)\n elif tag == EQUAL_S:\n for i, j in zip(range(i1, i2), range(j1, j2)):\n odiff = self._cmd_out_and_rtn_diff(i, j)\n if len(odiff) > 0:\n h = (\n \"cmd #{i} in {red}{aid}{no_color} input is the same as \\n\"\n \"cmd #{j} in {green}{bid}{no_color}, but output differs:\\n\"\n )\n s += h.format(\n i=i,\n aid=aid,\n j=j,\n bid=bid,\n red=RED_S,\n green=GREEN_S,\n no_color=NO_COLOR_S,\n )\n s += odiff + \"\\n\"\n else:\n raise RuntimeError(\"tag not understood\")\n if len(s) == 0:\n return s\n return \"Commands\\n--------\\n\" + s\n\n def format(self):\n \"\"\"Formats the difference between the two history files.\"\"\"\n s = self.header()\n ed = self.envdiff()\n if len(ed) > 0:\n s += \"\\n\\n\" + ed\n cd = self.cmdsdiff()\n if len(cd) > 0:\n s += \"\\n\\n\" + cd\n return s.rstrip()\n\n\n_HD_PARSER = None\n\n\ndef dh_create_parser(p=None):\n global _HD_PARSER\n p_was_none = p is None\n if _HD_PARSER is not None and p_was_none:\n return _HD_PARSER\n if p_was_none:\n p = argparse.ArgumentParser(\n \"diff-history\", description=\"diffs two xonsh history files\"\n )\n p.add_argument(\n \"--reopen\",\n dest=\"reopen\",\n default=False,\n action=\"store_true\",\n help=\"make lazy file loading reopen files each time\",\n )\n p.add_argument(\n \"-v\",\n \"--verbose\",\n dest=\"verbose\",\n default=False,\n action=\"store_true\",\n help=\"whether to print even more information\",\n )\n p.add_argument(\"a\", help=\"first file in diff\")\n p.add_argument(\"b\", help=\"second file in diff\")\n if p_was_none:\n _HD_PARSER = p\n return p\n\n\ndef dh_main_action(ns, hist=None, stdout=None, stderr=None):\n hd = HistoryDiffer(ns.a, ns.b, reopen=ns.reopen, verbose=ns.verbose)\n print_color(hd.format(), file=stdout)\n" }, { "path": "xonsh/dirstack.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"Directory stack and associated utilities for the xonsh shell.\"\"\"\nimport os\nimport glob\nimport argparse\nimport builtins\nimport subprocess\n\nfrom xonsh.lazyasd import lazyobject\nfrom xonsh.tools import get_sep\nfrom xonsh.events import events\nfrom xonsh.platform import ON_WINDOWS\n\nDIRSTACK = []\n\"\"\"A list containing the currently remembered directories.\"\"\"\n_unc_tempDrives = {}\n\"\"\" drive: sharePath for temp drive letters we create for UNC mapping\"\"\"\n\n\ndef _unc_check_enabled() -> bool:\n r\"\"\"Check whether CMD.EXE is enforcing no-UNC-as-working-directory check.\n\n Check can be disabled by setting {HKCU, HKLM}/SOFTWARE\\Microsoft\\Command Processor\\DisableUNCCheck:REG_DWORD=1\n\n Returns:\n True if `CMD.EXE` is enforcing the check (default Windows situation)\n False if check is explicitly disabled.\n \"\"\"\n if not ON_WINDOWS:\n return\n\n import winreg\n\n wval = None\n\n try:\n key = winreg.OpenKey(\n winreg.HKEY_CURRENT_USER, r\"software\\microsoft\\command processor\"\n )\n wval, wtype = winreg.QueryValueEx(key, \"DisableUNCCheck\")\n winreg.CloseKey(key)\n except OSError:\n pass\n\n if wval is None:\n try:\n key2 = winreg.OpenKey(\n winreg.HKEY_LOCAL_MACHINE, r\"software\\microsoft\\command processor\"\n )\n wval, wtype = winreg.QueryValueEx(key2, \"DisableUNCCheck\")\n winreg.CloseKey(key2)\n except OSError as e: # NOQA\n pass\n\n return False if wval else True\n\n\ndef _is_unc_path(some_path) -> bool:\n \"\"\"True if path starts with 2 backward (or forward, due to python path hacking) slashes.\"\"\"\n return (\n len(some_path) > 1\n and some_path[0] == some_path[1]\n and some_path[0] in (os.sep, os.altsep)\n )\n\n\ndef _unc_map_temp_drive(unc_path) -> str:\n r\"\"\"Map a new temporary drive letter for each distinct share,\n unless `CMD.EXE` is not insisting on non-UNC working directory.\n\n Emulating behavior of `CMD.EXE` `pushd`, create a new mapped drive (starting from Z: towards A:, skipping existing\n drive letters) for each new UNC path user selects.\n\n Args:\n unc_path: the path specified by user. Assumed to be a UNC path of form \\\\\\share...\n\n Returns:\n a replacement for `unc_path` to be used as the actual new working directory.\n Note that the drive letter may be a the same as one already mapped if the server and share portion of `unc_path`\n is the same as one still active on the stack.\n \"\"\"\n global _unc_tempDrives\n assert unc_path[1] in (os.sep, os.altsep), \"unc_path is UNC form of path\"\n\n if not _unc_check_enabled():\n return unc_path\n else:\n unc_share, rem_path = os.path.splitdrive(unc_path)\n unc_share = unc_share.casefold()\n for d in _unc_tempDrives:\n if _unc_tempDrives[d] == unc_share:\n return os.path.join(d, rem_path)\n\n for dord in range(ord(\"z\"), ord(\"a\"), -1):\n d = chr(dord) + \":\"\n if not os.path.isdir(d): # find unused drive letter starting from z:\n subprocess.check_output(\n [\"NET\", \"USE\", d, unc_share], universal_newlines=True\n )\n _unc_tempDrives[d] = unc_share\n return os.path.join(d, rem_path)\n\n\ndef _unc_unmap_temp_drive(left_drive, cwd):\n \"\"\"Unmap a temporary drive letter if it is no longer needed.\n Called after popping `DIRSTACK` and changing to new working directory, so we need stack *and*\n new current working directory to be sure drive letter no longer needed.\n\n Args:\n left_drive: driveletter (and colon) of working directory we just left\n cwd: full path of new current working directory\n \"\"\"\n\n global _unc_tempDrives\n\n if left_drive not in _unc_tempDrives: # if not one we've mapped, don't unmap it\n return\n\n for p in DIRSTACK + [cwd]: # if still in use , don't unmap it.\n if p.casefold().startswith(left_drive):\n return\n\n _unc_tempDrives.pop(left_drive)\n subprocess.check_output(\n [\"NET\", \"USE\", left_drive, \"/delete\"], universal_newlines=True\n )\n\n\nevents.doc(\n \"on_chdir\",\n \"\"\"\non_chdir(olddir: str, newdir: str) -> None\n\nFires when the current directory is changed for any reason.\n\"\"\",\n)\n\n\ndef _get_cwd():\n try:\n return os.getcwd()\n except (OSError, FileNotFoundError):\n return None\n\n\ndef _change_working_directory(newdir, follow_symlinks=False):\n env = builtins.__xonsh__.env\n old = env[\"PWD\"]\n new = os.path.join(old, newdir)\n absnew = os.path.abspath(new)\n\n if follow_symlinks:\n absnew = os.path.realpath(absnew)\n\n try:\n os.chdir(absnew)\n except (OSError, FileNotFoundError):\n if new.endswith(get_sep()):\n new = new[:-1]\n if os.path.basename(new) == \"..\":\n env[\"PWD\"] = new\n else:\n if old is not None:\n env[\"OLDPWD\"] = old\n if new is not None:\n env[\"PWD\"] = absnew\n\n # Fire event if the path actually changed\n if old != env[\"PWD\"]:\n events.on_chdir.fire(olddir=old, newdir=env[\"PWD\"])\n\n\ndef _try_cdpath(apath):\n # NOTE: this CDPATH implementation differs from the bash one.\n # In bash if a CDPATH is set, an unqualified local folder\n # is considered after all CDPATHs, example:\n # CDPATH=$HOME/src (with src/xonsh/ inside)\n # $ cd xonsh -> src/xonsh (with xonsh/xonsh)\n # a second $ cd xonsh has no effects, to move in the nested xonsh\n # in bash a full $ cd ./xonsh is needed.\n # In xonsh a relative folder is always preferred.\n env = builtins.__xonsh__.env\n cdpaths = env.get(\"CDPATH\")\n for cdp in cdpaths:\n globber = builtins.__xonsh__.expand_path(os.path.join(cdp, apath))\n for cdpath_prefixed_path in glob.iglob(globber):\n return cdpath_prefixed_path\n return apath\n\n\ndef cd(args, stdin=None):\n \"\"\"Changes the directory.\n\n If no directory is specified (i.e. if `args` is None) then this\n changes to the current user's home directory.\n \"\"\"\n env = builtins.__xonsh__.env\n oldpwd = env.get(\"OLDPWD\", None)\n cwd = env[\"PWD\"]\n\n follow_symlinks = False\n if len(args) > 0 and args[0] == \"-P\":\n follow_symlinks = True\n del args[0]\n\n if len(args) == 0:\n d = os.path.expanduser(\"~\")\n elif len(args) == 1:\n d = os.path.expanduser(args[0])\n if not os.path.isdir(d):\n if d == \"-\":\n if oldpwd is not None:\n d = oldpwd\n else:\n return \"\", \"cd: no previous directory stored\\n\", 1\n elif d.startswith(\"-\"):\n try:\n num = int(d[1:])\n except ValueError:\n return \"\", \"cd: Invalid destination: {0}\\n\".format(d), 1\n if num == 0:\n return None, None, 0\n elif num < 0:\n return \"\", \"cd: Invalid destination: {0}\\n\".format(d), 1\n elif num > len(DIRSTACK):\n e = \"cd: Too few elements in dirstack ({0} elements)\\n\"\n return \"\", e.format(len(DIRSTACK)), 1\n else:\n d = DIRSTACK[num - 1]\n else:\n d = _try_cdpath(d)\n else:\n return (\n \"\",\n (\n \"cd takes 0 or 1 arguments, not {0}. An additional `-P` \"\n \"flag can be passed in first position to follow symlinks.\"\n \"\\n\".format(len(args))\n ),\n 1,\n )\n if not os.path.exists(d):\n return \"\", \"cd: no such file or directory: {0}\\n\".format(d), 1\n if not os.path.isdir(d):\n return \"\", \"cd: {0} is not a directory\\n\".format(d), 1\n if not os.access(d, os.X_OK):\n return \"\", \"cd: permission denied: {0}\\n\".format(d), 1\n if (\n ON_WINDOWS\n and _is_unc_path(d)\n and _unc_check_enabled()\n and (not env.get(\"AUTO_PUSHD\"))\n ):\n return (\n \"\",\n \"cd: can't cd to UNC path on Windows, unless $AUTO_PUSHD set or reg entry \"\n + r\"HKCU\\SOFTWARE\\MICROSOFT\\Command Processor\\DisableUNCCheck:DWORD = 1\"\n + \"\\n\",\n 1,\n )\n\n # now, push the directory onto the dirstack if AUTO_PUSHD is set\n if cwd is not None and env.get(\"AUTO_PUSHD\"):\n pushd([\"-n\", \"-q\", cwd])\n if ON_WINDOWS and _is_unc_path(d):\n d = _unc_map_temp_drive(d)\n _change_working_directory(d, follow_symlinks)\n return None, None, 0\n\n\n@lazyobject\ndef pushd_parser():\n parser = argparse.ArgumentParser(prog=\"pushd\")\n parser.add_argument(\"dir\", nargs=\"?\")\n parser.add_argument(\n \"-n\",\n dest=\"cd\",\n help=\"Suppresses the normal change of directory when\"\n \" adding directories to the stack, so that only the\"\n \" stack is manipulated.\",\n action=\"store_false\",\n )\n parser.add_argument(\n \"-q\",\n dest=\"quiet\",\n help=\"Do not call dirs, regardless of $PUSHD_SILENT\",\n action=\"store_true\",\n )\n return parser\n\n\ndef pushd(args, stdin=None):\n r\"\"\"xonsh command: pushd\n\n Adds a directory to the top of the directory stack, or rotates the stack,\n making the new top of the stack the current working directory.\n\n On Windows, if the path is a UNC path (begins with `\\\\\\`) and if the `DisableUNCCheck` registry\n value is not enabled, creates a temporary mapped drive letter and sets the working directory there, emulating\n behavior of `PUSHD` in `CMD.EXE`\n \"\"\"\n global DIRSTACK\n\n try:\n args = pushd_parser.parse_args(args)\n except SystemExit:\n return None, None, 1\n\n env = builtins.__xonsh__.env\n\n pwd = env[\"PWD\"]\n\n if env.get(\"PUSHD_MINUS\", False):\n BACKWARD = \"-\"\n FORWARD = \"+\"\n else:\n BACKWARD = \"+\"\n FORWARD = \"-\"\n\n if args.dir is None:\n try:\n new_pwd = DIRSTACK.pop(0)\n except IndexError:\n e = \"pushd: Directory stack is empty\\n\"\n return None, e, 1\n elif os.path.isdir(args.dir):\n new_pwd = args.dir\n else:\n try:\n num = int(args.dir[1:])\n except ValueError:\n e = \"Invalid argument to pushd: {0}\\n\"\n return None, e.format(args.dir), 1\n\n if num < 0:\n e = \"Invalid argument to pushd: {0}\\n\"\n return None, e.format(args.dir), 1\n\n if num > len(DIRSTACK):\n e = \"Too few elements in dirstack ({0} elements)\\n\"\n return None, e.format(len(DIRSTACK)), 1\n elif args.dir.startswith(FORWARD):\n if num == len(DIRSTACK):\n new_pwd = None\n else:\n new_pwd = DIRSTACK.pop(len(DIRSTACK) - 1 - num)\n elif args.dir.startswith(BACKWARD):\n if num == 0:\n new_pwd = None\n else:\n new_pwd = DIRSTACK.pop(num - 1)\n else:\n e = \"Invalid argument to pushd: {0}\\n\"\n return None, e.format(args.dir), 1\n if new_pwd is not None:\n if ON_WINDOWS and _is_unc_path(new_pwd):\n new_pwd = _unc_map_temp_drive(new_pwd)\n if args.cd:\n DIRSTACK.insert(0, os.path.expanduser(pwd))\n _change_working_directory(new_pwd)\n else:\n DIRSTACK.insert(0, os.path.expanduser(new_pwd))\n\n maxsize = env.get(\"DIRSTACK_SIZE\")\n if len(DIRSTACK) > maxsize:\n DIRSTACK = DIRSTACK[:maxsize]\n\n if not args.quiet and not env.get(\"PUSHD_SILENT\"):\n return dirs([], None)\n\n return None, None, 0\n\n\n@lazyobject\ndef popd_parser():\n parser = argparse.ArgumentParser(prog=\"popd\")\n parser.add_argument(\"dir\", nargs=\"?\")\n parser.add_argument(\n \"-n\",\n dest=\"cd\",\n help=\"Suppresses the normal change of directory when\"\n \" adding directories to the stack, so that only the\"\n \" stack is manipulated.\",\n action=\"store_false\",\n )\n parser.add_argument(\n \"-q\",\n dest=\"quiet\",\n help=\"Do not call dirs, regardless of $PUSHD_SILENT\",\n action=\"store_true\",\n )\n return parser\n\n\ndef popd(args, stdin=None):\n \"\"\"\n xonsh command: popd\n\n Removes entries from the directory stack.\n \"\"\"\n global DIRSTACK\n\n try:\n args = pushd_parser.parse_args(args)\n except SystemExit:\n return None, None, 1\n\n env = builtins.__xonsh__.env\n\n if env.get(\"PUSHD_MINUS\"):\n BACKWARD = \"-\"\n FORWARD = \"+\"\n else:\n BACKWARD = \"-\"\n FORWARD = \"+\"\n\n if args.dir is None:\n try:\n new_pwd = DIRSTACK.pop(0)\n except IndexError:\n e = \"popd: Directory stack is empty\\n\"\n return None, e, 1\n else:\n try:\n num = int(args.dir[1:])\n except ValueError:\n e = \"Invalid argument to popd: {0}\\n\"\n return None, e.format(args.dir), 1\n\n if num < 0:\n e = \"Invalid argument to popd: {0}\\n\"\n return None, e.format(args.dir), 1\n\n if num > len(DIRSTACK):\n e = \"Too few elements in dirstack ({0} elements)\\n\"\n return None, e.format(len(DIRSTACK)), 1\n elif args.dir.startswith(FORWARD):\n if num == len(DIRSTACK):\n new_pwd = DIRSTACK.pop(0)\n else:\n new_pwd = None\n DIRSTACK.pop(len(DIRSTACK) - 1 - num)\n elif args.dir.startswith(BACKWARD):\n if num == 0:\n new_pwd = DIRSTACK.pop(0)\n else:\n new_pwd = None\n DIRSTACK.pop(num - 1)\n else:\n e = \"Invalid argument to popd: {0}\\n\"\n return None, e.format(args.dir), 1\n\n if new_pwd is not None:\n e = None\n if args.cd:\n env = builtins.__xonsh__.env\n pwd = env[\"PWD\"]\n\n _change_working_directory(new_pwd)\n\n if ON_WINDOWS:\n drive, rem_path = os.path.splitdrive(pwd)\n _unc_unmap_temp_drive(drive.casefold(), new_pwd)\n\n if not args.quiet and not env.get(\"PUSHD_SILENT\"):\n return dirs([], None)\n\n return None, None, 0\n\n\n@lazyobject\ndef dirs_parser():\n parser = argparse.ArgumentParser(prog=\"dirs\")\n parser.add_argument(\"N\", nargs=\"?\")\n parser.add_argument(\n \"-c\",\n dest=\"clear\",\n help=\"Clears the directory stack by deleting all of\" \" the entries.\",\n action=\"store_true\",\n )\n parser.add_argument(\n \"-p\",\n dest=\"print_long\",\n help=\"Print the directory stack with one entry per\" \" line.\",\n action=\"store_true\",\n )\n parser.add_argument(\n \"-v\",\n dest=\"verbose\",\n help=\"Print the directory stack with one entry per\"\n \" line, prefixing each entry with its index in the\"\n \" stack.\",\n action=\"store_true\",\n )\n parser.add_argument(\n \"-l\",\n dest=\"long\",\n help=\"Produces a longer listing; the default listing\"\n \" format uses a tilde to denote the home directory.\",\n action=\"store_true\",\n )\n return parser\n\n\ndef dirs(args, stdin=None):\n \"\"\"xonsh command: dirs\n\n Displays the list of currently remembered directories. Can also be used\n to clear the directory stack.\n \"\"\"\n global DIRSTACK\n try:\n args = dirs_parser.parse_args(args)\n except SystemExit:\n return None, None\n\n env = builtins.__xonsh__.env\n dirstack = [os.path.expanduser(env[\"PWD\"])] + DIRSTACK\n\n if env.get(\"PUSHD_MINUS\"):\n BACKWARD = \"-\"\n FORWARD = \"+\"\n else:\n BACKWARD = \"-\"\n FORWARD = \"+\"\n\n if args.clear:\n DIRSTACK = []\n return None, None, 0\n\n if args.long:\n o = dirstack\n else:\n d = os.path.expanduser(\"~\")\n o = [i.replace(d, \"~\") for i in dirstack]\n\n if args.verbose:\n out = \"\"\n pad = len(str(len(o) - 1))\n for (ix, e) in enumerate(o):\n blanks = \" \" * (pad - len(str(ix)))\n out += \"\\n{0}{1} {2}\".format(blanks, ix, e)\n out = out[1:]\n elif args.print_long:\n out = \"\\n\".join(o)\n else:\n out = \" \".join(o)\n\n N = args.N\n if N is not None:\n try:\n num = int(N[1:])\n except ValueError:\n e = \"Invalid argument to dirs: {0}\\n\"\n return None, e.format(N), 1\n\n if num < 0:\n e = \"Invalid argument to dirs: {0}\\n\"\n return None, e.format(len(o)), 1\n\n if num >= len(o):\n e = \"Too few elements in dirstack ({0} elements)\\n\"\n return None, e.format(len(o)), 1\n\n if N.startswith(BACKWARD):\n idx = num\n elif N.startswith(FORWARD):\n idx = len(o) - 1 - num\n else:\n e = \"Invalid argument to dirs: {0}\\n\"\n return None, e.format(N), 1\n\n out = o[idx]\n\n return out + \"\\n\", None, 0\n" }, { "path": "xonsh/dumb_shell.py", "content": "\"\"\"A dumb shell for when $TERM == 'dumb', which usually happens in emacs.\"\"\"\nimport builtins\n\nfrom xonsh.readline_shell import ReadlineShell\n\n\nclass DumbShell(ReadlineShell):\n \"\"\"A dumb shell for when $TERM == 'dumb', which usually happens in emacs.\"\"\"\n\n def __init__(self, *args, **kwargs):\n builtins.__xonsh__.env[\"XONSH_COLOR_STYLE\"] = \"emacs\"\n super().__init__(*args, **kwargs)\n" }, { "path": "xonsh/environ.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"Environment for the xonsh shell.\"\"\"\nimport os\nimport re\nimport sys\nimport pprint\nimport textwrap\nimport locale\nimport builtins\nimport warnings\nimport contextlib\nimport collections\nimport collections.abc as cabc\nimport subprocess\n\nfrom xonsh import __version__ as XONSH_VERSION\nfrom xonsh.lazyasd import LazyObject, lazyobject\nfrom xonsh.codecache import run_script_with_cache\nfrom xonsh.dirstack import _get_cwd\nfrom xonsh.events import events\nfrom xonsh.platform import (\n BASH_COMPLETIONS_DEFAULT,\n DEFAULT_ENCODING,\n PATH_DEFAULT,\n ON_WINDOWS,\n ON_LINUX,\n os_environ,\n)\n\nfrom xonsh.style_tools import PTK2_STYLE\n\nfrom xonsh.tools import (\n always_true,\n always_false,\n detype,\n ensure_string,\n is_env_path,\n str_to_env_path,\n env_path_to_str,\n is_bool,\n to_bool,\n bool_to_str,\n is_history_tuple,\n to_history_tuple,\n history_tuple_to_str,\n is_float,\n is_string,\n is_string_or_callable,\n is_completions_display_value,\n to_completions_display_value,\n is_string_set,\n csv_to_set,\n set_to_csv,\n is_int,\n is_bool_seq,\n to_bool_or_int,\n bool_or_int_to_str,\n csv_to_bool_seq,\n bool_seq_to_csv,\n DefaultNotGiven,\n print_exception,\n setup_win_unicode_console,\n intensify_colors_on_win_setter,\n is_dynamic_cwd_width,\n to_dynamic_cwd_tuple,\n dynamic_cwd_tuple_to_str,\n is_logfile_opt,\n to_logfile_opt,\n logfile_opt_to_str,\n executables_in,\n is_nonstring_seq_of_strings,\n pathsep_to_upper_seq,\n seq_to_upper_pathsep,\n print_color,\n is_history_backend,\n to_itself,\n swap_values,\n ptk2_color_depth_setter,\n is_str_str_dict,\n to_str_str_dict,\n dict_to_str,\n)\nfrom xonsh.ansi_colors import (\n ansi_color_escape_code_to_name,\n ansi_reverse_style,\n ansi_style_by_name,\n)\nimport xonsh.prompt.base as prompt\n\n\nevents.doc(\n \"on_envvar_new\",\n \"\"\"\non_envvar_new(name: str, value: Any) -> None\n\nFires after a new environment variable is created.\nNote: Setting envvars inside the handler might\ncause a recursion until the limit.\n\"\"\",\n)\n\n\nevents.doc(\n \"on_envvar_change\",\n \"\"\"\non_envvar_change(name: str, oldvalue: Any, newvalue: Any) -> None\n\nFires after an environment variable is changed.\nNote: Setting envvars inside the handler might\ncause a recursion until the limit.\n\"\"\",\n)\n\n\nevents.doc(\n \"on_pre_spec_run_ls\",\n \"\"\"\non_pre_spec_run_ls(spec: xonsh.built_ins.SubprocSpec) -> None\n\nFires right before a SubprocSpec.run() is called for the ls\ncommand.\n\"\"\",\n)\n\n\n@lazyobject\ndef HELP_TEMPLATE():\n return (\n \"{{INTENSE_RED}}{envvar}{{NO_COLOR}}:\\n\\n\"\n \"{{INTENSE_YELLOW}}{docstr}{{NO_COLOR}}\\n\\n\"\n \"default: {{CYAN}}{default}{{NO_COLOR}}\\n\"\n \"configurable: {{CYAN}}{configurable}{{NO_COLOR}}\"\n )\n\n\n@lazyobject\ndef LOCALE_CATS():\n lc = {\n \"LC_CTYPE\": locale.LC_CTYPE,\n \"LC_COLLATE\": locale.LC_COLLATE,\n \"LC_NUMERIC\": locale.LC_NUMERIC,\n \"LC_MONETARY\": locale.LC_MONETARY,\n \"LC_TIME\": locale.LC_TIME,\n }\n if hasattr(locale, \"LC_MESSAGES\"):\n lc[\"LC_MESSAGES\"] = locale.LC_MESSAGES\n return lc\n\n\ndef locale_convert(key):\n \"\"\"Creates a converter for a locale key.\"\"\"\n\n def lc_converter(val):\n try:\n locale.setlocale(LOCALE_CATS[key], val)\n val = locale.setlocale(LOCALE_CATS[key])\n except (locale.Error, KeyError):\n msg = \"Failed to set locale {0!r} to {1!r}\".format(key, val)\n warnings.warn(msg, RuntimeWarning)\n return val\n\n return lc_converter\n\n\ndef to_debug(x):\n \"\"\"Converts value using to_bool_or_int() and sets this value on as the\n execer's debug level.\n \"\"\"\n val = to_bool_or_int(x)\n if (\n hasattr(builtins, \"__xonsh__\")\n and hasattr(builtins.__xonsh__, \"execer\")\n and builtins.__xonsh__.execer is not None\n ):\n builtins.__xonsh__.execer.debug_level = val\n return val\n\n\n#\n# $LS_COLORS tools\n#\n\n\nclass LsColors(cabc.MutableMapping):\n \"\"\"Helps convert to/from $LS_COLORS format, respecting the xonsh color style.\n This accepts the same inputs as dict().\n \"\"\"\n\n default_settings = {\n \"*.7z\": (\"BOLD_RED\",),\n \"*.Z\": (\"BOLD_RED\",),\n \"*.aac\": (\"CYAN\",),\n \"*.ace\": (\"BOLD_RED\",),\n \"*.alz\": (\"BOLD_RED\",),\n \"*.arc\": (\"BOLD_RED\",),\n \"*.arj\": (\"BOLD_RED\",),\n \"*.asf\": (\"BOLD_PURPLE\",),\n \"*.au\": (\"CYAN\",),\n \"*.avi\": (\"BOLD_PURPLE\",),\n \"*.bmp\": (\"BOLD_PURPLE\",),\n \"*.bz\": (\"BOLD_RED\",),\n \"*.bz2\": (\"BOLD_RED\",),\n \"*.cab\": (\"BOLD_RED\",),\n \"*.cgm\": (\"BOLD_PURPLE\",),\n \"*.cpio\": (\"BOLD_RED\",),\n \"*.deb\": (\"BOLD_RED\",),\n \"*.dl\": (\"BOLD_PURPLE\",),\n \"*.dwm\": (\"BOLD_RED\",),\n \"*.dz\": (\"BOLD_RED\",),\n \"*.ear\": (\"BOLD_RED\",),\n \"*.emf\": (\"BOLD_PURPLE\",),\n \"*.esd\": (\"BOLD_RED\",),\n \"*.flac\": (\"CYAN\",),\n \"*.flc\": (\"BOLD_PURPLE\",),\n \"*.fli\": (\"BOLD_PURPLE\",),\n \"*.flv\": (\"BOLD_PURPLE\",),\n \"*.gif\": (\"BOLD_PURPLE\",),\n \"*.gl\": (\"BOLD_PURPLE\",),\n \"*.gz\": (\"BOLD_RED\",),\n \"*.jar\": (\"BOLD_RED\",),\n \"*.jpeg\": (\"BOLD_PURPLE\",),\n \"*.jpg\": (\"BOLD_PURPLE\",),\n \"*.lha\": (\"BOLD_RED\",),\n \"*.lrz\": (\"BOLD_RED\",),\n \"*.lz\": (\"BOLD_RED\",),\n \"*.lz4\": (\"BOLD_RED\",),\n \"*.lzh\": (\"BOLD_RED\",),\n \"*.lzma\": (\"BOLD_RED\",),\n \"*.lzo\": (\"BOLD_RED\",),\n \"*.m2v\": (\"BOLD_PURPLE\",),\n \"*.m4a\": (\"CYAN\",),\n \"*.m4v\": (\"BOLD_PURPLE\",),\n \"*.mid\": (\"CYAN\",),\n \"*.midi\": (\"CYAN\",),\n \"*.mjpeg\": (\"BOLD_PURPLE\",),\n \"*.mjpg\": (\"BOLD_PURPLE\",),\n \"*.mka\": (\"CYAN\",),\n \"*.mkv\": (\"BOLD_PURPLE\",),\n \"*.mng\": (\"BOLD_PURPLE\",),\n \"*.mov\": (\"BOLD_PURPLE\",),\n \"*.mp3\": (\"CYAN\",),\n \"*.mp4\": (\"BOLD_PURPLE\",),\n \"*.mp4v\": (\"BOLD_PURPLE\",),\n \"*.mpc\": (\"CYAN\",),\n \"*.mpeg\": (\"BOLD_PURPLE\",),\n \"*.mpg\": (\"BOLD_PURPLE\",),\n \"*.nuv\": (\"BOLD_PURPLE\",),\n \"*.oga\": (\"CYAN\",),\n \"*.ogg\": (\"CYAN\",),\n \"*.ogm\": (\"BOLD_PURPLE\",),\n \"*.ogv\": (\"BOLD_PURPLE\",),\n \"*.ogx\": (\"BOLD_PURPLE\",),\n \"*.opus\": (\"CYAN\",),\n \"*.pbm\": (\"BOLD_PURPLE\",),\n \"*.pcx\": (\"BOLD_PURPLE\",),\n \"*.pgm\": (\"BOLD_PURPLE\",),\n \"*.png\": (\"BOLD_PURPLE\",),\n \"*.ppm\": (\"BOLD_PURPLE\",),\n \"*.qt\": (\"BOLD_PURPLE\",),\n \"*.ra\": (\"CYAN\",),\n \"*.rar\": (\"BOLD_RED\",),\n \"*.rm\": (\"BOLD_PURPLE\",),\n \"*.rmvb\": (\"BOLD_PURPLE\",),\n \"*.rpm\": (\"BOLD_RED\",),\n \"*.rz\": (\"BOLD_RED\",),\n \"*.sar\": (\"BOLD_RED\",),\n \"*.spx\": (\"CYAN\",),\n \"*.svg\": (\"BOLD_PURPLE\",),\n \"*.svgz\": (\"BOLD_PURPLE\",),\n \"*.swm\": (\"BOLD_RED\",),\n \"*.t7z\": (\"BOLD_RED\",),\n \"*.tar\": (\"BOLD_RED\",),\n \"*.taz\": (\"BOLD_RED\",),\n \"*.tbz\": (\"BOLD_RED\",),\n \"*.tbz2\": (\"BOLD_RED\",),\n \"*.tga\": (\"BOLD_PURPLE\",),\n \"*.tgz\": (\"BOLD_RED\",),\n \"*.tif\": (\"BOLD_PURPLE\",),\n \"*.tiff\": (\"BOLD_PURPLE\",),\n \"*.tlz\": (\"BOLD_RED\",),\n \"*.txz\": (\"BOLD_RED\",),\n \"*.tz\": (\"BOLD_RED\",),\n \"*.tzo\": (\"BOLD_RED\",),\n \"*.tzst\": (\"BOLD_RED\",),\n \"*.vob\": (\"BOLD_PURPLE\",),\n \"*.war\": (\"BOLD_RED\",),\n \"*.wav\": (\"CYAN\",),\n \"*.webm\": (\"BOLD_PURPLE\",),\n \"*.wim\": (\"BOLD_RED\",),\n \"*.wmv\": (\"BOLD_PURPLE\",),\n \"*.xbm\": (\"BOLD_PURPLE\",),\n \"*.xcf\": (\"BOLD_PURPLE\",),\n \"*.xpm\": (\"BOLD_PURPLE\",),\n \"*.xspf\": (\"CYAN\",),\n \"*.xwd\": (\"BOLD_PURPLE\",),\n \"*.xz\": (\"BOLD_RED\",),\n \"*.yuv\": (\"BOLD_PURPLE\",),\n \"*.z\": (\"BOLD_RED\",),\n \"*.zip\": (\"BOLD_RED\",),\n \"*.zoo\": (\"BOLD_RED\",),\n \"*.zst\": (\"BOLD_RED\",),\n \"bd\": (\"BACKGROUND_BLACK\", \"YELLOW\"),\n \"ca\": (\"BLACK\", \"BACKGROUND_RED\"),\n \"cd\": (\"BACKGROUND_BLACK\", \"YELLOW\"),\n \"di\": (\"BOLD_BLUE\",),\n \"do\": (\"BOLD_PURPLE\",),\n \"ex\": (\"BOLD_GREEN\",),\n \"ln\": (\"BOLD_CYAN\",),\n \"mh\": (\"NO_COLOR\",),\n \"mi\": (\"NO_COLOR\",),\n \"or\": (\"BACKGROUND_BLACK\", \"RED\"),\n \"ow\": (\"BLUE\", \"BACKGROUND_GREEN\"),\n \"pi\": (\"BACKGROUND_BLACK\", \"YELLOW\"),\n \"rs\": (\"NO_COLOR\",),\n \"sg\": (\"BLACK\", \"BACKGROUND_YELLOW\"),\n \"so\": (\"BOLD_PURPLE\",),\n \"st\": (\"WHITE\", \"BACKGROUND_BLUE\"),\n \"su\": (\"WHITE\", \"BACKGROUND_RED\"),\n \"tw\": (\"BLACK\", \"BACKGROUND_GREEN\"),\n }\n\n def __init__(self, *args, **kwargs):\n self._d = dict(*args, **kwargs)\n self._style = self._style_name = None\n self._detyped = None\n\n def __getitem__(self, key):\n return self._d[key]\n\n def __setitem__(self, key, value):\n self._detyped = None\n self._d[key] = value\n\n def __delitem__(self, key):\n self._detyped = None\n del self._d[key]\n\n def __len__(self):\n return len(self._d)\n\n def __iter__(self):\n yield from self._d\n\n def __str__(self):\n return str(self._d)\n\n def __repr__(self):\n return \"{0}.{1}(...)\".format(\n self.__class__.__module__, self.__class__.__name__, self._d\n )\n\n def _repr_pretty_(self, p, cycle):\n name = \"{0}.{1}\".format(self.__class__.__module__, self.__class__.__name__)\n with p.group(0, name + \"(\", \")\"):\n if cycle:\n p.text(\"...\")\n elif len(self):\n p.break_()\n p.pretty(dict(self))\n\n def detype(self):\n \"\"\"De-types the instance, allowing it to be exported to the environment.\"\"\"\n style = self.style\n if self._detyped is None:\n self._detyped = \":\".join(\n [\n key + \"=\" + \";\".join([style[v] or \"0\" for v in val])\n for key, val in sorted(self._d.items())\n ]\n )\n return self._detyped\n\n @property\n def style_name(self):\n \"\"\"Current XONSH_COLOR_STYLE value\"\"\"\n env = builtins.__xonsh__.env\n env_style_name = env.get(\"XONSH_COLOR_STYLE\")\n if self._style_name is None or self._style_name != env_style_name:\n self._style_name = env_style_name\n self._style = self._dtyped = None\n return self._style_name\n\n @property\n def style(self):\n \"\"\"The ANSI color style for the current XONSH_COLOR_STYLE\"\"\"\n style_name = self.style_name\n if self._style is None:\n self._style = ansi_style_by_name(style_name)\n self._detyped = None\n return self._style\n\n @classmethod\n def fromstring(cls, s):\n \"\"\"Creates a new instance of the LsColors class from a colon-separated\n string of dircolor-valid keys to ANSI color escape sequences.\n \"\"\"\n obj = cls()\n # string inputs always use default codes, so translating into\n # xonsh names should be done from defaults\n reversed_default = ansi_reverse_style(style=\"default\")\n data = {}\n for item in s.split(\":\"):\n key, eq, esc = item.partition(\"=\")\n if not eq:\n # not a valid item\n continue\n data[key] = ansi_color_escape_code_to_name(\n esc, \"default\", reversed_style=reversed_default\n )\n obj._d = data\n return obj\n\n @classmethod\n def fromdircolors(cls, filename=None):\n \"\"\"Constructs an LsColors instance by running dircolors.\n If a filename is provided, it is passed down to the dircolors command.\n \"\"\"\n # assemble command\n cmd = [\"dircolors\", \"-b\"]\n if filename is not None:\n cmd.append(filename)\n # get env\n if hasattr(builtins, \"__xonsh__\") and hasattr(builtins.__xonsh__, \"env\"):\n denv = builtins.__xonsh__.env.detype()\n else:\n denv = None\n # run dircolors\n try:\n out = subprocess.check_output(\n cmd, env=denv, universal_newlines=True, stderr=subprocess.DEVNULL\n )\n except (subprocess.CalledProcessError, FileNotFoundError):\n return cls(cls.default_settings)\n s = out.splitlines()[0]\n _, _, s = s.partition(\"'\")\n s, _, _ = s.rpartition(\"'\")\n return cls.fromstring(s)\n\n @classmethod\n def convert(cls, x):\n \"\"\"Converts an object to LsColors, if needed.\"\"\"\n if isinstance(x, cls):\n return x\n elif isinstance(x, str):\n return cls.fromstring(x)\n elif isinstance(x, bytes):\n return cls.fromstring(x.decode())\n else:\n return cls(x)\n\n\ndef is_lscolors(x):\n \"\"\"Checks if an object is an instance of LsColors\"\"\"\n return isinstance(x, LsColors)\n\n\n@events.on_pre_spec_run_ls\ndef ensure_ls_colors_in_env(spec=None, **kwargs):\n \"\"\"This ensures that the $LS_COLORS environment variable is in the\n environment. This fires exactly once upon the first time the\n ls command is called.\n \"\"\"\n env = builtins.__xonsh__.env\n if \"LS_COLORS\" not in env._d:\n # this adds it to the env too\n default_lscolors(env)\n events.on_pre_spec_run_ls.discard(ensure_ls_colors_in_env)\n\n\n#\n# Ensurerers\n#\n\nEnsurer = collections.namedtuple(\"Ensurer\", [\"validate\", \"convert\", \"detype\"])\nEnsurer.__doc__ = \"\"\"Named tuples whose elements are functions that\nrepresent environment variable validation, conversion, detyping.\n\"\"\"\n\n\n@lazyobject\ndef DEFAULT_ENSURERS():\n return {\n \"AUTO_CD\": (is_bool, to_bool, bool_to_str),\n \"AUTO_PUSHD\": (is_bool, to_bool, bool_to_str),\n \"AUTO_SUGGEST\": (is_bool, to_bool, bool_to_str),\n \"AUTO_SUGGEST_IN_COMPLETIONS\": (is_bool, to_bool, bool_to_str),\n \"BASH_COMPLETIONS\": (is_env_path, str_to_env_path, env_path_to_str),\n \"CASE_SENSITIVE_COMPLETIONS\": (is_bool, to_bool, bool_to_str),\n re.compile(r\"\\w*DIRS$\"): (is_env_path, str_to_env_path, env_path_to_str),\n \"COLOR_INPUT\": (is_bool, to_bool, bool_to_str),\n \"COLOR_RESULTS\": (is_bool, to_bool, bool_to_str),\n \"COMPLETIONS_BRACKETS\": (is_bool, to_bool, bool_to_str),\n \"COMPLETIONS_CONFIRM\": (is_bool, to_bool, bool_to_str),\n \"COMPLETIONS_DISPLAY\": (\n is_completions_display_value,\n to_completions_display_value,\n str,\n ),\n \"COMPLETIONS_MENU_ROWS\": (is_int, int, str),\n \"COMPLETION_QUERY_LIMIT\": (is_int, int, str),\n \"DIRSTACK_SIZE\": (is_int, int, str),\n \"DOTGLOB\": (is_bool, to_bool, bool_to_str),\n \"DYNAMIC_CWD_WIDTH\": (\n is_dynamic_cwd_width,\n to_dynamic_cwd_tuple,\n dynamic_cwd_tuple_to_str,\n ),\n \"DYNAMIC_CWD_ELISION_CHAR\": (is_string, ensure_string, ensure_string),\n \"EXPAND_ENV_VARS\": (is_bool, to_bool, bool_to_str),\n \"FORCE_POSIX_PATHS\": (is_bool, to_bool, bool_to_str),\n \"FOREIGN_ALIASES_SUPPRESS_SKIP_MESSAGE\": (is_bool, to_bool, bool_to_str),\n \"FOREIGN_ALIASES_OVERRIDE\": (is_bool, to_bool, bool_to_str),\n \"FUZZY_PATH_COMPLETION\": (is_bool, to_bool, bool_to_str),\n \"GLOB_SORTED\": (is_bool, to_bool, bool_to_str),\n \"HISTCONTROL\": (is_string_set, csv_to_set, set_to_csv),\n \"IGNOREEOF\": (is_bool, to_bool, bool_to_str),\n \"INTENSIFY_COLORS_ON_WIN\": (\n always_false,\n intensify_colors_on_win_setter,\n bool_to_str,\n ),\n \"LANG\": (is_string, ensure_string, ensure_string),\n \"LC_COLLATE\": (always_false, locale_convert(\"LC_COLLATE\"), ensure_string),\n \"LC_CTYPE\": (always_false, locale_convert(\"LC_CTYPE\"), ensure_string),\n \"LC_MESSAGES\": (always_false, locale_convert(\"LC_MESSAGES\"), ensure_string),\n \"LC_MONETARY\": (always_false, locale_convert(\"LC_MONETARY\"), ensure_string),\n \"LC_NUMERIC\": (always_false, locale_convert(\"LC_NUMERIC\"), ensure_string),\n \"LC_TIME\": (always_false, locale_convert(\"LC_TIME\"), ensure_string),\n \"LS_COLORS\": (is_lscolors, LsColors.convert, detype),\n \"LOADED_RC_FILES\": (is_bool_seq, csv_to_bool_seq, bool_seq_to_csv),\n \"MOUSE_SUPPORT\": (is_bool, to_bool, bool_to_str),\n \"MULTILINE_PROMPT\": (is_string_or_callable, ensure_string, ensure_string),\n re.compile(r\"\\w*PATH$\"): (is_env_path, str_to_env_path, env_path_to_str),\n \"PATHEXT\": (\n is_nonstring_seq_of_strings,\n pathsep_to_upper_seq,\n seq_to_upper_pathsep,\n ),\n \"PRETTY_PRINT_RESULTS\": (is_bool, to_bool, bool_to_str),\n \"PROMPT\": (is_string_or_callable, ensure_string, ensure_string),\n \"PROMPT_FIELDS\": (always_true, None, None),\n \"PROMPT_TOOLKIT_COLOR_DEPTH\": (\n always_false,\n ptk2_color_depth_setter,\n ensure_string,\n ),\n \"PUSHD_MINUS\": (is_bool, to_bool, bool_to_str),\n \"PUSHD_SILENT\": (is_bool, to_bool, bool_to_str),\n \"PTK_STYLE_OVERRIDES\": (is_str_str_dict, to_str_str_dict, dict_to_str),\n \"RAISE_SUBPROC_ERROR\": (is_bool, to_bool, bool_to_str),\n \"RIGHT_PROMPT\": (is_string_or_callable, ensure_string, ensure_string),\n \"BOTTOM_TOOLBAR\": (is_string_or_callable, ensure_string, ensure_string),\n \"SUBSEQUENCE_PATH_COMPLETION\": (is_bool, to_bool, bool_to_str),\n \"SUGGEST_COMMANDS\": (is_bool, to_bool, bool_to_str),\n \"SUGGEST_MAX_NUM\": (is_int, int, str),\n \"SUGGEST_THRESHOLD\": (is_int, int, str),\n \"SUPPRESS_BRANCH_TIMEOUT_MESSAGE\": (is_bool, to_bool, bool_to_str),\n \"UPDATE_COMPLETIONS_ON_KEYPRESS\": (is_bool, to_bool, bool_to_str),\n \"UPDATE_OS_ENVIRON\": (is_bool, to_bool, bool_to_str),\n \"UPDATE_PROMPT_ON_KEYPRESS\": (is_bool, to_bool, bool_to_str),\n \"VC_BRANCH_TIMEOUT\": (is_float, float, str),\n \"VC_HG_SHOW_BRANCH\": (is_bool, to_bool, bool_to_str),\n \"VI_MODE\": (is_bool, to_bool, bool_to_str),\n \"VIRTUAL_ENV\": (is_string, ensure_string, ensure_string),\n \"WIN_UNICODE_CONSOLE\": (always_false, setup_win_unicode_console, bool_to_str),\n \"XONSHRC\": (is_env_path, str_to_env_path, env_path_to_str),\n \"XONSH_APPEND_NEWLINE\": (is_bool, to_bool, bool_to_str),\n \"XONSH_AUTOPAIR\": (is_bool, to_bool, bool_to_str),\n \"XONSH_CACHE_SCRIPTS\": (is_bool, to_bool, bool_to_str),\n \"XONSH_CACHE_EVERYTHING\": (is_bool, to_bool, bool_to_str),\n \"XONSH_COLOR_STYLE\": (is_string, ensure_string, ensure_string),\n \"XONSH_DEBUG\": (always_false, to_debug, bool_or_int_to_str),\n \"XONSH_ENCODING\": (is_string, ensure_string, ensure_string),\n \"XONSH_ENCODING_ERRORS\": (is_string, ensure_string, ensure_string),\n \"XONSH_HISTORY_BACKEND\": (is_history_backend, to_itself, ensure_string),\n \"XONSH_HISTORY_FILE\": (is_string, ensure_string, ensure_string),\n \"XONSH_HISTORY_MATCH_ANYWHERE\": (is_bool, to_bool, bool_to_str),\n \"XONSH_HISTORY_SIZE\": (\n is_history_tuple,\n to_history_tuple,\n history_tuple_to_str,\n ),\n \"XONSH_LOGIN\": (is_bool, to_bool, bool_to_str),\n \"XONSH_PROC_FREQUENCY\": (is_float, float, str),\n \"XONSH_SHOW_TRACEBACK\": (is_bool, to_bool, bool_to_str),\n \"XONSH_STDERR_PREFIX\": (is_string, ensure_string, ensure_string),\n \"XONSH_STDERR_POSTFIX\": (is_string, ensure_string, ensure_string),\n \"XONSH_STORE_STDOUT\": (is_bool, to_bool, bool_to_str),\n \"XONSH_STORE_STDIN\": (is_bool, to_bool, bool_to_str),\n \"XONSH_TRACEBACK_LOGFILE\": (is_logfile_opt, to_logfile_opt, logfile_opt_to_str),\n \"XONSH_DATETIME_FORMAT\": (is_string, ensure_string, ensure_string),\n }\n\n\n#\n# Defaults\n#\ndef default_value(f):\n \"\"\"Decorator for making callable default values.\"\"\"\n f._xonsh_callable_default = True\n return f\n\n\ndef is_callable_default(x):\n \"\"\"Checks if a value is a callable default.\"\"\"\n return callable(x) and getattr(x, \"_xonsh_callable_default\", False)\n\n\nDEFAULT_TITLE = \"{current_job:{} | }{user}@{hostname}: {cwd} | xonsh\"\n\n\n@default_value\ndef xonsh_data_dir(env):\n \"\"\"Ensures and returns the $XONSH_DATA_DIR\"\"\"\n xdd = os.path.expanduser(os.path.join(env.get(\"XDG_DATA_HOME\"), \"xonsh\"))\n os.makedirs(xdd, exist_ok=True)\n return xdd\n\n\n@default_value\ndef xonsh_config_dir(env):\n \"\"\"Ensures and returns the $XONSH_CONFIG_DIR\"\"\"\n xcd = os.path.expanduser(os.path.join(env.get(\"XDG_CONFIG_HOME\"), \"xonsh\"))\n os.makedirs(xcd, exist_ok=True)\n return xcd\n\n\ndef xonshconfig(env):\n \"\"\"Ensures and returns the $XONSHCONFIG\"\"\"\n xcd = env.get(\"XONSH_CONFIG_DIR\")\n xc = os.path.join(xcd, \"config.json\")\n return xc\n\n\n@default_value\ndef default_xonshrc(env):\n \"\"\"Creates a new instance of the default xonshrc tuple.\"\"\"\n xcdrc = os.path.join(xonsh_config_dir(env), \"rc.xsh\")\n if ON_WINDOWS:\n dxrc = (\n os.path.join(os_environ[\"ALLUSERSPROFILE\"], \"xonsh\", \"xonshrc\"),\n xcdrc,\n os.path.expanduser(\"~/.xonshrc\"),\n )\n else:\n dxrc = (\"/etc/xonshrc\", xcdrc, os.path.expanduser(\"~/.xonshrc\"))\n # Check if old config file exists and issue warning\n old_config_filename = xonshconfig(env)\n if os.path.isfile(old_config_filename):\n print(\n \"WARNING! old style configuration (\"\n + old_config_filename\n + \") is no longer supported. \"\n + \"Please migrate to xonshrc.\"\n )\n return dxrc\n\n\n@default_value\ndef xonsh_append_newline(env):\n \"\"\"Appends a newline if we are in interactive mode\"\"\"\n return env.get(\"XONSH_INTERACTIVE\", False)\n\n\n@default_value\ndef default_lscolors(env):\n \"\"\"Gets a default instanse of LsColors\"\"\"\n inherited_lscolors = os_environ.get(\"LS_COLORS\", None)\n if inherited_lscolors is None:\n lsc = LsColors.fromdircolors()\n else:\n lsc = LsColors.fromstring(inherited_lscolors)\n # have to place this in the env, so it is applied\n env[\"LS_COLORS\"] = lsc\n return lsc\n\n\n# Default values should generally be immutable, that way if a user wants\n# to set them they have to do a copy and write them to the environment.\n# try to keep this sorted.\n@lazyobject\ndef DEFAULT_VALUES():\n dv = {\n \"AUTO_CD\": False,\n \"AUTO_PUSHD\": False,\n \"AUTO_SUGGEST\": True,\n \"AUTO_SUGGEST_IN_COMPLETIONS\": False,\n \"BASH_COMPLETIONS\": BASH_COMPLETIONS_DEFAULT,\n \"CASE_SENSITIVE_COMPLETIONS\": ON_LINUX,\n \"CDPATH\": (),\n \"COLOR_INPUT\": True,\n \"COLOR_RESULTS\": False,\n \"COMPLETIONS_BRACKETS\": True,\n \"COMPLETIONS_CONFIRM\": False,\n \"COMPLETIONS_DISPLAY\": \"single\",\n \"COMPLETIONS_MENU_ROWS\": 5,\n \"COMPLETION_QUERY_LIMIT\": 100,\n \"DIRSTACK_SIZE\": 20,\n \"DOTGLOB\": False,\n \"DYNAMIC_CWD_WIDTH\": (float(\"inf\"), \"c\"),\n \"DYNAMIC_CWD_ELISION_CHAR\": \"\",\n \"EXPAND_ENV_VARS\": True,\n \"FORCE_POSIX_PATHS\": False,\n \"FOREIGN_ALIASES_SUPPRESS_SKIP_MESSAGE\": False,\n \"FOREIGN_ALIASES_OVERRIDE\": False,\n \"PROMPT_FIELDS\": dict(prompt.PROMPT_FIELDS),\n \"FUZZY_PATH_COMPLETION\": True,\n \"GLOB_SORTED\": True,\n \"HISTCONTROL\": set(),\n \"IGNOREEOF\": False,\n \"INDENT\": \" \",\n \"INTENSIFY_COLORS_ON_WIN\": True,\n \"LANG\": \"C.UTF-8\",\n \"LC_CTYPE\": locale.setlocale(locale.LC_CTYPE),\n \"LC_COLLATE\": locale.setlocale(locale.LC_COLLATE),\n \"LC_TIME\": locale.setlocale(locale.LC_TIME),\n \"LC_MONETARY\": locale.setlocale(locale.LC_MONETARY),\n \"LC_NUMERIC\": locale.setlocale(locale.LC_NUMERIC),\n \"LS_COLORS\": default_lscolors,\n \"LOADED_RC_FILES\": (),\n \"MOUSE_SUPPORT\": False,\n \"MULTILINE_PROMPT\": \".\",\n \"PATH\": PATH_DEFAULT,\n \"PATHEXT\": [\".COM\", \".EXE\", \".BAT\", \".CMD\"] if ON_WINDOWS else [],\n \"PRETTY_PRINT_RESULTS\": True,\n \"PROMPT\": prompt.default_prompt(),\n \"PROMPT_TOOLKIT_COLOR_DEPTH\": \"\",\n \"PTK_STYLE_OVERRIDES\": dict(PTK2_STYLE),\n \"PUSHD_MINUS\": False,\n \"PUSHD_SILENT\": False,\n \"RAISE_SUBPROC_ERROR\": False,\n \"RIGHT_PROMPT\": \"\",\n \"BOTTOM_TOOLBAR\": \"\",\n \"SHELL_TYPE\": \"best\",\n \"SUBSEQUENCE_PATH_COMPLETION\": True,\n \"SUPPRESS_BRANCH_TIMEOUT_MESSAGE\": False,\n \"SUGGEST_COMMANDS\": True,\n \"SUGGEST_MAX_NUM\": 5,\n \"SUGGEST_THRESHOLD\": 3,\n \"TITLE\": DEFAULT_TITLE,\n \"UPDATE_COMPLETIONS_ON_KEYPRESS\": True,\n \"UPDATE_OS_ENVIRON\": False,\n \"UPDATE_PROMPT_ON_KEYPRESS\": False,\n \"VC_BRANCH_TIMEOUT\": 0.2 if ON_WINDOWS else 0.1,\n \"VC_HG_SHOW_BRANCH\": True,\n \"VI_MODE\": False,\n \"WIN_UNICODE_CONSOLE\": True,\n \"XDG_CONFIG_HOME\": os.path.expanduser(os.path.join(\"~\", \".config\")),\n \"XDG_DATA_HOME\": os.path.expanduser(os.path.join(\"~\", \".local\", \"share\")),\n \"XONSHRC\": default_xonshrc,\n \"XONSH_APPEND_NEWLINE\": xonsh_append_newline,\n \"XONSH_AUTOPAIR\": False,\n \"XONSH_CACHE_SCRIPTS\": True,\n \"XONSH_CACHE_EVERYTHING\": False,\n \"XONSH_COLOR_STYLE\": \"default\",\n \"XONSH_CONFIG_DIR\": xonsh_config_dir,\n \"XONSH_DATA_DIR\": xonsh_data_dir,\n \"XONSH_DEBUG\": 0,\n \"XONSH_ENCODING\": DEFAULT_ENCODING,\n \"XONSH_ENCODING_ERRORS\": \"surrogateescape\",\n \"XONSH_HISTORY_BACKEND\": \"json\",\n \"XONSH_HISTORY_FILE\": os.path.expanduser(\"~/.xonsh_history.json\"),\n \"XONSH_HISTORY_MATCH_ANYWHERE\": False,\n \"XONSH_HISTORY_SIZE\": (8128, \"commands\"),\n \"XONSH_LOGIN\": False,\n \"XONSH_PROC_FREQUENCY\": 1e-4,\n \"XONSH_SHOW_TRACEBACK\": False,\n \"XONSH_STDERR_PREFIX\": \"\",\n \"XONSH_STDERR_POSTFIX\": \"\",\n \"XONSH_STORE_STDIN\": False,\n \"XONSH_STORE_STDOUT\": False,\n \"XONSH_TRACEBACK_LOGFILE\": None,\n \"XONSH_DATETIME_FORMAT\": \"%Y-%m-%d %H:%M\",\n }\n if hasattr(locale, \"LC_MESSAGES\"):\n dv[\"LC_MESSAGES\"] = locale.setlocale(locale.LC_MESSAGES)\n return dv\n\n\nVarDocs = collections.namedtuple(\n \"VarDocs\", [\"docstr\", \"configurable\", \"default\", \"store_as_str\"]\n)\nVarDocs.__doc__ = \"\"\"Named tuple for environment variable documentation\n\nParameters\n----------\ndocstr : str\n The environment variable docstring.\nconfigurable : bool, optional\n Flag for whether the environment variable is configurable or not.\ndefault : str, optional\n Custom docstring for the default value for complex defaults.\n Is this is DefaultNotGiven, then the default will be looked up\n from DEFAULT_VALUES and converted to a str.\nstore_as_str : bool, optional\n Flag for whether the environment variable should be stored as a\n string. This is used when persisting a variable that is not JSON\n serializable to the config file. For example, sets, frozensets, and\n potentially other non-trivial data types. default, False.\n\"\"\"\n# iterates from back\nVarDocs.__new__.__defaults__ = (True, DefaultNotGiven, False)\n\n\n# Please keep the following in alphabetic order - scopatz\n@lazyobject\ndef DEFAULT_DOCS():\n return {\n \"ANSICON\": VarDocs(\n \"This is used on Windows to set the title, \" \"if available.\",\n configurable=False,\n ),\n \"AUTO_CD\": VarDocs(\n \"Flag to enable changing to a directory by entering the dirname or \"\n \"full path only (without the cd command).\"\n ),\n \"AUTO_PUSHD\": VarDocs(\n \"Flag for automatically pushing directories onto the directory stack.\"\n ),\n \"AUTO_SUGGEST\": VarDocs(\n \"Enable automatic command suggestions based on history, like in the fish \"\n \"shell.\\n\\nPressing the right arrow key inserts the currently \"\n \"displayed suggestion. Only usable with ``$SHELL_TYPE=prompt_toolkit.``\"\n ),\n \"AUTO_SUGGEST_IN_COMPLETIONS\": VarDocs(\n \"Places the auto-suggest result as the first option in the completions. \"\n \"This enables you to tab complete the auto-suggestion.\"\n ),\n \"BASH_COMPLETIONS\": VarDocs(\n \"This is a list (or tuple) of strings that specifies where the \"\n \"``bash_completion`` script may be found. \"\n \"The first valid path will be used. For better performance, \"\n \"bash-completion v2.x is recommended since it lazy-loads individual \"\n \"completion scripts. \"\n \"For both bash-completion v1.x and v2.x, paths of individual completion \"\n \"scripts (like ``.../completes/ssh``) do not need to be included here. \"\n \"The default values are platform \"\n \"dependent, but sane. To specify an alternate list, do so in the run \"\n \"control file.\",\n default=(\n \"Normally this is:\\n\\n\"\n \" ``('/usr/share/bash-completion/bash_completion', )``\\n\\n\"\n \"But, on Mac it is:\\n\\n\"\n \" ``('/usr/local/share/bash-completion/bash_completion', \"\n \"'/usr/local/etc/bash_completion')``\\n\\n\"\n \"Other OS-specific defaults may be added in the future.\"\n ),\n ),\n \"CASE_SENSITIVE_COMPLETIONS\": VarDocs(\n \"Sets whether completions should be case sensitive or case \" \"insensitive.\",\n default=\"True on Linux, False otherwise.\",\n ),\n \"CDPATH\": VarDocs(\n \"A list of paths to be used as roots for a cd, breaking compatibility \"\n \"with Bash, xonsh always prefer an existing relative path.\"\n ),\n \"COLOR_INPUT\": VarDocs(\"Flag for syntax highlighting interactive input.\"),\n \"COLOR_RESULTS\": VarDocs(\"Flag for syntax highlighting return values.\"),\n \"COMPLETIONS_BRACKETS\": VarDocs(\n \"Flag to enable/disable inclusion of square brackets and parentheses \"\n \"in Python attribute completions.\",\n default=\"True\",\n ),\n \"COMPLETIONS_DISPLAY\": VarDocs(\n \"Configure if and how Python completions are displayed by the \"\n \"``prompt_toolkit`` shell.\\n\\nThis option does not affect Bash \"\n \"completions, auto-suggestions, etc.\\n\\nChanging it at runtime will \"\n \"take immediate effect, so you can quickly disable and enable \"\n \"completions during shell sessions.\\n\\n\"\n \"- If ``$COMPLETIONS_DISPLAY`` is ``none`` or ``false``, do not display\\n\"\n \" those completions.\\n\"\n \"- If ``$COMPLETIONS_DISPLAY`` is ``single``, display completions in a\\n\"\n \" single column while typing.\\n\"\n \"- If ``$COMPLETIONS_DISPLAY`` is ``multi`` or ``true``, display completions\\n\"\n \" in multiple columns while typing.\\n\\n\"\n \"- If ``$COMPLETIONS_DISPLAY`` is ``readline``, display completions\\n\"\n \" will emulate the behavior of readline.\\n\\n\"\n \"These option values are not case- or type-sensitive, so e.g.\"\n \"writing ``$COMPLETIONS_DISPLAY = None`` \"\n \"and ``$COMPLETIONS_DISPLAY = 'none'`` are equivalent. Only usable with \"\n \"``$SHELL_TYPE=prompt_toolkit``\"\n ),\n \"COMPLETIONS_CONFIRM\": VarDocs(\n \"While tab-completions menu is displayed, press to confirm \"\n \"completion instead of running command. This only affects the \"\n \"prompt-toolkit shell.\"\n ),\n \"COMPLETIONS_MENU_ROWS\": VarDocs(\n \"Number of rows to reserve for tab-completions menu if \"\n \"``$COMPLETIONS_DISPLAY`` is ``single`` or ``multi``. This only affects the \"\n \"prompt-toolkit shell.\"\n ),\n \"COMPLETION_QUERY_LIMIT\": VarDocs(\n \"The number of completions to display before the user is asked \"\n \"for confirmation.\"\n ),\n \"DIRSTACK_SIZE\": VarDocs(\"Maximum size of the directory stack.\"),\n \"DOTGLOB\": VarDocs(\n 'Globbing files with \"*\" or \"**\" will also match '\n \"dotfiles, or those 'hidden' files whose names \"\n \"begin with a literal '.'. Such files are filtered \"\n \"out by default.\"\n ),\n \"DYNAMIC_CWD_WIDTH\": VarDocs(\n \"Maximum length in number of characters \"\n \"or as a percentage for the ``cwd`` prompt variable. For example, \"\n '\"20\" is a twenty character width and \"10%\" is ten percent of the '\n \"number of columns available.\"\n ),\n \"DYNAMIC_CWD_ELISION_CHAR\": VarDocs(\n \"The string used to show a shortened directory in a shortened cwd, \"\n \"e.g. ``'…'``.\"\n ),\n \"EXPAND_ENV_VARS\": VarDocs(\n \"Toggles whether environment variables are expanded inside of strings \"\n \"in subprocess mode.\"\n ),\n \"FORCE_POSIX_PATHS\": VarDocs(\n \"Forces forward slashes (``/``) on Windows systems when using auto \"\n \"completion if set to anything truthy.\",\n configurable=ON_WINDOWS,\n ),\n \"FOREIGN_ALIASES_SUPPRESS_SKIP_MESSAGE\": VarDocs(\n \"Whether or not foreign aliases should suppress the message \"\n \"that informs the user when a foreign alias has been skipped \"\n \"because it already exists in xonsh.\",\n configurable=True,\n ),\n \"FOREIGN_ALIASES_OVERRIDE\": VarDocs(\n \"Whether or not foreign aliases should override xonsh aliases \"\n \"with the same name. Note that setting of this must happen in the \"\n \"environment that xonsh was started from. \"\n \"It cannot be set in the ``.xonshrc`` as loading of foreign aliases happens before\"\n \"``.xonshrc`` is parsed\",\n configurable=True,\n ),\n \"PROMPT_FIELDS\": VarDocs(\n \"Dictionary containing variables to be used when formatting $PROMPT \"\n \"and $TITLE. See 'Customizing the Prompt' \"\n \"http://xon.sh/tutorial.html#customizing-the-prompt\",\n configurable=False,\n default=\"``xonsh.prompt.PROMPT_FIELDS``\",\n ),\n \"FUZZY_PATH_COMPLETION\": VarDocs(\n \"Toggles 'fuzzy' matching of paths for tab completion, which is only \"\n \"used as a fallback if no other completions succeed but can be used \"\n \"as a way to adjust for typographical errors. If ``True``, then, e.g.,\"\n \" ``xonhs`` will match ``xonsh``.\"\n ),\n \"GLOB_SORTED\": VarDocs(\n \"Toggles whether globbing results are manually sorted. If ``False``, \"\n \"the results are returned in arbitrary order.\"\n ),\n \"HISTCONTROL\": VarDocs(\n \"A set of strings (comma-separated list in string form) of options \"\n \"that determine what commands are saved to the history list. By \"\n \"default all commands are saved. The option ``ignoredups`` will not \"\n \"save the command if it matches the previous command. The option \"\n \"'ignoreerr' will cause any commands that fail (i.e. return non-zero \"\n \"exit status) to not be added to the history list.\",\n store_as_str=True,\n ),\n \"IGNOREEOF\": VarDocs(\"Prevents Ctrl-D from exiting the shell.\"),\n \"INDENT\": VarDocs(\"Indentation string for multiline input\"),\n \"INTENSIFY_COLORS_ON_WIN\": VarDocs(\n \"Enhance style colors for readability \"\n \"when using the default terminal (``cmd.exe``) on Windows. Blue colors, \"\n \"which are hard to read, are replaced with cyan. Other colors are \"\n \"generally replaced by their bright counter parts.\",\n configurable=ON_WINDOWS,\n ),\n \"LANG\": VarDocs(\"Fallback locale setting for systems where it matters\"),\n \"LS_COLORS\": VarDocs(\"Color settings for ``ls`` command line utility\"),\n \"LOADED_RC_FILES\": VarDocs(\n \"Whether or not any of the xonsh run control files were loaded at \"\n \"startup. This is a sequence of bools in Python that is converted \"\n \"to a CSV list in string form, ie ``[True, False]`` becomes \"\n \"``'True,False'``.\",\n configurable=False,\n ),\n \"MOUSE_SUPPORT\": VarDocs(\n \"Enable mouse support in the ``prompt_toolkit`` shell. This allows \"\n \"clicking for positioning the cursor or selecting a completion. In \"\n \"some terminals however, this disables the ability to scroll back \"\n \"through the history of the terminal. Only usable with \"\n \"``$SHELL_TYPE=prompt_toolkit``\"\n ),\n \"MULTILINE_PROMPT\": VarDocs(\n \"Prompt text for 2nd+ lines of input, may be str or function which \"\n \"returns a str.\"\n ),\n \"OLDPWD\": VarDocs(\n \"Used to represent a previous present working directory.\",\n configurable=False,\n ),\n \"PATH\": VarDocs(\"List of strings representing where to look for executables.\"),\n \"PATHEXT\": VarDocs(\n \"Sequence of extension strings (eg, ``.EXE``) for \"\n \"filtering valid executables by. Each element must be \"\n \"uppercase.\"\n ),\n \"PRETTY_PRINT_RESULTS\": VarDocs('Flag for \"pretty printing\" return values.'),\n \"PROMPT\": VarDocs(\n \"The prompt text. May contain keyword arguments which are \"\n \"auto-formatted, see 'Customizing the Prompt' at \"\n \"http://xon.sh/tutorial.html#customizing-the-prompt. \"\n \"This value is never inherited from parent processes.\",\n default=\"``xonsh.environ.DEFAULT_PROMPT``\",\n ),\n \"PROMPT_TOOLKIT_COLOR_DEPTH\": VarDocs(\n \"The color depth used by prompt toolkit 2. Possible values are: \"\n \"``DEPTH_1_BIT``, ``DEPTH_4_BIT``, ``DEPTH_8_BIT``, ``DEPTH_24_BIT`` \"\n \"colors. Default is an empty string which means that prompt toolkit decide.\"\n ),\n \"PTK_STYLE_OVERRIDES\": VarDocs(\n \"A dictionary containing custom prompt_toolkit style definitions.\"\n ),\n \"PUSHD_MINUS\": VarDocs(\n \"Flag for directory pushing functionality. False is the normal \" \"behavior.\"\n ),\n \"PUSHD_SILENT\": VarDocs(\n \"Whether or not to suppress directory stack manipulation output.\"\n ),\n \"RAISE_SUBPROC_ERROR\": VarDocs(\n \"Whether or not to raise an error if a subprocess (captured or \"\n \"uncaptured) returns a non-zero exit status, which indicates failure. \"\n \"This is most useful in xonsh scripts or modules where failures \"\n \"should cause an end to execution. This is less useful at a terminal. \"\n \"The error that is raised is a ``subprocess.CalledProcessError``.\"\n ),\n \"RIGHT_PROMPT\": VarDocs(\n \"Template string for right-aligned text \"\n \"at the prompt. This may be parametrized in the same way as \"\n \"the ``$PROMPT`` variable. Currently, this is only available in the \"\n \"prompt-toolkit shell.\"\n ),\n \"BOTTOM_TOOLBAR\": VarDocs(\n \"Template string for the bottom toolbar. \"\n \"This may be parametrized in the same way as \"\n \"the ``$PROMPT`` variable. Currently, this is only available in the \"\n \"prompt-toolkit shell.\"\n ),\n \"SHELL_TYPE\": VarDocs(\n \"Which shell is used. Currently two base shell types are supported:\\n\\n\"\n \" - ``readline`` that is backed by Python's readline module\\n\"\n \" - ``prompt_toolkit`` that uses external library of the same name\\n\"\n \" - ``random`` selects a random shell from the above on startup\\n\"\n \" - ``best`` selects the most feature-rich shell available on the\\n\"\n \" user's system\\n\\n\"\n \"To use the ``prompt_toolkit`` shell you need to have the \"\n \"`prompt_toolkit `_\"\n \" library installed. To specify which shell should be used, do so in \"\n \"the run control file.\",\n default=\"``best``\",\n ),\n \"SUBSEQUENCE_PATH_COMPLETION\": VarDocs(\n \"Toggles subsequence matching of paths for tab completion. \"\n \"If ``True``, then, e.g., ``~/u/ro`` can match ``~/lou/carcolh``.\"\n ),\n \"SUGGEST_COMMANDS\": VarDocs(\n \"When a user types an invalid command, xonsh will try to offer \"\n \"suggestions of similar valid commands if this is True.\"\n ),\n \"SUGGEST_MAX_NUM\": VarDocs(\n \"xonsh will show at most this many suggestions in response to an \"\n \"invalid command. If negative, there is no limit to how many \"\n \"suggestions are shown.\"\n ),\n \"SUGGEST_THRESHOLD\": VarDocs(\n \"An error threshold. If the Levenshtein distance between the entered \"\n \"command and a valid command is less than this value, the valid \"\n 'command will be offered as a suggestion. Also used for \"fuzzy\" '\n \"tab completion of paths.\"\n ),\n \"SUPPRESS_BRANCH_TIMEOUT_MESSAGE\": VarDocs(\n \"Whether or not to suppress branch timeout warning messages.\"\n ),\n \"TERM\": VarDocs(\n \"TERM is sometimes set by the terminal emulator. This is used (when \"\n \"valid) to determine whether or not to set the title. Users shouldn't \"\n \"need to set this themselves. Note that this variable should be set as \"\n \"early as possible in order to ensure it is effective. Here are a few \"\n \"options:\\n\\n\"\n \"* Set this from the program that launches xonsh. On POSIX systems, \\n\"\n \" this can be performed by using env, e.g. \\n\"\n \" ``/usr/bin/env TERM=xterm-color xonsh`` or similar.\\n\"\n \"* From the xonsh command line, namely ``xonsh -DTERM=xterm-color``.\\n\"\n '* In the config file with ``{\"env\": {\"TERM\": \"xterm-color\"}}``.\\n'\n \"* Lastly, in xonshrc with ``$TERM``\\n\\n\"\n \"Ideally, your terminal emulator will set this correctly but that does \"\n \"not always happen.\",\n configurable=False,\n ),\n \"TITLE\": VarDocs(\n \"The title text for the window in which xonsh is running. Formatted \"\n \"in the same manner as ``$PROMPT``, see 'Customizing the Prompt' \"\n \"http://xon.sh/tutorial.html#customizing-the-prompt.\",\n default=\"``xonsh.environ.DEFAULT_TITLE``\",\n ),\n \"UPDATE_COMPLETIONS_ON_KEYPRESS\": VarDocs(\n \"Completions display is evaluated and presented whenever a key is \"\n \"pressed. This avoids the need to press TAB, except to cycle through \"\n \"the possibilities. This currently only affects the prompt-toolkit shell.\"\n ),\n \"UPDATE_OS_ENVIRON\": VarDocs(\n \"If True ``os_environ`` will always be updated \"\n \"when the xonsh environment changes. The environment can be reset to \"\n \"the default value by calling ``__xonsh__.env.undo_replace_env()``\"\n ),\n \"UPDATE_PROMPT_ON_KEYPRESS\": VarDocs(\n \"Disables caching the prompt between commands, \"\n \"so that it would be reevaluated on each keypress. \"\n \"Disabled by default because of the incurred performance penalty.\"\n ),\n \"VC_BRANCH_TIMEOUT\": VarDocs(\n \"The timeout (in seconds) for version control \"\n \"branch computations. This is a timeout per subprocess call, so the \"\n \"total time to compute will be larger than this in many cases.\"\n ),\n \"VC_HG_SHOW_BRANCH\": VarDocs(\n \"Whether or not to show the Mercurial branch in the prompt.\"\n ),\n \"VI_MODE\": VarDocs(\n \"Flag to enable ``vi_mode`` in the ``prompt_toolkit`` shell.\"\n ),\n \"VIRTUAL_ENV\": VarDocs(\n \"Path to the currently active Python environment.\", configurable=False\n ),\n \"WIN_UNICODE_CONSOLE\": VarDocs(\n \"Enables unicode support in windows terminals. Requires the external \"\n \"library ``win_unicode_console``.\",\n configurable=ON_WINDOWS,\n ),\n \"XDG_CONFIG_HOME\": VarDocs(\n \"Open desktop standard configuration home dir. This is the same \"\n \"default as used in the standard.\",\n configurable=False,\n default=\"``~/.config``\",\n ),\n \"XDG_DATA_HOME\": VarDocs(\n \"Open desktop standard data home dir. This is the same default as \"\n \"used in the standard.\",\n default=\"``~/.local/share``\",\n ),\n \"XONSHRC\": VarDocs(\n \"A list of the locations of run control files, if they exist. User \"\n \"defined run control file will supersede values set in system-wide \"\n \"control file if there is a naming collision.\",\n default=(\n \"On Linux & Mac OSX: ``['/etc/xonshrc', '~/.config/xonsh/rc.xsh', '~/.xonshrc']``\\n\"\n \"\\nOn Windows: \"\n \"``['%ALLUSERSPROFILE%\\\\\\\\xonsh\\\\\\\\xonshrc', '~/.config/xonsh/rc.xsh', '~/.xonshrc']``\"\n ),\n ),\n \"XONSH_APPEND_NEWLINE\": VarDocs(\n \"Append new line when a partial line is preserved in output.\"\n ),\n \"XONSH_AUTOPAIR\": VarDocs(\n \"Whether Xonsh will auto-insert matching parentheses, brackets, and \"\n \"quotes. Only available under the prompt-toolkit shell.\"\n ),\n \"XONSH_CACHE_SCRIPTS\": VarDocs(\n \"Controls whether the code for scripts run from xonsh will be cached\"\n \" (``True``) or re-compiled each time (``False``).\"\n ),\n \"XONSH_CACHE_EVERYTHING\": VarDocs(\n \"Controls whether all code (including code entered at the interactive\"\n \" prompt) will be cached.\"\n ),\n \"XONSH_COLOR_STYLE\": VarDocs(\n \"Sets the color style for xonsh colors. This is a style name, not \"\n \"a color map. Run ``xonfig styles`` to see the available styles.\"\n ),\n \"XONSH_CONFIG_DIR\": VarDocs(\n \"This is the location where xonsh configuration information is stored.\",\n configurable=False,\n default=\"``$XDG_CONFIG_HOME/xonsh``\",\n ),\n \"XONSH_DEBUG\": VarDocs(\n \"Sets the xonsh debugging level. This may be an integer or a boolean. \"\n \"Setting this variable prior to stating xonsh to ``1`` or ``True`` \"\n \"will suppress amalgamated imports. Setting it to ``2`` will get some \"\n \"basic information like input transformation, command replacement. \"\n \"With ``3`` or a higher number will make more debugging information \"\n \"presented, like PLY parsing messages.\",\n configurable=False,\n ),\n \"XONSH_DATA_DIR\": VarDocs(\n \"This is the location where xonsh data files are stored, such as \"\n \"history.\",\n default=\"``$XDG_DATA_HOME/xonsh``\",\n ),\n \"XONSH_ENCODING\": VarDocs(\n \"This is the encoding that xonsh should use for subprocess operations.\",\n default=\"``sys.getdefaultencoding()``\",\n ),\n \"XONSH_ENCODING_ERRORS\": VarDocs(\n \"The flag for how to handle encoding errors should they happen. \"\n \"Any string flag that has been previously registered with Python \"\n \"is allowed. See the 'Python codecs documentation' \"\n \"(https://docs.python.org/3/library/codecs.html#error-handlers) \"\n \"for more information and available options.\",\n default=\"``surrogateescape``\",\n ),\n \"XONSH_GITSTATUS_*\": VarDocs(\n \"Symbols for gitstatus prompt. Default values are: \\n\\n\"\n \"* ``XONSH_GITSTATUS_HASH``: ``:``\\n\"\n \"* ``XONSH_GITSTATUS_BRANCH``: ``{CYAN}``\\n\"\n \"* ``XONSH_GITSTATUS_OPERATION``: ``{CYAN}``\\n\"\n \"* ``XONSH_GITSTATUS_STAGED``: ``{RED}●``\\n\"\n \"* ``XONSH_GITSTATUS_CONFLICTS``: ``{RED}×``\\n\"\n \"* ``XONSH_GITSTATUS_CHANGED``: ``{BLUE}+``\\n\"\n \"* ``XONSH_GITSTATUS_UNTRACKED``: ``…``\\n\"\n \"* ``XONSH_GITSTATUS_STASHED``: ``⚑``\\n\"\n \"* ``XONSH_GITSTATUS_CLEAN``: ``{BOLD_GREEN}✓``\\n\"\n \"* ``XONSH_GITSTATUS_AHEAD``: ``↑·``\\n\"\n \"* ``XONSH_GITSTATUS_BEHIND``: ``↓·``\\n\"\n ),\n \"XONSH_HISTORY_BACKEND\": VarDocs(\n \"Set which history backend to use. Options are: 'json', \"\n \"'sqlite', and 'dummy'. The default is 'json'. \"\n \"``XONSH_HISTORY_BACKEND`` also accepts a class type that inherits \"\n \"from ``xonsh.history.base.History``, or its instance.\"\n ),\n \"XONSH_HISTORY_FILE\": VarDocs(\n \"Location of history file (deprecated).\",\n configurable=False,\n default=\"``~/.xonsh_history``\",\n ),\n \"XONSH_HISTORY_MATCH_ANYWHERE\": VarDocs(\n \"When searching history from a partial string (by pressing up arrow), \"\n \"match command history anywhere in a given line (not just the start)\",\n default=\"False\",\n ),\n \"XONSH_HISTORY_SIZE\": VarDocs(\n \"Value and units tuple that sets the size of history after garbage \"\n \"collection. Canonical units are:\\n\\n\"\n \"- ``commands`` for the number of past commands executed,\\n\"\n \"- ``files`` for the number of history files to keep,\\n\"\n \"- ``s`` for the number of seconds in the past that are allowed, and\\n\"\n \"- ``b`` for the number of bytes that history may consume.\\n\\n\"\n \"Common abbreviations, such as '6 months' or '1 GB' are also allowed.\",\n default=\"``(8128, 'commands')`` or ``'8128 commands'``\",\n ),\n \"XONSH_INTERACTIVE\": VarDocs(\n \"``True`` if xonsh is running interactively, and ``False`` otherwise.\",\n configurable=False,\n ),\n \"XONSH_LOGIN\": VarDocs(\n \"``True`` if xonsh is running as a login shell, and ``False`` otherwise.\",\n configurable=False,\n ),\n \"XONSH_PROC_FREQUENCY\": VarDocs(\n \"The process frequency is the time that \"\n \"xonsh process threads sleep for while running command pipelines. \"\n \"The value has units of seconds [s].\"\n ),\n \"XONSH_SHOW_TRACEBACK\": VarDocs(\n \"Controls if a traceback is shown if exceptions occur in the shell. \"\n \"Set to ``True`` to always show traceback or ``False`` to always hide. \"\n \"If undefined then the traceback is hidden but a notice is shown on how \"\n \"to enable the full traceback.\"\n ),\n \"XONSH_SOURCE\": VarDocs(\n \"When running a xonsh script, this variable contains the absolute path \"\n \"to the currently executing script's file.\",\n configurable=False,\n ),\n \"XONSH_STDERR_PREFIX\": VarDocs(\n \"A format string, using the same keys and colors as ``$PROMPT``, that \"\n \"is prepended whenever stderr is displayed. This may be used in \"\n \"conjunction with ``$XONSH_STDERR_POSTFIX`` to close out the block.\"\n \"For example, to have stderr appear on a red background, the \"\n 'prefix & postfix pair would be \"{BACKGROUND_RED}\" & \"{NO_COLOR}\".'\n ),\n \"XONSH_STDERR_POSTFIX\": VarDocs(\n \"A format string, using the same keys and colors as ``$PROMPT``, that \"\n \"is appended whenever stderr is displayed. This may be used in \"\n \"conjunction with ``$XONSH_STDERR_PREFIX`` to start the block.\"\n \"For example, to have stderr appear on a red background, the \"\n 'prefix & postfix pair would be \"{BACKGROUND_RED}\" & \"{NO_COLOR}\".'\n ),\n \"XONSH_STORE_STDIN\": VarDocs(\n \"Whether or not to store the stdin that is supplied to the \"\n \"``!()`` and ``![]`` operators.\"\n ),\n \"XONSH_STORE_STDOUT\": VarDocs(\n \"Whether or not to store the ``stdout`` and ``stderr`` streams in the \"\n \"history files.\"\n ),\n \"XONSH_TRACEBACK_LOGFILE\": VarDocs(\n \"Specifies a file to store the traceback log to, regardless of whether \"\n \"``XONSH_SHOW_TRACEBACK`` has been set. Its value must be a writable file \"\n \"or None / the empty string if traceback logging is not desired. \"\n \"Logging to a file is not enabled by default.\"\n ),\n \"XONSH_DATETIME_FORMAT\": VarDocs(\n \"The format that is used for ``datetime.strptime()`` in various places\"\n \"i.e the history timestamp option\"\n ),\n }\n\n\n#\n# actual environment\n#\n\n\nclass Env(cabc.MutableMapping):\n \"\"\"A xonsh environment, whose variables have limited typing\n (unlike BASH). Most variables are, by default, strings (like BASH).\n However, the following rules also apply based on variable-name:\n\n * PATH: any variable whose name ends in PATH is a list of strings.\n * XONSH_HISTORY_SIZE: this variable is an (int | float, str) tuple.\n * LC_* (locale categories): locale category names get/set the Python\n locale via locale.getlocale() and locale.setlocale() functions.\n\n An Env instance may be converted to an untyped version suitable for\n use in a subprocess.\n \"\"\"\n\n _arg_regex = None\n\n def __init__(self, *args, **kwargs):\n \"\"\"If no initial environment is given, os_environ is used.\"\"\"\n self._d = {}\n # sentinel value for non existing envvars\n self._no_value = object()\n self._orig_env = None\n self._ensurers = {k: Ensurer(*v) for k, v in DEFAULT_ENSURERS.items()}\n self._defaults = DEFAULT_VALUES\n self._docs = DEFAULT_DOCS\n if len(args) == 0 and len(kwargs) == 0:\n args = (os_environ,)\n for key, val in dict(*args, **kwargs).items():\n self[key] = val\n if ON_WINDOWS:\n path_key = next((k for k in self._d if k.upper() == \"PATH\"), None)\n if path_key:\n self[\"PATH\"] = self._d.pop(path_key)\n if \"PATH\" not in self._d:\n # this is here so the PATH is accessible to subprocs and so that\n # it can be modified in-place in the xonshrc file\n self._d[\"PATH\"] = list(PATH_DEFAULT)\n self._detyped = None\n\n def detype(self):\n if self._detyped is not None:\n return self._detyped\n ctx = {}\n for key, val in self._d.items():\n if not isinstance(key, str):\n key = str(key)\n ensurer = self.get_ensurer(key)\n if ensurer.detype is None:\n # cannot be detyped\n continue\n deval = ensurer.detype(val)\n if deval is None:\n # cannot be detyped\n continue\n ctx[key] = deval\n self._detyped = ctx\n return ctx\n\n def replace_env(self):\n \"\"\"Replaces the contents of os_environ with a detyped version\n of the xonsh environment.\n \"\"\"\n if self._orig_env is None:\n self._orig_env = dict(os_environ)\n os_environ.clear()\n os_environ.update(self.detype())\n\n def undo_replace_env(self):\n \"\"\"Replaces the contents of os_environ with a detyped version\n of the xonsh environment.\n \"\"\"\n if self._orig_env is not None:\n os_environ.clear()\n os_environ.update(self._orig_env)\n self._orig_env = None\n\n def _get_default_ensurer(self, default=None):\n if default is not None:\n return default\n else:\n default = Ensurer(always_true, None, ensure_string)\n return default\n\n def get_ensurer(self, key, default=None):\n \"\"\"Gets an ensurer for the given key.\"\"\"\n if key in self._ensurers:\n return self._ensurers[key]\n for k, ensurer in self._ensurers.items():\n if isinstance(k, str):\n continue\n if k.match(key) is not None:\n break\n else:\n ensurer = self._get_default_ensurer(default=default)\n self._ensurers[key] = ensurer\n return ensurer\n\n def set_ensurer(self, key, value):\n \"\"\"Sets an ensurer.\"\"\"\n self._detyped = None\n self._ensurers[key] = value\n\n def get_docs(self, key, default=VarDocs(\"\")):\n \"\"\"Gets the documentation for the environment variable.\"\"\"\n vd = self._docs.get(key, None)\n if vd is None:\n return default\n if vd.default is DefaultNotGiven:\n dval = pprint.pformat(self._defaults.get(key, \"\"))\n vd = vd._replace(default=dval)\n self._docs[key] = vd\n return vd\n\n def help(self, key):\n \"\"\"Get information about a specific environment variable.\"\"\"\n vardocs = self.get_docs(key)\n width = min(79, os.get_terminal_size()[0])\n docstr = \"\\n\".join(textwrap.wrap(vardocs.docstr, width=width))\n template = HELP_TEMPLATE.format(\n envvar=key,\n docstr=docstr,\n default=vardocs.default,\n configurable=vardocs.configurable,\n )\n print_color(template)\n\n def is_manually_set(self, varname):\n \"\"\"\n Checks if an environment variable has been manually set.\n \"\"\"\n return varname in self._d\n\n @contextlib.contextmanager\n def swap(self, other=None, **kwargs):\n \"\"\"Provides a context manager for temporarily swapping out certain\n environment variables with other values. On exit from the context\n manager, the original values are restored.\n \"\"\"\n old = {}\n # single positional argument should be a dict-like object\n if other is not None:\n for k, v in other.items():\n old[k] = self.get(k, NotImplemented)\n self[k] = v\n # kwargs could also have been sent in\n for k, v in kwargs.items():\n old[k] = self.get(k, NotImplemented)\n self[k] = v\n\n exception = None\n try:\n yield self\n except Exception as e:\n exception = e\n finally:\n # restore the values\n for k, v in old.items():\n if v is NotImplemented:\n del self[k]\n else:\n self[k] = v\n if exception is not None:\n raise exception from None\n\n #\n # Mutable mapping interface\n #\n\n def __getitem__(self, key):\n # remove this block on next release\n if key is Ellipsis:\n return self\n elif key in self._d:\n val = self._d[key]\n elif key in self._defaults:\n val = self._defaults[key]\n if is_callable_default(val):\n val = val(self)\n else:\n e = \"Unknown environment variable: ${}\"\n raise KeyError(e.format(key))\n if isinstance(\n val, (cabc.MutableSet, cabc.MutableSequence, cabc.MutableMapping)\n ):\n self._detyped = None\n return val\n\n def __setitem__(self, key, val):\n ensurer = self.get_ensurer(key)\n if not ensurer.validate(val):\n val = ensurer.convert(val)\n # existing envvars can have any value including None\n old_value = self._d[key] if key in self._d else self._no_value\n self._d[key] = val\n self._detyped = None\n if self.get(\"UPDATE_OS_ENVIRON\"):\n if self._orig_env is None:\n self.replace_env()\n elif ensurer.detype is None:\n pass\n else:\n deval = ensurer.detype(val)\n if deval is not None:\n os_environ[key] = deval\n if old_value is self._no_value:\n events.on_envvar_new.fire(name=key, value=val)\n elif old_value != val:\n events.on_envvar_change.fire(name=key, oldvalue=old_value, newvalue=val)\n\n def __delitem__(self, key):\n del self._d[key]\n self._detyped = None\n if self.get(\"UPDATE_OS_ENVIRON\") and key in os_environ:\n del os_environ[key]\n\n def get(self, key, default=None):\n \"\"\"The environment will look up default values from its own defaults if a\n default is not given here.\n \"\"\"\n try:\n return self[key]\n except KeyError:\n return default\n\n def __iter__(self):\n yield from (set(self._d) | set(self._defaults))\n\n def __contains__(self, item):\n return item in self._d or item in self._defaults\n\n def __len__(self):\n return len(self._d)\n\n def __str__(self):\n return str(self._d)\n\n def __repr__(self):\n return \"{0}.{1}(...)\".format(\n self.__class__.__module__, self.__class__.__name__, self._d\n )\n\n def _repr_pretty_(self, p, cycle):\n name = \"{0}.{1}\".format(self.__class__.__module__, self.__class__.__name__)\n with p.group(0, name + \"(\", \")\"):\n if cycle:\n p.text(\"...\")\n elif len(self):\n p.break_()\n p.pretty(dict(self))\n\n\ndef _yield_executables(directory, name):\n if ON_WINDOWS:\n base_name, ext = os.path.splitext(name.lower())\n for fname in executables_in(directory):\n fbase, fext = os.path.splitext(fname.lower())\n if base_name == fbase and (len(ext) == 0 or ext == fext):\n yield os.path.join(directory, fname)\n else:\n for x in executables_in(directory):\n if x == name:\n yield os.path.join(directory, name)\n return\n\n\ndef locate_binary(name):\n \"\"\"Locates an executable on the file system.\"\"\"\n return builtins.__xonsh__.commands_cache.locate_binary(name)\n\n\nBASE_ENV = LazyObject(\n lambda: {\n \"BASH_COMPLETIONS\": list(DEFAULT_VALUES[\"BASH_COMPLETIONS\"]),\n \"PROMPT_FIELDS\": dict(DEFAULT_VALUES[\"PROMPT_FIELDS\"]),\n \"XONSH_VERSION\": XONSH_VERSION,\n },\n globals(),\n \"BASE_ENV\",\n)\n\n\ndef xonshrc_context(rcfiles=None, execer=None, ctx=None, env=None, login=True):\n \"\"\"Attempts to read in all xonshrc files and return the context.\"\"\"\n loaded = env[\"LOADED_RC_FILES\"] = []\n ctx = {} if ctx is None else ctx\n if rcfiles is None:\n return env\n env[\"XONSHRC\"] = tuple(rcfiles)\n for rcfile in rcfiles:\n if not os.path.isfile(rcfile):\n loaded.append(False)\n continue\n _, ext = os.path.splitext(rcfile)\n status = xonsh_script_run_control(rcfile, ctx, env, execer=execer, login=login)\n loaded.append(status)\n return ctx\n\n\ndef windows_foreign_env_fixes(ctx):\n \"\"\"Environment fixes for Windows. Operates in-place.\"\"\"\n # remove these bash variables which only cause problems.\n for ev in [\"HOME\", \"OLDPWD\"]:\n if ev in ctx:\n del ctx[ev]\n # Override path-related bash variables; on Windows bash uses\n # /c/Windows/System32 syntax instead of C:\\\\Windows\\\\System32\n # which messes up these environment variables for xonsh.\n for ev in [\"PATH\", \"TEMP\", \"TMP\"]:\n if ev in os_environ:\n ctx[ev] = os_environ[ev]\n elif ev in ctx:\n del ctx[ev]\n ctx[\"PWD\"] = _get_cwd() or \"\"\n\n\ndef foreign_env_fixes(ctx):\n \"\"\"Environment fixes for all operating systems\"\"\"\n if \"PROMPT\" in ctx:\n del ctx[\"PROMPT\"]\n\n\ndef xonsh_script_run_control(filename, ctx, env, execer=None, login=True):\n \"\"\"Loads a xonsh file and applies it as a run control.\"\"\"\n if execer is None:\n return False\n updates = {\"__file__\": filename, \"__name__\": os.path.abspath(filename)}\n try:\n with swap_values(ctx, updates):\n run_script_with_cache(filename, execer, ctx)\n loaded = True\n except SyntaxError as err:\n msg = \"syntax error in xonsh run control file {0!r}: {1!s}\"\n print_exception(msg.format(filename, err))\n loaded = False\n except Exception as err:\n msg = \"error running xonsh run control file {0!r}: {1!s}\"\n print_exception(msg.format(filename, err))\n loaded = False\n return loaded\n\n\ndef default_env(env=None):\n \"\"\"Constructs a default xonsh environment.\"\"\"\n # in order of increasing precedence\n ctx = dict(BASE_ENV)\n ctx.update(os_environ)\n ctx[\"PWD\"] = _get_cwd() or \"\"\n # These can cause problems for programs (#2543)\n ctx.pop(\"LINES\", None)\n ctx.pop(\"COLUMNS\", None)\n # other shells' PROMPT definitions generally don't work in XONSH:\n try:\n del ctx[\"PROMPT\"]\n except KeyError:\n pass\n # finalize env\n if env is not None:\n ctx.update(env)\n return ctx\n\n\ndef make_args_env(args=None):\n \"\"\"Makes a dictionary containing the $ARGS and $ARG environment\n variables. If the supplied ARGS is None, then sys.argv is used.\n \"\"\"\n if args is None:\n args = sys.argv\n env = {\"ARG\" + str(i): arg for i, arg in enumerate(args)}\n env[\"ARGS\"] = list(args) # make a copy so we don't interfere with original variable\n return env\n" }, { "path": "xonsh/events.py", "content": "\"\"\"\nEvents for xonsh.\n\nIn all likelihood, you want builtins.events\n\nThe best way to \"declare\" an event is something like::\n\n events.doc('on_spam', \"Comes with eggs\")\n\"\"\"\nimport abc\nimport builtins\nimport collections.abc\nimport inspect\n\nfrom xonsh.tools import print_exception\n\n\ndef has_kwargs(func):\n return any(\n p.kind == p.VAR_KEYWORD for p in inspect.signature(func).parameters.values()\n )\n\n\ndef debug_level():\n if hasattr(builtins, \"__xonsh__\") and hasattr(builtins.__xonsh__, \"env\"):\n return builtins.__xonsh__.env.get(\"XONSH_DEBUG\")\n # FIXME: Under py.test, return 1(?)\n else:\n return 0 # Optimize for speed, not guaranteed correctness\n\n\nclass AbstractEvent(collections.abc.MutableSet, abc.ABC):\n \"\"\"\n A given event that handlers can register against.\n\n Acts as a ``MutableSet`` for registered handlers.\n\n Note that ordering is never guaranteed.\n \"\"\"\n\n @property\n def species(self):\n \"\"\"\n The species (basically, class) of the event\n \"\"\"\n return type(self).__bases__[\n 0\n ] # events.on_chdir -> -> \n\n def __call__(self, handler):\n \"\"\"\n Registers a handler. It's suggested to use this as a decorator.\n\n A decorator method is added to the handler, validator(). If a validator\n function is added, it can filter if the handler will be considered. The\n validator takes the same arguments as the handler. If it returns False,\n the handler will not called or considered, as if it was not registered\n at all.\n\n Parameters\n ----------\n handler : callable\n The handler to register\n\n Returns\n -------\n rtn : callable\n The handler\n \"\"\"\n # Using Python's \"private\" munging to minimize hypothetical collisions\n handler.__validator = None\n if debug_level():\n if not has_kwargs(handler):\n raise ValueError(\"Event handlers need a **kwargs for future proofing\")\n self.add(handler)\n\n def validator(vfunc):\n \"\"\"\n Adds a validator function to a handler to limit when it is considered.\n \"\"\"\n if debug_level():\n if not has_kwargs(handler):\n raise ValueError(\n \"Event validators need a **kwargs for future proofing\"\n )\n handler.__validator = vfunc\n\n handler.validator = validator\n\n return handler\n\n def _filterhandlers(self, handlers, **kwargs):\n \"\"\"\n Helper method for implementing classes. Generates the handlers that pass validation.\n \"\"\"\n for handler in handlers:\n if handler.__validator is not None and not handler.__validator(**kwargs):\n continue\n yield handler\n\n @abc.abstractmethod\n def fire(self, **kwargs):\n \"\"\"\n Fires an event, calling registered handlers with the given arguments.\n\n Parameters\n ----------\n **kwargs :\n Keyword arguments to pass to each handler\n \"\"\"\n\n\nclass Event(AbstractEvent):\n \"\"\"\n An event species for notify and scatter-gather events.\n \"\"\"\n\n # Wish I could just pull from set...\n def __init__(self):\n self._handlers = set()\n self._firing = False\n self._delayed_adds = None\n self._delayed_discards = None\n\n def __len__(self):\n return len(self._handlers)\n\n def __contains__(self, item):\n return item in self._handlers\n\n def __iter__(self):\n yield from self._handlers\n\n def add(self, item):\n \"\"\"\n Add an element to a set.\n\n This has no effect if the element is already present.\n \"\"\"\n if self._firing:\n if self._delayed_adds is None:\n self._delayed_adds = set()\n self._delayed_adds.add(item)\n else:\n self._handlers.add(item)\n\n def discard(self, item):\n \"\"\"\n Remove an element from a set if it is a member.\n\n If the element is not a member, do nothing.\n \"\"\"\n if self._firing:\n if self._delayed_discards is None:\n self._delayed_discards = set()\n self._delayed_discards.add(item)\n else:\n self._handlers.discard(item)\n\n def fire(self, **kwargs):\n \"\"\"\n Fires an event, calling registered handlers with the given arguments. A non-unique iterable\n of the results is returned.\n\n Each handler is called immediately. Exceptions are turned in to warnings.\n\n Parameters\n ----------\n **kwargs :\n Keyword arguments to pass to each handler\n\n Returns\n -------\n vals : iterable\n Return values of each handler. If multiple handlers return the same value, it will\n appear multiple times.\n \"\"\"\n vals = []\n self._firing = True\n for handler in self._filterhandlers(self._handlers, **kwargs):\n try:\n rv = handler(**kwargs)\n except Exception:\n print_exception(\"Exception raised in event handler; ignored.\")\n else:\n vals.append(rv)\n # clean up\n self._firing = False\n if self._delayed_adds is not None:\n self._handlers.update(self._delayed_adds)\n self._delayed_adds = None\n if self._delayed_discards is not None:\n self._handlers.difference_update(self._delayed_discards)\n self._delayed_discards = None\n return vals\n\n\nclass LoadEvent(AbstractEvent):\n \"\"\"\n An event species where each handler is called exactly once, shortly after either the event is\n fired or the handler is registered (whichever is later). Additional firings are ignored.\n\n Note: Does not support scatter/gather, due to never knowing when we have all the handlers.\n\n Note: Maintains a strong reference to pargs/kwargs in case of the addition of future handlers.\n\n Note: This is currently NOT thread safe.\n \"\"\"\n\n def __init__(self):\n self._fired = set()\n self._unfired = set()\n self._hasfired = False\n\n def __len__(self):\n return len(self._fired) + len(self._unfired)\n\n def __contains__(self, item):\n return item in self._fired or item in self._unfired\n\n def __iter__(self):\n yield from self._fired\n yield from self._unfired\n\n def add(self, item):\n \"\"\"\n Add an element to a set.\n\n This has no effect if the element is already present.\n \"\"\"\n if self._hasfired:\n self._call(item)\n self._fired.add(item)\n else:\n self._unfired.add(item)\n\n def discard(self, item):\n \"\"\"\n Remove an element from a set if it is a member.\n\n If the element is not a member, do nothing.\n \"\"\"\n self._fired.discard(item)\n self._unfired.discard(item)\n\n def _call(self, handler):\n try:\n handler(**self._kwargs)\n except Exception:\n print_exception(\"Exception raised in event handler; ignored.\")\n\n def fire(self, **kwargs):\n if self._hasfired:\n return\n self._kwargs = kwargs\n while self._unfired:\n handler = self._unfired.pop()\n self._call(handler)\n self._hasfired = True\n return () # Entirely for API compatibility\n\n\nclass EventManager:\n \"\"\"\n Container for all events in a system.\n\n Meant to be a singleton, but doesn't enforce that itself.\n\n Each event is just an attribute. They're created dynamically on first use.\n \"\"\"\n\n def doc(self, name, docstring):\n \"\"\"\n Applies a docstring to an event.\n\n Parameters\n ----------\n name : str\n The name of the event, eg \"on_precommand\"\n docstring : str\n The docstring to apply to the event\n \"\"\"\n type(getattr(self, name)).__doc__ = docstring\n\n @staticmethod\n def _mkevent(name, species=Event, doc=None):\n # NOTE: Also used in `xonsh_events` test fixture\n # (A little bit of magic to enable docstrings to work right)\n return type(\n name,\n (species,),\n {\n \"__doc__\": doc,\n \"__module__\": \"xonsh.events\",\n \"__qualname__\": \"events.\" + name,\n },\n )()\n\n def transmogrify(self, name, species):\n \"\"\"\n Converts an event from one species to another, preserving handlers and docstring.\n\n Please note: Some species maintain specialized state. This is lost on transmogrification.\n\n Parameters\n ----------\n name : str\n The name of the event, eg \"on_precommand\"\n species : subclass of AbstractEvent\n The type to turn the event in to.\n \"\"\"\n if isinstance(species, str):\n species = globals()[species]\n\n if not issubclass(species, AbstractEvent):\n raise ValueError(\"Invalid event class; must be a subclass of AbstractEvent\")\n\n oldevent = getattr(self, name)\n newevent = self._mkevent(name, species, type(oldevent).__doc__)\n setattr(self, name, newevent)\n\n for handler in oldevent:\n newevent.add(handler)\n\n def exists(self, name):\n \"\"\"Checks if an event with a given name exist. If it does not exist, it\n will not be created. That is what makes this different than\n ``hasattr(events, name)``, which will create the event.\n \"\"\"\n return name in self.__dict__\n\n def __getattr__(self, name):\n \"\"\"Get an event, if it doesn't already exist.\"\"\"\n if name.startswith(\"_\"):\n raise AttributeError\n # This is only called if the attribute doesn't exist, so create the Event...\n e = self._mkevent(name)\n # ... and save it.\n setattr(self, name, e)\n # Now it exists, and we won't be called again.\n return e\n\n\n# Not lazy because:\n# 1. Initialization of EventManager can't be much cheaper\n# 2. It's expected to be used at load time, negating any benefits of using lazy object\nevents = EventManager()\n" }, { "path": "xonsh/execer.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"Implements the xonsh executer.\"\"\"\nimport sys\nimport types\nimport inspect\nimport builtins\nimport collections.abc as cabc\n\nfrom xonsh.ast import CtxAwareTransformer\nfrom xonsh.parser import Parser\nfrom xonsh.tools import (\n subproc_toks,\n find_next_break,\n get_logical_line,\n replace_logical_line,\n balanced_parens,\n starting_whitespace,\n)\nfrom xonsh.built_ins import load_builtins, unload_builtins, load_proxies, unload_proxies\n\n\nclass Execer(object):\n \"\"\"Executes xonsh code in a context.\"\"\"\n\n def __init__(\n self,\n filename=\"\",\n debug_level=0,\n parser_args=None,\n unload=True,\n xonsh_ctx=None,\n scriptcache=True,\n cacheall=False,\n ):\n \"\"\"Parameters\n ----------\n filename : str, optional\n File we are to execute.\n debug_level : int, optional\n Debugging level to use in lexing and parsing.\n parser_args : dict, optional\n Arguments to pass down to the parser.\n unload : bool, optional\n Whether or not to unload xonsh builtins upon deletion.\n xonsh_ctx : dict or None, optional\n Xonsh xontext to load as builtins.__xonsh__.ctx\n scriptcache : bool, optional\n Whether or not to use a precompiled bytecode cache when execing\n code, default: True.\n cacheall : bool, optional\n Whether or not to cache all xonsh code, and not just files. If this\n is set to true, it will cache command line input too, default: False.\n \"\"\"\n parser_args = parser_args or {}\n self.parser = Parser(**parser_args)\n self.filename = filename\n self.debug_level = debug_level\n self.unload = unload\n self.scriptcache = scriptcache\n self.cacheall = cacheall\n self.ctxtransformer = CtxAwareTransformer(self.parser)\n load_builtins(execer=self, ctx=xonsh_ctx)\n load_proxies()\n\n def __del__(self):\n if self.unload:\n unload_proxies()\n unload_builtins()\n\n def parse(self, input, ctx, mode=\"exec\", filename=None, transform=True):\n \"\"\"Parses xonsh code in a context-aware fashion. For context-free\n parsing, please use the Parser class directly or pass in\n transform=False.\n \"\"\"\n if filename is None:\n filename = self.filename\n if not transform:\n return self.parser.parse(\n input, filename=filename, mode=mode, debug_level=(self.debug_level > 2)\n )\n\n # Parsing actually happens in a couple of phases. The first is a\n # shortcut for a context-free parser. Normally, all subprocess\n # lines should be wrapped in $(), to indicate that they are a\n # subproc. But that would be super annoying. Unfortunately, Python\n # mode - after indentation - is whitespace agnostic while, using\n # the Python token, subproc mode is whitespace aware. That is to say,\n # in Python mode \"ls -l\", \"ls-l\", and \"ls - l\" all parse to the\n # same AST because whitespace doesn't matter to the minus binary op.\n # However, these phases all have very different meaning in subproc\n # mode. The 'right' way to deal with this is to make the entire\n # grammar whitespace aware, and then ignore all of the whitespace\n # tokens for all of the Python rules. The lazy way implemented here\n # is to parse a line a second time with a $() wrapper if it fails\n # the first time. This is a context-free phase.\n tree, input = self._parse_ctx_free(input, mode=mode, filename=filename)\n if tree is None:\n return None\n\n # Now we need to perform context-aware AST transformation. This is\n # because the \"ls -l\" is valid Python. The only way that we know\n # it is not actually Python is by checking to see if the first token\n # (ls) is part of the execution context. If it isn't, then we will\n # assume that this line is supposed to be a subprocess line, assuming\n # it also is valid as a subprocess line.\n if ctx is None:\n ctx = set()\n elif isinstance(ctx, cabc.Mapping):\n ctx = set(ctx.keys())\n tree = self.ctxtransformer.ctxvisit(\n tree, input, ctx, mode=mode, debug_level=self.debug_level\n )\n return tree\n\n def compile(\n self,\n input,\n mode=\"exec\",\n glbs=None,\n locs=None,\n stacklevel=2,\n filename=None,\n transform=True,\n ):\n \"\"\"Compiles xonsh code into a Python code object, which may then\n be execed or evaled.\n \"\"\"\n if filename is None:\n filename = self.filename\n if glbs is None or locs is None:\n frame = inspect.stack()[stacklevel][0]\n glbs = frame.f_globals if glbs is None else glbs\n locs = frame.f_locals if locs is None else locs\n ctx = set(dir(builtins)) | set(glbs.keys()) | set(locs.keys())\n tree = self.parse(input, ctx, mode=mode, filename=filename, transform=transform)\n if tree is None:\n return None # handles comment only input\n code = compile(tree, filename, mode)\n return code\n\n def eval(\n self, input, glbs=None, locs=None, stacklevel=2, filename=None, transform=True\n ):\n \"\"\"Evaluates (and returns) xonsh code.\"\"\"\n if isinstance(input, types.CodeType):\n code = input\n else:\n if filename is None:\n filename = self.filename\n code = self.compile(\n input=input,\n glbs=glbs,\n locs=locs,\n mode=\"eval\",\n stacklevel=stacklevel,\n filename=filename,\n transform=transform,\n )\n if code is None:\n return None # handles comment only input\n return eval(code, glbs, locs)\n\n def exec(\n self,\n input,\n mode=\"exec\",\n glbs=None,\n locs=None,\n stacklevel=2,\n filename=None,\n transform=True,\n ):\n \"\"\"Execute xonsh code.\"\"\"\n if isinstance(input, types.CodeType):\n code = input\n else:\n if filename is None:\n filename = self.filename\n code = self.compile(\n input=input,\n glbs=glbs,\n locs=locs,\n mode=mode,\n stacklevel=stacklevel,\n filename=filename,\n transform=transform,\n )\n if code is None:\n return None # handles comment only input\n return exec(code, glbs, locs)\n\n def _print_debug_wrapping(\n self, line, sbpline, last_error_line, last_error_col, maxcol=None\n ):\n \"\"\"print some debugging info if asked for.\"\"\"\n if self.debug_level > 1:\n msg = \"{0}:{1}:{2}{3} - {4}\\n\" \"{0}:{1}:{2}{3} + {5}\"\n mstr = \"\" if maxcol is None else \":\" + str(maxcol)\n msg = msg.format(\n self.filename, last_error_line, last_error_col, mstr, line, sbpline\n )\n print(msg, file=sys.stderr)\n\n def _parse_ctx_free(self, input, mode=\"exec\", filename=None, logical_input=False):\n last_error_line = last_error_col = -1\n parsed = False\n original_error = None\n greedy = False\n if filename is None:\n filename = self.filename\n if logical_input:\n beg_spaces = starting_whitespace(input)\n input = input[len(beg_spaces) :]\n while not parsed:\n try:\n tree = self.parser.parse(\n input,\n filename=filename,\n mode=mode,\n debug_level=(self.debug_level > 2),\n )\n parsed = True\n except IndentationError as e:\n if original_error is None:\n raise e\n else:\n raise original_error\n except SyntaxError as e:\n if original_error is None:\n original_error = e\n if (e.loc is None) or (\n last_error_line == e.loc.lineno\n and last_error_col in (e.loc.column + 1, e.loc.column)\n ):\n raise original_error from None\n elif last_error_line != e.loc.lineno:\n original_error = e\n last_error_col = e.loc.column\n last_error_line = e.loc.lineno\n idx = last_error_line - 1\n lines = input.splitlines()\n line, nlogical, idx = get_logical_line(lines, idx)\n if nlogical > 1 and not logical_input:\n _, sbpline = self._parse_ctx_free(\n line, mode=mode, filename=filename, logical_input=True\n )\n self._print_debug_wrapping(\n line, sbpline, last_error_line, last_error_col, maxcol=None\n )\n replace_logical_line(lines, sbpline, idx, nlogical)\n last_error_col += 3\n input = \"\\n\".join(lines)\n continue\n if input.endswith(\"\\n\"):\n lines.append(\"\")\n if len(line.strip()) == 0:\n # whitespace only lines are not valid syntax in Python's\n # interactive mode='single', who knew?! Just ignore them.\n # this might cause actual syntax errors to have bad line\n # numbers reported, but should only affect interactive mode\n del lines[idx]\n last_error_line = last_error_col = -1\n input = \"\\n\".join(lines)\n continue\n\n if last_error_line > 1 and lines[idx - 1].rstrip()[-1:] == \":\":\n # catch non-indented blocks and raise error.\n prev_indent = len(lines[idx - 1]) - len(lines[idx - 1].lstrip())\n curr_indent = len(lines[idx]) - len(lines[idx].lstrip())\n if prev_indent == curr_indent:\n raise original_error\n lexer = self.parser.lexer\n maxcol = (\n None\n if greedy\n else find_next_break(line, mincol=last_error_col, lexer=lexer)\n )\n if not greedy and maxcol in (e.loc.column + 1, e.loc.column):\n # go greedy the first time if the syntax error was because\n # we hit an end token out of place. This usually indicates\n # a subshell or maybe a macro.\n if not balanced_parens(line, maxcol=maxcol):\n greedy = True\n maxcol = None\n sbpline = subproc_toks(\n line, returnline=True, greedy=greedy, maxcol=maxcol, lexer=lexer\n )\n if sbpline is None:\n # subprocess line had no valid tokens,\n if len(line.partition(\"#\")[0].strip()) == 0:\n # likely because it only contained a comment.\n del lines[idx]\n last_error_line = last_error_col = -1\n input = \"\\n\".join(lines)\n continue\n elif not greedy:\n greedy = True\n continue\n else:\n # or for some other syntax error\n raise original_error\n elif sbpline[last_error_col:].startswith(\n \"![![\"\n ) or sbpline.lstrip().startswith(\"![![\"):\n # if we have already wrapped this in subproc tokens\n # and it still doesn't work, adding more won't help\n # anything\n if not greedy:\n greedy = True\n continue\n else:\n raise original_error\n # replace the line\n self._print_debug_wrapping(\n line, sbpline, last_error_line, last_error_col, maxcol=maxcol\n )\n replace_logical_line(lines, sbpline, idx, nlogical)\n last_error_col += 3\n input = \"\\n\".join(lines)\n if logical_input:\n input = beg_spaces + input\n return tree, input\n" }, { "path": "xonsh/foreign_shells.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"Tools to help interface with foreign shells, such as Bash.\"\"\"\nimport os\nimport re\nimport json\nimport shlex\nimport sys\nimport tempfile\nimport builtins\nimport subprocess\nimport warnings\nimport functools\nimport collections.abc as cabc\n\nfrom xonsh.lazyasd import lazyobject\nfrom xonsh.tools import to_bool, ensure_string\nfrom xonsh.platform import ON_WINDOWS, ON_CYGWIN, ON_MSYS\n\n\nCOMMAND = \"\"\"{seterrprevcmd}\n{prevcmd}\necho __XONSH_ENV_BEG__\n{envcmd}\necho __XONSH_ENV_END__\necho __XONSH_ALIAS_BEG__\n{aliascmd}\necho __XONSH_ALIAS_END__\necho __XONSH_FUNCS_BEG__\n{funcscmd}\necho __XONSH_FUNCS_END__\n{postcmd}\n{seterrpostcmd}\"\"\"\n\nDEFAULT_BASH_FUNCSCMD = r\"\"\"# get function names from declare\ndeclstr=$(declare -F)\nread -r -a decls <<< $declstr\nfuncnames=\"\"\nfor((n=0;n<${#decls[@]};n++)); do\n if (( $(($n % 3 )) == 2 )); then\n # get every 3rd entry\n funcnames=\"$funcnames ${decls[$n]}\"\n fi\ndone\n\n# get functions locations: funcname lineno filename\nshopt -s extdebug\nnamelocfilestr=$(declare -F $funcnames)\nshopt -u extdebug\n\n# print just names and files as JSON object\nread -r -a namelocfile <<< $namelocfilestr\nsep=\" \"\nnamefile=\"{\"\nwhile IFS='' read -r line || [[ -n \"$line\" ]]; do\n name=${line%%\"$sep\"*}\n locfile=${line#*\"$sep\"}\n loc=${locfile%%\"$sep\"*}\n file=${locfile#*\"$sep\"}\n namefile=\"${namefile}\\\"${name}\\\":\\\"${file//\\\\/\\\\\\\\}\\\",\"\ndone <<< \"$namelocfilestr\"\nif [[ \"{\" == \"${namefile}\" ]]; then\n namefile=\"${namefile}}\"\nelse\n namefile=\"${namefile%?}}\"\nfi\necho $namefile\"\"\"\n\nDEFAULT_ZSH_FUNCSCMD = \"\"\"# get function names\nautoload -U is-at-least # We'll need to version check zsh\nnamefile=\"{\"\nfor name in ${(ok)functions}; do\n # force zsh to load the func in order to get the filename,\n # but use +X so that it isn't executed.\n autoload +X $name || continue\n loc=$(whence -v $name)\n loc=${(z)loc}\n if is-at-least 5.2; then\n file=${loc[-1]}\n else\n file=${loc[7,-1]}\n fi\n namefile=\"${namefile}\\\\\"${name}\\\\\":\\\\\"${(Q)file:A}\\\\\",\"\ndone\nif [[ \"{\" == \"${namefile}\" ]]; then\n namefile=\"${namefile}}\"\nelse\n namefile=\"${namefile%?}}\"\nfi\necho ${namefile}\"\"\"\n\n\n# mapping of shell name aliases to keys in other lookup dictionaries.\n@lazyobject\ndef CANON_SHELL_NAMES():\n return {\n \"bash\": \"bash\",\n \"/bin/bash\": \"bash\",\n \"zsh\": \"zsh\",\n \"/bin/zsh\": \"zsh\",\n \"/usr/bin/zsh\": \"zsh\",\n \"cmd\": \"cmd\",\n \"cmd.exe\": \"cmd\",\n }\n\n\n@lazyobject\ndef DEFAULT_ENVCMDS():\n return {\"bash\": \"env\", \"zsh\": \"env\", \"cmd\": \"set\"}\n\n\n@lazyobject\ndef DEFAULT_ALIASCMDS():\n return {\"bash\": \"alias\", \"zsh\": \"alias -L\", \"cmd\": \"\"}\n\n\n@lazyobject\ndef DEFAULT_FUNCSCMDS():\n return {\"bash\": DEFAULT_BASH_FUNCSCMD, \"zsh\": DEFAULT_ZSH_FUNCSCMD, \"cmd\": \"\"}\n\n\n@lazyobject\ndef DEFAULT_SOURCERS():\n return {\"bash\": \"source\", \"zsh\": \"source\", \"cmd\": \"call\"}\n\n\n@lazyobject\ndef DEFAULT_TMPFILE_EXT():\n return {\"bash\": \".sh\", \"zsh\": \".zsh\", \"cmd\": \".bat\"}\n\n\n@lazyobject\ndef DEFAULT_RUNCMD():\n return {\"bash\": \"-c\", \"zsh\": \"-c\", \"cmd\": \"/C\"}\n\n\n@lazyobject\ndef DEFAULT_SETERRPREVCMD():\n return {\"bash\": \"set -e\", \"zsh\": \"set -e\", \"cmd\": \"@echo off\"}\n\n\n@lazyobject\ndef DEFAULT_SETERRPOSTCMD():\n return {\"bash\": \"\", \"zsh\": \"\", \"cmd\": \"if errorlevel 1 exit 1\"}\n\n\n@functools.lru_cache()\ndef foreign_shell_data(\n shell,\n interactive=True,\n login=False,\n envcmd=None,\n aliascmd=None,\n extra_args=(),\n currenv=None,\n safe=True,\n prevcmd=\"\",\n postcmd=\"\",\n funcscmd=None,\n sourcer=None,\n use_tmpfile=False,\n tmpfile_ext=None,\n runcmd=None,\n seterrprevcmd=None,\n seterrpostcmd=None,\n show=False,\n dryrun=False,\n):\n \"\"\"Extracts data from a foreign (non-xonsh) shells. Currently this gets\n the environment, aliases, and functions but may be extended in the future.\n\n Parameters\n ----------\n shell : str\n The name of the shell, such as 'bash' or '/bin/sh'.\n interactive : bool, optional\n Whether the shell should be run in interactive mode.\n login : bool, optional\n Whether the shell should be a login shell.\n envcmd : str or None, optional\n The command to generate environment output with.\n aliascmd : str or None, optional\n The command to generate alias output with.\n extra_args : tuple of str, optional\n Additional command line options to pass into the shell.\n currenv : tuple of items or None, optional\n Manual override for the current environment.\n safe : bool, optional\n Flag for whether or not to safely handle exceptions and other errors.\n prevcmd : str, optional\n A command to run in the shell before anything else, useful for\n sourcing and other commands that may require environment recovery.\n postcmd : str, optional\n A command to run after everything else, useful for cleaning up any\n damage that the prevcmd may have caused.\n funcscmd : str or None, optional\n This is a command or script that can be used to determine the names\n and locations of any functions that are native to the foreign shell.\n This command should print *only* a JSON object that maps\n function names to the filenames where the functions are defined.\n If this is None, then a default script will attempted to be looked\n up based on the shell name. Callable wrappers for these functions\n will be returned in the aliases dictionary.\n sourcer : str or None, optional\n How to source a foreign shell file for purposes of calling functions\n in that shell. If this is None, a default value will attempt to be\n looked up based on the shell name.\n use_tmpfile : bool, optional\n This specifies if the commands are written to a tmp file or just\n parsed directly to the shell\n tmpfile_ext : str or None, optional\n If tmpfile is True this sets specifies the extension used.\n runcmd : str or None, optional\n Command line switches to use when running the script, such as\n -c for Bash and /C for cmd.exe.\n seterrprevcmd : str or None, optional\n Command that enables exit-on-error for the shell that is run at the\n start of the script. For example, this is \"set -e\" in Bash. To disable\n exit-on-error behavior, simply pass in an empty string.\n seterrpostcmd : str or None, optional\n Command that enables exit-on-error for the shell that is run at the end\n of the script. For example, this is \"if errorlevel 1 exit 1\" in\n cmd.exe. To disable exit-on-error behavior, simply pass in an\n empty string.\n show : bool, optional\n Whether or not to display the script that will be run.\n dryrun : bool, optional\n Whether or not to actually run and process the command.\n\n\n Returns\n -------\n env : dict\n Dictionary of shell's environment. (None if the subproc command fails)\n aliases : dict\n Dictionary of shell's aliases, this includes foreign function\n wrappers.(None if the subproc command fails)\n \"\"\"\n cmd = [shell]\n cmd.extend(extra_args) # needs to come here for GNU long options\n if interactive:\n cmd.append(\"-i\")\n if login:\n cmd.append(\"-l\")\n shkey = CANON_SHELL_NAMES[shell]\n envcmd = DEFAULT_ENVCMDS.get(shkey, \"env\") if envcmd is None else envcmd\n aliascmd = DEFAULT_ALIASCMDS.get(shkey, \"alias\") if aliascmd is None else aliascmd\n funcscmd = DEFAULT_FUNCSCMDS.get(shkey, \"echo {}\") if funcscmd is None else funcscmd\n tmpfile_ext = (\n DEFAULT_TMPFILE_EXT.get(shkey, \"sh\") if tmpfile_ext is None else tmpfile_ext\n )\n runcmd = DEFAULT_RUNCMD.get(shkey, \"-c\") if runcmd is None else runcmd\n seterrprevcmd = (\n DEFAULT_SETERRPREVCMD.get(shkey, \"\") if seterrprevcmd is None else seterrprevcmd\n )\n seterrpostcmd = (\n DEFAULT_SETERRPOSTCMD.get(shkey, \"\") if seterrpostcmd is None else seterrpostcmd\n )\n command = COMMAND.format(\n envcmd=envcmd,\n aliascmd=aliascmd,\n prevcmd=prevcmd,\n postcmd=postcmd,\n funcscmd=funcscmd,\n seterrprevcmd=seterrprevcmd,\n seterrpostcmd=seterrpostcmd,\n ).strip()\n if show:\n print(command)\n if dryrun:\n return None, None\n cmd.append(runcmd)\n if not use_tmpfile:\n cmd.append(command)\n else:\n tmpfile = tempfile.NamedTemporaryFile(suffix=tmpfile_ext, delete=False)\n tmpfile.write(command.encode(\"utf8\"))\n tmpfile.close()\n cmd.append(tmpfile.name)\n if currenv is None and hasattr(builtins.__xonsh__, \"env\"):\n currenv = builtins.__xonsh__.env.detype()\n elif currenv is not None:\n currenv = dict(currenv)\n try:\n s = subprocess.check_output(\n cmd,\n stderr=subprocess.PIPE,\n env=currenv,\n # start new session to avoid hangs\n # (doesn't work on Cygwin though)\n start_new_session=((not ON_CYGWIN) and (not ON_MSYS)),\n universal_newlines=True,\n )\n except (subprocess.CalledProcessError, FileNotFoundError):\n if not safe:\n raise\n return None, None\n finally:\n if use_tmpfile:\n os.remove(tmpfile.name)\n env = parse_env(s)\n aliases = parse_aliases(s, shell=shell, sourcer=sourcer, extra_args=extra_args)\n funcs = parse_funcs(s, shell=shell, sourcer=sourcer, extra_args=extra_args)\n aliases.update(funcs)\n return env, aliases\n\n\n@lazyobject\ndef ENV_RE():\n return re.compile(\"__XONSH_ENV_BEG__\\n(.*)\" \"__XONSH_ENV_END__\", flags=re.DOTALL)\n\n\n@lazyobject\ndef ENV_SPLIT_RE():\n return re.compile(\"^([^=]+)=([^=]*|[^\\n]*)$\", flags=re.DOTALL | re.MULTILINE)\n\n\ndef parse_env(s):\n \"\"\"Parses the environment portion of string into a dict.\"\"\"\n m = ENV_RE.search(s)\n if m is None:\n return {}\n g1 = m.group(1)\n g1 = g1[:-1] if g1.endswith(\"\\n\") else g1\n env = dict(ENV_SPLIT_RE.findall(g1))\n return env\n\n\n@lazyobject\ndef ALIAS_RE():\n return re.compile(\n \"__XONSH_ALIAS_BEG__\\n(.*)\" \"__XONSH_ALIAS_END__\", flags=re.DOTALL\n )\n\n\n@lazyobject\ndef FS_EXEC_ALIAS_RE():\n return re.compile(r\";|`|\\$\\(\")\n\n\ndef parse_aliases(s, shell, sourcer=None, extra_args=()):\n \"\"\"Parses the aliases portion of string into a dict.\"\"\"\n m = ALIAS_RE.search(s)\n if m is None:\n return {}\n g1 = m.group(1)\n items = [\n line.split(\"=\", 1)\n for line in g1.splitlines()\n if line.startswith(\"alias \") and \"=\" in line\n ]\n aliases = {}\n for key, value in items:\n try:\n key = key[6:] # lstrip 'alias '\n # undo bash's weird quoting of single quotes (sh_single_quote)\n value = value.replace(\"'\\\\''\", \"'\")\n # strip one single quote at the start and end of value\n if value[0] == \"'\" and value[-1] == \"'\":\n value = value[1:-1]\n # now compute actual alias\n if FS_EXEC_ALIAS_RE.search(value) is None:\n # simple list of args alias\n value = shlex.split(value)\n else:\n # alias is more complex, use ExecAlias, but via shell\n filename = \"\"\n value = ForeignShellExecAlias(\n src=value,\n shell=shell,\n filename=filename,\n sourcer=sourcer,\n extra_args=extra_args,\n )\n except ValueError as exc:\n warnings.warn(\n 'could not parse alias \"{0}\": {1!r}'.format(key, exc), RuntimeWarning\n )\n continue\n aliases[key] = value\n return aliases\n\n\n@lazyobject\ndef FUNCS_RE():\n return re.compile(\n \"__XONSH_FUNCS_BEG__\\n(.+)\\n\" \"__XONSH_FUNCS_END__\", flags=re.DOTALL\n )\n\n\ndef parse_funcs(s, shell, sourcer=None, extra_args=()):\n \"\"\"Parses the funcs portion of a string into a dict of callable foreign\n function wrappers.\n \"\"\"\n m = FUNCS_RE.search(s)\n if m is None:\n return {}\n g1 = m.group(1)\n if ON_WINDOWS:\n g1 = g1.replace(os.sep, os.altsep)\n try:\n namefiles = json.loads(g1.strip())\n except json.decoder.JSONDecodeError as exc:\n msg = (\n \"{0!r}\\n\\ncould not parse {1} functions:\\n\"\n \" s = {2!r}\\n\"\n \" g1 = {3!r}\\n\\n\"\n \"Note: you may be seeing this error if you use zsh with \"\n \"prezto. Prezto overwrites GNU coreutils functions (like echo) \"\n \"with its own zsh functions. Please try disabling prezto.\"\n )\n warnings.warn(msg.format(exc, shell, s, g1), RuntimeWarning)\n return {}\n sourcer = DEFAULT_SOURCERS.get(shell, \"source\") if sourcer is None else sourcer\n funcs = {}\n for funcname, filename in namefiles.items():\n if funcname.startswith(\"_\") or not filename:\n continue # skip private functions and invalid files\n if not os.path.isabs(filename):\n filename = os.path.abspath(filename)\n wrapper = ForeignShellFunctionAlias(\n funcname=funcname,\n shell=shell,\n sourcer=sourcer,\n filename=filename,\n extra_args=extra_args,\n )\n funcs[funcname] = wrapper\n return funcs\n\n\nclass ForeignShellBaseAlias(object):\n \"\"\"This class is responsible for calling foreign shell functions as if\n they were aliases. This does not currently support taking stdin.\n \"\"\"\n\n INPUT = \"echo ForeignShellBaseAlias {shell} {filename} {args}\\n\"\n\n def __init__(self, shell, filename, sourcer=None, extra_args=()):\n \"\"\"\n Parameters\n ----------\n shell : str\n Name or path to shell\n filename : str\n Where the function is defined, path to source.\n sourcer : str or None, optional\n Command to source foreign files with.\n extra_args : tuple of str, optional\n Additional command line options to pass into the shell.\n \"\"\"\n sourcer = DEFAULT_SOURCERS.get(shell, \"source\") if sourcer is None else sourcer\n self.shell = shell\n self.filename = filename\n self.sourcer = sourcer\n self.extra_args = extra_args\n\n def _input_kwargs(self):\n return {\n \"shell\": self.shell,\n \"filename\": self.filename,\n \"sourcer\": self.sourcer,\n \"extra_args\": self.extra_args,\n }\n\n def __eq__(self, other):\n if not hasattr(other, \"_input_kwargs\") or not callable(other._input_kwargs):\n return NotImplemented\n return self._input_kwargs() == other._input_kwargs()\n\n def __call__(\n self, args, stdin=None, stdout=None, stderr=None, spec=None, stack=None\n ):\n args, streaming = self._is_streaming(args)\n input = self.INPUT.format(args=\" \".join(args), **self._input_kwargs())\n cmd = [self.shell] + list(self.extra_args) + [\"-c\", input]\n env = builtins.__xonsh__.env\n denv = env.detype()\n if streaming:\n subprocess.check_call(cmd, env=denv)\n out = None\n else:\n out = subprocess.check_output(cmd, env=denv, stderr=subprocess.STDOUT)\n out = out.decode(\n encoding=env.get(\"XONSH_ENCODING\"),\n errors=env.get(\"XONSH_ENCODING_ERRORS\"),\n )\n out = out.replace(\"\\r\\n\", \"\\n\")\n return out\n\n def __repr__(self):\n return (\n self.__class__.__name__\n + \"(\"\n + \", \".join(\n [\n \"{k}={v!r}\".format(k=k, v=v)\n for k, v in sorted(self._input_kwargs().items())\n ]\n )\n + \")\"\n )\n\n @staticmethod\n def _is_streaming(args):\n \"\"\"Test and modify args if --xonsh-stream is present.\"\"\"\n if \"--xonsh-stream\" not in args:\n return args, False\n args = list(args)\n args.remove(\"--xonsh-stream\")\n return args, True\n\n\nclass ForeignShellFunctionAlias(ForeignShellBaseAlias):\n \"\"\"This class is responsible for calling foreign shell functions as if\n they were aliases. This does not currently support taking stdin.\n \"\"\"\n\n INPUT = '{sourcer} \"{filename}\"\\n' \"{funcname} {args}\\n\"\n\n def __init__(self, funcname, shell, filename, sourcer=None, extra_args=()):\n \"\"\"\n Parameters\n ----------\n funcname : str\n function name\n shell : str\n Name or path to shell\n filename : str\n Where the function is defined, path to source.\n sourcer : str or None, optional\n Command to source foreign files with.\n extra_args : tuple of str, optional\n Additional command line options to pass into the shell.\n \"\"\"\n super().__init__(\n shell=shell, filename=filename, sourcer=sourcer, extra_args=extra_args\n )\n self.funcname = funcname\n\n def _input_kwargs(self):\n inp = super()._input_kwargs()\n inp[\"funcname\"] = self.funcname\n return inp\n\n\nclass ForeignShellExecAlias(ForeignShellBaseAlias):\n \"\"\"Provides a callable alias for source code in a foreign shell.\"\"\"\n\n INPUT = \"{src} {args}\\n\"\n\n def __init__(\n self,\n src,\n shell,\n filename=\"\",\n sourcer=None,\n extra_args=(),\n ):\n \"\"\"\n Parameters\n ----------\n src : str\n Source code in the shell language\n shell : str\n Name or path to shell\n filename : str\n Where the function is defined, path to source.\n sourcer : str or None, optional\n Command to source foreign files with.\n extra_args : tuple of str, optional\n Additional command line options to pass into the shell.\n \"\"\"\n super().__init__(\n shell=shell, filename=filename, sourcer=sourcer, extra_args=extra_args\n )\n self.src = src.strip()\n\n def _input_kwargs(self):\n inp = super()._input_kwargs()\n inp[\"src\"] = self.src\n return inp\n\n\n@lazyobject\ndef VALID_SHELL_PARAMS():\n return frozenset(\n [\n \"shell\",\n \"interactive\",\n \"login\",\n \"envcmd\",\n \"aliascmd\",\n \"extra_args\",\n \"currenv\",\n \"safe\",\n \"prevcmd\",\n \"postcmd\",\n \"funcscmd\",\n \"sourcer\",\n ]\n )\n\n\ndef ensure_shell(shell):\n \"\"\"Ensures that a mapping follows the shell specification.\"\"\"\n if not isinstance(shell, cabc.MutableMapping):\n shell = dict(shell)\n shell_keys = set(shell.keys())\n if not (shell_keys <= VALID_SHELL_PARAMS):\n msg = \"unknown shell keys: {0}\"\n raise KeyError(msg.format(shell_keys - VALID_SHELL_PARAMS))\n shell[\"shell\"] = ensure_string(shell[\"shell\"]).lower()\n if \"interactive\" in shell_keys:\n shell[\"interactive\"] = to_bool(shell[\"interactive\"])\n if \"login\" in shell_keys:\n shell[\"login\"] = to_bool(shell[\"login\"])\n if \"envcmd\" in shell_keys:\n shell[\"envcmd\"] = (\n None if shell[\"envcmd\"] is None else ensure_string(shell[\"envcmd\"])\n )\n if \"aliascmd\" in shell_keys:\n shell[\"aliascmd\"] = (\n None if shell[\"aliascmd\"] is None else ensure_string(shell[\"aliascmd\"])\n )\n if \"extra_args\" in shell_keys and not isinstance(shell[\"extra_args\"], tuple):\n shell[\"extra_args\"] = tuple(map(ensure_string, shell[\"extra_args\"]))\n if \"currenv\" in shell_keys and not isinstance(shell[\"currenv\"], tuple):\n ce = shell[\"currenv\"]\n if isinstance(ce, cabc.Mapping):\n ce = tuple([(ensure_string(k), v) for k, v in ce.items()])\n elif isinstance(ce, cabc.Sequence):\n ce = tuple([(ensure_string(k), v) for k, v in ce])\n else:\n raise RuntimeError(\"unrecognized type for currenv\")\n shell[\"currenv\"] = ce\n if \"safe\" in shell_keys:\n shell[\"safe\"] = to_bool(shell[\"safe\"])\n if \"prevcmd\" in shell_keys:\n shell[\"prevcmd\"] = ensure_string(shell[\"prevcmd\"])\n if \"postcmd\" in shell_keys:\n shell[\"postcmd\"] = ensure_string(shell[\"postcmd\"])\n if \"funcscmd\" in shell_keys:\n shell[\"funcscmd\"] = (\n None if shell[\"funcscmd\"] is None else ensure_string(shell[\"funcscmd\"])\n )\n if \"sourcer\" in shell_keys:\n shell[\"sourcer\"] = (\n None if shell[\"sourcer\"] is None else ensure_string(shell[\"sourcer\"])\n )\n if \"seterrprevcmd\" in shell_keys:\n shell[\"seterrprevcmd\"] = (\n None\n if shell[\"seterrprevcmd\"] is None\n else ensure_string(shell[\"seterrprevcmd\"])\n )\n if \"seterrpostcmd\" in shell_keys:\n shell[\"seterrpostcmd\"] = (\n None\n if shell[\"seterrpostcmd\"] is None\n else ensure_string(shell[\"seterrpostcmd\"])\n )\n return shell\n\n\ndef load_foreign_envs(shells):\n \"\"\"Loads environments from foreign shells.\n\n Parameters\n ----------\n shells : sequence of dicts\n An iterable of dicts that can be passed into foreign_shell_data() as\n keyword arguments.\n\n Returns\n -------\n env : dict\n A dictionary of the merged environments.\n \"\"\"\n env = {}\n for shell in shells:\n shell = ensure_shell(shell)\n shenv, _ = foreign_shell_data(**shell)\n if shenv:\n env.update(shenv)\n return env\n\n\ndef load_foreign_aliases(shells):\n \"\"\"Loads aliases from foreign shells.\n\n Parameters\n ----------\n shells : sequence of dicts\n An iterable of dicts that can be passed into foreign_shell_data() as\n keyword arguments.\n\n Returns\n -------\n aliases : dict\n A dictionary of the merged aliases.\n \"\"\"\n aliases = {}\n xonsh_aliases = builtins.aliases\n for shell in shells:\n shell = ensure_shell(shell)\n _, shaliases = foreign_shell_data(**shell)\n if not builtins.__xonsh__.env.get(\"FOREIGN_ALIASES_OVERRIDE\"):\n shaliases = {} if shaliases is None else shaliases\n for alias in set(shaliases) & set(xonsh_aliases):\n del shaliases[alias]\n if builtins.__xonsh__.env.get(\"XONSH_DEBUG\") > 1:\n print(\n \"aliases: ignoring alias {!r} of shell {!r} \"\n \"which tries to override xonsh alias.\"\n \"\".format(alias, shell[\"shell\"]),\n file=sys.stderr,\n )\n aliases.update(shaliases)\n return aliases\n" }, { "path": "xonsh/fs.py", "content": "\"\"\"\nBackported functions to implement the PEP 519 (Adding a file system path protocol) API.\n\"\"\"\n\nimport abc\nimport sys\nimport io\nimport pathlib\n\ntry:\n from os import PathLike, fspath, fsencode, fsdecode\nexcept ImportError:\n\n class PathLike(abc.ABC):\n \"\"\"Abstract base class for implementing the file system path protocol.\"\"\"\n\n @abc.abstractmethod\n def __fspath__(self):\n \"\"\"Return the file system path representation of the object.\"\"\"\n raise NotImplementedError\n\n PathLike.register(pathlib.Path)\n\n def fspath(path):\n \"\"\"Return the string representation of the path.\n\n If str or bytes is passed in, it is returned unchanged. If __fspath__()\n returns something other than str or bytes then TypeError is raised. If\n this function is given something that is not str, bytes, or os.PathLike\n then TypeError is raised.\n \"\"\"\n if isinstance(path, (str, bytes)):\n return path\n\n if isinstance(path, pathlib.Path):\n return str(path)\n\n # Work from the object's type to match method resolution of other magic\n # methods.\n path_type = type(path)\n try:\n path = path_type.__fspath__(path)\n except AttributeError:\n if hasattr(path_type, \"__fspath__\"):\n raise\n else:\n if isinstance(path, (str, bytes)):\n return path\n else:\n raise TypeError(\n \"expected __fspath__() to return str or bytes, \"\n \"not \" + type(path).__name__\n )\n\n raise TypeError(\n \"expected str, bytes or os.PathLike object, not \" + path_type.__name__\n )\n\n def _fscodec():\n encoding = sys.getfilesystemencoding()\n if encoding == \"mbcs\":\n errors = \"strict\"\n else:\n errors = \"surrogateescape\"\n\n def fsencode(filename):\n \"\"\"Encode filename (an os.PathLike, bytes, or str) to the filesystem\n encoding with 'surrogateescape' error handler, return bytes unchanged.\n On Windows, use 'strict' error handler if the file system encoding is\n 'mbcs' (which is the default encoding).\n \"\"\"\n filename = fspath(filename) # Does type-checking of `filename`.\n if isinstance(filename, str):\n return filename.encode(encoding, errors)\n else:\n return filename\n\n def fsdecode(filename):\n \"\"\"Decode filename (an os.PathLike, bytes, or str) from the filesystem\n encoding with 'surrogateescape' error handler, return str unchanged. On\n Windows, use 'strict' error handler if the file system encoding is\n 'mbcs' (which is the default encoding).\n \"\"\"\n filename = fspath(filename) # Does type-checking of `filename`.\n if isinstance(filename, bytes):\n return filename.decode(encoding, errors)\n else:\n return filename\n\n return fsencode, fsdecode\n\n fsencode, fsdecode = _fscodec()\n del _fscodec\n\n def open(file, *pargs, **kwargs):\n if isinstance(file, PathLike):\n file = fspath(file)\n return io.open(file, *pargs, **kwargs)\n" }, { "path": "xonsh/history/__init__.py", "content": "# amalgamate exclude\nimport os as _os\n\nif _os.getenv(\"XONSH_DEBUG\", \"\"):\n pass\nelse:\n import sys as _sys\n\n try:\n from xonsh.history import __amalgam__\n\n base = __amalgam__\n _sys.modules[\"xonsh.history.base\"] = __amalgam__\n dummy = __amalgam__\n _sys.modules[\"xonsh.history.dummy\"] = __amalgam__\n json = __amalgam__\n _sys.modules[\"xonsh.history.json\"] = __amalgam__\n sqlite = __amalgam__\n _sys.modules[\"xonsh.history.sqlite\"] = __amalgam__\n main = __amalgam__\n _sys.modules[\"xonsh.history.main\"] = __amalgam__\n del __amalgam__\n except ImportError:\n pass\n del _sys\ndel _os\n# amalgamate end\n" }, { "path": "xonsh/history/base.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"Base class of Xonsh History backends.\"\"\"\nimport types\nimport uuid\n\n\nclass HistoryEntry(types.SimpleNamespace):\n \"\"\"Represent a command in history.\n\n Attributes\n ----------\n cmd: str\n The command as typed by the user, including newlines\n out: str\n The output of the command, if xonsh is configured to save it\n rtn: int\n The return of the command (ie, 0 on success)\n ts: two-tuple of floats\n The timestamps of when the command started and finished, including\n fractions.\n\n \"\"\"\n\n\nclass History:\n \"\"\"Xonsh history backend base class.\n\n History objects should be created via a subclass of History.\n\n Indexing\n --------\n History acts like a sequence that can be indexed to return\n ``HistoryEntry`` objects.\n\n Note that the most recent command is the last item in history.\n\n Attributes\n ----------\n rtns : sequence of ints\n The return of the command (ie, 0 on success)\n inps : sequence of strings\n The command as typed by the user, including newlines\n tss : sequence of two-tuples of floats\n The timestamps of when the command started and finished, including\n fractions\n outs : sequence of strings\n The output of the command, if xonsh is configured to save it\n gc : A garbage collector or None\n The garbage collector\n\n In all of these sequences, index 0 is the oldest and -1 (the last item)\n is the newest.\n \"\"\"\n\n def __init__(self, sessionid=None, **kwargs):\n \"\"\"Represents a xonsh session's history.\n\n Parameters\n ----------\n sessionid : int, uuid, str, optional\n Current session identifier, will generate a new sessionid if not\n set.\n \"\"\"\n self.sessionid = uuid.uuid4() if sessionid is None else sessionid\n self.gc = None\n self.buffer = None\n self.filename = None\n self.inps = None\n self.rtns = None\n self.tss = None\n self.outs = None\n self.last_cmd_rtn = None\n self.last_cmd_out = None\n\n def __len__(self):\n \"\"\"Return the number of items in current session.\"\"\"\n return len(list(self.items()))\n\n def __getitem__(self, item):\n \"\"\"Retrieve history entries, see ``History`` docs for more info.\"\"\"\n if isinstance(item, int):\n if item >= len(self):\n raise IndexError(\"history index out of range\")\n return HistoryEntry(\n cmd=self.inps[item],\n out=self.outs[item],\n rtn=self.rtns[item],\n ts=self.tss[item],\n )\n elif isinstance(item, slice):\n cmds = self.inps[item]\n outs = self.outs[item]\n rtns = self.rtns[item]\n tss = self.tss[item]\n return [\n HistoryEntry(cmd=c, out=o, rtn=r, ts=t)\n for c, o, r, t in zip(cmds, outs, rtns, tss)\n ]\n else:\n raise TypeError(\n \"history indices must be integers \"\n \"or slices, not {}\".format(type(item))\n )\n\n def __setitem__(self, *args):\n raise PermissionError(\n \"You cannot change history! \" \"you can create new though.\"\n )\n\n def append(self, cmd):\n \"\"\"Append a command item into history.\n\n Parameters\n ----------\n cmd: dict\n This dict contains information about the command that is to be\n added to the history list. It should contain the keys ``inp``,\n ``rtn`` and ``ts``. These key names mirror the same names defined\n as instance variables in the ``HistoryEntry`` class.\n \"\"\"\n pass\n\n def flush(self, **kwargs):\n \"\"\"Flush the history items to disk from a buffer.\"\"\"\n pass\n\n def items(self, newest_first=False):\n \"\"\"Get history items of current session.\"\"\"\n raise NotImplementedError\n\n def all_items(self, newest_first=False):\n \"\"\"Get all history items.\"\"\"\n raise NotImplementedError\n\n def info(self):\n \"\"\"A collection of information about the shell history.\n\n Returns\n -------\n dict or collections.OrderedDict\n Contains history information as str key pairs.\n \"\"\"\n raise NotImplementedError\n\n def run_gc(self, size=None, blocking=True):\n \"\"\"Run the garbage collector.\n\n Parameters\n ----------\n size: None or tuple of a int and a string\n Determines the size and units of what would be allowed to remain.\n blocking: bool\n If set blocking, then wait until gc action finished.\n \"\"\"\n pass\n" }, { "path": "xonsh/history/dummy.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"Implements the xonsh history backend.\"\"\"\nimport collections\nfrom xonsh.history.base import History\n\n\nclass DummyHistory(History):\n \"\"\"A dummy implement of history backend.\"\"\"\n\n def append(self, cmd):\n pass\n\n def items(self, newest_first=False):\n yield {\"inp\": \"dummy in action\", \"ts\": 1464652800, \"ind\": 0}\n\n def all_items(self, newest_first=False):\n return self.items(newest_first=newest_first)\n\n def info(self):\n data = collections.OrderedDict()\n data[\"backend\"] = \"dummy\"\n data[\"sessionid\"] = str(self.sessionid)\n return data\n" }, { "path": "xonsh/history/json.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"Implements JSON version of xonsh history backend.\"\"\"\nimport os\nimport sys\nimport time\nimport json\nimport builtins\nimport collections\nimport threading\nimport collections.abc as cabc\n\nfrom xonsh.history.base import History\nimport xonsh.tools as xt\nimport xonsh.lazyjson as xlj\nimport xonsh.xoreutils.uptime as uptime\n\n\ndef _xhj_gc_commands_to_rmfiles(hsize, files):\n \"\"\"Return the history files to remove to get under the command limit.\"\"\"\n rmfiles = []\n n = 0\n ncmds = 0\n for ts, fcmds, f in files[::-1]:\n if fcmds == 0:\n # we need to make sure that 'empty' history files don't hang around\n rmfiles.append((ts, fcmds, f))\n if ncmds + fcmds > hsize:\n break\n ncmds += fcmds\n n += 1\n rmfiles += files[:-n]\n return rmfiles\n\n\ndef _xhj_gc_files_to_rmfiles(hsize, files):\n \"\"\"Return the history files to remove to get under the file limit.\"\"\"\n rmfiles = files[:-hsize] if len(files) > hsize else []\n return rmfiles\n\n\ndef _xhj_gc_seconds_to_rmfiles(hsize, files):\n \"\"\"Return the history files to remove to get under the age limit.\"\"\"\n rmfiles = []\n now = time.time()\n for ts, _, f in files:\n if (now - ts) < hsize:\n break\n rmfiles.append((None, None, f))\n return rmfiles\n\n\ndef _xhj_gc_bytes_to_rmfiles(hsize, files):\n \"\"\"Return the history files to remove to get under the byte limit.\"\"\"\n rmfiles = []\n n = 0\n nbytes = 0\n for _, _, f in files[::-1]:\n fsize = os.stat(f).st_size\n if nbytes + fsize > hsize:\n break\n nbytes += fsize\n n += 1\n rmfiles = files[:-n]\n return rmfiles\n\n\ndef _xhj_get_history_files(sort=True, newest_first=False):\n \"\"\"Find and return the history files. Optionally sort files by\n modify time.\n \"\"\"\n data_dir = builtins.__xonsh__.env.get(\"XONSH_DATA_DIR\")\n data_dir = xt.expanduser_abs_path(data_dir)\n try:\n files = [\n os.path.join(data_dir, f)\n for f in os.listdir(data_dir)\n if f.startswith(\"xonsh-\") and f.endswith(\".json\")\n ]\n except OSError:\n files = []\n if builtins.__xonsh__.env.get(\"XONSH_DEBUG\"):\n xt.print_exception(\"Could not collect xonsh history files.\")\n if sort:\n files.sort(key=lambda x: os.path.getmtime(x), reverse=newest_first)\n return files\n\n\nclass JsonHistoryGC(threading.Thread):\n \"\"\"Shell history garbage collection.\"\"\"\n\n def __init__(self, wait_for_shell=True, size=None, *args, **kwargs):\n \"\"\"Thread responsible for garbage collecting old history.\n\n May wait for shell (and for xonshrc to have been loaded) to start work.\n \"\"\"\n super().__init__(*args, **kwargs)\n self.daemon = True\n self.size = size\n self.wait_for_shell = wait_for_shell\n self.start()\n self.gc_units_to_rmfiles = {\n \"commands\": _xhj_gc_commands_to_rmfiles,\n \"files\": _xhj_gc_files_to_rmfiles,\n \"s\": _xhj_gc_seconds_to_rmfiles,\n \"b\": _xhj_gc_bytes_to_rmfiles,\n }\n\n def run(self):\n while self.wait_for_shell:\n time.sleep(0.01)\n env = builtins.__xonsh__.env # pylint: disable=no-member\n if self.size is None:\n hsize, units = env.get(\"XONSH_HISTORY_SIZE\")\n else:\n hsize, units = xt.to_history_tuple(self.size)\n files = self.files(only_unlocked=True)\n rmfiles_fn = self.gc_units_to_rmfiles.get(units)\n if rmfiles_fn is None:\n raise ValueError(\"Units type {0!r} not understood\".format(units))\n\n for _, _, f in rmfiles_fn(hsize, files):\n try:\n os.remove(f)\n except OSError:\n pass\n\n def files(self, only_unlocked=False):\n \"\"\"Find and return the history files. Optionally locked files may be\n excluded.\n\n This is sorted by the last closed time. Returns a list of\n (timestamp, number of cmds, file name) tuples.\n \"\"\"\n # pylint: disable=no-member\n env = getattr(builtins, \"__xonsh__.env\", None)\n if env is None:\n return []\n boot = uptime.boottime()\n fs = _xhj_get_history_files(sort=False)\n files = []\n for f in fs:\n try:\n if os.path.getsize(f) == 0:\n # collect empty files (for gc)\n files.append((time.time(), 0, f))\n continue\n lj = xlj.LazyJSON(f, reopen=False)\n if lj[\"locked\"] and lj[\"ts\"][0] < boot:\n # computer was rebooted between when this history was created\n # and now and so this history should be unlocked.\n hist = lj.load()\n lj.close()\n hist[\"locked\"] = False\n with open(f, \"w\", newline=\"\\n\") as fp:\n xlj.ljdump(hist, fp, sort_keys=True)\n lj = xlj.LazyJSON(f, reopen=False)\n if only_unlocked and lj[\"locked\"]:\n continue\n # info: closing timestamp, number of commands, filename\n files.append((lj[\"ts\"][1] or lj[\"ts\"][0], len(lj.sizes[\"cmds\"]) - 1, f))\n lj.close()\n except (IOError, OSError, ValueError):\n continue\n files.sort()\n return files\n\n\nclass JsonHistoryFlusher(threading.Thread):\n \"\"\"Flush shell history to disk periodically.\"\"\"\n\n def __init__(self, filename, buffer, queue, cond, at_exit=False, *args, **kwargs):\n \"\"\"Thread for flushing history.\"\"\"\n super(JsonHistoryFlusher, self).__init__(*args, **kwargs)\n self.filename = filename\n self.buffer = buffer\n self.queue = queue\n queue.append(self)\n self.cond = cond\n self.at_exit = at_exit\n if at_exit:\n self.dump()\n queue.popleft()\n else:\n self.start()\n\n def run(self):\n with self.cond:\n self.cond.wait_for(self.i_am_at_the_front)\n self.dump()\n self.queue.popleft()\n\n def i_am_at_the_front(self):\n \"\"\"Tests if the flusher is at the front of the queue.\"\"\"\n return self is self.queue[0]\n\n def dump(self):\n \"\"\"Write the cached history to external storage.\"\"\"\n opts = builtins.__xonsh__.env.get(\"HISTCONTROL\")\n last_inp = None\n cmds = []\n for cmd in self.buffer:\n if \"ignoredups\" in opts and cmd[\"inp\"] == last_inp:\n # Skipping dup cmd\n continue\n if \"ignoreerr\" in opts and cmd[\"rtn\"] != 0:\n # Skipping failed cmd\n continue\n cmds.append(cmd)\n last_inp = cmd[\"inp\"]\n with open(self.filename, \"r\", newline=\"\\n\") as f:\n hist = xlj.LazyJSON(f).load()\n load_hist_len = len(hist[\"cmds\"])\n hist[\"cmds\"].extend(cmds)\n if self.at_exit:\n hist[\"ts\"][1] = time.time() # apply end time\n hist[\"locked\"] = False\n if not builtins.__xonsh__.env.get(\"XONSH_STORE_STDOUT\", False):\n [cmd.pop(\"out\") for cmd in hist[\"cmds\"][load_hist_len:] if \"out\" in cmd]\n with open(self.filename, \"w\", newline=\"\\n\") as f:\n xlj.ljdump(hist, f, sort_keys=True)\n\n\nclass JsonCommandField(cabc.Sequence):\n \"\"\"A field in the 'cmds' portion of history.\"\"\"\n\n def __init__(self, field, hist, default=None):\n \"\"\"Represents a field in the 'cmds' portion of history.\n\n Will query the buffer for the relevant data, if possible. Otherwise it\n will lazily acquire data from the file.\n\n Parameters\n ----------\n field : str\n The name of the field to query.\n hist : History object\n The history object to query.\n default : optional\n The default value to return if key is not present.\n \"\"\"\n self.field = field\n self.hist = hist\n self.default = default\n\n def __len__(self):\n return len(self.hist)\n\n def __getitem__(self, key):\n size = len(self)\n if isinstance(key, slice):\n return [self[i] for i in range(*key.indices(size))]\n elif not isinstance(key, int):\n raise IndexError(\"JsonCommandField may only be indexed by int or slice.\")\n elif size == 0:\n raise IndexError(\"JsonCommandField is empty.\")\n # now we know we have an int\n key = size + key if key < 0 else key # ensure key is non-negative\n bufsize = len(self.hist.buffer)\n if size - bufsize <= key: # key is in buffer\n return self.hist.buffer[key + bufsize - size].get(self.field, self.default)\n # now we know we have to go into the file\n queue = self.hist._queue\n queue.append(self)\n with self.hist._cond:\n self.hist._cond.wait_for(self.i_am_at_the_front)\n with open(self.hist.filename, \"r\", newline=\"\\n\") as f:\n lj = xlj.LazyJSON(f, reopen=False)\n rtn = lj[\"cmds\"][key].get(self.field, self.default)\n if isinstance(rtn, xlj.LJNode):\n rtn = rtn.load()\n queue.popleft()\n return rtn\n\n def i_am_at_the_front(self):\n \"\"\"Tests if the command field is at the front of the queue.\"\"\"\n return self is self.hist._queue[0]\n\n\nclass JsonHistory(History):\n \"\"\"Xonsh history backend implemented with JSON files.\n\n JsonHistory implements two extra actions: ``diff``, and ``replay``.\n \"\"\"\n\n def __init__(self, filename=None, sessionid=None, buffersize=100, gc=True, **meta):\n \"\"\"Represents a xonsh session's history as an in-memory buffer that is\n periodically flushed to disk.\n\n Parameters\n ----------\n filename : str, optional\n Location of history file, defaults to\n ``$XONSH_DATA_DIR/xonsh-{sessionid}.json``.\n sessionid : int, uuid, str, optional\n Current session identifier, will generate a new sessionid if not\n set.\n buffersize : int, optional\n Maximum buffersize in memory.\n meta : optional\n Top-level metadata to store along with the history. The kwargs\n 'cmds' and 'sessionid' are not allowed and will be overwritten.\n gc : bool, optional\n Run garbage collector flag.\n \"\"\"\n super().__init__(sessionid=sessionid, **meta)\n if filename is None:\n # pylint: disable=no-member\n data_dir = builtins.__xonsh__.env.get(\"XONSH_DATA_DIR\")\n data_dir = os.path.expanduser(data_dir)\n self.filename = os.path.join(\n data_dir, \"xonsh-{0}.json\".format(self.sessionid)\n )\n else:\n self.filename = filename\n self.buffer = []\n self.buffersize = buffersize\n self._queue = collections.deque()\n self._cond = threading.Condition()\n self._len = 0\n self.last_cmd_out = None\n self.last_cmd_rtn = None\n meta[\"cmds\"] = []\n meta[\"sessionid\"] = str(self.sessionid)\n with open(self.filename, \"w\", newline=\"\\n\") as f:\n xlj.ljdump(meta, f, sort_keys=True)\n self.gc = JsonHistoryGC() if gc else None\n # command fields that are known\n self.tss = JsonCommandField(\"ts\", self)\n self.inps = JsonCommandField(\"inp\", self)\n self.outs = JsonCommandField(\"out\", self)\n self.rtns = JsonCommandField(\"rtn\", self)\n\n def __len__(self):\n return self._len\n\n def append(self, cmd):\n \"\"\"Appends command to history. Will periodically flush the history to file.\n\n Parameters\n ----------\n cmd : dict\n This dict contains information about the command that is to be\n added to the history list. It should contain the keys ``inp``,\n ``rtn`` and ``ts``. These key names mirror the same names defined\n as instance variables in the ``HistoryEntry`` class.\n\n Returns\n -------\n hf : JsonHistoryFlusher or None\n The thread that was spawned to flush history\n \"\"\"\n self.buffer.append(cmd)\n self._len += 1 # must come before flushing\n if len(self.buffer) >= self.buffersize:\n hf = self.flush()\n else:\n hf = None\n return hf\n\n def flush(self, at_exit=False):\n \"\"\"Flushes the current command buffer to disk.\n\n Parameters\n ----------\n at_exit : bool, optional\n Whether the JsonHistoryFlusher should act as a thread in the\n background, or execute immediately and block.\n\n Returns\n -------\n hf : JsonHistoryFlusher or None\n The thread that was spawned to flush history\n \"\"\"\n if len(self.buffer) == 0:\n return\n hf = JsonHistoryFlusher(\n self.filename, tuple(self.buffer), self._queue, self._cond, at_exit=at_exit\n )\n self.buffer.clear()\n return hf\n\n def items(self, newest_first=False):\n \"\"\"Display history items of current session.\"\"\"\n if newest_first:\n items = zip(reversed(self.inps), reversed(self.tss))\n else:\n items = zip(self.inps, self.tss)\n for item, tss in items:\n yield {\"inp\": item.rstrip(), \"ts\": tss[0]}\n\n def all_items(self, newest_first=False, **kwargs):\n \"\"\"\n Returns all history as found in XONSH_DATA_DIR.\n\n yield format: {'inp': cmd, 'rtn': 0, ...}\n \"\"\"\n while self.gc and self.gc.is_alive():\n time.sleep(0.011) # gc sleeps for 0.01 secs, sleep a beat longer\n for f in _xhj_get_history_files(newest_first=newest_first):\n try:\n json_file = xlj.LazyJSON(f, reopen=False)\n except ValueError:\n # Invalid json file\n continue\n try:\n commands = json_file.load()[\"cmds\"]\n except json.decoder.JSONDecodeError:\n # file is corrupted somehow\n if builtins.__xonsh__.env.get(\"XONSH_DEBUG\") > 0:\n msg = \"xonsh history file {0!r} is not valid JSON\"\n print(msg.format(f), file=sys.stderr)\n continue\n if newest_first:\n commands = reversed(commands)\n for c in commands:\n yield {\"inp\": c[\"inp\"].rstrip(), \"ts\": c[\"ts\"][0]}\n # all items should also include session items\n yield from self.items()\n\n def info(self):\n data = collections.OrderedDict()\n data[\"backend\"] = \"json\"\n data[\"sessionid\"] = str(self.sessionid)\n data[\"filename\"] = self.filename\n data[\"length\"] = len(self)\n data[\"buffersize\"] = self.buffersize\n data[\"bufferlength\"] = len(self.buffer)\n envs = builtins.__xonsh__.env\n data[\"gc options\"] = envs.get(\"XONSH_HISTORY_SIZE\")\n return data\n\n def run_gc(self, size=None, blocking=True):\n self.gc = JsonHistoryGC(wait_for_shell=False, size=size)\n if blocking:\n while self.gc.is_alive():\n continue\n" }, { "path": "xonsh/history/main.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"Main entry points of the xonsh history.\"\"\"\nimport argparse\nimport builtins\nimport datetime\nimport functools\nimport json\nimport os\nimport sys\n\nfrom xonsh.history.base import History\nfrom xonsh.history.dummy import DummyHistory\nfrom xonsh.history.json import JsonHistory\nfrom xonsh.history.sqlite import SqliteHistory\nimport xonsh.diff_history as xdh\nimport xonsh.lazyasd as xla\nimport xonsh.tools as xt\n\nHISTORY_BACKENDS = {\"dummy\": DummyHistory, \"json\": JsonHistory, \"sqlite\": SqliteHistory}\n\n\ndef construct_history(**kwargs):\n \"\"\"Construct the history backend object.\"\"\"\n env = builtins.__xonsh__.env\n backend = env.get(\"XONSH_HISTORY_BACKEND\")\n if isinstance(backend, str) and backend in HISTORY_BACKENDS:\n kls_history = HISTORY_BACKENDS[backend]\n elif xt.is_class(backend):\n kls_history = backend\n elif isinstance(backend, History):\n return backend\n else:\n print(\n \"Unknown history backend: {}. Using JSON version\".format(backend),\n file=sys.stderr,\n )\n kls_history = JsonHistory\n return kls_history(**kwargs)\n\n\ndef _xh_session_parser(hist=None, newest_first=False, **kwargs):\n \"\"\"Returns history items of current session.\"\"\"\n if hist is None:\n hist = builtins.__xonsh__.history\n return hist.items()\n\n\ndef _xh_all_parser(hist=None, newest_first=False, **kwargs):\n \"\"\"Returns all history items.\"\"\"\n if hist is None:\n hist = builtins.__xonsh__.history\n return hist.all_items(newest_first=newest_first)\n\n\ndef _xh_find_histfile_var(file_list, default=None):\n \"\"\"Return the path of the history file\n from the value of the envvar HISTFILE.\n \"\"\"\n for f in file_list:\n f = xt.expanduser_abs_path(f)\n if not os.path.isfile(f):\n continue\n with open(f, \"r\") as rc_file:\n for line in rc_file:\n if line.startswith(\"HISTFILE=\"):\n hist_file = line.split(\"=\", 1)[1].strip(\"'\\\"\\n\")\n hist_file = xt.expanduser_abs_path(hist_file)\n if os.path.isfile(hist_file):\n return hist_file\n else:\n if default:\n default = xt.expanduser_abs_path(default)\n if os.path.isfile(default):\n return default\n\n\ndef _xh_bash_hist_parser(location=None, **kwargs):\n \"\"\"Yield commands from bash history file\"\"\"\n if location is None:\n location = _xh_find_histfile_var(\n [os.path.join(\"~\", \".bashrc\"), os.path.join(\"~\", \".bash_profile\")],\n os.path.join(\"~\", \".bash_history\"),\n )\n if location:\n with open(location, \"r\", errors=\"backslashreplace\") as bash_hist:\n for ind, line in enumerate(bash_hist):\n yield {\"inp\": line.rstrip(), \"ts\": 0.0, \"ind\": ind}\n else:\n print(\"No bash history file\", file=sys.stderr)\n\n\ndef _xh_zsh_hist_parser(location=None, **kwargs):\n \"\"\"Yield commands from zsh history file\"\"\"\n if location is None:\n location = _xh_find_histfile_var(\n [os.path.join(\"~\", \".zshrc\"), os.path.join(\"~\", \".zprofile\")],\n os.path.join(\"~\", \".zsh_history\"),\n )\n if location:\n with open(location, \"r\", errors=\"backslashreplace\") as zsh_hist:\n for ind, line in enumerate(zsh_hist):\n if line.startswith(\":\"):\n try:\n start_time, command = line.split(\";\", 1)\n except ValueError:\n # Invalid history entry\n continue\n try:\n start_time = float(start_time.split(\":\")[1])\n except ValueError:\n start_time = 0.0\n yield {\"inp\": command.rstrip(), \"ts\": start_time, \"ind\": ind}\n else:\n yield {\"inp\": line.rstrip(), \"ts\": 0.0, \"ind\": ind}\n\n else:\n print(\"No zsh history file found\", file=sys.stderr)\n\n\ndef _xh_filter_ts(commands, start_time, end_time):\n \"\"\"Yield only the commands between start and end time.\"\"\"\n for cmd in commands:\n if start_time <= cmd[\"ts\"] < end_time:\n yield cmd\n\n\ndef _xh_get_history(\n session=\"session\",\n *,\n slices=None,\n datetime_format=None,\n start_time=None,\n end_time=None,\n location=None\n):\n \"\"\"Get the requested portion of shell history.\n\n Parameters\n ----------\n session: {'session', 'all', 'xonsh', 'bash', 'zsh'}\n The history session to get.\n slices : list of slice-like objects, optional\n Get only portions of history.\n start_time, end_time: float, optional\n Filter commands by timestamp.\n location: string, optional\n The history file location (bash or zsh)\n\n Returns\n -------\n generator\n A filtered list of commands\n \"\"\"\n cmds = []\n for i, item in enumerate(_XH_HISTORY_SESSIONS[session](location=location)):\n item[\"ind\"] = i\n cmds.append(item)\n if slices:\n # transform/check all slices\n slices = [xt.ensure_slice(s) for s in slices]\n cmds = xt.get_portions(cmds, slices)\n if start_time or end_time:\n if start_time is None:\n start_time = 0.0\n else:\n start_time = xt.ensure_timestamp(start_time, datetime_format)\n if end_time is None:\n end_time = float(\"inf\")\n else:\n end_time = xt.ensure_timestamp(end_time, datetime_format)\n cmds = _xh_filter_ts(cmds, start_time, end_time)\n return cmds\n\n\ndef _xh_show_history(hist, ns, stdout=None, stderr=None):\n \"\"\"Show the requested portion of shell history.\n Accepts same parameters with `_xh_get_history`.\n \"\"\"\n try:\n commands = _xh_get_history(\n ns.session,\n slices=ns.slices,\n start_time=ns.start_time,\n end_time=ns.end_time,\n datetime_format=ns.datetime_format,\n )\n except Exception as err:\n print(\"history: error: {}\".format(err), file=stderr)\n return\n if ns.reverse:\n commands = reversed(list(commands))\n end = \"\\0\" if ns.null_byte else \"\\n\"\n if ns.numerate and ns.timestamp:\n for c in commands:\n dt = datetime.datetime.fromtimestamp(c[\"ts\"])\n print(\n \"{}:({}) {}\".format(c[\"ind\"], xt.format_datetime(dt), c[\"inp\"]),\n file=stdout,\n end=end,\n )\n elif ns.numerate:\n for c in commands:\n print(\"{}: {}\".format(c[\"ind\"], c[\"inp\"]), file=stdout, end=end)\n elif ns.timestamp:\n for c in commands:\n dt = datetime.datetime.fromtimestamp(c[\"ts\"])\n print(\n \"({}) {}\".format(xt.format_datetime(dt), c[\"inp\"]), file=stdout, end=end\n )\n else:\n for c in commands:\n print(c[\"inp\"], file=stdout, end=end)\n\n\n@xla.lazyobject\ndef _XH_HISTORY_SESSIONS():\n return {\n \"session\": _xh_session_parser,\n \"xonsh\": _xh_all_parser,\n \"all\": _xh_all_parser,\n \"zsh\": _xh_zsh_hist_parser,\n \"bash\": _xh_bash_hist_parser,\n }\n\n\n_XH_MAIN_ACTIONS = {\"show\", \"id\", \"file\", \"info\", \"diff\", \"gc\"}\n\n\n@functools.lru_cache()\ndef _xh_create_parser():\n \"\"\"Create a parser for the \"history\" command.\"\"\"\n p = argparse.ArgumentParser(\n prog=\"history\", description=\"try 'history --help' \" \"for more info\"\n )\n subp = p.add_subparsers(title=\"commands\", dest=\"action\")\n # session action\n show = subp.add_parser(\n \"show\", prefix_chars=\"-+\", help=\"display history of a session, default command\"\n )\n show.add_argument(\n \"-r\",\n dest=\"reverse\",\n default=False,\n action=\"store_true\",\n help=\"reverses the direction\",\n )\n show.add_argument(\n \"-n\",\n dest=\"numerate\",\n default=False,\n action=\"store_true\",\n help=\"numerate each command\",\n )\n show.add_argument(\n \"-t\",\n dest=\"timestamp\",\n default=False,\n action=\"store_true\",\n help=\"show command timestamps\",\n )\n show.add_argument(\n \"-T\", dest=\"end_time\", default=None, help=\"show only commands before timestamp\"\n )\n show.add_argument(\n \"+T\", dest=\"start_time\", default=None, help=\"show only commands after timestamp\"\n )\n show.add_argument(\n \"-f\",\n dest=\"datetime_format\",\n default=None,\n help=\"the datetime format to be used for\" \"filtering and printing\",\n )\n show.add_argument(\n \"-0\",\n dest=\"null_byte\",\n default=False,\n action=\"store_true\",\n help=\"separate commands by the null character for piping \"\n \"history to external filters\",\n )\n show.add_argument(\n \"session\",\n nargs=\"?\",\n choices=_XH_HISTORY_SESSIONS.keys(),\n default=\"session\",\n metavar=\"session\",\n help=\"{} (default: current session, all is an alias for xonsh)\"\n \"\".format(\", \".join(map(repr, _XH_HISTORY_SESSIONS.keys()))),\n )\n show.add_argument(\n \"slices\",\n nargs=\"*\",\n default=None,\n metavar=\"slice\",\n help=\"integer or slice notation\",\n )\n # 'id' subcommand\n subp.add_parser(\"id\", help=\"display the current session id\")\n # 'file' subcommand\n subp.add_parser(\"file\", help=\"display the current history filename\")\n # 'info' subcommand\n info = subp.add_parser(\n \"info\", help=(\"display information about the \" \"current history\")\n )\n info.add_argument(\n \"--json\",\n dest=\"json\",\n default=False,\n action=\"store_true\",\n help=\"print in JSON format\",\n )\n\n # gc\n gcp = subp.add_parser(\"gc\", help=\"launches a new history garbage collector\")\n gcp.add_argument(\n \"--size\",\n nargs=2,\n dest=\"size\",\n default=None,\n help=(\n \"next two arguments represent the history size and \"\n 'units; e.g. \"--size 8128 commands\"'\n ),\n )\n bgcp = gcp.add_mutually_exclusive_group()\n bgcp.add_argument(\n \"--blocking\",\n dest=\"blocking\",\n default=True,\n action=\"store_true\",\n help=(\"ensures that the gc blocks the main thread, \" \"default True\"),\n )\n bgcp.add_argument(\n \"--non-blocking\",\n dest=\"blocking\",\n action=\"store_false\",\n help=\"makes the gc non-blocking, and thus return sooner\",\n )\n\n hist = builtins.__xonsh__.history\n if isinstance(hist, JsonHistory):\n # add actions belong only to JsonHistory\n diff = subp.add_parser(\"diff\", help=\"diff two xonsh history files\")\n xdh.dh_create_parser(p=diff)\n\n import xonsh.replay as xrp\n\n replay = subp.add_parser(\"replay\", help=\"replay a xonsh history file\")\n xrp.replay_create_parser(p=replay)\n _XH_MAIN_ACTIONS.add(\"replay\")\n\n return p\n\n\ndef _xh_parse_args(args):\n \"\"\"Prepare and parse arguments for the history command.\n\n Add default action for ``history`` and\n default session for ``history show``.\n \"\"\"\n parser = _xh_create_parser()\n if not args:\n args = [\"show\", \"session\"]\n elif args[0] not in _XH_MAIN_ACTIONS and args[0] not in (\"-h\", \"--help\"):\n args = [\"show\", \"session\"] + args\n if args[0] == \"show\":\n if not any(a in _XH_HISTORY_SESSIONS for a in args):\n args.insert(1, \"session\")\n ns, slices = parser.parse_known_args(args)\n if slices:\n if not ns.slices:\n ns.slices = slices\n else:\n ns.slices.extend(slices)\n else:\n ns = parser.parse_args(args)\n return ns\n\n\ndef history_main(\n args=None, stdin=None, stdout=None, stderr=None, spec=None, stack=None\n):\n \"\"\"This is the history command entry point.\"\"\"\n hist = builtins.__xonsh__.history\n ns = _xh_parse_args(args)\n if not ns or not ns.action:\n return\n if ns.action == \"show\":\n _xh_show_history(hist, ns, stdout=stdout, stderr=stderr)\n elif ns.action == \"info\":\n data = hist.info()\n if ns.json:\n s = json.dumps(data)\n print(s, file=stdout)\n else:\n lines = [\"{0}: {1}\".format(k, v) for k, v in data.items()]\n print(\"\\n\".join(lines), file=stdout)\n elif ns.action == \"id\":\n if not hist.sessionid:\n return\n print(str(hist.sessionid), file=stdout)\n elif ns.action == \"file\":\n if not hist.filename:\n return\n print(str(hist.filename), file=stdout)\n elif ns.action == \"gc\":\n hist.run_gc(size=ns.size, blocking=ns.blocking)\n elif ns.action == \"diff\":\n if isinstance(hist, JsonHistory):\n xdh.dh_main_action(ns)\n elif ns.action == \"replay\":\n if isinstance(hist, JsonHistory):\n import xonsh.replay as xrp\n\n xrp.replay_main_action(hist, ns, stdout=stdout, stderr=stderr)\n else:\n print(\"Unknown history action {}\".format(ns.action), file=sys.stderr)\n" }, { "path": "xonsh/history/sqlite.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"Implements the xonsh history backend via sqlite3.\"\"\"\nimport builtins\nimport collections\nimport json\nimport os\nimport sqlite3\nimport sys\nimport threading\nimport time\n\nfrom xonsh.history.base import History\nimport xonsh.tools as xt\n\n\ndef _xh_sqlite_get_file_name():\n envs = builtins.__xonsh__.env\n file_name = envs.get(\"XONSH_HISTORY_SQLITE_FILE\")\n if not file_name:\n data_dir = envs.get(\"XONSH_DATA_DIR\")\n file_name = os.path.join(data_dir, \"xonsh-history.sqlite\")\n return xt.expanduser_abs_path(file_name)\n\n\ndef _xh_sqlite_get_conn(filename=None):\n if filename is None:\n filename = _xh_sqlite_get_file_name()\n return sqlite3.connect(filename)\n\n\ndef _xh_sqlite_create_history_table(cursor):\n \"\"\"Create Table for history items.\n\n Columns:\n info - JSON formatted, reserved for future extension.\n \"\"\"\n cursor.execute(\n \"\"\"\n CREATE TABLE IF NOT EXISTS xonsh_history\n (inp TEXT,\n rtn INTEGER,\n tsb REAL,\n tse REAL,\n sessionid TEXT,\n out TEXT,\n info TEXT\n )\n \"\"\"\n )\n\n\ndef _xh_sqlite_insert_command(cursor, cmd, sessionid, store_stdout):\n sql = \"INSERT INTO xonsh_history (inp, rtn, tsb, tse, sessionid\"\n tss = cmd.get(\"ts\", [None, None])\n params = [cmd[\"inp\"].rstrip(), cmd[\"rtn\"], tss[0], tss[1], sessionid]\n if store_stdout and \"out\" in cmd:\n sql += \", out\"\n params.append(cmd[\"out\"])\n if \"info\" in cmd:\n sql += \", info\"\n info = json.dumps(cmd[\"info\"])\n params.append(info)\n sql += \") VALUES (\" + (\"?, \" * len(params)).rstrip(\", \") + \")\"\n cursor.execute(sql, tuple(params))\n\n\ndef _xh_sqlite_get_count(cursor, sessionid=None):\n sql = \"SELECT count(*) FROM xonsh_history \"\n params = []\n if sessionid is not None:\n sql += \"WHERE sessionid = ? \"\n params.append(str(sessionid))\n cursor.execute(sql, tuple(params))\n return cursor.fetchone()[0]\n\n\ndef _xh_sqlite_get_records(cursor, sessionid=None, limit=None, newest_first=False):\n sql = \"SELECT inp, tsb, rtn FROM xonsh_history \"\n params = []\n if sessionid is not None:\n sql += \"WHERE sessionid = ? \"\n params.append(sessionid)\n sql += \"ORDER BY tsb \"\n if newest_first:\n sql += \"DESC \"\n if limit is not None:\n sql += \"LIMIT %d \" % limit\n cursor.execute(sql, tuple(params))\n return cursor.fetchall()\n\n\ndef _xh_sqlite_delete_records(cursor, size_to_keep):\n sql = \"SELECT min(tsb) FROM (\"\n sql += \"SELECT tsb FROM xonsh_history ORDER BY tsb DESC \"\n sql += \"LIMIT %d)\" % size_to_keep\n cursor.execute(sql)\n result = cursor.fetchone()\n if not result:\n return\n max_tsb = result[0]\n sql = \"DELETE FROM xonsh_history WHERE tsb < ?\"\n result = cursor.execute(sql, (max_tsb,))\n return result.rowcount\n\n\ndef xh_sqlite_append_history(cmd, sessionid, store_stdout, filename=None):\n with _xh_sqlite_get_conn(filename=filename) as conn:\n c = conn.cursor()\n _xh_sqlite_create_history_table(c)\n _xh_sqlite_insert_command(c, cmd, sessionid, store_stdout)\n conn.commit()\n\n\ndef xh_sqlite_get_count(sessionid=None, filename=None):\n with _xh_sqlite_get_conn(filename=filename) as conn:\n c = conn.cursor()\n return _xh_sqlite_get_count(c, sessionid=sessionid)\n\n\ndef xh_sqlite_items(sessionid=None, filename=None, newest_first=False):\n with _xh_sqlite_get_conn(filename=filename) as conn:\n c = conn.cursor()\n _xh_sqlite_create_history_table(c)\n return _xh_sqlite_get_records(c, sessionid=sessionid, newest_first=newest_first)\n\n\ndef xh_sqlite_delete_items(size_to_keep, filename=None):\n with _xh_sqlite_get_conn(filename=filename) as conn:\n c = conn.cursor()\n _xh_sqlite_create_history_table(c)\n return _xh_sqlite_delete_records(c, size_to_keep)\n\n\nclass SqliteHistoryGC(threading.Thread):\n \"\"\"Shell history garbage collection.\"\"\"\n\n def __init__(self, wait_for_shell=True, size=None, filename=None, *args, **kwargs):\n \"\"\"Thread responsible for garbage collecting old history.\n\n May wait for shell (and for xonshrc to have been loaded) to start work.\n \"\"\"\n super().__init__(*args, **kwargs)\n self.daemon = True\n self.filename = filename\n self.size = size\n self.wait_for_shell = wait_for_shell\n self.start()\n\n def run(self):\n while self.wait_for_shell:\n time.sleep(0.01)\n if self.size is not None:\n hsize, units = xt.to_history_tuple(self.size)\n else:\n envs = builtins.__xonsh__.env\n hsize, units = envs.get(\"XONSH_HISTORY_SIZE\")\n if units != \"commands\":\n print(\n \"sqlite backed history gc currently only supports \"\n '\"commands\" as units',\n file=sys.stderr,\n )\n return\n if hsize < 0:\n return\n xh_sqlite_delete_items(hsize, filename=self.filename)\n\n\nclass SqliteHistory(History):\n \"\"\"Xonsh history backend implemented with sqlite3.\"\"\"\n\n def __init__(self, gc=True, filename=None, **kwargs):\n super().__init__(**kwargs)\n if filename is None:\n filename = _xh_sqlite_get_file_name()\n self.filename = filename\n self.gc = SqliteHistoryGC() if gc else None\n self._last_hist_inp = None\n self.inps = []\n self.rtns = []\n self.outs = []\n self.tss = []\n\n def append(self, cmd):\n envs = builtins.__xonsh__.env\n opts = envs.get(\"HISTCONTROL\")\n inp = cmd[\"inp\"].rstrip()\n self.inps.append(inp)\n store_stdout = envs.get(\"XONSH_STORE_STDOUT\", False)\n if store_stdout:\n self.outs.append(cmd.get(\"out\"))\n else:\n self.outs.append(None)\n self.rtns.append(cmd[\"rtn\"])\n self.tss.append(cmd.get(\"ts\", (None, None)))\n\n opts = envs.get(\"HISTCONTROL\")\n if \"ignoredups\" in opts and inp == self._last_hist_inp:\n # Skipping dup cmd\n return\n if \"ignoreerr\" in opts and cmd[\"rtn\"] != 0:\n # Skipping failed cmd\n return\n self._last_hist_inp = inp\n xh_sqlite_append_history(\n cmd, str(self.sessionid), store_stdout, filename=self.filename\n )\n\n def all_items(self, newest_first=False):\n \"\"\"Display all history items.\"\"\"\n for item in xh_sqlite_items(filename=self.filename, newest_first=newest_first):\n yield {\"inp\": item[0], \"ts\": item[1], \"rtn\": item[2]}\n\n def items(self, newest_first=False):\n \"\"\"Display history items of current session.\"\"\"\n for item in xh_sqlite_items(\n sessionid=str(self.sessionid),\n filename=self.filename,\n newest_first=newest_first,\n ):\n yield {\"inp\": item[0], \"ts\": item[1], \"rtn\": item[2]}\n\n def info(self):\n data = collections.OrderedDict()\n data[\"backend\"] = \"sqlite\"\n data[\"sessionid\"] = str(self.sessionid)\n data[\"filename\"] = self.filename\n data[\"session items\"] = xh_sqlite_get_count(\n sessionid=self.sessionid, filename=self.filename\n )\n data[\"all items\"] = xh_sqlite_get_count(filename=self.filename)\n envs = builtins.__xonsh__.env\n data[\"gc options\"] = envs.get(\"XONSH_HISTORY_SIZE\")\n return data\n\n def run_gc(self, size=None, blocking=True):\n self.gc = SqliteHistoryGC(wait_for_shell=False, size=size)\n if blocking:\n while self.gc.is_alive():\n continue\n" }, { "path": "xonsh/imphooks.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"Import hooks for importing xonsh source files.\n\nThis module registers the hooks it defines when it is imported.\n\"\"\"\nimport os\nimport re\nimport sys\nimport types\nimport builtins\nimport contextlib\nimport importlib\nfrom importlib.machinery import ModuleSpec\nfrom importlib.abc import MetaPathFinder, SourceLoader, Loader\n\nfrom xonsh.events import events\nfrom xonsh.execer import Execer\nfrom xonsh.platform import scandir\nfrom xonsh.lazyasd import lazyobject\n\n\n@lazyobject\ndef ENCODING_LINE():\n # this regex comes from PEP 263\n # https://www.python.org/dev/peps/pep-0263/#defining-the-encoding\n return re.compile(b\"^[ tv]*#.*?coding[:=][ t]*([-_.a-zA-Z0-9]+)\")\n\n\ndef find_source_encoding(src):\n \"\"\"Finds the source encoding given bytes representing a file. If\n no encoding is found, UTF-8 will be returned as per the docs\n https://docs.python.org/3/howto/unicode.html#unicode-literals-in-python-source-code\n \"\"\"\n utf8 = \"UTF-8\"\n first, _, rest = src.partition(b\"\\n\")\n m = ENCODING_LINE.match(first)\n if m is not None:\n return m.group(1).decode(utf8)\n second, _, _ = rest.partition(b\"\\n\")\n m = ENCODING_LINE.match(second)\n if m is not None:\n return m.group(1).decode(utf8)\n return utf8\n\n\nclass XonshImportHook(MetaPathFinder, SourceLoader):\n \"\"\"Implements the import hook for xonsh source files.\"\"\"\n\n def __init__(self, *args, **kwargs):\n super(XonshImportHook, self).__init__(*args, **kwargs)\n self._filenames = {}\n self._execer = None\n\n @property\n def execer(self):\n if (\n hasattr(builtins, \"__xonsh__\")\n and hasattr(builtins.__xonsh__, \"execer\")\n and builtins.__xonsh__.execer is not None\n ):\n execer = builtins.__xonsh__.execer\n if self._execer is not None:\n self._execer = None\n elif self._execer is None:\n self._execer = execer = Execer(unload=False)\n else:\n execer = self._execer\n return execer\n\n #\n # MetaPathFinder methods\n #\n def find_spec(self, fullname, path, target=None):\n \"\"\"Finds the spec for a xonsh module if it exists.\"\"\"\n dot = \".\"\n spec = None\n path = sys.path if path is None else path\n if dot not in fullname and dot not in path:\n path = [dot] + path\n name = fullname.rsplit(dot, 1)[-1]\n fname = name + \".xsh\"\n for p in path:\n if not isinstance(p, str):\n continue\n if not os.path.isdir(p) or not os.access(p, os.R_OK):\n continue\n if fname not in {x.name for x in scandir(p)}:\n continue\n spec = ModuleSpec(fullname, self)\n self._filenames[fullname] = os.path.join(p, fname)\n break\n return spec\n\n #\n # SourceLoader methods\n #\n def create_module(self, spec):\n \"\"\"Create a xonsh module with the appropriate attributes.\"\"\"\n mod = types.ModuleType(spec.name)\n mod.__file__ = self.get_filename(spec.name)\n mod.__loader__ = self\n mod.__package__ = spec.parent or \"\"\n return mod\n\n def get_filename(self, fullname):\n \"\"\"Returns the filename for a module's fullname.\"\"\"\n return self._filenames[fullname]\n\n def get_data(self, path):\n \"\"\"Gets the bytes for a path.\"\"\"\n raise NotImplementedError\n\n def get_code(self, fullname):\n \"\"\"Gets the code object for a xonsh file.\"\"\"\n filename = self.get_filename(fullname)\n if filename is None:\n msg = \"xonsh file {0!r} could not be found\".format(fullname)\n raise ImportError(msg)\n with open(filename, \"rb\") as f:\n src = f.read()\n enc = find_source_encoding(src)\n src = src.decode(encoding=enc)\n src = src if src.endswith(\"\\n\") else src + \"\\n\"\n execer = self.execer\n execer.filename = filename\n ctx = {} # dummy for modules\n code = execer.compile(src, glbs=ctx, locs=ctx)\n return code\n\n\n#\n# Import events\n#\nevents.doc(\n \"on_import_pre_find_spec\",\n \"\"\"\non_import_pre_find_spec(fullname: str, path: str, target: module or None) -> None\n\nFires before any import find_spec() calls have been executed. The parameters\nhere are the same as importlib.abc.MetaPathFinder.find_spec(). Namely,\n\n:``fullname``: The full name of the module to import.\n:``path``: None if a top-level import, otherwise the ``__path__`` of the parent\n package.\n:``target``: Target module used to make a better guess about the package spec.\n\"\"\",\n)\n\nevents.doc(\n \"on_import_post_find_spec\",\n \"\"\"\non_import_post_find_spec(spec, fullname, path, target) -> None\n\nFires after all import find_spec() calls have been executed. The parameters\nhere the spec and the arguments importlib.abc.MetaPathFinder.find_spec(). Namely,\n\n:``spec``: A ModuleSpec object if the spec was found, or None if it was not.\n:``fullname``: The full name of the module to import.\n:``path``: None if a top-level import, otherwise the ``__path__`` of the parent\n package.\n:``target``: Target module used to make a better guess about the package spec.\n\"\"\",\n)\n\nevents.doc(\n \"on_import_pre_create_module\",\n \"\"\"\non_import_pre_create_module(spec: ModuleSpec) -> None\n\nFires right before a module is created by its loader. The only parameter\nis the spec object. See importlib for more details.\n\"\"\",\n)\n\nevents.doc(\n \"on_import_post_create_module\",\n \"\"\"\non_import_post_create_module(module: Module, spec: ModuleSpec) -> None\n\nFires after a module is created by its loader but before the loader returns it.\nThe parameters here are the module object itself and the spec object.\nSee importlib for more details.\n\"\"\",\n)\n\nevents.doc(\n \"on_import_pre_exec_module\",\n \"\"\"\non_import_pre_exec_module(module: Module) -> None\n\nFires right before a module is executed by its loader. The only parameter\nis the module itself. See importlib for more details.\n\"\"\",\n)\n\nevents.doc(\n \"on_import_post_exec_module\",\n \"\"\"\non_import_post_create_module(module: Module) -> None\n\nFires after a module is executed by its loader but before the loader returns it.\nThe only parameter is the module itself. See importlib for more details.\n\"\"\",\n)\n\n\ndef _should_dispatch_xonsh_import_event_loader():\n \"\"\"Figures out if we should dispatch to a load event\"\"\"\n return (\n len(events.on_import_pre_create_module) > 0\n or len(events.on_import_post_create_module) > 0\n or len(events.on_import_pre_exec_module) > 0\n or len(events.on_import_post_exec_module) > 0\n )\n\n\nclass XonshImportEventHook(MetaPathFinder):\n \"\"\"Implements the import hook for firing xonsh events on import.\"\"\"\n\n def __init__(self, *args, **kwargs):\n super().__init__(*args, **kwargs)\n self._fullname_stack = []\n\n @contextlib.contextmanager\n def append_stack(self, fullname):\n \"\"\"A context manager for appending and then removing a name from the\n fullname stack.\n \"\"\"\n self._fullname_stack.append(fullname)\n yield\n del self._fullname_stack[-1]\n\n #\n # MetaPathFinder methods\n #\n def find_spec(self, fullname, path, target=None):\n \"\"\"Finds the spec for a xonsh module if it exists.\"\"\"\n if fullname in reversed(self._fullname_stack):\n # don't execute if we are already in the stack.\n return None\n npre = len(events.on_import_pre_find_spec)\n npost = len(events.on_import_post_find_spec)\n dispatch_load = _should_dispatch_xonsh_import_event_loader()\n if npre > 0:\n events.on_import_pre_find_spec.fire(\n fullname=fullname, path=path, target=target\n )\n elif npost == 0 and not dispatch_load:\n # no events to fire, proceed normally and prevent recursion\n return None\n # now find the spec\n with self.append_stack(fullname):\n spec = importlib.util.find_spec(fullname)\n # fire post event\n if npost > 0:\n events.on_import_post_find_spec.fire(\n spec=spec, fullname=fullname, path=path, target=target\n )\n if dispatch_load and spec is not None and hasattr(spec.loader, \"create_module\"):\n spec.loader = XonshImportEventLoader(spec.loader)\n return spec\n\n\nclass XonshImportEventLoader(Loader):\n \"\"\"A class that dispatches loader calls to another loader and fires relevant\n xonsh events.\n \"\"\"\n\n def __init__(self, loader):\n self.loader = loader\n\n #\n # Loader methods\n #\n def create_module(self, spec):\n \"\"\"Creates and returns the module object.\"\"\"\n events.on_import_pre_create_module.fire(spec=spec)\n mod = self.loader.create_module(spec)\n events.on_import_post_create_module.fire(module=mod, spec=spec)\n return mod\n\n def exec_module(self, module):\n \"\"\"Executes the module in its own namespace.\"\"\"\n events.on_import_pre_exec_module.fire(module=module)\n rtn = self.loader.exec_module(module)\n events.on_import_post_exec_module.fire(module=module)\n return rtn\n\n def load_module(self, fullname):\n \"\"\"Legacy module loading, provided for backwards compatibility.\"\"\"\n return self.loader.load_module(fullname)\n\n def module_repr(self, module):\n \"\"\"Legacy module repr, provided for backwards compatibility.\"\"\"\n return self.loader.module_repr(module)\n\n\ndef install_import_hooks():\n \"\"\"\n Install Xonsh import hooks in ``sys.meta_path`` in order for ``.xsh`` files\n to be importable and import events to be fired.\n\n Can safely be called many times, will be no-op if xonsh import hooks are\n already present.\n \"\"\"\n found_imp = found_event = False\n for hook in sys.meta_path:\n if isinstance(hook, XonshImportHook):\n found_imp = True\n elif isinstance(hook, XonshImportEventHook):\n found_event = True\n if not found_imp:\n sys.meta_path.append(XonshImportHook())\n if not found_event:\n sys.meta_path.insert(0, XonshImportEventHook())\n\n\n# alias to deprecated name\ninstall_hook = install_import_hooks\n" }, { "path": "xonsh/inspectors.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"Tools for inspecting Python objects.\n\nThis file was forked from the IPython project:\n\n* Copyright (c) 2008-2014, IPython Development Team\n* Copyright (C) 2001-2007 Fernando Perez \n* Copyright (c) 2001, Janko Hauser \n* Copyright (c) 2001, Nathaniel Gray \n\"\"\"\nimport os\nimport io\nimport sys\nimport types\nimport inspect\nimport itertools\nimport linecache\nimport collections\n\nfrom xonsh.lazyasd import LazyObject\nfrom xonsh.tokenize import detect_encoding\nfrom xonsh.openpy import read_py_file\nfrom xonsh.tools import cast_unicode, safe_hasattr, indent, print_color, format_color\nfrom xonsh.platform import HAS_PYGMENTS, PYTHON_VERSION_INFO\nfrom xonsh.lazyimps import pygments, pyghooks\nfrom xonsh.style_tools import partial_color_tokenize\n\n\n# builtin docstrings to ignore\n_func_call_docstring = LazyObject(\n lambda: types.FunctionType.__call__.__doc__, globals(), \"_func_call_docstring\"\n)\n_object_init_docstring = LazyObject(\n lambda: object.__init__.__doc__, globals(), \"_object_init_docstring\"\n)\n_builtin_type_docstrings = LazyObject(\n lambda: {\n t.__doc__ for t in (types.ModuleType, types.MethodType, types.FunctionType)\n },\n globals(),\n \"_builtin_type_docstrings\",\n)\n\n_builtin_func_type = LazyObject(lambda: type(all), globals(), \"_builtin_func_type\")\n# Bound methods have the same type as builtin functions\n_builtin_meth_type = LazyObject(\n lambda: type(str.upper), globals(), \"_builtin_meth_type\"\n)\n\ninfo_fields = LazyObject(\n lambda: [\n \"type_name\",\n \"base_class\",\n \"string_form\",\n \"namespace\",\n \"length\",\n \"file\",\n \"definition\",\n \"docstring\",\n \"source\",\n \"init_definition\",\n \"class_docstring\",\n \"init_docstring\",\n \"call_def\",\n \"call_docstring\",\n # These won't be printed but will be used to determine how to\n # format the object\n \"ismagic\",\n \"isalias\",\n \"isclass\",\n \"argspec\",\n \"found\",\n \"name\",\n ],\n globals(),\n \"info_fields\",\n)\n\n\ndef object_info(**kw):\n \"\"\"Make an object info dict with all fields present.\"\"\"\n infodict = dict(itertools.zip_longest(info_fields, [None]))\n infodict.update(kw)\n return infodict\n\n\ndef get_encoding(obj):\n \"\"\"Get encoding for python source file defining obj\n\n Returns None if obj is not defined in a sourcefile.\n \"\"\"\n ofile = find_file(obj)\n # run contents of file through pager starting at line where the object\n # is defined, as long as the file isn't binary and is actually on the\n # filesystem.\n if ofile is None:\n return None\n elif ofile.endswith((\".so\", \".dll\", \".pyd\")):\n return None\n elif not os.path.isfile(ofile):\n return None\n else:\n # Print only text files, not extension binaries. Note that\n # getsourcelines returns lineno with 1-offset and page() uses\n # 0-offset, so we must adjust.\n with io.open(ofile, \"rb\") as buf: # Tweaked to use io.open for Python 2\n encoding, _ = detect_encoding(buf.readline)\n return encoding\n\n\ndef getdoc(obj):\n \"\"\"Stable wrapper around inspect.getdoc.\n\n This can't crash because of attribute problems.\n\n It also attempts to call a getdoc() method on the given object. This\n allows objects which provide their docstrings via non-standard mechanisms\n (like Pyro proxies) to still be inspected by ipython's ? system.\"\"\"\n # Allow objects to offer customized documentation via a getdoc method:\n try:\n ds = obj.getdoc()\n except Exception: # pylint:disable=broad-except\n pass\n else:\n # if we get extra info, we add it to the normal docstring.\n if isinstance(ds, str):\n return inspect.cleandoc(ds)\n\n try:\n docstr = inspect.getdoc(obj)\n encoding = get_encoding(obj)\n return cast_unicode(docstr, encoding=encoding)\n except Exception: # pylint:disable=broad-except\n # Harden against an inspect failure, which can occur with\n # SWIG-wrapped extensions.\n raise\n\n\ndef getsource(obj, is_binary=False):\n \"\"\"Wrapper around inspect.getsource.\n\n This can be modified by other projects to provide customized source\n extraction.\n\n Inputs:\n\n - obj: an object whose source code we will attempt to extract.\n\n Optional inputs:\n\n - is_binary: whether the object is known to come from a binary source.\n This implementation will skip returning any output for binary objects,\n but custom extractors may know how to meaningfully process them.\"\"\"\n\n if is_binary:\n return None\n else:\n # get source if obj was decorated with @decorator\n if hasattr(obj, \"__wrapped__\"):\n obj = obj.__wrapped__\n try:\n src = inspect.getsource(obj)\n except TypeError:\n if hasattr(obj, \"__class__\"):\n src = inspect.getsource(obj.__class__)\n encoding = get_encoding(obj)\n return cast_unicode(src, encoding=encoding)\n\n\ndef is_simple_callable(obj):\n \"\"\"True if obj is a function ()\"\"\"\n return (\n inspect.isfunction(obj)\n or inspect.ismethod(obj)\n or isinstance(obj, _builtin_func_type)\n or isinstance(obj, _builtin_meth_type)\n )\n\n\ndef getargspec(obj):\n \"\"\"Wrapper around :func:`inspect.getfullargspec` on Python 3, and\n :func:inspect.getargspec` on Python 2.\n\n In addition to functions and methods, this can also handle objects with a\n ``__call__`` attribute.\n \"\"\"\n if safe_hasattr(obj, \"__call__\") and not is_simple_callable(obj):\n obj = obj.__call__\n\n return inspect.getfullargspec(obj)\n\n\ndef format_argspec(argspec):\n \"\"\"Format argspect, convenience wrapper around inspect's.\n\n This takes a dict instead of ordered arguments and calls\n inspect.format_argspec with the arguments in the necessary order.\n \"\"\"\n return inspect.formatargspec(\n argspec[\"args\"], argspec[\"varargs\"], argspec[\"varkw\"], argspec[\"defaults\"]\n )\n\n\ndef call_tip(oinfo, format_call=True):\n \"\"\"Extract call tip data from an oinfo dict.\n\n Parameters\n ----------\n oinfo : dict\n\n format_call : bool, optional\n If True, the call line is formatted and returned as a string. If not, a\n tuple of (name, argspec) is returned.\n\n Returns\n -------\n call_info : None, str or (str, dict) tuple.\n When format_call is True, the whole call information is formatted as a\n single string. Otherwise, the object's name and its argspec dict are\n returned. If no call information is available, None is returned.\n\n docstring : str or None\n The most relevant docstring for calling purposes is returned, if\n available. The priority is: call docstring for callable instances, then\n constructor docstring for classes, then main object's docstring otherwise\n (regular functions).\n \"\"\"\n # Get call definition\n argspec = oinfo.get(\"argspec\")\n if argspec is None:\n call_line = None\n else:\n # Callable objects will have 'self' as their first argument, prune\n # it out if it's there for clarity (since users do *not* pass an\n # extra first argument explicitly).\n try:\n has_self = argspec[\"args\"][0] == \"self\"\n except (KeyError, IndexError):\n pass\n else:\n if has_self:\n argspec[\"args\"] = argspec[\"args\"][1:]\n\n call_line = oinfo[\"name\"] + format_argspec(argspec)\n\n # Now get docstring.\n # The priority is: call docstring, constructor docstring, main one.\n doc = oinfo.get(\"call_docstring\")\n if doc is None:\n doc = oinfo.get(\"init_docstring\")\n if doc is None:\n doc = oinfo.get(\"docstring\", \"\")\n\n return call_line, doc\n\n\ndef find_file(obj):\n \"\"\"Find the absolute path to the file where an object was defined.\n\n This is essentially a robust wrapper around `inspect.getabsfile`.\n\n Returns None if no file can be found.\n\n Parameters\n ----------\n obj : any Python object\n\n Returns\n -------\n fname : str\n The absolute path to the file where the object was defined.\n \"\"\"\n # get source if obj was decorated with @decorator\n if safe_hasattr(obj, \"__wrapped__\"):\n obj = obj.__wrapped__\n\n fname = None\n try:\n fname = inspect.getabsfile(obj)\n except TypeError:\n # For an instance, the file that matters is where its class was\n # declared.\n if hasattr(obj, \"__class__\"):\n try:\n fname = inspect.getabsfile(obj.__class__)\n except TypeError:\n # Can happen for builtins\n pass\n except: # pylint:disable=bare-except\n pass\n return cast_unicode(fname)\n\n\ndef find_source_lines(obj):\n \"\"\"Find the line number in a file where an object was defined.\n\n This is essentially a robust wrapper around `inspect.getsourcelines`.\n\n Returns None if no file can be found.\n\n Parameters\n ----------\n obj : any Python object\n\n Returns\n -------\n lineno : int\n The line number where the object definition starts.\n \"\"\"\n # get source if obj was decorated with @decorator\n if safe_hasattr(obj, \"__wrapped__\"):\n obj = obj.__wrapped__\n\n try:\n try:\n lineno = inspect.getsourcelines(obj)[1]\n except TypeError:\n # For instances, try the class object like getsource() does\n if hasattr(obj, \"__class__\"):\n lineno = inspect.getsourcelines(obj.__class__)[1]\n else:\n lineno = None\n except: # pylint:disable=bare-except\n return None\n\n return lineno\n\n\nif PYTHON_VERSION_INFO < (3, 5, 0):\n FrameInfo = collections.namedtuple(\n \"FrameInfo\",\n [\"frame\", \"filename\", \"lineno\", \"function\", \"code_context\", \"index\"],\n )\n\n def getouterframes(frame, context=1):\n \"\"\"Wrapper for getouterframes so that it acts like the Python v3.5 version.\"\"\"\n return [FrameInfo(*f) for f in inspect.getouterframes(frame, context=context)]\n\n\nelse:\n getouterframes = inspect.getouterframes\n\n\nclass Inspector(object):\n \"\"\"Inspects objects.\"\"\"\n\n def __init__(self, str_detail_level=0):\n self.str_detail_level = str_detail_level\n\n def _getdef(self, obj, oname=\"\"):\n \"\"\"Return the call signature for any callable object.\n\n If any exception is generated, None is returned instead and the\n exception is suppressed.\n \"\"\"\n try:\n hdef = oname + inspect.signature(*getargspec(obj))\n return cast_unicode(hdef)\n except: # pylint:disable=bare-except\n return None\n\n def noinfo(self, msg, oname):\n \"\"\"Generic message when no information is found.\"\"\"\n print(\"No %s found\" % msg, end=\" \")\n if oname:\n print(\"for %s\" % oname)\n else:\n print()\n\n def pdef(self, obj, oname=\"\"):\n \"\"\"Print the call signature for any callable object.\n\n If the object is a class, print the constructor information.\n \"\"\"\n\n if not callable(obj):\n print(\"Object is not callable.\")\n return\n\n header = \"\"\n\n if inspect.isclass(obj):\n header = self.__head(\"Class constructor information:\\n\")\n obj = obj.__init__\n\n output = self._getdef(obj, oname)\n if output is None:\n self.noinfo(\"definition header\", oname)\n else:\n print(header, output, end=\" \", file=sys.stdout)\n\n def pdoc(self, obj, oname=\"\"):\n \"\"\"Print the docstring for any object.\n\n Optional\n\n -formatter: a function to run the docstring through for specially\n formatted docstrings.\n \"\"\"\n\n head = self.__head # For convenience\n lines = []\n ds = getdoc(obj)\n if ds:\n lines.append(head(\"Class docstring:\"))\n lines.append(indent(ds))\n if inspect.isclass(obj) and hasattr(obj, \"__init__\"):\n init_ds = getdoc(obj.__init__)\n if init_ds is not None:\n lines.append(head(\"Init docstring:\"))\n lines.append(indent(init_ds))\n elif hasattr(obj, \"__call__\"):\n call_ds = getdoc(obj.__call__)\n if call_ds:\n lines.append(head(\"Call docstring:\"))\n lines.append(indent(call_ds))\n\n if not lines:\n self.noinfo(\"documentation\", oname)\n else:\n print(\"\\n\".join(lines))\n\n def psource(self, obj, oname=\"\"):\n \"\"\"Print the source code for an object.\"\"\"\n # Flush the source cache because inspect can return out-of-date source\n linecache.checkcache()\n try:\n src = getsource(obj)\n except: # pylint:disable=bare-except\n self.noinfo(\"source\", oname)\n else:\n print(src)\n\n def pfile(self, obj, oname=\"\"):\n \"\"\"Show the whole file where an object was defined.\"\"\"\n lineno = find_source_lines(obj)\n if lineno is None:\n self.noinfo(\"file\", oname)\n return\n\n ofile = find_file(obj)\n # run contents of file through pager starting at line where the object\n # is defined, as long as the file isn't binary and is actually on the\n # filesystem.\n if ofile.endswith((\".so\", \".dll\", \".pyd\")):\n print(\"File %r is binary, not printing.\" % ofile)\n elif not os.path.isfile(ofile):\n print(\"File %r does not exist, not printing.\" % ofile)\n else:\n # Print only text files, not extension binaries. Note that\n # getsourcelines returns lineno with 1-offset and page() uses\n # 0-offset, so we must adjust.\n o = read_py_file(ofile, skip_encoding_cookie=False)\n print(o, lineno - 1)\n\n def _format_fields_str(self, fields, title_width=0):\n \"\"\"Formats a list of fields for display using color strings.\n\n Parameters\n ----------\n fields : list\n A list of 2-tuples: (field_title, field_content)\n title_width : int\n How many characters to pad titles to. Default to longest title.\n \"\"\"\n out = []\n if title_width == 0:\n title_width = max(len(title) + 2 for title, _ in fields)\n for title, content in fields:\n title_len = len(title)\n title = \"{BOLD_RED}\" + title + \":{NO_COLOR}\"\n if len(content.splitlines()) > 1:\n title += \"\\n\"\n else:\n title += \" \".ljust(title_width - title_len)\n out.append(cast_unicode(title) + cast_unicode(content))\n return format_color(\"\\n\".join(out) + \"\\n\")\n\n def _format_fields_tokens(self, fields, title_width=0):\n \"\"\"Formats a list of fields for display using color tokens from\n pygments.\n\n Parameters\n ----------\n fields : list\n A list of 2-tuples: (field_title, field_content)\n title_width : int\n How many characters to pad titles to. Default to longest title.\n \"\"\"\n out = []\n if title_width == 0:\n title_width = max(len(title) + 2 for title, _ in fields)\n for title, content in fields:\n title_len = len(title)\n title = \"{BOLD_RED}\" + title + \":{NO_COLOR}\"\n if not isinstance(content, str) or len(content.splitlines()) > 1:\n title += \"\\n\"\n else:\n title += \" \".ljust(title_width - title_len)\n out += partial_color_tokenize(title)\n if isinstance(content, str):\n out[-1] = (out[-1][0], out[-1][1] + content + \"\\n\")\n else:\n out += content\n out[-1] = (out[-1][0], out[-1][1] + \"\\n\")\n out[-1] = (out[-1][0], out[-1][1] + \"\\n\")\n return out\n\n def _format_fields(self, fields, title_width=0):\n \"\"\"Formats a list of fields for display using color tokens from\n pygments.\n\n Parameters\n ----------\n fields : list\n A list of 2-tuples: (field_title, field_content)\n title_width : int\n How many characters to pad titles to. Default to longest title.\n \"\"\"\n if HAS_PYGMENTS:\n rtn = self._format_fields_tokens(fields, title_width=title_width)\n else:\n rtn = self._format_fields_str(fields, title_width=title_width)\n return rtn\n\n # The fields to be displayed by pinfo: (fancy_name, key_in_info_dict)\n pinfo_fields1 = [(\"Type\", \"type_name\")]\n\n pinfo_fields2 = [(\"String form\", \"string_form\")]\n\n pinfo_fields3 = [\n (\"Length\", \"length\"),\n (\"File\", \"file\"),\n (\"Definition\", \"definition\"),\n ]\n\n pinfo_fields_obj = [\n (\"Class docstring\", \"class_docstring\"),\n (\"Init docstring\", \"init_docstring\"),\n (\"Call def\", \"call_def\"),\n (\"Call docstring\", \"call_docstring\"),\n ]\n\n def pinfo(self, obj, oname=\"\", info=None, detail_level=0):\n \"\"\"Show detailed information about an object.\n\n Parameters\n ----------\n obj : object\n oname : str, optional\n name of the variable pointing to the object.\n info : dict, optional\n a structure with some information fields which may have been\n precomputed already.\n detail_level : int, optional\n if set to 1, more information is given.\n \"\"\"\n info = self.info(obj, oname=oname, info=info, detail_level=detail_level)\n displayfields = []\n\n def add_fields(fields):\n for title, key in fields:\n field = info[key]\n if field is not None:\n displayfields.append((title, field.rstrip()))\n\n add_fields(self.pinfo_fields1)\n add_fields(self.pinfo_fields2)\n\n # Namespace\n if info[\"namespace\"] is not None and info[\"namespace\"] != \"Interactive\":\n displayfields.append((\"Namespace\", info[\"namespace\"].rstrip()))\n\n add_fields(self.pinfo_fields3)\n if info[\"isclass\"] and info[\"init_definition\"]:\n displayfields.append((\"Init definition\", info[\"init_definition\"].rstrip()))\n\n # Source or docstring, depending on detail level and whether\n # source found.\n if detail_level > 0 and info[\"source\"] is not None:\n displayfields.append((\"Source\", cast_unicode(info[\"source\"])))\n elif info[\"docstring\"] is not None:\n displayfields.append((\"Docstring\", info[\"docstring\"]))\n\n # Constructor info for classes\n if info[\"isclass\"]:\n if info[\"init_docstring\"] is not None:\n displayfields.append((\"Init docstring\", info[\"init_docstring\"]))\n\n # Info for objects:\n else:\n add_fields(self.pinfo_fields_obj)\n\n # Finally send to printer/pager:\n if displayfields:\n print_color(self._format_fields(displayfields))\n\n def info(self, obj, oname=\"\", info=None, detail_level=0):\n \"\"\"Compute a dict with detailed information about an object.\n\n Optional arguments:\n\n - oname: name of the variable pointing to the object.\n\n - info: a structure with some information fields which may have been\n precomputed already.\n\n - detail_level: if set to 1, more information is given.\n \"\"\"\n obj_type = type(obj)\n if info is None:\n ismagic = 0\n isalias = 0\n ospace = \"\"\n else:\n ismagic = info.ismagic\n isalias = info.isalias\n ospace = info.namespace\n # Get docstring, special-casing aliases:\n if isalias:\n if not callable(obj):\n if len(obj) >= 2 and isinstance(obj[1], str):\n ds = \"Alias to the system command:\\n {0}\".format(obj[1])\n else: # pylint:disable=bare-except\n ds = \"Alias: \" + str(obj)\n else:\n ds = \"Alias to \" + str(obj)\n if obj.__doc__:\n ds += \"\\nDocstring:\\n\" + obj.__doc__\n else:\n ds = getdoc(obj)\n if ds is None:\n ds = \"\"\n\n # store output in a dict, we initialize it here and fill it as we go\n out = dict(name=oname, found=True, isalias=isalias, ismagic=ismagic)\n\n string_max = 200 # max size of strings to show (snipped if longer)\n shalf = int((string_max - 5) / 2)\n\n if ismagic:\n obj_type_name = \"Magic function\"\n elif isalias:\n obj_type_name = \"System alias\"\n else:\n obj_type_name = obj_type.__name__\n out[\"type_name\"] = obj_type_name\n\n try:\n bclass = obj.__class__\n out[\"base_class\"] = str(bclass)\n except: # pylint:disable=bare-except\n pass\n\n # String form, but snip if too long in ? form (full in ??)\n if detail_level >= self.str_detail_level:\n try:\n ostr = str(obj)\n str_head = \"string_form\"\n if not detail_level and len(ostr) > string_max:\n ostr = ostr[:shalf] + \" <...> \" + ostr[-shalf:]\n ostr = (\"\\n\" + \" \" * len(str_head.expandtabs())).join(\n q.strip() for q in ostr.split(\"\\n\")\n )\n out[str_head] = ostr\n except: # pylint:disable=bare-except\n pass\n\n if ospace:\n out[\"namespace\"] = ospace\n\n # Length (for strings and lists)\n try:\n out[\"length\"] = str(len(obj))\n except: # pylint:disable=bare-except\n pass\n\n # Filename where object was defined\n binary_file = False\n fname = find_file(obj)\n if fname is None:\n # if anything goes wrong, we don't want to show source, so it's as\n # if the file was binary\n binary_file = True\n else:\n if fname.endswith((\".so\", \".dll\", \".pyd\")):\n binary_file = True\n elif fname.endswith(\"\"):\n fname = \"Dynamically generated function. \" \"No source code available.\"\n out[\"file\"] = fname\n\n # Docstrings only in detail 0 mode, since source contains them (we\n # avoid repetitions). If source fails, we add them back, see below.\n if ds and detail_level == 0:\n out[\"docstring\"] = ds\n\n # Original source code for any callable\n if detail_level:\n # Flush the source cache because inspect can return out-of-date\n # source\n linecache.checkcache()\n source = None\n try:\n try:\n source = getsource(obj, binary_file)\n except TypeError:\n if hasattr(obj, \"__class__\"):\n source = getsource(obj.__class__, binary_file)\n if source is not None:\n source = source.rstrip()\n if HAS_PYGMENTS:\n lexer = pyghooks.XonshLexer()\n source = list(pygments.lex(source, lexer=lexer))\n out[\"source\"] = source\n except Exception: # pylint:disable=broad-except\n pass\n\n if ds and source is None:\n out[\"docstring\"] = ds\n\n # Constructor docstring for classes\n if inspect.isclass(obj):\n out[\"isclass\"] = True\n # reconstruct the function definition and print it:\n try:\n obj_init = obj.__init__\n except AttributeError:\n init_def = init_ds = None\n else:\n init_def = self._getdef(obj_init, oname)\n init_ds = getdoc(obj_init)\n # Skip Python's auto-generated docstrings\n if init_ds == _object_init_docstring:\n init_ds = None\n\n if init_def or init_ds:\n if init_def:\n out[\"init_definition\"] = init_def\n if init_ds:\n out[\"init_docstring\"] = init_ds\n\n # and class docstring for instances:\n else:\n # reconstruct the function definition and print it:\n defln = self._getdef(obj, oname)\n if defln:\n out[\"definition\"] = defln\n\n # First, check whether the instance docstring is identical to the\n # class one, and print it separately if they don't coincide. In\n # most cases they will, but it's nice to print all the info for\n # objects which use instance-customized docstrings.\n if ds:\n try:\n cls = getattr(obj, \"__class__\")\n except: # pylint:disable=bare-except\n class_ds = None\n else:\n class_ds = getdoc(cls)\n # Skip Python's auto-generated docstrings\n if class_ds in _builtin_type_docstrings:\n class_ds = None\n if class_ds and ds != class_ds:\n out[\"class_docstring\"] = class_ds\n\n # Next, try to show constructor docstrings\n try:\n init_ds = getdoc(obj.__init__)\n # Skip Python's auto-generated docstrings\n if init_ds == _object_init_docstring:\n init_ds = None\n except AttributeError:\n init_ds = None\n if init_ds:\n out[\"init_docstring\"] = init_ds\n\n # Call form docstring for callable instances\n if safe_hasattr(obj, \"__call__\") and not is_simple_callable(obj):\n call_def = self._getdef(obj.__call__, oname)\n if call_def:\n call_def = call_def\n # it may never be the case that call def and definition\n # differ, but don't include the same signature twice\n if call_def != out.get(\"definition\"):\n out[\"call_def\"] = call_def\n call_ds = getdoc(obj.__call__)\n # Skip Python's auto-generated docstrings\n if call_ds == _func_call_docstring:\n call_ds = None\n if call_ds:\n out[\"call_docstring\"] = call_ds\n\n # Compute the object's argspec as a callable. The key is to decide\n # whether to pull it from the object itself, from its __init__ or\n # from its __call__ method.\n\n if inspect.isclass(obj):\n # Old-style classes need not have an __init__\n callable_obj = getattr(obj, \"__init__\", None)\n elif callable(obj):\n callable_obj = obj\n else:\n callable_obj = None\n\n if callable_obj:\n try:\n argspec = getargspec(callable_obj)\n except (TypeError, AttributeError):\n # For extensions/builtins we can't retrieve the argspec\n pass\n else:\n # named tuples' _asdict() method returns an OrderedDict, but we\n # we want a normal\n out[\"argspec\"] = argspec_dict = dict(argspec._asdict())\n # We called this varkw before argspec became a named tuple.\n # With getfullargspec it's also called varkw.\n if \"varkw\" not in argspec_dict:\n argspec_dict[\"varkw\"] = argspec_dict.pop(\"keywords\")\n\n return object_info(**out)\n" }, { "path": "xonsh/jobs.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"Job control for the xonsh shell.\"\"\"\nimport os\nimport sys\nimport time\nimport ctypes\nimport signal\nimport builtins\nimport subprocess\nimport collections\n\nfrom xonsh.lazyasd import LazyObject\nfrom xonsh.platform import FD_STDERR, ON_DARWIN, ON_WINDOWS, ON_CYGWIN, ON_MSYS, LIBC\nfrom xonsh.tools import unthreadable\n\n\ntasks = LazyObject(collections.deque, globals(), \"tasks\")\n# Track time stamp of last exit command, so that two consecutive attempts to\n# exit can kill all jobs and exit.\n_last_exit_time = None\n\n\nif ON_DARWIN:\n\n def _send_signal(job, signal):\n # On OS X, os.killpg() may cause PermissionError when there are\n # any zombie processes in the process group.\n # See github issue #1012 for details\n for pid in job[\"pids\"]:\n if pid is None: # the pid of an aliased proc is None\n continue\n try:\n os.kill(pid, signal)\n except ProcessLookupError:\n pass\n\n\nelif ON_WINDOWS:\n pass\nelif ON_CYGWIN or ON_MSYS:\n # Similar to what happened on OSX, more issues on Cygwin\n # (see Github issue #514).\n def _send_signal(job, signal):\n try:\n os.killpg(job[\"pgrp\"], signal)\n except Exception:\n for pid in job[\"pids\"]:\n try:\n os.kill(pid, signal)\n except Exception:\n pass\n\n\nelse:\n\n def _send_signal(job, signal):\n pgrp = job[\"pgrp\"]\n if pgrp is None:\n for pid in job[\"pids\"]:\n try:\n os.kill(pid, signal)\n except Exception:\n pass\n else:\n os.killpg(job[\"pgrp\"], signal)\n\n\nif ON_WINDOWS:\n\n def _continue(job):\n job[\"status\"] = \"running\"\n\n def _kill(job):\n subprocess.check_output([\"taskkill\", \"/F\", \"/T\", \"/PID\", str(job[\"obj\"].pid)])\n\n def ignore_sigtstp():\n pass\n\n def give_terminal_to(pgid):\n pass\n\n def wait_for_active_job(last_task=None, backgrounded=False):\n \"\"\"\n Wait for the active job to finish, to be killed by SIGINT, or to be\n suspended by ctrl-z.\n \"\"\"\n _clear_dead_jobs()\n active_task = get_next_task()\n # Return when there are no foreground active task\n if active_task is None:\n return last_task\n obj = active_task[\"obj\"]\n _continue(active_task)\n while obj.returncode is None:\n try:\n obj.wait(0.01)\n except subprocess.TimeoutExpired:\n pass\n except KeyboardInterrupt:\n _kill(active_task)\n return wait_for_active_job(last_task=active_task)\n\n\nelse:\n\n def _continue(job):\n _send_signal(job, signal.SIGCONT)\n\n def _kill(job):\n _send_signal(job, signal.SIGKILL)\n\n def ignore_sigtstp():\n signal.signal(signal.SIGTSTP, signal.SIG_IGN)\n\n _shell_pgrp = os.getpgrp()\n\n _block_when_giving = LazyObject(\n lambda: (signal.SIGTTOU, signal.SIGTTIN, signal.SIGTSTP, signal.SIGCHLD),\n globals(),\n \"_block_when_giving\",\n )\n\n if ON_CYGWIN or ON_MSYS:\n # on cygwin, signal.pthread_sigmask does not exist in Python, even\n # though pthread_sigmask is defined in the kernel. thus, we use\n # ctypes to mimic the calls in the \"normal\" version below.\n LIBC.pthread_sigmask.restype = ctypes.c_int\n LIBC.pthread_sigmask.argtypes = [\n ctypes.c_int,\n ctypes.POINTER(ctypes.c_ulong),\n ctypes.POINTER(ctypes.c_ulong),\n ]\n\n def _pthread_sigmask(how, signals):\n mask = 0\n for sig in signals:\n mask |= 1 << sig\n oldmask = ctypes.c_ulong()\n mask = ctypes.c_ulong(mask)\n result = LIBC.pthread_sigmask(\n how, ctypes.byref(mask), ctypes.byref(oldmask)\n )\n if result:\n raise OSError(result, \"Sigmask error.\")\n\n return {\n sig\n for sig in getattr(signal, \"Signals\", range(0, 65))\n if (oldmask.value >> sig) & 1\n }\n\n else:\n _pthread_sigmask = signal.pthread_sigmask\n\n # give_terminal_to is a simplified version of:\n # give_terminal_to from bash 4.3 source, jobs.c, line 4030\n # this will give the terminal to the process group pgid\n def give_terminal_to(pgid):\n if pgid is None:\n return False\n oldmask = _pthread_sigmask(signal.SIG_BLOCK, _block_when_giving)\n try:\n os.tcsetpgrp(FD_STDERR, pgid)\n return True\n except ProcessLookupError:\n # when the process finished before giving terminal to it,\n # see issue #2288\n return False\n except OSError as e:\n if e.errno == 22: # [Errno 22] Invalid argument\n # there are cases that all the processes of pgid have\n # finished, then we don't need to do anything here, see\n # issue #2220\n return False\n elif e.errno == 25: # [Errno 25] Inappropriate ioctl for device\n # There are also cases where we are not connected to a\n # real TTY, even though we may be run in interactive\n # mode. See issue #2267 for an example with emacs\n return False\n else:\n raise\n finally:\n if oldmask:\n # only reset the mask if it is non-empty! See #2989\n _pthread_sigmask(signal.SIG_SETMASK, oldmask)\n\n def wait_for_active_job(last_task=None, backgrounded=False, _nochild=False):\n \"\"\"\n Wait for the active job to finish, to be killed by SIGINT, or to be\n suspended by ctrl-z.\n \"\"\"\n _clear_dead_jobs()\n active_task = get_next_task()\n # Return when there are no foreground active task\n if active_task is None:\n return last_task\n obj = active_task[\"obj\"]\n backgrounded = False\n try:\n _, wcode = os.waitpid(obj.pid, os.WUNTRACED)\n except ChildProcessError: # No child processes\n if _nochild:\n return active_task\n else:\n return wait_for_active_job(\n last_task=active_task, backgrounded=backgrounded, _nochild=True\n )\n if os.WIFSTOPPED(wcode):\n print(\"^Z\")\n active_task[\"status\"] = \"stopped\"\n backgrounded = True\n elif os.WIFSIGNALED(wcode):\n print() # get a newline because ^C will have been printed\n obj.signal = (os.WTERMSIG(wcode), os.WCOREDUMP(wcode))\n obj.returncode = None\n else:\n obj.returncode = os.WEXITSTATUS(wcode)\n obj.signal = None\n return wait_for_active_job(last_task=active_task, backgrounded=backgrounded)\n\n\ndef get_next_task():\n \"\"\" Get the next active task and put it on top of the queue\"\"\"\n selected_task = None\n for tid in tasks:\n task = get_task(tid)\n if not task[\"bg\"] and task[\"status\"] == \"running\":\n selected_task = tid\n break\n if selected_task is None:\n return\n tasks.remove(selected_task)\n tasks.appendleft(selected_task)\n return get_task(selected_task)\n\n\ndef get_task(tid):\n return builtins.__xonsh__.all_jobs[tid]\n\n\ndef _clear_dead_jobs():\n to_remove = set()\n for tid in tasks:\n obj = get_task(tid)[\"obj\"]\n if obj is None or obj.poll() is not None:\n to_remove.add(tid)\n for job in to_remove:\n tasks.remove(job)\n del builtins.__xonsh__.all_jobs[job]\n\n\ndef print_one_job(num, outfile=sys.stdout):\n \"\"\"Print a line describing job number ``num``.\"\"\"\n try:\n job = builtins.__xonsh__.all_jobs[num]\n except KeyError:\n return\n pos = \"+\" if tasks[0] == num else \"-\" if tasks[1] == num else \" \"\n status = job[\"status\"]\n cmd = [\" \".join(i) if isinstance(i, list) else i for i in job[\"cmds\"]]\n cmd = \" \".join(cmd)\n pid = job[\"pids\"][-1]\n bg = \" &\" if job[\"bg\"] else \"\"\n print(\"[{}]{} {}: {}{} ({})\".format(num, pos, status, cmd, bg, pid), file=outfile)\n\n\ndef get_next_job_number():\n \"\"\"Get the lowest available unique job number (for the next job created).\n \"\"\"\n _clear_dead_jobs()\n i = 1\n while i in builtins.__xonsh__.all_jobs:\n i += 1\n return i\n\n\ndef add_job(info):\n \"\"\"Add a new job to the jobs dictionary.\"\"\"\n num = get_next_job_number()\n info[\"started\"] = time.time()\n info[\"status\"] = \"running\"\n tasks.appendleft(num)\n builtins.__xonsh__.all_jobs[num] = info\n if info[\"bg\"] and builtins.__xonsh__.env.get(\"XONSH_INTERACTIVE\"):\n print_one_job(num)\n\n\ndef clean_jobs():\n \"\"\"Clean up jobs for exiting shell\n\n In non-interactive mode, kill all jobs.\n\n In interactive mode, check for suspended or background jobs, print a\n warning if any exist, and return False. Otherwise, return True.\n \"\"\"\n jobs_clean = True\n if builtins.__xonsh__.env[\"XONSH_INTERACTIVE\"]:\n _clear_dead_jobs()\n\n if builtins.__xonsh__.all_jobs:\n global _last_exit_time\n hist = builtins.__xonsh__.history\n if hist is not None and len(hist.tss) > 0:\n last_cmd_start = hist.tss[-1][0]\n else:\n last_cmd_start = None\n\n if _last_exit_time and last_cmd_start and _last_exit_time > last_cmd_start:\n # Exit occurred after last command started, so it was called as\n # part of the last command and is now being called again\n # immediately. Kill jobs and exit without reminder about\n # unfinished jobs in this case.\n kill_all_jobs()\n else:\n if len(builtins.__xonsh__.all_jobs) > 1:\n msg = \"there are unfinished jobs\"\n else:\n msg = \"there is an unfinished job\"\n\n if builtins.__xonsh__.env[\"SHELL_TYPE\"] != \"prompt_toolkit\":\n # The Ctrl+D binding for prompt_toolkit already inserts a\n # newline\n print()\n print(\"xonsh: {}\".format(msg), file=sys.stderr)\n print(\"-\" * 5, file=sys.stderr)\n jobs([], stdout=sys.stderr)\n print(\"-\" * 5, file=sys.stderr)\n print(\n 'Type \"exit\" or press \"ctrl-d\" again to force quit.',\n file=sys.stderr,\n )\n jobs_clean = False\n _last_exit_time = time.time()\n else:\n kill_all_jobs()\n\n return jobs_clean\n\n\ndef kill_all_jobs():\n \"\"\"\n Send SIGKILL to all child processes (called when exiting xonsh).\n \"\"\"\n _clear_dead_jobs()\n for job in builtins.__xonsh__.all_jobs.values():\n _kill(job)\n\n\ndef jobs(args, stdin=None, stdout=sys.stdout, stderr=None):\n \"\"\"\n xonsh command: jobs\n\n Display a list of all current jobs.\n \"\"\"\n _clear_dead_jobs()\n for j in tasks:\n print_one_job(j, outfile=stdout)\n return None, None\n\n\ndef resume_job(args, wording):\n \"\"\"\n used by fg and bg to resume a job either in the foreground or in the background.\n \"\"\"\n _clear_dead_jobs()\n if len(tasks) == 0:\n return \"\", \"There are currently no suspended jobs\"\n\n if len(args) == 0:\n tid = tasks[0] # take the last manipulated task by default\n elif len(args) == 1:\n try:\n if args[0] == \"+\": # take the last manipulated task\n tid = tasks[0]\n elif args[0] == \"-\": # take the second to last manipulated task\n tid = tasks[1]\n else:\n tid = int(args[0])\n except (ValueError, IndexError):\n return \"\", \"Invalid job: {}\\n\".format(args[0])\n\n if tid not in builtins.__xonsh__.all_jobs:\n return \"\", \"Invalid job: {}\\n\".format(args[0])\n else:\n return \"\", \"{} expects 0 or 1 arguments, not {}\\n\".format(wording, len(args))\n\n # Put this one on top of the queue\n tasks.remove(tid)\n tasks.appendleft(tid)\n\n job = get_task(tid)\n job[\"bg\"] = False\n job[\"status\"] = \"running\"\n if builtins.__xonsh__.env.get(\"XONSH_INTERACTIVE\"):\n print_one_job(tid)\n pipeline = job[\"pipeline\"]\n pipeline.resume(job)\n\n\n@unthreadable\ndef fg(args, stdin=None):\n \"\"\"\n xonsh command: fg\n\n Bring the currently active job to the foreground, or, if a single number is\n given as an argument, bring that job to the foreground. Additionally,\n specify \"+\" for the most recent job and \"-\" for the second most recent job.\n \"\"\"\n return resume_job(args, wording=\"fg\")\n\n\ndef bg(args, stdin=None):\n \"\"\"xonsh command: bg\n\n Resume execution of the currently active job in the background, or, if a\n single number is given as an argument, resume that job in the background.\n \"\"\"\n res = resume_job(args, wording=\"bg\")\n if res is None:\n curtask = get_task(tasks[0])\n curtask[\"bg\"] = True\n _continue(curtask)\n else:\n return res\n" }, { "path": "xonsh/jsonutils.py", "content": "\"\"\"Custom tools for managing JSON serialization / deserialization of xonsh\nobjects.\n\"\"\"\nimport functools\n\nfrom xonsh.tools import EnvPath\n\n\n@functools.singledispatch\ndef serialize_xonsh_json(val):\n \"\"\"JSON serializer for xonsh custom data structures. This is only\n called when another normal JSON types are not found.\n \"\"\"\n return str(val)\n\n\n@serialize_xonsh_json.register(EnvPath)\ndef _serialize_xonsh_json_env_path(val):\n return val.paths\n" }, { "path": "xonsh/jupyter_kernel.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"Hooks for Jupyter Xonsh Kernel.\"\"\"\nimport sys\nimport json\nimport hmac\nimport uuid\nimport errno\nimport hashlib\nimport datetime\nimport builtins\nimport threading\nfrom pprint import pformat\nfrom argparse import ArgumentParser\nfrom collections.abc import Set\n\nimport zmq\nfrom zmq.eventloop import ioloop, zmqstream\nfrom zmq.error import ZMQError\n\nfrom xonsh import __version__ as version\nfrom xonsh.main import setup\nfrom xonsh.completer import Completer\nfrom xonsh.commands_cache import predict_true\n\n\nMAX_SIZE = 8388608 # 8 Mb\nDELIM = b\"\"\n\n\ndef dump_bytes(*args, **kwargs):\n \"\"\"Converts an object to JSON and returns the bytes.\"\"\"\n return json.dumps(*args, **kwargs).encode(\"ascii\")\n\n\ndef load_bytes(b):\n \"\"\"Converts bytes of JSON to an object.\"\"\"\n return json.loads(b.decode(\"ascii\"))\n\n\ndef bind(socket, connection, port):\n \"\"\"Binds a socket to a port, or a random port if needed. Returns the port.\"\"\"\n if port <= 0:\n return socket.bind_to_random_port(connection)\n else:\n socket.bind(\"{}:{}\".format(connection, port))\n return port\n\n\nclass XonshKernel:\n \"\"\"Xonsh xernal for Jupyter\"\"\"\n\n implementation = \"Xonsh \" + version\n implementation_version = version\n language = \"xonsh\"\n language_version = version.split(\".\")[:3]\n banner = \"Xonsh - Python-powered, cross-platform shell\"\n language_info = {\n \"name\": \"xonsh\",\n \"version\": version,\n \"pygments_lexer\": \"xonsh\",\n \"codemirror_mode\": \"shell\",\n \"mimetype\": \"text/x-sh\",\n \"file_extension\": \".xsh\",\n }\n signature_schemes = {\"hmac-sha256\": hashlib.sha256}\n\n def __init__(self, debug_level=0, session_id=None, config=None, **kwargs):\n \"\"\"\n Parameters\n ----------\n debug_level : int, optional\n Integer from 0 (no debugging) to 3 (all debugging), default: 0.\n session_id : str or None, optional\n Unique string id representing the kernel session. If None, this will\n be replaced with a random UUID.\n config : dict or None, optional\n Configuration dictionary to start server with. BY default will\n search the command line for options (if given) or use default\n configuration.\n \"\"\"\n self.debug_level = debug_level\n self.session_id = str(uuid.uuid4()) if session_id is None else session_id\n self._parser = None\n self.config = self.make_default_config() if config is None else config\n\n self.exiting = False\n self.execution_count = 1\n self.completer = Completer()\n\n @property\n def parser(self):\n if self._parser is None:\n p = ArgumentParser(\"jupyter_kerenel\")\n p.add_argument(\"-f\", dest=\"config_file\", default=None)\n self._parser = p\n return self._parser\n\n def make_default_config(self):\n \"\"\"Provides default configuration\"\"\"\n ns, unknown = self.parser.parse_known_args(sys.argv)\n if ns.config_file is None:\n self.dprint(1, \"Starting xonsh kernel with default args...\")\n config = {\n \"control_port\": 0,\n \"hb_port\": 0,\n \"iopub_port\": 0,\n \"ip\": \"127.0.0.1\",\n \"key\": str(uuid.uuid4()),\n \"shell_port\": 0,\n \"signature_scheme\": \"hmac-sha256\",\n \"stdin_port\": 0,\n \"transport\": \"tcp\",\n }\n else:\n self.dprint(1, \"Loading simple_kernel with args:\", sys.argv)\n self.dprint(1, \"Reading config file {!r}...\".format(ns.config_file))\n with open(ns.config_file) as f:\n config = json.load(f)\n return config\n\n def iopub_handler(self, message):\n \"\"\"Handles iopub requests.\"\"\"\n self.dprint(2, \"iopub received:\", message)\n\n def control_handler(self, wire_message):\n \"\"\"Handles control requests\"\"\"\n self.dprint(1, \"control received:\", wire_message)\n identities, msg = self.deserialize_wire_message(wire_message)\n if msg[\"header\"][\"msg_type\"] == \"shutdown_request\":\n self.shutdown()\n\n def stdin_handler(self, message):\n self.dprint(2, \"stdin received:\", message)\n\n def start(self):\n \"\"\"Starts the server\"\"\"\n ioloop.install()\n connection = self.config[\"transport\"] + \"://\" + self.config[\"ip\"]\n secure_key = self.config[\"key\"].encode()\n digestmod = self.signature_schemes[self.config[\"signature_scheme\"]]\n self.auth = hmac.HMAC(secure_key, digestmod=digestmod)\n\n # Heartbeat\n ctx = zmq.Context()\n self.heartbeat_socket = ctx.socket(zmq.REP)\n self.config[\"hb_port\"] = bind(\n self.heartbeat_socket, connection, self.config[\"hb_port\"]\n )\n\n # IOPub/Sub, aslo called SubSocketChannel in IPython sources\n self.iopub_socket = ctx.socket(zmq.PUB)\n self.config[\"iopub_port\"] = bind(\n self.iopub_socket, connection, self.config[\"iopub_port\"]\n )\n self.iopub_stream = zmqstream.ZMQStream(self.iopub_socket)\n self.iopub_stream.on_recv(self.iopub_handler)\n\n # Control\n self.control_socket = ctx.socket(zmq.ROUTER)\n self.config[\"control_port\"] = bind(\n self.control_socket, connection, self.config[\"control_port\"]\n )\n self.control_stream = zmqstream.ZMQStream(self.control_socket)\n self.control_stream.on_recv(self.control_handler)\n\n # Stdin:\n self.stdin_socket = ctx.socket(zmq.ROUTER)\n self.config[\"stdin_port\"] = bind(\n self.stdin_socket, connection, self.config[\"stdin_port\"]\n )\n self.stdin_stream = zmqstream.ZMQStream(self.stdin_socket)\n self.stdin_stream.on_recv(self.stdin_handler)\n\n # Shell\n self.shell_socket = ctx.socket(zmq.ROUTER)\n self.config[\"shell_port\"] = bind(\n self.shell_socket, connection, self.config[\"shell_port\"]\n )\n self.shell_stream = zmqstream.ZMQStream(self.shell_socket)\n self.shell_stream.on_recv(self.shell_handler)\n\n # start up configurtation\n self.dprint(2, \"Config:\", json.dumps(self.config))\n self.dprint(1, \"Starting loops...\")\n self.hb_thread = threading.Thread(target=self.heartbeat_loop)\n self.hb_thread.daemon = True\n self.hb_thread.start()\n self.dprint(1, \"Ready! Listening...\")\n ioloop.IOLoop.instance().start()\n\n def shutdown(self):\n \"\"\"Shutsdown the kernel\"\"\"\n self.exiting = True\n ioloop.IOLoop.instance().stop()\n\n def dprint(self, level, *args, **kwargs):\n \"\"\"Print but with debug information.\"\"\"\n if level <= self.debug_level:\n print(\"DEBUG\" + str(level) + \":\", file=sys.__stdout__, *args, **kwargs)\n sys.__stdout__.flush()\n\n def sign(self, messages):\n \"\"\"Sign a message list with a secure signature.\"\"\"\n h = self.auth.copy()\n for m in messages:\n h.update(m)\n return h.hexdigest().encode(\"ascii\")\n\n def new_header(self, message_type):\n \"\"\"Make a new header\"\"\"\n return {\n \"date\": datetime.datetime.now().isoformat(),\n \"msg_id\": str(uuid.uuid4()),\n \"username\": \"kernel\",\n \"session\": self.session_id,\n \"msg_type\": message_type,\n \"version\": \"5.0\",\n }\n\n def send(\n self,\n stream,\n message_type,\n content=None,\n parent_header=None,\n metadata=None,\n identities=None,\n ):\n \"\"\"Send data to the client via a stream\"\"\"\n header = self.new_header(message_type)\n if content is None:\n content = {}\n if parent_header is None:\n parent_header = {}\n if metadata is None:\n metadata = {}\n\n messages = list(map(dump_bytes, [header, parent_header, metadata, content]))\n signature = self.sign(messages)\n parts = [DELIM, signature] + messages\n if identities:\n parts = identities + parts\n self.dprint(3, \"send parts:\", parts)\n stream.send_multipart(parts)\n if isinstance(stream, zmqstream.ZMQStream):\n stream.flush()\n\n def deserialize_wire_message(self, wire_message):\n \"\"\"Split the routing prefix and message frames from a message on the wire\"\"\"\n delim_idx = wire_message.index(DELIM)\n identities = wire_message[:delim_idx]\n m_signature = wire_message[delim_idx + 1]\n msg_frames = wire_message[delim_idx + 2 :]\n\n keys = (\"header\", \"parent_header\", \"metadata\", \"content\")\n m = {k: load_bytes(v) for k, v in zip(keys, msg_frames)}\n check_sig = self.sign(msg_frames)\n if check_sig != m_signature:\n raise ValueError(\"Signatures do not match\")\n return identities, m\n\n def run_thread(self, loop, name):\n \"\"\"Run main thread\"\"\"\n self.dprint(2, \"Starting loop for {name!r}...\".format(name=name))\n while not self.exiting:\n self.dprint(2, \"{} Loop!\".format(name))\n try:\n loop.start()\n except ZMQError as e:\n self.dprint(1, \"{} ZMQError!\\n {}\".format(name, e))\n if e.errno == errno.EINTR:\n continue\n else:\n raise\n except Exception:\n self.dprint(2, \"{} Exception!\".format(name))\n if self.exiting:\n break\n else:\n raise\n else:\n self.dprint(2, \"{} Break!\".format(name))\n break\n\n def heartbeat_loop(self):\n \"\"\"Run heartbeat\"\"\"\n self.dprint(2, \"Starting heartbeat loop...\")\n while not self.exiting:\n self.dprint(3, \".\", end=\"\")\n try:\n zmq.device(zmq.FORWARDER, self.heartbeat_socket, self.heartbeat_socket)\n except zmq.ZMQError as e:\n if e.errno == errno.EINTR:\n continue\n else:\n raise\n else:\n break\n\n def shell_handler(self, message):\n \"\"\"Dispatch shell messages to their handlers\"\"\"\n self.dprint(1, \"received:\", message)\n identities, msg = self.deserialize_wire_message(message)\n handler = getattr(self, \"handle_\" + msg[\"header\"][\"msg_type\"], None)\n if handler is None:\n self.dprint(0, \"unknown message type:\", msg[\"header\"][\"msg_type\"])\n return\n handler(msg, identities)\n\n def handle_execute_request(self, message, identities):\n \"\"\"Handle execute request messages.\"\"\"\n self.dprint(2, \"Xonsh Kernel Executing:\", pformat(message[\"content\"][\"code\"]))\n # Start by sending busy signal\n content = {\"execution_state\": \"busy\"}\n self.send(self.iopub_stream, \"status\", content, parent_header=message[\"header\"])\n\n # confirm the input that we are executing\n content = {\n \"execution_count\": self.execution_count,\n \"code\": message[\"content\"][\"code\"],\n }\n self.send(\n self.iopub_stream, \"execute_input\", content, parent_header=message[\"header\"]\n )\n\n # execute the code\n metadata = {\n \"dependencies_met\": True,\n \"engine\": self.session_id,\n \"status\": \"ok\",\n \"started\": datetime.datetime.now().isoformat(),\n }\n content = self.do_execute(parent_header=message[\"header\"], **message[\"content\"])\n self.send(\n self.shell_stream,\n \"execute_reply\",\n content,\n metadata=metadata,\n parent_header=message[\"header\"],\n identities=identities,\n )\n self.execution_count += 1\n\n # once we are done, send a signal that we are idle\n content = {\"execution_state\": \"idle\"}\n self.send(self.iopub_stream, \"status\", content, parent_header=message[\"header\"])\n\n def do_execute(\n self,\n code=\"\",\n silent=False,\n store_history=True,\n user_expressions=None,\n allow_stdin=False,\n parent_header=None,\n **kwargs\n ):\n \"\"\"Execute user code.\"\"\"\n if len(code.strip()) == 0:\n return {\n \"status\": \"ok\",\n \"execution_count\": self.execution_count,\n \"payload\": [],\n \"user_expressions\": {},\n }\n shell = builtins.__xonsh__.shell\n hist = builtins.__xonsh__.history\n try:\n shell.default(code, self, parent_header)\n interrupted = False\n except KeyboardInterrupt:\n interrupted = True\n\n if interrupted:\n return {\"status\": \"abort\", \"execution_count\": self.execution_count}\n\n rtn = 0 if (hist is None or len(hist) == 0) else hist.rtns[-1]\n if 0 < rtn:\n message = {\n \"status\": \"error\",\n \"execution_count\": self.execution_count,\n \"ename\": \"\",\n \"evalue\": str(rtn),\n \"traceback\": [],\n }\n else:\n message = {\n \"status\": \"ok\",\n \"execution_count\": self.execution_count,\n \"payload\": [],\n \"user_expressions\": {},\n }\n return message\n\n def _respond_in_chunks(self, name, s, chunksize=1024, parent_header=None):\n if s is None:\n return\n n = len(s)\n if n == 0:\n return\n lower = range(0, n, chunksize)\n upper = range(chunksize, n + chunksize, chunksize)\n for l, u in zip(lower, upper):\n response = {\"name\": name, \"text\": s[l:u]}\n self.send(\n self.iopub_socket, \"stream\", response, parent_header=parent_header\n )\n\n def handle_complete_request(self, message, identities):\n \"\"\"Handles kernel info requests.\"\"\"\n content = self.do_complete(\n message[\"content\"][\"code\"], message[\"content\"][\"cursor_pos\"]\n )\n self.send(\n self.shell_stream,\n \"complete_reply\",\n content,\n parent_header=message[\"header\"],\n identities=identities,\n )\n\n def do_complete(self, code, pos):\n \"\"\"Get completions.\"\"\"\n shell = builtins.__xonsh__.shell\n line = code.split(\"\\n\")[-1]\n line = builtins.aliases.expand_alias(line)\n prefix = line.split(\" \")[-1]\n endidx = pos\n begidx = pos - len(prefix)\n rtn, _ = self.completer.complete(prefix, line, begidx, endidx, shell.ctx)\n if isinstance(rtn, Set):\n rtn = list(rtn)\n message = {\n \"matches\": rtn,\n \"cursor_start\": begidx,\n \"cursor_end\": endidx,\n \"metadata\": {},\n \"status\": \"ok\",\n }\n return message\n\n def handle_kernel_info_request(self, message, identities):\n \"\"\"Handles kernel info requests.\"\"\"\n content = {\n \"protocol_version\": \"5.0\",\n \"ipython_version\": [1, 1, 0, \"\"],\n \"language\": self.language,\n \"language_version\": self.language_version,\n \"implementation\": self.implementation,\n \"implementation_version\": self.implementation_version,\n \"language_info\": self.language_info,\n \"banner\": self.banner,\n }\n self.send(\n self.shell_stream,\n \"kernel_info_reply\",\n content,\n parent_header=message[\"header\"],\n identities=identities,\n )\n\n\nif __name__ == \"__main__\":\n setup(\n shell_type=\"jupyter\",\n env={\"PAGER\": \"cat\"},\n aliases={\"less\": \"cat\"},\n xontribs=[\"coreutils\"],\n threadable_predictors={\"git\": predict_true, \"man\": predict_true},\n )\n if builtins.__xonsh__.commands_cache.is_only_functional_alias(\"cat\"):\n # this is needed if the underlying system doesn't have cat\n # we supply our own, because we can\n builtins.aliases[\"cat\"] = \"xonsh-cat\"\n builtins.__xonsh__.env[\"PAGER\"] = \"xonsh-cat\"\n shell = builtins.__xonsh__.shell\n kernel = shell.kernel = XonshKernel()\n kernel.start()\n" }, { "path": "xonsh/jupyter_shell.py", "content": "\"\"\"An interactive shell for the Jupyter kernel.\"\"\"\nimport io\nimport sys\nimport builtins\n\nfrom xonsh.base_shell import BaseShell\n\n\nclass StdJupyterRedirectBuf(io.RawIOBase):\n \"\"\"Redirects standard I/O buffers to the Jupyter kernel.\"\"\"\n\n def __init__(self, redirect):\n self.redirect = redirect\n self.encoding = redirect.encoding\n self.errors = redirect.errors\n\n def fileno(self):\n \"\"\"Returns the file descriptor of the std buffer.\"\"\"\n return self.redirect.fileno()\n\n def seek(self, offset, whence=io.SEEK_SET):\n \"\"\"Sets the location in both the stdbuf and the membuf.\"\"\"\n raise io.UnsupportedOperation(\"cannot seek Jupyter redirect\")\n\n def truncate(self, size=None):\n \"\"\"Truncate both buffers.\"\"\"\n raise io.UnsupportedOperation(\"cannot truncate Jupyter redirect\")\n\n def readinto(self, b):\n \"\"\"Read bytes into buffer from both streams.\"\"\"\n raise io.UnsupportedOperation(\"cannot read into Jupyter redirect\")\n\n def write(self, b):\n \"\"\"Write bytes to kernel.\"\"\"\n s = b if isinstance(b, str) else b.decode(self.encoding, self.errors)\n self.redirect.write(s)\n\n\nclass StdJupyterRedirect(io.TextIOBase):\n \"\"\"Redirects a standard I/O stream to the Jupyter kernel.\"\"\"\n\n def __init__(self, name, kernel, parent_header=None):\n \"\"\"\n Parameters\n ----------\n name : str\n The name of the buffer in the sys module, e.g. 'stdout'.\n kernel : XonshKernel\n Instance of a Jupyter kernel\n parent_header : dict or None, optional\n parent header information to pass along with the kernel\n \"\"\"\n self._name = name\n self.kernel = kernel\n self.parent_header = parent_header\n\n self.std = getattr(sys, name)\n self.buffer = StdJupyterRedirectBuf(self)\n setattr(sys, name, self)\n\n @property\n def encoding(self):\n \"\"\"The encoding of the stream\"\"\"\n env = builtins.__xonsh__.env\n return getattr(self.std, \"encoding\", env.get(\"XONSH_ENCODING\"))\n\n @property\n def errors(self):\n \"\"\"The encoding errors of the stream\"\"\"\n env = builtins.__xonsh__.env\n return getattr(self.std, \"errors\", env.get(\"XONSH_ENCODING_ERRORS\"))\n\n @property\n def newlines(self):\n \"\"\"The newlines of the standard buffer.\"\"\"\n return self.std.newlines\n\n def _replace_std(self):\n std = self.std\n if std is None:\n return\n setattr(sys, self._name, std)\n self.std = None\n\n def __del__(self):\n self._replace_std()\n\n def close(self):\n \"\"\"Restores the original std stream.\"\"\"\n self._replace_std()\n\n def __enter__(self):\n return self\n\n def __exit__(self, *args, **kwargs):\n self.close()\n\n def write(self, s):\n \"\"\"Writes data to the original kernel stream.\"\"\"\n self.kernel._respond_in_chunks(self._name, s, parent_header=self.parent_header)\n\n def flush(self):\n \"\"\"Flushes kernel iopub_stream.\"\"\"\n self.kernel.iopub_stream.flush()\n\n def fileno(self):\n \"\"\"Tunnel fileno() calls to the std stream.\"\"\"\n return self.std.fileno()\n\n def seek(self, offset, whence=io.SEEK_SET):\n \"\"\"Seek to a location.\"\"\"\n raise io.UnsupportedOperation(\"cannot seek Jupyter redirect\")\n\n def truncate(self, size=None):\n \"\"\"Truncate the streams.\"\"\"\n raise io.UnsupportedOperation(\"cannot truncate Jupyter redirect\")\n\n def detach(self):\n \"\"\"This operation is not supported.\"\"\"\n raise io.UnsupportedOperation(\"cannot detach a Jupyter redirect\")\n\n def read(self, size=None):\n \"\"\"Read from the stream\"\"\"\n raise io.UnsupportedOperation(\"cannot read a Jupyter redirect\")\n\n def readline(self, size=-1):\n \"\"\"Read a line.\"\"\"\n raise io.UnsupportedOperation(\"cannot read a line from a Jupyter redirect\")\n\n\nclass JupyterShell(BaseShell):\n \"\"\"A shell for the Jupyter kernel.\"\"\"\n\n def __init__(self, *args, **kwargs):\n super().__init__(*args, **kwargs)\n self.kernel = None\n\n def default(self, line, kernel, parent_header=None):\n \"\"\"Executes code, but redirects output to Jupyter client\"\"\"\n stdout = StdJupyterRedirect(\"stdout\", kernel, parent_header)\n stderr = StdJupyterRedirect(\"stderr\", kernel, parent_header)\n with stdout, stderr:\n rtn = super().default(line)\n return rtn\n" }, { "path": "xonsh/lazyasd.py", "content": "\"\"\"Lazy and self destructive containers for speeding up module import.\"\"\"\n# Copyright 2015-2016, the xonsh developers. All rights reserved.\nimport os\nimport sys\nimport time\nimport types\nimport builtins\nimport threading\nimport importlib\nimport importlib.util\nimport collections.abc as cabc\n\n__version__ = \"0.1.3\"\n\n\nclass LazyObject(object):\n def __init__(self, load, ctx, name):\n \"\"\"Lazily loads an object via the load function the first time an\n attribute is accessed. Once loaded it will replace itself in the\n provided context (typically the globals of the call site) with the\n given name.\n\n For example, you can prevent the compilation of a regular expression\n until it is actually used::\n\n DOT = LazyObject((lambda: re.compile('.')), globals(), 'DOT')\n\n Parameters\n ----------\n load : function with no arguments\n A loader function that performs the actual object construction.\n ctx : Mapping\n Context to replace the LazyObject instance in\n with the object returned by load().\n name : str\n Name in the context to give the loaded object. This *should*\n be the name on the LHS of the assignment.\n \"\"\"\n self._lasdo = {\"loaded\": False, \"load\": load, \"ctx\": ctx, \"name\": name}\n\n def _lazy_obj(self):\n d = self._lasdo\n if d[\"loaded\"]:\n obj = d[\"obj\"]\n else:\n obj = d[\"load\"]()\n d[\"ctx\"][d[\"name\"]] = d[\"obj\"] = obj\n d[\"loaded\"] = True\n return obj\n\n def __getattribute__(self, name):\n if name == \"_lasdo\" or name == \"_lazy_obj\":\n return super().__getattribute__(name)\n obj = self._lazy_obj()\n return getattr(obj, name)\n\n def __bool__(self):\n obj = self._lazy_obj()\n return bool(obj)\n\n def __iter__(self):\n obj = self._lazy_obj()\n yield from obj\n\n def __getitem__(self, item):\n obj = self._lazy_obj()\n return obj[item]\n\n def __setitem__(self, key, value):\n obj = self._lazy_obj()\n obj[key] = value\n\n def __delitem__(self, item):\n obj = self._lazy_obj()\n del obj[item]\n\n def __call__(self, *args, **kwargs):\n obj = self._lazy_obj()\n return obj(*args, **kwargs)\n\n def __lt__(self, other):\n obj = self._lazy_obj()\n return obj < other\n\n def __le__(self, other):\n obj = self._lazy_obj()\n return obj <= other\n\n def __eq__(self, other):\n obj = self._lazy_obj()\n return obj == other\n\n def __ne__(self, other):\n obj = self._lazy_obj()\n return obj != other\n\n def __gt__(self, other):\n obj = self._lazy_obj()\n return obj > other\n\n def __ge__(self, other):\n obj = self._lazy_obj()\n return obj >= other\n\n def __hash__(self):\n obj = self._lazy_obj()\n return hash(obj)\n\n def __or__(self, other):\n obj = self._lazy_obj()\n return obj | other\n\n def __str__(self):\n return str(self._lazy_obj())\n\n def __repr__(self):\n return repr(self._lazy_obj())\n\n\ndef lazyobject(f):\n \"\"\"Decorator for constructing lazy objects from a function.\"\"\"\n return LazyObject(f, f.__globals__, f.__name__)\n\n\nclass LazyDict(cabc.MutableMapping):\n def __init__(self, loaders, ctx, name):\n \"\"\"Dictionary like object that lazily loads its values from an initial\n dict of key-loader function pairs. Each key is loaded when its value\n is first accessed. Once fully loaded, this object will replace itself\n in the provided context (typically the globals of the call site) with\n the given name.\n\n For example, you can prevent the compilation of a bunch of regular\n expressions until they are actually used::\n\n RES = LazyDict({\n 'dot': lambda: re.compile('.'),\n 'all': lambda: re.compile('.*'),\n 'two': lambda: re.compile('..'),\n }, globals(), 'RES')\n\n Parameters\n ----------\n loaders : Mapping of keys to functions with no arguments\n A mapping of loader function that performs the actual value\n construction upon access.\n ctx : Mapping\n Context to replace the LazyDict instance in\n with the the fully loaded mapping.\n name : str\n Name in the context to give the loaded mapping. This *should*\n be the name on the LHS of the assignment.\n \"\"\"\n self._loaders = loaders\n self._ctx = ctx\n self._name = name\n self._d = type(loaders)() # make sure to return the same type\n\n def _destruct(self):\n if len(self._loaders) == 0:\n self._ctx[self._name] = self._d\n\n def __getitem__(self, key):\n d = self._d\n if key in d:\n val = d[key]\n else:\n # pop will raise a key error for us\n loader = self._loaders.pop(key)\n d[key] = val = loader()\n self._destruct()\n return val\n\n def __setitem__(self, key, value):\n self._d[key] = value\n if key in self._loaders:\n del self._loaders[key]\n self._destruct()\n\n def __delitem__(self, key):\n if key in self._d:\n del self._d[key]\n else:\n del self._loaders[key]\n self._destruct()\n\n def __iter__(self):\n yield from (set(self._d.keys()) | set(self._loaders.keys()))\n\n def __len__(self):\n return len(self._d) + len(self._loaders)\n\n\ndef lazydict(f):\n \"\"\"Decorator for constructing lazy dicts from a function.\"\"\"\n return LazyDict(f, f.__globals__, f.__name__)\n\n\nclass LazyBool(object):\n def __init__(self, load, ctx, name):\n \"\"\"Boolean like object that lazily computes it boolean value when it is\n first asked. Once loaded, this result will replace itself\n in the provided context (typically the globals of the call site) with\n the given name.\n\n For example, you can prevent the complex boolean until it is actually\n used::\n\n ALIVE = LazyDict(lambda: not DEAD, globals(), 'ALIVE')\n\n Parameters\n ----------\n load : function with no arguments\n A loader function that performs the actual boolean evaluation.\n ctx : Mapping\n Context to replace the LazyBool instance in\n with the the fully loaded mapping.\n name : str\n Name in the context to give the loaded mapping. This *should*\n be the name on the LHS of the assignment.\n \"\"\"\n self._load = load\n self._ctx = ctx\n self._name = name\n self._result = None\n\n def __bool__(self):\n if self._result is None:\n res = self._ctx[self._name] = self._result = self._load()\n else:\n res = self._result\n return res\n\n\ndef lazybool(f):\n \"\"\"Decorator for constructing lazy booleans from a function.\"\"\"\n return LazyBool(f, f.__globals__, f.__name__)\n\n\n#\n# Background module loaders\n#\n\n\nclass BackgroundModuleProxy(types.ModuleType):\n \"\"\"Proxy object for modules loaded in the background that block attribute\n access until the module is loaded..\n \"\"\"\n\n def __init__(self, modname):\n self.__dct__ = {\"loaded\": False, \"modname\": modname}\n\n def __getattribute__(self, name):\n passthrough = frozenset({\"__dct__\", \"__class__\", \"__spec__\"})\n if name in passthrough:\n return super().__getattribute__(name)\n dct = self.__dct__\n modname = dct[\"modname\"]\n if dct[\"loaded\"]:\n mod = sys.modules[modname]\n else:\n delay_types = (BackgroundModuleProxy, type(None))\n while isinstance(sys.modules.get(modname, None), delay_types):\n time.sleep(0.001)\n mod = sys.modules[modname]\n dct[\"loaded\"] = True\n # some modules may do construction after import, give them a second\n stall = 0\n while not hasattr(mod, name) and stall < 1000:\n stall += 1\n time.sleep(0.001)\n return getattr(mod, name)\n\n\nclass BackgroundModuleLoader(threading.Thread):\n \"\"\"Thread to load modules in the background.\"\"\"\n\n def __init__(self, name, package, replacements, *args, **kwargs):\n super().__init__(*args, **kwargs)\n self.daemon = True\n self.name = name\n self.package = package\n self.replacements = replacements\n self.start()\n\n def run(self):\n # wait for other modules to stop being imported\n # We assume that module loading is finished when sys.modules doesn't\n # get longer in 5 consecutive 1ms waiting steps\n counter = 0\n last = -1\n while counter < 5:\n new = len(sys.modules)\n if new == last:\n counter += 1\n else:\n last = new\n counter = 0\n time.sleep(0.001)\n # now import module properly\n modname = importlib.util.resolve_name(self.name, self.package)\n if isinstance(sys.modules[modname], BackgroundModuleProxy):\n del sys.modules[modname]\n mod = importlib.import_module(self.name, package=self.package)\n for targname, varname in self.replacements.items():\n if targname in sys.modules:\n targmod = sys.modules[targname]\n setattr(targmod, varname, mod)\n\n\ndef load_module_in_background(\n name, package=None, debug=\"DEBUG\", env=None, replacements=None\n):\n \"\"\"Entry point for loading modules in background thread.\n\n Parameters\n ----------\n name : str\n Module name to load in background thread.\n package : str or None, optional\n Package name, has the same meaning as in importlib.import_module().\n debug : str, optional\n Debugging symbol name to look up in the environment.\n env : Mapping or None, optional\n Environment this will default to __xonsh__.env, if available, and\n os.environ otherwise.\n replacements : Mapping or None, optional\n Dictionary mapping fully qualified module names (eg foo.bar.baz) that\n import the lazily loaded module, with the variable name in that\n module. For example, suppose that foo.bar imports module a as b,\n this dict is then {'foo.bar': 'b'}.\n\n Returns\n -------\n module : ModuleType\n This is either the original module that is found in sys.modules or\n a proxy module that will block until delay attribute access until the\n module is fully loaded.\n \"\"\"\n modname = importlib.util.resolve_name(name, package)\n if modname in sys.modules:\n return sys.modules[modname]\n if env is None:\n xonsh_obj = getattr(builtins, \"__xonsh__\", None)\n env = os.environ if xonsh_obj is None else getattr(xonsh_obj, \"env\", os.environ)\n if env.get(debug, None):\n mod = importlib.import_module(name, package=package)\n return mod\n proxy = sys.modules[modname] = BackgroundModuleProxy(modname)\n BackgroundModuleLoader(name, package, replacements or {})\n return proxy\n" }, { "path": "xonsh/lazyimps.py", "content": "\"\"\"Lazy imports that may apply across the xonsh package.\"\"\"\nimport importlib\n\nfrom xonsh.platform import ON_WINDOWS, ON_DARWIN\nfrom xonsh.lazyasd import LazyObject, lazyobject\n\npygments = LazyObject(\n lambda: importlib.import_module(\"pygments\"), globals(), \"pygments\"\n)\npyghooks = LazyObject(\n lambda: importlib.import_module(\"xonsh.pyghooks\"), globals(), \"pyghooks\"\n)\n\n\n@lazyobject\ndef pty():\n if ON_WINDOWS:\n return\n else:\n return importlib.import_module(\"pty\")\n\n\n@lazyobject\ndef termios():\n if ON_WINDOWS:\n return\n else:\n return importlib.import_module(\"termios\")\n\n\n@lazyobject\ndef fcntl():\n if ON_WINDOWS:\n return\n else:\n return importlib.import_module(\"fcntl\")\n\n\n@lazyobject\ndef tty():\n if ON_WINDOWS:\n return\n else:\n return importlib.import_module(\"tty\")\n\n\n@lazyobject\ndef _winapi():\n if ON_WINDOWS:\n import _winapi as m\n else:\n m = None\n return m\n\n\n@lazyobject\ndef msvcrt():\n if ON_WINDOWS:\n import msvcrt as m\n else:\n m = None\n return m\n\n\n@lazyobject\ndef winutils():\n if ON_WINDOWS:\n import xonsh.winutils as m\n else:\n m = None\n return m\n\n\n@lazyobject\ndef macutils():\n if ON_DARWIN:\n import xonsh.macutils as m\n else:\n m = None\n return m\n\n\n@lazyobject\ndef terminal256():\n return importlib.import_module(\"pygments.formatters.terminal256\")\n" }, { "path": "xonsh/lazyjson.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"Implements a lazy JSON file class that wraps around json data.\"\"\"\nimport io\nimport json\nimport weakref\nimport contextlib\nimport collections.abc as cabc\n\n\ndef _to_json_with_size(obj, offset=0, sort_keys=False):\n if isinstance(obj, str):\n s = json.dumps(obj)\n o = offset\n n = size = len(s.encode()) # size in bytes\n elif isinstance(obj, cabc.Mapping):\n s = \"{\"\n j = offset + 1\n o = {}\n size = {}\n items = sorted(obj.items()) if sort_keys else obj.items()\n for key, val in items:\n s_k, o_k, n_k, size_k = _to_json_with_size(\n key, offset=j, sort_keys=sort_keys\n )\n s += s_k + \": \"\n j += n_k + 2\n s_v, o_v, n_v, size_v = _to_json_with_size(\n val, offset=j, sort_keys=sort_keys\n )\n o[key] = o_v\n size[key] = size_v\n s += s_v + \", \"\n j += n_v + 2\n if s.endswith(\", \"):\n s = s[:-2]\n s += \"}\\n\"\n n = len(s)\n o[\"__total__\"] = offset\n size[\"__total__\"] = n\n elif isinstance(obj, cabc.Sequence):\n s = \"[\"\n j = offset + 1\n o = []\n size = []\n for x in obj:\n s_x, o_x, n_x, size_x = _to_json_with_size(x, offset=j, sort_keys=sort_keys)\n o.append(o_x)\n size.append(size_x)\n s += s_x + \", \"\n j += n_x + 2\n if s.endswith(\", \"):\n s = s[:-2]\n s += \"]\\n\"\n n = len(s)\n o.append(offset)\n size.append(n)\n else:\n s = json.dumps(obj, sort_keys=sort_keys)\n o = offset\n n = size = len(s)\n return s, o, n, size\n\n\ndef index(obj, sort_keys=False):\n \"\"\"Creates an index for a JSON file.\"\"\"\n idx = {}\n json_obj = _to_json_with_size(obj, sort_keys=sort_keys)\n s, idx[\"offsets\"], _, idx[\"sizes\"] = json_obj\n return s, idx\n\n\nJSON_FORMAT = \"\"\"{{\"locs\": [{iloc:>10}, {ilen:>10}, {dloc:>10}, {dlen:>10}],\n \"index\": {index},\n \"data\": {data}\n}}\n\"\"\"\n\n\ndef dumps(obj, sort_keys=False):\n \"\"\"Dumps an object to JSON with an index.\"\"\"\n data, idx = index(obj, sort_keys=sort_keys)\n jdx = json.dumps(idx, sort_keys=sort_keys)\n iloc = 69\n ilen = len(jdx)\n dloc = iloc + ilen + 11\n dlen = len(data)\n s = JSON_FORMAT.format(\n index=jdx, data=data, iloc=iloc, ilen=ilen, dloc=dloc, dlen=dlen\n )\n return s\n\n\ndef ljdump(obj, fp, sort_keys=False):\n \"\"\"Dumps an object to JSON file.\"\"\"\n s = dumps(obj, sort_keys=sort_keys)\n fp.write(s)\n\n\nclass LJNode(cabc.Mapping, cabc.Sequence):\n \"\"\"A proxy node for JSON nodes. Acts as both sequence and mapping.\"\"\"\n\n def __init__(self, offsets, sizes, root):\n \"\"\"Parameters\n ----------\n offsets : dict, list, or int\n offsets of corresponding data structure, in bytes\n sizes : dict, list, or int\n sizes of corresponding data structure, in bytes\n root : weakref.proxy of LazyJSON\n weakref back to root node, which should be a LazyJSON object.\n \"\"\"\n self.offsets = offsets\n self.sizes = sizes\n self.root = root\n self.is_mapping = isinstance(self.offsets, cabc.Mapping)\n self.is_sequence = isinstance(self.offsets, cabc.Sequence)\n\n def __len__(self):\n # recall that for maps, the '__total__' key is added and for\n # sequences the last element represents the total size/offset.\n return len(self.sizes) - 1\n\n def load(self):\n \"\"\"Returns the Python data structure represented by the node.\"\"\"\n if self.is_mapping:\n offset = self.offsets[\"__total__\"]\n size = self.sizes[\"__total__\"]\n elif self.is_sequence:\n offset = self.offsets[-1]\n size = self.sizes[-1]\n elif isinstance(self.offsets, int):\n offset = self.offsets\n size = self.sizes\n return self._load_or_node(offset, size)\n\n def _load_or_node(self, offset, size):\n if isinstance(offset, int):\n with self.root._open(newline=\"\\n\") as f:\n f.seek(self.root.dloc + offset)\n s = f.read(size)\n val = json.loads(s)\n elif isinstance(offset, (cabc.Mapping, cabc.Sequence)):\n val = LJNode(offset, size, self.root)\n else:\n raise TypeError(\"incorrect types for offset node\")\n return val\n\n def _getitem_mapping(self, key):\n if key == \"__total__\":\n raise KeyError('\"__total__\" is a special LazyJSON key!')\n offset = self.offsets[key]\n size = self.sizes[key]\n return self._load_or_node(offset, size)\n\n def _getitem_sequence(self, key):\n if isinstance(key, int):\n rtn = self._load_or_node(self.offsets[key], self.sizes[key])\n elif isinstance(key, slice):\n key = slice(*key.indices(len(self)))\n rtn = list(map(self._load_or_node, self.offsets[key], self.sizes[key]))\n else:\n raise TypeError(\"only integer indexing available\")\n return rtn\n\n def __getitem__(self, key):\n if self.is_mapping:\n rtn = self._getitem_mapping(key)\n elif self.is_sequence:\n rtn = self._getitem_sequence(key)\n else:\n raise NotImplementedError\n return rtn\n\n def __iter__(self):\n if self.is_mapping:\n keys = set(self.offsets.keys())\n keys.discard(\"__total__\")\n yield from iter(keys)\n elif self.is_sequence:\n i = 0\n n = len(self)\n while i < n:\n yield self._load_or_node(self.offsets[i], self.sizes[i])\n i += 1\n else:\n raise NotImplementedError\n\n\nclass LazyJSON(LJNode):\n \"\"\"Represents a lazy json file. Can be used like a normal Python\n dict or list.\n \"\"\"\n\n def __init__(self, f, reopen=True):\n \"\"\"Parameters\n ----------\n f : file handle or str\n JSON file to open.\n reopen : bool, optional\n Whether new file handle should be opened for each load.\n \"\"\"\n self._f = f\n self.reopen = reopen\n if not reopen and isinstance(f, str):\n self._f = open(f, \"r\", newline=\"\\n\")\n self._load_index()\n self.root = weakref.proxy(self)\n self.is_mapping = isinstance(self.offsets, cabc.Mapping)\n self.is_sequence = isinstance(self.offsets, cabc.Sequence)\n\n def __del__(self):\n self.close()\n\n def close(self):\n \"\"\"Close the file handle, if appropriate.\"\"\"\n if not self.reopen and isinstance(self._f, io.IOBase):\n try:\n self._f.close()\n except OSError:\n pass\n\n @contextlib.contextmanager\n def _open(self, *args, **kwargs):\n if self.reopen and isinstance(self._f, str):\n f = open(self._f, *args, **kwargs)\n yield f\n f.close()\n else:\n yield self._f\n\n def _load_index(self):\n \"\"\"Loads the index from the start of the file.\"\"\"\n with self._open(newline=\"\\n\") as f:\n # read in the location data\n f.seek(9)\n locs = f.read(48)\n locs = json.loads(locs)\n self.iloc, self.ilen, self.dloc, self.dlen = locs\n # read in the index\n f.seek(self.iloc)\n idx = f.read(self.ilen)\n idx = json.loads(idx)\n self.offsets = idx[\"offsets\"]\n self.sizes = idx[\"sizes\"]\n\n def __enter__(self):\n return self\n\n def __exit__(self, exc_type, exc_value, traceback):\n self.close()\n" }, { "path": "xonsh/lexer.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"Lexer for xonsh code.\n\nWritten using a hybrid of ``tokenize`` and PLY.\n\"\"\"\nimport io\nimport re\n\n# 'keyword' interferes with ast.keyword\nimport keyword as kwmod\n\nfrom xonsh.ply.ply.lex import LexToken\n\nfrom xonsh.lazyasd import lazyobject\nfrom xonsh.platform import PYTHON_VERSION_INFO\nfrom xonsh.tokenize import (\n OP,\n IOREDIRECT,\n STRING,\n DOLLARNAME,\n NUMBER,\n SEARCHPATH,\n NEWLINE,\n INDENT,\n DEDENT,\n NL,\n COMMENT,\n ENCODING,\n ENDMARKER,\n NAME,\n ERRORTOKEN,\n GREATER,\n LESS,\n RIGHTSHIFT,\n tokenize,\n TokenError,\n)\n\n\n@lazyobject\ndef token_map():\n \"\"\"Mapping from ``tokenize`` tokens (or token types) to PLY token types. If\n a simple one-to-one mapping from ``tokenize`` to PLY exists, the lexer will\n look it up here and generate a single PLY token of the given type.\n Otherwise, it will fall back to handling that token using one of the\n handlers in``special_handlers``.\n \"\"\"\n tm = {}\n # operators\n _op_map = {\n # punctuation\n \",\": \"COMMA\",\n \".\": \"PERIOD\",\n \";\": \"SEMI\",\n \":\": \"COLON\",\n \"...\": \"ELLIPSIS\",\n # basic operators\n \"+\": \"PLUS\",\n \"-\": \"MINUS\",\n \"*\": \"TIMES\",\n \"@\": \"AT\",\n \"/\": \"DIVIDE\",\n \"//\": \"DOUBLEDIV\",\n \"%\": \"MOD\",\n \"**\": \"POW\",\n \"|\": \"PIPE\",\n \"~\": \"TILDE\",\n \"^\": \"XOR\",\n \"<<\": \"LSHIFT\",\n \">>\": \"RSHIFT\",\n \"<\": \"LT\",\n \"<=\": \"LE\",\n \">\": \"GT\",\n \">=\": \"GE\",\n \"==\": \"EQ\",\n \"!=\": \"NE\",\n \"->\": \"RARROW\",\n # assignment operators\n \"=\": \"EQUALS\",\n \"+=\": \"PLUSEQUAL\",\n \"-=\": \"MINUSEQUAL\",\n \"*=\": \"TIMESEQUAL\",\n \"@=\": \"ATEQUAL\",\n \"/=\": \"DIVEQUAL\",\n \"%=\": \"MODEQUAL\",\n \"**=\": \"POWEQUAL\",\n \"<<=\": \"LSHIFTEQUAL\",\n \">>=\": \"RSHIFTEQUAL\",\n \"&=\": \"AMPERSANDEQUAL\",\n \"^=\": \"XOREQUAL\",\n \"|=\": \"PIPEEQUAL\",\n \"//=\": \"DOUBLEDIVEQUAL\",\n # extra xonsh operators\n \"?\": \"QUESTION\",\n \"??\": \"DOUBLE_QUESTION\",\n \"@$\": \"ATDOLLAR\",\n \"&\": \"AMPERSAND\",\n }\n for (op, typ) in _op_map.items():\n tm[(OP, op)] = typ\n tm[IOREDIRECT] = \"IOREDIRECT\"\n tm[STRING] = \"STRING\"\n tm[DOLLARNAME] = \"DOLLAR_NAME\"\n tm[NUMBER] = \"NUMBER\"\n tm[SEARCHPATH] = \"SEARCHPATH\"\n tm[NEWLINE] = \"NEWLINE\"\n tm[INDENT] = \"INDENT\"\n tm[DEDENT] = \"DEDENT\"\n if (3, 5, 0) <= PYTHON_VERSION_INFO < (3, 7, 0):\n from xonsh.tokenize import ASYNC, AWAIT\n\n tm[ASYNC] = \"ASYNC\"\n tm[AWAIT] = \"AWAIT\"\n return tm\n\n\nNEED_WHITESPACE = frozenset([\"and\", \"or\"])\n\n\n@lazyobject\ndef RE_NEED_WHITESPACE():\n pattern = r\"\\s?(\" + \"|\".join(NEED_WHITESPACE) + r\")(\\s|[\\\\]$)\"\n return re.compile(pattern)\n\n\ndef handle_name(state, token):\n \"\"\"Function for handling name tokens\"\"\"\n typ = \"NAME\"\n state[\"last\"] = token\n needs_whitespace = token.string in NEED_WHITESPACE\n has_whitespace = needs_whitespace and RE_NEED_WHITESPACE.match(\n token.line[max(0, token.start[1] - 1) :]\n )\n if state[\"pymode\"][-1][0]:\n if needs_whitespace and not has_whitespace:\n pass\n elif token.string in kwmod.kwlist:\n typ = token.string.upper()\n yield _new_token(typ, token.string, token.start)\n else:\n if has_whitespace and token.string == \"and\":\n yield _new_token(\"AND\", token.string, token.start)\n elif has_whitespace and token.string == \"or\":\n yield _new_token(\"OR\", token.string, token.start)\n else:\n yield _new_token(\"NAME\", token.string, token.start)\n\n\ndef _end_delimiter(state, token):\n py = state[\"pymode\"]\n s = token.string\n l, c = token.start\n if len(py) > 1:\n mode, orig, match, pos = py.pop()\n if s != match:\n e = '\"{}\" at {} ends \"{}\" at {} (expected \"{}\")'\n return e.format(s, (l, c), orig, pos, match)\n else:\n return 'Unmatched \"{}\" at line {}, column {}'.format(s, l, c)\n\n\ndef handle_rparen(state, token):\n \"\"\"\n Function for handling ``)``\n \"\"\"\n e = _end_delimiter(state, token)\n if e is None:\n state[\"last\"] = token\n yield _new_token(\"RPAREN\", \")\", token.start)\n else:\n yield _new_token(\"ERRORTOKEN\", e, token.start)\n\n\ndef handle_rbrace(state, token):\n \"\"\"Function for handling ``}``\"\"\"\n e = _end_delimiter(state, token)\n if e is None:\n state[\"last\"] = token\n yield _new_token(\"RBRACE\", \"}\", token.start)\n else:\n yield _new_token(\"ERRORTOKEN\", e, token.start)\n\n\ndef handle_rbracket(state, token):\n \"\"\"\n Function for handling ``]``\n \"\"\"\n e = _end_delimiter(state, token)\n if e is None:\n state[\"last\"] = token\n yield _new_token(\"RBRACKET\", \"]\", token.start)\n else:\n yield _new_token(\"ERRORTOKEN\", e, token.start)\n\n\ndef handle_error_space(state, token):\n \"\"\"\n Function for handling special whitespace characters in subprocess mode\n \"\"\"\n if not state[\"pymode\"][-1][0]:\n state[\"last\"] = token\n yield _new_token(\"WS\", token.string, token.start)\n else:\n yield from []\n\n\ndef handle_error_linecont(state, token):\n \"\"\"Function for handling special line continuations as whitespace\n characters in subprocess mode.\n \"\"\"\n if state[\"pymode\"][-1][0]:\n return\n prev = state[\"last\"]\n if prev.end != token.start:\n return # previous token is separated by whitespace\n state[\"last\"] = token\n yield _new_token(\"WS\", \"\\\\\", token.start)\n\n\ndef handle_error_token(state, token):\n \"\"\"\n Function for handling error tokens\n \"\"\"\n state[\"last\"] = token\n if token.string == \"!\":\n typ = \"BANG\"\n elif not state[\"pymode\"][-1][0]:\n typ = \"NAME\"\n else:\n typ = \"ERRORTOKEN\"\n yield _new_token(typ, token.string, token.start)\n\n\ndef handle_ignore(state, token):\n \"\"\"Function for handling tokens that should be ignored\"\"\"\n yield from []\n\n\ndef handle_double_amps(state, token):\n yield _new_token(\"AND\", \"and\", token.start)\n\n\ndef handle_double_pipe(state, token):\n yield _new_token(\"OR\", \"or\", token.start)\n\n\ndef handle_redirect(state, token):\n # The parser expects whitespace after a redirection in subproc mode.\n # If whitespace does not exist, we'll issue an empty whitespace\n # token before proceeding.\n state[\"last\"] = token\n typ = token.type\n st = token.string\n key = (typ, st) if (typ, st) in token_map else typ\n yield _new_token(token_map[key], st, token.start)\n if state[\"pymode\"][-1][0]:\n return\n # add a whitespace token after a redirection, if we need to\n next_tok = next(state[\"stream\"])\n if next_tok.start == token.end:\n yield _new_token(\"WS\", \"\", token.end)\n yield from handle_token(state, next_tok)\n\n\ndef _make_matcher_handler(tok, typ, pymode, ender, handlers):\n matcher = (\n \")\"\n if tok.endswith(\"(\")\n else \"}\"\n if tok.endswith(\"{\")\n else \"]\"\n if tok.endswith(\"[\")\n else None\n )\n\n def _inner_handler(state, token):\n state[\"pymode\"].append((pymode, tok, matcher, token.start))\n state[\"last\"] = token\n yield _new_token(typ, tok, token.start)\n\n handlers[(OP, tok)] = _inner_handler\n\n\n@lazyobject\ndef special_handlers():\n \"\"\"Mapping from ``tokenize`` tokens (or token types) to the proper\n function for generating PLY tokens from them. In addition to\n yielding PLY tokens, these functions may manipulate the Lexer's state.\n \"\"\"\n sh = {\n NL: handle_ignore,\n COMMENT: handle_ignore,\n ENCODING: handle_ignore,\n ENDMARKER: handle_ignore,\n NAME: handle_name,\n ERRORTOKEN: handle_error_token,\n LESS: handle_redirect,\n GREATER: handle_redirect,\n RIGHTSHIFT: handle_redirect,\n IOREDIRECT: handle_redirect,\n (OP, \"<\"): handle_redirect,\n (OP, \">\"): handle_redirect,\n (OP, \">>\"): handle_redirect,\n (OP, \")\"): handle_rparen,\n (OP, \"}\"): handle_rbrace,\n (OP, \"]\"): handle_rbracket,\n (OP, \"&&\"): handle_double_amps,\n (OP, \"||\"): handle_double_pipe,\n (ERRORTOKEN, \" \"): handle_error_space,\n (ERRORTOKEN, \"\\\\\\n\"): handle_error_linecont,\n (ERRORTOKEN, \"\\\\\\r\\n\"): handle_error_linecont,\n }\n _make_matcher_handler(\"(\", \"LPAREN\", True, \")\", sh)\n _make_matcher_handler(\"[\", \"LBRACKET\", True, \"]\", sh)\n _make_matcher_handler(\"{\", \"LBRACE\", True, \"}\", sh)\n _make_matcher_handler(\"$(\", \"DOLLAR_LPAREN\", False, \")\", sh)\n _make_matcher_handler(\"$[\", \"DOLLAR_LBRACKET\", False, \"]\", sh)\n _make_matcher_handler(\"${\", \"DOLLAR_LBRACE\", True, \"}\", sh)\n _make_matcher_handler(\"!(\", \"BANG_LPAREN\", False, \")\", sh)\n _make_matcher_handler(\"![\", \"BANG_LBRACKET\", False, \"]\", sh)\n _make_matcher_handler(\"@(\", \"AT_LPAREN\", True, \")\", sh)\n _make_matcher_handler(\"@$(\", \"ATDOLLAR_LPAREN\", False, \")\", sh)\n return sh\n\n\ndef handle_token(state, token):\n \"\"\"\n General-purpose token handler. Makes use of ``token_map`` or\n ``special_map`` to yield one or more PLY tokens from the given input.\n\n Parameters\n ----------\n\n state :\n The current state of the lexer, including information about whether\n we are in Python mode or subprocess mode, which changes the lexer's\n behavior. Also includes the stream of tokens yet to be considered.\n token :\n The token (from ``tokenize``) currently under consideration\n \"\"\"\n typ = token.type\n st = token.string\n pymode = state[\"pymode\"][-1][0]\n if not pymode:\n if state[\"last\"] is not None and state[\"last\"].end != token.start:\n cur = token.start\n old = state[\"last\"].end\n if cur[0] == old[0] and cur[1] > old[1]:\n yield _new_token(\"WS\", token.line[old[1] : cur[1]], old)\n if (typ, st) in special_handlers:\n yield from special_handlers[(typ, st)](state, token)\n elif (typ, st) in token_map:\n state[\"last\"] = token\n yield _new_token(token_map[(typ, st)], st, token.start)\n elif typ in special_handlers:\n yield from special_handlers[typ](state, token)\n elif typ in token_map:\n state[\"last\"] = token\n yield _new_token(token_map[typ], st, token.start)\n else:\n m = \"Unexpected token: {0}\".format(token)\n yield _new_token(\"ERRORTOKEN\", m, token.start)\n\n\ndef get_tokens(s):\n \"\"\"\n Given a string containing xonsh code, generates a stream of relevant PLY\n tokens using ``handle_token``.\n \"\"\"\n state = {\n \"indents\": [0],\n \"last\": None,\n \"pymode\": [(True, \"\", \"\", (0, 0))],\n \"stream\": tokenize(io.BytesIO(s.encode(\"utf-8\")).readline),\n }\n while True:\n try:\n token = next(state[\"stream\"])\n yield from handle_token(state, token)\n except StopIteration:\n if len(state[\"pymode\"]) > 1:\n pm, o, m, p = state[\"pymode\"][-1]\n l, c = p\n e = 'Unmatched \"{}\" at line {}, column {}'\n yield _new_token(\"ERRORTOKEN\", e.format(o, l, c), (0, 0))\n break\n except TokenError as e:\n # this is recoverable in single-line mode (from the shell)\n # (e.g., EOF while scanning string literal)\n yield _new_token(\"ERRORTOKEN\", e.args[0], (0, 0))\n break\n except IndentationError as e:\n # this is never recoverable\n yield _new_token(\"ERRORTOKEN\", e, (0, 0))\n break\n\n\n# synthesize a new PLY token\ndef _new_token(type, value, pos):\n o = LexToken()\n o.type = type\n o.value = value\n o.lineno, o.lexpos = pos\n return o\n\n\nclass Lexer(object):\n \"\"\"Implements a lexer for the xonsh language.\"\"\"\n\n _tokens = None\n\n def __init__(self):\n \"\"\"\n Attributes\n ----------\n fname : str\n Filename\n last : token\n The last token seen.\n lineno : int\n The last line number seen.\n\n \"\"\"\n self.fname = \"\"\n self.last = None\n self.beforelast = None\n\n def build(self, **kwargs):\n \"\"\"Part of the PLY lexer API.\"\"\"\n pass\n\n def reset(self):\n pass\n\n def input(self, s):\n \"\"\"Calls the lexer on the string s.\"\"\"\n self.token_stream = get_tokens(s)\n\n def token(self):\n \"\"\"Retrieves the next token.\"\"\"\n self.beforelast = self.last\n self.last = next(self.token_stream, None)\n return self.last\n\n def __iter__(self):\n t = self.token()\n while t is not None:\n yield t\n t = self.token()\n\n def split(self, s):\n \"\"\"Splits a string into a list of strings which are whitespace-separated\n tokens.\n \"\"\"\n vals = []\n self.input(s)\n l = c = -1\n ws = \"WS\"\n nl = \"\\n\"\n for t in self:\n if t.type == ws:\n continue\n elif l < t.lineno:\n vals.append(t.value)\n elif len(vals) > 0 and c == t.lexpos:\n vals[-1] = vals[-1] + t.value\n else:\n vals.append(t.value)\n nnl = t.value.count(nl)\n if nnl == 0:\n l = t.lineno\n c = t.lexpos + len(t.value)\n else:\n l = t.lineno + nnl\n c = len(t.value.rpartition(nl)[-1])\n return vals\n\n #\n # All the tokens recognized by the lexer\n #\n @property\n def tokens(self):\n if self._tokens is None:\n t = (\n tuple(token_map.values())\n + (\n \"NAME\", # name tokens\n \"BANG\", # ! tokens\n \"WS\", # whitespace in subprocess mode\n \"LPAREN\",\n \"RPAREN\", # ( )\n \"LBRACKET\",\n \"RBRACKET\", # [ ]\n \"LBRACE\",\n \"RBRACE\", # { }\n \"AT_LPAREN\", # @(\n \"BANG_LPAREN\", # !(\n \"BANG_LBRACKET\", # ![\n \"DOLLAR_LPAREN\", # $(\n \"DOLLAR_LBRACE\", # ${\n \"DOLLAR_LBRACKET\", # $[\n \"ATDOLLAR_LPAREN\", # @$(\n \"ERRORTOKEN\", # whoops!\n )\n + tuple(i.upper() for i in kwmod.kwlist)\n )\n self._tokens = t\n return self._tokens\n" }, { "path": "xonsh/macutils.py", "content": "\"\"\"Provides some Mac / Darwin based utility functions for xonsh.\"\"\"\nfrom ctypes import c_uint, byref, create_string_buffer\n\nfrom xonsh.platform import LIBC\n\n\ndef sysctlbyname(name, return_str=True):\n \"\"\"Gets a sysctl value by name. If return_str is true, this will return\n a string representation, else it will return the raw value.\n \"\"\"\n # forked from https://gist.github.com/pudquick/581a71425439f2cf8f09\n size = c_uint(0)\n # Find out how big our buffer will be\n LIBC.sysctlbyname(name, None, byref(size), None, 0)\n # Make the buffer\n buf = create_string_buffer(size.value)\n # Re-run, but provide the buffer\n LIBC.sysctlbyname(name, buf, byref(size), None, 0)\n if return_str:\n return buf.value\n else:\n return buf.raw\n" }, { "path": "xonsh/main.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"The main xonsh script.\"\"\"\nimport os\nimport sys\nimport enum\nimport argparse\nimport builtins\nimport contextlib\nimport signal\nimport traceback\n\nfrom xonsh import __version__\nfrom xonsh.timings import setup_timings\nfrom xonsh.lazyasd import lazyobject\nfrom xonsh.shell import Shell\nfrom xonsh.pretty import pretty\nfrom xonsh.execer import Execer\nfrom xonsh.proc import HiddenCommandPipeline\nfrom xonsh.jobs import ignore_sigtstp\nfrom xonsh.tools import setup_win_unicode_console, print_color, to_bool_or_int\nfrom xonsh.platform import HAS_PYGMENTS, ON_WINDOWS\nfrom xonsh.codecache import run_script_with_cache, run_code_with_cache\nfrom xonsh.xonfig import print_welcome_screen\nfrom xonsh.xontribs import xontribs_load\nfrom xonsh.lazyimps import pygments, pyghooks\nfrom xonsh.imphooks import install_import_hooks\nfrom xonsh.events import events\nfrom xonsh.environ import xonshrc_context, make_args_env\nfrom xonsh.built_ins import XonshSession, load_builtins, load_proxies\n\nfrom gitsome import __version__ as gitsome_version\n\n\nevents.transmogrify(\"on_post_init\", \"LoadEvent\")\nevents.doc(\n \"on_post_init\",\n \"\"\"\non_post_init() -> None\n\nFired after all initialization is finished and we're ready to do work.\n\nNOTE: This is fired before the wizard is automatically started.\n\"\"\",\n)\n\nevents.transmogrify(\"on_exit\", \"LoadEvent\")\nevents.doc(\n \"on_exit\",\n \"\"\"\non_exit() -> None\n\nFired after all commands have been executed, before tear-down occurs.\n\nNOTE: All the caveats of the ``atexit`` module also apply to this event.\n\"\"\",\n)\n\n\nevents.transmogrify(\"on_pre_cmdloop\", \"LoadEvent\")\nevents.doc(\n \"on_pre_cmdloop\",\n \"\"\"\non_pre_cmdloop() -> None\n\nFired just before the command loop is started, if it is.\n\"\"\",\n)\n\nevents.transmogrify(\"on_post_cmdloop\", \"LoadEvent\")\nevents.doc(\n \"on_post_cmdloop\",\n \"\"\"\non_post_cmdloop() -> None\n\nFired just after the command loop finishes, if it is.\n\nNOTE: All the caveats of the ``atexit`` module also apply to this event.\n\"\"\",\n)\n\nevents.transmogrify(\"on_pre_rc\", \"LoadEvent\")\nevents.doc(\n \"on_pre_rc\",\n \"\"\"\non_pre_rc() -> None\n\nFired just before rc files are loaded, if they are.\n\"\"\",\n)\n\nevents.transmogrify(\"on_post_rc\", \"LoadEvent\")\nevents.doc(\n \"on_post_rc\",\n \"\"\"\non_post_rc() -> None\n\nFired just after rc files are loaded, if they are.\n\"\"\",\n)\n\n\ndef get_setproctitle():\n \"\"\"Proxy function for loading process title\"\"\"\n try:\n from setproctitle import setproctitle as spt\n except ImportError:\n return\n return spt\n\n\ndef path_argument(s):\n \"\"\"Return a path only if the path is actually legal\n\n This is very similar to argparse.FileType, except that it doesn't return\n an open file handle, but rather simply validates the path.\"\"\"\n\n s = os.path.abspath(os.path.expanduser(s))\n if not os.path.isfile(s):\n msg = \"{0!r} must be a valid path to a file\".format(s)\n raise argparse.ArgumentTypeError(msg)\n return s\n\n\n@lazyobject\ndef parser():\n p = argparse.ArgumentParser(description=\"xonsh\", add_help=False)\n p.add_argument(\n \"-h\",\n \"--help\",\n dest=\"help\",\n action=\"store_true\",\n default=False,\n help=\"show help and exit\",\n )\n p.add_argument(\n \"-V\",\n \"--version\",\n dest=\"version\",\n action=\"store_true\",\n default=False,\n help=\"show version information and exit\",\n )\n p.add_argument(\n \"-c\",\n help=\"Run a single command and exit\",\n dest=\"command\",\n required=False,\n default=None,\n )\n p.add_argument(\n \"-i\",\n \"--interactive\",\n help=\"force running in interactive mode\",\n dest=\"force_interactive\",\n action=\"store_true\",\n default=False,\n )\n p.add_argument(\n \"-l\",\n \"--login\",\n help=\"run as a login shell\",\n dest=\"login\",\n action=\"store_true\",\n default=False,\n )\n p.add_argument(\n \"--config-path\",\n help=\"DEPRECATED: static configuration files may now be used \"\n \"in the XONSHRC file list, see the --rc option.\",\n dest=\"config_path\",\n default=None,\n type=path_argument,\n )\n p.add_argument(\n \"--rc\",\n help=\"The xonshrc files to load, these may be either xonsh \"\n \"files or JSON-based static configuration files.\",\n dest=\"rc\",\n nargs=\"+\",\n type=path_argument,\n default=None,\n )\n p.add_argument(\n \"--no-rc\",\n help=\"Do not load the .xonshrc files\",\n dest=\"norc\",\n action=\"store_true\",\n default=False,\n )\n p.add_argument(\n \"--no-script-cache\",\n help=\"Do not cache scripts as they are run\",\n dest=\"scriptcache\",\n action=\"store_false\",\n default=True,\n )\n p.add_argument(\n \"--cache-everything\",\n help=\"Use a cache, even for interactive commands\",\n dest=\"cacheall\",\n action=\"store_true\",\n default=False,\n )\n p.add_argument(\n \"-D\",\n dest=\"defines\",\n help=\"define an environment variable, in the form of \"\n \"-DNAME=VAL. May be used many times.\",\n metavar=\"ITEM\",\n action=\"append\",\n default=None,\n )\n p.add_argument(\n \"--shell-type\",\n help=\"What kind of shell should be used. \"\n \"Possible options: readline, prompt_toolkit, random. \"\n \"Warning! If set this overrides $SHELL_TYPE variable.\",\n dest=\"shell_type\",\n choices=tuple(Shell.shell_type_aliases.keys()),\n default=None,\n )\n p.add_argument(\n \"--timings\",\n help=\"Prints timing information before the prompt is shown. \"\n \"This is useful while tracking down performance issues \"\n \"and investigating startup times.\",\n dest=\"timings\",\n action=\"store_true\",\n default=None,\n )\n p.add_argument(\n \"file\",\n metavar=\"script-file\",\n help=\"If present, execute the script in script-file\" \" and exit\",\n nargs=\"?\",\n default=None,\n )\n p.add_argument(\n \"args\",\n metavar=\"args\",\n help=\"Additional arguments to the script specified \" \"by script-file\",\n nargs=argparse.REMAINDER,\n default=[],\n )\n return p\n\n\ndef _pprint_displayhook(value):\n if value is None:\n return\n builtins._ = None # Set '_' to None to avoid recursion\n if isinstance(value, HiddenCommandPipeline):\n builtins._ = value\n return\n env = builtins.__xonsh__.env\n if env.get(\"PRETTY_PRINT_RESULTS\"):\n printed_val = pretty(value)\n else:\n printed_val = repr(value)\n if HAS_PYGMENTS and env.get(\"COLOR_RESULTS\"):\n tokens = list(pygments.lex(printed_val, lexer=pyghooks.XonshLexer()))\n end = \"\" if env.get(\"SHELL_TYPE\") == \"prompt_toolkit2\" else \"\\n\"\n print_color(tokens, end=end)\n else:\n print(printed_val) # black & white case\n builtins._ = value\n\n\nclass XonshMode(enum.Enum):\n single_command = 0\n script_from_file = 1\n script_from_stdin = 2\n interactive = 3\n\n\ndef start_services(shell_kwargs, args):\n \"\"\"Starts up the essential services in the proper order.\n This returns the environment instance as a convenience.\n \"\"\"\n install_import_hooks()\n # create execer, which loads builtins\n ctx = shell_kwargs.get(\"ctx\", {})\n debug = to_bool_or_int(os.getenv(\"XONSH_DEBUG\", \"0\"))\n events.on_timingprobe.fire(name=\"pre_execer_init\")\n execer = Execer(\n xonsh_ctx=ctx,\n debug_level=debug,\n scriptcache=shell_kwargs.get(\"scriptcache\", True),\n cacheall=shell_kwargs.get(\"cacheall\", False),\n )\n events.on_timingprobe.fire(name=\"post_execer_init\")\n # load rc files\n login = shell_kwargs.get(\"login\", True)\n env = builtins.__xonsh__.env\n rc = shell_kwargs.get(\"rc\", None)\n rc = env.get(\"XONSHRC\") if rc is None else rc\n if args.mode != XonshMode.interactive and not args.force_interactive:\n # Don't load xonshrc if not interactive shell\n rc = None\n events.on_pre_rc.fire()\n xonshrc_context(rcfiles=rc, execer=execer, ctx=ctx, env=env, login=login)\n events.on_post_rc.fire()\n # create shell\n builtins.__xonsh__.shell = Shell(execer=execer, **shell_kwargs)\n ctx[\"__name__\"] = \"__main__\"\n return env\n\n\ndef premain(argv=None):\n \"\"\"Setup for main xonsh entry point. Returns parsed arguments.\"\"\"\n if argv is None:\n argv = sys.argv[1:]\n builtins.__xonsh__ = XonshSession()\n setup_timings(argv)\n setproctitle = get_setproctitle()\n if setproctitle is not None:\n setproctitle(\" \".join([\"xonsh\"] + argv))\n args = parser.parse_args(argv)\n if args.help:\n parser.print_help()\n parser.exit()\n if args.version:\n version = \"/\".join((\"xonsh\", __version__))\n print(version)\n parser.exit()\n shell_kwargs = {\n \"shell_type\": args.shell_type,\n \"completer\": False,\n \"login\": False,\n \"scriptcache\": args.scriptcache,\n \"cacheall\": args.cacheall,\n \"ctx\": builtins.__xonsh__.ctx,\n }\n if args.login:\n shell_kwargs[\"login\"] = True\n if args.norc:\n shell_kwargs[\"rc\"] = ()\n elif args.rc:\n shell_kwargs[\"rc\"] = args.rc\n setattr(sys, \"displayhook\", _pprint_displayhook)\n print('Gitsome Version: ' + gitsome_version)\n if args.command is not None:\n args.mode = XonshMode.single_command\n shell_kwargs[\"shell_type\"] = \"none\"\n elif args.file is not None:\n args.mode = XonshMode.script_from_file\n shell_kwargs[\"shell_type\"] = \"none\"\n elif not sys.stdin.isatty() and not args.force_interactive:\n args.mode = XonshMode.script_from_stdin\n shell_kwargs[\"shell_type\"] = \"none\"\n else:\n args.mode = XonshMode.interactive\n shell_kwargs[\"completer\"] = True\n shell_kwargs[\"login\"] = True\n env = start_services(shell_kwargs, args)\n env[\"XONSH_LOGIN\"] = shell_kwargs[\"login\"]\n if args.defines is not None:\n env.update([x.split(\"=\", 1) for x in args.defines])\n env[\"XONSH_INTERACTIVE\"] = args.force_interactive or (\n args.mode == XonshMode.interactive\n )\n if ON_WINDOWS:\n setup_win_unicode_console(env.get(\"WIN_UNICODE_CONSOLE\", True))\n return args\n\n\ndef _failback_to_other_shells(args, err):\n # only failback for interactive shell; if we cannot tell, treat it\n # as an interactive one for safe.\n if hasattr(args, \"mode\") and args.mode != XonshMode.interactive:\n raise err\n foreign_shell = None\n shells_file = \"/etc/shells\"\n if not os.path.exists(shells_file):\n # right now, it will always break here on Windows\n raise err\n excluded_list = [\"xonsh\", \"screen\"]\n with open(shells_file) as f:\n for line in f:\n line = line.strip()\n if not line or line.startswith(\"#\"):\n continue\n if \"/\" not in line:\n continue\n _, shell = line.rsplit(\"/\", 1)\n if shell in excluded_list:\n continue\n if not os.path.exists(line):\n continue\n foreign_shell = line\n break\n if foreign_shell:\n traceback.print_exc()\n print(\"Xonsh encountered an issue during launch\", file=sys.stderr)\n print(\"Failback to {}\".format(foreign_shell), file=sys.stderr)\n os.execlp(foreign_shell, foreign_shell)\n else:\n raise err\n\n\ndef main(argv=None):\n args = None\n try:\n args = premain(argv)\n return main_xonsh(args)\n except Exception as err:\n _failback_to_other_shells(args, err)\n\n\ndef main_xonsh(args):\n \"\"\"Main entry point for xonsh cli.\"\"\"\n if not ON_WINDOWS:\n\n def func_sig_ttin_ttou(n, f):\n pass\n\n signal.signal(signal.SIGTTIN, func_sig_ttin_ttou)\n signal.signal(signal.SIGTTOU, func_sig_ttin_ttou)\n\n events.on_post_init.fire()\n env = builtins.__xonsh__.env\n shell = builtins.__xonsh__.shell\n try:\n if args.mode == XonshMode.interactive:\n # enter the shell\n env[\"XONSH_INTERACTIVE\"] = True\n ignore_sigtstp()\n if env[\"XONSH_INTERACTIVE\"] and not any(\n os.path.isfile(i) for i in env[\"XONSHRC\"]\n ):\n print_welcome_screen()\n events.on_pre_cmdloop.fire()\n try:\n shell.shell.cmdloop()\n finally:\n events.on_post_cmdloop.fire()\n elif args.mode == XonshMode.single_command:\n # run a single command and exit\n run_code_with_cache(args.command.lstrip(), shell.execer, mode=\"single\")\n elif args.mode == XonshMode.script_from_file:\n # run a script contained in a file\n path = os.path.abspath(os.path.expanduser(args.file))\n if os.path.isfile(path):\n sys.argv = [args.file] + args.args\n env.update(make_args_env()) # $ARGS is not sys.argv\n env[\"XONSH_SOURCE\"] = path\n shell.ctx.update({\"__file__\": args.file, \"__name__\": \"__main__\"})\n run_script_with_cache(\n args.file, shell.execer, glb=shell.ctx, loc=None, mode=\"exec\"\n )\n else:\n print(\"xonsh: {0}: No such file or directory.\".format(args.file))\n elif args.mode == XonshMode.script_from_stdin:\n # run a script given on stdin\n code = sys.stdin.read()\n run_code_with_cache(\n code, shell.execer, glb=shell.ctx, loc=None, mode=\"exec\"\n )\n finally:\n events.on_exit.fire()\n postmain(args)\n\n\ndef postmain(args=None):\n \"\"\"Teardown for main xonsh entry point, accepts parsed arguments.\"\"\"\n if ON_WINDOWS:\n setup_win_unicode_console(enable=False)\n builtins.__xonsh__.shell = None\n\n\n@contextlib.contextmanager\ndef main_context(argv=None):\n \"\"\"Generator that runs pre- and post-main() functions. This has two iterations.\n The first yields the shell. The second returns None but cleans\n up the shell.\n \"\"\"\n args = premain(argv)\n yield builtins.__xonsh__.shell\n postmain(args)\n\n\ndef setup(\n ctx=None,\n shell_type=\"none\",\n env=((\"RAISE_SUBPROC_ERROR\", True),),\n aliases=(),\n xontribs=(),\n threadable_predictors=(),\n):\n \"\"\"Starts up a new xonsh shell. Calling this in function in another\n packages __init__.py will allow xonsh to be fully used in the\n package in headless or headed mode. This function is primarily indended to\n make starting up xonsh for 3rd party packages easier.\n\n Parameters\n ----------\n ctx : dict-like or None, optional\n The xonsh context to start with. If None, an empty dictionary\n is provided.\n shell_type : str, optional\n The type of shell to start. By default this is 'none', indicating\n we should start in headless mode.\n env : dict-like, optional\n Environment to update the current environment with after the shell\n has been initialized.\n aliases : dict-like, optional\n Aliases to add after the shell has been initialized.\n xontribs : iterable of str, optional\n Xontrib names to load.\n threadable_predictors : dict-like, optional\n Threadable predictors to start up with. These overide the defaults.\n \"\"\"\n ctx = {} if ctx is None else ctx\n # setup xonsh ctx and execer\n if not hasattr(builtins, \"__xonsh__\"):\n execer = Execer(xonsh_ctx=ctx)\n builtins.__xonsh__ = XonshSession(ctx=ctx, execer=execer)\n load_builtins(ctx=ctx, execer=execer)\n load_proxies()\n builtins.__xonsh__.shell = Shell(execer, ctx=ctx, shell_type=shell_type)\n builtins.__xonsh__.env.update(env)\n install_import_hooks()\n builtins.aliases.update(aliases)\n if xontribs:\n xontribs_load(xontribs)\n tp = builtins.__xonsh__.commands_cache.threadable_predictors\n tp.update(threadable_predictors)\n" }, { "path": "xonsh/openpy.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"Tools to open ``*.py`` files as Unicode.\n\nUses the encoding specified within the file, as per PEP 263.\n\nMuch of the code is taken from the tokenize module in Python 3.2.\n\nThis file was forked from the IPython project:\n\n* Copyright (c) 2008-2014, IPython Development Team\n* Copyright (C) 2001-2007 Fernando Perez \n* Copyright (c) 2001, Janko Hauser \n* Copyright (c) 2001, Nathaniel Gray \n\"\"\"\nimport io\nimport re\n\nfrom xonsh.lazyasd import LazyObject\nfrom xonsh.tokenize import detect_encoding, tokopen\n\n\ncookie_comment_re = LazyObject(\n lambda: re.compile(r\"^\\s*#.*coding[:=]\\s*([-\\w.]+)\", re.UNICODE),\n globals(),\n \"cookie_comment_re\",\n)\n\n\ndef source_to_unicode(txt, errors=\"replace\", skip_encoding_cookie=True):\n \"\"\"Converts a bytes string with python source code to unicode.\n\n Unicode strings are passed through unchanged. Byte strings are checked\n for the python source file encoding cookie to determine encoding.\n txt can be either a bytes buffer or a string containing the source\n code.\n \"\"\"\n if isinstance(txt, str):\n return txt\n if isinstance(txt, bytes):\n buf = io.BytesIO(txt)\n else:\n buf = txt\n try:\n encoding, _ = detect_encoding(buf.readline)\n except SyntaxError:\n encoding = \"ascii\"\n buf.seek(0)\n text = io.TextIOWrapper(buf, encoding, errors=errors, line_buffering=True)\n text.mode = \"r\"\n if skip_encoding_cookie:\n return u\"\".join(strip_encoding_cookie(text))\n else:\n return text.read()\n\n\ndef strip_encoding_cookie(filelike):\n \"\"\"Generator to pull lines from a text-mode file, skipping the encoding\n cookie if it is found in the first two lines.\n \"\"\"\n it = iter(filelike)\n try:\n first = next(it)\n if not cookie_comment_re.match(first):\n yield first\n second = next(it)\n if not cookie_comment_re.match(second):\n yield second\n except StopIteration:\n return\n for line in it:\n yield line\n\n\ndef read_py_file(filename, skip_encoding_cookie=True):\n \"\"\"Read a Python file, using the encoding declared inside the file.\n\n Parameters\n ----------\n filename : str\n The path to the file to read.\n skip_encoding_cookie : bool\n If True (the default), and the encoding declaration is found in the first\n two lines, that line will be excluded from the output - compiling a\n unicode string with an encoding declaration is a SyntaxError in Python 2.\n\n Returns\n -------\n A unicode string containing the contents of the file.\n \"\"\"\n with tokopen(filename) as f: # the open function defined in this module.\n if skip_encoding_cookie:\n return \"\".join(strip_encoding_cookie(f))\n else:\n return f.read()\n\n\ndef read_py_url(url, errors=\"replace\", skip_encoding_cookie=True):\n \"\"\"Read a Python file from a URL, using the encoding declared inside the file.\n\n Parameters\n ----------\n url : str\n The URL from which to fetch the file.\n errors : str\n How to handle decoding errors in the file. Options are the same as for\n bytes.decode(), but here 'replace' is the default.\n skip_encoding_cookie : bool\n If True (the default), and the encoding declaration is found in the first\n two lines, that line will be excluded from the output - compiling a\n unicode string with an encoding declaration is a SyntaxError in Python 2.\n\n Returns\n -------\n A unicode string containing the contents of the file.\n \"\"\"\n # Deferred import for faster start\n try:\n from urllib.request import urlopen # Py 3\n except ImportError:\n from urllib import urlopen\n response = urlopen(url)\n buf = io.BytesIO(response.read())\n return source_to_unicode(buf, errors, skip_encoding_cookie)\n\n\ndef _list_readline(x):\n \"\"\"Given a list, returns a readline() function that returns the next element\n with each call.\n \"\"\"\n x = iter(x)\n\n def readline():\n return next(x)\n\n return readline\n" }, { "path": "xonsh/parser.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"Implements the xonsh parser.\"\"\"\nfrom xonsh.lazyasd import lazyobject\nfrom xonsh.platform import PYTHON_VERSION_INFO\n\n\n@lazyobject\ndef Parser():\n if PYTHON_VERSION_INFO > (3, 6):\n from xonsh.parsers.v36 import Parser as p\n elif PYTHON_VERSION_INFO > (3, 5):\n from xonsh.parsers.v35 import Parser as p\n else:\n from xonsh.parsers.v34 import Parser as p\n return p\n" }, { "path": "xonsh/parsers/__init__.py", "content": "" }, { "path": "xonsh/parsers/base.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"Implements the base xonsh parser.\"\"\"\nimport os\nimport re\nimport time\nimport textwrap\nfrom threading import Thread\nfrom ast import parse as pyparse\nfrom collections.abc import Iterable, Sequence, Mapping\n\nfrom xonsh.ply.ply import yacc\n\nfrom xonsh.tools import FORMATTER\nfrom xonsh import ast\nfrom xonsh.ast import has_elts, xonsh_call, load_attribute_chain\nfrom xonsh.lexer import Lexer, LexToken\nfrom xonsh.platform import PYTHON_VERSION_INFO\nfrom xonsh.tokenize import SearchPath, StringPrefix\nfrom xonsh.lazyasd import LazyObject, lazyobject\nfrom xonsh.parsers.context_check import check_contexts\n\n\nRE_SEARCHPATH = LazyObject(lambda: re.compile(SearchPath), globals(), \"RE_SEARCHPATH\")\nRE_STRINGPREFIX = LazyObject(\n lambda: re.compile(StringPrefix), globals(), \"RE_STRINGPREFIX\"\n)\n\n\n@lazyobject\ndef RE_FSTR_EVAL_CHARS():\n return re.compile(\".*?[!@$`]\")\n\n\nclass Location(object):\n \"\"\"Location in a file.\"\"\"\n\n def __init__(self, fname, lineno, column=None):\n \"\"\"Takes a filename, line number, and optionally a column number.\"\"\"\n self.fname = fname\n self.lineno = lineno\n self.column = column\n\n def __str__(self):\n s = \"{0}:{1}\".format(self.fname, self.lineno)\n if self.column is not None:\n s += \":{0}\".format(self.column)\n return s\n\n\ndef ensure_has_elts(x, lineno=None, col_offset=None):\n \"\"\"Ensures that x is an AST node with elements.\"\"\"\n if not has_elts(x):\n if not isinstance(x, Iterable):\n x = [x]\n lineno = x[0].lineno if lineno is None else lineno\n col_offset = x[0].col_offset if col_offset is None else col_offset\n x = ast.Tuple(elts=x, ctx=ast.Load(), lineno=lineno, col_offset=col_offset)\n return x\n\n\ndef empty_list(lineno=None, col=None):\n \"\"\"Creates the AST node for an empty list.\"\"\"\n return ast.List(elts=[], ctx=ast.Load(), lineno=lineno, col_offset=col)\n\n\ndef binop(x, op, y, lineno=None, col=None):\n \"\"\"Creates the AST node for a binary operation.\"\"\"\n lineno = x.lineno if lineno is None else lineno\n col = x.col_offset if col is None else col\n return ast.BinOp(left=x, op=op, right=y, lineno=lineno, col_offset=col)\n\n\ndef call_split_lines(x, lineno=None, col=None):\n \"\"\"Creates the AST node for calling the 'splitlines' attribute of an\n object, nominally a string.\n \"\"\"\n return ast.Call(\n func=ast.Attribute(\n value=x, attr=\"splitlines\", ctx=ast.Load(), lineno=lineno, col_offset=col\n ),\n args=[],\n keywords=[],\n starargs=None,\n kwargs=None,\n lineno=lineno,\n col_offset=col,\n )\n\n\ndef ensure_list_from_str_or_list(x, lineno=None, col=None):\n \"\"\"Creates the AST node for the following expression::\n\n [x] if isinstance(x, str) else x\n\n Somewhat useful.\n \"\"\"\n return ast.IfExp(\n test=ast.Call(\n func=ast.Name(\n id=\"isinstance\", ctx=ast.Load(), lineno=lineno, col_offset=col\n ),\n args=[x, ast.Name(id=\"str\", ctx=ast.Load(), lineno=lineno, col_offset=col)],\n keywords=[],\n starargs=None,\n kwargs=None,\n lineno=lineno,\n col_offset=col,\n ),\n body=ast.List(elts=[x], ctx=ast.Load(), lineno=lineno, col_offset=col),\n orelse=x,\n lineno=lineno,\n col_offset=col,\n )\n\n\ndef xonsh_help(x, lineno=None, col=None):\n \"\"\"Creates the AST node for calling the __xonsh__.help() function.\"\"\"\n return xonsh_call(\"__xonsh__.help\", [x], lineno=lineno, col=col)\n\n\ndef xonsh_superhelp(x, lineno=None, col=None):\n \"\"\"Creates the AST node for calling the __xonsh__.superhelp() function.\"\"\"\n return xonsh_call(\"__xonsh__.superhelp\", [x], lineno=lineno, col=col)\n\n\ndef xonsh_pathsearch(pattern, pymode=False, lineno=None, col=None):\n \"\"\"Creates the AST node for calling the __xonsh__.pathsearch() function.\n The pymode argument indicate if it is called from subproc or python mode\"\"\"\n pymode = ast.NameConstant(value=pymode, lineno=lineno, col_offset=col)\n searchfunc, pattern = RE_SEARCHPATH.match(pattern).groups()\n pattern = ast.Str(s=pattern, lineno=lineno, col_offset=col)\n pathobj = False\n if searchfunc.startswith(\"@\"):\n func = searchfunc[1:]\n elif \"g\" in searchfunc:\n func = \"__xonsh__.globsearch\"\n pathobj = \"p\" in searchfunc\n else:\n func = \"__xonsh__.regexsearch\"\n pathobj = \"p\" in searchfunc\n func = load_attribute_chain(func, lineno=lineno, col=col)\n pathobj = ast.NameConstant(value=pathobj, lineno=lineno, col_offset=col)\n return xonsh_call(\n \"__xonsh__.pathsearch\",\n args=[func, pattern, pymode, pathobj],\n lineno=lineno,\n col=col,\n )\n\n\ndef load_ctx(x):\n \"\"\"Recursively sets ctx to ast.Load()\"\"\"\n if not hasattr(x, \"ctx\"):\n return\n x.ctx = ast.Load()\n if isinstance(x, (ast.Tuple, ast.List)):\n for e in x.elts:\n load_ctx(e)\n elif isinstance(x, ast.Starred):\n load_ctx(x.value)\n\n\ndef store_ctx(x):\n \"\"\"Recursively sets ctx to ast.Store()\"\"\"\n if not hasattr(x, \"ctx\"):\n return\n x.ctx = ast.Store()\n if isinstance(x, (ast.Tuple, ast.List)):\n for e in x.elts:\n store_ctx(e)\n elif isinstance(x, ast.Starred):\n store_ctx(x.value)\n\n\ndef del_ctx(x):\n \"\"\"Recursively sets ctx to ast.Del()\"\"\"\n if not hasattr(x, \"ctx\"):\n return\n x.ctx = ast.Del()\n if isinstance(x, (ast.Tuple, ast.List)):\n for e in x.elts:\n del_ctx(e)\n elif isinstance(x, ast.Starred):\n del_ctx(x.value)\n\n\ndef empty_list_if_newline(x):\n return [] if x == \"\\n\" else x\n\n\ndef lopen_loc(x):\n \"\"\"Extracts the line and column number for a node that may have an opening\n parenthesis, brace, or bracket.\n \"\"\"\n lineno = x._lopen_lineno if hasattr(x, \"_lopen_lineno\") else x.lineno\n col = x._lopen_col if hasattr(x, \"_lopen_col\") else x.col_offset\n return lineno, col\n\n\ndef hasglobstar(x):\n \"\"\"Returns True if a node has literal '*' for globbing.\"\"\"\n if isinstance(x, ast.Str):\n return \"*\" in x.s\n elif isinstance(x, list):\n for e in x:\n if hasglobstar(e):\n return True\n else:\n return False\n else:\n return False\n\n\ndef _wrap_fstr_field(field, spec, conv):\n rtn = \"{\" + field\n if conv:\n rtn += \"!\" + conv\n if spec:\n rtn += \":\" + spec\n rtn += \"}\"\n return rtn\n\n\ndef eval_fstr_fields(fstring, prefix, filename=None):\n \"\"\"Takes an fstring (and its prefix, ie f\") that may contain\n xonsh expressions as its field values and\n substitues them for a xonsh eval() call as needed. Roughly,\n for example, this will take f\"{$HOME}\" and transform it to\n be f\"{__xonsh__.execer.eval(r'$HOME')}\".\n \"\"\"\n last = fstring[-1]\n q, r = (\"'\", r\"\\'\") if last == '\"' else ('\"', r\"\\\"\")\n prelen = len(prefix)\n postlen = len(fstring) - len(fstring.rstrip(last))\n template = fstring[prelen:-postlen]\n repl = prefix\n for literal, field, spec, conv in FORMATTER.parse(template):\n repl += literal\n if field is None:\n continue\n elif RE_FSTR_EVAL_CHARS.match(field) is None:\n # just a normal python field, simply reconstruct.\n repl += _wrap_fstr_field(field, spec, conv)\n else:\n # the field has a special xonsh character, so we must eval it\n eval_field = \"__xonsh__.execer.eval(r\" + q\n eval_field += field.lstrip().replace(q, r)\n eval_field += q + \", glbs=globals(), locs=locals()\"\n if filename is not None:\n eval_field += \", filename=\" + q + filename + q\n eval_field += \")\"\n repl += _wrap_fstr_field(eval_field, spec, conv)\n repl += last * postlen\n return repl\n\n\nclass YaccLoader(Thread):\n \"\"\"Thread to load (but not shave) the yacc parser.\"\"\"\n\n def __init__(self, parser, yacc_kwargs, *args, **kwargs):\n super().__init__(*args, **kwargs)\n self.daemon = True\n self.parser = parser\n self.yacc_kwargs = yacc_kwargs\n self.start()\n\n def run(self):\n self.parser.parser = yacc.yacc(**self.yacc_kwargs)\n\n\nclass BaseParser(object):\n \"\"\"A base class that parses the xonsh language.\"\"\"\n\n def __init__(\n self,\n lexer_optimize=True,\n lexer_table=\"xonsh.lexer_table\",\n yacc_optimize=True,\n yacc_table=\"xonsh.parser_table\",\n yacc_debug=False,\n outputdir=None,\n ):\n \"\"\"Parameters\n ----------\n lexer_optimize : bool, optional\n Set to false when unstable and true when lexer is stable.\n lexer_table : str, optional\n Lexer module used when optimized.\n yacc_optimize : bool, optional\n Set to false when unstable and true when parser is stable.\n yacc_table : str, optional\n Parser module used when optimized.\n yacc_debug : debug, optional\n Dumps extra debug info.\n outputdir : str or None, optional\n The directory to place generated tables within. Defaults to the root\n xonsh dir.\n \"\"\"\n self.lexer = lexer = Lexer()\n self.tokens = lexer.tokens\n\n self._lines = None\n self.xonsh_code = None\n self._attach_nocomma_tok_rules()\n self._attach_nocloser_base_rules()\n self._attach_nodedent_base_rules()\n self._attach_nonewline_base_rules()\n self._attach_subproc_arg_part_rules()\n\n opt_rules = [\n \"newlines\",\n \"arglist\",\n \"func_call\",\n \"rarrow_test\",\n \"typedargslist\",\n \"equals_test\",\n \"colon_test\",\n \"tfpdef\",\n \"comma_tfpdef_list\",\n \"comma_pow_tfpdef\",\n \"vfpdef\",\n \"comma_vfpdef_list\",\n \"comma_pow_vfpdef\",\n \"equals_yield_expr_or_testlist_list\",\n \"testlist\",\n \"as_name\",\n \"period_or_ellipsis_list\",\n \"comma_import_as_name_list\",\n \"comma_dotted_as_name_list\",\n \"comma_name_list\",\n \"comma_test\",\n \"elif_part_list\",\n \"finally_part\",\n \"varargslist\",\n \"or_and_test_list\",\n \"and_not_test_list\",\n \"comp_op_expr_list\",\n \"xor_and_expr_list\",\n \"ampersand_shift_expr_list\",\n \"shift_arith_expr_list\",\n \"op_factor_list\",\n \"trailer_list\",\n \"testlist_comp\",\n \"yield_expr_or_testlist_comp\",\n \"dictorsetmaker\",\n \"comma_subscript_list\",\n \"test\",\n \"sliceop\",\n \"comp_iter\",\n \"yield_arg\",\n \"test_comma_list\",\n \"macroarglist\",\n \"any_raw_toks\",\n ]\n for rule in opt_rules:\n self._opt_rule(rule)\n\n list_rules = [\n \"comma_tfpdef\",\n \"comma_vfpdef\",\n \"semi_small_stmt\",\n \"comma_test_or_star_expr\",\n \"period_or_ellipsis\",\n \"comma_import_as_name\",\n \"comma_dotted_as_name\",\n \"period_name\",\n \"comma_name\",\n \"elif_part\",\n \"except_part\",\n \"comma_with_item\",\n \"or_and_test\",\n \"and_not_test\",\n \"comp_op_expr\",\n \"pipe_xor_expr\",\n \"xor_and_expr\",\n \"ampersand_shift_expr\",\n \"shift_arith_expr\",\n \"pm_term\",\n \"op_factor\",\n \"trailer\",\n \"comma_subscript\",\n \"comma_expr_or_star_expr\",\n \"comma_test\",\n \"comma_argument\",\n \"comma_item\",\n \"attr_period_name\",\n \"test_comma\",\n \"equals_yield_expr_or_testlist\",\n \"comma_nocomma\",\n ]\n for rule in list_rules:\n self._list_rule(rule)\n\n tok_rules = [\n \"def\",\n \"class\",\n \"return\",\n \"number\",\n \"name\",\n \"bang\",\n \"none\",\n \"true\",\n \"false\",\n \"ellipsis\",\n \"if\",\n \"del\",\n \"assert\",\n \"lparen\",\n \"lbrace\",\n \"lbracket\",\n \"string\",\n \"times\",\n \"plus\",\n \"minus\",\n \"divide\",\n \"doublediv\",\n \"mod\",\n \"at\",\n \"lshift\",\n \"rshift\",\n \"pipe\",\n \"xor\",\n \"ampersand\",\n \"for\",\n \"colon\",\n \"import\",\n \"except\",\n \"nonlocal\",\n \"global\",\n \"yield\",\n \"from\",\n \"raise\",\n \"with\",\n \"dollar_lparen\",\n \"dollar_lbrace\",\n \"dollar_lbracket\",\n \"try\",\n \"bang_lparen\",\n \"bang_lbracket\",\n \"comma\",\n \"rparen\",\n \"rbracket\",\n \"at_lparen\",\n \"atdollar_lparen\",\n \"indent\",\n \"dedent\",\n \"newline\",\n \"lambda\",\n \"ampersandequal\",\n \"as\",\n \"atdollar\",\n \"atequal\",\n \"break\",\n \"continue\",\n \"divequal\",\n \"dollar_name\",\n \"double_question\",\n \"doubledivequal\",\n \"elif\",\n \"else\",\n \"eq\",\n \"equals\",\n \"errortoken\",\n \"finally\",\n \"ge\",\n \"in\",\n \"is\",\n \"le\",\n \"lshiftequal\",\n \"minusequal\",\n \"modequal\",\n \"ne\",\n \"pass\",\n \"period\",\n \"pipeequal\",\n \"plusequal\",\n \"pow\",\n \"powequal\",\n \"question\",\n \"rarrow\",\n \"rshiftequal\",\n \"semi\",\n \"tilde\",\n \"timesequal\",\n \"while\",\n \"xorequal\",\n ]\n for rule in tok_rules:\n self._tok_rule(rule)\n\n yacc_kwargs = dict(\n module=self,\n debug=yacc_debug,\n start=\"start_symbols\",\n optimize=yacc_optimize,\n tabmodule=yacc_table,\n )\n if not yacc_debug:\n yacc_kwargs[\"errorlog\"] = yacc.NullLogger()\n if outputdir is None:\n outputdir = os.path.dirname(os.path.dirname(__file__))\n yacc_kwargs[\"outputdir\"] = outputdir\n if yacc_debug:\n # create parser on main thread\n self.parser = yacc.yacc(**yacc_kwargs)\n else:\n self.parser = None\n YaccLoader(self, yacc_kwargs)\n\n # Keeps track of the last token given to yacc (the lookahead token)\n self._last_yielded_token = None\n\n def reset(self):\n \"\"\"Resets for clean parsing.\"\"\"\n self.lexer.reset()\n self._last_yielded_token = None\n self._lines = None\n self.xonsh_code = None\n\n def parse(self, s, filename=\"\", mode=\"exec\", debug_level=0):\n \"\"\"Returns an abstract syntax tree of xonsh code.\n\n Parameters\n ----------\n s : str\n The xonsh code.\n filename : str, optional\n Name of the file.\n mode : str, optional\n Execution mode, one of: exec, eval, or single.\n debug_level : str, optional\n Debugging level passed down to yacc.\n\n Returns\n -------\n tree : AST\n \"\"\"\n self.reset()\n self.xonsh_code = s\n self.lexer.fname = filename\n while self.parser is None:\n time.sleep(0.01) # block until the parser is ready\n tree = self.parser.parse(input=s, lexer=self.lexer, debug=debug_level)\n if tree is not None:\n check_contexts(tree)\n # hack for getting modes right\n if mode == \"single\":\n if isinstance(tree, ast.Expression):\n tree = ast.Interactive(body=[self.expr(tree.body)])\n elif isinstance(tree, ast.Module):\n tree = ast.Interactive(body=tree.body)\n return tree\n\n def _lexer_errfunc(self, msg, line, column):\n self._parse_error(msg, self.currloc(line, column))\n\n def _yacc_lookahead_token(self):\n \"\"\"Gets the next-to-last and last token seen by the lexer.\"\"\"\n return self.lexer.beforelast, self.lexer.last\n\n def _opt_rule(self, rulename):\n \"\"\"For a rule name, creates an associated optional rule.\n '_opt' is appended to the rule name.\n \"\"\"\n\n def optfunc(self, p):\n p[0] = p[1]\n\n optfunc.__doc__ = (\"{0}_opt : empty\\n\" \" | {0}\").format(rulename)\n optfunc.__name__ = \"p_\" + rulename + \"_opt\"\n setattr(self.__class__, optfunc.__name__, optfunc)\n\n def _list_rule(self, rulename):\n \"\"\"For a rule name, creates an associated list rule.\n '_list' is appended to the rule name.\n \"\"\"\n\n def listfunc(self, p):\n p[0] = p[1] if len(p) == 2 else p[1] + p[2]\n\n listfunc.__doc__ = (\"{0}_list : {0}\\n\" \" | {0}_list {0}\").format(\n rulename\n )\n listfunc.__name__ = \"p_\" + rulename + \"_list\"\n setattr(self.__class__, listfunc.__name__, listfunc)\n\n def _tok_rule(self, rulename):\n \"\"\"For a rule name, creates a rule that returns the corresponding token.\n '_tok' is appended to the rule name.\n \"\"\"\n\n def tokfunc(self, p):\n s, t = self._yacc_lookahead_token()\n uprule = rulename.upper()\n if s is not None and s.type == uprule:\n p[0] = s\n elif t is not None and t.type == uprule:\n p[0] = t\n else:\n raise TypeError(\"token for {0!r} not found.\".format(rulename))\n\n tokfunc.__doc__ = \"{0}_tok : {1}\".format(rulename, rulename.upper())\n tokfunc.__name__ = \"p_\" + rulename + \"_tok\"\n setattr(self.__class__, tokfunc.__name__, tokfunc)\n\n def currloc(self, lineno, column=None):\n \"\"\"Returns the current location.\"\"\"\n return Location(fname=self.lexer.fname, lineno=lineno, column=column)\n\n def expr(self, p):\n \"\"\"Creates an expression for a token.\"\"\"\n expr = ast.Expr(value=p, lineno=p.lineno, col_offset=p.col_offset)\n expr.max_lineno = self.lineno\n expr.max_col = self.col\n return expr\n\n def token_col(self, t):\n \"\"\"Gets ths token column\"\"\"\n return t.lexpos\n\n @property\n def lineno(self):\n if self.lexer.last is None:\n return 1\n else:\n return self.lexer.last.lineno\n\n @property\n def col(self):\n s, t = self._yacc_lookahead_token()\n if t is not None:\n if t.type == \"NEWLINE\":\n t = s\n return self.token_col(t)\n return 0\n\n @property\n def lines(self):\n if self._lines is None and self.xonsh_code is not None:\n self._lines = self.xonsh_code.splitlines(keepends=True)\n return self._lines\n\n def source_slice(self, start, stop):\n \"\"\"Gets the original source code from two (line, col) tuples in\n source-space (i.e. lineno start at 1).\n \"\"\"\n bline, bcol = start\n eline, ecol = stop\n bline -= 1\n lines = self.lines[bline:eline]\n if ecol == 0:\n explen = eline - bline\n if explen == len(lines) and explen > 1:\n lines[-1] = \"\"\n else:\n lines[-1] = lines[-1][:ecol]\n lines[0] = lines[0][bcol:]\n return \"\".join(lines)\n\n def _parse_error(self, msg, loc):\n if self.xonsh_code is None or loc is None:\n err_line_pointer = \"\"\n else:\n col = loc.column + 1\n lines = self.lines\n if loc.lineno == 0:\n loc.lineno = len(lines)\n i = loc.lineno - 1\n if 0 <= i < len(lines):\n err_line = lines[i].rstrip()\n err_line_pointer = \"\\n{}\\n{: >{}}\".format(err_line, \"^\", col)\n else:\n err_line_pointer = \"\"\n err = SyntaxError(\"{0}: {1}{2}\".format(loc, msg, err_line_pointer))\n err.loc = loc\n raise err\n\n #\n # Precedence of operators\n #\n precedence = (\n (\"left\", \"PIPE\"),\n (\"left\", \"XOR\"),\n (\"left\", \"AMPERSAND\"),\n (\"left\", \"EQ\", \"NE\"),\n (\"left\", \"GT\", \"GE\", \"LT\", \"LE\"),\n (\"left\", \"RSHIFT\", \"LSHIFT\"),\n (\"left\", \"PLUS\", \"MINUS\"),\n (\"left\", \"TIMES\", \"DIVIDE\", \"DOUBLEDIV\", \"MOD\"),\n (\"left\", \"POW\"),\n )\n\n #\n # Grammar as defined by BNF\n #\n\n def p_start_symbols(self, p):\n \"\"\"start_symbols : single_input\n | file_input\n | eval_input\n | empty\n \"\"\"\n p[0] = p[1]\n\n def p_single_input(self, p):\n \"\"\"single_input : compound_stmt NEWLINE\n \"\"\"\n p1 = empty_list_if_newline(p[1])\n p0 = ast.Interactive(body=p1)\n p[0] = p0\n\n def p_file_input(self, p):\n \"\"\"file_input : file_stmts\"\"\"\n p[0] = ast.Module(body=p[1])\n\n def p_file_stmts_nl(self, p):\n \"\"\"file_stmts : newline_or_stmt\"\"\"\n # newline_or_stmt ENDMARKER\n p[0] = empty_list_if_newline(p[1])\n\n def p_file_stmts_files(self, p):\n \"\"\"file_stmts : file_stmts newline_or_stmt\"\"\"\n # file_input newline_or_stmt ENDMARKER\n p2 = empty_list_if_newline(p[2])\n p[0] = p[1] + p2\n\n def p_newline_or_stmt(self, p):\n \"\"\"newline_or_stmt : NEWLINE\n | stmt\n \"\"\"\n p[0] = p[1]\n\n def p_newlines(self, p):\n \"\"\"newlines : NEWLINE\n | newlines NEWLINE\n \"\"\"\n p[0] = p[1] if len(p) == 2 else p[1] + p[2]\n\n def p_eval_input(self, p):\n \"\"\"eval_input : testlist newlines_opt\n \"\"\"\n p1 = p[1]\n p[0] = ast.Expression(body=p1, lineno=p1.lineno, col_offset=p1.col_offset)\n\n def p_func_call(self, p):\n \"\"\"func_call : LPAREN arglist_opt RPAREN\"\"\"\n p[0] = p[2]\n\n def p_attr_period_name(self, p):\n \"\"\"attr_period_name : PERIOD NAME\"\"\"\n p[0] = [p[2]]\n\n def p_attr_name_alone(self, p):\n \"\"\"attr_name : name_tok\"\"\"\n p1 = p[1]\n p[0] = ast.Name(\n id=p1.value, ctx=ast.Load(), lineno=p1.lineno, col_offset=p1.lexpos\n )\n\n def p_attr_name_with(self, p):\n \"\"\"attr_name : name_tok attr_period_name_list\"\"\"\n p1 = p[1]\n name = ast.Name(\n id=p1.value, ctx=ast.Load(), lineno=p1.lineno, col_offset=p1.lexpos\n )\n p2 = p[2]\n p0 = ast.Attribute(\n value=name,\n attr=p2[0],\n ctx=ast.Load(),\n lineno=p1.lineno,\n col_offset=p1.lexpos,\n )\n for a in p2[1:]:\n p0 = ast.Attribute(\n value=p0,\n attr=a,\n ctx=ast.Load(),\n lineno=p0.lineno,\n col_offset=p0.col_offset,\n )\n p[0] = p0\n\n def p_decorator_no_call(self, p):\n \"\"\"decorator : at_tok attr_name NEWLINE\"\"\"\n p[0] = p[2]\n\n def p_decorator_call(self, p):\n \"\"\"decorator : at_tok attr_name func_call NEWLINE\"\"\"\n p1, name, p3 = p[1], p[2], p[3]\n if isinstance(name, ast.Attribute) or (p3 is not None):\n lineno, col = name.lineno, name.col_offset\n else:\n lineno, col = p1.lineno, p1.lexpos\n if p3 is None:\n p0 = ast.Call(\n func=name,\n args=[],\n keywords=[],\n starargs=None,\n kwargs=None,\n lineno=lineno,\n col_offset=col,\n )\n else:\n p0 = ast.Call(func=name, lineno=lineno, col_offset=col, **p3)\n p[0] = p0\n\n def p_decorators(self, p):\n \"\"\"decorators : decorator\n | decorators decorator\n \"\"\"\n p[0] = [p[1]] if len(p) == 2 else p[1] + [p[2]]\n\n def p_decorated(self, p):\n \"\"\"decorated : decorators classdef_or_funcdef\"\"\"\n p1, p2 = p[1], p[2]\n targ = p2[0]\n targ.decorator_list = p1\n # this is silly, CPython. This claims a func or class starts on\n # the line of the first decorator, rather than the 'def' or 'class'\n # line. However, it retains the original col_offset.\n targ.lineno = p1[0].lineno\n # async functions take the col number of the 'def', unless they are\n # decorated, in which case they have the col of the 'async'. WAT?\n if hasattr(targ, \"_async_tok\"):\n targ.col_offset = targ._async_tok.lexpos\n del targ._async_tok\n p[0] = p2\n\n def p_rarrow_test(self, p):\n \"\"\"rarrow_test : RARROW test\"\"\"\n p[0] = p[2]\n\n def p_funcdef(self, p):\n \"\"\"funcdef : def_tok NAME parameters rarrow_test_opt COLON suite\"\"\"\n f = ast.FunctionDef(\n name=p[2],\n args=p[3],\n returns=p[4],\n body=p[6],\n decorator_list=[],\n lineno=p[1].lineno,\n col_offset=p[1].lexpos,\n )\n p[0] = [f]\n\n def p_parameters(self, p):\n \"\"\"parameters : LPAREN typedargslist_opt RPAREN\"\"\"\n p2 = p[2]\n if p2 is None:\n p2 = ast.arguments(\n args=[],\n vararg=None,\n kwonlyargs=[],\n kw_defaults=[],\n kwarg=None,\n defaults=[],\n )\n p[0] = p2\n\n def p_equals_test(self, p):\n \"\"\"equals_test : EQUALS test\"\"\"\n p[0] = p[2]\n\n def p_typedargslist_kwarg(self, p):\n \"\"\"typedargslist : POW tfpdef\"\"\"\n p[0] = ast.arguments(\n args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=p[2], defaults=[]\n )\n\n def p_typedargslist_times4_tfpdef(self, p):\n \"\"\"typedargslist : TIMES tfpdef comma_pow_tfpdef_opt\"\"\"\n # *args, **kwargs\n p0 = ast.arguments(\n args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=p[3], defaults=[]\n )\n self._set_var_args(p0, p[2], None)\n p[0] = p0\n\n def p_typedargslist_times4_comma(self, p):\n \"\"\"typedargslist : TIMES comma_pow_tfpdef\"\"\"\n # *, **kwargs\n p0 = ast.arguments(\n args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=p[2], defaults=[]\n )\n p[0] = p0\n\n def p_typedargslist_times5_tdpdef(self, p):\n \"\"\"typedargslist : TIMES tfpdef comma_tfpdef_list comma_pow_tfpdef_opt\"\"\"\n # *args, x, **kwargs\n p0 = ast.arguments(\n args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=p[4], defaults=[]\n )\n self._set_var_args(p0, p[2], p[3]) # *args\n p[0] = p0\n\n def p_typedargslist_times5_comma(self, p):\n \"\"\"typedargslist : TIMES comma_tfpdef_list comma_pow_tfpdef_opt\"\"\"\n # *, x, **kwargs\n p0 = ast.arguments(\n args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=p[3], defaults=[]\n )\n self._set_var_args(p0, None, p[2]) # *args\n p[0] = p0\n\n def p_typedargslist_t5(self, p):\n \"\"\"typedargslist : tfpdef equals_test_opt comma_tfpdef_list_opt comma_opt\"\"\"\n # x\n p0 = ast.arguments(\n args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]\n )\n self._set_regular_args(p0, p[1], p[2], p[3], p[4])\n p[0] = p0\n\n def p_typedargslist_t7(self, p):\n \"\"\"typedargslist : tfpdef equals_test_opt comma_tfpdef_list_opt comma_opt POW tfpdef\"\"\"\n # x, **kwargs\n p0 = ast.arguments(\n args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=p[6], defaults=[]\n )\n self._set_regular_args(p0, p[1], p[2], p[3], p[4])\n p[0] = p0\n\n def p_typedargslist_t8(self, p):\n \"\"\"typedargslist : tfpdef equals_test_opt comma_tfpdef_list_opt comma_opt TIMES tfpdef_opt comma_tfpdef_list_opt\"\"\"\n p0 = ast.arguments(\n args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]\n )\n self._set_regular_args(p0, p[1], p[2], p[3], p[4])\n self._set_var_args(p0, p[6], p[7])\n p[0] = p0\n\n def p_typedargslist_t10(self, p):\n \"\"\"typedargslist : tfpdef equals_test_opt comma_tfpdef_list_opt comma_opt TIMES tfpdef_opt COMMA POW vfpdef\"\"\"\n # x, *args, **kwargs\n p0 = ast.arguments(\n args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=p[9], defaults=[]\n )\n self._set_regular_args(p0, p[1], p[2], p[3], p[4])\n self._set_var_args(p0, p[6], None)\n p[0] = p0\n\n def p_typedargslist_t11(self, p):\n \"\"\"typedargslist : tfpdef equals_test_opt comma_tfpdef_list_opt comma_opt TIMES tfpdef_opt comma_tfpdef_list COMMA POW tfpdef\"\"\"\n # x, *args, **kwargs\n p0 = ast.arguments(\n args=[],\n vararg=None,\n kwonlyargs=[],\n kw_defaults=[],\n kwarg=p[10],\n defaults=[],\n )\n self._set_regular_args(p0, p[1], p[2], p[3], p[4])\n self._set_var_args(p0, p[6], p[7])\n p[0] = p0\n\n def p_colon_test(self, p):\n \"\"\"colon_test : COLON test\"\"\"\n p[0] = p[2]\n\n def p_tfpdef(self, p):\n \"\"\"tfpdef : name_tok colon_test_opt\"\"\"\n p1 = p[1]\n kwargs = {\"arg\": p1.value, \"annotation\": p[2]}\n if PYTHON_VERSION_INFO >= (3, 5, 1):\n kwargs.update({\"lineno\": p1.lineno, \"col_offset\": p1.lexpos})\n p[0] = ast.arg(**kwargs)\n\n def p_comma_tfpdef_empty(self, p):\n \"\"\"comma_tfpdef : COMMA\"\"\"\n p[0] = []\n\n def p_comma_tfpdef_args(self, p):\n \"\"\"comma_tfpdef : COMMA tfpdef equals_test_opt\"\"\"\n p[0] = [{\"arg\": p[2], \"default\": p[3]}]\n\n def p_comma_pow_tfpdef(self, p):\n \"\"\"comma_pow_tfpdef : COMMA POW tfpdef\"\"\"\n p[0] = p[3]\n\n def _set_args_def(self, argmts, vals, kwargs=False):\n args, defs = (\n (argmts.kwonlyargs, argmts.kw_defaults)\n if kwargs\n else (argmts.args, argmts.defaults)\n )\n if vals is None and kwargs:\n loc = self.currloc(self.lineno, self.col)\n self._parse_error(\"named arguments must follow bare *\", loc)\n for v in vals:\n args.append(v[\"arg\"])\n d = v[\"default\"]\n if kwargs or (d is not None):\n defs.append(d)\n\n def _set_regular_args(self, p0, p1, p2, p3, p4):\n if p2 is None and p3 is None:\n # x\n p0.args.append(p1)\n elif p2 is not None and p3 is None:\n # x=42\n p0.args.append(p1)\n p0.defaults.append(p2)\n elif p2 is None and p3 is not None:\n # x, y and x, y=42\n p0.args.append(p1)\n self._set_args_def(p0, p3)\n else:\n # x=42, y=42\n p0.args.append(p1)\n p0.defaults.append(p2)\n self._set_args_def(p0, p3)\n\n def _set_var_args(self, p0, vararg, kwargs):\n if vararg is None and kwargs is not None:\n self._set_args_def(p0, kwargs, kwargs=True)\n elif vararg is not None and kwargs is None:\n # *args\n p0.vararg = vararg\n elif vararg is not None and kwargs is not None:\n # *args, x and *args, x, y and *args, x=10 and *args, x=10, y\n # and *args, x, y=10, and *args, x=42, y=65\n p0.vararg = vararg\n self._set_args_def(p0, kwargs, kwargs=True)\n else:\n assert False\n\n def p_varargslist_kwargs(self, p):\n \"\"\"varargslist : POW vfpdef\"\"\"\n p[0] = ast.arguments(\n args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=p[2], defaults=[]\n )\n\n def p_varargslist_times4(self, p):\n \"\"\"varargslist : TIMES vfpdef_opt comma_pow_vfpdef_opt\"\"\"\n p0 = ast.arguments(\n args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=p[3], defaults=[]\n )\n self._set_var_args(p0, p[2], None)\n p[0] = p0\n\n def p_varargslist_times5(self, p):\n \"\"\"varargslist : TIMES vfpdef_opt comma_vfpdef_list comma_pow_vfpdef_opt\"\"\"\n # *args, x, **kwargs\n p0 = ast.arguments(\n args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=p[4], defaults=[]\n )\n self._set_var_args(p0, p[2], p[3]) # *args\n p[0] = p0\n\n def p_varargslist_v5(self, p):\n \"\"\"varargslist : vfpdef equals_test_opt comma_vfpdef_list_opt comma_opt\"\"\"\n # x\n p0 = ast.arguments(\n args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]\n )\n self._set_regular_args(p0, p[1], p[2], p[3], p[4])\n p[0] = p0\n\n def p_varargslist_v7(self, p):\n \"\"\"varargslist : vfpdef equals_test_opt comma_vfpdef_list_opt comma_opt POW vfpdef\"\"\"\n # x, **kwargs\n p0 = ast.arguments(\n args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=p[6], defaults=[]\n )\n self._set_regular_args(p0, p[1], p[2], p[3], p[4])\n p[0] = p0\n\n def p_varargslist_v8(self, p):\n \"\"\"varargslist : vfpdef equals_test_opt comma_vfpdef_list_opt comma_opt TIMES vfpdef_opt comma_vfpdef_list_opt\"\"\"\n # x, *args\n p0 = ast.arguments(\n args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]\n )\n self._set_regular_args(p0, p[1], p[2], p[3], p[4])\n self._set_var_args(p0, p[6], p[7])\n p[0] = p0\n\n def p_varargslist_v10(self, p):\n \"\"\"varargslist : vfpdef equals_test_opt comma_vfpdef_list_opt comma_opt TIMES vfpdef_opt COMMA POW vfpdef\"\"\"\n # x, *args, **kwargs\n p0 = ast.arguments(\n args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=p[9], defaults=[]\n )\n self._set_regular_args(p0, p[1], p[2], p[3], p[4])\n self._set_var_args(p0, p[6], None)\n p[0] = p0\n\n def p_varargslist_v11(self, p):\n \"\"\"varargslist : vfpdef equals_test_opt comma_vfpdef_list_opt comma_opt TIMES vfpdef_opt comma_vfpdef_list COMMA POW vfpdef\"\"\"\n p0 = ast.arguments(\n args=[],\n vararg=None,\n kwonlyargs=[],\n kw_defaults=[],\n kwarg=p[10],\n defaults=[],\n )\n self._set_regular_args(p0, p[1], p[2], p[3], p[4])\n self._set_var_args(p0, p[6], p[7])\n p[0] = p0\n\n def p_vfpdef(self, p):\n \"\"\"vfpdef : name_tok\"\"\"\n p1 = p[1]\n kwargs = {\"arg\": p1.value, \"annotation\": None}\n if PYTHON_VERSION_INFO >= (3, 5, 1):\n kwargs.update({\"lineno\": p1.lineno, \"col_offset\": p1.lexpos})\n p[0] = ast.arg(**kwargs)\n\n def p_comma_vfpdef_empty(self, p):\n \"\"\"comma_vfpdef : COMMA\"\"\"\n p[0] = []\n\n def p_comma_vfpdef_value(self, p):\n \"\"\"comma_vfpdef : COMMA vfpdef equals_test_opt\"\"\"\n p[0] = [{\"arg\": p[2], \"default\": p[3]}]\n\n def p_comma_pow_vfpdef(self, p):\n \"\"\"comma_pow_vfpdef : COMMA POW vfpdef\"\"\"\n p[0] = p[3]\n\n def p_stmt(self, p):\n \"\"\"stmt : simple_stmt\n | compound_stmt\n \"\"\"\n p[0] = p[1]\n\n def p_stmt_list(self, p):\n \"\"\"stmt_list : stmt\n | stmt_list stmt\n \"\"\"\n if len(p) == 2:\n p[0] = p[1]\n else:\n p[0] = p[1] + p[2]\n\n def p_semi_opt(self, p):\n \"\"\"semi_opt : SEMI\n | empty\n \"\"\"\n if len(p) == 2:\n p[0] = p[1]\n\n def p_semi_small_stmt(self, p):\n \"\"\"semi_small_stmt : SEMI small_stmt\"\"\"\n p[0] = [p[2]]\n\n def p_simple_stmt_single(self, p):\n \"\"\"simple_stmt : small_stmt semi_opt NEWLINE\"\"\"\n p[0] = [p[1]]\n\n def p_simple_stmt_many(self, p):\n \"\"\"simple_stmt : small_stmt semi_small_stmt_list semi_opt NEWLINE\"\"\"\n p[0] = [p[1]] + p[2]\n\n def p_small_stmt(self, p):\n \"\"\"small_stmt : expr_stmt\n | del_stmt\n | pass_stmt\n | flow_stmt\n | import_stmt\n | global_stmt\n | nonlocal_stmt\n | assert_stmt\n \"\"\"\n p[0] = p[1]\n\n _augassign_op = {\n \"+=\": ast.Add,\n \"-=\": ast.Sub,\n \"*=\": ast.Mult,\n \"@=\": ast.MatMult,\n \"/=\": ast.Div,\n \"%=\": ast.Mod,\n \"//=\": ast.FloorDiv,\n \"**=\": ast.Pow,\n \"^=\": ast.BitXor,\n \"&=\": ast.BitAnd,\n \"|=\": ast.BitOr,\n \"<<=\": ast.LShift,\n \">>=\": ast.RShift,\n }\n\n def p_expr_stmt_testlist_assign(self, p):\n \"\"\"expr_stmt : testlist_star_expr equals_yield_expr_or_testlist_list_opt\n | testlist equals_yield_expr_or_testlist_list_opt\n \"\"\"\n p1, p2 = p[1], p[2]\n if isinstance(p1, ast.Tuple):\n p1 = [p1]\n if p2 is None and len(p1) == 1:\n p[0] = self.expr(p1[0])\n elif p2 is None:\n assert False\n else:\n for targ in p1:\n store_ctx(targ)\n list(map(store_ctx, p2[:-1]))\n lineno, col = lopen_loc(p1[0])\n p[0] = ast.Assign(\n targets=p1 + p2[:-1], value=p2[-1], lineno=lineno, col_offset=col\n )\n\n def p_expr_stmt_augassign(self, p):\n \"\"\"expr_stmt : testlist_star_expr augassign yield_expr_or_testlist\"\"\"\n p1, p2 = p[1], p[2]\n if not isinstance(p1, ast.Tuple):\n p1 = p1[0]\n store_ctx(p1)\n op = self._augassign_op[p2]\n if op is None:\n self._parse_error(\n \"operation {0!r} not supported\".format(p2),\n self.currloc(lineno=p.lineno, column=p.lexpos),\n )\n p[0] = ast.AugAssign(\n target=p1, op=op(), value=p[3], lineno=p1.lineno, col_offset=p1.col_offset\n )\n\n def store_star_expr(self, p1, p2, targs, rhs):\n \"\"\"Stores complex unpacking statements that target *x variables.\"\"\"\n p1 = [] if p1 is None else p1\n if isinstance(p1, ast.Tuple):\n p1 = [p1]\n for targ in p1:\n store_ctx(targ)\n store_ctx(p2)\n for targ in targs:\n store_ctx(targ)\n p1.append(p2)\n p1.extend(targs)\n p1 = [\n ast.Tuple(\n elts=p1,\n ctx=ast.Store(),\n lineno=p1[0].lineno,\n col_offset=p1[0].col_offset,\n )\n ]\n p0 = ast.Assign(\n targets=p1, value=rhs, lineno=p1[0].lineno, col_offset=p1[0].col_offset\n )\n return p0\n\n def p_expr_stmt_star5(self, p):\n \"\"\"expr_stmt : test_comma_list_opt star_expr comma_test_list equals_yield_expr_or_testlist\"\"\"\n targs, rhs = p[3], p[4][0]\n p[0] = self.store_star_expr(p[1], p[2], targs, rhs)\n\n def p_expr_stmt_star6(self, p):\n \"\"\"expr_stmt : test_comma_list_opt star_expr comma_opt test_comma_list_opt equals_yield_expr_or_testlist\"\"\"\n targs, rhs = (p[4] or []), p[5][0]\n p[0] = self.store_star_expr(p[1], p[2], targs, rhs)\n\n def p_test_comma(self, p):\n \"\"\"test_comma : test COMMA\"\"\"\n p[0] = [p[1]]\n\n def p_comma_opt(self, p):\n \"\"\"comma_opt : COMMA\n | empty\n \"\"\"\n if len(p) == 2:\n p[0] = p[1]\n\n def p_test_or_star_expr(self, p):\n \"\"\"test_or_star_expr : test\n | star_expr\n \"\"\"\n p[0] = p[1]\n\n def p_comma_test_or_star_expr(self, p):\n \"\"\"comma_test_or_star_expr : COMMA test_or_star_expr\"\"\"\n p[0] = [p[2]]\n\n def p_testlist_star_expr(self, p):\n \"\"\"testlist_star_expr : test_or_star_expr comma_test_or_star_expr_list comma_opt\n | test_or_star_expr comma_opt\n \"\"\"\n p1, p2 = p[1], p[2]\n if p2 is None:\n p0 = [p1]\n elif p2 == \",\":\n p0 = [\n ast.Tuple(\n elts=[p1],\n ctx=ast.Load(),\n lineno=p1.lineno,\n col_offset=p1.col_offset,\n )\n ]\n else:\n p0 = [\n ast.Tuple(\n elts=[p1] + p2,\n ctx=ast.Load(),\n lineno=p1.lineno,\n col_offset=p1.col_offset,\n )\n ]\n p[0] = p0\n\n def p_augassign(self, p):\n \"\"\"augassign : PLUSEQUAL\n | MINUSEQUAL\n | TIMESEQUAL\n | ATEQUAL\n | DIVEQUAL\n | MODEQUAL\n | AMPERSANDEQUAL\n | PIPEEQUAL\n | XOREQUAL\n | LSHIFTEQUAL\n | RSHIFTEQUAL\n | POWEQUAL\n | DOUBLEDIVEQUAL\n \"\"\"\n p[0] = p[1]\n\n def p_yield_expr_or_testlist(self, p):\n \"\"\"yield_expr_or_testlist : yield_expr\n | testlist\n \"\"\"\n p[0] = p[1]\n\n def p_equals_yield_expr_or_testlist(self, p):\n \"\"\"equals_yield_expr_or_testlist : EQUALS yield_expr_or_testlist\"\"\"\n p[0] = [p[2]]\n\n #\n # For normal assignments, additional restrictions enforced\n # by the interpreter\n #\n def p_del_stmt(self, p):\n \"\"\"del_stmt : del_tok exprlist\"\"\"\n p1 = p[1]\n p2 = p[2]\n for targ in p2:\n del_ctx(targ)\n p0 = ast.Delete(\n targets=p2, ctx=ast.Del(), lineno=p1.lineno, col_offset=p1.lexpos\n )\n p[0] = p0\n\n def p_pass_stmt(self, p):\n \"\"\"pass_stmt : PASS\"\"\"\n p[0] = ast.Pass(lineno=self.lineno, col_offset=self.col)\n\n def p_flow_stmt(self, p):\n \"\"\"flow_stmt : break_stmt\n | continue_stmt\n | return_stmt\n | raise_stmt\n | yield_stmt\n \"\"\"\n p[0] = p[1]\n\n def p_break_stmt(self, p):\n \"\"\"break_stmt : BREAK\"\"\"\n p[0] = ast.Break(lineno=self.lineno, col_offset=self.col)\n\n def p_continue_stmt(self, p):\n \"\"\"continue_stmt : CONTINUE\"\"\"\n p[0] = ast.Continue(lineno=self.lineno, col_offset=self.col)\n\n def p_return_stmt(self, p):\n \"\"\"return_stmt : return_tok testlist_opt\"\"\"\n p1 = p[1]\n p[0] = ast.Return(value=p[2], lineno=p1.lineno, col_offset=p1.lexpos)\n\n def p_yield_stmt(self, p):\n \"\"\"yield_stmt : yield_expr\"\"\"\n p[0] = self.expr(p[1])\n\n def p_raise_stmt_r1(self, p):\n \"\"\"raise_stmt : raise_tok\"\"\"\n p1 = p[1]\n p[0] = ast.Raise(exc=None, cause=None, lineno=p1.lineno, col_offset=p1.lexpos)\n\n def p_raise_stmt_r2(self, p):\n \"\"\"raise_stmt : raise_tok test\"\"\"\n p1 = p[1]\n p[0] = ast.Raise(exc=p[2], cause=None, lineno=p1.lineno, col_offset=p1.lexpos)\n\n def p_raise_stmt_r3(self, p):\n \"\"\"raise_stmt : raise_tok test FROM test\"\"\"\n p1 = p[1]\n p[0] = ast.Raise(exc=p[2], cause=p[4], lineno=p1.lineno, col_offset=p1.lexpos)\n\n def p_import_stmt(self, p):\n \"\"\"import_stmt : import_name\n | import_from\n \"\"\"\n p[0] = p[1]\n\n def p_import_name(self, p):\n \"\"\"import_name : import_tok dotted_as_names\n \"\"\"\n p1 = p[1]\n p[0] = ast.Import(names=p[2], lineno=p1.lineno, col_offset=p1.lexpos)\n\n def p_import_from_pre_f3(self, p):\n \"\"\"import_from_pre : from_tok period_or_ellipsis_list\"\"\"\n p1 = p[1]\n p[0] = (p[2], p1.lineno, p1.lexpos)\n\n def p_import_from_pre_f4(self, p):\n \"\"\"import_from_pre : from_tok period_or_ellipsis_list_opt dotted_name\"\"\"\n p1, p2, p3 = p[1], p[2], p[3]\n p0 = p3 if p2 is None else p2 + p3\n p[0] = (p0, p1.lineno, p1.lexpos)\n\n def p_import_from_post_times(self, p):\n \"\"\"import_from_post : TIMES\"\"\"\n p[0] = [ast.alias(name=\"*\", asname=None)]\n\n def p_import_from_post_as(self, p):\n \"\"\"import_from_post : import_as_names\"\"\"\n p[0] = p[1]\n\n def p_import_from_post_paren(self, p):\n \"\"\"import_from_post : LPAREN import_as_names\"\"\"\n p[0] = p[2]\n\n def p_import_from(self, p):\n \"\"\"import_from : import_from_pre IMPORT import_from_post\"\"\"\n # note below: the ('.' | '...') is necessary because '...' is\n # tokenized as ELLIPSIS\n p1, lineno, col = p[1]\n mod = p1.lstrip(\".\")\n lvl = len(p1) - len(mod)\n mod = mod or None\n p[0] = ast.ImportFrom(\n module=mod, names=p[3], level=lvl, lineno=lineno, col_offset=col\n )\n\n def p_period_or_ellipsis(self, p):\n \"\"\"period_or_ellipsis : PERIOD\n | ELLIPSIS\n \"\"\"\n p[0] = p[1]\n\n def p_as_name(self, p):\n \"\"\"as_name : AS NAME\"\"\"\n p[0] = p[2]\n\n def p_import_as_name(self, p):\n \"\"\"import_as_name : NAME as_name_opt\"\"\"\n p[0] = ast.alias(name=p[1], asname=p[2])\n\n def p_comma_import_as_name(self, p):\n \"\"\"comma_import_as_name : COMMA import_as_name\n \"\"\"\n p[0] = [p[2]]\n\n def p_comma_import_as_name_tail(self, p):\n \"\"\"comma_import_as_name : comma_opt RPAREN\"\"\"\n p[0] = list()\n\n def p_dotted_as_name(self, p):\n \"\"\"dotted_as_name : dotted_name as_name_opt\"\"\"\n p0 = ast.alias(name=p[1], asname=p[2])\n p[0] = p0\n\n def p_comma_dotted_as_name(self, p):\n \"\"\"comma_dotted_as_name : COMMA dotted_as_name\"\"\"\n p[0] = [p[2]]\n\n def p_import_as_names(self, p):\n \"\"\"import_as_names : import_as_name comma_import_as_name_list_opt\"\"\"\n p1, p2 = p[1], p[2]\n p0 = [p1]\n if p2 is not None:\n p0.extend(p2)\n p[0] = p0\n\n def p_dotted_as_names(self, p):\n \"\"\"dotted_as_names : dotted_as_name comma_dotted_as_name_list_opt\"\"\"\n p1, p2 = p[1], p[2]\n p0 = [p1]\n if p2 is not None:\n p0.extend(p2)\n p[0] = p0\n\n def p_period_name(self, p):\n \"\"\"period_name : PERIOD NAME\"\"\"\n p[0] = p[1] + p[2]\n\n def p_dotted_name(self, p):\n \"\"\"dotted_name : NAME\n | NAME period_name_list\n \"\"\"\n p[0] = p[1] if len(p) == 2 else p[1] + p[2]\n\n def p_comma_name(self, p):\n \"\"\"comma_name : COMMA NAME\"\"\"\n p[0] = [p[2]]\n\n def p_global_stmt(self, p):\n \"\"\"global_stmt : global_tok NAME comma_name_list_opt\"\"\"\n p1, p2, p3 = p[1], p[2], p[3]\n names = [p2]\n if p3 is not None:\n names += p3\n p[0] = ast.Global(names=names, lineno=p1.lineno, col_offset=p1.lexpos)\n\n def p_nonlocal_stmt(self, p):\n \"\"\"nonlocal_stmt : nonlocal_tok NAME comma_name_list_opt\"\"\"\n p1, p2, p3 = p[1], p[2], p[3]\n names = [p2]\n if p3 is not None:\n names += p3\n p[0] = ast.Nonlocal(names=names, lineno=p1.lineno, col_offset=p1.lexpos)\n\n def p_comma_test(self, p):\n \"\"\"comma_test : COMMA test\"\"\"\n p[0] = [p[2]]\n\n def p_assert_stmt(self, p):\n \"\"\"assert_stmt : assert_tok test comma_test_opt\"\"\"\n p1, p2, p3 = p[1], p[2], p[3]\n if p3 is not None:\n if len(p3) != 1:\n assert False\n p3 = p3[0]\n p[0] = ast.Assert(test=p2, msg=p3, lineno=p1.lineno, col_offset=p1.lexpos)\n\n def p_compound_stmt(self, p):\n \"\"\"compound_stmt : if_stmt\n | while_stmt\n | for_stmt\n | try_stmt\n | with_stmt\n | funcdef\n | classdef\n | decorated\n \"\"\"\n p[0] = p[1]\n\n def p_elif_part(self, p):\n \"\"\"elif_part : ELIF test COLON suite\"\"\"\n p2 = p[2]\n p[0] = [\n ast.If(\n test=p2,\n body=p[4],\n orelse=[],\n lineno=p2.lineno,\n col_offset=p2.col_offset,\n )\n ]\n\n def p_else_part(self, p):\n \"\"\"else_part : ELSE COLON suite\"\"\"\n p[0] = p[3]\n\n def p_if_stmt(self, p):\n \"\"\"if_stmt : if_tok test COLON suite elif_part_list_opt\n | if_tok test COLON suite elif_part_list_opt else_part\n \"\"\"\n p1 = p[1]\n lastif = ast.If(\n test=p[2], body=p[4], orelse=[], lineno=p1.lineno, col_offset=p1.lexpos\n )\n p0 = [lastif]\n p5 = p[5]\n p6 = p[6] if len(p) > 6 else []\n if p5 is not None:\n for elseif in p5:\n lastif.orelse.append(elseif)\n lastif = elseif\n lastif.orelse = p6\n p[0] = p0\n\n def p_while_stmt(self, p):\n \"\"\"while_stmt : WHILE test COLON suite\n | WHILE test COLON suite else_part\n \"\"\"\n p5 = p[5] if len(p) > 5 else []\n p[0] = [\n ast.While(\n test=p[2], body=p[4], orelse=p5, lineno=self.lineno, col_offset=self.col\n )\n ]\n\n def p_for_stmt(self, p):\n \"\"\"for_stmt : for_tok exprlist IN testlist COLON suite\n | for_tok exprlist IN testlist COLON suite else_part\n \"\"\"\n p1, p2 = p[1], p[2]\n p7 = p[7] if len(p) > 7 else []\n if len(p2) == 1:\n p2 = p2[0]\n store_ctx(p2)\n else:\n for x in p2:\n store_ctx(x)\n p2 = ast.Tuple(\n elts=p2,\n ctx=ast.Store(),\n lineno=p2[0].lineno,\n col_offset=p2[0].col_offset,\n )\n p[0] = [\n ast.For(\n target=p2,\n iter=p[4],\n body=p[6],\n orelse=p7,\n lineno=p1.lineno,\n col_offset=p1.lexpos,\n )\n ]\n\n def p_except_part(self, p):\n \"\"\"except_part : except_clause COLON suite\"\"\"\n p0 = p[1]\n p0.body = p[3]\n p[0] = [p0]\n\n def p_finally_part(self, p):\n \"\"\"finally_part : FINALLY COLON suite\"\"\"\n p[0] = p[3]\n\n def p_try_stmt_t5(self, p):\n \"\"\"try_stmt : try_tok COLON suite finally_part\"\"\"\n p1 = p[1]\n p[0] = [\n ast.Try(\n body=p[3],\n handlers=[],\n orelse=[],\n finalbody=p[4],\n lineno=p1.lineno,\n col_offset=p1.lexpos,\n )\n ]\n\n def p_try_stmt_t6(self, p):\n \"\"\"try_stmt : try_tok COLON suite except_part_list finally_part_opt\"\"\"\n p1 = p[1]\n p[0] = [\n ast.Try(\n body=p[3],\n handlers=p[4],\n orelse=[],\n finalbody=([] if p[5] is None else p[5]),\n lineno=p1.lineno,\n col_offset=p1.lexpos,\n )\n ]\n\n def p_try_stmt_t7(self, p):\n \"\"\"try_stmt : try_tok COLON suite except_part_list else_part finally_part_opt\"\"\"\n p1 = p[1]\n p[0] = [\n ast.Try(\n body=p[3],\n handlers=p[4],\n orelse=([] if p[5] is None else p[5]),\n finalbody=([] if p[6] is None else p[6]),\n lineno=p1.lineno,\n col_offset=p1.lexpos,\n )\n ]\n\n def p_with_stmt_w5(self, p):\n \"\"\"with_stmt : with_tok with_item COLON suite\"\"\"\n p1 = p[1]\n p[0] = [\n ast.With(items=[p[2]], body=p[4], lineno=p1.lineno, col_offset=p1.lexpos)\n ]\n\n def p_with_stmt_p6(self, p):\n \"\"\"with_stmt : with_tok with_item comma_with_item_list COLON suite\"\"\"\n p1 = p[1]\n p[0] = [\n ast.With(\n items=[p[2]] + p[3], body=p[5], lineno=p1.lineno, col_offset=p1.lexpos\n )\n ]\n\n def p_with_bang_stmt_single_suite(self, p):\n \"\"\"with_stmt : with_tok BANG with_item rawsuite\"\"\"\n p1, p3, p4 = p[1], p[3], p[4]\n expr = p3.context_expr\n l, c = expr.lineno, expr.col_offset\n gblcall = xonsh_call(\"globals\", [], lineno=l, col=c)\n loccall = xonsh_call(\"locals\", [], lineno=l, col=c)\n margs = [expr, p4, gblcall, loccall]\n p3.context_expr = xonsh_call(\"__xonsh__.enter_macro\", margs, lineno=l, col=c)\n body = [ast.Pass(lineno=p4.lineno, col_offset=p4.col_offset)]\n p[0] = [ast.With(items=[p3], body=body, lineno=p1.lineno, col_offset=p1.lexpos)]\n\n def p_with_bang_stmt_many_suite(self, p):\n \"\"\"with_stmt : with_tok BANG with_item comma_with_item_list rawsuite\"\"\"\n p1, p3, p4, p5 = p[1], p[3], p[4], p[5]\n items = [p3] + p4\n for item in items:\n expr = item.context_expr\n l, c = expr.lineno, expr.col_offset\n gblcall = xonsh_call(\"globals\", [], lineno=l, col=c)\n loccall = xonsh_call(\"locals\", [], lineno=l, col=c)\n margs = [expr, p5, gblcall, loccall]\n item.context_expr = xonsh_call(\n \"__xonsh__.enter_macro\", margs, lineno=l, col=c\n )\n body = [ast.Pass(lineno=p5.lineno, col_offset=p5.col_offset)]\n p[0] = [\n ast.With(items=items, body=body, lineno=p1.lineno, col_offset=p1.lexpos)\n ]\n\n def p_as_expr(self, p):\n \"\"\"as_expr : AS expr\"\"\"\n p2 = p[2]\n store_ctx(p2)\n p[0] = p2\n\n def p_with_item(self, p):\n \"\"\"with_item : test\n | test as_expr\n \"\"\"\n p2 = p[2] if len(p) > 2 else None\n p[0] = ast.withitem(context_expr=p[1], optional_vars=p2)\n\n def p_comma_with_item(self, p):\n \"\"\"comma_with_item : COMMA with_item\"\"\"\n p[0] = [p[2]]\n\n def p_except_clause_e2(self, p):\n \"\"\"except_clause : except_tok\"\"\"\n p1 = p[1]\n p[0] = ast.ExceptHandler(\n type=None, name=None, lineno=p1.lineno, col_offset=p1.lexpos\n )\n\n def p_except_clause(self, p):\n \"\"\"except_clause : except_tok test as_name_opt\"\"\"\n p1 = p[1]\n p[0] = ast.ExceptHandler(\n type=p[2], name=p[3], lineno=p1.lineno, col_offset=p1.lexpos\n )\n\n def p_suite(self, p):\n \"\"\"suite : simple_stmt\n | NEWLINE INDENT stmt_list DEDENT\n \"\"\"\n p[0] = p[1] if len(p) == 2 else p[3]\n\n def p_rawsuite_indent(self, p):\n \"\"\"rawsuite : COLON NEWLINE indent_tok nodedent dedent_tok\"\"\"\n p3, p5 = p[3], p[5]\n beg = (p3.lineno, p3.lexpos)\n end = (p5.lineno, p5.lexpos)\n s = self.source_slice(beg, end)\n s = textwrap.dedent(s)\n p[0] = ast.Str(s=s, lineno=beg[0], col_offset=beg[1])\n\n def p_rawsuite_simple_stmt(self, p):\n \"\"\"rawsuite : colon_tok nonewline newline_tok\"\"\"\n p1, p3 = p[1], p[3]\n beg = (p1.lineno, p1.lexpos + 1)\n end = (p3.lineno, p3.lexpos)\n s = self.source_slice(beg, end).strip()\n p[0] = ast.Str(s=s, lineno=beg[0], col_offset=beg[1])\n\n def _attach_nodedent_base_rules(self):\n toks = set(self.tokens)\n toks.remove(\"DEDENT\")\n ts = \"\\n | \".join(sorted(toks))\n doc = \"nodedent : \" + ts + \"\\n\"\n self.p_nodedent_base.__func__.__doc__ = doc\n\n def p_nodedent_base(self, p):\n # see above attachment function\n pass\n\n def p_nodedent_any(self, p):\n \"\"\"nodedent : INDENT any_dedent_toks DEDENT\"\"\"\n pass\n\n def p_nodedent_many(self, p):\n \"\"\"nodedent : nodedent nodedent\"\"\"\n pass\n\n def p_any_dedent_tok(self, p):\n \"\"\"any_dedent_tok : nodedent\n | DEDENT\n \"\"\"\n pass\n\n def p_any_dedent_toks(self, p):\n \"\"\"any_dedent_toks : any_dedent_tok\n | any_dedent_toks any_dedent_tok\n \"\"\"\n pass\n\n def _attach_nonewline_base_rules(self):\n toks = set(self.tokens)\n toks -= {\n \"NEWLINE\",\n \"LPAREN\",\n \"RPAREN\",\n \"LBRACE\",\n \"RBRACE\",\n \"LBRACKET\",\n \"RBRACKET\",\n \"AT_LPAREN\",\n \"BANG_LPAREN\",\n \"BANG_LBRACKET\",\n \"DOLLAR_LPAREN\",\n \"DOLLAR_LBRACE\",\n \"DOLLAR_LBRACKET\",\n \"ATDOLLAR_LPAREN\",\n }\n ts = \"\\n | \".join(sorted(toks))\n doc = \"nonewline : \" + ts + \"\\n\"\n self.p_nonewline_base.__func__.__doc__ = doc\n\n def p_nonewline_base(self, p):\n # see above attachment function\n pass\n\n def p_nonewline_any(self, p):\n \"\"\"nonewline : any_nested_raw\"\"\"\n pass\n\n def p_nonewline_many(self, p):\n \"\"\"nonewline : nonewline nonewline\"\"\"\n pass\n\n def p_test_ol(self, p):\n \"\"\"test : or_test\n | lambdef\n \"\"\"\n p[0] = p[1]\n\n def p_test_o5(self, p):\n \"\"\"test : or_test IF or_test ELSE test\"\"\"\n p[0] = ast.IfExp(\n test=p[3], body=p[1], orelse=p[5], lineno=self.lineno, col_offset=self.col\n )\n\n def p_test_nocond(self, p):\n \"\"\"test_nocond : or_test\n | lambdef_nocond\n \"\"\"\n p[0] = p[1]\n\n def p_lambdef(self, p):\n \"\"\"lambdef : lambda_tok varargslist_opt COLON test\"\"\"\n p1, p2, p4 = p[1], p[2], p[4]\n if p2 is None:\n args = ast.arguments(\n args=[],\n vararg=None,\n kwonlyargs=[],\n kw_defaults=[],\n kwarg=None,\n defaults=[],\n )\n else:\n args = p2\n p0 = ast.Lambda(args=args, body=p4, lineno=p1.lineno, col_offset=p1.lexpos)\n p[0] = p0\n\n def p_lambdef_nocond(self, p):\n \"\"\"lambdef_nocond : LAMBDA varargslist_opt COLON test_nocond\"\"\"\n assert False\n\n def p_or_test(self, p):\n \"\"\"or_test : and_test or_and_test_list_opt\"\"\"\n p1, p2 = p[1], p[2]\n if p2 is None:\n p0 = p1\n elif len(p2) == 2:\n lineno, col = lopen_loc(p1)\n p0 = ast.BoolOp(op=p2[0], values=[p1, p2[1]], lineno=lineno, col_offset=col)\n else:\n lineno, col = lopen_loc(p1)\n p0 = ast.BoolOp(\n op=p2[0], values=[p[1]] + p2[1::2], lineno=lineno, col_offset=col\n )\n p[0] = p0\n\n def p_or_and_test(self, p):\n \"\"\"or_and_test : OR and_test\"\"\"\n p[0] = [ast.Or(), p[2]]\n\n def p_and_test(self, p):\n \"\"\"and_test : not_test and_not_test_list_opt\"\"\"\n p1, p2 = p[1], p[2]\n if p2 is None:\n p0 = p1\n elif len(p2) == 2:\n lineno, col = lopen_loc(p1)\n p0 = ast.BoolOp(op=p2[0], values=[p1, p2[1]], lineno=lineno, col_offset=col)\n else:\n lineno, col = lopen_loc(p1)\n p0 = ast.BoolOp(\n op=p2[0], values=[p1] + p2[1::2], lineno=lineno, col_offset=col\n )\n p[0] = p0\n\n def p_and_not_test(self, p):\n \"\"\"and_not_test : AND not_test\"\"\"\n p[0] = [ast.And(), p[2]]\n\n def p_not_test_not(self, p):\n \"\"\"not_test : NOT not_test\"\"\"\n p[0] = ast.UnaryOp(\n op=ast.Not(), operand=p[2], lineno=self.lineno, col_offset=self.col\n )\n\n def p_not_test(self, p):\n \"\"\"not_test : comparison\"\"\"\n p[0] = p[1]\n\n def p_comparison(self, p):\n \"\"\"comparison : expr comp_op_expr_list_opt\"\"\"\n p1, p2 = p[1], p[2]\n if p2 is None:\n p0 = p1\n else:\n p0 = ast.Compare(\n left=p1,\n ops=p2[::2],\n comparators=p2[1::2],\n lineno=p1.lineno,\n col_offset=p1.col_offset,\n )\n p[0] = p0\n\n def p_comp_op_expr(self, p):\n \"\"\"comp_op_expr : comp_op expr\"\"\"\n p[0] = [p[1], p[2]]\n\n _comp_ops = {\n \"<\": ast.Lt,\n \">\": ast.Gt,\n \"==\": ast.Eq,\n \">=\": ast.GtE,\n \"<=\": ast.LtE,\n \"!=\": ast.NotEq,\n \"in\": ast.In,\n (\"not\", \"in\"): ast.NotIn,\n \"is\": ast.Is,\n (\"is\", \"not\"): ast.IsNot,\n }\n\n def p_comp_op_monograph(self, p):\n \"\"\"comp_op : LT\n | GT\n | EQ\n | GE\n | LE\n | NE\n | IN\n | IS\n \"\"\"\n p[0] = self._comp_ops[p[1]]()\n\n def p_comp_op_digraph(self, p):\n \"\"\"comp_op : NOT IN\n | IS NOT\n \"\"\"\n p[0] = self._comp_ops[(p[1], p[2])]()\n\n def p_star_expr(self, p):\n \"\"\"star_expr : times_tok expr\"\"\"\n p1 = p[1]\n p[0] = ast.Starred(\n value=p[2], ctx=ast.Load(), lineno=p1.lineno, col_offset=p1.lexpos\n )\n\n def _binop_combine(self, p1, p2):\n \"\"\"Combines binary operations\"\"\"\n if p2 is None:\n p0 = p1\n elif isinstance(p2, ast.BinOp):\n p2.left = p1\n p0 = p2\n elif isinstance(p2, Sequence) and isinstance(p2[0], ast.BinOp):\n p0 = p2[0]\n p0.left = p1\n p0.lineno, p0.col_offset = lopen_loc(p1)\n for bop in p2[1:]:\n locer = p1 if p0.left is p1 else p0\n bop.left = p0\n p0.lineno, p0.col_offset = lopen_loc(locer)\n p0 = bop\n else:\n p0 = p1 + p2\n return p0\n\n def p_expr(self, p):\n \"\"\"expr : xor_expr\n | xor_expr pipe_xor_expr_list\n \"\"\"\n p[0] = self._binop_combine(p[1], p[2] if len(p) > 2 else None)\n\n def p_pipe_xor_expr(self, p):\n \"\"\"pipe_xor_expr : pipe_tok xor_expr\"\"\"\n p1 = p[1]\n p[0] = [\n ast.BinOp(\n left=None,\n op=ast.BitOr(),\n right=p[2],\n lineno=p1.lineno,\n col_offset=p1.lexpos,\n )\n ]\n\n def p_xor_expr(self, p):\n \"\"\"xor_expr : and_expr xor_and_expr_list_opt\"\"\"\n p[0] = self._binop_combine(p[1], p[2])\n\n def p_xor_and_expr(self, p):\n \"\"\"xor_and_expr : xor_tok and_expr\"\"\"\n p1 = p[1]\n p[0] = [\n ast.BinOp(\n left=None,\n op=ast.BitXor(),\n right=p[2],\n lineno=p1.lineno,\n col_offset=p1.lexpos,\n )\n ]\n\n def p_and_expr(self, p):\n \"\"\"and_expr : shift_expr ampersand_shift_expr_list_opt\"\"\"\n p[0] = self._binop_combine(p[1], p[2])\n\n def p_ampersand_shift_expr(self, p):\n \"\"\"ampersand_shift_expr : ampersand_tok shift_expr\"\"\"\n p1 = p[1]\n p[0] = [\n ast.BinOp(\n left=None,\n op=ast.BitAnd(),\n right=p[2],\n lineno=p1.lineno,\n col_offset=p1.lexpos,\n )\n ]\n\n def p_shift_expr(self, p):\n \"\"\"shift_expr : arith_expr shift_arith_expr_list_opt\"\"\"\n p[0] = self._binop_combine(p[1], p[2])\n\n def p_shift_arith_expr(self, p):\n \"\"\"shift_arith_expr : lshift_tok arith_expr\n | rshift_tok arith_expr\n \"\"\"\n p1 = p[1]\n op = ast.LShift() if p1.value == \"<<\" else ast.RShift()\n p[0] = [\n ast.BinOp(\n left=None, op=op, right=p[2], lineno=p1.lineno, col_offset=p1.lexpos\n )\n ]\n\n def p_arith_expr_single(self, p):\n \"\"\"arith_expr : term\"\"\"\n p[0] = p[1]\n\n def p_arith_expr_many(self, p):\n \"\"\"arith_expr : term pm_term_list\"\"\"\n p1, p2 = p[1], p[2]\n if len(p2) == 2:\n lineno, col = lopen_loc(p1)\n p0 = ast.BinOp(\n left=p1, op=p2[0], right=p2[1], lineno=lineno, col_offset=col\n )\n else:\n left = p1\n for op, right in zip(p2[::2], p2[1::2]):\n locer = left if left is p1 else op\n lineno, col = lopen_loc(locer)\n left = ast.BinOp(\n left=left, op=op, right=right, lineno=lineno, col_offset=col\n )\n p0 = left\n p[0] = p0\n\n _term_binops = {\n \"+\": ast.Add,\n \"-\": ast.Sub,\n \"*\": ast.Mult,\n \"@\": ast.MatMult,\n \"/\": ast.Div,\n \"%\": ast.Mod,\n \"//\": ast.FloorDiv,\n }\n\n def p_pm_term(self, p):\n \"\"\"pm_term : plus_tok term\n | minus_tok term\n \"\"\"\n p1 = p[1]\n op = self._term_binops[p1.value](lineno=p1.lineno, col_offset=p1.lexpos)\n p[0] = [op, p[2]]\n\n def p_term(self, p):\n \"\"\"term : factor op_factor_list_opt\"\"\"\n p1, p2 = p[1], p[2]\n if p2 is None:\n p0 = p1\n elif len(p2) == 2:\n lineno, col = lopen_loc(p1)\n p0 = ast.BinOp(\n left=p1, op=p2[0], right=p2[1], lineno=lineno, col_offset=col\n )\n else:\n left = p1\n for op, right in zip(p2[::2], p2[1::2]):\n locer = left if left is p1 else op\n lineno, col = lopen_loc(locer)\n left = ast.BinOp(\n left=left, op=op, right=right, lineno=lineno, col_offset=col\n )\n p0 = left\n p[0] = p0\n\n def p_op_factor(self, p):\n \"\"\"op_factor : times_tok factor\n | at_tok factor\n | divide_tok factor\n | mod_tok factor\n | doublediv_tok factor\n \"\"\"\n p1 = p[1]\n op = self._term_binops[p1.value]\n if op is None:\n self._parse_error(\n \"operation {0!r} not supported\".format(p1),\n self.currloc(lineno=p.lineno, column=p.lexpos),\n )\n p[0] = [op(lineno=p1.lineno, col_offset=p1.lexpos), p[2]]\n\n _factor_ops = {\"+\": ast.UAdd, \"-\": ast.USub, \"~\": ast.Invert}\n\n def p_factor_power(self, p):\n \"\"\"factor : power\"\"\"\n p[0] = p[1]\n\n def p_factor_unary(self, p):\n \"\"\"factor : PLUS factor\n | MINUS factor\n | TILDE factor\n \"\"\"\n op = self._factor_ops[p[1]]()\n p[0] = ast.UnaryOp(op=op, operand=p[2], lineno=self.lineno, col_offset=self.col)\n\n def p_power_atom(self, p):\n \"\"\"power : atom_expr\"\"\"\n p[0] = p[1]\n\n def p_power(self, p):\n \"\"\"power : atom_expr POW factor\"\"\"\n p1 = p[1]\n p[0] = ast.BinOp(\n left=p1,\n op=ast.Pow(),\n right=p[3],\n lineno=p1.lineno,\n col_offset=p1.col_offset,\n )\n\n def p_yield_expr_or_testlist_comp(self, p):\n \"\"\"yield_expr_or_testlist_comp : yield_expr\n | testlist_comp\n \"\"\"\n p[0] = p[1]\n\n def _list_or_elts_if_not_real_tuple(self, x):\n if isinstance(x, ast.Tuple) and not (\n hasattr(x, \"_real_tuple\") and x._real_tuple\n ):\n rtn = x.elts\n else:\n rtn = [x]\n return rtn\n\n def apply_trailers(self, leader, trailers):\n \"\"\"Helper function for atom expr.\"\"\"\n if trailers is None:\n return leader\n p0 = leader\n for trailer in trailers:\n if isinstance(trailer, (ast.Index, ast.Slice, ast.ExtSlice)):\n p0 = ast.Subscript(\n value=leader,\n slice=trailer,\n ctx=ast.Load(),\n lineno=leader.lineno,\n col_offset=leader.col_offset,\n )\n elif isinstance(trailer, Mapping):\n # call normal functions\n p0 = ast.Call(\n func=leader,\n lineno=leader.lineno,\n col_offset=leader.col_offset,\n **trailer\n )\n elif isinstance(trailer, (ast.Tuple, tuple)):\n # call macro functions\n l, c = leader.lineno, leader.col_offset\n gblcall = xonsh_call(\"globals\", [], lineno=l, col=c)\n loccall = xonsh_call(\"locals\", [], lineno=l, col=c)\n if isinstance(trailer, tuple):\n trailer, arglist = trailer\n margs = [leader, trailer, gblcall, loccall]\n p0 = xonsh_call(\"__xonsh__.call_macro\", margs, lineno=l, col=c)\n elif isinstance(trailer, str):\n if trailer == \"?\":\n p0 = xonsh_help(leader, lineno=leader.lineno, col=leader.col_offset)\n elif trailer == \"??\":\n p0 = xonsh_superhelp(\n leader, lineno=leader.lineno, col=leader.col_offset\n )\n else:\n p0 = ast.Attribute(\n value=leader,\n attr=trailer,\n ctx=ast.Load(),\n lineno=leader.lineno,\n col_offset=leader.col_offset,\n )\n else:\n assert False\n leader = p0\n return p0\n\n def p_atom_expr(self, p):\n \"\"\"atom_expr : atom trailer_list_opt\"\"\"\n p[0] = self.apply_trailers(p[1], p[2])\n\n #\n # Atom rules! (So does Adam!)\n #\n def p_atom_lparen(self, p):\n \"\"\"atom : lparen_tok yield_expr_or_testlist_comp_opt RPAREN\"\"\"\n p1, p2 = p[1], p[2]\n p1, p1_tok = p1.value, p1\n if p2 is None:\n # empty container atom\n p0 = ast.Tuple(\n elts=[], ctx=ast.Load(), lineno=self.lineno, col_offset=self.col\n )\n p0._real_tuple = True\n elif isinstance(p2, ast.AST):\n p0 = p2\n p0._lopen_lineno, p0._lopen_col = p1_tok.lineno, p1_tok.lexpos\n p0._real_tuple = True\n elif len(p2) == 1 and isinstance(p2[0], ast.AST):\n p0 = p2[0]\n p0._lopen_lineno, p0._lopen_col = p1_tok.lineno, p1_tok.lexpos\n else:\n self.p_error(p)\n p[0] = p0\n\n def p_atom_lbraket(self, p):\n \"\"\"atom : lbracket_tok testlist_comp_opt RBRACKET\"\"\"\n p1, p2 = p[1], p[2]\n p1, p1_tok = p1.value, p1\n if p2 is None:\n p0 = ast.List(\n elts=[], ctx=ast.Load(), lineno=self.lineno, col_offset=self.col\n )\n\n elif isinstance(p2, ast.GeneratorExp):\n p0 = ast.ListComp(\n elt=p2.elt,\n generators=p2.generators,\n lineno=p2.lineno,\n col_offset=p2.col_offset,\n )\n else:\n if isinstance(p2, ast.Tuple):\n if hasattr(p2, \"_real_tuple\") and p2._real_tuple:\n elts = [p2]\n else:\n elts = p2.elts\n else:\n elts = [p2]\n p0 = ast.List(\n elts=elts,\n ctx=ast.Load(),\n lineno=p1_tok.lineno,\n col_offset=p1_tok.lexpos,\n )\n p[0] = p0\n\n def p_atom_lbrace(self, p):\n \"\"\"atom : lbrace_tok dictorsetmaker_opt RBRACE\"\"\"\n p1, p2 = p[1], p[2]\n p1, p1_tok = p1.value, p1\n if p2 is None:\n p0 = ast.Dict(\n keys=[],\n values=[],\n ctx=ast.Load(),\n lineno=self.lineno,\n col_offset=self.col,\n )\n else:\n p0 = p2\n p0.lineno, p0.col_offset = p1_tok.lineno, p1_tok.lexpos\n p[0] = p0\n\n def p_atom_ns(self, p):\n \"\"\"atom : number\n | string_literal_list\n \"\"\"\n p[0] = p[1]\n\n def p_atom_name(self, p):\n \"\"\"atom : name_tok\"\"\"\n p1 = p[1]\n p[0] = ast.Name(\n id=p1.value, ctx=ast.Load(), lineno=p1.lineno, col_offset=p1.lexpos\n )\n\n def p_atom_ellip(self, p):\n \"\"\"atom : ellipsis_tok\"\"\"\n p1 = p[1]\n p[0] = ast.EllipsisNode(lineno=p1.lineno, col_offset=p1.lexpos)\n\n def p_atom_none(self, p):\n \"\"\"atom : none_tok\"\"\"\n p1 = p[1]\n p[0] = ast.NameConstant(value=None, lineno=p1.lineno, col_offset=p1.lexpos)\n\n def p_atom_true(self, p):\n \"\"\"atom : true_tok\"\"\"\n p1 = p[1]\n p[0] = ast.NameConstant(value=True, lineno=p1.lineno, col_offset=p1.lexpos)\n\n def p_atom_false(self, p):\n \"\"\"atom : false_tok\"\"\"\n p1 = p[1]\n p[0] = ast.NameConstant(value=False, lineno=p1.lineno, col_offset=p1.lexpos)\n\n def p_atom_pathsearch(self, p):\n \"\"\"atom : SEARCHPATH\"\"\"\n p[0] = xonsh_pathsearch(p[1], pymode=True, lineno=self.lineno, col=self.col)\n\n def p_atom_dname(self, p):\n \"\"\"atom : DOLLAR_NAME\"\"\"\n p[0] = self._envvar_by_name(p[1][1:], lineno=self.lineno, col=self.col)\n\n def p_atom_fistful_of_dollars(self, p):\n \"\"\"atom : dollar_lbrace_tok test RBRACE\n | bang_lparen_tok subproc RPAREN\n | dollar_lparen_tok subproc RPAREN\n | bang_lbracket_tok subproc RBRACKET\n | dollar_lbracket_tok subproc RBRACKET\n \"\"\"\n p[0] = self._dollar_rules(p)\n\n def p_atom_bang_empty_fistful_of_dollars(self, p):\n \"\"\"atom : bang_lparen_tok subproc bang_tok RPAREN\n | dollar_lparen_tok subproc bang_tok RPAREN\n | bang_lbracket_tok subproc bang_tok RBRACKET\n | dollar_lbracket_tok subproc bang_tok RBRACKET\n \"\"\"\n self._append_subproc_bang_empty(p)\n p[0] = self._dollar_rules(p)\n\n def p_atom_bang_fistful_of_dollars(self, p):\n \"\"\"atom : bang_lparen_tok subproc bang_tok nocloser rparen_tok\n | dollar_lparen_tok subproc bang_tok nocloser rparen_tok\n | bang_lbracket_tok subproc bang_tok nocloser rbracket_tok\n | dollar_lbracket_tok subproc bang_tok nocloser rbracket_tok\n \"\"\"\n self._append_subproc_bang(p)\n p[0] = self._dollar_rules(p)\n\n def _attach_nocloser_base_rules(self):\n toks = set(self.tokens)\n toks -= {\n \"LPAREN\",\n \"RPAREN\",\n \"LBRACE\",\n \"RBRACE\",\n \"LBRACKET\",\n \"RBRACKET\",\n \"AT_LPAREN\",\n \"BANG_LPAREN\",\n \"BANG_LBRACKET\",\n \"DOLLAR_LPAREN\",\n \"DOLLAR_LBRACE\",\n \"DOLLAR_LBRACKET\",\n \"ATDOLLAR_LPAREN\",\n }\n ts = \"\\n | \".join(sorted(toks))\n doc = \"nocloser : \" + ts + \"\\n\"\n self.p_nocloser_base.__func__.__doc__ = doc\n\n def p_nocloser_base(self, p):\n # see above attachment function\n pass\n\n def p_nocloser_any(self, p):\n \"\"\"nocloser : any_nested_raw\"\"\"\n pass\n\n def p_nocloser_many(self, p):\n \"\"\"nocloser : nocloser nocloser\"\"\"\n pass\n\n def p_string_literal(self, p):\n \"\"\"string_literal : string_tok\"\"\"\n p1 = p[1]\n prefix = RE_STRINGPREFIX.match(p1.value).group().lower()\n if \"p\" in prefix and \"f\" in prefix:\n new_pref = prefix.replace(\"p\", \"\")\n value_without_p = new_pref + p1.value[len(prefix):]\n s = eval_fstr_fields(value_without_p, new_pref, filename=self.lexer.fname)\n s = pyparse(s).body[0].value\n s = ast.increment_lineno(s, p1.lineno - 1)\n p[0] = xonsh_call(\n \"__xonsh__.path_literal\", [s], lineno=p1.lineno, col=p1.lexpos\n )\n\n elif \"p\" in prefix:\n value_without_p = prefix.replace(\"p\", \"\") + p1.value[len(prefix) :]\n s = ast.Str(\n s=ast.literal_eval(value_without_p),\n lineno=p1.lineno,\n col_offset=p1.lexpos,\n )\n p[0] = xonsh_call(\n \"__xonsh__.path_literal\", [s], lineno=p1.lineno, col=p1.lexpos\n )\n elif \"f\" in prefix:\n s = eval_fstr_fields(p1.value, prefix, filename=self.lexer.fname)\n s = pyparse(s).body[0].value\n s = ast.increment_lineno(s, p1.lineno - 1)\n p[0] = s\n else:\n s = ast.literal_eval(p1.value)\n is_bytes = \"b\" in prefix\n cls = ast.Bytes if is_bytes else ast.Str\n p[0] = cls(s=s, lineno=p1.lineno, col_offset=p1.lexpos)\n\n def p_string_literal_list(self, p):\n \"\"\"string_literal_list : string_literal\n | string_literal_list string_literal\n \"\"\"\n if len(p) == 3:\n p[1].s += p[2].s\n p[0] = p[1]\n\n def p_number(self, p):\n \"\"\"number : number_tok\"\"\"\n p1 = p[1]\n p[0] = ast.Num(\n n=ast.literal_eval(p1.value.replace(\"_\", \"\")),\n lineno=p1.lineno,\n col_offset=p1.lexpos,\n )\n\n def p_testlist_comp_comp(self, p):\n \"\"\"testlist_comp : test_or_star_expr comp_for\"\"\"\n p1, p2 = p[1], p[2]\n p[0] = ast.GeneratorExp(\n elt=p1, generators=p2[\"comps\"], lineno=p1.lineno, col_offset=p1.col_offset\n )\n\n def p_testlist_comp_comma(self, p):\n \"\"\"testlist_comp : test_or_star_expr comma_opt\"\"\"\n p1, p2 = p[1], p[2]\n if p2 is None: # split out grouping parentheses.\n p[0] = p1\n else:\n p[0] = ast.Tuple(\n elts=[p1], ctx=ast.Load(), lineno=p1.lineno, col_offset=p1.col_offset\n )\n\n def p_testlist_comp_many(self, p):\n \"\"\"testlist_comp : test_or_star_expr comma_test_or_star_expr_list comma_opt\"\"\"\n p1, p2 = p[1], p[2]\n p[0] = ast.Tuple(\n elts=[p1] + p2, ctx=ast.Load(), lineno=p1.lineno, col_offset=p1.col_offset\n )\n\n def p_trailer_lparen(self, p):\n \"\"\"trailer : LPAREN arglist_opt RPAREN\"\"\"\n p[0] = [p[2] or dict(args=[], keywords=[], starargs=None, kwargs=None)]\n\n def p_trailer_bang_lparen(self, p):\n \"\"\"trailer : bang_lparen_tok macroarglist_opt rparen_tok\n | bang_lparen_tok nocomma comma_tok rparen_tok\n | bang_lparen_tok nocomma comma_tok WS rparen_tok\n | bang_lparen_tok macroarglist comma_tok rparen_tok\n | bang_lparen_tok macroarglist comma_tok WS rparen_tok\n \"\"\"\n p1, p2, p3 = p[1], p[2], p[3]\n begins = [(p1.lineno, p1.lexpos + 2)]\n ends = [(p3.lineno, p3.lexpos)]\n if p2:\n begins.extend([(x[0], x[1] + 1) for x in p2])\n ends = p2 + ends\n elts = []\n for beg, end in zip(begins, ends):\n s = self.source_slice(beg, end).strip()\n if not s:\n if len(begins) == 1:\n break\n else:\n msg = \"empty macro arguments not allowed\"\n self._parse_error(msg, self.currloc(*beg))\n node = ast.Str(s=s, lineno=beg[0], col_offset=beg[1])\n elts.append(node)\n p0 = ast.Tuple(\n elts=elts, ctx=ast.Load(), lineno=p1.lineno, col_offset=p1.lexpos\n )\n p[0] = [p0]\n\n def p_trailer_p3(self, p):\n \"\"\"trailer : LBRACKET subscriptlist RBRACKET\n | PERIOD NAME\n \"\"\"\n p[0] = [p[2]]\n\n def p_trailer_quest(self, p):\n \"\"\"trailer : DOUBLE_QUESTION\n | QUESTION\n \"\"\"\n p[0] = [p[1]]\n\n def _attach_nocomma_tok_rules(self):\n toks = set(self.tokens)\n toks -= {\n \"COMMA\",\n \"LPAREN\",\n \"RPAREN\",\n \"LBRACE\",\n \"RBRACE\",\n \"LBRACKET\",\n \"RBRACKET\",\n \"AT_LPAREN\",\n \"BANG_LPAREN\",\n \"BANG_LBRACKET\",\n \"DOLLAR_LPAREN\",\n \"DOLLAR_LBRACE\",\n \"DOLLAR_LBRACKET\",\n \"ATDOLLAR_LPAREN\",\n }\n ts = \"\\n | \".join(sorted(toks))\n doc = \"nocomma_tok : \" + ts + \"\\n\"\n self.p_nocomma_tok.__func__.__doc__ = doc\n\n # The following grammar rules are no-ops because we don't need to glue the\n # source code back together piece-by-piece. Instead, we simply look for\n # top-level commas and record their positions. With these positions and the\n # respective positions of the bounding parentheses, we can use the\n # source_slice() method. This does a much better job of capturing exactly\n # the source code that was provided. The tokenizer & lexer can be a little\n # lossy, especially with respect to whitespace.\n\n def p_nocomma_tok(self, p):\n # see attachment function above for docstring\n pass\n\n def p_any_raw_tok(self, p):\n \"\"\"any_raw_tok : nocomma\n | COMMA\n \"\"\"\n pass\n\n def p_any_raw_toks_one(self, p):\n \"\"\"any_raw_toks : any_raw_tok\"\"\"\n pass\n\n def p_any_raw_toks_many(self, p):\n \"\"\"any_raw_toks : any_raw_toks any_raw_tok\"\"\"\n pass\n\n def p_nocomma_part_tok(self, p):\n \"\"\"nocomma_part : nocomma_tok\"\"\"\n pass\n\n def p_any_nested_raw(self, p):\n \"\"\"any_nested_raw : LPAREN any_raw_toks_opt RPAREN\n | LBRACE any_raw_toks_opt RBRACE\n | LBRACKET any_raw_toks_opt RBRACKET\n | AT_LPAREN any_raw_toks_opt RPAREN\n | BANG_LPAREN any_raw_toks_opt RPAREN\n | BANG_LBRACKET any_raw_toks_opt RBRACKET\n | DOLLAR_LPAREN any_raw_toks_opt RPAREN\n | DOLLAR_LBRACE any_raw_toks_opt RBRACE\n | DOLLAR_LBRACKET any_raw_toks_opt RBRACKET\n | ATDOLLAR_LPAREN any_raw_toks_opt RPAREN\n \"\"\"\n pass\n\n def p_nocomma_part_any(self, p):\n \"\"\"nocomma_part : any_nested_raw\"\"\"\n pass\n\n def p_nocomma_base(self, p):\n \"\"\"nocomma : nocomma_part\"\"\"\n pass\n\n def p_nocomma_append(self, p):\n \"\"\"nocomma : nocomma nocomma_part\"\"\"\n pass\n\n def p_comma_nocomma(self, p):\n \"\"\"comma_nocomma : comma_tok nocomma\"\"\"\n p1 = p[1]\n p[0] = [(p1.lineno, p1.lexpos)]\n\n def p_macroarglist_single(self, p):\n \"\"\"macroarglist : nocomma\"\"\"\n p[0] = []\n\n def p_macroarglist_many(self, p):\n \"\"\"macroarglist : nocomma comma_nocomma_list\"\"\"\n p[0] = p[2]\n\n def p_subscriptlist(self, p):\n \"\"\"subscriptlist : subscript comma_subscript_list_opt comma_opt\"\"\"\n p1, p2 = p[1], p[2]\n if p2 is None:\n pass\n elif isinstance(p1, ast.Slice) or any([isinstance(x, ast.Slice) for x in p2]):\n p1 = ast.ExtSlice(dims=[p1] + p2)\n else:\n p1.value = ast.Tuple(\n elts=[p1.value] + [x.value for x in p2],\n ctx=ast.Load(),\n lineno=p1.lineno,\n col_offset=p1.col_offset,\n )\n p[0] = p1\n\n def p_comma_subscript(self, p):\n \"\"\"comma_subscript : COMMA subscript\"\"\"\n p[0] = [p[2]]\n\n def p_subscript_test(self, p):\n \"\"\"subscript : test\"\"\"\n p1 = p[1]\n p[0] = ast.Index(value=p1, lineno=p1.lineno, col_offset=p1.col_offset)\n\n def p_subscript_tok(self, p):\n \"\"\"subscript : test_opt colon_tok test_opt sliceop_opt\"\"\"\n p1 = p[1]\n if p1 is None:\n p2 = p[2]\n lineno, col = p2.lineno, p2.lexpos\n else:\n lineno, col = p1.lineno, p1.col_offset\n p[0] = ast.Slice(lower=p1, upper=p[3], step=p[4], lineno=lineno, col_offset=col)\n\n def p_sliceop(self, p):\n \"\"\"sliceop : COLON test_opt\"\"\"\n p[0] = p[2]\n\n def p_expr_or_star_expr(self, p):\n \"\"\"expr_or_star_expr : expr\n | star_expr\n \"\"\"\n p[0] = p[1]\n\n def p_comma_expr_or_star_expr(self, p):\n \"\"\"comma_expr_or_star_expr : COMMA expr_or_star_expr\"\"\"\n p[0] = [p[2]]\n\n def p_exprlist_e3(self, p):\n \"\"\"exprlist : expr_or_star_expr comma_opt\"\"\"\n p[0] = [p[1]]\n\n def p_exprlist_many(self, p):\n \"\"\"exprlist : expr_or_star_expr comma_expr_or_star_expr_list comma_opt\"\"\"\n p2 = p[2]\n p2.insert(0, p[1])\n p[0] = p2\n\n def p_testlist_test(self, p):\n \"\"\"testlist : test\"\"\"\n p1 = p[1]\n if isinstance(p1, ast.Tuple) and (\n hasattr(p1, \"_real_tuple\") and p1._real_tuple and p1.elts\n ):\n p1.lineno, p1.col_offset = lopen_loc(p1.elts[0])\n p[0] = p1\n\n def p_testlist_single(self, p):\n \"\"\"testlist : test COMMA\"\"\"\n p1 = p[1]\n if isinstance(p1, ast.List) or (\n isinstance(p1, ast.Tuple) and hasattr(p1, \"_real_tuple\") and p1._real_tuple\n ):\n lineno, col = lopen_loc(p1)\n p[0] = ast.Tuple(\n elts=[p1], ctx=ast.Load(), lineno=p1.lineno, col_offset=p1.col_offset\n )\n else:\n p[0] = ensure_has_elts(p[1])\n\n def p_testlist_many(self, p):\n \"\"\"testlist : test comma_test_list COMMA\n | test comma_test_list\n \"\"\"\n p1 = p[1]\n if isinstance(p1, ast.List) or (\n isinstance(p1, ast.Tuple) and hasattr(p1, \"_real_tuple\") and p1._real_tuple\n ):\n lineno, col = lopen_loc(p1)\n p1 = ast.Tuple(\n elts=[p1], ctx=ast.Load(), lineno=p1.lineno, col_offset=p1.col_offset\n )\n else:\n p1 = ensure_has_elts(p1)\n p1.elts += p[2]\n p[0] = p1\n\n def p_comma_item(self, p):\n \"\"\"comma_item : COMMA item\"\"\"\n p[0] = p[2]\n\n #\n # Dict or set maker\n #\n def p_dictorsetmaker_t6(self, p):\n \"\"\"dictorsetmaker : test COLON test comma_item_list comma_opt\"\"\"\n p1, p4 = p[1], p[4]\n keys = [p1]\n vals = [p[3]]\n for k, v in zip(p4[::2], p4[1::2]):\n keys.append(k)\n vals.append(v)\n lineno, col = lopen_loc(p1)\n p[0] = ast.Dict(\n keys=keys, values=vals, ctx=ast.Load(), lineno=lineno, col_offset=col\n )\n\n def p_dictorsetmaker_i4(self, p):\n \"\"\"dictorsetmaker : item comma_item_list comma_opt\"\"\"\n p1, p2 = p[1], p[2]\n keys = [p1[0]]\n vals = [p1[1]]\n for k, v in zip(p2[::2], p2[1::2]):\n keys.append(k)\n vals.append(v)\n lineno, col = lopen_loc(p1[0] or p2[0])\n p[0] = ast.Dict(\n keys=keys, values=vals, ctx=ast.Load(), lineno=lineno, col_offset=col\n )\n\n def p_dictorsetmaker_t4_dict(self, p):\n \"\"\"dictorsetmaker : test COLON testlist\"\"\"\n keys = [p[1]]\n vals = self._list_or_elts_if_not_real_tuple(p[3])\n lineno, col = lopen_loc(p[1])\n p[0] = ast.Dict(\n keys=keys, values=vals, ctx=ast.Load(), lineno=lineno, col_offset=col\n )\n\n def p_dictorsetmaker_t4_set(self, p):\n \"\"\"dictorsetmaker : test_or_star_expr comma_test_or_star_expr_list comma_opt\"\"\"\n p[0] = ast.Set(\n elts=[p[1]] + p[2], ctx=ast.Load(), lineno=self.lineno, col_offset=self.col\n )\n\n def p_dictorsetmaker_test_comma(self, p):\n \"\"\"dictorsetmaker : test_or_star_expr comma_opt\"\"\"\n elts = self._list_or_elts_if_not_real_tuple(p[1])\n p[0] = ast.Set(\n elts=elts, ctx=ast.Load(), lineno=self.lineno, col_offset=self.col\n )\n\n def p_dictorsetmaker_testlist(self, p):\n \"\"\"dictorsetmaker : testlist\"\"\"\n elts = self._list_or_elts_if_not_real_tuple(p[1])\n p[0] = ast.Set(\n elts=elts, ctx=ast.Load(), lineno=self.lineno, col_offset=self.col\n )\n\n def p_dictorsetmaker_comp(self, p):\n \"\"\"dictorsetmaker : item comp_for\n | test_or_star_expr comp_for\n \"\"\"\n p1 = p[1]\n comps = p[2].get(\"comps\", [])\n if isinstance(p1, list) and len(p1) == 2:\n p[0] = ast.DictComp(\n key=p1[0],\n value=p1[1],\n generators=comps,\n lineno=self.lineno,\n col_offset=self.col,\n )\n else:\n p[0] = ast.SetComp(\n elt=p1, generators=comps, lineno=self.lineno, col_offset=self.col\n )\n\n def p_classdef(self, p):\n \"\"\"classdef : class_tok NAME func_call_opt COLON suite\"\"\"\n p1, p3 = p[1], p[3]\n b, kw = ([], []) if p3 is None else (p3[\"args\"], p3[\"keywords\"])\n c = ast.ClassDef(\n name=p[2],\n bases=b,\n keywords=kw,\n starargs=None,\n kwargs=None,\n body=p[5],\n decorator_list=[],\n lineno=p1.lineno,\n col_offset=p1.lexpos,\n )\n p[0] = [c]\n\n def p_comma_argument(self, p):\n \"\"\"comma_argument : COMMA argument\"\"\"\n p[0] = [p[2]]\n\n def p_comp_iter(self, p):\n \"\"\"comp_iter : comp_for\n | comp_if\n \"\"\"\n p[0] = p[1]\n\n def p_comp_for(self, p):\n \"\"\"comp_for : FOR exprlist IN or_test comp_iter_opt\"\"\"\n targs, it, p5 = p[2], p[4], p[5]\n if len(targs) == 1:\n targ = targs[0]\n else:\n targ = ensure_has_elts(targs)\n store_ctx(targ)\n comp = ast.comprehension(target=targ, iter=it, ifs=[])\n comps = [comp]\n p0 = {\"comps\": comps}\n if p5 is not None:\n comps += p5.get(\"comps\", [])\n comp.ifs += p5.get(\"if\", [])\n p[0] = p0\n\n def p_comp_if(self, p):\n \"\"\"comp_if : IF test_nocond comp_iter_opt\"\"\"\n p2, p3 = p[2], p[3]\n p0 = {\"if\": [p2]}\n if p3 is not None:\n p0[\"comps\"] = p3.get(\"comps\", [])\n p[0] = p0\n\n def p_yield_expr(self, p):\n \"\"\"yield_expr : yield_tok yield_arg_opt\"\"\"\n p1, p2 = p[1], p[2]\n if p2 is None:\n p0 = ast.Yield(value=p2, lineno=p1.lineno, col_offset=p1.lexpos)\n elif p2[\"from\"]:\n p0 = ast.YieldFrom(value=p2[\"val\"], lineno=p1.lineno, col_offset=p1.lexpos)\n else:\n p0 = ast.Yield(value=p2[\"val\"], lineno=p1.lineno, col_offset=p1.lexpos)\n p[0] = p0\n\n def p_yield_arg_from(self, p):\n \"\"\"yield_arg : FROM test\"\"\"\n p[0] = {\"from\": True, \"val\": p[2]}\n\n def p_yield_arg_testlist(self, p):\n \"\"\"yield_arg : testlist\"\"\"\n p[0] = {\"from\": False, \"val\": p[1]}\n\n #\n # subprocess\n #\n\n def _dollar_rules(self, p):\n \"\"\"These handle the special xonsh $ shell atoms by looking up\n in a special __xonsh__.env dictionary injected in the __builtin__.\n \"\"\"\n lenp = len(p)\n p1, p2 = p[1], p[2]\n if isinstance(p1, LexToken):\n p1, p1_tok = p1.value, p1\n lineno, col = p1_tok.lineno, p1_tok.lexpos\n else:\n lineno, col = self.lineno, self.col\n if lenp == 3: # $NAME\n p0 = self._envvar_by_name(p2, lineno=lineno, col=col)\n elif p1 == \"${\":\n xenv = load_attribute_chain(\"__xonsh__.env\", lineno=lineno, col=col)\n idx = ast.Index(value=p2)\n p0 = ast.Subscript(\n value=xenv, slice=idx, ctx=ast.Load(), lineno=lineno, col_offset=col\n )\n elif p1 == \"$(\":\n p0 = xonsh_call(\n \"__xonsh__.subproc_captured_stdout\", p2, lineno=lineno, col=col\n )\n elif p1 == \"!(\":\n p0 = xonsh_call(\n \"__xonsh__.subproc_captured_object\", p2, lineno=lineno, col=col\n )\n elif p1 == \"![\":\n p0 = xonsh_call(\n \"__xonsh__.subproc_captured_hiddenobject\", p2, lineno=lineno, col=col\n )\n elif p1 == \"$[\":\n p0 = xonsh_call(\"__xonsh__.subproc_uncaptured\", p2, lineno=lineno, col=col)\n else:\n assert False\n return p0\n\n def _envvar_getter_by_name(self, var, lineno=None, col=None):\n xenv = load_attribute_chain(\"__xonsh__.env\", lineno=lineno, col=col)\n func = ast.Attribute(\n value=xenv, attr=\"get\", ctx=ast.Load(), lineno=lineno, col_offset=col\n )\n return ast.Call(\n func=func,\n args=[\n ast.Str(s=var, lineno=lineno, col_offset=col),\n ast.Str(s=\"\", lineno=lineno, col_offset=col),\n ],\n keywords=[],\n starargs=None,\n kwargs=None,\n lineno=lineno,\n col_offset=col,\n )\n\n def _envvar_by_name(self, var, lineno=None, col=None):\n \"\"\"Looks up a xonsh variable by name.\"\"\"\n xenv = load_attribute_chain(\"__xonsh__.env\", lineno=lineno, col=col)\n idx = ast.Index(value=ast.Str(s=var, lineno=lineno, col_offset=col))\n return ast.Subscript(\n value=xenv, slice=idx, ctx=ast.Load(), lineno=lineno, col_offset=col\n )\n\n def _subproc_cliargs(self, args, lineno=None, col=None):\n \"\"\"Creates an expression for subprocess CLI arguments.\"\"\"\n cliargs = currlist = empty_list(lineno=lineno, col=col)\n for arg in args:\n action = arg._cliarg_action\n if action == \"append\":\n if currlist is None:\n currlist = empty_list(lineno=lineno, col=col)\n cliargs = binop(\n cliargs, ast.Add(), currlist, lineno=lineno, col=col\n )\n currlist.elts.append(arg)\n elif action == \"extend\":\n cliargs = binop(cliargs, ast.Add(), arg, lineno=lineno, col=col)\n currlist = None\n elif action == \"splitlines\":\n sl = call_split_lines(arg, lineno=lineno, col=col)\n cliargs = binop(cliargs, ast.Add(), sl, lineno=lineno, col=col)\n currlist = None\n elif action == \"ensure_list\":\n x = ensure_list_from_str_or_list(arg, lineno=lineno, col=col)\n cliargs = binop(cliargs, ast.Add(), x, lineno=lineno, col=col)\n currlist = None\n else:\n raise ValueError(\"action not understood: \" + action)\n del arg._cliarg_action\n return cliargs\n\n def p_pipe(self, p):\n \"\"\"pipe : PIPE\n | WS PIPE\n | PIPE WS\n | WS PIPE WS\n \"\"\"\n p[0] = ast.Str(s=\"|\", lineno=self.lineno, col_offset=self.col)\n\n def p_subproc_s2(self, p):\n \"\"\"subproc : subproc_atoms\n | subproc_atoms WS\n \"\"\"\n p1 = p[1]\n p[0] = [self._subproc_cliargs(p1, lineno=self.lineno, col=self.col)]\n\n def p_subproc_amp(self, p):\n \"\"\"subproc : subproc AMPERSAND\"\"\"\n p1 = p[1]\n p[0] = p1 + [ast.Str(s=p[2], lineno=self.lineno, col_offset=self.col)]\n\n def p_subproc_pipe(self, p):\n \"\"\"subproc : subproc pipe subproc_atoms\n | subproc pipe subproc_atoms WS\n \"\"\"\n p1 = p[1]\n if len(p1) > 1 and hasattr(p1[-2], \"s\") and p1[-2].s != \"|\":\n msg = \"additional redirect following non-pipe redirect\"\n self._parse_error(msg, self.currloc(lineno=self.lineno, column=self.col))\n cliargs = self._subproc_cliargs(p[3], lineno=self.lineno, col=self.col)\n p[0] = p1 + [p[2], cliargs]\n\n def p_subproc_atoms_single(self, p):\n \"\"\"subproc_atoms : subproc_atom\"\"\"\n p[0] = [p[1]]\n\n def p_subproc_atoms_many(self, p):\n \"\"\"subproc_atoms : subproc_atoms WS subproc_atom\"\"\"\n p1 = p[1]\n p1.append(p[3])\n p[0] = p1\n\n def p_subproc_atoms_subshell(self, p):\n \"\"\"subproc_atoms : lparen_tok any_raw_tok rparen_tok\n | lparen_tok any_raw_toks rparen_tok\n \"\"\"\n p1 = p[1]\n p3 = p[3]\n l = p1.lineno\n c = p1.lexpos + 1\n subcmd = self.source_slice((l, c), (p3.lineno, p3.lexpos))\n subcmd = subcmd.strip() + \"\\n\"\n p0 = [\n ast.Str(s=\"xonsh\", lineno=l, col_offset=c),\n ast.Str(s=\"-c\", lineno=l, col_offset=c),\n ast.Str(s=subcmd, lineno=l, col_offset=c),\n ]\n for arg in p0:\n arg._cliarg_action = \"append\"\n p[0] = p0\n\n #\n # Subproc atom rules\n #\n def _append_subproc_bang_empty(self, p):\n \"\"\"Appends an empty string in subprocess mode to the argument list.\"\"\"\n p3 = p[3]\n node = ast.Str(s=\"\", lineno=p3.lineno, col_offset=p3.lexpos + 1)\n p[2][-1].elts.append(node)\n\n def _append_subproc_bang(self, p):\n \"\"\"Appends the part between ! and the ) or ] in subprocess mode to the\n argument list.\n \"\"\"\n p3, p5 = p[3], p[5]\n beg = (p3.lineno, p3.lexpos + 1)\n end = (p5.lineno, p5.lexpos)\n s = self.source_slice(beg, end).strip()\n node = ast.Str(s=s, lineno=beg[0], col_offset=beg[1])\n p[2][-1].elts.append(node)\n\n def p_subproc_atom_uncaptured(self, p):\n \"\"\"subproc_atom : dollar_lbracket_tok subproc RBRACKET\"\"\"\n p1 = p[1]\n p0 = xonsh_call(\n \"__xonsh__.subproc_uncaptured\", args=p[2], lineno=p1.lineno, col=p1.lexpos\n )\n p0._cliarg_action = \"splitlines\"\n p[0] = p0\n\n def p_subproc_atom_uncaptured_bang_empty(self, p):\n \"\"\"subproc_atom : dollar_lbracket_tok subproc bang_tok RBRACKET\"\"\"\n self._append_subproc_bang_empty(p)\n self.p_subproc_atom_uncaptured(p)\n\n def p_subproc_atom_uncaptured_bang(self, p):\n \"\"\"subproc_atom : dollar_lbracket_tok subproc bang_tok nocloser rbracket_tok\"\"\"\n self._append_subproc_bang(p)\n self.p_subproc_atom_uncaptured(p)\n\n def p_subproc_atom_captured_stdout(self, p):\n \"\"\"subproc_atom : dollar_lparen_tok subproc RPAREN\"\"\"\n p1 = p[1]\n p0 = xonsh_call(\n \"__xonsh__.subproc_captured_stdout\",\n args=p[2],\n lineno=p1.lineno,\n col=p1.lexpos,\n )\n p0._cliarg_action = \"append\"\n p[0] = p0\n\n def p_subproc_atom_captured_stdout_bang_empty(self, p):\n \"\"\"subproc_atom : dollar_lparen_tok subproc bang_tok RPAREN\"\"\"\n self._append_subproc_bang_empty(p)\n self.p_subproc_atom_captured_stdout(p)\n\n def p_subproc_atom_captured_stdout_bang(self, p):\n \"\"\"subproc_atom : dollar_lparen_tok subproc bang_tok nocloser rparen_tok\"\"\"\n self._append_subproc_bang(p)\n self.p_subproc_atom_captured_stdout(p)\n\n def p_subproc_atom_pyenv_lookup(self, p):\n \"\"\"subproc_atom : dollar_lbrace_tok test RBRACE\"\"\"\n p1 = p[1]\n lineno, col = p1.lineno, p1.lexpos\n xenv = load_attribute_chain(\"__xonsh__.env\", lineno=lineno, col=col)\n func = ast.Attribute(\n value=xenv, attr=\"get\", ctx=ast.Load(), lineno=lineno, col_offset=col\n )\n p0 = ast.Call(\n func=func,\n args=[p[2], ast.Str(s=\"\", lineno=lineno, col_offset=col)],\n keywords=[],\n starargs=None,\n kwargs=None,\n lineno=lineno,\n col_offset=col,\n )\n p0._cliarg_action = \"append\"\n p[0] = p0\n\n def p_subproc_atom_pyeval(self, p):\n \"\"\"subproc_atom : at_lparen_tok testlist_comp RPAREN\n subproc_arg_part : at_lparen_tok testlist_comp RPAREN\n \"\"\"\n p1 = p[1]\n p0 = xonsh_call(\n \"__xonsh__.list_of_strs_or_callables\",\n [p[2]],\n lineno=p1.lineno,\n col=p1.lexpos,\n )\n p0._cliarg_action = \"extend\"\n p[0] = p0\n\n def p_subproc_atom_subproc_inject(self, p):\n \"\"\"subproc_atom : atdollar_lparen_tok subproc RPAREN\"\"\"\n p1 = p[1]\n p0 = xonsh_call(\n \"__xonsh__.subproc_captured_inject\", p[2], lineno=p1.lineno, col=p1.lexpos\n )\n p0._cliarg_action = \"extend\"\n p[0] = p0\n\n def p_subproc_atom_subproc_inject_bang_empty(self, p):\n \"\"\"subproc_atom : atdollar_lparen_tok subproc bang_tok RPAREN\"\"\"\n self._append_subproc_bang_empty(p)\n self.p_subproc_atom_subproc_inject(p)\n\n def p_subproc_atom_subproc_inject_bang(self, p):\n \"\"\"subproc_atom : atdollar_lparen_tok subproc bang_tok nocloser rparen_tok\"\"\"\n self._append_subproc_bang(p)\n self.p_subproc_atom_subproc_inject(p)\n\n def p_subproc_atom_redirect(self, p):\n \"\"\"subproc_atom : GT\n | LT\n | RSHIFT\n | IOREDIRECT\n \"\"\"\n p0 = ast.Str(s=p[1], lineno=self.lineno, col_offset=self.col)\n p0._cliarg_action = \"append\"\n p[0] = p0\n\n def p_subproc_atom_re(self, p):\n \"\"\"subproc_atom : SEARCHPATH\"\"\"\n p0 = xonsh_pathsearch(p[1], pymode=False, lineno=self.lineno, col=self.col)\n p0._cliarg_action = \"extend\"\n p[0] = p0\n\n def p_subproc_atom_str(self, p):\n \"\"\"subproc_atom : string_literal\"\"\"\n p0 = xonsh_call(\n \"__xonsh__.expand_path\", args=[p[1]], lineno=self.lineno, col=self.col\n )\n p0._cliarg_action = \"append\"\n p[0] = p0\n\n def p_subproc_atom_arg(self, p):\n \"\"\"subproc_atom : subproc_arg\"\"\"\n p1 = p[1]\n if isinstance(p1, list):\n # has an expanding function call, such as @(x)\n p0 = xonsh_call(\n \"__xonsh__.list_of_list_of_strs_outer_product\",\n args=[ensure_has_elts(p1)],\n lineno=p1[0].lineno,\n col=p1[0].col_offset,\n )\n p0._cliarg_action = \"extend\"\n elif hasglobstar(p1):\n # globbed literal argument\n p0 = xonsh_call(\n \"__xonsh__.glob\", args=[p1], lineno=p1.lineno, col=p1.col_offset\n )\n p0._cliarg_action = \"extend\"\n else:\n # literal str argument\n p0 = xonsh_call(\n \"__xonsh__.expand_path\", args=[p1], lineno=p1.lineno, col=p1.col_offset\n )\n p0._cliarg_action = \"append\"\n p[0] = p0\n\n def p_subproc_arg_single(self, p):\n \"\"\"subproc_arg : subproc_arg_part\"\"\"\n p[0] = p[1]\n\n def p_subproc_arg_many(self, p):\n \"\"\"subproc_arg : subproc_arg subproc_arg_part\"\"\"\n # This glues the string together after parsing\n p1 = p[1]\n p2 = p[2]\n if isinstance(p1, ast.Str) and isinstance(p2, ast.Str):\n p0 = ast.Str(p1.s + p2.s, lineno=p1.lineno, col_offset=p1.col_offset)\n elif isinstance(p1, list):\n if isinstance(p2, list):\n p1.extend(p2)\n else:\n p1.append(p2)\n p0 = p1\n elif isinstance(p2, list):\n p2.insert(0, p1)\n p0 = p2\n else:\n p0 = [p1, p2]\n p[0] = p0\n\n def _attach_subproc_arg_part_rules(self):\n toks = set(self.tokens)\n toks -= {\n \"AND\",\n \"OR\",\n \"NOT\",\n \"BANG\",\n \"PIPE\",\n \"WS\",\n \"GT\",\n \"LT\",\n \"LSHIFT\",\n \"RSHIFT\",\n \"IOREDIRECT\",\n \"SEARCHPATH\",\n \"INDENT\",\n \"DEDENT\",\n \"LPAREN\",\n \"RPAREN\",\n \"LBRACE\",\n \"RBRACE\",\n \"LBRACKET\",\n \"RBRACKET\",\n \"AT_LPAREN\",\n \"BANG_LPAREN\",\n \"BANG_LBRACKET\",\n \"DOLLAR_LPAREN\",\n \"DOLLAR_LBRACE\",\n \"DOLLAR_LBRACKET\",\n \"ATDOLLAR_LPAREN\",\n }\n ts = \"\\n | \".join(sorted([t.lower() + \"_tok\" for t in toks]))\n doc = \"subproc_arg_part : \" + ts + \"\\n\"\n self.p_subproc_arg_part.__func__.__doc__ = doc\n\n def p_subproc_arg_part(self, p):\n # Many tokens cannot be part of this rule, such as $, ', \", ()\n # Use a string atom instead. See above attachment functions\n p1 = p[1]\n p[0] = ast.Str(s=p1.value, lineno=p1.lineno, col_offset=p1.lexpos)\n\n #\n # Helpers\n #\n\n def p_test_comma_combine(self, p):\n \"\"\"test_comma_list : test comma_test_list\n | test comma_test_list COMMA\n \"\"\"\n p2 = p[2]\n p2.insert(0, p[1])\n p[0] = p2\n\n def p_empty(self, p):\n \"empty : \"\n p[0] = None\n\n def p_error(self, p):\n if p is None:\n self._parse_error(\"no further code\", None)\n elif p.type == \"ERRORTOKEN\":\n if isinstance(p.value, BaseException):\n raise p.value\n else:\n self._parse_error(\n p.value, self.currloc(lineno=p.lineno, column=p.lexpos)\n )\n else:\n msg = (\"code: {0}\".format(p.value),)\n self._parse_error(msg, self.currloc(lineno=p.lineno, column=p.lexpos))\n" }, { "path": "xonsh/parsers/context_check.py", "content": "import ast\nimport keyword\nimport collections\n\n_all_keywords = frozenset(keyword.kwlist)\n\n\ndef _not_assignable(x, augassign=False):\n \"\"\"\n If ``x`` represents a value that can be assigned to, return ``None``.\n Otherwise, return a string describing the object. For use in generating\n meaningful syntax errors.\n \"\"\"\n if augassign and isinstance(x, (ast.Tuple, ast.List)):\n return \"literal\"\n elif isinstance(x, (ast.Tuple, ast.List)):\n if len(x.elts) == 0:\n return \"()\"\n for i in x.elts:\n res = _not_assignable(i)\n if res is not None:\n return res\n elif isinstance(x, (ast.Set, ast.Dict, ast.Num, ast.Str, ast.Bytes)):\n return \"literal\"\n elif isinstance(x, ast.Call):\n return \"function call\"\n elif isinstance(x, ast.Lambda):\n return \"lambda\"\n elif isinstance(x, (ast.BoolOp, ast.BinOp, ast.UnaryOp)):\n return \"operator\"\n elif isinstance(x, ast.IfExp):\n return \"conditional expression\"\n elif isinstance(x, ast.ListComp):\n return \"list comprehension\"\n elif isinstance(x, ast.DictComp):\n return \"dictionary comprehension\"\n elif isinstance(x, ast.SetComp):\n return \"set comprehension\"\n elif isinstance(x, ast.GeneratorExp):\n return \"generator expression\"\n elif isinstance(x, ast.Compare):\n return \"comparison\"\n elif isinstance(x, ast.Name) and x.id in _all_keywords:\n return \"keyword\"\n elif isinstance(x, ast.NameConstant):\n return \"keyword\"\n\n\n_loc = collections.namedtuple(\"_loc\", [\"lineno\", \"column\"])\n\n\ndef check_contexts(tree):\n c = ContextCheckingVisitor()\n c.visit(tree)\n if c.error is not None:\n e = SyntaxError(c.error[0])\n e.loc = _loc(c.error[1], c.error[2])\n raise e\n\n\nclass ContextCheckingVisitor(ast.NodeVisitor):\n def __init__(self):\n self.error = None\n\n def visit_Delete(self, node):\n for i in node.targets:\n err = _not_assignable(i)\n if err is not None:\n msg = \"can't delete {}\".format(err)\n self.error = msg, i.lineno, i.col_offset\n break\n\n def visit_Assign(self, node):\n for i in node.targets:\n err = _not_assignable(i)\n if err is not None:\n msg = \"can't assign to {}\".format(err)\n self.error = msg, i.lineno, i.col_offset\n break\n\n def visit_AugAssign(self, node):\n err = _not_assignable(node.target, True)\n if err is not None:\n msg = \"illegal target for augmented assignment: {}\".format(err)\n self.error = msg, node.target.lineno, node.target.col_offset\n" }, { "path": "xonsh/parsers/v34.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"Implements the xonsh parser for Python v3.4.\"\"\"\nimport xonsh.ast as ast\nfrom xonsh.parsers.base import BaseParser\n\n\nclass Parser(BaseParser):\n \"\"\"A Python v3.4 compliant parser for the xonsh language.\"\"\"\n\n def __init__(\n self,\n lexer_optimize=True,\n lexer_table=\"xonsh.lexer_table\",\n yacc_optimize=True,\n yacc_table=\"xonsh.parser_table\",\n yacc_debug=False,\n outputdir=None,\n ):\n \"\"\"Parameters\n ----------\n lexer_optimize : bool, optional\n Set to false when unstable and true when lexer is stable.\n lexer_table : str, optional\n Lexer module used when optimized.\n yacc_optimize : bool, optional\n Set to false when unstable and true when parser is stable.\n yacc_table : str, optional\n Parser module used when optimized.\n yacc_debug : debug, optional\n Dumps extra debug info.\n outputdir : str or None, optional\n The directory to place generated tables within.\n \"\"\"\n # Rule creation and modification *must* take place before super()\n opt_rules = [\"argument_comma_list\", \"comma_argument_list\"]\n for rule in opt_rules:\n self._opt_rule(rule)\n\n list_rules = [\"argument_comma\"]\n for rule in list_rules:\n self._list_rule(rule)\n\n super().__init__(\n lexer_optimize=lexer_optimize,\n lexer_table=lexer_table,\n yacc_optimize=yacc_optimize,\n yacc_table=yacc_table,\n yacc_debug=yacc_debug,\n outputdir=outputdir,\n )\n\n def p_classdef_or_funcdef(self, p):\n \"\"\"classdef_or_funcdef : classdef\n | funcdef\n \"\"\"\n p[0] = p[1]\n\n def p_item(self, p):\n \"\"\"item : test COLON test\"\"\"\n lenp = len(p)\n if lenp == 4:\n p0 = [p[1], p[3]]\n elif lenp == 3:\n p0 = [None, p[2]]\n else:\n assert False\n p[0] = p0\n\n def _set_arg(self, args, arg, ensure_kw=False):\n if isinstance(arg, ast.keyword):\n args[\"keywords\"].append(arg)\n elif ensure_kw:\n args[\"kwargs\"] = arg\n else:\n args[\"args\"].append(arg)\n\n def p_arglist(self, p):\n \"\"\"arglist : argument comma_opt\n | argument_comma_list argument comma_opt\n | argument_comma_list_opt TIMES test comma_argument_list_opt\n | argument_comma_list_opt TIMES test COMMA POW test\n | argument_comma_list_opt TIMES test comma_argument_list COMMA POW test\n | argument_comma_list_opt POW test\n \"\"\"\n lenp = len(p)\n p1, p2 = p[1], p[2]\n p0 = {\"args\": [], \"keywords\": [], \"starargs\": None, \"kwargs\": None}\n if lenp == 3:\n self._set_arg(p0, p1)\n elif lenp == 4 and p2 != \"**\":\n for arg in p1:\n self._set_arg(p0, arg)\n self._set_arg(p0, p2)\n elif lenp == 4 and p2 == \"**\":\n if p1 is not None:\n for arg in p1:\n self._set_arg(p0, arg)\n self._set_arg(p0, p[3], ensure_kw=True)\n elif lenp == 5:\n p0[\"starargs\"], p4 = p[3], p[4]\n if p1 is not None:\n for arg in p1:\n self._set_arg(p0, arg)\n if p4 is not None:\n for arg in p4:\n self._set_arg(p0, arg, ensure_kw=True)\n elif lenp == 7:\n p0[\"starargs\"] = p[3]\n if p1 is not None:\n for arg in p1:\n self._set_arg(p0, arg)\n self._set_arg(p0, p[6], ensure_kw=True)\n elif lenp == 8:\n p0[\"starargs\"], p4 = p[3], p[4]\n if p1 is not None:\n for arg in p1:\n self._set_arg(p0, arg)\n for arg in p4:\n self._set_arg(p0, arg, ensure_kw=True)\n self._set_arg(p0, p[7], ensure_kw=True)\n else:\n assert False\n p[0] = p0\n\n def p_argument_comma(self, p):\n \"\"\"argument_comma : argument COMMA\"\"\"\n p[0] = [p[1]]\n\n def p_argument(self, p):\n \"\"\"argument : test\n | test comp_for\n | test EQUALS test\n \"\"\"\n # Really [keyword '='] test\n # The reason that keywords are test nodes instead of NAME is that using\n # NAME results in an ambiguity.\n p1 = p[1]\n lenp = len(p)\n if lenp == 2:\n p0 = p1\n elif lenp == 3:\n if p1 == \"**\":\n p0 = ast.keyword(arg=None, value=p[2])\n elif p1 == \"*\":\n p0 = ast.Starred(value=p[2])\n else:\n p0 = ast.GeneratorExp(\n elt=p1,\n generators=p[2][\"comps\"],\n lineno=p1.lineno,\n col_offset=p1.col_offset,\n )\n elif lenp == 4:\n p0 = ast.keyword(arg=p1.id, value=p[3])\n else:\n assert False\n p[0] = p0\n" }, { "path": "xonsh/parsers/v35.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"Implements the xonsh parser for Python v3.5.\"\"\"\nimport xonsh.ast as ast\nfrom xonsh.parsers.base import BaseParser\n\n\nclass Parser(BaseParser):\n \"\"\"A Python v3.5 compliant parser for the xonsh language.\"\"\"\n\n def __init__(\n self,\n lexer_optimize=True,\n lexer_table=\"xonsh.lexer_table\",\n yacc_optimize=True,\n yacc_table=\"xonsh.parser_table\",\n yacc_debug=False,\n outputdir=None,\n ):\n \"\"\"Parameters\n ----------\n lexer_optimize : bool, optional\n Set to false when unstable and true when lexer is stable.\n lexer_table : str, optional\n Lexer module used when optimized.\n yacc_optimize : bool, optional\n Set to false when unstable and true when parser is stable.\n yacc_table : str, optional\n Parser module used when optimized.\n yacc_debug : debug, optional\n Dumps extra debug info.\n outputdir : str or None, optional\n The directory to place generated tables within.\n \"\"\"\n # Rule creation and modification *must* take place before super()\n tok_rules = [\"await\", \"async\"]\n for rule in tok_rules:\n self._tok_rule(rule)\n super().__init__(\n lexer_optimize=lexer_optimize,\n lexer_table=lexer_table,\n yacc_optimize=yacc_optimize,\n yacc_table=yacc_table,\n yacc_debug=yacc_debug,\n outputdir=outputdir,\n )\n\n def p_classdef_or_funcdef(self, p):\n \"\"\"classdef_or_funcdef : classdef\n | funcdef\n | async_funcdef\n \"\"\"\n p[0] = p[1]\n\n def p_async_funcdef(self, p):\n \"\"\"async_funcdef : async_tok funcdef\"\"\"\n p1, f = p[1], p[2][0]\n p[0] = [ast.AsyncFunctionDef(**f.__dict__)]\n p[0][0]._async_tok = p1\n\n def p_async_compound_stmt(self, p):\n \"\"\"compound_stmt : async_stmt\"\"\"\n p[0] = p[1]\n\n def p_async_for_stmt(self, p):\n \"\"\"async_for_stmt : ASYNC for_stmt\"\"\"\n f = p[2][0]\n p[0] = [ast.AsyncFor(**f.__dict__)]\n\n def p_async_with_stmt(self, p):\n \"\"\"async_with_stmt : ASYNC with_stmt\"\"\"\n w = p[2][0]\n p[0] = [ast.AsyncWith(**w.__dict__)]\n\n def p_atom_expr_await(self, p):\n \"\"\"atom_expr : await_tok atom trailer_list_opt\"\"\"\n p0 = self.apply_trailers(p[2], p[3])\n p1 = p[1]\n p0 = ast.Await(value=p0, ctx=ast.Load(), lineno=p1.lineno, col_offset=p1.lexpos)\n p[0] = p0\n\n def p_async_stmt(self, p):\n \"\"\"async_stmt : async_funcdef\n | async_with_stmt\n | async_for_stmt\n \"\"\"\n p[0] = p[1]\n\n def p_item_test(self, p):\n \"\"\"item : test COLON test\"\"\"\n p[0] = [p[1], p[3]]\n\n def p_item_pow(self, p):\n \"\"\"item : POW expr\"\"\"\n p[0] = [None, p[2]]\n\n def _set_arg(self, args, arg, ensure_kw=False):\n if isinstance(arg, ast.keyword):\n args[\"keywords\"].append(arg)\n elif ensure_kw:\n args[\"keywords\"].append(ast.keyword(arg=None, value=arg))\n else:\n args[\"args\"].append(arg)\n\n def p_arglist_single(self, p):\n \"\"\"arglist : argument comma_opt\"\"\"\n p0 = {\"args\": [], \"keywords\": []}\n self._set_arg(p0, p[1])\n p[0] = p0\n\n def p_arglist_many(self, p):\n \"\"\"arglist : argument comma_argument_list comma_opt\n \"\"\"\n p0 = {\"args\": [], \"keywords\": []}\n self._set_arg(p0, p[1])\n for arg in p[2]:\n self._set_arg(p0, arg)\n p[0] = p0\n\n # Argument rules\n # \"test '=' test\" is really \"keyword '=' test\", but we have no such token.\n # These need to be in a single rule to avoid grammar that is ambiguous\n # to our LL(1) parser. Even though 'test' includes '*expr' in star_expr,\n # we explicitly match '*' here, too, to give it proper precedence.\n # Illegal combinations and orderings are blocked in ast.c:\n # multiple (test comp_for) arguments are blocked; keyword unpackings\n # that precede iterable unpackings are blocked; etc.\n def p_argument_test_or_star(self, p):\n \"\"\"argument : test_or_star_expr\"\"\"\n p[0] = p[1]\n\n def p_argument_kwargs(self, p):\n \"\"\"argument : POW test\"\"\"\n p[0] = ast.keyword(arg=None, value=p[2])\n\n def p_argument_args(self, p):\n \"\"\"argument : TIMES test\"\"\"\n p[0] = ast.Starred(value=p[2])\n\n def p_argument(self, p):\n \"\"\"argument : test comp_for\"\"\"\n p1 = p[1]\n p[0] = ast.GeneratorExp(\n elt=p1, generators=p[2][\"comps\"], lineno=p1.lineno, col_offset=p1.col_offset\n )\n\n def p_argument_eq(self, p):\n \"\"\"argument : test EQUALS test\"\"\"\n p[0] = ast.keyword(arg=p[1].id, value=p[3])\n" }, { "path": "xonsh/parsers/v36.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"Implements the xonsh parser for Python v3.6.\"\"\"\nimport xonsh.ast as ast\nfrom xonsh.parsers.v35 import Parser as ThreeFiveParser\nfrom xonsh.parsers.base import store_ctx, ensure_has_elts, lopen_loc\n\n\nclass Parser(ThreeFiveParser):\n \"\"\"A Python v3.6 compliant parser for the xonsh language.\"\"\"\n\n def p_comp_for(self, p):\n \"\"\"comp_for : FOR exprlist IN or_test comp_iter_opt\"\"\"\n targs, it, p5 = p[2], p[4], p[5]\n if len(targs) == 1:\n targ = targs[0]\n else:\n targ = ensure_has_elts(targs)\n store_ctx(targ)\n # only difference with base should be the is_async=0\n comp = ast.comprehension(target=targ, iter=it, ifs=[], is_async=0)\n comps = [comp]\n p0 = {\"comps\": comps}\n if p5 is not None:\n comps += p5.get(\"comps\", [])\n comp.ifs += p5.get(\"if\", [])\n p[0] = p0\n\n def p_expr_stmt_annassign(self, p):\n \"\"\"expr_stmt : testlist_star_expr COLON test EQUALS test\"\"\"\n p1 = p[1][0]\n lineno, col = lopen_loc(p1)\n if len(p[1]) > 1 or not isinstance(p1, ast.Name):\n loc = self.currloc(lineno, col)\n self._parse_error(\"only single target can be annotated\", loc)\n store_ctx(p1)\n p[0] = ast.AnnAssign(\n target=p1,\n annotation=p[3],\n value=p[5],\n simple=1,\n lineno=lineno,\n col_offset=col,\n )\n" }, { "path": "xonsh/platform.py", "content": "\"\"\"Module for platform-specific constants and implementations, as well as\ncompatibility layers to make use of the 'best' implementation available\non a platform.\n\"\"\"\nimport os\nimport sys\nimport ctypes\nimport signal\nimport pathlib\nimport builtins\nimport platform\nimport functools\nimport subprocess\nimport collections\nimport collections.abc as cabc\nimport importlib.util\n\nfrom xonsh.lazyasd import LazyBool, lazyobject, lazybool\n\n# do not import any xonsh-modules here to avoid circular dependencies\n\nFD_STDIN = 0\nFD_STDOUT = 1\nFD_STDERR = 2\n\n\n@lazyobject\ndef distro():\n try:\n import distro as d\n except ImportError:\n d = None\n except Exception:\n raise\n return d\n\n\n#\n# OS\n#\nON_DARWIN = LazyBool(lambda: platform.system() == \"Darwin\", globals(), \"ON_DARWIN\")\n\"\"\"``True`` if executed on a Darwin platform, else ``False``. \"\"\"\nON_LINUX = LazyBool(lambda: platform.system() == \"Linux\", globals(), \"ON_LINUX\")\n\"\"\"``True`` if executed on a Linux platform, else ``False``. \"\"\"\nON_WINDOWS = LazyBool(lambda: platform.system() == \"Windows\", globals(), \"ON_WINDOWS\")\n\"\"\"``True`` if executed on a native Windows platform, else ``False``. \"\"\"\nON_CYGWIN = LazyBool(lambda: sys.platform == \"cygwin\", globals(), \"ON_CYGWIN\")\n\"\"\"``True`` if executed on a Cygwin Windows platform, else ``False``. \"\"\"\nON_MSYS = LazyBool(lambda: sys.platform == \"msys\", globals(), \"ON_MSYS\")\n\"\"\"``True`` if executed on a MSYS Windows platform, else ``False``. \"\"\"\nON_POSIX = LazyBool(lambda: (os.name == \"posix\"), globals(), \"ON_POSIX\")\n\"\"\"``True`` if executed on a POSIX-compliant platform, else ``False``. \"\"\"\nON_FREEBSD = LazyBool(\n lambda: (sys.platform.startswith(\"freebsd\")), globals(), \"ON_FREEBSD\"\n)\n\"\"\"``True`` if on a FreeBSD operating system, else ``False``.\"\"\"\nON_DRAGONFLY = LazyBool(\n lambda: (sys.platform.startswith(\"dragonfly\")), globals(), \"ON_DRAGONFLY\"\n)\n\"\"\"``True`` if on a DragonFly BSD operating system, else ``False``.\"\"\"\nON_NETBSD = LazyBool(\n lambda: (sys.platform.startswith(\"netbsd\")), globals(), \"ON_NETBSD\"\n)\n\"\"\"``True`` if on a NetBSD operating system, else ``False``.\"\"\"\n\n\n@lazybool\ndef ON_BSD():\n \"\"\"``True`` if on a BSD operating system, else ``False``.\"\"\"\n return bool(ON_FREEBSD) or bool(ON_NETBSD) or bool(ON_DRAGONFLY)\n\n\n@lazybool\ndef ON_BEOS():\n \"\"\"True if we are on BeOS or Haiku.\"\"\"\n return sys.platform == \"beos5\" or sys.platform == \"haiku1\"\n\n\n@lazybool\ndef ON_WSL():\n \"\"\"True if we are on Windows Subsystem for Linux (WSL)\"\"\"\n return \"Microsoft\" in platform.release()\n\n\n#\n# Python & packages\n#\n\nPYTHON_VERSION_INFO = sys.version_info[:3]\n\"\"\" Version of Python interpreter as three-value tuple. \"\"\"\n\n\n@lazyobject\ndef PYTHON_VERSION_INFO_BYTES():\n \"\"\"The python version info tuple in a canonical bytes form.\"\"\"\n return \".\".join(map(str, sys.version_info)).encode()\n\n\nON_ANACONDA = LazyBool(\n lambda: pathlib.Path(sys.prefix).joinpath(\"conda-meta\").exists(),\n globals(),\n \"ON_ANACONDA\",\n)\n\"\"\" ``True`` if executed in an Anaconda instance, else ``False``. \"\"\"\nCAN_RESIZE_WINDOW = LazyBool(\n lambda: hasattr(signal, \"SIGWINCH\"), globals(), \"CAN_RESIZE_WINDOW\"\n)\n\"\"\"``True`` if we can resize terminal window, as provided by the presense of\nsignal.SIGWINCH, else ``False``.\n\"\"\"\n\n\n@lazybool\ndef HAS_PYGMENTS():\n \"\"\"``True`` if `pygments` is available, else ``False``.\"\"\"\n spec = importlib.util.find_spec(\"pygments\")\n return spec is not None\n\n\n@functools.lru_cache(1)\ndef pygments_version():\n \"\"\"pygments.__version__ version if available, else None.\"\"\"\n if HAS_PYGMENTS:\n import pygments\n\n v = pygments.__version__\n else:\n v = None\n return v\n\n\n@functools.lru_cache(1)\ndef pygments_version_info():\n \"\"\" Returns `pygments`'s version as tuple of integers. \"\"\"\n if HAS_PYGMENTS:\n return tuple(int(x) for x in pygments_version().strip(\"<>+-=.\").split(\".\"))\n else:\n return None\n\n\n@functools.lru_cache(1)\ndef has_prompt_toolkit():\n \"\"\"Tests if the `prompt_toolkit` is available.\"\"\"\n spec = importlib.util.find_spec(\"prompt_toolkit\")\n return spec is not None\n\n\n@functools.lru_cache(1)\ndef ptk_version():\n \"\"\"Returns `prompt_toolkit.__version__` if available, else ``None``.\"\"\"\n if has_prompt_toolkit():\n import prompt_toolkit\n\n return getattr(prompt_toolkit, \"__version__\", \"<0.57\")\n else:\n return None\n\n\n@functools.lru_cache(1)\ndef ptk_version_info():\n \"\"\" Returns `prompt_toolkit`'s version as tuple of integers. \"\"\"\n if has_prompt_toolkit():\n return tuple(int(x) for x in ptk_version().strip(\"<>+-=.\").split(\".\"))\n else:\n return None\n\n\n@functools.lru_cache(1)\ndef ptk_above_min_supported():\n minimum_required_ptk_version = (1, 0)\n return ptk_version_info()[:2] >= minimum_required_ptk_version\n\n\n@functools.lru_cache(1)\ndef ptk_shell_type():\n \"\"\"Returns the prompt_toolkit shell type based on the installed version.\"\"\"\n if ptk_version_info()[:2] < (2, 0):\n return \"prompt_toolkit1\"\n else:\n return \"prompt_toolkit2\"\n\n\n@functools.lru_cache(1)\ndef win_ansi_support():\n if ON_WINDOWS:\n try:\n from prompt_toolkit.utils import is_windows_vt100_supported, is_conemu_ansi\n except ImportError:\n return False\n return is_conemu_ansi() or is_windows_vt100_supported()\n else:\n return False\n\n\n@functools.lru_cache(1)\ndef ptk_below_max_supported():\n ptk_max_version_cutoff = (2, 0)\n return ptk_version_info()[:2] < ptk_max_version_cutoff\n\n\n@functools.lru_cache(1)\ndef best_shell_type():\n if builtins.__xonsh__.env.get(\"TERM\", \"\") == \"dumb\":\n return \"dumb\"\n elif ON_WINDOWS or has_prompt_toolkit():\n return \"prompt_toolkit\"\n else:\n return \"readline\"\n\n\n@functools.lru_cache(1)\ndef is_readline_available():\n \"\"\"Checks if readline is available to import.\"\"\"\n spec = importlib.util.find_spec(\"readline\")\n return spec is not None\n\n\n@lazyobject\ndef seps():\n \"\"\"String of all path separators.\"\"\"\n s = os.path.sep\n if os.path.altsep is not None:\n s += os.path.altsep\n return s\n\n\ndef pathsplit(p):\n \"\"\"This is a safe version of os.path.split(), which does not work on input\n without a drive.\n \"\"\"\n n = len(p)\n while n and p[n - 1] not in seps:\n n -= 1\n pre = p[:n]\n pre = pre.rstrip(seps) or pre\n post = p[n:]\n return pre, post\n\n\ndef pathbasename(p):\n \"\"\"This is a safe version of os.path.basename(), which does not work on\n input without a drive. This version does.\n \"\"\"\n return pathsplit(p)[-1]\n\n\n@lazyobject\ndef expanduser():\n \"\"\"Dispatches to the correct platform-dependent expanduser() function.\"\"\"\n if ON_WINDOWS:\n return windows_expanduser\n else:\n return os.path.expanduser\n\n\ndef windows_expanduser(path):\n \"\"\"A Windows-specific expanduser() function for xonsh. This is needed\n since os.path.expanduser() does not check on Windows if the user actually\n exists. This restricts expanding the '~' if it is not followed by a\n separator. That is only '~/' and '~\\' are expanded.\n \"\"\"\n if not path.startswith(\"~\"):\n return path\n elif len(path) < 2 or path[1] in seps:\n return os.path.expanduser(path)\n else:\n return path\n\n\n# termios tc(get|set)attr indexes.\nIFLAG = 0\nOFLAG = 1\nCFLAG = 2\nLFLAG = 3\nISPEED = 4\nOSPEED = 5\nCC = 6\n\n\n#\n# Dev release info\n#\n\n\n@functools.lru_cache(1)\ndef githash():\n \"\"\"Returns a tuple contains two strings: the hash and the date.\"\"\"\n install_base = os.path.dirname(__file__)\n githash_file = \"{}/dev.githash\".format(install_base)\n if not os.path.exists(githash_file):\n return None, None\n sha = None\n date_ = None\n try:\n with open(githash_file) as f:\n sha, date_ = f.read().strip().split(\"|\")\n except ValueError:\n pass\n return sha, date_\n\n\n#\n# Encoding\n#\n\nDEFAULT_ENCODING = sys.getdefaultencoding()\n\"\"\" Default string encoding. \"\"\"\n\n\nif PYTHON_VERSION_INFO < (3, 5, 0):\n\n class DirEntry:\n def __init__(self, directory, name):\n self.__path__ = pathlib.Path(directory) / name\n self.name = name\n self.path = str(self.__path__)\n self.is_symlink = self.__path__.is_symlink\n\n def inode(self):\n return os.stat(self.path, follow_symlinks=False).st_ino\n\n def is_dir(self, *, follow_symlinks=True):\n if follow_symlinks:\n return self.__path__.is_dir()\n else:\n return not self.__path__.is_symlink() and self.__path__.is_dir()\n\n def is_file(self, *, follow_symlinks=True):\n if follow_symlinks:\n return self.__path__.is_file()\n else:\n return not self.__path__.is_symlink() and self.__path__.is_file()\n\n def stat(self, *, follow_symlinks=True):\n return os.stat(self.path, follow_symlinks=follow_symlinks)\n\n def scandir(path):\n \"\"\" Compatibility layer for `os.scandir` from Python 3.5+. \"\"\"\n return (DirEntry(path, x) for x in os.listdir(path))\n\n\nelse:\n scandir = os.scandir\n\n\n#\n# Linux distro\n#\n\n\n@functools.lru_cache(1)\ndef linux_distro():\n \"\"\"The id of the Linux distribution running on, possibly 'unknown'.\n None on non-Linux platforms.\n \"\"\"\n if ON_LINUX:\n if distro:\n ld = distro.id()\n elif PYTHON_VERSION_INFO < (3, 6, 6):\n ld = platform.linux_distribution()[0] or \"unknown\"\n elif \"-ARCH-\" in platform.platform():\n ld = \"arch\" # that's the only one we need to know for now\n else:\n ld = \"unknown\"\n else:\n ld = None\n return ld\n\n\n#\n# Windows\n#\n\n\n@functools.lru_cache(1)\ndef git_for_windows_path():\n \"\"\"Returns the path to git for windows, if available and None otherwise.\"\"\"\n import winreg\n\n try:\n key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, \"SOFTWARE\\\\GitForWindows\")\n gfwp, _ = winreg.QueryValueEx(key, \"InstallPath\")\n except FileNotFoundError:\n gfwp = None\n return gfwp\n\n\n@functools.lru_cache(1)\ndef windows_bash_command():\n \"\"\"Determines the command for Bash on windows.\"\"\"\n # Check that bash is on path otherwise try the default directory\n # used by Git for windows\n wbc = \"bash\"\n cmd_cache = builtins.__xonsh__.commands_cache\n bash_on_path = cmd_cache.lazy_locate_binary(\"bash\", ignore_alias=True)\n if bash_on_path:\n try:\n out = subprocess.check_output(\n [bash_on_path, \"--version\"],\n stderr=subprocess.PIPE,\n universal_newlines=True,\n )\n except subprocess.CalledProcessError:\n bash_works = False\n else:\n # Check if Bash is from the \"Windows Subsystem for Linux\" (WSL)\n # which can't be used by xonsh foreign-shell/completer\n bash_works = out and \"pc-linux-gnu\" not in out.splitlines()[0]\n\n if bash_works:\n wbc = bash_on_path\n else:\n gfwp = git_for_windows_path()\n if gfwp:\n bashcmd = os.path.join(gfwp, \"bin\\\\bash.exe\")\n if os.path.isfile(bashcmd):\n wbc = bashcmd\n return wbc\n\n\n#\n# Environment variables defaults\n#\n\nif ON_WINDOWS:\n\n class OSEnvironCasePreserving(cabc.MutableMapping):\n \"\"\" Case-preserving wrapper for os.environ on Windows.\n It uses nt.environ to get the correct cased keys on\n initialization. It also preserves the case of any variables\n add after initialization.\n \"\"\"\n\n def __init__(self):\n import nt\n\n self._upperkeys = dict((k.upper(), k) for k in nt.environ)\n\n def _sync(self):\n \"\"\" Ensure that the case sensitive map of the keys are\n in sync with os.environ\n \"\"\"\n envkeys = set(os.environ.keys())\n for key in envkeys.difference(self._upperkeys):\n self._upperkeys[key] = key.upper()\n for key in set(self._upperkeys).difference(envkeys):\n del self._upperkeys[key]\n\n def __contains__(self, k):\n self._sync()\n return k.upper() in self._upperkeys\n\n def __len__(self):\n self._sync()\n return len(self._upperkeys)\n\n def __iter__(self):\n self._sync()\n return iter(self._upperkeys.values())\n\n def __getitem__(self, k):\n self._sync()\n return os.environ[k]\n\n def __setitem__(self, k, v):\n self._sync()\n self._upperkeys[k.upper()] = k\n os.environ[k] = v\n\n def __delitem__(self, k):\n self._sync()\n if k.upper() in self._upperkeys:\n del self._upperkeys[k.upper()]\n del os.environ[k]\n\n def getkey_actual_case(self, k):\n self._sync()\n return self._upperkeys.get(k.upper())\n\n\n@lazyobject\ndef os_environ():\n \"\"\"This dispatches to the correct, case-sensitive version of os.environ.\n This is mainly a problem for Windows. See #2024 for more details.\n This can probably go away once support for Python v3.5 or v3.6 is\n dropped.\n \"\"\"\n if ON_WINDOWS:\n return OSEnvironCasePreserving()\n else:\n return os.environ\n\n\n@functools.lru_cache(1)\ndef bash_command():\n \"\"\"Determines the command for Bash on the current platform.\"\"\"\n if ON_WINDOWS:\n bc = windows_bash_command()\n else:\n bc = \"bash\"\n return bc\n\n\n@lazyobject\ndef BASH_COMPLETIONS_DEFAULT():\n \"\"\"A possibly empty tuple with default paths to Bash completions known for\n the current platform.\n \"\"\"\n if ON_LINUX or ON_CYGWIN or ON_MSYS:\n bcd = (\"/usr/share/bash-completion/bash_completion\",)\n elif ON_DARWIN:\n bcd = (\n \"/usr/local/share/bash-completion/bash_completion\", # v2.x\n \"/usr/local/etc/bash_completion\",\n \"/opt/local/etc/profile.d/bash_completion.sh\",\n \"/usr/local/etc/bash_completion.d/git-completion.bash\",\n ) # v1.x\n elif ON_WINDOWS and git_for_windows_path():\n bcd = (\n os.path.join(\n git_for_windows_path(), \"usr\\\\share\\\\bash-completion\\\\bash_completion\"\n ),\n os.path.join(\n git_for_windows_path(),\n \"mingw64\\\\share\\\\git\\\\completion\\\\\" \"git-completion.bash\",\n ),\n )\n else:\n bcd = ()\n return bcd\n\n\n@lazyobject\ndef PATH_DEFAULT():\n if ON_LINUX or ON_CYGWIN or ON_MSYS:\n if linux_distro() == \"arch\":\n pd = (\n \"/usr/local/sbin\",\n \"/usr/local/bin\",\n \"/usr/bin\",\n \"/usr/bin/site_perl\",\n \"/usr/bin/vendor_perl\",\n \"/usr/bin/core_perl\",\n )\n else:\n pd = (\n os.path.expanduser(\"~/bin\"),\n \"/usr/local/sbin\",\n \"/usr/local/bin\",\n \"/usr/sbin\",\n \"/usr/bin\",\n \"/sbin\",\n \"/bin\",\n \"/usr/games\",\n \"/usr/local/games\",\n )\n elif ON_DARWIN:\n pd = (\"/usr/local/bin\", \"/usr/bin\", \"/bin\", \"/usr/sbin\", \"/sbin\")\n elif ON_WINDOWS:\n import winreg\n\n key = winreg.OpenKey(\n winreg.HKEY_LOCAL_MACHINE,\n r\"SYSTEM\\CurrentControlSet\\Control\\Session Manager\\Environment\",\n )\n pd = tuple(winreg.QueryValueEx(key, \"Path\")[0].split(os.pathsep))\n else:\n pd = ()\n return pd\n\n\n#\n# libc\n#\n@lazyobject\ndef LIBC():\n \"\"\"The platform dependent libc implementation.\"\"\"\n global ctypes\n if ON_DARWIN:\n import ctypes.util\n\n libc = ctypes.CDLL(ctypes.util.find_library(\"c\"))\n elif ON_CYGWIN:\n libc = ctypes.CDLL(\"cygwin1.dll\")\n elif ON_MSYS:\n libc = ctypes.CDLL(\"msys-2.0.dll\")\n elif ON_BSD:\n try:\n libc = ctypes.CDLL(\"libc.so\")\n except AttributeError:\n libc = None\n except OSError:\n # OS X; can't use ctypes.util.find_library because that creates\n # a new process on Linux, which is undesirable.\n try:\n libc = ctypes.CDLL(\"libc.dylib\")\n except OSError:\n libc = None\n elif ON_POSIX:\n try:\n libc = ctypes.CDLL(\"libc.so\")\n except AttributeError:\n libc = None\n except OSError:\n # Debian and derivatives do the wrong thing because /usr/lib/libc.so\n # is a GNU ld script rather than an ELF object. To get around this, we\n # have to be more specific.\n # We don't want to use ctypes.util.find_library because that creates a\n # new process on Linux. We also don't want to try too hard because at\n # this point we're already pretty sure this isn't Linux.\n try:\n libc = ctypes.CDLL(\"libc.so.6\")\n except OSError:\n libc = None\n if not hasattr(libc, \"sysinfo\"):\n # Not Linux.\n libc = None\n elif ON_WINDOWS:\n if hasattr(ctypes, \"windll\") and hasattr(ctypes.windll, \"kernel32\"):\n libc = ctypes.windll.kernel32\n else:\n try:\n # Windows CE uses the cdecl calling convention.\n libc = ctypes.CDLL(\"coredll.lib\")\n except (AttributeError, OSError):\n libc = None\n elif ON_BEOS:\n libc = ctypes.CDLL(\"libroot.so\")\n else:\n libc = None\n return libc\n" }, { "path": "xonsh/ply/.gitignore", "content": "*.pyc\n*.pyo\n__pycache__\n*.out\n*.dif\n*~\n/dist\n/build\n/*.egg-info\n" }, { "path": "xonsh/ply/.travis.yml", "content": "language: python\npython:\n - \"2.6\"\n - \"2.7\"\n - \"3.3\"\n - \"3.4\"\n - \"3.5\"\n - \"3.6\"\ninstall: true\nscript: \"cd test && python testlex.py && python testyacc.py\"\n" }, { "path": "xonsh/ply/CHANGES", "content": "Current Version\n---------------\nIMPORTANT NOTE (2018-12-22): PLY is no longer be released in any\npackage-installable format. If you want to use the latest version, you\nneed to COPY the contents of the ply/ directory into your own project\nand use it. Although PLY is no longer distributed as a package, it is\nmaintained as a mature library. No new features are planned, but \nissues and pull requests for bugs are still welcome. Any changes to the \nsoftware will be noted here.\n\n01/19/19 Some improvements to the preprocessor module contributed by\n Rob Reilink. Issue #195 fixes the evaluation of expressions\n such as #if a != b. Issue #196 fixes some some issues\n related to the lack of a specified Unicode encoding on reading\n text files.\n\n12/22/18 Incorporated some minor changes to eliminate warning messages\n related to open() calls.\n\nVersion 3.11\n---------------------\n02/15/18 beazley\n Fixed some minor bugs related to re flags and token order. \n Github pull requests #151 and #153.\n\n02/15/18 beazley\n Added a set_lexpos() method to grammar symbols. Github issue #148.\n\n\n04/13/17 beazley\n Mostly minor bug fixes and small code cleanups.\n\nVersion 3.10\n---------------------\n01/31/17: beazley\n Changed grammar signature computation to not involve hashing\n functions. Parts are just combined into a big string.\n\n10/07/16: beazley\n Fixed Issue #101: Incorrect shift-reduce conflict resolution with\n precedence specifier.\n\n PLY was incorrectly resolving shift-reduce conflicts in certain\n cases. For example, in the example/calc/calc.py example, you \n could trigger it doing this:\n\n calc > -3 - 4 \n 1 (correct answer should be -7)\n calc >\n\n Issue and suggested patch contributed by https://github.com/RomaVis\n \nVersion 3.9\n---------------------\n08/30/16: beazley\n Exposed the parser state number as the parser.state attribute\n in productions and error functions. For example:\n\n def p_somerule(p):\n '''\n rule : A B C\n '''\n print('State:', p.parser.state)\n\n May address issue #65 (publish current state in error callback).\n\n08/30/16: beazley \n Fixed Issue #88. Python3 compatibility with ply/cpp.\n\n08/30/16: beazley\n Fixed Issue #93. Ply can crash if SyntaxError is raised inside\n a production. Not actually sure if the original implementation\n worked as documented at all. Yacc has been modified to follow\n the spec as outlined in the CHANGES noted for 11/27/07 below.\n\n08/30/16: beazley\n Fixed Issue #97. Failure with code validation when the original\n source files aren't present. Validation step now ignores\n the missing file.\n\n08/30/16: beazley\n Minor fixes to version numbers.\n\nVersion 3.8\n---------------------\n10/02/15: beazley\n Fixed issues related to Python 3.5. Patch contributed by Barry Warsaw.\n\nVersion 3.7\n---------------------\n08/25/15: beazley\n Fixed problems when reading table files from pickled data.\n\n05/07/15: beazley\n Fixed regression in handling of table modules if specified as module\n objects. See https://github.com/dabeaz/ply/issues/63\n\nVersion 3.6\n---------------------\n04/25/15: beazley\n If PLY is unable to create the 'parser.out' or 'parsetab.py' files due\n to permission issues, it now just issues a warning message and\n continues to operate. This could happen if a module using PLY\n\t is installed in a funny way where tables have to be regenerated, but\n for whatever reason, the user doesn't have write permission on\n the directory where PLY wants to put them. \n\n04/24/15: beazley\n Fixed some issues related to use of packages and table file\n modules. Just to emphasize, PLY now generates its special\n files such as 'parsetab.py' and 'lextab.py' in the *SAME*\n directory as the source file that uses lex() and yacc().\n\n\t If for some reason, you want to change the name of the table\n module, use the tabmodule and lextab options:\n\n lexer = lex.lex(lextab='spamlextab')\n parser = yacc.yacc(tabmodule='spamparsetab')\n\n If you specify a simple name as shown, the module will still be\n created in the same directory as the file invoking lex() or yacc().\n If you want the table files to be placed into a different package,\n then give a fully qualified package name. For example:\n\n lexer = lex.lex(lextab='pkgname.files.lextab')\n parser = yacc.yacc(tabmodule='pkgname.files.parsetab')\n\n For this to work, 'pkgname.files' must already exist as a valid \n Python package (i.e., the directories must already exist and be\n set up with the proper __init__.py files, etc.).\n\nVersion 3.5\n---------------------\n04/21/15: beazley\n Added support for defaulted_states in the parser. A\n defaulted_state is a state where the only legal action is a\n reduction of a single grammar rule across all valid input\n tokens. For such states, the rule is reduced and the\n reading of the next lookahead token is delayed until it is\n actually needed at a later point in time.\n\n\t This delay in consuming the next lookahead token is a\n\t potentially important feature in advanced parsing\n\t applications that require tight interaction between the\n\t lexer and the parser. For example, a grammar rule change\n\t modify the lexer state upon reduction and have such changes\n\t take effect before the next input token is read.\n\n\t *** POTENTIAL INCOMPATIBILITY ***\n\t One potential danger of defaulted_states is that syntax\n\t errors might be deferred to a a later point of processing\n\t than where they were detected in past versions of PLY.\n\t Thus, it's possible that your error handling could change\n\t slightly on the same inputs. defaulted_states do not change\n\t the overall parsing of the input (i.e., the same grammar is\n\t accepted).\n\n\t If for some reason, you need to disable defaulted states,\n\t you can do this:\n\n parser = yacc.yacc()\n parser.defaulted_states = {}\n\n04/21/15: beazley\n Fixed debug logging in the parser. It wasn't properly reporting goto states\n on grammar rule reductions.\n\n04/20/15: beazley\n Added actions to be defined to character literals (Issue #32). For example:\n\n literals = [ '{', '}' ]\n\n def t_lbrace(t):\n r'\\{'\n # Some action\n t.type = '{'\n return t\n\n def t_rbrace(t):\n r'\\}'\n # Some action\n t.type = '}'\n return t\n\n04/19/15: beazley\n Import of the 'parsetab.py' file is now constrained to only consider the\n directory specified by the outputdir argument to yacc(). If not supplied,\n the import will only consider the directory in which the grammar is defined.\n This should greatly reduce problems with the wrong parsetab.py file being\n imported by mistake. For example, if it's found somewhere else on the path\n by accident.\n\n\t *** POTENTIAL INCOMPATIBILITY *** It's possible that this might break some\n packaging/deployment setup if PLY was instructed to place its parsetab.py\n in a different location. You'll have to specify a proper outputdir= argument\n to yacc() to fix this if needed.\n\n04/19/15: beazley\n Changed default output directory to be the same as that in which the\n yacc grammar is defined. If your grammar is in a file 'calc.py',\n then the parsetab.py and parser.out files should be generated in the\n same directory as that file. The destination directory can be changed\n using the outputdir= argument to yacc().\n\n04/19/15: beazley\n Changed the parsetab.py file signature slightly so that the parsetab won't\n regenerate if created on a different major version of Python (ie., a \n parsetab created on Python 2 will work with Python 3).\n\n04/16/15: beazley\n Fixed Issue #44 call_errorfunc() should return the result of errorfunc()\n\n04/16/15: beazley\n Support for versions of Python <2.7 is officially dropped. PLY may work, but\n the unit tests requires Python 2.7 or newer.\n\n04/16/15: beazley\n Fixed bug related to calling yacc(start=...). PLY wasn't regenerating the\n table file correctly for this case.\n\n04/16/15: beazley\n Added skipped tests for PyPy and Java. Related to use of Python's -O option.\n\n05/29/13: beazley\n Added filter to make unit tests pass under 'python -3'.\n Reported by Neil Muller.\n \n05/29/13: beazley\n Fixed CPP_INTEGER regex in ply/cpp.py (Issue 21).\n\t Reported by @vbraun.\n \n05/29/13: beazley\n Fixed yacc validation bugs when from __future__ import unicode_literals\n is being used. Reported by Kenn Knowles.\n\n05/29/13: beazley\n Added support for Travis-CI. Contributed by Kenn Knowles.\n\n05/29/13: beazley\n Added a .gitignore file. Suggested by Kenn Knowles.\n\n05/29/13: beazley\n\t Fixed validation problems for source files that include a \n different source code encoding specifier. Fix relies on\n the inspect module. Should work on Python 2.6 and newer.\n Not sure about older versions of Python.\n Contributed by Michael Droettboom\n\n05/21/13: beazley\n Fixed unit tests for yacc to eliminate random failures due to dict hash value\n\t randomization in Python 3.3\n\t Reported by Arfrever\n\n10/15/12: beazley\n Fixed comment whitespace processing bugs in ply/cpp.py.\n Reported by Alexei Pososin.\n\n10/15/12: beazley\n Fixed token names in ply/ctokens.py to match rule names.\n Reported by Alexei Pososin.\n\n04/26/12: beazley\n Changes to functions available in panic mode error recover. In previous versions\n of PLY, the following global functions were available for use in the p_error() rule:\n\n yacc.errok() # Reset error state\n yacc.token() # Get the next token\n yacc.restart() # Reset the parsing stack\n\n The use of global variables was problematic for code involving multiple parsers\n and frankly was a poor design overall. These functions have been moved to methods\n of the parser instance created by the yacc() function. You should write code like\n this:\n\n def p_error(p):\n ...\n parser.errok()\n\n parser = yacc.yacc()\n\n *** POTENTIAL INCOMPATIBILITY *** The original global functions now issue a\n DeprecationWarning.\n \n04/19/12: beazley\n Fixed some problems with line and position tracking and the use of error\n symbols. If you have a grammar rule involving an error rule like this:\n\n def p_assignment_bad(p):\n '''assignment : location EQUALS error SEMI'''\n ...\n\n You can now do line and position tracking on the error token. For example:\n\n def p_assignment_bad(p):\n '''assignment : location EQUALS error SEMI'''\n start_line = p.lineno(3)\n start_pos = p.lexpos(3)\n\n If the trackng=True option is supplied to parse(), you can additionally get\n spans:\n\n def p_assignment_bad(p):\n '''assignment : location EQUALS error SEMI'''\n start_line, end_line = p.linespan(3)\n start_pos, end_pos = p.lexspan(3)\n\n Note that error handling is still a hairy thing in PLY. This won't work\n unless your lexer is providing accurate information. Please report bugs.\n Suggested by a bug reported by Davis Herring.\n \n04/18/12: beazley\n Change to doc string handling in lex module. Regex patterns are now first\n pulled from a function's .regex attribute. If that doesn't exist, then\n\t .doc is checked as a fallback. The @TOKEN decorator now sets the .regex\n\t attribute of a function instead of its doc string.\n\t Changed suggested by Kristoffer Ellersgaard Koch.\n\n04/18/12: beazley\n Fixed issue #1: Fixed _tabversion. It should use __tabversion__ instead of __version__\n Reported by Daniele Tricoli\n\n04/18/12: beazley\n Fixed issue #8: Literals empty list causes IndexError\n Reported by Walter Nissen.\n\n04/18/12: beazley\n Fixed issue #12: Typo in code snippet in documentation\n Reported by florianschanda.\n\n04/18/12: beazley\n Fixed issue #10: Correctly escape t_XOREQUAL pattern.\n Reported by Andy Kittner.\n\nVersion 3.4\n---------------------\n02/17/11: beazley\n Minor patch to make cpp.py compatible with Python 3. Note: This\n is an experimental file not currently used by the rest of PLY.\n\n02/17/11: beazley\n Fixed setup.py trove classifiers to properly list PLY as\n Python 3 compatible.\n\n01/02/11: beazley\n Migration of repository to github.\n\nVersion 3.3\n-----------------------------\n08/25/09: beazley\n Fixed issue 15 related to the set_lineno() method in yacc. Reported by\n\t mdsherry.\n\n08/25/09: beazley\n Fixed a bug related to regular expression compilation flags not being\n properly stored in lextab.py files created by the lexer when running\n in optimize mode. Reported by Bruce Frederiksen.\n\n\nVersion 3.2\n-----------------------------\n03/24/09: beazley\n Added an extra check to not print duplicated warning messages\n about reduce/reduce conflicts.\n\n03/24/09: beazley\n Switched PLY over to a BSD-license.\n\n03/23/09: beazley\n Performance optimization. Discovered a few places to make\n speedups in LR table generation.\n\n03/23/09: beazley\n New warning message. PLY now warns about rules never \n reduced due to reduce/reduce conflicts. Suggested by\n Bruce Frederiksen.\n\n03/23/09: beazley\n Some clean-up of warning messages related to reduce/reduce errors.\n\n03/23/09: beazley\n Added a new picklefile option to yacc() to write the parsing\n tables to a filename using the pickle module. Here is how\n it works:\n\n yacc(picklefile=\"parsetab.p\")\n\n This option can be used if the normal parsetab.py file is\n extremely large. For example, on jython, it is impossible\n to read parsing tables if the parsetab.py exceeds a certain\n threshold.\n\n The filename supplied to the picklefile option is opened\n relative to the current working directory of the Python \n interpreter. If you need to refer to the file elsewhere,\n you will need to supply an absolute or relative path.\n\n For maximum portability, the pickle file is written\n using protocol 0. \n \n03/13/09: beazley\n Fixed a bug in parser.out generation where the rule numbers\n where off by one.\n\n03/13/09: beazley\n Fixed a string formatting bug with one of the error messages.\n Reported by Richard Reitmeyer\n\nVersion 3.1\n-----------------------------\n02/28/09: beazley\n Fixed broken start argument to yacc(). PLY-3.0 broke this\n feature by accident.\n\n02/28/09: beazley\n Fixed debugging output. yacc() no longer reports shift/reduce\n or reduce/reduce conflicts if debugging is turned off. This\n restores similar behavior in PLY-2.5. Reported by Andrew Waters.\n\nVersion 3.0\n-----------------------------\n02/03/09: beazley\n Fixed missing lexer attribute on certain tokens when\n invoking the parser p_error() function. Reported by \n Bart Whiteley.\n\n02/02/09: beazley\n The lex() command now does all error-reporting and diagonistics\n using the logging module interface. Pass in a Logger object\n using the errorlog parameter to specify a different logger.\n\n02/02/09: beazley\n Refactored ply.lex to use a more object-oriented and organized\n approach to collecting lexer information.\n\n02/01/09: beazley\n Removed the nowarn option from lex(). All output is controlled\n by passing in a logger object. Just pass in a logger with a high\n level setting to suppress output. This argument was never \n documented to begin with so hopefully no one was relying upon it.\n\n02/01/09: beazley\n Discovered and removed a dead if-statement in the lexer. This\n resulted in a 6-7% speedup in lexing when I tested it.\n\n01/13/09: beazley\n Minor change to the procedure for signalling a syntax error in a\n production rule. A normal SyntaxError exception should be raised\n instead of yacc.SyntaxError.\n\n01/13/09: beazley\n Added a new method p.set_lineno(n,lineno) that can be used to set the\n line number of symbol n in grammar rules. This simplifies manual\n tracking of line numbers.\n\n01/11/09: beazley\n Vastly improved debugging support for yacc.parse(). Instead of passing\n debug as an integer, you can supply a Logging object (see the logging\n module). Messages will be generated at the ERROR, INFO, and DEBUG\n\t logging levels, each level providing progressively more information.\n The debugging trace also shows states, grammar rule, values passed\n into grammar rules, and the result of each reduction.\n\n01/09/09: beazley\n The yacc() command now does all error-reporting and diagnostics using\n the interface of the logging module. Use the errorlog parameter to\n specify a logging object for error messages. Use the debuglog parameter\n to specify a logging object for the 'parser.out' output.\n\n01/09/09: beazley\n *HUGE* refactoring of the the ply.yacc() implementation. The high-level\n\t user interface is backwards compatible, but the internals are completely\n reorganized into classes. No more global variables. The internals\n are also more extensible. For example, you can use the classes to \n construct a LALR(1) parser in an entirely different manner than \n what is currently the case. Documentation is forthcoming.\n\n01/07/09: beazley\n Various cleanup and refactoring of yacc internals. \n\n01/06/09: beazley\n Fixed a bug with precedence assignment. yacc was assigning the precedence\n each rule based on the left-most token, when in fact, it should have been\n using the right-most token. Reported by Bruce Frederiksen.\n\n11/27/08: beazley\n Numerous changes to support Python 3.0 including removal of deprecated \n statements (e.g., has_key) and the additional of compatibility code\n to emulate features from Python 2 that have been removed, but which\n are needed. Fixed the unit testing suite to work with Python 3.0.\n The code should be backwards compatible with Python 2.\n\n11/26/08: beazley\n Loosened the rules on what kind of objects can be passed in as the\n \"module\" parameter to lex() and yacc(). Previously, you could only use\n a module or an instance. Now, PLY just uses dir() to get a list of\n symbols on whatever the object is without regard for its type.\n\n11/26/08: beazley\n Changed all except: statements to be compatible with Python2.x/3.x syntax.\n\n11/26/08: beazley\n Changed all raise Exception, value statements to raise Exception(value) for\n forward compatibility.\n\n11/26/08: beazley\n Removed all print statements from lex and yacc, using sys.stdout and sys.stderr\n directly. Preparation for Python 3.0 support.\n\n11/04/08: beazley\n Fixed a bug with referring to symbols on the the parsing stack using negative\n indices.\n\n05/29/08: beazley\n Completely revamped the testing system to use the unittest module for everything.\n Added additional tests to cover new errors/warnings.\n\nVersion 2.5\n-----------------------------\n05/28/08: beazley\n Fixed a bug with writing lex-tables in optimized mode and start states.\n Reported by Kevin Henry.\n\nVersion 2.4\n-----------------------------\n05/04/08: beazley\n A version number is now embedded in the table file signature so that \n yacc can more gracefully accomodate changes to the output format\n in the future.\n\n05/04/08: beazley\n Removed undocumented .pushback() method on grammar productions. I'm\n not sure this ever worked and can't recall ever using it. Might have\n been an abandoned idea that never really got fleshed out. This\n feature was never described or tested so removing it is hopefully\n harmless.\n\n05/04/08: beazley\n Added extra error checking to yacc() to detect precedence rules defined\n for undefined terminal symbols. This allows yacc() to detect a potential\n problem that can be really tricky to debug if no warning message or error\n message is generated about it.\n\n05/04/08: beazley\n lex() now has an outputdir that can specify the output directory for\n tables when running in optimize mode. For example:\n\n lexer = lex.lex(optimize=True, lextab=\"ltab\", outputdir=\"foo/bar\")\n \n The behavior of specifying a table module and output directory are\n more aligned with the behavior of yacc().\n \n05/04/08: beazley\n [Issue 9]\n Fixed filename bug in when specifying the modulename in lex() and yacc(). \n If you specified options such as the following:\n\n parser = yacc.yacc(tabmodule=\"foo.bar.parsetab\",outputdir=\"foo/bar\")\n\n yacc would create a file \"foo.bar.parsetab.py\" in the given directory.\n Now, it simply generates a file \"parsetab.py\" in that directory. \n Bug reported by cptbinho.\n\n05/04/08: beazley\n Slight modification to lex() and yacc() to allow their table files\n\t to be loaded from a previously loaded module. This might make\n\t it easier to load the parsing tables from a complicated package\n structure. For example:\n\n\t import foo.bar.spam.parsetab as parsetab\n parser = yacc.yacc(tabmodule=parsetab)\n\n Note: lex and yacc will never regenerate the table file if used\n in the form---you will get a warning message instead. \n This idea suggested by Brian Clapper.\n\n\n04/28/08: beazley\n Fixed a big with p_error() functions being picked up correctly\n when running in yacc(optimize=1) mode. Patch contributed by\n Bart Whiteley.\n\n02/28/08: beazley\n Fixed a bug with 'nonassoc' precedence rules. Basically the\n non-precedence was being ignored and not producing the correct\n run-time behavior in the parser.\n\n02/16/08: beazley\n Slight relaxation of what the input() method to a lexer will\n accept as a string. Instead of testing the input to see\n if the input is a string or unicode string, it checks to see\n if the input object looks like it contains string data.\n This change makes it possible to pass string-like objects\n in as input. For example, the object returned by mmap.\n\n import mmap, os\n data = mmap.mmap(os.open(filename,os.O_RDONLY),\n os.path.getsize(filename),\n access=mmap.ACCESS_READ)\n lexer.input(data)\n \n\n11/29/07: beazley\n Modification of ply.lex to allow token functions to aliased.\n This is subtle, but it makes it easier to create libraries and\n to reuse token specifications. For example, suppose you defined\n a function like this:\n\n def number(t):\n r'\\d+'\n t.value = int(t.value)\n return t\n\n This change would allow you to define a token rule as follows:\n\n t_NUMBER = number\n\n In this case, the token type will be set to 'NUMBER' and use\n the associated number() function to process tokens. \n \n11/28/07: beazley\n Slight modification to lex and yacc to grab symbols from both\n the local and global dictionaries of the caller. This\n modification allows lexers and parsers to be defined using\n inner functions and closures.\n\n11/28/07: beazley\n Performance optimization: The lexer.lexmatch and t.lexer\n attributes are no longer set for lexer tokens that are not\n defined by functions. The only normal use of these attributes\n would be in lexer rules that need to perform some kind of \n special processing. Thus, it doesn't make any sense to set\n them on every token.\n\n *** POTENTIAL INCOMPATIBILITY *** This might break code\n that is mucking around with internal lexer state in some\n sort of magical way.\n\n11/27/07: beazley\n Added the ability to put the parser into error-handling mode\n from within a normal production. To do this, simply raise\n a yacc.SyntaxError exception like this:\n\n def p_some_production(p):\n 'some_production : prod1 prod2'\n ...\n raise yacc.SyntaxError # Signal an error\n\n A number of things happen after this occurs:\n\n - The last symbol shifted onto the symbol stack is discarded\n and parser state backed up to what it was before the\n the rule reduction.\n\n - The current lookahead symbol is saved and replaced by\n the 'error' symbol.\n\n - The parser enters error recovery mode where it tries\n to either reduce the 'error' rule or it starts \n discarding items off of the stack until the parser\n resets.\n\n When an error is manually set, the parser does *not* call \n the p_error() function (if any is defined).\n *** NEW FEATURE *** Suggested on the mailing list\n\n11/27/07: beazley\n Fixed structure bug in examples/ansic. Reported by Dion Blazakis.\n\n11/27/07: beazley\n Fixed a bug in the lexer related to start conditions and ignored\n token rules. If a rule was defined that changed state, but\n returned no token, the lexer could be left in an inconsistent\n state. Reported by \n \n11/27/07: beazley\n Modified setup.py to support Python Eggs. Patch contributed by\n Simon Cross.\n\n11/09/07: beazely\n Fixed a bug in error handling in yacc. If a syntax error occurred and the\n parser rolled the entire parse stack back, the parser would be left in in\n inconsistent state that would cause it to trigger incorrect actions on\n subsequent input. Reported by Ton Biegstraaten, Justin King, and others.\n\n11/09/07: beazley\n Fixed a bug when passing empty input strings to yacc.parse(). This \n would result in an error message about \"No input given\". Reported\n by Andrew Dalke.\n\nVersion 2.3\n-----------------------------\n02/20/07: beazley\n Fixed a bug with character literals if the literal '.' appeared as the\n last symbol of a grammar rule. Reported by Ales Smrcka.\n\n02/19/07: beazley\n Warning messages are now redirected to stderr instead of being printed\n to standard output.\n\n02/19/07: beazley\n Added a warning message to lex.py if it detects a literal backslash\n character inside the t_ignore declaration. This is to help\n problems that might occur if someone accidentally defines t_ignore\n as a Python raw string. For example:\n\n t_ignore = r' \\t'\n\n The idea for this is from an email I received from David Cimimi who\n reported bizarre behavior in lexing as a result of defining t_ignore\n as a raw string by accident.\n\n02/18/07: beazley\n Performance improvements. Made some changes to the internal\n table organization and LR parser to improve parsing performance.\n\n02/18/07: beazley\n Automatic tracking of line number and position information must now be \n enabled by a special flag to parse(). For example:\n\n yacc.parse(data,tracking=True)\n\n In many applications, it's just not that important to have the\n parser automatically track all line numbers. By making this an \n optional feature, it allows the parser to run significantly faster\n (more than a 20% speed increase in many cases). Note: positional\n information is always available for raw tokens---this change only\n applies to positional information associated with nonterminal\n grammar symbols.\n *** POTENTIAL INCOMPATIBILITY ***\n\t \n02/18/07: beazley\n Yacc no longer supports extended slices of grammar productions.\n However, it does support regular slices. For example:\n\n def p_foo(p):\n '''foo: a b c d e'''\n p[0] = p[1:3]\n\n This change is a performance improvement to the parser--it streamlines\n normal access to the grammar values since slices are now handled in\n a __getslice__() method as opposed to __getitem__().\n\n02/12/07: beazley\n Fixed a bug in the handling of token names when combined with\n start conditions. Bug reported by Todd O'Bryan.\n\nVersion 2.2\n------------------------------\n11/01/06: beazley\n Added lexpos() and lexspan() methods to grammar symbols. These\n mirror the same functionality of lineno() and linespan(). For\n example:\n\n def p_expr(p):\n 'expr : expr PLUS expr'\n p.lexpos(1) # Lexing position of left-hand-expression\n p.lexpos(1) # Lexing position of PLUS\n start,end = p.lexspan(3) # Lexing range of right hand expression\n\n11/01/06: beazley\n Minor change to error handling. The recommended way to skip characters\n in the input is to use t.lexer.skip() as shown here:\n\n def t_error(t):\n print \"Illegal character '%s'\" % t.value[0]\n t.lexer.skip(1)\n \n The old approach of just using t.skip(1) will still work, but won't\n be documented.\n\n10/31/06: beazley\n Discarded tokens can now be specified as simple strings instead of\n functions. To do this, simply include the text \"ignore_\" in the\n token declaration. For example:\n\n t_ignore_cppcomment = r'//.*'\n \n Previously, this had to be done with a function. For example:\n\n def t_ignore_cppcomment(t):\n r'//.*'\n pass\n\n If start conditions/states are being used, state names should appear\n before the \"ignore_\" text.\n\n10/19/06: beazley\n The Lex module now provides support for flex-style start conditions\n as described at http://www.gnu.org/software/flex/manual/html_chapter/flex_11.html.\n Please refer to this document to understand this change note. Refer to\n the PLY documentation for PLY-specific explanation of how this works.\n\n To use start conditions, you first need to declare a set of states in\n your lexer file:\n\n states = (\n ('foo','exclusive'),\n ('bar','inclusive')\n )\n\n This serves the same role as the %s and %x specifiers in flex.\n\n One a state has been declared, tokens for that state can be \n declared by defining rules of the form t_state_TOK. For example:\n\n t_PLUS = '\\+' # Rule defined in INITIAL state\n t_foo_NUM = '\\d+' # Rule defined in foo state\n t_bar_NUM = '\\d+' # Rule defined in bar state\n\n t_foo_bar_NUM = '\\d+' # Rule defined in both foo and bar\n t_ANY_NUM = '\\d+' # Rule defined in all states\n\n In addition to defining tokens for each state, the t_ignore and t_error\n specifications can be customized for specific states. For example:\n\n t_foo_ignore = \" \" # Ignored characters for foo state\n def t_bar_error(t): \n # Handle errors in bar state\n\n With token rules, the following methods can be used to change states\n \n def t_TOKNAME(t):\n t.lexer.begin('foo') # Begin state 'foo'\n t.lexer.push_state('foo') # Begin state 'foo', push old state\n # onto a stack\n t.lexer.pop_state() # Restore previous state\n t.lexer.current_state() # Returns name of current state\n\n These methods mirror the BEGIN(), yy_push_state(), yy_pop_state(), and\n yy_top_state() functions in flex.\n\n The use of start states can be used as one way to write sub-lexers.\n For example, the lexer or parser might instruct the lexer to start\n generating a different set of tokens depending on the context.\n \n example/yply/ylex.py shows the use of start states to grab C/C++ \n code fragments out of traditional yacc specification files.\n\n *** NEW FEATURE *** Suggested by Daniel Larraz with whom I also\n discussed various aspects of the design.\n\n10/19/06: beazley\n Minor change to the way in which yacc.py was reporting shift/reduce\n conflicts. Although the underlying LALR(1) algorithm was correct,\n PLY was under-reporting the number of conflicts compared to yacc/bison\n when precedence rules were in effect. This change should make PLY\n report the same number of conflicts as yacc.\n\n10/19/06: beazley\n Modified yacc so that grammar rules could also include the '-' \n character. For example:\n\n def p_expr_list(p):\n 'expression-list : expression-list expression'\n\n Suggested by Oldrich Jedlicka.\n\n10/18/06: beazley\n Attribute lexer.lexmatch added so that token rules can access the re \n match object that was generated. For example:\n\n def t_FOO(t):\n r'some regex'\n m = t.lexer.lexmatch\n # Do something with m\n\n\n This may be useful if you want to access named groups specified within\n the regex for a specific token. Suggested by Oldrich Jedlicka.\n \n10/16/06: beazley\n Changed the error message that results if an illegal character\n is encountered and no default error function is defined in lex.\n The exception is now more informative about the actual cause of\n the error.\n \nVersion 2.1\n------------------------------\n10/02/06: beazley\n The last Lexer object built by lex() can be found in lex.lexer.\n The last Parser object built by yacc() can be found in yacc.parser.\n\n10/02/06: beazley\n New example added: examples/yply\n\n This example uses PLY to convert Unix-yacc specification files to\n PLY programs with the same grammar. This may be useful if you\n want to convert a grammar from bison/yacc to use with PLY.\n \n10/02/06: beazley\n Added support for a start symbol to be specified in the yacc\n input file itself. Just do this:\n\n start = 'name'\n\n where 'name' matches some grammar rule. For example:\n\n def p_name(p):\n 'name : A B C'\n ...\n\n This mirrors the functionality of the yacc %start specifier.\n\n09/30/06: beazley\n Some new examples added.:\n\n examples/GardenSnake : A simple indentation based language similar\n to Python. Shows how you might handle \n whitespace. Contributed by Andrew Dalke.\n\n examples/BASIC : An implementation of 1964 Dartmouth BASIC.\n Contributed by Dave against his better\n judgement.\n\n09/28/06: beazley\n Minor patch to allow named groups to be used in lex regular\n expression rules. For example:\n\n t_QSTRING = r'''(?P['\"]).*?(?P=quote)'''\n\n Patch submitted by Adam Ring.\n \n09/28/06: beazley\n LALR(1) is now the default parsing method. To use SLR, use\n yacc.yacc(method=\"SLR\"). Note: there is no performance impact\n on parsing when using LALR(1) instead of SLR. However, constructing\n the parsing tables will take a little longer.\n\n09/26/06: beazley\n Change to line number tracking. To modify line numbers, modify\n the line number of the lexer itself. For example:\n\n def t_NEWLINE(t):\n r'\\n'\n t.lexer.lineno += 1\n\n This modification is both cleanup and a performance optimization.\n In past versions, lex was monitoring every token for changes in\n the line number. This extra processing is unnecessary for a vast\n majority of tokens. Thus, this new approach cleans it up a bit.\n\n *** POTENTIAL INCOMPATIBILITY ***\n You will need to change code in your lexer that updates the line\n number. For example, \"t.lineno += 1\" becomes \"t.lexer.lineno += 1\"\n \n09/26/06: beazley\n Added the lexing position to tokens as an attribute lexpos. This\n is the raw index into the input text at which a token appears.\n This information can be used to compute column numbers and other\n details (e.g., scan backwards from lexpos to the first newline\n to get a column position).\n\n09/25/06: beazley\n Changed the name of the __copy__() method on the Lexer class\n to clone(). This is used to clone a Lexer object (e.g., if\n you're running different lexers at the same time).\n\n09/21/06: beazley\n Limitations related to the use of the re module have been eliminated.\n Several users reported problems with regular expressions exceeding\n more than 100 named groups. To solve this, lex.py is now capable\n of automatically splitting its master regular regular expression into\n smaller expressions as needed. This should, in theory, make it\n possible to specify an arbitrarily large number of tokens.\n\n09/21/06: beazley\n Improved error checking in lex.py. Rules that match the empty string\n are now rejected (otherwise they cause the lexer to enter an infinite\n loop). An extra check for rules containing '#' has also been added.\n Since lex compiles regular expressions in verbose mode, '#' is interpreted\n as a regex comment, it is critical to use '\\#' instead. \n\n09/18/06: beazley\n Added a @TOKEN decorator function to lex.py that can be used to \n define token rules where the documentation string might be computed\n in some way.\n \n digit = r'([0-9])'\n nondigit = r'([_A-Za-z])'\n identifier = r'(' + nondigit + r'(' + digit + r'|' + nondigit + r')*)' \n\n from ply.lex import TOKEN\n\n @TOKEN(identifier)\n def t_ID(t):\n # Do whatever\n\n The @TOKEN decorator merely sets the documentation string of the\n associated token function as needed for lex to work. \n\n Note: An alternative solution is the following:\n\n def t_ID(t):\n # Do whatever\n \n t_ID.__doc__ = identifier\n\n Note: Decorators require the use of Python 2.4 or later. If compatibility\n with old versions is needed, use the latter solution.\n\n The need for this feature was suggested by Cem Karan.\n\n09/14/06: beazley\n Support for single-character literal tokens has been added to yacc.\n These literals must be enclosed in quotes. For example:\n\n def p_expr(p):\n \"expr : expr '+' expr\"\n ...\n\n def p_expr(p):\n 'expr : expr \"-\" expr'\n ...\n\n In addition to this, it is necessary to tell the lexer module about\n literal characters. This is done by defining the variable 'literals'\n as a list of characters. This should be defined in the module that\n invokes the lex.lex() function. For example:\n\n literals = ['+','-','*','/','(',')','=']\n \n or simply\n\n literals = '+=*/()='\n\n It is important to note that literals can only be a single character.\n When the lexer fails to match a token using its normal regular expression\n rules, it will check the current character against the literal list.\n If found, it will be returned with a token type set to match the literal\n character. Otherwise, an illegal character will be signalled.\n\n\n09/14/06: beazley\n Modified PLY to install itself as a proper Python package called 'ply'.\n This will make it a little more friendly to other modules. This\n changes the usage of PLY only slightly. Just do this to import the\n modules\n\n import ply.lex as lex\n import ply.yacc as yacc\n\n Alternatively, you can do this:\n\n from ply import *\n\n Which imports both the lex and yacc modules.\n Change suggested by Lee June.\n\n09/13/06: beazley\n Changed the handling of negative indices when used in production rules.\n A negative production index now accesses already parsed symbols on the\n parsing stack. For example, \n\n def p_foo(p):\n \"foo: A B C D\"\n print p[1] # Value of 'A' symbol\n print p[2] # Value of 'B' symbol\n print p[-1] # Value of whatever symbol appears before A\n # on the parsing stack.\n\n p[0] = some_val # Sets the value of the 'foo' grammer symbol\n \n This behavior makes it easier to work with embedded actions within the\n parsing rules. For example, in C-yacc, it is possible to write code like\n this:\n\n bar: A { printf(\"seen an A = %d\\n\", $1); } B { do_stuff; }\n\n In this example, the printf() code executes immediately after A has been\n parsed. Within the embedded action code, $1 refers to the A symbol on\n the stack.\n\n To perform this equivalent action in PLY, you need to write a pair\n of rules like this:\n\n def p_bar(p):\n \"bar : A seen_A B\"\n do_stuff\n\n def p_seen_A(p):\n \"seen_A :\"\n print \"seen an A =\", p[-1]\n\n The second rule \"seen_A\" is merely a empty production which should be\n reduced as soon as A is parsed in the \"bar\" rule above. The use \n of the negative index p[-1] is used to access whatever symbol appeared\n before the seen_A symbol.\n\n This feature also makes it possible to support inherited attributes.\n For example:\n\n def p_decl(p):\n \"decl : scope name\"\n\n def p_scope(p):\n \"\"\"scope : GLOBAL\n | LOCAL\"\"\"\n p[0] = p[1]\n\n def p_name(p):\n \"name : ID\"\n if p[-1] == \"GLOBAL\":\n # ...\n else if p[-1] == \"LOCAL\":\n #...\n\n In this case, the name rule is inheriting an attribute from the\n scope declaration that precedes it.\n \n *** POTENTIAL INCOMPATIBILITY ***\n If you are currently using negative indices within existing grammar rules,\n your code will break. This should be extremely rare if non-existent in\n most cases. The argument to various grammar rules is not usually not\n processed in the same way as a list of items.\n \nVersion 2.0\n------------------------------\n09/07/06: beazley\n Major cleanup and refactoring of the LR table generation code. Both SLR\n and LALR(1) table generation is now performed by the same code base with\n only minor extensions for extra LALR(1) processing.\n\n09/07/06: beazley\n Completely reimplemented the entire LALR(1) parsing engine to use the\n DeRemer and Pennello algorithm for calculating lookahead sets. This\n significantly improves the performance of generating LALR(1) tables\n and has the added feature of actually working correctly! If you\n experienced weird behavior with LALR(1) in prior releases, this should\n hopefully resolve all of those problems. Many thanks to \n Andrew Waters and Markus Schoepflin for submitting bug reports\n and helping me test out the revised LALR(1) support.\n\nVersion 1.8\n------------------------------\n08/02/06: beazley\n Fixed a problem related to the handling of default actions in LALR(1)\n parsing. If you experienced subtle and/or bizarre behavior when trying\n to use the LALR(1) engine, this may correct those problems. Patch\n contributed by Russ Cox. Note: This patch has been superceded by \n revisions for LALR(1) parsing in Ply-2.0.\n\n08/02/06: beazley\n Added support for slicing of productions in yacc. \n Patch contributed by Patrick Mezard.\n\nVersion 1.7\n------------------------------\n03/02/06: beazley\n Fixed infinite recursion problem ReduceToTerminals() function that\n would sometimes come up in LALR(1) table generation. Reported by \n Markus Schoepflin.\n\n03/01/06: beazley\n Added \"reflags\" argument to lex(). For example:\n\n lex.lex(reflags=re.UNICODE)\n\n This can be used to specify optional flags to the re.compile() function\n used inside the lexer. This may be necessary for special situations such\n as processing Unicode (e.g., if you want escapes like \\w and \\b to consult\n the Unicode character property database). The need for this suggested by\n Andreas Jung.\n\n03/01/06: beazley\n Fixed a bug with an uninitialized variable on repeated instantiations of parser\n objects when the write_tables=0 argument was used. Reported by Michael Brown.\n\n03/01/06: beazley\n Modified lex.py to accept Unicode strings both as the regular expressions for\n tokens and as input. Hopefully this is the only change needed for Unicode support.\n Patch contributed by Johan Dahl.\n\n03/01/06: beazley\n Modified the class-based interface to work with new-style or old-style classes.\n Patch contributed by Michael Brown (although I tweaked it slightly so it would work\n with older versions of Python).\n\nVersion 1.6\n------------------------------\n05/27/05: beazley\n Incorporated patch contributed by Christopher Stawarz to fix an extremely\n devious bug in LALR(1) parser generation. This patch should fix problems\n numerous people reported with LALR parsing.\n\n05/27/05: beazley\n Fixed problem with lex.py copy constructor. Reported by Dave Aitel, Aaron Lav,\n and Thad Austin. \n\n05/27/05: beazley\n Added outputdir option to yacc() to control output directory. Contributed\n by Christopher Stawarz.\n\n05/27/05: beazley\n Added rununit.py test script to run tests using the Python unittest module.\n Contributed by Miki Tebeka.\n\nVersion 1.5\n------------------------------\n05/26/04: beazley\n Major enhancement. LALR(1) parsing support is now working.\n This feature was implemented by Elias Ioup (ezioup@alumni.uchicago.edu)\n and optimized by David Beazley. To use LALR(1) parsing do\n the following:\n\n yacc.yacc(method=\"LALR\")\n\n Computing LALR(1) parsing tables takes about twice as long as\n the default SLR method. However, LALR(1) allows you to handle\n more complex grammars. For example, the ANSI C grammar\n (in example/ansic) has 13 shift-reduce conflicts with SLR, but\n only has 1 shift-reduce conflict with LALR(1).\n\n05/20/04: beazley\n Added a __len__ method to parser production lists. Can\n be used in parser rules like this:\n\n def p_somerule(p):\n \"\"\"a : B C D\n | E F\"\n if (len(p) == 3):\n # Must have been first rule\n elif (len(p) == 2):\n # Must be second rule\n\n Suggested by Joshua Gerth and others.\n\nVersion 1.4\n------------------------------\n04/23/04: beazley\n Incorporated a variety of patches contributed by Eric Raymond.\n These include:\n\n 0. Cleans up some comments so they don't wrap on an 80-column display.\n 1. Directs compiler errors to stderr where they belong.\n 2. Implements and documents automatic line counting when \\n is ignored.\n 3. Changes the way progress messages are dumped when debugging is on. \n The new format is both less verbose and conveys more information than\n the old, including shift and reduce actions.\n\n04/23/04: beazley\n Added a Python setup.py file to simply installation. Contributed\n by Adam Kerrison.\n\n04/23/04: beazley\n Added patches contributed by Adam Kerrison.\n \n - Some output is now only shown when debugging is enabled. This\n means that PLY will be completely silent when not in debugging mode.\n \n - An optional parameter \"write_tables\" can be passed to yacc() to\n control whether or not parsing tables are written. By default,\n it is true, but it can be turned off if you don't want the yacc\n table file. Note: disabling this will cause yacc() to regenerate\n the parsing table each time.\n\n04/23/04: beazley\n Added patches contributed by David McNab. This patch addes two\n features:\n\n - The parser can be supplied as a class instead of a module.\n For an example of this, see the example/classcalc directory.\n\n - Debugging output can be directed to a filename of the user's\n choice. Use\n\n yacc(debugfile=\"somefile.out\")\n\n \nVersion 1.3\n------------------------------\n12/10/02: jmdyck\n Various minor adjustments to the code that Dave checked in today.\n Updated test/yacc_{inf,unused}.exp to reflect today's changes.\n\n12/10/02: beazley\n Incorporated a variety of minor bug fixes to empty production\n handling and infinite recursion checking. Contributed by\n Michael Dyck.\n\n12/10/02: beazley\n Removed bogus recover() method call in yacc.restart()\n\nVersion 1.2\n------------------------------\n11/27/02: beazley\n Lexer and parser objects are now available as an attribute\n of tokens and slices respectively. For example:\n \n def t_NUMBER(t):\n r'\\d+'\n print t.lexer\n\n def p_expr_plus(t):\n 'expr: expr PLUS expr'\n print t.lexer\n print t.parser\n\n This can be used for state management (if needed).\n \n10/31/02: beazley\n Modified yacc.py to work with Python optimize mode. To make\n this work, you need to use\n\n yacc.yacc(optimize=1)\n\n Furthermore, you need to first run Python in normal mode\n to generate the necessary parsetab.py files. After that,\n you can use python -O or python -OO. \n\n Note: optimized mode turns off a lot of error checking.\n Only use when you are sure that your grammar is working.\n Make sure parsetab.py is up to date!\n\n10/30/02: beazley\n Added cloning of Lexer objects. For example:\n\n import copy\n l = lex.lex()\n lc = copy.copy(l)\n\n l.input(\"Some text\")\n lc.input(\"Some other text\")\n ...\n\n This might be useful if the same \"lexer\" is meant to\n be used in different contexts---or if multiple lexers\n are running concurrently.\n \n10/30/02: beazley\n Fixed subtle bug with first set computation and empty productions.\n Patch submitted by Michael Dyck.\n\n10/30/02: beazley\n Fixed error messages to use \"filename:line: message\" instead\n of \"filename:line. message\". This makes error reporting more\n friendly to emacs. Patch submitted by Franois Pinard.\n\n10/30/02: beazley\n Improvements to parser.out file. Terminals and nonterminals\n are sorted instead of being printed in random order.\n Patch submitted by Franois Pinard.\n\n10/30/02: beazley\n Improvements to parser.out file output. Rules are now printed\n in a way that's easier to understand. Contributed by Russ Cox.\n\n10/30/02: beazley\n Added 'nonassoc' associativity support. This can be used\n to disable the chaining of operators like a < b < c.\n To use, simply specify 'nonassoc' in the precedence table\n\n precedence = (\n ('nonassoc', 'LESSTHAN', 'GREATERTHAN'), # Nonassociative operators\n ('left', 'PLUS', 'MINUS'),\n ('left', 'TIMES', 'DIVIDE'),\n ('right', 'UMINUS'), # Unary minus operator\n )\n\n Patch contributed by Russ Cox.\n\n10/30/02: beazley\n Modified the lexer to provide optional support for Python -O and -OO\n modes. To make this work, Python *first* needs to be run in\n unoptimized mode. This reads the lexing information and creates a\n file \"lextab.py\". Then, run lex like this:\n\n # module foo.py\n ...\n ...\n lex.lex(optimize=1)\n\n Once the lextab file has been created, subsequent calls to\n lex.lex() will read data from the lextab file instead of using \n introspection. In optimized mode (-O, -OO) everything should\n work normally despite the loss of doc strings.\n\n To change the name of the file 'lextab.py' use the following:\n\n lex.lex(lextab=\"footab\")\n\n (this creates a file footab.py)\n \n\nVersion 1.1 October 25, 2001\n------------------------------\n\n10/25/01: beazley\n Modified the table generator to produce much more compact data.\n This should greatly reduce the size of the parsetab.py[c] file.\n Caveat: the tables still need to be constructed so a little more\n work is done in parsetab on import. \n\n10/25/01: beazley\n There may be a possible bug in the cycle detector that reports errors\n about infinite recursion. I'm having a little trouble tracking it\n down, but if you get this problem, you can disable the cycle\n detector as follows:\n\n yacc.yacc(check_recursion = 0)\n\n10/25/01: beazley\n Fixed a bug in lex.py that sometimes caused illegal characters to be\n reported incorrectly. Reported by Sverre Jrgensen.\n\n7/8/01 : beazley\n Added a reference to the underlying lexer object when tokens are handled by\n functions. The lexer is available as the 'lexer' attribute. This\n was added to provide better lexing support for languages such as Fortran\n where certain types of tokens can't be conveniently expressed as regular \n expressions (and where the tokenizing function may want to perform a \n little backtracking). Suggested by Pearu Peterson.\n\n6/20/01 : beazley\n Modified yacc() function so that an optional starting symbol can be specified.\n For example:\n \n yacc.yacc(start=\"statement\")\n\n Normally yacc always treats the first production rule as the starting symbol.\n However, if you are debugging your grammar it may be useful to specify\n an alternative starting symbol. Idea suggested by Rich Salz.\n \nVersion 1.0 June 18, 2001\n--------------------------\nInitial public offering\n\n" }, { "path": "xonsh/ply/CONTRIBUTING.md", "content": "Contributing to PLY\n===================\n\nPLY is a mature project that no longer makes releases. New features\nare no longer being added to it. However, if you feel that you have\nfound a bug in PLY or its documentation, please submit an issue or a\npull request.\n\nImportant note: The Github repo for PLY always contains the most\nup-to-date version of the software. If you want to use the current\nversion, you should COPY the contents of the `ply/` directory into\nyour own project and use it. There will be no future package-installable\nreleases of PLY.\n\n\n\n\n\n" }, { "path": "xonsh/ply/Makefile", "content": "PYTHON ?= python\n\ntest:\n\tcd test && $(PYTHON) testlex.py\n\tcd test && $(PYTHON) testyacc.py\n\nwheel:\n\t$(PYTHON) setup.py bdist_wheel\n\nsdist:\n\t$(PYTHON) setup.py sdist\n\nupload: wheel sdist\n\t$(PYTHON) setup.py bdist_wheel upload\n\t$(PYTHON) setup.py sdist upload\n\n.PHONY: test wheel sdist upload\n" }, { "path": "xonsh/ply/README.md", "content": "# PLY (Python Lex-Yacc)\n\n[![Build Status](https://travis-ci.org/dabeaz/ply.svg?branch=master)](https://travis-ci.org/dabeaz/ply)\n\nCopyright (C) 2001-2019\nDavid M. Beazley (Dabeaz LLC)\nAll rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are\nmet:\n\n* Redistributions of source code must retain the above copyright notice,\n this list of conditions and the following disclaimer. \n* Redistributions in binary form must reproduce the above copyright notice, \n this list of conditions and the following disclaimer in the documentation\n and/or other materials provided with the distribution. \n* Neither the name of the David Beazley or Dabeaz LLC may be used to\n endorse or promote products derived from this software without\n specific prior written permission. \n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n\"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\nLIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\nA PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\nOWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\nSPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\nLIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\nDATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\nTHEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nIntroduction\n============\n\nPLY is a 100% Python implementation of the common parsing tools lex\nand yacc. Here are a few highlights:\n\n - PLY is very closely modeled after traditional lex/yacc.\n If you know how to use these tools in C, you will find PLY\n to be similar.\n\n - PLY provides *very* extensive error reporting and diagnostic \n information to assist in parser construction. The original\n implementation was developed for instructional purposes. As\n a result, the system tries to identify the most common types\n of errors made by novice users. \n\n - PLY provides full support for empty productions, error recovery,\n precedence specifiers, and moderately ambiguous grammars.\n\n - Parsing is based on LR-parsing which is fast, memory efficient, \n better suited to large grammars, and which has a number of nice\n properties when dealing with syntax errors and other parsing problems.\n Currently, PLY builds its parsing tables using the LALR(1)\n algorithm used in yacc.\n\n - PLY uses Python introspection features to build lexers and parsers. \n This greatly simplifies the task of parser construction since it reduces \n the number of files and eliminates the need to run a separate lex/yacc \n tool before running your program.\n\n - PLY can be used to build parsers for \"real\" programming languages.\n Although it is not ultra-fast due to its Python implementation,\n PLY can be used to parse grammars consisting of several hundred\n rules (as might be found for a language like C). The lexer and LR \n parser are also reasonably efficient when parsing typically\n sized programs. People have used PLY to build parsers for\n C, C++, ADA, and other real programming languages.\n\nHow to Use\n==========\n\nPLY consists of two files : lex.py and yacc.py. These are contained\nwithin the 'ply' directory which may also be used as a Python package.\nTo use PLY, simply copy the 'ply' directory to your project and import\nlex and yacc from the associated 'ply' package. For example:\n\n from .ply import lex\n from .ply import yacc\n\nAlternatively, you can copy just the files lex.py and yacc.py\nindividually and use them as modules however you see fit. For example:\n\n import lex\n import yacc\n\nPLY has no third-party dependencies. \n\nThe file doc/ply.html contains complete documentation on how to use\nthe system.\n\nThe example directory contains several different examples including a\nPLY specification for ANSI C as given in K&R 2nd Ed. \n\nA simple example is found at the end of this document\n\nRequirements\n============\nPLY requires the use of Python 2.6 or greater. However, you should\nuse the latest Python release if possible. It should work on just\nabout any platform. PLY has been tested with both CPython and Jython.\nIt also seems to work with IronPython.\n\nResources\n=========\nMore information about PLY can be obtained on the PLY webpage at:\n\n http://www.dabeaz.com/ply\n\nFor a detailed overview of parsing theory, consult the excellent\nbook \"Compilers : Principles, Techniques, and Tools\" by Aho, Sethi, and\nUllman. The topics found in \"Lex & Yacc\" by Levine, Mason, and Brown\nmay also be useful.\n\nThe GitHub page for PLY can be found at:\n\n https://github.com/dabeaz/ply\n\nAn old and inactive discussion group for PLY is found at:\n\n http://groups.google.com/group/ply-hack\n\nAcknowledgments\n===============\nA special thanks is in order for all of the students in CS326 who\nsuffered through about 25 different versions of these tools :-).\n\nThe CHANGES file acknowledges those who have contributed patches.\n\nElias Ioup did the first implementation of LALR(1) parsing in PLY-1.x. \nAndrew Waters and Markus Schoepflin were instrumental in reporting bugs\nand testing a revised LALR(1) implementation for PLY-2.0.\n\nSpecial Note for PLY-3.0\n========================\nPLY-3.0 the first PLY release to support Python 3. However, backwards\ncompatibility with Python 2.6 is still preserved. PLY provides dual\nPython 2/3 compatibility by restricting its implementation to a common\nsubset of basic language features. You should not convert PLY using\n2to3--it is not necessary and may in fact break the implementation.\n\nExample\n=======\n\nHere is a simple example showing a PLY implementation of a calculator\nwith variables.\n\n # -----------------------------------------------------------------------------\n # calc.py\n #\n # A simple calculator with variables.\n # -----------------------------------------------------------------------------\n\n tokens = (\n 'NAME','NUMBER',\n 'PLUS','MINUS','TIMES','DIVIDE','EQUALS',\n 'LPAREN','RPAREN',\n )\n\n # Tokens\n\n t_PLUS = r'\\+'\n t_MINUS = r'-'\n t_TIMES = r'\\*'\n t_DIVIDE = r'/'\n t_EQUALS = r'='\n t_LPAREN = r'\\('\n t_RPAREN = r'\\)'\n t_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*'\n\n def t_NUMBER(t):\n r'\\d+'\n t.value = int(t.value)\n return t\n\n # Ignored characters\n t_ignore = \" \\t\"\n\n def t_newline(t):\n r'\\n+'\n t.lexer.lineno += t.value.count(\"\\n\")\n\n def t_error(t):\n print(\"Illegal character '%s'\" % t.value[0])\n t.lexer.skip(1)\n\n # Build the lexer\n import ply.lex as lex\n lex.lex()\n\n # Precedence rules for the arithmetic operators\n precedence = (\n ('left','PLUS','MINUS'),\n ('left','TIMES','DIVIDE'),\n ('right','UMINUS'),\n )\n\n # dictionary of names (for storing variables)\n names = { }\n\n def p_statement_assign(p):\n 'statement : NAME EQUALS expression'\n names[p[1]] = p[3]\n\n def p_statement_expr(p):\n 'statement : expression'\n print(p[1])\n\n def p_expression_binop(p):\n '''expression : expression PLUS expression\n | expression MINUS expression\n | expression TIMES expression\n | expression DIVIDE expression'''\n if p[2] == '+' : p[0] = p[1] + p[3]\n elif p[2] == '-': p[0] = p[1] - p[3]\n elif p[2] == '*': p[0] = p[1] * p[3]\n elif p[2] == '/': p[0] = p[1] / p[3]\n\n def p_expression_uminus(p):\n 'expression : MINUS expression %prec UMINUS'\n p[0] = -p[2]\n\n def p_expression_group(p):\n 'expression : LPAREN expression RPAREN'\n p[0] = p[2]\n\n def p_expression_number(p):\n 'expression : NUMBER'\n p[0] = p[1]\n\n def p_expression_name(p):\n 'expression : NAME'\n try:\n p[0] = names[p[1]]\n except LookupError:\n print(\"Undefined name '%s'\" % p[1])\n p[0] = 0\n\n def p_error(p):\n print(\"Syntax error at '%s'\" % p.value)\n\n import ply.yacc as yacc\n yacc.yacc()\n\n while True:\n try:\n s = raw_input('calc > ') # use input() on Python 3\n except EOFError:\n break\n yacc.parse(s)\n\n\nBug Reports and Patches\n=======================\nMy goal with PLY is to simply have a decent lex/yacc implementation\nfor Python. As a general rule, I don't spend huge amounts of time\nworking on it unless I receive very specific bug reports and/or\npatches to fix problems. At this time, PLY is mature software and new\nfeatures are no longer being added. If you think you have found a\nbug, please visit the PLY Github page at https://github.com/dabeaz/ply\nto report an issue.\n \n-- Dave\n\n\n\n\n\n\n\n\n\n" }, { "path": "xonsh/ply/__init__.py", "content": "" }, { "path": "xonsh/ply/doc/internal.html", "content": "\n\nPLY Internals\n\n\n\n

PLY Internals

\n \n\nDavid M. Beazley
\ndave@dabeaz.com
\n\n\n

\nPLY Version: 3.11\n

\n\n\n

\n\n
\n\n\n\n

1. Introduction

\n\n\nThis document describes classes and functions that make up the internal\noperation of PLY. Using this programming interface, it is possible to\nmanually build an parser using a different interface specification\nthan what PLY normally uses. For example, you could build a gramar\nfrom information parsed in a completely different input format. Some of\nthese objects may be useful for building more advanced parsing engines\nsuch as GLR.\n\n

\nIt should be stressed that using PLY at this level is not for the\nfaint of heart. Generally, it's assumed that you know a bit of\nthe underlying compiler theory and how an LR parser is put together.\n\n

2. Grammar Class

\n\n\nThe file ply.yacc defines a class Grammar that \nis used to hold and manipulate information about a grammar\nspecification. It encapsulates the same basic information\nabout a grammar that is put into a YACC file including \nthe list of tokens, precedence rules, and grammar rules. \nVarious operations are provided to perform different validations\non the grammar. In addition, there are operations to compute\nthe first and follow sets that are needed by the various table\ngeneration algorithms.\n\n

\nGrammar(terminals)\n\n

\nCreates a new grammar object. terminals is a list of strings\nspecifying the terminals for the grammar. An instance g of\nGrammar has the following methods:\n
\n\n

\ng.set_precedence(term,assoc,level)\n

\nSets the precedence level and associativity for a given terminal term. \nassoc is one of 'right',\n'left', or 'nonassoc' and level is a positive integer. The higher\nthe value of level, the higher the precedence. Here is an example of typical\nprecedence settings:\n\n
\ng.set_precedence('PLUS',  'left',1)\ng.set_precedence('MINUS', 'left',1)\ng.set_precedence('TIMES', 'left',2)\ng.set_precedence('DIVIDE','left',2)\ng.set_precedence('UMINUS','left',3)\n
\n\nThis method must be called prior to adding any productions to the\ngrammar with g.add_production(). The precedence of individual grammar\nrules is determined by the precedence of the right-most terminal.\n\n
\n

\ng.add_production(name,syms,func=None,file='',line=0)\n

\nAdds a new grammar rule. name is the name of the rule,\nsyms is a list of symbols making up the right hand\nside of the rule, func is the function to call when\nreducing the rule. file and line specify\nthe filename and line number of the rule and are used for\ngenerating error messages. \n\n

\nThe list of symbols in syms may include character\nliterals and %prec specifiers. Here are some\nexamples:\n\n

\ng.add_production('expr',['expr','PLUS','term'],func,file,line)\ng.add_production('expr',['expr','\"+\"','term'],func,file,line)\ng.add_production('expr',['MINUS','expr','%prec','UMINUS'],func,file,line)\n
\n\n

\nIf any kind of error is detected, a GrammarError exception\nis raised with a message indicating the reason for the failure.\n

\n\n

\ng.set_start(start=None)\n

\nSets the starting rule for the grammar. start is a string\nspecifying the name of the start rule. If start is omitted,\nthe first grammar rule added with add_production() is taken to be\nthe starting rule. This method must always be called after all\nproductions have been added.\n
\n\n

\ng.find_unreachable()\n

\nDiagnostic function. Returns a list of all unreachable non-terminals\ndefined in the grammar. This is used to identify inactive parts of\nthe grammar specification.\n
\n\n

\ng.infinite_cycle()\n

\nDiagnostic function. Returns a list of all non-terminals in the\ngrammar that result in an infinite cycle. This condition occurs if\nthere is no way for a grammar rule to expand to a string containing\nonly terminal symbols.\n
\n\n

\ng.undefined_symbols()\n

\nDiagnostic function. Returns a list of tuples (name, prod)\ncorresponding to undefined symbols in the grammar. name is the\nname of the undefined symbol and prod is an instance of \nProduction which has information about the production rule\nwhere the undefined symbol was used.\n
\n\n

\ng.unused_terminals()\n

\nDiagnostic function. Returns a list of terminals that were defined,\nbut never used in the grammar.\n
\n\n

\ng.unused_rules()\n

\nDiagnostic function. Returns a list of Production instances\ncorresponding to production rules that were defined in the grammar,\nbut never used anywhere. This is slightly different\nthan find_unreachable().\n
\n\n

\ng.unused_precedence()\n

\nDiagnostic function. Returns a list of tuples (term, assoc) \ncorresponding to precedence rules that were set, but never used the\ngrammar. term is the terminal name and assoc is the\nprecedence associativity (e.g., 'left', 'right', \nor 'nonassoc'.\n
\n\n

\ng.compute_first()\n

\nCompute all of the first sets for all symbols in the grammar. Returns a dictionary\nmapping symbol names to a list of all first symbols.\n
\n\n

\ng.compute_follow()\n

\nCompute all of the follow sets for all non-terminals in the grammar.\nThe follow set is the set of all possible symbols that might follow a\ngiven non-terminal. Returns a dictionary mapping non-terminal names\nto a list of symbols.\n
\n\n

\ng.build_lritems()\n

\nCalculates all of the LR items for all productions in the grammar. This\nstep is required before using the grammar for any kind of table generation.\nSee the section on LR items below.\n
\n\n

\nThe following attributes are set by the above methods and may be useful\nin code that works with the grammar. All of these attributes should be\nassumed to be read-only. Changing their values directly will likely \nbreak the grammar.\n\n

\ng.Productions\n

\nA list of all productions added. The first entry is reserved for\na production representing the starting rule. The objects in this list\nare instances of the Production class, described shortly.\n
\n\n

\ng.Prodnames\n

\nA dictionary mapping the names of nonterminals to a list of all\nproductions of that nonterminal.\n
\n\n

\ng.Terminals\n

\nA dictionary mapping the names of terminals to a list of the\nproduction numbers where they are used.\n
\n\n

\ng.Nonterminals\n

\nA dictionary mapping the names of nonterminals to a list of the\nproduction numbers where they are used.\n
\n\n

\ng.First\n

\nA dictionary representing the first sets for all grammar symbols. This is\ncomputed and returned by the compute_first() method.\n
\n\n

\ng.Follow\n

\nA dictionary representing the follow sets for all grammar rules. This is\ncomputed and returned by the compute_follow() method.\n
\n\n

\ng.Start\n

\nStarting symbol for the grammar. Set by the set_start() method.\n
\n\nFor the purposes of debugging, a Grammar object supports the __len__() and\n__getitem__() special methods. Accessing g[n] returns the nth production\nfrom the grammar.\n\n\n

3. Productions

\n\n\nGrammar objects store grammar rules as instances of a Production class. This\nclass has no public constructor--you should only create productions by calling Grammar.add_production().\nThe following attributes are available on a Production instance p.\n\n

\np.name\n

\nThe name of the production. For a grammar rule such as A : B C D, this is 'A'.\n
\n\n

\np.prod\n

\nA tuple of symbols making up the right-hand side of the production. For a grammar rule such as A : B C D, this is ('B','C','D').\n
\n\n

\np.number\n

\nProduction number. An integer containing the index of the production in the grammar's Productions list.\n
\n\n

\np.func\n

\nThe name of the reduction function associated with the production.\nThis is the function that will execute when reducing the entire\ngrammar rule during parsing.\n
\n\n

\np.callable\n

\nThe callable object associated with the name in p.func. This is None\nunless the production has been bound using bind().\n
\n\n

\np.file\n

\nFilename associated with the production. Typically this is the file where the production was defined. Used for error messages.\n
\n\n

\np.lineno\n

\nLine number associated with the production. Typically this is the line number in p.file where the production was defined. Used for error messages.\n
\n\n

\np.prec\n

\nPrecedence and associativity associated with the production. This is a tuple (assoc,level) where\nassoc is one of 'left','right', or 'nonassoc' and level is\nan integer. This value is determined by the precedence of the right-most terminal symbol in the production\nor by use of the %prec specifier when adding the production.\n
\n\n

\np.usyms\n

\nA list of all unique symbols found in the production.\n
\n\n

\np.lr_items\n

\nA list of all LR items for this production. This attribute only has a meaningful value if the\nGrammar.build_lritems() method has been called. The items in this list are \ninstances of LRItem described below.\n
\n\n

\np.lr_next\n

\nThe head of a linked-list representation of the LR items in p.lr_items. \nThis attribute only has a meaningful value if the Grammar.build_lritems() \nmethod has been called. Each LRItem instance has a lr_next attribute\nto move to the next item. The list is terminated by None.\n
\n\n

\np.bind(dict)\n

\nBinds the production function name in p.func to a callable object in \ndict. This operation is typically carried out in the last step\nprior to running the parsing engine and is needed since parsing tables are typically\nread from files which only include the function names, not the functions themselves.\n
\n\n

\nProduction objects support\nthe __len__(), __getitem__(), and __str__()\nspecial methods.\nlen(p) returns the number of symbols in p.prod\nand p[n] is the same as p.prod[n]. \n\n

4. LRItems

\n\n\nThe construction of parsing tables in an LR-based parser generator is primarily\ndone over a set of \"LR Items\". An LR item represents a stage of parsing one\nof the grammar rules. To compute the LR items, it is first necessary to\ncall Grammar.build_lritems(). Once this step, all of the productions\nin the grammar will have their LR items attached to them.\n\n

\nHere is an interactive example that shows what LR items look like if you\ninteractively experiment. In this example, g is a Grammar \nobject.\n\n

\n
\n>>> g.build_lritems()\n>>> p = g[1]\n>>> p\nProduction(statement -> ID = expr)\n>>>\n
\n
\n\nIn the above code, p represents the first grammar rule. In\nthis case, a rule 'statement -> ID = expr'.\n\n

\nNow, let's look at the LR items for p.\n\n

\n
\n>>> p.lr_items\n[LRItem(statement -> . ID = expr), \n LRItem(statement -> ID . = expr), \n LRItem(statement -> ID = . expr), \n LRItem(statement -> ID = expr .)]\n>>>\n
\n
\n\nIn each LR item, the dot (.) represents a specific stage of parsing. In each LR item, the dot\nis advanced by one symbol. It is only when the dot reaches the very end that a production\nis successfully parsed.\n\n

\nAn instance lr of LRItem has the following\nattributes that hold information related to that specific stage of\nparsing.\n\n

\nlr.name\n

\nThe name of the grammar rule. For example, 'statement' in the above example.\n
\n\n

\nlr.prod\n

\nA tuple of symbols representing the right-hand side of the production, including the\nspecial '.' character. For example, ('ID','.','=','expr').\n
\n\n

\nlr.number\n

\nAn integer representing the production number in the grammar.\n
\n\n

\nlr.usyms\n

\nA set of unique symbols in the production. Inherited from the original Production instance.\n
\n\n

\nlr.lr_index\n

\nAn integer representing the position of the dot (.). You should never use lr.prod.index()\nto search for it--the result will be wrong if the grammar happens to also use (.) as a character\nliteral.\n
\n\n

\nlr.lr_after\n

\nA list of all productions that can legally appear immediately to the right of the\ndot (.). This list contains Production instances. This attribute\nrepresents all of the possible branches a parse can take from the current position.\nFor example, suppose that lr represents a stage immediately before\nan expression like this:\n\n
\n>>> lr\nLRItem(statement -> ID = . expr)\n>>>\n
\n\nThen, the value of lr.lr_after might look like this, showing all productions that\ncan legally appear next:\n\n
\n>>> lr.lr_after\n[Production(expr -> expr PLUS expr), \n Production(expr -> expr MINUS expr), \n Production(expr -> expr TIMES expr), \n Production(expr -> expr DIVIDE expr), \n Production(expr -> MINUS expr), \n Production(expr -> LPAREN expr RPAREN), \n Production(expr -> NUMBER), \n Production(expr -> ID)]\n>>>\n
\n\n
\n\n

\nlr.lr_before\n

\nThe grammar symbol that appears immediately before the dot (.) or None if\nat the beginning of the parse. \n
\n\n

\nlr.lr_next\n

\nA link to the next LR item, representing the next stage of the parse. None if lr\nis the last LR item.\n
\n\nLRItem instances also support the __len__() and __getitem__() special methods.\nlen(lr) returns the number of items in lr.prod including the dot (.). lr[n]\nreturns lr.prod[n].\n\n

\nIt goes without saying that all of the attributes associated with LR\nitems should be assumed to be read-only. Modifications will very\nlikely create a small black-hole that will consume you and your code.\n\n

5. LRTable

\n\n\nThe LRTable class is used to represent LR parsing table data. This\nminimally includes the production list, action table, and goto table. \n\n

\nLRTable()\n

\nCreate an empty LRTable object. This object contains only the information needed to\nrun an LR parser. \n
\n\nAn instance lrtab of LRTable has the following methods:\n\n

\nlrtab.read_table(module)\n

\nPopulates the LR table with information from the module specified in module.\nmodule is either a module object already loaded with import or\nthe name of a Python module. If it's a string containing a module name, it is\nloaded and parsing data is extracted. Returns the signature value that was used\nwhen initially writing the tables. Raises a VersionError exception if\nthe module was created using an incompatible version of PLY.\n
\n\n

\nlrtab.bind_callables(dict)\n

\nThis binds all of the function names used in productions to callable objects\nfound in the dictionary dict. During table generation and when reading\nLR tables from files, PLY only uses the names of action functions such as 'p_expr',\n'p_statement', etc. In order to actually run the parser, these names\nhave to be bound to callable objects. This method is always called prior to\nrunning a parser.\n
\n\nAfter lrtab has been populated, the following attributes are defined.\n\n

\nlrtab.lr_method\n

\nThe LR parsing method used (e.g., 'LALR')\n
\n\n\n

\nlrtab.lr_productions\n

\nThe production list. If the parsing tables have been newly\nconstructed, this will be a list of Production instances. If\nthe parsing tables have been read from a file, it's a list\nof MiniProduction instances. This, together\nwith lr_action and lr_goto contain all of the\ninformation needed by the LR parsing engine.\n
\n\n

\nlrtab.lr_action\n

\nThe LR action dictionary that implements the underlying state machine.\nThe keys of this dictionary are the LR states.\n
\n\n

\nlrtab.lr_goto\n

\nThe LR goto table that contains information about grammar rule reductions.\n
\n\n\n

6. LRGeneratedTable

\n\n\nThe LRGeneratedTable class represents constructed LR parsing tables on a\ngrammar. It is a subclass of LRTable.\n\n

\nLRGeneratedTable(grammar, method='LALR',log=None)\n

\nCreate the LR parsing tables on a grammar. grammar is an instance of Grammar,\nmethod is a string with the parsing method ('SLR' or 'LALR'), and\nlog is a logger object used to write debugging information. The debugging information\nwritten to log is the same as what appears in the parser.out file created\nby yacc. By supplying a custom logger with a different message format, it is possible to get\nmore information (e.g., the line number in yacc.py used for issuing each line of\noutput in the log). The result is an instance of LRGeneratedTable.\n
\n\n

\nAn instance lr of LRGeneratedTable has the following attributes.\n\n

\nlr.grammar\n

\nA link to the Grammar object used to construct the parsing tables.\n
\n\n

\nlr.lr_method\n

\nThe LR parsing method used (e.g., 'LALR')\n
\n\n\n

\nlr.lr_productions\n

\nA reference to grammar.Productions. This, together with lr_action and lr_goto\ncontain all of the information needed by the LR parsing engine.\n
\n\n

\nlr.lr_action\n

\nThe LR action dictionary that implements the underlying state machine. The keys of this dictionary are\nthe LR states.\n
\n\n

\nlr.lr_goto\n

\nThe LR goto table that contains information about grammar rule reductions.\n
\n\n

\nlr.sr_conflicts\n

\nA list of tuples (state,token,resolution) identifying all shift/reduce conflicts. state is the LR state\nnumber where the conflict occurred, token is the token causing the conflict, and resolution is\na string describing the resolution taken. resolution is either 'shift' or 'reduce'.\n
\n\n

\nlr.rr_conflicts\n

\nA list of tuples (state,rule,rejected) identifying all reduce/reduce conflicts. state is the\nLR state number where the conflict occurred, rule is the production rule that was selected\nand rejected is the production rule that was rejected. Both rule and rejected are\ninstances of Production. They can be inspected to provide the user with more information.\n
\n\n

\nThere are two public methods of LRGeneratedTable.\n\n

\nlr.write_table(modulename,outputdir=\"\",signature=\"\")\n

\nWrites the LR parsing table information to a Python module. modulename is a string \nspecifying the name of a module such as \"parsetab\". outputdir is the name of a \ndirectory where the module should be created. signature is a string representing a\ngrammar signature that's written into the output file. This can be used to detect when\nthe data stored in a module file is out-of-sync with the the grammar specification (and that\nthe tables need to be regenerated). If modulename is a string \"parsetab\",\nthis function creates a file called parsetab.py. If the module name represents a\npackage such as \"foo.bar.parsetab\", then only the last component, \"parsetab\" is\nused.\n
\n\n\n

7. LRParser

\n\n\nThe LRParser class implements the low-level LR parsing engine.\n\n\n

\nLRParser(lrtab, error_func)\n

\nCreate an LRParser. lrtab is an instance of LRTable\ncontaining the LR production and state tables. error_func is the\nerror function to invoke in the event of a parsing error.\n
\n\nAn instance p of LRParser has the following methods:\n\n

\np.parse(input=None,lexer=None,debug=0,tracking=0,tokenfunc=None)\n

\nRun the parser. input is a string, which if supplied is fed into the\nlexer using its input() method. lexer is an instance of the\nLexer class to use for tokenizing. If not supplied, the last lexer\ncreated with the lex module is used. debug is a boolean flag\nthat enables debugging. tracking is a boolean flag that tells the\nparser to perform additional line number tracking. tokenfunc is a callable\nfunction that returns the next token. If supplied, the parser will use it to get\nall tokens.\n
\n\n

\np.restart()\n

\nResets the parser state for a parse already in progress.\n
\n\n

8. ParserReflect

\n\n\n

\nThe ParserReflect class is used to collect parser specification data\nfrom a Python module or object. This class is what collects all of the\np_rule() functions in a PLY file, performs basic error checking,\nand collects all of the needed information to build a grammar. Most of the\nhigh-level PLY interface as used by the yacc() function is actually\nimplemented by this class.\n\n

\nParserReflect(pdict, log=None)\n

\nCreates a ParserReflect instance. pdict is a dictionary\ncontaining parser specification data. This dictionary typically corresponds\nto the module or class dictionary of code that implements a PLY parser.\nlog is a logger instance that will be used to report error\nmessages.\n
\n\nAn instance p of ParserReflect has the following methods:\n\n

\np.get_all()\n

\nCollect and store all required parsing information.\n
\n\n

\np.validate_all()\n

\nValidate all of the collected parsing information. This is a seprate step\nfrom p.get_all() as a performance optimization. In order to\nincrease parser start-up time, a parser can elect to only validate the\nparsing data when regenerating the parsing tables. The validation\nstep tries to collect as much information as possible rather than\nraising an exception at the first sign of trouble. The attribute\np.error is set if there are any validation errors. The\nvalue of this attribute is also returned.\n
\n\n

\np.signature()\n

\nCompute a signature representing the contents of the collected parsing\ndata. The signature value should change if anything in the parser\nspecification has changed in a way that would justify parser table\nregeneration. This method can be called after p.get_all(),\nbut before p.validate_all().\n
\n\nThe following attributes are set in the process of collecting data:\n\n

\np.start\n

\nThe grammar start symbol, if any. Taken from pdict['start'].\n
\n\n

\np.error_func\n

\nThe error handling function or None. Taken from pdict['p_error'].\n
\n\n

\np.tokens\n

\nThe token list. Taken from pdict['tokens'].\n
\n\n

\np.prec\n

\nThe precedence specifier. Taken from pdict['precedence'].\n
\n\n

\np.preclist\n

\nA parsed version of the precedence specified. A list of tuples of the form\n(token,assoc,level) where token is the terminal symbol,\nassoc is the associativity (e.g., 'left') and level\nis a numeric precedence level.\n
\n\n

\np.grammar\n

\nA list of tuples (name, rules) representing the grammar rules. name is the\nname of a Python function or method in pdict that starts with \"p_\".\nrules is a list of tuples (filename,line,prodname,syms) representing\nthe grammar rules found in the documentation string of that function. filename and line contain location\ninformation that can be used for debugging. prodname is the name of the \nproduction. syms is the right-hand side of the production. If you have a\nfunction like this\n\n
\ndef p_expr(p):\n    '''expr : expr PLUS expr\n            | expr MINUS expr\n            | expr TIMES expr\n            | expr DIVIDE expr'''\n
\n\nthen the corresponding entry in p.grammar might look like this:\n\n
\n('p_expr', [ ('calc.py',10,'expr', ['expr','PLUS','expr']),\n             ('calc.py',11,'expr', ['expr','MINUS','expr']),\n             ('calc.py',12,'expr', ['expr','TIMES','expr']),\n             ('calc.py',13,'expr', ['expr','DIVIDE','expr'])\n           ])\n
\n
\n\n

\np.pfuncs\n

\nA sorted list of tuples (line, file, name, doc) representing all of\nthe p_ functions found. line and file give location\ninformation. name is the name of the function. doc is the\ndocumentation string. This list is sorted in ascending order by line number.\n
\n\n

\np.files\n

\nA dictionary holding all of the source filenames that were encountered\nwhile collecting parser information. Only the keys of this dictionary have\nany meaning.\n
\n\n

\np.error\n

\nAn attribute that indicates whether or not any critical errors \noccurred in validation. If this is set, it means that that some kind\nof problem was detected and that no further processing should be\nperformed.\n
\n\n\n

9. High-level operation

\n\n\nUsing all of the above classes requires some attention to detail. The yacc()\nfunction carries out a very specific sequence of operations to create a grammar.\nThis same sequence should be emulated if you build an alternative PLY interface.\n\n
    \n
  1. A ParserReflect object is created and raw grammar specification data is\ncollected.\n
  2. A Grammar object is created and populated with information\nfrom the specification data.\n
  3. A LRGenerator object is created to run the LALR algorithm over\nthe Grammar object.\n
  4. Productions in the LRGenerator and bound to callables using the bind_callables()\nmethod.\n
  5. A LRParser object is created from from the information in the\nLRGenerator object.\n
\n\n\n\n\n\n\n\n\n\n\n" }, { "path": "xonsh/ply/doc/makedoc.py", "content": "#!/usr/local/bin/python\n\n###############################################################################\n# Takes a chapter as input and adds internal links and numbering to all\n# of the H1, H2, H3, H4 and H5 sections.\n#\n# Every heading HTML tag (H1, H2 etc) is given an autogenerated name to link\n# to. However, if the name is not an autogenerated name from a previous run,\n# it will be kept. If it is autogenerated, it might change on subsequent runs\n# of this program. Thus if you want to create links to one of the headings,\n# then change the heading link name to something that does not look like an\n# autogenerated link name.\n###############################################################################\n\nimport sys\nimport re\nimport string\n\n###############################################################################\n# Functions\n###############################################################################\n\n# Regexs for \nalink = re.compile(r\"\", re.IGNORECASE)\nheading = re.compile(r\"(_nn\\d)\", re.IGNORECASE)\n\ndef getheadingname(m):\n autogeneratedheading = True\n if m.group(1) != None:\n amatch = alink.match(m.group(1))\n if amatch:\n # A non-autogenerated heading - keep it\n headingname = amatch.group(1)\n autogeneratedheading = heading.match(headingname)\n if autogeneratedheading:\n # The heading name was either non-existent or autogenerated,\n # We can create a new heading / change the existing heading\n headingname = \"%s_nn%d\" % (filenamebase, nameindex)\n return headingname\n\n###############################################################################\n# Main program\n###############################################################################\n\nif len(sys.argv) != 2:\n print \"usage: makedoc.py filename\"\n sys.exit(1)\n\nfilename = sys.argv[1]\nfilenamebase = string.split(filename,\".\")[0]\n\nsection = 0\nsubsection = 0\nsubsubsection = 0\nsubsubsubsection = 0\nnameindex = 0\n\nname = \"\"\n\n# Regexs for

,...

sections\n\nh1 = re.compile(r\".*?

()*[\\d\\.\\s]*(.*?)

\", re.IGNORECASE)\nh2 = re.compile(r\".*?

()*[\\d\\.\\s]*(.*?)

\", re.IGNORECASE)\nh3 = re.compile(r\".*?

()*[\\d\\.\\s]*(.*?)

\", re.IGNORECASE)\nh4 = re.compile(r\".*?

()*[\\d\\.\\s]*(.*?)

\", re.IGNORECASE)\nh5 = re.compile(r\".*?
()*[\\d\\.\\s]*(.*?)
\", re.IGNORECASE)\n\n# Make backup\nwith open(filename) as src, open(filename+\".bak\",\"w\") as dst:\n dst.write(src.read())\n\nlines = data.splitlines()\nresult = [ ] # This is the result of postprocessing the file\nindex = \"\\n
\\n\" # index contains the index for adding at the top of the file. Also printed to stdout.\n\nskip = 0\nskipspace = 0\n\nfor s in lines:\n if s == \"\":\n if not skip:\n result.append(\"@INDEX@\")\n skip = 1\n else:\n skip = 0\n continue\n if skip:\n continue\n\n if not s and skipspace:\n continue\n\n if skipspace:\n result.append(\"\")\n result.append(\"\")\n skipspace = 0\n \n m = h2.match(s)\n if m:\n prevheadingtext = m.group(2)\n nameindex += 1\n section += 1\n headingname = getheadingname(m)\n result.append(\"\"\"

%d. %s

\"\"\" % (headingname,section, prevheadingtext))\n\n if subsubsubsection:\n index += \"\\n\"\n if subsubsection:\n index += \"\\n\"\n if subsection:\n index += \"\\n\"\n if section == 1:\n index += \"
    \\n\"\n\n index += \"\"\"
  • %s\\n\"\"\" % (headingname,prevheadingtext)\n subsection = 0\n subsubsection = 0\n subsubsubsection = 0\n skipspace = 1 \n continue\n m = h3.match(s)\n if m:\n prevheadingtext = m.group(2)\n nameindex += 1\n subsection += 1\n headingname = getheadingname(m)\n result.append(\"\"\"

    %d.%d %s

    \"\"\" % (headingname,section, subsection, prevheadingtext))\n\n if subsubsubsection:\n index += \"
\\n\"\n if subsubsection:\n index += \"\\n\"\n if subsection == 1:\n index += \"
    \\n\"\n\n index += \"\"\"
  • %s\\n\"\"\" % (headingname,prevheadingtext)\n subsubsection = 0\n skipspace = 1 \n continue\n m = h4.match(s)\n if m:\n prevheadingtext = m.group(2)\n nameindex += 1\n subsubsection += 1\n subsubsubsection = 0\n headingname = getheadingname(m)\n result.append(\"\"\"

    %d.%d.%d %s

    \"\"\" % (headingname,section, subsection, subsubsection, prevheadingtext))\n\n if subsubsubsection:\n index += \"
\\n\"\n if subsubsection == 1:\n index += \"
    \\n\"\n\n index += \"\"\"
  • %s\\n\"\"\" % (headingname,prevheadingtext)\n skipspace = 1 \n continue\n m = h5.match(s)\n if m:\n prevheadingtext = m.group(2)\n nameindex += 1\n subsubsubsection += 1\n headingname = getheadingname(m)\n result.append(\"\"\"
    %d.%d.%d.%d %s
    \"\"\" % (headingname,section, subsection, subsubsection, subsubsubsection, prevheadingtext))\n\n if subsubsubsection == 1:\n index += \"
      \\n\"\n\n index += \"\"\"
    • %s\\n\"\"\" % (headingname,prevheadingtext)\n skipspace = 1\n continue\n \n result.append(s)\n\nif subsubsubsection:\n index += \"
    \\n\"\n\nif subsubsection:\n index += \"
\\n\"\n\nif subsection:\n index += \"\\n\"\n\nif section:\n index += \"\\n\"\n\nindex += \"
\\n\\n\"\n\ndata = \"\\n\".join(result)\n\ndata = data.replace(\"@INDEX@\",index) + \"\\n\"\n\n# Write the file back out\nwith open(filename,\"w\") as f:\n f.write(data)\n" }, { "path": "xonsh/ply/doc/ply.html", "content": "\n\nPLY (Python Lex-Yacc)\n\n\n\n

PLY (Python Lex-Yacc)

\n \n\nDavid M. Beazley
\ndave@dabeaz.com
\n\n\n

\nPLY Version: 3.11\n

\n\n\n

\n\n
\n\n\n\n\n\n\n\n

1. Preface and Requirements

\n\n\n

\nThis document provides an overview of lexing and parsing with PLY.\nGiven the intrinsic complexity of parsing, I would strongly advise \nthat you read (or at least skim) this entire document before jumping\ninto a big development project with PLY. \n

\n\n

\nPLY-3.5 is compatible with both Python 2 and Python 3. If you are using\nPython 2, you have to use Python 2.6 or newer.\n

\n\n

2. Introduction

\n\n\nPLY is a pure-Python implementation of the popular compiler\nconstruction tools lex and yacc. The main goal of PLY is to stay\nfairly faithful to the way in which traditional lex/yacc tools work.\nThis includes supporting LALR(1) parsing as well as providing\nextensive input validation, error reporting, and diagnostics. Thus,\nif you've used yacc in another programming language, it should be\nrelatively straightforward to use PLY. \n\n

\nEarly versions of PLY were developed to support an Introduction to\nCompilers Course I taught in 2001 at the University of Chicago. \nSince PLY was primarily developed as an instructional tool, you will\nfind it to be fairly picky about token and grammar rule\nspecification. In part, this\nadded formality is meant to catch common programming mistakes made by\nnovice users. However, advanced users will also find such features to\nbe useful when building complicated grammars for real programming\nlanguages. It should also be noted that PLY does not provide much in\nthe way of bells and whistles (e.g., automatic construction of\nabstract syntax trees, tree traversal, etc.). Nor would I consider it\nto be a parsing framework. Instead, you will find a bare-bones, yet\nfully capable lex/yacc implementation written entirely in Python.\n\n

\nThe rest of this document assumes that you are somewhat familiar with\nparsing theory, syntax directed translation, and the use of compiler\nconstruction tools such as lex and yacc in other programming\nlanguages. If you are unfamiliar with these topics, you will probably\nwant to consult an introductory text such as \"Compilers: Principles,\nTechniques, and Tools\", by Aho, Sethi, and Ullman. O'Reilly's \"Lex\nand Yacc\" by John Levine may also be handy. In fact, the O'Reilly book can be\nused as a reference for PLY as the concepts are virtually identical.\n\n

3. PLY Overview

\n\n\n

\nPLY consists of two separate modules; lex.py and\nyacc.py, both of which are found in a Python package\ncalled ply. The lex.py module is used to break input text into a\ncollection of tokens specified by a collection of regular expression\nrules. yacc.py is used to recognize language syntax that has\nbeen specified in the form of a context free grammar.\n

\n\n

\nThe two tools are meant to work together. Specifically,\nlex.py provides an external interface in the form of a\ntoken() function that returns the next valid token on the\ninput stream. yacc.py calls this repeatedly to retrieve\ntokens and invoke grammar rules. The output of yacc.py is\noften an Abstract Syntax Tree (AST). However, this is entirely up to\nthe user. If desired, yacc.py can also be used to implement\nsimple one-pass compilers. \n\n

\nLike its Unix counterpart, yacc.py provides most of the\nfeatures you expect including extensive error checking, grammar\nvalidation, support for empty productions, error tokens, and ambiguity\nresolution via precedence rules. In fact, almost everything that is possible in traditional yacc \nshould be supported in PLY.\n\n

\nThe primary difference between\nyacc.py and Unix yacc is that yacc.py \ndoesn't involve a separate code-generation process. \nInstead, PLY relies on reflection (introspection)\nto build its lexers and parsers. Unlike traditional lex/yacc which\nrequire a special input file that is converted into a separate source\nfile, the specifications given to PLY are valid Python\nprograms. This means that there are no extra source files nor is\nthere a special compiler construction step (e.g., running yacc to\ngenerate Python code for the compiler). Since the generation of the\nparsing tables is relatively expensive, PLY caches the results and\nsaves them to a file. If no changes are detected in the input source,\nthe tables are read from the cache. Otherwise, they are regenerated.\n\n

4. Lex

\n\n\nlex.py is used to tokenize an input string. For example, suppose\nyou're writing a programming language and a user supplied the following input string:\n\n
\n
\nx = 3 + 42 * (s - t)\n
\n
\n\nA tokenizer splits the string into individual tokens\n\n
\n
\n'x','=', '3', '+', '42', '*', '(', 's', '-', 't', ')'\n
\n
\n\nTokens are usually given names to indicate what they are. For example:\n\n
\n
\n'ID','EQUALS','NUMBER','PLUS','NUMBER','TIMES',\n'LPAREN','ID','MINUS','ID','RPAREN'\n
\n
\n\nMore specifically, the input is broken into pairs of token types and values. For example:\n\n
\n
\n('ID','x'), ('EQUALS','='), ('NUMBER','3'), \n('PLUS','+'), ('NUMBER','42), ('TIMES','*'),\n('LPAREN','('), ('ID','s'), ('MINUS','-'),\n('ID','t'), ('RPAREN',')'\n
\n
\n\nThe identification of tokens is typically done by writing a series of regular expression\nrules. The next section shows how this is done using lex.py.\n\n

4.1 Lex Example

\n\n\nThe following example shows how lex.py is used to write a simple tokenizer.\n\n
\n
\n# ------------------------------------------------------------\n# calclex.py\n#\n# tokenizer for a simple expression evaluator for\n# numbers and +,-,*,/\n# ------------------------------------------------------------\nimport ply.lex as lex\n\n# List of token names.   This is always required\ntokens = (\n   'NUMBER',\n   'PLUS',\n   'MINUS',\n   'TIMES',\n   'DIVIDE',\n   'LPAREN',\n   'RPAREN',\n)\n\n# Regular expression rules for simple tokens\nt_PLUS    = r'\\+'\nt_MINUS   = r'-'\nt_TIMES   = r'\\*'\nt_DIVIDE  = r'/'\nt_LPAREN  = r'\\('\nt_RPAREN  = r'\\)'\n\n# A regular expression rule with some action code\ndef t_NUMBER(t):\n    r'\\d+'\n    t.value = int(t.value)    \n    return t\n\n# Define a rule so we can track line numbers\ndef t_newline(t):\n    r'\\n+'\n    t.lexer.lineno += len(t.value)\n\n# A string containing ignored characters (spaces and tabs)\nt_ignore  = ' \\t'\n\n# Error handling rule\ndef t_error(t):\n    print(\"Illegal character '%s'\" % t.value[0])\n    t.lexer.skip(1)\n\n# Build the lexer\nlexer = lex.lex()\n\n
\n
\nTo use the lexer, you first need to feed it some input text using\nits input() method. After that, repeated calls\nto token() produce tokens. The following code shows how this\nworks:\n\n
\n
\n\n# Test it out\ndata = '''\n3 + 4 * 10\n  + -20 *2\n'''\n\n# Give the lexer some input\nlexer.input(data)\n\n# Tokenize\nwhile True:\n    tok = lexer.token()\n    if not tok: \n        break      # No more input\n    print(tok)\n
\n
\n\nWhen executed, the example will produce the following output:\n\n
\n
\n$ python example.py\nLexToken(NUMBER,3,2,1)\nLexToken(PLUS,'+',2,3)\nLexToken(NUMBER,4,2,5)\nLexToken(TIMES,'*',2,7)\nLexToken(NUMBER,10,2,10)\nLexToken(PLUS,'+',3,14)\nLexToken(MINUS,'-',3,16)\nLexToken(NUMBER,20,3,18)\nLexToken(TIMES,'*',3,20)\nLexToken(NUMBER,2,3,21)\n
\n
\n\nLexers also support the iteration protocol. So, you can write the above loop as follows:\n\n
\n
\nfor tok in lexer:\n    print(tok)\n
\n
\n\nThe tokens returned by lexer.token() are instances\nof LexToken. This object has\nattributes tok.type, tok.value,\ntok.lineno, and tok.lexpos. The following code shows an example of\naccessing these attributes:\n\n
\n
\n# Tokenize\nwhile True:\n    tok = lexer.token()\n    if not tok: \n        break      # No more input\n    print(tok.type, tok.value, tok.lineno, tok.lexpos)\n
\n
\n\nThe tok.type and tok.value attributes contain the\ntype and value of the token itself. \ntok.lineno and tok.lexpos contain information about\nthe location of the token. tok.lexpos is the index of the\ntoken relative to the start of the input text.\n\n

4.2 The tokens list

\n\n\n

\nAll lexers must provide a list tokens that defines all of the possible token\nnames that can be produced by the lexer. This list is always required\nand is used to perform a variety of validation checks. The tokens list is also used by the\nyacc.py module to identify terminals.\n

\n\n

\nIn the example, the following code specified the token names:\n\n

\n
\ntokens = (\n   'NUMBER',\n   'PLUS',\n   'MINUS',\n   'TIMES',\n   'DIVIDE',\n   'LPAREN',\n   'RPAREN',\n)\n
\n
\n\n

4.3 Specification of tokens

\n\n\nEach token is specified by writing a regular expression rule compatible with Python's re module. Each of these rules\nare defined by making declarations with a special prefix t_ to indicate that it\ndefines a token. For simple tokens, the regular expression can\nbe specified as strings such as this (note: Python raw strings are used since they are the\nmost convenient way to write regular expression strings):\n\n
\n
\nt_PLUS = r'\\+'\n
\n
\n\nIn this case, the name following the t_ must exactly match one of the\nnames supplied in tokens. If some kind of action needs to be performed,\na token rule can be specified as a function. For example, this rule matches numbers and\nconverts the string into a Python integer.\n\n
\n
\ndef t_NUMBER(t):\n    r'\\d+'\n    t.value = int(t.value)\n    return t\n
\n
\n\nWhen a function is used, the regular expression rule is specified in the function documentation string. \nThe function always takes a single argument which is an instance of \nLexToken. This object has attributes of t.type which is the token type (as a string),\nt.value which is the lexeme (the actual text matched), t.lineno which is the current line number, and t.lexpos which\nis the position of the token relative to the beginning of the input text.\nBy default, t.type is set to the name following the t_ prefix. The action\nfunction can modify the contents of the LexToken object as appropriate. However, \nwhen it is done, the resulting token should be returned. If no value is returned by the action\nfunction, the token is simply discarded and the next token read.\n\n

\nInternally, lex.py uses the re module to do its pattern matching. Patterns are compiled\nusing the re.VERBOSE flag which can be used to help readability. However, be aware that unescaped\nwhitespace is ignored and comments are allowed in this mode. If your pattern involves whitespace, make sure you\nuse \\s. If you need to match the # character, use [#].\n

\n\n

\nWhen building the master regular expression,\nrules are added in the following order:\n

\n\n

\n

    \n
  1. All tokens defined by functions are added in the same order as they appear in the lexer file.\n
  2. Tokens defined by strings are added next by sorting them in order of decreasing regular expression length (longer expressions\nare added first).\n
\n

\nWithout this ordering, it can be difficult to correctly match certain types of tokens. For example, if you \nwanted to have separate tokens for \"=\" and \"==\", you need to make sure that \"==\" is checked first. By sorting regular\nexpressions in order of decreasing length, this problem is solved for rules defined as strings. For functions,\nthe order can be explicitly controlled since rules appearing first are checked first.\n\n

\nTo handle reserved words, you should write a single rule to match an\nidentifier and do a special name lookup in a function like this:\n\n

\n
\nreserved = {\n   'if' : 'IF',\n   'then' : 'THEN',\n   'else' : 'ELSE',\n   'while' : 'WHILE',\n   ...\n}\n\ntokens = ['LPAREN','RPAREN',...,'ID'] + list(reserved.values())\n\ndef t_ID(t):\n    r'[a-zA-Z_][a-zA-Z_0-9]*'\n    t.type = reserved.get(t.value,'ID')    # Check for reserved words\n    return t\n
\n
\n\nThis approach greatly reduces the number of regular expression rules and is likely to make things a little faster.\n\n

\nNote: You should avoid writing individual rules for reserved words. For example, if you write rules like this,\n\n

\n
\nt_FOR   = r'for'\nt_PRINT = r'print'\n
\n
\n\nthose rules will be triggered for identifiers that include those words as a prefix such as \"forget\" or \"printed\". This is probably not\nwhat you want.\n\n

4.4 Token values

\n\n\nWhen tokens are returned by lex, they have a value that is stored in the value attribute. Normally, the value is the text\nthat was matched. However, the value can be assigned to any Python object. For instance, when lexing identifiers, you may\nwant to return both the identifier name and information from some sort of symbol table. To do this, you might write a rule like this:\n\n
\n
\ndef t_ID(t):\n    ...\n    # Look up symbol table information and return a tuple\n    t.value = (t.value, symbol_lookup(t.value))\n    ...\n    return t\n
\n
\n\nIt is important to note that storing data in other attribute names is not recommended. The yacc.py module only exposes the\ncontents of the value attribute. Thus, accessing other attributes may be unnecessarily awkward. If you\nneed to store multiple values on a token, assign a tuple, dictionary, or instance to value.\n\n

4.5 Discarded tokens

\n\n\nTo discard a token, such as a comment, simply define a token rule that returns no value. For example:\n\n
\n
\ndef t_COMMENT(t):\n    r'\\#.*'\n    pass\n    # No return value. Token discarded\n
\n
\n\nAlternatively, you can include the prefix \"ignore_\" in the token declaration to force a token to be ignored. For example:\n\n
\n
\nt_ignore_COMMENT = r'\\#.*'\n
\n
\n\nBe advised that if you are ignoring many different kinds of text, you may still want to use functions since these provide more precise\ncontrol over the order in which regular expressions are matched (i.e., functions are matched in order of specification whereas strings are\nsorted by regular expression length).\n\n

4.6 Line numbers and positional information

\n\n\n

By default, lex.py knows nothing about line numbers. This is because lex.py doesn't know anything\nabout what constitutes a \"line\" of input (e.g., the newline character or even if the input is textual data).\nTo update this information, you need to write a special rule. In the example, the t_newline() rule shows how to do this.\n\n

\n
\n# Define a rule so we can track line numbers\ndef t_newline(t):\n    r'\\n+'\n    t.lexer.lineno += len(t.value)\n
\n
\nWithin the rule, the lineno attribute of the underlying lexer t.lexer is updated.\nAfter the line number is updated, the token is simply discarded since nothing is returned.\n\n

\nlex.py does not perform any kind of automatic column tracking. However, it does record positional\ninformation related to each token in the lexpos attribute. Using this, it is usually possible to compute \ncolumn information as a separate step. For instance, just count backwards until you reach a newline.\n\n

\n
\n# Compute column.\n#     input is the input text string\n#     token is a token instance\ndef find_column(input, token):\n    line_start = input.rfind('\\n', 0, token.lexpos) + 1\n    return (token.lexpos - line_start) + 1\n
\n
\n\nSince column information is often only useful in the context of error handling, calculating the column\nposition can be performed when needed as opposed to doing it for each token.\n\n

4.7 Ignored characters

\n\n\n

\nThe special t_ignore rule is reserved by lex.py for characters\nthat should be completely ignored in the input stream. \nUsually this is used to skip over whitespace and other non-essential characters. \nAlthough it is possible to define a regular expression rule for whitespace in a manner\nsimilar to t_newline(), the use of t_ignore provides substantially better\nlexing performance because it is handled as a special case and is checked in a much\nmore efficient manner than the normal regular expression rules.\n

\n\n

\nThe characters given in t_ignore are not ignored when such characters are part of\nother regular expression patterns. For example, if you had a rule to capture quoted text,\nthat pattern can include the ignored characters (which will be captured in the normal way). The\nmain purpose of t_ignore is to ignore whitespace and other padding between the\ntokens that you actually want to parse.\n

\n\n

4.8 Literal characters

\n\n\n

\nLiteral characters can be specified by defining a variable literals in your lexing module. For example:\n\n

\n
\nliterals = [ '+','-','*','/' ]\n
\n
\n\nor alternatively\n\n
\n
\nliterals = \"+-*/\"\n
\n
\n\nA literal character is simply a single character that is returned \"as is\" when encountered by the lexer. Literals are checked\nafter all of the defined regular expression rules. Thus, if a rule starts with one of the literal characters, it will always \ntake precedence.\n\n

\nWhen a literal token is returned, both its type and value attributes are set to the character itself. For example, '+'.\n

\n\n

\nIt's possible to write token functions that perform additional actions\nwhen literals are matched. However, you'll need to set the token type\nappropriately. For example:\n

\n\n
\n
\nliterals = [ '{', '}' ]\n\ndef t_lbrace(t):\n    r'\\{'\n    t.type = '{'      # Set token type to the expected literal\n    return t\n\ndef t_rbrace(t):\n    r'\\}'\n    t.type = '}'      # Set token type to the expected literal\n    return t\n
\n
\n\n

4.9 Error handling

\n\n\n

\nThe t_error()\nfunction is used to handle lexing errors that occur when illegal\ncharacters are detected. In this case, the t.value attribute contains the\nrest of the input string that has not been tokenized. In the example, the error function\nwas defined as follows:\n\n

\n
\n# Error handling rule\ndef t_error(t):\n    print(\"Illegal character '%s'\" % t.value[0])\n    t.lexer.skip(1)\n
\n
\n\nIn this case, we simply print the offending character and skip ahead one character by calling t.lexer.skip(1).\n\n

4.10 EOF Handling

\n\n\n

\nThe t_eof() function is used to handle an end-of-file (EOF) condition in the input. As input, it\nreceives a token type 'eof' with the lineno and lexpos attributes set appropriately.\nThe main use of this function is provide more input to the lexer so that it can continue to parse. Here is an\nexample of how this works:\n

\n\n
\n
\n# EOF handling rule\ndef t_eof(t):\n    # Get more input (Example)\n    more = raw_input('... ')\n    if more:\n        self.lexer.input(more)\n        return self.lexer.token()\n    return None\n
\n
\n\n

\nThe EOF function should return the next available token (by calling self.lexer.token()) or None to\nindicate no more data. Be aware that setting more input with the self.lexer.input() method does\nNOT reset the lexer state or the lineno attribute used for position tracking. The lexpos \nattribute is reset so be aware of that if you're using it in error reporting.\n

\n\n

4.11 Building and using the lexer

\n\n\n

\nTo build the lexer, the function lex.lex() is used. For example:

\n\n
\n
\nlexer = lex.lex()\n
\n
\n\n

This function\nuses Python reflection (or introspection) to read the regular expression rules\nout of the calling context and build the lexer. Once the lexer has been built, two methods can\nbe used to control the lexer.\n

\n
    \n
  • lexer.input(data). Reset the lexer and store a new input string.\n
  • lexer.token(). Return the next token. Returns a special LexToken instance on success or\nNone if the end of the input text has been reached.\n
\n\n

4.12 The @TOKEN decorator

\n\n\nIn some applications, you may want to define build tokens from as a series of\nmore complex regular expression rules. For example:\n\n
\n
\ndigit            = r'([0-9])'\nnondigit         = r'([_A-Za-z])'\nidentifier       = r'(' + nondigit + r'(' + digit + r'|' + nondigit + r')*)'        \n\ndef t_ID(t):\n    # want docstring to be identifier above. ?????\n    ...\n
\n
\n\nIn this case, we want the regular expression rule for ID to be one of the variables above. However, there is no\nway to directly specify this using a normal documentation string. To solve this problem, you can use the @TOKEN\ndecorator. For example:\n\n
\n
\nfrom ply.lex import TOKEN\n\n@TOKEN(identifier)\ndef t_ID(t):\n    ...\n
\n
\n\n

\nThis will attach identifier to the docstring for t_ID() allowing lex.py to work normally. \n

\n\n

4.13 Optimized mode

\n\n\nFor improved performance, it may be desirable to use Python's\noptimized mode (e.g., running Python with the -O\noption). However, doing so causes Python to ignore documentation\nstrings. This presents special problems for lex.py. To\nhandle this case, you can create your lexer using\nthe optimize option as follows:\n\n
\n
\nlexer = lex.lex(optimize=1)\n
\n
\n\nNext, run Python in its normal operating mode. When you do\nthis, lex.py will write a file called lextab.py in\nthe same directory as the module containing the lexer specification.\nThis file contains all of the regular\nexpression rules and tables used during lexing. On subsequent\nexecutions,\nlextab.py will simply be imported to build the lexer. This\napproach substantially improves the startup time of the lexer and it\nworks in Python's optimized mode.\n\n

\nTo change the name of the lexer-generated module, use the lextab keyword argument. For example:\n

\n\n
\n
\nlexer = lex.lex(optimize=1,lextab=\"footab\")\n
\n
\n\nWhen running in optimized mode, it is important to note that lex disables most error checking. Thus, this is really only recommended\nif you're sure everything is working correctly and you're ready to start releasing production code.\n\n

4.14 Debugging

\n\n\nFor the purpose of debugging, you can run lex() in a debugging mode as follows:\n\n
\n
\nlexer = lex.lex(debug=1)\n
\n
\n\n

\nThis will produce various sorts of debugging information including all of the added rules,\nthe master regular expressions used by the lexer, and tokens generating during lexing.\n

\n\n

\nIn addition, lex.py comes with a simple main function which\nwill either tokenize input read from standard input or from a file specified\non the command line. To use it, simply put this in your lexer:\n

\n\n
\n
\nif __name__ == '__main__':\n     lex.runmain()\n
\n
\n\nPlease refer to the \"Debugging\" section near the end for some more advanced details \nof debugging.\n\n

4.15 Alternative specification of lexers

\n\n\nAs shown in the example, lexers are specified all within one Python module. If you want to\nput token rules in a different module from the one in which you invoke lex(), use the\nmodule keyword argument.\n\n

\nFor example, you might have a dedicated module that just contains\nthe token rules:\n\n

\n
\n# module: tokrules.py\n# This module just contains the lexing rules\n\n# List of token names.   This is always required\ntokens = (\n   'NUMBER',\n   'PLUS',\n   'MINUS',\n   'TIMES',\n   'DIVIDE',\n   'LPAREN',\n   'RPAREN',\n)\n\n# Regular expression rules for simple tokens\nt_PLUS    = r'\\+'\nt_MINUS   = r'-'\nt_TIMES   = r'\\*'\nt_DIVIDE  = r'/'\nt_LPAREN  = r'\\('\nt_RPAREN  = r'\\)'\n\n# A regular expression rule with some action code\ndef t_NUMBER(t):\n    r'\\d+'\n    t.value = int(t.value)    \n    return t\n\n# Define a rule so we can track line numbers\ndef t_newline(t):\n    r'\\n+'\n    t.lexer.lineno += len(t.value)\n\n# A string containing ignored characters (spaces and tabs)\nt_ignore  = ' \\t'\n\n# Error handling rule\ndef t_error(t):\n    print(\"Illegal character '%s'\" % t.value[0])\n    t.lexer.skip(1)\n
\n
\n\nNow, if you wanted to build a tokenizer from these rules from within a different module, you would do the following (shown for Python interactive mode):\n\n
\n
\n>>> import tokrules\n>>> lexer = lex.lex(module=tokrules)\n>>> lexer.input(\"3 + 4\")\n>>> lexer.token()\nLexToken(NUMBER,3,1,1,0)\n>>> lexer.token()\nLexToken(PLUS,'+',1,2)\n>>> lexer.token()\nLexToken(NUMBER,4,1,4)\n>>> lexer.token()\nNone\n>>>\n
\n
\n\nThe module option can also be used to define lexers from instances of a class. For example:\n\n
\n
\nimport ply.lex as lex\n\nclass MyLexer(object):\n    # List of token names.   This is always required\n    tokens = (\n       'NUMBER',\n       'PLUS',\n       'MINUS',\n       'TIMES',\n       'DIVIDE',\n       'LPAREN',\n       'RPAREN',\n    )\n\n    # Regular expression rules for simple tokens\n    t_PLUS    = r'\\+'\n    t_MINUS   = r'-'\n    t_TIMES   = r'\\*'\n    t_DIVIDE  = r'/'\n    t_LPAREN  = r'\\('\n    t_RPAREN  = r'\\)'\n\n    # A regular expression rule with some action code\n    # Note addition of self parameter since we're in a class\n    def t_NUMBER(self,t):\n        r'\\d+'\n        t.value = int(t.value)    \n        return t\n\n    # Define a rule so we can track line numbers\n    def t_newline(self,t):\n        r'\\n+'\n        t.lexer.lineno += len(t.value)\n\n    # A string containing ignored characters (spaces and tabs)\n    t_ignore  = ' \\t'\n\n    # Error handling rule\n    def t_error(self,t):\n        print(\"Illegal character '%s'\" % t.value[0])\n        t.lexer.skip(1)\n\n    # Build the lexer\n    def build(self,**kwargs):\n        self.lexer = lex.lex(module=self, **kwargs)\n    \n    # Test it output\n    def test(self,data):\n        self.lexer.input(data)\n        while True:\n             tok = self.lexer.token()\n             if not tok: \n                 break\n             print(tok)\n\n# Build the lexer and try it out\nm = MyLexer()\nm.build()           # Build the lexer\nm.test(\"3 + 4\")     # Test it\n
\n
\n\n\nWhen building a lexer from class, you should construct the lexer from\nan instance of the class, not the class object itself. This is because\nPLY only works properly if the lexer actions are defined by bound-methods.\n\n

\nWhen using the module option to lex(), PLY collects symbols\nfrom the underlying object using the dir() function. There is no\ndirect access to the __dict__ attribute of the object supplied as a \nmodule value.

\n\n

\nFinally, if you want to keep things nicely encapsulated, but don't want to use a \nfull-fledged class definition, lexers can be defined using closures. For example:\n\n

\n
\nimport ply.lex as lex\n\n# List of token names.   This is always required\ntokens = (\n  'NUMBER',\n  'PLUS',\n  'MINUS',\n  'TIMES',\n  'DIVIDE',\n  'LPAREN',\n  'RPAREN',\n)\n\ndef MyLexer():\n    # Regular expression rules for simple tokens\n    t_PLUS    = r'\\+'\n    t_MINUS   = r'-'\n    t_TIMES   = r'\\*'\n    t_DIVIDE  = r'/'\n    t_LPAREN  = r'\\('\n    t_RPAREN  = r'\\)'\n\n    # A regular expression rule with some action code\n    def t_NUMBER(t):\n        r'\\d+'\n        t.value = int(t.value)    \n        return t\n\n    # Define a rule so we can track line numbers\n    def t_newline(t):\n        r'\\n+'\n        t.lexer.lineno += len(t.value)\n\n    # A string containing ignored characters (spaces and tabs)\n    t_ignore  = ' \\t'\n\n    # Error handling rule\n    def t_error(t):\n        print(\"Illegal character '%s'\" % t.value[0])\n        t.lexer.skip(1)\n\n    # Build the lexer from my environment and return it    \n    return lex.lex()\n
\n
\n\n

\nImportant note: If you are defining a lexer using a class or closure, be aware that PLY still requires you to only\ndefine a single lexer per module (source file). There are extensive validation/error checking parts of the PLY that \nmay falsely report error messages if you don't follow this rule.\n

\n\n

4.16 Maintaining state

\n\n\nIn your lexer, you may want to maintain a variety of state\ninformation. This might include mode settings, symbol tables, and\nother details. As an example, suppose that you wanted to keep\ntrack of how many NUMBER tokens had been encountered. \n\n

\nOne way to do this is to keep a set of global variables in the module\nwhere you created the lexer. For example: \n\n

\n
\nnum_count = 0\ndef t_NUMBER(t):\n    r'\\d+'\n    global num_count\n    num_count += 1\n    t.value = int(t.value)    \n    return t\n
\n
\n\nIf you don't like the use of a global variable, another place to store\ninformation is inside the Lexer object created by lex().\nTo this, you can use the lexer attribute of tokens passed to\nthe various rules. For example:\n\n
\n
\ndef t_NUMBER(t):\n    r'\\d+'\n    t.lexer.num_count += 1     # Note use of lexer attribute\n    t.value = int(t.value)    \n    return t\n\nlexer = lex.lex()\nlexer.num_count = 0            # Set the initial count\n
\n
\n\nThis latter approach has the advantage of being simple and working \ncorrectly in applications where multiple instantiations of a given\nlexer exist in the same application. However, this might also feel\nlike a gross violation of encapsulation to OO purists. \nJust to put your mind at some ease, all\ninternal attributes of the lexer (with the exception of lineno) have names that are prefixed\nby lex (e.g., lexdata,lexpos, etc.). Thus,\nit is perfectly safe to store attributes in the lexer that\ndon't have names starting with that prefix or a name that conflicts with one of the\npredefined methods (e.g., input(), token(), etc.).\n\n

\nIf you don't like assigning values on the lexer object, you can define your lexer as a class as\nshown in the previous section:\n\n

\n
\nclass MyLexer:\n    ...\n    def t_NUMBER(self,t):\n        r'\\d+'\n        self.num_count += 1\n        t.value = int(t.value)    \n        return t\n\n    def build(self, **kwargs):\n        self.lexer = lex.lex(object=self,**kwargs)\n\n    def __init__(self):\n        self.num_count = 0\n
\n
\n\nThe class approach may be the easiest to manage if your application is\ngoing to be creating multiple instances of the same lexer and you need\nto manage a lot of state.\n\n

\nState can also be managed through closures. For example, in Python 3:\n\n

\n
\ndef MyLexer():\n    num_count = 0\n    ...\n    def t_NUMBER(t):\n        r'\\d+'\n        nonlocal num_count\n        num_count += 1\n        t.value = int(t.value)    \n        return t\n    ...\n
\n
\n\n

4.17 Lexer cloning

\n\n\n

\nIf necessary, a lexer object can be duplicated by invoking its clone() method. For example:\n\n

\n
\nlexer = lex.lex()\n...\nnewlexer = lexer.clone()\n
\n
\n\nWhen a lexer is cloned, the copy is exactly identical to the original lexer\nincluding any input text and internal state. However, the clone allows a\ndifferent set of input text to be supplied which may be processed separately.\nThis may be useful in situations when you are writing a parser/compiler that\ninvolves recursive or reentrant processing. For instance, if you\nneeded to scan ahead in the input for some reason, you could create a\nclone and use it to look ahead. Or, if you were implementing some kind of preprocessor,\ncloned lexers could be used to handle different input files.\n\n

\nCreating a clone is different than calling lex.lex() in that\nPLY doesn't regenerate any of the internal tables or regular expressions.\n\n

\nSpecial considerations need to be made when cloning lexers that also\nmaintain their own internal state using classes or closures. Namely,\nyou need to be aware that the newly created lexers will share all of\nthis state with the original lexer. For example, if you defined a\nlexer as a class and did this:\n\n

\n
\nm = MyLexer()\na = lex.lex(object=m)      # Create a lexer\n\nb = a.clone()              # Clone the lexer\n
\n
\n\nThen both a and b are going to be bound to the same\nobject m and any changes to m will be reflected in both lexers. It's\nimportant to emphasize that clone() is only meant to create a new lexer\nthat reuses the regular expressions and environment of another lexer. If you\nneed to make a totally new copy of a lexer, then call lex() again.\n\n

4.18 Internal lexer state

\n\n\nA Lexer object lexer has a number of internal attributes that may be useful in certain\nsituations. \n\n

\nlexer.lexpos\n

\nThis attribute is an integer that contains the current position within the input text. If you modify\nthe value, it will change the result of the next call to token(). Within token rule functions, this points\nto the first character after the matched text. If the value is modified within a rule, the next returned token will be\nmatched at the new position.\n
\n\n

\nlexer.lineno\n

\nThe current value of the line number attribute stored in the lexer. PLY only specifies that the attribute\nexists---it never sets, updates, or performs any processing with it. If you want to track line numbers,\nyou will need to add code yourself (see the section on line numbers and positional information).\n
\n\n

\nlexer.lexdata\n

\nThe current input text stored in the lexer. This is the string passed with the input() method. It\nwould probably be a bad idea to modify this unless you really know what you're doing.\n
\n\n

\nlexer.lexmatch\n

\nThis is the raw Match object returned by the Python re.match() function (used internally by PLY) for the\ncurrent token. If you have written a regular expression that contains named groups, you can use this to retrieve those values.\nNote: This attribute is only updated when tokens are defined and processed by functions. \n
\n\n

4.19 Conditional lexing and start conditions

\n\n\nIn advanced parsing applications, it may be useful to have different\nlexing states. For instance, you may want the occurrence of a certain\ntoken or syntactic construct to trigger a different kind of lexing.\nPLY supports a feature that allows the underlying lexer to be put into\na series of different states. Each state can have its own tokens,\nlexing rules, and so forth. The implementation is based largely on\nthe \"start condition\" feature of GNU flex. Details of this can be found\nat http://flex.sourceforge.net/manual/Start-Conditions.html.\n\n

\nTo define a new lexing state, it must first be declared. This is done by including a \"states\" declaration in your\nlex file. For example:\n\n

\n
\nstates = (\n   ('foo','exclusive'),\n   ('bar','inclusive'),\n)\n
\n
\n\nThis declaration declares two states, 'foo'\nand 'bar'. States may be of two types; 'exclusive'\nand 'inclusive'. An exclusive state completely overrides the\ndefault behavior of the lexer. That is, lex will only return tokens\nand apply rules defined specifically for that state. An inclusive\nstate adds additional tokens and rules to the default set of rules.\nThus, lex will return both the tokens defined by default in addition\nto those defined for the inclusive state.\n\n

\nOnce a state has been declared, tokens and rules are declared by including the\nstate name in token/rule declaration. For example:\n\n

\n
\nt_foo_NUMBER = r'\\d+'                      # Token 'NUMBER' in state 'foo'        \nt_bar_ID     = r'[a-zA-Z_][a-zA-Z0-9_]*'   # Token 'ID' in state 'bar'\n\ndef t_foo_newline(t):\n    r'\\n'\n    t.lexer.lineno += 1\n
\n
\n\nA token can be declared in multiple states by including multiple state names in the declaration. For example:\n\n
\n
\nt_foo_bar_NUMBER = r'\\d+'         # Defines token 'NUMBER' in both state 'foo' and 'bar'\n
\n
\n\nAlternative, a token can be declared in all states using the 'ANY' in the name.\n\n
\n
\nt_ANY_NUMBER = r'\\d+'         # Defines a token 'NUMBER' in all states\n
\n
\n\nIf no state name is supplied, as is normally the case, the token is associated with a special state 'INITIAL'. For example,\nthese two declarations are identical:\n\n
\n
\nt_NUMBER = r'\\d+'\nt_INITIAL_NUMBER = r'\\d+'\n
\n
\n\n

\nStates are also associated with the special t_ignore, t_error(), and t_eof() declarations. For example, if a state treats\nthese differently, you can declare:

\n\n
\n
\nt_foo_ignore = \" \\t\\n\"       # Ignored characters for state 'foo'\n\ndef t_bar_error(t):          # Special error handler for state 'bar'\n    pass \n
\n
\n\nBy default, lexing operates in the 'INITIAL' state. This state includes all of the normally defined tokens. \nFor users who aren't using different states, this fact is completely transparent. If, during lexing or parsing, you want to change\nthe lexing state, use the begin() method. For example:\n\n
\n
\ndef t_begin_foo(t):\n    r'start_foo'\n    t.lexer.begin('foo')             # Starts 'foo' state\n
\n
\n\nTo get out of a state, you use begin() to switch back to the initial state. For example:\n\n
\n
\ndef t_foo_end(t):\n    r'end_foo'\n    t.lexer.begin('INITIAL')        # Back to the initial state\n
\n
\n\nThe management of states can also be done with a stack. For example:\n\n
\n
\ndef t_begin_foo(t):\n    r'start_foo'\n    t.lexer.push_state('foo')             # Starts 'foo' state\n\ndef t_foo_end(t):\n    r'end_foo'\n    t.lexer.pop_state()                   # Back to the previous state\n
\n
\n\n

\nThe use of a stack would be useful in situations where there are many ways of entering a new lexing state and you merely want to go back\nto the previous state afterwards.\n\n

\nAn example might help clarify. Suppose you were writing a parser and you wanted to grab sections of arbitrary C code enclosed by\ncurly braces. That is, whenever you encounter a starting brace '{', you want to read all of the enclosed code up to the ending brace '}' \nand return it as a string. Doing this with a normal regular expression rule is nearly (if not actually) impossible. This is because braces can\nbe nested and can be included in comments and strings. Thus, simply matching up to the first matching '}' character isn't good enough. Here is how\nyou might use lexer states to do this:\n\n

\n
\n# Declare the state\nstates = (\n  ('ccode','exclusive'),\n)\n\n# Match the first {. Enter ccode state.\ndef t_ccode(t):\n    r'\\{'\n    t.lexer.code_start = t.lexer.lexpos        # Record the starting position\n    t.lexer.level = 1                          # Initial brace level\n    t.lexer.begin('ccode')                     # Enter 'ccode' state\n\n# Rules for the ccode state\ndef t_ccode_lbrace(t):     \n    r'\\{'\n    t.lexer.level +=1                \n\ndef t_ccode_rbrace(t):\n    r'\\}'\n    t.lexer.level -=1\n\n    # If closing brace, return the code fragment\n    if t.lexer.level == 0:\n         t.value = t.lexer.lexdata[t.lexer.code_start:t.lexer.lexpos+1]\n         t.type = \"CCODE\"\n         t.lexer.lineno += t.value.count('\\n')\n         t.lexer.begin('INITIAL')           \n         return t\n\n# C or C++ comment (ignore)    \ndef t_ccode_comment(t):\n    r'(/\\*(.|\\n)*?\\*/)|(//.*)'\n    pass\n\n# C string\ndef t_ccode_string(t):\n   r'\\\"([^\\\\\\n]|(\\\\.))*?\\\"'\n\n# C character literal\ndef t_ccode_char(t):\n   r'\\'([^\\\\\\n]|(\\\\.))*?\\''\n\n# Any sequence of non-whitespace characters (not braces, strings)\ndef t_ccode_nonspace(t):\n   r'[^\\s\\{\\}\\'\\\"]+'\n\n# Ignored characters (whitespace)\nt_ccode_ignore = \" \\t\\n\"\n\n# For bad characters, we just skip over it\ndef t_ccode_error(t):\n    t.lexer.skip(1)\n
\n
\n\nIn this example, the occurrence of the first '{' causes the lexer to record the starting position and enter a new state 'ccode'. A collection of rules then match\nvarious parts of the input that follow (comments, strings, etc.). All of these rules merely discard the token (by not returning a value).\nHowever, if the closing right brace is encountered, the rule t_ccode_rbrace collects all of the code (using the earlier recorded starting\nposition), stores it, and returns a token 'CCODE' containing all of that text. When returning the token, the lexing state is restored back to its\ninitial state.\n\n

4.20 Miscellaneous Issues

\n\n\n

\n

  • The lexer requires input to be supplied as a single input string. Since most machines have more than enough memory, this \nrarely presents a performance concern. However, it means that the lexer currently can't be used with streaming data\nsuch as open files or sockets. This limitation is primarily a side-effect of using the re module. You might be\nable to work around this by implementing an appropriate def t_eof() end-of-file handling rule. The main complication\nhere is that you'll probably need to ensure that data is fed to the lexer in a way so that it doesn't split in in the middle\nof a token.

    \n\n

    \n

  • The lexer should work properly with both Unicode strings given as token and pattern matching rules as\nwell as for input text.\n\n

    \n

  • If you need to supply optional flags to the re.compile() function, use the reflags option to lex. For example:\n\n
    \n
    \nlex.lex(reflags=re.UNICODE | re.VERBOSE)\n
    \n
    \n\nNote: by default, reflags is set to re.VERBOSE. If you provide\nyour own flags, you may need to include this for PLY to preserve its normal behavior.\n\n

    \n

  • Since the lexer is written entirely in Python, its performance is\nlargely determined by that of the Python re module. Although\nthe lexer has been written to be as efficient as possible, it's not\nblazingly fast when used on very large input files. If\nperformance is concern, you might consider upgrading to the most\nrecent version of Python, creating a hand-written lexer, or offloading\nthe lexer into a C extension module. \n\n

    \nIf you are going to create a hand-written lexer and you plan to use it with yacc.py, \nit only needs to conform to the following requirements:\n\n

      \n
    • It must provide a token() method that returns the next token or None if no more\ntokens are available.\n
    • The token() method must return an object tok that has type and value attributes. If \nline number tracking is being used, then the token should also define a lineno attribute.\n
    \n\n

    5. Parsing basics

    \n\n\nyacc.py is used to parse language syntax. Before showing an\nexample, there are a few important bits of background that must be\nmentioned. First, syntax is usually specified in terms of a BNF grammar.\nFor example, if you wanted to parse\nsimple arithmetic expressions, you might first write an unambiguous\ngrammar specification like this:\n\n
    \n
     \nexpression : expression + term\n           | expression - term\n           | term\n\nterm       : term * factor\n           | term / factor\n           | factor\n\nfactor     : NUMBER\n           | ( expression )\n
    \n
    \n\nIn the grammar, symbols such as NUMBER, +, -, *, and / are known\nas terminals and correspond to raw input tokens. Identifiers such as term and factor refer to \ngrammar rules comprised of a collection of terminals and other rules. These identifiers are known as non-terminals.\n

    \n\nThe semantic behavior of a language is often specified using a\ntechnique known as syntax directed translation. In syntax directed\ntranslation, attributes are attached to each symbol in a given grammar\nrule along with an action. Whenever a particular grammar rule is\nrecognized, the action describes what to do. For example, given the\nexpression grammar above, you might write the specification for a\nsimple calculator like this:\n\n

    \n
     \nGrammar                             Action\n--------------------------------    -------------------------------------------- \nexpression0 : expression1 + term    expression0.val = expression1.val + term.val\n            | expression1 - term    expression0.val = expression1.val - term.val\n            | term                  expression0.val = term.val\n\nterm0       : term1 * factor        term0.val = term1.val * factor.val\n            | term1 / factor        term0.val = term1.val / factor.val\n            | factor                term0.val = factor.val\n\nfactor      : NUMBER                factor.val = int(NUMBER.lexval)\n            | ( expression )        factor.val = expression.val\n
    \n
    \n\nA good way to think about syntax directed translation is to \nview each symbol in the grammar as a kind of object. Associated\nwith each symbol is a value representing its \"state\" (for example, the\nval attribute above). Semantic\nactions are then expressed as a collection of functions or methods\nthat operate on the symbols and associated values.\n\n

    \nYacc uses a parsing technique known as LR-parsing or shift-reduce parsing. LR parsing is a\nbottom up technique that tries to recognize the right-hand-side of various grammar rules.\nWhenever a valid right-hand-side is found in the input, the appropriate action code is triggered and the\ngrammar symbols are replaced by the grammar symbol on the left-hand-side. \n\n

    \nLR parsing is commonly implemented by shifting grammar symbols onto a\nstack and looking at the stack and the next input token for patterns that\nmatch one of the grammar rules.\nThe details of the algorithm can be found in a compiler textbook, but the\nfollowing example illustrates the steps that are performed if you\nwanted to parse the expression\n3 + 5 * (10 - 20) using the grammar defined above. In the example,\nthe special symbol $ represents the end of input.\n\n\n

    \n
    \nStep Symbol Stack           Input Tokens            Action\n---- ---------------------  ---------------------   -------------------------------\n1                           3 + 5 * ( 10 - 20 )$    Shift 3\n2    3                        + 5 * ( 10 - 20 )$    Reduce factor : NUMBER\n3    factor                   + 5 * ( 10 - 20 )$    Reduce term   : factor\n4    term                     + 5 * ( 10 - 20 )$    Reduce expr : term\n5    expr                     + 5 * ( 10 - 20 )$    Shift +\n6    expr +                     5 * ( 10 - 20 )$    Shift 5\n7    expr + 5                     * ( 10 - 20 )$    Reduce factor : NUMBER\n8    expr + factor                * ( 10 - 20 )$    Reduce term   : factor\n9    expr + term                  * ( 10 - 20 )$    Shift *\n10   expr + term *                  ( 10 - 20 )$    Shift (\n11   expr + term * (                  10 - 20 )$    Shift 10\n12   expr + term * ( 10                  - 20 )$    Reduce factor : NUMBER\n13   expr + term * ( factor              - 20 )$    Reduce term : factor\n14   expr + term * ( term                - 20 )$    Reduce expr : term\n15   expr + term * ( expr                - 20 )$    Shift -\n16   expr + term * ( expr -                20 )$    Shift 20\n17   expr + term * ( expr - 20                )$    Reduce factor : NUMBER\n18   expr + term * ( expr - factor            )$    Reduce term : factor\n19   expr + term * ( expr - term              )$    Reduce expr : expr - term\n20   expr + term * ( expr                     )$    Shift )\n21   expr + term * ( expr )                    $    Reduce factor : (expr)\n22   expr + term * factor                      $    Reduce term : term * factor\n23   expr + term                               $    Reduce expr : expr + term\n24   expr                                      $    Reduce expr\n25                                             $    Success!\n
    \n
    \n\nWhen parsing the expression, an underlying state machine and the\ncurrent input token determine what happens next. If the next token\nlooks like part of a valid grammar rule (based on other items on the\nstack), it is generally shifted onto the stack. If the top of the\nstack contains a valid right-hand-side of a grammar rule, it is\nusually \"reduced\" and the symbols replaced with the symbol on the\nleft-hand-side. When this reduction occurs, the appropriate action is\ntriggered (if defined). If the input token can't be shifted and the\ntop of stack doesn't match any grammar rules, a syntax error has\noccurred and the parser must take some kind of recovery step (or bail\nout). A parse is only successful if the parser reaches a state where\nthe symbol stack is empty and there are no more input tokens.\n\n

    \nIt is important to note that the underlying implementation is built\naround a large finite-state machine that is encoded in a collection of\ntables. The construction of these tables is non-trivial and\nbeyond the scope of this discussion. However, subtle details of this\nprocess explain why, in the example above, the parser chooses to shift\na token onto the stack in step 9 rather than reducing the\nrule expr : expr + term.\n\n

    6. Yacc

    \n\n\nThe ply.yacc module implements the parsing component of PLY.\nThe name \"yacc\" stands for \"Yet Another Compiler Compiler\" and is\nborrowed from the Unix tool of the same name.\n\n

    6.1 An example

    \n\n\nSuppose you wanted to make a grammar for simple arithmetic expressions as previously described. Here is\nhow you would do it with yacc.py:\n\n
    \n
    \n# Yacc example\n\nimport ply.yacc as yacc\n\n# Get the token map from the lexer.  This is required.\nfrom calclex import tokens\n\ndef p_expression_plus(p):\n    'expression : expression PLUS term'\n    p[0] = p[1] + p[3]\n\ndef p_expression_minus(p):\n    'expression : expression MINUS term'\n    p[0] = p[1] - p[3]\n\ndef p_expression_term(p):\n    'expression : term'\n    p[0] = p[1]\n\ndef p_term_times(p):\n    'term : term TIMES factor'\n    p[0] = p[1] * p[3]\n\ndef p_term_div(p):\n    'term : term DIVIDE factor'\n    p[0] = p[1] / p[3]\n\ndef p_term_factor(p):\n    'term : factor'\n    p[0] = p[1]\n\ndef p_factor_num(p):\n    'factor : NUMBER'\n    p[0] = p[1]\n\ndef p_factor_expr(p):\n    'factor : LPAREN expression RPAREN'\n    p[0] = p[2]\n\n# Error rule for syntax errors\ndef p_error(p):\n    print(\"Syntax error in input!\")\n\n# Build the parser\nparser = yacc.yacc()\n\nwhile True:\n   try:\n       s = raw_input('calc > ')\n   except EOFError:\n       break\n   if not s: continue\n   result = parser.parse(s)\n   print(result)\n
    \n
    \n\nIn this example, each grammar rule is defined by a Python function\nwhere the docstring to that function contains the appropriate\ncontext-free grammar specification. The statements that make up the\nfunction body implement the semantic actions of the rule. Each function\naccepts a single argument p that is a sequence containing the\nvalues of each grammar symbol in the corresponding rule. The values\nof p[i] are mapped to grammar symbols as shown here:\n\n
    \n
    \ndef p_expression_plus(p):\n    'expression : expression PLUS term'\n    #   ^            ^        ^    ^\n    #  p[0]         p[1]     p[2] p[3]\n\n    p[0] = p[1] + p[3]\n
    \n
    \n\n

    \nFor tokens, the \"value\" of the corresponding p[i] is the\nsame as the p.value attribute assigned in the lexer\nmodule. For non-terminals, the value is determined by whatever is\nplaced in p[0] when rules are reduced. This value can be\nanything at all. However, it probably most common for the value to be\na simple Python type, a tuple, or an instance. In this example, we\nare relying on the fact that the NUMBER token stores an\ninteger value in its value field. All of the other rules simply\nperform various types of integer operations and propagate the result.\n

    \n\n

    \nNote: The use of negative indices have a special meaning in\nyacc---specially p[-1] does not have the same value\nas p[3] in this example. Please see the section on \"Embedded\nActions\" for further details.\n

    \n\n

    \nThe first rule defined in the yacc specification determines the\nstarting grammar symbol (in this case, a rule for expression\nappears first). Whenever the starting rule is reduced by the parser\nand no more input is available, parsing stops and the final value is\nreturned (this value will be whatever the top-most rule placed\nin p[0]). Note: an alternative starting symbol can be\nspecified using the start keyword argument to\nyacc().\n\n

    The p_error(p) rule is defined to catch syntax errors.\nSee the error handling section below for more detail.\n\n

    \nTo build the parser, call the yacc.yacc() function. This\nfunction looks at the module and attempts to construct all of the LR\nparsing tables for the grammar you have specified. The first\ntime yacc.yacc() is invoked, you will get a message such as\nthis:\n\n

    \n
    \n$ python calcparse.py\nGenerating LALR tables\ncalc > \n
    \n
    \n\n

    \nSince table construction is relatively expensive (especially for large\ngrammars), the resulting parsing table is written to \na file called parsetab.py. In addition, a\ndebugging file called parser.out is created. On subsequent\nexecutions, yacc will reload the table from\nparsetab.py unless it has detected a change in the underlying\ngrammar (in which case the tables and parsetab.py file are\nregenerated). Both of these files are written to the same directory\nas the module in which the parser is specified. \nThe name of the parsetab module can be changed using the\ntabmodule keyword argument to yacc(). For example:\n

    \n\n
    \n
    \nparser = yacc.yacc(tabmodule='fooparsetab')\n
    \n
    \n\n

    \nIf any errors are detected in your grammar specification, yacc.py will produce\ndiagnostic messages and possibly raise an exception. Some of the errors that can be detected include:\n\n

      \n
    • Duplicated function names (if more than one rule function have the same name in the grammar file).\n
    • Shift/reduce and reduce/reduce conflicts generated by ambiguous grammars.\n
    • Badly specified grammar rules.\n
    • Infinite recursion (rules that can never terminate).\n
    • Unused rules and tokens\n
    • Undefined rules and tokens\n
    \n\nThe next few sections discuss grammar specification in more detail.\n\n

    \nThe final part of the example shows how to actually run the parser\ncreated by\nyacc(). To run the parser, you simply have to call\nthe parse() with a string of input text. This will run all\nof the grammar rules and return the result of the entire parse. This\nresult return is the value assigned to p[0] in the starting\ngrammar rule.\n\n

    6.2 Combining Grammar Rule Functions

    \n\n\nWhen grammar rules are similar, they can be combined into a single function.\nFor example, consider the two rules in our earlier example:\n\n
    \n
    \ndef p_expression_plus(p):\n    'expression : expression PLUS term'\n    p[0] = p[1] + p[3]\n\ndef p_expression_minus(t):\n    'expression : expression MINUS term'\n    p[0] = p[1] - p[3]\n
    \n
    \n\nInstead of writing two functions, you might write a single function like this:\n\n
    \n
    \ndef p_expression(p):\n    '''expression : expression PLUS term\n                  | expression MINUS term'''\n    if p[2] == '+':\n        p[0] = p[1] + p[3]\n    elif p[2] == '-':\n        p[0] = p[1] - p[3]\n
    \n
    \n\nIn general, the doc string for any given function can contain multiple grammar rules. So, it would\nhave also been legal (although possibly confusing) to write this:\n\n
    \n
    \ndef p_binary_operators(p):\n    '''expression : expression PLUS term\n                  | expression MINUS term\n       term       : term TIMES factor\n                  | term DIVIDE factor'''\n    if p[2] == '+':\n        p[0] = p[1] + p[3]\n    elif p[2] == '-':\n        p[0] = p[1] - p[3]\n    elif p[2] == '*':\n        p[0] = p[1] * p[3]\n    elif p[2] == '/':\n        p[0] = p[1] / p[3]\n
    \n
    \n\nWhen combining grammar rules into a single function, it is usually a good idea for all of the rules to have\na similar structure (e.g., the same number of terms). Otherwise, the corresponding action code may be more \ncomplicated than necessary. However, it is possible to handle simple cases using len(). For example:\n\n
    \n
    \ndef p_expressions(p):\n    '''expression : expression MINUS expression\n                  | MINUS expression'''\n    if (len(p) == 4):\n        p[0] = p[1] - p[3]\n    elif (len(p) == 3):\n        p[0] = -p[2]\n
    \n
    \n\nIf parsing performance is a concern, you should resist the urge to put\ntoo much conditional processing into a single grammar rule as shown in\nthese examples. When you add checks to see which grammar rule is\nbeing handled, you are actually duplicating the work that the parser\nhas already performed (i.e., the parser already knows exactly what rule it\nmatched). You can eliminate this overhead by using a\nseparate p_rule() function for each grammar rule.\n\n

    6.3 Character Literals

    \n\n\nIf desired, a grammar may contain tokens defined as single character literals. For example:\n\n
    \n
    \ndef p_binary_operators(p):\n    '''expression : expression '+' term\n                  | expression '-' term\n       term       : term '*' factor\n                  | term '/' factor'''\n    if p[2] == '+':\n        p[0] = p[1] + p[3]\n    elif p[2] == '-':\n        p[0] = p[1] - p[3]\n    elif p[2] == '*':\n        p[0] = p[1] * p[3]\n    elif p[2] == '/':\n        p[0] = p[1] / p[3]\n
    \n
    \n\nA character literal must be enclosed in quotes such as '+'. In addition, if literals are used, they must be declared in the\ncorresponding lex file through the use of a special literals declaration.\n\n
    \n
    \n# Literals.  Should be placed in module given to lex()\nliterals = ['+','-','*','/' ]\n
    \n
    \n\nCharacter literals are limited to a single character. Thus, it is not legal to specify literals such as '<=' or '=='. For this, use\nthe normal lexing rules (e.g., define a rule such as t_EQ = r'==').\n\n

    6.4 Empty Productions

    \n\n\nyacc.py can handle empty productions by defining a rule like this:\n\n
    \n
    \ndef p_empty(p):\n    'empty :'\n    pass\n
    \n
    \n\nNow to use the empty production, simply use 'empty' as a symbol. For example:\n\n
    \n
    \ndef p_optitem(p):\n    'optitem : item'\n    '        | empty'\n    ...\n
    \n
    \n\nNote: You can write empty rules anywhere by simply specifying an empty\nright hand side. However, I personally find that writing an \"empty\"\nrule and using \"empty\" to denote an empty production is easier to read\nand more clearly states your intentions.\n\n

    6.5 Changing the starting symbol

    \n\n\nNormally, the first rule found in a yacc specification defines the starting grammar rule (top level rule). To change this, simply\nsupply a start specifier in your file. For example:\n\n
    \n
    \nstart = 'foo'\n\ndef p_bar(p):\n    'bar : A B'\n\n# This is the starting rule due to the start specifier above\ndef p_foo(p):\n    'foo : bar X'\n...\n
    \n
    \n\nThe use of a start specifier may be useful during debugging\nsince you can use it to have yacc build a subset of a larger grammar.\nFor this purpose, it is also possible to specify a starting symbol as\nan argument to yacc(). For example:\n\n
    \n
    \nparser = yacc.yacc(start='foo')\n
    \n
    \n\n

    6.6 Dealing With Ambiguous Grammars

    \n\n\nThe expression grammar given in the earlier example has been written\nin a special format to eliminate ambiguity. However, in many\nsituations, it is extremely difficult or awkward to write grammars in\nthis format. A much more natural way to express the grammar is in a\nmore compact form like this:\n\n
    \n
    \nexpression : expression PLUS expression\n           | expression MINUS expression\n           | expression TIMES expression\n           | expression DIVIDE expression\n           | LPAREN expression RPAREN\n           | NUMBER\n
    \n
    \n\nUnfortunately, this grammar specification is ambiguous. For example,\nif you are parsing the string \"3 * 4 + 5\", there is no way to tell how\nthe operators are supposed to be grouped. For example, does the\nexpression mean \"(3 * 4) + 5\" or is it \"3 * (4+5)\"?\n\n

    \nWhen an ambiguous grammar is given to yacc.py it will print\nmessages about \"shift/reduce conflicts\" or \"reduce/reduce conflicts\".\nA shift/reduce conflict is caused when the parser generator can't\ndecide whether or not to reduce a rule or shift a symbol on the\nparsing stack. For example, consider the string \"3 * 4 + 5\" and the\ninternal parsing stack:\n\n

    \n
    \nStep Symbol Stack           Input Tokens            Action\n---- ---------------------  ---------------------   -------------------------------\n1    $                                3 * 4 + 5$    Shift 3\n2    $ 3                                * 4 + 5$    Reduce : expression : NUMBER\n3    $ expr                             * 4 + 5$    Shift *\n4    $ expr *                             4 + 5$    Shift 4\n5    $ expr * 4                             + 5$    Reduce: expression : NUMBER\n6    $ expr * expr                          + 5$    SHIFT/REDUCE CONFLICT ????\n
    \n
    \n\nIn this case, when the parser reaches step 6, it has two options. One\nis to reduce the rule expr : expr * expr on the stack. The\nother option is to shift the token + on the stack. Both\noptions are perfectly legal from the rules of the\ncontext-free-grammar.\n\n

    \nBy default, all shift/reduce conflicts are resolved in favor of\nshifting. Therefore, in the above example, the parser will always\nshift the + instead of reducing. Although this strategy\nworks in many cases (for example, the case of \n\"if-then\" versus \"if-then-else\"), it is not enough for arithmetic expressions. In fact,\nin the above example, the decision to shift + is completely\nwrong---we should have reduced expr * expr since\nmultiplication has higher mathematical precedence than addition.\n\n

    To resolve ambiguity, especially in expression\ngrammars, yacc.py allows individual tokens to be assigned a\nprecedence level and associativity. This is done by adding a variable\nprecedence to the grammar file like this:\n\n

    \n
    \nprecedence = (\n    ('left', 'PLUS', 'MINUS'),\n    ('left', 'TIMES', 'DIVIDE'),\n)\n
    \n
    \n\nThis declaration specifies that PLUS/MINUS have the\nsame precedence level and are left-associative and that\nTIMES/DIVIDE have the same precedence and are\nleft-associative. Within the precedence declaration, tokens\nare ordered from lowest to highest precedence. Thus, this declaration\nspecifies that TIMES/DIVIDE have higher precedence\nthan PLUS/MINUS (since they appear later in the\nprecedence specification).\n\n

    \nThe precedence specification works by associating a numerical\nprecedence level value and associativity direction to the listed\ntokens. For example, in the above example you get:\n\n

    \n
    \nPLUS      : level = 1,  assoc = 'left'\nMINUS     : level = 1,  assoc = 'left'\nTIMES     : level = 2,  assoc = 'left'\nDIVIDE    : level = 2,  assoc = 'left'\n
    \n
    \n\nThese values are then used to attach a numerical precedence value and\nassociativity direction to each grammar rule. This is always\ndetermined by looking at the precedence of the right-most terminal\nsymbol. For example:\n\n
    \n
    \nexpression : expression PLUS expression                 # level = 1, left\n           | expression MINUS expression                # level = 1, left\n           | expression TIMES expression                # level = 2, left\n           | expression DIVIDE expression               # level = 2, left\n           | LPAREN expression RPAREN                   # level = None (not specified)\n           | NUMBER                                     # level = None (not specified)\n
    \n
    \n\nWhen shift/reduce conflicts are encountered, the parser generator resolves the conflict by\nlooking at the precedence rules and associativity specifiers.\n\n

    \n

      \n
    1. If the current token has higher precedence than the rule on the stack, it is shifted.\n
    2. If the grammar rule on the stack has higher precedence, the rule is reduced.\n
    3. If the current token and the grammar rule have the same precedence, the\nrule is reduced for left associativity, whereas the token is shifted for right associativity.\n
    4. If nothing is known about the precedence, shift/reduce conflicts are resolved in\nfavor of shifting (the default).\n
    \n\nFor example, if \"expression PLUS expression\" has been parsed and the\nnext token is \"TIMES\", the action is going to be a shift because\n\"TIMES\" has a higher precedence level than \"PLUS\". On the other hand,\nif \"expression TIMES expression\" has been parsed and the next token is\n\"PLUS\", the action is going to be reduce because \"PLUS\" has a lower\nprecedence than \"TIMES.\"\n\n

    \nWhen shift/reduce conflicts are resolved using the first three\ntechniques (with the help of precedence rules), yacc.py will\nreport no errors or conflicts in the grammar (although it will print\nsome information in the parser.out debugging file).\n\n

    \nOne problem with the precedence specifier technique is that it is\nsometimes necessary to change the precedence of an operator in certain\ncontexts. For example, consider a unary-minus operator in \"3 + 4 *\n-5\". Mathematically, the unary minus is normally given a very high\nprecedence--being evaluated before the multiply. However, in our\nprecedence specifier, MINUS has a lower precedence than TIMES. To\ndeal with this, precedence rules can be given for so-called \"fictitious tokens\"\nlike this:\n\n

    \n
    \nprecedence = (\n    ('left', 'PLUS', 'MINUS'),\n    ('left', 'TIMES', 'DIVIDE'),\n    ('right', 'UMINUS'),            # Unary minus operator\n)\n
    \n
    \n\nNow, in the grammar file, we can write our unary minus rule like this:\n\n
    \n
    \ndef p_expr_uminus(p):\n    'expression : MINUS expression %prec UMINUS'\n    p[0] = -p[2]\n
    \n
    \n\nIn this case, %prec UMINUS overrides the default rule precedence--setting it to that\nof UMINUS in the precedence specifier.\n\n

    \nAt first, the use of UMINUS in this example may appear very confusing.\nUMINUS is not an input token or a grammar rule. Instead, you should\nthink of it as the name of a special marker in the precedence table. When you use the %prec qualifier, you're simply\ntelling yacc that you want the precedence of the expression to be the same as for this special marker instead of the usual precedence.\n\n

    \nIt is also possible to specify non-associativity in the precedence table. This would\nbe used when you don't want operations to chain together. For example, suppose\nyou wanted to support comparison operators like < and > but you didn't want to allow\ncombinations like a < b < c. To do this, simply specify a rule like this:\n\n

    \n
    \nprecedence = (\n    ('nonassoc', 'LESSTHAN', 'GREATERTHAN'),  # Nonassociative operators\n    ('left', 'PLUS', 'MINUS'),\n    ('left', 'TIMES', 'DIVIDE'),\n    ('right', 'UMINUS'),            # Unary minus operator\n)\n
    \n
    \n\n

    \nIf you do this, the occurrence of input text such as a < b < c will result in a syntax error. However, simple\nexpressions such as a < b will still be fine.\n\n

    \nReduce/reduce conflicts are caused when there are multiple grammar\nrules that can be applied to a given set of symbols. This kind of\nconflict is almost always bad and is always resolved by picking the\nrule that appears first in the grammar file. Reduce/reduce conflicts\nare almost always caused when different sets of grammar rules somehow\ngenerate the same set of symbols. For example:\n\n

    \n
    \nassignment :  ID EQUALS NUMBER\n           |  ID EQUALS expression\n           \nexpression : expression PLUS expression\n           | expression MINUS expression\n           | expression TIMES expression\n           | expression DIVIDE expression\n           | LPAREN expression RPAREN\n           | NUMBER\n
    \n
    \n\nIn this case, a reduce/reduce conflict exists between these two rules:\n\n
    \n
    \nassignment  : ID EQUALS NUMBER\nexpression  : NUMBER\n
    \n
    \n\nFor example, if you wrote \"a = 5\", the parser can't figure out if this\nis supposed to be reduced as assignment : ID EQUALS NUMBER or\nwhether it's supposed to reduce the 5 as an expression and then reduce\nthe rule assignment : ID EQUALS expression.\n\n

    \nIt should be noted that reduce/reduce conflicts are notoriously\ndifficult to spot simply looking at the input grammar. When a\nreduce/reduce conflict occurs, yacc() will try to help by\nprinting a warning message such as this:\n\n

    \n
    \nWARNING: 1 reduce/reduce conflict\nWARNING: reduce/reduce conflict in state 15 resolved using rule (assignment -> ID EQUALS NUMBER)\nWARNING: rejected rule (expression -> NUMBER)\n
    \n
    \n\nThis message identifies the two rules that are in conflict. However,\nit may not tell you how the parser arrived at such a state. To try\nand figure it out, you'll probably have to look at your grammar and\nthe contents of the\nparser.out debugging file with an appropriately high level of\ncaffeination.\n\n

    6.7 The parser.out file

    \n\n\nTracking down shift/reduce and reduce/reduce conflicts is one of the finer pleasures of using an LR\nparsing algorithm. To assist in debugging, yacc.py creates a debugging file called\n'parser.out' when it generates the parsing table. The contents of this file look like the following:\n\n
    \n
    \nUnused terminals:\n\n\nGrammar\n\nRule 1     expression -> expression PLUS expression\nRule 2     expression -> expression MINUS expression\nRule 3     expression -> expression TIMES expression\nRule 4     expression -> expression DIVIDE expression\nRule 5     expression -> NUMBER\nRule 6     expression -> LPAREN expression RPAREN\n\nTerminals, with rules where they appear\n\nTIMES                : 3\nerror                : \nMINUS                : 2\nRPAREN               : 6\nLPAREN               : 6\nDIVIDE               : 4\nPLUS                 : 1\nNUMBER               : 5\n\nNonterminals, with rules where they appear\n\nexpression           : 1 1 2 2 3 3 4 4 6 0\n\n\nParsing method: LALR\n\n\nstate 0\n\n    S' -> . expression\n    expression -> . expression PLUS expression\n    expression -> . expression MINUS expression\n    expression -> . expression TIMES expression\n    expression -> . expression DIVIDE expression\n    expression -> . NUMBER\n    expression -> . LPAREN expression RPAREN\n\n    NUMBER          shift and go to state 3\n    LPAREN          shift and go to state 2\n\n\nstate 1\n\n    S' -> expression .\n    expression -> expression . PLUS expression\n    expression -> expression . MINUS expression\n    expression -> expression . TIMES expression\n    expression -> expression . DIVIDE expression\n\n    PLUS            shift and go to state 6\n    MINUS           shift and go to state 5\n    TIMES           shift and go to state 4\n    DIVIDE          shift and go to state 7\n\n\nstate 2\n\n    expression -> LPAREN . expression RPAREN\n    expression -> . expression PLUS expression\n    expression -> . expression MINUS expression\n    expression -> . expression TIMES expression\n    expression -> . expression DIVIDE expression\n    expression -> . NUMBER\n    expression -> . LPAREN expression RPAREN\n\n    NUMBER          shift and go to state 3\n    LPAREN          shift and go to state 2\n\n\nstate 3\n\n    expression -> NUMBER .\n\n    $               reduce using rule 5\n    PLUS            reduce using rule 5\n    MINUS           reduce using rule 5\n    TIMES           reduce using rule 5\n    DIVIDE          reduce using rule 5\n    RPAREN          reduce using rule 5\n\n\nstate 4\n\n    expression -> expression TIMES . expression\n    expression -> . expression PLUS expression\n    expression -> . expression MINUS expression\n    expression -> . expression TIMES expression\n    expression -> . expression DIVIDE expression\n    expression -> . NUMBER\n    expression -> . LPAREN expression RPAREN\n\n    NUMBER          shift and go to state 3\n    LPAREN          shift and go to state 2\n\n\nstate 5\n\n    expression -> expression MINUS . expression\n    expression -> . expression PLUS expression\n    expression -> . expression MINUS expression\n    expression -> . expression TIMES expression\n    expression -> . expression DIVIDE expression\n    expression -> . NUMBER\n    expression -> . LPAREN expression RPAREN\n\n    NUMBER          shift and go to state 3\n    LPAREN          shift and go to state 2\n\n\nstate 6\n\n    expression -> expression PLUS . expression\n    expression -> . expression PLUS expression\n    expression -> . expression MINUS expression\n    expression -> . expression TIMES expression\n    expression -> . expression DIVIDE expression\n    expression -> . NUMBER\n    expression -> . LPAREN expression RPAREN\n\n    NUMBER          shift and go to state 3\n    LPAREN          shift and go to state 2\n\n\nstate 7\n\n    expression -> expression DIVIDE . expression\n    expression -> . expression PLUS expression\n    expression -> . expression MINUS expression\n    expression -> . expression TIMES expression\n    expression -> . expression DIVIDE expression\n    expression -> . NUMBER\n    expression -> . LPAREN expression RPAREN\n\n    NUMBER          shift and go to state 3\n    LPAREN          shift and go to state 2\n\n\nstate 8\n\n    expression -> LPAREN expression . RPAREN\n    expression -> expression . PLUS expression\n    expression -> expression . MINUS expression\n    expression -> expression . TIMES expression\n    expression -> expression . DIVIDE expression\n\n    RPAREN          shift and go to state 13\n    PLUS            shift and go to state 6\n    MINUS           shift and go to state 5\n    TIMES           shift and go to state 4\n    DIVIDE          shift and go to state 7\n\n\nstate 9\n\n    expression -> expression TIMES expression .\n    expression -> expression . PLUS expression\n    expression -> expression . MINUS expression\n    expression -> expression . TIMES expression\n    expression -> expression . DIVIDE expression\n\n    $               reduce using rule 3\n    PLUS            reduce using rule 3\n    MINUS           reduce using rule 3\n    TIMES           reduce using rule 3\n    DIVIDE          reduce using rule 3\n    RPAREN          reduce using rule 3\n\n  ! PLUS            [ shift and go to state 6 ]\n  ! MINUS           [ shift and go to state 5 ]\n  ! TIMES           [ shift and go to state 4 ]\n  ! DIVIDE          [ shift and go to state 7 ]\n\nstate 10\n\n    expression -> expression MINUS expression .\n    expression -> expression . PLUS expression\n    expression -> expression . MINUS expression\n    expression -> expression . TIMES expression\n    expression -> expression . DIVIDE expression\n\n    $               reduce using rule 2\n    PLUS            reduce using rule 2\n    MINUS           reduce using rule 2\n    RPAREN          reduce using rule 2\n    TIMES           shift and go to state 4\n    DIVIDE          shift and go to state 7\n\n  ! TIMES           [ reduce using rule 2 ]\n  ! DIVIDE          [ reduce using rule 2 ]\n  ! PLUS            [ shift and go to state 6 ]\n  ! MINUS           [ shift and go to state 5 ]\n\nstate 11\n\n    expression -> expression PLUS expression .\n    expression -> expression . PLUS expression\n    expression -> expression . MINUS expression\n    expression -> expression . TIMES expression\n    expression -> expression . DIVIDE expression\n\n    $               reduce using rule 1\n    PLUS            reduce using rule 1\n    MINUS           reduce using rule 1\n    RPAREN          reduce using rule 1\n    TIMES           shift and go to state 4\n    DIVIDE          shift and go to state 7\n\n  ! TIMES           [ reduce using rule 1 ]\n  ! DIVIDE          [ reduce using rule 1 ]\n  ! PLUS            [ shift and go to state 6 ]\n  ! MINUS           [ shift and go to state 5 ]\n\nstate 12\n\n    expression -> expression DIVIDE expression .\n    expression -> expression . PLUS expression\n    expression -> expression . MINUS expression\n    expression -> expression . TIMES expression\n    expression -> expression . DIVIDE expression\n\n    $               reduce using rule 4\n    PLUS            reduce using rule 4\n    MINUS           reduce using rule 4\n    TIMES           reduce using rule 4\n    DIVIDE          reduce using rule 4\n    RPAREN          reduce using rule 4\n\n  ! PLUS            [ shift and go to state 6 ]\n  ! MINUS           [ shift and go to state 5 ]\n  ! TIMES           [ shift and go to state 4 ]\n  ! DIVIDE          [ shift and go to state 7 ]\n\nstate 13\n\n    expression -> LPAREN expression RPAREN .\n\n    $               reduce using rule 6\n    PLUS            reduce using rule 6\n    MINUS           reduce using rule 6\n    TIMES           reduce using rule 6\n    DIVIDE          reduce using rule 6\n    RPAREN          reduce using rule 6\n
    \n
    \n\nThe different states that appear in this file are a representation of\nevery possible sequence of valid input tokens allowed by the grammar.\nWhen receiving input tokens, the parser is building up a stack and\nlooking for matching rules. Each state keeps track of the grammar\nrules that might be in the process of being matched at that point. Within each\nrule, the \".\" character indicates the current location of the parse\nwithin that rule. In addition, the actions for each valid input token\nare listed. When a shift/reduce or reduce/reduce conflict arises,\nrules not selected are prefixed with an !. For example:\n\n
    \n
    \n  ! TIMES           [ reduce using rule 2 ]\n  ! DIVIDE          [ reduce using rule 2 ]\n  ! PLUS            [ shift and go to state 6 ]\n  ! MINUS           [ shift and go to state 5 ]\n
    \n
    \n\nBy looking at these rules (and with a little practice), you can usually track down the source\nof most parsing conflicts. It should also be stressed that not all shift-reduce conflicts are\nbad. However, the only way to be sure that they are resolved correctly is to look at parser.out.\n \n

    6.8 Syntax Error Handling

    \n\n\nIf you are creating a parser for production use, the handling of\nsyntax errors is important. As a general rule, you don't want a\nparser to simply throw up its hands and stop at the first sign of\ntrouble. Instead, you want it to report the error, recover if possible, and\ncontinue parsing so that all of the errors in the input get reported\nto the user at once. This is the standard behavior found in compilers\nfor languages such as C, C++, and Java.\n\nIn PLY, when a syntax error occurs during parsing, the error is immediately\ndetected (i.e., the parser does not read any more tokens beyond the\nsource of the error). However, at this point, the parser enters a\nrecovery mode that can be used to try and continue further parsing.\nAs a general rule, error recovery in LR parsers is a delicate\ntopic that involves ancient rituals and black-magic. The recovery mechanism\nprovided by yacc.py is comparable to Unix yacc so you may want\nconsult a book like O'Reilly's \"Lex and Yacc\" for some of the finer details.\n\n

    \nWhen a syntax error occurs, yacc.py performs the following steps:\n\n

      \n
    1. On the first occurrence of an error, the user-defined p_error() function\nis called with the offending token as an argument. However, if the syntax error is due to\nreaching the end-of-file, p_error() is called with an\n argument of None.\nAfterwards, the parser enters\nan \"error-recovery\" mode in which it will not make future calls to p_error() until it\nhas successfully shifted at least 3 tokens onto the parsing stack.\n\n

      \n

    2. If no recovery action is taken in p_error(), the offending lookahead token is replaced\nwith a special error token.\n\n

      \n

    3. If the offending lookahead token is already set to error, the top item of the parsing stack is\ndeleted.\n\n

      \n

    4. If the entire parsing stack is unwound, the parser enters a restart state and attempts to start\nparsing from its initial state.\n\n

      \n

    5. If a grammar rule accepts error as a token, it will be\nshifted onto the parsing stack.\n\n

      \n

    6. If the top item of the parsing stack is error, lookahead tokens will be discarded until the\nparser can successfully shift a new symbol or reduce a rule involving error.\n
    \n\n

    6.8.1 Recovery and resynchronization with error rules

    \n\n\nThe most well-behaved approach for handling syntax errors is to write grammar rules that include the error\ntoken. For example, suppose your language had a grammar rule for a print statement like this:\n\n
    \n
    \ndef p_statement_print(p):\n     'statement : PRINT expr SEMI'\n     ...\n
    \n
    \n\nTo account for the possibility of a bad expression, you might write an additional grammar rule like this:\n\n
    \n
    \ndef p_statement_print_error(p):\n     'statement : PRINT error SEMI'\n     print(\"Syntax error in print statement. Bad expression\")\n\n
    \n
    \n\nIn this case, the error token will match any sequence of\ntokens that might appear up to the first semicolon that is\nencountered. Once the semicolon is reached, the rule will be\ninvoked and the error token will go away.\n\n

    \nThis type of recovery is sometimes known as parser resynchronization.\nThe error token acts as a wildcard for any bad input text and\nthe token immediately following error acts as a\nsynchronization token.\n\n

    \nIt is important to note that the error token usually does not appear as the last token\non the right in an error rule. For example:\n\n

    \n
    \ndef p_statement_print_error(p):\n    'statement : PRINT error'\n    print(\"Syntax error in print statement. Bad expression\")\n
    \n
    \n\nThis is because the first bad token encountered will cause the rule to\nbe reduced--which may make it difficult to recover if more bad tokens\nimmediately follow. \n\n

    6.8.2 Panic mode recovery

    \n\n\nAn alternative error recovery scheme is to enter a panic mode recovery in which tokens are\ndiscarded to a point where the parser might be able to recover in some sensible manner.\n\n

    \nPanic mode recovery is implemented entirely in the p_error() function. For example, this\nfunction starts discarding tokens until it reaches a closing '}'. Then, it restarts the \nparser in its initial state.\n\n

    \n
    \ndef p_error(p):\n    print(\"Whoa. You are seriously hosed.\")\n    if not p:\n        print(\"End of File!\")\n        return\n\n    # Read ahead looking for a closing '}'\n    while True:\n        tok = parser.token()             # Get the next token\n        if not tok or tok.type == 'RBRACE': \n            break\n    parser.restart()\n
    \n
    \n\n

    \nThis function simply discards the bad token and tells the parser that the error was ok.\n\n

    \n
    \ndef p_error(p):\n    if p:\n         print(\"Syntax error at token\", p.type)\n         # Just discard the token and tell the parser it's okay.\n         parser.errok()\n    else:\n         print(\"Syntax error at EOF\")\n
    \n
    \n\n

    \nMore information on these methods is as follows:\n

    \n\n

    \n

      \n
    • parser.errok(). This resets the parser state so it doesn't think it's in error-recovery\nmode. This will prevent an error token from being generated and will reset the internal\nerror counters so that the next syntax error will call p_error() again.\n\n

      \n

    • parser.token(). This returns the next token on the input stream.\n\n

      \n

    • parser.restart(). This discards the entire parsing stack and resets the parser\nto its initial state. \n
    \n\n

    \nTo supply the next lookahead token to the parser, p_error() can return a token. This might be\nuseful if trying to synchronize on special characters. For example:\n\n

    \n
    \ndef p_error(p):\n    # Read ahead looking for a terminating \";\"\n    while True:\n        tok = parser.token()             # Get the next token\n        if not tok or tok.type == 'SEMI': break\n    parser.errok()\n\n    # Return SEMI to the parser as the next lookahead token\n    return tok  \n
    \n
    \n\n

    \nKeep in mind in that the above error handling functions,\nparser is an instance of the parser created by\nyacc(). You'll need to save this instance someplace in your\ncode so that you can refer to it during error handling.\n

    \n\n

    6.8.3 Signalling an error from a production

    \n\n\nIf necessary, a production rule can manually force the parser to enter error recovery. This\nis done by raising the SyntaxError exception like this:\n\n
    \n
    \ndef p_production(p):\n    'production : some production ...'\n    raise SyntaxError\n
    \n
    \n\nThe effect of raising SyntaxError is the same as if the last symbol shifted onto the\nparsing stack was actually a syntax error. Thus, when you do this, the last symbol shifted is popped off\nof the parsing stack and the current lookahead token is set to an error token. The parser\nthen enters error-recovery mode where it tries to reduce rules that can accept error tokens. \nThe steps that follow from this point are exactly the same as if a syntax error were detected and \np_error() were called.\n\n

    \nOne important aspect of manually setting an error is that the p_error() function will NOT be\ncalled in this case. If you need to issue an error message, make sure you do it in the production that\nraises SyntaxError.\n\n

    \nNote: This feature of PLY is meant to mimic the behavior of the YYERROR macro in yacc.\n\n

    6.8.4 When Do Syntax Errors Get Reported

    \n\n\n

    \nIn most cases, yacc will handle errors as soon as a bad input token is\ndetected on the input. However, be aware that yacc may choose to\ndelay error handling until after it has reduced one or more grammar\nrules first. This behavior might be unexpected, but it's related to\nspecial states in the underlying parsing table known as \"defaulted\nstates.\" A defaulted state is parsing condition where the same\ngrammar rule will be reduced regardless of what valid token\ncomes next on the input. For such states, yacc chooses to go ahead\nand reduce the grammar rule without reading the next input\ntoken. If the next token is bad, yacc will eventually get around to reading it and \nreport a syntax error. It's just a little unusual in that you might\nsee some of your grammar rules firing immediately prior to the syntax \nerror.\n

    \n\n

    \nUsually, the delayed error reporting with defaulted states is harmless\n(and there are other reasons for wanting PLY to behave in this way).\nHowever, if you need to turn this behavior off for some reason. You\ncan clear the defaulted states table like this:\n

    \n\n
    \n
    \nparser = yacc.yacc()\nparser.defaulted_states = {}\n
    \n
    \n\n

    \nDisabling defaulted states is not recommended if your grammar makes use\nof embedded actions as described in Section 6.11.

    \n\n

    6.8.5 General comments on error handling

    \n\n\nFor normal types of languages, error recovery with error rules and resynchronization characters is probably the most reliable\ntechnique. This is because you can instrument the grammar to catch errors at selected places where it is relatively easy \nto recover and continue parsing. Panic mode recovery is really only useful in certain specialized applications where you might want\nto discard huge portions of the input text to find a valid restart point.\n\n

    6.9 Line Number and Position Tracking

    \n\n\nPosition tracking is often a tricky problem when writing compilers.\nBy default, PLY tracks the line number and position of all tokens.\nThis information is available using the following functions:\n\n
      \n
    • p.lineno(num). Return the line number for symbol num\n
    • p.lexpos(num). Return the lexing position for symbol num\n
    \n\nFor example:\n\n
    \n
    \ndef p_expression(p):\n    'expression : expression PLUS expression'\n    line   = p.lineno(2)        # line number of the PLUS token\n    index  = p.lexpos(2)        # Position of the PLUS token\n
    \n
    \n\nAs an optional feature, yacc.py can automatically track line\nnumbers and positions for all of the grammar symbols as well.\nHowever, this extra tracking requires extra processing and can\nsignificantly slow down parsing. Therefore, it must be enabled by\npassing the\ntracking=True option to yacc.parse(). For example:\n\n
    \n
    \nyacc.parse(data,tracking=True)\n
    \n
    \n\nOnce enabled, the lineno() and lexpos() methods work\nfor all grammar symbols. In addition, two additional methods can be\nused:\n\n
      \n
    • p.linespan(num). Return a tuple (startline,endline) with the starting and ending line number for symbol num.\n
    • p.lexspan(num). Return a tuple (start,end) with the starting and ending positions for symbol num.\n
    \n\nFor example:\n\n
    \n
    \ndef p_expression(p):\n    'expression : expression PLUS expression'\n    p.lineno(1)        # Line number of the left expression\n    p.lineno(2)        # line number of the PLUS operator\n    p.lineno(3)        # line number of the right expression\n    ...\n    start,end = p.linespan(3)    # Start,end lines of the right expression\n    starti,endi = p.lexspan(3)   # Start,end positions of right expression\n\n
    \n
    \n\nNote: The lexspan() function only returns the range of values up to the start of the last grammar symbol. \n\n

    \nAlthough it may be convenient for PLY to track position information on\nall grammar symbols, this is often unnecessary. For example, if you\nare merely using line number information in an error message, you can\noften just key off of a specific token in the grammar rule. For\nexample:\n\n

    \n
    \ndef p_bad_func(p):\n    'funccall : fname LPAREN error RPAREN'\n    # Line number reported from LPAREN token\n    print(\"Bad function call at line\", p.lineno(2))\n
    \n
    \n\n

    \nSimilarly, you may get better parsing performance if you only\nselectively propagate line number information where it's needed using\nthe p.set_lineno() method. For example:\n\n

    \n
    \ndef p_fname(p):\n    'fname : ID'\n    p[0] = p[1]\n    p.set_lineno(0,p.lineno(1))\n
    \n
    \n\nPLY doesn't retain line number information from rules that have already been\nparsed. If you are building an abstract syntax tree and need to have line numbers,\nyou should make sure that the line numbers appear in the tree itself.\n\n

    6.10 AST Construction

    \n\n\nyacc.py provides no special functions for constructing an\nabstract syntax tree. However, such construction is easy enough to do\non your own. \n\n

    A minimal way to construct a tree is to simply create and\npropagate a tuple or list in each grammar rule function. There\nare many possible ways to do this, but one example would be something\nlike this:\n\n

    \n
    \ndef p_expression_binop(p):\n    '''expression : expression PLUS expression\n                  | expression MINUS expression\n                  | expression TIMES expression\n                  | expression DIVIDE expression'''\n\n    p[0] = ('binary-expression',p[2],p[1],p[3])\n\ndef p_expression_group(p):\n    'expression : LPAREN expression RPAREN'\n    p[0] = ('group-expression',p[2])\n\ndef p_expression_number(p):\n    'expression : NUMBER'\n    p[0] = ('number-expression',p[1])\n
    \n
    \n\n

    \nAnother approach is to create a set of data structure for different\nkinds of abstract syntax tree nodes and assign nodes to p[0]\nin each rule. For example:\n\n

    \n
    \nclass Expr: pass\n\nclass BinOp(Expr):\n    def __init__(self,left,op,right):\n        self.type = \"binop\"\n        self.left = left\n        self.right = right\n        self.op = op\n\nclass Number(Expr):\n    def __init__(self,value):\n        self.type = \"number\"\n        self.value = value\n\ndef p_expression_binop(p):\n    '''expression : expression PLUS expression\n                  | expression MINUS expression\n                  | expression TIMES expression\n                  | expression DIVIDE expression'''\n\n    p[0] = BinOp(p[1],p[2],p[3])\n\ndef p_expression_group(p):\n    'expression : LPAREN expression RPAREN'\n    p[0] = p[2]\n\ndef p_expression_number(p):\n    'expression : NUMBER'\n    p[0] = Number(p[1])\n
    \n
    \n\nThe advantage to this approach is that it may make it easier to attach more complicated\nsemantics, type checking, code generation, and other features to the node classes.\n\n

    \nTo simplify tree traversal, it may make sense to pick a very generic\ntree structure for your parse tree nodes. For example:\n\n

    \n
    \nclass Node:\n    def __init__(self,type,children=None,leaf=None):\n         self.type = type\n         if children:\n              self.children = children\n         else:\n              self.children = [ ]\n         self.leaf = leaf\n\t \ndef p_expression_binop(p):\n    '''expression : expression PLUS expression\n                  | expression MINUS expression\n                  | expression TIMES expression\n                  | expression DIVIDE expression'''\n\n    p[0] = Node(\"binop\", [p[1],p[3]], p[2])\n
    \n
    \n\n

    6.11 Embedded Actions

    \n\n\nThe parsing technique used by yacc only allows actions to be executed at the end of a rule. For example,\nsuppose you have a rule like this:\n\n
    \n
    \ndef p_foo(p):\n    \"foo : A B C D\"\n    print(\"Parsed a foo\", p[1],p[2],p[3],p[4])\n
    \n
    \n\n

    \nIn this case, the supplied action code only executes after all of the\nsymbols A, B, C, and D have been\nparsed. Sometimes, however, it is useful to execute small code\nfragments during intermediate stages of parsing. For example, suppose\nyou wanted to perform some action immediately after A has\nbeen parsed. To do this, write an empty rule like this:\n\n

    \n
    \ndef p_foo(p):\n    \"foo : A seen_A B C D\"\n    print(\"Parsed a foo\", p[1],p[3],p[4],p[5])\n    print(\"seen_A returned\", p[2])\n\ndef p_seen_A(p):\n    \"seen_A :\"\n    print(\"Saw an A = \", p[-1])   # Access grammar symbol to left\n    p[0] = some_value            # Assign value to seen_A\n\n
    \n
    \n\n

    \nIn this example, the empty seen_A rule executes immediately\nafter A is shifted onto the parsing stack. Within this\nrule, p[-1] refers to the symbol on the stack that appears\nimmediately to the left of the seen_A symbol. In this case,\nit would be the value of A in the foo rule\nimmediately above. Like other rules, a value can be returned from an\nembedded action by simply assigning it to p[0]\n\n

    \nThe use of embedded actions can sometimes introduce extra shift/reduce conflicts. For example,\nthis grammar has no conflicts:\n\n

    \n
    \ndef p_foo(p):\n    \"\"\"foo : abcd\n           | abcx\"\"\"\n\ndef p_abcd(p):\n    \"abcd : A B C D\"\n\ndef p_abcx(p):\n    \"abcx : A B C X\"\n
    \n
    \n\nHowever, if you insert an embedded action into one of the rules like this,\n\n
    \n
    \ndef p_foo(p):\n    \"\"\"foo : abcd\n           | abcx\"\"\"\n\ndef p_abcd(p):\n    \"abcd : A B C D\"\n\ndef p_abcx(p):\n    \"abcx : A B seen_AB C X\"\n\ndef p_seen_AB(p):\n    \"seen_AB :\"\n
    \n
    \n\nan extra shift-reduce conflict will be introduced. This conflict is\ncaused by the fact that the same symbol C appears next in\nboth the abcd and abcx rules. The parser can either\nshift the symbol (abcd rule) or reduce the empty\nrule seen_AB (abcx rule).\n\n

    \nA common use of embedded rules is to control other aspects of parsing\nsuch as scoping of local variables. For example, if you were parsing C code, you might\nwrite code like this:\n\n

    \n
    \ndef p_statements_block(p):\n    \"statements: LBRACE new_scope statements RBRACE\"\"\"\n    # Action code\n    ...\n    pop_scope()        # Return to previous scope\n\ndef p_new_scope(p):\n    \"new_scope :\"\n    # Create a new scope for local variables\n    s = new_scope()\n    push_scope(s)\n    ...\n
    \n
    \n\nIn this case, the embedded action new_scope executes\nimmediately after a LBRACE ({) symbol is parsed.\nThis might adjust internal symbol tables and other aspects of the\nparser. Upon completion of the rule statements_block, code\nmight undo the operations performed in the embedded action\n(e.g., pop_scope()).\n\n

    6.12 Miscellaneous Yacc Notes

    \n\n\n
      \n\n
    • By default, yacc.py relies on lex.py for tokenizing. However, an alternative tokenizer\ncan be supplied as follows:\n\n
      \n
      \nparser = yacc.parse(lexer=x)\n
      \n
      \nin this case, x must be a Lexer object that minimally has a x.token() method for retrieving the next\ntoken. If an input string is given to yacc.parse(), the lexer must also have an x.input() method.\n\n

      \n

    • By default, the yacc generates tables in debugging mode (which produces the parser.out file and other output).\nTo disable this, use\n\n
      \n
      \nparser = yacc.yacc(debug=False)\n
      \n
      \n\n

      \n

    • To change the name of the parsetab.py file, use:\n\n
      \n
      \nparser = yacc.yacc(tabmodule=\"foo\")\n
      \n
      \n\n

      \nNormally, the parsetab.py file is placed into the same directory as\nthe module where the parser is defined. If you want it to go somewhere else, you can\ngiven an absolute package name for tabmodule instead. In that case, the \ntables will be written there.\n

      \n\n

      \n

    • To change the directory in which the parsetab.py file (and other output files) are written, use:\n
      \n
      \nparser = yacc.yacc(tabmodule=\"foo\",outputdir=\"somedirectory\")\n
      \n
      \n\n

      \nNote: Be aware that unless the directory specified is also on Python's path (sys.path), subsequent\nimports of the table file will fail. As a general rule, it's better to specify a destination using the\ntabmodule argument instead of directly specifying a directory using the outputdir argument.\n

      \n\n

      \n

    • To prevent yacc from generating any kind of parser table file, use:\n
      \n
      \nparser = yacc.yacc(write_tables=False)\n
      \n
      \n\nNote: If you disable table generation, yacc() will regenerate the parsing tables\neach time it runs (which may take awhile depending on how large your grammar is).\n\n

      \n

    • To print copious amounts of debugging during parsing, use:\n\n
      \n
      \nparser.parse(input_text, debug=True)     \n
      \n
      \n\n

      \n

    • Since the generation of the LALR tables is relatively expensive, previously generated tables are\ncached and reused if possible. The decision to regenerate the tables is determined by taking an MD5\nchecksum of all grammar rules and precedence rules. Only in the event of a mismatch are the tables regenerated.\n\n

      \nIt should be noted that table generation is reasonably efficient, even for grammars that involve around a 100 rules\nand several hundred states.

      • \n\n\n

      \n

    • Since LR parsing is driven by tables, the performance of the parser is largely independent of the\nsize of the grammar. The biggest bottlenecks will be the lexer and the complexity of the code in your grammar rules.\n
      • \n

      \n\n

      \n

    • yacc() also allows parsers to be defined as classes and as closures (see the section on alternative specification of\nlexers). However, be aware that only one parser may be defined in a single module (source file). There are various \nerror checks and validation steps that may issue confusing error messages if you try to define multiple parsers\nin the same source file.\n
      • \n

      \n\n

      \n

    • Decorators of production rules have to update the wrapped function's line number. wrapper.co_firstlineno = func.__code__.co_firstlineno:\n\n
      \n
      \nfrom functools import wraps\nfrom nodes import Collection\n\n\ndef strict(*types):\n    def decorate(func):\n        @wraps(func)\n        def wrapper(p):\n            func(p)\n            if not isinstance(p[0], types):\n                raise TypeError\n\n        wrapper.co_firstlineno = func.__code__.co_firstlineno\n        return wrapper\n\n    return decorate\n\n@strict(Collection)\ndef p_collection(p):\n    \"\"\"\n    collection  : sequence\n                | map\n    \"\"\"\n    p[0] = p[1]\n
      \n
      \n\n
      • \n

      \n\n\n
    \n

    \n\n\n

    7. Multiple Parsers and Lexers

    \n\n\nIn advanced parsing applications, you may want to have multiple\nparsers and lexers. \n\n

    \nAs a general rules this isn't a problem. However, to make it work,\nyou need to carefully make sure everything gets hooked up correctly.\nFirst, make sure you save the objects returned by lex() and\nyacc(). For example:\n\n

    \n
    \nlexer  = lex.lex()       # Return lexer object\nparser = yacc.yacc()     # Return parser object\n
    \n
    \n\nNext, when parsing, make sure you give the parse() function a reference to the lexer it\nshould be using. For example:\n\n
    \n
    \nparser.parse(text,lexer=lexer)\n
    \n
    \n\nIf you forget to do this, the parser will use the last lexer\ncreated--which is not always what you want.\n\n

    \nWithin lexer and parser rule functions, these objects are also\navailable. In the lexer, the \"lexer\" attribute of a token refers to\nthe lexer object that triggered the rule. For example:\n\n

    \n
    \ndef t_NUMBER(t):\n   r'\\d+'\n   ...\n   print(t.lexer)           # Show lexer object\n
    \n
    \n\nIn the parser, the \"lexer\" and \"parser\" attributes refer to the lexer\nand parser objects respectively.\n\n
    \n
    \ndef p_expr_plus(p):\n   'expr : expr PLUS expr'\n   ...\n   print(p.parser)          # Show parser object\n   print(p.lexer)           # Show lexer object\n
    \n
    \n\nIf necessary, arbitrary attributes can be attached to the lexer or parser object.\nFor example, if you wanted to have different parsing modes, you could attach a mode\nattribute to the parser object and look at it later.\n\n

    8. Using Python's Optimized Mode

    \n\n\nBecause PLY uses information from doc-strings, parsing and lexing\ninformation must be gathered while running the Python interpreter in\nnormal mode (i.e., not with the -O or -OO options). However, if you\nspecify optimized mode like this:\n\n
    \n
    \nlex.lex(optimize=1)\nyacc.yacc(optimize=1)\n
    \n
    \n\nthen PLY can later be used when Python runs in optimized mode. To make this work,\nmake sure you first run Python in normal mode. Once the lexing and parsing tables\nhave been generated the first time, run Python in optimized mode. PLY will use\nthe tables without the need for doc strings.\n\n

    \nBeware: running PLY in optimized mode disables a lot of error\nchecking. You should only do this when your project has stabilized\nand you don't need to do any debugging. One of the purposes of\noptimized mode is to substantially decrease the startup time of\nyour compiler (by assuming that everything is already properly\nspecified and works).\n\n

    9. Advanced Debugging

    \n\n\n

    \nDebugging a compiler is typically not an easy task. PLY provides some\nadvanced diagostic capabilities through the use of Python's\nlogging module. The next two sections describe this:\n\n

    9.1 Debugging the lex() and yacc() commands

    \n\n\n

    \nBoth the lex() and yacc() commands have a debugging\nmode that can be enabled using the debug flag. For example:\n\n

    \n
    \nlex.lex(debug=True)\nyacc.yacc(debug=True)\n
    \n
    \n\nNormally, the output produced by debugging is routed to either\nstandard error or, in the case of yacc(), to a file\nparser.out. This output can be more carefully controlled\nby supplying a logging object. Here is an example that adds\ninformation about where different debugging messages are coming from:\n\n
    \n
    \n# Set up a logging object\nimport logging\nlogging.basicConfig(\n    level = logging.DEBUG,\n    filename = \"parselog.txt\",\n    filemode = \"w\",\n    format = \"%(filename)10s:%(lineno)4d:%(message)s\"\n)\nlog = logging.getLogger()\n\nlex.lex(debug=True,debuglog=log)\nyacc.yacc(debug=True,debuglog=log)\n
    \n
    \n\nIf you supply a custom logger, the amount of debugging\ninformation produced can be controlled by setting the logging level.\nTypically, debugging messages are either issued at the DEBUG,\nINFO, or WARNING levels.\n\n

    \nPLY's error messages and warnings are also produced using the logging\ninterface. This can be controlled by passing a logging object\nusing the errorlog parameter.\n\n

    \n
    \nlex.lex(errorlog=log)\nyacc.yacc(errorlog=log)\n
    \n
    \n\nIf you want to completely silence warnings, you can either pass in a\nlogging object with an appropriate filter level or use the NullLogger\nobject defined in either lex or yacc. For example:\n\n
    \n
    \nyacc.yacc(errorlog=yacc.NullLogger())\n
    \n
    \n\n

    9.2 Run-time Debugging

    \n\n\n

    \nTo enable run-time debugging of a parser, use the debug option to parse. This\noption can either be an integer (which simply turns debugging on or off) or an instance\nof a logger object. For example:\n\n

    \n
    \nlog = logging.getLogger()\nparser.parse(input,debug=log)\n
    \n
    \n\nIf a logging object is passed, you can use its filtering level to control how much\noutput gets generated. The INFO level is used to produce information\nabout rule reductions. The DEBUG level will show information about the\nparsing stack, token shifts, and other details. The ERROR level shows information\nrelated to parsing errors.\n\n

    \nFor very complicated problems, you should pass in a logging object that\nredirects to a file where you can more easily inspect the output after\nexecution.\n\n

    10. Packaging Advice

    \n\n\n

    \nIf you are distributing a package that makes use of PLY, you should\nspend a few moments thinking about how you want to handle the files\nthat are automatically generated. For example, the parsetab.py\nfile generated by the yacc() function.

    \n\n

    \nStarting in PLY-3.6, the table files are created in the same directory\nas the file where a parser is defined. This means that the\nparsetab.py file will live side-by-side with your parser\nspecification. In terms of packaging, this is probably the easiest and\nmost sane approach to manage. You don't need to give yacc()\nany extra arguments and it should just \"work.\"

    \n\n

    \nOne concern is the management of the parsetab.py file itself.\nFor example, should you have this file checked into version control (e.g., GitHub),\nshould it be included in a package distribution as a normal file, or should you\njust let PLY generate it automatically for the user when they install your package?\n

    \n\n

    \nAs of PLY-3.6, the parsetab.py file should be compatible across all versions\nof Python including Python 2 and 3. Thus, a table file generated in Python 2 should\nwork fine if it's used on Python 3. Because of this, it should be relatively harmless \nto distribute the parsetab.py file yourself if you need to. However, be aware\nthat older/newer versions of PLY may try to regenerate the file if there are future \nenhancements or changes to its format.\n

    \n\n

    \nTo make the generation of table files easier for the purposes of installation, you might\nway to make your parser files executable using the -m option or similar. For\nexample:\n

    \n\n
    \n
    \n# calc.py\n...\n...\ndef make_parser():\n    parser = yacc.yacc()\n    return parser\n\nif __name__ == '__main__':\n    make_parser()\n
    \n
    \n\n

    \nYou can then use a command such as python -m calc.py to generate the tables. Alternatively,\na setup.py script, can import the module and use make_parser() to create the\nparsing tables.\n

    \n\n

    \nIf you're willing to sacrifice a little startup time, you can also instruct PLY to never write the\ntables using yacc.yacc(write_tables=False, debug=False). In this mode, PLY will regenerate\nthe parsing tables from scratch each time. For a small grammar, you probably won't notice. For a \nlarge grammar, you should probably reconsider--the parsing tables are meant to dramatically speed up this process.\n

    \n\n

    \nDuring operation, it is normal for PLY to produce diagnostic error\nmessages (usually printed to standard error). These are generated\nentirely using the logging module. If you want to redirect\nthese messages or silence them, you can provide your own logging\nobject to yacc(). For example:\n

    \n\n
    \n
    \nimport logging\nlog = logging.getLogger('ply')\n...\nparser = yacc.yacc(errorlog=log)\n
    \n
    \n\n

    11. Where to go from here?

    \n\n\nThe examples directory of the PLY distribution contains several simple examples. Please consult a\ncompilers textbook for the theory and underlying implementation details or LR parsing.\n\n\n\n\n\n\n\n\n\n\n" }, { "path": "xonsh/ply/example/BASIC/README", "content": "Inspired by a September 14, 2006 Salon article \"Why Johnny Can't Code\" by\nDavid Brin (http://www.salon.com/tech/feature/2006/09/14/basic/index.html),\nI thought that a fully working BASIC interpreter might be an interesting,\nif not questionable, PLY example. Uh, okay, so maybe it's just a bad idea,\nbut in any case, here it is.\n\nIn this example, you'll find a rough implementation of 1964 Dartmouth BASIC\nas described in the manual at:\n\n http://www.bitsavers.org/pdf/dartmouth/BASIC_Oct64.pdf\n\nSee also:\n\n http://en.wikipedia.org/wiki/Dartmouth_BASIC\n\nThis dialect is downright primitive---there are no string variables\nand no facilities for interactive input. Moreover, subroutines and functions\nare brain-dead even more than they usually are for BASIC. Of course,\nthe GOTO statement is provided.\n\nNevertheless, there are a few interesting aspects of this example:\n\n - It illustrates a fully working interpreter including lexing, parsing,\n and interpretation of instructions.\n \n - The parser shows how to catch and report various kinds of parsing\n errors in a more graceful way.\n\n - The example both parses files (supplied on command line) and\n interactive input entered line by line.\n\n - It shows how you might represent parsed information. In this case,\n each BASIC statement is encoded into a Python tuple containing the\n statement type and parameters. These tuples are then stored in\n a dictionary indexed by program line numbers.\n\n - Even though it's just BASIC, the parser contains more than 80\n rules and 150 parsing states. Thus, it's a little more meaty than\n the calculator example.\n\nTo use the example, run it as follows:\n\n % python basic.py hello.bas\n HELLO WORLD\n %\n\nor use it interactively:\n\n % python basic.py\n [BASIC] 10 PRINT \"HELLO WORLD\"\n [BASIC] 20 END\n [BASIC] RUN\n HELLO WORLD\n [BASIC]\n\nThe following files are defined:\n\n basic.py - High level script that controls everything\n basiclex.py - BASIC tokenizer\n basparse.py - BASIC parser\n basinterp.py - BASIC interpreter that runs parsed programs.\n\nIn addition, a number of sample BASIC programs (.bas suffix) are\nprovided. These were taken out of the Dartmouth manual.\n\nDisclaimer: I haven't spent a ton of time testing this and it's likely that\nI've skimped here and there on a few finer details (e.g., strictly enforcing\nvariable naming rules). However, the interpreter seems to be able to run\nthe examples in the BASIC manual.\n\nHave fun!\n\n-Dave\n\n\n\n\n\n\n" }, { "path": "xonsh/ply/example/BASIC/basic.py", "content": "# An implementation of Dartmouth BASIC (1964)\n#\n\nimport sys\nsys.path.insert(0, \"../..\")\n\nif sys.version_info[0] >= 3:\n raw_input = input\n\nimport basiclex\nimport basparse\nimport basinterp\n\n# If a filename has been specified, we try to run it.\n# If a runtime error occurs, we bail out and enter\n# interactive mode below\nif len(sys.argv) == 2:\n with open(sys.argv[1]) as f:\n data = f.read()\n prog = basparse.parse(data)\n if not prog:\n raise SystemExit\n b = basinterp.BasicInterpreter(prog)\n try:\n b.run()\n raise SystemExit\n except RuntimeError:\n pass\n\nelse:\n b = basinterp.BasicInterpreter({})\n\n# Interactive mode. This incrementally adds/deletes statements\n# from the program stored in the BasicInterpreter object. In\n# addition, special commands 'NEW','LIST',and 'RUN' are added.\n# Specifying a line number with no code deletes that line from\n# the program.\n\nwhile 1:\n try:\n line = raw_input(\"[BASIC] \")\n except EOFError:\n raise SystemExit\n if not line:\n continue\n line += \"\\n\"\n prog = basparse.parse(line)\n if not prog:\n continue\n\n keys = list(prog)\n if keys[0] > 0:\n b.add_statements(prog)\n else:\n stat = prog[keys[0]]\n if stat[0] == 'RUN':\n try:\n b.run()\n except RuntimeError:\n pass\n elif stat[0] == 'LIST':\n b.list()\n elif stat[0] == 'BLANK':\n b.del_line(stat[1])\n elif stat[0] == 'NEW':\n b.new()\n" }, { "path": "xonsh/ply/example/BASIC/basiclex.py", "content": "# An implementation of Dartmouth BASIC (1964)\n\nfrom ply import *\n\nkeywords = (\n 'LET', 'READ', 'DATA', 'PRINT', 'GOTO', 'IF', 'THEN', 'FOR', 'NEXT', 'TO', 'STEP',\n 'END', 'STOP', 'DEF', 'GOSUB', 'DIM', 'REM', 'RETURN', 'RUN', 'LIST', 'NEW',\n)\n\ntokens = keywords + (\n 'EQUALS', 'PLUS', 'MINUS', 'TIMES', 'DIVIDE', 'POWER',\n 'LPAREN', 'RPAREN', 'LT', 'LE', 'GT', 'GE', 'NE',\n 'COMMA', 'SEMI', 'INTEGER', 'FLOAT', 'STRING',\n 'ID', 'NEWLINE'\n)\n\nt_ignore = ' \\t'\n\n\ndef t_REM(t):\n r'REM .*'\n return t\n\n\ndef t_ID(t):\n r'[A-Z][A-Z0-9]*'\n if t.value in keywords:\n t.type = t.value\n return t\n\nt_EQUALS = r'='\nt_PLUS = r'\\+'\nt_MINUS = r'-'\nt_TIMES = r'\\*'\nt_POWER = r'\\^'\nt_DIVIDE = r'/'\nt_LPAREN = r'\\('\nt_RPAREN = r'\\)'\nt_LT = r'<'\nt_LE = r'<='\nt_GT = r'>'\nt_GE = r'>='\nt_NE = r'<>'\nt_COMMA = r'\\,'\nt_SEMI = r';'\nt_INTEGER = r'\\d+'\nt_FLOAT = r'((\\d*\\.\\d+)(E[\\+-]?\\d+)?|([1-9]\\d*E[\\+-]?\\d+))'\nt_STRING = r'\\\".*?\\\"'\n\n\ndef t_NEWLINE(t):\n r'\\n'\n t.lexer.lineno += 1\n return t\n\n\ndef t_error(t):\n print(\"Illegal character %s\" % t.value[0])\n t.lexer.skip(1)\n\nlex.lex(debug=0)\n" }, { "path": "xonsh/ply/example/BASIC/basiclog.py", "content": "# An implementation of Dartmouth BASIC (1964)\n#\n\nimport sys\nsys.path.insert(0, \"../..\")\n\nif sys.version_info[0] >= 3:\n raw_input = input\n\nimport logging\nlogging.basicConfig(\n level=logging.INFO,\n filename=\"parselog.txt\",\n filemode=\"w\"\n)\nlog = logging.getLogger()\n\nimport basiclex\nimport basparse\nimport basinterp\n\n# If a filename has been specified, we try to run it.\n# If a runtime error occurs, we bail out and enter\n# interactive mode below\nif len(sys.argv) == 2:\n with open(sys.argv[1]) as f:\n data = f.read()\n prog = basparse.parse(data, debug=log)\n if not prog:\n raise SystemExit\n b = basinterp.BasicInterpreter(prog)\n try:\n b.run()\n raise SystemExit\n except RuntimeError:\n pass\n\nelse:\n b = basinterp.BasicInterpreter({})\n\n# Interactive mode. This incrementally adds/deletes statements\n# from the program stored in the BasicInterpreter object. In\n# addition, special commands 'NEW','LIST',and 'RUN' are added.\n# Specifying a line number with no code deletes that line from\n# the program.\n\nwhile 1:\n try:\n line = raw_input(\"[BASIC] \")\n except EOFError:\n raise SystemExit\n if not line:\n continue\n line += \"\\n\"\n prog = basparse.parse(line, debug=log)\n if not prog:\n continue\n\n keys = list(prog)\n if keys[0] > 0:\n b.add_statements(prog)\n else:\n stat = prog[keys[0]]\n if stat[0] == 'RUN':\n try:\n b.run()\n except RuntimeError:\n pass\n elif stat[0] == 'LIST':\n b.list()\n elif stat[0] == 'BLANK':\n b.del_line(stat[1])\n elif stat[0] == 'NEW':\n b.new()\n" }, { "path": "xonsh/ply/example/BASIC/basinterp.py", "content": "# This file provides the runtime support for running a basic program\n# Assumes the program has been parsed using basparse.py\n\nimport sys\nimport math\nimport random\n\n\nclass BasicInterpreter:\n\n # Initialize the interpreter. prog is a dictionary\n # containing (line,statement) mappings\n def __init__(self, prog):\n self.prog = prog\n\n self.functions = { # Built-in function table\n 'SIN': lambda z: math.sin(self.eval(z)),\n 'COS': lambda z: math.cos(self.eval(z)),\n 'TAN': lambda z: math.tan(self.eval(z)),\n 'ATN': lambda z: math.atan(self.eval(z)),\n 'EXP': lambda z: math.exp(self.eval(z)),\n 'ABS': lambda z: abs(self.eval(z)),\n 'LOG': lambda z: math.log(self.eval(z)),\n 'SQR': lambda z: math.sqrt(self.eval(z)),\n 'INT': lambda z: int(self.eval(z)),\n 'RND': lambda z: random.random()\n }\n\n # Collect all data statements\n def collect_data(self):\n self.data = []\n for lineno in self.stat:\n if self.prog[lineno][0] == 'DATA':\n self.data = self.data + self.prog[lineno][1]\n self.dc = 0 # Initialize the data counter\n\n # Check for end statements\n def check_end(self):\n has_end = 0\n for lineno in self.stat:\n if self.prog[lineno][0] == 'END' and not has_end:\n has_end = lineno\n if not has_end:\n print(\"NO END INSTRUCTION\")\n self.error = 1\n return\n if has_end != lineno:\n print(\"END IS NOT LAST\")\n self.error = 1\n\n # Check loops\n def check_loops(self):\n for pc in range(len(self.stat)):\n lineno = self.stat[pc]\n if self.prog[lineno][0] == 'FOR':\n forinst = self.prog[lineno]\n loopvar = forinst[1]\n for i in range(pc + 1, len(self.stat)):\n if self.prog[self.stat[i]][0] == 'NEXT':\n nextvar = self.prog[self.stat[i]][1]\n if nextvar != loopvar:\n continue\n self.loopend[pc] = i\n break\n else:\n print(\"FOR WITHOUT NEXT AT LINE %s\" % self.stat[pc])\n self.error = 1\n\n # Evaluate an expression\n def eval(self, expr):\n etype = expr[0]\n if etype == 'NUM':\n return expr[1]\n elif etype == 'GROUP':\n return self.eval(expr[1])\n elif etype == 'UNARY':\n if expr[1] == '-':\n return -self.eval(expr[2])\n elif etype == 'BINOP':\n if expr[1] == '+':\n return self.eval(expr[2]) + self.eval(expr[3])\n elif expr[1] == '-':\n return self.eval(expr[2]) - self.eval(expr[3])\n elif expr[1] == '*':\n return self.eval(expr[2]) * self.eval(expr[3])\n elif expr[1] == '/':\n return float(self.eval(expr[2])) / self.eval(expr[3])\n elif expr[1] == '^':\n return abs(self.eval(expr[2]))**self.eval(expr[3])\n elif etype == 'VAR':\n var, dim1, dim2 = expr[1]\n if not dim1 and not dim2:\n if var in self.vars:\n return self.vars[var]\n else:\n print(\"UNDEFINED VARIABLE %s AT LINE %s\" %\n (var, self.stat[self.pc]))\n raise RuntimeError\n # May be a list lookup or a function evaluation\n if dim1 and not dim2:\n if var in self.functions:\n # A function\n return self.functions[var](dim1)\n else:\n # A list evaluation\n if var in self.lists:\n dim1val = self.eval(dim1)\n if dim1val < 1 or dim1val > len(self.lists[var]):\n print(\"LIST INDEX OUT OF BOUNDS AT LINE %s\" %\n self.stat[self.pc])\n raise RuntimeError\n return self.lists[var][dim1val - 1]\n if dim1 and dim2:\n if var in self.tables:\n dim1val = self.eval(dim1)\n dim2val = self.eval(dim2)\n if dim1val < 1 or dim1val > len(self.tables[var]) or dim2val < 1 or dim2val > len(self.tables[var][0]):\n print(\"TABLE INDEX OUT OUT BOUNDS AT LINE %s\" %\n self.stat[self.pc])\n raise RuntimeError\n return self.tables[var][dim1val - 1][dim2val - 1]\n print(\"UNDEFINED VARIABLE %s AT LINE %s\" %\n (var, self.stat[self.pc]))\n raise RuntimeError\n\n # Evaluate a relational expression\n def releval(self, expr):\n etype = expr[1]\n lhs = self.eval(expr[2])\n rhs = self.eval(expr[3])\n if etype == '<':\n if lhs < rhs:\n return 1\n else:\n return 0\n\n elif etype == '<=':\n if lhs <= rhs:\n return 1\n else:\n return 0\n\n elif etype == '>':\n if lhs > rhs:\n return 1\n else:\n return 0\n\n elif etype == '>=':\n if lhs >= rhs:\n return 1\n else:\n return 0\n\n elif etype == '=':\n if lhs == rhs:\n return 1\n else:\n return 0\n\n elif etype == '<>':\n if lhs != rhs:\n return 1\n else:\n return 0\n\n # Assignment\n def assign(self, target, value):\n var, dim1, dim2 = target\n if not dim1 and not dim2:\n self.vars[var] = self.eval(value)\n elif dim1 and not dim2:\n # List assignment\n dim1val = self.eval(dim1)\n if not var in self.lists:\n self.lists[var] = [0] * 10\n\n if dim1val > len(self.lists[var]):\n print (\"DIMENSION TOO LARGE AT LINE %s\" % self.stat[self.pc])\n raise RuntimeError\n self.lists[var][dim1val - 1] = self.eval(value)\n elif dim1 and dim2:\n dim1val = self.eval(dim1)\n dim2val = self.eval(dim2)\n if not var in self.tables:\n temp = [0] * 10\n v = []\n for i in range(10):\n v.append(temp[:])\n self.tables[var] = v\n # Variable already exists\n if dim1val > len(self.tables[var]) or dim2val > len(self.tables[var][0]):\n print(\"DIMENSION TOO LARGE AT LINE %s\" % self.stat[self.pc])\n raise RuntimeError\n self.tables[var][dim1val - 1][dim2val - 1] = self.eval(value)\n\n # Change the current line number\n def goto(self, linenum):\n if not linenum in self.prog:\n print(\"UNDEFINED LINE NUMBER %d AT LINE %d\" %\n (linenum, self.stat[self.pc]))\n raise RuntimeError\n self.pc = self.stat.index(linenum)\n\n # Run it\n def run(self):\n self.vars = {} # All variables\n self.lists = {} # List variables\n self.tables = {} # Tables\n self.loops = [] # Currently active loops\n self.loopend = {} # Mapping saying where loops end\n self.gosub = None # Gosub return point (if any)\n self.error = 0 # Indicates program error\n\n self.stat = list(self.prog) # Ordered list of all line numbers\n self.stat.sort()\n self.pc = 0 # Current program counter\n\n # Processing prior to running\n\n self.collect_data() # Collect all of the data statements\n self.check_end()\n self.check_loops()\n\n if self.error:\n raise RuntimeError\n\n while 1:\n line = self.stat[self.pc]\n instr = self.prog[line]\n\n op = instr[0]\n\n # END and STOP statements\n if op == 'END' or op == 'STOP':\n break # We're done\n\n # GOTO statement\n elif op == 'GOTO':\n newline = instr[1]\n self.goto(newline)\n continue\n\n # PRINT statement\n elif op == 'PRINT':\n plist = instr[1]\n out = \"\"\n for label, val in plist:\n if out:\n out += ' ' * (15 - (len(out) % 15))\n out += label\n if val:\n if label:\n out += \" \"\n eval = self.eval(val)\n out += str(eval)\n sys.stdout.write(out)\n end = instr[2]\n if not (end == ',' or end == ';'):\n sys.stdout.write(\"\\n\")\n if end == ',':\n sys.stdout.write(\" \" * (15 - (len(out) % 15)))\n if end == ';':\n sys.stdout.write(\" \" * (3 - (len(out) % 3)))\n\n # LET statement\n elif op == 'LET':\n target = instr[1]\n value = instr[2]\n self.assign(target, value)\n\n # READ statement\n elif op == 'READ':\n for target in instr[1]:\n if self.dc < len(self.data):\n value = ('NUM', self.data[self.dc])\n self.assign(target, value)\n self.dc += 1\n else:\n # No more data. Program ends\n return\n elif op == 'IF':\n relop = instr[1]\n newline = instr[2]\n if (self.releval(relop)):\n self.goto(newline)\n continue\n\n elif op == 'FOR':\n loopvar = instr[1]\n initval = instr[2]\n finval = instr[3]\n stepval = instr[4]\n\n # Check to see if this is a new loop\n if not self.loops or self.loops[-1][0] != self.pc:\n # Looks like a new loop. Make the initial assignment\n newvalue = initval\n self.assign((loopvar, None, None), initval)\n if not stepval:\n stepval = ('NUM', 1)\n stepval = self.eval(stepval) # Evaluate step here\n self.loops.append((self.pc, stepval))\n else:\n # It's a repeat of the previous loop\n # Update the value of the loop variable according to the\n # step\n stepval = ('NUM', self.loops[-1][1])\n newvalue = (\n 'BINOP', '+', ('VAR', (loopvar, None, None)), stepval)\n\n if self.loops[-1][1] < 0:\n relop = '>='\n else:\n relop = '<='\n if not self.releval(('RELOP', relop, newvalue, finval)):\n # Loop is done. Jump to the NEXT\n self.pc = self.loopend[self.pc]\n self.loops.pop()\n else:\n self.assign((loopvar, None, None), newvalue)\n\n elif op == 'NEXT':\n if not self.loops:\n print(\"NEXT WITHOUT FOR AT LINE %s\" % line)\n return\n\n nextvar = instr[1]\n self.pc = self.loops[-1][0]\n loopinst = self.prog[self.stat[self.pc]]\n forvar = loopinst[1]\n if nextvar != forvar:\n print(\"NEXT DOESN'T MATCH FOR AT LINE %s\" % line)\n return\n continue\n elif op == 'GOSUB':\n newline = instr[1]\n if self.gosub:\n print(\"ALREADY IN A SUBROUTINE AT LINE %s\" % line)\n return\n self.gosub = self.stat[self.pc]\n self.goto(newline)\n continue\n\n elif op == 'RETURN':\n if not self.gosub:\n print(\"RETURN WITHOUT A GOSUB AT LINE %s\" % line)\n return\n self.goto(self.gosub)\n self.gosub = None\n\n elif op == 'FUNC':\n fname = instr[1]\n pname = instr[2]\n expr = instr[3]\n\n def eval_func(pvalue, name=pname, self=self, expr=expr):\n self.assign((pname, None, None), pvalue)\n return self.eval(expr)\n self.functions[fname] = eval_func\n\n elif op == 'DIM':\n for vname, x, y in instr[1]:\n if y == 0:\n # Single dimension variable\n self.lists[vname] = [0] * x\n else:\n # Double dimension variable\n temp = [0] * y\n v = []\n for i in range(x):\n v.append(temp[:])\n self.tables[vname] = v\n\n self.pc += 1\n\n # Utility functions for program listing\n def expr_str(self, expr):\n etype = expr[0]\n if etype == 'NUM':\n return str(expr[1])\n elif etype == 'GROUP':\n return \"(%s)\" % self.expr_str(expr[1])\n elif etype == 'UNARY':\n if expr[1] == '-':\n return \"-\" + str(expr[2])\n elif etype == 'BINOP':\n return \"%s %s %s\" % (self.expr_str(expr[2]), expr[1], self.expr_str(expr[3]))\n elif etype == 'VAR':\n return self.var_str(expr[1])\n\n def relexpr_str(self, expr):\n return \"%s %s %s\" % (self.expr_str(expr[2]), expr[1], self.expr_str(expr[3]))\n\n def var_str(self, var):\n varname, dim1, dim2 = var\n if not dim1 and not dim2:\n return varname\n if dim1 and not dim2:\n return \"%s(%s)\" % (varname, self.expr_str(dim1))\n return \"%s(%s,%s)\" % (varname, self.expr_str(dim1), self.expr_str(dim2))\n\n # Create a program listing\n def list(self):\n stat = list(self.prog) # Ordered list of all line numbers\n stat.sort()\n for line in stat:\n instr = self.prog[line]\n op = instr[0]\n if op in ['END', 'STOP', 'RETURN']:\n print(\"%s %s\" % (line, op))\n continue\n elif op == 'REM':\n print(\"%s %s\" % (line, instr[1]))\n elif op == 'PRINT':\n _out = \"%s %s \" % (line, op)\n first = 1\n for p in instr[1]:\n if not first:\n _out += \", \"\n if p[0] and p[1]:\n _out += '\"%s\"%s' % (p[0], self.expr_str(p[1]))\n elif p[1]:\n _out += self.expr_str(p[1])\n else:\n _out += '\"%s\"' % (p[0],)\n first = 0\n if instr[2]:\n _out += instr[2]\n print(_out)\n elif op == 'LET':\n print(\"%s LET %s = %s\" %\n (line, self.var_str(instr[1]), self.expr_str(instr[2])))\n elif op == 'READ':\n _out = \"%s READ \" % line\n first = 1\n for r in instr[1]:\n if not first:\n _out += \",\"\n _out += self.var_str(r)\n first = 0\n print(_out)\n elif op == 'IF':\n print(\"%s IF %s THEN %d\" %\n (line, self.relexpr_str(instr[1]), instr[2]))\n elif op == 'GOTO' or op == 'GOSUB':\n print(\"%s %s %s\" % (line, op, instr[1]))\n elif op == 'FOR':\n _out = \"%s FOR %s = %s TO %s\" % (\n line, instr[1], self.expr_str(instr[2]), self.expr_str(instr[3]))\n if instr[4]:\n _out += \" STEP %s\" % (self.expr_str(instr[4]))\n print(_out)\n elif op == 'NEXT':\n print(\"%s NEXT %s\" % (line, instr[1]))\n elif op == 'FUNC':\n print(\"%s DEF %s(%s) = %s\" %\n (line, instr[1], instr[2], self.expr_str(instr[3])))\n elif op == 'DIM':\n _out = \"%s DIM \" % line\n first = 1\n for vname, x, y in instr[1]:\n if not first:\n _out += \",\"\n first = 0\n if y == 0:\n _out += \"%s(%d)\" % (vname, x)\n else:\n _out += \"%s(%d,%d)\" % (vname, x, y)\n\n print(_out)\n elif op == 'DATA':\n _out = \"%s DATA \" % line\n first = 1\n for v in instr[1]:\n if not first:\n _out += \",\"\n first = 0\n _out += v\n print(_out)\n\n # Erase the current program\n def new(self):\n self.prog = {}\n\n # Insert statements\n def add_statements(self, prog):\n for line, stat in prog.items():\n self.prog[line] = stat\n\n # Delete a statement\n def del_line(self, lineno):\n try:\n del self.prog[lineno]\n except KeyError:\n pass\n" }, { "path": "xonsh/ply/example/BASIC/basparse.py", "content": "# An implementation of Dartmouth BASIC (1964)\n#\n\nfrom ply import *\nimport basiclex\n\ntokens = basiclex.tokens\n\nprecedence = (\n ('left', 'PLUS', 'MINUS'),\n ('left', 'TIMES', 'DIVIDE'),\n ('left', 'POWER'),\n ('right', 'UMINUS')\n)\n\n# A BASIC program is a series of statements. We represent the program as a\n# dictionary of tuples indexed by line number.\n\n\ndef p_program(p):\n '''program : program statement\n | statement'''\n\n if len(p) == 2 and p[1]:\n p[0] = {}\n line, stat = p[1]\n p[0][line] = stat\n elif len(p) == 3:\n p[0] = p[1]\n if not p[0]:\n p[0] = {}\n if p[2]:\n line, stat = p[2]\n p[0][line] = stat\n\n# This catch-all rule is used for any catastrophic errors. In this case,\n# we simply return nothing\n\n\ndef p_program_error(p):\n '''program : error'''\n p[0] = None\n p.parser.error = 1\n\n# Format of all BASIC statements.\n\n\ndef p_statement(p):\n '''statement : INTEGER command NEWLINE'''\n if isinstance(p[2], str):\n print(\"%s %s %s\" % (p[2], \"AT LINE\", p[1]))\n p[0] = None\n p.parser.error = 1\n else:\n lineno = int(p[1])\n p[0] = (lineno, p[2])\n\n# Interactive statements.\n\n\ndef p_statement_interactive(p):\n '''statement : RUN NEWLINE\n | LIST NEWLINE\n | NEW NEWLINE'''\n p[0] = (0, (p[1], 0))\n\n# Blank line number\n\n\ndef p_statement_blank(p):\n '''statement : INTEGER NEWLINE'''\n p[0] = (0, ('BLANK', int(p[1])))\n\n# Error handling for malformed statements\n\n\ndef p_statement_bad(p):\n '''statement : INTEGER error NEWLINE'''\n print(\"MALFORMED STATEMENT AT LINE %s\" % p[1])\n p[0] = None\n p.parser.error = 1\n\n# Blank line\n\n\ndef p_statement_newline(p):\n '''statement : NEWLINE'''\n p[0] = None\n\n# LET statement\n\n\ndef p_command_let(p):\n '''command : LET variable EQUALS expr'''\n p[0] = ('LET', p[2], p[4])\n\n\ndef p_command_let_bad(p):\n '''command : LET variable EQUALS error'''\n p[0] = \"BAD EXPRESSION IN LET\"\n\n# READ statement\n\n\ndef p_command_read(p):\n '''command : READ varlist'''\n p[0] = ('READ', p[2])\n\n\ndef p_command_read_bad(p):\n '''command : READ error'''\n p[0] = \"MALFORMED VARIABLE LIST IN READ\"\n\n# DATA statement\n\n\ndef p_command_data(p):\n '''command : DATA numlist'''\n p[0] = ('DATA', p[2])\n\n\ndef p_command_data_bad(p):\n '''command : DATA error'''\n p[0] = \"MALFORMED NUMBER LIST IN DATA\"\n\n# PRINT statement\n\n\ndef p_command_print(p):\n '''command : PRINT plist optend'''\n p[0] = ('PRINT', p[2], p[3])\n\n\ndef p_command_print_bad(p):\n '''command : PRINT error'''\n p[0] = \"MALFORMED PRINT STATEMENT\"\n\n# Optional ending on PRINT. Either a comma (,) or semicolon (;)\n\n\ndef p_optend(p):\n '''optend : COMMA \n | SEMI\n |'''\n if len(p) == 2:\n p[0] = p[1]\n else:\n p[0] = None\n\n# PRINT statement with no arguments\n\n\ndef p_command_print_empty(p):\n '''command : PRINT'''\n p[0] = ('PRINT', [], None)\n\n# GOTO statement\n\n\ndef p_command_goto(p):\n '''command : GOTO INTEGER'''\n p[0] = ('GOTO', int(p[2]))\n\n\ndef p_command_goto_bad(p):\n '''command : GOTO error'''\n p[0] = \"INVALID LINE NUMBER IN GOTO\"\n\n# IF-THEN statement\n\n\ndef p_command_if(p):\n '''command : IF relexpr THEN INTEGER'''\n p[0] = ('IF', p[2], int(p[4]))\n\n\ndef p_command_if_bad(p):\n '''command : IF error THEN INTEGER'''\n p[0] = \"BAD RELATIONAL EXPRESSION\"\n\n\ndef p_command_if_bad2(p):\n '''command : IF relexpr THEN error'''\n p[0] = \"INVALID LINE NUMBER IN THEN\"\n\n# FOR statement\n\n\ndef p_command_for(p):\n '''command : FOR ID EQUALS expr TO expr optstep'''\n p[0] = ('FOR', p[2], p[4], p[6], p[7])\n\n\ndef p_command_for_bad_initial(p):\n '''command : FOR ID EQUALS error TO expr optstep'''\n p[0] = \"BAD INITIAL VALUE IN FOR STATEMENT\"\n\n\ndef p_command_for_bad_final(p):\n '''command : FOR ID EQUALS expr TO error optstep'''\n p[0] = \"BAD FINAL VALUE IN FOR STATEMENT\"\n\n\ndef p_command_for_bad_step(p):\n '''command : FOR ID EQUALS expr TO expr STEP error'''\n p[0] = \"MALFORMED STEP IN FOR STATEMENT\"\n\n# Optional STEP qualifier on FOR statement\n\n\ndef p_optstep(p):\n '''optstep : STEP expr\n | empty'''\n if len(p) == 3:\n p[0] = p[2]\n else:\n p[0] = None\n\n# NEXT statement\n\n\ndef p_command_next(p):\n '''command : NEXT ID'''\n\n p[0] = ('NEXT', p[2])\n\n\ndef p_command_next_bad(p):\n '''command : NEXT error'''\n p[0] = \"MALFORMED NEXT\"\n\n# END statement\n\n\ndef p_command_end(p):\n '''command : END'''\n p[0] = ('END',)\n\n# REM statement\n\n\ndef p_command_rem(p):\n '''command : REM'''\n p[0] = ('REM', p[1])\n\n# STOP statement\n\n\ndef p_command_stop(p):\n '''command : STOP'''\n p[0] = ('STOP',)\n\n# DEF statement\n\n\ndef p_command_def(p):\n '''command : DEF ID LPAREN ID RPAREN EQUALS expr'''\n p[0] = ('FUNC', p[2], p[4], p[7])\n\n\ndef p_command_def_bad_rhs(p):\n '''command : DEF ID LPAREN ID RPAREN EQUALS error'''\n p[0] = \"BAD EXPRESSION IN DEF STATEMENT\"\n\n\ndef p_command_def_bad_arg(p):\n '''command : DEF ID LPAREN error RPAREN EQUALS expr'''\n p[0] = \"BAD ARGUMENT IN DEF STATEMENT\"\n\n# GOSUB statement\n\n\ndef p_command_gosub(p):\n '''command : GOSUB INTEGER'''\n p[0] = ('GOSUB', int(p[2]))\n\n\ndef p_command_gosub_bad(p):\n '''command : GOSUB error'''\n p[0] = \"INVALID LINE NUMBER IN GOSUB\"\n\n# RETURN statement\n\n\ndef p_command_return(p):\n '''command : RETURN'''\n p[0] = ('RETURN',)\n\n# DIM statement\n\n\ndef p_command_dim(p):\n '''command : DIM dimlist'''\n p[0] = ('DIM', p[2])\n\n\ndef p_command_dim_bad(p):\n '''command : DIM error'''\n p[0] = \"MALFORMED VARIABLE LIST IN DIM\"\n\n# List of variables supplied to DIM statement\n\n\ndef p_dimlist(p):\n '''dimlist : dimlist COMMA dimitem\n | dimitem'''\n if len(p) == 4:\n p[0] = p[1]\n p[0].append(p[3])\n else:\n p[0] = [p[1]]\n\n# DIM items\n\n\ndef p_dimitem_single(p):\n '''dimitem : ID LPAREN INTEGER RPAREN'''\n p[0] = (p[1], eval(p[3]), 0)\n\n\ndef p_dimitem_double(p):\n '''dimitem : ID LPAREN INTEGER COMMA INTEGER RPAREN'''\n p[0] = (p[1], eval(p[3]), eval(p[5]))\n\n# Arithmetic expressions\n\n\ndef p_expr_binary(p):\n '''expr : expr PLUS expr\n | expr MINUS expr\n | expr TIMES expr\n | expr DIVIDE expr\n | expr POWER expr'''\n\n p[0] = ('BINOP', p[2], p[1], p[3])\n\n\ndef p_expr_number(p):\n '''expr : INTEGER\n | FLOAT'''\n p[0] = ('NUM', eval(p[1]))\n\n\ndef p_expr_variable(p):\n '''expr : variable'''\n p[0] = ('VAR', p[1])\n\n\ndef p_expr_group(p):\n '''expr : LPAREN expr RPAREN'''\n p[0] = ('GROUP', p[2])\n\n\ndef p_expr_unary(p):\n '''expr : MINUS expr %prec UMINUS'''\n p[0] = ('UNARY', '-', p[2])\n\n# Relational expressions\n\n\ndef p_relexpr(p):\n '''relexpr : expr LT expr\n | expr LE expr\n | expr GT expr\n | expr GE expr\n | expr EQUALS expr\n | expr NE expr'''\n p[0] = ('RELOP', p[2], p[1], p[3])\n\n# Variables\n\n\ndef p_variable(p):\n '''variable : ID\n | ID LPAREN expr RPAREN\n | ID LPAREN expr COMMA expr RPAREN'''\n if len(p) == 2:\n p[0] = (p[1], None, None)\n elif len(p) == 5:\n p[0] = (p[1], p[3], None)\n else:\n p[0] = (p[1], p[3], p[5])\n\n# Builds a list of variable targets as a Python list\n\n\ndef p_varlist(p):\n '''varlist : varlist COMMA variable\n | variable'''\n if len(p) > 2:\n p[0] = p[1]\n p[0].append(p[3])\n else:\n p[0] = [p[1]]\n\n\n# Builds a list of numbers as a Python list\n\ndef p_numlist(p):\n '''numlist : numlist COMMA number\n | number'''\n\n if len(p) > 2:\n p[0] = p[1]\n p[0].append(p[3])\n else:\n p[0] = [p[1]]\n\n# A number. May be an integer or a float\n\n\ndef p_number(p):\n '''number : INTEGER\n | FLOAT'''\n p[0] = eval(p[1])\n\n# A signed number.\n\n\ndef p_number_signed(p):\n '''number : MINUS INTEGER\n | MINUS FLOAT'''\n p[0] = eval(\"-\" + p[2])\n\n# List of targets for a print statement\n# Returns a list of tuples (label,expr)\n\n\ndef p_plist(p):\n '''plist : plist COMMA pitem\n | pitem'''\n if len(p) > 3:\n p[0] = p[1]\n p[0].append(p[3])\n else:\n p[0] = [p[1]]\n\n\ndef p_item_string(p):\n '''pitem : STRING'''\n p[0] = (p[1][1:-1], None)\n\n\ndef p_item_string_expr(p):\n '''pitem : STRING expr'''\n p[0] = (p[1][1:-1], p[2])\n\n\ndef p_item_expr(p):\n '''pitem : expr'''\n p[0] = (\"\", p[1])\n\n# Empty\n\n\ndef p_empty(p):\n '''empty : '''\n\n# Catastrophic error handler\n\n\ndef p_error(p):\n if not p:\n print(\"SYNTAX ERROR AT EOF\")\n\nbparser = yacc.yacc()\n\n\ndef parse(data, debug=0):\n bparser.error = 0\n p = bparser.parse(data, debug=debug)\n if bparser.error:\n return None\n return p\n" }, { "path": "xonsh/ply/example/BASIC/dim.bas", "content": "5 DIM A(50,15)\n10 FOR I = 1 TO 50\n20 FOR J = 1 TO 15\n30 LET A(I,J) = I + J\n35 REM PRINT I,J, A(I,J)\n40 NEXT J\n50 NEXT I\n100 FOR I = 1 TO 50\n110 FOR J = 1 TO 15\n120 PRINT A(I,J),\n130 NEXT J\n140 PRINT \n150 NEXT I\n999 END\n" }, { "path": "xonsh/ply/example/BASIC/func.bas", "content": "10 DEF FDX(X) = 2*X\n20 FOR I = 0 TO 100\n30 PRINT FDX(I)\n40 NEXT I\n50 END\n" }, { "path": "xonsh/ply/example/BASIC/gcd.bas", "content": "10 PRINT \"A\",\"B\",\"C\",\"GCD\"\n20 READ A,B,C\n30 LET X = A\n40 LET Y = B\n50 GOSUB 200\n60 LET X = G\n70 LET Y = C\n80 GOSUB 200\n90 PRINT A, B, C, G\n100 GOTO 20\n110 DATA 60, 90, 120\n120 DATA 38456, 64872, 98765\n130 DATA 32, 384, 72\n200 LET Q = INT(X/Y)\n210 LET R = X - Q*Y\n220 IF R = 0 THEN 300\n230 LET X = Y\n240 LET Y = R\n250 GOTO 200\n300 LET G = Y\n310 RETURN\n999 END\n" }, { "path": "xonsh/ply/example/BASIC/gosub.bas", "content": "100 LET X = 3\n110 GOSUB 400\n120 PRINT U, V, W\n200 LET X = 5\n210 GOSUB 400\n220 LET Z = U + 2*V + 3*W\n230 PRINT Z\n240 GOTO 999\n400 LET U = X*X\n410 LET V = X*X*X\n420 LET W = X*X*X*X + X*X*X + X*X + X\n430 RETURN\n999 END\n" }, { "path": "xonsh/ply/example/BASIC/hello.bas", "content": "5 REM HELLO WORLD PROGAM\n10 PRINT \"HELLO WORLD\"\n99 END\n\n" }, { "path": "xonsh/ply/example/BASIC/linear.bas", "content": "1 REM ::: SOLVE A SYSTEM OF LINEAR EQUATIONS\n2 REM ::: A1*X1 + A2*X2 = B1\n3 REM ::: A3*X1 + A4*X2 = B2\n4 REM --------------------------------------\n10 READ A1, A2, A3, A4\n15 LET D = A1 * A4 - A3 * A2\n20 IF D = 0 THEN 65\n30 READ B1, B2\n37 LET X1 = (B1*A4 - B2*A2) / D\n42 LET X2 = (A1*B2 - A3*B1) / D\n55 PRINT X1, X2\n60 GOTO 30\n65 PRINT \"NO UNIQUE SOLUTION\"\n70 DATA 1, 2, 4\n80 DATA 2, -7, 5\n85 DATA 1, 3, 4, -7\n90 END\n" }, { "path": "xonsh/ply/example/BASIC/maxsin.bas", "content": "5 PRINT \"X VALUE\", \"SINE\", \"RESOLUTION\"\n10 READ D\n20 LET M = -1\n30 FOR X = 0 TO 3 STEP D\n40 IF SIN(X) <= M THEN 80\n50 LET X0 = X\n60 LET M = SIN(X)\n80 NEXT X\n85 PRINT X0, M, D\n90 GOTO 10\n100 DATA .1, .01, .001\n110 END\n" }, { "path": "xonsh/ply/example/BASIC/powers.bas", "content": "5 PRINT \"THIS PROGRAM COMPUTES AND PRINTS THE NTH POWERS\"\n6 PRINT \"OF THE NUMBERS LESS THAN OR EQUAL TO N FOR VARIOUS\"\n7 PRINT \"N FROM 1 THROUGH 7\"\n8 PRINT\n10 FOR N = 1 TO 7\n15 PRINT \"N = \"N\n20 FOR I = 1 TO N\n30 PRINT I^N,\n40 NEXT I\n50 PRINT\n60 PRINT\n70 NEXT N\n80 END\n" }, { "path": "xonsh/ply/example/BASIC/rand.bas", "content": "10 FOR I = 1 TO 20\n20 PRINT INT(10*RND(0))\n30 NEXT I\n40 END\n" }, { "path": "xonsh/ply/example/BASIC/sales.bas", "content": "10 FOR I = 1 TO 3\n20 READ P(I)\n30 NEXT I\n40 FOR I = 1 TO 3\n50 FOR J = 1 TO 5\n60 READ S(I,J)\n70 NEXT J\n80 NEXT I\n90 FOR J = 1 TO 5\n100 LET S = 0\n110 FOR I = 1 TO 3\n120 LET S = S + P(I) * S(I,J)\n130 NEXT I\n140 PRINT \"TOTAL SALES FOR SALESMAN\"J, \"$\"S\n150 NEXT J\n200 DATA 1.25, 4.30, 2.50\n210 DATA 40, 20, 37, 29, 42\n220 DATA 10, 16, 3, 21, 8\n230 DATA 35, 47, 29, 16, 33\n300 END\n" }, { "path": "xonsh/ply/example/BASIC/sears.bas", "content": "1 REM :: THIS PROGRAM COMPUTES HOW MANY TIMES YOU HAVE TO FOLD\n2 REM :: A PIECE OF PAPER SO THAT IT IS TALLER THAN THE\n3 REM :: SEARS TOWER.\n4 REM :: S = HEIGHT OF TOWER (METERS)\n5 REM :: T = THICKNESS OF PAPER (MILLIMETERS)\n10 LET S = 442\n20 LET T = 0.1\n30 REM CONVERT T TO METERS\n40 LET T = T * .001\n50 LET F = 1\n60 LET H = T\n100 IF H > S THEN 200\n120 LET H = 2 * H\n125 LET F = F + 1\n130 GOTO 100\n200 PRINT \"NUMBER OF FOLDS =\"F\n220 PRINT \"FINAL HEIGHT =\"H\n999 END\n" }, { "path": "xonsh/ply/example/BASIC/sqrt1.bas", "content": "10 LET X = 0\n20 LET X = X + 1\n30 PRINT X, SQR(X)\n40 IF X < 100 THEN 20\n50 END\n" }, { "path": "xonsh/ply/example/BASIC/sqrt2.bas", "content": "10 FOR X = 1 TO 100\n20 PRINT X, SQR(X)\n30 NEXT X\n40 END\n" }, { "path": "xonsh/ply/example/GardenSnake/GardenSnake.py", "content": "# GardenSnake - a parser generator demonstration program\n#\n# This implements a modified version of a subset of Python:\n# - only 'def', 'return' and 'if' statements\n# - 'if' only has 'then' clause (no elif nor else)\n# - single-quoted strings only, content in raw format\n# - numbers are decimal.Decimal instances (not integers or floats)\n# - no print statment; use the built-in 'print' function\n# - only < > == + - / * implemented (and unary + -)\n# - assignment and tuple assignment work\n# - no generators of any sort\n# - no ... well, no quite a lot\n\n# Why? I'm thinking about a new indentation-based configuration\n# language for a project and wanted to figure out how to do it. Once\n# I got that working I needed a way to test it out. My original AST\n# was dumb so I decided to target Python's AST and compile it into\n# Python code. Plus, it's pretty cool that it only took a day or so\n# from sitting down with Ply to having working code.\n\n# This uses David Beazley's Ply from http://www.dabeaz.com/ply/\n\n# This work is hereby released into the Public Domain. To view a copy of\n# the public domain dedication, visit\n# http://creativecommons.org/licenses/publicdomain/ or send a letter to\n# Creative Commons, 543 Howard Street, 5th Floor, San Francisco,\n# California, 94105, USA.\n#\n# Portions of this work are derived from Python's Grammar definition\n# and may be covered under the Python copyright and license\n#\n# Andrew Dalke / Dalke Scientific Software, LLC\n# 30 August 2006 / Cape Town, South Africa\n\n# Changelog:\n# 30 August - added link to CC license; removed the \"swapcase\" encoding\n\n# Modifications for inclusion in PLY distribution\nimport sys\nsys.path.insert(0, \"../..\")\nfrom ply import *\n\n##### Lexer ######\n#import lex\nimport decimal\n\ntokens = (\n 'DEF',\n 'IF',\n 'NAME',\n 'NUMBER', # Python decimals\n 'STRING', # single quoted strings only; syntax of raw strings\n 'LPAR',\n 'RPAR',\n 'COLON',\n 'EQ',\n 'ASSIGN',\n 'LT',\n 'GT',\n 'PLUS',\n 'MINUS',\n 'MULT',\n 'DIV',\n 'RETURN',\n 'WS',\n 'NEWLINE',\n 'COMMA',\n 'SEMICOLON',\n 'INDENT',\n 'DEDENT',\n 'ENDMARKER',\n)\n\n#t_NUMBER = r'\\d+'\n# taken from decmial.py but without the leading sign\n\n\ndef t_NUMBER(t):\n r\"\"\"(\\d+(\\.\\d*)?|\\.\\d+)([eE][-+]? \\d+)?\"\"\"\n t.value = decimal.Decimal(t.value)\n return t\n\n\ndef t_STRING(t):\n r\"'([^\\\\']+|\\\\'|\\\\\\\\)*'\" # I think this is right ...\n t.value = t.value[1:-1].decode(\"string-escape\") # .swapcase() # for fun\n return t\n\nt_COLON = r':'\nt_EQ = r'=='\nt_ASSIGN = r'='\nt_LT = r'<'\nt_GT = r'>'\nt_PLUS = r'\\+'\nt_MINUS = r'-'\nt_MULT = r'\\*'\nt_DIV = r'/'\nt_COMMA = r','\nt_SEMICOLON = r';'\n\n# Ply nicely documented how to do this.\n\nRESERVED = {\n \"def\": \"DEF\",\n \"if\": \"IF\",\n \"return\": \"RETURN\",\n}\n\n\ndef t_NAME(t):\n r'[a-zA-Z_][a-zA-Z0-9_]*'\n t.type = RESERVED.get(t.value, \"NAME\")\n return t\n\n# Putting this before t_WS let it consume lines with only comments in\n# them so the latter code never sees the WS part. Not consuming the\n# newline. Needed for \"if 1: #comment\"\n\n\ndef t_comment(t):\n r\"[ ]*\\043[^\\n]*\" # \\043 is '#'\n pass\n\n\n# Whitespace\ndef t_WS(t):\n r' [ ]+ '\n if t.lexer.at_line_start and t.lexer.paren_count == 0:\n return t\n\n# Don't generate newline tokens when inside of parenthesis, eg\n# a = (1,\n# 2, 3)\n\n\ndef t_newline(t):\n r'\\n+'\n t.lexer.lineno += len(t.value)\n t.type = \"NEWLINE\"\n if t.lexer.paren_count == 0:\n return t\n\n\ndef t_LPAR(t):\n r'\\('\n t.lexer.paren_count += 1\n return t\n\n\ndef t_RPAR(t):\n r'\\)'\n # check for underflow? should be the job of the parser\n t.lexer.paren_count -= 1\n return t\n\n\ndef t_error(t):\n raise SyntaxError(\"Unknown symbol %r\" % (t.value[0],))\n print \"Skipping\", repr(t.value[0])\n t.lexer.skip(1)\n\n# I implemented INDENT / DEDENT generation as a post-processing filter\n\n# The original lex token stream contains WS and NEWLINE characters.\n# WS will only occur before any other tokens on a line.\n\n# I have three filters. One tags tokens by adding two attributes.\n# \"must_indent\" is True if the token must be indented from the\n# previous code. The other is \"at_line_start\" which is True for WS\n# and the first non-WS/non-NEWLINE on a line. It flags the check so\n# see if the new line has changed indication level.\n\n# Python's syntax has three INDENT states\n# 0) no colon hence no need to indent\n# 1) \"if 1: go()\" - simple statements have a COLON but no need for an indent\n# 2) \"if 1:\\n go()\" - complex statements have a COLON NEWLINE and must indent\nNO_INDENT = 0\nMAY_INDENT = 1\nMUST_INDENT = 2\n\n# only care about whitespace at the start of a line\n\n\ndef track_tokens_filter(lexer, tokens):\n lexer.at_line_start = at_line_start = True\n indent = NO_INDENT\n saw_colon = False\n for token in tokens:\n token.at_line_start = at_line_start\n\n if token.type == \"COLON\":\n at_line_start = False\n indent = MAY_INDENT\n token.must_indent = False\n\n elif token.type == \"NEWLINE\":\n at_line_start = True\n if indent == MAY_INDENT:\n indent = MUST_INDENT\n token.must_indent = False\n\n elif token.type == \"WS\":\n assert token.at_line_start == True\n at_line_start = True\n token.must_indent = False\n\n else:\n # A real token; only indent after COLON NEWLINE\n if indent == MUST_INDENT:\n token.must_indent = True\n else:\n token.must_indent = False\n at_line_start = False\n indent = NO_INDENT\n\n yield token\n lexer.at_line_start = at_line_start\n\n\ndef _new_token(type, lineno):\n tok = lex.LexToken()\n tok.type = type\n tok.value = None\n tok.lineno = lineno\n return tok\n\n# Synthesize a DEDENT tag\n\n\ndef DEDENT(lineno):\n return _new_token(\"DEDENT\", lineno)\n\n# Synthesize an INDENT tag\n\n\ndef INDENT(lineno):\n return _new_token(\"INDENT\", lineno)\n\n\n# Track the indentation level and emit the right INDENT / DEDENT events.\ndef indentation_filter(tokens):\n # A stack of indentation levels; will never pop item 0\n levels = [0]\n token = None\n depth = 0\n prev_was_ws = False\n for token in tokens:\n # if 1:\n # print \"Process\", token,\n # if token.at_line_start:\n # print \"at_line_start\",\n # if token.must_indent:\n # print \"must_indent\",\n # print\n\n # WS only occurs at the start of the line\n # There may be WS followed by NEWLINE so\n # only track the depth here. Don't indent/dedent\n # until there's something real.\n if token.type == \"WS\":\n assert depth == 0\n depth = len(token.value)\n prev_was_ws = True\n # WS tokens are never passed to the parser\n continue\n\n if token.type == \"NEWLINE\":\n depth = 0\n if prev_was_ws or token.at_line_start:\n # ignore blank lines\n continue\n # pass the other cases on through\n yield token\n continue\n\n # then it must be a real token (not WS, not NEWLINE)\n # which can affect the indentation level\n\n prev_was_ws = False\n if token.must_indent:\n # The current depth must be larger than the previous level\n if not (depth > levels[-1]):\n raise IndentationError(\"expected an indented block\")\n\n levels.append(depth)\n yield INDENT(token.lineno)\n\n elif token.at_line_start:\n # Must be on the same level or one of the previous levels\n if depth == levels[-1]:\n # At the same level\n pass\n elif depth > levels[-1]:\n raise IndentationError(\n \"indentation increase but not in new block\")\n else:\n # Back up; but only if it matches a previous level\n try:\n i = levels.index(depth)\n except ValueError:\n raise IndentationError(\"inconsistent indentation\")\n for _ in range(i + 1, len(levels)):\n yield DEDENT(token.lineno)\n levels.pop()\n\n yield token\n\n ### Finished processing ###\n\n # Must dedent any remaining levels\n if len(levels) > 1:\n assert token is not None\n for _ in range(1, len(levels)):\n yield DEDENT(token.lineno)\n\n\n# The top-level filter adds an ENDMARKER, if requested.\n# Python's grammar uses it.\ndef filter(lexer, add_endmarker=True):\n token = None\n tokens = iter(lexer.token, None)\n tokens = track_tokens_filter(lexer, tokens)\n for token in indentation_filter(tokens):\n yield token\n\n if add_endmarker:\n lineno = 1\n if token is not None:\n lineno = token.lineno\n yield _new_token(\"ENDMARKER\", lineno)\n\n# Combine Ply and my filters into a new lexer\n\n\nclass IndentLexer(object):\n\n def __init__(self, debug=0, optimize=0, lextab='lextab', reflags=0):\n self.lexer = lex.lex(debug=debug, optimize=optimize,\n lextab=lextab, reflags=reflags)\n self.token_stream = None\n\n def input(self, s, add_endmarker=True):\n self.lexer.paren_count = 0\n self.lexer.input(s)\n self.token_stream = filter(self.lexer, add_endmarker)\n\n def token(self):\n try:\n return self.token_stream.next()\n except StopIteration:\n return None\n\n########## Parser (tokens -> AST) ######\n\n# also part of Ply\n#import yacc\n\n# I use the Python AST\nfrom compiler import ast\n\n# Helper function\n\n\ndef Assign(left, right):\n names = []\n if isinstance(left, ast.Name):\n # Single assignment on left\n return ast.Assign([ast.AssName(left.name, 'OP_ASSIGN')], right)\n elif isinstance(left, ast.Tuple):\n # List of things - make sure they are Name nodes\n names = []\n for child in left.getChildren():\n if not isinstance(child, ast.Name):\n raise SyntaxError(\"that assignment not supported\")\n names.append(child.name)\n ass_list = [ast.AssName(name, 'OP_ASSIGN') for name in names]\n return ast.Assign([ast.AssTuple(ass_list)], right)\n else:\n raise SyntaxError(\"Can't do that yet\")\n\n\n# The grammar comments come from Python's Grammar/Grammar file\n\n# NB: compound_stmt in single_input is followed by extra NEWLINE!\n# file_input: (NEWLINE | stmt)* ENDMARKER\ndef p_file_input_end(p):\n \"\"\"file_input_end : file_input ENDMARKER\"\"\"\n p[0] = ast.Stmt(p[1])\n\n\ndef p_file_input(p):\n \"\"\"file_input : file_input NEWLINE\n | file_input stmt\n | NEWLINE\n | stmt\"\"\"\n if isinstance(p[len(p) - 1], basestring):\n if len(p) == 3:\n p[0] = p[1]\n else:\n p[0] = [] # p == 2 --> only a blank line\n else:\n if len(p) == 3:\n p[0] = p[1] + p[2]\n else:\n p[0] = p[1]\n\n\n# funcdef: [decorators] 'def' NAME parameters ':' suite\n# ignoring decorators\ndef p_funcdef(p):\n \"funcdef : DEF NAME parameters COLON suite\"\n p[0] = ast.Function(None, p[2], tuple(p[3]), (), 0, None, p[5])\n\n# parameters: '(' [varargslist] ')'\n\n\ndef p_parameters(p):\n \"\"\"parameters : LPAR RPAR\n | LPAR varargslist RPAR\"\"\"\n if len(p) == 3:\n p[0] = []\n else:\n p[0] = p[2]\n\n\n# varargslist: (fpdef ['=' test] ',')* ('*' NAME [',' '**' NAME] | '**' NAME) |\n# highly simplified\ndef p_varargslist(p):\n \"\"\"varargslist : varargslist COMMA NAME\n | NAME\"\"\"\n if len(p) == 4:\n p[0] = p[1] + p[3]\n else:\n p[0] = [p[1]]\n\n# stmt: simple_stmt | compound_stmt\n\n\ndef p_stmt_simple(p):\n \"\"\"stmt : simple_stmt\"\"\"\n # simple_stmt is a list\n p[0] = p[1]\n\n\ndef p_stmt_compound(p):\n \"\"\"stmt : compound_stmt\"\"\"\n p[0] = [p[1]]\n\n# simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE\n\n\ndef p_simple_stmt(p):\n \"\"\"simple_stmt : small_stmts NEWLINE\n | small_stmts SEMICOLON NEWLINE\"\"\"\n p[0] = p[1]\n\n\ndef p_small_stmts(p):\n \"\"\"small_stmts : small_stmts SEMICOLON small_stmt\n | small_stmt\"\"\"\n if len(p) == 4:\n p[0] = p[1] + [p[3]]\n else:\n p[0] = [p[1]]\n\n# small_stmt: expr_stmt | print_stmt | del_stmt | pass_stmt | flow_stmt |\n# import_stmt | global_stmt | exec_stmt | assert_stmt\n\n\ndef p_small_stmt(p):\n \"\"\"small_stmt : flow_stmt\n | expr_stmt\"\"\"\n p[0] = p[1]\n\n# expr_stmt: testlist (augassign (yield_expr|testlist) |\n# ('=' (yield_expr|testlist))*)\n# augassign: ('+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' |\n# '<<=' | '>>=' | '**=' | '//=')\n\n\ndef p_expr_stmt(p):\n \"\"\"expr_stmt : testlist ASSIGN testlist\n | testlist \"\"\"\n if len(p) == 2:\n # a list of expressions\n p[0] = ast.Discard(p[1])\n else:\n p[0] = Assign(p[1], p[3])\n\n\ndef p_flow_stmt(p):\n \"flow_stmt : return_stmt\"\n p[0] = p[1]\n\n# return_stmt: 'return' [testlist]\n\n\ndef p_return_stmt(p):\n \"return_stmt : RETURN testlist\"\n p[0] = ast.Return(p[2])\n\n\ndef p_compound_stmt(p):\n \"\"\"compound_stmt : if_stmt\n | funcdef\"\"\"\n p[0] = p[1]\n\n\ndef p_if_stmt(p):\n 'if_stmt : IF test COLON suite'\n p[0] = ast.If([(p[2], p[4])], None)\n\n\ndef p_suite(p):\n \"\"\"suite : simple_stmt\n | NEWLINE INDENT stmts DEDENT\"\"\"\n if len(p) == 2:\n p[0] = ast.Stmt(p[1])\n else:\n p[0] = ast.Stmt(p[3])\n\n\ndef p_stmts(p):\n \"\"\"stmts : stmts stmt\n | stmt\"\"\"\n if len(p) == 3:\n p[0] = p[1] + p[2]\n else:\n p[0] = p[1]\n\n# No using Python's approach because Ply supports precedence\n\n# comparison: expr (comp_op expr)*\n# arith_expr: term (('+'|'-') term)*\n# term: factor (('*'|'/'|'%'|'//') factor)*\n# factor: ('+'|'-'|'~') factor | power\n# comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'\n\n\ndef make_lt_compare((left, right)):\n return ast.Compare(left, [('<', right), ])\n\n\ndef make_gt_compare((left, right)):\n return ast.Compare(left, [('>', right), ])\n\n\ndef make_eq_compare((left, right)):\n return ast.Compare(left, [('==', right), ])\n\n\nbinary_ops = {\n \"+\": ast.Add,\n \"-\": ast.Sub,\n \"*\": ast.Mul,\n \"/\": ast.Div,\n \"<\": make_lt_compare,\n \">\": make_gt_compare,\n \"==\": make_eq_compare,\n}\nunary_ops = {\n \"+\": ast.UnaryAdd,\n \"-\": ast.UnarySub,\n}\nprecedence = (\n (\"left\", \"EQ\", \"GT\", \"LT\"),\n (\"left\", \"PLUS\", \"MINUS\"),\n (\"left\", \"MULT\", \"DIV\"),\n)\n\n\ndef p_comparison(p):\n \"\"\"comparison : comparison PLUS comparison\n | comparison MINUS comparison\n | comparison MULT comparison\n | comparison DIV comparison\n | comparison LT comparison\n | comparison EQ comparison\n | comparison GT comparison\n | PLUS comparison\n | MINUS comparison\n | power\"\"\"\n if len(p) == 4:\n p[0] = binary_ops[p[2]]((p[1], p[3]))\n elif len(p) == 3:\n p[0] = unary_ops[p[1]](p[2])\n else:\n p[0] = p[1]\n\n# power: atom trailer* ['**' factor]\n# trailers enables function calls. I only allow one level of calls\n# so this is 'trailer'\n\n\ndef p_power(p):\n \"\"\"power : atom\n | atom trailer\"\"\"\n if len(p) == 2:\n p[0] = p[1]\n else:\n if p[2][0] == \"CALL\":\n p[0] = ast.CallFunc(p[1], p[2][1], None, None)\n else:\n raise AssertionError(\"not implemented\")\n\n\ndef p_atom_name(p):\n \"\"\"atom : NAME\"\"\"\n p[0] = ast.Name(p[1])\n\n\ndef p_atom_number(p):\n \"\"\"atom : NUMBER\n | STRING\"\"\"\n p[0] = ast.Const(p[1])\n\n\ndef p_atom_tuple(p):\n \"\"\"atom : LPAR testlist RPAR\"\"\"\n p[0] = p[2]\n\n# trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME\n\n\ndef p_trailer(p):\n \"trailer : LPAR arglist RPAR\"\n p[0] = (\"CALL\", p[2])\n\n# testlist: test (',' test)* [',']\n# Contains shift/reduce error\n\n\ndef p_testlist(p):\n \"\"\"testlist : testlist_multi COMMA\n | testlist_multi \"\"\"\n if len(p) == 2:\n p[0] = p[1]\n else:\n # May need to promote singleton to tuple\n if isinstance(p[1], list):\n p[0] = p[1]\n else:\n p[0] = [p[1]]\n # Convert into a tuple?\n if isinstance(p[0], list):\n p[0] = ast.Tuple(p[0])\n\n\ndef p_testlist_multi(p):\n \"\"\"testlist_multi : testlist_multi COMMA test\n | test\"\"\"\n if len(p) == 2:\n # singleton\n p[0] = p[1]\n else:\n if isinstance(p[1], list):\n p[0] = p[1] + [p[3]]\n else:\n # singleton -> tuple\n p[0] = [p[1], p[3]]\n\n\n# test: or_test ['if' or_test 'else' test] | lambdef\n# as I don't support 'and', 'or', and 'not' this works down to 'comparison'\ndef p_test(p):\n \"test : comparison\"\n p[0] = p[1]\n\n\n# arglist: (argument ',')* (argument [',']| '*' test [',' '**' test] | '**' test)\n# XXX INCOMPLETE: this doesn't allow the trailing comma\ndef p_arglist(p):\n \"\"\"arglist : arglist COMMA argument\n | argument\"\"\"\n if len(p) == 4:\n p[0] = p[1] + [p[3]]\n else:\n p[0] = [p[1]]\n\n# argument: test [gen_for] | test '=' test # Really [keyword '='] test\n\n\ndef p_argument(p):\n \"argument : test\"\n p[0] = p[1]\n\n\ndef p_error(p):\n # print \"Error!\", repr(p)\n raise SyntaxError(p)\n\n\nclass GardenSnakeParser(object):\n\n def __init__(self, lexer=None):\n if lexer is None:\n lexer = IndentLexer()\n self.lexer = lexer\n self.parser = yacc.yacc(start=\"file_input_end\")\n\n def parse(self, code):\n self.lexer.input(code)\n result = self.parser.parse(lexer=self.lexer)\n return ast.Module(None, result)\n\n\n###### Code generation ######\n\nfrom compiler import misc, syntax, pycodegen\n\n\nclass GardenSnakeCompiler(object):\n\n def __init__(self):\n self.parser = GardenSnakeParser()\n\n def compile(self, code, filename=\"\"):\n tree = self.parser.parse(code)\n # print tree\n misc.set_filename(filename, tree)\n syntax.check(tree)\n gen = pycodegen.ModuleCodeGenerator(tree)\n code = gen.getCode()\n return code\n\n####### Test code #######\n\ncompile = GardenSnakeCompiler().compile\n\ncode = r\"\"\"\n\nprint('LET\\'S TRY THIS \\\\OUT')\n \n#Comment here\ndef x(a):\n print('called with',a)\n if a == 1:\n return 2\n if a*2 > 10: return 999 / 4\n # Another comment here\n\n return a+2*3\n\nints = (1, 2,\n 3, 4,\n5)\nprint('mutiline-expression', ints)\n\nt = 4+1/3*2+6*(9-5+1)\nprint('predence test; should be 34+2/3:', t, t==(34+2/3))\n\nprint('numbers', 1,2,3,4,5)\nif 1:\n 8\n a=9\n print(x(a))\n\nprint(x(1))\nprint(x(2))\nprint(x(8),'3')\nprint('this is decimal', 1/5)\nprint('BIG DECIMAL', 1.234567891234567e12345)\n\n\"\"\"\n\n# Set up the GardenSnake run-time environment\n\n\ndef print_(*args):\n print \"-->\", \" \".join(map(str, args))\n\nglobals()[\"print\"] = print_\n\ncompiled_code = compile(code)\n\nexec compiled_code in globals()\nprint \"Done\"\n" }, { "path": "xonsh/ply/example/GardenSnake/README", "content": "This example is Andrew Dalke's GardenSnake language. It shows how to process an\nindentation-like language like Python. Further details can be found here:\n\nhttp://dalkescientific.com/writings/diary/archive/2006/08/30/gardensnake_language.html\n\n" }, { "path": "xonsh/ply/example/README", "content": "Simple examples:\n calc - Simple calculator\n classcalc - Simple calculate defined as a class\n \nComplex examples\n ansic - ANSI C grammar from K&R\n BASIC - A small BASIC interpreter\n GardenSnake - A simple python-like language\n yply - Converts Unix yacc files to PLY programs.\n\n" }, { "path": "xonsh/ply/example/ansic/README", "content": "This example is incomplete. Was going to specify an ANSI C parser.\nThis is part of it.\n" }, { "path": "xonsh/ply/example/ansic/clex.py", "content": "# ----------------------------------------------------------------------\n# clex.py\n#\n# A lexer for ANSI C.\n# ----------------------------------------------------------------------\n\nimport sys\nsys.path.insert(0, \"../..\")\n\nimport ply.lex as lex\n\n# Reserved words\nreserved = (\n 'AUTO', 'BREAK', 'CASE', 'CHAR', 'CONST', 'CONTINUE', 'DEFAULT', 'DO', 'DOUBLE',\n 'ELSE', 'ENUM', 'EXTERN', 'FLOAT', 'FOR', 'GOTO', 'IF', 'INT', 'LONG', 'REGISTER',\n 'RETURN', 'SHORT', 'SIGNED', 'SIZEOF', 'STATIC', 'STRUCT', 'SWITCH', 'TYPEDEF',\n 'UNION', 'UNSIGNED', 'VOID', 'VOLATILE', 'WHILE',\n)\n\ntokens = reserved + (\n # Literals (identifier, integer constant, float constant, string constant,\n # char const)\n 'ID', 'TYPEID', 'ICONST', 'FCONST', 'SCONST', 'CCONST',\n\n # Operators (+,-,*,/,%,|,&,~,^,<<,>>, ||, &&, !, <, <=, >, >=, ==, !=)\n 'PLUS', 'MINUS', 'TIMES', 'DIVIDE', 'MOD',\n 'OR', 'AND', 'NOT', 'XOR', 'LSHIFT', 'RSHIFT',\n 'LOR', 'LAND', 'LNOT',\n 'LT', 'LE', 'GT', 'GE', 'EQ', 'NE',\n\n # Assignment (=, *=, /=, %=, +=, -=, <<=, >>=, &=, ^=, |=)\n 'EQUALS', 'TIMESEQUAL', 'DIVEQUAL', 'MODEQUAL', 'PLUSEQUAL', 'MINUSEQUAL',\n 'LSHIFTEQUAL', 'RSHIFTEQUAL', 'ANDEQUAL', 'XOREQUAL', 'OREQUAL',\n\n # Increment/decrement (++,--)\n 'PLUSPLUS', 'MINUSMINUS',\n\n # Structure dereference (->)\n 'ARROW',\n\n # Conditional operator (?)\n 'CONDOP',\n\n # Delimeters ( ) [ ] { } , . ; :\n 'LPAREN', 'RPAREN',\n 'LBRACKET', 'RBRACKET',\n 'LBRACE', 'RBRACE',\n 'COMMA', 'PERIOD', 'SEMI', 'COLON',\n\n # Ellipsis (...)\n 'ELLIPSIS',\n)\n\n# Completely ignored characters\nt_ignore = ' \\t\\x0c'\n\n# Newlines\n\n\ndef t_NEWLINE(t):\n r'\\n+'\n t.lexer.lineno += t.value.count(\"\\n\")\n\n# Operators\nt_PLUS = r'\\+'\nt_MINUS = r'-'\nt_TIMES = r'\\*'\nt_DIVIDE = r'/'\nt_MOD = r'%'\nt_OR = r'\\|'\nt_AND = r'&'\nt_NOT = r'~'\nt_XOR = r'\\^'\nt_LSHIFT = r'<<'\nt_RSHIFT = r'>>'\nt_LOR = r'\\|\\|'\nt_LAND = r'&&'\nt_LNOT = r'!'\nt_LT = r'<'\nt_GT = r'>'\nt_LE = r'<='\nt_GE = r'>='\nt_EQ = r'=='\nt_NE = r'!='\n\n# Assignment operators\n\nt_EQUALS = r'='\nt_TIMESEQUAL = r'\\*='\nt_DIVEQUAL = r'/='\nt_MODEQUAL = r'%='\nt_PLUSEQUAL = r'\\+='\nt_MINUSEQUAL = r'-='\nt_LSHIFTEQUAL = r'<<='\nt_RSHIFTEQUAL = r'>>='\nt_ANDEQUAL = r'&='\nt_OREQUAL = r'\\|='\nt_XOREQUAL = r'\\^='\n\n# Increment/decrement\nt_PLUSPLUS = r'\\+\\+'\nt_MINUSMINUS = r'--'\n\n# ->\nt_ARROW = r'->'\n\n# ?\nt_CONDOP = r'\\?'\n\n# Delimeters\nt_LPAREN = r'\\('\nt_RPAREN = r'\\)'\nt_LBRACKET = r'\\['\nt_RBRACKET = r'\\]'\nt_LBRACE = r'\\{'\nt_RBRACE = r'\\}'\nt_COMMA = r','\nt_PERIOD = r'\\.'\nt_SEMI = r';'\nt_COLON = r':'\nt_ELLIPSIS = r'\\.\\.\\.'\n\n# Identifiers and reserved words\n\nreserved_map = {}\nfor r in reserved:\n reserved_map[r.lower()] = r\n\n\ndef t_ID(t):\n r'[A-Za-z_][\\w_]*'\n t.type = reserved_map.get(t.value, \"ID\")\n return t\n\n# Integer literal\nt_ICONST = r'\\d+([uU]|[lL]|[uU][lL]|[lL][uU])?'\n\n# Floating literal\nt_FCONST = r'((\\d+)(\\.\\d+)(e(\\+|-)?(\\d+))? | (\\d+)e(\\+|-)?(\\d+))([lL]|[fF])?'\n\n# String literal\nt_SCONST = r'\\\"([^\\\\\\n]|(\\\\.))*?\\\"'\n\n# Character constant 'c' or L'c'\nt_CCONST = r'(L)?\\'([^\\\\\\n]|(\\\\.))*?\\''\n\n# Comments\n\n\ndef t_comment(t):\n r'/\\*(.|\\n)*?\\*/'\n t.lexer.lineno += t.value.count('\\n')\n\n# Preprocessor directive (ignored)\n\n\ndef t_preprocessor(t):\n r'\\#(.)*?\\n'\n t.lexer.lineno += 1\n\n\ndef t_error(t):\n print(\"Illegal character %s\" % repr(t.value[0]))\n t.lexer.skip(1)\n\nlexer = lex.lex()\nif __name__ == \"__main__\":\n lex.runmain(lexer)\n" }, { "path": "xonsh/ply/example/ansic/cparse.py", "content": "# -----------------------------------------------------------------------------\n# cparse.py\n#\n# Simple parser for ANSI C. Based on the grammar in K&R, 2nd Ed.\n# -----------------------------------------------------------------------------\n\nimport sys\nimport clex\nimport ply.yacc as yacc\n\n# Get the token map\ntokens = clex.tokens\n\n# translation-unit:\n\n\ndef p_translation_unit_1(t):\n 'translation_unit : external_declaration'\n pass\n\n\ndef p_translation_unit_2(t):\n 'translation_unit : translation_unit external_declaration'\n pass\n\n# external-declaration:\n\n\ndef p_external_declaration_1(t):\n 'external_declaration : function_definition'\n pass\n\n\ndef p_external_declaration_2(t):\n 'external_declaration : declaration'\n pass\n\n# function-definition:\n\n\ndef p_function_definition_1(t):\n 'function_definition : declaration_specifiers declarator declaration_list compound_statement'\n pass\n\n\ndef p_function_definition_2(t):\n 'function_definition : declarator declaration_list compound_statement'\n pass\n\n\ndef p_function_definition_3(t):\n 'function_definition : declarator compound_statement'\n pass\n\n\ndef p_function_definition_4(t):\n 'function_definition : declaration_specifiers declarator compound_statement'\n pass\n\n# declaration:\n\n\ndef p_declaration_1(t):\n 'declaration : declaration_specifiers init_declarator_list SEMI'\n pass\n\n\ndef p_declaration_2(t):\n 'declaration : declaration_specifiers SEMI'\n pass\n\n# declaration-list:\n\n\ndef p_declaration_list_1(t):\n 'declaration_list : declaration'\n pass\n\n\ndef p_declaration_list_2(t):\n 'declaration_list : declaration_list declaration '\n pass\n\n# declaration-specifiers\n\n\ndef p_declaration_specifiers_1(t):\n 'declaration_specifiers : storage_class_specifier declaration_specifiers'\n pass\n\n\ndef p_declaration_specifiers_2(t):\n 'declaration_specifiers : type_specifier declaration_specifiers'\n pass\n\n\ndef p_declaration_specifiers_3(t):\n 'declaration_specifiers : type_qualifier declaration_specifiers'\n pass\n\n\ndef p_declaration_specifiers_4(t):\n 'declaration_specifiers : storage_class_specifier'\n pass\n\n\ndef p_declaration_specifiers_5(t):\n 'declaration_specifiers : type_specifier'\n pass\n\n\ndef p_declaration_specifiers_6(t):\n 'declaration_specifiers : type_qualifier'\n pass\n\n# storage-class-specifier\n\n\ndef p_storage_class_specifier(t):\n '''storage_class_specifier : AUTO\n | REGISTER\n | STATIC\n | EXTERN\n | TYPEDEF\n '''\n pass\n\n# type-specifier:\n\n\ndef p_type_specifier(t):\n '''type_specifier : VOID\n | CHAR\n | SHORT\n | INT\n | LONG\n | FLOAT\n | DOUBLE\n | SIGNED\n | UNSIGNED\n | struct_or_union_specifier\n | enum_specifier\n | TYPEID\n '''\n pass\n\n# type-qualifier:\n\n\ndef p_type_qualifier(t):\n '''type_qualifier : CONST\n | VOLATILE'''\n pass\n\n# struct-or-union-specifier\n\n\ndef p_struct_or_union_specifier_1(t):\n 'struct_or_union_specifier : struct_or_union ID LBRACE struct_declaration_list RBRACE'\n pass\n\n\ndef p_struct_or_union_specifier_2(t):\n 'struct_or_union_specifier : struct_or_union LBRACE struct_declaration_list RBRACE'\n pass\n\n\ndef p_struct_or_union_specifier_3(t):\n 'struct_or_union_specifier : struct_or_union ID'\n pass\n\n# struct-or-union:\n\n\ndef p_struct_or_union(t):\n '''struct_or_union : STRUCT\n | UNION\n '''\n pass\n\n# struct-declaration-list:\n\n\ndef p_struct_declaration_list_1(t):\n 'struct_declaration_list : struct_declaration'\n pass\n\n\ndef p_struct_declaration_list_2(t):\n 'struct_declaration_list : struct_declaration_list struct_declaration'\n pass\n\n# init-declarator-list:\n\n\ndef p_init_declarator_list_1(t):\n 'init_declarator_list : init_declarator'\n pass\n\n\ndef p_init_declarator_list_2(t):\n 'init_declarator_list : init_declarator_list COMMA init_declarator'\n pass\n\n# init-declarator\n\n\ndef p_init_declarator_1(t):\n 'init_declarator : declarator'\n pass\n\n\ndef p_init_declarator_2(t):\n 'init_declarator : declarator EQUALS initializer'\n pass\n\n# struct-declaration:\n\n\ndef p_struct_declaration(t):\n 'struct_declaration : specifier_qualifier_list struct_declarator_list SEMI'\n pass\n\n# specifier-qualifier-list:\n\n\ndef p_specifier_qualifier_list_1(t):\n 'specifier_qualifier_list : type_specifier specifier_qualifier_list'\n pass\n\n\ndef p_specifier_qualifier_list_2(t):\n 'specifier_qualifier_list : type_specifier'\n pass\n\n\ndef p_specifier_qualifier_list_3(t):\n 'specifier_qualifier_list : type_qualifier specifier_qualifier_list'\n pass\n\n\ndef p_specifier_qualifier_list_4(t):\n 'specifier_qualifier_list : type_qualifier'\n pass\n\n# struct-declarator-list:\n\n\ndef p_struct_declarator_list_1(t):\n 'struct_declarator_list : struct_declarator'\n pass\n\n\ndef p_struct_declarator_list_2(t):\n 'struct_declarator_list : struct_declarator_list COMMA struct_declarator'\n pass\n\n# struct-declarator:\n\n\ndef p_struct_declarator_1(t):\n 'struct_declarator : declarator'\n pass\n\n\ndef p_struct_declarator_2(t):\n 'struct_declarator : declarator COLON constant_expression'\n pass\n\n\ndef p_struct_declarator_3(t):\n 'struct_declarator : COLON constant_expression'\n pass\n\n# enum-specifier:\n\n\ndef p_enum_specifier_1(t):\n 'enum_specifier : ENUM ID LBRACE enumerator_list RBRACE'\n pass\n\n\ndef p_enum_specifier_2(t):\n 'enum_specifier : ENUM LBRACE enumerator_list RBRACE'\n pass\n\n\ndef p_enum_specifier_3(t):\n 'enum_specifier : ENUM ID'\n pass\n\n# enumerator_list:\n\n\ndef p_enumerator_list_1(t):\n 'enumerator_list : enumerator'\n pass\n\n\ndef p_enumerator_list_2(t):\n 'enumerator_list : enumerator_list COMMA enumerator'\n pass\n\n# enumerator:\n\n\ndef p_enumerator_1(t):\n 'enumerator : ID'\n pass\n\n\ndef p_enumerator_2(t):\n 'enumerator : ID EQUALS constant_expression'\n pass\n\n# declarator:\n\n\ndef p_declarator_1(t):\n 'declarator : pointer direct_declarator'\n pass\n\n\ndef p_declarator_2(t):\n 'declarator : direct_declarator'\n pass\n\n# direct-declarator:\n\n\ndef p_direct_declarator_1(t):\n 'direct_declarator : ID'\n pass\n\n\ndef p_direct_declarator_2(t):\n 'direct_declarator : LPAREN declarator RPAREN'\n pass\n\n\ndef p_direct_declarator_3(t):\n 'direct_declarator : direct_declarator LBRACKET constant_expression_opt RBRACKET'\n pass\n\n\ndef p_direct_declarator_4(t):\n 'direct_declarator : direct_declarator LPAREN parameter_type_list RPAREN '\n pass\n\n\ndef p_direct_declarator_5(t):\n 'direct_declarator : direct_declarator LPAREN identifier_list RPAREN '\n pass\n\n\ndef p_direct_declarator_6(t):\n 'direct_declarator : direct_declarator LPAREN RPAREN '\n pass\n\n# pointer:\n\n\ndef p_pointer_1(t):\n 'pointer : TIMES type_qualifier_list'\n pass\n\n\ndef p_pointer_2(t):\n 'pointer : TIMES'\n pass\n\n\ndef p_pointer_3(t):\n 'pointer : TIMES type_qualifier_list pointer'\n pass\n\n\ndef p_pointer_4(t):\n 'pointer : TIMES pointer'\n pass\n\n# type-qualifier-list:\n\n\ndef p_type_qualifier_list_1(t):\n 'type_qualifier_list : type_qualifier'\n pass\n\n\ndef p_type_qualifier_list_2(t):\n 'type_qualifier_list : type_qualifier_list type_qualifier'\n pass\n\n# parameter-type-list:\n\n\ndef p_parameter_type_list_1(t):\n 'parameter_type_list : parameter_list'\n pass\n\n\ndef p_parameter_type_list_2(t):\n 'parameter_type_list : parameter_list COMMA ELLIPSIS'\n pass\n\n# parameter-list:\n\n\ndef p_parameter_list_1(t):\n 'parameter_list : parameter_declaration'\n pass\n\n\ndef p_parameter_list_2(t):\n 'parameter_list : parameter_list COMMA parameter_declaration'\n pass\n\n# parameter-declaration:\n\n\ndef p_parameter_declaration_1(t):\n 'parameter_declaration : declaration_specifiers declarator'\n pass\n\n\ndef p_parameter_declaration_2(t):\n 'parameter_declaration : declaration_specifiers abstract_declarator_opt'\n pass\n\n# identifier-list:\n\n\ndef p_identifier_list_1(t):\n 'identifier_list : ID'\n pass\n\n\ndef p_identifier_list_2(t):\n 'identifier_list : identifier_list COMMA ID'\n pass\n\n# initializer:\n\n\ndef p_initializer_1(t):\n 'initializer : assignment_expression'\n pass\n\n\ndef p_initializer_2(t):\n '''initializer : LBRACE initializer_list RBRACE\n | LBRACE initializer_list COMMA RBRACE'''\n pass\n\n# initializer-list:\n\n\ndef p_initializer_list_1(t):\n 'initializer_list : initializer'\n pass\n\n\ndef p_initializer_list_2(t):\n 'initializer_list : initializer_list COMMA initializer'\n pass\n\n# type-name:\n\n\ndef p_type_name(t):\n 'type_name : specifier_qualifier_list abstract_declarator_opt'\n pass\n\n\ndef p_abstract_declarator_opt_1(t):\n 'abstract_declarator_opt : empty'\n pass\n\n\ndef p_abstract_declarator_opt_2(t):\n 'abstract_declarator_opt : abstract_declarator'\n pass\n\n# abstract-declarator:\n\n\ndef p_abstract_declarator_1(t):\n 'abstract_declarator : pointer '\n pass\n\n\ndef p_abstract_declarator_2(t):\n 'abstract_declarator : pointer direct_abstract_declarator'\n pass\n\n\ndef p_abstract_declarator_3(t):\n 'abstract_declarator : direct_abstract_declarator'\n pass\n\n# direct-abstract-declarator:\n\n\ndef p_direct_abstract_declarator_1(t):\n 'direct_abstract_declarator : LPAREN abstract_declarator RPAREN'\n pass\n\n\ndef p_direct_abstract_declarator_2(t):\n 'direct_abstract_declarator : direct_abstract_declarator LBRACKET constant_expression_opt RBRACKET'\n pass\n\n\ndef p_direct_abstract_declarator_3(t):\n 'direct_abstract_declarator : LBRACKET constant_expression_opt RBRACKET'\n pass\n\n\ndef p_direct_abstract_declarator_4(t):\n 'direct_abstract_declarator : direct_abstract_declarator LPAREN parameter_type_list_opt RPAREN'\n pass\n\n\ndef p_direct_abstract_declarator_5(t):\n 'direct_abstract_declarator : LPAREN parameter_type_list_opt RPAREN'\n pass\n\n# Optional fields in abstract declarators\n\n\ndef p_constant_expression_opt_1(t):\n 'constant_expression_opt : empty'\n pass\n\n\ndef p_constant_expression_opt_2(t):\n 'constant_expression_opt : constant_expression'\n pass\n\n\ndef p_parameter_type_list_opt_1(t):\n 'parameter_type_list_opt : empty'\n pass\n\n\ndef p_parameter_type_list_opt_2(t):\n 'parameter_type_list_opt : parameter_type_list'\n pass\n\n# statement:\n\n\ndef p_statement(t):\n '''\n statement : labeled_statement\n | expression_statement\n | compound_statement\n | selection_statement\n | iteration_statement\n | jump_statement\n '''\n pass\n\n# labeled-statement:\n\n\ndef p_labeled_statement_1(t):\n 'labeled_statement : ID COLON statement'\n pass\n\n\ndef p_labeled_statement_2(t):\n 'labeled_statement : CASE constant_expression COLON statement'\n pass\n\n\ndef p_labeled_statement_3(t):\n 'labeled_statement : DEFAULT COLON statement'\n pass\n\n# expression-statement:\n\n\ndef p_expression_statement(t):\n 'expression_statement : expression_opt SEMI'\n pass\n\n# compound-statement:\n\n\ndef p_compound_statement_1(t):\n 'compound_statement : LBRACE declaration_list statement_list RBRACE'\n pass\n\n\ndef p_compound_statement_2(t):\n 'compound_statement : LBRACE statement_list RBRACE'\n pass\n\n\ndef p_compound_statement_3(t):\n 'compound_statement : LBRACE declaration_list RBRACE'\n pass\n\n\ndef p_compound_statement_4(t):\n 'compound_statement : LBRACE RBRACE'\n pass\n\n# statement-list:\n\n\ndef p_statement_list_1(t):\n 'statement_list : statement'\n pass\n\n\ndef p_statement_list_2(t):\n 'statement_list : statement_list statement'\n pass\n\n# selection-statement\n\n\ndef p_selection_statement_1(t):\n 'selection_statement : IF LPAREN expression RPAREN statement'\n pass\n\n\ndef p_selection_statement_2(t):\n 'selection_statement : IF LPAREN expression RPAREN statement ELSE statement '\n pass\n\n\ndef p_selection_statement_3(t):\n 'selection_statement : SWITCH LPAREN expression RPAREN statement '\n pass\n\n# iteration_statement:\n\n\ndef p_iteration_statement_1(t):\n 'iteration_statement : WHILE LPAREN expression RPAREN statement'\n pass\n\n\ndef p_iteration_statement_2(t):\n 'iteration_statement : FOR LPAREN expression_opt SEMI expression_opt SEMI expression_opt RPAREN statement '\n pass\n\n\ndef p_iteration_statement_3(t):\n 'iteration_statement : DO statement WHILE LPAREN expression RPAREN SEMI'\n pass\n\n# jump_statement:\n\n\ndef p_jump_statement_1(t):\n 'jump_statement : GOTO ID SEMI'\n pass\n\n\ndef p_jump_statement_2(t):\n 'jump_statement : CONTINUE SEMI'\n pass\n\n\ndef p_jump_statement_3(t):\n 'jump_statement : BREAK SEMI'\n pass\n\n\ndef p_jump_statement_4(t):\n 'jump_statement : RETURN expression_opt SEMI'\n pass\n\n\ndef p_expression_opt_1(t):\n 'expression_opt : empty'\n pass\n\n\ndef p_expression_opt_2(t):\n 'expression_opt : expression'\n pass\n\n# expression:\n\n\ndef p_expression_1(t):\n 'expression : assignment_expression'\n pass\n\n\ndef p_expression_2(t):\n 'expression : expression COMMA assignment_expression'\n pass\n\n# assigment_expression:\n\n\ndef p_assignment_expression_1(t):\n 'assignment_expression : conditional_expression'\n pass\n\n\ndef p_assignment_expression_2(t):\n 'assignment_expression : unary_expression assignment_operator assignment_expression'\n pass\n\n# assignment_operator:\n\n\ndef p_assignment_operator(t):\n '''\n assignment_operator : EQUALS\n | TIMESEQUAL\n | DIVEQUAL\n | MODEQUAL\n | PLUSEQUAL\n | MINUSEQUAL\n | LSHIFTEQUAL\n | RSHIFTEQUAL\n | ANDEQUAL\n | OREQUAL\n | XOREQUAL\n '''\n pass\n\n# conditional-expression\n\n\ndef p_conditional_expression_1(t):\n 'conditional_expression : logical_or_expression'\n pass\n\n\ndef p_conditional_expression_2(t):\n 'conditional_expression : logical_or_expression CONDOP expression COLON conditional_expression '\n pass\n\n# constant-expression\n\n\ndef p_constant_expression(t):\n 'constant_expression : conditional_expression'\n pass\n\n# logical-or-expression\n\n\ndef p_logical_or_expression_1(t):\n 'logical_or_expression : logical_and_expression'\n pass\n\n\ndef p_logical_or_expression_2(t):\n 'logical_or_expression : logical_or_expression LOR logical_and_expression'\n pass\n\n# logical-and-expression\n\n\ndef p_logical_and_expression_1(t):\n 'logical_and_expression : inclusive_or_expression'\n pass\n\n\ndef p_logical_and_expression_2(t):\n 'logical_and_expression : logical_and_expression LAND inclusive_or_expression'\n pass\n\n# inclusive-or-expression:\n\n\ndef p_inclusive_or_expression_1(t):\n 'inclusive_or_expression : exclusive_or_expression'\n pass\n\n\ndef p_inclusive_or_expression_2(t):\n 'inclusive_or_expression : inclusive_or_expression OR exclusive_or_expression'\n pass\n\n# exclusive-or-expression:\n\n\ndef p_exclusive_or_expression_1(t):\n 'exclusive_or_expression : and_expression'\n pass\n\n\ndef p_exclusive_or_expression_2(t):\n 'exclusive_or_expression : exclusive_or_expression XOR and_expression'\n pass\n\n# AND-expression\n\n\ndef p_and_expression_1(t):\n 'and_expression : equality_expression'\n pass\n\n\ndef p_and_expression_2(t):\n 'and_expression : and_expression AND equality_expression'\n pass\n\n\n# equality-expression:\ndef p_equality_expression_1(t):\n 'equality_expression : relational_expression'\n pass\n\n\ndef p_equality_expression_2(t):\n 'equality_expression : equality_expression EQ relational_expression'\n pass\n\n\ndef p_equality_expression_3(t):\n 'equality_expression : equality_expression NE relational_expression'\n pass\n\n\n# relational-expression:\ndef p_relational_expression_1(t):\n 'relational_expression : shift_expression'\n pass\n\n\ndef p_relational_expression_2(t):\n 'relational_expression : relational_expression LT shift_expression'\n pass\n\n\ndef p_relational_expression_3(t):\n 'relational_expression : relational_expression GT shift_expression'\n pass\n\n\ndef p_relational_expression_4(t):\n 'relational_expression : relational_expression LE shift_expression'\n pass\n\n\ndef p_relational_expression_5(t):\n 'relational_expression : relational_expression GE shift_expression'\n pass\n\n# shift-expression\n\n\ndef p_shift_expression_1(t):\n 'shift_expression : additive_expression'\n pass\n\n\ndef p_shift_expression_2(t):\n 'shift_expression : shift_expression LSHIFT additive_expression'\n pass\n\n\ndef p_shift_expression_3(t):\n 'shift_expression : shift_expression RSHIFT additive_expression'\n pass\n\n# additive-expression\n\n\ndef p_additive_expression_1(t):\n 'additive_expression : multiplicative_expression'\n pass\n\n\ndef p_additive_expression_2(t):\n 'additive_expression : additive_expression PLUS multiplicative_expression'\n pass\n\n\ndef p_additive_expression_3(t):\n 'additive_expression : additive_expression MINUS multiplicative_expression'\n pass\n\n# multiplicative-expression\n\n\ndef p_multiplicative_expression_1(t):\n 'multiplicative_expression : cast_expression'\n pass\n\n\ndef p_multiplicative_expression_2(t):\n 'multiplicative_expression : multiplicative_expression TIMES cast_expression'\n pass\n\n\ndef p_multiplicative_expression_3(t):\n 'multiplicative_expression : multiplicative_expression DIVIDE cast_expression'\n pass\n\n\ndef p_multiplicative_expression_4(t):\n 'multiplicative_expression : multiplicative_expression MOD cast_expression'\n pass\n\n# cast-expression:\n\n\ndef p_cast_expression_1(t):\n 'cast_expression : unary_expression'\n pass\n\n\ndef p_cast_expression_2(t):\n 'cast_expression : LPAREN type_name RPAREN cast_expression'\n pass\n\n# unary-expression:\n\n\ndef p_unary_expression_1(t):\n 'unary_expression : postfix_expression'\n pass\n\n\ndef p_unary_expression_2(t):\n 'unary_expression : PLUSPLUS unary_expression'\n pass\n\n\ndef p_unary_expression_3(t):\n 'unary_expression : MINUSMINUS unary_expression'\n pass\n\n\ndef p_unary_expression_4(t):\n 'unary_expression : unary_operator cast_expression'\n pass\n\n\ndef p_unary_expression_5(t):\n 'unary_expression : SIZEOF unary_expression'\n pass\n\n\ndef p_unary_expression_6(t):\n 'unary_expression : SIZEOF LPAREN type_name RPAREN'\n pass\n\n# unary-operator\n\n\ndef p_unary_operator(t):\n '''unary_operator : AND\n | TIMES\n | PLUS \n | MINUS\n | NOT\n | LNOT '''\n pass\n\n# postfix-expression:\n\n\ndef p_postfix_expression_1(t):\n 'postfix_expression : primary_expression'\n pass\n\n\ndef p_postfix_expression_2(t):\n 'postfix_expression : postfix_expression LBRACKET expression RBRACKET'\n pass\n\n\ndef p_postfix_expression_3(t):\n 'postfix_expression : postfix_expression LPAREN argument_expression_list RPAREN'\n pass\n\n\ndef p_postfix_expression_4(t):\n 'postfix_expression : postfix_expression LPAREN RPAREN'\n pass\n\n\ndef p_postfix_expression_5(t):\n 'postfix_expression : postfix_expression PERIOD ID'\n pass\n\n\ndef p_postfix_expression_6(t):\n 'postfix_expression : postfix_expression ARROW ID'\n pass\n\n\ndef p_postfix_expression_7(t):\n 'postfix_expression : postfix_expression PLUSPLUS'\n pass\n\n\ndef p_postfix_expression_8(t):\n 'postfix_expression : postfix_expression MINUSMINUS'\n pass\n\n# primary-expression:\n\n\ndef p_primary_expression(t):\n '''primary_expression : ID\n | constant\n | SCONST\n | LPAREN expression RPAREN'''\n pass\n\n# argument-expression-list:\n\n\ndef p_argument_expression_list(t):\n '''argument_expression_list : assignment_expression\n | argument_expression_list COMMA assignment_expression'''\n pass\n\n# constant:\n\n\ndef p_constant(t):\n '''constant : ICONST\n | FCONST\n | CCONST'''\n pass\n\n\ndef p_empty(t):\n 'empty : '\n pass\n\n\ndef p_error(t):\n print(\"Whoa. We're hosed\")\n\nimport profile\n# Build the grammar\n\nyacc.yacc()\n#yacc.yacc(method='LALR',write_tables=False,debug=False)\n\n#profile.run(\"yacc.yacc(method='LALR')\")\n" }, { "path": "xonsh/ply/example/calc/calc.py", "content": "# -----------------------------------------------------------------------------\n# calc.py\n#\n# A simple calculator with variables. This is from O'Reilly's\n# \"Lex and Yacc\", p. 63.\n# -----------------------------------------------------------------------------\n\nimport sys\nsys.path.insert(0, \"../..\")\n\nif sys.version_info[0] >= 3:\n raw_input = input\n\ntokens = (\n 'NAME', 'NUMBER',\n)\n\nliterals = ['=', '+', '-', '*', '/', '(', ')']\n\n# Tokens\n\nt_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*'\n\n\ndef t_NUMBER(t):\n r'\\d+'\n t.value = int(t.value)\n return t\n\nt_ignore = \" \\t\"\n\n\ndef t_newline(t):\n r'\\n+'\n t.lexer.lineno += t.value.count(\"\\n\")\n\n\ndef t_error(t):\n print(\"Illegal character '%s'\" % t.value[0])\n t.lexer.skip(1)\n\n# Build the lexer\nimport ply.lex as lex\nlex.lex()\n\n# Parsing rules\n\nprecedence = (\n ('left', '+', '-'),\n ('left', '*', '/'),\n ('right', 'UMINUS'),\n)\n\n# dictionary of names\nnames = {}\n\n\ndef p_statement_assign(p):\n 'statement : NAME \"=\" expression'\n names[p[1]] = p[3]\n\n\ndef p_statement_expr(p):\n 'statement : expression'\n print(p[1])\n\n\ndef p_expression_binop(p):\n '''expression : expression '+' expression\n | expression '-' expression\n | expression '*' expression\n | expression '/' expression'''\n if p[2] == '+':\n p[0] = p[1] + p[3]\n elif p[2] == '-':\n p[0] = p[1] - p[3]\n elif p[2] == '*':\n p[0] = p[1] * p[3]\n elif p[2] == '/':\n p[0] = p[1] / p[3]\n\n\ndef p_expression_uminus(p):\n \"expression : '-' expression %prec UMINUS\"\n p[0] = -p[2]\n\n\ndef p_expression_group(p):\n \"expression : '(' expression ')'\"\n p[0] = p[2]\n\n\ndef p_expression_number(p):\n \"expression : NUMBER\"\n p[0] = p[1]\n\n\ndef p_expression_name(p):\n \"expression : NAME\"\n try:\n p[0] = names[p[1]]\n except LookupError:\n print(\"Undefined name '%s'\" % p[1])\n p[0] = 0\n\n\ndef p_error(p):\n if p:\n print(\"Syntax error at '%s'\" % p.value)\n else:\n print(\"Syntax error at EOF\")\n\nimport ply.yacc as yacc\nyacc.yacc()\n\nwhile 1:\n try:\n s = raw_input('calc > ')\n except EOFError:\n break\n if not s:\n continue\n yacc.parse(s)\n" }, { "path": "xonsh/ply/example/calcdebug/calc.py", "content": "# -----------------------------------------------------------------------------\n# calc.py\n#\n# This example shows how to run the parser in a debugging mode\n# with output routed to a logging object.\n# -----------------------------------------------------------------------------\n\nimport sys\nsys.path.insert(0, \"../..\")\n\nif sys.version_info[0] >= 3:\n raw_input = input\n\ntokens = (\n 'NAME', 'NUMBER',\n)\n\nliterals = ['=', '+', '-', '*', '/', '(', ')']\n\n# Tokens\n\nt_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*'\n\n\ndef t_NUMBER(t):\n r'\\d+'\n t.value = int(t.value)\n return t\n\nt_ignore = \" \\t\"\n\n\ndef t_newline(t):\n r'\\n+'\n t.lexer.lineno += t.value.count(\"\\n\")\n\n\ndef t_error(t):\n print(\"Illegal character '%s'\" % t.value[0])\n t.lexer.skip(1)\n\n# Build the lexer\nimport ply.lex as lex\nlex.lex()\n\n# Parsing rules\n\nprecedence = (\n ('left', '+', '-'),\n ('left', '*', '/'),\n ('right', 'UMINUS'),\n)\n\n# dictionary of names\nnames = {}\n\n\ndef p_statement_assign(p):\n 'statement : NAME \"=\" expression'\n names[p[1]] = p[3]\n\n\ndef p_statement_expr(p):\n 'statement : expression'\n print(p[1])\n\n\ndef p_expression_binop(p):\n '''expression : expression '+' expression\n | expression '-' expression\n | expression '*' expression\n | expression '/' expression'''\n if p[2] == '+':\n p[0] = p[1] + p[3]\n elif p[2] == '-':\n p[0] = p[1] - p[3]\n elif p[2] == '*':\n p[0] = p[1] * p[3]\n elif p[2] == '/':\n p[0] = p[1] / p[3]\n\n\ndef p_expression_uminus(p):\n \"expression : '-' expression %prec UMINUS\"\n p[0] = -p[2]\n\n\ndef p_expression_group(p):\n \"expression : '(' expression ')'\"\n p[0] = p[2]\n\n\ndef p_expression_number(p):\n \"expression : NUMBER\"\n p[0] = p[1]\n\n\ndef p_expression_name(p):\n \"expression : NAME\"\n try:\n p[0] = names[p[1]]\n except LookupError:\n print(\"Undefined name '%s'\" % p[1])\n p[0] = 0\n\n\ndef p_error(p):\n if p:\n print(\"Syntax error at '%s'\" % p.value)\n else:\n print(\"Syntax error at EOF\")\n\nimport ply.yacc as yacc\nyacc.yacc()\n\nimport logging\nlogging.basicConfig(\n level=logging.INFO,\n filename=\"parselog.txt\"\n)\n\nwhile 1:\n try:\n s = raw_input('calc > ')\n except EOFError:\n break\n if not s:\n continue\n yacc.parse(s, debug=logging.getLogger())\n" }, { "path": "xonsh/ply/example/calceof/calc.py", "content": "# -----------------------------------------------------------------------------\n# calc.py\n#\n# A simple calculator with variables. Asks the user for more input and\n# demonstrates the use of the t_eof() rule.\n# -----------------------------------------------------------------------------\n\nimport sys\nsys.path.insert(0, \"../..\")\n\nif sys.version_info[0] >= 3:\n raw_input = input\n\ntokens = (\n 'NAME', 'NUMBER',\n)\n\nliterals = ['=', '+', '-', '*', '/', '(', ')']\n\n# Tokens\n\nt_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*'\n\n\ndef t_NUMBER(t):\n r'\\d+'\n t.value = int(t.value)\n return t\n\nt_ignore = \" \\t\"\n\n\ndef t_newline(t):\n r'\\n+'\n t.lexer.lineno += t.value.count(\"\\n\")\n\n\ndef t_eof(t):\n more = raw_input('... ')\n if more:\n t.lexer.input(more + '\\n')\n return t.lexer.token()\n else:\n return None\n\n\ndef t_error(t):\n print(\"Illegal character '%s'\" % t.value[0])\n t.lexer.skip(1)\n\n# Build the lexer\nimport ply.lex as lex\nlex.lex()\n\n# Parsing rules\n\nprecedence = (\n ('left', '+', '-'),\n ('left', '*', '/'),\n ('right', 'UMINUS'),\n)\n\n# dictionary of names\nnames = {}\n\n\ndef p_statement_assign(p):\n 'statement : NAME \"=\" expression'\n names[p[1]] = p[3]\n\n\ndef p_statement_expr(p):\n 'statement : expression'\n print(p[1])\n\n\ndef p_expression_binop(p):\n '''expression : expression '+' expression\n | expression '-' expression\n | expression '*' expression\n | expression '/' expression'''\n if p[2] == '+':\n p[0] = p[1] + p[3]\n elif p[2] == '-':\n p[0] = p[1] - p[3]\n elif p[2] == '*':\n p[0] = p[1] * p[3]\n elif p[2] == '/':\n p[0] = p[1] / p[3]\n\n\ndef p_expression_uminus(p):\n \"expression : '-' expression %prec UMINUS\"\n p[0] = -p[2]\n\n\ndef p_expression_group(p):\n \"expression : '(' expression ')'\"\n p[0] = p[2]\n\n\ndef p_expression_number(p):\n \"expression : NUMBER\"\n p[0] = p[1]\n\n\ndef p_expression_name(p):\n \"expression : NAME\"\n try:\n p[0] = names[p[1]]\n except LookupError:\n print(\"Undefined name '%s'\" % p[1])\n p[0] = 0\n\n\ndef p_error(p):\n if p:\n print(\"Syntax error at '%s'\" % p.value)\n else:\n print(\"Syntax error at EOF\")\n\nimport ply.yacc as yacc\nyacc.yacc()\n\nwhile 1:\n try:\n s = raw_input('calc > ')\n except EOFError:\n break\n if not s:\n continue\n yacc.parse(s + '\\n')\n" }, { "path": "xonsh/ply/example/classcalc/calc.py", "content": "#!/usr/bin/env python\n\n# -----------------------------------------------------------------------------\n# calc.py\n#\n# A simple calculator with variables. This is from O'Reilly's\n# \"Lex and Yacc\", p. 63.\n#\n# Class-based example contributed to PLY by David McNab\n# -----------------------------------------------------------------------------\n\nimport sys\nsys.path.insert(0, \"../..\")\n\nif sys.version_info[0] >= 3:\n raw_input = input\n\nimport ply.lex as lex\nimport ply.yacc as yacc\nimport os\n\n\nclass Parser:\n \"\"\"\n Base class for a lexer/parser that has the rules defined as methods\n \"\"\"\n tokens = ()\n precedence = ()\n\n def __init__(self, **kw):\n self.debug = kw.get('debug', 0)\n self.names = {}\n try:\n modname = os.path.split(os.path.splitext(__file__)[0])[\n 1] + \"_\" + self.__class__.__name__\n except:\n modname = \"parser\" + \"_\" + self.__class__.__name__\n self.debugfile = modname + \".dbg\"\n self.tabmodule = modname + \"_\" + \"parsetab\"\n # print self.debugfile, self.tabmodule\n\n # Build the lexer and parser\n lex.lex(module=self, debug=self.debug)\n yacc.yacc(module=self,\n debug=self.debug,\n debugfile=self.debugfile,\n tabmodule=self.tabmodule)\n\n def run(self):\n while 1:\n try:\n s = raw_input('calc > ')\n except EOFError:\n break\n if not s:\n continue\n yacc.parse(s)\n\n\nclass Calc(Parser):\n\n tokens = (\n 'NAME', 'NUMBER',\n 'PLUS', 'MINUS', 'EXP', 'TIMES', 'DIVIDE', 'EQUALS',\n 'LPAREN', 'RPAREN',\n )\n\n # Tokens\n\n t_PLUS = r'\\+'\n t_MINUS = r'-'\n t_EXP = r'\\*\\*'\n t_TIMES = r'\\*'\n t_DIVIDE = r'/'\n t_EQUALS = r'='\n t_LPAREN = r'\\('\n t_RPAREN = r'\\)'\n t_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*'\n\n def t_NUMBER(self, t):\n r'\\d+'\n try:\n t.value = int(t.value)\n except ValueError:\n print(\"Integer value too large %s\" % t.value)\n t.value = 0\n # print \"parsed number %s\" % repr(t.value)\n return t\n\n t_ignore = \" \\t\"\n\n def t_newline(self, t):\n r'\\n+'\n t.lexer.lineno += t.value.count(\"\\n\")\n\n def t_error(self, t):\n print(\"Illegal character '%s'\" % t.value[0])\n t.lexer.skip(1)\n\n # Parsing rules\n\n precedence = (\n ('left', 'PLUS', 'MINUS'),\n ('left', 'TIMES', 'DIVIDE'),\n ('left', 'EXP'),\n ('right', 'UMINUS'),\n )\n\n def p_statement_assign(self, p):\n 'statement : NAME EQUALS expression'\n self.names[p[1]] = p[3]\n\n def p_statement_expr(self, p):\n 'statement : expression'\n print(p[1])\n\n def p_expression_binop(self, p):\n \"\"\"\n expression : expression PLUS expression\n | expression MINUS expression\n | expression TIMES expression\n | expression DIVIDE expression\n | expression EXP expression\n \"\"\"\n # print [repr(p[i]) for i in range(0,4)]\n if p[2] == '+':\n p[0] = p[1] + p[3]\n elif p[2] == '-':\n p[0] = p[1] - p[3]\n elif p[2] == '*':\n p[0] = p[1] * p[3]\n elif p[2] == '/':\n p[0] = p[1] / p[3]\n elif p[2] == '**':\n p[0] = p[1] ** p[3]\n\n def p_expression_uminus(self, p):\n 'expression : MINUS expression %prec UMINUS'\n p[0] = -p[2]\n\n def p_expression_group(self, p):\n 'expression : LPAREN expression RPAREN'\n p[0] = p[2]\n\n def p_expression_number(self, p):\n 'expression : NUMBER'\n p[0] = p[1]\n\n def p_expression_name(self, p):\n 'expression : NAME'\n try:\n p[0] = self.names[p[1]]\n except LookupError:\n print(\"Undefined name '%s'\" % p[1])\n p[0] = 0\n\n def p_error(self, p):\n if p:\n print(\"Syntax error at '%s'\" % p.value)\n else:\n print(\"Syntax error at EOF\")\n\nif __name__ == '__main__':\n calc = Calc()\n calc.run()\n" }, { "path": "xonsh/ply/example/cleanup.sh", "content": "#!/bin/sh\nrm -f */*.pyc */parsetab.py */parser.out */*~ */*.class\n" }, { "path": "xonsh/ply/example/closurecalc/calc.py", "content": "# -----------------------------------------------------------------------------\n# calc.py\n#\n# A calculator parser that makes use of closures. The function make_calculator()\n# returns a function that accepts an input string and returns a result. All\n# lexing rules, parsing rules, and internal state are held inside the function.\n# -----------------------------------------------------------------------------\n\nimport sys\nsys.path.insert(0, \"../..\")\n\nif sys.version_info[0] >= 3:\n raw_input = input\n\n# Make a calculator function\n\n\ndef make_calculator():\n import ply.lex as lex\n import ply.yacc as yacc\n\n # ------- Internal calculator state\n\n variables = {} # Dictionary of stored variables\n\n # ------- Calculator tokenizing rules\n\n tokens = (\n 'NAME', 'NUMBER',\n )\n\n literals = ['=', '+', '-', '*', '/', '(', ')']\n\n t_ignore = \" \\t\"\n\n t_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*'\n\n def t_NUMBER(t):\n r'\\d+'\n t.value = int(t.value)\n return t\n\n def t_newline(t):\n r'\\n+'\n t.lexer.lineno += t.value.count(\"\\n\")\n\n def t_error(t):\n print(\"Illegal character '%s'\" % t.value[0])\n t.lexer.skip(1)\n\n # Build the lexer\n lexer = lex.lex()\n\n # ------- Calculator parsing rules\n\n precedence = (\n ('left', '+', '-'),\n ('left', '*', '/'),\n ('right', 'UMINUS'),\n )\n\n def p_statement_assign(p):\n 'statement : NAME \"=\" expression'\n variables[p[1]] = p[3]\n p[0] = None\n\n def p_statement_expr(p):\n 'statement : expression'\n p[0] = p[1]\n\n def p_expression_binop(p):\n '''expression : expression '+' expression\n | expression '-' expression\n | expression '*' expression\n | expression '/' expression'''\n if p[2] == '+':\n p[0] = p[1] + p[3]\n elif p[2] == '-':\n p[0] = p[1] - p[3]\n elif p[2] == '*':\n p[0] = p[1] * p[3]\n elif p[2] == '/':\n p[0] = p[1] / p[3]\n\n def p_expression_uminus(p):\n \"expression : '-' expression %prec UMINUS\"\n p[0] = -p[2]\n\n def p_expression_group(p):\n \"expression : '(' expression ')'\"\n p[0] = p[2]\n\n def p_expression_number(p):\n \"expression : NUMBER\"\n p[0] = p[1]\n\n def p_expression_name(p):\n \"expression : NAME\"\n try:\n p[0] = variables[p[1]]\n except LookupError:\n print(\"Undefined name '%s'\" % p[1])\n p[0] = 0\n\n def p_error(p):\n if p:\n print(\"Syntax error at '%s'\" % p.value)\n else:\n print(\"Syntax error at EOF\")\n\n # Build the parser\n parser = yacc.yacc()\n\n # ------- Input function\n\n def input(text):\n result = parser.parse(text, lexer=lexer)\n return result\n\n return input\n\n# Make a calculator object and use it\ncalc = make_calculator()\n\nwhile True:\n try:\n s = raw_input(\"calc > \")\n except EOFError:\n break\n r = calc(s)\n if r:\n print(r)\n" }, { "path": "xonsh/ply/example/hedit/hedit.py", "content": "# -----------------------------------------------------------------------------\n# hedit.py\n#\n# Paring of Fortran H Edit descriptions (Contributed by Pearu Peterson)\n#\n# These tokens can't be easily tokenized because they are of the following\n# form:\n#\n# nHc1...cn\n#\n# where n is a positive integer and c1 ... cn are characters.\n#\n# This example shows how to modify the state of the lexer to parse\n# such tokens\n# -----------------------------------------------------------------------------\n\nimport sys\nsys.path.insert(0, \"../..\")\n\n\ntokens = (\n 'H_EDIT_DESCRIPTOR',\n)\n\n# Tokens\nt_ignore = \" \\t\\n\"\n\n\ndef t_H_EDIT_DESCRIPTOR(t):\n r\"\\d+H.*\" # This grabs all of the remaining text\n i = t.value.index('H')\n n = eval(t.value[:i])\n\n # Adjust the tokenizing position\n t.lexer.lexpos -= len(t.value) - (i + 1 + n)\n\n t.value = t.value[i + 1:i + 1 + n]\n return t\n\n\ndef t_error(t):\n print(\"Illegal character '%s'\" % t.value[0])\n t.lexer.skip(1)\n\n# Build the lexer\nimport ply.lex as lex\nlex.lex()\nlex.runmain()\n" }, { "path": "xonsh/ply/example/newclasscalc/calc.py", "content": "#!/usr/bin/env python\n\n# -----------------------------------------------------------------------------\n# calc.py\n#\n# A simple calculator with variables. This is from O'Reilly's\n# \"Lex and Yacc\", p. 63.\n#\n# Class-based example contributed to PLY by David McNab.\n#\n# Modified to use new-style classes. Test case.\n# -----------------------------------------------------------------------------\n\nimport sys\nsys.path.insert(0, \"../..\")\n\nif sys.version_info[0] >= 3:\n raw_input = input\n\nimport ply.lex as lex\nimport ply.yacc as yacc\nimport os\n\n\nclass Parser(object):\n \"\"\"\n Base class for a lexer/parser that has the rules defined as methods\n \"\"\"\n tokens = ()\n precedence = ()\n\n def __init__(self, **kw):\n self.debug = kw.get('debug', 0)\n self.names = {}\n try:\n modname = os.path.split(os.path.splitext(__file__)[0])[\n 1] + \"_\" + self.__class__.__name__\n except:\n modname = \"parser\" + \"_\" + self.__class__.__name__\n self.debugfile = modname + \".dbg\"\n self.tabmodule = modname + \"_\" + \"parsetab\"\n # print self.debugfile, self.tabmodule\n\n # Build the lexer and parser\n lex.lex(module=self, debug=self.debug)\n yacc.yacc(module=self,\n debug=self.debug,\n debugfile=self.debugfile,\n tabmodule=self.tabmodule)\n\n def run(self):\n while 1:\n try:\n s = raw_input('calc > ')\n except EOFError:\n break\n if not s:\n continue\n yacc.parse(s)\n\n\nclass Calc(Parser):\n\n tokens = (\n 'NAME', 'NUMBER',\n 'PLUS', 'MINUS', 'EXP', 'TIMES', 'DIVIDE', 'EQUALS',\n 'LPAREN', 'RPAREN',\n )\n\n # Tokens\n\n t_PLUS = r'\\+'\n t_MINUS = r'-'\n t_EXP = r'\\*\\*'\n t_TIMES = r'\\*'\n t_DIVIDE = r'/'\n t_EQUALS = r'='\n t_LPAREN = r'\\('\n t_RPAREN = r'\\)'\n t_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*'\n\n def t_NUMBER(self, t):\n r'\\d+'\n try:\n t.value = int(t.value)\n except ValueError:\n print(\"Integer value too large %s\" % t.value)\n t.value = 0\n # print \"parsed number %s\" % repr(t.value)\n return t\n\n t_ignore = \" \\t\"\n\n def t_newline(self, t):\n r'\\n+'\n t.lexer.lineno += t.value.count(\"\\n\")\n\n def t_error(self, t):\n print(\"Illegal character '%s'\" % t.value[0])\n t.lexer.skip(1)\n\n # Parsing rules\n\n precedence = (\n ('left', 'PLUS', 'MINUS'),\n ('left', 'TIMES', 'DIVIDE'),\n ('left', 'EXP'),\n ('right', 'UMINUS'),\n )\n\n def p_statement_assign(self, p):\n 'statement : NAME EQUALS expression'\n self.names[p[1]] = p[3]\n\n def p_statement_expr(self, p):\n 'statement : expression'\n print(p[1])\n\n def p_expression_binop(self, p):\n \"\"\"\n expression : expression PLUS expression\n | expression MINUS expression\n | expression TIMES expression\n | expression DIVIDE expression\n | expression EXP expression\n \"\"\"\n # print [repr(p[i]) for i in range(0,4)]\n if p[2] == '+':\n p[0] = p[1] + p[3]\n elif p[2] == '-':\n p[0] = p[1] - p[3]\n elif p[2] == '*':\n p[0] = p[1] * p[3]\n elif p[2] == '/':\n p[0] = p[1] / p[3]\n elif p[2] == '**':\n p[0] = p[1] ** p[3]\n\n def p_expression_uminus(self, p):\n 'expression : MINUS expression %prec UMINUS'\n p[0] = -p[2]\n\n def p_expression_group(self, p):\n 'expression : LPAREN expression RPAREN'\n p[0] = p[2]\n\n def p_expression_number(self, p):\n 'expression : NUMBER'\n p[0] = p[1]\n\n def p_expression_name(self, p):\n 'expression : NAME'\n try:\n p[0] = self.names[p[1]]\n except LookupError:\n print(\"Undefined name '%s'\" % p[1])\n p[0] = 0\n\n def p_error(self, p):\n if p:\n print(\"Syntax error at '%s'\" % p.value)\n else:\n print(\"Syntax error at EOF\")\n\nif __name__ == '__main__':\n calc = Calc()\n calc.run()\n" }, { "path": "xonsh/ply/example/optcalc/README", "content": "An example showing how to use Python optimized mode.\nTo run:\n\n - First run 'python calc.py'\n\n - Then run 'python -OO calc.py'\n\nIf working correctly, the second version should run the\nsame way.\n" }, { "path": "xonsh/ply/example/optcalc/calc.py", "content": "# -----------------------------------------------------------------------------\n# calc.py\n#\n# A simple calculator with variables. This is from O'Reilly's\n# \"Lex and Yacc\", p. 63.\n# -----------------------------------------------------------------------------\n\nimport sys\nsys.path.insert(0, \"../..\")\n\nif sys.version_info[0] >= 3:\n raw_input = input\n\ntokens = (\n 'NAME', 'NUMBER',\n 'PLUS', 'MINUS', 'TIMES', 'DIVIDE', 'EQUALS',\n 'LPAREN', 'RPAREN',\n)\n\n# Tokens\n\nt_PLUS = r'\\+'\nt_MINUS = r'-'\nt_TIMES = r'\\*'\nt_DIVIDE = r'/'\nt_EQUALS = r'='\nt_LPAREN = r'\\('\nt_RPAREN = r'\\)'\nt_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*'\n\n\ndef t_NUMBER(t):\n r'\\d+'\n try:\n t.value = int(t.value)\n except ValueError:\n print(\"Integer value too large %s\" % t.value)\n t.value = 0\n return t\n\nt_ignore = \" \\t\"\n\n\ndef t_newline(t):\n r'\\n+'\n t.lexer.lineno += t.value.count(\"\\n\")\n\n\ndef t_error(t):\n print(\"Illegal character '%s'\" % t.value[0])\n t.lexer.skip(1)\n\n# Build the lexer\nimport ply.lex as lex\nlex.lex(optimize=1)\n\n# Parsing rules\n\nprecedence = (\n ('left', 'PLUS', 'MINUS'),\n ('left', 'TIMES', 'DIVIDE'),\n ('right', 'UMINUS'),\n)\n\n# dictionary of names\nnames = {}\n\n\ndef p_statement_assign(t):\n 'statement : NAME EQUALS expression'\n names[t[1]] = t[3]\n\n\ndef p_statement_expr(t):\n 'statement : expression'\n print(t[1])\n\n\ndef p_expression_binop(t):\n '''expression : expression PLUS expression\n | expression MINUS expression\n | expression TIMES expression\n | expression DIVIDE expression'''\n if t[2] == '+':\n t[0] = t[1] + t[3]\n elif t[2] == '-':\n t[0] = t[1] - t[3]\n elif t[2] == '*':\n t[0] = t[1] * t[3]\n elif t[2] == '/':\n t[0] = t[1] / t[3]\n elif t[2] == '<':\n t[0] = t[1] < t[3]\n\n\ndef p_expression_uminus(t):\n 'expression : MINUS expression %prec UMINUS'\n t[0] = -t[2]\n\n\ndef p_expression_group(t):\n 'expression : LPAREN expression RPAREN'\n t[0] = t[2]\n\n\ndef p_expression_number(t):\n 'expression : NUMBER'\n t[0] = t[1]\n\n\ndef p_expression_name(t):\n 'expression : NAME'\n try:\n t[0] = names[t[1]]\n except LookupError:\n print(\"Undefined name '%s'\" % t[1])\n t[0] = 0\n\n\ndef p_error(t):\n if t:\n print(\"Syntax error at '%s'\" % t.value)\n else:\n print(\"Syntax error at EOF\")\n\nimport ply.yacc as yacc\nyacc.yacc(optimize=1)\n\nwhile 1:\n try:\n s = raw_input('calc > ')\n except EOFError:\n break\n yacc.parse(s)\n" }, { "path": "xonsh/ply/example/unicalc/calc.py", "content": "# -----------------------------------------------------------------------------\n# calc.py\n#\n# A simple calculator with variables. This is from O'Reilly's\n# \"Lex and Yacc\", p. 63.\n#\n# This example uses unicode strings for tokens, docstrings, and input.\n# -----------------------------------------------------------------------------\n\nimport sys\nsys.path.insert(0, \"../..\")\n\ntokens = (\n 'NAME', 'NUMBER',\n 'PLUS', 'MINUS', 'TIMES', 'DIVIDE', 'EQUALS',\n 'LPAREN', 'RPAREN',\n)\n\n# Tokens\n\nt_PLUS = ur'\\+'\nt_MINUS = ur'-'\nt_TIMES = ur'\\*'\nt_DIVIDE = ur'/'\nt_EQUALS = ur'='\nt_LPAREN = ur'\\('\nt_RPAREN = ur'\\)'\nt_NAME = ur'[a-zA-Z_][a-zA-Z0-9_]*'\n\n\ndef t_NUMBER(t):\n ur'\\d+'\n try:\n t.value = int(t.value)\n except ValueError:\n print \"Integer value too large\", t.value\n t.value = 0\n return t\n\nt_ignore = u\" \\t\"\n\n\ndef t_newline(t):\n ur'\\n+'\n t.lexer.lineno += t.value.count(\"\\n\")\n\n\ndef t_error(t):\n print \"Illegal character '%s'\" % t.value[0]\n t.lexer.skip(1)\n\n# Build the lexer\nimport ply.lex as lex\nlex.lex()\n\n# Parsing rules\n\nprecedence = (\n ('left', 'PLUS', 'MINUS'),\n ('left', 'TIMES', 'DIVIDE'),\n ('right', 'UMINUS'),\n)\n\n# dictionary of names\nnames = {}\n\n\ndef p_statement_assign(p):\n 'statement : NAME EQUALS expression'\n names[p[1]] = p[3]\n\n\ndef p_statement_expr(p):\n 'statement : expression'\n print p[1]\n\n\ndef p_expression_binop(p):\n '''expression : expression PLUS expression\n | expression MINUS expression\n | expression TIMES expression\n | expression DIVIDE expression'''\n if p[2] == u'+':\n p[0] = p[1] + p[3]\n elif p[2] == u'-':\n p[0] = p[1] - p[3]\n elif p[2] == u'*':\n p[0] = p[1] * p[3]\n elif p[2] == u'/':\n p[0] = p[1] / p[3]\n\n\ndef p_expression_uminus(p):\n 'expression : MINUS expression %prec UMINUS'\n p[0] = -p[2]\n\n\ndef p_expression_group(p):\n 'expression : LPAREN expression RPAREN'\n p[0] = p[2]\n\n\ndef p_expression_number(p):\n 'expression : NUMBER'\n p[0] = p[1]\n\n\ndef p_expression_name(p):\n 'expression : NAME'\n try:\n p[0] = names[p[1]]\n except LookupError:\n print \"Undefined name '%s'\" % p[1]\n p[0] = 0\n\n\ndef p_error(p):\n if p:\n print \"Syntax error at '%s'\" % p.value\n else:\n print \"Syntax error at EOF\"\n\nimport ply.yacc as yacc\nyacc.yacc()\n\nwhile 1:\n try:\n s = raw_input('calc > ')\n except EOFError:\n break\n if not s:\n continue\n yacc.parse(unicode(s))\n" }, { "path": "xonsh/ply/example/yply/README", "content": "yply.py\n\nThis example implements a program yply.py that converts a UNIX-yacc\nspecification file into a PLY-compatible program. To use, simply\nrun it like this:\n\n % python yply.py [-nocode] inputfile.y >myparser.py\n\nThe output of this program is Python code. In the output,\nany C code in the original file is included, but is commented out.\nIf you use the -nocode option, then all of the C code in the\noriginal file is just discarded.\n\nTo use the resulting grammer with PLY, you'll need to edit the\nmyparser.py file. Within this file, some stub code is included that\ncan be used to test the construction of the parsing tables. However,\nyou'll need to do more editing to make a workable parser.\n\nDisclaimer: This just an example I threw together in an afternoon.\nIt might have some bugs. However, it worked when I tried it on\na yacc-specified C++ parser containing 442 rules and 855 parsing\nstates.\n\nComments:\n\n1. This example does not parse specification files meant for lex/flex.\n You'll need to specify the tokenizer on your own.\n\n2. This example shows a number of interesting PLY features including\n \n - Parsing of literal text delimited by nested parentheses\n - Some interaction between the parser and the lexer.\n - Use of literals in the grammar specification\n - One pass compilation. The program just emits the result,\n there is no intermediate parse tree.\n\n3. This program could probably be cleaned up and enhanced a lot.\n It would be great if someone wanted to work on this (hint).\n\n-Dave\n \n" }, { "path": "xonsh/ply/example/yply/ylex.py", "content": "# lexer for yacc-grammars\n#\n# Author: David Beazley (dave@dabeaz.com)\n# Date : October 2, 2006\n\nimport sys\nsys.path.append(\"../..\")\n\nfrom ply import *\n\ntokens = (\n 'LITERAL', 'SECTION', 'TOKEN', 'LEFT', 'RIGHT', 'PREC', 'START', 'TYPE', 'NONASSOC', 'UNION', 'CODE',\n 'ID', 'QLITERAL', 'NUMBER',\n)\n\nstates = (('code', 'exclusive'),)\n\nliterals = [';', ',', '<', '>', '|', ':']\nt_ignore = ' \\t'\n\nt_TOKEN = r'%token'\nt_LEFT = r'%left'\nt_RIGHT = r'%right'\nt_NONASSOC = r'%nonassoc'\nt_PREC = r'%prec'\nt_START = r'%start'\nt_TYPE = r'%type'\nt_UNION = r'%union'\nt_ID = r'[a-zA-Z_][a-zA-Z_0-9]*'\nt_QLITERAL = r'''(?P['\"]).*?(?P=quote)'''\nt_NUMBER = r'\\d+'\n\n\ndef t_SECTION(t):\n r'%%'\n if getattr(t.lexer, \"lastsection\", 0):\n t.value = t.lexer.lexdata[t.lexpos + 2:]\n t.lexer.lexpos = len(t.lexer.lexdata)\n else:\n t.lexer.lastsection = 0\n return t\n\n# Comments\n\n\ndef t_ccomment(t):\n r'/\\*(.|\\n)*?\\*/'\n t.lexer.lineno += t.value.count('\\n')\n\nt_ignore_cppcomment = r'//.*'\n\n\ndef t_LITERAL(t):\n r'%\\{(.|\\n)*?%\\}'\n t.lexer.lineno += t.value.count(\"\\n\")\n return t\n\n\ndef t_NEWLINE(t):\n r'\\n'\n t.lexer.lineno += 1\n\n\ndef t_code(t):\n r'\\{'\n t.lexer.codestart = t.lexpos\n t.lexer.level = 1\n t.lexer.begin('code')\n\n\ndef t_code_ignore_string(t):\n r'\\\"([^\\\\\\n]|(\\\\.))*?\\\"'\n\n\ndef t_code_ignore_char(t):\n r'\\'([^\\\\\\n]|(\\\\.))*?\\''\n\n\ndef t_code_ignore_comment(t):\n r'/\\*(.|\\n)*?\\*/'\n\n\ndef t_code_ignore_cppcom(t):\n r'//.*'\n\n\ndef t_code_lbrace(t):\n r'\\{'\n t.lexer.level += 1\n\n\ndef t_code_rbrace(t):\n r'\\}'\n t.lexer.level -= 1\n if t.lexer.level == 0:\n t.type = 'CODE'\n t.value = t.lexer.lexdata[t.lexer.codestart:t.lexpos + 1]\n t.lexer.begin('INITIAL')\n t.lexer.lineno += t.value.count('\\n')\n return t\n\nt_code_ignore_nonspace = r'[^\\s\\}\\'\\\"\\{]+'\nt_code_ignore_whitespace = r'\\s+'\nt_code_ignore = \"\"\n\n\ndef t_code_error(t):\n raise RuntimeError\n\n\ndef t_error(t):\n print(\"%d: Illegal character '%s'\" % (t.lexer.lineno, t.value[0]))\n print(t.value)\n t.lexer.skip(1)\n\nlex.lex()\n\nif __name__ == '__main__':\n lex.runmain()\n" }, { "path": "xonsh/ply/example/yply/yparse.py", "content": "# parser for Unix yacc-based grammars\n#\n# Author: David Beazley (dave@dabeaz.com)\n# Date : October 2, 2006\n\nimport ylex\ntokens = ylex.tokens\n\nfrom ply import *\n\ntokenlist = []\npreclist = []\n\nemit_code = 1\n\n\ndef p_yacc(p):\n '''yacc : defsection rulesection'''\n\n\ndef p_defsection(p):\n '''defsection : definitions SECTION\n | SECTION'''\n p.lexer.lastsection = 1\n print(\"tokens = \", repr(tokenlist))\n print()\n print(\"precedence = \", repr(preclist))\n print()\n print(\"# -------------- RULES ----------------\")\n print()\n\n\ndef p_rulesection(p):\n '''rulesection : rules SECTION'''\n\n print(\"# -------------- RULES END ----------------\")\n print_code(p[2], 0)\n\n\ndef p_definitions(p):\n '''definitions : definitions definition\n | definition'''\n\n\ndef p_definition_literal(p):\n '''definition : LITERAL'''\n print_code(p[1], 0)\n\n\ndef p_definition_start(p):\n '''definition : START ID'''\n print(\"start = '%s'\" % p[2])\n\n\ndef p_definition_token(p):\n '''definition : toktype opttype idlist optsemi '''\n for i in p[3]:\n if i[0] not in \"'\\\"\":\n tokenlist.append(i)\n if p[1] == '%left':\n preclist.append(('left',) + tuple(p[3]))\n elif p[1] == '%right':\n preclist.append(('right',) + tuple(p[3]))\n elif p[1] == '%nonassoc':\n preclist.append(('nonassoc',) + tuple(p[3]))\n\n\ndef p_toktype(p):\n '''toktype : TOKEN\n | LEFT\n | RIGHT\n | NONASSOC'''\n p[0] = p[1]\n\n\ndef p_opttype(p):\n '''opttype : '<' ID '>'\n | empty'''\n\n\ndef p_idlist(p):\n '''idlist : idlist optcomma tokenid\n | tokenid'''\n if len(p) == 2:\n p[0] = [p[1]]\n else:\n p[0] = p[1]\n p[1].append(p[3])\n\n\ndef p_tokenid(p):\n '''tokenid : ID \n | ID NUMBER\n | QLITERAL\n | QLITERAL NUMBER'''\n p[0] = p[1]\n\n\ndef p_optsemi(p):\n '''optsemi : ';'\n | empty'''\n\n\ndef p_optcomma(p):\n '''optcomma : ','\n | empty'''\n\n\ndef p_definition_type(p):\n '''definition : TYPE '<' ID '>' namelist optsemi'''\n # type declarations are ignored\n\n\ndef p_namelist(p):\n '''namelist : namelist optcomma ID\n | ID'''\n\n\ndef p_definition_union(p):\n '''definition : UNION CODE optsemi'''\n # Union declarations are ignored\n\n\ndef p_rules(p):\n '''rules : rules rule\n | rule'''\n if len(p) == 2:\n rule = p[1]\n else:\n rule = p[2]\n\n # Print out a Python equivalent of this rule\n\n embedded = [] # Embedded actions (a mess)\n embed_count = 0\n\n rulename = rule[0]\n rulecount = 1\n for r in rule[1]:\n # r contains one of the rule possibilities\n print(\"def p_%s_%d(p):\" % (rulename, rulecount))\n prod = []\n prodcode = \"\"\n for i in range(len(r)):\n item = r[i]\n if item[0] == '{': # A code block\n if i == len(r) - 1:\n prodcode = item\n break\n else:\n # an embedded action\n embed_name = \"_embed%d_%s\" % (embed_count, rulename)\n prod.append(embed_name)\n embedded.append((embed_name, item))\n embed_count += 1\n else:\n prod.append(item)\n print(\" '''%s : %s'''\" % (rulename, \" \".join(prod)))\n # Emit code\n print_code(prodcode, 4)\n print()\n rulecount += 1\n\n for e, code in embedded:\n print(\"def p_%s(p):\" % e)\n print(\" '''%s : '''\" % e)\n print_code(code, 4)\n print()\n\n\ndef p_rule(p):\n '''rule : ID ':' rulelist ';' '''\n p[0] = (p[1], [p[3]])\n\n\ndef p_rule2(p):\n '''rule : ID ':' rulelist morerules ';' '''\n p[4].insert(0, p[3])\n p[0] = (p[1], p[4])\n\n\ndef p_rule_empty(p):\n '''rule : ID ':' ';' '''\n p[0] = (p[1], [[]])\n\n\ndef p_rule_empty2(p):\n '''rule : ID ':' morerules ';' '''\n\n p[3].insert(0, [])\n p[0] = (p[1], p[3])\n\n\ndef p_morerules(p):\n '''morerules : morerules '|' rulelist\n | '|' rulelist\n | '|' '''\n\n if len(p) == 2:\n p[0] = [[]]\n elif len(p) == 3:\n p[0] = [p[2]]\n else:\n p[0] = p[1]\n p[0].append(p[3])\n\n# print(\"morerules\", len(p), p[0])\n\n\ndef p_rulelist(p):\n '''rulelist : rulelist ruleitem\n | ruleitem'''\n\n if len(p) == 2:\n p[0] = [p[1]]\n else:\n p[0] = p[1]\n p[1].append(p[2])\n\n\ndef p_ruleitem(p):\n '''ruleitem : ID\n | QLITERAL\n | CODE\n | PREC'''\n p[0] = p[1]\n\n\ndef p_empty(p):\n '''empty : '''\n\n\ndef p_error(p):\n pass\n\nyacc.yacc(debug=0)\n\n\ndef print_code(code, indent):\n if not emit_code:\n return\n codelines = code.splitlines()\n for c in codelines:\n print(\"%s# %s\" % (\" \" * indent, c))\n" }, { "path": "xonsh/ply/example/yply/yply.py", "content": "#!/usr/local/bin/python\n# yply.py\n#\n# Author: David Beazley (dave@dabeaz.com)\n# Date : October 2, 2006\n#\n# Converts a UNIX-yacc specification file into a PLY-compatible\n# specification. To use, simply do this:\n#\n# % python yply.py [-nocode] inputfile.y >myparser.py\n#\n# The output of this program is Python code. In the output,\n# any C code in the original file is included, but is commented.\n# If you use the -nocode option, then all of the C code in the\n# original file is discarded.\n#\n# Disclaimer: This just an example I threw together in an afternoon.\n# It might have some bugs. However, it worked when I tried it on\n# a yacc-specified C++ parser containing 442 rules and 855 parsing\n# states.\n#\n\nimport sys\nsys.path.insert(0, \"../..\")\n\nimport ylex\nimport yparse\n\nfrom ply import *\n\nif len(sys.argv) == 1:\n print(\"usage : yply.py [-nocode] inputfile\")\n raise SystemExit\n\nif len(sys.argv) == 3:\n if sys.argv[1] == '-nocode':\n yparse.emit_code = 0\n else:\n print(\"Unknown option '%s'\" % sys.argv[1])\n raise SystemExit\n filename = sys.argv[2]\nelse:\n filename = sys.argv[1]\n\nyacc.parse(open(filename).read())\n\nprint(\"\"\"\nif __name__ == '__main__':\n from ply import *\n yacc.yacc()\n\"\"\")\n" }, { "path": "xonsh/ply/ply/__init__.py", "content": "# PLY package\n# Author: David Beazley (dave@dabeaz.com)\n\n__version__ = '3.11'\n__all__ = ['lex','yacc']\n" }, { "path": "xonsh/ply/ply/cpp.py", "content": "# -----------------------------------------------------------------------------\n# ply: cpp.py\n#\n# Copyright (C) 2001-2019\n# David M. Beazley (Dabeaz LLC)\n# All rights reserved.\n#\n# Latest version: https://github.com/dabeaz/ply\n#\n# Redistribution and use in source and binary forms, with or without\n# modification, are permitted provided that the following conditions are\n# met:\n#\n# * Redistributions of source code must retain the above copyright notice,\n# this list of conditions and the following disclaimer.\n# * Redistributions in binary form must reproduce the above copyright notice,\n# this list of conditions and the following disclaimer in the documentation\n# and/or other materials provided with the distribution.\n# * Neither the name of David Beazley or Dabeaz LLC may be used to\n# endorse or promote products derived from this software without\n# specific prior written permission.\n#\n# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n# \"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\n# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\n# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\n# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\n# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\n# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n# -----------------------------------------------------------------------------\n\n# This module implements an ANSI-C style lexical preprocessor for PLY.\n# -----------------------------------------------------------------------------\nfrom __future__ import generators\n\nimport sys\n\n# Some Python 3 compatibility shims\nif sys.version_info.major < 3:\n STRING_TYPES = (str, unicode)\nelse:\n STRING_TYPES = str\n xrange = range\n\n# -----------------------------------------------------------------------------\n# Default preprocessor lexer definitions. These tokens are enough to get\n# a basic preprocessor working. Other modules may import these if they want\n# -----------------------------------------------------------------------------\n\ntokens = (\n 'CPP_ID','CPP_INTEGER', 'CPP_FLOAT', 'CPP_STRING', 'CPP_CHAR', 'CPP_WS', 'CPP_COMMENT1', 'CPP_COMMENT2', 'CPP_POUND','CPP_DPOUND'\n)\n\nliterals = \"+-*/%|&~^<>=!?()[]{}.,;:\\\\\\'\\\"\"\n\n# Whitespace\ndef t_CPP_WS(t):\n r'\\s+'\n t.lexer.lineno += t.value.count(\"\\n\")\n return t\n\nt_CPP_POUND = r'\\#'\nt_CPP_DPOUND = r'\\#\\#'\n\n# Identifier\nt_CPP_ID = r'[A-Za-z_][\\w_]*'\n\n# Integer literal\ndef CPP_INTEGER(t):\n r'(((((0x)|(0X))[0-9a-fA-F]+)|(\\d+))([uU][lL]|[lL][uU]|[uU]|[lL])?)'\n return t\n\nt_CPP_INTEGER = CPP_INTEGER\n\n# Floating literal\nt_CPP_FLOAT = r'((\\d+)(\\.\\d+)(e(\\+|-)?(\\d+))? | (\\d+)e(\\+|-)?(\\d+))([lL]|[fF])?'\n\n# String literal\ndef t_CPP_STRING(t):\n r'\\\"([^\\\\\\n]|(\\\\(.|\\n)))*?\\\"'\n t.lexer.lineno += t.value.count(\"\\n\")\n return t\n\n# Character constant 'c' or L'c'\ndef t_CPP_CHAR(t):\n r'(L)?\\'([^\\\\\\n]|(\\\\(.|\\n)))*?\\''\n t.lexer.lineno += t.value.count(\"\\n\")\n return t\n\n# Comment\ndef t_CPP_COMMENT1(t):\n r'(/\\*(.|\\n)*?\\*/)'\n ncr = t.value.count(\"\\n\")\n t.lexer.lineno += ncr\n # replace with one space or a number of '\\n'\n t.type = 'CPP_WS'; t.value = '\\n' * ncr if ncr else ' '\n return t\n\n# Line comment\ndef t_CPP_COMMENT2(t):\n r'(//.*?(\\n|$))'\n # replace with '/n'\n t.type = 'CPP_WS'; t.value = '\\n'\n return t\n\ndef t_error(t):\n t.type = t.value[0]\n t.value = t.value[0]\n t.lexer.skip(1)\n return t\n\nimport re\nimport copy\nimport time\nimport os.path\n\n# -----------------------------------------------------------------------------\n# trigraph()\n#\n# Given an input string, this function replaces all trigraph sequences.\n# The following mapping is used:\n#\n# ??= #\n# ??/ \\\n# ??' ^\n# ??( [\n# ??) ]\n# ??! |\n# ??< {\n# ??> }\n# ??- ~\n# -----------------------------------------------------------------------------\n\n_trigraph_pat = re.compile(r'''\\?\\?[=/\\'\\(\\)\\!<>\\-]''')\n_trigraph_rep = {\n '=':'#',\n '/':'\\\\',\n \"'\":'^',\n '(':'[',\n ')':']',\n '!':'|',\n '<':'{',\n '>':'}',\n '-':'~'\n}\n\ndef trigraph(input):\n return _trigraph_pat.sub(lambda g: _trigraph_rep[g.group()[-1]],input)\n\n# ------------------------------------------------------------------\n# Macro object\n#\n# This object holds information about preprocessor macros\n#\n# .name - Macro name (string)\n# .value - Macro value (a list of tokens)\n# .arglist - List of argument names\n# .variadic - Boolean indicating whether or not variadic macro\n# .vararg - Name of the variadic parameter\n#\n# When a macro is created, the macro replacement token sequence is\n# pre-scanned and used to create patch lists that are later used\n# during macro expansion\n# ------------------------------------------------------------------\n\nclass Macro(object):\n def __init__(self,name,value,arglist=None,variadic=False):\n self.name = name\n self.value = value\n self.arglist = arglist\n self.variadic = variadic\n if variadic:\n self.vararg = arglist[-1]\n self.source = None\n\n# ------------------------------------------------------------------\n# Preprocessor object\n#\n# Object representing a preprocessor. Contains macro definitions,\n# include directories, and other information\n# ------------------------------------------------------------------\n\nclass Preprocessor(object):\n def __init__(self,lexer=None):\n if lexer is None:\n lexer = lex.lexer\n self.lexer = lexer\n self.macros = { }\n self.path = []\n self.temp_path = []\n\n # Probe the lexer for selected tokens\n self.lexprobe()\n\n tm = time.localtime()\n self.define(\"__DATE__ \\\"%s\\\"\" % time.strftime(\"%b %d %Y\",tm))\n self.define(\"__TIME__ \\\"%s\\\"\" % time.strftime(\"%H:%M:%S\",tm))\n self.parser = None\n\n # -----------------------------------------------------------------------------\n # tokenize()\n #\n # Utility function. Given a string of text, tokenize into a list of tokens\n # -----------------------------------------------------------------------------\n\n def tokenize(self,text):\n tokens = []\n self.lexer.input(text)\n while True:\n tok = self.lexer.token()\n if not tok: break\n tokens.append(tok)\n return tokens\n\n # ---------------------------------------------------------------------\n # error()\n #\n # Report a preprocessor error/warning of some kind\n # ----------------------------------------------------------------------\n\n def error(self,file,line,msg):\n print(\"%s:%d %s\" % (file,line,msg))\n\n # ----------------------------------------------------------------------\n # lexprobe()\n #\n # This method probes the preprocessor lexer object to discover\n # the token types of symbols that are important to the preprocessor.\n # If this works right, the preprocessor will simply \"work\"\n # with any suitable lexer regardless of how tokens have been named.\n # ----------------------------------------------------------------------\n\n def lexprobe(self):\n\n # Determine the token type for identifiers\n self.lexer.input(\"identifier\")\n tok = self.lexer.token()\n if not tok or tok.value != \"identifier\":\n print(\"Couldn't determine identifier type\")\n else:\n self.t_ID = tok.type\n\n # Determine the token type for integers\n self.lexer.input(\"12345\")\n tok = self.lexer.token()\n if not tok or int(tok.value) != 12345:\n print(\"Couldn't determine integer type\")\n else:\n self.t_INTEGER = tok.type\n self.t_INTEGER_TYPE = type(tok.value)\n\n # Determine the token type for strings enclosed in double quotes\n self.lexer.input(\"\\\"filename\\\"\")\n tok = self.lexer.token()\n if not tok or tok.value != \"\\\"filename\\\"\":\n print(\"Couldn't determine string type\")\n else:\n self.t_STRING = tok.type\n\n # Determine the token type for whitespace--if any\n self.lexer.input(\" \")\n tok = self.lexer.token()\n if not tok or tok.value != \" \":\n self.t_SPACE = None\n else:\n self.t_SPACE = tok.type\n\n # Determine the token type for newlines\n self.lexer.input(\"\\n\")\n tok = self.lexer.token()\n if not tok or tok.value != \"\\n\":\n self.t_NEWLINE = None\n print(\"Couldn't determine token for newlines\")\n else:\n self.t_NEWLINE = tok.type\n\n self.t_WS = (self.t_SPACE, self.t_NEWLINE)\n\n # Check for other characters used by the preprocessor\n chars = [ '<','>','#','##','\\\\','(',')',',','.']\n for c in chars:\n self.lexer.input(c)\n tok = self.lexer.token()\n if not tok or tok.value != c:\n print(\"Unable to lex '%s' required for preprocessor\" % c)\n\n # ----------------------------------------------------------------------\n # add_path()\n #\n # Adds a search path to the preprocessor.\n # ----------------------------------------------------------------------\n\n def add_path(self,path):\n self.path.append(path)\n\n # ----------------------------------------------------------------------\n # group_lines()\n #\n # Given an input string, this function splits it into lines. Trailing whitespace\n # is removed. Any line ending with \\ is grouped with the next line. This\n # function forms the lowest level of the preprocessor---grouping into text into\n # a line-by-line format.\n # ----------------------------------------------------------------------\n\n def group_lines(self,input):\n lex = self.lexer.clone()\n lines = [x.rstrip() for x in input.splitlines()]\n for i in xrange(len(lines)):\n j = i+1\n while lines[i].endswith('\\\\') and (j < len(lines)):\n lines[i] = lines[i][:-1]+lines[j]\n lines[j] = \"\"\n j += 1\n\n input = \"\\n\".join(lines)\n lex.input(input)\n lex.lineno = 1\n\n current_line = []\n while True:\n tok = lex.token()\n if not tok:\n break\n current_line.append(tok)\n if tok.type in self.t_WS and '\\n' in tok.value:\n yield current_line\n current_line = []\n\n if current_line:\n yield current_line\n\n # ----------------------------------------------------------------------\n # tokenstrip()\n #\n # Remove leading/trailing whitespace tokens from a token list\n # ----------------------------------------------------------------------\n\n def tokenstrip(self,tokens):\n i = 0\n while i < len(tokens) and tokens[i].type in self.t_WS:\n i += 1\n del tokens[:i]\n i = len(tokens)-1\n while i >= 0 and tokens[i].type in self.t_WS:\n i -= 1\n del tokens[i+1:]\n return tokens\n\n\n # ----------------------------------------------------------------------\n # collect_args()\n #\n # Collects comma separated arguments from a list of tokens. The arguments\n # must be enclosed in parenthesis. Returns a tuple (tokencount,args,positions)\n # where tokencount is the number of tokens consumed, args is a list of arguments,\n # and positions is a list of integers containing the starting index of each\n # argument. Each argument is represented by a list of tokens.\n #\n # When collecting arguments, leading and trailing whitespace is removed\n # from each argument.\n #\n # This function properly handles nested parenthesis and commas---these do not\n # define new arguments.\n # ----------------------------------------------------------------------\n\n def collect_args(self,tokenlist):\n args = []\n positions = []\n current_arg = []\n nesting = 1\n tokenlen = len(tokenlist)\n\n # Search for the opening '('.\n i = 0\n while (i < tokenlen) and (tokenlist[i].type in self.t_WS):\n i += 1\n\n if (i < tokenlen) and (tokenlist[i].value == '('):\n positions.append(i+1)\n else:\n self.error(self.source,tokenlist[0].lineno,\"Missing '(' in macro arguments\")\n return 0, [], []\n\n i += 1\n\n while i < tokenlen:\n t = tokenlist[i]\n if t.value == '(':\n current_arg.append(t)\n nesting += 1\n elif t.value == ')':\n nesting -= 1\n if nesting == 0:\n if current_arg:\n args.append(self.tokenstrip(current_arg))\n positions.append(i)\n return i+1,args,positions\n current_arg.append(t)\n elif t.value == ',' and nesting == 1:\n args.append(self.tokenstrip(current_arg))\n positions.append(i+1)\n current_arg = []\n else:\n current_arg.append(t)\n i += 1\n\n # Missing end argument\n self.error(self.source,tokenlist[-1].lineno,\"Missing ')' in macro arguments\")\n return 0, [],[]\n\n # ----------------------------------------------------------------------\n # macro_prescan()\n #\n # Examine the macro value (token sequence) and identify patch points\n # This is used to speed up macro expansion later on---we'll know\n # right away where to apply patches to the value to form the expansion\n # ----------------------------------------------------------------------\n\n def macro_prescan(self,macro):\n macro.patch = [] # Standard macro arguments\n macro.str_patch = [] # String conversion expansion\n macro.var_comma_patch = [] # Variadic macro comma patch\n i = 0\n while i < len(macro.value):\n if macro.value[i].type == self.t_ID and macro.value[i].value in macro.arglist:\n argnum = macro.arglist.index(macro.value[i].value)\n # Conversion of argument to a string\n if i > 0 and macro.value[i-1].value == '#':\n macro.value[i] = copy.copy(macro.value[i])\n macro.value[i].type = self.t_STRING\n del macro.value[i-1]\n macro.str_patch.append((argnum,i-1))\n continue\n # Concatenation\n elif (i > 0 and macro.value[i-1].value == '##'):\n macro.patch.append(('c',argnum,i-1))\n del macro.value[i-1]\n i -= 1\n continue\n elif ((i+1) < len(macro.value) and macro.value[i+1].value == '##'):\n macro.patch.append(('c',argnum,i))\n del macro.value[i + 1]\n continue\n # Standard expansion\n else:\n macro.patch.append(('e',argnum,i))\n elif macro.value[i].value == '##':\n if macro.variadic and (i > 0) and (macro.value[i-1].value == ',') and \\\n ((i+1) < len(macro.value)) and (macro.value[i+1].type == self.t_ID) and \\\n (macro.value[i+1].value == macro.vararg):\n macro.var_comma_patch.append(i-1)\n i += 1\n macro.patch.sort(key=lambda x: x[2],reverse=True)\n\n # ----------------------------------------------------------------------\n # macro_expand_args()\n #\n # Given a Macro and list of arguments (each a token list), this method\n # returns an expanded version of a macro. The return value is a token sequence\n # representing the replacement macro tokens\n # ----------------------------------------------------------------------\n\n def macro_expand_args(self,macro,args,expanded):\n # Make a copy of the macro token sequence\n rep = [copy.copy(_x) for _x in macro.value]\n\n # Make string expansion patches. These do not alter the length of the replacement sequence\n\n str_expansion = {}\n for argnum, i in macro.str_patch:\n if argnum not in str_expansion:\n str_expansion[argnum] = ('\"%s\"' % \"\".join([x.value for x in args[argnum]])).replace(\"\\\\\",\"\\\\\\\\\")\n rep[i] = copy.copy(rep[i])\n rep[i].value = str_expansion[argnum]\n\n # Make the variadic macro comma patch. If the variadic macro argument is empty, we get rid\n comma_patch = False\n if macro.variadic and not args[-1]:\n for i in macro.var_comma_patch:\n rep[i] = None\n comma_patch = True\n\n # Make all other patches. The order of these matters. It is assumed that the patch list\n # has been sorted in reverse order of patch location since replacements will cause the\n # size of the replacement sequence to expand from the patch point.\n\n expanded_args = { }\n for ptype, argnum, i in macro.patch:\n # Concatenation. Argument is left unexpanded\n if ptype == 'c':\n rep[i:i+1] = args[argnum]\n # Normal expansion. Argument is macro expanded first\n elif ptype == 'e':\n if argnum not in expanded_args:\n expanded_args[argnum] = self.expand_macros(args[argnum],expanded)\n rep[i:i+1] = expanded_args[argnum]\n\n # Get rid of removed comma if necessary\n if comma_patch:\n rep = [_i for _i in rep if _i]\n\n return rep\n\n\n # ----------------------------------------------------------------------\n # expand_macros()\n #\n # Given a list of tokens, this function performs macro expansion.\n # The expanded argument is a dictionary that contains macros already\n # expanded. This is used to prevent infinite recursion.\n # ----------------------------------------------------------------------\n\n def expand_macros(self,tokens,expanded=None):\n if expanded is None:\n expanded = {}\n i = 0\n while i < len(tokens):\n t = tokens[i]\n if t.type == self.t_ID:\n if t.value in self.macros and t.value not in expanded:\n # Yes, we found a macro match\n expanded[t.value] = True\n\n m = self.macros[t.value]\n if not m.arglist:\n # A simple macro\n ex = self.expand_macros([copy.copy(_x) for _x in m.value],expanded)\n for e in ex:\n e.lineno = t.lineno\n tokens[i:i+1] = ex\n i += len(ex)\n else:\n # A macro with arguments\n j = i + 1\n while j < len(tokens) and tokens[j].type in self.t_WS:\n j += 1\n if j < len(tokens) and tokens[j].value == '(':\n tokcount,args,positions = self.collect_args(tokens[j:])\n if not m.variadic and len(args) != len(m.arglist):\n self.error(self.source,t.lineno,\"Macro %s requires %d arguments\" % (t.value,len(m.arglist)))\n i = j + tokcount\n elif m.variadic and len(args) < len(m.arglist)-1:\n if len(m.arglist) > 2:\n self.error(self.source,t.lineno,\"Macro %s must have at least %d arguments\" % (t.value, len(m.arglist)-1))\n else:\n self.error(self.source,t.lineno,\"Macro %s must have at least %d argument\" % (t.value, len(m.arglist)-1))\n i = j + tokcount\n else:\n if m.variadic:\n if len(args) == len(m.arglist)-1:\n args.append([])\n else:\n args[len(m.arglist)-1] = tokens[j+positions[len(m.arglist)-1]:j+tokcount-1]\n del args[len(m.arglist):]\n\n # Get macro replacement text\n rep = self.macro_expand_args(m,args,expanded)\n rep = self.expand_macros(rep,expanded)\n for r in rep:\n r.lineno = t.lineno\n tokens[i:j+tokcount] = rep\n i += len(rep)\n else:\n # This is not a macro. It is just a word which\n # equals to name of the macro. Hence, go to the\n # next token.\n i += 1\n\n del expanded[t.value]\n continue\n elif t.value == '__LINE__':\n t.type = self.t_INTEGER\n t.value = self.t_INTEGER_TYPE(t.lineno)\n\n i += 1\n return tokens\n\n # ----------------------------------------------------------------------\n # evalexpr()\n #\n # Evaluate an expression token sequence for the purposes of evaluating\n # integral expressions.\n # ----------------------------------------------------------------------\n\n def evalexpr(self,tokens):\n # tokens = tokenize(line)\n # Search for defined macros\n i = 0\n while i < len(tokens):\n if tokens[i].type == self.t_ID and tokens[i].value == 'defined':\n j = i + 1\n needparen = False\n result = \"0L\"\n while j < len(tokens):\n if tokens[j].type in self.t_WS:\n j += 1\n continue\n elif tokens[j].type == self.t_ID:\n if tokens[j].value in self.macros:\n result = \"1L\"\n else:\n result = \"0L\"\n if not needparen: break\n elif tokens[j].value == '(':\n needparen = True\n elif tokens[j].value == ')':\n break\n else:\n self.error(self.source,tokens[i].lineno,\"Malformed defined()\")\n j += 1\n tokens[i].type = self.t_INTEGER\n tokens[i].value = self.t_INTEGER_TYPE(result)\n del tokens[i+1:j+1]\n i += 1\n tokens = self.expand_macros(tokens)\n return self.evalexpr_expanded(tokens)\n\n # ----------------------------------------------------------------------\n # evalexpr_expanded()\n #\n # Helper for evalexpr that evaluates the expression that had its macros\n # and defined(...) expressions expanded by evalexpr\n # ----------------------------------------------------------------------\n\n def evalexpr_expanded(self, tokens):\n for i,t in enumerate(tokens):\n if t.type == self.t_ID:\n tokens[i] = copy.copy(t)\n tokens[i].type = self.t_INTEGER\n tokens[i].value = self.t_INTEGER_TYPE(\"0\")\n elif t.type == self.t_INTEGER:\n tokens[i] = copy.copy(t)\n # Strip off any trailing suffixes\n tokens[i].value = str(tokens[i].value)\n while tokens[i].value[-1] not in \"0123456789abcdefABCDEF\":\n tokens[i].value = tokens[i].value[:-1]\n\n return self.evalexpr_string(\"\".join([str(x.value) for x in tokens]))\n\n # ----------------------------------------------------------------------\n # evalexpr_string()\n #\n # Helper for evalexpr that evaluates a string expression\n # This implementation does basic C->python conversion and then uses eval()\n # ----------------------------------------------------------------------\n def evalexpr_string(self, expr):\n expr = expr.replace(\"&&\",\" and \")\n expr = expr.replace(\"||\",\" or \")\n expr = expr.replace(\"!\",\" not \")\n expr = expr.replace(\" not =\",\" !=\")\n try:\n result = eval(expr)\n except Exception:\n self.error(self.source,tokens[0].lineno,\"Couldn't evaluate expression\")\n result = 0\n return result\n\n # ----------------------------------------------------------------------\n # parsegen()\n #\n # Parse an input string/\n # ----------------------------------------------------------------------\n def parsegen(self,input,source=None):\n\n # Replace trigraph sequences\n t = trigraph(input)\n lines = self.group_lines(t)\n\n if not source:\n source = \"\"\n\n self.define(\"__FILE__ \\\"%s\\\"\" % source)\n\n self.source = source\n chunk = []\n enable = True\n iftrigger = False\n ifstack = []\n\n for x in lines:\n for i,tok in enumerate(x):\n if tok.type not in self.t_WS: break\n if tok.value == '#':\n # Preprocessor directive\n\n # insert necessary whitespace instead of eaten tokens\n for tok in x:\n if tok.type in self.t_WS and '\\n' in tok.value:\n chunk.append(tok)\n\n dirtokens = self.tokenstrip(x[i+1:])\n if dirtokens:\n name = dirtokens[0].value\n args = self.tokenstrip(dirtokens[1:])\n else:\n name = \"\"\n args = []\n\n if name == 'define':\n if enable:\n for tok in self.expand_macros(chunk):\n yield tok\n chunk = []\n self.define(args)\n elif name == 'include':\n if enable:\n for tok in self.expand_macros(chunk):\n yield tok\n chunk = []\n oldfile = self.macros['__FILE__']\n for tok in self.include(args):\n yield tok\n self.macros['__FILE__'] = oldfile\n self.source = source\n elif name == 'undef':\n if enable:\n for tok in self.expand_macros(chunk):\n yield tok\n chunk = []\n self.undef(args)\n elif name == 'ifdef':\n ifstack.append((enable,iftrigger))\n if enable:\n if not args[0].value in self.macros:\n enable = False\n iftrigger = False\n else:\n iftrigger = True\n elif name == 'ifndef':\n ifstack.append((enable,iftrigger))\n if enable:\n if args[0].value in self.macros:\n enable = False\n iftrigger = False\n else:\n iftrigger = True\n elif name == 'if':\n ifstack.append((enable,iftrigger))\n if enable:\n result = self.evalexpr(args)\n if not result:\n enable = False\n iftrigger = False\n else:\n iftrigger = True\n elif name == 'elif':\n if ifstack:\n if ifstack[-1][0]: # We only pay attention if outer \"if\" allows this\n if enable: # If already true, we flip enable False\n enable = False\n elif not iftrigger: # If False, but not triggered yet, we'll check expression\n result = self.evalexpr(args)\n if result:\n enable = True\n iftrigger = True\n else:\n self.error(self.source,dirtokens[0].lineno,\"Misplaced #elif\")\n\n elif name == 'else':\n if ifstack:\n if ifstack[-1][0]:\n if enable:\n enable = False\n elif not iftrigger:\n enable = True\n iftrigger = True\n else:\n self.error(self.source,dirtokens[0].lineno,\"Misplaced #else\")\n\n elif name == 'endif':\n if ifstack:\n enable,iftrigger = ifstack.pop()\n else:\n self.error(self.source,dirtokens[0].lineno,\"Misplaced #endif\")\n else:\n # Unknown preprocessor directive\n pass\n\n else:\n # Normal text\n if enable:\n chunk.extend(x)\n\n for tok in self.expand_macros(chunk):\n yield tok\n chunk = []\n\n # ----------------------------------------------------------------------\n # include()\n #\n # Implementation of file-inclusion\n # ----------------------------------------------------------------------\n\n def include(self,tokens):\n # Try to extract the filename and then process an include file\n if not tokens:\n return\n if tokens:\n if tokens[0].value != '<' and tokens[0].type != self.t_STRING:\n tokens = self.expand_macros(tokens)\n\n if tokens[0].value == '<':\n # Include <...>\n i = 1\n while i < len(tokens):\n if tokens[i].value == '>':\n break\n i += 1\n else:\n print(\"Malformed #include <...>\")\n return\n filename = \"\".join([x.value for x in tokens[1:i]])\n path = self.path + [\"\"] + self.temp_path\n elif tokens[0].type == self.t_STRING:\n filename = tokens[0].value[1:-1]\n path = self.temp_path + [\"\"] + self.path\n else:\n print(\"Malformed #include statement\")\n return\n for p in path:\n iname = os.path.join(p,filename)\n try:\n data = self.read_include_file(iname)\n dname = os.path.dirname(iname)\n if dname:\n self.temp_path.insert(0,dname)\n for tok in self.parsegen(data,filename):\n yield tok\n if dname:\n del self.temp_path[0]\n break\n except IOError:\n pass\n else:\n print(\"Couldn't find '%s'\" % filename)\n\n # ----------------------------------------------------------------------\n # read_include_file()\n #\n # Reads a source file for inclusion using #include\n # Could be overridden to e.g. customize encoding, limit access to\n # certain paths on the filesystem, or provide the contents of system\n # include files\n # ----------------------------------------------------------------------\n\n def read_include_file(self, filepath):\n with open(filepath, 'r', encoding='utf-8', errors='surrogateescape') as file:\n return file.read()\n\n # ----------------------------------------------------------------------\n # define()\n #\n # Define a new macro\n # ----------------------------------------------------------------------\n\n def define(self,tokens):\n if isinstance(tokens,STRING_TYPES):\n tokens = self.tokenize(tokens)\n\n linetok = tokens\n try:\n name = linetok[0]\n if len(linetok) > 1:\n mtype = linetok[1]\n else:\n mtype = None\n if not mtype:\n m = Macro(name.value,[])\n self.macros[name.value] = m\n elif mtype.type in self.t_WS:\n # A normal macro\n m = Macro(name.value,self.tokenstrip(linetok[2:]))\n self.macros[name.value] = m\n elif mtype.value == '(':\n # A macro with arguments\n tokcount, args, positions = self.collect_args(linetok[1:])\n variadic = False\n for a in args:\n if variadic:\n print(\"No more arguments may follow a variadic argument\")\n break\n astr = \"\".join([str(_i.value) for _i in a])\n if astr == \"...\":\n variadic = True\n a[0].type = self.t_ID\n a[0].value = '__VA_ARGS__'\n variadic = True\n del a[1:]\n continue\n elif astr[-3:] == \"...\" and a[0].type == self.t_ID:\n variadic = True\n del a[1:]\n # If, for some reason, \".\" is part of the identifier, strip off the name for the purposes\n # of macro expansion\n if a[0].value[-3:] == '...':\n a[0].value = a[0].value[:-3]\n continue\n if len(a) > 1 or a[0].type != self.t_ID:\n print(\"Invalid macro argument\")\n break\n else:\n mvalue = self.tokenstrip(linetok[1+tokcount:])\n i = 0\n while i < len(mvalue):\n if i+1 < len(mvalue):\n if mvalue[i].type in self.t_WS and mvalue[i+1].value == '##':\n del mvalue[i]\n continue\n elif mvalue[i].value == '##' and mvalue[i+1].type in self.t_WS:\n del mvalue[i+1]\n i += 1\n m = Macro(name.value,mvalue,[x[0].value for x in args],variadic)\n self.macro_prescan(m)\n self.macros[name.value] = m\n else:\n print(\"Bad macro definition\")\n except LookupError:\n print(\"Bad macro definition\")\n\n # ----------------------------------------------------------------------\n # undef()\n #\n # Undefine a macro\n # ----------------------------------------------------------------------\n\n def undef(self,tokens):\n id = tokens[0].value\n try:\n del self.macros[id]\n except LookupError:\n pass\n\n # ----------------------------------------------------------------------\n # parse()\n #\n # Parse input text.\n # ----------------------------------------------------------------------\n def parse(self,input,source=None,ignore={}):\n self.ignore = ignore\n self.parser = self.parsegen(input,source)\n\n # ----------------------------------------------------------------------\n # token()\n #\n # Method to return individual tokens\n # ----------------------------------------------------------------------\n def token(self):\n try:\n while True:\n tok = next(self.parser)\n if tok.type not in self.ignore: return tok\n except StopIteration:\n self.parser = None\n return None\n\nif __name__ == '__main__':\n import ply.lex as lex\n lexer = lex.lex()\n\n # Run a preprocessor\n import sys\n with open(sys.argv[1]) as f:\n input = f.read()\n\n p = Preprocessor(lexer)\n p.parse(input,sys.argv[1])\n while True:\n tok = p.token()\n if not tok: break\n print(p.source, tok)\n" }, { "path": "xonsh/ply/ply/ctokens.py", "content": "# ----------------------------------------------------------------------\n# ctokens.py\n#\n# Token specifications for symbols in ANSI C and C++. This file is\n# meant to be used as a library in other tokenizers.\n# ----------------------------------------------------------------------\n\n# Reserved words\n\ntokens = [\n # Literals (identifier, integer constant, float constant, string constant, char const)\n 'ID', 'TYPEID', 'INTEGER', 'FLOAT', 'STRING', 'CHARACTER',\n\n # Operators (+,-,*,/,%,|,&,~,^,<<,>>, ||, &&, !, <, <=, >, >=, ==, !=)\n 'PLUS', 'MINUS', 'TIMES', 'DIVIDE', 'MODULO',\n 'OR', 'AND', 'NOT', 'XOR', 'LSHIFT', 'RSHIFT',\n 'LOR', 'LAND', 'LNOT',\n 'LT', 'LE', 'GT', 'GE', 'EQ', 'NE',\n\n # Assignment (=, *=, /=, %=, +=, -=, <<=, >>=, &=, ^=, |=)\n 'EQUALS', 'TIMESEQUAL', 'DIVEQUAL', 'MODEQUAL', 'PLUSEQUAL', 'MINUSEQUAL',\n 'LSHIFTEQUAL','RSHIFTEQUAL', 'ANDEQUAL', 'XOREQUAL', 'OREQUAL',\n\n # Increment/decrement (++,--)\n 'INCREMENT', 'DECREMENT',\n\n # Structure dereference (->)\n 'ARROW',\n\n # Ternary operator (?)\n 'TERNARY',\n\n # Delimeters ( ) [ ] { } , . ; :\n 'LPAREN', 'RPAREN',\n 'LBRACKET', 'RBRACKET',\n 'LBRACE', 'RBRACE',\n 'COMMA', 'PERIOD', 'SEMI', 'COLON',\n\n # Ellipsis (...)\n 'ELLIPSIS',\n]\n\n# Operators\nt_PLUS = r'\\+'\nt_MINUS = r'-'\nt_TIMES = r'\\*'\nt_DIVIDE = r'/'\nt_MODULO = r'%'\nt_OR = r'\\|'\nt_AND = r'&'\nt_NOT = r'~'\nt_XOR = r'\\^'\nt_LSHIFT = r'<<'\nt_RSHIFT = r'>>'\nt_LOR = r'\\|\\|'\nt_LAND = r'&&'\nt_LNOT = r'!'\nt_LT = r'<'\nt_GT = r'>'\nt_LE = r'<='\nt_GE = r'>='\nt_EQ = r'=='\nt_NE = r'!='\n\n# Assignment operators\n\nt_EQUALS = r'='\nt_TIMESEQUAL = r'\\*='\nt_DIVEQUAL = r'/='\nt_MODEQUAL = r'%='\nt_PLUSEQUAL = r'\\+='\nt_MINUSEQUAL = r'-='\nt_LSHIFTEQUAL = r'<<='\nt_RSHIFTEQUAL = r'>>='\nt_ANDEQUAL = r'&='\nt_OREQUAL = r'\\|='\nt_XOREQUAL = r'\\^='\n\n# Increment/decrement\nt_INCREMENT = r'\\+\\+'\nt_DECREMENT = r'--'\n\n# ->\nt_ARROW = r'->'\n\n# ?\nt_TERNARY = r'\\?'\n\n# Delimeters\nt_LPAREN = r'\\('\nt_RPAREN = r'\\)'\nt_LBRACKET = r'\\['\nt_RBRACKET = r'\\]'\nt_LBRACE = r'\\{'\nt_RBRACE = r'\\}'\nt_COMMA = r','\nt_PERIOD = r'\\.'\nt_SEMI = r';'\nt_COLON = r':'\nt_ELLIPSIS = r'\\.\\.\\.'\n\n# Identifiers\nt_ID = r'[A-Za-z_][A-Za-z0-9_]*'\n\n# Integer literal\nt_INTEGER = r'\\d+([uU]|[lL]|[uU][lL]|[lL][uU])?'\n\n# Floating literal\nt_FLOAT = r'((\\d+)(\\.\\d+)(e(\\+|-)?(\\d+))? | (\\d+)e(\\+|-)?(\\d+))([lL]|[fF])?'\n\n# String literal\nt_STRING = r'\\\"([^\\\\\\n]|(\\\\.))*?\\\"'\n\n# Character constant 'c' or L'c'\nt_CHARACTER = r'(L)?\\'([^\\\\\\n]|(\\\\.))*?\\''\n\n# Comment (C-Style)\ndef t_COMMENT(t):\n r'/\\*(.|\\n)*?\\*/'\n t.lexer.lineno += t.value.count('\\n')\n return t\n\n# Comment (C++-Style)\ndef t_CPPCOMMENT(t):\n r'//.*\\n'\n t.lexer.lineno += 1\n return t\n" }, { "path": "xonsh/ply/ply/lex.py", "content": "# -----------------------------------------------------------------------------\n# ply: lex.py\n#\n# Copyright (C) 2001-2019\n# David M. Beazley (Dabeaz LLC)\n# All rights reserved.\n#\n# Latest version: https://github.com/dabeaz/ply\n#\n# Redistribution and use in source and binary forms, with or without\n# modification, are permitted provided that the following conditions are\n# met:\n#\n# * Redistributions of source code must retain the above copyright notice,\n# this list of conditions and the following disclaimer.\n# * Redistributions in binary form must reproduce the above copyright notice,\n# this list of conditions and the following disclaimer in the documentation\n# and/or other materials provided with the distribution.\n# * Neither the name of David Beazley or Dabeaz LLC may be used to\n# endorse or promote products derived from this software without\n# specific prior written permission.\n#\n# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n# \"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\n# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\n# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\n# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\n# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\n# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n# -----------------------------------------------------------------------------\n\n__version__ = '3.11'\n__tabversion__ = '3.10'\n\nimport re\nimport sys\nimport types\nimport copy\nimport os\nimport inspect\n\n# This tuple contains known string types\ntry:\n # Python 2.6\n StringTypes = (types.StringType, types.UnicodeType)\nexcept AttributeError:\n # Python 3.0\n StringTypes = (str, bytes)\n\n# This regular expression is used to match valid token names\n_is_identifier = re.compile(r'^[a-zA-Z0-9_]+$')\n\n# Exception thrown when invalid token encountered and no default error\n# handler is defined.\nclass LexError(Exception):\n def __init__(self, message, s):\n self.args = (message,)\n self.text = s\n\n\n# Token class. This class is used to represent the tokens produced.\nclass LexToken(object):\n def __str__(self):\n return 'LexToken(%s,%r,%d,%d)' % (self.type, self.value, self.lineno, self.lexpos)\n\n def __repr__(self):\n return str(self)\n\n\n# This object is a stand-in for a logging object created by the\n# logging module.\n\nclass PlyLogger(object):\n def __init__(self, f):\n self.f = f\n\n def critical(self, msg, *args, **kwargs):\n self.f.write((msg % args) + '\\n')\n\n def warning(self, msg, *args, **kwargs):\n self.f.write('WARNING: ' + (msg % args) + '\\n')\n\n def error(self, msg, *args, **kwargs):\n self.f.write('ERROR: ' + (msg % args) + '\\n')\n\n info = critical\n debug = critical\n\n\n# Null logger is used when no output is generated. Does nothing.\nclass NullLogger(object):\n def __getattribute__(self, name):\n return self\n\n def __call__(self, *args, **kwargs):\n return self\n\n\n# -----------------------------------------------------------------------------\n# === Lexing Engine ===\n#\n# The following Lexer class implements the lexer runtime. There are only\n# a few public methods and attributes:\n#\n# input() - Store a new string in the lexer\n# token() - Get the next token\n# clone() - Clone the lexer\n#\n# lineno - Current line number\n# lexpos - Current position in the input string\n# -----------------------------------------------------------------------------\n\nclass Lexer:\n def __init__(self):\n self.lexre = None # Master regular expression. This is a list of\n # tuples (re, findex) where re is a compiled\n # regular expression and findex is a list\n # mapping regex group numbers to rules\n self.lexretext = None # Current regular expression strings\n self.lexstatere = {} # Dictionary mapping lexer states to master regexs\n self.lexstateretext = {} # Dictionary mapping lexer states to regex strings\n self.lexstaterenames = {} # Dictionary mapping lexer states to symbol names\n self.lexstate = 'INITIAL' # Current lexer state\n self.lexstatestack = [] # Stack of lexer states\n self.lexstateinfo = None # State information\n self.lexstateignore = {} # Dictionary of ignored characters for each state\n self.lexstateerrorf = {} # Dictionary of error functions for each state\n self.lexstateeoff = {} # Dictionary of eof functions for each state\n self.lexreflags = 0 # Optional re compile flags\n self.lexdata = None # Actual input data (as a string)\n self.lexpos = 0 # Current position in input text\n self.lexlen = 0 # Length of the input text\n self.lexerrorf = None # Error rule (if any)\n self.lexeoff = None # EOF rule (if any)\n self.lextokens = None # List of valid tokens\n self.lexignore = '' # Ignored characters\n self.lexliterals = '' # Literal characters that can be passed through\n self.lexmodule = None # Module\n self.lineno = 1 # Current line number\n self.lexoptimize = False # Optimized mode\n\n def clone(self, object=None):\n c = copy.copy(self)\n\n # If the object parameter has been supplied, it means we are attaching the\n # lexer to a new object. In this case, we have to rebind all methods in\n # the lexstatere and lexstateerrorf tables.\n\n if object:\n newtab = {}\n for key, ritem in self.lexstatere.items():\n newre = []\n for cre, findex in ritem:\n newfindex = []\n for f in findex:\n if not f or not f[0]:\n newfindex.append(f)\n continue\n newfindex.append((getattr(object, f[0].__name__), f[1]))\n newre.append((cre, newfindex))\n newtab[key] = newre\n c.lexstatere = newtab\n c.lexstateerrorf = {}\n for key, ef in self.lexstateerrorf.items():\n c.lexstateerrorf[key] = getattr(object, ef.__name__)\n c.lexmodule = object\n return c\n\n # ------------------------------------------------------------\n # writetab() - Write lexer information to a table file\n # ------------------------------------------------------------\n def writetab(self, lextab, outputdir=''):\n if isinstance(lextab, types.ModuleType):\n raise IOError(\"Won't overwrite existing lextab module\")\n basetabmodule = lextab.split('.')[-1]\n filename = os.path.join(outputdir, basetabmodule) + '.py'\n with open(filename, 'w') as tf:\n tf.write('# %s.py. This file automatically created by PLY (version %s). Don\\'t edit!\\n' % (basetabmodule, __version__))\n tf.write('_tabversion = %s\\n' % repr(__tabversion__))\n tf.write('_lextokens = set(%s)\\n' % repr(tuple(sorted(self.lextokens))))\n tf.write('_lexreflags = %s\\n' % repr(int(self.lexreflags)))\n tf.write('_lexliterals = %s\\n' % repr(self.lexliterals))\n tf.write('_lexstateinfo = %s\\n' % repr(self.lexstateinfo))\n\n # Rewrite the lexstatere table, replacing function objects with function names\n tabre = {}\n for statename, lre in self.lexstatere.items():\n titem = []\n for (pat, func), retext, renames in zip(lre, self.lexstateretext[statename], self.lexstaterenames[statename]):\n titem.append((retext, _funcs_to_names(func, renames)))\n tabre[statename] = titem\n\n tf.write('_lexstatere = %s\\n' % repr(tabre))\n tf.write('_lexstateignore = %s\\n' % repr(self.lexstateignore))\n\n taberr = {}\n for statename, ef in self.lexstateerrorf.items():\n taberr[statename] = ef.__name__ if ef else None\n tf.write('_lexstateerrorf = %s\\n' % repr(taberr))\n\n tabeof = {}\n for statename, ef in self.lexstateeoff.items():\n tabeof[statename] = ef.__name__ if ef else None\n tf.write('_lexstateeoff = %s\\n' % repr(tabeof))\n\n # ------------------------------------------------------------\n # readtab() - Read lexer information from a tab file\n # ------------------------------------------------------------\n def readtab(self, tabfile, fdict):\n if isinstance(tabfile, types.ModuleType):\n lextab = tabfile\n else:\n exec('import %s' % tabfile)\n lextab = sys.modules[tabfile]\n\n if getattr(lextab, '_tabversion', '0.0') != __tabversion__:\n raise ImportError('Inconsistent PLY version')\n\n self.lextokens = lextab._lextokens\n self.lexreflags = lextab._lexreflags\n self.lexliterals = lextab._lexliterals\n self.lextokens_all = self.lextokens | set(self.lexliterals)\n self.lexstateinfo = lextab._lexstateinfo\n self.lexstateignore = lextab._lexstateignore\n self.lexstatere = {}\n self.lexstateretext = {}\n for statename, lre in lextab._lexstatere.items():\n titem = []\n txtitem = []\n for pat, func_name in lre:\n titem.append((re.compile(pat, lextab._lexreflags), _names_to_funcs(func_name, fdict)))\n\n self.lexstatere[statename] = titem\n self.lexstateretext[statename] = txtitem\n\n self.lexstateerrorf = {}\n for statename, ef in lextab._lexstateerrorf.items():\n self.lexstateerrorf[statename] = fdict[ef]\n\n self.lexstateeoff = {}\n for statename, ef in lextab._lexstateeoff.items():\n self.lexstateeoff[statename] = fdict[ef]\n\n self.begin('INITIAL')\n\n # ------------------------------------------------------------\n # input() - Push a new string into the lexer\n # ------------------------------------------------------------\n def input(self, s):\n # Pull off the first character to see if s looks like a string\n c = s[:1]\n if not isinstance(c, StringTypes):\n raise ValueError('Expected a string')\n self.lexdata = s\n self.lexpos = 0\n self.lexlen = len(s)\n\n # ------------------------------------------------------------\n # begin() - Changes the lexing state\n # ------------------------------------------------------------\n def begin(self, state):\n if state not in self.lexstatere:\n raise ValueError('Undefined state')\n self.lexre = self.lexstatere[state]\n self.lexretext = self.lexstateretext[state]\n self.lexignore = self.lexstateignore.get(state, '')\n self.lexerrorf = self.lexstateerrorf.get(state, None)\n self.lexeoff = self.lexstateeoff.get(state, None)\n self.lexstate = state\n\n # ------------------------------------------------------------\n # push_state() - Changes the lexing state and saves old on stack\n # ------------------------------------------------------------\n def push_state(self, state):\n self.lexstatestack.append(self.lexstate)\n self.begin(state)\n\n # ------------------------------------------------------------\n # pop_state() - Restores the previous state\n # ------------------------------------------------------------\n def pop_state(self):\n self.begin(self.lexstatestack.pop())\n\n # ------------------------------------------------------------\n # current_state() - Returns the current lexing state\n # ------------------------------------------------------------\n def current_state(self):\n return self.lexstate\n\n # ------------------------------------------------------------\n # skip() - Skip ahead n characters\n # ------------------------------------------------------------\n def skip(self, n):\n self.lexpos += n\n\n # ------------------------------------------------------------\n # opttoken() - Return the next token from the Lexer\n #\n # Note: This function has been carefully implemented to be as fast\n # as possible. Don't make changes unless you really know what\n # you are doing\n # ------------------------------------------------------------\n def token(self):\n # Make local copies of frequently referenced attributes\n lexpos = self.lexpos\n lexlen = self.lexlen\n lexignore = self.lexignore\n lexdata = self.lexdata\n\n while lexpos < lexlen:\n # This code provides some short-circuit code for whitespace, tabs, and other ignored characters\n if lexdata[lexpos] in lexignore:\n lexpos += 1\n continue\n\n # Look for a regular expression match\n for lexre, lexindexfunc in self.lexre:\n m = lexre.match(lexdata, lexpos)\n if not m:\n continue\n\n # Create a token for return\n tok = LexToken()\n tok.value = m.group()\n tok.lineno = self.lineno\n tok.lexpos = lexpos\n\n i = m.lastindex\n func, tok.type = lexindexfunc[i]\n\n if not func:\n # If no token type was set, it's an ignored token\n if tok.type:\n self.lexpos = m.end()\n return tok\n else:\n lexpos = m.end()\n break\n\n lexpos = m.end()\n\n # If token is processed by a function, call it\n\n tok.lexer = self # Set additional attributes useful in token rules\n self.lexmatch = m\n self.lexpos = lexpos\n\n newtok = func(tok)\n\n # Every function must return a token, if nothing, we just move to next token\n if not newtok:\n lexpos = self.lexpos # This is here in case user has updated lexpos.\n lexignore = self.lexignore # This is here in case there was a state change\n break\n\n # Verify type of the token. If not in the token map, raise an error\n if not self.lexoptimize:\n if newtok.type not in self.lextokens_all:\n raise LexError(\"%s:%d: Rule '%s' returned an unknown token type '%s'\" % (\n func.__code__.co_filename, func.__code__.co_firstlineno,\n func.__name__, newtok.type), lexdata[lexpos:])\n\n return newtok\n else:\n # No match, see if in literals\n if lexdata[lexpos] in self.lexliterals:\n tok = LexToken()\n tok.value = lexdata[lexpos]\n tok.lineno = self.lineno\n tok.type = tok.value\n tok.lexpos = lexpos\n self.lexpos = lexpos + 1\n return tok\n\n # No match. Call t_error() if defined.\n if self.lexerrorf:\n tok = LexToken()\n tok.value = self.lexdata[lexpos:]\n tok.lineno = self.lineno\n tok.type = 'error'\n tok.lexer = self\n tok.lexpos = lexpos\n self.lexpos = lexpos\n newtok = self.lexerrorf(tok)\n if lexpos == self.lexpos:\n # Error method didn't change text position at all. This is an error.\n raise LexError(\"Scanning error. Illegal character '%s'\" % (lexdata[lexpos]), lexdata[lexpos:])\n lexpos = self.lexpos\n if not newtok:\n continue\n return newtok\n\n self.lexpos = lexpos\n raise LexError(\"Illegal character '%s' at index %d\" % (lexdata[lexpos], lexpos), lexdata[lexpos:])\n\n if self.lexeoff:\n tok = LexToken()\n tok.type = 'eof'\n tok.value = ''\n tok.lineno = self.lineno\n tok.lexpos = lexpos\n tok.lexer = self\n self.lexpos = lexpos\n newtok = self.lexeoff(tok)\n return newtok\n\n self.lexpos = lexpos + 1\n if self.lexdata is None:\n raise RuntimeError('No input string given with input()')\n return None\n\n # Iterator interface\n def __iter__(self):\n return self\n\n def next(self):\n t = self.token()\n if t is None:\n raise StopIteration\n return t\n\n __next__ = next\n\n# -----------------------------------------------------------------------------\n# ==== Lex Builder ===\n#\n# The functions and classes below are used to collect lexing information\n# and build a Lexer object from it.\n# -----------------------------------------------------------------------------\n\n# -----------------------------------------------------------------------------\n# _get_regex(func)\n#\n# Returns the regular expression assigned to a function either as a doc string\n# or as a .regex attribute attached by the @TOKEN decorator.\n# -----------------------------------------------------------------------------\ndef _get_regex(func):\n return getattr(func, 'regex', func.__doc__)\n\n# -----------------------------------------------------------------------------\n# get_caller_module_dict()\n#\n# This function returns a dictionary containing all of the symbols defined within\n# a caller further down the call stack. This is used to get the environment\n# associated with the yacc() call if none was provided.\n# -----------------------------------------------------------------------------\ndef get_caller_module_dict(levels):\n f = sys._getframe(levels)\n ldict = f.f_globals.copy()\n if f.f_globals != f.f_locals:\n ldict.update(f.f_locals)\n return ldict\n\n# -----------------------------------------------------------------------------\n# _funcs_to_names()\n#\n# Given a list of regular expression functions, this converts it to a list\n# suitable for output to a table file\n# -----------------------------------------------------------------------------\ndef _funcs_to_names(funclist, namelist):\n result = []\n for f, name in zip(funclist, namelist):\n if f and f[0]:\n result.append((name, f[1]))\n else:\n result.append(f)\n return result\n\n# -----------------------------------------------------------------------------\n# _names_to_funcs()\n#\n# Given a list of regular expression function names, this converts it back to\n# functions.\n# -----------------------------------------------------------------------------\ndef _names_to_funcs(namelist, fdict):\n result = []\n for n in namelist:\n if n and n[0]:\n result.append((fdict[n[0]], n[1]))\n else:\n result.append(n)\n return result\n\n# -----------------------------------------------------------------------------\n# _form_master_re()\n#\n# This function takes a list of all of the regex components and attempts to\n# form the master regular expression. Given limitations in the Python re\n# module, it may be necessary to break the master regex into separate expressions.\n# -----------------------------------------------------------------------------\ndef _form_master_re(relist, reflags, ldict, toknames):\n if not relist:\n return []\n regex = '|'.join(relist)\n try:\n lexre = re.compile(regex, reflags)\n\n # Build the index to function map for the matching engine\n lexindexfunc = [None] * (max(lexre.groupindex.values()) + 1)\n lexindexnames = lexindexfunc[:]\n\n for f, i in lexre.groupindex.items():\n handle = ldict.get(f, None)\n if type(handle) in (types.FunctionType, types.MethodType):\n lexindexfunc[i] = (handle, toknames[f])\n lexindexnames[i] = f\n elif handle is not None:\n lexindexnames[i] = f\n if f.find('ignore_') > 0:\n lexindexfunc[i] = (None, None)\n else:\n lexindexfunc[i] = (None, toknames[f])\n\n return [(lexre, lexindexfunc)], [regex], [lexindexnames]\n except Exception:\n m = int(len(relist)/2)\n if m == 0:\n m = 1\n llist, lre, lnames = _form_master_re(relist[:m], reflags, ldict, toknames)\n rlist, rre, rnames = _form_master_re(relist[m:], reflags, ldict, toknames)\n return (llist+rlist), (lre+rre), (lnames+rnames)\n\n# -----------------------------------------------------------------------------\n# def _statetoken(s,names)\n#\n# Given a declaration name s of the form \"t_\" and a dictionary whose keys are\n# state names, this function returns a tuple (states,tokenname) where states\n# is a tuple of state names and tokenname is the name of the token. For example,\n# calling this with s = \"t_foo_bar_SPAM\" might return (('foo','bar'),'SPAM')\n# -----------------------------------------------------------------------------\ndef _statetoken(s, names):\n parts = s.split('_')\n for i, part in enumerate(parts[1:], 1):\n if part not in names and part != 'ANY':\n break\n\n if i > 1:\n states = tuple(parts[1:i])\n else:\n states = ('INITIAL',)\n\n if 'ANY' in states:\n states = tuple(names)\n\n tokenname = '_'.join(parts[i:])\n return (states, tokenname)\n\n\n# -----------------------------------------------------------------------------\n# LexerReflect()\n#\n# This class represents information needed to build a lexer as extracted from a\n# user's input file.\n# -----------------------------------------------------------------------------\nclass LexerReflect(object):\n def __init__(self, ldict, log=None, reflags=0):\n self.ldict = ldict\n self.error_func = None\n self.tokens = []\n self.reflags = reflags\n self.stateinfo = {'INITIAL': 'inclusive'}\n self.modules = set()\n self.error = False\n self.log = PlyLogger(sys.stderr) if log is None else log\n\n # Get all of the basic information\n def get_all(self):\n self.get_tokens()\n self.get_literals()\n self.get_states()\n self.get_rules()\n\n # Validate all of the information\n def validate_all(self):\n self.validate_tokens()\n self.validate_literals()\n self.validate_rules()\n return self.error\n\n # Get the tokens map\n def get_tokens(self):\n tokens = self.ldict.get('tokens', None)\n if not tokens:\n self.log.error('No token list is defined')\n self.error = True\n return\n\n if not isinstance(tokens, (list, tuple)):\n self.log.error('tokens must be a list or tuple')\n self.error = True\n return\n\n if not tokens:\n self.log.error('tokens is empty')\n self.error = True\n return\n\n self.tokens = tokens\n\n # Validate the tokens\n def validate_tokens(self):\n terminals = {}\n for n in self.tokens:\n if not _is_identifier.match(n):\n self.log.error(\"Bad token name '%s'\", n)\n self.error = True\n if n in terminals:\n self.log.warning(\"Token '%s' multiply defined\", n)\n terminals[n] = 1\n\n # Get the literals specifier\n def get_literals(self):\n self.literals = self.ldict.get('literals', '')\n if not self.literals:\n self.literals = ''\n\n # Validate literals\n def validate_literals(self):\n try:\n for c in self.literals:\n if not isinstance(c, StringTypes) or len(c) > 1:\n self.log.error('Invalid literal %s. Must be a single character', repr(c))\n self.error = True\n\n except TypeError:\n self.log.error('Invalid literals specification. literals must be a sequence of characters')\n self.error = True\n\n def get_states(self):\n self.states = self.ldict.get('states', None)\n # Build statemap\n if self.states:\n if not isinstance(self.states, (tuple, list)):\n self.log.error('states must be defined as a tuple or list')\n self.error = True\n else:\n for s in self.states:\n if not isinstance(s, tuple) or len(s) != 2:\n self.log.error(\"Invalid state specifier %s. Must be a tuple (statename,'exclusive|inclusive')\", repr(s))\n self.error = True\n continue\n name, statetype = s\n if not isinstance(name, StringTypes):\n self.log.error('State name %s must be a string', repr(name))\n self.error = True\n continue\n if not (statetype == 'inclusive' or statetype == 'exclusive'):\n self.log.error(\"State type for state %s must be 'inclusive' or 'exclusive'\", name)\n self.error = True\n continue\n if name in self.stateinfo:\n self.log.error(\"State '%s' already defined\", name)\n self.error = True\n continue\n self.stateinfo[name] = statetype\n\n # Get all of the symbols with a t_ prefix and sort them into various\n # categories (functions, strings, error functions, and ignore characters)\n\n def get_rules(self):\n tsymbols = [f for f in self.ldict if f[:2] == 't_']\n\n # Now build up a list of functions and a list of strings\n self.toknames = {} # Mapping of symbols to token names\n self.funcsym = {} # Symbols defined as functions\n self.strsym = {} # Symbols defined as strings\n self.ignore = {} # Ignore strings by state\n self.errorf = {} # Error functions by state\n self.eoff = {} # EOF functions by state\n\n for s in self.stateinfo:\n self.funcsym[s] = []\n self.strsym[s] = []\n\n if len(tsymbols) == 0:\n self.log.error('No rules of the form t_rulename are defined')\n self.error = True\n return\n\n for f in tsymbols:\n t = self.ldict[f]\n states, tokname = _statetoken(f, self.stateinfo)\n self.toknames[f] = tokname\n\n if hasattr(t, '__call__'):\n if tokname == 'error':\n for s in states:\n self.errorf[s] = t\n elif tokname == 'eof':\n for s in states:\n self.eoff[s] = t\n elif tokname == 'ignore':\n line = t.__code__.co_firstlineno\n file = t.__code__.co_filename\n self.log.error(\"%s:%d: Rule '%s' must be defined as a string\", file, line, t.__name__)\n self.error = True\n else:\n for s in states:\n self.funcsym[s].append((f, t))\n elif isinstance(t, StringTypes):\n if tokname == 'ignore':\n for s in states:\n self.ignore[s] = t\n if '\\\\' in t:\n self.log.warning(\"%s contains a literal backslash '\\\\'\", f)\n\n elif tokname == 'error':\n self.log.error(\"Rule '%s' must be defined as a function\", f)\n self.error = True\n else:\n for s in states:\n self.strsym[s].append((f, t))\n else:\n self.log.error('%s not defined as a function or string', f)\n self.error = True\n\n # Sort the functions by line number\n for f in self.funcsym.values():\n f.sort(key=lambda x: x[1].__code__.co_firstlineno)\n\n # Sort the strings by regular expression length\n for s in self.strsym.values():\n s.sort(key=lambda x: len(x[1]), reverse=True)\n\n # Validate all of the t_rules collected\n def validate_rules(self):\n for state in self.stateinfo:\n # Validate all rules defined by functions\n\n for fname, f in self.funcsym[state]:\n line = f.__code__.co_firstlineno\n file = f.__code__.co_filename\n module = inspect.getmodule(f)\n self.modules.add(module)\n\n tokname = self.toknames[fname]\n if isinstance(f, types.MethodType):\n reqargs = 2\n else:\n reqargs = 1\n nargs = f.__code__.co_argcount\n if nargs > reqargs:\n self.log.error(\"%s:%d: Rule '%s' has too many arguments\", file, line, f.__name__)\n self.error = True\n continue\n\n if nargs < reqargs:\n self.log.error(\"%s:%d: Rule '%s' requires an argument\", file, line, f.__name__)\n self.error = True\n continue\n\n if not _get_regex(f):\n self.log.error(\"%s:%d: No regular expression defined for rule '%s'\", file, line, f.__name__)\n self.error = True\n continue\n\n try:\n c = re.compile('(?P<%s>%s)' % (fname, _get_regex(f)), self.reflags)\n if c.match(''):\n self.log.error(\"%s:%d: Regular expression for rule '%s' matches empty string\", file, line, f.__name__)\n self.error = True\n except re.error as e:\n self.log.error(\"%s:%d: Invalid regular expression for rule '%s'. %s\", file, line, f.__name__, e)\n if '#' in _get_regex(f):\n self.log.error(\"%s:%d. Make sure '#' in rule '%s' is escaped with '\\\\#'\", file, line, f.__name__)\n self.error = True\n\n # Validate all rules defined by strings\n for name, r in self.strsym[state]:\n tokname = self.toknames[name]\n if tokname == 'error':\n self.log.error(\"Rule '%s' must be defined as a function\", name)\n self.error = True\n continue\n\n if tokname not in self.tokens and tokname.find('ignore_') < 0:\n self.log.error(\"Rule '%s' defined for an unspecified token %s\", name, tokname)\n self.error = True\n continue\n\n try:\n c = re.compile('(?P<%s>%s)' % (name, r), self.reflags)\n if (c.match('')):\n self.log.error(\"Regular expression for rule '%s' matches empty string\", name)\n self.error = True\n except re.error as e:\n self.log.error(\"Invalid regular expression for rule '%s'. %s\", name, e)\n if '#' in r:\n self.log.error(\"Make sure '#' in rule '%s' is escaped with '\\\\#'\", name)\n self.error = True\n\n if not self.funcsym[state] and not self.strsym[state]:\n self.log.error(\"No rules defined for state '%s'\", state)\n self.error = True\n\n # Validate the error function\n efunc = self.errorf.get(state, None)\n if efunc:\n f = efunc\n line = f.__code__.co_firstlineno\n file = f.__code__.co_filename\n module = inspect.getmodule(f)\n self.modules.add(module)\n\n if isinstance(f, types.MethodType):\n reqargs = 2\n else:\n reqargs = 1\n nargs = f.__code__.co_argcount\n if nargs > reqargs:\n self.log.error(\"%s:%d: Rule '%s' has too many arguments\", file, line, f.__name__)\n self.error = True\n\n if nargs < reqargs:\n self.log.error(\"%s:%d: Rule '%s' requires an argument\", file, line, f.__name__)\n self.error = True\n\n for module in self.modules:\n self.validate_module(module)\n\n # -----------------------------------------------------------------------------\n # validate_module()\n #\n # This checks to see if there are duplicated t_rulename() functions or strings\n # in the parser input file. This is done using a simple regular expression\n # match on each line in the source code of the given module.\n # -----------------------------------------------------------------------------\n\n def validate_module(self, module):\n try:\n lines, linen = inspect.getsourcelines(module)\n except IOError:\n return\n\n fre = re.compile(r'\\s*def\\s+(t_[a-zA-Z_0-9]*)\\(')\n sre = re.compile(r'\\s*(t_[a-zA-Z_0-9]*)\\s*=')\n\n counthash = {}\n linen += 1\n for line in lines:\n m = fre.match(line)\n if not m:\n m = sre.match(line)\n if m:\n name = m.group(1)\n prev = counthash.get(name)\n if not prev:\n counthash[name] = linen\n else:\n filename = inspect.getsourcefile(module)\n self.log.error('%s:%d: Rule %s redefined. Previously defined on line %d', filename, linen, name, prev)\n self.error = True\n linen += 1\n\n# -----------------------------------------------------------------------------\n# lex(module)\n#\n# Build all of the regular expression rules from definitions in the supplied module\n# -----------------------------------------------------------------------------\ndef lex(module=None, object=None, debug=False, optimize=False, lextab='lextab',\n reflags=int(re.VERBOSE), nowarn=False, outputdir=None, debuglog=None, errorlog=None):\n\n if lextab is None:\n lextab = 'lextab'\n\n global lexer\n\n ldict = None\n stateinfo = {'INITIAL': 'inclusive'}\n lexobj = Lexer()\n lexobj.lexoptimize = optimize\n global token, input\n\n if errorlog is None:\n errorlog = PlyLogger(sys.stderr)\n\n if debug:\n if debuglog is None:\n debuglog = PlyLogger(sys.stderr)\n\n # Get the module dictionary used for the lexer\n if object:\n module = object\n\n # Get the module dictionary used for the parser\n if module:\n _items = [(k, getattr(module, k)) for k in dir(module)]\n ldict = dict(_items)\n # If no __file__ attribute is available, try to obtain it from the __module__ instead\n if '__file__' not in ldict:\n ldict['__file__'] = sys.modules[ldict['__module__']].__file__\n else:\n ldict = get_caller_module_dict(2)\n\n # Determine if the module is package of a package or not.\n # If so, fix the tabmodule setting so that tables load correctly\n pkg = ldict.get('__package__')\n if pkg and isinstance(lextab, str):\n if '.' not in lextab:\n lextab = pkg + '.' + lextab\n\n # Collect parser information from the dictionary\n linfo = LexerReflect(ldict, log=errorlog, reflags=reflags)\n linfo.get_all()\n if not optimize:\n if linfo.validate_all():\n raise SyntaxError(\"Can't build lexer\")\n\n if optimize and lextab:\n try:\n lexobj.readtab(lextab, ldict)\n token = lexobj.token\n input = lexobj.input\n lexer = lexobj\n return lexobj\n\n except ImportError:\n pass\n\n # Dump some basic debugging information\n if debug:\n debuglog.info('lex: tokens = %r', linfo.tokens)\n debuglog.info('lex: literals = %r', linfo.literals)\n debuglog.info('lex: states = %r', linfo.stateinfo)\n\n # Build a dictionary of valid token names\n lexobj.lextokens = set()\n for n in linfo.tokens:\n lexobj.lextokens.add(n)\n\n # Get literals specification\n if isinstance(linfo.literals, (list, tuple)):\n lexobj.lexliterals = type(linfo.literals[0])().join(linfo.literals)\n else:\n lexobj.lexliterals = linfo.literals\n\n lexobj.lextokens_all = lexobj.lextokens | set(lexobj.lexliterals)\n\n # Get the stateinfo dictionary\n stateinfo = linfo.stateinfo\n\n regexs = {}\n # Build the master regular expressions\n for state in stateinfo:\n regex_list = []\n\n # Add rules defined by functions first\n for fname, f in linfo.funcsym[state]:\n regex_list.append('(?P<%s>%s)' % (fname, _get_regex(f)))\n if debug:\n debuglog.info(\"lex: Adding rule %s -> '%s' (state '%s')\", fname, _get_regex(f), state)\n\n # Now add all of the simple rules\n for name, r in linfo.strsym[state]:\n regex_list.append('(?P<%s>%s)' % (name, r))\n if debug:\n debuglog.info(\"lex: Adding rule %s -> '%s' (state '%s')\", name, r, state)\n\n regexs[state] = regex_list\n\n # Build the master regular expressions\n\n if debug:\n debuglog.info('lex: ==== MASTER REGEXS FOLLOW ====')\n\n for state in regexs:\n lexre, re_text, re_names = _form_master_re(regexs[state], reflags, ldict, linfo.toknames)\n lexobj.lexstatere[state] = lexre\n lexobj.lexstateretext[state] = re_text\n lexobj.lexstaterenames[state] = re_names\n if debug:\n for i, text in enumerate(re_text):\n debuglog.info(\"lex: state '%s' : regex[%d] = '%s'\", state, i, text)\n\n # For inclusive states, we need to add the regular expressions from the INITIAL state\n for state, stype in stateinfo.items():\n if state != 'INITIAL' and stype == 'inclusive':\n lexobj.lexstatere[state].extend(lexobj.lexstatere['INITIAL'])\n lexobj.lexstateretext[state].extend(lexobj.lexstateretext['INITIAL'])\n lexobj.lexstaterenames[state].extend(lexobj.lexstaterenames['INITIAL'])\n\n lexobj.lexstateinfo = stateinfo\n lexobj.lexre = lexobj.lexstatere['INITIAL']\n lexobj.lexretext = lexobj.lexstateretext['INITIAL']\n lexobj.lexreflags = reflags\n\n # Set up ignore variables\n lexobj.lexstateignore = linfo.ignore\n lexobj.lexignore = lexobj.lexstateignore.get('INITIAL', '')\n\n # Set up error functions\n lexobj.lexstateerrorf = linfo.errorf\n lexobj.lexerrorf = linfo.errorf.get('INITIAL', None)\n if not lexobj.lexerrorf:\n errorlog.warning('No t_error rule is defined')\n\n # Set up eof functions\n lexobj.lexstateeoff = linfo.eoff\n lexobj.lexeoff = linfo.eoff.get('INITIAL', None)\n\n # Check state information for ignore and error rules\n for s, stype in stateinfo.items():\n if stype == 'exclusive':\n if s not in linfo.errorf:\n errorlog.warning(\"No error rule is defined for exclusive state '%s'\", s)\n if s not in linfo.ignore and lexobj.lexignore:\n errorlog.warning(\"No ignore rule is defined for exclusive state '%s'\", s)\n elif stype == 'inclusive':\n if s not in linfo.errorf:\n linfo.errorf[s] = linfo.errorf.get('INITIAL', None)\n if s not in linfo.ignore:\n linfo.ignore[s] = linfo.ignore.get('INITIAL', '')\n\n # Create global versions of the token() and input() functions\n token = lexobj.token\n input = lexobj.input\n lexer = lexobj\n\n # If in optimize mode, we write the lextab\n if lextab and optimize:\n if outputdir is None:\n # If no output directory is set, the location of the output files\n # is determined according to the following rules:\n # - If lextab specifies a package, files go into that package directory\n # - Otherwise, files go in the same directory as the specifying module\n if isinstance(lextab, types.ModuleType):\n srcfile = lextab.__file__\n else:\n if '.' not in lextab:\n srcfile = ldict['__file__']\n else:\n parts = lextab.split('.')\n pkgname = '.'.join(parts[:-1])\n exec('import %s' % pkgname)\n srcfile = getattr(sys.modules[pkgname], '__file__', '')\n outputdir = os.path.dirname(srcfile)\n try:\n lexobj.writetab(lextab, outputdir)\n if lextab in sys.modules:\n del sys.modules[lextab]\n except IOError as e:\n errorlog.warning(\"Couldn't write lextab module %r. %s\" % (lextab, e))\n\n return lexobj\n\n# -----------------------------------------------------------------------------\n# runmain()\n#\n# This runs the lexer as a main program\n# -----------------------------------------------------------------------------\n\ndef runmain(lexer=None, data=None):\n if not data:\n try:\n filename = sys.argv[1]\n with open(filename) as f:\n data = f.read()\n except IndexError:\n sys.stdout.write('Reading from standard input (type EOF to end):\\n')\n data = sys.stdin.read()\n\n if lexer:\n _input = lexer.input\n else:\n _input = input\n _input(data)\n if lexer:\n _token = lexer.token\n else:\n _token = token\n\n while True:\n tok = _token()\n if not tok:\n break\n sys.stdout.write('(%s,%r,%d,%d)\\n' % (tok.type, tok.value, tok.lineno, tok.lexpos))\n\n# -----------------------------------------------------------------------------\n# @TOKEN(regex)\n#\n# This decorator function can be used to set the regex expression on a function\n# when its docstring might need to be set in an alternative way\n# -----------------------------------------------------------------------------\n\ndef TOKEN(r):\n def set_regex(f):\n if hasattr(r, '__call__'):\n f.regex = _get_regex(r)\n else:\n f.regex = r\n return f\n return set_regex\n\n# Alternative spelling of the TOKEN decorator\nToken = TOKEN\n" }, { "path": "xonsh/ply/ply/yacc.py", "content": "# -----------------------------------------------------------------------------\n# ply: yacc.py\n#\n# Copyright (C) 2001-2019\n# David M. Beazley (Dabeaz LLC)\n# All rights reserved.\n#\n# Latest version: https://github.com/dabeaz/ply\n#\n# Redistribution and use in source and binary forms, with or without\n# modification, are permitted provided that the following conditions are\n# met:\n#\n# * Redistributions of source code must retain the above copyright notice,\n# this list of conditions and the following disclaimer.\n# * Redistributions in binary form must reproduce the above copyright notice,\n# this list of conditions and the following disclaimer in the documentation\n# and/or other materials provided with the distribution.\n# * Neither the name of David Beazley or Dabeaz LLC may be used to\n# endorse or promote products derived from this software without\n# specific prior written permission.\n#\n# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n# \"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\n# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\n# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\n# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\n# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\n# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n# -----------------------------------------------------------------------------\n#\n# This implements an LR parser that is constructed from grammar rules defined\n# as Python functions. The grammar is specified by supplying the BNF inside\n# Python documentation strings. The inspiration for this technique was borrowed\n# from John Aycock's Spark parsing system. PLY might be viewed as cross between\n# Spark and the GNU bison utility.\n#\n# The current implementation is only somewhat object-oriented. The\n# LR parser itself is defined in terms of an object (which allows multiple\n# parsers to co-exist). However, most of the variables used during table\n# construction are defined in terms of global variables. Users shouldn't\n# notice unless they are trying to define multiple parsers at the same\n# time using threads (in which case they should have their head examined).\n#\n# This implementation supports both SLR and LALR(1) parsing. LALR(1)\n# support was originally implemented by Elias Ioup (ezioup@alumni.uchicago.edu),\n# using the algorithm found in Aho, Sethi, and Ullman \"Compilers: Principles,\n# Techniques, and Tools\" (The Dragon Book). LALR(1) has since been replaced\n# by the more efficient DeRemer and Pennello algorithm.\n#\n# :::::::: WARNING :::::::\n#\n# Construction of LR parsing tables is fairly complicated and expensive.\n# To make this module run fast, a *LOT* of work has been put into\n# optimization---often at the expensive of readability and what might\n# consider to be good Python \"coding style.\" Modify the code at your\n# own risk!\n# ----------------------------------------------------------------------------\n\nimport re\nimport types\nimport sys\nimport os.path\nimport inspect\nimport warnings\n\n__version__ = '3.11'\n__tabversion__ = '3.10'\n\n#-----------------------------------------------------------------------------\n# === User configurable parameters ===\n#\n# Change these to modify the default behavior of yacc (if you wish)\n#-----------------------------------------------------------------------------\n\nyaccdebug = True # Debugging mode. If set, yacc generates a\n # a 'parser.out' file in the current directory\n\ndebug_file = 'parser.out' # Default name of the debugging file\ntab_module = 'parsetab' # Default name of the table module\ndefault_lr = 'LALR' # Default LR table generation method\n\nerror_count = 3 # Number of symbols that must be shifted to leave recovery mode\n\nyaccdevel = False # Set to True if developing yacc. This turns off optimized\n # implementations of certain functions.\n\nresultlimit = 40 # Size limit of results when running in debug mode.\n\npickle_protocol = 0 # Protocol to use when writing pickle files\n\n# String type-checking compatibility\nif sys.version_info[0] < 3:\n string_types = basestring\nelse:\n string_types = str\n\nMAXINT = sys.maxsize\n\n# This object is a stand-in for a logging object created by the\n# logging module. PLY will use this by default to create things\n# such as the parser.out file. If a user wants more detailed\n# information, they can create their own logging object and pass\n# it into PLY.\n\nclass PlyLogger(object):\n def __init__(self, f):\n self.f = f\n\n def debug(self, msg, *args, **kwargs):\n self.f.write((msg % args) + '\\n')\n\n info = debug\n\n def warning(self, msg, *args, **kwargs):\n self.f.write('WARNING: ' + (msg % args) + '\\n')\n\n def error(self, msg, *args, **kwargs):\n self.f.write('ERROR: ' + (msg % args) + '\\n')\n\n critical = debug\n\n# Null logger is used when no output is generated. Does nothing.\nclass NullLogger(object):\n def __getattribute__(self, name):\n return self\n\n def __call__(self, *args, **kwargs):\n return self\n\n# Exception raised for yacc-related errors\nclass YaccError(Exception):\n pass\n\n# Format the result message that the parser produces when running in debug mode.\ndef format_result(r):\n repr_str = repr(r)\n if '\\n' in repr_str:\n repr_str = repr(repr_str)\n if len(repr_str) > resultlimit:\n repr_str = repr_str[:resultlimit] + ' ...'\n result = '<%s @ 0x%x> (%s)' % (type(r).__name__, id(r), repr_str)\n return result\n\n# Format stack entries when the parser is running in debug mode\ndef format_stack_entry(r):\n repr_str = repr(r)\n if '\\n' in repr_str:\n repr_str = repr(repr_str)\n if len(repr_str) < 16:\n return repr_str\n else:\n return '<%s @ 0x%x>' % (type(r).__name__, id(r))\n\n# Panic mode error recovery support. This feature is being reworked--much of the\n# code here is to offer a deprecation/backwards compatible transition\n\n_errok = None\n_token = None\n_restart = None\n_warnmsg = '''PLY: Don't use global functions errok(), token(), and restart() in p_error().\nInstead, invoke the methods on the associated parser instance:\n\n def p_error(p):\n ...\n # Use parser.errok(), parser.token(), parser.restart()\n ...\n\n parser = yacc.yacc()\n'''\n\ndef errok():\n warnings.warn(_warnmsg)\n return _errok()\n\ndef restart():\n warnings.warn(_warnmsg)\n return _restart()\n\ndef token():\n warnings.warn(_warnmsg)\n return _token()\n\n# Utility function to call the p_error() function with some deprecation hacks\ndef call_errorfunc(errorfunc, token, parser):\n global _errok, _token, _restart\n _errok = parser.errok\n _token = parser.token\n _restart = parser.restart\n r = errorfunc(token)\n try:\n del _errok, _token, _restart\n except NameError:\n pass\n return r\n\n#-----------------------------------------------------------------------------\n# === LR Parsing Engine ===\n#\n# The following classes are used for the LR parser itself. These are not\n# used during table construction and are independent of the actual LR\n# table generation algorithm\n#-----------------------------------------------------------------------------\n\n# This class is used to hold non-terminal grammar symbols during parsing.\n# It normally has the following attributes set:\n# .type = Grammar symbol type\n# .value = Symbol value\n# .lineno = Starting line number\n# .endlineno = Ending line number (optional, set automatically)\n# .lexpos = Starting lex position\n# .endlexpos = Ending lex position (optional, set automatically)\n\nclass YaccSymbol:\n def __str__(self):\n return self.type\n\n def __repr__(self):\n return str(self)\n\n# This class is a wrapper around the objects actually passed to each\n# grammar rule. Index lookup and assignment actually assign the\n# .value attribute of the underlying YaccSymbol object.\n# The lineno() method returns the line number of a given\n# item (or 0 if not defined). The linespan() method returns\n# a tuple of (startline,endline) representing the range of lines\n# for a symbol. The lexspan() method returns a tuple (lexpos,endlexpos)\n# representing the range of positional information for a symbol.\n\nclass YaccProduction:\n def __init__(self, s, stack=None):\n self.slice = s\n self.stack = stack\n self.lexer = None\n self.parser = None\n\n def __getitem__(self, n):\n if isinstance(n, slice):\n return [s.value for s in self.slice[n]]\n elif n >= 0:\n return self.slice[n].value\n else:\n return self.stack[n].value\n\n def __setitem__(self, n, v):\n self.slice[n].value = v\n\n def __getslice__(self, i, j):\n return [s.value for s in self.slice[i:j]]\n\n def __len__(self):\n return len(self.slice)\n\n def lineno(self, n):\n return getattr(self.slice[n], 'lineno', 0)\n\n def set_lineno(self, n, lineno):\n self.slice[n].lineno = lineno\n\n def linespan(self, n):\n startline = getattr(self.slice[n], 'lineno', 0)\n endline = getattr(self.slice[n], 'endlineno', startline)\n return startline, endline\n\n def lexpos(self, n):\n return getattr(self.slice[n], 'lexpos', 0)\n\n def set_lexpos(self, n, lexpos):\n self.slice[n].lexpos = lexpos\n\n def lexspan(self, n):\n startpos = getattr(self.slice[n], 'lexpos', 0)\n endpos = getattr(self.slice[n], 'endlexpos', startpos)\n return startpos, endpos\n\n def error(self):\n raise SyntaxError\n\n# -----------------------------------------------------------------------------\n# == LRParser ==\n#\n# The LR Parsing engine.\n# -----------------------------------------------------------------------------\n\nclass LRParser:\n def __init__(self, lrtab, errorf):\n self.productions = lrtab.lr_productions\n self.action = lrtab.lr_action\n self.goto = lrtab.lr_goto\n self.errorfunc = errorf\n self.set_defaulted_states()\n self.errorok = True\n\n def errok(self):\n self.errorok = True\n\n def restart(self):\n del self.statestack[:]\n del self.symstack[:]\n sym = YaccSymbol()\n sym.type = '$end'\n self.symstack.append(sym)\n self.statestack.append(0)\n\n # Defaulted state support.\n # This method identifies parser states where there is only one possible reduction action.\n # For such states, the parser can make a choose to make a rule reduction without consuming\n # the next look-ahead token. This delayed invocation of the tokenizer can be useful in\n # certain kinds of advanced parsing situations where the lexer and parser interact with\n # each other or change states (i.e., manipulation of scope, lexer states, etc.).\n #\n # See: http://www.gnu.org/software/bison/manual/html_node/Default-Reductions.html#Default-Reductions\n def set_defaulted_states(self):\n self.defaulted_states = {}\n for state, actions in self.action.items():\n rules = list(actions.values())\n if len(rules) == 1 and rules[0] < 0:\n self.defaulted_states[state] = rules[0]\n\n def disable_defaulted_states(self):\n self.defaulted_states = {}\n\n def parse(self, input=None, lexer=None, debug=False, tracking=False, tokenfunc=None):\n if debug or yaccdevel:\n if isinstance(debug, int):\n debug = PlyLogger(sys.stderr)\n return self.parsedebug(input, lexer, debug, tracking, tokenfunc)\n elif tracking:\n return self.parseopt(input, lexer, debug, tracking, tokenfunc)\n else:\n return self.parseopt_notrack(input, lexer, debug, tracking, tokenfunc)\n\n\n # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n # parsedebug().\n #\n # This is the debugging enabled version of parse(). All changes made to the\n # parsing engine should be made here. Optimized versions of this function\n # are automatically created by the ply/ygen.py script. This script cuts out\n # sections enclosed in markers such as this:\n #\n # #--! DEBUG\n # statements\n # #--! DEBUG\n #\n # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n\n def parsedebug(self, input=None, lexer=None, debug=False, tracking=False, tokenfunc=None):\n #--! parsedebug-start\n lookahead = None # Current lookahead symbol\n lookaheadstack = [] # Stack of lookahead symbols\n actions = self.action # Local reference to action table (to avoid lookup on self.)\n goto = self.goto # Local reference to goto table (to avoid lookup on self.)\n prod = self.productions # Local reference to production list (to avoid lookup on self.)\n defaulted_states = self.defaulted_states # Local reference to defaulted states\n pslice = YaccProduction(None) # Production object passed to grammar rules\n errorcount = 0 # Used during error recovery\n\n #--! DEBUG\n debug.info('PLY: PARSE DEBUG START')\n #--! DEBUG\n\n # If no lexer was given, we will try to use the lex module\n if not lexer:\n from . import lex\n lexer = lex.lexer\n\n # Set up the lexer and parser objects on pslice\n pslice.lexer = lexer\n pslice.parser = self\n\n # If input was supplied, pass to lexer\n if input is not None:\n lexer.input(input)\n\n if tokenfunc is None:\n # Tokenize function\n get_token = lexer.token\n else:\n get_token = tokenfunc\n\n # Set the parser() token method (sometimes used in error recovery)\n self.token = get_token\n\n # Set up the state and symbol stacks\n\n statestack = [] # Stack of parsing states\n self.statestack = statestack\n symstack = [] # Stack of grammar symbols\n self.symstack = symstack\n\n pslice.stack = symstack # Put in the production\n errtoken = None # Err token\n\n # The start state is assumed to be (0,$end)\n\n statestack.append(0)\n sym = YaccSymbol()\n sym.type = '$end'\n symstack.append(sym)\n state = 0\n while True:\n # Get the next symbol on the input. If a lookahead symbol\n # is already set, we just use that. Otherwise, we'll pull\n # the next token off of the lookaheadstack or from the lexer\n\n #--! DEBUG\n debug.debug('')\n debug.debug('State : %s', state)\n #--! DEBUG\n\n if state not in defaulted_states:\n if not lookahead:\n if not lookaheadstack:\n lookahead = get_token() # Get the next token\n else:\n lookahead = lookaheadstack.pop()\n if not lookahead:\n lookahead = YaccSymbol()\n lookahead.type = '$end'\n\n # Check the action table\n ltype = lookahead.type\n t = actions[state].get(ltype)\n else:\n t = defaulted_states[state]\n #--! DEBUG\n debug.debug('Defaulted state %s: Reduce using %d', state, -t)\n #--! DEBUG\n\n #--! DEBUG\n debug.debug('Stack : %s',\n ('%s . %s' % (' '.join([xx.type for xx in symstack][1:]), str(lookahead))).lstrip())\n #--! DEBUG\n\n if t is not None:\n if t > 0:\n # shift a symbol on the stack\n statestack.append(t)\n state = t\n\n #--! DEBUG\n debug.debug('Action : Shift and goto state %s', t)\n #--! DEBUG\n\n symstack.append(lookahead)\n lookahead = None\n\n # Decrease error count on successful shift\n if errorcount:\n errorcount -= 1\n continue\n\n if t < 0:\n # reduce a symbol on the stack, emit a production\n p = prod[-t]\n pname = p.name\n plen = p.len\n\n # Get production function\n sym = YaccSymbol()\n sym.type = pname # Production name\n sym.value = None\n\n #--! DEBUG\n if plen:\n debug.info('Action : Reduce rule [%s] with %s and goto state %d', p.str,\n '['+','.join([format_stack_entry(_v.value) for _v in symstack[-plen:]])+']',\n goto[statestack[-1-plen]][pname])\n else:\n debug.info('Action : Reduce rule [%s] with %s and goto state %d', p.str, [],\n goto[statestack[-1]][pname])\n\n #--! DEBUG\n\n if plen:\n targ = symstack[-plen-1:]\n targ[0] = sym\n\n #--! TRACKING\n if tracking:\n t1 = targ[1]\n sym.lineno = t1.lineno\n sym.lexpos = t1.lexpos\n t1 = targ[-1]\n sym.endlineno = getattr(t1, 'endlineno', t1.lineno)\n sym.endlexpos = getattr(t1, 'endlexpos', t1.lexpos)\n #--! TRACKING\n\n # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n # The code enclosed in this section is duplicated\n # below as a performance optimization. Make sure\n # changes get made in both locations.\n\n pslice.slice = targ\n\n try:\n # Call the grammar rule with our special slice object\n del symstack[-plen:]\n self.state = state\n p.callable(pslice)\n del statestack[-plen:]\n #--! DEBUG\n debug.info('Result : %s', format_result(pslice[0]))\n #--! DEBUG\n symstack.append(sym)\n state = goto[statestack[-1]][pname]\n statestack.append(state)\n except SyntaxError:\n # If an error was set. Enter error recovery state\n lookaheadstack.append(lookahead) # Save the current lookahead token\n symstack.extend(targ[1:-1]) # Put the production slice back on the stack\n statestack.pop() # Pop back one state (before the reduce)\n state = statestack[-1]\n sym.type = 'error'\n sym.value = 'error'\n lookahead = sym\n errorcount = error_count\n self.errorok = False\n\n continue\n # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n\n else:\n\n #--! TRACKING\n if tracking:\n sym.lineno = lexer.lineno\n sym.lexpos = lexer.lexpos\n #--! TRACKING\n\n targ = [sym]\n\n # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n # The code enclosed in this section is duplicated\n # above as a performance optimization. Make sure\n # changes get made in both locations.\n\n pslice.slice = targ\n\n try:\n # Call the grammar rule with our special slice object\n self.state = state\n p.callable(pslice)\n #--! DEBUG\n debug.info('Result : %s', format_result(pslice[0]))\n #--! DEBUG\n symstack.append(sym)\n state = goto[statestack[-1]][pname]\n statestack.append(state)\n except SyntaxError:\n # If an error was set. Enter error recovery state\n lookaheadstack.append(lookahead) # Save the current lookahead token\n statestack.pop() # Pop back one state (before the reduce)\n state = statestack[-1]\n sym.type = 'error'\n sym.value = 'error'\n lookahead = sym\n errorcount = error_count\n self.errorok = False\n\n continue\n # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n\n if t == 0:\n n = symstack[-1]\n result = getattr(n, 'value', None)\n #--! DEBUG\n debug.info('Done : Returning %s', format_result(result))\n debug.info('PLY: PARSE DEBUG END')\n #--! DEBUG\n return result\n\n if t is None:\n\n #--! DEBUG\n debug.error('Error : %s',\n ('%s . %s' % (' '.join([xx.type for xx in symstack][1:]), str(lookahead))).lstrip())\n #--! DEBUG\n\n # We have some kind of parsing error here. To handle\n # this, we are going to push the current token onto\n # the tokenstack and replace it with an 'error' token.\n # If there are any synchronization rules, they may\n # catch it.\n #\n # In addition to pushing the error token, we call call\n # the user defined p_error() function if this is the\n # first syntax error. This function is only called if\n # errorcount == 0.\n if errorcount == 0 or self.errorok:\n errorcount = error_count\n self.errorok = False\n errtoken = lookahead\n if errtoken.type == '$end':\n errtoken = None # End of file!\n if self.errorfunc:\n if errtoken and not hasattr(errtoken, 'lexer'):\n errtoken.lexer = lexer\n self.state = state\n tok = call_errorfunc(self.errorfunc, errtoken, self)\n if self.errorok:\n # User must have done some kind of panic\n # mode recovery on their own. The\n # returned token is the next lookahead\n lookahead = tok\n errtoken = None\n continue\n else:\n if errtoken:\n if hasattr(errtoken, 'lineno'):\n lineno = lookahead.lineno\n else:\n lineno = 0\n if lineno:\n sys.stderr.write('yacc: Syntax error at line %d, token=%s\\n' % (lineno, errtoken.type))\n else:\n sys.stderr.write('yacc: Syntax error, token=%s' % errtoken.type)\n else:\n sys.stderr.write('yacc: Parse error in input. EOF\\n')\n return\n\n else:\n errorcount = error_count\n\n # case 1: the statestack only has 1 entry on it. If we're in this state, the\n # entire parse has been rolled back and we're completely hosed. The token is\n # discarded and we just keep going.\n\n if len(statestack) <= 1 and lookahead.type != '$end':\n lookahead = None\n errtoken = None\n state = 0\n # Nuke the pushback stack\n del lookaheadstack[:]\n continue\n\n # case 2: the statestack has a couple of entries on it, but we're\n # at the end of the file. nuke the top entry and generate an error token\n\n # Start nuking entries on the stack\n if lookahead.type == '$end':\n # Whoa. We're really hosed here. Bail out\n return\n\n if lookahead.type != 'error':\n sym = symstack[-1]\n if sym.type == 'error':\n # Hmmm. Error is on top of stack, we'll just nuke input\n # symbol and continue\n #--! TRACKING\n if tracking:\n sym.endlineno = getattr(lookahead, 'lineno', sym.lineno)\n sym.endlexpos = getattr(lookahead, 'lexpos', sym.lexpos)\n #--! TRACKING\n lookahead = None\n continue\n\n # Create the error symbol for the first time and make it the new lookahead symbol\n t = YaccSymbol()\n t.type = 'error'\n\n if hasattr(lookahead, 'lineno'):\n t.lineno = t.endlineno = lookahead.lineno\n if hasattr(lookahead, 'lexpos'):\n t.lexpos = t.endlexpos = lookahead.lexpos\n t.value = lookahead\n lookaheadstack.append(lookahead)\n lookahead = t\n else:\n sym = symstack.pop()\n #--! TRACKING\n if tracking:\n lookahead.lineno = sym.lineno\n lookahead.lexpos = sym.lexpos\n #--! TRACKING\n statestack.pop()\n state = statestack[-1]\n\n continue\n\n # Call an error function here\n raise RuntimeError('yacc: internal parser error!!!\\n')\n\n #--! parsedebug-end\n\n # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n # parseopt().\n #\n # Optimized version of parse() method. DO NOT EDIT THIS CODE DIRECTLY!\n # This code is automatically generated by the ply/ygen.py script. Make\n # changes to the parsedebug() method instead.\n # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n\n def parseopt(self, input=None, lexer=None, debug=False, tracking=False, tokenfunc=None):\n #--! parseopt-start\n lookahead = None # Current lookahead symbol\n lookaheadstack = [] # Stack of lookahead symbols\n actions = self.action # Local reference to action table (to avoid lookup on self.)\n goto = self.goto # Local reference to goto table (to avoid lookup on self.)\n prod = self.productions # Local reference to production list (to avoid lookup on self.)\n defaulted_states = self.defaulted_states # Local reference to defaulted states\n pslice = YaccProduction(None) # Production object passed to grammar rules\n errorcount = 0 # Used during error recovery\n\n\n # If no lexer was given, we will try to use the lex module\n if not lexer:\n from . import lex\n lexer = lex.lexer\n\n # Set up the lexer and parser objects on pslice\n pslice.lexer = lexer\n pslice.parser = self\n\n # If input was supplied, pass to lexer\n if input is not None:\n lexer.input(input)\n\n if tokenfunc is None:\n # Tokenize function\n get_token = lexer.token\n else:\n get_token = tokenfunc\n\n # Set the parser() token method (sometimes used in error recovery)\n self.token = get_token\n\n # Set up the state and symbol stacks\n\n statestack = [] # Stack of parsing states\n self.statestack = statestack\n symstack = [] # Stack of grammar symbols\n self.symstack = symstack\n\n pslice.stack = symstack # Put in the production\n errtoken = None # Err token\n\n # The start state is assumed to be (0,$end)\n\n statestack.append(0)\n sym = YaccSymbol()\n sym.type = '$end'\n symstack.append(sym)\n state = 0\n while True:\n # Get the next symbol on the input. If a lookahead symbol\n # is already set, we just use that. Otherwise, we'll pull\n # the next token off of the lookaheadstack or from the lexer\n\n\n if state not in defaulted_states:\n if not lookahead:\n if not lookaheadstack:\n lookahead = get_token() # Get the next token\n else:\n lookahead = lookaheadstack.pop()\n if not lookahead:\n lookahead = YaccSymbol()\n lookahead.type = '$end'\n\n # Check the action table\n ltype = lookahead.type\n t = actions[state].get(ltype)\n else:\n t = defaulted_states[state]\n\n\n if t is not None:\n if t > 0:\n # shift a symbol on the stack\n statestack.append(t)\n state = t\n\n\n symstack.append(lookahead)\n lookahead = None\n\n # Decrease error count on successful shift\n if errorcount:\n errorcount -= 1\n continue\n\n if t < 0:\n # reduce a symbol on the stack, emit a production\n p = prod[-t]\n pname = p.name\n plen = p.len\n\n # Get production function\n sym = YaccSymbol()\n sym.type = pname # Production name\n sym.value = None\n\n\n if plen:\n targ = symstack[-plen-1:]\n targ[0] = sym\n\n #--! TRACKING\n if tracking:\n t1 = targ[1]\n sym.lineno = t1.lineno\n sym.lexpos = t1.lexpos\n t1 = targ[-1]\n sym.endlineno = getattr(t1, 'endlineno', t1.lineno)\n sym.endlexpos = getattr(t1, 'endlexpos', t1.lexpos)\n #--! TRACKING\n\n # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n # The code enclosed in this section is duplicated\n # below as a performance optimization. Make sure\n # changes get made in both locations.\n\n pslice.slice = targ\n\n try:\n # Call the grammar rule with our special slice object\n del symstack[-plen:]\n self.state = state\n p.callable(pslice)\n del statestack[-plen:]\n symstack.append(sym)\n state = goto[statestack[-1]][pname]\n statestack.append(state)\n except SyntaxError:\n # If an error was set. Enter error recovery state\n lookaheadstack.append(lookahead) # Save the current lookahead token\n symstack.extend(targ[1:-1]) # Put the production slice back on the stack\n statestack.pop() # Pop back one state (before the reduce)\n state = statestack[-1]\n sym.type = 'error'\n sym.value = 'error'\n lookahead = sym\n errorcount = error_count\n self.errorok = False\n\n continue\n # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n\n else:\n\n #--! TRACKING\n if tracking:\n sym.lineno = lexer.lineno\n sym.lexpos = lexer.lexpos\n #--! TRACKING\n\n targ = [sym]\n\n # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n # The code enclosed in this section is duplicated\n # above as a performance optimization. Make sure\n # changes get made in both locations.\n\n pslice.slice = targ\n\n try:\n # Call the grammar rule with our special slice object\n self.state = state\n p.callable(pslice)\n symstack.append(sym)\n state = goto[statestack[-1]][pname]\n statestack.append(state)\n except SyntaxError:\n # If an error was set. Enter error recovery state\n lookaheadstack.append(lookahead) # Save the current lookahead token\n statestack.pop() # Pop back one state (before the reduce)\n state = statestack[-1]\n sym.type = 'error'\n sym.value = 'error'\n lookahead = sym\n errorcount = error_count\n self.errorok = False\n\n continue\n # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n\n if t == 0:\n n = symstack[-1]\n result = getattr(n, 'value', None)\n return result\n\n if t is None:\n\n\n # We have some kind of parsing error here. To handle\n # this, we are going to push the current token onto\n # the tokenstack and replace it with an 'error' token.\n # If there are any synchronization rules, they may\n # catch it.\n #\n # In addition to pushing the error token, we call call\n # the user defined p_error() function if this is the\n # first syntax error. This function is only called if\n # errorcount == 0.\n if errorcount == 0 or self.errorok:\n errorcount = error_count\n self.errorok = False\n errtoken = lookahead\n if errtoken.type == '$end':\n errtoken = None # End of file!\n if self.errorfunc:\n if errtoken and not hasattr(errtoken, 'lexer'):\n errtoken.lexer = lexer\n self.state = state\n tok = call_errorfunc(self.errorfunc, errtoken, self)\n if self.errorok:\n # User must have done some kind of panic\n # mode recovery on their own. The\n # returned token is the next lookahead\n lookahead = tok\n errtoken = None\n continue\n else:\n if errtoken:\n if hasattr(errtoken, 'lineno'):\n lineno = lookahead.lineno\n else:\n lineno = 0\n if lineno:\n sys.stderr.write('yacc: Syntax error at line %d, token=%s\\n' % (lineno, errtoken.type))\n else:\n sys.stderr.write('yacc: Syntax error, token=%s' % errtoken.type)\n else:\n sys.stderr.write('yacc: Parse error in input. EOF\\n')\n return\n\n else:\n errorcount = error_count\n\n # case 1: the statestack only has 1 entry on it. If we're in this state, the\n # entire parse has been rolled back and we're completely hosed. The token is\n # discarded and we just keep going.\n\n if len(statestack) <= 1 and lookahead.type != '$end':\n lookahead = None\n errtoken = None\n state = 0\n # Nuke the pushback stack\n del lookaheadstack[:]\n continue\n\n # case 2: the statestack has a couple of entries on it, but we're\n # at the end of the file. nuke the top entry and generate an error token\n\n # Start nuking entries on the stack\n if lookahead.type == '$end':\n # Whoa. We're really hosed here. Bail out\n return\n\n if lookahead.type != 'error':\n sym = symstack[-1]\n if sym.type == 'error':\n # Hmmm. Error is on top of stack, we'll just nuke input\n # symbol and continue\n #--! TRACKING\n if tracking:\n sym.endlineno = getattr(lookahead, 'lineno', sym.lineno)\n sym.endlexpos = getattr(lookahead, 'lexpos', sym.lexpos)\n #--! TRACKING\n lookahead = None\n continue\n\n # Create the error symbol for the first time and make it the new lookahead symbol\n t = YaccSymbol()\n t.type = 'error'\n\n if hasattr(lookahead, 'lineno'):\n t.lineno = t.endlineno = lookahead.lineno\n if hasattr(lookahead, 'lexpos'):\n t.lexpos = t.endlexpos = lookahead.lexpos\n t.value = lookahead\n lookaheadstack.append(lookahead)\n lookahead = t\n else:\n sym = symstack.pop()\n #--! TRACKING\n if tracking:\n lookahead.lineno = sym.lineno\n lookahead.lexpos = sym.lexpos\n #--! TRACKING\n statestack.pop()\n state = statestack[-1]\n\n continue\n\n # Call an error function here\n raise RuntimeError('yacc: internal parser error!!!\\n')\n\n #--! parseopt-end\n\n # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n # parseopt_notrack().\n #\n # Optimized version of parseopt() with line number tracking removed.\n # DO NOT EDIT THIS CODE DIRECTLY. This code is automatically generated\n # by the ply/ygen.py script. Make changes to the parsedebug() method instead.\n # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n\n def parseopt_notrack(self, input=None, lexer=None, debug=False, tracking=False, tokenfunc=None):\n #--! parseopt-notrack-start\n lookahead = None # Current lookahead symbol\n lookaheadstack = [] # Stack of lookahead symbols\n actions = self.action # Local reference to action table (to avoid lookup on self.)\n goto = self.goto # Local reference to goto table (to avoid lookup on self.)\n prod = self.productions # Local reference to production list (to avoid lookup on self.)\n defaulted_states = self.defaulted_states # Local reference to defaulted states\n pslice = YaccProduction(None) # Production object passed to grammar rules\n errorcount = 0 # Used during error recovery\n\n\n # If no lexer was given, we will try to use the lex module\n if not lexer:\n from . import lex\n lexer = lex.lexer\n\n # Set up the lexer and parser objects on pslice\n pslice.lexer = lexer\n pslice.parser = self\n\n # If input was supplied, pass to lexer\n if input is not None:\n lexer.input(input)\n\n if tokenfunc is None:\n # Tokenize function\n get_token = lexer.token\n else:\n get_token = tokenfunc\n\n # Set the parser() token method (sometimes used in error recovery)\n self.token = get_token\n\n # Set up the state and symbol stacks\n\n statestack = [] # Stack of parsing states\n self.statestack = statestack\n symstack = [] # Stack of grammar symbols\n self.symstack = symstack\n\n pslice.stack = symstack # Put in the production\n errtoken = None # Err token\n\n # The start state is assumed to be (0,$end)\n\n statestack.append(0)\n sym = YaccSymbol()\n sym.type = '$end'\n symstack.append(sym)\n state = 0\n while True:\n # Get the next symbol on the input. If a lookahead symbol\n # is already set, we just use that. Otherwise, we'll pull\n # the next token off of the lookaheadstack or from the lexer\n\n\n if state not in defaulted_states:\n if not lookahead:\n if not lookaheadstack:\n lookahead = get_token() # Get the next token\n else:\n lookahead = lookaheadstack.pop()\n if not lookahead:\n lookahead = YaccSymbol()\n lookahead.type = '$end'\n\n # Check the action table\n ltype = lookahead.type\n t = actions[state].get(ltype)\n else:\n t = defaulted_states[state]\n\n\n if t is not None:\n if t > 0:\n # shift a symbol on the stack\n statestack.append(t)\n state = t\n\n\n symstack.append(lookahead)\n lookahead = None\n\n # Decrease error count on successful shift\n if errorcount:\n errorcount -= 1\n continue\n\n if t < 0:\n # reduce a symbol on the stack, emit a production\n p = prod[-t]\n pname = p.name\n plen = p.len\n\n # Get production function\n sym = YaccSymbol()\n sym.type = pname # Production name\n sym.value = None\n\n\n if plen:\n targ = symstack[-plen-1:]\n targ[0] = sym\n\n\n # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n # The code enclosed in this section is duplicated\n # below as a performance optimization. Make sure\n # changes get made in both locations.\n\n pslice.slice = targ\n\n try:\n # Call the grammar rule with our special slice object\n del symstack[-plen:]\n self.state = state\n p.callable(pslice)\n del statestack[-plen:]\n symstack.append(sym)\n state = goto[statestack[-1]][pname]\n statestack.append(state)\n except SyntaxError:\n # If an error was set. Enter error recovery state\n lookaheadstack.append(lookahead) # Save the current lookahead token\n symstack.extend(targ[1:-1]) # Put the production slice back on the stack\n statestack.pop() # Pop back one state (before the reduce)\n state = statestack[-1]\n sym.type = 'error'\n sym.value = 'error'\n lookahead = sym\n errorcount = error_count\n self.errorok = False\n\n continue\n # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n\n else:\n\n\n targ = [sym]\n\n # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n # The code enclosed in this section is duplicated\n # above as a performance optimization. Make sure\n # changes get made in both locations.\n\n pslice.slice = targ\n\n try:\n # Call the grammar rule with our special slice object\n self.state = state\n p.callable(pslice)\n symstack.append(sym)\n state = goto[statestack[-1]][pname]\n statestack.append(state)\n except SyntaxError:\n # If an error was set. Enter error recovery state\n lookaheadstack.append(lookahead) # Save the current lookahead token\n statestack.pop() # Pop back one state (before the reduce)\n state = statestack[-1]\n sym.type = 'error'\n sym.value = 'error'\n lookahead = sym\n errorcount = error_count\n self.errorok = False\n\n continue\n # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n\n if t == 0:\n n = symstack[-1]\n result = getattr(n, 'value', None)\n return result\n\n if t is None:\n\n\n # We have some kind of parsing error here. To handle\n # this, we are going to push the current token onto\n # the tokenstack and replace it with an 'error' token.\n # If there are any synchronization rules, they may\n # catch it.\n #\n # In addition to pushing the error token, we call call\n # the user defined p_error() function if this is the\n # first syntax error. This function is only called if\n # errorcount == 0.\n if errorcount == 0 or self.errorok:\n errorcount = error_count\n self.errorok = False\n errtoken = lookahead\n if errtoken.type == '$end':\n errtoken = None # End of file!\n if self.errorfunc:\n if errtoken and not hasattr(errtoken, 'lexer'):\n errtoken.lexer = lexer\n self.state = state\n tok = call_errorfunc(self.errorfunc, errtoken, self)\n if self.errorok:\n # User must have done some kind of panic\n # mode recovery on their own. The\n # returned token is the next lookahead\n lookahead = tok\n errtoken = None\n continue\n else:\n if errtoken:\n if hasattr(errtoken, 'lineno'):\n lineno = lookahead.lineno\n else:\n lineno = 0\n if lineno:\n sys.stderr.write('yacc: Syntax error at line %d, token=%s\\n' % (lineno, errtoken.type))\n else:\n sys.stderr.write('yacc: Syntax error, token=%s' % errtoken.type)\n else:\n sys.stderr.write('yacc: Parse error in input. EOF\\n')\n return\n\n else:\n errorcount = error_count\n\n # case 1: the statestack only has 1 entry on it. If we're in this state, the\n # entire parse has been rolled back and we're completely hosed. The token is\n # discarded and we just keep going.\n\n if len(statestack) <= 1 and lookahead.type != '$end':\n lookahead = None\n errtoken = None\n state = 0\n # Nuke the pushback stack\n del lookaheadstack[:]\n continue\n\n # case 2: the statestack has a couple of entries on it, but we're\n # at the end of the file. nuke the top entry and generate an error token\n\n # Start nuking entries on the stack\n if lookahead.type == '$end':\n # Whoa. We're really hosed here. Bail out\n return\n\n if lookahead.type != 'error':\n sym = symstack[-1]\n if sym.type == 'error':\n # Hmmm. Error is on top of stack, we'll just nuke input\n # symbol and continue\n lookahead = None\n continue\n\n # Create the error symbol for the first time and make it the new lookahead symbol\n t = YaccSymbol()\n t.type = 'error'\n\n if hasattr(lookahead, 'lineno'):\n t.lineno = t.endlineno = lookahead.lineno\n if hasattr(lookahead, 'lexpos'):\n t.lexpos = t.endlexpos = lookahead.lexpos\n t.value = lookahead\n lookaheadstack.append(lookahead)\n lookahead = t\n else:\n sym = symstack.pop()\n statestack.pop()\n state = statestack[-1]\n\n continue\n\n # Call an error function here\n raise RuntimeError('yacc: internal parser error!!!\\n')\n\n #--! parseopt-notrack-end\n\n# -----------------------------------------------------------------------------\n# === Grammar Representation ===\n#\n# The following functions, classes, and variables are used to represent and\n# manipulate the rules that make up a grammar.\n# -----------------------------------------------------------------------------\n\n# regex matching identifiers\n_is_identifier = re.compile(r'^[a-zA-Z0-9_-]+$')\n\n# -----------------------------------------------------------------------------\n# class Production:\n#\n# This class stores the raw information about a single production or grammar rule.\n# A grammar rule refers to a specification such as this:\n#\n# expr : expr PLUS term\n#\n# Here are the basic attributes defined on all productions\n#\n# name - Name of the production. For example 'expr'\n# prod - A list of symbols on the right side ['expr','PLUS','term']\n# prec - Production precedence level\n# number - Production number.\n# func - Function that executes on reduce\n# file - File where production function is defined\n# lineno - Line number where production function is defined\n#\n# The following attributes are defined or optional.\n#\n# len - Length of the production (number of symbols on right hand side)\n# usyms - Set of unique symbols found in the production\n# -----------------------------------------------------------------------------\n\nclass Production(object):\n reduced = 0\n def __init__(self, number, name, prod, precedence=('right', 0), func=None, file='', line=0):\n self.name = name\n self.prod = tuple(prod)\n self.number = number\n self.func = func\n self.callable = None\n self.file = file\n self.line = line\n self.prec = precedence\n\n # Internal settings used during table construction\n\n self.len = len(self.prod) # Length of the production\n\n # Create a list of unique production symbols used in the production\n self.usyms = []\n for s in self.prod:\n if s not in self.usyms:\n self.usyms.append(s)\n\n # List of all LR items for the production\n self.lr_items = []\n self.lr_next = None\n\n # Create a string representation\n if self.prod:\n self.str = '%s -> %s' % (self.name, ' '.join(self.prod))\n else:\n self.str = '%s -> ' % self.name\n\n def __str__(self):\n return self.str\n\n def __repr__(self):\n return 'Production(' + str(self) + ')'\n\n def __len__(self):\n return len(self.prod)\n\n def __nonzero__(self):\n return 1\n\n def __getitem__(self, index):\n return self.prod[index]\n\n # Return the nth lr_item from the production (or None if at the end)\n def lr_item(self, n):\n if n > len(self.prod):\n return None\n p = LRItem(self, n)\n # Precompute the list of productions immediately following.\n try:\n p.lr_after = self.Prodnames[p.prod[n+1]]\n except (IndexError, KeyError):\n p.lr_after = []\n try:\n p.lr_before = p.prod[n-1]\n except IndexError:\n p.lr_before = None\n return p\n\n # Bind the production function name to a callable\n def bind(self, pdict):\n if self.func:\n self.callable = pdict[self.func]\n\n# This class serves as a minimal standin for Production objects when\n# reading table data from files. It only contains information\n# actually used by the LR parsing engine, plus some additional\n# debugging information.\nclass MiniProduction(object):\n def __init__(self, str, name, len, func, file, line):\n self.name = name\n self.len = len\n self.func = func\n self.callable = None\n self.file = file\n self.line = line\n self.str = str\n\n def __str__(self):\n return self.str\n\n def __repr__(self):\n return 'MiniProduction(%s)' % self.str\n\n # Bind the production function name to a callable\n def bind(self, pdict):\n if self.func:\n self.callable = pdict[self.func]\n\n\n# -----------------------------------------------------------------------------\n# class LRItem\n#\n# This class represents a specific stage of parsing a production rule. For\n# example:\n#\n# expr : expr . PLUS term\n#\n# In the above, the \".\" represents the current location of the parse. Here\n# basic attributes:\n#\n# name - Name of the production. For example 'expr'\n# prod - A list of symbols on the right side ['expr','.', 'PLUS','term']\n# number - Production number.\n#\n# lr_next Next LR item. Example, if we are ' expr -> expr . PLUS term'\n# then lr_next refers to 'expr -> expr PLUS . term'\n# lr_index - LR item index (location of the \".\") in the prod list.\n# lookaheads - LALR lookahead symbols for this item\n# len - Length of the production (number of symbols on right hand side)\n# lr_after - List of all productions that immediately follow\n# lr_before - Grammar symbol immediately before\n# -----------------------------------------------------------------------------\n\nclass LRItem(object):\n def __init__(self, p, n):\n self.name = p.name\n self.prod = list(p.prod)\n self.number = p.number\n self.lr_index = n\n self.lookaheads = {}\n self.prod.insert(n, '.')\n self.prod = tuple(self.prod)\n self.len = len(self.prod)\n self.usyms = p.usyms\n\n def __str__(self):\n if self.prod:\n s = '%s -> %s' % (self.name, ' '.join(self.prod))\n else:\n s = '%s -> ' % self.name\n return s\n\n def __repr__(self):\n return 'LRItem(' + str(self) + ')'\n\n# -----------------------------------------------------------------------------\n# rightmost_terminal()\n#\n# Return the rightmost terminal from a list of symbols. Used in add_production()\n# -----------------------------------------------------------------------------\ndef rightmost_terminal(symbols, terminals):\n i = len(symbols) - 1\n while i >= 0:\n if symbols[i] in terminals:\n return symbols[i]\n i -= 1\n return None\n\n# -----------------------------------------------------------------------------\n# === GRAMMAR CLASS ===\n#\n# The following class represents the contents of the specified grammar along\n# with various computed properties such as first sets, follow sets, LR items, etc.\n# This data is used for critical parts of the table generation process later.\n# -----------------------------------------------------------------------------\n\nclass GrammarError(YaccError):\n pass\n\nclass Grammar(object):\n def __init__(self, terminals):\n self.Productions = [None] # A list of all of the productions. The first\n # entry is always reserved for the purpose of\n # building an augmented grammar\n\n self.Prodnames = {} # A dictionary mapping the names of nonterminals to a list of all\n # productions of that nonterminal.\n\n self.Prodmap = {} # A dictionary that is only used to detect duplicate\n # productions.\n\n self.Terminals = {} # A dictionary mapping the names of terminal symbols to a\n # list of the rules where they are used.\n\n for term in terminals:\n self.Terminals[term] = []\n\n self.Terminals['error'] = []\n\n self.Nonterminals = {} # A dictionary mapping names of nonterminals to a list\n # of rule numbers where they are used.\n\n self.First = {} # A dictionary of precomputed FIRST(x) symbols\n\n self.Follow = {} # A dictionary of precomputed FOLLOW(x) symbols\n\n self.Precedence = {} # Precedence rules for each terminal. Contains tuples of the\n # form ('right',level) or ('nonassoc', level) or ('left',level)\n\n self.UsedPrecedence = set() # Precedence rules that were actually used by the grammer.\n # This is only used to provide error checking and to generate\n # a warning about unused precedence rules.\n\n self.Start = None # Starting symbol for the grammar\n\n\n def __len__(self):\n return len(self.Productions)\n\n def __getitem__(self, index):\n return self.Productions[index]\n\n # -----------------------------------------------------------------------------\n # set_precedence()\n #\n # Sets the precedence for a given terminal. assoc is the associativity such as\n # 'left','right', or 'nonassoc'. level is a numeric level.\n #\n # -----------------------------------------------------------------------------\n\n def set_precedence(self, term, assoc, level):\n assert self.Productions == [None], 'Must call set_precedence() before add_production()'\n if term in self.Precedence:\n raise GrammarError('Precedence already specified for terminal %r' % term)\n if assoc not in ['left', 'right', 'nonassoc']:\n raise GrammarError(\"Associativity must be one of 'left','right', or 'nonassoc'\")\n self.Precedence[term] = (assoc, level)\n\n # -----------------------------------------------------------------------------\n # add_production()\n #\n # Given an action function, this function assembles a production rule and\n # computes its precedence level.\n #\n # The production rule is supplied as a list of symbols. For example,\n # a rule such as 'expr : expr PLUS term' has a production name of 'expr' and\n # symbols ['expr','PLUS','term'].\n #\n # Precedence is determined by the precedence of the right-most non-terminal\n # or the precedence of a terminal specified by %prec.\n #\n # A variety of error checks are performed to make sure production symbols\n # are valid and that %prec is used correctly.\n # -----------------------------------------------------------------------------\n\n def add_production(self, prodname, syms, func=None, file='', line=0):\n\n if prodname in self.Terminals:\n raise GrammarError('%s:%d: Illegal rule name %r. Already defined as a token' % (file, line, prodname))\n if prodname == 'error':\n raise GrammarError('%s:%d: Illegal rule name %r. error is a reserved word' % (file, line, prodname))\n if not _is_identifier.match(prodname):\n raise GrammarError('%s:%d: Illegal rule name %r' % (file, line, prodname))\n\n # Look for literal tokens\n for n, s in enumerate(syms):\n if s[0] in \"'\\\"\":\n try:\n c = eval(s)\n if (len(c) > 1):\n raise GrammarError('%s:%d: Literal token %s in rule %r may only be a single character' %\n (file, line, s, prodname))\n if c not in self.Terminals:\n self.Terminals[c] = []\n syms[n] = c\n continue\n except SyntaxError:\n pass\n if not _is_identifier.match(s) and s != '%prec':\n raise GrammarError('%s:%d: Illegal name %r in rule %r' % (file, line, s, prodname))\n\n # Determine the precedence level\n if '%prec' in syms:\n if syms[-1] == '%prec':\n raise GrammarError('%s:%d: Syntax error. Nothing follows %%prec' % (file, line))\n if syms[-2] != '%prec':\n raise GrammarError('%s:%d: Syntax error. %%prec can only appear at the end of a grammar rule' %\n (file, line))\n precname = syms[-1]\n prodprec = self.Precedence.get(precname)\n if not prodprec:\n raise GrammarError('%s:%d: Nothing known about the precedence of %r' % (file, line, precname))\n else:\n self.UsedPrecedence.add(precname)\n del syms[-2:] # Drop %prec from the rule\n else:\n # If no %prec, precedence is determined by the rightmost terminal symbol\n precname = rightmost_terminal(syms, self.Terminals)\n prodprec = self.Precedence.get(precname, ('right', 0))\n\n # See if the rule is already in the rulemap\n map = '%s -> %s' % (prodname, syms)\n if map in self.Prodmap:\n m = self.Prodmap[map]\n raise GrammarError('%s:%d: Duplicate rule %s. ' % (file, line, m) +\n 'Previous definition at %s:%d' % (m.file, m.line))\n\n # From this point on, everything is valid. Create a new Production instance\n pnumber = len(self.Productions)\n if prodname not in self.Nonterminals:\n self.Nonterminals[prodname] = []\n\n # Add the production number to Terminals and Nonterminals\n for t in syms:\n if t in self.Terminals:\n self.Terminals[t].append(pnumber)\n else:\n if t not in self.Nonterminals:\n self.Nonterminals[t] = []\n self.Nonterminals[t].append(pnumber)\n\n # Create a production and add it to the list of productions\n p = Production(pnumber, prodname, syms, prodprec, func, file, line)\n self.Productions.append(p)\n self.Prodmap[map] = p\n\n # Add to the global productions list\n try:\n self.Prodnames[prodname].append(p)\n except KeyError:\n self.Prodnames[prodname] = [p]\n\n # -----------------------------------------------------------------------------\n # set_start()\n #\n # Sets the starting symbol and creates the augmented grammar. Production\n # rule 0 is S' -> start where start is the start symbol.\n # -----------------------------------------------------------------------------\n\n def set_start(self, start=None):\n if not start:\n start = self.Productions[1].name\n if start not in self.Nonterminals:\n raise GrammarError('start symbol %s undefined' % start)\n self.Productions[0] = Production(0, \"S'\", [start])\n self.Nonterminals[start].append(0)\n self.Start = start\n\n # -----------------------------------------------------------------------------\n # find_unreachable()\n #\n # Find all of the nonterminal symbols that can't be reached from the starting\n # symbol. Returns a list of nonterminals that can't be reached.\n # -----------------------------------------------------------------------------\n\n def find_unreachable(self):\n\n # Mark all symbols that are reachable from a symbol s\n def mark_reachable_from(s):\n if s in reachable:\n return\n reachable.add(s)\n for p in self.Prodnames.get(s, []):\n for r in p.prod:\n mark_reachable_from(r)\n\n reachable = set()\n mark_reachable_from(self.Productions[0].prod[0])\n return [s for s in self.Nonterminals if s not in reachable]\n\n # -----------------------------------------------------------------------------\n # infinite_cycles()\n #\n # This function looks at the various parsing rules and tries to detect\n # infinite recursion cycles (grammar rules where there is no possible way\n # to derive a string of only terminals).\n # -----------------------------------------------------------------------------\n\n def infinite_cycles(self):\n terminates = {}\n\n # Terminals:\n for t in self.Terminals:\n terminates[t] = True\n\n terminates['$end'] = True\n\n # Nonterminals:\n\n # Initialize to false:\n for n in self.Nonterminals:\n terminates[n] = False\n\n # Then propagate termination until no change:\n while True:\n some_change = False\n for (n, pl) in self.Prodnames.items():\n # Nonterminal n terminates iff any of its productions terminates.\n for p in pl:\n # Production p terminates iff all of its rhs symbols terminate.\n for s in p.prod:\n if not terminates[s]:\n # The symbol s does not terminate,\n # so production p does not terminate.\n p_terminates = False\n break\n else:\n # didn't break from the loop,\n # so every symbol s terminates\n # so production p terminates.\n p_terminates = True\n\n if p_terminates:\n # symbol n terminates!\n if not terminates[n]:\n terminates[n] = True\n some_change = True\n # Don't need to consider any more productions for this n.\n break\n\n if not some_change:\n break\n\n infinite = []\n for (s, term) in terminates.items():\n if not term:\n if s not in self.Prodnames and s not in self.Terminals and s != 'error':\n # s is used-but-not-defined, and we've already warned of that,\n # so it would be overkill to say that it's also non-terminating.\n pass\n else:\n infinite.append(s)\n\n return infinite\n\n # -----------------------------------------------------------------------------\n # undefined_symbols()\n #\n # Find all symbols that were used the grammar, but not defined as tokens or\n # grammar rules. Returns a list of tuples (sym, prod) where sym in the symbol\n # and prod is the production where the symbol was used.\n # -----------------------------------------------------------------------------\n def undefined_symbols(self):\n result = []\n for p in self.Productions:\n if not p:\n continue\n\n for s in p.prod:\n if s not in self.Prodnames and s not in self.Terminals and s != 'error':\n result.append((s, p))\n return result\n\n # -----------------------------------------------------------------------------\n # unused_terminals()\n #\n # Find all terminals that were defined, but not used by the grammar. Returns\n # a list of all symbols.\n # -----------------------------------------------------------------------------\n def unused_terminals(self):\n unused_tok = []\n for s, v in self.Terminals.items():\n if s != 'error' and not v:\n unused_tok.append(s)\n\n return unused_tok\n\n # ------------------------------------------------------------------------------\n # unused_rules()\n #\n # Find all grammar rules that were defined, but not used (maybe not reachable)\n # Returns a list of productions.\n # ------------------------------------------------------------------------------\n\n def unused_rules(self):\n unused_prod = []\n for s, v in self.Nonterminals.items():\n if not v:\n p = self.Prodnames[s][0]\n unused_prod.append(p)\n return unused_prod\n\n # -----------------------------------------------------------------------------\n # unused_precedence()\n #\n # Returns a list of tuples (term,precedence) corresponding to precedence\n # rules that were never used by the grammar. term is the name of the terminal\n # on which precedence was applied and precedence is a string such as 'left' or\n # 'right' corresponding to the type of precedence.\n # -----------------------------------------------------------------------------\n\n def unused_precedence(self):\n unused = []\n for termname in self.Precedence:\n if not (termname in self.Terminals or termname in self.UsedPrecedence):\n unused.append((termname, self.Precedence[termname][0]))\n\n return unused\n\n # -------------------------------------------------------------------------\n # _first()\n #\n # Compute the value of FIRST1(beta) where beta is a tuple of symbols.\n #\n # During execution of compute_first1, the result may be incomplete.\n # Afterward (e.g., when called from compute_follow()), it will be complete.\n # -------------------------------------------------------------------------\n def _first(self, beta):\n\n # We are computing First(x1,x2,x3,...,xn)\n result = []\n for x in beta:\n x_produces_empty = False\n\n # Add all the non- symbols of First[x] to the result.\n for f in self.First[x]:\n if f == '':\n x_produces_empty = True\n else:\n if f not in result:\n result.append(f)\n\n if x_produces_empty:\n # We have to consider the next x in beta,\n # i.e. stay in the loop.\n pass\n else:\n # We don't have to consider any further symbols in beta.\n break\n else:\n # There was no 'break' from the loop,\n # so x_produces_empty was true for all x in beta,\n # so beta produces empty as well.\n result.append('')\n\n return result\n\n # -------------------------------------------------------------------------\n # compute_first()\n #\n # Compute the value of FIRST1(X) for all symbols\n # -------------------------------------------------------------------------\n def compute_first(self):\n if self.First:\n return self.First\n\n # Terminals:\n for t in self.Terminals:\n self.First[t] = [t]\n\n self.First['$end'] = ['$end']\n\n # Nonterminals:\n\n # Initialize to the empty set:\n for n in self.Nonterminals:\n self.First[n] = []\n\n # Then propagate symbols until no change:\n while True:\n some_change = False\n for n in self.Nonterminals:\n for p in self.Prodnames[n]:\n for f in self._first(p.prod):\n if f not in self.First[n]:\n self.First[n].append(f)\n some_change = True\n if not some_change:\n break\n\n return self.First\n\n # ---------------------------------------------------------------------\n # compute_follow()\n #\n # Computes all of the follow sets for every non-terminal symbol. The\n # follow set is the set of all symbols that might follow a given\n # non-terminal. See the Dragon book, 2nd Ed. p. 189.\n # ---------------------------------------------------------------------\n def compute_follow(self, start=None):\n # If already computed, return the result\n if self.Follow:\n return self.Follow\n\n # If first sets not computed yet, do that first.\n if not self.First:\n self.compute_first()\n\n # Add '$end' to the follow list of the start symbol\n for k in self.Nonterminals:\n self.Follow[k] = []\n\n if not start:\n start = self.Productions[1].name\n\n self.Follow[start] = ['$end']\n\n while True:\n didadd = False\n for p in self.Productions[1:]:\n # Here is the production set\n for i, B in enumerate(p.prod):\n if B in self.Nonterminals:\n # Okay. We got a non-terminal in a production\n fst = self._first(p.prod[i+1:])\n hasempty = False\n for f in fst:\n if f != '' and f not in self.Follow[B]:\n self.Follow[B].append(f)\n didadd = True\n if f == '':\n hasempty = True\n if hasempty or i == (len(p.prod)-1):\n # Add elements of follow(a) to follow(b)\n for f in self.Follow[p.name]:\n if f not in self.Follow[B]:\n self.Follow[B].append(f)\n didadd = True\n if not didadd:\n break\n return self.Follow\n\n\n # -----------------------------------------------------------------------------\n # build_lritems()\n #\n # This function walks the list of productions and builds a complete set of the\n # LR items. The LR items are stored in two ways: First, they are uniquely\n # numbered and placed in the list _lritems. Second, a linked list of LR items\n # is built for each production. For example:\n #\n # E -> E PLUS E\n #\n # Creates the list\n #\n # [E -> . E PLUS E, E -> E . PLUS E, E -> E PLUS . E, E -> E PLUS E . ]\n # -----------------------------------------------------------------------------\n\n def build_lritems(self):\n for p in self.Productions:\n lastlri = p\n i = 0\n lr_items = []\n while True:\n if i > len(p):\n lri = None\n else:\n lri = LRItem(p, i)\n # Precompute the list of productions immediately following\n try:\n lri.lr_after = self.Prodnames[lri.prod[i+1]]\n except (IndexError, KeyError):\n lri.lr_after = []\n try:\n lri.lr_before = lri.prod[i-1]\n except IndexError:\n lri.lr_before = None\n\n lastlri.lr_next = lri\n if not lri:\n break\n lr_items.append(lri)\n lastlri = lri\n i += 1\n p.lr_items = lr_items\n\n# -----------------------------------------------------------------------------\n# == Class LRTable ==\n#\n# This basic class represents a basic table of LR parsing information.\n# Methods for generating the tables are not defined here. They are defined\n# in the derived class LRGeneratedTable.\n# -----------------------------------------------------------------------------\n\nclass VersionError(YaccError):\n pass\n\nclass LRTable(object):\n def __init__(self):\n self.lr_action = None\n self.lr_goto = None\n self.lr_productions = None\n self.lr_method = None\n\n def read_table(self, module):\n if isinstance(module, types.ModuleType):\n parsetab = module\n else:\n exec('import %s' % module)\n parsetab = sys.modules[module]\n\n if parsetab._tabversion != __tabversion__:\n raise VersionError('yacc table file version is out of date')\n\n self.lr_action = parsetab._lr_action\n self.lr_goto = parsetab._lr_goto\n\n self.lr_productions = []\n for p in parsetab._lr_productions:\n self.lr_productions.append(MiniProduction(*p))\n\n self.lr_method = parsetab._lr_method\n return parsetab._lr_signature\n\n def read_pickle(self, filename):\n try:\n import cPickle as pickle\n except ImportError:\n import pickle\n\n if not os.path.exists(filename):\n raise ImportError\n\n in_f = open(filename, 'rb')\n\n tabversion = pickle.load(in_f)\n if tabversion != __tabversion__:\n raise VersionError('yacc table file version is out of date')\n self.lr_method = pickle.load(in_f)\n signature = pickle.load(in_f)\n self.lr_action = pickle.load(in_f)\n self.lr_goto = pickle.load(in_f)\n productions = pickle.load(in_f)\n\n self.lr_productions = []\n for p in productions:\n self.lr_productions.append(MiniProduction(*p))\n\n in_f.close()\n return signature\n\n # Bind all production function names to callable objects in pdict\n def bind_callables(self, pdict):\n for p in self.lr_productions:\n p.bind(pdict)\n\n\n# -----------------------------------------------------------------------------\n# === LR Generator ===\n#\n# The following classes and functions are used to generate LR parsing tables on\n# a grammar.\n# -----------------------------------------------------------------------------\n\n# -----------------------------------------------------------------------------\n# digraph()\n# traverse()\n#\n# The following two functions are used to compute set valued functions\n# of the form:\n#\n# F(x) = F'(x) U U{F(y) | x R y}\n#\n# This is used to compute the values of Read() sets as well as FOLLOW sets\n# in LALR(1) generation.\n#\n# Inputs: X - An input set\n# R - A relation\n# FP - Set-valued function\n# ------------------------------------------------------------------------------\n\ndef digraph(X, R, FP):\n N = {}\n for x in X:\n N[x] = 0\n stack = []\n F = {}\n for x in X:\n if N[x] == 0:\n traverse(x, N, stack, F, X, R, FP)\n return F\n\ndef traverse(x, N, stack, F, X, R, FP):\n stack.append(x)\n d = len(stack)\n N[x] = d\n F[x] = FP(x) # F(X) <- F'(x)\n\n rel = R(x) # Get y's related to x\n for y in rel:\n if N[y] == 0:\n traverse(y, N, stack, F, X, R, FP)\n N[x] = min(N[x], N[y])\n for a in F.get(y, []):\n if a not in F[x]:\n F[x].append(a)\n if N[x] == d:\n N[stack[-1]] = MAXINT\n F[stack[-1]] = F[x]\n element = stack.pop()\n while element != x:\n N[stack[-1]] = MAXINT\n F[stack[-1]] = F[x]\n element = stack.pop()\n\nclass LALRError(YaccError):\n pass\n\n# -----------------------------------------------------------------------------\n# == LRGeneratedTable ==\n#\n# This class implements the LR table generation algorithm. There are no\n# public methods except for write()\n# -----------------------------------------------------------------------------\n\nclass LRGeneratedTable(LRTable):\n def __init__(self, grammar, method='LALR', log=None):\n if method not in ['SLR', 'LALR']:\n raise LALRError('Unsupported method %s' % method)\n\n self.grammar = grammar\n self.lr_method = method\n\n # Set up the logger\n if not log:\n log = NullLogger()\n self.log = log\n\n # Internal attributes\n self.lr_action = {} # Action table\n self.lr_goto = {} # Goto table\n self.lr_productions = grammar.Productions # Copy of grammar Production array\n self.lr_goto_cache = {} # Cache of computed gotos\n self.lr0_cidhash = {} # Cache of closures\n\n self._add_count = 0 # Internal counter used to detect cycles\n\n # Diagonistic information filled in by the table generator\n self.sr_conflict = 0\n self.rr_conflict = 0\n self.conflicts = [] # List of conflicts\n\n self.sr_conflicts = []\n self.rr_conflicts = []\n\n # Build the tables\n self.grammar.build_lritems()\n self.grammar.compute_first()\n self.grammar.compute_follow()\n self.lr_parse_table()\n\n # Compute the LR(0) closure operation on I, where I is a set of LR(0) items.\n\n def lr0_closure(self, I):\n self._add_count += 1\n\n # Add everything in I to J\n J = I[:]\n didadd = True\n while didadd:\n didadd = False\n for j in J:\n for x in j.lr_after:\n if getattr(x, 'lr0_added', 0) == self._add_count:\n continue\n # Add B --> .G to J\n J.append(x.lr_next)\n x.lr0_added = self._add_count\n didadd = True\n\n return J\n\n # Compute the LR(0) goto function goto(I,X) where I is a set\n # of LR(0) items and X is a grammar symbol. This function is written\n # in a way that guarantees uniqueness of the generated goto sets\n # (i.e. the same goto set will never be returned as two different Python\n # objects). With uniqueness, we can later do fast set comparisons using\n # id(obj) instead of element-wise comparison.\n\n def lr0_goto(self, I, x):\n # First we look for a previously cached entry\n g = self.lr_goto_cache.get((id(I), x))\n if g:\n return g\n\n # Now we generate the goto set in a way that guarantees uniqueness\n # of the result\n\n s = self.lr_goto_cache.get(x)\n if not s:\n s = {}\n self.lr_goto_cache[x] = s\n\n gs = []\n for p in I:\n n = p.lr_next\n if n and n.lr_before == x:\n s1 = s.get(id(n))\n if not s1:\n s1 = {}\n s[id(n)] = s1\n gs.append(n)\n s = s1\n g = s.get('$end')\n if not g:\n if gs:\n g = self.lr0_closure(gs)\n s['$end'] = g\n else:\n s['$end'] = gs\n self.lr_goto_cache[(id(I), x)] = g\n return g\n\n # Compute the LR(0) sets of item function\n def lr0_items(self):\n C = [self.lr0_closure([self.grammar.Productions[0].lr_next])]\n i = 0\n for I in C:\n self.lr0_cidhash[id(I)] = i\n i += 1\n\n # Loop over the items in C and each grammar symbols\n i = 0\n while i < len(C):\n I = C[i]\n i += 1\n\n # Collect all of the symbols that could possibly be in the goto(I,X) sets\n asyms = {}\n for ii in I:\n for s in ii.usyms:\n asyms[s] = None\n\n for x in asyms:\n g = self.lr0_goto(I, x)\n if not g or id(g) in self.lr0_cidhash:\n continue\n self.lr0_cidhash[id(g)] = len(C)\n C.append(g)\n\n return C\n\n # -----------------------------------------------------------------------------\n # ==== LALR(1) Parsing ====\n #\n # LALR(1) parsing is almost exactly the same as SLR except that instead of\n # relying upon Follow() sets when performing reductions, a more selective\n # lookahead set that incorporates the state of the LR(0) machine is utilized.\n # Thus, we mainly just have to focus on calculating the lookahead sets.\n #\n # The method used here is due to DeRemer and Pennelo (1982).\n #\n # DeRemer, F. L., and T. J. Pennelo: \"Efficient Computation of LALR(1)\n # Lookahead Sets\", ACM Transactions on Programming Languages and Systems,\n # Vol. 4, No. 4, Oct. 1982, pp. 615-649\n #\n # Further details can also be found in:\n #\n # J. Tremblay and P. Sorenson, \"The Theory and Practice of Compiler Writing\",\n # McGraw-Hill Book Company, (1985).\n #\n # -----------------------------------------------------------------------------\n\n # -----------------------------------------------------------------------------\n # compute_nullable_nonterminals()\n #\n # Creates a dictionary containing all of the non-terminals that might produce\n # an empty production.\n # -----------------------------------------------------------------------------\n\n def compute_nullable_nonterminals(self):\n nullable = set()\n num_nullable = 0\n while True:\n for p in self.grammar.Productions[1:]:\n if p.len == 0:\n nullable.add(p.name)\n continue\n for t in p.prod:\n if t not in nullable:\n break\n else:\n nullable.add(p.name)\n if len(nullable) == num_nullable:\n break\n num_nullable = len(nullable)\n return nullable\n\n # -----------------------------------------------------------------------------\n # find_nonterminal_trans(C)\n #\n # Given a set of LR(0) items, this functions finds all of the non-terminal\n # transitions. These are transitions in which a dot appears immediately before\n # a non-terminal. Returns a list of tuples of the form (state,N) where state\n # is the state number and N is the nonterminal symbol.\n #\n # The input C is the set of LR(0) items.\n # -----------------------------------------------------------------------------\n\n def find_nonterminal_transitions(self, C):\n trans = []\n for stateno, state in enumerate(C):\n for p in state:\n if p.lr_index < p.len - 1:\n t = (stateno, p.prod[p.lr_index+1])\n if t[1] in self.grammar.Nonterminals:\n if t not in trans:\n trans.append(t)\n return trans\n\n # -----------------------------------------------------------------------------\n # dr_relation()\n #\n # Computes the DR(p,A) relationships for non-terminal transitions. The input\n # is a tuple (state,N) where state is a number and N is a nonterminal symbol.\n #\n # Returns a list of terminals.\n # -----------------------------------------------------------------------------\n\n def dr_relation(self, C, trans, nullable):\n state, N = trans\n terms = []\n\n g = self.lr0_goto(C[state], N)\n for p in g:\n if p.lr_index < p.len - 1:\n a = p.prod[p.lr_index+1]\n if a in self.grammar.Terminals:\n if a not in terms:\n terms.append(a)\n\n # This extra bit is to handle the start state\n if state == 0 and N == self.grammar.Productions[0].prod[0]:\n terms.append('$end')\n\n return terms\n\n # -----------------------------------------------------------------------------\n # reads_relation()\n #\n # Computes the READS() relation (p,A) READS (t,C).\n # -----------------------------------------------------------------------------\n\n def reads_relation(self, C, trans, empty):\n # Look for empty transitions\n rel = []\n state, N = trans\n\n g = self.lr0_goto(C[state], N)\n j = self.lr0_cidhash.get(id(g), -1)\n for p in g:\n if p.lr_index < p.len - 1:\n a = p.prod[p.lr_index + 1]\n if a in empty:\n rel.append((j, a))\n\n return rel\n\n # -----------------------------------------------------------------------------\n # compute_lookback_includes()\n #\n # Determines the lookback and includes relations\n #\n # LOOKBACK:\n #\n # This relation is determined by running the LR(0) state machine forward.\n # For example, starting with a production \"N : . A B C\", we run it forward\n # to obtain \"N : A B C .\" We then build a relationship between this final\n # state and the starting state. These relationships are stored in a dictionary\n # lookdict.\n #\n # INCLUDES:\n #\n # Computes the INCLUDE() relation (p,A) INCLUDES (p',B).\n #\n # This relation is used to determine non-terminal transitions that occur\n # inside of other non-terminal transition states. (p,A) INCLUDES (p', B)\n # if the following holds:\n #\n # B -> LAT, where T -> epsilon and p' -L-> p\n #\n # L is essentially a prefix (which may be empty), T is a suffix that must be\n # able to derive an empty string. State p' must lead to state p with the string L.\n #\n # -----------------------------------------------------------------------------\n\n def compute_lookback_includes(self, C, trans, nullable):\n lookdict = {} # Dictionary of lookback relations\n includedict = {} # Dictionary of include relations\n\n # Make a dictionary of non-terminal transitions\n dtrans = {}\n for t in trans:\n dtrans[t] = 1\n\n # Loop over all transitions and compute lookbacks and includes\n for state, N in trans:\n lookb = []\n includes = []\n for p in C[state]:\n if p.name != N:\n continue\n\n # Okay, we have a name match. We now follow the production all the way\n # through the state machine until we get the . on the right hand side\n\n lr_index = p.lr_index\n j = state\n while lr_index < p.len - 1:\n lr_index = lr_index + 1\n t = p.prod[lr_index]\n\n # Check to see if this symbol and state are a non-terminal transition\n if (j, t) in dtrans:\n # Yes. Okay, there is some chance that this is an includes relation\n # the only way to know for certain is whether the rest of the\n # production derives empty\n\n li = lr_index + 1\n while li < p.len:\n if p.prod[li] in self.grammar.Terminals:\n break # No forget it\n if p.prod[li] not in nullable:\n break\n li = li + 1\n else:\n # Appears to be a relation between (j,t) and (state,N)\n includes.append((j, t))\n\n g = self.lr0_goto(C[j], t) # Go to next set\n j = self.lr0_cidhash.get(id(g), -1) # Go to next state\n\n # When we get here, j is the final state, now we have to locate the production\n for r in C[j]:\n if r.name != p.name:\n continue\n if r.len != p.len:\n continue\n i = 0\n # This look is comparing a production \". A B C\" with \"A B C .\"\n while i < r.lr_index:\n if r.prod[i] != p.prod[i+1]:\n break\n i = i + 1\n else:\n lookb.append((j, r))\n for i in includes:\n if i not in includedict:\n includedict[i] = []\n includedict[i].append((state, N))\n lookdict[(state, N)] = lookb\n\n return lookdict, includedict\n\n # -----------------------------------------------------------------------------\n # compute_read_sets()\n #\n # Given a set of LR(0) items, this function computes the read sets.\n #\n # Inputs: C = Set of LR(0) items\n # ntrans = Set of nonterminal transitions\n # nullable = Set of empty transitions\n #\n # Returns a set containing the read sets\n # -----------------------------------------------------------------------------\n\n def compute_read_sets(self, C, ntrans, nullable):\n FP = lambda x: self.dr_relation(C, x, nullable)\n R = lambda x: self.reads_relation(C, x, nullable)\n F = digraph(ntrans, R, FP)\n return F\n\n # -----------------------------------------------------------------------------\n # compute_follow_sets()\n #\n # Given a set of LR(0) items, a set of non-terminal transitions, a readset,\n # and an include set, this function computes the follow sets\n #\n # Follow(p,A) = Read(p,A) U U {Follow(p',B) | (p,A) INCLUDES (p',B)}\n #\n # Inputs:\n # ntrans = Set of nonterminal transitions\n # readsets = Readset (previously computed)\n # inclsets = Include sets (previously computed)\n #\n # Returns a set containing the follow sets\n # -----------------------------------------------------------------------------\n\n def compute_follow_sets(self, ntrans, readsets, inclsets):\n FP = lambda x: readsets[x]\n R = lambda x: inclsets.get(x, [])\n F = digraph(ntrans, R, FP)\n return F\n\n # -----------------------------------------------------------------------------\n # add_lookaheads()\n #\n # Attaches the lookahead symbols to grammar rules.\n #\n # Inputs: lookbacks - Set of lookback relations\n # followset - Computed follow set\n #\n # This function directly attaches the lookaheads to productions contained\n # in the lookbacks set\n # -----------------------------------------------------------------------------\n\n def add_lookaheads(self, lookbacks, followset):\n for trans, lb in lookbacks.items():\n # Loop over productions in lookback\n for state, p in lb:\n if state not in p.lookaheads:\n p.lookaheads[state] = []\n f = followset.get(trans, [])\n for a in f:\n if a not in p.lookaheads[state]:\n p.lookaheads[state].append(a)\n\n # -----------------------------------------------------------------------------\n # add_lalr_lookaheads()\n #\n # This function does all of the work of adding lookahead information for use\n # with LALR parsing\n # -----------------------------------------------------------------------------\n\n def add_lalr_lookaheads(self, C):\n # Determine all of the nullable nonterminals\n nullable = self.compute_nullable_nonterminals()\n\n # Find all non-terminal transitions\n trans = self.find_nonterminal_transitions(C)\n\n # Compute read sets\n readsets = self.compute_read_sets(C, trans, nullable)\n\n # Compute lookback/includes relations\n lookd, included = self.compute_lookback_includes(C, trans, nullable)\n\n # Compute LALR FOLLOW sets\n followsets = self.compute_follow_sets(trans, readsets, included)\n\n # Add all of the lookaheads\n self.add_lookaheads(lookd, followsets)\n\n # -----------------------------------------------------------------------------\n # lr_parse_table()\n #\n # This function constructs the parse tables for SLR or LALR\n # -----------------------------------------------------------------------------\n def lr_parse_table(self):\n Productions = self.grammar.Productions\n Precedence = self.grammar.Precedence\n goto = self.lr_goto # Goto array\n action = self.lr_action # Action array\n log = self.log # Logger for output\n\n actionp = {} # Action production array (temporary)\n\n log.info('Parsing method: %s', self.lr_method)\n\n # Step 1: Construct C = { I0, I1, ... IN}, collection of LR(0) items\n # This determines the number of states\n\n C = self.lr0_items()\n\n if self.lr_method == 'LALR':\n self.add_lalr_lookaheads(C)\n\n # Build the parser table, state by state\n st = 0\n for I in C:\n # Loop over each production in I\n actlist = [] # List of actions\n st_action = {}\n st_actionp = {}\n st_goto = {}\n log.info('')\n log.info('state %d', st)\n log.info('')\n for p in I:\n log.info(' (%d) %s', p.number, p)\n log.info('')\n\n for p in I:\n if p.len == p.lr_index + 1:\n if p.name == \"S'\":\n # Start symbol. Accept!\n st_action['$end'] = 0\n st_actionp['$end'] = p\n else:\n # We are at the end of a production. Reduce!\n if self.lr_method == 'LALR':\n laheads = p.lookaheads[st]\n else:\n laheads = self.grammar.Follow[p.name]\n for a in laheads:\n actlist.append((a, p, 'reduce using rule %d (%s)' % (p.number, p)))\n r = st_action.get(a)\n if r is not None:\n # Whoa. Have a shift/reduce or reduce/reduce conflict\n if r > 0:\n # Need to decide on shift or reduce here\n # By default we favor shifting. Need to add\n # some precedence rules here.\n\n # Shift precedence comes from the token\n sprec, slevel = Precedence.get(a, ('right', 0))\n\n # Reduce precedence comes from rule being reduced (p)\n rprec, rlevel = Productions[p.number].prec\n\n if (slevel < rlevel) or ((slevel == rlevel) and (rprec == 'left')):\n # We really need to reduce here.\n st_action[a] = -p.number\n st_actionp[a] = p\n if not slevel and not rlevel:\n log.info(' ! shift/reduce conflict for %s resolved as reduce', a)\n self.sr_conflicts.append((st, a, 'reduce'))\n Productions[p.number].reduced += 1\n elif (slevel == rlevel) and (rprec == 'nonassoc'):\n st_action[a] = None\n else:\n # Hmmm. Guess we'll keep the shift\n if not rlevel:\n log.info(' ! shift/reduce conflict for %s resolved as shift', a)\n self.sr_conflicts.append((st, a, 'shift'))\n elif r < 0:\n # Reduce/reduce conflict. In this case, we favor the rule\n # that was defined first in the grammar file\n oldp = Productions[-r]\n pp = Productions[p.number]\n if oldp.line > pp.line:\n st_action[a] = -p.number\n st_actionp[a] = p\n chosenp, rejectp = pp, oldp\n Productions[p.number].reduced += 1\n Productions[oldp.number].reduced -= 1\n else:\n chosenp, rejectp = oldp, pp\n self.rr_conflicts.append((st, chosenp, rejectp))\n log.info(' ! reduce/reduce conflict for %s resolved using rule %d (%s)',\n a, st_actionp[a].number, st_actionp[a])\n else:\n raise LALRError('Unknown conflict in state %d' % st)\n else:\n st_action[a] = -p.number\n st_actionp[a] = p\n Productions[p.number].reduced += 1\n else:\n i = p.lr_index\n a = p.prod[i+1] # Get symbol right after the \".\"\n if a in self.grammar.Terminals:\n g = self.lr0_goto(I, a)\n j = self.lr0_cidhash.get(id(g), -1)\n if j >= 0:\n # We are in a shift state\n actlist.append((a, p, 'shift and go to state %d' % j))\n r = st_action.get(a)\n if r is not None:\n # Whoa have a shift/reduce or shift/shift conflict\n if r > 0:\n if r != j:\n raise LALRError('Shift/shift conflict in state %d' % st)\n elif r < 0:\n # Do a precedence check.\n # - if precedence of reduce rule is higher, we reduce.\n # - if precedence of reduce is same and left assoc, we reduce.\n # - otherwise we shift\n\n # Shift precedence comes from the token\n sprec, slevel = Precedence.get(a, ('right', 0))\n\n # Reduce precedence comes from the rule that could have been reduced\n rprec, rlevel = Productions[st_actionp[a].number].prec\n\n if (slevel > rlevel) or ((slevel == rlevel) and (rprec == 'right')):\n # We decide to shift here... highest precedence to shift\n Productions[st_actionp[a].number].reduced -= 1\n st_action[a] = j\n st_actionp[a] = p\n if not rlevel:\n log.info(' ! shift/reduce conflict for %s resolved as shift', a)\n self.sr_conflicts.append((st, a, 'shift'))\n elif (slevel == rlevel) and (rprec == 'nonassoc'):\n st_action[a] = None\n else:\n # Hmmm. Guess we'll keep the reduce\n if not slevel and not rlevel:\n log.info(' ! shift/reduce conflict for %s resolved as reduce', a)\n self.sr_conflicts.append((st, a, 'reduce'))\n\n else:\n raise LALRError('Unknown conflict in state %d' % st)\n else:\n st_action[a] = j\n st_actionp[a] = p\n\n # Print the actions associated with each terminal\n _actprint = {}\n for a, p, m in actlist:\n if a in st_action:\n if p is st_actionp[a]:\n log.info(' %-15s %s', a, m)\n _actprint[(a, m)] = 1\n log.info('')\n # Print the actions that were not used. (debugging)\n not_used = 0\n for a, p, m in actlist:\n if a in st_action:\n if p is not st_actionp[a]:\n if not (a, m) in _actprint:\n log.debug(' ! %-15s [ %s ]', a, m)\n not_used = 1\n _actprint[(a, m)] = 1\n if not_used:\n log.debug('')\n\n # Construct the goto table for this state\n\n nkeys = {}\n for ii in I:\n for s in ii.usyms:\n if s in self.grammar.Nonterminals:\n nkeys[s] = None\n for n in nkeys:\n g = self.lr0_goto(I, n)\n j = self.lr0_cidhash.get(id(g), -1)\n if j >= 0:\n st_goto[n] = j\n log.info(' %-30s shift and go to state %d', n, j)\n\n action[st] = st_action\n actionp[st] = st_actionp\n goto[st] = st_goto\n st += 1\n\n # -----------------------------------------------------------------------------\n # write()\n #\n # This function writes the LR parsing tables to a file\n # -----------------------------------------------------------------------------\n\n def write_table(self, tabmodule, outputdir='', signature=''):\n if isinstance(tabmodule, types.ModuleType):\n raise IOError(\"Won't overwrite existing tabmodule\")\n\n basemodulename = tabmodule.split('.')[-1]\n filename = os.path.join(outputdir, basemodulename) + '.py'\n try:\n f = open(filename, 'w')\n\n f.write('''\n# %s\n# This file is automatically generated. Do not edit.\n# pylint: disable=W,C,R\n_tabversion = %r\n\n_lr_method = %r\n\n_lr_signature = %r\n ''' % (os.path.basename(filename), __tabversion__, self.lr_method, signature))\n\n # Change smaller to 0 to go back to original tables\n smaller = 1\n\n # Factor out names to try and make smaller\n if smaller:\n items = {}\n\n for s, nd in self.lr_action.items():\n for name, v in nd.items():\n i = items.get(name)\n if not i:\n i = ([], [])\n items[name] = i\n i[0].append(s)\n i[1].append(v)\n\n f.write('\\n_lr_action_items = {')\n for k, v in items.items():\n f.write('%r:([' % k)\n for i in v[0]:\n f.write('%r,' % i)\n f.write('],[')\n for i in v[1]:\n f.write('%r,' % i)\n\n f.write(']),')\n f.write('}\\n')\n\n f.write('''\n_lr_action = {}\nfor _k, _v in _lr_action_items.items():\n for _x,_y in zip(_v[0],_v[1]):\n if not _x in _lr_action: _lr_action[_x] = {}\n _lr_action[_x][_k] = _y\ndel _lr_action_items\n''')\n\n else:\n f.write('\\n_lr_action = { ')\n for k, v in self.lr_action.items():\n f.write('(%r,%r):%r,' % (k[0], k[1], v))\n f.write('}\\n')\n\n if smaller:\n # Factor out names to try and make smaller\n items = {}\n\n for s, nd in self.lr_goto.items():\n for name, v in nd.items():\n i = items.get(name)\n if not i:\n i = ([], [])\n items[name] = i\n i[0].append(s)\n i[1].append(v)\n\n f.write('\\n_lr_goto_items = {')\n for k, v in items.items():\n f.write('%r:([' % k)\n for i in v[0]:\n f.write('%r,' % i)\n f.write('],[')\n for i in v[1]:\n f.write('%r,' % i)\n\n f.write(']),')\n f.write('}\\n')\n\n f.write('''\n_lr_goto = {}\nfor _k, _v in _lr_goto_items.items():\n for _x, _y in zip(_v[0], _v[1]):\n if not _x in _lr_goto: _lr_goto[_x] = {}\n _lr_goto[_x][_k] = _y\ndel _lr_goto_items\n''')\n else:\n f.write('\\n_lr_goto = { ')\n for k, v in self.lr_goto.items():\n f.write('(%r,%r):%r,' % (k[0], k[1], v))\n f.write('}\\n')\n\n # Write production table\n f.write('_lr_productions = [\\n')\n for p in self.lr_productions:\n if p.func:\n f.write(' (%r,%r,%d,%r,%r,%d),\\n' % (p.str, p.name, p.len,\n p.func, os.path.basename(p.file), p.line))\n else:\n f.write(' (%r,%r,%d,None,None,None),\\n' % (str(p), p.name, p.len))\n f.write(']\\n')\n f.close()\n\n except IOError as e:\n raise\n\n\n # -----------------------------------------------------------------------------\n # pickle_table()\n #\n # This function pickles the LR parsing tables to a supplied file object\n # -----------------------------------------------------------------------------\n\n def pickle_table(self, filename, signature=''):\n try:\n import cPickle as pickle\n except ImportError:\n import pickle\n with open(filename, 'wb') as outf:\n pickle.dump(__tabversion__, outf, pickle_protocol)\n pickle.dump(self.lr_method, outf, pickle_protocol)\n pickle.dump(signature, outf, pickle_protocol)\n pickle.dump(self.lr_action, outf, pickle_protocol)\n pickle.dump(self.lr_goto, outf, pickle_protocol)\n\n outp = []\n for p in self.lr_productions:\n if p.func:\n outp.append((p.str, p.name, p.len, p.func, os.path.basename(p.file), p.line))\n else:\n outp.append((str(p), p.name, p.len, None, None, None))\n pickle.dump(outp, outf, pickle_protocol)\n\n# -----------------------------------------------------------------------------\n# === INTROSPECTION ===\n#\n# The following functions and classes are used to implement the PLY\n# introspection features followed by the yacc() function itself.\n# -----------------------------------------------------------------------------\n\n# -----------------------------------------------------------------------------\n# get_caller_module_dict()\n#\n# This function returns a dictionary containing all of the symbols defined within\n# a caller further down the call stack. This is used to get the environment\n# associated with the yacc() call if none was provided.\n# -----------------------------------------------------------------------------\n\ndef get_caller_module_dict(levels):\n f = sys._getframe(levels)\n ldict = f.f_globals.copy()\n if f.f_globals != f.f_locals:\n ldict.update(f.f_locals)\n return ldict\n\n# -----------------------------------------------------------------------------\n# parse_grammar()\n#\n# This takes a raw grammar rule string and parses it into production data\n# -----------------------------------------------------------------------------\ndef parse_grammar(doc, file, line):\n grammar = []\n # Split the doc string into lines\n pstrings = doc.splitlines()\n lastp = None\n dline = line\n for ps in pstrings:\n dline += 1\n p = ps.split()\n if not p:\n continue\n try:\n if p[0] == '|':\n # This is a continuation of a previous rule\n if not lastp:\n raise SyntaxError(\"%s:%d: Misplaced '|'\" % (file, dline))\n prodname = lastp\n syms = p[1:]\n else:\n prodname = p[0]\n lastp = prodname\n syms = p[2:]\n assign = p[1]\n if assign != ':' and assign != '::=':\n raise SyntaxError(\"%s:%d: Syntax error. Expected ':'\" % (file, dline))\n\n grammar.append((file, dline, prodname, syms))\n except SyntaxError:\n raise\n except Exception:\n raise SyntaxError('%s:%d: Syntax error in rule %r' % (file, dline, ps.strip()))\n\n return grammar\n\n# -----------------------------------------------------------------------------\n# ParserReflect()\n#\n# This class represents information extracted for building a parser including\n# start symbol, error function, tokens, precedence list, action functions,\n# etc.\n# -----------------------------------------------------------------------------\nclass ParserReflect(object):\n def __init__(self, pdict, log=None):\n self.pdict = pdict\n self.start = None\n self.error_func = None\n self.tokens = None\n self.modules = set()\n self.grammar = []\n self.error = False\n\n if log is None:\n self.log = PlyLogger(sys.stderr)\n else:\n self.log = log\n\n # Get all of the basic information\n def get_all(self):\n self.get_start()\n self.get_error_func()\n self.get_tokens()\n self.get_precedence()\n self.get_pfunctions()\n\n # Validate all of the information\n def validate_all(self):\n self.validate_start()\n self.validate_error_func()\n self.validate_tokens()\n self.validate_precedence()\n self.validate_pfunctions()\n self.validate_modules()\n return self.error\n\n # Compute a signature over the grammar\n def signature(self):\n parts = []\n try:\n if self.start:\n parts.append(self.start)\n if self.prec:\n parts.append(''.join([''.join(p) for p in self.prec]))\n if self.tokens:\n parts.append(' '.join(self.tokens))\n for f in self.pfuncs:\n if f[3]:\n parts.append(f[3])\n except (TypeError, ValueError):\n pass\n return ''.join(parts)\n\n # -----------------------------------------------------------------------------\n # validate_modules()\n #\n # This method checks to see if there are duplicated p_rulename() functions\n # in the parser module file. Without this function, it is really easy for\n # users to make mistakes by cutting and pasting code fragments (and it's a real\n # bugger to try and figure out why the resulting parser doesn't work). Therefore,\n # we just do a little regular expression pattern matching of def statements\n # to try and detect duplicates.\n # -----------------------------------------------------------------------------\n\n def validate_modules(self):\n # Match def p_funcname(\n fre = re.compile(r'\\s*def\\s+(p_[a-zA-Z_0-9]*)\\(')\n\n for module in self.modules:\n try:\n lines, linen = inspect.getsourcelines(module)\n except IOError:\n continue\n\n counthash = {}\n for linen, line in enumerate(lines):\n linen += 1\n m = fre.match(line)\n if m:\n name = m.group(1)\n prev = counthash.get(name)\n if not prev:\n counthash[name] = linen\n else:\n filename = inspect.getsourcefile(module)\n self.log.warning('%s:%d: Function %s redefined. Previously defined on line %d',\n filename, linen, name, prev)\n\n # Get the start symbol\n def get_start(self):\n self.start = self.pdict.get('start')\n\n # Validate the start symbol\n def validate_start(self):\n if self.start is not None:\n if not isinstance(self.start, string_types):\n self.log.error(\"'start' must be a string\")\n\n # Look for error handler\n def get_error_func(self):\n self.error_func = self.pdict.get('p_error')\n\n # Validate the error function\n def validate_error_func(self):\n if self.error_func:\n if isinstance(self.error_func, types.FunctionType):\n ismethod = 0\n elif isinstance(self.error_func, types.MethodType):\n ismethod = 1\n else:\n self.log.error(\"'p_error' defined, but is not a function or method\")\n self.error = True\n return\n\n eline = self.error_func.__code__.co_firstlineno\n efile = self.error_func.__code__.co_filename\n module = inspect.getmodule(self.error_func)\n self.modules.add(module)\n\n argcount = self.error_func.__code__.co_argcount - ismethod\n if argcount != 1:\n self.log.error('%s:%d: p_error() requires 1 argument', efile, eline)\n self.error = True\n\n # Get the tokens map\n def get_tokens(self):\n tokens = self.pdict.get('tokens')\n if not tokens:\n self.log.error('No token list is defined')\n self.error = True\n return\n\n if not isinstance(tokens, (list, tuple)):\n self.log.error('tokens must be a list or tuple')\n self.error = True\n return\n\n if not tokens:\n self.log.error('tokens is empty')\n self.error = True\n return\n\n self.tokens = sorted(tokens)\n\n # Validate the tokens\n def validate_tokens(self):\n # Validate the tokens.\n if 'error' in self.tokens:\n self.log.error(\"Illegal token name 'error'. Is a reserved word\")\n self.error = True\n return\n\n terminals = set()\n for n in self.tokens:\n if n in terminals:\n self.log.warning('Token %r multiply defined', n)\n terminals.add(n)\n\n # Get the precedence map (if any)\n def get_precedence(self):\n self.prec = self.pdict.get('precedence')\n\n # Validate and parse the precedence map\n def validate_precedence(self):\n preclist = []\n if self.prec:\n if not isinstance(self.prec, (list, tuple)):\n self.log.error('precedence must be a list or tuple')\n self.error = True\n return\n for level, p in enumerate(self.prec):\n if not isinstance(p, (list, tuple)):\n self.log.error('Bad precedence table')\n self.error = True\n return\n\n if len(p) < 2:\n self.log.error('Malformed precedence entry %s. Must be (assoc, term, ..., term)', p)\n self.error = True\n return\n assoc = p[0]\n if not isinstance(assoc, string_types):\n self.log.error('precedence associativity must be a string')\n self.error = True\n return\n for term in p[1:]:\n if not isinstance(term, string_types):\n self.log.error('precedence items must be strings')\n self.error = True\n return\n preclist.append((term, assoc, level+1))\n self.preclist = preclist\n\n # Get all p_functions from the grammar\n def get_pfunctions(self):\n p_functions = []\n for name, item in self.pdict.items():\n if not name.startswith('p_') or name == 'p_error':\n continue\n if isinstance(item, (types.FunctionType, types.MethodType)):\n line = getattr(item, 'co_firstlineno', item.__code__.co_firstlineno)\n module = inspect.getmodule(item)\n p_functions.append((line, module, name, item.__doc__))\n\n # Sort all of the actions by line number; make sure to stringify\n # modules to make them sortable, since `line` may not uniquely sort all\n # p functions\n p_functions.sort(key=lambda p_function: (\n p_function[0],\n str(p_function[1]),\n p_function[2],\n p_function[3]))\n self.pfuncs = p_functions\n\n # Validate all of the p_functions\n def validate_pfunctions(self):\n grammar = []\n # Check for non-empty symbols\n if len(self.pfuncs) == 0:\n self.log.error('no rules of the form p_rulename are defined')\n self.error = True\n return\n\n for line, module, name, doc in self.pfuncs:\n file = inspect.getsourcefile(module)\n func = self.pdict[name]\n if isinstance(func, types.MethodType):\n reqargs = 2\n else:\n reqargs = 1\n if func.__code__.co_argcount > reqargs:\n self.log.error('%s:%d: Rule %r has too many arguments', file, line, func.__name__)\n self.error = True\n elif func.__code__.co_argcount < reqargs:\n self.log.error('%s:%d: Rule %r requires an argument', file, line, func.__name__)\n self.error = True\n elif not func.__doc__:\n self.log.warning('%s:%d: No documentation string specified in function %r (ignored)',\n file, line, func.__name__)\n else:\n try:\n parsed_g = parse_grammar(doc, file, line)\n for g in parsed_g:\n grammar.append((name, g))\n except SyntaxError as e:\n self.log.error(str(e))\n self.error = True\n\n # Looks like a valid grammar rule\n # Mark the file in which defined.\n self.modules.add(module)\n\n # Secondary validation step that looks for p_ definitions that are not functions\n # or functions that look like they might be grammar rules.\n\n for n, v in self.pdict.items():\n if n.startswith('p_') and isinstance(v, (types.FunctionType, types.MethodType)):\n continue\n if n.startswith('t_'):\n continue\n if n.startswith('p_') and n != 'p_error':\n self.log.warning('%r not defined as a function', n)\n if ((isinstance(v, types.FunctionType) and v.__code__.co_argcount == 1) or\n (isinstance(v, types.MethodType) and v.__func__.__code__.co_argcount == 2)):\n if v.__doc__:\n try:\n doc = v.__doc__.split(' ')\n if doc[1] == ':':\n self.log.warning('%s:%d: Possible grammar rule %r defined without p_ prefix',\n v.__code__.co_filename, v.__code__.co_firstlineno, n)\n except IndexError:\n pass\n\n self.grammar = grammar\n\n# -----------------------------------------------------------------------------\n# yacc(module)\n#\n# Build a parser\n# -----------------------------------------------------------------------------\n\ndef yacc(method='LALR', debug=yaccdebug, module=None, tabmodule=tab_module, start=None,\n check_recursion=True, optimize=False, write_tables=True, debugfile=debug_file,\n outputdir=None, debuglog=None, errorlog=None, picklefile=None):\n\n if tabmodule is None:\n tabmodule = tab_module\n\n # Reference to the parsing method of the last built parser\n global parse\n\n # If pickling is enabled, table files are not created\n if picklefile:\n write_tables = 0\n\n if errorlog is None:\n errorlog = PlyLogger(sys.stderr)\n\n # Get the module dictionary used for the parser\n if module:\n _items = [(k, getattr(module, k)) for k in dir(module)]\n pdict = dict(_items)\n # If no __file__ or __package__ attributes are available, try to obtain them\n # from the __module__ instead\n if '__file__' not in pdict:\n pdict['__file__'] = sys.modules[pdict['__module__']].__file__\n if '__package__' not in pdict and '__module__' in pdict:\n if hasattr(sys.modules[pdict['__module__']], '__package__'):\n pdict['__package__'] = sys.modules[pdict['__module__']].__package__\n else:\n pdict = get_caller_module_dict(2)\n\n if outputdir is None:\n # If no output directory is set, the location of the output files\n # is determined according to the following rules:\n # - If tabmodule specifies a package, files go into that package directory\n # - Otherwise, files go in the same directory as the specifying module\n if isinstance(tabmodule, types.ModuleType):\n srcfile = tabmodule.__file__\n else:\n if '.' not in tabmodule:\n srcfile = pdict['__file__']\n else:\n parts = tabmodule.split('.')\n pkgname = '.'.join(parts[:-1])\n exec('import %s' % pkgname)\n srcfile = getattr(sys.modules[pkgname], '__file__', '')\n outputdir = os.path.dirname(srcfile)\n\n # Determine if the module is package of a package or not.\n # If so, fix the tabmodule setting so that tables load correctly\n pkg = pdict.get('__package__')\n if pkg and isinstance(tabmodule, str):\n if '.' not in tabmodule:\n tabmodule = pkg + '.' + tabmodule\n\n\n\n # Set start symbol if it's specified directly using an argument\n if start is not None:\n pdict['start'] = start\n\n # Collect parser information from the dictionary\n pinfo = ParserReflect(pdict, log=errorlog)\n pinfo.get_all()\n\n if pinfo.error:\n raise YaccError('Unable to build parser')\n\n # Check signature against table files (if any)\n signature = pinfo.signature()\n\n # Read the tables\n try:\n lr = LRTable()\n if picklefile:\n read_signature = lr.read_pickle(picklefile)\n else:\n read_signature = lr.read_table(tabmodule)\n if optimize or (read_signature == signature):\n try:\n lr.bind_callables(pinfo.pdict)\n parser = LRParser(lr, pinfo.error_func)\n parse = parser.parse\n return parser\n except Exception as e:\n errorlog.warning('There was a problem loading the table file: %r', e)\n except VersionError as e:\n errorlog.warning(str(e))\n except ImportError:\n pass\n\n if debuglog is None:\n if debug:\n try:\n debuglog = PlyLogger(open(os.path.join(outputdir, debugfile), 'w'))\n except IOError as e:\n errorlog.warning(\"Couldn't open %r. %s\" % (debugfile, e))\n debuglog = NullLogger()\n else:\n debuglog = NullLogger()\n\n debuglog.info('Created by PLY version %s (http://www.dabeaz.com/ply)', __version__)\n\n errors = False\n\n # Validate the parser information\n if pinfo.validate_all():\n raise YaccError('Unable to build parser')\n\n if not pinfo.error_func:\n errorlog.warning('no p_error() function is defined')\n\n # Create a grammar object\n grammar = Grammar(pinfo.tokens)\n\n # Set precedence level for terminals\n for term, assoc, level in pinfo.preclist:\n try:\n grammar.set_precedence(term, assoc, level)\n except GrammarError as e:\n errorlog.warning('%s', e)\n\n # Add productions to the grammar\n for funcname, gram in pinfo.grammar:\n file, line, prodname, syms = gram\n try:\n grammar.add_production(prodname, syms, funcname, file, line)\n except GrammarError as e:\n errorlog.error('%s', e)\n errors = True\n\n # Set the grammar start symbols\n try:\n if start is None:\n grammar.set_start(pinfo.start)\n else:\n grammar.set_start(start)\n except GrammarError as e:\n errorlog.error(str(e))\n errors = True\n\n if errors:\n raise YaccError('Unable to build parser')\n\n # Verify the grammar structure\n undefined_symbols = grammar.undefined_symbols()\n for sym, prod in undefined_symbols:\n errorlog.error('%s:%d: Symbol %r used, but not defined as a token or a rule', prod.file, prod.line, sym)\n errors = True\n\n unused_terminals = grammar.unused_terminals()\n if unused_terminals:\n debuglog.info('')\n debuglog.info('Unused terminals:')\n debuglog.info('')\n for term in unused_terminals:\n errorlog.warning('Token %r defined, but not used', term)\n debuglog.info(' %s', term)\n\n # Print out all productions to the debug log\n if debug:\n debuglog.info('')\n debuglog.info('Grammar')\n debuglog.info('')\n for n, p in enumerate(grammar.Productions):\n debuglog.info('Rule %-5d %s', n, p)\n\n # Find unused non-terminals\n unused_rules = grammar.unused_rules()\n for prod in unused_rules:\n errorlog.warning('%s:%d: Rule %r defined, but not used', prod.file, prod.line, prod.name)\n\n if len(unused_terminals) == 1:\n errorlog.warning('There is 1 unused token')\n if len(unused_terminals) > 1:\n errorlog.warning('There are %d unused tokens', len(unused_terminals))\n\n if len(unused_rules) == 1:\n errorlog.warning('There is 1 unused rule')\n if len(unused_rules) > 1:\n errorlog.warning('There are %d unused rules', len(unused_rules))\n\n if debug:\n debuglog.info('')\n debuglog.info('Terminals, with rules where they appear')\n debuglog.info('')\n terms = list(grammar.Terminals)\n terms.sort()\n for term in terms:\n debuglog.info('%-20s : %s', term, ' '.join([str(s) for s in grammar.Terminals[term]]))\n\n debuglog.info('')\n debuglog.info('Nonterminals, with rules where they appear')\n debuglog.info('')\n nonterms = list(grammar.Nonterminals)\n nonterms.sort()\n for nonterm in nonterms:\n debuglog.info('%-20s : %s', nonterm, ' '.join([str(s) for s in grammar.Nonterminals[nonterm]]))\n debuglog.info('')\n\n if check_recursion:\n unreachable = grammar.find_unreachable()\n for u in unreachable:\n errorlog.warning('Symbol %r is unreachable', u)\n\n infinite = grammar.infinite_cycles()\n for inf in infinite:\n errorlog.error('Infinite recursion detected for symbol %r', inf)\n errors = True\n\n unused_prec = grammar.unused_precedence()\n for term, assoc in unused_prec:\n errorlog.error('Precedence rule %r defined for unknown symbol %r', assoc, term)\n errors = True\n\n if errors:\n raise YaccError('Unable to build parser')\n\n # Run the LRGeneratedTable on the grammar\n if debug:\n errorlog.debug('Generating %s tables', method)\n\n lr = LRGeneratedTable(grammar, method, debuglog)\n\n if debug:\n num_sr = len(lr.sr_conflicts)\n\n # Report shift/reduce and reduce/reduce conflicts\n if num_sr == 1:\n errorlog.warning('1 shift/reduce conflict')\n elif num_sr > 1:\n errorlog.warning('%d shift/reduce conflicts', num_sr)\n\n num_rr = len(lr.rr_conflicts)\n if num_rr == 1:\n errorlog.warning('1 reduce/reduce conflict')\n elif num_rr > 1:\n errorlog.warning('%d reduce/reduce conflicts', num_rr)\n\n # Write out conflicts to the output file\n if debug and (lr.sr_conflicts or lr.rr_conflicts):\n debuglog.warning('')\n debuglog.warning('Conflicts:')\n debuglog.warning('')\n\n for state, tok, resolution in lr.sr_conflicts:\n debuglog.warning('shift/reduce conflict for %s in state %d resolved as %s', tok, state, resolution)\n\n already_reported = set()\n for state, rule, rejected in lr.rr_conflicts:\n if (state, id(rule), id(rejected)) in already_reported:\n continue\n debuglog.warning('reduce/reduce conflict in state %d resolved using rule (%s)', state, rule)\n debuglog.warning('rejected rule (%s) in state %d', rejected, state)\n errorlog.warning('reduce/reduce conflict in state %d resolved using rule (%s)', state, rule)\n errorlog.warning('rejected rule (%s) in state %d', rejected, state)\n already_reported.add((state, id(rule), id(rejected)))\n\n warned_never = []\n for state, rule, rejected in lr.rr_conflicts:\n if not rejected.reduced and (rejected not in warned_never):\n debuglog.warning('Rule (%s) is never reduced', rejected)\n errorlog.warning('Rule (%s) is never reduced', rejected)\n warned_never.append(rejected)\n\n # Write the table file if requested\n if write_tables:\n try:\n lr.write_table(tabmodule, outputdir, signature)\n if tabmodule in sys.modules:\n del sys.modules[tabmodule]\n except IOError as e:\n errorlog.warning(\"Couldn't create %r. %s\" % (tabmodule, e))\n\n # Write a pickled version of the tables\n if picklefile:\n try:\n lr.pickle_table(picklefile, signature)\n except IOError as e:\n errorlog.warning(\"Couldn't create %r. %s\" % (picklefile, e))\n\n # Build the parser\n lr.bind_callables(pinfo.pdict)\n parser = LRParser(lr, pinfo.error_func)\n\n parse = parser.parse\n return parser\n" }, { "path": "xonsh/ply/ply/ygen.py", "content": "# ply: ygen.py\n#\n# This is a support program that auto-generates different versions of the YACC parsing\n# function with different features removed for the purposes of performance.\n#\n# Users should edit the method LRParser.parsedebug() in yacc.py. The source code\n# for that method is then used to create the other methods. See the comments in\n# yacc.py for further details.\n\nimport os.path\nimport shutil\n\ndef get_source_range(lines, tag):\n srclines = enumerate(lines)\n start_tag = '#--! %s-start' % tag\n end_tag = '#--! %s-end' % tag\n\n for start_index, line in srclines:\n if line.strip().startswith(start_tag):\n break\n\n for end_index, line in srclines:\n if line.strip().endswith(end_tag):\n break\n\n return (start_index + 1, end_index)\n\ndef filter_section(lines, tag):\n filtered_lines = []\n include = True\n tag_text = '#--! %s' % tag\n for line in lines:\n if line.strip().startswith(tag_text):\n include = not include\n elif include:\n filtered_lines.append(line)\n return filtered_lines\n\ndef main():\n dirname = os.path.dirname(__file__)\n shutil.copy2(os.path.join(dirname, 'yacc.py'), os.path.join(dirname, 'yacc.py.bak'))\n with open(os.path.join(dirname, 'yacc.py'), 'r') as f:\n lines = f.readlines()\n\n parse_start, parse_end = get_source_range(lines, 'parsedebug')\n parseopt_start, parseopt_end = get_source_range(lines, 'parseopt')\n parseopt_notrack_start, parseopt_notrack_end = get_source_range(lines, 'parseopt-notrack')\n\n # Get the original source\n orig_lines = lines[parse_start:parse_end]\n\n # Filter the DEBUG sections out\n parseopt_lines = filter_section(orig_lines, 'DEBUG')\n\n # Filter the TRACKING sections out\n parseopt_notrack_lines = filter_section(parseopt_lines, 'TRACKING')\n\n # Replace the parser source sections with updated versions\n lines[parseopt_notrack_start:parseopt_notrack_end] = parseopt_notrack_lines\n lines[parseopt_start:parseopt_end] = parseopt_lines\n\n lines = [line.rstrip()+'\\n' for line in lines]\n with open(os.path.join(dirname, 'yacc.py'), 'w') as f:\n f.writelines(lines)\n\n print('Updated yacc.py')\n\nif __name__ == '__main__':\n main()\n" }, { "path": "xonsh/ply/setup.md", "content": "# Maintained, No Package Releases\n\nPLY is maintained software, but no longer produces package releases.\nThere is no `setup.py` file. It is not something that you install\nwith `pip` or a similar tool. You must COPY the necessary code from\nPLY into your project and take ownership of it.\n\nWhy this policy? PLY is a highly specialized tool for expert-level\nprogrammers who are writing parsers and compilers. If you are writing\na compiler, there's a good chance that it's part of a substantially\nlarger project. Managing external dependencies (such as PLY) in such\nprojects is an ongoing challenge. However, the truth of the matter is\nthat PLY just isn't that big. All of the core functionality is\ncontained in just two files. PLY has no external dependencies of its\nown. It changes very rarely. Plus, there are various customizations\nthat you might want to apply to how it works. So, all things equal,\nit's probably better for you to copy it. \n\nBut what about getting all of the latest improvements and bug fixes?\nWhat improvements? PLY is implementing a 1970s-era parsing algorithm.\nIt's not cutting edge. As for bug fixes, you'll know pretty rapidly\nif PLY works for your project or not. If it's working, there's\nliterally no reason to ever upgrade it. Keep using the version of code\nthat you copied. If you think you've found a bug, check back with the\nrepository to see if it's been fixed. Or submit it as an issue so that\nit can be looked at.\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" }, { "path": "xonsh/ply/test/README", "content": "This directory mostly contains tests for various types of error\nconditions. To run:\n\n $ python testlex.py \n $ python testyacc.py \n $ python testcpp.py\n\nThe script 'cleanup.sh' cleans up this directory to its original state.\n" }, { "path": "xonsh/ply/test/calclex.py", "content": "# -----------------------------------------------------------------------------\n# calclex.py\n# -----------------------------------------------------------------------------\nimport sys\n\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\nimport ply.lex as lex\n\ntokens = (\n 'NAME','NUMBER',\n 'PLUS','MINUS','TIMES','DIVIDE','EQUALS',\n 'LPAREN','RPAREN',\n )\n\n# Tokens\n\nt_PLUS = r'\\+'\nt_MINUS = r'-'\nt_TIMES = r'\\*'\nt_DIVIDE = r'/'\nt_EQUALS = r'='\nt_LPAREN = r'\\('\nt_RPAREN = r'\\)'\nt_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*'\n\ndef t_NUMBER(t):\n r'\\d+'\n try:\n t.value = int(t.value)\n except ValueError:\n print(\"Integer value too large %s\" % t.value)\n t.value = 0\n return t\n\nt_ignore = \" \\t\"\n\ndef t_newline(t):\n r'\\n+'\n t.lexer.lineno += t.value.count(\"\\n\")\n \ndef t_error(t):\n print(\"Illegal character '%s'\" % t.value[0])\n t.lexer.skip(1)\n \n# Build the lexer\nlexer = lex.lex()\n\n\n\n" }, { "path": "xonsh/ply/test/cleanup.sh", "content": "#!/bin/sh\n\nrm -rf *~ *.pyc *.pyo *.dif *.out __pycache__\n\n" }, { "path": "xonsh/ply/test/lex_closure.py", "content": "# -----------------------------------------------------------------------------\n# lex_closure.py\n# -----------------------------------------------------------------------------\nimport sys\n\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\nimport ply.lex as lex\n\ntokens = (\n 'NAME','NUMBER',\n 'PLUS','MINUS','TIMES','DIVIDE','EQUALS',\n 'LPAREN','RPAREN',\n )\n\ndef make_calc():\n\n # Tokens\n\n t_PLUS = r'\\+'\n t_MINUS = r'-'\n t_TIMES = r'\\*'\n t_DIVIDE = r'/'\n t_EQUALS = r'='\n t_LPAREN = r'\\('\n t_RPAREN = r'\\)'\n t_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*'\n\n def t_NUMBER(t):\n r'\\d+'\n try:\n t.value = int(t.value)\n except ValueError:\n print(\"Integer value too large %s\" % t.value)\n t.value = 0\n return t\n\n t_ignore = \" \\t\"\n\n def t_newline(t):\n r'\\n+'\n t.lineno += t.value.count(\"\\n\")\n \n def t_error(t):\n print(\"Illegal character '%s'\" % t.value[0])\n t.lexer.skip(1)\n \n # Build the lexer\n return lex.lex()\n\nmake_calc()\nlex.runmain(data=\"3+4\")\n\n\n\n" }, { "path": "xonsh/ply/test/lex_doc1.py", "content": "# lex_doc1.py\n#\n# Missing documentation string\n\nimport sys\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\n\nimport ply.lex as lex\n\ntokens = [\n \"PLUS\",\n \"MINUS\",\n \"NUMBER\",\n ]\n\nt_PLUS = r'\\+'\nt_MINUS = r'-'\ndef t_NUMBER(t):\n pass\n\ndef t_error(t):\n pass\n\nlex.lex()\n\n\n" }, { "path": "xonsh/ply/test/lex_dup1.py", "content": "# lex_dup1.py\n#\n# Duplicated rule specifiers\n\nimport sys\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\n\nimport ply.lex as lex\n\ntokens = [\n \"PLUS\",\n \"MINUS\",\n \"NUMBER\",\n ]\n\nt_PLUS = r'\\+'\nt_MINUS = r'-'\nt_NUMBER = r'\\d+'\n\nt_NUMBER = r'\\d+'\n\ndef t_error(t):\n pass\n\n\n\nlex.lex()\n\n\n" }, { "path": "xonsh/ply/test/lex_dup2.py", "content": "# lex_dup2.py\n#\n# Duplicated rule specifiers\n\nimport sys\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\n\nimport ply.lex as lex\n\ntokens = [\n \"PLUS\",\n \"MINUS\",\n \"NUMBER\",\n ]\n\nt_PLUS = r'\\+'\nt_MINUS = r'-'\ndef t_NUMBER(t):\n r'\\d+'\n pass\n\ndef t_NUMBER(t):\n r'\\d+'\n pass\n\ndef t_error(t):\n pass\n\n\n\nlex.lex()\n\n\n" }, { "path": "xonsh/ply/test/lex_dup3.py", "content": "# lex_dup3.py\n#\n# Duplicated rule specifiers\n\nimport sys\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\n\nimport ply.lex as lex\n\ntokens = [\n \"PLUS\",\n \"MINUS\",\n \"NUMBER\",\n ]\n\nt_PLUS = r'\\+'\nt_MINUS = r'-'\nt_NUMBER = r'\\d+'\n\ndef t_NUMBER(t):\n r'\\d+'\n pass\n\ndef t_error(t):\n pass\n\n\n\nlex.lex()\n\n\n" }, { "path": "xonsh/ply/test/lex_empty.py", "content": "# lex_empty.py\n#\n# No rules defined\n\nimport sys\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\n\nimport ply.lex as lex\n\ntokens = [\n \"PLUS\",\n \"MINUS\",\n \"NUMBER\",\n ]\n\n\n\nlex.lex()\n\n\n" }, { "path": "xonsh/ply/test/lex_error1.py", "content": "# lex_error1.py\n#\n# Missing t_error() rule\n\nimport sys\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\n\nimport ply.lex as lex\n\ntokens = [\n \"PLUS\",\n \"MINUS\",\n \"NUMBER\",\n ]\n\nt_PLUS = r'\\+'\nt_MINUS = r'-'\nt_NUMBER = r'\\d+'\n\n\n\nlex.lex()\n\n\n" }, { "path": "xonsh/ply/test/lex_error2.py", "content": "# lex_error2.py\n#\n# t_error defined, but not function\n\nimport sys\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\n\nimport ply.lex as lex\n\ntokens = [\n \"PLUS\",\n \"MINUS\",\n \"NUMBER\",\n ]\n\nt_PLUS = r'\\+'\nt_MINUS = r'-'\nt_NUMBER = r'\\d+'\n\nt_error = \"foo\"\n\n\n\nlex.lex()\n\n\n" }, { "path": "xonsh/ply/test/lex_error3.py", "content": "# lex_error3.py\n#\n# t_error defined as function, but with wrong # args\n\nimport sys\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\n\nimport ply.lex as lex\n\ntokens = [\n \"PLUS\",\n \"MINUS\",\n \"NUMBER\",\n ]\n\nt_PLUS = r'\\+'\nt_MINUS = r'-'\nt_NUMBER = r'\\d+'\n\ndef t_error():\n pass\n\n\n\nlex.lex()\n\n\n" }, { "path": "xonsh/ply/test/lex_error4.py", "content": "# lex_error4.py\n#\n# t_error defined as function, but too many args\n\nimport sys\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\n\nimport ply.lex as lex\n\ntokens = [\n \"PLUS\",\n \"MINUS\",\n \"NUMBER\",\n ]\n\nt_PLUS = r'\\+'\nt_MINUS = r'-'\nt_NUMBER = r'\\d+'\n\ndef t_error(t,s):\n pass\n\n\n\nlex.lex()\n\n\n" }, { "path": "xonsh/ply/test/lex_hedit.py", "content": "# -----------------------------------------------------------------------------\n# hedit.py\n#\n# Paring of Fortran H Edit descriptions (Contributed by Pearu Peterson)\n#\n# These tokens can't be easily tokenized because they are of the following\n# form:\n#\n# nHc1...cn\n#\n# where n is a positive integer and c1 ... cn are characters.\n#\n# This example shows how to modify the state of the lexer to parse\n# such tokens\n# -----------------------------------------------------------------------------\nimport sys\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\n\nimport ply.lex as lex\n\ntokens = (\n 'H_EDIT_DESCRIPTOR',\n )\n\n# Tokens\nt_ignore = \" \\t\\n\"\n\ndef t_H_EDIT_DESCRIPTOR(t):\n r\"\\d+H.*\" # This grabs all of the remaining text\n i = t.value.index('H')\n n = eval(t.value[:i])\n \n # Adjust the tokenizing position\n t.lexer.lexpos -= len(t.value) - (i+1+n)\n t.value = t.value[i+1:i+1+n]\n return t \n \ndef t_error(t):\n print(\"Illegal character '%s'\" % t.value[0])\n t.lexer.skip(1)\n \n# Build the lexer\nlex.lex()\nlex.runmain(data=\"3Habc 10Habcdefghij 2Hxy\")\n\n\n\n" }, { "path": "xonsh/ply/test/lex_ignore.py", "content": "# lex_ignore.py\n#\n# Improperly specific ignore declaration\n\nimport sys\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\n\nimport ply.lex as lex\n\ntokens = [\n \"PLUS\",\n \"MINUS\",\n \"NUMBER\",\n ]\n\nt_PLUS = r'\\+'\nt_MINUS = r'-'\nt_NUMBER = r'\\d+'\n\ndef t_ignore(t):\n ' \\t'\n pass\n\ndef t_error(t):\n pass\n\nimport sys\n\nlex.lex()\n\n\n" }, { "path": "xonsh/ply/test/lex_ignore2.py", "content": "# lex_ignore2.py\n#\n# ignore declaration as a raw string\n\nimport sys\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\n\nimport ply.lex as lex\n\ntokens = [\n \"PLUS\",\n \"MINUS\",\n \"NUMBER\",\n ]\n\nt_PLUS = r'\\+'\nt_MINUS = r'-'\nt_NUMBER = r'\\d+'\n\nt_ignore = r' \\t'\n\ndef t_error(t):\n pass\n\n\n\nlex.lex()\n\n\n" }, { "path": "xonsh/ply/test/lex_literal1.py", "content": "# lex_literal1.py\n#\n# Bad literal specification\n\nimport sys\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\n\nimport ply.lex as lex\n\ntokens = [\n \"NUMBER\",\n ]\n\nliterals = [\"+\",\"-\",\"**\"]\n\ndef t_NUMBER(t):\n r'\\d+'\n return t\n\ndef t_error(t):\n pass\n\nlex.lex()\n\n\n" }, { "path": "xonsh/ply/test/lex_literal2.py", "content": "# lex_literal2.py\n#\n# Bad literal specification\n\nimport sys\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\n\nimport ply.lex as lex\n\ntokens = [\n \"NUMBER\",\n ]\n\nliterals = 23\n\ndef t_NUMBER(t):\n r'\\d+'\n return t\n\ndef t_error(t):\n pass\n\nlex.lex()\n\n\n" }, { "path": "xonsh/ply/test/lex_literal3.py", "content": "# lex_literal3.py\n#\n# An empty literal specification given as a list\n# Issue 8 : Literals empty list causes IndexError\n\nimport sys\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\n\nimport ply.lex as lex\n\ntokens = [\n \"NUMBER\",\n ]\n\nliterals = []\n\ndef t_NUMBER(t):\n r'\\d+'\n return t\n\ndef t_error(t):\n pass\n\nlex.lex()\n\n\n" }, { "path": "xonsh/ply/test/lex_many_tokens.py", "content": "# lex_many_tokens.py\n#\n# Test lex's ability to handle a large number of tokens (beyond the\n# 100-group limit of the re module)\n\nimport sys\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\n\nimport ply.lex as lex\n\ntokens = [\"TOK%d\" % i for i in range(1000)]\n\nfor tok in tokens:\n if sys.version_info[0] < 3:\n exec(\"t_%s = '%s:'\" % (tok,tok))\n else:\n exec(\"t_%s = '%s:'\" % (tok,tok), globals())\n\nt_ignore = \" \\t\"\n\ndef t_error(t):\n pass\n\nlex.lex(optimize=1,lextab=\"manytab\")\nlex.runmain(data=\"TOK34: TOK143: TOK269: TOK372: TOK452: TOK561: TOK999:\")\n\n\n" }, { "path": "xonsh/ply/test/lex_module.py", "content": "# lex_module.py\n#\n\nimport sys\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\n\nimport ply.lex as lex\nimport lex_module_import\nlex.lex(module=lex_module_import)\nlex.runmain(data=\"3+4\")\n" }, { "path": "xonsh/ply/test/lex_module_import.py", "content": "# -----------------------------------------------------------------------------\n# lex_module_import.py\n#\n# A lexer defined in a module, but built in lex_module.py\n# -----------------------------------------------------------------------------\n\ntokens = (\n 'NAME','NUMBER',\n 'PLUS','MINUS','TIMES','DIVIDE','EQUALS',\n 'LPAREN','RPAREN',\n )\n\n# Tokens\n\nt_PLUS = r'\\+'\nt_MINUS = r'-'\nt_TIMES = r'\\*'\nt_DIVIDE = r'/'\nt_EQUALS = r'='\nt_LPAREN = r'\\('\nt_RPAREN = r'\\)'\nt_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*'\n\ndef t_NUMBER(t):\n r'\\d+'\n try:\n t.value = int(t.value)\n except ValueError:\n print(\"Integer value too large %s\" % t.value)\n t.value = 0\n return t\n\nt_ignore = \" \\t\"\n\ndef t_newline(t):\n r'\\n+'\n t.lineno += t.value.count(\"\\n\")\n \ndef t_error(t):\n print(\"Illegal character '%s'\" % t.value[0])\n t.lexer.skip(1)\n \n" }, { "path": "xonsh/ply/test/lex_object.py", "content": "# -----------------------------------------------------------------------------\n# lex_object.py\n# -----------------------------------------------------------------------------\nimport sys\n\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\nimport ply.lex as lex\n\nclass CalcLexer:\n tokens = (\n 'NAME','NUMBER',\n 'PLUS','MINUS','TIMES','DIVIDE','EQUALS',\n 'LPAREN','RPAREN',\n )\n\n # Tokens\n\n t_PLUS = r'\\+'\n t_MINUS = r'-'\n t_TIMES = r'\\*'\n t_DIVIDE = r'/'\n t_EQUALS = r'='\n t_LPAREN = r'\\('\n t_RPAREN = r'\\)'\n t_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*'\n\n def t_NUMBER(self,t):\n r'\\d+'\n try:\n t.value = int(t.value)\n except ValueError:\n print(\"Integer value too large %s\" % t.value)\n t.value = 0\n return t\n\n t_ignore = \" \\t\"\n\n def t_newline(self,t):\n r'\\n+'\n t.lineno += t.value.count(\"\\n\")\n \n def t_error(self,t):\n print(\"Illegal character '%s'\" % t.value[0])\n t.lexer.skip(1)\n \n\ncalc = CalcLexer()\n\n# Build the lexer\nlex.lex(object=calc)\nlex.runmain(data=\"3+4\")\n\n\n\n\n" }, { "path": "xonsh/ply/test/lex_opt_alias.py", "content": "# -----------------------------------------------------------------------------\n# lex_opt_alias.py\n#\n# Tests ability to match up functions with states, aliases, and\n# lexing tables.\n# -----------------------------------------------------------------------------\n\nimport sys\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\n\ntokens = (\n 'NAME','NUMBER',\n )\n\nstates = (('instdef','inclusive'),('spam','exclusive'))\n\nliterals = ['=','+','-','*','/', '(',')']\n\n# Tokens\n\ndef t_instdef_spam_BITS(t):\n r'[01-]+'\n return t\n\nt_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*'\n\ndef NUMBER(t):\n r'\\d+'\n try:\n t.value = int(t.value)\n except ValueError:\n print(\"Integer value too large %s\" % t.value)\n t.value = 0\n return t\n\nt_ANY_NUMBER = NUMBER\n\nt_ignore = \" \\t\"\nt_spam_ignore = t_ignore\n\ndef t_newline(t):\n r'\\n+'\n t.lexer.lineno += t.value.count(\"\\n\")\n \ndef t_error(t):\n print(\"Illegal character '%s'\" % t.value[0])\n t.lexer.skip(1)\n\nt_spam_error = t_error\n\n# Build the lexer\nimport ply.lex as lex\nlex.lex(optimize=1,lextab=\"aliastab\")\nlex.runmain(data=\"3+4\")\n" }, { "path": "xonsh/ply/test/lex_optimize.py", "content": "# -----------------------------------------------------------------------------\n# lex_optimize.py\n# -----------------------------------------------------------------------------\nimport sys\n\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\nimport ply.lex as lex\n\ntokens = (\n 'NAME','NUMBER',\n 'PLUS','MINUS','TIMES','DIVIDE','EQUALS',\n 'LPAREN','RPAREN',\n )\n\n# Tokens\n\nt_PLUS = r'\\+'\nt_MINUS = r'-'\nt_TIMES = r'\\*'\nt_DIVIDE = r'/'\nt_EQUALS = r'='\nt_LPAREN = r'\\('\nt_RPAREN = r'\\)'\nt_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*'\n\ndef t_NUMBER(t):\n r'\\d+'\n try:\n t.value = int(t.value)\n except ValueError:\n print(\"Integer value too large %s\" % t.value)\n t.value = 0\n return t\n\nt_ignore = \" \\t\"\n\ndef t_newline(t):\n r'\\n+'\n t.lineno += t.value.count(\"\\n\")\n \ndef t_error(t):\n print(\"Illegal character '%s'\" % t.value[0])\n t.lexer.skip(1)\n \n# Build the lexer\nlex.lex(optimize=1)\nlex.runmain(data=\"3+4\")\n\n\n\n" }, { "path": "xonsh/ply/test/lex_optimize2.py", "content": "# -----------------------------------------------------------------------------\n# lex_optimize2.py\n# -----------------------------------------------------------------------------\nimport sys\n\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\nimport ply.lex as lex\n\ntokens = (\n 'NAME','NUMBER',\n 'PLUS','MINUS','TIMES','DIVIDE','EQUALS',\n 'LPAREN','RPAREN',\n )\n\n# Tokens\n\nt_PLUS = r'\\+'\nt_MINUS = r'-'\nt_TIMES = r'\\*'\nt_DIVIDE = r'/'\nt_EQUALS = r'='\nt_LPAREN = r'\\('\nt_RPAREN = r'\\)'\nt_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*'\n\ndef t_NUMBER(t):\n r'\\d+'\n try:\n t.value = int(t.value)\n except ValueError:\n print(\"Integer value too large %s\" % t.value)\n t.value = 0\n return t\n\nt_ignore = \" \\t\"\n\ndef t_newline(t):\n r'\\n+'\n t.lineno += t.value.count(\"\\n\")\n \ndef t_error(t):\n print(\"Illegal character '%s'\" % t.value[0])\n t.lexer.skip(1)\n \n# Build the lexer\nlex.lex(optimize=1,lextab=\"opt2tab\")\nlex.runmain(data=\"3+4\")\n\n\n\n" }, { "path": "xonsh/ply/test/lex_optimize3.py", "content": "# -----------------------------------------------------------------------------\n# lex_optimize3.py\n#\n# Writes table in a subdirectory structure.\n# -----------------------------------------------------------------------------\nimport sys\n\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\nimport ply.lex as lex\n\ntokens = (\n 'NAME','NUMBER',\n 'PLUS','MINUS','TIMES','DIVIDE','EQUALS',\n 'LPAREN','RPAREN',\n )\n\n# Tokens\n\nt_PLUS = r'\\+'\nt_MINUS = r'-'\nt_TIMES = r'\\*'\nt_DIVIDE = r'/'\nt_EQUALS = r'='\nt_LPAREN = r'\\('\nt_RPAREN = r'\\)'\nt_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*'\n\ndef t_NUMBER(t):\n r'\\d+'\n try:\n t.value = int(t.value)\n except ValueError:\n print(\"Integer value too large %s\" % t.value)\n t.value = 0\n return t\n\nt_ignore = \" \\t\"\n\ndef t_newline(t):\n r'\\n+'\n t.lineno += t.value.count(\"\\n\")\n \ndef t_error(t):\n print(\"Illegal character '%s'\" % t.value[0])\n t.lexer.skip(1)\n \n# Build the lexer\nlex.lex(optimize=1,lextab=\"lexdir.sub.calctab\" ,outputdir=\"lexdir/sub\")\nlex.runmain(data=\"3+4\")\n\n\n\n" }, { "path": "xonsh/ply/test/lex_optimize4.py", "content": "# -----------------------------------------------------------------------------\n# lex_optimize4.py\n# -----------------------------------------------------------------------------\nimport re\nimport sys\n\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\nimport ply.lex as lex\n\ntokens = [\n \"PLUS\",\n \"MINUS\",\n \"NUMBER\",\n ]\n\nt_PLUS = r'\\+?'\nt_MINUS = r'-'\nt_NUMBER = r'(\\d+)'\n\ndef t_error(t):\n pass\n\n\n# Build the lexer\nlex.lex(optimize=True, lextab=\"opt4tab\", reflags=re.UNICODE)\nlex.runmain(data=\"3+4\")\n" }, { "path": "xonsh/ply/test/lex_re1.py", "content": "# lex_re1.py\n#\n# Bad regular expression in a string\n\nimport sys\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\n\nimport ply.lex as lex\n\ntokens = [\n \"PLUS\",\n \"MINUS\",\n \"NUMBER\",\n ]\n\nt_PLUS = r'\\+'\nt_MINUS = r'-'\nt_NUMBER = r'(\\d+'\n\ndef t_error(t):\n pass\n\n\n\nlex.lex()\n\n\n" }, { "path": "xonsh/ply/test/lex_re2.py", "content": "# lex_re2.py\n#\n# Regular expression rule matches empty string\n\nimport sys\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\n\nimport ply.lex as lex\n\ntokens = [\n \"PLUS\",\n \"MINUS\",\n \"NUMBER\",\n ]\n\nt_PLUS = r'\\+?'\nt_MINUS = r'-'\nt_NUMBER = r'(\\d+)'\n\ndef t_error(t):\n pass\n\n\n\nlex.lex()\n\n\n" }, { "path": "xonsh/ply/test/lex_re3.py", "content": "# lex_re3.py\n#\n# Regular expression rule matches empty string\n\nimport sys\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\n\nimport ply.lex as lex\n\ntokens = [\n \"PLUS\",\n \"MINUS\",\n \"NUMBER\",\n \"POUND\",\n ]\n\nt_PLUS = r'\\+'\nt_MINUS = r'-'\nt_NUMBER = r'(\\d+)'\nt_POUND = r'#'\n\ndef t_error(t):\n pass\n\n\n\nlex.lex()\n\n\n" }, { "path": "xonsh/ply/test/lex_rule1.py", "content": "# lex_rule1.py\n#\n# Rule function with incorrect number of arguments\n\nimport sys\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\n\nimport ply.lex as lex\n\ntokens = [\n \"PLUS\",\n \"MINUS\",\n \"NUMBER\",\n ]\n\nt_PLUS = r'\\+'\nt_MINUS = r'-'\nt_NUMBER = 1\n\ndef t_error(t):\n pass\n\n\n\nlex.lex()\n\n\n" }, { "path": "xonsh/ply/test/lex_rule2.py", "content": "# lex_rule2.py\n#\n# Rule function with incorrect number of arguments\n\nimport sys\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\n\nimport ply.lex as lex\n\ntokens = [\n \"PLUS\",\n \"MINUS\",\n \"NUMBER\",\n ]\n\nt_PLUS = r'\\+'\nt_MINUS = r'-'\ndef t_NUMBER():\n r'\\d+'\n return t\n\ndef t_error(t):\n pass\n\n\n\nlex.lex()\n\n\n" }, { "path": "xonsh/ply/test/lex_rule3.py", "content": "# lex_rule3.py\n#\n# Rule function with incorrect number of arguments\n\nimport sys\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\n\nimport ply.lex as lex\n\ntokens = [\n \"PLUS\",\n \"MINUS\",\n \"NUMBER\",\n ]\n\nt_PLUS = r'\\+'\nt_MINUS = r'-'\ndef t_NUMBER(t,s):\n r'\\d+'\n return t\n\ndef t_error(t):\n pass\n\nlex.lex()\n\n\n" }, { "path": "xonsh/ply/test/lex_state1.py", "content": "# lex_state1.py\n#\n# Bad state declaration\n\nimport sys\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\n\nimport ply.lex as lex\n\ntokens = [ \n \"PLUS\",\n \"MINUS\",\n \"NUMBER\",\n ]\n\nstates = 'comment'\n\nt_PLUS = r'\\+'\nt_MINUS = r'-'\nt_NUMBER = r'\\d+'\n\n# Comments\ndef t_comment(t):\n r'/\\*'\n t.lexer.begin('comment')\n print(\"Entering comment state\")\n\ndef t_comment_body_part(t):\n r'(.|\\n)*\\*/'\n print(\"comment body %s\" % t)\n t.lexer.begin('INITIAL')\n\ndef t_error(t):\n pass\n\n\n\nlex.lex()\n\n\n" }, { "path": "xonsh/ply/test/lex_state2.py", "content": "# lex_state2.py\n#\n# Bad state declaration\n\nimport sys\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\n\nimport ply.lex as lex\n\ntokens = [ \n \"PLUS\",\n \"MINUS\",\n \"NUMBER\",\n ]\n\nstates = ('comment','example')\n\nt_PLUS = r'\\+'\nt_MINUS = r'-'\nt_NUMBER = r'\\d+'\n\n# Comments\ndef t_comment(t):\n r'/\\*'\n t.lexer.begin('comment')\n print(\"Entering comment state\")\n\ndef t_comment_body_part(t):\n r'(.|\\n)*\\*/'\n print(\"comment body %s\" % t)\n t.lexer.begin('INITIAL')\n\ndef t_error(t):\n pass\n\n\n\nlex.lex()\n\n\n" }, { "path": "xonsh/ply/test/lex_state3.py", "content": "# lex_state3.py\n#\n# Bad state declaration\n\nimport sys\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\n\nimport ply.lex as lex\n\ntokens = [ \n \"PLUS\",\n \"MINUS\",\n \"NUMBER\",\n ]\n\ncomment = 1\nstates = ((comment, 'inclusive'),\n ('example', 'exclusive'))\n\nt_PLUS = r'\\+'\nt_MINUS = r'-'\nt_NUMBER = r'\\d+'\n\n# Comments\ndef t_comment(t):\n r'/\\*'\n t.lexer.begin('comment')\n print(\"Entering comment state\")\n\ndef t_comment_body_part(t):\n r'(.|\\n)*\\*/'\n print(\"comment body %s\" % t)\n t.lexer.begin('INITIAL')\n\ndef t_error(t):\n pass\n\n\n\nlex.lex()\n\n\n" }, { "path": "xonsh/ply/test/lex_state4.py", "content": "# lex_state4.py\n#\n# Bad state declaration\n\nimport sys\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\n\nimport ply.lex as lex\n\ntokens = [ \n \"PLUS\",\n \"MINUS\",\n \"NUMBER\",\n ]\n\n\nstates = (('comment', 'exclsive'),)\n\nt_PLUS = r'\\+'\nt_MINUS = r'-'\nt_NUMBER = r'\\d+'\n\n# Comments\ndef t_comment(t):\n r'/\\*'\n t.lexer.begin('comment')\n print(\"Entering comment state\")\n\ndef t_comment_body_part(t):\n r'(.|\\n)*\\*/'\n print(\"comment body %s\" % t)\n t.lexer.begin('INITIAL')\n\ndef t_error(t):\n pass\n\n\n\nlex.lex()\n\n\n" }, { "path": "xonsh/ply/test/lex_state5.py", "content": "# lex_state5.py\n#\n# Bad state declaration\n\nimport sys\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\n\nimport ply.lex as lex\n\ntokens = [ \n \"PLUS\",\n \"MINUS\",\n \"NUMBER\",\n ]\n\nstates = (('comment', 'exclusive'),\n ('comment', 'exclusive'))\n\nt_PLUS = r'\\+'\nt_MINUS = r'-'\nt_NUMBER = r'\\d+'\n\n# Comments\ndef t_comment(t):\n r'/\\*'\n t.lexer.begin('comment')\n print(\"Entering comment state\")\n\ndef t_comment_body_part(t):\n r'(.|\\n)*\\*/'\n print(\"comment body %s\" % t)\n t.lexer.begin('INITIAL')\n\ndef t_error(t):\n pass\n\n\nlex.lex()\n\n\n" }, { "path": "xonsh/ply/test/lex_state_noerror.py", "content": "# lex_state_noerror.py\n#\n# Declaration of a state for which no rules are defined\n\nimport sys\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\n\nimport ply.lex as lex\n\ntokens = [ \n \"PLUS\",\n \"MINUS\",\n \"NUMBER\",\n ]\n\nstates = (('comment', 'exclusive'),)\n\nt_PLUS = r'\\+'\nt_MINUS = r'-'\nt_NUMBER = r'\\d+'\n\n# Comments\ndef t_comment(t):\n r'/\\*'\n t.lexer.begin('comment')\n print(\"Entering comment state\")\n\ndef t_comment_body_part(t):\n r'(.|\\n)*\\*/'\n print(\"comment body %s\" % t)\n t.lexer.begin('INITIAL')\n\ndef t_error(t):\n pass\n\n\nlex.lex()\n\n\n" }, { "path": "xonsh/ply/test/lex_state_norule.py", "content": "# lex_state_norule.py\n#\n# Declaration of a state for which no rules are defined\n\nimport sys\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\n\nimport ply.lex as lex\n\ntokens = [ \n \"PLUS\",\n \"MINUS\",\n \"NUMBER\",\n ]\n\nstates = (('comment', 'exclusive'),\n ('example', 'exclusive'))\n\nt_PLUS = r'\\+'\nt_MINUS = r'-'\nt_NUMBER = r'\\d+'\n\n# Comments\ndef t_comment(t):\n r'/\\*'\n t.lexer.begin('comment')\n print(\"Entering comment state\")\n\ndef t_comment_body_part(t):\n r'(.|\\n)*\\*/'\n print(\"comment body %s\" % t)\n t.lexer.begin('INITIAL')\n\ndef t_error(t):\n pass\n\n\nlex.lex()\n\n\n" }, { "path": "xonsh/ply/test/lex_state_try.py", "content": "# lex_state_try.py\n#\n# Declaration of a state for which no rules are defined\n\nimport sys\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\n\nimport ply.lex as lex\n\ntokens = [ \n \"PLUS\",\n \"MINUS\",\n \"NUMBER\",\n ]\n\nstates = (('comment', 'exclusive'),)\n\nt_PLUS = r'\\+'\nt_MINUS = r'-'\nt_NUMBER = r'\\d+'\n\nt_ignore = \" \\t\"\n\n# Comments\ndef t_comment(t):\n r'/\\*'\n t.lexer.begin('comment')\n print(\"Entering comment state\")\n\ndef t_comment_body_part(t):\n r'(.|\\n)*\\*/'\n print(\"comment body %s\" % t)\n t.lexer.begin('INITIAL')\n\ndef t_error(t):\n pass\n\nt_comment_error = t_error\nt_comment_ignore = t_ignore\n\nlex.lex()\n\ndata = \"3 + 4 /* This is a comment */ + 10\"\n\nlex.runmain(data=data)\n" }, { "path": "xonsh/ply/test/lex_token1.py", "content": "# lex_token1.py\n#\n# Tests for absence of tokens variable\n\nimport sys\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\n\nimport ply.lex as lex\n\nt_PLUS = r'\\+'\nt_MINUS = r'-'\nt_NUMBER = r'\\d+'\n\ndef t_error(t):\n pass\n\nlex.lex()\n\n\n" }, { "path": "xonsh/ply/test/lex_token2.py", "content": "# lex_token2.py\n#\n# Tests for tokens of wrong type\n\nimport sys\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\n\nimport ply.lex as lex\n\ntokens = \"PLUS MINUS NUMBER\"\n\nt_PLUS = r'\\+'\nt_MINUS = r'-'\nt_NUMBER = r'\\d+'\n\ndef t_error(t):\n pass\n\n\nlex.lex()\n\n\n" }, { "path": "xonsh/ply/test/lex_token3.py", "content": "# lex_token3.py\n#\n# tokens is right type, but is missing a token for one rule\n\nimport sys\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\n\nimport ply.lex as lex\n\ntokens = [\n \"PLUS\",\n \"NUMBER\",\n ]\n\nt_PLUS = r'\\+'\nt_MINUS = r'-'\nt_NUMBER = r'\\d+'\n\ndef t_error(t):\n pass\n\nlex.lex()\n\n\n" }, { "path": "xonsh/ply/test/lex_token4.py", "content": "# lex_token4.py\n#\n# Bad token name\n\nimport sys\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\n\nimport ply.lex as lex\n\ntokens = [\n \"PLUS\",\n \"MINUS\",\n \"-\",\n \"NUMBER\",\n ]\n\nt_PLUS = r'\\+'\nt_MINUS = r'-'\nt_NUMBER = r'\\d+'\n\ndef t_error(t):\n pass\n\nlex.lex()\n\n\n" }, { "path": "xonsh/ply/test/lex_token5.py", "content": "# lex_token5.py\n#\n# Return a bad token name\n\nimport sys\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\n\nimport ply.lex as lex\n\ntokens = [\n \"PLUS\",\n \"MINUS\",\n \"NUMBER\",\n ]\n\nt_PLUS = r'\\+'\nt_MINUS = r'-'\n\ndef t_NUMBER(t):\n r'\\d+'\n t.type = \"NUM\"\n return t\n\ndef t_error(t):\n pass\n\nlex.lex()\nlex.input(\"1234\")\nt = lex.token()\n\n\n" }, { "path": "xonsh/ply/test/lex_token_dup.py", "content": "# lex_token_dup.py\n#\n# Duplicate token name in tokens\n\nimport sys\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\n\nimport ply.lex as lex\n\ntokens = [\n \"PLUS\",\n \"MINUS\",\n \"NUMBER\",\n \"MINUS\"\n ]\n\nt_PLUS = r'\\+'\nt_MINUS = r'-'\n\ndef t_NUMBER(t):\n r'\\d+'\n return t\n\ndef t_error(t):\n pass\n\nlex.lex()\n\n\n" }, { "path": "xonsh/ply/test/pkg_test1/__init__.py", "content": "# Tests proper handling of lextab and parsetab files in package structures\n\n# Here for testing purposes\nimport sys\nif '..' not in sys.path: \n sys.path.insert(0, '..')\n\nfrom .parsing.calcparse import parser\n\n" }, { "path": "xonsh/ply/test/pkg_test1/parsing/__init__.py", "content": "" }, { "path": "xonsh/ply/test/pkg_test1/parsing/calclex.py", "content": "# -----------------------------------------------------------------------------\n# calclex.py\n# -----------------------------------------------------------------------------\n\nimport ply.lex as lex\n\ntokens = (\n 'NAME','NUMBER',\n 'PLUS','MINUS','TIMES','DIVIDE','EQUALS',\n 'LPAREN','RPAREN',\n )\n\n# Tokens\n\nt_PLUS = r'\\+'\nt_MINUS = r'-'\nt_TIMES = r'\\*'\nt_DIVIDE = r'/'\nt_EQUALS = r'='\nt_LPAREN = r'\\('\nt_RPAREN = r'\\)'\nt_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*'\n\ndef t_NUMBER(t):\n r'\\d+'\n try:\n t.value = int(t.value)\n except ValueError:\n print(\"Integer value too large %s\" % t.value)\n t.value = 0\n return t\n\nt_ignore = \" \\t\"\n\ndef t_newline(t):\n r'\\n+'\n t.lexer.lineno += t.value.count(\"\\n\")\n \ndef t_error(t):\n print(\"Illegal character '%s'\" % t.value[0])\n t.lexer.skip(1)\n \n# Build the lexer\nlexer = lex.lex(optimize=True)\n\n\n\n" }, { "path": "xonsh/ply/test/pkg_test1/parsing/calcparse.py", "content": "# -----------------------------------------------------------------------------\n# yacc_simple.py\n#\n# A simple, properly specifier grammar\n# -----------------------------------------------------------------------------\n\nfrom .calclex import tokens\nfrom ply import yacc\n\n# Parsing rules\nprecedence = (\n ('left','PLUS','MINUS'),\n ('left','TIMES','DIVIDE'),\n ('right','UMINUS'),\n )\n\n# dictionary of names\nnames = { }\n\ndef p_statement_assign(t):\n 'statement : NAME EQUALS expression'\n names[t[1]] = t[3]\n\ndef p_statement_expr(t):\n 'statement : expression'\n t[0] = t[1]\n\ndef p_expression_binop(t):\n '''expression : expression PLUS expression\n | expression MINUS expression\n | expression TIMES expression\n | expression DIVIDE expression'''\n if t[2] == '+' : t[0] = t[1] + t[3]\n elif t[2] == '-': t[0] = t[1] - t[3]\n elif t[2] == '*': t[0] = t[1] * t[3]\n elif t[2] == '/': t[0] = t[1] / t[3]\n\ndef p_expression_uminus(t):\n 'expression : MINUS expression %prec UMINUS'\n t[0] = -t[2]\n\ndef p_expression_group(t):\n 'expression : LPAREN expression RPAREN'\n t[0] = t[2]\n\ndef p_expression_number(t):\n 'expression : NUMBER'\n t[0] = t[1]\n\ndef p_expression_name(t):\n 'expression : NAME'\n try:\n t[0] = names[t[1]]\n except LookupError:\n print(\"Undefined name '%s'\" % t[1])\n t[0] = 0\n\ndef p_error(t):\n print(\"Syntax error at '%s'\" % t.value)\n\nparser = yacc.yacc()\n\n\n\n\n\n" }, { "path": "xonsh/ply/test/pkg_test2/__init__.py", "content": "# Tests proper handling of lextab and parsetab files in package structures\n\n# Here for testing purposes\nimport sys\nif '..' not in sys.path: \n sys.path.insert(0, '..')\n\nfrom .parsing.calcparse import parser\n\n" }, { "path": "xonsh/ply/test/pkg_test2/parsing/__init__.py", "content": "" }, { "path": "xonsh/ply/test/pkg_test2/parsing/calclex.py", "content": "# -----------------------------------------------------------------------------\n# calclex.py\n# -----------------------------------------------------------------------------\n\nimport ply.lex as lex\n\ntokens = (\n 'NAME','NUMBER',\n 'PLUS','MINUS','TIMES','DIVIDE','EQUALS',\n 'LPAREN','RPAREN',\n )\n\n# Tokens\n\nt_PLUS = r'\\+'\nt_MINUS = r'-'\nt_TIMES = r'\\*'\nt_DIVIDE = r'/'\nt_EQUALS = r'='\nt_LPAREN = r'\\('\nt_RPAREN = r'\\)'\nt_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*'\n\ndef t_NUMBER(t):\n r'\\d+'\n try:\n t.value = int(t.value)\n except ValueError:\n print(\"Integer value too large %s\" % t.value)\n t.value = 0\n return t\n\nt_ignore = \" \\t\"\n\ndef t_newline(t):\n r'\\n+'\n t.lexer.lineno += t.value.count(\"\\n\")\n \ndef t_error(t):\n print(\"Illegal character '%s'\" % t.value[0])\n t.lexer.skip(1)\n \n# Build the lexer\nlexer = lex.lex(optimize=True, lextab='calclextab')\n\n\n\n" }, { "path": "xonsh/ply/test/pkg_test2/parsing/calcparse.py", "content": "# -----------------------------------------------------------------------------\n# yacc_simple.py\n#\n# A simple, properly specifier grammar\n# -----------------------------------------------------------------------------\n\nfrom .calclex import tokens\nfrom ply import yacc\n\n# Parsing rules\nprecedence = (\n ('left','PLUS','MINUS'),\n ('left','TIMES','DIVIDE'),\n ('right','UMINUS'),\n )\n\n# dictionary of names\nnames = { }\n\ndef p_statement_assign(t):\n 'statement : NAME EQUALS expression'\n names[t[1]] = t[3]\n\ndef p_statement_expr(t):\n 'statement : expression'\n t[0] = t[1]\n\ndef p_expression_binop(t):\n '''expression : expression PLUS expression\n | expression MINUS expression\n | expression TIMES expression\n | expression DIVIDE expression'''\n if t[2] == '+' : t[0] = t[1] + t[3]\n elif t[2] == '-': t[0] = t[1] - t[3]\n elif t[2] == '*': t[0] = t[1] * t[3]\n elif t[2] == '/': t[0] = t[1] / t[3]\n\ndef p_expression_uminus(t):\n 'expression : MINUS expression %prec UMINUS'\n t[0] = -t[2]\n\ndef p_expression_group(t):\n 'expression : LPAREN expression RPAREN'\n t[0] = t[2]\n\ndef p_expression_number(t):\n 'expression : NUMBER'\n t[0] = t[1]\n\ndef p_expression_name(t):\n 'expression : NAME'\n try:\n t[0] = names[t[1]]\n except LookupError:\n print(\"Undefined name '%s'\" % t[1])\n t[0] = 0\n\ndef p_error(t):\n print(\"Syntax error at '%s'\" % t.value)\n\nparser = yacc.yacc(tabmodule='calcparsetab')\n\n\n\n\n\n" }, { "path": "xonsh/ply/test/pkg_test3/__init__.py", "content": "# Tests proper handling of lextab and parsetab files in package structures\n\n# Here for testing purposes\nimport sys\nif '..' not in sys.path: \n sys.path.insert(0, '..')\n\nfrom .parsing.calcparse import parser\n\n" }, { "path": "xonsh/ply/test/pkg_test3/generated/__init__.py", "content": "" }, { "path": "xonsh/ply/test/pkg_test3/parsing/__init__.py", "content": "" }, { "path": "xonsh/ply/test/pkg_test3/parsing/calclex.py", "content": "# -----------------------------------------------------------------------------\n# calclex.py\n# -----------------------------------------------------------------------------\n\nimport ply.lex as lex\n\ntokens = (\n 'NAME','NUMBER',\n 'PLUS','MINUS','TIMES','DIVIDE','EQUALS',\n 'LPAREN','RPAREN',\n )\n\n# Tokens\n\nt_PLUS = r'\\+'\nt_MINUS = r'-'\nt_TIMES = r'\\*'\nt_DIVIDE = r'/'\nt_EQUALS = r'='\nt_LPAREN = r'\\('\nt_RPAREN = r'\\)'\nt_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*'\n\ndef t_NUMBER(t):\n r'\\d+'\n try:\n t.value = int(t.value)\n except ValueError:\n print(\"Integer value too large %s\" % t.value)\n t.value = 0\n return t\n\nt_ignore = \" \\t\"\n\ndef t_newline(t):\n r'\\n+'\n t.lexer.lineno += t.value.count(\"\\n\")\n \ndef t_error(t):\n print(\"Illegal character '%s'\" % t.value[0])\n t.lexer.skip(1)\n \n# Build the lexer\nlexer = lex.lex(optimize=True, lextab='pkg_test3.generated.lextab')\n\n\n\n" }, { "path": "xonsh/ply/test/pkg_test3/parsing/calcparse.py", "content": "# -----------------------------------------------------------------------------\n# yacc_simple.py\n#\n# A simple, properly specifier grammar\n# -----------------------------------------------------------------------------\n\nfrom .calclex import tokens\nfrom ply import yacc\n\n# Parsing rules\nprecedence = (\n ('left','PLUS','MINUS'),\n ('left','TIMES','DIVIDE'),\n ('right','UMINUS'),\n )\n\n# dictionary of names\nnames = { }\n\ndef p_statement_assign(t):\n 'statement : NAME EQUALS expression'\n names[t[1]] = t[3]\n\ndef p_statement_expr(t):\n 'statement : expression'\n t[0] = t[1]\n\ndef p_expression_binop(t):\n '''expression : expression PLUS expression\n | expression MINUS expression\n | expression TIMES expression\n | expression DIVIDE expression'''\n if t[2] == '+' : t[0] = t[1] + t[3]\n elif t[2] == '-': t[0] = t[1] - t[3]\n elif t[2] == '*': t[0] = t[1] * t[3]\n elif t[2] == '/': t[0] = t[1] / t[3]\n\ndef p_expression_uminus(t):\n 'expression : MINUS expression %prec UMINUS'\n t[0] = -t[2]\n\ndef p_expression_group(t):\n 'expression : LPAREN expression RPAREN'\n t[0] = t[2]\n\ndef p_expression_number(t):\n 'expression : NUMBER'\n t[0] = t[1]\n\ndef p_expression_name(t):\n 'expression : NAME'\n try:\n t[0] = names[t[1]]\n except LookupError:\n print(\"Undefined name '%s'\" % t[1])\n t[0] = 0\n\ndef p_error(t):\n print(\"Syntax error at '%s'\" % t.value)\n\nparser = yacc.yacc(tabmodule='pkg_test3.generated.parsetab')\n\n\n\n\n\n" }, { "path": "xonsh/ply/test/pkg_test4/__init__.py", "content": "# Tests proper handling of lextab and parsetab files in package structures\n# Check of warning messages when files aren't writable\n\n# Here for testing purposes\nimport sys\nif '..' not in sys.path: \n sys.path.insert(0, '..')\n\nimport ply.lex\nimport ply.yacc\n\ndef patched_open(filename, mode):\n if 'w' in mode:\n raise IOError(\"Permission denied %r\" % filename)\n return open(filename, mode)\n\nply.lex.open = patched_open\nply.yacc.open = patched_open\ntry:\n from .parsing.calcparse import parser\nfinally:\n del ply.lex.open\n del ply.yacc.open\n\n\n" }, { "path": "xonsh/ply/test/pkg_test4/parsing/__init__.py", "content": "" }, { "path": "xonsh/ply/test/pkg_test4/parsing/calclex.py", "content": "# -----------------------------------------------------------------------------\n# calclex.py\n# -----------------------------------------------------------------------------\n\nimport ply.lex as lex\n\ntokens = (\n 'NAME','NUMBER',\n 'PLUS','MINUS','TIMES','DIVIDE','EQUALS',\n 'LPAREN','RPAREN',\n )\n\n# Tokens\n\nt_PLUS = r'\\+'\nt_MINUS = r'-'\nt_TIMES = r'\\*'\nt_DIVIDE = r'/'\nt_EQUALS = r'='\nt_LPAREN = r'\\('\nt_RPAREN = r'\\)'\nt_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*'\n\ndef t_NUMBER(t):\n r'\\d+'\n try:\n t.value = int(t.value)\n except ValueError:\n print(\"Integer value too large %s\" % t.value)\n t.value = 0\n return t\n\nt_ignore = \" \\t\"\n\ndef t_newline(t):\n r'\\n+'\n t.lexer.lineno += t.value.count(\"\\n\")\n \ndef t_error(t):\n print(\"Illegal character '%s'\" % t.value[0])\n t.lexer.skip(1)\n \n# Build the lexer\nlexer = lex.lex(optimize=True)\n\n\n\n" }, { "path": "xonsh/ply/test/pkg_test4/parsing/calcparse.py", "content": "# -----------------------------------------------------------------------------\n# yacc_simple.py\n#\n# A simple, properly specifier grammar\n# -----------------------------------------------------------------------------\n\nfrom .calclex import tokens\nfrom ply import yacc\n\n# Parsing rules\nprecedence = (\n ('left','PLUS','MINUS'),\n ('left','TIMES','DIVIDE'),\n ('right','UMINUS'),\n )\n\n# dictionary of names\nnames = { }\n\ndef p_statement_assign(t):\n 'statement : NAME EQUALS expression'\n names[t[1]] = t[3]\n\ndef p_statement_expr(t):\n 'statement : expression'\n t[0] = t[1]\n\ndef p_expression_binop(t):\n '''expression : expression PLUS expression\n | expression MINUS expression\n | expression TIMES expression\n | expression DIVIDE expression'''\n if t[2] == '+' : t[0] = t[1] + t[3]\n elif t[2] == '-': t[0] = t[1] - t[3]\n elif t[2] == '*': t[0] = t[1] * t[3]\n elif t[2] == '/': t[0] = t[1] / t[3]\n\ndef p_expression_uminus(t):\n 'expression : MINUS expression %prec UMINUS'\n t[0] = -t[2]\n\ndef p_expression_group(t):\n 'expression : LPAREN expression RPAREN'\n t[0] = t[2]\n\ndef p_expression_number(t):\n 'expression : NUMBER'\n t[0] = t[1]\n\ndef p_expression_name(t):\n 'expression : NAME'\n try:\n t[0] = names[t[1]]\n except LookupError:\n print(\"Undefined name '%s'\" % t[1])\n t[0] = 0\n\ndef p_error(t):\n print(\"Syntax error at '%s'\" % t.value)\n\nparser = yacc.yacc()\n\n\n\n\n\n" }, { "path": "xonsh/ply/test/pkg_test5/__init__.py", "content": "# Tests proper handling of lextab and parsetab files in package structures\n\n# Here for testing purposes\nimport sys\nif '..' not in sys.path: \n sys.path.insert(0, '..')\n\nfrom .parsing.calcparse import parser\n\n" }, { "path": "xonsh/ply/test/pkg_test5/parsing/__init__.py", "content": "" }, { "path": "xonsh/ply/test/pkg_test5/parsing/calclex.py", "content": "# -----------------------------------------------------------------------------\n# calclex.py\n# -----------------------------------------------------------------------------\n\nimport ply.lex as lex\n\ntokens = (\n 'NAME','NUMBER',\n 'PLUS','MINUS','TIMES','DIVIDE','EQUALS',\n 'LPAREN','RPAREN',\n )\n\n# Tokens\n\nt_PLUS = r'\\+'\nt_MINUS = r'-'\nt_TIMES = r'\\*'\nt_DIVIDE = r'/'\nt_EQUALS = r'='\nt_LPAREN = r'\\('\nt_RPAREN = r'\\)'\nt_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*'\n\ndef t_NUMBER(t):\n r'\\d+'\n try:\n t.value = int(t.value)\n except ValueError:\n print(\"Integer value too large %s\" % t.value)\n t.value = 0\n return t\n\nt_ignore = \" \\t\"\n\ndef t_newline(t):\n r'\\n+'\n t.lexer.lineno += t.value.count(\"\\n\")\n \ndef t_error(t):\n print(\"Illegal character '%s'\" % t.value[0])\n t.lexer.skip(1)\n \n# Build the lexer\nimport os.path\nlexer = lex.lex(optimize=True, outputdir=os.path.dirname(__file__))\n\n\n\n" }, { "path": "xonsh/ply/test/pkg_test5/parsing/calcparse.py", "content": "# -----------------------------------------------------------------------------\n# yacc_simple.py\n#\n# A simple, properly specifier grammar\n# -----------------------------------------------------------------------------\n\nfrom .calclex import tokens\nfrom ply import yacc\n\n# Parsing rules\nprecedence = (\n ('left','PLUS','MINUS'),\n ('left','TIMES','DIVIDE'),\n ('right','UMINUS'),\n )\n\n# dictionary of names\nnames = { }\n\ndef p_statement_assign(t):\n 'statement : NAME EQUALS expression'\n names[t[1]] = t[3]\n\ndef p_statement_expr(t):\n 'statement : expression'\n t[0] = t[1]\n\ndef p_expression_binop(t):\n '''expression : expression PLUS expression\n | expression MINUS expression\n | expression TIMES expression\n | expression DIVIDE expression'''\n if t[2] == '+' : t[0] = t[1] + t[3]\n elif t[2] == '-': t[0] = t[1] - t[3]\n elif t[2] == '*': t[0] = t[1] * t[3]\n elif t[2] == '/': t[0] = t[1] / t[3]\n\ndef p_expression_uminus(t):\n 'expression : MINUS expression %prec UMINUS'\n t[0] = -t[2]\n\ndef p_expression_group(t):\n 'expression : LPAREN expression RPAREN'\n t[0] = t[2]\n\ndef p_expression_number(t):\n 'expression : NUMBER'\n t[0] = t[1]\n\ndef p_expression_name(t):\n 'expression : NAME'\n try:\n t[0] = names[t[1]]\n except LookupError:\n print(\"Undefined name '%s'\" % t[1])\n t[0] = 0\n\ndef p_error(t):\n print(\"Syntax error at '%s'\" % t.value)\n\nimport os.path\nparser = yacc.yacc(outputdir=os.path.dirname(__file__))\n\n\n\n\n\n" }, { "path": "xonsh/ply/test/pkg_test6/__init__.py", "content": "# Tests proper sorting of modules in yacc.ParserReflect.get_pfunctions\n\n# Here for testing purposes\nimport sys\nif '..' not in sys.path:\n sys.path.insert(0, '..')\n\nfrom .parsing.calcparse import parser\n\n" }, { "path": "xonsh/ply/test/pkg_test6/parsing/__init__.py", "content": "" }, { "path": "xonsh/ply/test/pkg_test6/parsing/calclex.py", "content": "# -----------------------------------------------------------------------------\n# calclex.py\n# -----------------------------------------------------------------------------\n\nimport ply.lex as lex\n\ntokens = (\n 'NAME','NUMBER',\n 'PLUS','MINUS','TIMES','DIVIDE','EQUALS',\n 'LPAREN','RPAREN',\n )\n\n# Tokens\n\nt_PLUS = r'\\+'\nt_MINUS = r'-'\nt_TIMES = r'\\*'\nt_DIVIDE = r'/'\nt_EQUALS = r'='\nt_LPAREN = r'\\('\nt_RPAREN = r'\\)'\nt_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*'\n\ndef t_NUMBER(t):\n r'\\d+'\n try:\n t.value = int(t.value)\n except ValueError:\n print(\"Integer value too large %s\" % t.value)\n t.value = 0\n return t\n\nt_ignore = \" \\t\"\n\ndef t_newline(t):\n r'\\n+'\n t.lexer.lineno += t.value.count(\"\\n\")\n \ndef t_error(t):\n print(\"Illegal character '%s'\" % t.value[0])\n t.lexer.skip(1)\n \n# Build the lexer\nimport os.path\nlexer = lex.lex(optimize=True, outputdir=os.path.dirname(__file__))\n\n\n\n" }, { "path": "xonsh/ply/test/pkg_test6/parsing/calcparse.py", "content": "# -----------------------------------------------------------------------------\n# yacc_simple.py\n#\n# A simple, properly specifier grammar\n# -----------------------------------------------------------------------------\n\nfrom .calclex import tokens\nfrom ply import yacc\n\n# Parsing rules\nprecedence = (\n ('left','PLUS','MINUS'),\n ('left','TIMES','DIVIDE'),\n ('right','UMINUS'),\n )\n\n# dictionary of names\nnames = { }\n\nfrom .statement import *\n\nfrom .expression import *\n\ndef p_error(t):\n print(\"Syntax error at '%s'\" % t.value)\n\nimport os.path\nparser = yacc.yacc(outputdir=os.path.dirname(__file__))\n\n\n\n\n\n" }, { "path": "xonsh/ply/test/pkg_test6/parsing/expression.py", "content": "# This file contains definitions of expression grammar\n\ndef p_expression_binop(t):\n '''expression : expression PLUS expression\n | expression MINUS expression\n | expression TIMES expression\n | expression DIVIDE expression'''\n if t[2] == '+' : t[0] = t[1] + t[3]\n elif t[2] == '-': t[0] = t[1] - t[3]\n elif t[2] == '*': t[0] = t[1] * t[3]\n elif t[2] == '/': t[0] = t[1] / t[3]\n\ndef p_expression_uminus(t):\n 'expression : MINUS expression %prec UMINUS'\n t[0] = -t[2]\n\ndef p_expression_group(t):\n 'expression : LPAREN expression RPAREN'\n t[0] = t[2]\n\ndef p_expression_number(t):\n 'expression : NUMBER'\n t[0] = t[1]\n\ndef p_expression_name(t):\n 'expression : NAME'\n try:\n t[0] = names[t[1]]\n except LookupError:\n print(\"Undefined name '%s'\" % t[1])\n t[0] = 0\n" }, { "path": "xonsh/ply/test/pkg_test6/parsing/statement.py", "content": "# This file contains definitions of statement grammar\n\ndef p_statement_assign(t):\n 'statement : NAME EQUALS expression'\n names[t[1]] = t[3]\n\ndef p_statement_expr(t):\n 'statement : expression'\n t[0] = t[1]\n" }, { "path": "xonsh/ply/test/test_cpp_nonascii.c", "content": "/* ë */\n#define x 1" }, { "path": "xonsh/ply/test/testcpp.py", "content": "from unittest import TestCase, main\n\nfrom multiprocessing import Process, Queue\nfrom six.moves.queue import Empty\n\nimport sys\nimport locale\n\nif \"..\" not in sys.path:\n sys.path.insert(0, \"..\")\n\nfrom ply.lex import lex\nfrom ply.cpp import *\n\n\ndef preprocessing(in_, out_queue):\n out = None\n\n try:\n p = Preprocessor(lex())\n p.parse(in_)\n tokens = [t.value for t in p.parser]\n out = \"\".join(tokens)\n finally:\n out_queue.put(out)\n\nclass CPPTests(TestCase):\n \"Tests related to ANSI-C style lexical preprocessor.\"\n\n def __test_preprocessing(self, in_, expected, time_limit = 1.0):\n out_queue = Queue()\n\n preprocessor = Process(\n name = \"PLY`s C preprocessor\",\n target = preprocessing,\n args = (in_, out_queue)\n )\n\n preprocessor.start()\n\n try:\n out = out_queue.get(timeout = time_limit)\n except Empty:\n preprocessor.terminate()\n raise RuntimeError(\"Time limit exceeded!\")\n else:\n self.assertMultiLineEqual(out, expected)\n\n def test_infinite_argument_expansion(self):\n # CPP does not drags set of currently expanded macros through macro\n # arguments expansion. If there is a match between an argument value\n # and name of an already expanded macro then CPP falls into infinite\n # recursion.\n self.__test_preprocessing(\"\"\"\\\n#define a(x) x\n#define b a(b)\nb\n\"\"\" , \"\"\"\\\n\n\nb\"\"\"\n )\n\n\n def test_concatenation(self):\n self.__test_preprocessing(\"\"\"\\\n#define a(x) x##_\n#define b(x) _##x\n#define c(x) _##x##_\n#define d(x,y) _##x##y##_\n\na(i)\nb(j)\nc(k)\nd(q,s)\"\"\"\n , \"\"\"\\\n\n\n\n\n\ni_\n_j\n_k_\n_qs_\"\"\"\n )\n\n def test_deadloop_macro(self):\n # If there is a word which equals to name of a parametrized macro, then\n # attempt to expand such word as a macro manages the parser to fall\n # into an infinite loop.\n\n self.__test_preprocessing(\"\"\"\\\n#define a(x) x\n\na;\"\"\"\n , \"\"\"\\\n\n\na;\"\"\"\n )\n\n def test_index_error(self):\n # If there are no tokens after a word (\"a\") which equals to name of\n # a parameterized macro, then attempt to expand this word leads to\n # IndexError.\n\n self.__test_preprocessing(\"\"\"\\\n#define a(x) x\n\na\"\"\"\n , \"\"\"\\\n\n\na\"\"\"\n )\n\n def test_evalexpr(self):\n # #if 1 != 2 is not processed correctly; undefined values are converted\n # to 0L instead of 0 (issue #195)\n #\n self.__test_preprocessing(\"\"\"\\\n#if (1!=0) && (!x || (!(1==2)))\na;\n#else\nb;\n#endif\n\"\"\"\n , \"\"\"\\\n\na;\n\n\"\"\"\n )\n\n def test_include_nonascii(self):\n # Issue #196: #included files are read using the current locale's\n # getdefaultencoding. if a #included file contains non-ascii characters,\n # while default encoding is e.g. US_ASCII, this causes an error\n locale.setlocale(locale.LC_ALL, 'C')\n self.__test_preprocessing(\"\"\"\\\n#include \"test_cpp_nonascii.c\"\nx;\n\n\"\"\"\n , \"\"\"\\\n\n \n1;\n\"\"\"\n )\n\nmain()\n" }, { "path": "xonsh/ply/test/testlex.py", "content": "# testlex.py\n\nimport unittest\ntry:\n import StringIO\nexcept ImportError:\n import io as StringIO\n\nimport sys\nimport os\nimport warnings\nimport platform\n\nsys.path.insert(0,\"..\")\nsys.tracebacklimit = 0\n\nimport ply.lex\n\ntry:\n from importlib.util import cache_from_source\nexcept ImportError:\n # Python 2.7, but we don't care.\n cache_from_source = None\n\n\ndef make_pymodule_path(filename, optimization=None):\n path = os.path.dirname(filename)\n file = os.path.basename(filename)\n mod, ext = os.path.splitext(file)\n\n if sys.hexversion >= 0x3050000:\n fullpath = cache_from_source(filename, optimization=optimization)\n elif sys.hexversion >= 0x3040000:\n fullpath = cache_from_source(filename, ext=='.pyc')\n elif sys.hexversion >= 0x3020000:\n import imp\n modname = mod+\".\"+imp.get_tag()+ext\n fullpath = os.path.join(path,'__pycache__',modname)\n else:\n fullpath = filename\n return fullpath\n\ndef pymodule_out_exists(filename, optimization=None):\n return os.path.exists(make_pymodule_path(filename,\n optimization=optimization))\n\ndef pymodule_out_remove(filename, optimization=None):\n os.remove(make_pymodule_path(filename, optimization=optimization))\n\ndef implementation():\n if platform.system().startswith(\"Java\"):\n return \"Jython\"\n elif hasattr(sys, \"pypy_version_info\"):\n return \"PyPy\"\n else:\n return \"CPython\"\n\ntest_pyo = (implementation() == 'CPython')\n\ndef check_expected(result, expected, contains=False):\n if sys.version_info[0] >= 3:\n if isinstance(result,str):\n result = result.encode('ascii')\n if isinstance(expected,str):\n expected = expected.encode('ascii')\n resultlines = result.splitlines()\n expectedlines = expected.splitlines()\n\n if len(resultlines) != len(expectedlines):\n return False\n\n for rline,eline in zip(resultlines,expectedlines):\n if contains:\n if eline not in rline:\n return False\n else:\n if not rline.endswith(eline):\n return False\n return True\n\ndef run_import(module):\n code = \"import \"+module\n exec(code)\n del sys.modules[module]\n \n# Tests related to errors and warnings when building lexers\nclass LexErrorWarningTests(unittest.TestCase):\n def setUp(self):\n sys.stderr = StringIO.StringIO()\n sys.stdout = StringIO.StringIO()\n if sys.hexversion >= 0x3020000:\n warnings.filterwarnings('ignore',category=ResourceWarning)\n\n def tearDown(self):\n sys.stderr = sys.__stderr__\n sys.stdout = sys.__stdout__\n def test_lex_doc1(self):\n self.assertRaises(SyntaxError,run_import,\"lex_doc1\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"lex_doc1.py:18: No regular expression defined for rule 't_NUMBER'\\n\"))\n def test_lex_dup1(self):\n self.assertRaises(SyntaxError,run_import,\"lex_dup1\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"lex_dup1.py:20: Rule t_NUMBER redefined. Previously defined on line 18\\n\" )) \n \n def test_lex_dup2(self):\n self.assertRaises(SyntaxError,run_import,\"lex_dup2\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"lex_dup2.py:22: Rule t_NUMBER redefined. Previously defined on line 18\\n\" ))\n \n def test_lex_dup3(self):\n self.assertRaises(SyntaxError,run_import,\"lex_dup3\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"lex_dup3.py:20: Rule t_NUMBER redefined. Previously defined on line 18\\n\" ))\n\n def test_lex_empty(self):\n self.assertRaises(SyntaxError,run_import,\"lex_empty\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"No rules of the form t_rulename are defined\\n\"\n \"No rules defined for state 'INITIAL'\\n\"))\n\n def test_lex_error1(self):\n run_import(\"lex_error1\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"No t_error rule is defined\\n\"))\n\n def test_lex_error2(self):\n self.assertRaises(SyntaxError,run_import,\"lex_error2\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"Rule 't_error' must be defined as a function\\n\")\n )\n\n def test_lex_error3(self):\n self.assertRaises(SyntaxError,run_import,\"lex_error3\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"lex_error3.py:20: Rule 't_error' requires an argument\\n\"))\n\n def test_lex_error4(self):\n self.assertRaises(SyntaxError,run_import,\"lex_error4\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"lex_error4.py:20: Rule 't_error' has too many arguments\\n\"))\n\n def test_lex_ignore(self):\n self.assertRaises(SyntaxError,run_import,\"lex_ignore\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"lex_ignore.py:20: Rule 't_ignore' must be defined as a string\\n\"))\n\n def test_lex_ignore2(self):\n run_import(\"lex_ignore2\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"t_ignore contains a literal backslash '\\\\'\\n\"))\n\n\n def test_lex_re1(self):\n self.assertRaises(SyntaxError,run_import,\"lex_re1\")\n result = sys.stderr.getvalue()\n if sys.hexversion < 0x3050000:\n msg = \"Invalid regular expression for rule 't_NUMBER'. unbalanced parenthesis\\n\"\n else:\n msg = \"Invalid regular expression for rule 't_NUMBER'. missing ), unterminated subpattern at position 0\"\n self.assert_(check_expected(result,\n msg,\n contains=True))\n\n def test_lex_re2(self):\n self.assertRaises(SyntaxError,run_import,\"lex_re2\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"Regular expression for rule 't_PLUS' matches empty string\\n\"))\n\n def test_lex_re3(self):\n self.assertRaises(SyntaxError,run_import,\"lex_re3\")\n result = sys.stderr.getvalue()\n# self.assert_(check_expected(result,\n# \"Invalid regular expression for rule 't_POUND'. unbalanced parenthesis\\n\"\n# \"Make sure '#' in rule 't_POUND' is escaped with '\\\\#'\\n\"))\n\n if sys.hexversion < 0x3050000:\n msg = (\"Invalid regular expression for rule 't_POUND'. unbalanced parenthesis\\n\"\n \"Make sure '#' in rule 't_POUND' is escaped with '\\\\#'\\n\")\n else:\n msg = (\"Invalid regular expression for rule 't_POUND'. missing ), unterminated subpattern at position 0\\n\"\n \"ERROR: Make sure '#' in rule 't_POUND' is escaped with '\\#'\")\n self.assert_(check_expected(result,\n msg,\n contains=True), result)\n\n def test_lex_rule1(self):\n self.assertRaises(SyntaxError,run_import,\"lex_rule1\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"t_NUMBER not defined as a function or string\\n\"))\n\n def test_lex_rule2(self):\n self.assertRaises(SyntaxError,run_import,\"lex_rule2\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"lex_rule2.py:18: Rule 't_NUMBER' requires an argument\\n\"))\n\n def test_lex_rule3(self):\n self.assertRaises(SyntaxError,run_import,\"lex_rule3\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"lex_rule3.py:18: Rule 't_NUMBER' has too many arguments\\n\"))\n\n\n def test_lex_state1(self):\n self.assertRaises(SyntaxError,run_import,\"lex_state1\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"states must be defined as a tuple or list\\n\"))\n\n def test_lex_state2(self):\n self.assertRaises(SyntaxError,run_import,\"lex_state2\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"Invalid state specifier 'comment'. Must be a tuple (statename,'exclusive|inclusive')\\n\"\n \"Invalid state specifier 'example'. Must be a tuple (statename,'exclusive|inclusive')\\n\"))\n\n def test_lex_state3(self):\n self.assertRaises(SyntaxError,run_import,\"lex_state3\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"State name 1 must be a string\\n\"\n \"No rules defined for state 'example'\\n\"))\n\n def test_lex_state4(self):\n self.assertRaises(SyntaxError,run_import,\"lex_state4\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"State type for state comment must be 'inclusive' or 'exclusive'\\n\"))\n\n\n def test_lex_state5(self):\n self.assertRaises(SyntaxError,run_import,\"lex_state5\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"State 'comment' already defined\\n\"))\n\n def test_lex_state_noerror(self):\n run_import(\"lex_state_noerror\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"No error rule is defined for exclusive state 'comment'\\n\")) \n\n def test_lex_state_norule(self):\n self.assertRaises(SyntaxError,run_import,\"lex_state_norule\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"No rules defined for state 'example'\\n\"))\n\n def test_lex_token1(self):\n self.assertRaises(SyntaxError,run_import,\"lex_token1\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"No token list is defined\\n\"\n \"Rule 't_NUMBER' defined for an unspecified token NUMBER\\n\"\n \"Rule 't_PLUS' defined for an unspecified token PLUS\\n\"\n \"Rule 't_MINUS' defined for an unspecified token MINUS\\n\"\n))\n\n def test_lex_token2(self):\n self.assertRaises(SyntaxError,run_import,\"lex_token2\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"tokens must be a list or tuple\\n\"\n \"Rule 't_NUMBER' defined for an unspecified token NUMBER\\n\"\n \"Rule 't_PLUS' defined for an unspecified token PLUS\\n\"\n \"Rule 't_MINUS' defined for an unspecified token MINUS\\n\"\n))\n \n def test_lex_token3(self):\n self.assertRaises(SyntaxError,run_import,\"lex_token3\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"Rule 't_MINUS' defined for an unspecified token MINUS\\n\"))\n\n\n def test_lex_token4(self):\n self.assertRaises(SyntaxError,run_import,\"lex_token4\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"Bad token name '-'\\n\"))\n\n\n def test_lex_token5(self):\n try:\n run_import(\"lex_token5\")\n except ply.lex.LexError:\n e = sys.exc_info()[1]\n self.assert_(check_expected(str(e),\"lex_token5.py:19: Rule 't_NUMBER' returned an unknown token type 'NUM'\"))\n\n def test_lex_token_dup(self):\n run_import(\"lex_token_dup\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"Token 'MINUS' multiply defined\\n\")) \n\n \n def test_lex_literal1(self):\n self.assertRaises(SyntaxError,run_import,\"lex_literal1\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"Invalid literal '**'. Must be a single character\\n\"))\n\n def test_lex_literal2(self):\n self.assertRaises(SyntaxError,run_import,\"lex_literal2\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"Invalid literals specification. literals must be a sequence of characters\\n\"))\n\nimport os\nimport subprocess\nimport shutil\n\n# Tests related to various build options associated with lexers\nclass LexBuildOptionTests(unittest.TestCase):\n def setUp(self):\n sys.stderr = StringIO.StringIO()\n sys.stdout = StringIO.StringIO()\n def tearDown(self):\n sys.stderr = sys.__stderr__\n sys.stdout = sys.__stdout__\n try:\n shutil.rmtree(\"lexdir\")\n except OSError:\n pass\n\n def test_lex_module(self):\n run_import(\"lex_module\")\n result = sys.stdout.getvalue()\n self.assert_(check_expected(result,\n \"(NUMBER,3,1,0)\\n\"\n \"(PLUS,'+',1,1)\\n\"\n \"(NUMBER,4,1,2)\\n\"))\n \n def test_lex_object(self):\n run_import(\"lex_object\")\n result = sys.stdout.getvalue()\n self.assert_(check_expected(result,\n \"(NUMBER,3,1,0)\\n\"\n \"(PLUS,'+',1,1)\\n\"\n \"(NUMBER,4,1,2)\\n\"))\n\n def test_lex_closure(self):\n run_import(\"lex_closure\")\n result = sys.stdout.getvalue()\n self.assert_(check_expected(result,\n \"(NUMBER,3,1,0)\\n\"\n \"(PLUS,'+',1,1)\\n\"\n \"(NUMBER,4,1,2)\\n\"))\n\n def test_lex_optimize(self):\n try:\n os.remove(\"lextab.py\")\n except OSError:\n pass\n try:\n os.remove(\"lextab.pyc\")\n except OSError:\n pass\n try:\n os.remove(\"lextab.pyo\")\n except OSError:\n pass\n run_import(\"lex_optimize\")\n\n result = sys.stdout.getvalue()\n self.assert_(check_expected(result,\n \"(NUMBER,3,1,0)\\n\"\n \"(PLUS,'+',1,1)\\n\"\n \"(NUMBER,4,1,2)\\n\"))\n self.assert_(os.path.exists(\"lextab.py\"))\n\n p = subprocess.Popen([sys.executable,'-O','lex_optimize.py'],\n stdout=subprocess.PIPE)\n result = p.stdout.read()\n\n self.assert_(check_expected(result,\n \"(NUMBER,3,1,0)\\n\"\n \"(PLUS,'+',1,1)\\n\"\n \"(NUMBER,4,1,2)\\n\"))\n if test_pyo:\n self.assert_(pymodule_out_exists(\"lextab.pyo\", 1))\n pymodule_out_remove(\"lextab.pyo\", 1)\n\n p = subprocess.Popen([sys.executable,'-OO','lex_optimize.py'],\n stdout=subprocess.PIPE)\n result = p.stdout.read()\n self.assert_(check_expected(result,\n \"(NUMBER,3,1,0)\\n\"\n \"(PLUS,'+',1,1)\\n\"\n \"(NUMBER,4,1,2)\\n\"))\n\n if test_pyo:\n self.assert_(pymodule_out_exists(\"lextab.pyo\", 2))\n try:\n os.remove(\"lextab.py\")\n except OSError:\n pass\n try:\n pymodule_out_remove(\"lextab.pyc\")\n except OSError:\n pass\n try:\n pymodule_out_remove(\"lextab.pyo\", 2)\n except OSError:\n pass\n\n def test_lex_optimize2(self):\n try:\n os.remove(\"opt2tab.py\")\n except OSError:\n pass\n try:\n os.remove(\"opt2tab.pyc\")\n except OSError:\n pass\n try:\n os.remove(\"opt2tab.pyo\")\n except OSError:\n pass\n run_import(\"lex_optimize2\")\n result = sys.stdout.getvalue()\n self.assert_(check_expected(result,\n \"(NUMBER,3,1,0)\\n\"\n \"(PLUS,'+',1,1)\\n\"\n \"(NUMBER,4,1,2)\\n\"))\n self.assert_(os.path.exists(\"opt2tab.py\"))\n\n p = subprocess.Popen([sys.executable,'-O','lex_optimize2.py'],\n stdout=subprocess.PIPE)\n result = p.stdout.read()\n self.assert_(check_expected(result,\n \"(NUMBER,3,1,0)\\n\"\n \"(PLUS,'+',1,1)\\n\"\n \"(NUMBER,4,1,2)\\n\"))\n if test_pyo:\n self.assert_(pymodule_out_exists(\"opt2tab.pyo\", 1))\n pymodule_out_remove(\"opt2tab.pyo\", 1)\n p = subprocess.Popen([sys.executable,'-OO','lex_optimize2.py'],\n stdout=subprocess.PIPE)\n result = p.stdout.read()\n self.assert_(check_expected(result,\n \"(NUMBER,3,1,0)\\n\"\n \"(PLUS,'+',1,1)\\n\"\n \"(NUMBER,4,1,2)\\n\"))\n if test_pyo:\n self.assert_(pymodule_out_exists(\"opt2tab.pyo\", 2))\n try:\n os.remove(\"opt2tab.py\")\n except OSError:\n pass\n try:\n pymodule_out_remove(\"opt2tab.pyc\")\n except OSError:\n pass\n try:\n pymodule_out_remove(\"opt2tab.pyo\", 2)\n except OSError:\n pass\n\n def test_lex_optimize3(self):\n try:\n shutil.rmtree(\"lexdir\")\n except OSError:\n pass\n \n os.mkdir(\"lexdir\")\n os.mkdir(\"lexdir/sub\")\n with open(\"lexdir/__init__.py\",\"w\") as f:\n f.write(\"\")\n with open(\"lexdir/sub/__init__.py\",\"w\") as f:\n f.write(\"\")\n run_import(\"lex_optimize3\")\n result = sys.stdout.getvalue()\n self.assert_(check_expected(result,\n \"(NUMBER,3,1,0)\\n\"\n \"(PLUS,'+',1,1)\\n\"\n \"(NUMBER,4,1,2)\\n\"))\n self.assert_(os.path.exists(\"lexdir/sub/calctab.py\"))\n\n p = subprocess.Popen([sys.executable,'-O','lex_optimize3.py'],\n stdout=subprocess.PIPE)\n result = p.stdout.read()\n self.assert_(check_expected(result,\n \"(NUMBER,3,1,0)\\n\"\n \"(PLUS,'+',1,1)\\n\"\n \"(NUMBER,4,1,2)\\n\"))\n if test_pyo:\n self.assert_(pymodule_out_exists(\"lexdir/sub/calctab.pyo\", 1))\n pymodule_out_remove(\"lexdir/sub/calctab.pyo\", 1)\n\n p = subprocess.Popen([sys.executable,'-OO','lex_optimize3.py'],\n stdout=subprocess.PIPE)\n result = p.stdout.read()\n self.assert_(check_expected(result,\n \"(NUMBER,3,1,0)\\n\"\n \"(PLUS,'+',1,1)\\n\"\n \"(NUMBER,4,1,2)\\n\"))\n if test_pyo:\n self.assert_(pymodule_out_exists(\"lexdir/sub/calctab.pyo\", 2))\n try:\n shutil.rmtree(\"lexdir\")\n except OSError:\n pass\n\n def test_lex_optimize4(self):\n\n # Regression test to make sure that reflags works correctly\n # on Python 3.\n\n for extension in ['py', 'pyc']:\n try:\n os.remove(\"opt4tab.{0}\".format(extension))\n except OSError:\n pass\n\n run_import(\"lex_optimize4\")\n run_import(\"lex_optimize4\")\n\n for extension in ['py', 'pyc']:\n try:\n os.remove(\"opt4tab.{0}\".format(extension))\n except OSError:\n pass\n\n def test_lex_opt_alias(self):\n try:\n os.remove(\"aliastab.py\")\n except OSError:\n pass\n try:\n os.remove(\"aliastab.pyc\")\n except OSError:\n pass\n try:\n os.remove(\"aliastab.pyo\")\n except OSError:\n pass\n run_import(\"lex_opt_alias\")\n result = sys.stdout.getvalue()\n self.assert_(check_expected(result,\n \"(NUMBER,3,1,0)\\n\"\n \"(+,'+',1,1)\\n\"\n \"(NUMBER,4,1,2)\\n\"))\n self.assert_(os.path.exists(\"aliastab.py\"))\n\n p = subprocess.Popen([sys.executable,'-O','lex_opt_alias.py'],\n stdout=subprocess.PIPE)\n result = p.stdout.read()\n self.assert_(check_expected(result,\n \"(NUMBER,3,1,0)\\n\"\n \"(+,'+',1,1)\\n\"\n \"(NUMBER,4,1,2)\\n\"))\n if test_pyo:\n self.assert_(pymodule_out_exists(\"aliastab.pyo\", 1))\n pymodule_out_remove(\"aliastab.pyo\", 1)\n\n p = subprocess.Popen([sys.executable,'-OO','lex_opt_alias.py'],\n stdout=subprocess.PIPE)\n result = p.stdout.read()\n self.assert_(check_expected(result,\n \"(NUMBER,3,1,0)\\n\"\n \"(+,'+',1,1)\\n\"\n \"(NUMBER,4,1,2)\\n\"))\n\n if test_pyo:\n self.assert_(pymodule_out_exists(\"aliastab.pyo\", 2))\n try:\n os.remove(\"aliastab.py\")\n except OSError:\n pass\n try:\n pymodule_out_remove(\"aliastab.pyc\")\n except OSError:\n pass\n try:\n pymodule_out_remove(\"aliastab.pyo\", 2)\n except OSError:\n pass\n\n def test_lex_many_tokens(self):\n try:\n os.remove(\"manytab.py\")\n except OSError:\n pass\n try:\n os.remove(\"manytab.pyc\")\n except OSError:\n pass\n try:\n os.remove(\"manytab.pyo\")\n except OSError:\n pass\n run_import(\"lex_many_tokens\")\n result = sys.stdout.getvalue()\n self.assert_(check_expected(result,\n \"(TOK34,'TOK34:',1,0)\\n\"\n \"(TOK143,'TOK143:',1,7)\\n\"\n \"(TOK269,'TOK269:',1,15)\\n\"\n \"(TOK372,'TOK372:',1,23)\\n\"\n \"(TOK452,'TOK452:',1,31)\\n\"\n \"(TOK561,'TOK561:',1,39)\\n\"\n \"(TOK999,'TOK999:',1,47)\\n\"\n ))\n\n self.assert_(os.path.exists(\"manytab.py\"))\n\n if implementation() == 'CPython':\n p = subprocess.Popen([sys.executable,'-O','lex_many_tokens.py'],\n stdout=subprocess.PIPE)\n result = p.stdout.read()\n self.assert_(check_expected(result,\n \"(TOK34,'TOK34:',1,0)\\n\"\n \"(TOK143,'TOK143:',1,7)\\n\"\n \"(TOK269,'TOK269:',1,15)\\n\"\n \"(TOK372,'TOK372:',1,23)\\n\"\n \"(TOK452,'TOK452:',1,31)\\n\"\n \"(TOK561,'TOK561:',1,39)\\n\"\n \"(TOK999,'TOK999:',1,47)\\n\"\n ))\n\n self.assert_(pymodule_out_exists(\"manytab.pyo\", 1))\n pymodule_out_remove(\"manytab.pyo\", 1)\n try:\n os.remove(\"manytab.py\")\n except OSError:\n pass\n try:\n os.remove(\"manytab.pyc\")\n except OSError:\n pass\n try:\n os.remove(\"manytab.pyo\")\n except OSError:\n pass\n \n# Tests related to run-time behavior of lexers\nclass LexRunTests(unittest.TestCase):\n def setUp(self):\n sys.stderr = StringIO.StringIO()\n sys.stdout = StringIO.StringIO()\n def tearDown(self):\n sys.stderr = sys.__stderr__\n sys.stdout = sys.__stdout__\n\n def test_lex_hedit(self):\n run_import(\"lex_hedit\")\n result = sys.stdout.getvalue()\n self.assert_(check_expected(result,\n \"(H_EDIT_DESCRIPTOR,'abc',1,0)\\n\"\n \"(H_EDIT_DESCRIPTOR,'abcdefghij',1,6)\\n\"\n \"(H_EDIT_DESCRIPTOR,'xy',1,20)\\n\"))\n \n def test_lex_state_try(self):\n run_import(\"lex_state_try\")\n result = sys.stdout.getvalue()\n self.assert_(check_expected(result,\n \"(NUMBER,'3',1,0)\\n\"\n \"(PLUS,'+',1,2)\\n\"\n \"(NUMBER,'4',1,4)\\n\"\n \"Entering comment state\\n\"\n \"comment body LexToken(body_part,'This is a comment */',1,9)\\n\"\n \"(PLUS,'+',1,30)\\n\"\n \"(NUMBER,'10',1,32)\\n\"\n )) \n\n\n\nunittest.main()\n" }, { "path": "xonsh/ply/test/testyacc.py", "content": "# testyacc.py\n\nimport unittest\ntry:\n import StringIO\nexcept ImportError:\n import io as StringIO\n\nimport sys\nimport os\nimport warnings\nimport re\nimport platform\n\nsys.path.insert(0,\"..\")\nsys.tracebacklimit = 0\n\nimport ply.yacc\n\ndef make_pymodule_path(filename):\n path = os.path.dirname(filename)\n file = os.path.basename(filename)\n mod, ext = os.path.splitext(file)\n\n if sys.hexversion >= 0x3040000:\n import importlib.util\n fullpath = importlib.util.cache_from_source(filename, ext=='.pyc')\n elif sys.hexversion >= 0x3020000:\n import imp\n modname = mod+\".\"+imp.get_tag()+ext\n fullpath = os.path.join(path,'__pycache__',modname)\n else:\n fullpath = filename\n return fullpath\n\ndef pymodule_out_exists(filename):\n return os.path.exists(make_pymodule_path(filename))\n\ndef pymodule_out_remove(filename):\n os.remove(make_pymodule_path(filename))\n\ndef implementation():\n if platform.system().startswith(\"Java\"):\n return \"Jython\"\n elif hasattr(sys, \"pypy_version_info\"):\n return \"PyPy\"\n else:\n return \"CPython\"\n\n# Check the output to see if it contains all of a set of expected output lines.\n# This alternate implementation looks weird, but is needed to properly handle\n# some variations in error message order that occurs due to dict hash table\n# randomization that was introduced in Python 3.3\ndef check_expected(result, expected):\n # Normalize 'state n' text to account for randomization effects in Python 3.3\n expected = re.sub(r' state \\d+', 'state ', expected)\n result = re.sub(r' state \\d+', 'state ', result)\n\n resultlines = set()\n for line in result.splitlines():\n if line.startswith(\"WARNING: \"):\n line = line[9:]\n elif line.startswith(\"ERROR: \"):\n line = line[7:]\n resultlines.add(line)\n\n # Selectively remove expected lines from the output\n for eline in expected.splitlines():\n resultlines = set(line for line in resultlines if not line.endswith(eline))\n\n # Return True if no result lines remain\n return not bool(resultlines)\n\ndef run_import(module):\n code = \"import \"+module\n exec(code)\n del sys.modules[module]\n \n# Tests related to errors and warnings when building parsers\nclass YaccErrorWarningTests(unittest.TestCase):\n def setUp(self):\n sys.stderr = StringIO.StringIO()\n sys.stdout = StringIO.StringIO()\n try:\n os.remove(\"parsetab.py\")\n pymodule_out_remove(\"parsetab.pyc\")\n except OSError:\n pass\n \n if sys.hexversion >= 0x3020000:\n warnings.filterwarnings('ignore', category=ResourceWarning)\n warnings.filterwarnings('ignore', category=DeprecationWarning)\n\n def tearDown(self):\n sys.stderr = sys.__stderr__\n sys.stdout = sys.__stdout__\n def test_yacc_badargs(self):\n self.assertRaises(ply.yacc.YaccError,run_import,\"yacc_badargs\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"yacc_badargs.py:23: Rule 'p_statement_assign' has too many arguments\\n\"\n \"yacc_badargs.py:27: Rule 'p_statement_expr' requires an argument\\n\"\n )) \n def test_yacc_badid(self):\n self.assertRaises(ply.yacc.YaccError,run_import,\"yacc_badid\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"yacc_badid.py:32: Illegal name 'bad&rule' in rule 'statement'\\n\"\n \"yacc_badid.py:36: Illegal rule name 'bad&rule'\\n\"\n ))\n\n def test_yacc_badprec(self):\n try:\n run_import(\"yacc_badprec\")\n except ply.yacc.YaccError:\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"precedence must be a list or tuple\\n\"\n ))\n def test_yacc_badprec2(self):\n self.assertRaises(ply.yacc.YaccError,run_import,\"yacc_badprec2\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"Bad precedence table\\n\"\n ))\n\n def test_yacc_badprec3(self):\n run_import(\"yacc_badprec3\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"Precedence already specified for terminal 'MINUS'\\n\"\n \"Generating LALR tables\\n\"\n\n ))\n \n def test_yacc_badrule(self):\n self.assertRaises(ply.yacc.YaccError,run_import,\"yacc_badrule\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"yacc_badrule.py:24: Syntax error. Expected ':'\\n\"\n \"yacc_badrule.py:28: Syntax error in rule 'statement'\\n\"\n \"yacc_badrule.py:33: Syntax error. Expected ':'\\n\"\n \"yacc_badrule.py:42: Syntax error. Expected ':'\\n\"\n ))\n\n def test_yacc_badtok(self):\n try:\n run_import(\"yacc_badtok\")\n except ply.yacc.YaccError:\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"tokens must be a list or tuple\\n\"))\n\n def test_yacc_dup(self):\n run_import(\"yacc_dup\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"yacc_dup.py:27: Function p_statement redefined. Previously defined on line 23\\n\"\n \"Token 'EQUALS' defined, but not used\\n\"\n \"There is 1 unused token\\n\"\n \"Generating LALR tables\\n\"\n\n ))\n def test_yacc_error1(self):\n try:\n run_import(\"yacc_error1\")\n except ply.yacc.YaccError:\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"yacc_error1.py:61: p_error() requires 1 argument\\n\"))\n\n def test_yacc_error2(self):\n try:\n run_import(\"yacc_error2\")\n except ply.yacc.YaccError:\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"yacc_error2.py:61: p_error() requires 1 argument\\n\"))\n\n def test_yacc_error3(self):\n try:\n run_import(\"yacc_error3\")\n except ply.yacc.YaccError:\n e = sys.exc_info()[1]\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"'p_error' defined, but is not a function or method\\n\"))\n \n def test_yacc_error4(self):\n self.assertRaises(ply.yacc.YaccError,run_import,\"yacc_error4\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"yacc_error4.py:62: Illegal rule name 'error'. Already defined as a token\\n\"\n ))\n\n\n def test_yacc_error5(self):\n run_import(\"yacc_error5\")\n result = sys.stdout.getvalue()\n self.assert_(check_expected(result,\n \"Group at 3:10 to 3:12\\n\"\n \"Undefined name 'a'\\n\"\n \"Syntax error at 'b'\\n\"\n \"Syntax error at 4:18 to 4:22\\n\"\n \"Assignment Error at 2:5 to 5:27\\n\"\n \"13\\n\"\n ))\n\n def test_yacc_error6(self):\n run_import(\"yacc_error6\")\n result = sys.stdout.getvalue()\n self.assert_(check_expected(result,\n \"a=7\\n\"\n \"Line 3: Syntax error at '*'\\n\"\n \"c=21\\n\"\n ))\n\n def test_yacc_error7(self):\n run_import(\"yacc_error7\")\n result = sys.stdout.getvalue()\n self.assert_(check_expected(result,\n \"a=7\\n\"\n \"Line 3: Syntax error at '*'\\n\"\n \"c=21\\n\"\n ))\n\n def test_yacc_inf(self):\n self.assertRaises(ply.yacc.YaccError,run_import,\"yacc_inf\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"Token 'NUMBER' defined, but not used\\n\"\n \"There is 1 unused token\\n\"\n \"Infinite recursion detected for symbol 'statement'\\n\"\n \"Infinite recursion detected for symbol 'expression'\\n\"\n ))\n def test_yacc_literal(self):\n self.assertRaises(ply.yacc.YaccError,run_import,\"yacc_literal\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"yacc_literal.py:36: Literal token '**' in rule 'expression' may only be a single character\\n\"\n ))\n def test_yacc_misplaced(self):\n self.assertRaises(ply.yacc.YaccError,run_import,\"yacc_misplaced\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"yacc_misplaced.py:32: Misplaced '|'\\n\"\n ))\n\n def test_yacc_missing1(self):\n self.assertRaises(ply.yacc.YaccError,run_import,\"yacc_missing1\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"yacc_missing1.py:24: Symbol 'location' used, but not defined as a token or a rule\\n\"\n ))\n\n def test_yacc_nested(self):\n run_import(\"yacc_nested\")\n result = sys.stdout.getvalue()\n self.assert_(check_expected(result,\n \"A\\n\"\n \"A\\n\"\n \"A\\n\",\n ))\n\n def test_yacc_nodoc(self):\n run_import(\"yacc_nodoc\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"yacc_nodoc.py:27: No documentation string specified in function 'p_statement_expr' (ignored)\\n\"\n \"Generating LALR tables\\n\"\n ))\n\n def test_yacc_noerror(self):\n run_import(\"yacc_noerror\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"no p_error() function is defined\\n\"\n \"Generating LALR tables\\n\"\n ))\n\n def test_yacc_nop(self):\n run_import(\"yacc_nop\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"yacc_nop.py:27: Possible grammar rule 'statement_expr' defined without p_ prefix\\n\"\n \"Generating LALR tables\\n\"\n ))\n\n def test_yacc_notfunc(self):\n run_import(\"yacc_notfunc\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"'p_statement_assign' not defined as a function\\n\"\n \"Token 'EQUALS' defined, but not used\\n\"\n \"There is 1 unused token\\n\"\n \"Generating LALR tables\\n\"\n ))\n def test_yacc_notok(self):\n try:\n run_import(\"yacc_notok\")\n except ply.yacc.YaccError:\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"No token list is defined\\n\"))\n\n def test_yacc_rr(self):\n run_import(\"yacc_rr\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"Generating LALR tables\\n\"\n \"1 reduce/reduce conflict\\n\"\n \"reduce/reduce conflict in state 15 resolved using rule (statement -> NAME EQUALS NUMBER)\\n\"\n \"rejected rule (expression -> NUMBER) in state 15\\n\"\n\n ))\n\n def test_yacc_rr_unused(self):\n run_import(\"yacc_rr_unused\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"no p_error() function is defined\\n\"\n \"Generating LALR tables\\n\"\n \"3 reduce/reduce conflicts\\n\"\n \"reduce/reduce conflict in state 1 resolved using rule (rule3 -> A)\\n\"\n \"rejected rule (rule4 -> A) in state 1\\n\"\n \"reduce/reduce conflict in state 1 resolved using rule (rule3 -> A)\\n\"\n \"rejected rule (rule5 -> A) in state 1\\n\"\n \"reduce/reduce conflict in state 1 resolved using rule (rule4 -> A)\\n\"\n \"rejected rule (rule5 -> A) in state 1\\n\"\n \"Rule (rule5 -> A) is never reduced\\n\"\n ))\n\n def test_yacc_simple(self):\n run_import(\"yacc_simple\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"Generating LALR tables\\n\"\n ))\n\n def test_yacc_sr(self):\n run_import(\"yacc_sr\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"Generating LALR tables\\n\"\n \"20 shift/reduce conflicts\\n\"\n ))\n\n def test_yacc_term1(self):\n self.assertRaises(ply.yacc.YaccError,run_import,\"yacc_term1\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"yacc_term1.py:24: Illegal rule name 'NUMBER'. Already defined as a token\\n\"\n ))\n\n def test_yacc_unicode_literals(self):\n run_import(\"yacc_unicode_literals\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"Generating LALR tables\\n\"\n ))\n\n def test_yacc_unused(self):\n self.assertRaises(ply.yacc.YaccError,run_import,\"yacc_unused\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"yacc_unused.py:62: Symbol 'COMMA' used, but not defined as a token or a rule\\n\"\n \"Symbol 'COMMA' is unreachable\\n\"\n \"Symbol 'exprlist' is unreachable\\n\"\n ))\n def test_yacc_unused_rule(self):\n run_import(\"yacc_unused_rule\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"yacc_unused_rule.py:62: Rule 'integer' defined, but not used\\n\"\n \"There is 1 unused rule\\n\"\n \"Symbol 'integer' is unreachable\\n\"\n \"Generating LALR tables\\n\"\n ))\n\n def test_yacc_uprec(self):\n self.assertRaises(ply.yacc.YaccError,run_import,\"yacc_uprec\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"yacc_uprec.py:37: Nothing known about the precedence of 'UMINUS'\\n\"\n ))\n\n def test_yacc_uprec2(self):\n self.assertRaises(ply.yacc.YaccError,run_import,\"yacc_uprec2\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"yacc_uprec2.py:37: Syntax error. Nothing follows %prec\\n\"\n ))\n\n def test_yacc_prec1(self):\n self.assertRaises(ply.yacc.YaccError,run_import,\"yacc_prec1\")\n result = sys.stderr.getvalue()\n self.assert_(check_expected(result,\n \"Precedence rule 'left' defined for unknown symbol '+'\\n\"\n \"Precedence rule 'left' defined for unknown symbol '*'\\n\"\n \"Precedence rule 'left' defined for unknown symbol '-'\\n\"\n \"Precedence rule 'left' defined for unknown symbol '/'\\n\"\n ))\n\n def test_pkg_test1(self):\n from pkg_test1 import parser\n self.assertTrue(os.path.exists('pkg_test1/parsing/parsetab.py'))\n self.assertTrue(os.path.exists('pkg_test1/parsing/lextab.py'))\n self.assertTrue(os.path.exists('pkg_test1/parsing/parser.out'))\n r = parser.parse('3+4+5')\n self.assertEqual(r, 12)\n\n def test_pkg_test2(self):\n from pkg_test2 import parser\n self.assertTrue(os.path.exists('pkg_test2/parsing/calcparsetab.py'))\n self.assertTrue(os.path.exists('pkg_test2/parsing/calclextab.py'))\n self.assertTrue(os.path.exists('pkg_test2/parsing/parser.out'))\n r = parser.parse('3+4+5')\n self.assertEqual(r, 12)\n\n def test_pkg_test3(self):\n from pkg_test3 import parser\n self.assertTrue(os.path.exists('pkg_test3/generated/parsetab.py'))\n self.assertTrue(os.path.exists('pkg_test3/generated/lextab.py'))\n self.assertTrue(os.path.exists('pkg_test3/generated/parser.out'))\n r = parser.parse('3+4+5')\n self.assertEqual(r, 12)\n\n def test_pkg_test4(self):\n from pkg_test4 import parser\n self.assertFalse(os.path.exists('pkg_test4/parsing/parsetab.py'))\n self.assertFalse(os.path.exists('pkg_test4/parsing/lextab.py'))\n self.assertFalse(os.path.exists('pkg_test4/parsing/parser.out'))\n r = parser.parse('3+4+5')\n self.assertEqual(r, 12)\n\n def test_pkg_test5(self):\n from pkg_test5 import parser\n self.assertTrue(os.path.exists('pkg_test5/parsing/parsetab.py'))\n self.assertTrue(os.path.exists('pkg_test5/parsing/lextab.py'))\n self.assertTrue(os.path.exists('pkg_test5/parsing/parser.out'))\n r = parser.parse('3+4+5')\n self.assertEqual(r, 12)\n\n def test_pkg_test6(self):\n from pkg_test6 import parser\n self.assertTrue(os.path.exists('pkg_test6/parsing/parsetab.py'))\n self.assertTrue(os.path.exists('pkg_test6/parsing/lextab.py'))\n self.assertTrue(os.path.exists('pkg_test6/parsing/parser.out'))\n r = parser.parse('3+4+5')\n self.assertEqual(r, 12)\n\nunittest.main()\n" }, { "path": "xonsh/ply/test/yacc_badargs.py", "content": "# -----------------------------------------------------------------------------\n# yacc_badargs.py\n#\n# Rules with wrong # args\n# -----------------------------------------------------------------------------\nimport sys\nsys.tracebacklimit = 0\nsys.path.insert(0,\"..\")\nimport ply.yacc as yacc\n\nfrom calclex import tokens\n\n# Parsing rules\nprecedence = (\n ('left','PLUS','MINUS'),\n ('left','TIMES','DIVIDE'),\n ('right','UMINUS'),\n )\n\n# dictionary of names\nnames = { }\n\ndef p_statement_assign(t,s):\n 'statement : NAME EQUALS expression'\n names[t[1]] = t[3]\n\ndef p_statement_expr():\n 'statement : expression'\n print(t[1])\n\ndef p_expression_binop(t):\n '''expression : expression PLUS expression\n | expression MINUS expression\n | expression TIMES expression\n | expression DIVIDE expression'''\n if t[2] == '+' : t[0] = t[1] + t[3]\n elif t[2] == '-': t[0] = t[1] - t[3]\n elif t[2] == '*': t[0] = t[1] * t[3]\n elif t[2] == '/': t[0] = t[1] / t[3]\n\ndef p_expression_uminus(t):\n 'expression : MINUS expression %prec UMINUS'\n t[0] = -t[2]\n\ndef p_expression_group(t):\n 'expression : LPAREN expression RPAREN'\n t[0] = t[2]\n\ndef p_expression_number(t):\n 'expression : NUMBER'\n t[0] = t[1]\n\ndef p_expression_name(t):\n 'expression : NAME'\n try:\n t[0] = names[t[1]]\n except LookupError:\n print(\"Undefined name '%s'\" % t[1])\n t[0] = 0\n\ndef p_error(t):\n print(\"Syntax error at '%s'\" % t.value)\n\nyacc.yacc()\n\n\n\n\n" }, { "path": "xonsh/ply/test/yacc_badid.py", "content": "# -----------------------------------------------------------------------------\n# yacc_badid.py\n#\n# Attempt to define a rule with a bad-identifier name\n# -----------------------------------------------------------------------------\nimport sys\n\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\nimport ply.yacc as yacc\n\nfrom calclex import tokens\n\n# Parsing rules\nprecedence = (\n ('left','PLUS','MINUS'),\n ('left','TIMES','DIVIDE'),\n ('right','UMINUS'),\n )\n\n# dictionary of names\nnames = { }\n\ndef p_statement_assign(t):\n 'statement : NAME EQUALS expression'\n names[t[1]] = t[3]\n\ndef p_statement_expr(t):\n 'statement : expression'\n print(t[1])\n\ndef p_statement_expr2(t):\n 'statement : bad&rule'\n pass\n\ndef p_badrule(t):\n 'bad&rule : expression'\n pass\n\n\ndef p_expression_binop(t):\n '''expression : expression PLUS expression\n | expression MINUS expression\n | expression TIMES expression\n | expression DIVIDE expression'''\n if t[2] == '+' : t[0] = t[1] + t[3]\n elif t[2] == '-': t[0] = t[1] - t[3]\n elif t[2] == '*': t[0] = t[1] * t[3]\n elif t[2] == '/': t[0] = t[1] / t[3]\n\ndef p_expression_uminus(t):\n 'expression : MINUS expression %prec UMINUS'\n t[0] = -t[2]\n\ndef p_expression_group(t):\n 'expression : LPAREN expression RPAREN'\n t[0] = t[2]\n\ndef p_expression_number(t):\n 'expression : NUMBER'\n t[0] = t[1]\n\ndef p_expression_name(t):\n 'expression : NAME'\n try:\n t[0] = names[t[1]]\n except LookupError:\n print(\"Undefined name '%s'\" % t[1])\n t[0] = 0\n\ndef p_error(t):\n pass\n\nyacc.yacc()\n\n\n\n\n" }, { "path": "xonsh/ply/test/yacc_badprec.py", "content": "# -----------------------------------------------------------------------------\n# yacc_badprec.py\n#\n# Bad precedence specifier\n# -----------------------------------------------------------------------------\nimport sys\n\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\nimport ply.yacc as yacc\n\nfrom calclex import tokens\n\n# Parsing rules\nprecedence = \"blah\"\n\n# dictionary of names\nnames = { }\n\ndef p_statement_assign(t):\n 'statement : NAME EQUALS expression'\n names[t[1]] = t[3]\n\ndef p_statement_expr(t):\n 'statement : expression'\n print(t[1])\n\ndef p_expression_binop(t):\n '''expression : expression PLUS expression\n | expression MINUS expression\n | expression TIMES expression\n | expression DIVIDE expression'''\n if t[2] == '+' : t[0] = t[1] + t[3]\n elif t[2] == '-': t[0] = t[1] - t[3]\n elif t[2] == '*': t[0] = t[1] * t[3]\n elif t[2] == '/': t[0] = t[1] / t[3]\n\ndef p_expression_uminus(t):\n 'expression : MINUS expression %prec UMINUS'\n t[0] = -t[2]\n\ndef p_expression_group(t):\n 'expression : LPAREN expression RPAREN'\n t[0] = t[2]\n\ndef p_expression_number(t):\n 'expression : NUMBER'\n t[0] = t[1]\n\ndef p_expression_name(t):\n 'expression : NAME'\n try:\n t[0] = names[t[1]]\n except LookupError:\n print(\"Undefined name '%s'\" % t[1])\n t[0] = 0\n\ndef p_error(t):\n print(\"Syntax error at '%s'\" % t.value)\n\nyacc.yacc()\n\n\n\n\n" }, { "path": "xonsh/ply/test/yacc_badprec2.py", "content": "# -----------------------------------------------------------------------------\n# yacc_badprec2.py\n#\n# Bad precedence\n# -----------------------------------------------------------------------------\nimport sys\n\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\nimport ply.yacc as yacc\n\nfrom calclex import tokens\n\n# Parsing rules\nprecedence = (\n 42,\n ('left','TIMES','DIVIDE'),\n ('right','UMINUS'),\n )\n\n# dictionary of names\nnames = { }\n\ndef p_statement_assign(t):\n 'statement : NAME EQUALS expression'\n names[t[1]] = t[3]\n\ndef p_statement_expr(t):\n 'statement : expression'\n print(t[1])\n\ndef p_expression_binop(t):\n '''expression : expression PLUS expression\n | expression MINUS expression\n | expression TIMES expression\n | expression DIVIDE expression'''\n if t[2] == '+' : t[0] = t[1] + t[3]\n elif t[2] == '-': t[0] = t[1] - t[3]\n elif t[2] == '*': t[0] = t[1] * t[3]\n elif t[2] == '/': t[0] = t[1] / t[3]\n\ndef p_expression_uminus(t):\n 'expression : MINUS expression %prec UMINUS'\n t[0] = -t[2]\n\ndef p_expression_group(t):\n 'expression : LPAREN expression RPAREN'\n t[0] = t[2]\n\ndef p_expression_number(t):\n 'expression : NUMBER'\n t[0] = t[1]\n\ndef p_expression_name(t):\n 'expression : NAME'\n try:\n t[0] = names[t[1]]\n except LookupError:\n print(\"Undefined name '%s'\" % t[1])\n t[0] = 0\n\ndef p_error(t):\n print(\"Syntax error at '%s'\" % t.value)\n\nyacc.yacc()\n\n\n\n\n" }, { "path": "xonsh/ply/test/yacc_badprec3.py", "content": "# -----------------------------------------------------------------------------\n# yacc_badprec3.py\n#\n# Bad precedence\n# -----------------------------------------------------------------------------\nimport sys\n\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\nimport ply.yacc as yacc\n\nfrom calclex import tokens\n\n# Parsing rules\nprecedence = (\n ('left','PLUS','MINUS'),\n ('left','TIMES','DIVIDE','MINUS'),\n ('right','UMINUS'),\n )\n\n# dictionary of names\nnames = { }\n\ndef p_statement_assign(t):\n 'statement : NAME EQUALS expression'\n names[t[1]] = t[3]\n\ndef p_statement_expr(t):\n 'statement : expression'\n print(t[1])\n\ndef p_expression_binop(t):\n '''expression : expression PLUS expression\n | expression MINUS expression\n | expression TIMES expression\n | expression DIVIDE expression'''\n if t[2] == '+' : t[0] = t[1] + t[3]\n elif t[2] == '-': t[0] = t[1] - t[3]\n elif t[2] == '*': t[0] = t[1] * t[3]\n elif t[3] == '/': t[0] = t[1] / t[3]\n\ndef p_expression_uminus(t):\n 'expression : MINUS expression %prec UMINUS'\n t[0] = -t[2]\n\ndef p_expression_group(t):\n 'expression : LPAREN expression RPAREN'\n t[0] = t[2]\n\ndef p_expression_number(t):\n 'expression : NUMBER'\n t[0] = t[1]\n\ndef p_expression_name(t):\n 'expression : NAME'\n try:\n t[0] = names[t[1]]\n except LookupError:\n print(\"Undefined name '%s'\" % t[1])\n t[0] = 0\n\ndef p_error(t):\n print(\"Syntax error at '%s'\" % t.value)\n\nyacc.yacc()\n\n\n\n\n" }, { "path": "xonsh/ply/test/yacc_badrule.py", "content": "# -----------------------------------------------------------------------------\n# yacc_badrule.py\n#\n# Syntax problems in the rule strings\n# -----------------------------------------------------------------------------\nimport sys\n\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\nimport ply.yacc as yacc\n\nfrom calclex import tokens\n\n# Parsing rules\nprecedence = (\n ('left','PLUS','MINUS'),\n ('left','TIMES','DIVIDE'),\n ('right','UMINUS'),\n )\n\n# dictionary of names\nnames = { }\n\ndef p_statement_assign(t):\n 'statement NAME EQUALS expression'\n names[t[1]] = t[3]\n\ndef p_statement_expr(t):\n 'statement'\n print(t[1])\n\ndef p_expression_binop(t):\n '''expression : expression PLUS expression\n expression MINUS expression\n | expression TIMES expression\n | expression DIVIDE expression'''\n if t[2] == '+' : t[0] = t[1] + t[3]\n elif t[2] == '-': t[0] = t[1] - t[3]\n elif t[2] == '*': t[0] = t[1] * t[3]\n elif t[2] == '/': t[0] = t[1] / t[3]\n\ndef p_expression_uminus(t):\n 'expression: MINUS expression %prec UMINUS'\n t[0] = -t[2]\n\ndef p_expression_group(t):\n 'expression : LPAREN expression RPAREN'\n t[0] = t[2]\n\ndef p_expression_number(t):\n 'expression : NUMBER'\n t[0] = t[1]\n\ndef p_expression_name(t):\n 'expression : NAME'\n try:\n t[0] = names[t[1]]\n except LookupError:\n print(\"Undefined name '%s'\" % t[1])\n t[0] = 0\n\ndef p_error(t):\n print(\"Syntax error at '%s'\" % t.value)\n\nyacc.yacc()\n\n\n\n\n" }, { "path": "xonsh/ply/test/yacc_badtok.py", "content": "# -----------------------------------------------------------------------------\n# yacc_badtok.py\n#\n# A grammar, but tokens is a bad datatype\n# -----------------------------------------------------------------------------\n\nimport sys\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\nimport ply.yacc as yacc\n\ntokens = \"Hello\"\n\n# Parsing rules\nprecedence = (\n ('left','PLUS','MINUS'),\n ('left','TIMES','DIVIDE'),\n ('right','UMINUS'),\n )\n\n# dictionary of names\nnames = { }\n\ndef p_statement_assign(t):\n 'statement : NAME EQUALS expression'\n names[t[1]] = t[3]\n\ndef p_statement_expr(t):\n 'statement : expression'\n print(t[1])\n\ndef p_expression_binop(t):\n '''expression : expression PLUS expression\n | expression MINUS expression\n | expression TIMES expression\n | expression DIVIDE expression'''\n if t[2] == '+' : t[0] = t[1] + t[3]\n elif t[2] == '-': t[0] = t[1] - t[3]\n elif t[2] == '*': t[0] = t[1] * t[3]\n elif t[2] == '/': t[0] = t[1] / t[3]\n\ndef p_expression_uminus(t):\n 'expression : MINUS expression %prec UMINUS'\n t[0] = -t[2]\n\ndef p_expression_group(t):\n 'expression : LPAREN expression RPAREN'\n t[0] = t[2]\n\ndef p_expression_number(t):\n 'expression : NUMBER'\n t[0] = t[1]\n\ndef p_expression_name(t):\n 'expression : NAME'\n try:\n t[0] = names[t[1]]\n except LookupError:\n print(\"Undefined name '%s'\" % t[1])\n t[0] = 0\n\ndef p_error(t):\n print(\"Syntax error at '%s'\" % t.value)\n\nyacc.yacc()\n\n\n\n\n" }, { "path": "xonsh/ply/test/yacc_dup.py", "content": "# -----------------------------------------------------------------------------\n# yacc_dup.py\n#\n# Duplicated rule name\n# -----------------------------------------------------------------------------\nimport sys\n\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\nimport ply.yacc as yacc\n\nfrom calclex import tokens\n\n# Parsing rules\nprecedence = (\n ('left','PLUS','MINUS'),\n ('left','TIMES','DIVIDE'),\n ('right','UMINUS'),\n )\n\n# dictionary of names\nnames = { }\n\ndef p_statement(t):\n 'statement : NAME EQUALS expression'\n names[t[1]] = t[3]\n\ndef p_statement(t):\n 'statement : expression'\n print(t[1])\n\ndef p_expression_binop(t):\n '''expression : expression PLUS expression\n | expression MINUS expression\n | expression TIMES expression\n | expression DIVIDE expression'''\n if t[2] == '+' : t[0] = t[1] + t[3]\n elif t[2] == '-': t[0] = t[1] - t[3]\n elif t[2] == '*': t[0] = t[1] * t[3]\n elif t[2] == '/': t[0] = t[1] / t[3]\n\ndef p_expression_uminus(t):\n 'expression : MINUS expression %prec UMINUS'\n t[0] = -t[2]\n\ndef p_expression_group(t):\n 'expression : LPAREN expression RPAREN'\n t[0] = t[2]\n\ndef p_expression_number(t):\n 'expression : NUMBER'\n t[0] = t[1]\n\ndef p_expression_name(t):\n 'expression : NAME'\n try:\n t[0] = names[t[1]]\n except LookupError:\n print(\"Undefined name '%s'\" % t[1])\n t[0] = 0\n\ndef p_error(t):\n print(\"Syntax error at '%s'\" % t.value)\n\nyacc.yacc()\n\n\n\n\n" }, { "path": "xonsh/ply/test/yacc_error1.py", "content": "# -----------------------------------------------------------------------------\n# yacc_error1.py\n#\n# Bad p_error() function\n# -----------------------------------------------------------------------------\nimport sys\n\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\nimport ply.yacc as yacc\n\nfrom calclex import tokens\n\n# Parsing rules\nprecedence = (\n ('left','PLUS','MINUS'),\n ('left','TIMES','DIVIDE'),\n ('right','UMINUS'),\n )\n\n# dictionary of names\nnames = { }\n\ndef p_statement_assign(t):\n 'statement : NAME EQUALS expression'\n names[t[1]] = t[3]\n\ndef p_statement_expr(t):\n 'statement : expression'\n print(t[1])\n\ndef p_expression_binop(t):\n '''expression : expression PLUS expression\n | expression MINUS expression\n | expression TIMES expression\n | expression DIVIDE expression'''\n if t[2] == '+' : t[0] = t[1] + t[3]\n elif t[2] == '-': t[0] = t[1] - t[3]\n elif t[2] == '*': t[0] = t[1] * t[3]\n elif t[2] == '/': t[0] = t[1] / t[3]\n\ndef p_expression_uminus(t):\n 'expression : MINUS expression %prec UMINUS'\n t[0] = -t[2]\n\ndef p_expression_group(t):\n 'expression : LPAREN expression RPAREN'\n t[0] = t[2]\n\ndef p_expression_number(t):\n 'expression : NUMBER'\n t[0] = t[1]\n\ndef p_expression_name(t):\n 'expression : NAME'\n try:\n t[0] = names[t[1]]\n except LookupError:\n print(\"Undefined name '%s'\" % t[1])\n t[0] = 0\n\ndef p_error(t,s):\n print(\"Syntax error at '%s'\" % t.value)\n\nyacc.yacc()\n\n\n\n\n" }, { "path": "xonsh/ply/test/yacc_error2.py", "content": "# -----------------------------------------------------------------------------\n# yacc_error2.py\n#\n# Bad p_error() function\n# -----------------------------------------------------------------------------\nimport sys\n\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\nimport ply.yacc as yacc\n\nfrom calclex import tokens\n\n# Parsing rules\nprecedence = (\n ('left','PLUS','MINUS'),\n ('left','TIMES','DIVIDE'),\n ('right','UMINUS'),\n )\n\n# dictionary of names\nnames = { }\n\ndef p_statement_assign(t):\n 'statement : NAME EQUALS expression'\n names[t[1]] = t[3]\n\ndef p_statement_expr(t):\n 'statement : expression'\n print(t[1])\n\ndef p_expression_binop(t):\n '''expression : expression PLUS expression\n | expression MINUS expression\n | expression TIMES expression\n | expression DIVIDE expression'''\n if t[2] == '+' : t[0] = t[1] + t[3]\n elif t[2] == '-': t[0] = t[1] - t[3]\n elif t[2] == '*': t[0] = t[1] * t[3]\n elif t[2] == '/': t[0] = t[1] / t[3]\n\ndef p_expression_uminus(t):\n 'expression : MINUS expression %prec UMINUS'\n t[0] = -t[2]\n\ndef p_expression_group(t):\n 'expression : LPAREN expression RPAREN'\n t[0] = t[2]\n\ndef p_expression_number(t):\n 'expression : NUMBER'\n t[0] = t[1]\n\ndef p_expression_name(t):\n 'expression : NAME'\n try:\n t[0] = names[t[1]]\n except LookupError:\n print(\"Undefined name '%s'\" % t[1])\n t[0] = 0\n\ndef p_error():\n print(\"Syntax error at '%s'\" % t.value)\n\nyacc.yacc()\n\n\n\n\n" }, { "path": "xonsh/ply/test/yacc_error3.py", "content": "# -----------------------------------------------------------------------------\n# yacc_error3.py\n#\n# Bad p_error() function\n# -----------------------------------------------------------------------------\nimport sys\n\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\nimport ply.yacc as yacc\n\nfrom calclex import tokens\n\n# Parsing rules\nprecedence = (\n ('left','PLUS','MINUS'),\n ('left','TIMES','DIVIDE'),\n ('right','UMINUS'),\n )\n\n# dictionary of names\nnames = { }\n\ndef p_statement_assign(t):\n 'statement : NAME EQUALS expression'\n names[t[1]] = t[3]\n\ndef p_statement_expr(t):\n 'statement : expression'\n print(t[1])\n\ndef p_expression_binop(t):\n '''expression : expression PLUS expression\n | expression MINUS expression\n | expression TIMES expression\n | expression DIVIDE expression'''\n if t[2] == '+' : t[0] = t[1] + t[3]\n elif t[2] == '-': t[0] = t[1] - t[3]\n elif t[2] == '*': t[0] = t[1] * t[3]\n elif t[2] == '/': t[0] = t[1] / t[3]\n\ndef p_expression_uminus(t):\n 'expression : MINUS expression %prec UMINUS'\n t[0] = -t[2]\n\ndef p_expression_group(t):\n 'expression : LPAREN expression RPAREN'\n t[0] = t[2]\n\ndef p_expression_number(t):\n 'expression : NUMBER'\n t[0] = t[1]\n\ndef p_expression_name(t):\n 'expression : NAME'\n try:\n t[0] = names[t[1]]\n except LookupError:\n print(\"Undefined name '%s'\" % t[1])\n t[0] = 0\n\np_error = \"blah\"\n\nyacc.yacc()\n\n\n\n\n" }, { "path": "xonsh/ply/test/yacc_error4.py", "content": "# -----------------------------------------------------------------------------\n# yacc_error4.py\n#\n# Attempt to define a rule named 'error'\n# -----------------------------------------------------------------------------\nimport sys\n\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\nimport ply.yacc as yacc\n\nfrom calclex import tokens\n\n# Parsing rules\nprecedence = (\n ('left','PLUS','MINUS'),\n ('left','TIMES','DIVIDE'),\n ('right','UMINUS'),\n )\n\n# dictionary of names\nnames = { }\n\ndef p_statement_assign(t):\n 'statement : NAME EQUALS expression'\n names[t[1]] = t[3]\n\ndef p_statement_expr(t):\n 'statement : expression'\n print(t[1])\n\ndef p_expression_binop(t):\n '''expression : expression PLUS expression\n | expression MINUS expression\n | expression TIMES expression\n | expression DIVIDE expression'''\n if t[2] == '+' : t[0] = t[1] + t[3]\n elif t[2] == '-': t[0] = t[1] - t[3]\n elif t[2] == '*': t[0] = t[1] * t[3]\n elif t[2] == '/': t[0] = t[1] / t[3]\n\ndef p_expression_uminus(t):\n 'expression : MINUS expression %prec UMINUS'\n t[0] = -t[2]\n\ndef p_expression_group(t):\n 'expression : LPAREN expression RPAREN'\n t[0] = t[2]\n\ndef p_expression_number(t):\n 'expression : NUMBER'\n t[0] = t[1]\n\ndef p_expression_name(t):\n 'expression : NAME'\n try:\n t[0] = names[t[1]]\n except LookupError:\n print(\"Undefined name '%s'\" % t[1])\n t[0] = 0\n\ndef p_error_handler(t):\n 'error : NAME'\n pass\n\ndef p_error(t):\n pass\n\nyacc.yacc()\n\n\n\n\n" }, { "path": "xonsh/ply/test/yacc_error5.py", "content": "# -----------------------------------------------------------------------------\n# yacc_error5.py\n#\n# Lineno and position tracking with error tokens\n# -----------------------------------------------------------------------------\nimport sys\n\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\nimport ply.yacc as yacc\n\nfrom calclex import tokens\n\n# Parsing rules\nprecedence = (\n ('left','PLUS','MINUS'),\n ('left','TIMES','DIVIDE'),\n ('right','UMINUS'),\n )\n\n# dictionary of names\nnames = { }\n\ndef p_statement_assign(t):\n 'statement : NAME EQUALS expression'\n names[t[1]] = t[3]\n\ndef p_statement_assign_error(t):\n 'statement : NAME EQUALS error'\n line_start, line_end = t.linespan(3)\n pos_start, pos_end = t.lexspan(3)\n print(\"Assignment Error at %d:%d to %d:%d\" % (line_start,pos_start,line_end,pos_end))\n\ndef p_statement_expr(t):\n 'statement : expression'\n print(t[1])\n\ndef p_expression_binop(t):\n '''expression : expression PLUS expression\n | expression MINUS expression\n | expression TIMES expression\n | expression DIVIDE expression'''\n if t[2] == '+' : t[0] = t[1] + t[3]\n elif t[2] == '-': t[0] = t[1] - t[3]\n elif t[2] == '*': t[0] = t[1] * t[3]\n elif t[2] == '/': t[0] = t[1] / t[3]\n\ndef p_expression_uminus(t):\n 'expression : MINUS expression %prec UMINUS'\n t[0] = -t[2]\n\ndef p_expression_group(t):\n 'expression : LPAREN expression RPAREN'\n line_start, line_end = t.linespan(2)\n pos_start, pos_end = t.lexspan(2)\n print(\"Group at %d:%d to %d:%d\" % (line_start,pos_start, line_end, pos_end))\n t[0] = t[2]\n\ndef p_expression_group_error(t):\n 'expression : LPAREN error RPAREN'\n line_start, line_end = t.linespan(2)\n pos_start, pos_end = t.lexspan(2)\n print(\"Syntax error at %d:%d to %d:%d\" % (line_start,pos_start, line_end, pos_end))\n t[0] = 0\n \ndef p_expression_number(t):\n 'expression : NUMBER'\n t[0] = t[1]\n\ndef p_expression_name(t):\n 'expression : NAME'\n try:\n t[0] = names[t[1]]\n except LookupError:\n print(\"Undefined name '%s'\" % t[1])\n t[0] = 0\n\ndef p_error(t):\n print(\"Syntax error at '%s'\" % t.value)\n\nparser = yacc.yacc()\nimport calclex\ncalclex.lexer.lineno=1\nparser.parse(\"\"\"\na = 3 +\n(4*5) +\n(a b c) +\n+ 6 + 7\n\"\"\", tracking=True)\n\n\n\n\n\n\n" }, { "path": "xonsh/ply/test/yacc_error6.py", "content": "# -----------------------------------------------------------------------------\n# yacc_error6.py\n#\n# Panic mode recovery test\n# -----------------------------------------------------------------------------\nimport sys\n\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\nimport ply.yacc as yacc\n\nfrom calclex import tokens\n\n# Parsing rules\nprecedence = (\n ('left','PLUS','MINUS'),\n ('left','TIMES','DIVIDE'),\n ('right','UMINUS'),\n )\n\ndef p_statements(t):\n 'statements : statements statement'\n pass\n\ndef p_statements_1(t):\n 'statements : statement'\n pass\n\ndef p_statement_assign(p):\n 'statement : LPAREN NAME EQUALS expression RPAREN'\n print(\"%s=%s\" % (p[2],p[4]))\n\ndef p_statement_expr(t):\n 'statement : LPAREN expression RPAREN'\n print(t[1])\n\ndef p_expression_binop(t):\n '''expression : expression PLUS expression\n | expression MINUS expression\n | expression TIMES expression\n | expression DIVIDE expression'''\n if t[2] == '+' : t[0] = t[1] + t[3]\n elif t[2] == '-': t[0] = t[1] - t[3]\n elif t[2] == '*': t[0] = t[1] * t[3]\n elif t[2] == '/': t[0] = t[1] / t[3]\n\ndef p_expression_uminus(t):\n 'expression : MINUS expression %prec UMINUS'\n t[0] = -t[2]\n\ndef p_expression_number(t):\n 'expression : NUMBER'\n t[0] = t[1]\n\ndef p_error(p):\n if p:\n print(\"Line %d: Syntax error at '%s'\" % (p.lineno, p.value))\n # Scan ahead looking for a name token\n while True:\n tok = parser.token()\n if not tok or tok.type == 'RPAREN':\n break\n if tok:\n parser.restart()\n return None\n\nparser = yacc.yacc()\nimport calclex\ncalclex.lexer.lineno=1\n\nparser.parse(\"\"\"\n(a = 3 + 4)\n(b = 4 + * 5 - 6 + *)\n(c = 10 + 11)\n\"\"\")\n\n\n\n\n\n\n" }, { "path": "xonsh/ply/test/yacc_error7.py", "content": "# -----------------------------------------------------------------------------\n# yacc_error7.py\n#\n# Panic mode recovery test using deprecated functionality\n# -----------------------------------------------------------------------------\nimport sys\n\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\nimport ply.yacc as yacc\n\nfrom calclex import tokens\n\n# Parsing rules\nprecedence = (\n ('left','PLUS','MINUS'),\n ('left','TIMES','DIVIDE'),\n ('right','UMINUS'),\n )\n\ndef p_statements(t):\n 'statements : statements statement'\n pass\n\ndef p_statements_1(t):\n 'statements : statement'\n pass\n\ndef p_statement_assign(p):\n 'statement : LPAREN NAME EQUALS expression RPAREN'\n print(\"%s=%s\" % (p[2],p[4]))\n\ndef p_statement_expr(t):\n 'statement : LPAREN expression RPAREN'\n print(t[1])\n\ndef p_expression_binop(t):\n '''expression : expression PLUS expression\n | expression MINUS expression\n | expression TIMES expression\n | expression DIVIDE expression'''\n if t[2] == '+' : t[0] = t[1] + t[3]\n elif t[2] == '-': t[0] = t[1] - t[3]\n elif t[2] == '*': t[0] = t[1] * t[3]\n elif t[2] == '/': t[0] = t[1] / t[3]\n\ndef p_expression_uminus(t):\n 'expression : MINUS expression %prec UMINUS'\n t[0] = -t[2]\n\ndef p_expression_number(t):\n 'expression : NUMBER'\n t[0] = t[1]\n\ndef p_error(p):\n if p:\n print(\"Line %d: Syntax error at '%s'\" % (p.lineno, p.value))\n # Scan ahead looking for a name token\n while True:\n tok = yacc.token()\n if not tok or tok.type == 'RPAREN':\n break\n if tok:\n yacc.restart()\n return None\n\nparser = yacc.yacc()\nimport calclex\ncalclex.lexer.lineno=1\n\nparser.parse(\"\"\"\n(a = 3 + 4)\n(b = 4 + * 5 - 6 + *)\n(c = 10 + 11)\n\"\"\")\n\n\n\n\n\n\n" }, { "path": "xonsh/ply/test/yacc_inf.py", "content": "# -----------------------------------------------------------------------------\n# yacc_inf.py\n#\n# Infinite recursion\n# -----------------------------------------------------------------------------\nimport sys\n\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\nimport ply.yacc as yacc\n\nfrom calclex import tokens\n\n# Parsing rules\nprecedence = (\n ('left','PLUS','MINUS'),\n ('left','TIMES','DIVIDE'),\n ('right','UMINUS'),\n )\n\n# dictionary of names\nnames = { }\n\ndef p_statement_assign(t):\n 'statement : NAME EQUALS expression'\n names[t[1]] = t[3]\n\ndef p_statement_expr(t):\n 'statement : expression'\n print(t[1])\n\ndef p_expression_binop(t):\n '''expression : expression PLUS expression\n | expression MINUS expression\n | expression TIMES expression\n | expression DIVIDE expression'''\n if t[2] == '+' : t[0] = t[1] + t[3]\n elif t[2] == '-': t[0] = t[1] - t[3]\n elif t[2] == '*': t[0] = t[1] * t[3]\n elif t[2] == '/': t[0] = t[1] / t[3]\n\ndef p_expression_uminus(t):\n 'expression : MINUS expression %prec UMINUS'\n t[0] = -t[2]\n\ndef p_expression_group(t):\n 'expression : LPAREN expression RPAREN'\n t[0] = t[2]\n\ndef p_error(t):\n print(\"Syntax error at '%s'\" % t.value)\n\nyacc.yacc()\n\n\n\n\n" }, { "path": "xonsh/ply/test/yacc_literal.py", "content": "# -----------------------------------------------------------------------------\n# yacc_literal.py\n#\n# Grammar with bad literal characters\n# -----------------------------------------------------------------------------\nimport sys\n\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\nimport ply.yacc as yacc\n\nfrom calclex import tokens\n\n# Parsing rules\nprecedence = (\n ('left','+','-'),\n ('left','*','/'),\n ('right','UMINUS'),\n )\n\n# dictionary of names\nnames = { }\n\ndef p_statement_assign(t):\n 'statement : NAME EQUALS expression'\n names[t[1]] = t[3]\n\ndef p_statement_expr(t):\n 'statement : expression'\n print(t[1])\n\ndef p_expression_binop(t):\n '''expression : expression '+' expression\n | expression '-' expression\n | expression '*' expression\n | expression '/' expression\n | expression '**' expression '''\n if t[2] == '+' : t[0] = t[1] + t[3]\n elif t[2] == '-': t[0] = t[1] - t[3]\n elif t[2] == '*': t[0] = t[1] * t[3]\n elif t[2] == '/': t[0] = t[1] / t[3]\n\ndef p_expression_uminus(t):\n 'expression : MINUS expression %prec UMINUS'\n t[0] = -t[2]\n\ndef p_expression_group(t):\n 'expression : LPAREN expression RPAREN'\n t[0] = t[2]\n\ndef p_expression_number(t):\n 'expression : NUMBER'\n t[0] = t[1]\n\ndef p_expression_name(t):\n 'expression : NAME'\n try:\n t[0] = names[t[1]]\n except LookupError:\n print(\"Undefined name '%s'\" % t[1])\n t[0] = 0\n\ndef p_error(t):\n print(\"Syntax error at '%s'\" % t.value)\n\nyacc.yacc()\n\n\n\n\n" }, { "path": "xonsh/ply/test/yacc_misplaced.py", "content": "# -----------------------------------------------------------------------------\n# yacc_misplaced.py\n#\n# A misplaced | in grammar rules\n# -----------------------------------------------------------------------------\nimport sys\n\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\nimport ply.yacc as yacc\n\nfrom calclex import tokens\n\n# Parsing rules\nprecedence = (\n ('left','PLUS','MINUS'),\n ('left','TIMES','DIVIDE'),\n ('right','UMINUS'),\n )\n\n# dictionary of names\nnames = { }\n\ndef p_statement_assign(t):\n 'statement : NAME EQUALS expression'\n names[t[1]] = t[3]\n\ndef p_statement_expr(t):\n 'statement : expression'\n print(t[1])\n\ndef p_expression_binop(t):\n ''' | expression PLUS expression\n | expression MINUS expression\n | expression TIMES expression\n | expression DIVIDE expression'''\n if t[2] == '+' : t[0] = t[1] + t[3]\n elif t[2] == '-': t[0] = t[1] - t[3]\n elif t[2] == '*': t[0] = t[1] * t[3]\n elif t[2] == '/': t[0] = t[1] / t[3]\n\ndef p_expression_uminus(t):\n 'expression : MINUS expression %prec UMINUS'\n t[0] = -t[2]\n\ndef p_expression_group(t):\n 'expression : LPAREN expression RPAREN'\n t[0] = t[2]\n\ndef p_expression_number(t):\n 'expression : NUMBER'\n t[0] = t[1]\n\ndef p_expression_name(t):\n 'expression : NAME'\n try:\n t[0] = names[t[1]]\n except LookupError:\n print(\"Undefined name '%s'\" % t[1])\n t[0] = 0\n\ndef p_error(t):\n print(\"Syntax error at '%s'\" % t.value)\n\nyacc.yacc()\n\n\n\n\n" }, { "path": "xonsh/ply/test/yacc_missing1.py", "content": "# -----------------------------------------------------------------------------\n# yacc_missing1.py\n#\n# Grammar with a missing rule\n# -----------------------------------------------------------------------------\nimport sys\n\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\nimport ply.yacc as yacc\n\nfrom calclex import tokens\n\n# Parsing rules\nprecedence = (\n ('left','PLUS','MINUS'),\n ('left','TIMES','DIVIDE'),\n ('right','UMINUS'),\n )\n\n# dictionary of names\nnames = { }\n\ndef p_statement_assign(t):\n 'statement : location EQUALS expression'\n names[t[1]] = t[3]\n\ndef p_statement_expr(t):\n 'statement : expression'\n print(t[1])\n\ndef p_expression_binop(t):\n '''expression : expression PLUS expression\n | expression MINUS expression\n | expression TIMES expression\n | expression DIVIDE expression'''\n if t[2] == '+' : t[0] = t[1] + t[3]\n elif t[2] == '-': t[0] = t[1] - t[3]\n elif t[2] == '*': t[0] = t[1] * t[3]\n elif t[2] == '/': t[0] = t[1] / t[3]\n\ndef p_expression_uminus(t):\n 'expression : MINUS expression %prec UMINUS'\n t[0] = -t[2]\n\ndef p_expression_group(t):\n 'expression : LPAREN expression RPAREN'\n t[0] = t[2]\n\ndef p_expression_number(t):\n 'expression : NUMBER'\n t[0] = t[1]\n\ndef p_expression_name(t):\n 'expression : NAME'\n try:\n t[0] = names[t[1]]\n except LookupError:\n print(\"Undefined name '%s'\" % t[1])\n t[0] = 0\n\ndef p_error(t):\n print(\"Syntax error at '%s'\" % t.value)\n\nyacc.yacc()\n\n\n\n\n" }, { "path": "xonsh/ply/test/yacc_nested.py", "content": "import sys\n\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\n\nfrom ply import lex, yacc\n\nt_A = 'A'\nt_B = 'B'\nt_C = 'C'\n\ntokens = ('A', 'B', 'C')\n\nthe_lexer = lex.lex()\n\ndef t_error(t):\n pass\n\ndef p_error(p):\n pass\n\ndef p_start(t):\n '''start : A nest C'''\n pass\n\ndef p_nest(t):\n '''nest : B'''\n print(t[-1])\n\nthe_parser = yacc.yacc(debug = False, write_tables = False)\n\nthe_parser.parse('ABC', the_lexer)\nthe_parser.parse('ABC', the_lexer, tracking=True)\nthe_parser.parse('ABC', the_lexer, tracking=True, debug=1)\n" }, { "path": "xonsh/ply/test/yacc_nodoc.py", "content": "# -----------------------------------------------------------------------------\n# yacc_nodoc.py\n#\n# Rule with a missing doc-string\n# -----------------------------------------------------------------------------\nimport sys\n\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\nimport ply.yacc as yacc\n\nfrom calclex import tokens\n\n# Parsing rules\nprecedence = (\n ('left','PLUS','MINUS'),\n ('left','TIMES','DIVIDE'),\n ('right','UMINUS'),\n )\n\n# dictionary of names\nnames = { }\n\ndef p_statement_assign(t):\n 'statement : NAME EQUALS expression'\n names[t[1]] = t[3]\n\ndef p_statement_expr(t):\n print(t[1])\n\ndef p_expression_binop(t):\n '''expression : expression PLUS expression\n | expression MINUS expression\n | expression TIMES expression\n | expression DIVIDE expression'''\n if t[2] == '+' : t[0] = t[1] + t[3]\n elif t[2] == '-': t[0] = t[1] - t[3]\n elif t[2] == '*': t[0] = t[1] * t[3]\n elif t[2] == '/': t[0] = t[1] / t[3]\n\ndef p_expression_uminus(t):\n 'expression : MINUS expression %prec UMINUS'\n t[0] = -t[2]\n\ndef p_expression_group(t):\n 'expression : LPAREN expression RPAREN'\n t[0] = t[2]\n\ndef p_expression_number(t):\n 'expression : NUMBER'\n t[0] = t[1]\n\ndef p_expression_name(t):\n 'expression : NAME'\n try:\n t[0] = names[t[1]]\n except LookupError:\n print(\"Undefined name '%s'\" % t[1])\n t[0] = 0\n\ndef p_error(t):\n print(\"Syntax error at '%s'\" % t.value)\n\nyacc.yacc()\n\n\n\n\n" }, { "path": "xonsh/ply/test/yacc_noerror.py", "content": "# -----------------------------------------------------------------------------\n# yacc_noerror.py\n#\n# No p_error() rule defined.\n# -----------------------------------------------------------------------------\nimport sys\n\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\nimport ply.yacc as yacc\n\nfrom calclex import tokens\n\n# Parsing rules\nprecedence = (\n ('left','PLUS','MINUS'),\n ('left','TIMES','DIVIDE'),\n ('right','UMINUS'),\n )\n\n# dictionary of names\nnames = { }\n\ndef p_statement_assign(t):\n 'statement : NAME EQUALS expression'\n names[t[1]] = t[3]\n\ndef p_statement_expr(t):\n 'statement : expression'\n print(t[1])\n\ndef p_expression_binop(t):\n '''expression : expression PLUS expression\n | expression MINUS expression\n | expression TIMES expression\n | expression DIVIDE expression'''\n if t[2] == '+' : t[0] = t[1] + t[3]\n elif t[2] == '-': t[0] = t[1] - t[3]\n elif t[2] == '*': t[0] = t[1] * t[3]\n elif t[2] == '/': t[0] = t[1] / t[3]\n\ndef p_expression_uminus(t):\n 'expression : MINUS expression %prec UMINUS'\n t[0] = -t[2]\n\ndef p_expression_group(t):\n 'expression : LPAREN expression RPAREN'\n t[0] = t[2]\n\ndef p_expression_number(t):\n 'expression : NUMBER'\n t[0] = t[1]\n\ndef p_expression_name(t):\n 'expression : NAME'\n try:\n t[0] = names[t[1]]\n except LookupError:\n print(\"Undefined name '%s'\" % t[1])\n t[0] = 0\n\n\nyacc.yacc()\n\n\n\n\n" }, { "path": "xonsh/ply/test/yacc_nop.py", "content": "# -----------------------------------------------------------------------------\n# yacc_nop.py\n#\n# Possible grammar rule defined without p_ prefix\n# -----------------------------------------------------------------------------\nimport sys\n\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\nimport ply.yacc as yacc\n\nfrom calclex import tokens\n\n# Parsing rules\nprecedence = (\n ('left','PLUS','MINUS'),\n ('left','TIMES','DIVIDE'),\n ('right','UMINUS'),\n )\n\n# dictionary of names\nnames = { }\n\ndef p_statement_assign(t):\n 'statement : NAME EQUALS expression'\n names[t[1]] = t[3]\n\ndef statement_expr(t):\n 'statement : expression'\n print(t[1])\n\ndef p_expression_binop(t):\n '''expression : expression PLUS expression\n | expression MINUS expression\n | expression TIMES expression\n | expression DIVIDE expression'''\n if t[2] == '+' : t[0] = t[1] + t[3]\n elif t[2] == '-': t[0] = t[1] - t[3]\n elif t[2] == '*': t[0] = t[1] * t[3]\n elif t[2] == '/': t[0] = t[1] / t[3]\n\ndef p_expression_uminus(t):\n 'expression : MINUS expression %prec UMINUS'\n t[0] = -t[2]\n\ndef p_expression_group(t):\n 'expression : LPAREN expression RPAREN'\n t[0] = t[2]\n\ndef p_expression_number(t):\n 'expression : NUMBER'\n t[0] = t[1]\n\ndef p_expression_name(t):\n 'expression : NAME'\n try:\n t[0] = names[t[1]]\n except LookupError:\n print(\"Undefined name '%s'\" % t[1])\n t[0] = 0\n\ndef p_error(t):\n print(\"Syntax error at '%s'\" % t.value)\n\nyacc.yacc()\n\n\n\n\n" }, { "path": "xonsh/ply/test/yacc_notfunc.py", "content": "# -----------------------------------------------------------------------------\n# yacc_notfunc.py\n#\n# p_rule not defined as a function\n# -----------------------------------------------------------------------------\nimport sys\n\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\nimport ply.yacc as yacc\n\nfrom calclex import tokens\n\n# Parsing rules\nprecedence = (\n ('left','PLUS','MINUS'),\n ('left','TIMES','DIVIDE'),\n ('right','UMINUS'),\n )\n\n# dictionary of names\nnames = { }\n\np_statement_assign = \"Blah\"\n\ndef p_statement_expr(t):\n 'statement : expression'\n print(t[1])\n\ndef p_expression_binop(t):\n '''expression : expression PLUS expression\n | expression MINUS expression\n | expression TIMES expression\n | expression DIVIDE expression'''\n if t[2] == '+' : t[0] = t[1] + t[3]\n elif t[2] == '-': t[0] = t[1] - t[3]\n elif t[2] == '*': t[0] = t[1] * t[3]\n elif t[2] == '/': t[0] = t[1] / t[3]\n\ndef p_expression_uminus(t):\n 'expression : MINUS expression %prec UMINUS'\n t[0] = -t[2]\n\ndef p_expression_group(t):\n 'expression : LPAREN expression RPAREN'\n t[0] = t[2]\n\ndef p_expression_number(t):\n 'expression : NUMBER'\n t[0] = t[1]\n\ndef p_expression_name(t):\n 'expression : NAME'\n try:\n t[0] = names[t[1]]\n except LookupError:\n print(\"Undefined name '%s'\" % t[1])\n t[0] = 0\n\ndef p_error(t):\n print(\"Syntax error at '%s'\" % t.value)\n\nyacc.yacc()\n\n\n\n\n" }, { "path": "xonsh/ply/test/yacc_notok.py", "content": "# -----------------------------------------------------------------------------\n# yacc_notok.py\n#\n# A grammar, but we forgot to import the tokens list\n# -----------------------------------------------------------------------------\n\nimport sys\n\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\nimport ply.yacc as yacc\n\n# Parsing rules\nprecedence = (\n ('left','PLUS','MINUS'),\n ('left','TIMES','DIVIDE'),\n ('right','UMINUS'),\n )\n\n# dictionary of names\nnames = { }\n\ndef p_statement_assign(t):\n 'statement : NAME EQUALS expression'\n names[t[1]] = t[3]\n\ndef p_statement_expr(t):\n 'statement : expression'\n print(t[1])\n\ndef p_expression_binop(t):\n '''expression : expression PLUS expression\n | expression MINUS expression\n | expression TIMES expression\n | expression DIVIDE expression'''\n if t[2] == '+' : t[0] = t[1] + t[3]\n elif t[2] == '-': t[0] = t[1] - t[3]\n elif t[2] == '*': t[0] = t[1] * t[3]\n elif t[2] == '/': t[0] = t[1] / t[3]\n\ndef p_expression_uminus(t):\n 'expression : MINUS expression %prec UMINUS'\n t[0] = -t[2]\n\ndef p_expression_group(t):\n 'expression : LPAREN expression RPAREN'\n t[0] = t[2]\n\ndef p_expression_number(t):\n 'expression : NUMBER'\n t[0] = t[1]\n\ndef p_expression_name(t):\n 'expression : NAME'\n try:\n t[0] = names[t[1]]\n except LookupError:\n print(\"Undefined name '%s'\" % t[1])\n t[0] = 0\n\ndef p_error(t):\n print(\"Syntax error at '%s'\" % t.value)\n\nyacc.yacc()\n\n\n\n\n" }, { "path": "xonsh/ply/test/yacc_prec1.py", "content": "# -----------------------------------------------------------------------------\n# yacc_prec1.py\n#\n# Tests case where precedence specifier doesn't match up to terminals\n# -----------------------------------------------------------------------------\nimport sys\n\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\nimport ply.yacc as yacc\n\nfrom calclex import tokens\n\n# Parsing rules\nprecedence = (\n ('left', '+', '-'),\n ('left', '*', '/'),\n ('right','UMINUS'),\n )\n\n# dictionary of names\nnames = { }\n\ndef p_statement_assign(t):\n 'statement : NAME EQUALS expression'\n names[t[1]] = t[3]\n\ndef p_statement_expr(t):\n 'statement : expression'\n print(t[1])\n\ndef p_expression_binop(t):\n '''expression : expression PLUS expression\n | expression MINUS expression\n | expression TIMES expression\n | expression DIVIDE expression'''\n if t[2] == '+' : t[0] = t[1] + t[3]\n elif t[2] == '-': t[0] = t[1] - t[3]\n elif t[2] == '*': t[0] = t[1] * t[3]\n elif t[2] == '/': t[0] = t[1] / t[3]\n\ndef p_expression_uminus(t):\n 'expression : MINUS expression %prec UMINUS'\n t[0] = -t[2]\n\ndef p_expression_group(t):\n 'expression : LPAREN expression RPAREN'\n t[0] = t[2]\n\ndef p_expression_number(t):\n 'expression : NUMBER'\n t[0] = t[1]\n\ndef p_expression_name(t):\n 'expression : NAME'\n try:\n t[0] = names[t[1]]\n except LookupError:\n print(\"Undefined name '%s'\" % t[1])\n t[0] = 0\n\ndef p_error(t):\n print(\"Syntax error at '%s'\" % t.value)\n\nyacc.yacc()\n\n\n\n\n" }, { "path": "xonsh/ply/test/yacc_rr.py", "content": "# -----------------------------------------------------------------------------\n# yacc_rr.py\n#\n# A grammar with a reduce/reduce conflict\n# -----------------------------------------------------------------------------\nimport sys\n\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\nimport ply.yacc as yacc\n\nfrom calclex import tokens\n\n# Parsing rules\nprecedence = (\n ('left','PLUS','MINUS'),\n ('left','TIMES','DIVIDE'),\n ('right','UMINUS'),\n )\n\n# dictionary of names\nnames = { }\n\ndef p_statement_assign(t):\n 'statement : NAME EQUALS expression'\n names[t[1]] = t[3]\n\ndef p_statement_assign_2(t):\n 'statement : NAME EQUALS NUMBER'\n names[t[1]] = t[3]\n\ndef p_statement_expr(t):\n 'statement : expression'\n print(t[1])\n\ndef p_expression_binop(t):\n '''expression : expression PLUS expression\n | expression MINUS expression\n | expression TIMES expression\n | expression DIVIDE expression'''\n if t[2] == '+' : t[0] = t[1] + t[3]\n elif t[2] == '-': t[0] = t[1] - t[3]\n elif t[2] == '*': t[0] = t[1] * t[3]\n elif t[2] == '/': t[0] = t[1] / t[3]\n\ndef p_expression_uminus(t):\n 'expression : MINUS expression %prec UMINUS'\n t[0] = -t[2]\n\ndef p_expression_group(t):\n 'expression : LPAREN expression RPAREN'\n t[0] = t[2]\n\ndef p_expression_number(t):\n 'expression : NUMBER'\n t[0] = t[1]\n\ndef p_expression_name(t):\n 'expression : NAME'\n try:\n t[0] = names[t[1]]\n except LookupError:\n print(\"Undefined name '%s'\" % t[1])\n t[0] = 0\n\ndef p_error(t):\n print(\"Syntax error at '%s'\" % t.value)\n\nyacc.yacc()\n\n\n\n\n" }, { "path": "xonsh/ply/test/yacc_rr_unused.py", "content": "# -----------------------------------------------------------------------------\n# yacc_rr_unused.py\n#\n# A grammar with reduce/reduce conflicts and a rule that never\n# gets reduced.\n# -----------------------------------------------------------------------------\nimport sys\n\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\nimport ply.yacc as yacc\n\ntokens = ('A', 'B', 'C')\n\ndef p_grammar(p):\n '''\n rule1 : rule2 B\n | rule2 C\n\n rule2 : rule3 B\n | rule4\n | rule5\n\n rule3 : A\n\n rule4 : A\n\n rule5 : A\n '''\n\nyacc.yacc()\n" }, { "path": "xonsh/ply/test/yacc_simple.py", "content": "# -----------------------------------------------------------------------------\n# yacc_simple.py\n#\n# A simple, properly specifier grammar\n# -----------------------------------------------------------------------------\nimport sys\n\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\nimport ply.yacc as yacc\n\nfrom calclex import tokens\n\n# Parsing rules\nprecedence = (\n ('left','PLUS','MINUS'),\n ('left','TIMES','DIVIDE'),\n ('right','UMINUS'),\n )\n\n# dictionary of names\nnames = { }\n\ndef p_statement_assign(t):\n 'statement : NAME EQUALS expression'\n names[t[1]] = t[3]\n\ndef p_statement_expr(t):\n 'statement : expression'\n print(t[1])\n\ndef p_expression_binop(t):\n '''expression : expression PLUS expression\n | expression MINUS expression\n | expression TIMES expression\n | expression DIVIDE expression'''\n if t[2] == '+' : t[0] = t[1] + t[3]\n elif t[2] == '-': t[0] = t[1] - t[3]\n elif t[2] == '*': t[0] = t[1] * t[3]\n elif t[2] == '/': t[0] = t[1] / t[3]\n\ndef p_expression_uminus(t):\n 'expression : MINUS expression %prec UMINUS'\n t[0] = -t[2]\n\ndef p_expression_group(t):\n 'expression : LPAREN expression RPAREN'\n t[0] = t[2]\n\ndef p_expression_number(t):\n 'expression : NUMBER'\n t[0] = t[1]\n\ndef p_expression_name(t):\n 'expression : NAME'\n try:\n t[0] = names[t[1]]\n except LookupError:\n print(\"Undefined name '%s'\" % t[1])\n t[0] = 0\n\ndef p_error(t):\n print(\"Syntax error at '%s'\" % t.value)\n\nyacc.yacc()\n\n\n\n\n" }, { "path": "xonsh/ply/test/yacc_sr.py", "content": "# -----------------------------------------------------------------------------\n# yacc_sr.py\n#\n# A grammar with shift-reduce conflicts\n# -----------------------------------------------------------------------------\nimport sys\n\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\nimport ply.yacc as yacc\n\nfrom calclex import tokens\n\n# Parsing rules\n\n# dictionary of names\nnames = { }\n\ndef p_statement_assign(t):\n 'statement : NAME EQUALS expression'\n names[t[1]] = t[3]\n\ndef p_statement_expr(t):\n 'statement : expression'\n print(t[1])\n\ndef p_expression_binop(t):\n '''expression : expression PLUS expression\n | expression MINUS expression\n | expression TIMES expression\n | expression DIVIDE expression'''\n if t[2] == '+' : t[0] = t[1] + t[3]\n elif t[2] == '-': t[0] = t[1] - t[3]\n elif t[2] == '*': t[0] = t[1] * t[3]\n elif t[2] == '/': t[0] = t[1] / t[3]\n\ndef p_expression_uminus(t):\n 'expression : MINUS expression'\n t[0] = -t[2]\n\ndef p_expression_group(t):\n 'expression : LPAREN expression RPAREN'\n t[0] = t[2]\n\ndef p_expression_number(t):\n 'expression : NUMBER'\n t[0] = t[1]\n\ndef p_expression_name(t):\n 'expression : NAME'\n try:\n t[0] = names[t[1]]\n except LookupError:\n print(\"Undefined name '%s'\" % t[1])\n t[0] = 0\n\ndef p_error(t):\n print(\"Syntax error at '%s'\" % t.value)\n\nyacc.yacc()\n\n\n\n\n" }, { "path": "xonsh/ply/test/yacc_term1.py", "content": "# -----------------------------------------------------------------------------\n# yacc_term1.py\n#\n# Terminal used on the left-hand-side\n# -----------------------------------------------------------------------------\nimport sys\n\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\nimport ply.yacc as yacc\n\nfrom calclex import tokens\n\n# Parsing rules\nprecedence = (\n ('left','PLUS','MINUS'),\n ('left','TIMES','DIVIDE'),\n ('right','UMINUS'),\n )\n\n# dictionary of names\nnames = { }\n\ndef p_statement_assign(t):\n 'NUMBER : NAME EQUALS expression'\n names[t[1]] = t[3]\n\ndef p_statement_expr(t):\n 'statement : expression'\n print(t[1])\n\ndef p_expression_binop(t):\n '''expression : expression PLUS expression\n | expression MINUS expression\n | expression TIMES expression\n | expression DIVIDE expression'''\n if t[2] == '+' : t[0] = t[1] + t[3]\n elif t[2] == '-': t[0] = t[1] - t[3]\n elif t[2] == '*': t[0] = t[1] * t[3]\n elif t[2] == '/': t[0] = t[1] / t[3]\n\ndef p_expression_uminus(t):\n 'expression : MINUS expression %prec UMINUS'\n t[0] = -t[2]\n\ndef p_expression_group(t):\n 'expression : LPAREN expression RPAREN'\n t[0] = t[2]\n\ndef p_expression_number(t):\n 'expression : NUMBER'\n t[0] = t[1]\n\ndef p_expression_name(t):\n 'expression : NAME'\n try:\n t[0] = names[t[1]]\n except LookupError:\n print(\"Undefined name '%s'\" % t[1])\n t[0] = 0\n\ndef p_error(t):\n print(\"Syntax error at '%s'\" % t.value)\n\nyacc.yacc()\n\n\n\n\n" }, { "path": "xonsh/ply/test/yacc_unicode_literals.py", "content": "# -----------------------------------------------------------------------------\n# yacc_unicode_literals\n#\n# Test for unicode literals on Python 2.x\n# -----------------------------------------------------------------------------\nfrom __future__ import unicode_literals\n\nimport sys\n\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\nimport ply.yacc as yacc\n\nfrom calclex import tokens\n\n# Parsing rules\nprecedence = (\n ('left','PLUS','MINUS'),\n ('left','TIMES','DIVIDE'),\n ('right','UMINUS'),\n )\n\n# dictionary of names\nnames = { }\n\ndef p_statement_assign(t):\n 'statement : NAME EQUALS expression'\n names[t[1]] = t[3]\n\ndef p_statement_expr(t):\n 'statement : expression'\n print(t[1])\n\ndef p_expression_binop(t):\n '''expression : expression PLUS expression\n | expression MINUS expression\n | expression TIMES expression\n | expression DIVIDE expression'''\n if t[2] == '+' : t[0] = t[1] + t[3]\n elif t[2] == '-': t[0] = t[1] - t[3]\n elif t[2] == '*': t[0] = t[1] * t[3]\n elif t[2] == '/': t[0] = t[1] / t[3]\n\ndef p_expression_uminus(t):\n 'expression : MINUS expression %prec UMINUS'\n t[0] = -t[2]\n\ndef p_expression_group(t):\n 'expression : LPAREN expression RPAREN'\n t[0] = t[2]\n\ndef p_expression_number(t):\n 'expression : NUMBER'\n t[0] = t[1]\n\ndef p_expression_name(t):\n 'expression : NAME'\n try:\n t[0] = names[t[1]]\n except LookupError:\n print(\"Undefined name '%s'\" % t[1])\n t[0] = 0\n\ndef p_error(t):\n print(\"Syntax error at '%s'\" % t.value)\n\nyacc.yacc()\n\n\n\n\n" }, { "path": "xonsh/ply/test/yacc_unused.py", "content": "# -----------------------------------------------------------------------------\n# yacc_unused.py\n#\n# A grammar with an unused rule\n# -----------------------------------------------------------------------------\nimport sys\n\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\nimport ply.yacc as yacc\n\nfrom calclex import tokens\n\n# Parsing rules\nprecedence = (\n ('left','PLUS','MINUS'),\n ('left','TIMES','DIVIDE'),\n ('right','UMINUS'),\n )\n\n# dictionary of names\nnames = { }\n\ndef p_statement_assign(t):\n 'statement : NAME EQUALS expression'\n names[t[1]] = t[3]\n\ndef p_statement_expr(t):\n 'statement : expression'\n print(t[1])\n\ndef p_expression_binop(t):\n '''expression : expression PLUS expression\n | expression MINUS expression\n | expression TIMES expression\n | expression DIVIDE expression'''\n if t[2] == '+' : t[0] = t[1] + t[3]\n elif t[2] == '-': t[0] = t[1] - t[3]\n elif t[2] == '*': t[0] = t[1] * t[3]\n elif t[2] == '/': t[0] = t[1] / t[3]\n\ndef p_expression_uminus(t):\n 'expression : MINUS expression %prec UMINUS'\n t[0] = -t[2]\n\ndef p_expression_group(t):\n 'expression : LPAREN expression RPAREN'\n t[0] = t[2]\n\ndef p_expression_number(t):\n 'expression : NUMBER'\n t[0] = t[1]\n\ndef p_expression_name(t):\n 'expression : NAME'\n try:\n t[0] = names[t[1]]\n except LookupError:\n print(\"Undefined name '%s'\" % t[1])\n t[0] = 0\n\ndef p_expr_list(t):\n 'exprlist : exprlist COMMA expression'\n pass\n\ndef p_expr_list_2(t):\n 'exprlist : expression'\n pass\n\n\ndef p_error(t):\n print(\"Syntax error at '%s'\" % t.value)\n\nyacc.yacc()\n\n\n\n\n" }, { "path": "xonsh/ply/test/yacc_unused_rule.py", "content": "# -----------------------------------------------------------------------------\n# yacc_unused_rule.py\n#\n# Grammar with an unused rule\n# -----------------------------------------------------------------------------\nimport sys\n\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\nimport ply.yacc as yacc\n\nfrom calclex import tokens\n\n# Parsing rules\nprecedence = (\n ('left','PLUS','MINUS'),\n ('left','TIMES','DIVIDE'),\n ('right','UMINUS'),\n )\n\n# dictionary of names\nnames = { }\n\ndef p_statement_assign(t):\n 'statement : NAME EQUALS expression'\n names[t[1]] = t[3]\n\ndef p_statement_expr(t):\n 'statement : expression'\n print(t[1])\n\ndef p_expression_binop(t):\n '''expression : expression PLUS expression\n | expression MINUS expression\n | expression TIMES expression\n | expression DIVIDE expression'''\n if t[2] == '+' : t[0] = t[1] + t[3]\n elif t[2] == '-': t[0] = t[1] - t[3]\n elif t[2] == '*': t[0] = t[1] * t[3]\n elif t[2] == '/': t[0] = t[1] / t[3]\n\ndef p_expression_uminus(t):\n 'expression : MINUS expression %prec UMINUS'\n t[0] = -t[2]\n\ndef p_expression_group(t):\n 'expression : LPAREN expression RPAREN'\n t[0] = t[2]\n\ndef p_expression_number(t):\n 'expression : NUMBER'\n t[0] = t[1]\n\ndef p_expression_name(t):\n 'expression : NAME'\n try:\n t[0] = names[t[1]]\n except LookupError:\n print(\"Undefined name '%s'\" % t[1])\n t[0] = 0\n\ndef p_integer(t):\n 'integer : NUMBER'\n t[0] = t[1]\n\ndef p_error(t):\n print(\"Syntax error at '%s'\" % t.value)\n\nyacc.yacc()\n\n\n\n\n" }, { "path": "xonsh/ply/test/yacc_uprec.py", "content": "# -----------------------------------------------------------------------------\n# yacc_uprec.py\n#\n# A grammar with a bad %prec specifier\n# -----------------------------------------------------------------------------\nimport sys\n\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\nimport ply.yacc as yacc\n\nfrom calclex import tokens\n\n# Parsing rules\n\n# dictionary of names\nnames = { }\n\ndef p_statement_assign(t):\n 'statement : NAME EQUALS expression'\n names[t[1]] = t[3]\n\ndef p_statement_expr(t):\n 'statement : expression'\n print(t[1])\n\ndef p_expression_binop(t):\n '''expression : expression PLUS expression\n | expression MINUS expression\n | expression TIMES expression\n | expression DIVIDE expression'''\n if t[2] == '+' : t[0] = t[1] + t[3]\n elif t[2] == '-': t[0] = t[1] - t[3]\n elif t[2] == '*': t[0] = t[1] * t[3]\n elif t[2] == '/': t[0] = t[1] / t[3]\n\ndef p_expression_uminus(t):\n 'expression : MINUS expression %prec UMINUS'\n t[0] = -t[2]\n\ndef p_expression_group(t):\n 'expression : LPAREN expression RPAREN'\n t[0] = t[2]\n\ndef p_expression_number(t):\n 'expression : NUMBER'\n t[0] = t[1]\n\ndef p_expression_name(t):\n 'expression : NAME'\n try:\n t[0] = names[t[1]]\n except LookupError:\n print(\"Undefined name '%s'\" % t[1])\n t[0] = 0\n\ndef p_error(t):\n print(\"Syntax error at '%s'\" % t.value)\n\nyacc.yacc()\n\n\n\n\n" }, { "path": "xonsh/ply/test/yacc_uprec2.py", "content": "# -----------------------------------------------------------------------------\n# yacc_uprec2.py\n#\n# A grammar with a bad %prec specifier\n# -----------------------------------------------------------------------------\nimport sys\n\nif \"..\" not in sys.path: sys.path.insert(0,\"..\")\nimport ply.yacc as yacc\n\nfrom calclex import tokens\n\n# Parsing rules\n\n# dictionary of names\nnames = { }\n\ndef p_statement_assign(t):\n 'statement : NAME EQUALS expression'\n names[t[1]] = t[3]\n\ndef p_statement_expr(t):\n 'statement : expression'\n print(t[1])\n\ndef p_expression_binop(t):\n '''expression : expression PLUS expression\n | expression MINUS expression\n | expression TIMES expression\n | expression DIVIDE expression'''\n if t[2] == '+' : t[0] = t[1] + t[3]\n elif t[2] == '-': t[0] = t[1] - t[3]\n elif t[2] == '*': t[0] = t[1] * t[3]\n elif t[2] == '/': t[0] = t[1] / t[3]\n\ndef p_expression_uminus(t):\n 'expression : MINUS expression %prec'\n t[0] = -t[2]\n\ndef p_expression_group(t):\n 'expression : LPAREN expression RPAREN'\n t[0] = t[2]\n\ndef p_expression_number(t):\n 'expression : NUMBER'\n t[0] = t[1]\n\ndef p_expression_name(t):\n 'expression : NAME'\n try:\n t[0] = names[t[1]]\n except LookupError:\n print(\"Undefined name '%s'\" % t[1])\n t[0] = 0\n\ndef p_error(t):\n print(\"Syntax error at '%s'\" % t.value)\n\nyacc.yacc()\n\n\n\n\n" }, { "path": "xonsh/pretty.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nPython advanced pretty printer. This pretty printer is intended to\nreplace the old `pprint` python module which does not allow developers\nto provide their own pretty print callbacks.\n\nThis module is based on ruby's `prettyprint.rb` library by `Tanaka Akira`.\n\nThe following implementations were forked from the IPython project:\n* Copyright (c) 2008-2014, IPython Development Team\n* Copyright (C) 2001-2007 Fernando Perez \n* Copyright (c) 2001, Janko Hauser \n* Copyright (c) 2001, Nathaniel Gray \n\nExample Usage\n-------------\n\nTo directly print the representation of an object use `pprint`::\n\n from pretty import pretty_print\n pretty_pprint(complex_object)\n\nTo get a string of the output use `pretty`::\n\n from pretty import pretty\n string = pretty(complex_object)\n\n\nExtending\n---------\n\nThe pretty library allows developers to add pretty printing rules for their\nown objects. This process is straightforward. All you have to do is to\nadd a `_repr_pretty_` method to your object and call the methods on the\npretty printer passed::\n\n class MyObject(object):\n\n def _repr_pretty_(self, p, cycle):\n ...\n\nHere is an example implementation of a `_repr_pretty_` method for a list\nsubclass::\n\n class MyList(list):\n\n def _repr_pretty_(self, p, cycle):\n if cycle:\n p.text('MyList(...)')\n else:\n with p.group(8, 'MyList([', '])'):\n for idx, item in enumerate(self):\n if idx:\n p.text(',')\n p.breakable()\n p.pretty(item)\n\nThe `cycle` parameter is `True` if pretty detected a cycle. You *have* to\nreact to that or the result is an infinite loop. `p.text()` just adds\nnon breaking text to the output, `p.breakable()` either adds a whitespace\nor breaks here. If you pass it an argument it's used instead of the\ndefault space. `p.pretty` prettyprints another object using the pretty print\nmethod.\n\nThe first parameter to the `group` function specifies the extra indentation\nof the next line. In this example the next item will either be on the same\nline (if the items are short enough) or aligned with the right edge of the\nopening bracket of `MyList`.\n\nIf you just want to indent something you can use the group function\nwithout open / close parameters. You can also use this code::\n\n with p.indent(2):\n ...\n\n\n:copyright: 2007 by Armin Ronacher.\n Portions (c) 2009 by Robert Kern.\n:license: BSD License.\n\"\"\"\nimport io\nimport re\nimport sys\nimport types\nimport datetime\nimport contextlib\nimport collections\n\nfrom xonsh.lazyasd import LazyObject, lazyobject\n\n__all__ = [\n \"pretty\",\n \"pretty_print\",\n \"PrettyPrinter\",\n \"RepresentationPrinter\",\n \"for_type\",\n \"for_type_by_name\",\n]\n\n\nMAX_SEQ_LENGTH = 1000\n\n\ndef _safe_getattr(obj, attr, default=None):\n \"\"\"Safe version of getattr.\n\n Same as getattr, but will return ``default`` on any Exception,\n rather than raising.\n \"\"\"\n try:\n return getattr(obj, attr, default)\n except Exception:\n return default\n\n\nCUnicodeIO = io.StringIO\n\n\ndef pretty(\n obj, verbose=False, max_width=79, newline=\"\\n\", max_seq_length=MAX_SEQ_LENGTH\n):\n \"\"\"\n Pretty print the object's representation.\n \"\"\"\n if hasattr(obj, \"xonsh_display\"):\n return obj.xonsh_display()\n\n stream = CUnicodeIO()\n printer = RepresentationPrinter(\n stream, verbose, max_width, newline, max_seq_length=max_seq_length\n )\n printer.pretty(obj)\n printer.flush()\n return stream.getvalue()\n\n\ndef pretty_print(\n obj, verbose=False, max_width=79, newline=\"\\n\", max_seq_length=MAX_SEQ_LENGTH\n):\n \"\"\"\n Like pretty() but print to stdout.\n \"\"\"\n printer = RepresentationPrinter(\n sys.stdout, verbose, max_width, newline, max_seq_length=max_seq_length\n )\n printer.pretty(obj)\n printer.flush()\n sys.stdout.write(newline)\n sys.stdout.flush()\n\n\nclass _PrettyPrinterBase(object):\n @contextlib.contextmanager\n def indent(self, indent):\n \"\"\"with statement support for indenting/dedenting.\"\"\"\n self.indentation += indent\n try:\n yield\n finally:\n self.indentation -= indent\n\n @contextlib.contextmanager\n def group(self, indent=0, open=\"\", close=\"\"):\n \"\"\"like begin_group / end_group but for the with statement.\"\"\"\n self.begin_group(indent, open)\n try:\n yield\n finally:\n self.end_group(indent, close)\n\n\nclass PrettyPrinter(_PrettyPrinterBase):\n \"\"\"\n Baseclass for the `RepresentationPrinter` prettyprinter that is used to\n generate pretty reprs of objects. Contrary to the `RepresentationPrinter`\n this printer knows nothing about the default pprinters or the `_repr_pretty_`\n callback method.\n \"\"\"\n\n def __init__(\n self, output, max_width=79, newline=\"\\n\", max_seq_length=MAX_SEQ_LENGTH\n ):\n self.output = output\n self.max_width = max_width\n self.newline = newline\n self.max_seq_length = max_seq_length\n self.output_width = 0\n self.buffer_width = 0\n self.buffer = collections.deque()\n\n root_group = Group(0)\n self.group_stack = [root_group]\n self.group_queue = GroupQueue(root_group)\n self.indentation = 0\n\n def _break_outer_groups(self):\n while self.max_width < self.output_width + self.buffer_width:\n group = self.group_queue.deq()\n if not group:\n return\n while group.breakables:\n x = self.buffer.popleft()\n self.output_width = x.output(self.output, self.output_width)\n self.buffer_width -= x.width\n while self.buffer and isinstance(self.buffer[0], Text):\n x = self.buffer.popleft()\n self.output_width = x.output(self.output, self.output_width)\n self.buffer_width -= x.width\n\n def text(self, obj):\n \"\"\"Add literal text to the output.\"\"\"\n width = len(obj)\n if self.buffer:\n text = self.buffer[-1]\n if not isinstance(text, Text):\n text = Text()\n self.buffer.append(text)\n text.add(obj, width)\n self.buffer_width += width\n self._break_outer_groups()\n else:\n self.output.write(obj)\n self.output_width += width\n\n def breakable(self, sep=\" \"):\n \"\"\"\n Add a breakable separator to the output. This does not mean that it\n will automatically break here. If no breaking on this position takes\n place the `sep` is inserted which default to one space.\n \"\"\"\n width = len(sep)\n group = self.group_stack[-1]\n if group.want_break:\n self.flush()\n self.output.write(self.newline)\n self.output.write(\" \" * self.indentation)\n self.output_width = self.indentation\n self.buffer_width = 0\n else:\n self.buffer.append(Breakable(sep, width, self))\n self.buffer_width += width\n self._break_outer_groups()\n\n def break_(self):\n \"\"\"\n Explicitly insert a newline into the output, maintaining correct indentation.\n \"\"\"\n self.flush()\n self.output.write(self.newline)\n self.output.write(\" \" * self.indentation)\n self.output_width = self.indentation\n self.buffer_width = 0\n\n def begin_group(self, indent=0, open=\"\"):\n \"\"\"\n Begin a group. If you want support for python < 2.5 which doesn't has\n the with statement this is the preferred way:\n\n p.begin_group(1, '{')\n ...\n p.end_group(1, '}')\n\n The python 2.5 expression would be this:\n\n with p.group(1, '{', '}'):\n ...\n\n The first parameter specifies the indentation for the next line (usually\n the width of the opening text), the second the opening text. All\n parameters are optional.\n \"\"\"\n if open:\n self.text(open)\n group = Group(self.group_stack[-1].depth + 1)\n self.group_stack.append(group)\n self.group_queue.enq(group)\n self.indentation += indent\n\n def _enumerate(self, seq):\n \"\"\"like enumerate, but with an upper limit on the number of items\"\"\"\n for idx, x in enumerate(seq):\n if self.max_seq_length and idx >= self.max_seq_length:\n self.text(\",\")\n self.breakable()\n self.text(\"...\")\n return\n yield idx, x\n\n def end_group(self, dedent=0, close=\"\"):\n \"\"\"End a group. See `begin_group` for more details.\"\"\"\n self.indentation -= dedent\n group = self.group_stack.pop()\n if not group.breakables:\n self.group_queue.remove(group)\n if close:\n self.text(close)\n\n def flush(self):\n \"\"\"Flush data that is left in the buffer.\"\"\"\n for data in self.buffer:\n self.output_width += data.output(self.output, self.output_width)\n self.buffer.clear()\n self.buffer_width = 0\n\n\ndef _get_mro(obj_class):\n \"\"\" Get a reasonable method resolution order of a class and its superclasses\n for both old-style and new-style classes.\n \"\"\"\n if not hasattr(obj_class, \"__mro__\"):\n # Old-style class. Mix in object to make a fake new-style class.\n try:\n obj_class = type(obj_class.__name__, (obj_class, object), {})\n except TypeError:\n # Old-style extension type that does not descend from object.\n # FIXME: try to construct a more thorough MRO.\n mro = [obj_class]\n else:\n mro = obj_class.__mro__[1:-1]\n else:\n mro = obj_class.__mro__\n return mro\n\n\nclass RepresentationPrinter(PrettyPrinter):\n \"\"\"\n Special pretty printer that has a `pretty` method that calls the pretty\n printer for a python object.\n\n This class stores processing data on `self` so you must *never* use\n this class in a threaded environment. Always lock it or reinstantiate\n it.\n\n Instances also have a verbose flag callbacks can access to control their\n output. For example the default instance repr prints all attributes and\n methods that are not prefixed by an underscore if the printer is in\n verbose mode.\n \"\"\"\n\n def __init__(\n self,\n output,\n verbose=False,\n max_width=79,\n newline=\"\\n\",\n singleton_pprinters=None,\n type_pprinters=None,\n deferred_pprinters=None,\n max_seq_length=MAX_SEQ_LENGTH,\n ):\n\n PrettyPrinter.__init__(\n self, output, max_width, newline, max_seq_length=max_seq_length\n )\n self.verbose = verbose\n self.stack = []\n if singleton_pprinters is None:\n singleton_pprinters = _singleton_pprinters.copy()\n self.singleton_pprinters = singleton_pprinters\n if type_pprinters is None:\n type_pprinters = _type_pprinters.copy()\n self.type_pprinters = type_pprinters\n if deferred_pprinters is None:\n deferred_pprinters = _deferred_type_pprinters.copy()\n self.deferred_pprinters = deferred_pprinters\n\n def pretty(self, obj):\n \"\"\"Pretty print the given object.\"\"\"\n obj_id = id(obj)\n cycle = obj_id in self.stack\n self.stack.append(obj_id)\n self.begin_group()\n try:\n obj_class = _safe_getattr(obj, \"__class__\", None) or type(obj)\n # First try to find registered singleton printers for the type.\n try:\n printer = self.singleton_pprinters[obj_id]\n except (TypeError, KeyError):\n pass\n else:\n return printer(obj, self, cycle)\n # Next walk the mro and check for either:\n # 1) a registered printer\n # 2) a _repr_pretty_ method\n for cls in _get_mro(obj_class):\n if cls in self.type_pprinters:\n # printer registered in self.type_pprinters\n return self.type_pprinters[cls](obj, self, cycle)\n else:\n # deferred printer\n printer = self._in_deferred_types(cls)\n if printer is not None:\n return printer(obj, self, cycle)\n else:\n # Finally look for special method names.\n # Some objects automatically create any requested\n # attribute. Try to ignore most of them by checking for\n # callability.\n if \"_repr_pretty_\" in cls.__dict__:\n meth = cls._repr_pretty_\n if callable(meth):\n return meth(obj, self, cycle)\n return _default_pprint(obj, self, cycle)\n finally:\n self.end_group()\n self.stack.pop()\n\n def _in_deferred_types(self, cls):\n \"\"\"\n Check if the given class is specified in the deferred type registry.\n\n Returns the printer from the registry if it exists, and None if the\n class is not in the registry. Successful matches will be moved to the\n regular type registry for future use.\n \"\"\"\n mod = _safe_getattr(cls, \"__module__\", None)\n name = _safe_getattr(cls, \"__name__\", None)\n key = (mod, name)\n printer = None\n if key in self.deferred_pprinters:\n # Move the printer over to the regular registry.\n printer = self.deferred_pprinters.pop(key)\n self.type_pprinters[cls] = printer\n return printer\n\n\nclass Printable(object):\n def output(self, stream, output_width):\n return output_width\n\n\nclass Text(Printable):\n def __init__(self):\n self.objs = []\n self.width = 0\n\n def output(self, stream, output_width):\n for obj in self.objs:\n stream.write(obj)\n return output_width + self.width\n\n def add(self, obj, width):\n self.objs.append(obj)\n self.width += width\n\n\nclass Breakable(Printable):\n def __init__(self, seq, width, pretty):\n self.obj = seq\n self.width = width\n self.pretty = pretty\n self.indentation = pretty.indentation\n self.group = pretty.group_stack[-1]\n self.group.breakables.append(self)\n\n def output(self, stream, output_width):\n self.group.breakables.popleft()\n if self.group.want_break:\n stream.write(self.pretty.newline)\n stream.write(\" \" * self.indentation)\n return self.indentation\n if not self.group.breakables:\n self.pretty.group_queue.remove(self.group)\n stream.write(self.obj)\n return output_width + self.width\n\n\nclass Group(Printable):\n def __init__(self, depth):\n self.depth = depth\n self.breakables = collections.deque()\n self.want_break = False\n\n\nclass GroupQueue(object):\n def __init__(self, *groups):\n self.queue = []\n for group in groups:\n self.enq(group)\n\n def enq(self, group):\n depth = group.depth\n while depth > len(self.queue) - 1:\n self.queue.append([])\n self.queue[depth].append(group)\n\n def deq(self):\n for stack in self.queue:\n for idx, group in enumerate(reversed(stack)):\n if group.breakables:\n del stack[idx]\n group.want_break = True\n return group\n for group in stack:\n group.want_break = True\n del stack[:]\n\n def remove(self, group):\n try:\n self.queue[group.depth].remove(group)\n except ValueError:\n pass\n\n\n@lazyobject\ndef _baseclass_reprs():\n try:\n br = (object.__repr__, types.InstanceType.__repr__)\n except AttributeError: # Python 3\n br = (object.__repr__,)\n return br\n\n\ndef _default_pprint(obj, p, cycle):\n \"\"\"\n The default print function. Used if an object does not provide one and\n it's none of the builtin objects.\n \"\"\"\n klass = _safe_getattr(obj, \"__class__\", None) or type(obj)\n if _safe_getattr(klass, \"__repr__\", None) not in _baseclass_reprs:\n # A user-provided repr. Find newlines and replace them with p.break_()\n _repr_pprint(obj, p, cycle)\n return\n p.begin_group(1, \"<\")\n p.pretty(klass)\n p.text(\" at 0x%x\" % id(obj))\n if cycle:\n p.text(\" ...\")\n elif p.verbose:\n first = True\n for key in dir(obj):\n if not key.startswith(\"_\"):\n try:\n value = getattr(obj, key)\n except AttributeError:\n continue\n if isinstance(value, types.MethodType):\n continue\n if not first:\n p.text(\",\")\n p.breakable()\n p.text(key)\n p.text(\"=\")\n step = len(key) + 1\n p.indentation += step\n p.pretty(value)\n p.indentation -= step\n first = False\n p.end_group(1, \">\")\n\n\ndef _seq_pprinter_factory(start, end, basetype):\n \"\"\"\n Factory that returns a pprint function useful for sequences. Used by\n the default pprint for tuples, dicts, and lists.\n \"\"\"\n\n def inner(obj, p, cycle):\n typ = type(obj)\n if (\n basetype is not None\n and typ is not basetype\n and typ.__repr__ != basetype.__repr__\n ):\n # If the subclass provides its own repr, use it instead.\n return p.text(typ.__repr__(obj))\n\n if cycle:\n return p.text(start + \"...\" + end)\n step = len(start)\n p.begin_group(step, start)\n for idx, x in p._enumerate(obj):\n if idx:\n p.text(\",\")\n p.breakable()\n p.pretty(x)\n if len(obj) == 1 and type(obj) is tuple:\n # Special case for 1-item tuples.\n p.text(\",\")\n p.end_group(step, end)\n\n return inner\n\n\ndef _set_pprinter_factory(start, end, basetype):\n \"\"\"\n Factory that returns a pprint function useful for sets and frozensets.\n \"\"\"\n\n def inner(obj, p, cycle):\n typ = type(obj)\n if (\n basetype is not None\n and typ is not basetype\n and typ.__repr__ != basetype.__repr__\n ):\n # If the subclass provides its own repr, use it instead.\n return p.text(typ.__repr__(obj))\n\n if cycle:\n return p.text(start + \"...\" + end)\n if len(obj) == 0:\n # Special case.\n p.text(basetype.__name__ + \"()\")\n else:\n step = len(start)\n p.begin_group(step, start)\n # Like dictionary keys, we will try to sort the items if there aren't too many\n items = obj\n if not (p.max_seq_length and len(obj) >= p.max_seq_length):\n try:\n items = sorted(obj)\n except Exception:\n # Sometimes the items don't sort.\n pass\n for idx, x in p._enumerate(items):\n if idx:\n p.text(\",\")\n p.breakable()\n p.pretty(x)\n p.end_group(step, end)\n\n return inner\n\n\ndef _dict_pprinter_factory(start, end, basetype=None):\n \"\"\"\n Factory that returns a pprint function used by the default pprint of\n dicts and dict proxies.\n \"\"\"\n\n def inner(obj, p, cycle):\n typ = type(obj)\n if (\n basetype is not None\n and typ is not basetype\n and typ.__repr__ != basetype.__repr__\n ):\n # If the subclass provides its own repr, use it instead.\n return p.text(typ.__repr__(obj))\n\n if cycle:\n return p.text(\"{...}\")\n p.begin_group(1, start)\n keys = obj.keys()\n # if dict isn't large enough to be truncated, sort keys before displaying\n if not (p.max_seq_length and len(obj) >= p.max_seq_length):\n try:\n keys = sorted(keys)\n except Exception:\n # Sometimes the keys don't sort.\n pass\n for idx, key in p._enumerate(keys):\n if idx:\n p.text(\",\")\n p.breakable()\n p.pretty(key)\n p.text(\": \")\n p.pretty(obj[key])\n p.end_group(1, end)\n\n return inner\n\n\ndef _super_pprint(obj, p, cycle):\n \"\"\"The pprint for the super type.\"\"\"\n p.begin_group(8, \"\")\n\n\ndef _re_pattern_pprint(obj, p, cycle):\n \"\"\"The pprint function for regular expression patterns.\"\"\"\n p.text(\"re.compile(\")\n pattern = repr(obj.pattern)\n if pattern[:1] in \"uU\":\n pattern = pattern[1:]\n prefix = \"ur\"\n else:\n prefix = \"r\"\n pattern = prefix + pattern.replace(\"\\\\\\\\\", \"\\\\\")\n p.text(pattern)\n if obj.flags:\n p.text(\",\")\n p.breakable()\n done_one = False\n for flag in (\n \"TEMPLATE\",\n \"IGNORECASE\",\n \"LOCALE\",\n \"MULTILINE\",\n \"DOTALL\",\n \"UNICODE\",\n \"VERBOSE\",\n \"DEBUG\",\n ):\n if obj.flags & getattr(re, flag):\n if done_one:\n p.text(\"|\")\n p.text(\"re.\" + flag)\n done_one = True\n p.text(\")\")\n\n\ndef _type_pprint(obj, p, cycle):\n \"\"\"The pprint for classes and types.\"\"\"\n # Heap allocated types might not have the module attribute,\n # and others may set it to None.\n\n # Checks for a __repr__ override in the metaclass\n if type(obj).__repr__ is not type.__repr__:\n _repr_pprint(obj, p, cycle)\n return\n\n mod = _safe_getattr(obj, \"__module__\", None)\n try:\n name = obj.__qualname__\n if not isinstance(name, str):\n # This can happen if the type implements __qualname__ as a property\n # or other descriptor in Python 2.\n raise Exception(\"Try __name__\")\n except Exception:\n name = obj.__name__\n if not isinstance(name, str):\n name = \"\"\n\n if mod in (None, \"__builtin__\", \"builtins\", \"exceptions\"):\n p.text(name)\n else:\n p.text(mod + \".\" + name)\n\n\ndef _repr_pprint(obj, p, cycle):\n \"\"\"A pprint that just redirects to the normal repr function.\"\"\"\n # Find newlines and replace them with p.break_()\n output = repr(obj)\n for idx, output_line in enumerate(output.splitlines()):\n if idx:\n p.break_()\n p.text(output_line)\n\n\ndef _function_pprint(obj, p, cycle):\n \"\"\"Base pprint for all functions and builtin functions.\"\"\"\n name = _safe_getattr(obj, \"__qualname__\", obj.__name__)\n mod = obj.__module__\n if mod and mod not in (\"__builtin__\", \"builtins\", \"exceptions\"):\n name = mod + \".\" + name\n p.text(\"\" % name)\n\n\ndef _exception_pprint(obj, p, cycle):\n \"\"\"Base pprint for all exceptions.\"\"\"\n name = getattr(obj.__class__, \"__qualname__\", obj.__class__.__name__)\n if obj.__class__.__module__ not in (\"exceptions\", \"builtins\"):\n name = \"%s.%s\" % (obj.__class__.__module__, name)\n step = len(name) + 1\n p.begin_group(step, name + \"(\")\n for idx, arg in enumerate(getattr(obj, \"args\", ())):\n if idx:\n p.text(\",\")\n p.breakable()\n p.pretty(arg)\n p.end_group(step, \")\")\n\n\n@lazyobject\ndef _type_pprinters():\n #: printers for builtin types\n tp = {\n int: _repr_pprint,\n float: _repr_pprint,\n str: _repr_pprint,\n tuple: _seq_pprinter_factory(\"(\", \")\", tuple),\n list: _seq_pprinter_factory(\"[\", \"]\", list),\n dict: _dict_pprinter_factory(\"{\", \"}\", dict),\n set: _set_pprinter_factory(\"{\", \"}\", set),\n frozenset: _set_pprinter_factory(\"frozenset({\", \"})\", frozenset),\n super: _super_pprint,\n type(re.compile(\"\")): _re_pattern_pprint,\n type: _type_pprint,\n types.FunctionType: _function_pprint,\n types.BuiltinFunctionType: _function_pprint,\n types.MethodType: _repr_pprint,\n datetime.datetime: _repr_pprint,\n datetime.timedelta: _repr_pprint,\n }\n #: the exception base\n try:\n _exception_base = BaseException\n except NameError:\n _exception_base = Exception\n tp[_exception_base] = _exception_pprint\n try:\n tp[types.DictProxyType] = _dict_pprinter_factory(\"\")\n tp[types.ClassType] = _type_pprint\n tp[types.SliceType] = _repr_pprint\n except AttributeError: # Python 3\n tp[slice] = _repr_pprint\n try:\n tp[xrange] = _repr_pprint\n tp[long] = _repr_pprint\n tp[unicode] = _repr_pprint\n except NameError:\n tp[range] = _repr_pprint\n tp[bytes] = _repr_pprint\n return tp\n\n\n#: printers for types specified by name\n@lazyobject\ndef _deferred_type_pprinters():\n dtp = {}\n for_type_by_name(\"collections\", \"defaultdict\", _defaultdict_pprint, dtp=dtp)\n for_type_by_name(\"collections\", \"OrderedDict\", _ordereddict_pprint, dtp=dtp)\n for_type_by_name(\"collections\", \"deque\", _deque_pprint, dtp=dtp)\n for_type_by_name(\"collections\", \"Counter\", _counter_pprint, dtp=dtp)\n return dtp\n\n\ndef for_type(typ, func):\n \"\"\"\n Add a pretty printer for a given type.\n \"\"\"\n oldfunc = _type_pprinters.get(typ, None)\n if func is not None:\n # To support easy restoration of old pprinters, we need to ignore Nones.\n _type_pprinters[typ] = func\n return oldfunc\n\n\ndef for_type_by_name(type_module, type_name, func, dtp=None):\n \"\"\"\n Add a pretty printer for a type specified by the module and name of a type\n rather than the type object itself.\n \"\"\"\n if dtp is None:\n dtp = _deferred_type_pprinters\n key = (type_module, type_name)\n oldfunc = dtp.get(key, None)\n if func is not None:\n # To support easy restoration of old pprinters, we need to ignore Nones.\n dtp[key] = func\n return oldfunc\n\n\n#: printers for the default singletons\n_singleton_pprinters = LazyObject(\n lambda: dict.fromkeys(\n map(id, [None, True, False, Ellipsis, NotImplemented]), _repr_pprint\n ),\n globals(),\n \"_singleton_pprinters\",\n)\n\n\ndef _defaultdict_pprint(obj, p, cycle):\n name = obj.__class__.__name__\n with p.group(len(name) + 1, name + \"(\", \")\"):\n if cycle:\n p.text(\"...\")\n else:\n p.pretty(obj.default_factory)\n p.text(\",\")\n p.breakable()\n p.pretty(dict(obj))\n\n\ndef _ordereddict_pprint(obj, p, cycle):\n name = obj.__class__.__name__\n with p.group(len(name) + 1, name + \"(\", \")\"):\n if cycle:\n p.text(\"...\")\n elif len(obj):\n p.pretty(list(obj.items()))\n\n\ndef _deque_pprint(obj, p, cycle):\n name = obj.__class__.__name__\n with p.group(len(name) + 1, name + \"(\", \")\"):\n if cycle:\n p.text(\"...\")\n else:\n p.pretty(list(obj))\n\n\ndef _counter_pprint(obj, p, cycle):\n name = obj.__class__.__name__\n with p.group(len(name) + 1, name + \"(\", \")\"):\n if cycle:\n p.text(\"...\")\n elif len(obj):\n p.pretty(dict(obj))\n" }, { "path": "xonsh/proc.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"Interface for running Python functions as subprocess-mode commands.\n\nCode for several helper methods in the `ProcProxy` class have been reproduced\nwithout modification from `subprocess.py` in the Python 3.4.2 standard library.\nThe contents of `subprocess.py` (and, thus, the reproduced methods) are\nCopyright (c) 2003-2005 by Peter Astrand and were\nlicensed to the Python Software foundation under a Contributor Agreement.\n\"\"\"\nimport io\nimport os\nimport re\nimport sys\nimport time\nimport queue\nimport array\nimport ctypes\nimport signal\nimport inspect\nimport builtins\nimport functools\nimport threading\nimport subprocess\nimport collections.abc as cabc\n\nfrom xonsh.platform import (\n ON_WINDOWS,\n ON_POSIX,\n ON_MSYS,\n ON_CYGWIN,\n CAN_RESIZE_WINDOW,\n LFLAG,\n CC,\n)\nfrom xonsh.tools import (\n redirect_stdout,\n redirect_stderr,\n print_exception,\n XonshCalledProcessError,\n findfirst,\n on_main_thread,\n XonshError,\n format_std_prepost,\n ALIAS_KWARG_NAMES,\n)\nfrom xonsh.lazyasd import lazyobject, LazyObject\nfrom xonsh.jobs import wait_for_active_job, give_terminal_to, _continue\nfrom xonsh.lazyimps import fcntl, termios, _winapi, msvcrt, winutils\n\n# these decorators are imported for users back-compatible\nfrom xonsh.tools import unthreadable, uncapturable # NOQA\n\n# foreground has be deprecated\nforeground = unthreadable\n\n\n@lazyobject\ndef STDOUT_CAPTURE_KINDS():\n return frozenset([\"stdout\", \"object\"])\n\n\n# The following escape codes are xterm codes.\n# See http://rtfm.etla.org/xterm/ctlseq.html for more.\nMODE_NUMS = (\"1049\", \"47\", \"1047\")\nSTART_ALTERNATE_MODE = LazyObject(\n lambda: frozenset(\"\\x1b[?{0}h\".format(i).encode() for i in MODE_NUMS),\n globals(),\n \"START_ALTERNATE_MODE\",\n)\nEND_ALTERNATE_MODE = LazyObject(\n lambda: frozenset(\"\\x1b[?{0}l\".format(i).encode() for i in MODE_NUMS),\n globals(),\n \"END_ALTERNATE_MODE\",\n)\nALTERNATE_MODE_FLAGS = LazyObject(\n lambda: tuple(START_ALTERNATE_MODE) + tuple(END_ALTERNATE_MODE),\n globals(),\n \"ALTERNATE_MODE_FLAGS\",\n)\nRE_HIDDEN_BYTES = LazyObject(\n lambda: re.compile(b\"(\\001.*?\\002)\"), globals(), \"RE_HIDDEN\"\n)\n\n\n@lazyobject\ndef RE_VT100_ESCAPE():\n return re.compile(b\"(\\x9B|\\x1B\\\\[)[0-?]*[ -\\\\/]*[@-~]\")\n\n\n@lazyobject\ndef RE_HIDE_ESCAPE():\n return re.compile(\n b\"(\" + RE_HIDDEN_BYTES.pattern + b\"|\" + RE_VT100_ESCAPE.pattern + b\")\"\n )\n\n\nclass QueueReader:\n \"\"\"Provides a file-like interface to reading from a queue.\"\"\"\n\n def __init__(self, fd, timeout=None):\n \"\"\"\n Parameters\n ----------\n fd : int\n A file descriptor\n timeout : float or None, optional\n The queue reading timeout.\n \"\"\"\n self.fd = fd\n self.timeout = timeout\n self.closed = False\n self.queue = queue.Queue()\n self.thread = None\n\n def close(self):\n \"\"\"close the reader\"\"\"\n self.closed = True\n\n def is_fully_read(self):\n \"\"\"Returns whether or not the queue is fully read and the reader is\n closed.\n \"\"\"\n return (\n self.closed\n and (self.thread is None or not self.thread.is_alive())\n and self.queue.empty()\n )\n\n def read_queue(self):\n \"\"\"Reads a single chunk from the queue. This is blocking if\n the timeout is None and non-blocking otherwise.\n \"\"\"\n try:\n return self.queue.get(block=True, timeout=self.timeout)\n except queue.Empty:\n return b\"\"\n\n def read(self, size=-1):\n \"\"\"Reads bytes from the file.\"\"\"\n i = 0\n buf = b\"\"\n while size < 0 or i != size:\n line = self.read_queue()\n if line:\n buf += line\n else:\n break\n i += len(line)\n return buf\n\n def readline(self, size=-1):\n \"\"\"Reads a line, or a partial line from the file descriptor.\"\"\"\n i = 0\n nl = b\"\\n\"\n buf = b\"\"\n while size < 0 or i != size:\n line = self.read_queue()\n if line:\n buf += line\n if line.endswith(nl):\n break\n else:\n break\n i += len(line)\n return buf\n\n def _read_all_lines(self):\n \"\"\"This reads all remaining lines in a blocking fashion.\"\"\"\n lines = []\n while not self.is_fully_read():\n chunk = self.read_queue()\n lines.extend(chunk.splitlines(keepends=True))\n return lines\n\n def readlines(self, hint=-1):\n \"\"\"Reads lines from the file descriptor. This is blocking for negative\n hints (i.e. read all the remaining lines) and non-blocking otherwise.\n \"\"\"\n if hint == -1:\n return self._read_all_lines()\n lines = []\n while len(lines) != hint:\n chunk = self.read_queue()\n if not chunk:\n break\n lines.extend(chunk.splitlines(keepends=True))\n return lines\n\n def fileno(self):\n \"\"\"Returns the file descriptor number.\"\"\"\n return self.fd\n\n @staticmethod\n def readable():\n \"\"\"Returns true, because this object is always readable.\"\"\"\n return True\n\n def iterqueue(self):\n \"\"\"Iterates through all remaining chunks in a blocking fashion.\"\"\"\n while not self.is_fully_read():\n chunk = self.read_queue()\n if not chunk:\n continue\n yield chunk\n\n\ndef populate_fd_queue(reader, fd, queue):\n \"\"\"Reads 1 kb of data from a file descriptor into a queue.\n If this ends or fails, it flags the calling reader object as closed.\n \"\"\"\n while True:\n try:\n c = os.read(fd, 1024)\n except OSError:\n reader.closed = True\n break\n if c:\n queue.put(c)\n else:\n reader.closed = True\n break\n\n\nclass NonBlockingFDReader(QueueReader):\n \"\"\"A class for reading characters from a file descriptor on a background\n thread. This has the advantages that the calling thread can close the\n file and that the reading does not block the calling thread.\n \"\"\"\n\n def __init__(self, fd, timeout=None):\n \"\"\"\n Parameters\n ----------\n fd : int\n A file descriptor\n timeout : float or None, optional\n The queue reading timeout.\n \"\"\"\n super().__init__(fd, timeout=timeout)\n # start reading from stream\n self.thread = threading.Thread(\n target=populate_fd_queue, args=(self, self.fd, self.queue)\n )\n self.thread.daemon = True\n self.thread.start()\n\n\ndef populate_buffer(reader, fd, buffer, chunksize):\n \"\"\"Reads bytes from the file descriptor and copies them into a buffer.\n\n The reads happen in parallel using the pread() syscall; which is only\n available on POSIX systems. If the read fails for any reason, the reader is\n flagged as closed.\n \"\"\"\n offset = 0\n while True:\n try:\n buf = os.pread(fd, chunksize, offset)\n except OSError:\n reader.closed = True\n break\n if buf:\n buffer.write(buf)\n offset += len(buf)\n else:\n reader.closed = True\n break\n\n\nclass BufferedFDParallelReader:\n \"\"\"Buffered, parallel background thread reader.\"\"\"\n\n def __init__(self, fd, buffer=None, chunksize=1024):\n \"\"\"\n Parameters\n ----------\n fd : int\n File descriptor from which to read.\n buffer : binary file-like or None, optional\n A buffer to write bytes into. If None, a new BytesIO object\n is created.\n chunksize : int, optional\n The max size of the parallel reads, default 1 kb.\n \"\"\"\n self.fd = fd\n self.buffer = io.BytesIO() if buffer is None else buffer\n self.chunksize = chunksize\n self.closed = False\n # start reading from stream\n self.thread = threading.Thread(\n target=populate_buffer, args=(self, fd, self.buffer, chunksize)\n )\n self.thread.daemon = True\n\n self.thread.start()\n\n\ndef _expand_console_buffer(cols, max_offset, expandsize, orig_posize, fd):\n # if we are getting close to the end of the console buffer,\n # expand it so that we can read from it successfully.\n if cols == 0:\n return orig_posize[-1], max_offset, orig_posize\n rows = ((max_offset + expandsize) // cols) + 1\n winutils.set_console_screen_buffer_size(cols, rows, fd=fd)\n orig_posize = orig_posize[:3] + (rows,)\n max_offset = (rows - 1) * cols\n return rows, max_offset, orig_posize\n\n\ndef populate_console(reader, fd, buffer, chunksize, queue, expandsize=None):\n \"\"\"Reads bytes from the file descriptor and puts lines into the queue.\n The reads happened in parallel,\n using xonsh.winutils.read_console_output_character(),\n and is thus only available on windows. If the read fails for any reason,\n the reader is flagged as closed.\n \"\"\"\n # OK, so this function is super annoying because Windows stores its\n # buffers as a 2D regular, dense array -- without trailing newlines.\n # Meanwhile, we want to add *lines* to the queue. Also, as is typical\n # with parallel reads, the entire buffer that you ask for may not be\n # filled. Thus we have to deal with the full generality.\n # 1. reads may end in the middle of a line\n # 2. excess whitespace at the end of a line may not be real, unless\n # 3. you haven't read to the end of the line yet!\n # So there are alignment issues everywhere. Also, Windows will automatically\n # read past the current cursor position, even though there is presumably\n # nothing to see there.\n #\n # These chunked reads basically need to happen like this because,\n # a. The default buffer size is HUGE for the console (90k lines x 120 cols)\n # as so we can't just read in everything at the end and see what we\n # care about without a noticeable performance hit.\n # b. Even with this huge size, it is still possible to write more lines than\n # this, so we should scroll along with the console.\n # Unfortunately, because we do not have control over the terminal emulator,\n # It is not possible to compute how far back we should set the beginning\n # read position because we don't know how many characters have been popped\n # off the top of the buffer. If we did somehow know this number we could do\n # something like the following:\n #\n # new_offset = (y*cols) + x\n # if new_offset == max_offset:\n # new_offset -= scrolled_offset\n # x = new_offset%cols\n # y = new_offset//cols\n # continue\n #\n # So this method is imperfect and only works as long as the screen has\n # room to expand to. Thus the trick here is to expand the screen size\n # when we get close enough to the end of the screen. There remain some\n # async issues related to not being able to set the cursor position.\n # but they just affect the alignment / capture of the output of the\n # first command run after a screen resize.\n if expandsize is None:\n expandsize = 100 * chunksize\n x, y, cols, rows = posize = winutils.get_position_size(fd)\n pre_x = pre_y = -1\n orig_posize = posize\n offset = (cols * y) + x\n max_offset = (rows - 1) * cols\n # I believe that there is a bug in PTK that if we reset the\n # cursor position, the cursor on the next prompt is accidentally on\n # the next line. If this is fixed, uncomment the following line.\n # if max_offset < offset + expandsize:\n # rows, max_offset, orig_posize = _expand_console_buffer(\n # cols, max_offset, expandsize,\n # orig_posize, fd)\n # winutils.set_console_cursor_position(x, y, fd=fd)\n while True:\n posize = winutils.get_position_size(fd)\n offset = (cols * y) + x\n if ((posize[1], posize[0]) <= (y, x) and posize[2:] == (cols, rows)) or (\n pre_x == x and pre_y == y\n ):\n # already at or ahead of the current cursor position.\n if reader.closed:\n break\n else:\n time.sleep(reader.timeout)\n continue\n elif max_offset <= offset + expandsize:\n ecb = _expand_console_buffer(cols, max_offset, expandsize, orig_posize, fd)\n rows, max_offset, orig_posize = ecb\n continue\n elif posize[2:] == (cols, rows):\n # cursor updated but screen size is the same.\n pass\n else:\n # screen size changed, which is offset preserving\n orig_posize = posize\n cols, rows = posize[2:]\n x = offset % cols\n y = offset // cols\n pre_x = pre_y = -1\n max_offset = (rows - 1) * cols\n continue\n try:\n buf = winutils.read_console_output_character(\n x=x, y=y, fd=fd, buf=buffer, bufsize=chunksize, raw=True\n )\n except (OSError, IOError):\n reader.closed = True\n break\n # cursor position and offset\n if not reader.closed:\n buf = buf.rstrip()\n nread = len(buf)\n if nread == 0:\n time.sleep(reader.timeout)\n continue\n cur_x, cur_y = posize[0], posize[1]\n cur_offset = (cols * cur_y) + cur_x\n beg_offset = (cols * y) + x\n end_offset = beg_offset + nread\n if end_offset > cur_offset and cur_offset != max_offset:\n buf = buf[: cur_offset - end_offset]\n # convert to lines\n xshift = cols - x\n yshift = (nread // cols) + (1 if nread % cols > 0 else 0)\n lines = [buf[:xshift]]\n lines += [\n buf[l * cols + xshift : (l + 1) * cols + xshift] for l in range(yshift)\n ]\n lines = [line for line in lines if line]\n if not lines:\n time.sleep(reader.timeout)\n continue\n # put lines in the queue\n nl = b\"\\n\"\n for line in lines[:-1]:\n queue.put(line.rstrip() + nl)\n if len(lines[-1]) == xshift:\n queue.put(lines[-1].rstrip() + nl)\n else:\n queue.put(lines[-1])\n # update x and y locations\n if (beg_offset + len(buf)) % cols == 0:\n new_offset = beg_offset + len(buf)\n else:\n new_offset = beg_offset + len(buf.rstrip())\n pre_x = x\n pre_y = y\n x = new_offset % cols\n y = new_offset // cols\n time.sleep(reader.timeout)\n\n\nclass ConsoleParallelReader(QueueReader):\n \"\"\"Parallel reader for consoles that runs in a background thread.\n This is only needed, available, and useful on Windows.\n \"\"\"\n\n def __init__(self, fd, buffer=None, chunksize=1024, timeout=None):\n \"\"\"\n Parameters\n ----------\n fd : int\n Standard buffer file descriptor, 0 for stdin, 1 for stdout (default),\n and 2 for stderr.\n buffer : ctypes.c_wchar_p, optional\n An existing buffer to (re-)use.\n chunksize : int, optional\n The max size of the parallel reads, default 1 kb.\n timeout : float, optional\n The queue reading timeout.\n \"\"\"\n timeout = timeout or builtins.__xonsh__.env.get(\"XONSH_PROC_FREQUENCY\")\n super().__init__(fd, timeout=timeout)\n self._buffer = buffer # this cannot be public\n if buffer is None:\n self._buffer = ctypes.c_char_p(b\" \" * chunksize)\n self.chunksize = chunksize\n # start reading from stream\n self.thread = threading.Thread(\n target=populate_console,\n args=(self, fd, self._buffer, chunksize, self.queue),\n )\n self.thread.daemon = True\n self.thread.start()\n\n\ndef safe_fdclose(handle, cache=None):\n \"\"\"Closes a file handle in the safest way possible, and potentially\n storing the result.\n \"\"\"\n if cache is not None and cache.get(handle, False):\n return\n status = True\n if handle is None:\n pass\n elif isinstance(handle, int):\n if handle >= 3:\n # don't close stdin, stdout, stderr, -1\n try:\n os.close(handle)\n except OSError:\n status = False\n elif handle is sys.stdin or handle is sys.stdout or handle is sys.stderr:\n # don't close stdin, stdout, or stderr\n pass\n else:\n try:\n handle.close()\n except OSError:\n status = False\n if cache is not None:\n cache[handle] = status\n\n\ndef safe_flush(handle):\n \"\"\"Attempts to safely flush a file handle, returns success bool.\"\"\"\n status = True\n try:\n handle.flush()\n except OSError:\n status = False\n return status\n\n\ndef still_writable(fd):\n \"\"\"Determines whether a file descriptor is still writable by trying to\n write an empty string and seeing if it fails.\n \"\"\"\n try:\n os.write(fd, b\"\")\n status = True\n except OSError:\n status = False\n return status\n\n\nclass PopenThread(threading.Thread):\n \"\"\"A thread for running and managing subprocess. This allows reading\n from the stdin, stdout, and stderr streams in a non-blocking fashion.\n\n This takes the same arguments and keyword arguments as regular Popen.\n This requires that the captured_stdout and captured_stderr attributes\n to be set following instantiation.\n \"\"\"\n\n def __init__(self, *args, stdin=None, stdout=None, stderr=None, **kwargs):\n super().__init__()\n self.lock = threading.RLock()\n env = builtins.__xonsh__.env\n # stdin setup\n self.orig_stdin = stdin\n if stdin is None:\n self.stdin_fd = 0\n elif isinstance(stdin, int):\n self.stdin_fd = stdin\n else:\n self.stdin_fd = stdin.fileno()\n self.store_stdin = env.get(\"XONSH_STORE_STDIN\")\n self.timeout = env.get(\"XONSH_PROC_FREQUENCY\")\n self.in_alt_mode = False\n self.stdin_mode = None\n # stdout setup\n self.orig_stdout = stdout\n self.stdout_fd = 1 if stdout is None else stdout.fileno()\n self._set_pty_size()\n # stderr setup\n self.orig_stderr = stderr\n # Set some signal handles, if we can. Must come before process\n # is started to prevent deadlock on windows\n self.proc = None # has to be here for closure for handles\n self.old_int_handler = self.old_winch_handler = None\n self.old_tstp_handler = self.old_quit_handler = None\n if on_main_thread():\n self.old_int_handler = signal.signal(signal.SIGINT, self._signal_int)\n if ON_POSIX:\n self.old_tstp_handler = signal.signal(signal.SIGTSTP, self._signal_tstp)\n self.old_quit_handler = signal.signal(signal.SIGQUIT, self._signal_quit)\n if CAN_RESIZE_WINDOW:\n self.old_winch_handler = signal.signal(\n signal.SIGWINCH, self._signal_winch\n )\n # start up process\n if ON_WINDOWS and stdout is not None:\n os.set_inheritable(stdout.fileno(), False)\n\n try:\n self.proc = proc = subprocess.Popen(\n *args, stdin=stdin, stdout=stdout, stderr=stderr, **kwargs\n )\n except Exception:\n self._clean_up()\n raise\n\n self.pid = proc.pid\n self.universal_newlines = uninew = proc.universal_newlines\n if uninew:\n self.encoding = enc = env.get(\"XONSH_ENCODING\")\n self.encoding_errors = err = env.get(\"XONSH_ENCODING_ERRORS\")\n self.stdin = io.BytesIO() # stdin is always bytes!\n self.stdout = io.TextIOWrapper(io.BytesIO(), encoding=enc, errors=err)\n self.stderr = io.TextIOWrapper(io.BytesIO(), encoding=enc, errors=err)\n else:\n self.encoding = self.encoding_errors = None\n self.stdin = io.BytesIO()\n self.stdout = io.BytesIO()\n self.stderr = io.BytesIO()\n self.suspended = False\n self.prevs_are_closed = False\n self.start()\n\n def run(self):\n \"\"\"Runs the subprocess by performing a parallel read on stdin if allowed,\n and copying bytes from captured_stdout to stdout and bytes from\n captured_stderr to stderr.\n \"\"\"\n proc = self.proc\n spec = self._wait_and_getattr(\"spec\")\n # get stdin and apply parallel reader if needed.\n stdin = self.stdin\n if self.orig_stdin is None:\n origin = None\n elif ON_POSIX and self.store_stdin:\n origin = self.orig_stdin\n origfd = origin if isinstance(origin, int) else origin.fileno()\n origin = BufferedFDParallelReader(origfd, buffer=stdin)\n else:\n origin = None\n # get non-blocking stdout\n stdout = self.stdout.buffer if self.universal_newlines else self.stdout\n capout = spec.captured_stdout\n if capout is None:\n procout = None\n else:\n procout = NonBlockingFDReader(capout.fileno(), timeout=self.timeout)\n # get non-blocking stderr\n stderr = self.stderr.buffer if self.universal_newlines else self.stderr\n caperr = spec.captured_stderr\n if caperr is None:\n procerr = None\n else:\n procerr = NonBlockingFDReader(caperr.fileno(), timeout=self.timeout)\n # initial read from buffer\n self._read_write(procout, stdout, sys.__stdout__)\n self._read_write(procerr, stderr, sys.__stderr__)\n # loop over reads while process is running.\n i = j = cnt = 1\n while proc.poll() is None:\n # this is here for CPU performance reasons.\n if i + j == 0:\n cnt = min(cnt + 1, 1000)\n tout = self.timeout * cnt\n if procout is not None:\n procout.timeout = tout\n if procerr is not None:\n procerr.timeout = tout\n elif cnt == 1:\n pass\n else:\n cnt = 1\n if procout is not None:\n procout.timeout = self.timeout\n if procerr is not None:\n procerr.timeout = self.timeout\n # redirect some output!\n i = self._read_write(procout, stdout, sys.__stdout__)\n j = self._read_write(procerr, stderr, sys.__stderr__)\n if self.suspended:\n break\n if self.suspended:\n return\n # close files to send EOF to non-blocking reader.\n # capout & caperr seem to be needed only by Windows, while\n # orig_stdout & orig_stderr are need by posix and Windows.\n # Also, order seems to matter here,\n # with orig_* needed to be closed before cap*\n safe_fdclose(self.orig_stdout)\n safe_fdclose(self.orig_stderr)\n if ON_WINDOWS:\n safe_fdclose(capout)\n safe_fdclose(caperr)\n # read in the remaining data in a blocking fashion.\n while (procout is not None and not procout.is_fully_read()) or (\n procerr is not None and not procerr.is_fully_read()\n ):\n self._read_write(procout, stdout, sys.__stdout__)\n self._read_write(procerr, stderr, sys.__stderr__)\n # kill the process if it is still alive. Happens when piping.\n if proc.poll() is None:\n proc.terminate()\n\n def _wait_and_getattr(self, name):\n \"\"\"make sure the instance has a certain attr, and return it.\"\"\"\n while not hasattr(self, name):\n time.sleep(1e-7)\n return getattr(self, name)\n\n def _read_write(self, reader, writer, stdbuf):\n \"\"\"Reads a chunk of bytes from a buffer and write into memory or back\n down to the standard buffer, as appropriate. Returns the number of\n successful reads.\n \"\"\"\n if reader is None:\n return 0\n i = -1\n for i, chunk in enumerate(iter(reader.read_queue, b\"\")):\n self._alt_mode_switch(chunk, writer, stdbuf)\n if i >= 0:\n writer.flush()\n stdbuf.flush()\n return i + 1\n\n def _alt_mode_switch(self, chunk, membuf, stdbuf):\n \"\"\"Enables recursively switching between normal capturing mode\n and 'alt' mode, which passes through values to the standard\n buffer. Pagers, text editors, curses applications, etc. use\n alternate mode.\n \"\"\"\n i, flag = findfirst(chunk, ALTERNATE_MODE_FLAGS)\n if flag is None:\n self._alt_mode_writer(chunk, membuf, stdbuf)\n else:\n # This code is executed when the child process switches the\n # terminal into or out of alternate mode. The line below assumes\n # that the user has opened vim, less, or similar, and writes writes\n # to stdin.\n j = i + len(flag)\n # write the first part of the chunk in the current mode.\n self._alt_mode_writer(chunk[:i], membuf, stdbuf)\n # switch modes\n # write the flag itself the current mode where alt mode is on\n # so that it is streamed to the terminal ASAP.\n # this is needed for terminal emulators to find the correct\n # positions before and after alt mode.\n alt_mode = flag in START_ALTERNATE_MODE\n if alt_mode:\n self.in_alt_mode = alt_mode\n self._alt_mode_writer(flag, membuf, stdbuf)\n self._enable_cbreak_stdin()\n else:\n self._alt_mode_writer(flag, membuf, stdbuf)\n self.in_alt_mode = alt_mode\n self._disable_cbreak_stdin()\n # recurse this function, but without the current flag.\n self._alt_mode_switch(chunk[j:], membuf, stdbuf)\n\n def _alt_mode_writer(self, chunk, membuf, stdbuf):\n \"\"\"Write bytes to the standard buffer if in alt mode or otherwise\n to the in-memory buffer.\n \"\"\"\n if not chunk:\n pass # don't write empty values\n elif self.in_alt_mode:\n stdbuf.buffer.write(chunk)\n else:\n with self.lock:\n p = membuf.tell()\n membuf.seek(0, io.SEEK_END)\n membuf.write(chunk)\n membuf.seek(p)\n\n #\n # Window resize handlers\n #\n\n def _signal_winch(self, signum, frame):\n \"\"\"Signal handler for SIGWINCH - window size has changed.\"\"\"\n self.send_signal(signal.SIGWINCH)\n self._set_pty_size()\n\n def _set_pty_size(self):\n \"\"\"Sets the window size of the child pty based on the window size of\n our own controlling terminal.\n \"\"\"\n if ON_WINDOWS or not os.isatty(self.stdout_fd):\n return\n # Get the terminal size of the real terminal, set it on the\n # pseudoterminal.\n buf = array.array(\"h\", [0, 0, 0, 0])\n # 1 = stdout here\n try:\n fcntl.ioctl(1, termios.TIOCGWINSZ, buf, True)\n fcntl.ioctl(self.stdout_fd, termios.TIOCSWINSZ, buf)\n except OSError:\n pass\n\n #\n # SIGINT handler\n #\n\n def _signal_int(self, signum, frame):\n \"\"\"Signal handler for SIGINT - Ctrl+C may have been pressed.\"\"\"\n self.send_signal(signum)\n if self.proc is not None and self.proc.poll() is not None:\n self._restore_sigint(frame=frame)\n if on_main_thread():\n signal.pthread_kill(threading.get_ident(), signal.SIGINT)\n\n def _restore_sigint(self, frame=None):\n old = self.old_int_handler\n if old is not None:\n if on_main_thread():\n signal.signal(signal.SIGINT, old)\n self.old_int_handler = None\n if frame is not None:\n self._disable_cbreak_stdin()\n if old is not None and old is not self._signal_int:\n old(signal.SIGINT, frame)\n\n #\n # SIGTSTP handler\n #\n\n def _signal_tstp(self, signum, frame):\n \"\"\"Signal handler for suspending SIGTSTP - Ctrl+Z may have been pressed.\n \"\"\"\n self.suspended = True\n self.send_signal(signum)\n self._restore_sigtstp(frame=frame)\n\n def _restore_sigtstp(self, frame=None):\n old = self.old_tstp_handler\n if old is not None:\n if on_main_thread():\n signal.signal(signal.SIGTSTP, old)\n self.old_tstp_handler = None\n if frame is not None:\n self._disable_cbreak_stdin()\n\n #\n # SIGQUIT handler\n #\n\n def _signal_quit(self, signum, frame):\n r\"\"\"Signal handler for quiting SIGQUIT - Ctrl+\\ may have been pressed.\n \"\"\"\n self.send_signal(signum)\n self._restore_sigquit(frame=frame)\n\n def _restore_sigquit(self, frame=None):\n old = self.old_quit_handler\n if old is not None:\n if on_main_thread():\n signal.signal(signal.SIGQUIT, old)\n self.old_quit_handler = None\n if frame is not None:\n self._disable_cbreak_stdin()\n\n #\n # cbreak mode handlers\n #\n\n def _enable_cbreak_stdin(self):\n if not ON_POSIX:\n return\n try:\n self.stdin_mode = termios.tcgetattr(self.stdin_fd)[:]\n except termios.error:\n # this can happen for cases where another process is controlling\n # xonsh's tty device, such as in testing.\n self.stdin_mode = None\n return\n new = self.stdin_mode[:]\n new[LFLAG] &= ~(termios.ECHO | termios.ICANON)\n new[CC][termios.VMIN] = 1\n new[CC][termios.VTIME] = 0\n try:\n # termios.TCSAFLUSH may be less reliable than termios.TCSANOW\n termios.tcsetattr(self.stdin_fd, termios.TCSANOW, new)\n except termios.error:\n self._disable_cbreak_stdin()\n\n def _disable_cbreak_stdin(self):\n if not ON_POSIX or self.stdin_mode is None:\n return\n new = self.stdin_mode[:]\n new[LFLAG] |= termios.ECHO | termios.ICANON\n new[CC][termios.VMIN] = 1\n new[CC][termios.VTIME] = 0\n try:\n termios.tcsetattr(self.stdin_fd, termios.TCSANOW, new)\n except termios.error:\n pass\n\n #\n # Dispatch methods\n #\n\n def poll(self):\n \"\"\"Dispatches to Popen.returncode.\"\"\"\n return self.proc.returncode\n\n def wait(self, timeout=None):\n \"\"\"Dispatches to Popen.wait(), but also does process cleanup such as\n joining this thread and replacing the original window size signal\n handler.\n \"\"\"\n self._disable_cbreak_stdin()\n rtn = self.proc.wait(timeout=timeout)\n self.join()\n # need to replace the old signal handlers somewhere...\n if self.old_winch_handler is not None and on_main_thread():\n signal.signal(signal.SIGWINCH, self.old_winch_handler)\n self.old_winch_handler = None\n self._clean_up()\n return rtn\n\n def _clean_up(self):\n self._restore_sigint()\n self._restore_sigtstp()\n self._restore_sigquit()\n\n @property\n def returncode(self):\n \"\"\"Process return code.\"\"\"\n return self.proc.returncode\n\n @returncode.setter\n def returncode(self, value):\n \"\"\"Process return code.\"\"\"\n self.proc.returncode = value\n\n @property\n def signal(self):\n \"\"\"Process signal, or None.\"\"\"\n s = getattr(self.proc, \"signal\", None)\n if s is None:\n rtn = self.returncode\n if rtn is not None and rtn != 0:\n s = (-1 * rtn, rtn < 0 if ON_WINDOWS else os.WCOREDUMP(rtn))\n return s\n\n @signal.setter\n def signal(self, value):\n \"\"\"Process signal, or None.\"\"\"\n self.proc.signal = value\n\n def send_signal(self, signal):\n \"\"\"Dispatches to Popen.send_signal().\"\"\"\n dt = 0.0\n while self.proc is None and dt < self.timeout:\n time.sleep(1e-7)\n dt += 1e-7\n if self.proc is None:\n return\n try:\n rtn = self.proc.send_signal(signal)\n except ProcessLookupError:\n # This can happen in the case of !(cmd) when the command has ended\n rtn = None\n return rtn\n\n def terminate(self):\n \"\"\"Dispatches to Popen.terminate().\"\"\"\n return self.proc.terminate()\n\n def kill(self):\n \"\"\"Dispatches to Popen.kill().\"\"\"\n return self.proc.kill()\n\n\nclass Handle(int):\n closed = False\n\n def Close(self, CloseHandle=None):\n CloseHandle = CloseHandle or _winapi.CloseHandle\n if not self.closed:\n self.closed = True\n CloseHandle(self)\n\n def Detach(self):\n if not self.closed:\n self.closed = True\n return int(self)\n raise ValueError(\"already closed\")\n\n def __repr__(self):\n return \"Handle(%d)\" % int(self)\n\n __del__ = Close\n __str__ = __repr__\n\n\nclass FileThreadDispatcher:\n \"\"\"Dispatches to different file handles depending on the\n current thread. Useful if you want file operation to go to different\n places for different threads.\n \"\"\"\n\n def __init__(self, default=None):\n \"\"\"\n Parameters\n ----------\n default : file-like or None, optional\n The file handle to write to if a thread cannot be found in\n the registry. If None, a new in-memory instance.\n\n Attributes\n ----------\n registry : dict\n Maps thread idents to file handles.\n \"\"\"\n if default is None:\n default = io.TextIOWrapper(io.BytesIO())\n self.default = default\n self.registry = {}\n\n def register(self, handle):\n \"\"\"Registers a file handle for the current thread. Returns self so\n that this method can be used in a with-statement.\n \"\"\"\n if handle is self:\n # prevent weird recurssion errors\n return self\n self.registry[threading.get_ident()] = handle\n return self\n\n def deregister(self):\n \"\"\"Removes the current thread from the registry.\"\"\"\n ident = threading.get_ident()\n if ident in self.registry:\n # don't remove if we have already been deregistered\n del self.registry[threading.get_ident()]\n\n @property\n def available(self):\n \"\"\"True if the thread is available in the registry.\"\"\"\n return threading.get_ident() in self.registry\n\n @property\n def handle(self):\n \"\"\"Gets the current handle for the thread.\"\"\"\n return self.registry.get(threading.get_ident(), self.default)\n\n def __enter__(self):\n pass\n\n def __exit__(self, ex_type, ex_value, ex_traceback):\n self.deregister()\n\n #\n # io.TextIOBase interface\n #\n\n @property\n def encoding(self):\n \"\"\"Gets the encoding for this thread's handle.\"\"\"\n return self.handle.encoding\n\n @property\n def errors(self):\n \"\"\"Gets the errors for this thread's handle.\"\"\"\n return self.handle.errors\n\n @property\n def newlines(self):\n \"\"\"Gets the newlines for this thread's handle.\"\"\"\n return self.handle.newlines\n\n @property\n def buffer(self):\n \"\"\"Gets the buffer for this thread's handle.\"\"\"\n return self.handle.buffer\n\n def detach(self):\n \"\"\"Detaches the buffer for the current thread.\"\"\"\n return self.handle.detach()\n\n def read(self, size=None):\n \"\"\"Reads from the handle for the current thread.\"\"\"\n return self.handle.read(size)\n\n def readline(self, size=-1):\n \"\"\"Reads a line from the handle for the current thread.\"\"\"\n return self.handle.readline(size)\n\n def readlines(self, hint=-1):\n \"\"\"Reads lines from the handle for the current thread.\"\"\"\n return self.handle.readlines(hint)\n\n def seek(self, offset, whence=io.SEEK_SET):\n \"\"\"Seeks the current file.\"\"\"\n return self.handle.seek(offset, whence)\n\n def tell(self):\n \"\"\"Reports the current position in the handle for the current thread.\"\"\"\n return self.handle.tell()\n\n def write(self, s):\n \"\"\"Writes to this thread's handle. This also flushes, just to be\n extra sure the string was written.\n \"\"\"\n h = self.handle\n try:\n r = h.write(s)\n h.flush()\n except OSError:\n r = None\n return r\n\n @property\n def line_buffering(self):\n \"\"\"Gets if line buffering for this thread's handle enabled.\"\"\"\n return self.handle.line_buffering\n\n #\n # io.IOBase interface\n #\n\n def close(self):\n \"\"\"Closes the current thread's handle.\"\"\"\n return self.handle.close()\n\n @property\n def closed(self):\n \"\"\"Is the thread's handle closed.\"\"\"\n return self.handle.closed\n\n def fileno(self):\n \"\"\"Returns the file descriptor for the current thread.\"\"\"\n return self.handle.fileno()\n\n def flush(self):\n \"\"\"Flushes the file descriptor for the current thread.\"\"\"\n return safe_flush(self.handle)\n\n def isatty(self):\n \"\"\"Returns if the file descriptor for the current thread is a tty.\"\"\"\n return self.handle.isatty()\n\n def readable(self):\n \"\"\"Returns if file descriptor for the current thread is readable.\"\"\"\n return self.handle.readable()\n\n def seekable(self):\n \"\"\"Returns if file descriptor for the current thread is seekable.\"\"\"\n return self.handle.seekable()\n\n def truncate(self, size=None):\n \"\"\"Truncates the file for for the current thread.\"\"\"\n return self.handle.truncate()\n\n def writable(self, size=None):\n \"\"\"Returns if file descriptor for the current thread is writable.\"\"\"\n return self.handle.writable(size)\n\n def writelines(self):\n \"\"\"Writes lines for the file descriptor for the current thread.\"\"\"\n return self.handle.writelines()\n\n\n# These should NOT be lazy since they *need* to get the true stdout from the\n# main thread. Also their creation time should be negligible.\nSTDOUT_DISPATCHER = FileThreadDispatcher(default=sys.stdout)\nSTDERR_DISPATCHER = FileThreadDispatcher(default=sys.stderr)\n\n\ndef parse_proxy_return(r, stdout, stderr):\n \"\"\"Proxies may return a variety of outputs. This handles them generally.\n\n Parameters\n ----------\n r : tuple, str, int, or None\n Return from proxy function\n stdout : file-like\n Current stdout stream\n stdout : file-like\n Current stderr stream\n\n Returns\n -------\n cmd_result : int\n The return code of the proxy\n \"\"\"\n cmd_result = 0\n if isinstance(r, str):\n stdout.write(r)\n stdout.flush()\n elif isinstance(r, int):\n cmd_result = r\n elif isinstance(r, cabc.Sequence):\n rlen = len(r)\n if rlen > 0 and r[0] is not None:\n stdout.write(r[0])\n stdout.flush()\n if rlen > 1 and r[1] is not None:\n stderr.write(r[1])\n stderr.flush()\n if rlen > 2 and r[2] is not None:\n cmd_result = r[2]\n elif r is not None:\n # for the random object...\n stdout.write(str(r))\n stdout.flush()\n return cmd_result\n\n\ndef proxy_zero(f, args, stdin, stdout, stderr, spec, stack):\n \"\"\"Calls a proxy function which takes no parameters.\"\"\"\n return f()\n\n\ndef proxy_one(f, args, stdin, stdout, stderr, spec, stack):\n \"\"\"Calls a proxy function which takes one parameter: args\"\"\"\n return f(args)\n\n\ndef proxy_two(f, args, stdin, stdout, stderr, spec, stack):\n \"\"\"Calls a proxy function which takes two parameter: args and stdin.\"\"\"\n return f(args, stdin)\n\n\ndef proxy_three(f, args, stdin, stdout, stderr, spec, stack):\n \"\"\"Calls a proxy function which takes three parameter: args, stdin, stdout.\n \"\"\"\n return f(args, stdin, stdout)\n\n\ndef proxy_four(f, args, stdin, stdout, stderr, spec, stack):\n \"\"\"Calls a proxy function which takes four parameter: args, stdin, stdout,\n and stderr.\n \"\"\"\n return f(args, stdin, stdout, stderr)\n\n\ndef proxy_five(f, args, stdin, stdout, stderr, spec, stack):\n \"\"\"Calls a proxy function which takes four parameter: args, stdin, stdout,\n stderr, and spec.\n \"\"\"\n return f(args, stdin, stdout, stderr, spec)\n\n\nPROXIES = (proxy_zero, proxy_one, proxy_two, proxy_three, proxy_four, proxy_five)\n\n\ndef partial_proxy(f):\n \"\"\"Dispatches the appropriate proxy function based on the number of args.\"\"\"\n numargs = 0\n for name, param in inspect.signature(f).parameters.items():\n if (\n param.kind == param.POSITIONAL_ONLY\n or param.kind == param.POSITIONAL_OR_KEYWORD\n ):\n numargs += 1\n elif name in ALIAS_KWARG_NAMES and param.kind == param.KEYWORD_ONLY:\n numargs += 1\n if numargs < 6:\n return functools.partial(PROXIES[numargs], f)\n elif numargs == 6:\n # don't need to partial.\n return f\n else:\n e = \"Expected proxy with 6 or fewer arguments for {}, not {}\"\n raise XonshError(e.format(\", \".join(ALIAS_KWARG_NAMES), numargs))\n\n\nclass ProcProxyThread(threading.Thread):\n \"\"\"\n Class representing a function to be run as a subprocess-mode command.\n \"\"\"\n\n def __init__(\n self,\n f,\n args,\n stdin=None,\n stdout=None,\n stderr=None,\n universal_newlines=False,\n close_fds=False,\n env=None,\n ):\n \"\"\"Parameters\n ----------\n f : function\n The function to be executed.\n args : list\n A (possibly empty) list containing the arguments that were given on\n the command line\n stdin : file-like, optional\n A file-like object representing stdin (input can be read from\n here). If `stdin` is not provided or if it is explicitly set to\n `None`, then an instance of `io.StringIO` representing an empty\n file is used.\n stdout : file-like, optional\n A file-like object representing stdout (normal output can be\n written here). If `stdout` is not provided or if it is explicitly\n set to `None`, then `sys.stdout` is used.\n stderr : file-like, optional\n A file-like object representing stderr (error output can be\n written here). If `stderr` is not provided or if it is explicitly\n set to `None`, then `sys.stderr` is used.\n universal_newlines : bool, optional\n Whether or not to use universal newlines.\n close_fds : bool, optional\n Whether or not to close file descriptors. This is here for Popen\n compatability and currently does nothing.\n env : Mapping, optional\n Environment mapping.\n \"\"\"\n self.orig_f = f\n self.f = partial_proxy(f)\n self.args = args\n self.pid = None\n self.returncode = None\n self._closed_handle_cache = {}\n\n handles = self._get_handles(stdin, stdout, stderr)\n (\n self.p2cread,\n self.p2cwrite,\n self.c2pread,\n self.c2pwrite,\n self.errread,\n self.errwrite,\n ) = handles\n\n # default values\n self.stdin = stdin\n self.stdout = stdout\n self.stderr = stderr\n self.close_fds = close_fds\n self.env = env or builtins.__xonsh__.env\n self._interrupted = False\n\n if ON_WINDOWS:\n if self.p2cwrite != -1:\n self.p2cwrite = msvcrt.open_osfhandle(self.p2cwrite.Detach(), 0)\n if self.c2pread != -1:\n self.c2pread = msvcrt.open_osfhandle(self.c2pread.Detach(), 0)\n if self.errread != -1:\n self.errread = msvcrt.open_osfhandle(self.errread.Detach(), 0)\n\n if self.p2cwrite != -1:\n self.stdin = io.open(self.p2cwrite, \"wb\", -1)\n if universal_newlines:\n self.stdin = io.TextIOWrapper(\n self.stdin, write_through=True, line_buffering=False\n )\n elif isinstance(stdin, int) and stdin != 0:\n self.stdin = io.open(stdin, \"wb\", -1)\n\n if self.c2pread != -1:\n self.stdout = io.open(self.c2pread, \"rb\", -1)\n if universal_newlines:\n self.stdout = io.TextIOWrapper(self.stdout)\n\n if self.errread != -1:\n self.stderr = io.open(self.errread, \"rb\", -1)\n if universal_newlines:\n self.stderr = io.TextIOWrapper(self.stderr)\n\n # Set some signal handles, if we can. Must come before process\n # is started to prevent deadlock on windows\n self.old_int_handler = None\n if on_main_thread():\n self.old_int_handler = signal.signal(signal.SIGINT, self._signal_int)\n # start up the proc\n super().__init__()\n self.start()\n\n def __del__(self):\n self._restore_sigint()\n\n def run(self):\n \"\"\"Set up input/output streams and execute the child function in a new\n thread. This is part of the `threading.Thread` interface and should\n not be called directly.\n \"\"\"\n if self.f is None:\n return\n spec = self._wait_and_getattr(\"spec\")\n last_in_pipeline = spec.last_in_pipeline\n if last_in_pipeline:\n capout = spec.captured_stdout # NOQA\n caperr = spec.captured_stderr # NOQA\n env = builtins.__xonsh__.env\n enc = env.get(\"XONSH_ENCODING\")\n err = env.get(\"XONSH_ENCODING_ERRORS\")\n if ON_WINDOWS:\n if self.p2cread != -1:\n self.p2cread = msvcrt.open_osfhandle(self.p2cread.Detach(), 0)\n if self.c2pwrite != -1:\n self.c2pwrite = msvcrt.open_osfhandle(self.c2pwrite.Detach(), 0)\n if self.errwrite != -1:\n self.errwrite = msvcrt.open_osfhandle(self.errwrite.Detach(), 0)\n # get stdin\n if self.stdin is None:\n sp_stdin = None\n elif self.p2cread != -1:\n sp_stdin = io.TextIOWrapper(\n io.open(self.p2cread, \"rb\", -1), encoding=enc, errors=err\n )\n else:\n sp_stdin = sys.stdin\n # stdout\n if self.c2pwrite != -1:\n sp_stdout = io.TextIOWrapper(\n io.open(self.c2pwrite, \"wb\", -1), encoding=enc, errors=err\n )\n else:\n sp_stdout = sys.stdout\n # stderr\n if self.errwrite == self.c2pwrite:\n sp_stderr = sp_stdout\n elif self.errwrite != -1:\n sp_stderr = io.TextIOWrapper(\n io.open(self.errwrite, \"wb\", -1), encoding=enc, errors=err\n )\n else:\n sp_stderr = sys.stderr\n # run the function itself\n try:\n with STDOUT_DISPATCHER.register(sp_stdout), STDERR_DISPATCHER.register(\n sp_stderr\n ), redirect_stdout(STDOUT_DISPATCHER), redirect_stderr(STDERR_DISPATCHER):\n r = self.f(self.args, sp_stdin, sp_stdout, sp_stderr, spec, spec.stack)\n except SystemExit as e:\n r = e.code if isinstance(e.code, int) else int(bool(e.code))\n except OSError:\n status = still_writable(self.c2pwrite) and still_writable(self.errwrite)\n if status:\n # stdout and stderr are still writable, so error must\n # come from function itself.\n print_exception()\n r = 1\n else:\n # stdout and stderr are no longer writable, so error must\n # come from the fact that the next process in the pipeline\n # has closed the other side of the pipe. The function then\n # attempted to write to this side of the pipe anyway. This\n # is not truly an error and we should exit gracefully.\n r = 0\n except Exception:\n print_exception()\n r = 1\n safe_flush(sp_stdout)\n safe_flush(sp_stderr)\n self.returncode = parse_proxy_return(r, sp_stdout, sp_stderr)\n if not last_in_pipeline and not ON_WINDOWS:\n # mac requires us *not to* close the handles here while\n # windows requires us *to* close the handles here\n return\n # clean up\n # scopz: not sure why this is needed, but stdin cannot go here\n # and stdout & stderr must.\n handles = [self.stdout, self.stderr]\n for handle in handles:\n safe_fdclose(handle, cache=self._closed_handle_cache)\n\n def _wait_and_getattr(self, name):\n \"\"\"make sure the instance has a certain attr, and return it.\"\"\"\n while not hasattr(self, name):\n time.sleep(1e-7)\n return getattr(self, name)\n\n def poll(self):\n \"\"\"Check if the function has completed.\n\n Returns\n -------\n None if the function is still executing, and the returncode otherwise\n \"\"\"\n return self.returncode\n\n def wait(self, timeout=None):\n \"\"\"Waits for the process to finish and returns the return code.\"\"\"\n self.join()\n self._restore_sigint()\n return self.returncode\n\n #\n # SIGINT handler\n #\n\n def _signal_int(self, signum, frame):\n \"\"\"Signal handler for SIGINT - Ctrl+C may have been pressed.\"\"\"\n # Check if we have already been interrupted. This should prevent\n # the possibility of infinite recursion.\n if self._interrupted:\n return\n self._interrupted = True\n # close file handles here to stop an processes piped to us.\n handles = (\n self.p2cread,\n self.p2cwrite,\n self.c2pread,\n self.c2pwrite,\n self.errread,\n self.errwrite,\n )\n for handle in handles:\n safe_fdclose(handle)\n if self.poll() is not None:\n self._restore_sigint(frame=frame)\n if on_main_thread():\n signal.pthread_kill(threading.get_ident(), signal.SIGINT)\n\n def _restore_sigint(self, frame=None):\n old = self.old_int_handler\n if old is not None:\n if on_main_thread():\n signal.signal(signal.SIGINT, old)\n self.old_int_handler = None\n if frame is not None:\n if old is not None and old is not self._signal_int:\n old(signal.SIGINT, frame)\n if self._interrupted:\n self.returncode = 1\n\n # The code below (_get_devnull, _get_handles, and _make_inheritable) comes\n # from subprocess.py in the Python 3.4.2 Standard Library\n def _get_devnull(self):\n if not hasattr(self, \"_devnull\"):\n self._devnull = os.open(os.devnull, os.O_RDWR)\n return self._devnull\n\n if ON_WINDOWS:\n\n def _make_inheritable(self, handle):\n \"\"\"Return a duplicate of handle, which is inheritable\"\"\"\n h = _winapi.DuplicateHandle(\n _winapi.GetCurrentProcess(),\n handle,\n _winapi.GetCurrentProcess(),\n 0,\n 1,\n _winapi.DUPLICATE_SAME_ACCESS,\n )\n return Handle(h)\n\n def _get_handles(self, stdin, stdout, stderr):\n \"\"\"Construct and return tuple with IO objects:\n p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite\n \"\"\"\n if stdin is None and stdout is None and stderr is None:\n return (-1, -1, -1, -1, -1, -1)\n\n p2cread, p2cwrite = -1, -1\n c2pread, c2pwrite = -1, -1\n errread, errwrite = -1, -1\n\n if stdin is None:\n p2cread = _winapi.GetStdHandle(_winapi.STD_INPUT_HANDLE)\n if p2cread is None:\n p2cread, _ = _winapi.CreatePipe(None, 0)\n p2cread = Handle(p2cread)\n _winapi.CloseHandle(_)\n elif stdin == subprocess.PIPE:\n p2cread, p2cwrite = Handle(p2cread), Handle(p2cwrite)\n elif stdin == subprocess.DEVNULL:\n p2cread = msvcrt.get_osfhandle(self._get_devnull())\n elif isinstance(stdin, int):\n p2cread = msvcrt.get_osfhandle(stdin)\n else:\n # Assuming file-like object\n p2cread = msvcrt.get_osfhandle(stdin.fileno())\n p2cread = self._make_inheritable(p2cread)\n\n if stdout is None:\n c2pwrite = _winapi.GetStdHandle(_winapi.STD_OUTPUT_HANDLE)\n if c2pwrite is None:\n _, c2pwrite = _winapi.CreatePipe(None, 0)\n c2pwrite = Handle(c2pwrite)\n _winapi.CloseHandle(_)\n elif stdout == subprocess.PIPE:\n c2pread, c2pwrite = _winapi.CreatePipe(None, 0)\n c2pread, c2pwrite = Handle(c2pread), Handle(c2pwrite)\n elif stdout == subprocess.DEVNULL:\n c2pwrite = msvcrt.get_osfhandle(self._get_devnull())\n elif isinstance(stdout, int):\n c2pwrite = msvcrt.get_osfhandle(stdout)\n else:\n # Assuming file-like object\n c2pwrite = msvcrt.get_osfhandle(stdout.fileno())\n c2pwrite = self._make_inheritable(c2pwrite)\n\n if stderr is None:\n errwrite = _winapi.GetStdHandle(_winapi.STD_ERROR_HANDLE)\n if errwrite is None:\n _, errwrite = _winapi.CreatePipe(None, 0)\n errwrite = Handle(errwrite)\n _winapi.CloseHandle(_)\n elif stderr == subprocess.PIPE:\n errread, errwrite = _winapi.CreatePipe(None, 0)\n errread, errwrite = Handle(errread), Handle(errwrite)\n elif stderr == subprocess.STDOUT:\n errwrite = c2pwrite\n elif stderr == subprocess.DEVNULL:\n errwrite = msvcrt.get_osfhandle(self._get_devnull())\n elif isinstance(stderr, int):\n errwrite = msvcrt.get_osfhandle(stderr)\n else:\n # Assuming file-like object\n errwrite = msvcrt.get_osfhandle(stderr.fileno())\n errwrite = self._make_inheritable(errwrite)\n\n return (p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite)\n\n else:\n # POSIX versions\n def _get_handles(self, stdin, stdout, stderr):\n \"\"\"Construct and return tuple with IO objects:\n p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite\n \"\"\"\n p2cread, p2cwrite = -1, -1\n c2pread, c2pwrite = -1, -1\n errread, errwrite = -1, -1\n\n if stdin is None:\n pass\n elif stdin == subprocess.PIPE:\n p2cread, p2cwrite = os.pipe()\n elif stdin == subprocess.DEVNULL:\n p2cread = self._get_devnull()\n elif isinstance(stdin, int):\n p2cread = stdin\n else:\n # Assuming file-like object\n p2cread = stdin.fileno()\n\n if stdout is None:\n pass\n elif stdout == subprocess.PIPE:\n c2pread, c2pwrite = os.pipe()\n elif stdout == subprocess.DEVNULL:\n c2pwrite = self._get_devnull()\n elif isinstance(stdout, int):\n c2pwrite = stdout\n else:\n # Assuming file-like object\n c2pwrite = stdout.fileno()\n\n if stderr is None:\n pass\n elif stderr == subprocess.PIPE:\n errread, errwrite = os.pipe()\n elif stderr == subprocess.STDOUT:\n errwrite = c2pwrite\n elif stderr == subprocess.DEVNULL:\n errwrite = self._get_devnull()\n elif isinstance(stderr, int):\n errwrite = stderr\n else:\n # Assuming file-like object\n errwrite = stderr.fileno()\n\n return (p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite)\n\n\n#\n# Foreground Thread Process Proxies\n#\n\n\nclass ProcProxy(object):\n \"\"\"This is process proxy class that runs its alias functions on the\n same thread that it was called from, which is typically the main thread.\n This prevents the process from running on a background thread, but enables\n debugger and profiler tools (functions) be run on the same thread that they\n are attempting to debug.\n \"\"\"\n\n def __init__(\n self,\n f,\n args,\n stdin=None,\n stdout=None,\n stderr=None,\n universal_newlines=False,\n close_fds=False,\n env=None,\n ):\n self.orig_f = f\n self.f = partial_proxy(f)\n self.args = args\n self.pid = os.getpid()\n self.returncode = None\n self.stdin = stdin\n self.stdout = stdout\n self.stderr = stderr\n self.universal_newlines = universal_newlines\n self.close_fds = close_fds\n self.env = env\n\n def poll(self):\n \"\"\"Check if the function has completed via the returncode or None.\n \"\"\"\n return self.returncode\n\n def wait(self, timeout=None):\n \"\"\"Runs the function and returns the result. Timeout argument only\n present for API compatibility.\n \"\"\"\n if self.f is None:\n return 0\n env = builtins.__xonsh__.env\n enc = env.get(\"XONSH_ENCODING\")\n err = env.get(\"XONSH_ENCODING_ERRORS\")\n spec = self._wait_and_getattr(\"spec\")\n # set file handles\n if self.stdin is None:\n stdin = None\n else:\n if isinstance(self.stdin, int):\n inbuf = io.open(self.stdin, \"rb\", -1)\n else:\n inbuf = self.stdin\n stdin = io.TextIOWrapper(inbuf, encoding=enc, errors=err)\n stdout = self._pick_buf(self.stdout, sys.stdout, enc, err)\n stderr = self._pick_buf(self.stderr, sys.stderr, enc, err)\n # run the actual function\n try:\n r = self.f(self.args, stdin, stdout, stderr, spec, spec.stack)\n except Exception:\n print_exception()\n r = 1\n self.returncode = parse_proxy_return(r, stdout, stderr)\n safe_flush(stdout)\n safe_flush(stderr)\n return self.returncode\n\n @staticmethod\n def _pick_buf(handle, sysbuf, enc, err):\n if handle is None or handle is sysbuf:\n buf = sysbuf\n elif isinstance(handle, int):\n if handle < 3:\n buf = sysbuf\n else:\n buf = io.TextIOWrapper(\n io.open(handle, \"wb\", -1), encoding=enc, errors=err\n )\n elif hasattr(handle, \"encoding\"):\n # must be a text stream, no need to wrap.\n buf = handle\n else:\n # must be a binary stream, should wrap it.\n buf = io.TextIOWrapper(handle, encoding=enc, errors=err)\n return buf\n\n def _wait_and_getattr(self, name):\n \"\"\"make sure the instance has a certain attr, and return it.\"\"\"\n while not hasattr(self, name):\n time.sleep(1e-7)\n return getattr(self, name)\n\n\n@lazyobject\ndef SIGNAL_MESSAGES():\n sm = {\n signal.SIGABRT: \"Aborted\",\n signal.SIGFPE: \"Floating point exception\",\n signal.SIGILL: \"Illegal instructions\",\n signal.SIGTERM: \"Terminated\",\n signal.SIGSEGV: \"Segmentation fault\",\n }\n if ON_POSIX:\n sm.update(\n {signal.SIGQUIT: \"Quit\", signal.SIGHUP: \"Hangup\", signal.SIGKILL: \"Killed\"}\n )\n return sm\n\n\ndef safe_readlines(handle, hint=-1):\n \"\"\"Attempts to read lines without throwing an error.\"\"\"\n try:\n lines = handle.readlines(hint)\n except OSError:\n lines = []\n return lines\n\n\ndef safe_readable(handle):\n \"\"\"Attempts to find if the handle is readable without throwing an error.\"\"\"\n try:\n status = handle.readable()\n except (OSError, ValueError):\n status = False\n return status\n\n\ndef update_fg_process_group(pipeline_group, background):\n if background:\n return False\n if not ON_POSIX:\n return False\n env = builtins.__xonsh__.env\n if not env.get(\"XONSH_INTERACTIVE\"):\n return False\n return give_terminal_to(pipeline_group)\n\n\nclass CommandPipeline:\n \"\"\"Represents a subprocess-mode command pipeline.\"\"\"\n\n attrnames = (\n \"stdin\",\n \"stdout\",\n \"stderr\",\n \"pid\",\n \"returncode\",\n \"args\",\n \"alias\",\n \"stdin_redirect\",\n \"stdout_redirect\",\n \"stderr_redirect\",\n \"timestamps\",\n \"executed_cmd\",\n \"input\",\n \"output\",\n \"errors\",\n )\n\n nonblocking = (io.BytesIO, NonBlockingFDReader, ConsoleParallelReader)\n\n def __init__(self, specs):\n \"\"\"\n Parameters\n ----------\n specs : list of SubprocSpec\n Process specifications\n\n Attributes\n ----------\n spec : SubprocSpec\n The last specification in specs\n proc : Popen-like\n The process in procs\n ended : bool\n Boolean for if the command has stopped executing.\n input : str\n A string of the standard input.\n output : str\n A string of the standard output.\n errors : str\n A string of the standard error.\n lines : list of str\n The output lines\n starttime : floats or None\n Pipeline start timestamp.\n \"\"\"\n self.starttime = None\n self.ended = False\n self.procs = []\n self.specs = specs\n self.spec = specs[-1]\n self.captured = specs[-1].captured\n self.input = self._output = self.errors = self.endtime = None\n self._closed_handle_cache = {}\n self.lines = []\n self._stderr_prefix = self._stderr_postfix = None\n self.term_pgid = None\n\n background = self.spec.background\n pipeline_group = None\n for spec in specs:\n if self.starttime is None:\n self.starttime = time.time()\n try:\n proc = spec.run(pipeline_group=pipeline_group)\n except Exception:\n print_exception()\n self._return_terminal()\n self.proc = None\n return\n if (\n proc.pid\n and pipeline_group is None\n and not spec.is_proxy\n and self.captured != \"object\"\n ):\n pipeline_group = proc.pid\n if update_fg_process_group(pipeline_group, background):\n self.term_pgid = pipeline_group\n self.procs.append(proc)\n self.proc = self.procs[-1]\n\n def __repr__(self):\n s = self.__class__.__name__ + \"(\"\n s += \", \".join(a + \"=\" + str(getattr(self, a)) for a in self.attrnames)\n s += \")\"\n return s\n\n def __bool__(self):\n return self.returncode == 0\n\n def __len__(self):\n return len(self.procs)\n\n def __iter__(self):\n \"\"\"Iterates through stdout and returns the lines, converting to\n strings and universal newlines if needed.\n \"\"\"\n if self.ended:\n yield from iter(self.lines)\n else:\n yield from self.tee_stdout()\n\n def iterraw(self):\n \"\"\"Iterates through the last stdout, and returns the lines\n exactly as found.\n \"\"\"\n # get appropriate handles\n spec = self.spec\n proc = self.proc\n if proc is None:\n return\n timeout = builtins.__xonsh__.env.get(\"XONSH_PROC_FREQUENCY\")\n # get the correct stdout\n stdout = proc.stdout\n if (\n stdout is None or spec.stdout is None or not safe_readable(stdout)\n ) and spec.captured_stdout is not None:\n stdout = spec.captured_stdout\n if hasattr(stdout, \"buffer\"):\n stdout = stdout.buffer\n if stdout is not None and not isinstance(stdout, self.nonblocking):\n stdout = NonBlockingFDReader(stdout.fileno(), timeout=timeout)\n if (\n not stdout\n or self.captured == \"stdout\"\n or not safe_readable(stdout)\n or not spec.threadable\n ):\n # we get here if the process is not threadable or the\n # class is the real Popen\n PrevProcCloser(pipeline=self)\n task = wait_for_active_job()\n if task is None or task[\"status\"] != \"stopped\":\n proc.wait()\n self._endtime()\n if self.captured == \"object\":\n self.end(tee_output=False)\n elif self.captured == \"hiddenobject\" and stdout:\n b = stdout.read()\n lines = b.splitlines(keepends=True)\n yield from lines\n self.end(tee_output=False)\n elif self.captured == \"stdout\":\n b = stdout.read()\n s = self._decode_uninew(b, universal_newlines=True)\n self.lines = s.splitlines(keepends=True)\n return\n # get the correct stderr\n stderr = proc.stderr\n if (\n stderr is None or spec.stderr is None or not safe_readable(stderr)\n ) and spec.captured_stderr is not None:\n stderr = spec.captured_stderr\n if hasattr(stderr, \"buffer\"):\n stderr = stderr.buffer\n if stderr is not None and not isinstance(stderr, self.nonblocking):\n stderr = NonBlockingFDReader(stderr.fileno(), timeout=timeout)\n # read from process while it is running\n check_prev_done = len(self.procs) == 1\n prev_end_time = None\n i = j = cnt = 1\n while proc.poll() is None:\n if getattr(proc, \"suspended\", False):\n return\n elif getattr(proc, \"in_alt_mode\", False):\n time.sleep(0.1) # probably not leaving any time soon\n continue\n elif not check_prev_done:\n # In the case of pipelines with more than one command\n # we should give the commands a little time\n # to start up fully. This is particularly true for\n # GNU Parallel, which has a long startup time.\n pass\n elif self._prev_procs_done():\n self._close_prev_procs()\n proc.prevs_are_closed = True\n break\n stdout_lines = safe_readlines(stdout, 1024)\n i = len(stdout_lines)\n if i != 0:\n yield from stdout_lines\n stderr_lines = safe_readlines(stderr, 1024)\n j = len(stderr_lines)\n if j != 0:\n self.stream_stderr(stderr_lines)\n if not check_prev_done:\n # if we are piping...\n if stdout_lines or stderr_lines:\n # see if we have some output.\n check_prev_done = True\n elif prev_end_time is None:\n # or see if we already know that the next-to-last\n # proc in the pipeline has ended.\n if self._prev_procs_done():\n # if it has, record the time\n prev_end_time = time.time()\n elif time.time() - prev_end_time >= 0.1:\n # if we still don't have any output, even though the\n # next-to-last proc has finished, wait a bit to make\n # sure we have fully started up, etc.\n check_prev_done = True\n # this is for CPU usage\n if i + j == 0:\n cnt = min(cnt + 1, 1000)\n else:\n cnt = 1\n time.sleep(timeout * cnt)\n # read from process now that it is over\n yield from safe_readlines(stdout)\n self.stream_stderr(safe_readlines(stderr))\n proc.wait()\n self._endtime()\n yield from safe_readlines(stdout)\n self.stream_stderr(safe_readlines(stderr))\n if self.captured == \"object\":\n self.end(tee_output=False)\n\n def itercheck(self):\n \"\"\"Iterates through the command lines and throws an error if the\n returncode is non-zero.\n \"\"\"\n yield from self\n if self.returncode:\n # I included self, as providing access to stderr and other details\n # useful when instance isn't assigned to a variable in the shell.\n raise XonshCalledProcessError(\n self.returncode, self.executed_cmd, self.stdout, self.stderr, self\n )\n\n def tee_stdout(self):\n \"\"\"Writes the process stdout to the output variable, line-by-line, and\n yields each line. This may optionally accept lines (in bytes) to iterate\n over, in which case it does not call iterraw().\n \"\"\"\n env = builtins.__xonsh__.env\n enc = env.get(\"XONSH_ENCODING\")\n err = env.get(\"XONSH_ENCODING_ERRORS\")\n lines = self.lines\n stream = self.captured not in STDOUT_CAPTURE_KINDS\n if stream and not self.spec.stdout:\n stream = False\n stdout_has_buffer = hasattr(sys.stdout, \"buffer\")\n nl = b\"\\n\"\n cr = b\"\\r\"\n crnl = b\"\\r\\n\"\n for line in self.iterraw():\n # write to stdout line ASAP, if needed\n if stream:\n if stdout_has_buffer:\n sys.stdout.buffer.write(line)\n else:\n sys.stdout.write(line.decode(encoding=enc, errors=err))\n sys.stdout.flush()\n # do some munging of the line before we return it\n if line.endswith(crnl):\n line = line[:-2] + nl\n elif line.endswith(cr):\n line = line[:-1] + nl\n line = RE_HIDE_ESCAPE.sub(b\"\", line)\n line = line.decode(encoding=enc, errors=err)\n # tee it up!\n lines.append(line)\n yield line\n\n def stream_stderr(self, lines):\n \"\"\"Streams lines to sys.stderr and the errors attribute.\"\"\"\n if not lines:\n return\n env = builtins.__xonsh__.env\n enc = env.get(\"XONSH_ENCODING\")\n err = env.get(\"XONSH_ENCODING_ERRORS\")\n b = b\"\".join(lines)\n if self.stderr_prefix:\n b = self.stderr_prefix + b\n if self.stderr_postfix:\n b += self.stderr_postfix\n stderr_has_buffer = hasattr(sys.stderr, \"buffer\")\n # write bytes to std stream\n if stderr_has_buffer:\n sys.stderr.buffer.write(b)\n else:\n sys.stderr.write(b.decode(encoding=enc, errors=err))\n sys.stderr.flush()\n # do some munging of the line before we save it to the attr\n b = b.replace(b\"\\r\\n\", b\"\\n\").replace(b\"\\r\", b\"\\n\")\n b = RE_HIDE_ESCAPE.sub(b\"\", b)\n env = builtins.__xonsh__.env\n s = b.decode(\n encoding=env.get(\"XONSH_ENCODING\"), errors=env.get(\"XONSH_ENCODING_ERRORS\")\n )\n # set the errors\n if self.errors is None:\n self.errors = s\n else:\n self.errors += s\n\n def _decode_uninew(self, b, universal_newlines=None):\n \"\"\"Decode bytes into a str and apply universal newlines as needed.\"\"\"\n if not b:\n return \"\"\n if isinstance(b, bytes):\n env = builtins.__xonsh__.env\n s = b.decode(\n encoding=env.get(\"XONSH_ENCODING\"),\n errors=env.get(\"XONSH_ENCODING_ERRORS\"),\n )\n else:\n s = b\n if universal_newlines or self.spec.universal_newlines:\n s = s.replace(\"\\r\\n\", \"\\n\").replace(\"\\r\", \"\\n\")\n return s\n\n #\n # Ending methods\n #\n\n def end(self, tee_output=True):\n \"\"\"\n End the pipeline, return the controlling terminal if needed.\n\n Main things done in self._end().\n \"\"\"\n if self.ended:\n return\n self._end(tee_output=tee_output)\n self._return_terminal()\n\n def _end(self, tee_output):\n \"\"\"Waits for the command to complete and then runs any closing and\n cleanup procedures that need to be run.\n \"\"\"\n if tee_output:\n for _ in self.tee_stdout():\n pass\n self._endtime()\n # since we are driven by getting output, input may not be available\n # until the command has completed.\n self._set_input()\n self._close_prev_procs()\n self._close_proc()\n self._check_signal()\n self._apply_to_history()\n self.ended = True\n self._raise_subproc_error()\n\n def _return_terminal(self):\n if ON_WINDOWS or not ON_POSIX:\n return\n pgid = os.getpgid(0)\n if self.term_pgid is None or pgid == self.term_pgid:\n return\n if give_terminal_to(pgid): # if gave term succeed\n self.term_pgid = pgid\n if builtins.__xonsh__.shell is not None:\n # restoring sanity could probably be called whenever we return\n # control to the shell. But it only seems to matter after a\n # ^Z event. This *has* to be called after we give the terminal\n # back to the shell.\n builtins.__xonsh__.shell.shell.restore_tty_sanity()\n\n def resume(self, job, tee_output=True):\n self.ended = False\n if give_terminal_to(job[\"pgrp\"]):\n self.term_pgid = job[\"pgrp\"]\n _continue(job)\n self.end(tee_output=tee_output)\n\n def _endtime(self):\n \"\"\"Sets the closing timestamp if it hasn't been already.\"\"\"\n if self.endtime is None:\n self.endtime = time.time()\n\n def _safe_close(self, handle):\n safe_fdclose(handle, cache=self._closed_handle_cache)\n\n def _prev_procs_done(self):\n \"\"\"Boolean for if all previous processes have completed. If there\n is only a single process in the pipeline, this returns False.\n \"\"\"\n any_running = False\n for s, p in zip(self.specs[:-1], self.procs[:-1]):\n if p.poll() is None:\n any_running = True\n continue\n self._safe_close(s.stdin)\n self._safe_close(s.stdout)\n self._safe_close(s.stderr)\n if p is None:\n continue\n self._safe_close(p.stdin)\n self._safe_close(p.stdout)\n self._safe_close(p.stderr)\n return False if any_running else (len(self) > 1)\n\n def _close_prev_procs(self):\n \"\"\"Closes all but the last proc's stdout.\"\"\"\n for s, p in zip(self.specs[:-1], self.procs[:-1]):\n self._safe_close(s.stdin)\n self._safe_close(s.stdout)\n self._safe_close(s.stderr)\n if p is None:\n continue\n self._safe_close(p.stdin)\n self._safe_close(p.stdout)\n self._safe_close(p.stderr)\n\n def _close_proc(self):\n \"\"\"Closes last proc's stdout.\"\"\"\n s = self.spec\n p = self.proc\n self._safe_close(s.stdin)\n self._safe_close(s.stdout)\n self._safe_close(s.stderr)\n self._safe_close(s.captured_stdout)\n self._safe_close(s.captured_stderr)\n if p is None:\n return\n self._safe_close(p.stdin)\n self._safe_close(p.stdout)\n self._safe_close(p.stderr)\n\n def _set_input(self):\n \"\"\"Sets the input variable.\"\"\"\n if self.proc is None:\n return\n stdin = self.proc.stdin\n if (\n stdin is None\n or isinstance(stdin, int)\n or stdin.closed\n or not stdin.seekable()\n or not safe_readable(stdin)\n ):\n input = b\"\"\n else:\n stdin.seek(0)\n input = stdin.read()\n self.input = self._decode_uninew(input)\n\n def _check_signal(self):\n \"\"\"Checks if a signal was received and issues a message.\"\"\"\n proc_signal = getattr(self.proc, \"signal\", None)\n if proc_signal is None:\n return\n sig, core = proc_signal\n sig_str = SIGNAL_MESSAGES.get(sig)\n if sig_str:\n if core:\n sig_str += \" (core dumped)\"\n print(sig_str, file=sys.stderr)\n if self.errors is not None:\n self.errors += sig_str + \"\\n\"\n\n def _apply_to_history(self):\n \"\"\"Applies the results to the current history object.\"\"\"\n hist = builtins.__xonsh__.history\n if hist is not None:\n hist.last_cmd_rtn = 1 if self.proc is None else self.proc.returncode\n\n def _raise_subproc_error(self):\n \"\"\"Raises a subprocess error, if we are supposed to.\"\"\"\n spec = self.spec\n rtn = self.returncode\n if (\n rtn is not None\n and rtn > 0\n and builtins.__xonsh__.env.get(\"RAISE_SUBPROC_ERROR\")\n ):\n try:\n raise subprocess.CalledProcessError(rtn, spec.args, output=self.output)\n finally:\n # this is need to get a working terminal in interactive mode\n self._return_terminal()\n\n #\n # Properties\n #\n\n @property\n def stdin(self):\n \"\"\"Process stdin.\"\"\"\n return self.proc.stdin\n\n @property\n def stdout(self):\n \"\"\"Process stdout.\"\"\"\n return self.proc.stdout\n\n @property\n def stderr(self):\n \"\"\"Process stderr.\"\"\"\n return self.proc.stderr\n\n @property\n def inp(self):\n \"\"\"Creates normalized input string from args.\"\"\"\n return \" \".join(self.args)\n\n @property\n def output(self):\n \"\"\"Non-blocking, lazy access to output\"\"\"\n if self.ended:\n if self._output is None:\n self._output = \"\".join(self.lines)\n return self._output\n else:\n return \"\".join(self.lines)\n\n @property\n def out(self):\n \"\"\"Output value as a str.\"\"\"\n self.end()\n return self.output\n\n @property\n def err(self):\n \"\"\"Error messages as a string.\"\"\"\n self.end()\n return self.errors\n\n @property\n def pid(self):\n \"\"\"Process identifier.\"\"\"\n return self.proc.pid\n\n @property\n def returncode(self):\n \"\"\"Process return code, waits until command is completed.\"\"\"\n self.end()\n if self.proc is None:\n return 1\n return self.proc.returncode\n\n rtn = returncode\n\n @property\n def args(self):\n \"\"\"Arguments to the process.\"\"\"\n return self.spec.args\n\n @property\n def rtn(self):\n \"\"\"Alias to return code.\"\"\"\n return self.returncode\n\n @property\n def alias(self):\n \"\"\"Alias the process used.\"\"\"\n return self.spec.alias\n\n @property\n def stdin_redirect(self):\n \"\"\"Redirection used for stdin.\"\"\"\n stdin = self.spec.stdin\n name = getattr(stdin, \"name\", \"\")\n mode = getattr(stdin, \"mode\", \"r\")\n return [name, mode]\n\n @property\n def stdout_redirect(self):\n \"\"\"Redirection used for stdout.\"\"\"\n stdout = self.spec.stdout\n name = getattr(stdout, \"name\", \"\")\n mode = getattr(stdout, \"mode\", \"a\")\n return [name, mode]\n\n @property\n def stderr_redirect(self):\n \"\"\"Redirection used for stderr.\"\"\"\n stderr = self.spec.stderr\n name = getattr(stderr, \"name\", \"\")\n mode = getattr(stderr, \"mode\", \"r\")\n return [name, mode]\n\n @property\n def timestamps(self):\n \"\"\"The start and end time stamps.\"\"\"\n return [self.starttime, self.endtime]\n\n @property\n def executed_cmd(self):\n \"\"\"The resolve and executed command.\"\"\"\n return self.spec.cmd\n\n @property\n def stderr_prefix(self):\n \"\"\"Prefix to print in front of stderr, as bytes.\"\"\"\n p = self._stderr_prefix\n if p is None:\n env = builtins.__xonsh__.env\n t = env.get(\"XONSH_STDERR_PREFIX\")\n s = format_std_prepost(t, env=env)\n p = s.encode(\n encoding=env.get(\"XONSH_ENCODING\"),\n errors=env.get(\"XONSH_ENCODING_ERRORS\"),\n )\n self._stderr_prefix = p\n return p\n\n @property\n def stderr_postfix(self):\n \"\"\"Postfix to print after stderr, as bytes.\"\"\"\n p = self._stderr_postfix\n if p is None:\n env = builtins.__xonsh__.env\n t = env.get(\"XONSH_STDERR_POSTFIX\")\n s = format_std_prepost(t, env=env)\n p = s.encode(\n encoding=env.get(\"XONSH_ENCODING\"),\n errors=env.get(\"XONSH_ENCODING_ERRORS\"),\n )\n self._stderr_postfix = p\n return p\n\n\nclass HiddenCommandPipeline(CommandPipeline):\n def __repr__(self):\n return \"\"\n\n\ndef pause_call_resume(p, f, *args, **kwargs):\n \"\"\"For a process p, this will call a function f with the remaining args and\n and kwargs. If the process cannot accept signals, the function will be called.\n\n Parameters\n ----------\n p : Popen object or similar\n f : callable\n args : remaining arguments\n kwargs : keyword arguments\n \"\"\"\n can_send_signal = (\n hasattr(p, \"send_signal\") and ON_POSIX and not ON_MSYS and not ON_CYGWIN\n )\n if can_send_signal:\n try:\n p.send_signal(signal.SIGSTOP)\n except PermissionError:\n pass\n try:\n f(*args, **kwargs)\n except Exception:\n pass\n if can_send_signal:\n p.send_signal(signal.SIGCONT)\n\n\nclass PrevProcCloser(threading.Thread):\n \"\"\"Previous process closer thread for pipelines whose last command\n is itself unthreadable. This makes sure that the pipeline is\n driven forward and does not deadlock.\n \"\"\"\n\n def __init__(self, pipeline):\n \"\"\"\n Parameters\n ----------\n pipeline : CommandPipeline\n The pipeline whose prev procs we should close.\n \"\"\"\n self.pipeline = pipeline\n super().__init__()\n self.daemon = True\n self.start()\n\n def run(self):\n \"\"\"Runs the closing algorithm.\"\"\"\n pipeline = self.pipeline\n check_prev_done = len(pipeline.procs) == 1\n if check_prev_done:\n return\n proc = pipeline.proc\n prev_end_time = None\n timeout = builtins.__xonsh__.env.get(\"XONSH_PROC_FREQUENCY\")\n sleeptime = min(timeout * 1000, 0.1)\n while proc.poll() is None:\n if not check_prev_done:\n # In the case of pipelines with more than one command\n # we should give the commands a little time\n # to start up fully. This is particularly true for\n # GNU Parallel, which has a long startup time.\n pass\n elif pipeline._prev_procs_done():\n pipeline._close_prev_procs()\n proc.prevs_are_closed = True\n break\n if not check_prev_done:\n # if we are piping...\n if prev_end_time is None:\n # or see if we already know that the next-to-last\n # proc in the pipeline has ended.\n if pipeline._prev_procs_done():\n # if it has, record the time\n prev_end_time = time.time()\n elif time.time() - prev_end_time >= 0.1:\n # if we still don't have any output, even though the\n # next-to-last proc has finished, wait a bit to make\n # sure we have fully started up, etc.\n check_prev_done = True\n # this is for CPU usage\n time.sleep(sleeptime)\n" }, { "path": "xonsh/prompt/__init__.py", "content": "# amalgamate exclude\nimport os as _os\n\nif _os.getenv(\"XONSH_DEBUG\", \"\"):\n pass\nelse:\n import sys as _sys\n\n try:\n from xonsh.prompt import __amalgam__\n\n cwd = __amalgam__\n _sys.modules[\"xonsh.prompt.cwd\"] = __amalgam__\n env = __amalgam__\n _sys.modules[\"xonsh.prompt.env\"] = __amalgam__\n gitstatus = __amalgam__\n _sys.modules[\"xonsh.prompt.gitstatus\"] = __amalgam__\n job = __amalgam__\n _sys.modules[\"xonsh.prompt.job\"] = __amalgam__\n vc = __amalgam__\n _sys.modules[\"xonsh.prompt.vc\"] = __amalgam__\n base = __amalgam__\n _sys.modules[\"xonsh.prompt.base\"] = __amalgam__\n del __amalgam__\n except ImportError:\n pass\n del _sys\ndel _os\n# amalgamate end\n" }, { "path": "xonsh/prompt/base.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"Base prompt, provides PROMPT_FIELDS and prompt related functions\"\"\"\n\nimport builtins\nimport itertools\nimport os\nimport re\nimport socket\nimport sys\n\nimport xonsh.lazyasd as xl\nimport xonsh.tools as xt\nimport xonsh.platform as xp\n\nfrom xonsh.prompt.cwd import (\n _collapsed_pwd,\n _replace_home_cwd,\n _dynamically_collapsed_pwd,\n)\nfrom xonsh.prompt.job import _current_job\nfrom xonsh.prompt.env import env_name, vte_new_tab_cwd\nfrom xonsh.prompt.vc import current_branch, branch_color, branch_bg_color\nfrom xonsh.prompt.gitstatus import gitstatus_prompt\n\n\n@xt.lazyobject\ndef DEFAULT_PROMPT():\n return default_prompt()\n\n\nclass PromptFormatter:\n \"\"\"Class that holds all the related prompt formatting methods,\n uses the ``PROMPT_FIELDS`` envvar (no color formatting).\n \"\"\"\n\n def __init__(self):\n self.cache = {}\n\n def __call__(self, template=DEFAULT_PROMPT, fields=None):\n \"\"\"Formats a xonsh prompt template string.\"\"\"\n if fields is None:\n self.fields = builtins.__xonsh__.env.get(\"PROMPT_FIELDS\", PROMPT_FIELDS)\n else:\n self.fields = fields\n try:\n prompt = self._format_prompt(template=template)\n except Exception:\n return _failover_template_format(template)\n # keep cache only during building prompt\n self.cache.clear()\n return prompt\n\n def _format_prompt(self, template=DEFAULT_PROMPT):\n template = template() if callable(template) else template\n toks = []\n for literal, field, spec, conv in xt.FORMATTER.parse(template):\n toks.append(literal)\n entry = self._format_field(field, spec, conv)\n if entry is not None:\n toks.append(entry)\n return \"\".join(toks)\n\n def _format_field(self, field, spec, conv):\n if field is None:\n return\n elif field.startswith(\"$\"):\n val = builtins.__xonsh__.env[field[1:]]\n return _format_value(val, spec, conv)\n elif field in self.fields:\n val = self._get_field_value(field)\n return _format_value(val, spec, conv)\n else:\n # color or unknown field, return as is\n return \"{\" + field + \"}\"\n\n def _get_field_value(self, field):\n field_value = self.fields[field]\n if field_value in self.cache:\n return self.cache[field_value]\n try:\n value = field_value() if callable(field_value) else field_value\n self.cache[field_value] = value\n except Exception:\n print(\"prompt: error: on field {!r}\" \"\".format(field), file=sys.stderr)\n xt.print_exception()\n value = \"(ERROR:{})\".format(field)\n return value\n\n\n@xl.lazyobject\ndef PROMPT_FIELDS():\n return dict(\n user=xp.os_environ.get(\"USERNAME\" if xp.ON_WINDOWS else \"USER\", \"\"),\n prompt_end=\"#\" if xt.is_superuser() else \"$\",\n hostname=socket.gethostname().split(\".\", 1)[0],\n cwd=_dynamically_collapsed_pwd,\n cwd_dir=lambda: os.path.dirname(_replace_home_cwd()),\n cwd_base=lambda: os.path.basename(_replace_home_cwd()),\n short_cwd=_collapsed_pwd,\n curr_branch=current_branch,\n branch_color=branch_color,\n branch_bg_color=branch_bg_color,\n current_job=_current_job,\n env_name=env_name,\n env_prefix=\"(\",\n env_postfix=\") \",\n vte_new_tab_cwd=vte_new_tab_cwd,\n gitstatus=gitstatus_prompt,\n )\n\n\ndef default_prompt():\n return ('{RED}{user} '\n '{BOLD_WHITE}at '\n '{YELLOW}{hostname} '\n '{BOLD_WHITE}in '\n '{GREEN}{cwd} '\n '{BOLD_WHITE}on '\n '{branch_color}{curr_branch} '\n '{BOLD_WHITE}\\n'\n '${NO_COLOR} ')\n\n\ndef _failover_template_format(template):\n if callable(template):\n try:\n # Exceptions raises from function of producing $PROMPT\n # in user's xonshrc should not crash xonsh\n return template()\n except Exception:\n xt.print_exception()\n return \"$ \"\n return template\n\n\n@xt.lazyobject\ndef RE_HIDDEN():\n return re.compile(\"\\001.*?\\002\")\n\n\ndef multiline_prompt(curr=\"\"):\n \"\"\"Returns the filler text for the prompt in multiline scenarios.\"\"\"\n line = curr.rsplit(\"\\n\", 1)[1] if \"\\n\" in curr else curr\n line = RE_HIDDEN.sub(\"\", line) # gets rid of colors\n # most prompts end in whitespace, head is the part before that.\n head = line.rstrip()\n headlen = len(head)\n # tail is the trailing whitespace\n tail = line if headlen == 0 else line.rsplit(head[-1], 1)[1]\n # now to construct the actual string\n dots = builtins.__xonsh__.env.get(\"MULTILINE_PROMPT\")\n dots = dots() if callable(dots) else dots\n if dots is None or len(dots) == 0:\n return \"\"\n tokstr = xt.format_color(dots, hide=True)\n baselen = 0\n basetoks = []\n for x in tokstr.split(\"\\001\"):\n pre, sep, post = x.partition(\"\\002\")\n if len(sep) == 0:\n basetoks.append((\"\", pre))\n baselen += len(pre)\n else:\n basetoks.append((\"\\001\" + pre + \"\\002\", post))\n baselen += len(post)\n if baselen == 0:\n return xt.format_color(\"{NO_COLOR}\" + tail, hide=True)\n toks = basetoks * (headlen // baselen)\n n = headlen % baselen\n count = 0\n for tok in basetoks:\n slen = len(tok[1])\n newcount = slen + count\n if slen == 0:\n continue\n elif newcount <= n:\n toks.append(tok)\n else:\n toks.append((tok[0], tok[1][: n - count]))\n count = newcount\n if n <= count:\n break\n toks.append((xt.format_color(\"{NO_COLOR}\", hide=True), tail))\n rtn = \"\".join(itertools.chain.from_iterable(toks))\n return rtn\n\n\ndef is_template_string(template, PROMPT_FIELDS=None):\n \"\"\"Returns whether or not the string is a valid template.\"\"\"\n template = template() if callable(template) else template\n try:\n included_names = set(i[1] for i in xt.FORMATTER.parse(template))\n except ValueError:\n return False\n included_names.discard(None)\n if PROMPT_FIELDS is None:\n fmtter = builtins.__xonsh__.env.get(\"PROMPT_FIELDS\", PROMPT_FIELDS)\n else:\n fmtter = PROMPT_FIELDS\n known_names = set(fmtter.keys())\n return included_names <= known_names\n\n\ndef _format_value(val, spec, conv):\n \"\"\"Formats a value from a template string {val!conv:spec}. The spec is\n applied as a format string itself, but if the value is None, the result\n will be empty. The purpose of this is to allow optional parts in a\n prompt string. For example, if the prompt contains '{current_job:{} | }',\n and 'current_job' returns 'sleep', the result is 'sleep | ', and if\n 'current_job' returns None, the result is ''.\n \"\"\"\n if val is None:\n return \"\"\n val = xt.FORMATTER.convert_field(val, conv)\n if spec:\n val = xt.FORMATTER.format(spec, val)\n if not isinstance(val, str):\n val = str(val)\n return val\n" }, { "path": "xonsh/prompt/cwd.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"CWD related prompt formatter\"\"\"\n\nimport os\nimport shutil\nimport builtins\n\nimport xonsh.tools as xt\nimport xonsh.platform as xp\n\n\ndef _replace_home(x):\n if xp.ON_WINDOWS:\n home = (\n builtins.__xonsh__.env[\"HOMEDRIVE\"] + builtins.__xonsh__.env[\"HOMEPATH\"][0]\n )\n if x.startswith(home):\n x = x.replace(home, \"~\", 1)\n\n if builtins.__xonsh__.env.get(\"FORCE_POSIX_PATHS\"):\n x = x.replace(os.sep, os.altsep)\n\n return x\n else:\n home = builtins.__xonsh__.env[\"HOME\"]\n if x.startswith(home):\n x = x.replace(home, \"~\", 1)\n return x\n\n\ndef _replace_home_cwd():\n return _replace_home(builtins.__xonsh__.env[\"PWD\"])\n\n\ndef _collapsed_pwd():\n sep = xt.get_sep()\n pwd = _replace_home_cwd().split(sep)\n l = len(pwd)\n leader = sep if l > 0 and len(pwd[0]) == 0 else \"\"\n base = [\n i[0] if ix != l - 1 and i[0] != \".\" else i[0:2] if ix != l - 1 else i\n for ix, i in enumerate(pwd)\n if len(i) > 0\n ]\n return leader + sep.join(base)\n\n\ndef _dynamically_collapsed_pwd():\n \"\"\"Return the compact current working directory. It respects the\n environment variable DYNAMIC_CWD_WIDTH.\n \"\"\"\n original_path = _replace_home_cwd()\n target_width, units = builtins.__xonsh__.env[\"DYNAMIC_CWD_WIDTH\"]\n elision_char = builtins.__xonsh__.env[\"DYNAMIC_CWD_ELISION_CHAR\"]\n if target_width == float(\"inf\"):\n return original_path\n if units == \"%\":\n cols, _ = shutil.get_terminal_size()\n target_width = (cols * target_width) // 100\n sep = xt.get_sep()\n pwd = original_path.split(sep)\n last = pwd.pop()\n remaining_space = target_width - len(last)\n # Reserve space for separators\n remaining_space_for_text = remaining_space - len(pwd)\n parts = []\n for i in range(len(pwd)):\n part = pwd[i]\n part_len = int(\n min(len(part), max(1, remaining_space_for_text // (len(pwd) - i)))\n )\n remaining_space_for_text -= part_len\n if len(part) > part_len:\n reduced_part = part[0 : part_len - len(elision_char)] + elision_char\n parts.append(reduced_part)\n else:\n parts.append(part)\n parts.append(last)\n full = sep.join(parts)\n truncature_char = elision_char if elision_char else \"...\"\n # If even if displaying one letter per dir we are too long\n if len(full) > target_width:\n # We truncate the left most part\n full = truncature_char + full[int(-target_width) + len(truncature_char) :]\n # if there is not even a single separator we still\n # want to display at least the beginning of the directory\n if full.find(sep) == -1:\n full = (truncature_char + sep + last)[\n 0 : int(target_width) - len(truncature_char)\n ] + truncature_char\n return full\n" }, { "path": "xonsh/prompt/env.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"Prompt formatter for virtualenv and others\"\"\"\n\nimport os\nimport builtins\n\nimport xonsh.platform as xp\n\n\ndef find_env_name():\n \"\"\"Finds the current environment name from $VIRTUAL_ENV or\n $CONDA_DEFAULT_ENV if that is set.\n \"\"\"\n env_path = builtins.__xonsh__.env.get(\"VIRTUAL_ENV\", \"\")\n if len(env_path) == 0 and xp.ON_ANACONDA:\n env_path = builtins.__xonsh__.env.get(\"CONDA_DEFAULT_ENV\", \"\")\n env_name = os.path.basename(env_path)\n return env_name\n\n\ndef env_name():\n \"\"\"Returns the current env_name if it non-empty, surrounded by the\n ``{env_prefix}`` and ``{env_postfix}`` fields.\n \"\"\"\n env_name = find_env_name()\n if (\n builtins.__xonsh__.env.get(\"VIRTUAL_ENV_DISABLE_PROMPT\")\n or not env_name\n ):\n # env name prompt printing disabled, or no environment; just return\n return\n\n venv_prompt = builtins.__xonsh__.env.get(\"VIRTUAL_ENV_PROMPT\")\n if venv_prompt is not None:\n return venv_prompt\n else:\n pf = builtins.__xonsh__.shell.prompt_formatter\n pre = pf._get_field_value(\"env_prefix\")\n post = pf._get_field_value(\"env_postfix\")\n return pre + env_name + post\n\n\ndef vte_new_tab_cwd():\n \"\"\"This prints an escape sequence that tells VTE terminals the hostname\n and pwd. This should not be needed in most cases, but sometimes is for\n certain Linux terminals that do not read the PWD from the environment\n on startup. Note that this does not return a string, it simply prints\n and flushes the escape sequence to stdout directly.\n \"\"\"\n env = builtins.__xonsh__.env\n t = \"\\033]7;file://{}{}\\007\"\n s = t.format(env.get(\"HOSTNAME\"), env.get(\"PWD\"))\n print(s, end=\"\", flush=True)\n" }, { "path": "xonsh/prompt/gitstatus.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"Informative git status prompt formatter\"\"\"\n\nimport builtins\nimport collections\nimport os\nimport subprocess\n\nimport xonsh.lazyasd as xl\n\n\nGitStatus = collections.namedtuple(\n \"GitStatus\",\n [\n \"branch\",\n \"num_ahead\",\n \"num_behind\",\n \"untracked\",\n \"changed\",\n \"conflicts\",\n \"staged\",\n \"stashed\",\n \"operations\",\n ],\n)\n\n\ndef _check_output(*args, **kwargs):\n kwargs.update(\n dict(\n env=builtins.__xonsh__.env.detype(),\n stdout=subprocess.PIPE,\n stderr=subprocess.DEVNULL,\n universal_newlines=True,\n )\n )\n timeout = builtins.__xonsh__.env[\"VC_BRANCH_TIMEOUT\"]\n # See https://docs.python.org/3/library/subprocess.html#subprocess.Popen.communicate\n with subprocess.Popen(*args, **kwargs) as proc:\n try:\n out, err = proc.communicate(timeout=timeout)\n if proc.returncode != 0:\n raise subprocess.CalledProcessError(\n proc.returncode, proc.args, output=out, stderr=err\n ) # note err will always be empty as we redirect stderr to DEVNULL abvoe\n return out\n except subprocess.TimeoutExpired:\n # We use `.terminate()` (SIGTERM) instead of `.kill()` (SIGKILL) here\n # because otherwise we guarantee that a `.git/index.lock` file will be\n # left over, and subsequent git operations will fail.\n # We don't want that.\n # As a result, we must rely on git to exit properly on SIGTERM.\n proc.terminate()\n # We wait() to ensure that git has finished before the next\n # `gitstatus` prompt is rendered (otherwise `index.lock` still exists,\n # and it will fail).\n # We don't technically have to call `wait()` here as the\n # `with subprocess.Popen()` context manager above would do that\n # for us, but we do it to be explicit that waiting is being done.\n proc.wait() # we ignore what git says after we sent it SIGTERM\n raise\n\n\n@xl.lazyobject\ndef _DEFS():\n DEFS = {\n \"HASH\": \":\",\n \"BRANCH\": \"{CYAN}\",\n \"OPERATION\": \"{CYAN}\",\n \"STAGED\": \"{RED}●\",\n \"CONFLICTS\": \"{RED}×\",\n \"CHANGED\": \"{BLUE}+\",\n \"UNTRACKED\": \"…\",\n \"STASHED\": \"⚑\",\n \"CLEAN\": \"{BOLD_GREEN}✓\",\n \"AHEAD\": \"↑·\",\n \"BEHIND\": \"↓·\",\n }\n return DEFS\n\n\ndef _get_def(key):\n def_ = builtins.__xonsh__.env.get(\"XONSH_GITSTATUS_\" + key)\n return def_ if def_ is not None else _DEFS[key]\n\n\ndef _get_tag_or_hash():\n tag_or_hash = _check_output([\"git\", \"describe\", \"--always\"]).strip()\n hash_ = _check_output([\"git\", \"rev-parse\", \"--short\", \"HEAD\"]).strip()\n have_tag_name = tag_or_hash != hash_\n return tag_or_hash if have_tag_name else _get_def(\"HASH\") + hash_\n\n\ndef _get_stash(gitdir):\n try:\n with open(os.path.join(gitdir, \"logs/refs/stash\")) as f:\n return sum(1 for _ in f)\n except IOError:\n return 0\n\n\ndef _gitoperation(gitdir):\n files = (\n (\"rebase-merge\", \"REBASE\"),\n (\"rebase-apply\", \"AM/REBASE\"),\n (\"MERGE_HEAD\", \"MERGING\"),\n (\"CHERRY_PICK_HEAD\", \"CHERRY-PICKING\"),\n (\"REVERT_HEAD\", \"REVERTING\"),\n (\"BISECT_LOG\", \"BISECTING\"),\n )\n return [f[1] for f in files if os.path.exists(os.path.join(gitdir, f[0]))]\n\n\ndef gitstatus():\n \"\"\"Return namedtuple with fields:\n branch name, number of ahead commit, number of behind commit,\n untracked number, changed number, conflicts number,\n staged number, stashed number, operation.\"\"\"\n status = _check_output([\"git\", \"status\", \"--porcelain\", \"--branch\"])\n branch = \"\"\n num_ahead, num_behind = 0, 0\n untracked, changed, conflicts, staged = 0, 0, 0, 0\n for line in status.splitlines():\n if line.startswith(\"##\"):\n line = line[2:].strip()\n if \"Initial commit on\" in line:\n branch = line.split()[-1]\n elif \"no branch\" in line:\n branch = _get_tag_or_hash()\n elif \"...\" not in line:\n branch = line\n else:\n branch, rest = line.split(\"...\")\n if \" \" in rest:\n divergence = rest.split(\" \", 1)[-1]\n divergence = divergence.strip(\"[]\")\n for div in divergence.split(\", \"):\n if \"ahead\" in div:\n num_ahead = int(div[len(\"ahead \") :].strip())\n elif \"behind\" in div:\n num_behind = int(div[len(\"behind \") :].strip())\n elif line.startswith(\"??\"):\n untracked += 1\n else:\n if len(line) > 1 and line[1] == \"M\":\n changed += 1\n\n if len(line) > 0 and line[0] == \"U\":\n conflicts += 1\n elif len(line) > 0 and line[0] != \" \":\n staged += 1\n\n gitdir = _check_output([\"git\", \"rev-parse\", \"--git-dir\"]).strip()\n stashed = _get_stash(gitdir)\n operations = _gitoperation(gitdir)\n\n return GitStatus(\n branch,\n num_ahead,\n num_behind,\n untracked,\n changed,\n conflicts,\n staged,\n stashed,\n operations,\n )\n\n\ndef gitstatus_prompt():\n \"\"\"Return str `BRANCH|OPERATOR|numbers`\"\"\"\n try:\n s = gitstatus()\n except subprocess.SubprocessError:\n return None\n\n ret = _get_def(\"BRANCH\") + s.branch\n if s.num_ahead > 0:\n ret += _get_def(\"AHEAD\") + str(s.num_ahead)\n if s.num_behind > 0:\n ret += _get_def(\"BEHIND\") + str(s.num_behind)\n if s.operations:\n ret += _get_def(\"OPERATION\") + \"|\" + \"|\".join(s.operations)\n ret += \"|\"\n if s.staged > 0:\n ret += _get_def(\"STAGED\") + str(s.staged) + \"{NO_COLOR}\"\n if s.conflicts > 0:\n ret += _get_def(\"CONFLICTS\") + str(s.conflicts) + \"{NO_COLOR}\"\n if s.changed > 0:\n ret += _get_def(\"CHANGED\") + str(s.changed) + \"{NO_COLOR}\"\n if s.untracked > 0:\n ret += _get_def(\"UNTRACKED\") + str(s.untracked) + \"{NO_COLOR}\"\n if s.stashed > 0:\n ret += _get_def(\"STASHED\") + str(s.stashed) + \"{NO_COLOR}\"\n if s.staged + s.conflicts + s.changed + s.untracked + s.stashed == 0:\n ret += _get_def(\"CLEAN\") + \"{NO_COLOR}\"\n ret += \"{NO_COLOR}\"\n\n return ret\n" }, { "path": "xonsh/prompt/job.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"Prompt formatter for current jobs\"\"\"\n\nimport xonsh.jobs as xj\n\n\ndef _current_job():\n j = xj.get_next_task()\n if j is not None:\n if not j[\"bg\"]:\n cmd = j[\"cmds\"][-1]\n s = cmd[0]\n if s == \"sudo\" and len(cmd) > 1:\n s = cmd[1]\n return s\n" }, { "path": "xonsh/prompt/vc.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"Prompt formatter for simple version control branches\"\"\"\n# pylint:disable=no-member, invalid-name\n\nimport os\nimport sys\nimport queue\nimport builtins\nimport threading\nimport subprocess\n\nimport xonsh.tools as xt\n\n\ndef _get_git_branch(q):\n denv = builtins.__xonsh__.env.detype()\n try:\n branches = xt.decode_bytes(\n subprocess.check_output(\n [\"git\", \"branch\"], env=denv, stderr=subprocess.DEVNULL\n )\n ).splitlines()\n except (subprocess.CalledProcessError, OSError, FileNotFoundError):\n q.put(None)\n else:\n for branch in branches:\n if not branch.startswith(\"* \"):\n continue\n elif branch.endswith(\")\"):\n branch = branch.split()[-1][:-1]\n else:\n branch = branch.split()[-1]\n\n q.put(branch)\n break\n else:\n q.put(None)\n\n\ndef get_git_branch():\n \"\"\"Attempts to find the current git branch. If this could not\n be determined (timeout, not in a git repo, etc.) then this returns None.\n \"\"\"\n branch = None\n timeout = builtins.__xonsh__.env.get(\"VC_BRANCH_TIMEOUT\")\n q = queue.Queue()\n\n t = threading.Thread(target=_get_git_branch, args=(q,))\n t.start()\n t.join(timeout=timeout)\n try:\n branch = q.get_nowait()\n except queue.Empty:\n branch = None\n return branch\n\n\ndef _get_hg_root(q):\n _curpwd = builtins.__xonsh__.env[\"PWD\"]\n while True:\n if not os.path.isdir(_curpwd):\n return False\n if any([b.name == \".hg\" for b in xt.scandir(_curpwd)]):\n q.put(_curpwd)\n break\n else:\n _oldpwd = _curpwd\n _curpwd = os.path.split(_curpwd)[0]\n if _oldpwd == _curpwd:\n return False\n\n\ndef get_hg_branch(root=None):\n \"\"\"Try to get the mercurial branch of the current directory,\n return None if not in a repo or subprocess.TimeoutExpired if timed out.\n \"\"\"\n env = builtins.__xonsh__.env\n timeout = env[\"VC_BRANCH_TIMEOUT\"]\n q = queue.Queue()\n t = threading.Thread(target=_get_hg_root, args=(q,))\n t.start()\n t.join(timeout=timeout)\n try:\n root = q.get_nowait()\n except queue.Empty:\n return None\n if env.get(\"VC_HG_SHOW_BRANCH\"):\n # get branch name\n branch_path = os.path.sep.join([root, \".hg\", \"branch\"])\n if os.path.exists(branch_path):\n with open(branch_path, \"r\") as branch_file:\n branch = branch_file.read()\n else:\n branch = \"default\"\n else:\n branch = \"\"\n # add bookmark, if we can\n bookmark_path = os.path.sep.join([root, \".hg\", \"bookmarks.current\"])\n if os.path.exists(bookmark_path):\n with open(bookmark_path, \"r\") as bookmark_file:\n active_bookmark = bookmark_file.read()\n if env.get(\"VC_HG_SHOW_BRANCH\") is True:\n branch = \"{0}, {1}\".format(\n *(b.strip(os.linesep) for b in (branch, active_bookmark))\n )\n else:\n branch = active_bookmark.strip(os.linesep)\n else:\n branch = branch.strip(os.linesep)\n return branch\n\n\n_FIRST_BRANCH_TIMEOUT = True\n\n\ndef _first_branch_timeout_message():\n global _FIRST_BRANCH_TIMEOUT\n sbtm = builtins.__xonsh__.env[\"SUPPRESS_BRANCH_TIMEOUT_MESSAGE\"]\n if not _FIRST_BRANCH_TIMEOUT or sbtm:\n return\n _FIRST_BRANCH_TIMEOUT = False\n print(\n \"xonsh: branch timeout: computing the branch name, color, or both \"\n \"timed out while formatting the prompt. You may avoid this by \"\n \"increasing the value of $VC_BRANCH_TIMEOUT or by removing branch \"\n \"fields, like {curr_branch}, from your $PROMPT. See the FAQ \"\n \"for more details. This message will be suppressed for the remainder \"\n \"of this session. To suppress this message permanently, set \"\n \"$SUPPRESS_BRANCH_TIMEOUT_MESSAGE = True in your xonshrc file.\",\n file=sys.stderr,\n )\n\n\ndef current_branch():\n \"\"\"Gets the branch for a current working directory. Returns an empty string\n if the cwd is not a repository. This currently only works for git and hg\n and should be extended in the future. If a timeout occurred, the string\n '' is returned.\n \"\"\"\n branch = None\n cmds = builtins.__xonsh__.commands_cache\n # check for binary only once\n if cmds.is_empty():\n has_git = bool(cmds.locate_binary(\"git\", ignore_alias=True))\n has_hg = bool(cmds.locate_binary(\"hg\", ignore_alias=True))\n else:\n has_git = bool(cmds.lazy_locate_binary(\"git\", ignore_alias=True))\n has_hg = bool(cmds.lazy_locate_binary(\"hg\", ignore_alias=True))\n if has_git:\n branch = get_git_branch()\n if not branch and has_hg:\n branch = get_hg_branch()\n if isinstance(branch, subprocess.TimeoutExpired):\n branch = \"\"\n _first_branch_timeout_message()\n return branch or None\n\n\ndef _git_dirty_working_directory(q, include_untracked):\n status = None\n denv = builtins.__xonsh__.env.detype()\n try:\n cmd = [\"git\", \"status\", \"--porcelain\"]\n if include_untracked:\n cmd.append(\"--untracked-files=normal\")\n else:\n cmd.append(\"--untracked-files=no\")\n status = subprocess.check_output(cmd, stderr=subprocess.DEVNULL, env=denv)\n except (subprocess.CalledProcessError, OSError, FileNotFoundError):\n q.put(None)\n if status is not None:\n return q.put(bool(status))\n\n\ndef git_dirty_working_directory(include_untracked=False):\n \"\"\"Returns whether or not the git directory is dirty. If this could not\n be determined (timeout, file not found, etc.) then this returns None.\n \"\"\"\n timeout = builtins.__xonsh__.env.get(\"VC_BRANCH_TIMEOUT\")\n q = queue.Queue()\n t = threading.Thread(\n target=_git_dirty_working_directory, args=(q, include_untracked)\n )\n t.start()\n t.join(timeout=timeout)\n try:\n return q.get_nowait()\n except queue.Empty:\n return None\n\n\ndef hg_dirty_working_directory():\n \"\"\"Computes whether or not the mercurial working directory is dirty or not.\n If this cannot be determined, None is returned.\n \"\"\"\n env = builtins.__xonsh__.env\n cwd = env[\"PWD\"]\n denv = env.detype()\n vcbt = env[\"VC_BRANCH_TIMEOUT\"]\n # Override user configurations settings and aliases\n denv[\"HGRCPATH\"] = \"\"\n try:\n s = subprocess.check_output(\n [\"hg\", \"identify\", \"--id\"],\n stderr=subprocess.PIPE,\n cwd=cwd,\n timeout=vcbt,\n universal_newlines=True,\n env=denv,\n )\n return s.strip(os.linesep).endswith(\"+\")\n except (\n subprocess.CalledProcessError,\n subprocess.TimeoutExpired,\n FileNotFoundError,\n ):\n return None\n\n\ndef dirty_working_directory():\n \"\"\"Returns a boolean as to whether there are uncommitted files in version\n control repository we are inside. If this cannot be determined, returns\n None. Currently supports git and hg.\n \"\"\"\n dwd = None\n cmds = builtins.__xonsh__.commands_cache\n if cmds.lazy_locate_binary(\"git\", ignore_alias=True):\n dwd = git_dirty_working_directory()\n if cmds.lazy_locate_binary(\"hg\", ignore_alias=True) and dwd is None:\n dwd = hg_dirty_working_directory()\n return dwd\n\n\ndef branch_color():\n \"\"\"Return red if the current branch is dirty, yellow if the dirtiness can\n not be determined, and green if it clean. These are bold, intense colors\n for the foreground.\n \"\"\"\n dwd = dirty_working_directory()\n if dwd is None:\n color = \"{YELLOW}\"\n elif dwd:\n color = \"{RED}\"\n else:\n color = \"{GREEN}\"\n return color\n\n\ndef branch_bg_color():\n \"\"\"Return red if the current branch is dirty, yellow if the dirtiness can\n not be determined, and green if it clean. These are background colors.\n \"\"\"\n dwd = dirty_working_directory()\n if dwd is None:\n color = \"{BACKGROUND_YELLOW}\"\n elif dwd:\n color = \"{BACKGROUND_RED}\"\n else:\n color = \"{BACKGROUND_GREEN}\"\n return color\n" }, { "path": "xonsh/ptk/__init__.py", "content": "" }, { "path": "xonsh/ptk/completer.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"Completer implementation to use with prompt_toolkit.\"\"\"\nimport os\nimport builtins\n\nfrom prompt_toolkit.layout.dimension import LayoutDimension\nfrom prompt_toolkit.completion import Completer, Completion\nfrom prompt_toolkit.auto_suggest import AutoSuggestFromHistory\n\n\nclass PromptToolkitCompleter(Completer):\n \"\"\"Simple prompt_toolkit Completer object.\n\n It just redirects requests to normal Xonsh completer.\n \"\"\"\n\n def __init__(self, completer, ctx, shell):\n \"\"\"Takes instance of xonsh.completer.Completer, the xonsh execution\n context, and the shell instance itself.\n \"\"\"\n self.completer = completer\n self.ctx = ctx\n self.shell = shell\n self.hist_suggester = AutoSuggestFromHistory()\n\n def get_completions(self, document, complete_event):\n \"\"\"Returns a generator for list of completions.\"\"\"\n env = builtins.__xonsh__.env\n should_complete = complete_event.completion_requested or env.get(\n \"UPDATE_COMPLETIONS_ON_KEYPRESS\"\n )\n # Only generate completions when the user hits tab.\n if not should_complete or self.completer is None:\n return\n # generate actual completions\n line = document.current_line.lstrip()\n line_ex = builtins.aliases.expand_alias(line)\n\n endidx = document.cursor_position_col\n begidx = line[:endidx].rfind(\" \") + 1 if line[:endidx].rfind(\" \") >= 0 else 0\n prefix = line[begidx:endidx]\n expand_offset = len(line_ex) - len(line)\n # get normal completions\n completions, l = self.completer.complete(\n prefix, line_ex, begidx + expand_offset, endidx + expand_offset, self.ctx\n )\n # completions from auto suggest\n sug_comp = None\n if env.get(\"AUTO_SUGGEST\") and env.get(\"AUTO_SUGGEST_IN_COMPLETIONS\"):\n sug_comp = self.suggestion_completion(document, line)\n if sug_comp is None:\n pass\n elif len(completions) == 0:\n completions = (sug_comp,)\n else:\n completions = set(completions)\n completions.discard(sug_comp)\n completions = (sug_comp,) + tuple(sorted(completions))\n # reserve space, if needed.\n if len(completions) <= 1:\n pass\n elif len(os.path.commonprefix(completions)) <= len(prefix):\n self.reserve_space()\n # Find common prefix (strip quoting)\n c_prefix = os.path.commonprefix([a.strip(\"'\\\"\") for a in completions])\n # Find last split symbol, do not trim the last part\n while c_prefix:\n if c_prefix[-1] in r\"/\\.:@,\":\n break\n c_prefix = c_prefix[:-1]\n # yield completions\n if sug_comp is None:\n pre = min(document.cursor_position_col - begidx, len(c_prefix))\n else:\n pre = len(c_prefix)\n for comp in completions:\n # do not display quote\n disp = comp[pre:].strip(\"'\\\"\")\n yield Completion(comp, -l, display=disp)\n\n def suggestion_completion(self, document, line):\n \"\"\"Provides a completion based on the current auto-suggestion.\"\"\"\n cli = self.shell.prompter.cli\n sug = self.hist_suggester.get_suggestion(cli, cli.current_buffer, document)\n if sug is None:\n return None\n comp, _, _ = sug.text.partition(\" \")\n _, _, prev = line.rpartition(\" \")\n return prev + comp\n\n def reserve_space(self):\n cli = builtins.__xonsh__.shell.shell.prompter.cli\n window = cli.application.layout.children[0].content.children[1]\n\n if window and window.render_info:\n h = window.render_info.content_height\n r = builtins.__xonsh__.env.get(\"COMPLETIONS_MENU_ROWS\")\n size = h + r\n\n def comp_height(cli):\n # If there is an autocompletion menu to be shown, make sure that o\n # layout has at least a minimal height in order to display it.\n if not cli.is_done:\n return LayoutDimension(min=size)\n else:\n return LayoutDimension()\n\n window._height = comp_height\n" }, { "path": "xonsh/ptk/history.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"History object for use with prompt_toolkit.\"\"\"\nimport builtins\nfrom threading import Thread\n\nimport prompt_toolkit.history\n\n\nclass PromptToolkitHistory(prompt_toolkit.history.History):\n \"\"\"History class that implements the prompt-toolkit history interface\n with the xonsh backend.\n \"\"\"\n\n def __init__(self, load_prev=True, wait_for_gc=True, *args, **kwargs):\n \"\"\"Initialize history object.\"\"\"\n super().__init__()\n self.strings = []\n if load_prev:\n PromptToolkitHistoryAdder(self, wait_for_gc=wait_for_gc)\n\n def append(self, entry):\n \"\"\"Append new entry to the history.\"\"\"\n self.strings.append(entry)\n\n def __getitem__(self, index):\n return self.strings[index]\n\n def __len__(self):\n return len(self.strings)\n\n def __iter__(self):\n return iter(self.strings)\n\n\nclass PromptToolkitHistoryAdder(Thread):\n def __init__(self, ptkhist, wait_for_gc=True, *args, **kwargs):\n \"\"\"Thread responsible for adding inputs from history to the current\n prompt-toolkit history instance. May wait for the history garbage\n collector to finish.\n \"\"\"\n super(PromptToolkitHistoryAdder, self).__init__(*args, **kwargs)\n self.daemon = True\n self.ptkhist = ptkhist\n self.wait_for_gc = wait_for_gc\n self.start()\n\n def run(self):\n hist = builtins.__xonsh__.history\n if hist is None:\n return\n buf = None\n ptkhist = self.ptkhist\n for cmd in hist.all_items():\n line = cmd[\"inp\"].rstrip()\n if len(ptkhist) == 0 or line != ptkhist[-1]:\n ptkhist.append(line)\n if buf is None:\n buf = self._buf()\n if buf is None:\n continue\n buf.reset(initial_document=buf.document)\n\n def _buf(self):\n # Thread-safe version of\n # buf = builtins.__xonsh__.shell.shell.prompter.cli.application.buffer\n path = [\n \"__xonsh__\",\n \"shell\",\n \"shell\",\n \"prompter\",\n \"cli\",\n \"application\",\n \"buffer\",\n ]\n buf = builtins\n for a in path:\n buf = getattr(buf, a, None)\n if buf is None:\n break\n return buf\n" }, { "path": "xonsh/ptk/key_bindings.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"Key bindings for prompt_toolkit xonsh shell.\"\"\"\nimport builtins\n\nfrom prompt_toolkit.enums import DEFAULT_BUFFER\nfrom prompt_toolkit.filters import (\n Condition,\n IsMultiline,\n HasSelection,\n EmacsInsertMode,\n ViInsertMode,\n)\nfrom prompt_toolkit.keys import Keys\n\nfrom xonsh.aliases import xonsh_exit\nfrom xonsh.tools import check_for_partial_string, get_line_continuation\nfrom xonsh.shell import transform_command\n\nenv = builtins.__xonsh__.env\nDEDENT_TOKENS = frozenset([\"raise\", \"return\", \"pass\", \"break\", \"continue\"])\n\n\ndef carriage_return(b, cli, *, autoindent=True):\n \"\"\"Preliminary parser to determine if 'Enter' key should send command to the\n xonsh parser for execution or should insert a newline for continued input.\n\n Current 'triggers' for inserting a newline are:\n - Not on first line of buffer and line is non-empty\n - Previous character is a colon (covers if, for, etc...)\n - User is in an open paren-block\n - Line ends with backslash\n - Any text exists below cursor position (relevant when editing previous\n multiline blocks)\n \"\"\"\n doc = b.document\n at_end_of_line = _is_blank(doc.current_line_after_cursor)\n current_line_blank = _is_blank(doc.current_line)\n\n indent = env.get(\"INDENT\") if autoindent else \"\"\n\n partial_string_info = check_for_partial_string(doc.text)\n in_partial_string = (\n partial_string_info[0] is not None and partial_string_info[1] is None\n )\n\n # indent after a colon\n if doc.current_line_before_cursor.strip().endswith(\":\") and at_end_of_line:\n b.newline(copy_margin=autoindent)\n b.insert_text(indent, fire_event=False)\n # if current line isn't blank, check dedent tokens\n elif (\n not current_line_blank\n and doc.current_line.split(maxsplit=1)[0] in DEDENT_TOKENS\n and doc.line_count > 1\n ):\n b.newline(copy_margin=autoindent)\n b.delete_before_cursor(count=len(indent))\n elif not doc.on_first_line and not current_line_blank:\n b.newline(copy_margin=autoindent)\n elif doc.current_line.endswith(get_line_continuation()):\n b.newline(copy_margin=autoindent)\n elif doc.find_next_word_beginning() is not None and (\n any(not _is_blank(i) for i in doc.lines_from_current[1:])\n ):\n b.newline(copy_margin=autoindent)\n elif not current_line_blank and not can_compile(doc.text):\n b.newline(copy_margin=autoindent)\n elif current_line_blank and in_partial_string:\n b.newline(copy_margin=autoindent)\n else:\n b.accept_action.validate_and_handle(cli, b)\n\n\ndef _is_blank(l):\n return len(l.strip()) == 0\n\n\ndef can_compile(src):\n \"\"\"Returns whether the code can be compiled, i.e. it is valid xonsh.\"\"\"\n src = src if src.endswith(\"\\n\") else src + \"\\n\"\n src = transform_command(src, show_diff=False)\n src = src.lstrip()\n try:\n builtins.__xonsh__.execer.compile(\n src, mode=\"single\", glbs=None, locs=builtins.__xonsh__.ctx\n )\n rtn = True\n except SyntaxError:\n rtn = False\n except Exception:\n rtn = True\n return rtn\n\n\n@Condition\ndef tab_insert_indent(cli):\n \"\"\"Check if should insert indent instead of starting autocompletion.\n Checks if there are only whitespaces before the cursor - if so indent\n should be inserted, otherwise autocompletion.\n\n \"\"\"\n before_cursor = cli.current_buffer.document.current_line_before_cursor\n\n return bool(before_cursor.isspace())\n\n\n@Condition\ndef beginning_of_line(cli):\n \"\"\"Check if cursor is at beginning of a line other than the first line in a\n multiline document\n \"\"\"\n before_cursor = cli.current_buffer.document.current_line_before_cursor\n\n return bool(\n len(before_cursor) == 0 and not cli.current_buffer.document.on_first_line\n )\n\n\n@Condition\ndef end_of_line(cli):\n \"\"\"Check if cursor is at the end of a line other than the last line in a\n multiline document\n \"\"\"\n d = cli.current_buffer.document\n at_end = d.is_cursor_at_the_end_of_line\n last_line = d.is_cursor_at_the_end\n\n return bool(at_end and not last_line)\n\n\n@Condition\ndef should_confirm_completion(cli):\n \"\"\"Check if completion needs confirmation\"\"\"\n return (\n builtins.__xonsh__.env.get(\"COMPLETIONS_CONFIRM\")\n and cli.current_buffer.complete_state\n )\n\n\n# Copied from prompt-toolkit's key_binding/bindings/basic.py\n@Condition\ndef ctrl_d_condition(cli):\n \"\"\"Ctrl-D binding is only active when the default buffer is selected and\n empty.\n \"\"\"\n if builtins.__xonsh__.env.get(\"IGNOREEOF\"):\n raise EOFError\n else:\n return cli.current_buffer_name == DEFAULT_BUFFER and not cli.current_buffer.text\n\n\n@Condition\ndef autopair_condition(cli):\n \"\"\"Check if XONSH_AUTOPAIR is set\"\"\"\n return builtins.__xonsh__.env.get(\"XONSH_AUTOPAIR\", False)\n\n\n@Condition\ndef whitespace_or_bracket_before(cli):\n \"\"\"Check if there is whitespace or an opening\n bracket to the left of the cursor\"\"\"\n d = cli.current_buffer.document\n return bool(\n d.cursor_position == 0\n or d.char_before_cursor.isspace()\n or d.char_before_cursor in \"([{\"\n )\n\n\n@Condition\ndef whitespace_or_bracket_after(cli):\n \"\"\"Check if there is whitespace or a closing\n bracket to the right of the cursor\"\"\"\n d = cli.current_buffer.document\n return bool(\n d.is_cursor_at_the_end_of_line\n or d.current_char.isspace()\n or d.current_char in \")]}\"\n )\n\n\ndef load_xonsh_bindings(key_bindings_manager):\n \"\"\"\n Load custom key bindings.\n \"\"\"\n handle = key_bindings_manager.registry.add_binding\n has_selection = HasSelection()\n insert_mode = ViInsertMode() | EmacsInsertMode()\n\n @handle(Keys.Tab, filter=tab_insert_indent)\n def insert_indent(event):\n \"\"\"\n If there are only whitespaces before current cursor position insert\n indent instead of autocompleting.\n \"\"\"\n event.cli.current_buffer.insert_text(env.get(\"INDENT\"))\n\n @handle(Keys.ControlX, Keys.ControlE, filter=~has_selection)\n def open_editor(event):\n \"\"\" Open current buffer in editor \"\"\"\n event.current_buffer.open_in_editor(event.cli)\n\n @handle(Keys.BackTab, filter=insert_mode)\n def insert_literal_tab(event):\n \"\"\" Insert literal tab on Shift+Tab instead of autocompleting \"\"\"\n b = event.current_buffer\n if b.complete_state:\n b.complete_previous()\n else:\n event.cli.current_buffer.insert_text(env.get(\"INDENT\"))\n\n @handle(\"(\", filter=autopair_condition & whitespace_or_bracket_after)\n def insert_right_parens(event):\n event.cli.current_buffer.insert_text(\"(\")\n event.cli.current_buffer.insert_text(\")\", move_cursor=False)\n\n @handle(\")\", filter=autopair_condition)\n def overwrite_right_parens(event):\n buffer = event.cli.current_buffer\n if buffer.document.current_char == \")\":\n buffer.cursor_position += 1\n else:\n buffer.insert_text(\")\")\n\n @handle(\"[\", filter=autopair_condition & whitespace_or_bracket_after)\n def insert_right_bracket(event):\n event.cli.current_buffer.insert_text(\"[\")\n event.cli.current_buffer.insert_text(\"]\", move_cursor=False)\n\n @handle(\"]\", filter=autopair_condition)\n def overwrite_right_bracket(event):\n buffer = event.cli.current_buffer\n\n if buffer.document.current_char == \"]\":\n buffer.cursor_position += 1\n else:\n buffer.insert_text(\"]\")\n\n @handle(\"{\", filter=autopair_condition & whitespace_or_bracket_after)\n def insert_right_brace(event):\n event.cli.current_buffer.insert_text(\"{\")\n event.cli.current_buffer.insert_text(\"}\", move_cursor=False)\n\n @handle(\"}\", filter=autopair_condition)\n def overwrite_right_brace(event):\n buffer = event.cli.current_buffer\n\n if buffer.document.current_char == \"}\":\n buffer.cursor_position += 1\n else:\n buffer.insert_text(\"}\")\n\n @handle(\"'\", filter=autopair_condition)\n def insert_right_quote(event):\n buffer = event.cli.current_buffer\n\n if buffer.document.current_char == \"'\":\n buffer.cursor_position += 1\n elif whitespace_or_bracket_before(event.cli) and whitespace_or_bracket_after(\n event.cli\n ):\n buffer.insert_text(\"'\")\n buffer.insert_text(\"'\", move_cursor=False)\n else:\n buffer.insert_text(\"'\")\n\n @handle('\"', filter=autopair_condition)\n def insert_right_double_quote(event):\n buffer = event.cli.current_buffer\n\n if buffer.document.current_char == '\"':\n buffer.cursor_position += 1\n elif whitespace_or_bracket_before(event.cli) and whitespace_or_bracket_after(\n event.cli\n ):\n buffer.insert_text('\"')\n buffer.insert_text('\"', move_cursor=False)\n else:\n buffer.insert_text('\"')\n\n @handle(Keys.Backspace, filter=autopair_condition)\n def delete_brackets_or_quotes(event):\n \"\"\"Delete empty pair of brackets or quotes\"\"\"\n buffer = event.cli.current_buffer\n before = buffer.document.char_before_cursor\n after = buffer.document.current_char\n\n if any(\n [before == b and after == a for (b, a) in [\"()\", \"[]\", \"{}\", \"''\", '\"\"']]\n ):\n buffer.delete(1)\n\n buffer.delete_before_cursor(1)\n\n @handle(Keys.ControlD, filter=ctrl_d_condition)\n def call_exit_alias(event):\n \"\"\"Use xonsh exit function\"\"\"\n b = event.cli.current_buffer\n b.accept_action.validate_and_handle(event.cli, b)\n xonsh_exit([])\n\n @handle(Keys.ControlJ, filter=IsMultiline())\n def multiline_carriage_return(event):\n \"\"\" Wrapper around carriage_return multiline parser \"\"\"\n b = event.cli.current_buffer\n carriage_return(b, event.cli)\n\n @handle(Keys.ControlJ, filter=should_confirm_completion)\n def enter_confirm_completion(event):\n \"\"\"Ignore (confirm completion)\"\"\"\n event.current_buffer.complete_state = None\n\n @handle(Keys.Escape, filter=should_confirm_completion)\n def esc_cancel_completion(event):\n \"\"\"Use to cancel completion\"\"\"\n event.cli.current_buffer.cancel_completion()\n\n @handle(Keys.Escape, Keys.ControlJ)\n def execute_block_now(event):\n \"\"\"Execute a block of text irrespective of cursor position\"\"\"\n b = event.cli.current_buffer\n b.accept_action.validate_and_handle(event.cli, b)\n\n @handle(Keys.Left, filter=beginning_of_line)\n def wrap_cursor_back(event):\n \"\"\"Move cursor to end of previous line unless at beginning of\n document\n \"\"\"\n b = event.cli.current_buffer\n b.cursor_up(count=1)\n relative_end_index = b.document.get_end_of_line_position()\n b.cursor_right(count=relative_end_index)\n\n @handle(Keys.Right, filter=end_of_line)\n def wrap_cursor_forward(event):\n \"\"\"Move cursor to beginning of next line unless at end of document\"\"\"\n b = event.cli.current_buffer\n relative_begin_index = b.document.get_start_of_line_position()\n b.cursor_left(count=abs(relative_begin_index))\n b.cursor_down(count=1)\n\n @handle(Keys.ControlI, filter=insert_mode)\n def generate_completions(event):\n \"\"\"\n Tab-completion: where the first tab completes the common suffix and the\n second tab lists all the completions.\n\n Notes\n -----\n This method was forked from the mainline prompt-toolkit repo.\n Copyright (c) 2014, Jonathan Slenders, All rights reserved.\n \"\"\"\n b = event.current_buffer\n\n def second_tab():\n if b.complete_state:\n b.complete_next()\n else:\n event.cli.start_completion(select_first=False)\n\n # On the second tab-press, or when already navigating through\n # completions.\n if event.is_repeat or b.complete_state:\n second_tab()\n else:\n event.cli.start_completion(insert_common_part=True, select_first=False)\n" }, { "path": "xonsh/ptk/shell.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"The prompt_toolkit based xonsh shell.\"\"\"\nimport sys\nimport builtins\n\nfrom prompt_toolkit.key_binding.manager import KeyBindingManager\nfrom prompt_toolkit.auto_suggest import AutoSuggestFromHistory\nfrom prompt_toolkit.layout.lexers import PygmentsLexer\nfrom prompt_toolkit.shortcuts import print_tokens\nfrom prompt_toolkit.styles import PygmentsStyle, style_from_dict\n\nfrom xonsh.base_shell import BaseShell\nfrom xonsh.tools import print_exception, carriage_return, ansicolors_to_ptk1_names\nfrom xonsh.ptk.completer import PromptToolkitCompleter\nfrom xonsh.ptk.history import PromptToolkitHistory\nfrom xonsh.ptk.key_bindings import load_xonsh_bindings\nfrom xonsh.ptk.shortcuts import Prompter\nfrom xonsh.events import events\nfrom xonsh.shell import transform_command\nfrom xonsh.platform import HAS_PYGMENTS, ON_WINDOWS\nfrom xonsh.style_tools import (\n partial_color_tokenize,\n _TokenType,\n DEFAULT_STYLE_DICT as _DEFAULT_STYLE_DICT,\n)\nfrom xonsh.lazyimps import pygments, pyghooks, winutils\nfrom xonsh.pygments_cache import get_all_styles\nfrom xonsh.lazyasd import LazyObject\n\n\nToken = _TokenType()\n\nevents.transmogrify(\"on_ptk_create\", \"LoadEvent\")\nevents.doc(\n \"on_ptk_create\",\n \"\"\"\non_ptk_create(prompter: Prompter, history: PromptToolkitHistory, completer: PromptToolkitCompleter, bindings: KeyBindingManager) ->\n\nFired after prompt toolkit has been initialized\n\"\"\",\n)\n\n# Convert new ansicolor names to names\n# understood by PTK1\nDEFAULT_STYLE_DICT = LazyObject(\n lambda: ansicolors_to_ptk1_names(_DEFAULT_STYLE_DICT),\n globals(),\n \"DEFAULT_STYLE_DICT\",\n)\n\n\nclass PromptToolkitShell(BaseShell):\n \"\"\"The xonsh shell.\"\"\"\n\n def __init__(self, **kwargs):\n super().__init__(**kwargs)\n if ON_WINDOWS:\n winutils.enable_virtual_terminal_processing()\n self._first_prompt = True\n self.prompter = Prompter()\n self.history = PromptToolkitHistory()\n self.pt_completer = PromptToolkitCompleter(self.completer, self.ctx, self)\n key_bindings_manager_args = {\n \"enable_auto_suggest_bindings\": True,\n \"enable_search\": True,\n \"enable_abort_and_exit_bindings\": True,\n }\n self.key_bindings_manager = KeyBindingManager(**key_bindings_manager_args)\n load_xonsh_bindings(self.key_bindings_manager)\n # This assumes that PromptToolkitShell is a singleton\n events.on_ptk_create.fire(\n prompter=self.prompter,\n history=self.history,\n completer=self.pt_completer,\n bindings=self.key_bindings_manager,\n )\n\n def singleline(\n self,\n store_in_history=True,\n auto_suggest=None,\n enable_history_search=True,\n multiline=True,\n **kwargs\n ):\n \"\"\"Reads a single line of input from the shell. The store_in_history\n kwarg flags whether the input should be stored in PTK's in-memory\n history.\n \"\"\"\n events.on_pre_prompt.fire()\n env = builtins.__xonsh__.env\n mouse_support = env.get(\"MOUSE_SUPPORT\")\n if store_in_history:\n history = self.history\n else:\n history = None\n enable_history_search = False\n auto_suggest = auto_suggest if env.get(\"AUTO_SUGGEST\") else None\n completions_display = env.get(\"COMPLETIONS_DISPLAY\")\n multicolumn = completions_display == \"multi\"\n complete_while_typing = env.get(\"UPDATE_COMPLETIONS_ON_KEYPRESS\")\n if complete_while_typing:\n # PTK requires history search to be none when completing while typing\n enable_history_search = False\n if HAS_PYGMENTS:\n self.styler.style_name = env.get(\"XONSH_COLOR_STYLE\")\n completer = None if completions_display == \"none\" else self.pt_completer\n if not env.get(\"UPDATE_PROMPT_ON_KEYPRESS\"):\n prompt_tokens_cached = self.prompt_tokens(None)\n get_prompt_tokens = lambda cli: prompt_tokens_cached\n rprompt_tokens_cached = self.rprompt_tokens(None)\n get_rprompt_tokens = lambda cli: rprompt_tokens_cached\n bottom_toolbar_tokens_cached = self.bottom_toolbar_tokens(None)\n get_bottom_toolbar_tokens = lambda cli: bottom_toolbar_tokens_cached\n else:\n get_prompt_tokens = self.prompt_tokens\n get_rprompt_tokens = self.rprompt_tokens\n get_bottom_toolbar_tokens = self.bottom_toolbar_tokens\n\n with self.prompter:\n prompt_args = {\n \"mouse_support\": mouse_support,\n \"auto_suggest\": auto_suggest,\n \"get_prompt_tokens\": get_prompt_tokens,\n \"get_rprompt_tokens\": get_rprompt_tokens,\n \"get_bottom_toolbar_tokens\": get_bottom_toolbar_tokens,\n \"completer\": completer,\n \"multiline\": multiline,\n \"get_continuation_tokens\": self.continuation_tokens,\n \"history\": history,\n \"enable_history_search\": enable_history_search,\n \"reserve_space_for_menu\": 0,\n \"key_bindings_registry\": self.key_bindings_manager.registry,\n \"display_completions_in_columns\": multicolumn,\n \"complete_while_typing\": complete_while_typing,\n }\n if builtins.__xonsh__.env.get(\"COLOR_INPUT\"):\n if HAS_PYGMENTS:\n prompt_args[\"lexer\"] = PygmentsLexer(pyghooks.XonshLexer)\n prompt_args[\"style\"] = PygmentsStyle(\n pyghooks.xonsh_style_proxy(self.styler)\n )\n else:\n prompt_args[\"style\"] = style_from_dict(DEFAULT_STYLE_DICT)\n line = self.prompter.prompt(**prompt_args)\n events.on_post_prompt.fire()\n return line\n\n def _push(self, line):\n \"\"\"Pushes a line onto the buffer and compiles the code in a way that\n enables multiline input.\n \"\"\"\n code = None\n self.buffer.append(line)\n if self.need_more_lines:\n return None, code\n src = \"\".join(self.buffer)\n src = transform_command(src)\n try:\n code = self.execer.compile(src, mode=\"single\", glbs=self.ctx, locs=None)\n self.reset_buffer()\n except Exception: # pylint: disable=broad-except\n self.reset_buffer()\n print_exception()\n return src, None\n return src, code\n\n def cmdloop(self, intro=None):\n \"\"\"Enters a loop that reads and execute input from user.\"\"\"\n if intro:\n print(intro)\n auto_suggest = AutoSuggestFromHistory()\n self.push = self._push\n while not builtins.__xonsh__.exit:\n try:\n line = self.singleline(auto_suggest=auto_suggest)\n if not line:\n self.emptyline()\n else:\n line = self.precmd(line)\n self.default(line)\n except (KeyboardInterrupt, SystemExit):\n self.reset_buffer()\n except EOFError:\n if builtins.__xonsh__.env.get(\"IGNOREEOF\"):\n print('Use \"exit\" to leave the shell.', file=sys.stderr)\n else:\n break\n\n def prompt_tokens(self, cli):\n \"\"\"Returns a list of (token, str) tuples for the current prompt.\"\"\"\n p = builtins.__xonsh__.env.get(\"PROMPT\")\n try:\n p = self.prompt_formatter(p)\n except Exception: # pylint: disable=broad-except\n print_exception()\n toks = partial_color_tokenize(p)\n if self._first_prompt:\n carriage_return()\n self._first_prompt = False\n self.settitle()\n return toks\n\n def rprompt_tokens(self, cli):\n \"\"\"Returns a list of (token, str) tuples for the current right\n prompt.\n \"\"\"\n p = builtins.__xonsh__.env.get(\"RIGHT_PROMPT\")\n # self.prompt_formatter does handle empty strings properly,\n # but this avoids descending into it in the common case of\n # $RIGHT_PROMPT == ''.\n if isinstance(p, str) and len(p) == 0:\n return []\n try:\n p = self.prompt_formatter(p)\n except Exception: # pylint: disable=broad-except\n print_exception()\n toks = partial_color_tokenize(p)\n return toks\n\n def bottom_toolbar_tokens(self, cli):\n \"\"\"Returns a list of (token, str) tuples for the current bottom\n toolbar.\n \"\"\"\n p = builtins.__xonsh__.env.get(\"BOTTOM_TOOLBAR\")\n # self.prompt_formatter does handle empty strings properly,\n # but this avoids descending into it in the common case of\n # $TOOLBAR == ''.\n if isinstance(p, str) and len(p) == 0:\n return []\n try:\n p = self.prompt_formatter(p)\n except Exception: # pylint: disable=broad-except\n print_exception()\n toks = partial_color_tokenize(p)\n return toks\n\n def continuation_tokens(self, cli, width):\n \"\"\"Displays dots in multiline prompt\"\"\"\n width = width - 1\n dots = builtins.__xonsh__.env.get(\"MULTILINE_PROMPT\")\n dots = dots() if callable(dots) else dots\n if dots is None:\n return [(Token, \" \" * (width + 1))]\n basetoks = self.format_color(dots)\n baselen = sum(len(t[1]) for t in basetoks)\n if baselen == 0:\n return [(Token, \" \" * (width + 1))]\n toks = basetoks * (width // baselen)\n n = width % baselen\n count = 0\n for tok in basetoks:\n slen = len(tok[1])\n newcount = slen + count\n if slen == 0:\n continue\n elif newcount <= n:\n toks.append(tok)\n else:\n toks.append((tok[0], tok[1][: n - count]))\n count = newcount\n if n <= count:\n break\n toks.append((Token, \" \")) # final space\n return toks\n\n def format_color(self, string, hide=False, force_string=False, **kwargs):\n \"\"\"Formats a color string using Pygments. This, therefore, returns\n a list of (Token, str) tuples. If force_string is set to true, though,\n this will return a color formatted string.\n \"\"\"\n tokens = partial_color_tokenize(string)\n if force_string and HAS_PYGMENTS:\n env = builtins.__xonsh__.env\n self.styler.style_name = env.get(\"XONSH_COLOR_STYLE\")\n proxy_style = pyghooks.xonsh_style_proxy(self.styler)\n formatter = pyghooks.XonshTerminal256Formatter(style=proxy_style)\n s = pygments.format(tokens, formatter)\n return s\n elif force_string:\n print(\"To force colorization of string, install Pygments\")\n return tokens\n else:\n return tokens\n\n def print_color(self, string, end=\"\\n\", **kwargs):\n \"\"\"Prints a color string using prompt-toolkit color management.\"\"\"\n if isinstance(string, str):\n tokens = partial_color_tokenize(string + end)\n else:\n # assume this is a list of (Token, str) tuples and just print\n tokens = string\n if HAS_PYGMENTS:\n env = builtins.__xonsh__.env\n self.styler.style_name = env.get(\"XONSH_COLOR_STYLE\")\n proxy_style = PygmentsStyle(pyghooks.xonsh_style_proxy(self.styler))\n else:\n proxy_style = style_from_dict(DEFAULT_STYLE_DICT)\n print_tokens(tokens, style=proxy_style)\n\n def color_style_names(self):\n \"\"\"Returns an iterable of all available style names.\"\"\"\n if not HAS_PYGMENTS:\n return [\"For other xonsh styles, please install pygments\"]\n return get_all_styles()\n\n def color_style(self):\n \"\"\"Returns the current color map.\"\"\"\n if not HAS_PYGMENTS:\n return DEFAULT_STYLE_DICT\n env = builtins.__xonsh__.env\n self.styler.style_name = env.get(\"XONSH_COLOR_STYLE\")\n return self.styler.styles\n\n def restore_tty_sanity(self):\n \"\"\"An interface for resetting the TTY stdin mode. This is highly\n dependent on the shell backend. Also it is mostly optional since\n it only affects ^Z backgrounding behaviour.\n \"\"\"\n # PTK does not seem to need any specialization here. However,\n # if it does for some reason in the future...\n # The following writes an ANSI escape sequence that sends the cursor\n # to the end of the line. This has the effect of restoring ECHO mode.\n # See http://unix.stackexchange.com/a/108014/129048 for more details.\n # This line can also be replaced by os.system(\"stty sane\"), as per\n # http://stackoverflow.com/questions/19777129/interactive-python-interpreter-run-in-background#comment29421919_19778355\n # However, it is important to note that not termios-based solution\n # seems to work. My guess is that this is because termios restoration\n # needs to be performed by the subprocess itself. This fix is important\n # when subprocesses don't properly restore the terminal attributes,\n # like Python in interactive mode. Also note that the sequences \"\\033M\"\n # and \"\\033E\" seem to work too, but these are technically VT100 codes.\n # I used the more primitive ANSI sequence to maximize compatibility.\n # -scopatz 2017-01-28\n # if not ON_POSIX:\n # return\n # sys.stdout.write('\\033[9999999C\\n')\n" }, { "path": "xonsh/ptk/shortcuts.py", "content": "\"\"\"A prompt-toolkit inspired shortcut collection.\"\"\"\nimport builtins\nimport textwrap\n\nfrom prompt_toolkit.interface import CommandLineInterface\nfrom prompt_toolkit.enums import EditingMode\nfrom prompt_toolkit.utils import DummyContext\nfrom prompt_toolkit.shortcuts import (\n create_prompt_application,\n create_eventloop,\n create_asyncio_eventloop,\n create_output,\n)\n\nfrom xonsh.platform import ptk_version_info\nimport xonsh.tools as xt\n\n\nclass Prompter(object):\n def __init__(self, cli=None, *args, **kwargs):\n \"\"\"Implements a prompt that statefully holds a command-line\n interface. When used as a context manager, it will return itself\n on entry and reset itself on exit.\n\n Parameters\n ----------\n cli : CommandLineInterface or None, optional\n If this is not a CommandLineInterface object, such an object\n will be created when the prompt() method is called.\n \"\"\"\n self.cli = cli\n self.major_minor = ptk_version_info()[:2]\n\n def __enter__(self):\n self.reset()\n return self\n\n def __exit__(self, exc_type, exc_value, traceback):\n pass\n\n def prompt(self, message=\"\", **kwargs):\n \"\"\"Get input from the user and return it.\n\n This is a wrapper around a lot of prompt_toolkit functionality and\n can be a replacement for raw_input. (or GNU readline.) If you want\n to keep your history across several calls, create one\n `~prompt_toolkit.history.History instance and pass it every\n time. This function accepts many keyword arguments. Except for the\n following. they are a proxy to the arguments of\n create_prompt_application().\n\n Parameters\n ----------\n patch_stdout : file-like, optional\n Replace ``sys.stdout`` by a proxy that ensures that print\n statements from other threads won't destroy the prompt. (They\n will be printed above the prompt instead.)\n return_asyncio_coroutine : bool, optional\n When True, return a asyncio coroutine. (Python >3.3)\n\n Notes\n -----\n This method was forked from the mainline prompt-toolkit repo.\n Copyright (c) 2014, Jonathan Slenders, All rights reserved.\n \"\"\"\n patch_stdout = kwargs.pop(\"patch_stdout\", False)\n return_asyncio_coroutine = kwargs.pop(\"return_asyncio_coroutine\", False)\n if return_asyncio_coroutine:\n eventloop = create_asyncio_eventloop()\n else:\n eventloop = kwargs.pop(\"eventloop\", None) or create_eventloop()\n\n # Create CommandLineInterface.\n if self.cli is None:\n if builtins.__xonsh__.env.get(\"VI_MODE\"):\n editing_mode = EditingMode.VI\n else:\n editing_mode = EditingMode.EMACS\n kwargs[\"editing_mode\"] = editing_mode\n cli = CommandLineInterface(\n application=create_prompt_application(message, **kwargs),\n eventloop=eventloop,\n output=create_output(),\n )\n self.cli = cli\n else:\n cli = self.cli\n\n # Replace stdout.\n patch_context = cli.patch_stdout_context() if patch_stdout else DummyContext()\n\n # Read input and return it.\n if return_asyncio_coroutine:\n # Create an asyncio coroutine and call it.\n exec_context = {\"patch_context\": patch_context, \"cli\": cli}\n exec(\n textwrap.dedent(\n \"\"\"\n import asyncio\n @asyncio.coroutine\n def prompt_coro():\n with patch_context:\n document = yield from cli.run_async(reset_current_buffer=False)\n if document:\n return document.text\n \"\"\"\n ),\n exec_context,\n )\n return exec_context[\"prompt_coro\"]()\n else:\n # Note: We pass `reset_current_buffer=False`, because that way\n # it's easy to give DEFAULT_BUFFER a default value, without it\n # getting erased. We don't have to reset anyway, because this is\n # the first and only time that this CommandLineInterface will run.\n try:\n with patch_context:\n document = cli.run(reset_current_buffer=False)\n if document:\n return document.text\n except Exception:\n xt.print_exception()\n # return something to prevent xonsh crash when any\n # exceptions raise\n return \"\"\n finally:\n eventloop.close()\n\n def reset(self):\n \"\"\"Resets the prompt and cli to a pristine state on this object.\"\"\"\n self.cli = None\n" }, { "path": "xonsh/ptk2/__init__.py", "content": "# must come before ptk / pygments imports\nfrom xonsh.lazyasd import load_module_in_background\n\nload_module_in_background(\n \"pkg_resources\",\n debug=\"XONSH_DEBUG\",\n replacements={\"pygments.plugin\": \"pkg_resources\"},\n)\n" }, { "path": "xonsh/ptk2/completer.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"Completer implementation to use with prompt_toolkit.\"\"\"\n\nfrom prompt_toolkit.completion import Completer\n\nfrom gitsome.completer import CompleterGitsome\n\n\nclass PromptToolkitCompleter(Completer):\n \"\"\"Simple prompt_toolkit Completer object.\n\n It just redirects requests to normal Xonsh completer.\n \"\"\"\n\n def __init__(self, completer, ctx, shell):\n \"\"\"Takes instance of xonsh.completer.Completer, the xonsh execution\n context, and the shell instance itself.\n \"\"\"\n self.completer = completer\n self.ctx = ctx\n self.shell = shell\n self.completer_gitsome = CompleterGitsome()\n\n def get_completions(self, document, complete_event):\n \"\"\"Returns a generator for list of completions.\"\"\"\n line = document.current_line.lstrip()\n endidx = document.cursor_position_col\n space_pos = document.find_backwards(' ')\n if space_pos is None:\n begidx = 0\n else:\n begidx = space_pos + endidx + 1\n prefix = line[begidx:endidx]\n completions, dummy = self.completer.complete(prefix,\n line,\n begidx,\n endidx,\n self.ctx)\n completions_gitsome = \\\n self.completer_gitsome.get_completions(document,\n complete_event)\n completions_with_meta = \\\n self.completer_gitsome.build_completions_with_meta(line,\n prefix,\n completions)\n completions_gitsome.extend(completions_with_meta)\n for comp in completions_gitsome:\n yield comp\n" }, { "path": "xonsh/ptk2/history.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"History object for use with prompt_toolkit.\"\"\"\nimport builtins\n\nimport prompt_toolkit.history\n\n\nclass PromptToolkitHistory(prompt_toolkit.history.History):\n \"\"\"History class that implements the prompt-toolkit history interface\n with the xonsh backend.\n \"\"\"\n\n def __init__(self, load_prev=True, *args, **kwargs):\n \"\"\"Initialize history object.\"\"\"\n super().__init__()\n self.load_prev = load_prev\n\n def store_string(self, entry):\n pass\n\n def load_history_strings(self):\n \"\"\"Loads synchronous history strings\"\"\"\n if not self.load_prev:\n return\n hist = builtins.__xonsh__.history\n if hist is None:\n return\n for cmd in hist.all_items(newest_first=True):\n line = cmd[\"inp\"].rstrip()\n strs = self.get_strings()\n if len(strs) == 0 or line != strs[-1]:\n yield line\n\n def __getitem__(self, index):\n return self.get_strings()[index]\n\n def __len__(self):\n return len(self.get_strings())\n\n def __iter__(self):\n return iter(self.get_strings())\n\n\ndef _cust_history_matches(self, i):\n \"\"\"Custom history search method for prompt_toolkit that matches previous\n commands anywhere on a line, not just at the start.\n\n This gets monkeypatched into the prompt_toolkit prompter if\n ``XONSH_HISTORY_MATCH_ANYWHERE=True``\"\"\"\n return (\n self.history_search_text is None\n or self.history_search_text in self._working_lines[i]\n )\n" }, { "path": "xonsh/ptk2/key_bindings.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"Key bindings for prompt_toolkit xonsh shell.\"\"\"\nimport builtins\n\nfrom prompt_toolkit import search\nfrom prompt_toolkit.enums import DEFAULT_BUFFER\nfrom prompt_toolkit.filters import (\n Condition,\n IsMultiline,\n HasSelection,\n EmacsInsertMode,\n ViInsertMode,\n IsSearching,\n)\nfrom prompt_toolkit.keys import Keys\nfrom prompt_toolkit.application.current import get_app\n\nfrom xonsh.aliases import xonsh_exit\nfrom xonsh.tools import check_for_partial_string, get_line_continuation\nfrom xonsh.shell import transform_command\n\nDEDENT_TOKENS = frozenset([\"raise\", \"return\", \"pass\", \"break\", \"continue\"])\n\n\ndef carriage_return(b, cli, *, autoindent=True):\n \"\"\"Preliminary parser to determine if 'Enter' key should send command to the\n xonsh parser for execution or should insert a newline for continued input.\n\n Current 'triggers' for inserting a newline are:\n - Not on first line of buffer and line is non-empty\n - Previous character is a colon (covers if, for, etc...)\n - User is in an open paren-block\n - Line ends with backslash\n - Any text exists below cursor position (relevant when editing previous\n multiline blocks)\n \"\"\"\n doc = b.document\n at_end_of_line = _is_blank(doc.current_line_after_cursor)\n current_line_blank = _is_blank(doc.current_line)\n\n env = builtins.__xonsh__.env\n indent = env.get(\"INDENT\") if autoindent else \"\"\n\n partial_string_info = check_for_partial_string(doc.text)\n in_partial_string = (\n partial_string_info[0] is not None and partial_string_info[1] is None\n )\n\n # indent after a colon\n if doc.current_line_before_cursor.strip().endswith(\":\") and at_end_of_line:\n b.newline(copy_margin=autoindent)\n b.insert_text(indent, fire_event=False)\n # if current line isn't blank, check dedent tokens\n elif (\n not current_line_blank\n and doc.current_line.split(maxsplit=1)[0] in DEDENT_TOKENS\n and doc.line_count > 1\n ):\n b.newline(copy_margin=autoindent)\n b.delete_before_cursor(count=len(indent))\n elif not doc.on_first_line and not current_line_blank:\n b.newline(copy_margin=autoindent)\n elif doc.current_line.endswith(get_line_continuation()):\n b.newline(copy_margin=autoindent)\n elif doc.find_next_word_beginning() is not None and (\n any(not _is_blank(i) for i in doc.lines_from_current[1:])\n ):\n b.newline(copy_margin=autoindent)\n elif not current_line_blank and not can_compile(doc.text):\n b.newline(copy_margin=autoindent)\n elif current_line_blank and in_partial_string:\n b.newline(copy_margin=autoindent)\n else:\n b.validate_and_handle()\n\n\ndef _is_blank(l):\n return len(l.strip()) == 0\n\n\ndef can_compile(src):\n \"\"\"Returns whether the code can be compiled, i.e. it is valid xonsh.\"\"\"\n src = src if src.endswith(\"\\n\") else src + \"\\n\"\n src = transform_command(src, show_diff=False)\n src = src.lstrip()\n try:\n builtins.__xonsh__.execer.compile(\n src, mode=\"single\", glbs=None, locs=builtins.__xonsh__.ctx\n )\n rtn = True\n except SyntaxError:\n rtn = False\n except Exception:\n rtn = True\n return rtn\n\n\n@Condition\ndef tab_insert_indent():\n \"\"\"Check if should insert indent instead of starting autocompletion.\n Checks if there are only whitespaces before the cursor - if so indent\n should be inserted, otherwise autocompletion.\n\n \"\"\"\n before_cursor = get_app().current_buffer.document.current_line_before_cursor\n\n return bool(before_cursor.isspace())\n\n\n@Condition\ndef beginning_of_line():\n \"\"\"Check if cursor is at beginning of a line other than the first line in a\n multiline document\n \"\"\"\n app = get_app()\n before_cursor = app.current_buffer.document.current_line_before_cursor\n\n return bool(\n len(before_cursor) == 0 and not app.current_buffer.document.on_first_line\n )\n\n\n@Condition\ndef end_of_line():\n \"\"\"Check if cursor is at the end of a line other than the last line in a\n multiline document\n \"\"\"\n d = get_app().current_buffer.document\n at_end = d.is_cursor_at_the_end_of_line\n last_line = d.is_cursor_at_the_end\n\n return bool(at_end and not last_line)\n\n\n@Condition\ndef should_confirm_completion():\n \"\"\"Check if completion needs confirmation\"\"\"\n return (\n builtins.__xonsh__.env.get(\"COMPLETIONS_CONFIRM\")\n and get_app().current_buffer.complete_state\n )\n\n\n# Copied from prompt-toolkit's key_binding/bindings/basic.py\n@Condition\ndef ctrl_d_condition():\n \"\"\"Ctrl-D binding is only active when the default buffer is selected and\n empty.\n \"\"\"\n if builtins.__xonsh__.env.get(\"IGNOREEOF\"):\n return False\n else:\n app = get_app()\n buffer_name = app.current_buffer.name\n\n return buffer_name == DEFAULT_BUFFER and not app.current_buffer.text\n\n\n@Condition\ndef autopair_condition():\n \"\"\"Check if XONSH_AUTOPAIR is set\"\"\"\n return builtins.__xonsh__.env.get(\"XONSH_AUTOPAIR\", False)\n\n\n@Condition\ndef whitespace_or_bracket_before():\n \"\"\"Check if there is whitespace or an opening\n bracket to the left of the cursor\"\"\"\n d = get_app().current_buffer.document\n return bool(\n d.cursor_position == 0\n or d.char_before_cursor.isspace()\n or d.char_before_cursor in \"([{\"\n )\n\n\n@Condition\ndef whitespace_or_bracket_after():\n \"\"\"Check if there is whitespace or a closing\n bracket to the right of the cursor\"\"\"\n d = get_app().current_buffer.document\n return bool(\n d.is_cursor_at_the_end_of_line\n or d.current_char.isspace()\n or d.current_char in \")]}\"\n )\n\n\ndef load_xonsh_bindings(key_bindings):\n \"\"\"\n Load custom key bindings.\n \"\"\"\n handle = key_bindings.add\n has_selection = HasSelection()\n insert_mode = ViInsertMode() | EmacsInsertMode()\n\n @handle(Keys.Tab, filter=tab_insert_indent)\n def insert_indent(event):\n \"\"\"\n If there are only whitespaces before current cursor position insert\n indent instead of autocompleting.\n \"\"\"\n env = builtins.__xonsh__.env\n event.cli.current_buffer.insert_text(env.get(\"INDENT\"))\n\n @handle(Keys.ControlX, Keys.ControlE, filter=~has_selection)\n def open_editor(event):\n \"\"\" Open current buffer in editor \"\"\"\n event.current_buffer.open_in_editor(event.cli)\n\n @handle(Keys.BackTab, filter=insert_mode)\n def insert_literal_tab(event):\n \"\"\" Insert literal tab on Shift+Tab instead of autocompleting \"\"\"\n b = event.current_buffer\n if b.complete_state:\n b.complete_previous()\n else:\n env = builtins.__xonsh__.env\n event.cli.current_buffer.insert_text(env.get(\"INDENT\"))\n\n @handle(\"(\", filter=autopair_condition & whitespace_or_bracket_after)\n def insert_right_parens(event):\n event.cli.current_buffer.insert_text(\"(\")\n event.cli.current_buffer.insert_text(\")\", move_cursor=False)\n\n @handle(\")\", filter=autopair_condition)\n def overwrite_right_parens(event):\n buffer = event.cli.current_buffer\n if buffer.document.current_char == \")\":\n buffer.cursor_position += 1\n else:\n buffer.insert_text(\")\")\n\n @handle(\"[\", filter=autopair_condition & whitespace_or_bracket_after)\n def insert_right_bracket(event):\n event.cli.current_buffer.insert_text(\"[\")\n event.cli.current_buffer.insert_text(\"]\", move_cursor=False)\n\n @handle(\"]\", filter=autopair_condition)\n def overwrite_right_bracket(event):\n buffer = event.cli.current_buffer\n\n if buffer.document.current_char == \"]\":\n buffer.cursor_position += 1\n else:\n buffer.insert_text(\"]\")\n\n @handle(\"{\", filter=autopair_condition & whitespace_or_bracket_after)\n def insert_right_brace(event):\n event.cli.current_buffer.insert_text(\"{\")\n event.cli.current_buffer.insert_text(\"}\", move_cursor=False)\n\n @handle(\"}\", filter=autopair_condition)\n def overwrite_right_brace(event):\n buffer = event.cli.current_buffer\n\n if buffer.document.current_char == \"}\":\n buffer.cursor_position += 1\n else:\n buffer.insert_text(\"}\")\n\n @handle(\"'\", filter=autopair_condition)\n def insert_right_quote(event):\n buffer = event.cli.current_buffer\n\n if buffer.document.current_char == \"'\":\n buffer.cursor_position += 1\n elif whitespace_or_bracket_before() and whitespace_or_bracket_after():\n buffer.insert_text(\"'\")\n buffer.insert_text(\"'\", move_cursor=False)\n else:\n buffer.insert_text(\"'\")\n\n @handle('\"', filter=autopair_condition)\n def insert_right_double_quote(event):\n buffer = event.cli.current_buffer\n\n if buffer.document.current_char == '\"':\n buffer.cursor_position += 1\n elif whitespace_or_bracket_before() and whitespace_or_bracket_after():\n buffer.insert_text('\"')\n buffer.insert_text('\"', move_cursor=False)\n else:\n buffer.insert_text('\"')\n\n @handle(Keys.Backspace, filter=autopair_condition)\n def delete_brackets_or_quotes(event):\n \"\"\"Delete empty pair of brackets or quotes\"\"\"\n buffer = event.cli.current_buffer\n before = buffer.document.char_before_cursor\n after = buffer.document.current_char\n\n if any(\n [before == b and after == a for (b, a) in [\"()\", \"[]\", \"{}\", \"''\", '\"\"']]\n ):\n buffer.delete(1)\n\n buffer.delete_before_cursor(1)\n\n @handle(Keys.ControlD, filter=ctrl_d_condition)\n def call_exit_alias(event):\n \"\"\"Use xonsh exit function\"\"\"\n b = event.cli.current_buffer\n b.validate_and_handle()\n xonsh_exit([])\n\n @handle(Keys.ControlJ, filter=IsMultiline())\n @handle(Keys.ControlM, filter=IsMultiline())\n def multiline_carriage_return(event):\n \"\"\" Wrapper around carriage_return multiline parser \"\"\"\n b = event.cli.current_buffer\n carriage_return(b, event.cli)\n\n @handle(Keys.ControlJ, filter=should_confirm_completion)\n @handle(Keys.ControlM, filter=should_confirm_completion)\n def enter_confirm_completion(event):\n \"\"\"Ignore (confirm completion)\"\"\"\n event.current_buffer.complete_state = None\n\n @handle(Keys.Escape, filter=should_confirm_completion)\n def esc_cancel_completion(event):\n \"\"\"Use to cancel completion\"\"\"\n event.cli.current_buffer.cancel_completion()\n\n @handle(Keys.Escape, Keys.ControlJ)\n def execute_block_now(event):\n \"\"\"Execute a block of text irrespective of cursor position\"\"\"\n b = event.cli.current_buffer\n b.validate_and_handle()\n\n @handle(Keys.Left, filter=beginning_of_line)\n def wrap_cursor_back(event):\n \"\"\"Move cursor to end of previous line unless at beginning of\n document\n \"\"\"\n b = event.cli.current_buffer\n b.cursor_up(count=1)\n relative_end_index = b.document.get_end_of_line_position()\n b.cursor_right(count=relative_end_index)\n\n @handle(Keys.Right, filter=end_of_line)\n def wrap_cursor_forward(event):\n \"\"\"Move cursor to beginning of next line unless at end of document\"\"\"\n b = event.cli.current_buffer\n relative_begin_index = b.document.get_start_of_line_position()\n b.cursor_left(count=abs(relative_begin_index))\n b.cursor_down(count=1)\n\n @handle(Keys.ControlM, filter=IsSearching())\n @handle(Keys.ControlJ, filter=IsSearching())\n def accept_search(event):\n search.accept_search()\n" }, { "path": "xonsh/ptk2/shell.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"The prompt_toolkit based xonsh shell.\"\"\"\nimport sys\nimport builtins\nfrom types import MethodType\n\nfrom xonsh.events import events\nfrom xonsh.base_shell import BaseShell\nfrom xonsh.shell import transform_command\nfrom xonsh.tools import print_exception, carriage_return\nfrom xonsh.platform import HAS_PYGMENTS, ON_WINDOWS\nfrom xonsh.style_tools import partial_color_tokenize, _TokenType, DEFAULT_STYLE_DICT\nfrom xonsh.lazyimps import pygments, pyghooks, winutils\nfrom xonsh.pygments_cache import get_all_styles\nfrom xonsh.ptk2.history import PromptToolkitHistory, _cust_history_matches\nfrom xonsh.ptk2.completer import PromptToolkitCompleter\nfrom xonsh.ptk2.key_bindings import load_xonsh_bindings\n\nfrom prompt_toolkit.auto_suggest import AutoSuggestFromHistory\nfrom prompt_toolkit.lexers import PygmentsLexer\nfrom prompt_toolkit.enums import EditingMode\nfrom prompt_toolkit.key_binding import KeyBindings\nfrom prompt_toolkit.history import ThreadedHistory\nfrom prompt_toolkit.shortcuts import print_formatted_text as ptk_print\nfrom prompt_toolkit.shortcuts import CompleteStyle\nfrom prompt_toolkit.shortcuts.prompt import PromptSession\nfrom prompt_toolkit.formatted_text import PygmentsTokens\nfrom prompt_toolkit.styles import merge_styles, Style\nfrom prompt_toolkit.styles.pygments import (\n style_from_pygments_cls,\n style_from_pygments_dict,\n)\n\n\nToken = _TokenType()\n\nevents.transmogrify(\"on_ptk_create\", \"LoadEvent\")\nevents.doc(\n \"on_ptk_create\",\n \"\"\"\non_ptk_create(prompter: PromptSession, history: PromptToolkitHistory, completer: PromptToolkitCompleter, bindings: KeyBindings) ->\n\nFired after prompt toolkit has been initialized\n\"\"\",\n)\n\n\nclass PromptToolkit2Shell(BaseShell):\n \"\"\"The xonsh shell for prompt_toolkit v2.\"\"\"\n\n completion_displays_to_styles = {\n \"multi\": CompleteStyle.MULTI_COLUMN,\n \"single\": CompleteStyle.COLUMN,\n \"readline\": CompleteStyle.READLINE_LIKE,\n \"none\": None,\n }\n\n def __init__(self, **kwargs):\n super().__init__(**kwargs)\n if ON_WINDOWS:\n winutils.enable_virtual_terminal_processing()\n self._first_prompt = True\n self.history = ThreadedHistory(PromptToolkitHistory())\n self.prompter = PromptSession(history=self.history)\n self.pt_completer = PromptToolkitCompleter(self.completer, self.ctx, self)\n self.key_bindings = KeyBindings()\n load_xonsh_bindings(self.key_bindings)\n # Store original `_history_matches` in case we need to restore it\n self._history_matches_orig = self.prompter.default_buffer._history_matches\n # This assumes that PromptToolkit2Shell is a singleton\n events.on_ptk_create.fire(\n prompter=self.prompter,\n history=self.history,\n completer=self.pt_completer,\n bindings=self.key_bindings,\n )\n\n def singleline(\n self, auto_suggest=None, enable_history_search=True, multiline=True, **kwargs\n ):\n \"\"\"Reads a single line of input from the shell. The store_in_history\n kwarg flags whether the input should be stored in PTK's in-memory\n history.\n \"\"\"\n events.on_pre_prompt.fire()\n env = builtins.__xonsh__.env\n mouse_support = env.get(\"MOUSE_SUPPORT\")\n auto_suggest = auto_suggest if env.get(\"AUTO_SUGGEST\") else None\n completions_display = env.get(\"COMPLETIONS_DISPLAY\")\n complete_style = self.completion_displays_to_styles[completions_display]\n\n complete_while_typing = env.get(\"UPDATE_COMPLETIONS_ON_KEYPRESS\")\n if complete_while_typing:\n # PTK requires history search to be none when completing while typing\n enable_history_search = False\n if HAS_PYGMENTS:\n self.styler.style_name = env.get(\"XONSH_COLOR_STYLE\")\n completer = None if completions_display == \"none\" else self.pt_completer\n\n if env.get(\"UPDATE_PROMPT_ON_KEYPRESS\"):\n get_prompt_tokens = self.prompt_tokens\n get_rprompt_tokens = self.rprompt_tokens\n get_bottom_toolbar_tokens = self.bottom_toolbar_tokens\n else:\n get_prompt_tokens = self.prompt_tokens()\n get_rprompt_tokens = self.rprompt_tokens()\n get_bottom_toolbar_tokens = self.bottom_toolbar_tokens()\n\n if env.get(\"VI_MODE\"):\n editing_mode = EditingMode.VI\n else:\n editing_mode = EditingMode.EMACS\n\n if env.get(\"XONSH_HISTORY_MATCH_ANYWHERE\"):\n self.prompter.default_buffer._history_matches = MethodType(\n _cust_history_matches, self.prompter.default_buffer\n )\n elif (\n self.prompter.default_buffer._history_matches\n is not self._history_matches_orig\n ):\n self.prompter.default_buffer._history_matches = self._history_matches_orig\n\n prompt_args = {\n \"mouse_support\": mouse_support,\n \"auto_suggest\": auto_suggest,\n \"message\": get_prompt_tokens,\n \"rprompt\": get_rprompt_tokens,\n \"bottom_toolbar\": get_bottom_toolbar_tokens,\n \"completer\": completer,\n \"multiline\": multiline,\n \"editing_mode\": editing_mode,\n \"prompt_continuation\": self.continuation_tokens,\n \"enable_history_search\": enable_history_search,\n \"reserve_space_for_menu\": 10,\n \"key_bindings\": self.key_bindings,\n \"complete_style\": complete_style,\n \"complete_while_typing\": complete_while_typing,\n \"include_default_pygments_style\": False,\n }\n if builtins.__xonsh__.env.get(\"COLOR_INPUT\"):\n style = style_from_pygments_dict(DEFAULT_STYLE_DICT)\n\n prompt_args[\"style\"] = style\n\n style_overrides_env = env.get(\"PTK_STYLE_OVERRIDES\")\n if style_overrides_env:\n try:\n style_overrides = Style.from_dict(style_overrides_env)\n prompt_args[\"style\"] = merge_styles([style, style_overrides])\n except (AttributeError, TypeError, ValueError):\n print_exception()\n\n line = self.prompter.prompt(**prompt_args)\n events.on_post_prompt.fire()\n return line\n\n def _push(self, line):\n \"\"\"Pushes a line onto the buffer and compiles the code in a way that\n enables multiline input.\n \"\"\"\n code = None\n self.buffer.append(line)\n if self.need_more_lines:\n return None, code\n src = \"\".join(self.buffer)\n src = transform_command(src)\n try:\n code = self.execer.compile(src, mode=\"single\", glbs=self.ctx, locs=None)\n self.reset_buffer()\n except Exception: # pylint: disable=broad-except\n self.reset_buffer()\n print_exception()\n return src, None\n return src, code\n\n def cmdloop(self, intro=None):\n \"\"\"Enters a loop that reads and execute input from user.\"\"\"\n if intro:\n print(intro)\n auto_suggest = AutoSuggestFromHistory()\n self.push = self._push\n while not builtins.__xonsh__.exit:\n try:\n line = self.singleline(auto_suggest=auto_suggest)\n if not line:\n self.emptyline()\n else:\n line = self.precmd(line)\n self.default(line)\n except (KeyboardInterrupt, SystemExit):\n self.reset_buffer()\n except EOFError:\n if builtins.__xonsh__.env.get(\"IGNOREEOF\"):\n print('Use \"exit\" to leave the shell.', file=sys.stderr)\n else:\n break\n\n def prompt_tokens(self):\n \"\"\"Returns a list of (token, str) tuples for the current prompt.\"\"\"\n p = builtins.__xonsh__.env.get(\"PROMPT\")\n try:\n p = self.prompt_formatter(p)\n except Exception: # pylint: disable=broad-except\n print_exception()\n toks = partial_color_tokenize(p)\n if self._first_prompt:\n carriage_return()\n self._first_prompt = False\n self.settitle()\n return PygmentsTokens(toks)\n\n def rprompt_tokens(self):\n \"\"\"Returns a list of (token, str) tuples for the current right\n prompt.\n \"\"\"\n p = builtins.__xonsh__.env.get(\"RIGHT_PROMPT\")\n # self.prompt_formatter does handle empty strings properly,\n # but this avoids descending into it in the common case of\n # $RIGHT_PROMPT == ''.\n if isinstance(p, str) and len(p) == 0:\n return []\n try:\n p = self.prompt_formatter(p)\n except Exception: # pylint: disable=broad-except\n print_exception()\n toks = partial_color_tokenize(p)\n return PygmentsTokens(toks)\n\n def bottom_toolbar_tokens(self):\n \"\"\"Returns a list of (token, str) tuples for the current bottom\n toolbar.\n \"\"\"\n p = builtins.__xonsh__.env.get(\"BOTTOM_TOOLBAR\")\n if not p:\n return\n try:\n p = self.prompt_formatter(p)\n except Exception: # pylint: disable=broad-except\n print_exception()\n toks = partial_color_tokenize(p)\n return PygmentsTokens(toks)\n\n def continuation_tokens(self, width, line_number, is_soft_wrap=False):\n \"\"\"Displays dots in multiline prompt\"\"\"\n if is_soft_wrap:\n return \"\"\n width = width - 1\n dots = builtins.__xonsh__.env.get(\"MULTILINE_PROMPT\")\n dots = dots() if callable(dots) else dots\n if dots is None:\n return [(Token, \" \" * (width + 1))]\n basetoks = self.format_color(dots)\n baselen = sum(len(t[1]) for t in basetoks)\n if baselen == 0:\n return [(Token, \" \" * (width + 1))]\n toks = basetoks * (width // baselen)\n n = width % baselen\n count = 0\n for tok in basetoks:\n slen = len(tok[1])\n newcount = slen + count\n if slen == 0:\n continue\n elif newcount <= n:\n toks.append(tok)\n else:\n toks.append((tok[0], tok[1][: n - count]))\n count = newcount\n if n <= count:\n break\n toks.append((Token, \" \")) # final space\n return PygmentsTokens(toks)\n\n def format_color(self, string, hide=False, force_string=False, **kwargs):\n \"\"\"Formats a color string using Pygments. This, therefore, returns\n a list of (Token, str) tuples. If force_string is set to true, though,\n this will return a color formatted string.\n \"\"\"\n tokens = partial_color_tokenize(string)\n if force_string and HAS_PYGMENTS:\n env = builtins.__xonsh__.env\n self.styler.style_name = env.get(\"XONSH_COLOR_STYLE\")\n proxy_style = pyghooks.xonsh_style_proxy(self.styler)\n formatter = pyghooks.XonshTerminal256Formatter(style=proxy_style)\n s = pygments.format(tokens, formatter)\n return s\n elif force_string:\n print(\"To force colorization of string, install Pygments\")\n return tokens\n else:\n return tokens\n\n def print_color(self, string, end=\"\\n\", **kwargs):\n \"\"\"Prints a color string using prompt-toolkit color management.\"\"\"\n if isinstance(string, str):\n tokens = partial_color_tokenize(string)\n else:\n # assume this is a list of (Token, str) tuples and just print\n tokens = string\n tokens = PygmentsTokens(tokens)\n if HAS_PYGMENTS:\n env = builtins.__xonsh__.env\n self.styler.style_name = env.get(\"XONSH_COLOR_STYLE\")\n proxy_style = style_from_pygments_cls(\n pyghooks.xonsh_style_proxy(self.styler)\n )\n else:\n proxy_style = style_from_pygments_dict(DEFAULT_STYLE_DICT)\n ptk_print(\n tokens, style=proxy_style, end=end, include_default_pygments_style=False\n )\n\n def color_style_names(self):\n \"\"\"Returns an iterable of all available style names.\"\"\"\n if not HAS_PYGMENTS:\n return [\"For other xonsh styles, please install pygments\"]\n return get_all_styles()\n\n def color_style(self):\n \"\"\"Returns the current color map.\"\"\"\n if not HAS_PYGMENTS:\n return DEFAULT_STYLE_DICT\n env = builtins.__xonsh__.env\n self.styler.style_name = env.get(\"XONSH_COLOR_STYLE\")\n return self.styler.styles\n\n def restore_tty_sanity(self):\n \"\"\"An interface for resetting the TTY stdin mode. This is highly\n dependent on the shell backend. Also it is mostly optional since\n it only affects ^Z backgrounding behaviour.\n \"\"\"\n # PTK does not seem to need any specialization here. However,\n # if it does for some reason in the future...\n # The following writes an ANSI escape sequence that sends the cursor\n # to the end of the line. This has the effect of restoring ECHO mode.\n # See http://unix.stackexchange.com/a/108014/129048 for more details.\n # This line can also be replaced by os.system(\"stty sane\"), as per\n # http://stackoverflow.com/questions/19777129/interactive-python-interpreter-run-in-background#comment29421919_19778355\n # However, it is important to note that not termios-based solution\n # seems to work. My guess is that this is because termios restoration\n # needs to be performed by the subprocess itself. This fix is important\n # when subprocesses don't properly restore the terminal attributes,\n # like Python in interactive mode. Also note that the sequences \"\\033M\"\n # and \"\\033E\" seem to work too, but these are technically VT100 codes.\n # I used the more primitive ANSI sequence to maximize compatibility.\n # -scopatz 2017-01-28\n # if not ON_POSIX:\n # return\n # sys.stdout.write('\\033[9999999C\\n')\n" }, { "path": "xonsh/pyghooks.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"Hooks for pygments syntax highlighting.\"\"\"\nimport os\nimport re\nimport sys\nimport builtins\nfrom collections import ChainMap\nfrom collections.abc import MutableMapping\n\nfrom pygments.lexer import inherit, bygroups, include\nfrom pygments.lexers.agile import PythonLexer\nfrom pygments.token import (\n Keyword,\n Name,\n Comment,\n String,\n Error,\n Number,\n Operator,\n Generic,\n Whitespace,\n Token,\n Punctuation,\n Text,\n)\nfrom pygments.style import Style\nimport pygments.util\n\nfrom xonsh.commands_cache import CommandsCache\nfrom xonsh.lazyasd import LazyObject, LazyDict, lazyobject\nfrom xonsh.tools import (\n ON_WINDOWS,\n intensify_colors_for_cmd_exe,\n ansicolors_to_ptk1_names,\n ANSICOLOR_NAMES_MAP,\n PTK_NEW_OLD_COLOR_MAP,\n hardcode_colors_for_win10,\n FORMATTER,\n)\n\nfrom xonsh.color_tools import (\n RE_BACKGROUND,\n BASE_XONSH_COLORS,\n make_palette,\n find_closest_color,\n)\nfrom xonsh.style_tools import norm_name\nfrom xonsh.lazyimps import terminal256\nfrom xonsh.platform import (\n os_environ,\n win_ansi_support,\n ptk_version_info,\n pygments_version_info,\n)\n\nfrom xonsh.pygments_cache import get_style_by_name\n\n\ndef _command_is_valid(cmd):\n try:\n cmd_abspath = os.path.abspath(os.path.expanduser(cmd))\n except (FileNotFoundError, OSError):\n return False\n return cmd in builtins.__xonsh__.commands_cache or (\n os.path.isfile(cmd_abspath) and os.access(cmd_abspath, os.X_OK)\n )\n\n\ndef _command_is_autocd(cmd):\n if not builtins.__xonsh__.env.get(\"AUTO_CD\", False):\n return False\n try:\n cmd_abspath = os.path.abspath(os.path.expanduser(cmd))\n except (FileNotFoundError, OSError):\n return False\n return os.path.isdir(cmd_abspath)\n\n\ndef subproc_cmd_callback(_, match):\n \"\"\"Yield Builtin token if match contains valid command,\n otherwise fallback to fallback lexer.\n \"\"\"\n cmd = match.group()\n yield match.start(), Name.Builtin if _command_is_valid(cmd) else Error, cmd\n\n\ndef subproc_arg_callback(_, match):\n \"\"\"Check if match contains valid path\"\"\"\n text = match.group()\n try:\n ispath = os.path.exists(os.path.expanduser(text))\n except (FileNotFoundError, OSError):\n ispath = False\n yield (match.start(), Name.Constant if ispath else Text, text)\n\n\nCOMMAND_TOKEN_RE = r'[^=\\s\\[\\]{}()$\"\\'`<&|;!]+(?=\\s|$|\\)|\\]|\\}|!)'\n\n\nclass XonshLexer(PythonLexer):\n \"\"\"Xonsh console lexer for pygments.\"\"\"\n\n name = \"Xonsh lexer\"\n aliases = [\"xonsh\", \"xsh\"]\n filenames = [\"*.xsh\", \"*xonshrc\"]\n\n def __init__(self, *args, **kwargs):\n # If the lexer is loaded as a pygment plugin, we have to mock\n # __xonsh__.env and __xonsh__.commands_cache\n if not hasattr(builtins, \"__xonsh__\"):\n from argparse import Namespace\n\n setattr(builtins, \"__xonsh__\", Namespace())\n if not hasattr(builtins.__xonsh__, \"env\"):\n setattr(builtins.__xonsh__, \"env\", {})\n if ON_WINDOWS:\n pathext = os_environ.get(\"PATHEXT\", [\".EXE\", \".BAT\", \".CMD\"])\n builtins.__xonsh__.env[\"PATHEXT\"] = pathext.split(os.pathsep)\n if not hasattr(builtins.__xonsh__, \"commands_cache\"):\n setattr(builtins.__xonsh__, \"commands_cache\", CommandsCache())\n _ = builtins.__xonsh__.commands_cache.all_commands # NOQA\n super().__init__(*args, **kwargs)\n\n tokens = {\n \"mode_switch_brackets\": [\n (r\"(\\$)(\\{)\", bygroups(Keyword, Punctuation), \"py_curly_bracket\"),\n (r\"(@)(\\()\", bygroups(Keyword, Punctuation), \"py_bracket\"),\n (\n r\"([\\!\\$])(\\()\",\n bygroups(Keyword, Punctuation),\n (\"subproc_bracket\", \"subproc_start\"),\n ),\n (\n r\"(@\\$)(\\()\",\n bygroups(Keyword, Punctuation),\n (\"subproc_bracket\", \"subproc_start\"),\n ),\n (\n r\"([\\!\\$])(\\[)\",\n bygroups(Keyword, Punctuation),\n (\"subproc_square_bracket\", \"subproc_start\"),\n ),\n (r\"(g?)(`)\", bygroups(String.Affix, String.Backtick), \"backtick_re\"),\n ],\n \"subproc_bracket\": [(r\"\\)\", Punctuation, \"#pop\"), include(\"subproc\")],\n \"subproc_square_bracket\": [(r\"\\]\", Punctuation, \"#pop\"), include(\"subproc\")],\n \"py_bracket\": [(r\"\\)\", Punctuation, \"#pop\"), include(\"root\")],\n \"py_curly_bracket\": [(r\"\\}\", Punctuation, \"#pop\"), include(\"root\")],\n \"backtick_re\": [\n (r\"[\\.\\^\\$\\*\\+\\?\\[\\]\\|]\", String.Regex),\n (r\"({[0-9]+}|{[0-9]+,[0-9]+})\\??\", String.Regex),\n (r\"\\\\([0-9]+|[AbBdDsSwWZabfnrtuUvx\\\\])\", String.Escape),\n (r\"`\", String.Backtick, \"#pop\"),\n (r\"[^`\\.\\^\\$\\*\\+\\?\\[\\]\\|]+\", String.Backtick),\n ],\n \"root\": [\n (r\"\\?\", Keyword),\n (r\"(?<=\\w)!\", Keyword),\n (r\"\\$\\w+\", Name.Variable),\n (r\"\\(\", Punctuation, \"py_bracket\"),\n (r\"\\{\", Punctuation, \"py_curly_bracket\"),\n include(\"mode_switch_brackets\"),\n inherit,\n ],\n \"subproc_start\": [\n (r\"\\s+\", Whitespace),\n (COMMAND_TOKEN_RE, subproc_cmd_callback, \"#pop\"),\n (r\"\", Whitespace, \"#pop\"),\n ],\n \"subproc\": [\n include(\"mode_switch_brackets\"),\n (r\"&&|\\|\\|\", Operator, \"subproc_start\"),\n (r'\"(\\\\\\\\|\\\\[0-7]+|\\\\.|[^\"\\\\])*\"', String.Double),\n (r\"'(\\\\\\\\|\\\\[0-7]+|\\\\.|[^'\\\\])*'\", String.Single),\n (r\"(?<=\\w|\\s)!\", Keyword, \"subproc_macro\"),\n (r\"^!\", Keyword, \"subproc_macro\"),\n (r\";\", Punctuation, \"subproc_start\"),\n (r\"&|=\", Punctuation),\n (r\"\\|\", Punctuation, \"subproc_start\"),\n (r\"\\s+\", Text),\n (r'[^=\\s\\[\\]{}()$\"\\'`<&|;]+', subproc_arg_callback),\n (r\"<\", Text),\n (r\"\\$\\w+\", Name.Variable),\n ],\n \"subproc_macro\": [\n (r\"(\\s*)([^\\n]+)\", bygroups(Whitespace, String)),\n (r\"\", Whitespace, \"#pop\"),\n ],\n }\n\n def get_tokens_unprocessed(self, text):\n \"\"\"Check first command, then call super.get_tokens_unprocessed\n with root or subproc state\"\"\"\n start = 0\n state = (\"root\",)\n m = re.match(r\"(\\s*)({})\".format(COMMAND_TOKEN_RE), text)\n if m is not None:\n yield m.start(1), Whitespace, m.group(1)\n cmd = m.group(2)\n cmd_is_valid = _command_is_valid(cmd)\n cmd_is_autocd = _command_is_autocd(cmd)\n\n if cmd_is_valid or cmd_is_autocd:\n yield (m.start(2), Name.Builtin if cmd_is_valid else Name.Constant, cmd)\n start = m.end(2)\n state = (\"subproc\",)\n\n for i, t, v in super().get_tokens_unprocessed(text[start:], state):\n yield i + start, t, v\n\n\nclass XonshConsoleLexer(XonshLexer):\n \"\"\"Xonsh console lexer for pygments.\"\"\"\n\n name = \"Xonsh console lexer\"\n aliases = [\"xonshcon\"]\n filenames = []\n\n tokens = {\n \"root\": [\n (r\"^(>>>|\\.\\.\\.) \", Generic.Prompt),\n (r\"\\n(>>>|\\.\\.\\.)\", Generic.Prompt),\n (r\"\\n(?![>.][>.][>.] )([^\\n]*)\", Generic.Output),\n (r\"\\n(?![>.][>.][>.] )(.*?)$\", Generic.Output),\n inherit,\n ]\n }\n\n\n#\n# Colors and Styles\n#\n\nColor = Token.Color # alias to new color token namespace\n\n\ndef color_by_name(name, fg=None, bg=None):\n \"\"\"Converts a color name to a color token, foreground name,\n and background name. Will take into consideration current foreground\n and background colors, if provided.\n\n Parameters\n ----------\n name : str\n Color name.\n fg : str, optional\n Foreground color name.\n bg : str, optional\n Background color name.\n\n Returns\n -------\n tok : Token\n Pygments Token.Color subclass\n fg : str or None\n New computed foreground color name.\n bg : str or None\n New computed background color name.\n \"\"\"\n name = name.upper()\n if name == \"NO_COLOR\":\n return Color.NO_COLOR, None, None\n m = RE_BACKGROUND.search(name)\n if m is None: # must be foreground color\n fg = norm_name(name)\n else:\n bg = norm_name(name)\n # assemble token\n if fg is None and bg is None:\n tokname = \"NO_COLOR\"\n elif fg is None:\n tokname = bg\n elif bg is None:\n tokname = fg\n else:\n tokname = fg + \"__\" + bg\n tok = getattr(Color, tokname)\n return tok, fg, bg\n\n\ndef code_by_name(name, styles):\n \"\"\"Converts a token name into a pygments-style color code.\n\n Parameters\n ----------\n name : str\n Color token name.\n styles : Mapping\n Mapping for looking up non-hex colors\n\n Returns\n -------\n code : str\n Pygments style color code.\n \"\"\"\n fg, _, bg = name.lower().partition(\"__\")\n if fg.startswith(\"background_\"):\n fg, bg = bg, fg\n codes = []\n # foreground color\n if len(fg) == 0:\n pass\n elif \"hex\" in fg:\n for p in fg.split(\"_\"):\n codes.append(\"#\" + p[3:] if p.startswith(\"hex\") else p)\n else:\n fgtok = getattr(Color, fg.upper())\n if fgtok in styles:\n codes.append(styles[fgtok])\n else:\n codes += fg.split(\"_\")\n # background color\n if len(bg) == 0:\n pass\n elif bg.startswith(\"background_hex\"):\n codes.append(\"bg:#\" + bg[14:])\n else:\n bgtok = getattr(Color, bg.upper())\n if bgtok in styles:\n codes.append(styles[bgtok])\n else:\n codes.append(bg.replace(\"background_\", \"bg:\"))\n code = \" \".join(codes)\n return code\n\n\ndef partial_color_tokenize(template):\n \"\"\"Tokenizes a template string containing colors. Will return a list\n of tuples mapping the token to the string which has that color.\n These sub-strings maybe templates themselves.\n \"\"\"\n if builtins.__xonsh__.shell is not None:\n styles = __xonsh__.shell.shell.styler.styles\n else:\n styles = None\n color = Color.NO_COLOR\n try:\n toks, color = _partial_color_tokenize_main(template, styles)\n except Exception:\n toks = [(Color.NO_COLOR, template)]\n if styles is not None:\n styles[color] # ensure color is available\n return toks\n\n\ndef _partial_color_tokenize_main(template, styles):\n bopen = \"{\"\n bclose = \"}\"\n colon = \":\"\n expl = \"!\"\n color = Color.NO_COLOR\n fg = bg = None\n value = \"\"\n toks = []\n for literal, field, spec, conv in FORMATTER.parse(template):\n if field is None:\n value += literal\n elif field in KNOWN_COLORS or \"#\" in field:\n value += literal\n next_color, fg, bg = color_by_name(field, fg, bg)\n if next_color is not color:\n if len(value) > 0:\n toks.append((color, value))\n if styles is not None:\n styles[color] # ensure color is available\n color = next_color\n value = \"\"\n elif field is not None:\n parts = [literal, bopen, field]\n if conv is not None and len(conv) > 0:\n parts.append(expl)\n parts.append(conv)\n if spec is not None and len(spec) > 0:\n parts.append(colon)\n parts.append(spec)\n parts.append(bclose)\n value += \"\".join(parts)\n else:\n value += literal\n toks.append((color, value))\n return toks, color\n\n\nclass CompoundColorMap(MutableMapping):\n \"\"\"Looks up color tokens by name, potentially generating the value\n from the lookup.\n \"\"\"\n\n def __init__(self, styles, *args, **kwargs):\n self.styles = styles\n self.colors = dict(*args, **kwargs)\n\n def __getitem__(self, key):\n if key in self.colors:\n return self.colors[key]\n if key in self.styles:\n value = self.styles[key]\n self[key] = value\n return value\n if key is Color:\n raise KeyError\n pre, _, name = str(key).rpartition(\".\")\n if pre != \"Token.Color\":\n raise KeyError\n value = code_by_name(name, self.styles)\n self[key] = value\n return value\n\n def __setitem__(self, key, value):\n self.colors[key] = value\n\n def __delitem__(self, key):\n del self.colors[key]\n\n def __iter__(self):\n yield from self.colors.keys()\n\n def __len__(self):\n return len(self.colors)\n\n\nclass XonshStyle(Style):\n \"\"\"A xonsh pygments style that will dispatch to the correct color map\n by using a ChainMap. The style_name property may be used to reset\n the current style.\n \"\"\"\n\n def __init__(self, style_name=\"default\"):\n \"\"\"\n Parameters\n ----------\n style_name : str, optional\n The style name to initialize with.\n \"\"\"\n self.trap = {} # for trapping custom colors set by user\n self._smap = {}\n self._style_name = \"\"\n self.style_name = style_name\n super().__init__()\n\n @property\n def style_name(self):\n return self._style_name\n\n @style_name.setter\n def style_name(self, value):\n if self._style_name == value:\n return\n if value not in STYLES:\n try: # loading style dynamically\n pygments_style_by_name(value)\n except Exception:\n print(\n \"Could not find style {0!r}, using default\".format(value),\n file=sys.stderr,\n )\n value = \"default\"\n builtins.__xonsh__.env[\"XONSH_COLOR_STYLE\"] = value\n cmap = STYLES[value]\n if value == \"default\":\n self._smap = XONSH_BASE_STYLE.copy()\n else:\n try:\n self._smap = get_style_by_name(value)().styles.copy()\n except (ImportError, pygments.util.ClassNotFound):\n self._smap = XONSH_BASE_STYLE.copy()\n compound = CompoundColorMap(ChainMap(self.trap, cmap, PTK_STYLE, self._smap))\n self.styles = ChainMap(self.trap, cmap, PTK_STYLE, self._smap, compound)\n self._style_name = value\n # Convert new ansicolor names to old PTK1 names\n # Can be remvoed when PTK1 support is dropped.\n if builtins.__xonsh__.shell.shell_type != \"prompt_toolkit2\":\n for smap in [self.trap, cmap, PTK_STYLE, self._smap]:\n smap.update(ansicolors_to_ptk1_names(smap))\n if ON_WINDOWS and \"prompt_toolkit\" in builtins.__xonsh__.shell.shell_type:\n self.enhance_colors_for_cmd_exe()\n\n @style_name.deleter\n def style_name(self):\n self._style_name = \"\"\n\n def enhance_colors_for_cmd_exe(self):\n \"\"\" Enhance colors when using cmd.exe on windows.\n When using the default style all blue and dark red colors\n are changed to CYAN and intense red.\n \"\"\"\n env = builtins.__xonsh__.env\n # Ensure we are not using ConEmu or Visual Stuio Code\n if \"CONEMUANSI\" in env or \"VSCODE_PID\" in env:\n return\n if env.get(\"INTENSIFY_COLORS_ON_WIN\", False):\n if win_ansi_support():\n newcolors = hardcode_colors_for_win10(self.styles)\n else:\n newcolors = intensify_colors_for_cmd_exe(self.styles)\n self.trap.update(newcolors)\n\n\ndef xonsh_style_proxy(styler):\n \"\"\"Factory for a proxy class to a xonsh style.\"\"\"\n # Monky patch pygments' list of known ansi colors\n # with the new ansi color names used by PTK2\n # Can be removed once pygment names get fixed.\n pygments.style.ansicolors.update(ANSICOLOR_NAMES_MAP)\n\n class XonshStyleProxy(Style):\n \"\"\"Simple proxy class to fool prompt toolkit.\"\"\"\n\n target = styler\n styles = styler.styles\n\n def __new__(cls, *args, **kwargs):\n return cls.target\n\n return XonshStyleProxy\n\n\nPTK_STYLE = LazyObject(\n lambda: {\n Token.Menu.Completions: \"bg:ansigray ansiblack\",\n Token.Menu.Completions.Completion: \"\",\n Token.Menu.Completions.Completion.Current: \"bg:ansibrightblack ansiwhite\",\n Token.Scrollbar: \"bg:ansibrightblack\",\n Token.Scrollbar.Button: \"bg:ansiblack\",\n Token.Scrollbar.Arrow: \"bg:ansiblack ansiwhite bold\",\n Token.AutoSuggestion: \"ansibrightblack\",\n Token.Aborted: \"ansibrightblack\",\n },\n globals(),\n \"PTK_STYLE\",\n)\n\n\nXONSH_BASE_STYLE = LazyObject(\n lambda: {\n Whitespace: \"ansigray\",\n Comment: \"underline ansicyan\",\n Comment.Preproc: \"underline ansiyellow\",\n Keyword: \"bold ansigreen\",\n Keyword.Pseudo: \"nobold\",\n Keyword.Type: \"nobold ansired\",\n Operator: \"ansibrightblack\",\n Operator.Word: \"bold ansimagenta\",\n Name.Builtin: \"ansigreen\",\n Name.Function: \"ansibrightblue\",\n Name.Class: \"bold ansibrightblue\",\n Name.Namespace: \"bold ansibrightblue\",\n Name.Exception: \"bold ansibrightred\",\n Name.Variable: \"ansiblue\",\n Name.Constant: \"ansired\",\n Name.Label: \"ansibrightyellow\",\n Name.Entity: \"bold ansigray\",\n Name.Attribute: \"ansibrightyellow\",\n Name.Tag: \"bold ansigreen\",\n Name.Decorator: \"ansibrightmagenta\",\n String: \"ansibrightred\",\n String.Doc: \"underline\",\n String.Interpol: \"bold ansimagenta\",\n String.Escape: \"bold ansiyellow\",\n String.Regex: \"ansimagenta\",\n String.Symbol: \"ansiyellow\",\n String.Other: \"ansigreen\",\n Number: \"ansibrightblack\",\n Generic.Heading: \"bold ansiblue\",\n Generic.Subheading: \"bold ansimagenta\",\n Generic.Deleted: \"ansired\",\n Generic.Inserted: \"ansibrightgreen\",\n Generic.Error: \"bold ansibrightred\",\n Generic.Emph: \"underline\",\n Generic.Prompt: \"bold ansiblue\",\n Generic.Output: \"ansiblue\",\n Generic.Traceback: \"ansiblue\",\n Error: \"ansibrightred\",\n },\n globals(),\n \"XONSH_BASE_STYLE\",\n)\n\n\nKNOWN_COLORS = LazyObject(\n lambda: frozenset(\n [\n \"BACKGROUND_BLACK\",\n \"BACKGROUND_BLUE\",\n \"BACKGROUND_CYAN\",\n \"BACKGROUND_GREEN\",\n \"BACKGROUND_INTENSE_BLACK\",\n \"BACKGROUND_INTENSE_BLUE\",\n \"BACKGROUND_INTENSE_CYAN\",\n \"BACKGROUND_INTENSE_GREEN\",\n \"BACKGROUND_INTENSE_PURPLE\",\n \"BACKGROUND_INTENSE_RED\",\n \"BACKGROUND_INTENSE_WHITE\",\n \"BACKGROUND_INTENSE_YELLOW\",\n \"BACKGROUND_PURPLE\",\n \"BACKGROUND_RED\",\n \"BACKGROUND_WHITE\",\n \"BACKGROUND_YELLOW\",\n \"BLACK\",\n \"BLUE\",\n \"BOLD_BLACK\",\n \"BOLD_BLUE\",\n \"BOLD_CYAN\",\n \"BOLD_GREEN\",\n \"BOLD_INTENSE_BLACK\",\n \"BOLD_INTENSE_BLUE\",\n \"BOLD_INTENSE_CYAN\",\n \"BOLD_INTENSE_GREEN\",\n \"BOLD_INTENSE_PURPLE\",\n \"BOLD_INTENSE_RED\",\n \"BOLD_INTENSE_WHITE\",\n \"BOLD_INTENSE_YELLOW\",\n \"BOLD_PURPLE\",\n \"BOLD_RED\",\n \"BOLD_UNDERLINE_BLACK\",\n \"BOLD_UNDERLINE_BLUE\",\n \"BOLD_UNDERLINE_CYAN\",\n \"BOLD_UNDERLINE_GREEN\",\n \"BOLD_UNDERLINE_INTENSE_BLACK\",\n \"BOLD_UNDERLINE_INTENSE_BLUE\",\n \"BOLD_UNDERLINE_INTENSE_CYAN\",\n \"BOLD_UNDERLINE_INTENSE_GREEN\",\n \"BOLD_UNDERLINE_INTENSE_PURPLE\",\n \"BOLD_UNDERLINE_INTENSE_RED\",\n \"BOLD_UNDERLINE_INTENSE_WHITE\",\n \"BOLD_UNDERLINE_INTENSE_YELLOW\",\n \"BOLD_UNDERLINE_PURPLE\",\n \"BOLD_UNDERLINE_RED\",\n \"BOLD_UNDERLINE_WHITE\",\n \"BOLD_UNDERLINE_YELLOW\",\n \"BOLD_WHITE\",\n \"BOLD_YELLOW\",\n \"CYAN\",\n \"GREEN\",\n \"INTENSE_BLACK\",\n \"INTENSE_BLUE\",\n \"INTENSE_CYAN\",\n \"INTENSE_GREEN\",\n \"INTENSE_PURPLE\",\n \"INTENSE_RED\",\n \"INTENSE_WHITE\",\n \"INTENSE_YELLOW\",\n \"NO_COLOR\",\n \"PURPLE\",\n \"RED\",\n \"UNDERLINE_BLACK\",\n \"UNDERLINE_BLUE\",\n \"UNDERLINE_CYAN\",\n \"UNDERLINE_GREEN\",\n \"UNDERLINE_INTENSE_BLACK\",\n \"UNDERLINE_INTENSE_BLUE\",\n \"UNDERLINE_INTENSE_CYAN\",\n \"UNDERLINE_INTENSE_GREEN\",\n \"UNDERLINE_INTENSE_PURPLE\",\n \"UNDERLINE_INTENSE_RED\",\n \"UNDERLINE_INTENSE_WHITE\",\n \"UNDERLINE_INTENSE_YELLOW\",\n \"UNDERLINE_PURPLE\",\n \"UNDERLINE_RED\",\n \"UNDERLINE_WHITE\",\n \"UNDERLINE_YELLOW\",\n \"WHITE\",\n \"YELLOW\",\n ]\n ),\n globals(),\n \"KNOWN_COLORS\",\n)\n\n\ndef _expand_style(cmap):\n \"\"\"Expands a style in order to more quickly make color map changes.\"\"\"\n for key, val in list(cmap.items()):\n if key is Color.NO_COLOR:\n continue\n _, _, key = str(key).rpartition(\".\")\n cmap[getattr(Color, \"BOLD_\" + key)] = \"bold \" + val\n cmap[getattr(Color, \"UNDERLINE_\" + key)] = \"underline \" + val\n cmap[getattr(Color, \"BOLD_UNDERLINE_\" + key)] = \"bold underline \" + val\n if val == \"noinherit\":\n cmap[getattr(Color, \"BACKGROUND_\" + key)] = val\n else:\n cmap[getattr(Color, \"BACKGROUND_\" + key)] = \"bg:\" + val\n\n\ndef _bw_style():\n style = {\n Color.BLACK: \"noinherit\",\n Color.BLUE: \"noinherit\",\n Color.CYAN: \"noinherit\",\n Color.GREEN: \"noinherit\",\n Color.INTENSE_BLACK: \"noinherit\",\n Color.INTENSE_BLUE: \"noinherit\",\n Color.INTENSE_CYAN: \"noinherit\",\n Color.INTENSE_GREEN: \"noinherit\",\n Color.INTENSE_PURPLE: \"noinherit\",\n Color.INTENSE_RED: \"noinherit\",\n Color.INTENSE_WHITE: \"noinherit\",\n Color.INTENSE_YELLOW: \"noinherit\",\n Color.NO_COLOR: \"noinherit\",\n Color.PURPLE: \"noinherit\",\n Color.RED: \"noinherit\",\n Color.WHITE: \"noinherit\",\n Color.YELLOW: \"noinherit\",\n }\n _expand_style(style)\n return style\n\n\ndef _default_style():\n style = {\n Color.BLACK: \"ansiblack\",\n Color.BLUE: \"ansiblue\",\n Color.CYAN: \"ansicyan\",\n Color.GREEN: \"ansigreen\",\n Color.INTENSE_BLACK: \"ansibrightblack\",\n Color.INTENSE_BLUE: \"ansibrightblue\",\n Color.INTENSE_CYAN: \"ansibrightcyan\",\n Color.INTENSE_GREEN: \"ansibrightgreen\",\n Color.INTENSE_PURPLE: \"ansibrightmagenta\",\n Color.INTENSE_RED: \"ansibrightred\",\n Color.INTENSE_WHITE: \"ansiwhite\",\n Color.INTENSE_YELLOW: \"ansibrightyellow\",\n Color.NO_COLOR: \"noinherit\",\n Color.PURPLE: \"ansimagenta\",\n Color.RED: \"ansired\",\n Color.WHITE: \"ansigray\",\n Color.YELLOW: \"ansiyellow\",\n }\n _expand_style(style)\n return style\n\n\ndef _monokai_style():\n style = {\n Color.BLACK: \"#1e0010\",\n Color.BLUE: \"#6666ef\",\n Color.CYAN: \"#66d9ef\",\n Color.GREEN: \"#2ee22e\",\n Color.INTENSE_BLACK: \"#5e5e5e\",\n Color.INTENSE_BLUE: \"#2626d7\",\n Color.INTENSE_CYAN: \"#2ed9d9\",\n Color.INTENSE_GREEN: \"#a6e22e\",\n Color.INTENSE_PURPLE: \"#ae81ff\",\n Color.INTENSE_RED: \"#f92672\",\n Color.INTENSE_WHITE: \"#f8f8f2\",\n Color.INTENSE_YELLOW: \"#e6db74\",\n Color.NO_COLOR: \"noinherit\",\n Color.PURPLE: \"#960050\",\n Color.RED: \"#AF0000\",\n Color.WHITE: \"#d7d7d7\",\n Color.YELLOW: \"#e2e22e\",\n }\n _expand_style(style)\n return style\n\n\n######################################\n# Auto-generated below this line #\n######################################\ndef _algol_style():\n style = {\n Color.BLACK: \"#666\",\n Color.BLUE: \"#666\",\n Color.CYAN: \"#666\",\n Color.GREEN: \"#666\",\n Color.INTENSE_BLACK: \"#666\",\n Color.INTENSE_BLUE: \"#888\",\n Color.INTENSE_CYAN: \"#888\",\n Color.INTENSE_GREEN: \"#888\",\n Color.INTENSE_PURPLE: \"#888\",\n Color.INTENSE_RED: \"#FF0000\",\n Color.INTENSE_WHITE: \"#888\",\n Color.INTENSE_YELLOW: \"#888\",\n Color.NO_COLOR: \"noinherit\",\n Color.PURPLE: \"#666\",\n Color.RED: \"#FF0000\",\n Color.WHITE: \"#888\",\n Color.YELLOW: \"#FF0000\",\n }\n _expand_style(style)\n return style\n\n\ndef _algol_nu_style():\n style = {\n Color.BLACK: \"#666\",\n Color.BLUE: \"#666\",\n Color.CYAN: \"#666\",\n Color.GREEN: \"#666\",\n Color.INTENSE_BLACK: \"#666\",\n Color.INTENSE_BLUE: \"#888\",\n Color.INTENSE_CYAN: \"#888\",\n Color.INTENSE_GREEN: \"#888\",\n Color.INTENSE_PURPLE: \"#888\",\n Color.INTENSE_RED: \"#FF0000\",\n Color.INTENSE_WHITE: \"#888\",\n Color.INTENSE_YELLOW: \"#888\",\n Color.NO_COLOR: \"noinherit\",\n Color.PURPLE: \"#666\",\n Color.RED: \"#FF0000\",\n Color.WHITE: \"#888\",\n Color.YELLOW: \"#FF0000\",\n }\n _expand_style(style)\n return style\n\n\ndef _autumn_style():\n style = {\n Color.BLACK: \"#000080\",\n Color.BLUE: \"#0000aa\",\n Color.CYAN: \"#00aaaa\",\n Color.GREEN: \"#00aa00\",\n Color.INTENSE_BLACK: \"#555555\",\n Color.INTENSE_BLUE: \"#1e90ff\",\n Color.INTENSE_CYAN: \"#1e90ff\",\n Color.INTENSE_GREEN: \"#4c8317\",\n Color.INTENSE_PURPLE: \"#FAA\",\n Color.INTENSE_RED: \"#aa5500\",\n Color.INTENSE_WHITE: \"#bbbbbb\",\n Color.INTENSE_YELLOW: \"#FAA\",\n Color.NO_COLOR: \"noinherit\",\n Color.PURPLE: \"#800080\",\n Color.RED: \"#aa0000\",\n Color.WHITE: \"#aaaaaa\",\n Color.YELLOW: \"#aa5500\",\n }\n _expand_style(style)\n return style\n\n\ndef _borland_style():\n style = {\n Color.BLACK: \"#000000\",\n Color.BLUE: \"#000080\",\n Color.CYAN: \"#008080\",\n Color.GREEN: \"#008800\",\n Color.INTENSE_BLACK: \"#555555\",\n Color.INTENSE_BLUE: \"#0000FF\",\n Color.INTENSE_CYAN: \"#ddffdd\",\n Color.INTENSE_GREEN: \"#888888\",\n Color.INTENSE_PURPLE: \"#e3d2d2\",\n Color.INTENSE_RED: \"#FF0000\",\n Color.INTENSE_WHITE: \"#ffdddd\",\n Color.INTENSE_YELLOW: \"#e3d2d2\",\n Color.NO_COLOR: \"noinherit\",\n Color.PURPLE: \"#800080\",\n Color.RED: \"#aa0000\",\n Color.WHITE: \"#aaaaaa\",\n Color.YELLOW: \"#a61717\",\n }\n _expand_style(style)\n return style\n\n\ndef _colorful_style():\n style = {\n Color.BLACK: \"#000\",\n Color.BLUE: \"#00C\",\n Color.CYAN: \"#0e84b5\",\n Color.GREEN: \"#00A000\",\n Color.INTENSE_BLACK: \"#555\",\n Color.INTENSE_BLUE: \"#33B\",\n Color.INTENSE_CYAN: \"#bbbbbb\",\n Color.INTENSE_GREEN: \"#888\",\n Color.INTENSE_PURPLE: \"#FAA\",\n Color.INTENSE_RED: \"#D42\",\n Color.INTENSE_WHITE: \"#fff0ff\",\n Color.INTENSE_YELLOW: \"#FAA\",\n Color.NO_COLOR: \"noinherit\",\n Color.PURPLE: \"#800080\",\n Color.RED: \"#A00000\",\n Color.WHITE: \"#bbbbbb\",\n Color.YELLOW: \"#A60\",\n }\n _expand_style(style)\n return style\n\n\ndef _emacs_style():\n style = {\n Color.BLACK: \"#008000\",\n Color.BLUE: \"#000080\",\n Color.CYAN: \"#04D\",\n Color.GREEN: \"#00A000\",\n Color.INTENSE_BLACK: \"#666666\",\n Color.INTENSE_BLUE: \"#04D\",\n Color.INTENSE_CYAN: \"#bbbbbb\",\n Color.INTENSE_GREEN: \"#00BB00\",\n Color.INTENSE_PURPLE: \"#AA22FF\",\n Color.INTENSE_RED: \"#D2413A\",\n Color.INTENSE_WHITE: \"#bbbbbb\",\n Color.INTENSE_YELLOW: \"#bbbbbb\",\n Color.NO_COLOR: \"noinherit\",\n Color.PURPLE: \"#800080\",\n Color.RED: \"#A00000\",\n Color.WHITE: \"#bbbbbb\",\n Color.YELLOW: \"#BB6622\",\n }\n _expand_style(style)\n return style\n\n\ndef _friendly_style():\n style = {\n Color.BLACK: \"#007020\",\n Color.BLUE: \"#000080\",\n Color.CYAN: \"#0e84b5\",\n Color.GREEN: \"#00A000\",\n Color.INTENSE_BLACK: \"#555555\",\n Color.INTENSE_BLUE: \"#70a0d0\",\n Color.INTENSE_CYAN: \"#60add5\",\n Color.INTENSE_GREEN: \"#40a070\",\n Color.INTENSE_PURPLE: \"#bb60d5\",\n Color.INTENSE_RED: \"#d55537\",\n Color.INTENSE_WHITE: \"#fff0f0\",\n Color.INTENSE_YELLOW: \"#bbbbbb\",\n Color.NO_COLOR: \"noinherit\",\n Color.PURPLE: \"#800080\",\n Color.RED: \"#A00000\",\n Color.WHITE: \"#bbbbbb\",\n Color.YELLOW: \"#c65d09\",\n }\n _expand_style(style)\n return style\n\n\ndef _fruity_style():\n style = {\n Color.BLACK: \"#0f140f\",\n Color.BLUE: \"#0086d2\",\n Color.CYAN: \"#0086d2\",\n Color.GREEN: \"#008800\",\n Color.INTENSE_BLACK: \"#444444\",\n Color.INTENSE_BLUE: \"#0086f7\",\n Color.INTENSE_CYAN: \"#0086f7\",\n Color.INTENSE_GREEN: \"#888888\",\n Color.INTENSE_PURPLE: \"#ff0086\",\n Color.INTENSE_RED: \"#fb660a\",\n Color.INTENSE_WHITE: \"#ffffff\",\n Color.INTENSE_YELLOW: \"#cdcaa9\",\n Color.NO_COLOR: \"noinherit\",\n Color.PURPLE: \"#ff0086\",\n Color.RED: \"#ff0007\",\n Color.WHITE: \"#cdcaa9\",\n Color.YELLOW: \"#fb660a\",\n }\n _expand_style(style)\n return style\n\n\ndef _igor_style():\n style = {\n Color.BLACK: \"#009C00\",\n Color.BLUE: \"#0000FF\",\n Color.CYAN: \"#007575\",\n Color.GREEN: \"#009C00\",\n Color.INTENSE_BLACK: \"#007575\",\n Color.INTENSE_BLUE: \"#0000FF\",\n Color.INTENSE_CYAN: \"#007575\",\n Color.INTENSE_GREEN: \"#009C00\",\n Color.INTENSE_PURPLE: \"#CC00A3\",\n Color.INTENSE_RED: \"#C34E00\",\n Color.INTENSE_WHITE: \"#CC00A3\",\n Color.INTENSE_YELLOW: \"#C34E00\",\n Color.NO_COLOR: \"noinherit\",\n Color.PURPLE: \"#CC00A3\",\n Color.RED: \"#C34E00\",\n Color.WHITE: \"#CC00A3\",\n Color.YELLOW: \"#C34E00\",\n }\n _expand_style(style)\n return style\n\n\ndef _lovelace_style():\n style = {\n Color.BLACK: \"#444444\",\n Color.BLUE: \"#2838b0\",\n Color.CYAN: \"#289870\",\n Color.GREEN: \"#388038\",\n Color.INTENSE_BLACK: \"#666666\",\n Color.INTENSE_BLUE: \"#2838b0\",\n Color.INTENSE_CYAN: \"#888888\",\n Color.INTENSE_GREEN: \"#289870\",\n Color.INTENSE_PURPLE: \"#a848a8\",\n Color.INTENSE_RED: \"#b83838\",\n Color.INTENSE_WHITE: \"#888888\",\n Color.INTENSE_YELLOW: \"#a89028\",\n Color.NO_COLOR: \"noinherit\",\n Color.PURPLE: \"#a848a8\",\n Color.RED: \"#c02828\",\n Color.WHITE: \"#888888\",\n Color.YELLOW: \"#b85820\",\n }\n _expand_style(style)\n return style\n\n\ndef _manni_style():\n style = {\n Color.BLACK: \"#000000\",\n Color.BLUE: \"#000099\",\n Color.CYAN: \"#009999\",\n Color.GREEN: \"#00CC00\",\n Color.INTENSE_BLACK: \"#555555\",\n Color.INTENSE_BLUE: \"#9999FF\",\n Color.INTENSE_CYAN: \"#00CCFF\",\n Color.INTENSE_GREEN: \"#99CC66\",\n Color.INTENSE_PURPLE: \"#CC00FF\",\n Color.INTENSE_RED: \"#FF6600\",\n Color.INTENSE_WHITE: \"#FFCCCC\",\n Color.INTENSE_YELLOW: \"#FFCC33\",\n Color.NO_COLOR: \"noinherit\",\n Color.PURPLE: \"#CC00FF\",\n Color.RED: \"#AA0000\",\n Color.WHITE: \"#AAAAAA\",\n Color.YELLOW: \"#CC3300\",\n }\n _expand_style(style)\n return style\n\n\ndef _murphy_style():\n style = {\n Color.BLACK: \"#000\",\n Color.BLUE: \"#000080\",\n Color.CYAN: \"#0e84b5\",\n Color.GREEN: \"#00A000\",\n Color.INTENSE_BLACK: \"#555\",\n Color.INTENSE_BLUE: \"#66f\",\n Color.INTENSE_CYAN: \"#5ed\",\n Color.INTENSE_GREEN: \"#5ed\",\n Color.INTENSE_PURPLE: \"#e9e\",\n Color.INTENSE_RED: \"#f84\",\n Color.INTENSE_WHITE: \"#eee\",\n Color.INTENSE_YELLOW: \"#fc8\",\n Color.NO_COLOR: \"noinherit\",\n Color.PURPLE: \"#800080\",\n Color.RED: \"#A00000\",\n Color.WHITE: \"#bbbbbb\",\n Color.YELLOW: \"#c65d09\",\n }\n _expand_style(style)\n return style\n\n\ndef _native_style():\n style = {\n Color.BLACK: \"#520000\",\n Color.BLUE: \"#3677a9\",\n Color.CYAN: \"#24909d\",\n Color.GREEN: \"#589819\",\n Color.INTENSE_BLACK: \"#666666\",\n Color.INTENSE_BLUE: \"#447fcf\",\n Color.INTENSE_CYAN: \"#40ffff\",\n Color.INTENSE_GREEN: \"#6ab825\",\n Color.INTENSE_PURPLE: \"#e3d2d2\",\n Color.INTENSE_RED: \"#cd2828\",\n Color.INTENSE_WHITE: \"#ffffff\",\n Color.INTENSE_YELLOW: \"#ed9d13\",\n Color.NO_COLOR: \"noinherit\",\n Color.PURPLE: \"#666666\",\n Color.RED: \"#a61717\",\n Color.WHITE: \"#aaaaaa\",\n Color.YELLOW: \"#a61717\",\n }\n _expand_style(style)\n return style\n\n\ndef _paraiso_dark_style():\n style = {\n Color.BLACK: \"#776e71\",\n Color.BLUE: \"#815ba4\",\n Color.CYAN: \"#06b6ef\",\n Color.GREEN: \"#48b685\",\n Color.INTENSE_BLACK: \"#776e71\",\n Color.INTENSE_BLUE: \"#815ba4\",\n Color.INTENSE_CYAN: \"#5bc4bf\",\n Color.INTENSE_GREEN: \"#48b685\",\n Color.INTENSE_PURPLE: \"#e7e9db\",\n Color.INTENSE_RED: \"#ef6155\",\n Color.INTENSE_WHITE: \"#e7e9db\",\n Color.INTENSE_YELLOW: \"#fec418\",\n Color.NO_COLOR: \"noinherit\",\n Color.PURPLE: \"#815ba4\",\n Color.RED: \"#ef6155\",\n Color.WHITE: \"#5bc4bf\",\n Color.YELLOW: \"#f99b15\",\n }\n _expand_style(style)\n return style\n\n\ndef _paraiso_light_style():\n style = {\n Color.BLACK: \"#2f1e2e\",\n Color.BLUE: \"#2f1e2e\",\n Color.CYAN: \"#06b6ef\",\n Color.GREEN: \"#48b685\",\n Color.INTENSE_BLACK: \"#2f1e2e\",\n Color.INTENSE_BLUE: \"#815ba4\",\n Color.INTENSE_CYAN: \"#5bc4bf\",\n Color.INTENSE_GREEN: \"#48b685\",\n Color.INTENSE_PURPLE: \"#815ba4\",\n Color.INTENSE_RED: \"#ef6155\",\n Color.INTENSE_WHITE: \"#5bc4bf\",\n Color.INTENSE_YELLOW: \"#fec418\",\n Color.NO_COLOR: \"noinherit\",\n Color.PURPLE: \"#815ba4\",\n Color.RED: \"#2f1e2e\",\n Color.WHITE: \"#8d8687\",\n Color.YELLOW: \"#f99b15\",\n }\n _expand_style(style)\n return style\n\n\ndef _pastie_style():\n style = {\n Color.BLACK: \"#000000\",\n Color.BLUE: \"#0000DD\",\n Color.CYAN: \"#0066bb\",\n Color.GREEN: \"#008800\",\n Color.INTENSE_BLACK: \"#555555\",\n Color.INTENSE_BLUE: \"#3333bb\",\n Color.INTENSE_CYAN: \"#ddffdd\",\n Color.INTENSE_GREEN: \"#22bb22\",\n Color.INTENSE_PURPLE: \"#e3d2d2\",\n Color.INTENSE_RED: \"#dd7700\",\n Color.INTENSE_WHITE: \"#fff0ff\",\n Color.INTENSE_YELLOW: \"#e3d2d2\",\n Color.NO_COLOR: \"noinherit\",\n Color.PURPLE: \"#bb0066\",\n Color.RED: \"#aa0000\",\n Color.WHITE: \"#bbbbbb\",\n Color.YELLOW: \"#aa6600\",\n }\n _expand_style(style)\n return style\n\n\ndef _perldoc_style():\n style = {\n Color.BLACK: \"#000080\",\n Color.BLUE: \"#000080\",\n Color.CYAN: \"#1e889b\",\n Color.GREEN: \"#00aa00\",\n Color.INTENSE_BLACK: \"#555555\",\n Color.INTENSE_BLUE: \"#B452CD\",\n Color.INTENSE_CYAN: \"#bbbbbb\",\n Color.INTENSE_GREEN: \"#228B22\",\n Color.INTENSE_PURPLE: \"#B452CD\",\n Color.INTENSE_RED: \"#CD5555\",\n Color.INTENSE_WHITE: \"#e3d2d2\",\n Color.INTENSE_YELLOW: \"#e3d2d2\",\n Color.NO_COLOR: \"noinherit\",\n Color.PURPLE: \"#8B008B\",\n Color.RED: \"#aa0000\",\n Color.WHITE: \"#a7a7a7\",\n Color.YELLOW: \"#cb6c20\",\n }\n _expand_style(style)\n return style\n\n\ndef _rrt_style():\n style = {\n Color.BLACK: \"#ff0000\",\n Color.BLUE: \"#87ceeb\",\n Color.CYAN: \"#87ceeb\",\n Color.GREEN: \"#00ff00\",\n Color.INTENSE_BLACK: \"#87ceeb\",\n Color.INTENSE_BLUE: \"#87ceeb\",\n Color.INTENSE_CYAN: \"#7fffd4\",\n Color.INTENSE_GREEN: \"#00ff00\",\n Color.INTENSE_PURPLE: \"#ee82ee\",\n Color.INTENSE_RED: \"#ff0000\",\n Color.INTENSE_WHITE: \"#e5e5e5\",\n Color.INTENSE_YELLOW: \"#eedd82\",\n Color.NO_COLOR: \"noinherit\",\n Color.PURPLE: \"#ee82ee\",\n Color.RED: \"#ff0000\",\n Color.WHITE: \"#87ceeb\",\n Color.YELLOW: \"#ff0000\",\n }\n _expand_style(style)\n return style\n\n\ndef _tango_style():\n style = {\n Color.BLACK: \"#000000\",\n Color.BLUE: \"#0000cf\",\n Color.CYAN: \"#3465a4\",\n Color.GREEN: \"#00A000\",\n Color.INTENSE_BLACK: \"#204a87\",\n Color.INTENSE_BLUE: \"#5c35cc\",\n Color.INTENSE_CYAN: \"#f8f8f8\",\n Color.INTENSE_GREEN: \"#4e9a06\",\n Color.INTENSE_PURPLE: \"#f8f8f8\",\n Color.INTENSE_RED: \"#ef2929\",\n Color.INTENSE_WHITE: \"#f8f8f8\",\n Color.INTENSE_YELLOW: \"#c4a000\",\n Color.NO_COLOR: \"noinherit\",\n Color.PURPLE: \"#800080\",\n Color.RED: \"#a40000\",\n Color.WHITE: \"#f8f8f8\",\n Color.YELLOW: \"#8f5902\",\n }\n _expand_style(style)\n return style\n\n\ndef _trac_style():\n style = {\n Color.BLACK: \"#000000\",\n Color.BLUE: \"#000080\",\n Color.CYAN: \"#009999\",\n Color.GREEN: \"#808000\",\n Color.INTENSE_BLACK: \"#555555\",\n Color.INTENSE_BLUE: \"#445588\",\n Color.INTENSE_CYAN: \"#ddffdd\",\n Color.INTENSE_GREEN: \"#999988\",\n Color.INTENSE_PURPLE: \"#e3d2d2\",\n Color.INTENSE_RED: \"#bb8844\",\n Color.INTENSE_WHITE: \"#ffdddd\",\n Color.INTENSE_YELLOW: \"#e3d2d2\",\n Color.NO_COLOR: \"noinherit\",\n Color.PURPLE: \"#800080\",\n Color.RED: \"#aa0000\",\n Color.WHITE: \"#aaaaaa\",\n Color.YELLOW: \"#808000\",\n }\n _expand_style(style)\n return style\n\n\ndef _vim_style():\n style = {\n Color.BLACK: \"#000080\",\n Color.BLUE: \"#000080\",\n Color.CYAN: \"#00cdcd\",\n Color.GREEN: \"#00cd00\",\n Color.INTENSE_BLACK: \"#666699\",\n Color.INTENSE_BLUE: \"#3399cc\",\n Color.INTENSE_CYAN: \"#00cdcd\",\n Color.INTENSE_GREEN: \"#00cd00\",\n Color.INTENSE_PURPLE: \"#cd00cd\",\n Color.INTENSE_RED: \"#FF0000\",\n Color.INTENSE_WHITE: \"#cccccc\",\n Color.INTENSE_YELLOW: \"#cdcd00\",\n Color.NO_COLOR: \"noinherit\",\n Color.PURPLE: \"#cd00cd\",\n Color.RED: \"#cd0000\",\n Color.WHITE: \"#cccccc\",\n Color.YELLOW: \"#cd0000\",\n }\n _expand_style(style)\n return style\n\n\ndef _vs_style():\n style = {\n Color.BLACK: \"#008000\",\n Color.BLUE: \"#0000ff\",\n Color.CYAN: \"#2b91af\",\n Color.GREEN: \"#008000\",\n Color.INTENSE_BLACK: \"#2b91af\",\n Color.INTENSE_BLUE: \"#2b91af\",\n Color.INTENSE_CYAN: \"#2b91af\",\n Color.INTENSE_GREEN: \"#2b91af\",\n Color.INTENSE_PURPLE: \"#2b91af\",\n Color.INTENSE_RED: \"#FF0000\",\n Color.INTENSE_WHITE: \"#2b91af\",\n Color.INTENSE_YELLOW: \"#2b91af\",\n Color.NO_COLOR: \"noinherit\",\n Color.PURPLE: \"#a31515\",\n Color.RED: \"#a31515\",\n Color.WHITE: \"#2b91af\",\n Color.YELLOW: \"#a31515\",\n }\n _expand_style(style)\n return style\n\n\ndef _xcode_style():\n style = {\n Color.BLACK: \"#000000\",\n Color.BLUE: \"#1C01CE\",\n Color.CYAN: \"#3F6E75\",\n Color.GREEN: \"#177500\",\n Color.INTENSE_BLACK: \"#3F6E75\",\n Color.INTENSE_BLUE: \"#2300CE\",\n Color.INTENSE_CYAN: \"#3F6E75\",\n Color.INTENSE_GREEN: \"#3F6E75\",\n Color.INTENSE_PURPLE: \"#A90D91\",\n Color.INTENSE_RED: \"#C41A16\",\n Color.INTENSE_WHITE: \"#3F6E75\",\n Color.INTENSE_YELLOW: \"#836C28\",\n Color.NO_COLOR: \"noinherit\",\n Color.PURPLE: \"#A90D91\",\n Color.RED: \"#C41A16\",\n Color.WHITE: \"#3F6E75\",\n Color.YELLOW: \"#836C28\",\n }\n _expand_style(style)\n return style\n\n\nSTYLES = LazyDict(\n {\n \"algol\": _algol_style,\n \"algol_nu\": _algol_nu_style,\n \"autumn\": _autumn_style,\n \"borland\": _borland_style,\n \"bw\": _bw_style,\n \"colorful\": _colorful_style,\n \"default\": _default_style,\n \"emacs\": _emacs_style,\n \"friendly\": _friendly_style,\n \"fruity\": _fruity_style,\n \"igor\": _igor_style,\n \"lovelace\": _lovelace_style,\n \"manni\": _manni_style,\n \"monokai\": _monokai_style,\n \"murphy\": _murphy_style,\n \"native\": _native_style,\n \"paraiso-dark\": _paraiso_dark_style,\n \"paraiso-light\": _paraiso_light_style,\n \"pastie\": _pastie_style,\n \"perldoc\": _perldoc_style,\n \"rrt\": _rrt_style,\n \"tango\": _tango_style,\n \"trac\": _trac_style,\n \"vim\": _vim_style,\n \"vs\": _vs_style,\n \"xcode\": _xcode_style,\n },\n globals(),\n \"STYLES\",\n)\n\ndel (\n _algol_style,\n _algol_nu_style,\n _autumn_style,\n _borland_style,\n _bw_style,\n _colorful_style,\n _default_style,\n _emacs_style,\n _friendly_style,\n _fruity_style,\n _igor_style,\n _lovelace_style,\n _manni_style,\n _monokai_style,\n _murphy_style,\n _native_style,\n _paraiso_dark_style,\n _paraiso_light_style,\n _pastie_style,\n _perldoc_style,\n _rrt_style,\n _tango_style,\n _trac_style,\n _vim_style,\n _vs_style,\n _xcode_style,\n)\n\n\n# dynamic styles\ndef make_pygments_style(palette):\n \"\"\"Makes a pygments style based on a color palette.\"\"\"\n global Color\n style = {getattr(Color, \"NO_COLOR\"): \"noinherit\"}\n for name, t in BASE_XONSH_COLORS.items():\n color = find_closest_color(t, palette)\n style[getattr(Color, name)] = \"#\" + color\n style[getattr(Color, \"BOLD_\" + name)] = \"bold #\" + color\n style[getattr(Color, \"UNDERLINE_\" + name)] = \"underline #\" + color\n style[getattr(Color, \"BOLD_UNDERLINE_\" + name)] = \"bold underline #\" + color\n style[getattr(Color, \"BACKGROUND_\" + name)] = \"bg:#\" + color\n return style\n\n\ndef pygments_style_by_name(name):\n \"\"\"Gets or makes a pygments color style by its name.\"\"\"\n if name in STYLES:\n return STYLES[name]\n pstyle = get_style_by_name(name)\n palette = make_palette(pstyle.styles.values())\n astyle = make_pygments_style(palette)\n STYLES[name] = astyle\n return astyle\n\n\ndef _monkey_patch_pygments_codes():\n \"\"\" Monky patch pygments' dict of console codes,\n with new color names\n \"\"\"\n import pygments.console\n\n if \"brightblack\" in pygments.console.codes:\n # Assume that colors are already fixed in pygments\n # for example when using pygments from source\n return\n\n if not getattr(pygments.console, \"_xonsh_patched\", False):\n patched_codes = {}\n for new, old in PTK_NEW_OLD_COLOR_MAP.items():\n if old in pygments.console.codes:\n patched_codes[new[1:]] = pygments.console.codes[old]\n pygments.console.codes.update(patched_codes)\n pygments.console._xonsh_patched = True\n\n\n#\n# Formatter\n#\n\n\n@lazyobject\ndef XonshTerminal256Formatter():\n\n if (\n ptk_version_info()\n and ptk_version_info() > (2, 0)\n and pygments_version_info()\n and (2, 2, 0) <= pygments_version_info() < (2, 4, 0)\n ):\n # Monky patch pygments' dict of console codes\n # with the new color names used by PTK2\n # Can be removed once pygment names get fixed.\n _monkey_patch_pygments_codes()\n\n class XonshTerminal256FormatterProxy(terminal256.Terminal256Formatter):\n \"\"\"Proxy class for xonsh terminal256 formatting that understands.\n xonsh color tokens.\n \"\"\"\n\n def __init__(self, *args, **kwargs):\n super().__init__(*args, **kwargs)\n # just keep the opening token for colors.\n color_names = set(map(str, Color.subtypes))\n for name, (opener, closer) in self.style_string.items():\n if name in color_names:\n self.style_string[name] = (opener, \"\")\n # special case NO_COLOR, because it is special.\n self.style_string[\"Token.Color.NO_COLOR\"] = (\"\\x1b[39m\", \"\")\n\n return XonshTerminal256FormatterProxy\n" }, { "path": "xonsh/pygments_cache.py", "content": "\"\"\"A fast, drop-in replacement for pygments ``get_*()`` and ``guess_*()`` funtions.\n\nThe following pygments API functions are currently supplied here::\n\n from pygments_cache import get_lexer_for_filename, guess_lexer_for_filename\n from pygments_cache import get_formatter_for_filename, get_formatter_by_name\n from pygments_cache import get_style_by_name, get_all_styles\n from pygments_cache import get_filter_by_name\n\nThe cache itself is stored at the location given by the ``$PYGMENTS_CACHE_FILE``\nenvironment variable, or by default at ``~/.local/share/pygments-cache/cache.py``.\nThe cache file is created on first use, if it does not already exist.\n\n\n\"\"\"\nimport os\nimport importlib\n\n\n# Global storage variables\n__version__ = \"0.1.1\"\nCACHE = None\nDEBUG = False\n\n\ndef _print_duplicate_message(duplicates):\n import sys\n\n for filename, vals in sorted(duplicates.items()):\n msg = \"for {0} ambiquity between:\\n \".format(filename)\n vals = [m + \":\" + c for m, c in vals]\n msg += \"\\n \".join(sorted(vals))\n print(msg, file=sys.stderr)\n\n\ndef _discover_lexers():\n import inspect\n from pygments.lexers import get_all_lexers, find_lexer_class\n\n # maps file extension (and names) to (module, classname) tuples\n default_exts = {\n # C / C++\n \".h\": (\"pygments.lexers.c_cpp\", \"CLexer\"),\n \".hh\": (\"pygments.lexers.c_cpp\", \"CppLexer\"),\n \".cp\": (\"pygments.lexers.c_cpp\", \"CppLexer\"),\n # python\n \".py\": (\"pygments.lexers.python\", \"Python3Lexer\"),\n \".pyw\": (\"pygments.lexers.python\", \"Python3Lexer\"),\n \".sc\": (\"pygments.lexers.python\", \"Python3Lexer\"),\n \".tac\": (\"pygments.lexers.python\", \"Python3Lexer\"),\n \"SConstruct\": (\"pygments.lexers.python\", \"Python3Lexer\"),\n \"SConscript\": (\"pygments.lexers.python\", \"Python3Lexer\"),\n \".sage\": (\"pygments.lexers.python\", \"Python3Lexer\"),\n \".pytb\": (\"pygments.lexers.python\", \"Python3TracebackLexer\"),\n # perl\n \".t\": (\"pygments.lexers.perl\", \"Perl6Lexer\"),\n \".pl\": (\"pygments.lexers.perl\", \"Perl6Lexer\"),\n \".pm\": (\"pygments.lexers.perl\", \"Perl6Lexer\"),\n # asm\n \".s\": (\"pygments.lexers.asm\", \"GasLexer\"),\n \".S\": (\"pygments.lexers.asm\", \"GasLexer\"),\n \".asm\": (\"pygments.lexers.asm\", \"NasmLexer\"),\n \".ASM\": (\"pygments.lexers.asm\", \"NasmLexer\"),\n # Antlr\n \".g\": (\"pygments.lexers.parsers\", \"AntlrCppLexer\"),\n \".G\": (\"pygments.lexers.parsers\", \"AntlrCppLexer\"),\n # XML\n \".xml\": (\"pygments.lexers.html\", \"XmlLexer\"),\n \".xsl\": (\"pygments.lexers.html\", \"XsltLexer\"),\n \".xslt\": (\"pygments.lexers.html\", \"XsltLexer\"),\n # ASP\n \".axd\": (\"pygments.lexers.dotnet\", \"CSharpAspxLexer\"),\n \".asax\": (\"pygments.lexers.dotnet\", \"CSharpAspxLexer\"),\n \".ascx\": (\"pygments.lexers.dotnet\", \"CSharpAspxLexer\"),\n \".ashx\": (\"pygments.lexers.dotnet\", \"CSharpAspxLexer\"),\n \".asmx\": (\"pygments.lexers.dotnet\", \"CSharpAspxLexer\"),\n \".aspx\": (\"pygments.lexers.dotnet\", \"CSharpAspxLexer\"),\n # misc\n \".b\": (\"pygments.lexers.esoteric\", \"BrainfuckLexer\"),\n \".j\": (\"pygments.lexers.jvm\", \"JasminLexer\"),\n \".m\": (\"pygments.lexers.matlab\", \"MatlabLexer\"),\n \".n\": (\"pygments.lexers.dotnet\", \"NemerleLexer\"),\n \".p\": (\"pygments.lexers.pawn\", \"PawnLexer\"),\n \".v\": (\"pygments.lexers.theorem\", \"CoqLexer\"),\n \".as\": (\"pygments.lexers.actionscript\", \"ActionScript3Lexer\"),\n \".fs\": (\"pygments.lexers.forth\", \"ForthLexer\"),\n \".hy\": (\"pygments.lexers.lisp\", \"HyLexer\"),\n \".ts\": (\"pygments.lexers.javascript\", \"TypeScriptLexer\"),\n \".rl\": (\"pygments.lexers.parsers\", \"RagelCppLexer\"),\n \".bas\": (\"pygments.lexers.basic\", \"QBasicLexer\"),\n \".bug\": (\"pygments.lexers.modeling\", \"BugsLexer\"),\n \".ecl\": (\"pygments.lexers.ecl\", \"ECLLexer\"),\n \".inc\": (\"pygments.lexers.php\", \"PhpLexer\"),\n \".inf\": (\"pygments.lexers.configs\", \"IniLexer\"),\n \".pro\": (\"pygments.lexers.prolog\", \"PrologLexer\"),\n \".sql\": (\"pygments.lexers.sql\", \"SqlLexer\"),\n \".txt\": (\"pygments.lexers.special\", \"TextLexer\"),\n \".html\": (\"pygments.lexers.html\", \"HtmlLexer\"),\n }\n exts = {}\n lexers = {\"exts\": exts}\n if DEBUG:\n from collections import defaultdict\n\n duplicates = defaultdict(set)\n for longname, aliases, filenames, mimetypes in get_all_lexers():\n cls = find_lexer_class(longname)\n mod = inspect.getmodule(cls)\n val = (mod.__name__, cls.__name__)\n for filename in filenames:\n if filename.startswith(\"*.\"):\n filename = filename[1:]\n if \"*\" in filename:\n continue\n if (\n DEBUG\n and filename in exts\n and exts[filename] != val\n and filename not in default_exts\n ):\n duplicates[filename].add(val)\n duplicates[filename].add(exts[filename])\n exts[filename] = val\n # remove some ambiquity\n exts.update(default_exts)\n # print duplicate message\n if DEBUG:\n _print_duplicate_message(duplicates)\n return lexers\n\n\ndef _discover_formatters():\n import inspect\n from pygments.formatters import get_all_formatters\n\n # maps file extension (and names) to (module, classname) tuples\n default_exts = {}\n exts = {}\n # maps formatter 'name' (not the class name) and alias to (module, classname) tuples\n default_names = {}\n names = {}\n formatters = {\"exts\": exts, \"names\": names}\n if DEBUG:\n from collections import defaultdict\n\n duplicates = defaultdict(set)\n for cls in get_all_formatters():\n mod = inspect.getmodule(cls)\n val = (mod.__name__, cls.__name__)\n # add extentions\n for filename in cls.filenames:\n if filename.startswith(\"*.\"):\n filename = filename[1:]\n if \"*\" in filename:\n continue\n if (\n DEBUG\n and filename in exts\n and exts[filename] != val\n and filename not in default_exts\n ):\n duplicates[filename].add(val)\n duplicates[filename].add(exts[filename])\n exts[filename] = val\n # add names and aliases\n names[cls.name] = val\n for alias in cls.aliases:\n if (\n DEBUG\n and alias in names\n and names[alias] != val\n and alias not in default_names\n ):\n duplicates[alias].add(val)\n duplicates[alias].add(names[alias])\n names[alias] = val\n # remove some ambiquity\n exts.update(default_exts)\n names.update(default_names)\n # print dumplicate message\n if DEBUG:\n _print_duplicate_message(duplicates)\n return formatters\n\n\ndef _discover_styles():\n import inspect\n from pygments.styles import get_all_styles, get_style_by_name\n\n # maps style 'name' (not the class name) and aliases to (module, classname) tuples\n default_names = {}\n names = {}\n styles = {\"names\": names}\n if DEBUG:\n from collections import defaultdict\n\n duplicates = defaultdict(set)\n for name in get_all_styles():\n cls = get_style_by_name(name)\n mod = inspect.getmodule(cls)\n val = (mod.__name__, cls.__name__)\n if DEBUG and name in names and names[name] != val and name not in default_names:\n duplicates[name].add(val)\n duplicates[name].add(names[name])\n names[name] = val\n # remove some ambiquity\n names.update(default_names)\n # print dumplicate message\n if DEBUG:\n _print_duplicate_message(duplicates)\n return styles\n\n\ndef _discover_filters():\n import inspect\n from pygments.filters import get_all_filters, get_filter_by_name\n\n # maps filter 'name' (not the class name) to (module, classname) tuples\n default_names = {}\n names = {}\n filters = {\"names\": names}\n if DEBUG:\n from collections import defaultdict\n\n duplicates = defaultdict(set)\n for name in get_all_filters():\n filter = get_filter_by_name(name)\n cls = type(filter)\n mod = inspect.getmodule(cls)\n val = (mod.__name__, cls.__name__)\n if DEBUG and name in names and names[name] != val and name not in default_names:\n duplicates[name].add(val)\n duplicates[name].add(names[name])\n names[name] = val\n # remove some ambiquity\n names.update(default_names)\n # print dumplicate message\n if DEBUG:\n _print_duplicate_message(duplicates)\n return filters\n\n\ndef build_cache():\n \"\"\"Does the hard work of building a cache from nothing.\"\"\"\n cache = {}\n cache[\"lexers\"] = _discover_lexers()\n cache[\"formatters\"] = _discover_formatters()\n cache[\"styles\"] = _discover_styles()\n cache[\"filters\"] = _discover_filters()\n return cache\n\n\ndef cache_filename():\n \"\"\"Gets the name of the cache file to use.\"\"\"\n # Configuration variables read from the environment\n if \"PYGMENTS_CACHE_FILE\" in os.environ:\n return os.environ[\"PYGMENTS_CACHE_FILE\"]\n else:\n return os.path.join(\n os.environ.get(\n \"XDG_DATA_HOME\",\n os.path.join(os.path.expanduser(\"~\"), \".local\", \"share\"),\n ),\n \"pygments-cache\",\n \"cache.py\",\n )\n\n\ndef load(filename):\n \"\"\"Loads the cache from a filename.\"\"\"\n global CACHE\n with open(filename) as f:\n s = f.read()\n ctx = globals()\n CACHE = eval(s, ctx, ctx)\n return CACHE\n\n\ndef write_cache(filename):\n \"\"\"Writes the current cache to the file\"\"\"\n from pprint import pformat\n\n d = os.path.dirname(filename)\n os.makedirs(d, exist_ok=True)\n s = pformat(CACHE)\n with open(filename, \"w\") as f:\n f.write(s)\n\n\ndef load_or_build():\n \"\"\"Loads the cache from disk. If the cache does not exist,\n this will build and write it out.\n \"\"\"\n global CACHE\n fname = cache_filename()\n if os.path.exists(fname):\n load(fname)\n else:\n import sys\n\n print(\"pygments cache not found, building...\", file=sys.stderr)\n CACHE = build_cache()\n print(\"...writing cache to \" + fname, file=sys.stderr)\n write_cache(fname)\n\n\n#\n# pygments interface\n#\n\n\ndef get_lexer_for_filename(filename, text=\"\", **options):\n \"\"\"Gets a lexer from a filename (usually via the filename extension).\n This mimics the behavior of ``pygments.lexers.get_lexer_for_filename()``\n and ``pygments.lexers.guess_lexer_for_filename()``.\n \"\"\"\n if CACHE is None:\n load_or_build()\n exts = CACHE[\"lexers\"][\"exts\"]\n fname = os.path.basename(filename)\n key = fname if fname in exts else os.path.splitext(fname)[1]\n if key in exts:\n modname, clsname = exts[key]\n mod = importlib.import_module(modname)\n cls = getattr(mod, clsname)\n lexer = cls(**options)\n else:\n # couldn't find lexer in cache, fallback to the hard way\n import inspect\n from pygments.lexers import guess_lexer_for_filename\n\n lexer = guess_lexer_for_filename(filename, text, **options)\n # add this filename to the cache for future use\n cls = type(lexer)\n mod = inspect.getmodule(cls)\n exts[fname] = (mod.__name__, cls.__name__)\n write_cache(cache_filename())\n return lexer\n\n\nguess_lexer_for_filename = get_lexer_for_filename\n\n\ndef get_formatter_for_filename(fn, **options):\n \"\"\"Gets a formatter instance from a filename (usually via the filename\n extension). This mimics the behavior of\n ``pygments.formatters.get_formatter_for_filename()``.\n \"\"\"\n if CACHE is None:\n load_or_build()\n exts = CACHE[\"formatters\"][\"exts\"]\n fname = os.path.basename(fn)\n key = fname if fname in exts else os.path.splitext(fname)[1]\n if key in exts:\n modname, clsname = exts[key]\n mod = importlib.import_module(modname)\n cls = getattr(mod, clsname)\n formatter = cls(**options)\n else:\n # couldn't find formatter in cache, fallback to the hard way\n import inspect\n from pygments.formatters import get_formatter_for_filename\n\n formatter = get_formatter_for_filename(fn, **options)\n # add this filename to the cache for future use\n cls = type(formatter)\n mod = inspect.getmodule(cls)\n exts[fname] = (mod.__name__, cls.__name__)\n write_cache(cache_filename())\n return formatter\n\n\ndef get_formatter_by_name(alias, **options):\n \"\"\"Gets a formatter instance from its name or alias.\n This mimics the behavior of ``pygments.formatters.get_formatter_by_name()``.\n \"\"\"\n if CACHE is None:\n load_or_build()\n names = CACHE[\"formatters\"][\"names\"]\n if alias in names:\n modname, clsname = names[alias]\n mod = importlib.import_module(modname)\n cls = getattr(mod, clsname)\n formatter = cls(**options)\n else:\n # couldn't find formatter in cache, fallback to the hard way\n import inspect\n from pygments.formatters import get_formatter_by_name\n\n formatter = get_formatter_by_name(alias, **options)\n # add this filename to the cache for future use\n cls = type(formatter)\n mod = inspect.getmodule(cls)\n names[alias] = (mod.__name__, cls.__name__)\n write_cache(cache_filename())\n return formatter\n\n\ndef get_style_by_name(name):\n \"\"\"Gets a style class from its name or alias.\n This mimics the behavior of ``pygments.styles.get_style_by_name()``.\n \"\"\"\n if CACHE is None:\n load_or_build()\n names = CACHE[\"styles\"][\"names\"]\n if name in names:\n modname, clsname = names[name]\n mod = importlib.import_module(modname)\n style = getattr(mod, clsname)\n else:\n # couldn't find style in cache, fallback to the hard way\n import inspect\n from pygments.styles import get_style_by_name\n\n style = get_style_by_name(name)\n # add this style to the cache for future use\n mod = inspect.getmodule(style)\n names[name] = (mod.__name__, style.__name__)\n write_cache(cache_filename())\n return style\n\n\ndef get_all_styles():\n \"\"\"Iterable through all known style names.\n This mimics the behavior of ``pygments.styles.get_all_styles``.\n \"\"\"\n if CACHE is None:\n load_or_build()\n yield from CACHE[\"styles\"][\"names\"]\n\n\ndef get_filter_by_name(filtername, **options):\n \"\"\"Gets a filter instance from its name. This mimics the behavior of\n ``pygments.filters.get_filtere_by_name()``.\n \"\"\"\n if CACHE is None:\n load_or_build()\n names = CACHE[\"filters\"][\"names\"]\n if filtername in names:\n modname, clsname = names[filtername]\n mod = importlib.import_module(modname)\n cls = getattr(mod, clsname)\n filter = cls(**options)\n else:\n # couldn't find style in cache, fallback to the hard way\n import inspect\n from pygments.filters import get_filter_by_name\n\n filter = get_filter_by_name(filtername, **options)\n # add this filter to the cache for future use\n cls = type(filter)\n mod = inspect.getmodule(cls)\n names[filtername] = (mod.__name__, cls.__name__)\n write_cache(cache_filename())\n return filter\n" }, { "path": "xonsh/pytest_plugin.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"Pytest plugin for testing xsh files.\"\"\"\nimport sys\nimport importlib\nfrom traceback import format_list, extract_tb\n\nimport pytest\n\nfrom xonsh.main import setup\n\n\ndef pytest_configure(config):\n setup()\n\n\ndef pytest_collection_modifyitems(items):\n items.sort(key=lambda x: 0 if isinstance(x, XshFunction) else 1)\n\n\ndef _limited_traceback(excinfo):\n \"\"\" Return a formatted traceback with all the stack\n from this frame (i.e __file__) up removed\n \"\"\"\n tb = extract_tb(excinfo.tb)\n try:\n idx = [__file__ in e for e in tb].index(True)\n return format_list(tb[idx + 1 :])\n except ValueError:\n return format_list(tb)\n\n\ndef pytest_collect_file(parent, path):\n if path.ext.lower() == \".xsh\" and path.basename.startswith(\"test_\"):\n return XshFile(path, parent)\n\n\nclass XshFile(pytest.File):\n def collect(self):\n sys.path.append(self.fspath.dirname)\n mod = importlib.import_module(self.fspath.purebasename)\n sys.path.pop(0)\n tests = [t for t in dir(mod) if t.startswith(\"test_\")]\n for test_name in tests:\n obj = getattr(mod, test_name)\n if hasattr(obj, \"__call__\"):\n yield XshFunction(\n name=test_name, parent=self, test_func=obj, test_module=mod\n )\n\n\nclass XshFunction(pytest.Item):\n def __init__(self, name, parent, test_func, test_module):\n super().__init__(name, parent)\n self._test_func = test_func\n self._test_module = test_module\n\n def runtest(self, *args, **kwargs):\n self._test_func(*args, **kwargs)\n\n def repr_failure(self, excinfo):\n \"\"\" called when self.runtest() raises an exception. \"\"\"\n formatted_tb = _limited_traceback(excinfo)\n formatted_tb.insert(0, \"xonsh execution failed\\n\")\n formatted_tb.append(\"{}: {}\".format(excinfo.type.__name__, excinfo.value))\n return \"\".join(formatted_tb)\n\n def reportinfo(self):\n return self.fspath, 0, \"xonsh test: {}\".format(self.name)\n" }, { "path": "xonsh/readline_shell.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"The readline based xonsh shell.\n\nPortions of this code related to initializing the readline library\nare included from the IPython project. The IPython project is:\n\n* Copyright (c) 2008-2014, IPython Development Team\n* Copyright (c) 2001-2007, Fernando Perez \n* Copyright (c) 2001, Janko Hauser \n* Copyright (c) 2001, Nathaniel Gray \n\n\"\"\"\nimport os\nimport sys\nimport cmd\nimport select\nimport shutil\nimport builtins\nimport importlib\nimport threading\nimport subprocess\nimport collections\n\nfrom xonsh.lazyasd import LazyObject, lazyobject\nfrom xonsh.base_shell import BaseShell\nfrom xonsh.ansi_colors import (\n ansi_partial_color_format,\n ansi_color_style_names,\n ansi_color_style,\n)\nfrom xonsh.prompt.base import multiline_prompt\nfrom xonsh.tools import (\n print_exception,\n check_for_partial_string,\n to_bool,\n columnize,\n carriage_return,\n)\nfrom xonsh.platform import (\n ON_WINDOWS,\n ON_CYGWIN,\n ON_MSYS,\n ON_DARWIN,\n ON_POSIX,\n os_environ,\n)\nfrom xonsh.lazyimps import pygments, pyghooks, winutils\nfrom xonsh.events import events\n\nreadline = None\nRL_COMPLETION_SUPPRESS_APPEND = RL_LIB = RL_STATE = None\nRL_COMPLETION_QUERY_ITEMS = None\nRL_CAN_RESIZE = False\nRL_DONE = None\nRL_VARIABLE_VALUE = None\n_RL_STATE_DONE = 0x1000000\n_RL_STATE_ISEARCH = 0x0000080\n\n_RL_PREV_CASE_SENSITIVE_COMPLETIONS = \"to-be-set\"\n\n\ndef setup_readline():\n \"\"\"Sets up the readline module and completion suppression, if available.\"\"\"\n global RL_COMPLETION_SUPPRESS_APPEND, RL_LIB, RL_CAN_RESIZE, RL_STATE, readline, RL_COMPLETION_QUERY_ITEMS\n if RL_COMPLETION_SUPPRESS_APPEND is not None:\n return\n for _rlmod_name in (\"gnureadline\", \"readline\"):\n try:\n readline = importlib.import_module(_rlmod_name)\n sys.modules[\"readline\"] = readline\n except ImportError:\n pass\n else:\n break\n\n if readline is None:\n print(\n \"\"\"Skipping setup. Because no `readline` implementation available.\n Please install a backend (`readline`, `prompt-toolkit`, etc) to use\n `xonsh` interactively.\n See https://github.com/xonsh/xonsh/issues/1170\"\"\"\n )\n return\n\n import ctypes\n import ctypes.util\n\n uses_libedit = readline.__doc__ and \"libedit\" in readline.__doc__\n readline.set_completer_delims(\" \\t\\n\")\n # Cygwin seems to hang indefinitely when querying the readline lib\n if (not ON_CYGWIN) and (not ON_MSYS) and (not readline.__file__.endswith(\".py\")):\n RL_LIB = lib = ctypes.cdll.LoadLibrary(readline.__file__)\n try:\n RL_COMPLETION_SUPPRESS_APPEND = ctypes.c_int.in_dll(\n lib, \"rl_completion_suppress_append\"\n )\n except ValueError:\n # not all versions of readline have this symbol, ie Macs sometimes\n RL_COMPLETION_SUPPRESS_APPEND = None\n try:\n RL_COMPLETION_QUERY_ITEMS = ctypes.c_int.in_dll(\n lib, \"rl_completion_query_items\"\n )\n except ValueError:\n # not all versions of readline have this symbol, ie Macs sometimes\n RL_COMPLETION_QUERY_ITEMS = None\n try:\n RL_STATE = ctypes.c_int.in_dll(lib, \"rl_readline_state\")\n except Exception:\n pass\n RL_CAN_RESIZE = hasattr(lib, \"rl_reset_screen_size\")\n env = builtins.__xonsh__.env\n # reads in history\n readline.set_history_length(-1)\n ReadlineHistoryAdder()\n # sets up IPython-like history matching with up and down\n readline.parse_and_bind('\"\\\\e[B\": history-search-forward')\n readline.parse_and_bind('\"\\\\e[A\": history-search-backward')\n # Setup Shift-Tab to indent\n readline.parse_and_bind('\"\\\\e[Z\": \"{0}\"'.format(env.get(\"INDENT\")))\n\n # handle tab completion differences found in libedit readline compatibility\n # as discussed at http://stackoverflow.com/a/7116997\n if uses_libedit and ON_DARWIN:\n readline.parse_and_bind(\"bind ^I rl_complete\")\n print(\n \"\\n\".join(\n [\n \"\",\n \"*\" * 78,\n \"libedit detected - readline will not be well behaved, including but not limited to:\",\n \" * crashes on tab completion\",\n \" * incorrect history navigation\",\n \" * corrupting long-lines\",\n \" * failure to wrap or indent lines properly\",\n \"\",\n \"It is highly recommended that you install gnureadline, which is installable with:\",\n \" xpip install gnureadline\",\n \"*\" * 78,\n ]\n ),\n file=sys.stderr,\n )\n else:\n readline.parse_and_bind(\"tab: complete\")\n # try to load custom user settings\n inputrc_name = os_environ.get(\"INPUTRC\")\n if inputrc_name is None:\n if uses_libedit:\n inputrc_name = \".editrc\"\n else:\n inputrc_name = \".inputrc\"\n inputrc_name = os.path.join(os.path.expanduser(\"~\"), inputrc_name)\n if (not ON_WINDOWS) and (not os.path.isfile(inputrc_name)):\n inputrc_name = \"/etc/inputrc\"\n if ON_WINDOWS:\n winutils.enable_virtual_terminal_processing()\n if os.path.isfile(inputrc_name):\n try:\n readline.read_init_file(inputrc_name)\n except Exception:\n # this seems to fail with libedit\n print_exception(\"xonsh: could not load readline default init file.\")\n # properly reset input typed before the first prompt\n readline.set_startup_hook(carriage_return)\n\n\ndef teardown_readline():\n \"\"\"Tears down up the readline module, if available.\"\"\"\n try:\n import readline\n except (ImportError, TypeError):\n return\n\n\ndef _rebind_case_sensitive_completions():\n # handle case sensitive, see Github issue #1342 for details\n global _RL_PREV_CASE_SENSITIVE_COMPLETIONS\n env = builtins.__xonsh__.env\n case_sensitive = env.get(\"CASE_SENSITIVE_COMPLETIONS\")\n if case_sensitive is _RL_PREV_CASE_SENSITIVE_COMPLETIONS:\n return\n if case_sensitive:\n readline.parse_and_bind(\"set completion-ignore-case off\")\n else:\n readline.parse_and_bind(\"set completion-ignore-case on\")\n _RL_PREV_CASE_SENSITIVE_COMPLETIONS = case_sensitive\n\n\ndef fix_readline_state_after_ctrl_c():\n \"\"\"\n Fix to allow Ctrl-C to exit reverse-i-search.\n\n Based on code from:\n http://bugs.python.org/file39467/raw_input__workaround_demo.py\n \"\"\"\n if ON_WINDOWS:\n # hack to make pyreadline mimic the desired behavior\n try:\n _q = readline.rl.mode.process_keyevent_queue\n if len(_q) > 1:\n _q.pop()\n except Exception:\n pass\n if RL_STATE is None:\n return\n if RL_STATE.value & _RL_STATE_ISEARCH:\n RL_STATE.value &= ~_RL_STATE_ISEARCH\n if not RL_STATE.value & _RL_STATE_DONE:\n RL_STATE.value |= _RL_STATE_DONE\n\n\ndef rl_completion_suppress_append(val=1):\n \"\"\"Sets the rl_completion_suppress_append variable, if possible.\n A value of 1 (default) means to suppress, a value of 0 means to enable.\n \"\"\"\n if RL_COMPLETION_SUPPRESS_APPEND is None:\n return\n RL_COMPLETION_SUPPRESS_APPEND.value = val\n\n\ndef rl_completion_query_items(val=None):\n \"\"\"Sets the rl_completion_query_items variable, if possible.\n A None value will set this to $COMPLETION_QUERY_LIMIT, otherwise any integer\n is accepted.\n \"\"\"\n if RL_COMPLETION_QUERY_ITEMS is None:\n return\n if val is None:\n val = builtins.__xonsh__.env.get(\"COMPLETION_QUERY_LIMIT\")\n RL_COMPLETION_QUERY_ITEMS.value = val\n\n\ndef rl_variable_dumper(readable=True):\n \"\"\"Dumps the currently set readline variables. If readable is True, then this\n output may be used in an inputrc file.\n \"\"\"\n RL_LIB.rl_variable_dumper(int(readable))\n\n\ndef rl_variable_value(variable):\n \"\"\"Returns the currently set value for a readline configuration variable.\"\"\"\n global RL_VARIABLE_VALUE\n if RL_VARIABLE_VALUE is None:\n import ctypes\n\n RL_VARIABLE_VALUE = RL_LIB.rl_variable_value\n RL_VARIABLE_VALUE.restype = ctypes.c_char_p\n env = builtins.__xonsh__.env\n enc, errors = env.get(\"XONSH_ENCODING\"), env.get(\"XONSH_ENCODING_ERRORS\")\n if isinstance(variable, str):\n variable = variable.encode(encoding=enc, errors=errors)\n rtn = RL_VARIABLE_VALUE(variable)\n return rtn.decode(encoding=enc, errors=errors)\n\n\n@lazyobject\ndef rl_on_new_line():\n \"\"\"Grabs one of a few possible redisplay functions in readline.\"\"\"\n names = [\"rl_on_new_line\", \"rl_forced_update_display\", \"rl_redisplay\"]\n for name in names:\n func = getattr(RL_LIB, name, None)\n if func is not None:\n break\n else:\n\n def print_for_newline():\n print()\n\n func = print_for_newline\n return func\n\n\ndef _insert_text_func(s, readline):\n \"\"\"Creates a function to insert text via readline.\"\"\"\n\n def inserter():\n readline.insert_text(s)\n readline.redisplay()\n\n return inserter\n\n\nDEDENT_TOKENS = LazyObject(\n lambda: frozenset([\"raise\", \"return\", \"pass\", \"break\", \"continue\"]),\n globals(),\n \"DEDENT_TOKENS\",\n)\n\n\nclass ReadlineShell(BaseShell, cmd.Cmd):\n \"\"\"The readline based xonsh shell.\"\"\"\n\n def __init__(self, completekey=\"tab\", stdin=None, stdout=None, **kwargs):\n super().__init__(completekey=completekey, stdin=stdin, stdout=stdout, **kwargs)\n setup_readline()\n self._current_indent = \"\"\n self._current_prompt = \"\"\n self._force_hide = None\n self._complete_only_last_table = {\n # Truth table for completions, keys are:\n # (prefix_begs_quote, prefix_ends_quote, i_ends_quote,\n # last_starts_with_prefix, i_has_space)\n (True, True, True, True, True): True,\n (True, True, True, True, False): True,\n (True, True, True, False, True): False,\n (True, True, True, False, False): True,\n (True, True, False, True, True): False,\n (True, True, False, True, False): False,\n (True, True, False, False, True): False,\n (True, True, False, False, False): False,\n (True, False, True, True, True): True,\n (True, False, True, True, False): False,\n (True, False, True, False, True): False,\n (True, False, True, False, False): True,\n (True, False, False, True, True): False,\n (True, False, False, True, False): False,\n (True, False, False, False, True): False,\n (True, False, False, False, False): False,\n (False, True, True, True, True): True,\n (False, True, True, True, False): True,\n (False, True, True, False, True): True,\n (False, True, True, False, False): True,\n (False, True, False, True, True): False,\n (False, True, False, True, False): False,\n (False, True, False, False, True): False,\n (False, True, False, False, False): False,\n (False, False, True, True, True): False,\n (False, False, True, True, False): False,\n (False, False, True, False, True): False,\n (False, False, True, False, False): True,\n (False, False, False, True, True): True,\n (False, False, False, True, False): False,\n (False, False, False, False, True): False,\n (False, False, False, False, False): False,\n }\n self.cmdqueue = collections.deque()\n\n def __del__(self):\n teardown_readline()\n\n def singleline(self, store_in_history=True, **kwargs):\n \"\"\"Reads a single line of input. The store_in_history kwarg\n flags whether the input should be stored in readline's in-memory\n history.\n \"\"\"\n if not store_in_history: # store current position to remove it later\n try:\n import readline\n except ImportError:\n store_in_history = True\n pos = readline.get_current_history_length() - 1\n events.on_pre_prompt.fire()\n rtn = input(self.prompt)\n events.on_post_prompt.fire()\n if not store_in_history and pos >= 0:\n readline.remove_history_item(pos)\n return rtn\n\n def parseline(self, line):\n \"\"\"Overridden to no-op.\"\"\"\n return \"\", line, line\n\n def _querycompletions(self, completions, loc):\n \"\"\"Returns whether or not we should show completions. 0 means that prefixes\n should not be shown, 1 means that there is a common prefix among all completions\n and they should be shown, while 2 means that there is no common prefix but\n we are under the query limit and they should be shown.\n \"\"\"\n if os.path.commonprefix([c[loc:] for c in completions]):\n return 1\n elif len(completions) <= builtins.__xonsh__.env.get(\"COMPLETION_QUERY_LIMIT\"):\n return 2\n msg = \"\\nDisplay all {} possibilities? \".format(len(completions))\n msg += \"({GREEN}y{NO_COLOR} or {RED}n{NO_COLOR})\"\n self.print_color(msg, end=\"\", flush=True, file=sys.stderr)\n yn = \"x\"\n while yn not in \"yn\":\n yn = sys.stdin.read(1)\n show_completions = to_bool(yn)\n print()\n if not show_completions:\n rl_on_new_line()\n return 0\n w, h = shutil.get_terminal_size()\n lines = columnize(completions, width=w)\n more_msg = self.format_color(\n \"{YELLOW}==={NO_COLOR} more or \"\n \"{PURPLE}({NO_COLOR}q{PURPLE}){NO_COLOR}uit \"\n \"{YELLOW}==={NO_COLOR}\"\n )\n while len(lines) > h - 1:\n print(\"\".join(lines[: h - 1]), end=\"\", flush=True, file=sys.stderr)\n lines = lines[h - 1 :]\n print(more_msg, end=\"\", flush=True, file=sys.stderr)\n q = sys.stdin.read(1).lower()\n print(flush=True, file=sys.stderr)\n if q == \"q\":\n rl_on_new_line()\n return 0\n print(\"\".join(lines), end=\"\", flush=True, file=sys.stderr)\n rl_on_new_line()\n return 0\n\n def completedefault(self, prefix, line, begidx, endidx):\n \"\"\"Implements tab-completion for text.\"\"\"\n if self.completer is None:\n return []\n rl_completion_suppress_append() # this needs to be called each time\n _rebind_case_sensitive_completions()\n rl_completion_query_items(val=999999999)\n completions, l = self.completer.complete(\n prefix, line, begidx, endidx, ctx=self.ctx\n )\n chopped = prefix[:-l]\n if chopped:\n rtn_completions = [chopped + i for i in completions]\n else:\n rtn_completions = completions\n rtn = []\n prefix_begs_quote = prefix.startswith(\"'\") or prefix.startswith('\"')\n prefix_ends_quote = prefix.endswith(\"'\") or prefix.endswith('\"')\n for i in rtn_completions:\n i_ends_quote = i.endswith(\"'\") or i.endswith('\"')\n last = i.rsplit(\" \", 1)[-1]\n last_starts_prefix = last.startswith(prefix)\n i_has_space = \" \" in i\n key = (\n prefix_begs_quote,\n prefix_ends_quote,\n i_ends_quote,\n last_starts_prefix,\n i_has_space,\n )\n rtn.append(last if self._complete_only_last_table[key] else i)\n # return based on show completions\n show_completions = self._querycompletions(completions, endidx - begidx)\n if show_completions == 0:\n return []\n elif show_completions == 1:\n return rtn\n elif show_completions == 2:\n return completions\n else:\n raise ValueError(\"query completions flag not understood.\")\n\n # tab complete on first index too\n completenames = completedefault\n\n def _load_remaining_input_into_queue(self):\n buf = b\"\"\n while True:\n r, w, x = select.select([self.stdin], [], [], 1e-6)\n if len(r) == 0:\n break\n buf += os.read(self.stdin.fileno(), 1024)\n if len(buf) > 0:\n buf = buf.decode().replace(\"\\r\\n\", \"\\n\").replace(\"\\r\", \"\\n\")\n self.cmdqueue.extend(buf.splitlines(keepends=True))\n\n def postcmd(self, stop, line):\n \"\"\"Called just before execution of line. For readline, this handles the\n automatic indentation of code blocks.\n \"\"\"\n try:\n import readline\n except ImportError:\n return stop\n if self.need_more_lines:\n if len(line.strip()) == 0:\n readline.set_pre_input_hook(None)\n self._current_indent = \"\"\n elif line.rstrip()[-1] == \":\":\n ind = line[: len(line) - len(line.lstrip())]\n ind += builtins.__xonsh__.env.get(\"INDENT\")\n readline.set_pre_input_hook(_insert_text_func(ind, readline))\n self._current_indent = ind\n elif line.split(maxsplit=1)[0] in DEDENT_TOKENS:\n env = builtins.__xonsh__.env\n ind = self._current_indent[: -len(env.get(\"INDENT\"))]\n readline.set_pre_input_hook(_insert_text_func(ind, readline))\n self._current_indent = ind\n else:\n ind = line[: len(line) - len(line.lstrip())]\n if ind != self._current_indent:\n insert_func = _insert_text_func(ind, readline)\n readline.set_pre_input_hook(insert_func)\n self._current_indent = ind\n else:\n readline.set_pre_input_hook(None)\n return stop\n\n def _cmdloop(self, intro=None):\n \"\"\"Repeatedly issue a prompt, accept input, parse an initial prefix\n off the received input, and dispatch to action methods, passing them\n the remainder of the line as argument.\n\n This was forked from Lib/cmd.py from the Python standard library v3.4.3,\n (C) Python Software Foundation, 2015.\n \"\"\"\n self.preloop()\n if self.use_rawinput and self.completekey:\n try:\n import readline\n\n self.old_completer = readline.get_completer()\n readline.set_completer(self.complete)\n readline.parse_and_bind(self.completekey + \": complete\")\n have_readline = True\n except ImportError:\n have_readline = False\n try:\n if intro is not None:\n self.intro = intro\n if self.intro:\n self.stdout.write(str(self.intro) + \"\\n\")\n stop = None\n while not stop:\n line = None\n exec_now = False\n if len(self.cmdqueue) > 0:\n line = self.cmdqueue.popleft()\n exec_now = line.endswith(\"\\n\")\n if self.use_rawinput and not exec_now:\n inserter = (\n None if line is None else _insert_text_func(line, readline)\n )\n if inserter is not None:\n readline.set_pre_input_hook(inserter)\n try:\n line = self.singleline()\n except EOFError:\n if builtins.__xonsh__.env.get(\"IGNOREEOF\"):\n self.stdout.write('Use \"exit\" to leave the shell.' \"\\n\")\n line = \"\"\n else:\n line = \"EOF\"\n if inserter is not None:\n readline.set_pre_input_hook(None)\n else:\n self.print_color(self.prompt, file=self.stdout)\n if line is not None:\n os.write(self.stdin.fileno(), line.encode())\n if not exec_now:\n line = self.stdin.readline()\n if len(line) == 0:\n line = \"EOF\"\n else:\n line = line.rstrip(\"\\r\\n\")\n if have_readline and line != \"EOF\":\n readline.add_history(line)\n if not ON_WINDOWS:\n # select() is not fully functional on windows\n self._load_remaining_input_into_queue()\n line = self.precmd(line)\n stop = self.onecmd(line)\n stop = self.postcmd(stop, line)\n if ON_WINDOWS:\n winutils.enable_virtual_terminal_processing()\n self.postloop()\n finally:\n if self.use_rawinput and self.completekey:\n try:\n import readline\n\n readline.set_completer(self.old_completer)\n except ImportError:\n pass\n\n def cmdloop(self, intro=None):\n while not builtins.__xonsh__.exit:\n try:\n self._cmdloop(intro=intro)\n except (KeyboardInterrupt, SystemExit):\n print() # Gives a newline\n fix_readline_state_after_ctrl_c()\n self.reset_buffer()\n intro = None\n\n @property\n def prompt(self):\n \"\"\"Obtains the current prompt string.\"\"\"\n global RL_LIB, RL_CAN_RESIZE\n if RL_CAN_RESIZE:\n # This is needed to support some system where line-wrapping doesn't\n # work. This is a bug in upstream Python, or possibly readline.\n RL_LIB.rl_reset_screen_size()\n if self.need_more_lines:\n if self.mlprompt is None:\n try:\n self.mlprompt = multiline_prompt(curr=self._current_prompt)\n except Exception: # pylint: disable=broad-except\n print_exception()\n self.mlprompt = \" \"\n return self.mlprompt\n env = builtins.__xonsh__.env # pylint: disable=no-member\n p = env.get(\"PROMPT\")\n try:\n p = self.prompt_formatter(p)\n except Exception: # pylint: disable=broad-except\n print_exception()\n hide = True if self._force_hide is None else self._force_hide\n p = ansi_partial_color_format(p, style=env.get(\"XONSH_COLOR_STYLE\"), hide=hide)\n self._current_prompt = p\n self.settitle()\n return p\n\n def format_color(self, string, hide=False, force_string=False, **kwargs):\n \"\"\"Readline implementation of color formatting. This uses ANSI color\n codes.\n \"\"\"\n hide = hide if self._force_hide is None else self._force_hide\n style = builtins.__xonsh__.env.get(\"XONSH_COLOR_STYLE\")\n return ansi_partial_color_format(string, hide=hide, style=style)\n\n def print_color(self, string, hide=False, **kwargs):\n if isinstance(string, str):\n s = self.format_color(string, hide=hide)\n else:\n # assume this is a list of (Token, str) tuples and format it\n env = builtins.__xonsh__.env\n self.styler.style_name = env.get(\"XONSH_COLOR_STYLE\")\n style_proxy = pyghooks.xonsh_style_proxy(self.styler)\n formatter = pyghooks.XonshTerminal256Formatter(style=style_proxy)\n s = pygments.format(string, formatter).rstrip()\n print(s, **kwargs)\n\n def color_style_names(self):\n \"\"\"Returns an iterable of all available style names.\"\"\"\n return ansi_color_style_names()\n\n def color_style(self):\n \"\"\"Returns the current color map.\"\"\"\n style = style = builtins.__xonsh__.env.get(\"XONSH_COLOR_STYLE\")\n return ansi_color_style(style=style)\n\n def restore_tty_sanity(self):\n \"\"\"An interface for resetting the TTY stdin mode. This is highly\n dependent on the shell backend. Also it is mostly optional since\n it only affects ^Z backgrounding behaviour.\n \"\"\"\n if not ON_POSIX:\n return\n stty, _ = builtins.__xonsh__.commands_cache.lazyget(\"stty\", None)\n if stty is None:\n return\n # If available, we should just call the stty utility. This call should\n # not throw even if stty fails. It should also be noted that subprocess\n # calls, like the following, seem to be ineffective:\n # subprocess.call([stty, 'sane'], shell=True)\n # My guess is that this is because Popen does some crazy redirecting\n # under the covers. This effectively hides the true TTY stdin handle\n # from stty. To get around this we have to use the lower level\n # os.system() function.\n os.system(stty + \" sane\")\n\n\nclass ReadlineHistoryAdder(threading.Thread):\n def __init__(self, wait_for_gc=True, *args, **kwargs):\n \"\"\"Thread responsible for adding inputs from history to the\n current readline instance. May wait for the history garbage\n collector to finish.\n \"\"\"\n super(ReadlineHistoryAdder, self).__init__(*args, **kwargs)\n self.daemon = True\n self.wait_for_gc = wait_for_gc\n self.start()\n\n def run(self):\n try:\n import readline\n except ImportError:\n return\n hist = builtins.__xonsh__.history\n if hist is None:\n return\n i = 1\n for h in hist.all_items():\n line = h[\"inp\"].rstrip()\n if i == 1:\n pass\n elif line == readline.get_history_item(i - 1):\n continue\n readline.add_history(line)\n if RL_LIB is not None:\n RL_LIB.history_set_pos(i)\n i += 1\n" }, { "path": "xonsh/replay.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"Tools to replay xonsh history files.\"\"\"\nimport json\nimport time\nimport builtins\nimport collections.abc as cabc\n\nfrom xonsh.tools import swap\nfrom xonsh.lazyjson import LazyJSON\nfrom xonsh.environ import Env\nimport xonsh.history.main as xhm\n\n\nDEFAULT_MERGE_ENVS = (\"replay\", \"native\")\n\n\nclass Replayer(object):\n \"\"\"Replays a xonsh history file.\"\"\"\n\n def __init__(self, f, reopen=True):\n \"\"\"\n Parameters\n ----------\n f : file handle or str\n Path to xonsh history file.\n reopen : bool, optional\n Whether new file handle should be opened for each load, passed directly into\n LazyJSON class.\n \"\"\"\n self._lj = LazyJSON(f, reopen=reopen)\n\n def __del__(self):\n self._lj.close()\n\n def replay(self, merge_envs=DEFAULT_MERGE_ENVS, target=None):\n \"\"\"Replays the history specified, returns the history object where the code\n was executed.\n\n Parameters\n ----------\n merge_env : tuple of str or Mappings, optional\n Describes how to merge the environments, in order of increasing precedence.\n Available strings are 'replay' and 'native'. The 'replay' env comes from the\n history file that we are replaying. The 'native' env comes from what this\n instance of xonsh was started up with. Instead of a string, a dict or other\n mapping may be passed in as well. Defaults to ('replay', 'native').\n target : str, optional\n Path to new history file.\n \"\"\"\n shell = builtins.__xonsh__.shell\n re_env = self._lj[\"env\"].load()\n new_env = self._merge_envs(merge_envs, re_env)\n new_hist = xhm.construct_history(\n env=new_env.detype(),\n locked=True,\n ts=[time.time(), None],\n gc=False,\n filename=target,\n )\n with swap(builtins.__xonsh__, \"env\", new_env), swap(\n builtins.__xonsh__, \"history\", new_hist\n ):\n for cmd in self._lj[\"cmds\"]:\n inp = cmd[\"inp\"]\n shell.default(inp)\n if builtins.__xonsh__.exit: # prevent premature exit\n builtins.__xonsh__.exit = False\n new_hist.flush(at_exit=True)\n return new_hist\n\n def _merge_envs(self, merge_envs, re_env):\n new_env = {}\n for e in merge_envs:\n if e == \"replay\":\n new_env.update(re_env)\n elif e == \"native\":\n new_env.update(builtins.__xonsh__.env)\n elif isinstance(e, cabc.Mapping):\n new_env.update(e)\n else:\n raise TypeError(\"Type of env not understood: {0!r}\".format(e))\n new_env = Env(**new_env)\n return new_env\n\n\n_REPLAY_PARSER = None\n\n\ndef replay_create_parser(p=None):\n global _REPLAY_PARSER\n p_was_none = p is None\n if _REPLAY_PARSER is not None and p_was_none:\n return _REPLAY_PARSER\n if p_was_none:\n from argparse import ArgumentParser\n\n p = ArgumentParser(\"replay\", description=\"replays a xonsh history file\")\n p.add_argument(\n \"--merge-envs\",\n dest=\"merge_envs\",\n default=DEFAULT_MERGE_ENVS,\n nargs=\"+\",\n help=\"Describes how to merge the environments, in order of \"\n \"increasing precedence. Available strings are 'replay' and \"\n \"'native'. The 'replay' env comes from the history file that we \"\n \"are replaying. The 'native' env comes from what this instance \"\n \"of xonsh was started up with. One or more of these options may \"\n \"be passed in. Defaults to '--merge-envs replay native'.\",\n )\n p.add_argument(\n \"--json\",\n dest=\"json\",\n default=False,\n action=\"store_true\",\n help=\"print history info in JSON format\",\n )\n p.add_argument(\n \"-o\", \"--target\", dest=\"target\", default=None, help=\"path to new history file\"\n )\n p.add_argument(\"path\", help=\"path to replay history file\")\n if p_was_none:\n _REPLAY_PARSER = p\n return p\n\n\ndef replay_main_action(h, ns, stdout=None, stderr=None):\n replayer = Replayer(ns.path)\n hist = replayer.replay(merge_envs=ns.merge_envs, target=ns.target)\n print(\"----------------------------------------------------------------\")\n print(\"Just replayed history, new history has the following information\")\n print(\"----------------------------------------------------------------\")\n data = hist.info()\n if ns.json:\n s = json.dumps(data)\n print(s, file=stdout)\n else:\n lines = [\"{0}: {1}\".format(k, v) for k, v in data.items()]\n print(\"\\n\".join(lines), file=stdout)\n\n\ndef replay_main(args, stdin=None):\n \"\"\"Acts as main function for replaying a xonsh history file.\"\"\"\n parser = replay_create_parser()\n ns = parser.parse_args(args)\n replay_main_action(ns)\n" }, { "path": "xonsh/shell.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"The xonsh shell\"\"\"\nimport sys\nimport random\nimport time\nimport difflib\nimport builtins\nimport warnings\n\nfrom xonsh.platform import (\n best_shell_type,\n has_prompt_toolkit,\n ptk_above_min_supported,\n ptk_shell_type,\n)\nfrom xonsh.tools import XonshError, print_exception\nfrom xonsh.events import events\nimport xonsh.history.main as xhm\n\n\nevents.doc(\n \"on_transform_command\",\n \"\"\"\non_transform_command(cmd: str) -> str\n\nFired to request xontribs to transform a command line. Return the transformed\ncommand, or the same command if no transformation occurs. Only done for\ninteractive sessions.\n\nThis may be fired multiple times per command, with other transformers input or\noutput, so design any handlers for this carefully.\n\"\"\",\n)\n\nevents.doc(\n \"on_precommand\",\n \"\"\"\non_precommand(cmd: str) -> None\n\nFires just before a command is executed.\n\"\"\",\n)\n\nevents.doc(\n \"on_postcommand\",\n \"\"\"\non_postcommand(cmd: str, rtn: int, out: str or None, ts: list) -> None\n\nFires just after a command is executed. The arguments are the same as history.\n\nParameters:\n\n* ``cmd``: The command that was executed (after transformation)\n* ``rtn``: The result of the command executed (``0`` for success)\n* ``out``: If xonsh stores command output, this is the output\n* ``ts``: Timestamps, in the order of ``[starting, ending]``\n\"\"\",\n)\n\nevents.doc(\n \"on_pre_prompt\",\n \"\"\"\non_pre_prompt() -> None\n\nFires just before the prompt is shown\n\"\"\",\n)\n\nevents.doc(\n \"on_post_prompt\",\n \"\"\"\non_post_prompt() -> None\n\nFires just after the prompt returns\n\"\"\",\n)\n\n\ndef transform_command(src, show_diff=True):\n \"\"\"Returns the results of firing the precommand handles.\"\"\"\n i = 0\n limit = sys.getrecursionlimit()\n lst = \"\"\n raw = src\n while src != lst:\n lst = src\n srcs = events.on_transform_command.fire(cmd=src)\n for s in srcs:\n if s != lst:\n src = s\n break\n i += 1\n if i == limit:\n print_exception(\n \"Modifications to source input took more than \"\n \"the recursion limit number of iterations to \"\n \"converge.\"\n )\n debug_level = builtins.__xonsh__.env.get(\"XONSH_DEBUG\")\n if show_diff and debug_level > 1 and src != raw:\n sys.stderr.writelines(\n difflib.unified_diff(\n raw.splitlines(keepends=True),\n src.splitlines(keepends=True),\n fromfile=\"before precommand event\",\n tofile=\"after precommand event\",\n )\n )\n return src\n\n\nclass Shell(object):\n \"\"\"Main xonsh shell.\n\n Initializes execution environment and decides if prompt_toolkit or\n readline version of shell should be used.\n \"\"\"\n\n shell_type_aliases = {\n \"b\": \"best\",\n \"best\": \"best\",\n \"d\": \"dumb\",\n \"dumb\": \"dumb\",\n \"ptk\": \"prompt_toolkit\",\n \"ptk1\": \"prompt_toolkit1\",\n \"ptk2\": \"prompt_toolkit2\",\n \"prompt-toolkit\": \"prompt_toolkit\",\n \"prompt_toolkit\": \"prompt_toolkit\",\n \"prompt-toolkit1\": \"prompt_toolkit1\",\n \"prompt-toolkit2\": \"prompt_toolkit2\",\n \"rand\": \"random\",\n \"random\": \"random\",\n \"rl\": \"readline\",\n \"readline\": \"readline\",\n }\n\n def __init__(self, execer, ctx=None, shell_type=None, **kwargs):\n \"\"\"\n Parameters\n ----------\n execer : Execer\n An execer instance capable of running xonsh code.\n ctx : Mapping, optional\n The execution context for the shell (e.g. the globals namespace).\n If none, this is computed by loading the rc files. If not None,\n this no additional context is computed and this is used\n directly.\n shell_type : str, optional\n The shell type to start, such as 'readline', 'prompt_toolkit1',\n or 'random'.\n \"\"\"\n self.execer = execer\n self.ctx = {} if ctx is None else ctx\n env = builtins.__xonsh__.env\n # build history backend before creating shell\n builtins.__xonsh__.history = hist = xhm.construct_history(\n env=env.detype(), ts=[time.time(), None], locked=True\n )\n\n # pick a valid shell -- if no shell is specified by the user,\n # shell type is pulled from env\n if shell_type is None:\n shell_type = env.get(\"SHELL_TYPE\")\n if shell_type == \"none\":\n # This bricks interactive xonsh\n # Can happen from the use of .xinitrc, .xsession, etc\n shell_type = \"best\"\n shell_type = self.shell_type_aliases.get(shell_type, shell_type)\n if shell_type == \"best\" or shell_type is None:\n shell_type = best_shell_type()\n elif env.get(\"TERM\", \"\") == \"dumb\":\n shell_type = \"dumb\"\n elif shell_type == \"random\":\n shell_type = random.choice((\"readline\", \"prompt_toolkit\"))\n if shell_type == \"prompt_toolkit\":\n if not has_prompt_toolkit():\n warnings.warn(\n \"prompt_toolkit is not available, using \" \"readline instead.\"\n )\n shell_type = \"readline\"\n elif not ptk_above_min_supported():\n warnings.warn(\n \"prompt-toolkit version < v1.0.0 is not \"\n \"supported. Please update prompt-toolkit. Using \"\n \"readline instead.\"\n )\n shell_type = \"readline\"\n else:\n shell_type = ptk_shell_type()\n self.shell_type = env[\"SHELL_TYPE\"] = shell_type\n # actually make the shell\n if shell_type == \"none\":\n from xonsh.base_shell import BaseShell as shell_class\n elif shell_type == \"prompt_toolkit2\":\n from xonsh.ptk2.shell import PromptToolkit2Shell as shell_class\n elif shell_type == \"prompt_toolkit1\":\n from xonsh.ptk.shell import PromptToolkitShell as shell_class\n elif shell_type == \"readline\":\n from xonsh.readline_shell import ReadlineShell as shell_class\n elif shell_type == \"jupyter\":\n from xonsh.jupyter_shell import JupyterShell as shell_class\n elif shell_type == \"dumb\":\n from xonsh.dumb_shell import DumbShell as shell_class\n else:\n raise XonshError(\"{} is not recognized as a shell type\".format(shell_type))\n self.shell = shell_class(execer=self.execer, ctx=self.ctx, **kwargs)\n # allows history garbage collector to start running\n if hist.gc is not None:\n hist.gc.wait_for_shell = False\n\n def __getattr__(self, attr):\n \"\"\"Delegates calls to appropriate shell instance.\"\"\"\n return getattr(self.shell, attr)\n" }, { "path": "xonsh/style_tools.py", "content": "\"\"\"Xonsh color styling tools that simulate pygments, when it is unavailable.\"\"\"\nimport builtins\nfrom collections import defaultdict\n\nfrom xonsh.platform import HAS_PYGMENTS\nfrom xonsh.lazyasd import LazyObject\nfrom xonsh.color_tools import RE_BACKGROUND\nfrom xonsh.tools import FORMATTER\n\n\nclass _TokenType(tuple):\n \"\"\"\n Forked from the pygments project\n https://bitbucket.org/birkenfeld/pygments-main\n Copyright (c) 2006-2017 by the respective authors, All rights reserved.\n See https://bitbucket.org/birkenfeld/pygments-main/raw/05818a4ef9891d9ac22c851f7b3ea4b4fce460ab/AUTHORS\n \"\"\"\n\n parent = None\n\n def split(self):\n buf = []\n node = self\n while node is not None:\n buf.append(node)\n node = node.parent\n buf.reverse()\n return buf\n\n def __init__(self, *args):\n # no need to call super.__init__\n self.subtypes = set()\n\n def __contains__(self, val):\n return self is val or (type(val) is self.__class__ and val[: len(self)] == self)\n\n def __getattr__(self, val):\n if not val or not val[0].isupper():\n return tuple.__getattribute__(self, val)\n new = _TokenType(self + (val,))\n setattr(self, val, new)\n self.subtypes.add(new)\n new.parent = self\n return new\n\n def __repr__(self):\n return \"Token\" + (self and \".\" or \"\") + \".\".join(self)\n\n def __copy__(self):\n # These instances are supposed to be singletons\n return self\n\n def __deepcopy__(self, memo):\n # These instances are supposed to be singletons\n return self\n\n\nToken = _TokenType()\nColor = Token.Color\n\n\ndef partial_color_tokenize(template):\n \"\"\"Tokenizes a template string containing colors. Will return a list\n of tuples mapping the token to the string which has that color.\n These sub-strings maybe templates themselves.\n \"\"\"\n if HAS_PYGMENTS and builtins.__xonsh__.shell is not None:\n styles = __xonsh__.shell.shell.styler.styles\n elif builtins.__xonsh__.shell is not None:\n styles = DEFAULT_STYLE_DICT\n else:\n styles = None\n color = Color.NO_COLOR\n try:\n toks, color = _partial_color_tokenize_main(template, styles)\n except Exception:\n toks = [(Color.NO_COLOR, template)]\n if styles is not None:\n styles[color] # ensure color is available\n return toks\n\n\ndef _partial_color_tokenize_main(template, styles):\n bopen = \"{\"\n bclose = \"}\"\n colon = \":\"\n expl = \"!\"\n color = Color.NO_COLOR\n fg = bg = None\n value = \"\"\n toks = []\n for literal, field, spec, conv in FORMATTER.parse(template):\n if field is None:\n value += literal\n elif field in KNOWN_COLORS or \"#\" in field:\n value += literal\n next_color, fg, bg = color_by_name(field, fg, bg)\n if next_color is not color:\n if len(value) > 0:\n toks.append((color, value))\n if styles is not None:\n styles[color] # ensure color is available\n color = next_color\n value = \"\"\n elif field is not None:\n parts = [literal, bopen, field]\n if conv is not None and len(conv) > 0:\n parts.append(expl)\n parts.append(conv)\n if spec is not None and len(spec) > 0:\n parts.append(colon)\n parts.append(spec)\n parts.append(bclose)\n value += \"\".join(parts)\n else:\n value += literal\n toks.append((color, value))\n return toks, color\n\n\ndef color_by_name(name, fg=None, bg=None):\n \"\"\"Converts a color name to a color token, foreground name,\n and background name. Will take into consideration current foreground\n and background colors, if provided.\n\n Parameters\n ----------\n name : str\n Color name.\n fg : str, optional\n Foreground color name.\n bg : str, optional\n Background color name.\n\n Returns\n -------\n tok : Token\n Pygments Token.Color subclass\n fg : str or None\n New computed foreground color name.\n bg : str or None\n New computed background color name.\n \"\"\"\n name = name.upper()\n if name == \"NO_COLOR\":\n return Color.NO_COLOR, None, None\n m = RE_BACKGROUND.search(name)\n if m is None: # must be foreground color\n fg = norm_name(name)\n else:\n bg = norm_name(name)\n # assemble token\n if fg is None and bg is None:\n tokname = \"NO_COLOR\"\n elif fg is None:\n tokname = bg\n elif bg is None:\n tokname = fg\n else:\n tokname = fg + \"__\" + bg\n tok = getattr(Color, tokname)\n return tok, fg, bg\n\n\ndef norm_name(name):\n \"\"\"Normalizes a color name.\"\"\"\n return name.replace(\"#\", \"HEX\").replace(\"BGHEX\", \"BACKGROUND_HEX\")\n\n\nKNOWN_COLORS = LazyObject(\n lambda: frozenset(\n [\n \"BACKGROUND_BLACK\",\n \"BACKGROUND_BLUE\",\n \"BACKGROUND_CYAN\",\n \"BACKGROUND_GREEN\",\n \"BACKGROUND_INTENSE_BLACK\",\n \"BACKGROUND_INTENSE_BLUE\",\n \"BACKGROUND_INTENSE_CYAN\",\n \"BACKGROUND_INTENSE_GREEN\",\n \"BACKGROUND_INTENSE_PURPLE\",\n \"BACKGROUND_INTENSE_RED\",\n \"BACKGROUND_INTENSE_WHITE\",\n \"BACKGROUND_INTENSE_YELLOW\",\n \"BACKGROUND_PURPLE\",\n \"BACKGROUND_RED\",\n \"BACKGROUND_WHITE\",\n \"BACKGROUND_YELLOW\",\n \"BLACK\",\n \"BLUE\",\n \"BOLD_BLACK\",\n \"BOLD_BLUE\",\n \"BOLD_CYAN\",\n \"BOLD_GREEN\",\n \"BOLD_INTENSE_BLACK\",\n \"BOLD_INTENSE_BLUE\",\n \"BOLD_INTENSE_CYAN\",\n \"BOLD_INTENSE_GREEN\",\n \"BOLD_INTENSE_PURPLE\",\n \"BOLD_INTENSE_RED\",\n \"BOLD_INTENSE_WHITE\",\n \"BOLD_INTENSE_YELLOW\",\n \"BOLD_PURPLE\",\n \"BOLD_RED\",\n \"BOLD_UNDERLINE_BLACK\",\n \"BOLD_UNDERLINE_BLUE\",\n \"BOLD_UNDERLINE_CYAN\",\n \"BOLD_UNDERLINE_GREEN\",\n \"BOLD_UNDERLINE_INTENSE_BLACK\",\n \"BOLD_UNDERLINE_INTENSE_BLUE\",\n \"BOLD_UNDERLINE_INTENSE_CYAN\",\n \"BOLD_UNDERLINE_INTENSE_GREEN\",\n \"BOLD_UNDERLINE_INTENSE_PURPLE\",\n \"BOLD_UNDERLINE_INTENSE_RED\",\n \"BOLD_UNDERLINE_INTENSE_WHITE\",\n \"BOLD_UNDERLINE_INTENSE_YELLOW\",\n \"BOLD_UNDERLINE_PURPLE\",\n \"BOLD_UNDERLINE_RED\",\n \"BOLD_UNDERLINE_WHITE\",\n \"BOLD_UNDERLINE_YELLOW\",\n \"BOLD_WHITE\",\n \"BOLD_YELLOW\",\n \"CYAN\",\n \"GREEN\",\n \"INTENSE_BLACK\",\n \"INTENSE_BLUE\",\n \"INTENSE_CYAN\",\n \"INTENSE_GREEN\",\n \"INTENSE_PURPLE\",\n \"INTENSE_RED\",\n \"INTENSE_WHITE\",\n \"INTENSE_YELLOW\",\n \"NO_COLOR\",\n \"PURPLE\",\n \"RED\",\n \"UNDERLINE_BLACK\",\n \"UNDERLINE_BLUE\",\n \"UNDERLINE_CYAN\",\n \"UNDERLINE_GREEN\",\n \"UNDERLINE_INTENSE_BLACK\",\n \"UNDERLINE_INTENSE_BLUE\",\n \"UNDERLINE_INTENSE_CYAN\",\n \"UNDERLINE_INTENSE_GREEN\",\n \"UNDERLINE_INTENSE_PURPLE\",\n \"UNDERLINE_INTENSE_RED\",\n \"UNDERLINE_INTENSE_WHITE\",\n \"UNDERLINE_INTENSE_YELLOW\",\n \"UNDERLINE_PURPLE\",\n \"UNDERLINE_RED\",\n \"UNDERLINE_WHITE\",\n \"UNDERLINE_YELLOW\",\n \"WHITE\",\n \"YELLOW\",\n ]\n ),\n globals(),\n \"KNOWN_COLORS\",\n)\n\nDEFAULT_STYLE_DICT = LazyObject(\n lambda: defaultdict(\n lambda: \"\",\n {\n Token: \"\",\n Token.Aborted: \"ansibrightblack\",\n Token.AutoSuggestion: \"ansibrightblack\",\n Token.Color.BACKGROUND_BLACK: \"bg:ansiblack\",\n Token.Color.BACKGROUND_BLUE: \"bg:ansiblue\",\n Token.Color.BACKGROUND_CYAN: \"bg:ansicyan\",\n Token.Color.BACKGROUND_GREEN: \"bg:ansigreen\",\n Token.Color.BACKGROUND_INTENSE_BLACK: \"bg:ansibrightblack\",\n Token.Color.BACKGROUND_INTENSE_BLUE: \"bg:ansibrightblue\",\n Token.Color.BACKGROUND_INTENSE_CYAN: \"bg:ansibrightcyan\",\n Token.Color.BACKGROUND_INTENSE_GREEN: \"bg:ansibrightgreen\",\n Token.Color.BACKGROUND_INTENSE_PURPLE: \"bg:ansibrightmagenta\",\n Token.Color.BACKGROUND_INTENSE_RED: \"bg:ansibrightred\",\n Token.Color.BACKGROUND_INTENSE_WHITE: \"bg:ansiwhite\",\n Token.Color.BACKGROUND_INTENSE_YELLOW: \"bg:ansibrightyellow\",\n Token.Color.BACKGROUND_PURPLE: \"bg:ansimagenta\",\n Token.Color.BACKGROUND_RED: \"bg:ansired\",\n Token.Color.BACKGROUND_WHITE: \"bg:ansigray\",\n Token.Color.BACKGROUND_YELLOW: \"bg:ansiyellow\",\n Token.Color.BLACK: \"ansiblack\",\n Token.Color.BLUE: \"ansiblue\",\n Token.Color.BOLD_BLACK: \"bold ansiblack\",\n Token.Color.BOLD_BLUE: \"bold ansiblue\",\n Token.Color.BOLD_CYAN: \"bold ansicyan\",\n Token.Color.BOLD_GREEN: \"bold ansigreen\",\n Token.Color.BOLD_INTENSE_BLACK: \"bold ansibrightblack\",\n Token.Color.BOLD_INTENSE_BLUE: \"bold ansibrightblue\",\n Token.Color.BOLD_INTENSE_CYAN: \"bold ansibrightcyan\",\n Token.Color.BOLD_INTENSE_GREEN: \"bold ansibrightgreen\",\n Token.Color.BOLD_INTENSE_PURPLE: \"bold ansibrightmagenta\",\n Token.Color.BOLD_INTENSE_RED: \"bold ansibrightred\",\n Token.Color.BOLD_INTENSE_WHITE: \"bold ansiwhite\",\n Token.Color.BOLD_INTENSE_YELLOW: \"bold ansibrightyellow\",\n Token.Color.BOLD_PURPLE: \"bold ansimagenta\",\n Token.Color.BOLD_RED: \"bold ansired\",\n Token.Color.BOLD_UNDERLINE_BLACK: \"bold underline ansiblack\",\n Token.Color.BOLD_UNDERLINE_BLUE: \"bold underline ansiblue\",\n Token.Color.BOLD_UNDERLINE_CYAN: \"bold underline ansicyan\",\n Token.Color.BOLD_UNDERLINE_GREEN: \"bold underline ansigreen\",\n Token.Color.BOLD_UNDERLINE_INTENSE_BLACK: \"bold underline ansibrightblack\",\n Token.Color.BOLD_UNDERLINE_INTENSE_BLUE: \"bold underline ansibrightblue\",\n Token.Color.BOLD_UNDERLINE_INTENSE_CYAN: \"bold underline ansibrightcyan\",\n Token.Color.BOLD_UNDERLINE_INTENSE_GREEN: \"bold underline ansibrightgreen\",\n Token.Color.BOLD_UNDERLINE_INTENSE_PURPLE: \"bold underline ansibrightmagenta\",\n Token.Color.BOLD_UNDERLINE_INTENSE_RED: \"bold underline ansibrightred\",\n Token.Color.BOLD_UNDERLINE_INTENSE_WHITE: \"bold underline ansiwhite\",\n Token.Color.BOLD_UNDERLINE_INTENSE_YELLOW: \"bold underline ansibrightyellow\",\n Token.Color.BOLD_UNDERLINE_PURPLE: \"bold underline ansimagenta\",\n Token.Color.BOLD_UNDERLINE_RED: \"bold underline ansired\",\n Token.Color.BOLD_UNDERLINE_WHITE: \"bold underline ansigray\",\n Token.Color.BOLD_UNDERLINE_YELLOW: \"bold underline ansiyellow\",\n Token.Color.BOLD_WHITE: \"bold ansigray\",\n Token.Color.BOLD_YELLOW: \"bold ansiyellow\",\n Token.Color.CYAN: \"ansicyan\",\n Token.Color.GREEN: \"ansigreen\",\n Token.Color.INTENSE_BLACK: \"ansibrightblack\",\n Token.Color.INTENSE_BLUE: \"ansibrightblue\",\n Token.Color.INTENSE_CYAN: \"ansibrightcyan\",\n Token.Color.INTENSE_GREEN: \"ansibrightgreen\",\n Token.Color.INTENSE_PURPLE: \"ansibrightmagenta\",\n Token.Color.INTENSE_RED: \"ansibrightred\",\n Token.Color.INTENSE_WHITE: \"ansiwhite\",\n Token.Color.INTENSE_YELLOW: \"ansibrightyellow\",\n Token.Color.NO_COLOR: \"noinherit\",\n Token.Color.PURPLE: \"ansimagenta\",\n Token.Color.RED: \"ansired\",\n Token.Color.UNDERLINE_BLACK: \"underline ansiblack\",\n Token.Color.UNDERLINE_BLUE: \"underline ansiblue\",\n Token.Color.UNDERLINE_CYAN: \"underline ansicyan\",\n Token.Color.UNDERLINE_GREEN: \"underline ansigreen\",\n Token.Color.UNDERLINE_INTENSE_BLACK: \"underline ansibrightblack\",\n Token.Color.UNDERLINE_INTENSE_BLUE: \"underline ansibrightblue\",\n Token.Color.UNDERLINE_INTENSE_CYAN: \"underline ansibrightcyan\",\n Token.Color.UNDERLINE_INTENSE_GREEN: \"underline ansibrightgreen\",\n Token.Color.UNDERLINE_INTENSE_PURPLE: \"underline ansibrightmagenta\",\n Token.Color.UNDERLINE_INTENSE_RED: \"underline ansibrightred\",\n Token.Color.UNDERLINE_INTENSE_WHITE: \"underline ansiwhite\",\n Token.Color.UNDERLINE_INTENSE_YELLOW: \"underline ansibrightyellow\",\n Token.Color.UNDERLINE_PURPLE: \"underline ansimagenta\",\n Token.Color.UNDERLINE_RED: \"underline ansired\",\n Token.Color.UNDERLINE_WHITE: \"underline ansigray\",\n Token.Color.UNDERLINE_YELLOW: \"underline ansiyellow\",\n Token.Color.WHITE: \"ansigray\",\n Token.Color.YELLOW: \"ansiyellow\",\n Token.Comment: \"underline ansicyan\",\n Token.Comment.Hashbang: \"\",\n Token.Comment.Multiline: \"\",\n Token.Comment.Preproc: \"underline ansiyellow\",\n Token.Comment.PreprocFile: \"\",\n Token.Comment.Single: \"\",\n Token.Comment.Special: \"\",\n Token.Error: \"ansibrightred\",\n Token.Escape: \"\",\n Token.Generic: \"\",\n Token.Generic.Deleted: \"ansired\",\n Token.Generic.Emph: \"underline\",\n Token.Generic.Error: \"bold ansibrightred\",\n Token.Generic.Heading: \"bold ansiblue\",\n Token.Generic.Inserted: \"ansibrightgreen\",\n Token.Generic.Output: \"ansiblue\",\n Token.Generic.Prompt: \"bold ansiblue\",\n Token.Generic.Strong: \"\",\n Token.Generic.Subheading: \"bold ansimagenta\",\n Token.Generic.Traceback: \"ansiblue\",\n Token.Keyword: \"bold ansigreen\",\n Token.Keyword.Constant: \"\",\n Token.Keyword.Declaration: \"\",\n Token.Keyword.Namespace: \"\",\n Token.Keyword.Pseudo: \"nobold\",\n Token.Keyword.Reserved: \"\",\n Token.Keyword.Type: \"nobold ansired\",\n Token.Literal: \"\",\n Token.Literal.Date: \"\",\n Token.Literal.Number: \"ansibrightblack\",\n Token.Literal.Number.Bin: \"\",\n Token.Literal.Number.Float: \"\",\n Token.Literal.Number.Hex: \"\",\n Token.Literal.Number.Integer: \"\",\n Token.Literal.Number.Integer.Long: \"\",\n Token.Literal.Number.Oct: \"\",\n Token.Literal.String: \"ansibrightred\",\n Token.Literal.String.Affix: \"\",\n Token.Literal.String.Backtick: \"\",\n Token.Literal.String.Char: \"\",\n Token.Literal.String.Delimiter: \"\",\n Token.Literal.String.Doc: \"underline\",\n Token.Literal.String.Double: \"\",\n Token.Literal.String.Escape: \"bold ansiyellow\",\n Token.Literal.String.Heredoc: \"\",\n Token.Literal.String.Interpol: \"bold ansimagenta\",\n Token.Literal.String.Other: \"ansigreen\",\n Token.Literal.String.Regex: \"ansimagenta\",\n Token.Literal.String.Single: \"\",\n Token.Literal.String.Symbol: \"ansiyellow\",\n Token.Menu.Completions: \"bg:ansigray ansiblack\",\n Token.Menu.Completions.Completion: \"\",\n Token.Menu.Completions.Completion.Current: \"bg:ansibrightblack ansiwhite\",\n Token.Name: \"\",\n Token.Name.Attribute: \"ansibrightyellow\",\n Token.Name.Builtin: \"ansigreen\",\n Token.Name.Builtin.Pseudo: \"\",\n Token.Name.Class: \"bold ansibrightblue\",\n Token.Name.Constant: \"ansired\",\n Token.Name.Decorator: \"ansibrightmagenta\",\n Token.Name.Entity: \"bold ansigray\",\n Token.Name.Exception: \"bold ansibrightred\",\n Token.Name.Function: \"ansibrightblue\",\n Token.Name.Function.Magic: \"\",\n Token.Name.Label: \"ansibrightyellow\",\n Token.Name.Namespace: \"bold ansibrightblue\",\n Token.Name.Other: \"\",\n Token.Name.Property: \"\",\n Token.Name.Tag: \"bold ansigreen\",\n Token.Name.Variable: \"ansiblue\",\n Token.Name.Variable.Class: \"\",\n Token.Name.Variable.Global: \"\",\n Token.Name.Variable.Instance: \"\",\n Token.Name.Variable.Magic: \"\",\n Token.Operator: \"ansibrightblack\",\n Token.Operator.Word: \"bold ansimagenta\",\n Token.Other: \"\",\n Token.Punctuation: \"\",\n Token.Scrollbar: \"bg:ansibrightblack\",\n Token.Scrollbar.Arrow: \"bg:ansiblack ansiwhite bold\",\n Token.Scrollbar.Button: \"bg:ansiblack\",\n Token.Text: \"\",\n Token.Text.Whitespace: \"ansigray\",\n },\n ),\n globals(),\n \"DEFAULT_STYLE_DICT\",\n)\n\nPTK2_STYLE = {\n \"completion-menu\": \"bg:ansicyan ansiwhite\",\n \"completion-menu.completion\": \"bg:#008888 #ffffff\",\n \"completion-menu.completion.current\": \"bg:ansibrightblack ansiwhite\",\n \"completion-menu.meta.completion\": \"bg:#00aaaa #ffffff\",\n \"completion-menu.meta.completion.current\": \"bg:#00aaaa #000000\",\n \"scrollbar.background\": \"bg:ansibrightblack\",\n \"scrollbar.arrow\": \"bg:ansiblack ansiwhite bold\",\n \"scrollbar.button\": \"bg:ansiblack\",\n}\n" }, { "path": "xonsh/timings.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"Timing related functionality for the xonsh shell.\n\nThe following time_it alias and Timer was forked from the IPython project:\n* Copyright (c) 2008-2014, IPython Development Team\n* Copyright (C) 2001-2007 Fernando Perez \n* Copyright (c) 2001, Janko Hauser \n* Copyright (c) 2001, Nathaniel Gray \n\"\"\"\nimport os\nimport gc\nimport sys\nimport math\nimport time\nimport timeit\nimport builtins\nimport itertools\n\nfrom xonsh.lazyasd import lazyobject, lazybool\nfrom xonsh.events import events\nfrom xonsh.platform import ON_WINDOWS\n\n\n@lazybool\ndef _HAVE_RESOURCE():\n try:\n import resource as r\n\n have = True\n except ImportError:\n # There is no distinction of user/system time under windows, so we\n # just use time.perf_counter() for everything...\n have = False\n return have\n\n\n@lazyobject\ndef resource():\n import resource as r\n\n return r\n\n\n@lazyobject\ndef clocku():\n if _HAVE_RESOURCE:\n\n def clocku():\n \"\"\"clocku() -> floating point number\n Return the *USER* CPU time in seconds since the start of the\n process.\"\"\"\n return resource.getrusage(resource.RUSAGE_SELF)[0]\n\n else:\n clocku = time.perf_counter\n return clocku\n\n\n@lazyobject\ndef clocks():\n if _HAVE_RESOURCE:\n\n def clocks():\n \"\"\"clocks() -> floating point number\n Return the *SYSTEM* CPU time in seconds since the start of the\n process.\"\"\"\n return resource.getrusage(resource.RUSAGE_SELF)[1]\n\n else:\n clocks = time.perf_counter\n return clocks\n\n\n@lazyobject\ndef clock():\n if _HAVE_RESOURCE:\n\n def clock():\n \"\"\"clock() -> floating point number\n Return the *TOTAL USER+SYSTEM* CPU time in seconds since the\n start of the process.\"\"\"\n u, s = resource.getrusage(resource.RUSAGE_SELF)[:2]\n return u + s\n\n else:\n clock = time.perf_counter\n return clock\n\n\n@lazyobject\ndef clock2():\n if _HAVE_RESOURCE:\n\n def clock2():\n \"\"\"clock2() -> (t_user,t_system)\n Similar to clock(), but return a tuple of user/system times.\"\"\"\n return resource.getrusage(resource.RUSAGE_SELF)[:2]\n\n else:\n\n def clock2():\n \"\"\"Under windows, system CPU time can't be measured.\n This just returns perf_counter() and zero.\"\"\"\n return time.perf_counter(), 0.0\n\n return clock2\n\n\ndef format_time(timespan, precision=3):\n \"\"\"Formats the timespan in a human readable form\"\"\"\n if timespan >= 60.0:\n # we have more than a minute, format that in a human readable form\n parts = [(\"d\", 60 * 60 * 24), (\"h\", 60 * 60), (\"min\", 60), (\"s\", 1)]\n time = []\n leftover = timespan\n for suffix, length in parts:\n value = int(leftover / length)\n if value > 0:\n leftover = leftover % length\n time.append(\"{0}{1}\".format(str(value), suffix))\n if leftover < 1:\n break\n return \" \".join(time)\n # Unfortunately the unicode 'micro' symbol can cause problems in\n # certain terminals.\n # See bug: https://bugs.launchpad.net/ipython/+bug/348466\n # Try to prevent crashes by being more secure than it needs to\n # E.g. eclipse is able to print a mu, but has no sys.stdout.encoding set.\n units = [\"s\", \"ms\", \"us\", \"ns\"] # the save value\n if hasattr(sys.stdout, \"encoding\") and sys.stdout.encoding:\n try:\n \"\\xb5\".encode(sys.stdout.encoding)\n units = [\"s\", \"ms\", \"\\xb5s\", \"ns\"]\n except Exception:\n pass\n scaling = [1, 1e3, 1e6, 1e9]\n\n if timespan > 0.0:\n order = min(-int(math.floor(math.log10(timespan)) // 3), 3)\n else:\n order = 3\n return \"{1:.{0}g} {2}\".format(precision, timespan * scaling[order], units[order])\n\n\nclass Timer(timeit.Timer):\n \"\"\"Timer class that explicitly uses self.inner\n which is an undocumented implementation detail of CPython,\n not shared by PyPy.\n \"\"\"\n\n # Timer.timeit copied from CPython 3.4.2\n def timeit(self, number=timeit.default_number):\n \"\"\"Time 'number' executions of the main statement.\n To be precise, this executes the setup statement once, and\n then returns the time it takes to execute the main statement\n a number of times, as a float measured in seconds. The\n argument is the number of times through the loop, defaulting\n to one million. The main statement, the setup statement and\n the timer function to be used are passed to the constructor.\n \"\"\"\n it = itertools.repeat(None, number)\n gcold = gc.isenabled()\n gc.disable()\n try:\n timing = self.inner(it, self.timer)\n finally:\n if gcold:\n gc.enable()\n return timing\n\n\nINNER_TEMPLATE = \"\"\"\ndef inner(_it, _timer):\n #setup\n _t0 = _timer()\n for _i in _it:\n {stmt}\n _t1 = _timer()\n return _t1 - _t0\n\"\"\"\n\n\ndef timeit_alias(args, stdin=None):\n \"\"\"Runs timing study on arguments.\"\"\"\n # some real args\n number = 0\n quiet = False\n repeat = 3\n precision = 3\n # setup\n ctx = builtins.__xonsh__.ctx\n timer = Timer(timer=clock)\n stmt = \" \".join(args)\n innerstr = INNER_TEMPLATE.format(stmt=stmt)\n # Track compilation time so it can be reported if too long\n # Minimum time above which compilation time will be reported\n tc_min = 0.1\n t0 = clock()\n innercode = builtins.compilex(\n innerstr, filename=\"\", mode=\"exec\", glbs=ctx\n )\n tc = clock() - t0\n # get inner func\n ns = {}\n builtins.execx(innercode, glbs=ctx, locs=ns, mode=\"exec\")\n timer.inner = ns[\"inner\"]\n # Check if there is a huge difference between the best and worst timings.\n worst_tuning = 0\n if number == 0:\n # determine number so that 0.2 <= total time < 2.0\n number = 1\n for _ in range(1, 10):\n time_number = timer.timeit(number)\n worst_tuning = max(worst_tuning, time_number / number)\n if time_number >= 0.2:\n break\n number *= 10\n all_runs = timer.repeat(repeat, number)\n best = min(all_runs) / number\n # print some debug info\n if not quiet:\n worst = max(all_runs) / number\n if worst_tuning:\n worst = max(worst, worst_tuning)\n # Check best timing is greater than zero to avoid a\n # ZeroDivisionError.\n # In cases where the slowest timing is less than 10 microseconds\n # we assume that it does not really matter if the fastest\n # timing is 4 times faster than the slowest timing or not.\n if worst > 4 * best and best > 0 and worst > 1e-5:\n print(\n (\n \"The slowest run took {0:0.2f} times longer than the \"\n \"fastest. This could mean that an intermediate result \"\n \"is being cached.\"\n ).format(worst / best)\n )\n print(\n \"{0} loops, best of {1}: {2} per loop\".format(\n number, repeat, format_time(best, precision)\n )\n )\n if tc > tc_min:\n print(\"Compiler time: {0:.2f} s\".format(tc))\n return\n\n\n_timings = {\"start\": clock()}\n\n\ndef setup_timings(argv):\n global _timings\n if \"--timings\" in argv:\n events.doc(\n \"on_timingprobe\",\n \"\"\"\n on_timingprobe(name: str) -> None\n\n Fired to insert some timings into the startuptime list\n \"\"\",\n )\n\n @events.on_timingprobe\n def timing_on_timingprobe(name, **kw):\n global _timings\n _timings[name] = clock()\n\n @events.on_post_cmdloop\n def timing_on_post_cmdloop(**kw):\n global _timings\n _timings[\"on_post_cmdloop\"] = clock()\n\n @events.on_post_init\n def timing_on_post_init(**kw):\n global _timings\n _timings[\"on_post_init\"] = clock()\n\n @events.on_post_rc\n def timing_on_post_rc(**kw):\n global _timings\n _timings[\"on_post_rc\"] = clock()\n\n @events.on_postcommand\n def timing_on_postcommand(**kw):\n global _timings\n _timings[\"on_postcommand\"] = clock()\n\n @events.on_pre_cmdloop\n def timing_on_pre_cmdloop(**kw):\n global _timings\n _timings[\"on_pre_cmdloop\"] = clock()\n\n @events.on_pre_rc\n def timing_on_pre_rc(**kw):\n global _timings\n _timings[\"on_pre_rc\"] = clock()\n\n @events.on_precommand\n def timing_on_precommand(**kw):\n global _timings\n _timings[\"on_precommand\"] = clock()\n\n @events.on_ptk_create\n def timing_on_ptk_create(**kw):\n global _timings\n _timings[\"on_ptk_create\"] = clock()\n\n @events.on_chdir\n def timing_on_chdir(**kw):\n global _timings\n _timings[\"on_chdir\"] = clock()\n\n @events.on_post_prompt\n def timing_on_post_prompt(**kw):\n global _timings\n _timings = {\"on_post_prompt\": clock()}\n\n @events.on_pre_prompt\n def timing_on_pre_prompt(**kw):\n global _timings\n _timings[\"on_pre_prompt\"] = clock()\n times = list(_timings.items())\n times = sorted(times, key=lambda x: x[1])\n width = max(len(s) for s, _ in times) + 2\n header_format = \"|{{:<{}}}|{{:^11}}|{{:^11}}|\".format(width)\n entry_format = \"|{{:<{}}}|{{:^11.3f}}|{{:^11.3f}}|\".format(width)\n sepline = \"|{}|{}|{}|\".format(\"-\" * width, \"-\" * 11, \"-\" * 11)\n # Print result table\n print(\" Debug level: {}\".format(os.getenv(\"XONSH_DEBUG\", \"Off\")))\n print(sepline)\n print(header_format.format(\"Event name\", \"Time (s)\", \"Delta (s)\"))\n print(sepline)\n prevtime = tstart = times[0][1]\n for name, ts in times:\n print(entry_format.format(name, ts - tstart, ts - prevtime))\n prevtime = ts\n print(sepline)\n" }, { "path": "xonsh/tokenize.py", "content": "\"\"\"Tokenization help for xonsh programs.\n\nThis file is a modified version of tokenize.py form the Python 3.4 and 3.5\nstandard libraries (licensed under the Python Software Foundation License,\nversion 2), which provides tokenization help for Python programs.\n\nIt is modified to properly tokenize xonsh code, including backtick regex\npath and several xonsh-specific operators.\n\nA few pieces of this file are specific to the version of Python being used.\nTo find these pieces, search the PY35.\n\nOriginal file credits:\n __author__ = 'Ka-Ping Yee '\n __credits__ = ('GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, '\n 'Skip Montanaro, Raymond Hettinger, Trent Nelson, '\n 'Michael Foord')\n\"\"\"\n\nimport re\nimport io\nimport sys\nimport codecs\nimport builtins\nimport itertools\nimport collections\nimport token\nfrom token import (\n AMPER,\n AMPEREQUAL,\n AT,\n CIRCUMFLEX,\n CIRCUMFLEXEQUAL,\n COLON,\n COMMA,\n DEDENT,\n DOT,\n DOUBLESLASH,\n DOUBLESLASHEQUAL,\n DOUBLESTAR,\n DOUBLESTAREQUAL,\n ENDMARKER,\n EQEQUAL,\n EQUAL,\n ERRORTOKEN,\n GREATER,\n GREATEREQUAL,\n INDENT,\n LBRACE,\n LEFTSHIFT,\n LEFTSHIFTEQUAL,\n LESS,\n LESSEQUAL,\n LPAR,\n LSQB,\n MINEQUAL,\n MINUS,\n NAME,\n NEWLINE,\n NOTEQUAL,\n NUMBER,\n N_TOKENS,\n OP,\n PERCENT,\n PERCENTEQUAL,\n PLUS,\n PLUSEQUAL,\n RBRACE,\n RIGHTSHIFT,\n RIGHTSHIFTEQUAL,\n RPAR,\n RSQB,\n SEMI,\n SLASH,\n SLASHEQUAL,\n STAR,\n STAREQUAL,\n STRING,\n TILDE,\n VBAR,\n VBAREQUAL,\n tok_name,\n)\n\nfrom xonsh.lazyasd import LazyObject\nfrom xonsh.platform import PYTHON_VERSION_INFO\n\ncookie_re = LazyObject(\n lambda: re.compile(r\"^[ \\t\\f]*#.*coding[:=][ \\t]*([-\\w.]+)\", re.ASCII),\n globals(),\n \"cookie_re\",\n)\nblank_re = LazyObject(\n lambda: re.compile(br\"^[ \\t\\f]*(?:[#\\r\\n]|$)\", re.ASCII), globals(), \"blank_re\"\n)\n\n#\n# token modifications\n#\ntok_name = tok_name.copy()\n__all__ = token.__all__ + [\n \"COMMENT\",\n \"tokenize\",\n \"detect_encoding\",\n \"NL\",\n \"untokenize\",\n \"ENCODING\",\n \"TokenInfo\",\n \"TokenError\",\n \"SEARCHPATH\",\n \"ATDOLLAR\",\n \"ATEQUAL\",\n \"DOLLARNAME\",\n \"IOREDIRECT\",\n]\nHAS_ASYNC = (3, 5, 0) <= PYTHON_VERSION_INFO < (3, 7, 0)\nif HAS_ASYNC:\n ASYNC = token.ASYNC\n AWAIT = token.AWAIT\n ADDSPACE_TOKS = (NAME, NUMBER, ASYNC, AWAIT)\nelse:\n ADDSPACE_TOKS = (NAME, NUMBER)\ndel token # must clean up token\nPY35 = (3, 5, 0) <= PYTHON_VERSION_INFO\nAUGASSIGN_OPS = r\"[+\\-*/%&@|^=<>]=?\"\nif not PY35:\n AUGASSIGN_OPS = AUGASSIGN_OPS.replace(\"@\", \"\")\n\n\nCOMMENT = N_TOKENS\ntok_name[COMMENT] = \"COMMENT\"\nNL = N_TOKENS + 1\ntok_name[NL] = \"NL\"\nENCODING = N_TOKENS + 2\ntok_name[ENCODING] = \"ENCODING\"\nN_TOKENS += 3\nSEARCHPATH = N_TOKENS\ntok_name[N_TOKENS] = \"SEARCHPATH\"\nN_TOKENS += 1\nIOREDIRECT = N_TOKENS\ntok_name[N_TOKENS] = \"IOREDIRECT\"\nN_TOKENS += 1\nDOLLARNAME = N_TOKENS\ntok_name[N_TOKENS] = \"DOLLARNAME\"\nN_TOKENS += 1\nATDOLLAR = N_TOKENS\ntok_name[N_TOKENS] = \"ATDOLLAR\"\nN_TOKENS += 1\nATEQUAL = N_TOKENS\ntok_name[N_TOKENS] = \"ATEQUAL\"\nN_TOKENS += 1\n_xonsh_tokens = {\n \"?\": \"QUESTION\",\n \"@=\": \"ATEQUAL\",\n \"@$\": \"ATDOLLAR\",\n \"||\": \"DOUBLEPIPE\",\n \"&&\": \"DOUBLEAMPER\",\n \"@(\": \"ATLPAREN\",\n \"!(\": \"BANGLPAREN\",\n \"![\": \"BANGLBRACKET\",\n \"$(\": \"DOLLARLPAREN\",\n \"$[\": \"DOLLARLBRACKET\",\n \"${\": \"DOLLARLBRACE\",\n \"??\": \"DOUBLEQUESTION\",\n \"@$(\": \"ATDOLLARLPAREN\",\n}\n\nadditional_parenlevs = frozenset({\"@(\", \"!(\", \"![\", \"$(\", \"$[\", \"${\", \"@$(\"})\n\n_glbs = globals()\nfor v in _xonsh_tokens.values():\n _glbs[v] = N_TOKENS\n tok_name[N_TOKENS] = v\n N_TOKENS += 1\n __all__.append(v)\ndel _glbs, v\n\nEXACT_TOKEN_TYPES = {\n \"(\": LPAR,\n \")\": RPAR,\n \"[\": LSQB,\n \"]\": RSQB,\n \":\": COLON,\n \",\": COMMA,\n \";\": SEMI,\n \"+\": PLUS,\n \"-\": MINUS,\n \"*\": STAR,\n \"/\": SLASH,\n \"|\": VBAR,\n \"&\": AMPER,\n \"<\": LESS,\n \">\": GREATER,\n \"=\": EQUAL,\n \".\": DOT,\n \"%\": PERCENT,\n \"{\": LBRACE,\n \"}\": RBRACE,\n \"==\": EQEQUAL,\n \"!=\": NOTEQUAL,\n \"<=\": LESSEQUAL,\n \">=\": GREATEREQUAL,\n \"~\": TILDE,\n \"^\": CIRCUMFLEX,\n \"<<\": LEFTSHIFT,\n \">>\": RIGHTSHIFT,\n \"**\": DOUBLESTAR,\n \"+=\": PLUSEQUAL,\n \"-=\": MINEQUAL,\n \"*=\": STAREQUAL,\n \"/=\": SLASHEQUAL,\n \"%=\": PERCENTEQUAL,\n \"&=\": AMPEREQUAL,\n \"|=\": VBAREQUAL,\n \"^=\": CIRCUMFLEXEQUAL,\n \"<<=\": LEFTSHIFTEQUAL,\n \">>=\": RIGHTSHIFTEQUAL,\n \"**=\": DOUBLESTAREQUAL,\n \"//\": DOUBLESLASH,\n \"//=\": DOUBLESLASHEQUAL,\n \"@\": AT,\n}\n\nEXACT_TOKEN_TYPES.update(_xonsh_tokens)\n\n\nclass TokenInfo(collections.namedtuple(\"TokenInfo\", \"type string start end line\")):\n def __repr__(self):\n annotated_type = \"%d (%s)\" % (self.type, tok_name[self.type])\n return (\n \"TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r)\"\n % self._replace(type=annotated_type)\n )\n\n @property\n def exact_type(self):\n if self.type == OP and self.string in EXACT_TOKEN_TYPES:\n return EXACT_TOKEN_TYPES[self.string]\n else:\n return self.type\n\n\ndef group(*choices):\n return \"(\" + \"|\".join(choices) + \")\"\n\n\ndef tokany(*choices):\n return group(*choices) + \"*\"\n\n\ndef maybe(*choices):\n return group(*choices) + \"?\"\n\n\n# Note: we use unicode matching for names (\"\\w\") but ascii matching for\n# number literals.\nWhitespace = r\"[ \\f\\t]*\"\nComment = r\"#[^\\r\\n]*\"\nIgnore = Whitespace + tokany(r\"\\\\\\r?\\n\" + Whitespace) + maybe(Comment)\nName_RE = r\"\\$?\\w+\"\n\nHexnumber = r\"0[xX](?:_?[0-9a-fA-F])+\"\nBinnumber = r\"0[bB](?:_?[01])+\"\nOctnumber = r\"0[oO](?:_?[0-7])+\"\nDecnumber = r\"(?:0(?:_?0)*|[1-9](?:_?[0-9])*)\"\nIntnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber)\nExponent = r\"[eE][-+]?[0-9](?:_?[0-9])*\"\nPointfloat = group(\n r\"[0-9](?:_?[0-9])*\\.(?:[0-9](?:_?[0-9])*)?\", r\"\\.[0-9](?:_?[0-9])*\"\n) + maybe(Exponent)\nExpfloat = r\"[0-9](?:_?[0-9])*\" + Exponent\nFloatnumber = group(Pointfloat, Expfloat)\nImagnumber = group(r\"[0-9](?:_?[0-9])*[jJ]\", Floatnumber + r\"[jJ]\")\nNumber = group(Imagnumber, Floatnumber, Intnumber)\n\nStringPrefix = r\"(?:[bB][rR]?|[p][fFrR]?|[rR][bBpfF]?|[uU]|[fF][rR]?[p]?)?\"\n\n# Tail end of ' string.\nSingle = r\"[^'\\\\]*(?:\\\\.[^'\\\\]*)*'\"\n# Tail end of \" string.\nDouble = r'[^\"\\\\]*(?:\\\\.[^\"\\\\]*)*\"'\n# Tail end of ''' string.\nSingle3 = r\"[^'\\\\]*(?:(?:\\\\.|'(?!''))[^'\\\\]*)*'''\"\n# Tail end of \"\"\" string.\nDouble3 = r'[^\"\\\\]*(?:(?:\\\\.|\"(?!\"\"))[^\"\\\\]*)*\"\"\"'\nTriple = group(StringPrefix + \"'''\", StringPrefix + '\"\"\"')\n# Single-line ' or \" string.\nString = group(\n StringPrefix + r\"'[^\\n'\\\\]*(?:\\\\.[^\\n'\\\\]*)*'\",\n StringPrefix + r'\"[^\\n\"\\\\]*(?:\\\\.[^\\n\"\\\\]*)*\"',\n)\n\n# Xonsh-specific Syntax\nSearchPath = r\"((?:[rgp]+|@\\w*)?)`([^\\n`\\\\]*(?:\\\\.[^\\n`\\\\]*)*)`\"\n\n# Because of leftmost-then-longest match semantics, be sure to put the\n# longest operators first (e.g., if = came before ==, == would get\n# recognized as two instances of =).\n_redir_names = (\"out\", \"all\", \"err\", \"e\", \"2\", \"a\", \"&\", \"1\", \"o\")\n_redir_map = (\n # stderr to stdout\n \"err>out\",\n \"err>&1\",\n \"2>out\",\n \"err>o\",\n \"err>1\",\n \"e>out\",\n \"e>&1\",\n \"2>&1\",\n \"e>o\",\n \"2>o\",\n \"e>1\",\n \"2>1\",\n # stdout to stderr\n \"out>err\",\n \"out>&2\",\n \"1>err\",\n \"out>e\",\n \"out>2\",\n \"o>err\",\n \"o>&2\",\n \"1>&2\",\n \"o>e\",\n \"1>e\",\n \"o>2\",\n \"1>2\",\n)\nIORedirect = group(group(*_redir_map), \"{}>>?\".format(group(*_redir_names)))\n_redir_check = set(_redir_map)\n_redir_check = {\"{}>\".format(i) for i in _redir_names}.union(_redir_check)\n_redir_check = {\"{}>>\".format(i) for i in _redir_names}.union(_redir_check)\n_redir_check = frozenset(_redir_check)\nOperator = group(\n r\"\\*\\*=?\",\n r\">>=?\",\n r\"<<=?\",\n r\"!=\",\n r\"//=?\",\n r\"->\",\n r\"@\\$\\(?\",\n r\"\\|\\|\",\n \"&&\",\n r\"@\\(\",\n r\"!\\(\",\n r\"!\\[\",\n r\"\\$\\(\",\n r\"\\$\\[\",\n r\"\\${\",\n r\"\\?\\?\",\n r\"\\?\",\n AUGASSIGN_OPS,\n r\"~\",\n)\n\nBracket = \"[][(){}]\"\nSpecial = group(r\"\\r?\\n\", r\"\\.\\.\\.\", r\"[:;.,@]\")\nFunny = group(Operator, Bracket, Special)\n\nPlainToken = group(IORedirect, Number, Funny, String, Name_RE, SearchPath)\n\n# First (or only) line of ' or \" string.\nContStr = group(\n StringPrefix + r\"'[^\\n'\\\\]*(?:\\\\.[^\\n'\\\\]*)*\" + group(\"'\", r\"\\\\\\r?\\n\"),\n StringPrefix + r'\"[^\\n\"\\\\]*(?:\\\\.[^\\n\"\\\\]*)*' + group('\"', r\"\\\\\\r?\\n\"),\n)\nPseudoExtras = group(r\"\\\\\\r?\\n|\\Z\", Comment, Triple, SearchPath)\nPseudoToken = Whitespace + group(\n PseudoExtras, IORedirect, Number, Funny, ContStr, Name_RE\n)\n\n\ndef _compile(expr):\n return re.compile(expr, re.UNICODE)\n\n\nendpats = {\n \"'\": Single,\n '\"': Double,\n \"'''\": Single3,\n '\"\"\"': Double3,\n \"r'''\": Single3,\n 'r\"\"\"': Double3,\n \"b'''\": Single3,\n 'b\"\"\"': Double3,\n \"f'''\": Single3,\n 'f\"\"\"': Double3,\n \"R'''\": Single3,\n 'R\"\"\"': Double3,\n \"B'''\": Single3,\n 'B\"\"\"': Double3,\n \"F'''\": Single3,\n 'F\"\"\"': Double3,\n \"br'''\": Single3,\n 'br\"\"\"': Double3,\n \"fr'''\": Single3,\n 'fr\"\"\"': Double3,\n \"fp'''\": Single3,\n 'fp\"\"\"': Double3,\n \"bR'''\": Single3,\n 'bR\"\"\"': Double3,\n \"Br'''\": Single3,\n 'Br\"\"\"': Double3,\n \"BR'''\": Single3,\n 'BR\"\"\"': Double3,\n \"rb'''\": Single3,\n 'rb\"\"\"': Double3,\n \"rf'''\": Single3,\n 'rf\"\"\"': Double3,\n \"Rb'''\": Single3,\n 'Rb\"\"\"': Double3,\n \"Fr'''\": Single3,\n 'Fr\"\"\"': Double3,\n \"Fp'''\": Single3,\n 'Fp\"\"\"': Double3,\n \"rB'''\": Single3,\n 'rB\"\"\"': Double3,\n \"rF'''\": Single3,\n 'rF\"\"\"': Double3,\n \"RB'''\": Single3,\n 'RB\"\"\"': Double3,\n \"RF'''\": Single3,\n 'RF\"\"\"': Double3,\n \"u'''\": Single3,\n 'u\"\"\"': Double3,\n \"U'''\": Single3,\n 'U\"\"\"': Double3,\n \"p'''\": Single3,\n 'p\"\"\"': Double3,\n \"pr'''\": Single3,\n 'pr\"\"\"': Double3,\n \"pf'''\": Single3,\n 'pf\"\"\"': Double3,\n \"pF'''\": Single3,\n 'pF\"\"\"': Double3,\n \"pR'''\": Single3,\n 'pR\"\"\"': Double3,\n \"rp'''\": Single3,\n 'rp\"\"\"': Double3,\n \"Rp'''\": Single3,\n 'Rp\"\"\"': Double3,\n \"r\": None,\n \"R\": None,\n \"b\": None,\n \"B\": None,\n \"u\": None,\n \"U\": None,\n \"p\": None,\n \"f\": None,\n \"F\": None,\n}\n\ntriple_quoted = {}\nfor t in (\n \"'''\",\n '\"\"\"',\n \"r'''\",\n 'r\"\"\"',\n \"R'''\",\n 'R\"\"\"',\n \"b'''\",\n 'b\"\"\"',\n \"B'''\",\n 'B\"\"\"',\n \"f'''\",\n 'f\"\"\"',\n \"F'''\",\n 'F\"\"\"',\n \"br'''\",\n 'br\"\"\"',\n \"Br'''\",\n 'Br\"\"\"',\n \"bR'''\",\n 'bR\"\"\"',\n \"BR'''\",\n 'BR\"\"\"',\n \"rb'''\",\n 'rb\"\"\"',\n \"rB'''\",\n 'rB\"\"\"',\n \"Rb'''\",\n 'Rb\"\"\"',\n \"RB'''\",\n 'RB\"\"\"',\n \"fr'''\",\n 'fr\"\"\"',\n \"Fr'''\",\n 'Fr\"\"\"',\n \"fR'''\",\n 'fR\"\"\"',\n \"FR'''\",\n 'FR\"\"\"',\n \"rf'''\",\n 'rf\"\"\"',\n \"rF'''\",\n 'rF\"\"\"',\n \"Rf'''\",\n 'Rf\"\"\"',\n \"RF'''\",\n 'RF\"\"\"',\n \"u'''\",\n 'u\"\"\"',\n \"U'''\",\n 'U\"\"\"',\n \"p'''\",\n 'p\"\"\"\"',\n \"pr'''\",\n 'pr\"\"\"\"',\n \"pR'''\",\n 'pR\"\"\"\"',\n \"rp'''\",\n 'rp\"\"\"\"',\n \"Rp'''\",\n 'Rp\"\"\"\"',\n \"pf'''\",\n 'pf\"\"\"\"',\n \"pF'''\",\n 'pF\"\"\"\"',\n \"fp'''\",\n 'fp\"\"\"\"',\n \"Fp'''\",\n 'Fp\"\"\"\"',\n):\n triple_quoted[t] = t\nsingle_quoted = {}\nfor t in (\n \"'\",\n '\"',\n \"r'\",\n 'r\"',\n \"R'\",\n 'R\"',\n \"b'\",\n 'b\"',\n \"B'\",\n 'B\"',\n \"f'\",\n 'f\"',\n \"F'\",\n 'F\"',\n \"br'\",\n 'br\"',\n \"Br'\",\n 'Br\"',\n \"bR'\",\n 'bR\"',\n \"BR'\",\n 'BR\"',\n \"rb'\",\n 'rb\"',\n \"rB'\",\n 'rB\"',\n \"Rb'\",\n 'Rb\"',\n \"RB'\",\n 'RB\"',\n \"fr'\",\n 'fr\"',\n \"Fr'\",\n 'Fr\"',\n \"fR'\",\n 'fR\"',\n \"FR'\",\n 'FR\"',\n \"rf'\",\n 'rf\"',\n \"rF'\",\n 'rF\"',\n \"Rf'\",\n 'Rf\"',\n \"RF'\",\n 'RF\"',\n \"u'\",\n 'u\"',\n \"U'\",\n 'U\"',\n \"p'\",\n 'p\"',\n \"pr'\",\n 'pr\"',\n \"pR'\",\n 'pR\"',\n \"rp'\",\n 'rp\"',\n \"Rp'\",\n 'Rp\"',\n \"pf'\",\n 'pf\"',\n \"pF'\",\n 'pF\"',\n \"fp'\",\n 'fp\"',\n \"Fp'\",\n 'Fp\"',\n):\n single_quoted[t] = t\n\ntabsize = 8\n\n\nclass TokenError(Exception):\n pass\n\n\nclass StopTokenizing(Exception):\n pass\n\n\nclass Untokenizer:\n def __init__(self):\n self.tokens = []\n self.prev_row = 1\n self.prev_col = 0\n self.encoding = None\n\n def add_whitespace(self, start):\n row, col = start\n if row < self.prev_row or row == self.prev_row and col < self.prev_col:\n raise ValueError(\n \"start ({},{}) precedes previous end ({},{})\".format(\n row, col, self.prev_row, self.prev_col\n )\n )\n row_offset = row - self.prev_row\n if row_offset:\n self.tokens.append(\"\\\\\\n\" * row_offset)\n self.prev_col = 0\n col_offset = col - self.prev_col\n if col_offset:\n self.tokens.append(\" \" * col_offset)\n\n def untokenize(self, iterable):\n it = iter(iterable)\n indents = []\n startline = False\n for t in it:\n if len(t) == 2:\n self.compat(t, it)\n break\n tok_type, token, start, end, line = t\n if tok_type == ENCODING:\n self.encoding = token\n continue\n if tok_type == ENDMARKER:\n break\n if tok_type == INDENT:\n indents.append(token)\n continue\n elif tok_type == DEDENT:\n indents.pop()\n self.prev_row, self.prev_col = end\n continue\n elif tok_type in (NEWLINE, NL):\n startline = True\n elif startline and indents:\n indent = indents[-1]\n if start[1] >= len(indent):\n self.tokens.append(indent)\n self.prev_col = len(indent)\n startline = False\n self.add_whitespace(start)\n self.tokens.append(token)\n self.prev_row, self.prev_col = end\n if tok_type in (NEWLINE, NL):\n self.prev_row += 1\n self.prev_col = 0\n return \"\".join(self.tokens)\n\n def compat(self, token, iterable):\n indents = []\n toks_append = self.tokens.append\n startline = token[0] in (NEWLINE, NL)\n prevstring = False\n\n for tok in itertools.chain([token], iterable):\n toknum, tokval = tok[:2]\n if toknum == ENCODING:\n self.encoding = tokval\n continue\n\n if toknum in ADDSPACE_TOKS:\n tokval += \" \"\n\n # Insert a space between two consecutive strings\n if toknum == STRING:\n if prevstring:\n tokval = \" \" + tokval\n prevstring = True\n else:\n prevstring = False\n\n if toknum == INDENT:\n indents.append(tokval)\n continue\n elif toknum == DEDENT:\n indents.pop()\n continue\n elif toknum in (NEWLINE, NL):\n startline = True\n elif startline and indents:\n toks_append(indents[-1])\n startline = False\n toks_append(tokval)\n\n\ndef untokenize(iterable):\n \"\"\"Transform tokens back into Python source code.\n It returns a bytes object, encoded using the ENCODING\n token, which is the first token sequence output by tokenize.\n\n Each element returned by the iterable must be a token sequence\n with at least two elements, a token number and token value. If\n only two tokens are passed, the resulting output is poor.\n\n Round-trip invariant for full input:\n Untokenized source will match input source exactly\n\n Round-trip invariant for limited intput:\n # Output bytes will tokenize the back to the input\n t1 = [tok[:2] for tok in tokenize(f.readline)]\n newcode = untokenize(t1)\n readline = BytesIO(newcode).readline\n t2 = [tok[:2] for tok in tokenize(readline)]\n assert t1 == t2\n \"\"\"\n ut = Untokenizer()\n out = ut.untokenize(iterable)\n if ut.encoding is not None:\n out = out.encode(ut.encoding)\n return out\n\n\ndef _get_normal_name(orig_enc):\n \"\"\"Imitates get_normal_name in tokenizer.c.\"\"\"\n # Only care about the first 12 characters.\n enc = orig_enc[:12].lower().replace(\"_\", \"-\")\n if enc == \"utf-8\" or enc.startswith(\"utf-8-\"):\n return \"utf-8\"\n if enc in (\"latin-1\", \"iso-8859-1\", \"iso-latin-1\") or enc.startswith(\n (\"latin-1-\", \"iso-8859-1-\", \"iso-latin-1-\")\n ):\n return \"iso-8859-1\"\n return orig_enc\n\n\ndef detect_encoding(readline):\n \"\"\"\n The detect_encoding() function is used to detect the encoding that should\n be used to decode a Python source file. It requires one argument, readline,\n in the same way as the tokenize() generator.\n\n It will call readline a maximum of twice, and return the encoding used\n (as a string) and a list of any lines (left as bytes) it has read in.\n\n It detects the encoding from the presence of a utf-8 bom or an encoding\n cookie as specified in pep-0263. If both a bom and a cookie are present,\n but disagree, a SyntaxError will be raised. If the encoding cookie is an\n invalid charset, raise a SyntaxError. Note that if a utf-8 bom is found,\n 'utf-8-sig' is returned.\n\n If no encoding is specified, then the default of 'utf-8' will be returned.\n \"\"\"\n try:\n filename = readline.__self__.name\n except AttributeError:\n filename = None\n bom_found = False\n encoding = None\n default = \"utf-8\"\n\n def read_or_stop():\n try:\n return readline()\n except StopIteration:\n return b\"\"\n\n def find_cookie(line):\n try:\n # Decode as UTF-8. Either the line is an encoding declaration,\n # in which case it should be pure ASCII, or it must be UTF-8\n # per default encoding.\n line_string = line.decode(\"utf-8\")\n except UnicodeDecodeError:\n msg = \"invalid or missing encoding declaration\"\n if filename is not None:\n msg = \"{} for {!r}\".format(msg, filename)\n raise SyntaxError(msg)\n\n match = cookie_re.match(line_string)\n if not match:\n return None\n encoding = _get_normal_name(match.group(1))\n try:\n codecs.lookup(encoding)\n except LookupError:\n # This behaviour mimics the Python interpreter\n if filename is None:\n msg = \"unknown encoding: \" + encoding\n else:\n msg = \"unknown encoding for {!r}: {}\".format(filename, encoding)\n raise SyntaxError(msg)\n\n if bom_found:\n if encoding != \"utf-8\":\n # This behaviour mimics the Python interpreter\n if filename is None:\n msg = \"encoding problem: utf-8\"\n else:\n msg = \"encoding problem for {!r}: utf-8\".format(filename)\n raise SyntaxError(msg)\n encoding += \"-sig\"\n return encoding\n\n first = read_or_stop()\n if first.startswith(codecs.BOM_UTF8):\n bom_found = True\n first = first[3:]\n default = \"utf-8-sig\"\n if not first:\n return default, []\n\n encoding = find_cookie(first)\n if encoding:\n return encoding, [first]\n if not blank_re.match(first):\n return default, [first]\n\n second = read_or_stop()\n if not second:\n return default, [first]\n\n encoding = find_cookie(second)\n if encoding:\n return encoding, [first, second]\n\n return default, [first, second]\n\n\ndef tokopen(filename):\n \"\"\"Open a file in read only mode using the encoding detected by\n detect_encoding().\n \"\"\"\n buffer = builtins.open(filename, \"rb\")\n try:\n encoding, lines = detect_encoding(buffer.readline)\n buffer.seek(0)\n text = io.TextIOWrapper(buffer, encoding, line_buffering=True)\n text.mode = \"r\"\n return text\n except Exception:\n buffer.close()\n raise\n\n\ndef _tokenize(readline, encoding):\n lnum = parenlev = continued = 0\n numchars = \"0123456789\"\n contstr, needcont = \"\", 0\n contline = None\n indents = [0]\n\n # 'stashed' and 'async_*' are used for async/await parsing\n stashed = None\n async_def = False\n async_def_indent = 0\n async_def_nl = False\n\n if encoding is not None:\n if encoding == \"utf-8-sig\":\n # BOM will already have been stripped.\n encoding = \"utf-8\"\n yield TokenInfo(ENCODING, encoding, (0, 0), (0, 0), \"\")\n while True: # loop over lines in stream\n try:\n line = readline()\n except StopIteration:\n line = b\"\"\n\n if encoding is not None:\n line = line.decode(encoding)\n lnum += 1\n pos, max = 0, len(line)\n\n if contstr: # continued string\n if not line:\n raise TokenError(\"EOF in multi-line string\", strstart)\n endmatch = endprog.match(line)\n if endmatch:\n pos = end = endmatch.end(0)\n yield TokenInfo(\n STRING, contstr + line[:end], strstart, (lnum, end), contline + line\n )\n contstr, needcont = \"\", 0\n contline = None\n elif needcont and line[-2:] != \"\\\\\\n\" and line[-3:] != \"\\\\\\r\\n\":\n yield TokenInfo(\n ERRORTOKEN, contstr + line, strstart, (lnum, len(line)), contline\n )\n contstr = \"\"\n contline = None\n continue\n else:\n contstr = contstr + line\n contline = contline + line\n continue\n\n elif parenlev == 0 and not continued: # new statement\n if not line:\n break\n column = 0\n while pos < max: # measure leading whitespace\n if line[pos] == \" \":\n column += 1\n elif line[pos] == \"\\t\":\n column = (column // tabsize + 1) * tabsize\n elif line[pos] == \"\\f\":\n column = 0\n else:\n break\n pos += 1\n if pos == max:\n break\n\n if line[pos] in \"#\\r\\n\": # skip comments or blank lines\n if line[pos] == \"#\":\n comment_token = line[pos:].rstrip(\"\\r\\n\")\n nl_pos = pos + len(comment_token)\n yield TokenInfo(\n COMMENT,\n comment_token,\n (lnum, pos),\n (lnum, pos + len(comment_token)),\n line,\n )\n yield TokenInfo(\n NL, line[nl_pos:], (lnum, nl_pos), (lnum, len(line)), line\n )\n else:\n yield TokenInfo(\n (NL, COMMENT)[line[pos] == \"#\"],\n line[pos:],\n (lnum, pos),\n (lnum, len(line)),\n line,\n )\n continue\n\n if column > indents[-1]: # count indents or dedents\n indents.append(column)\n yield TokenInfo(INDENT, line[:pos], (lnum, 0), (lnum, pos), line)\n while column < indents[-1]:\n if column not in indents:\n raise IndentationError(\n \"unindent does not match any outer indentation level\",\n (\"\", lnum, pos, line),\n )\n indents = indents[:-1]\n\n if async_def and async_def_indent >= indents[-1]:\n async_def = False\n async_def_nl = False\n async_def_indent = 0\n\n yield TokenInfo(DEDENT, \"\", (lnum, pos), (lnum, pos), line)\n\n if async_def and async_def_nl and async_def_indent >= indents[-1]:\n async_def = False\n async_def_nl = False\n async_def_indent = 0\n\n else: # continued statement\n if not line:\n raise TokenError(\"EOF in multi-line statement\", (lnum, 0))\n continued = 0\n\n while pos < max:\n pseudomatch = _compile(PseudoToken).match(line, pos)\n if pseudomatch: # scan for tokens\n start, end = pseudomatch.span(1)\n spos, epos, pos = (lnum, start), (lnum, end), end\n if start == end:\n continue\n token, initial = line[start:end], line[start]\n\n if token in _redir_check:\n yield TokenInfo(IOREDIRECT, token, spos, epos, line)\n elif initial in numchars or ( # ordinary number\n initial == \".\" and token != \".\" and token != \"...\"\n ):\n yield TokenInfo(NUMBER, token, spos, epos, line)\n elif initial in \"\\r\\n\":\n if stashed:\n yield stashed\n stashed = None\n if parenlev > 0:\n yield TokenInfo(NL, token, spos, epos, line)\n else:\n yield TokenInfo(NEWLINE, token, spos, epos, line)\n if async_def:\n async_def_nl = True\n\n elif initial == \"#\":\n assert not token.endswith(\"\\n\")\n if stashed:\n yield stashed\n stashed = None\n yield TokenInfo(COMMENT, token, spos, epos, line)\n # Xonsh-specific Regex Globbing\n elif re.match(SearchPath, token):\n yield TokenInfo(SEARCHPATH, token, spos, epos, line)\n elif token in triple_quoted:\n endprog = _compile(endpats[token])\n endmatch = endprog.match(line, pos)\n if endmatch: # all on one line\n pos = endmatch.end(0)\n token = line[start:pos]\n yield TokenInfo(STRING, token, spos, (lnum, pos), line)\n else:\n strstart = (lnum, start) # multiple lines\n contstr = line[start:]\n contline = line\n break\n elif (\n initial in single_quoted\n or token[:2] in single_quoted\n or token[:3] in single_quoted\n ):\n if token[-1] == \"\\n\": # continued string\n strstart = (lnum, start)\n endprog = _compile(\n endpats[initial] or endpats[token[1]] or endpats[token[2]]\n )\n contstr, needcont = line[start:], 1\n contline = line\n break\n else: # ordinary string\n yield TokenInfo(STRING, token, spos, epos, line)\n elif token.startswith(\"$\") and token[1:].isidentifier():\n yield TokenInfo(DOLLARNAME, token, spos, epos, line)\n elif initial.isidentifier(): # ordinary name\n if token in (\"async\", \"await\"):\n if async_def:\n yield TokenInfo(\n ASYNC if token == \"async\" else AWAIT,\n token,\n spos,\n epos,\n line,\n )\n continue\n\n tok = TokenInfo(NAME, token, spos, epos, line)\n if token == \"async\" and not stashed:\n stashed = tok\n continue\n\n if (\n HAS_ASYNC\n and token == \"def\"\n and (\n stashed\n and stashed.type == NAME\n and stashed.string == \"async\"\n )\n ):\n async_def = True\n async_def_indent = indents[-1]\n\n yield TokenInfo(\n ASYNC,\n stashed.string,\n stashed.start,\n stashed.end,\n stashed.line,\n )\n stashed = None\n\n if stashed:\n yield stashed\n stashed = None\n\n yield tok\n elif token == \"\\\\\\n\" or token == \"\\\\\\r\\n\": # continued stmt\n continued = 1\n yield TokenInfo(ERRORTOKEN, token, spos, epos, line)\n elif initial == \"\\\\\": # continued stmt\n # for cases like C:\\\\path\\\\to\\\\file\n continued = 1\n else:\n if initial in \"([{\":\n parenlev += 1\n elif initial in \")]}\":\n parenlev -= 1\n elif token in additional_parenlevs:\n parenlev += 1\n if stashed:\n yield stashed\n stashed = None\n yield TokenInfo(OP, token, spos, epos, line)\n else:\n yield TokenInfo(\n ERRORTOKEN, line[pos], (lnum, pos), (lnum, pos + 1), line\n )\n pos += 1\n\n if stashed:\n yield stashed\n stashed = None\n\n for indent in indents[1:]: # pop remaining indent levels\n yield TokenInfo(DEDENT, \"\", (lnum, 0), (lnum, 0), \"\")\n yield TokenInfo(ENDMARKER, \"\", (lnum, 0), (lnum, 0), \"\")\n\n\ndef tokenize(readline):\n \"\"\"\n The tokenize() generator requires one argument, readline, which\n must be a callable object which provides the same interface as the\n readline() method of built-in file objects. Each call to the function\n should return one line of input as bytes. Alternately, readline\n can be a callable function terminating with StopIteration:\n readline = open(myfile, 'rb').__next__ # Example of alternate readline\n\n The generator produces 5-tuples with these members: the token type; the\n token string; a 2-tuple (srow, scol) of ints specifying the row and\n column where the token begins in the source; a 2-tuple (erow, ecol) of\n ints specifying the row and column where the token ends in the source;\n and the line on which the token was found. The line passed is the\n logical line; continuation lines are included.\n\n The first token sequence will always be an ENCODING token\n which tells you which encoding was used to decode the bytes stream.\n \"\"\"\n encoding, consumed = detect_encoding(readline)\n rl_gen = iter(readline, b\"\")\n empty = itertools.repeat(b\"\")\n return _tokenize(itertools.chain(consumed, rl_gen, empty).__next__, encoding)\n\n\n# An undocumented, backwards compatible, API for all the places in the standard\n# library that expect to be able to use tokenize with strings\ndef generate_tokens(readline):\n return _tokenize(readline, None)\n\n\ndef tokenize_main():\n import argparse\n\n # Helper error handling routines\n def perror(message):\n print(message, file=sys.stderr)\n\n def error(message, filename=None, location=None):\n if location:\n args = (filename,) + location + (message,)\n perror(\"%s:%d:%d: error: %s\" % args)\n elif filename:\n perror(\"%s: error: %s\" % (filename, message))\n else:\n perror(\"error: %s\" % message)\n sys.exit(1)\n\n # Parse the arguments and options\n parser = argparse.ArgumentParser(prog=\"python -m tokenize\")\n parser.add_argument(\n dest=\"filename\",\n nargs=\"?\",\n metavar=\"filename.py\",\n help=\"the file to tokenize; defaults to stdin\",\n )\n parser.add_argument(\n \"-e\",\n \"--exact\",\n dest=\"exact\",\n action=\"store_true\",\n help=\"display token names using the exact type\",\n )\n args = parser.parse_args()\n\n try:\n # Tokenize the input\n if args.filename:\n filename = args.filename\n with builtins.open(filename, \"rb\") as f:\n tokens = list(tokenize(f.readline))\n else:\n filename = \"\"\n tokens = _tokenize(sys.stdin.readline, None)\n\n # Output the tokenization\n for token in tokens:\n token_type = token.type\n if args.exact:\n token_type = token.exact_type\n token_range = \"%d,%d-%d,%d:\" % (token.start + token.end)\n print(\"%-20s%-15s%-15r\" % (token_range, tok_name[token_type], token.string))\n except IndentationError as err:\n line, column = err.args[1][1:3]\n error(err.args[0], filename, (line, column))\n except TokenError as err:\n line, column = err.args[1]\n error(err.args[0], filename, (line, column))\n except SyntaxError as err:\n error(err, filename)\n except OSError as err:\n error(err)\n except KeyboardInterrupt:\n print(\"interrupted\\n\")\n except Exception as err:\n perror(\"unexpected error: %s\" % err)\n raise\n" }, { "path": "xonsh/tools.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"Misc. xonsh tools.\n\nThe following implementations were forked from the IPython project:\n\n* Copyright (c) 2008-2014, IPython Development Team\n* Copyright (C) 2001-2007 Fernando Perez \n* Copyright (c) 2001, Janko Hauser \n* Copyright (c) 2001, Nathaniel Gray \n\nImplementations:\n\n* decode()\n* encode()\n* cast_unicode()\n* safe_hasattr()\n* indent()\n\n\"\"\"\nimport builtins\nimport collections\nimport collections.abc as cabc\nimport contextlib\nimport ctypes\nimport datetime\nfrom distutils.version import LooseVersion\nimport functools\nimport glob\nimport itertools\nimport os\nimport pathlib\nimport re\nimport subprocess\nimport sys\nimport threading\nimport traceback\nimport warnings\nimport operator\nimport ast\nimport string\n\n# adding imports from further xonsh modules is discouraged to avoid circular\n# dependencies\nfrom xonsh import __version__\nfrom xonsh.lazyasd import LazyObject, LazyDict, lazyobject\nfrom xonsh.platform import (\n scandir,\n DEFAULT_ENCODING,\n ON_LINUX,\n ON_WINDOWS,\n PYTHON_VERSION_INFO,\n expanduser,\n os_environ,\n)\n\n\n@functools.lru_cache(1)\ndef is_superuser():\n if ON_WINDOWS:\n rtn = ctypes.windll.shell32.IsUserAnAdmin() != 0\n else:\n rtn = os.getuid() == 0\n return rtn\n\n\nclass XonshError(Exception):\n pass\n\n\nclass XonshCalledProcessError(XonshError, subprocess.CalledProcessError):\n \"\"\"Raised when there's an error with a called process\n\n Inherits from XonshError and subprocess.CalledProcessError, catching\n either will also catch this error.\n\n Raised *after* iterating over stdout of a captured command, if the\n returncode of the command is nonzero.\n\n Example:\n try:\n for line in !(ls):\n print(line)\n except subprocess.CalledProcessError as error:\n print(\"Error in process: {}.format(error.completed_command.pid))\n\n This also handles differences between Python3.4 and 3.5 where\n CalledProcessError is concerned.\n \"\"\"\n\n def __init__(\n self, returncode, command, output=None, stderr=None, completed_command=None\n ):\n super().__init__(returncode, command, output)\n self.stderr = stderr\n self.completed_command = completed_command\n\n\ndef expand_path(s, expand_user=True):\n \"\"\"Takes a string path and expands ~ to home if expand_user is set\n and environment vars if EXPAND_ENV_VARS is set.\"\"\"\n session = getattr(builtins, \"__xonsh__\", None)\n env = os_environ if session is None else getattr(session, \"env\", os_environ)\n if env.get(\"EXPAND_ENV_VARS\", False):\n s = expandvars(s)\n if expand_user:\n # expand ~ according to Bash unquoted rules \"Each variable assignment is\n # checked for unquoted tilde-prefixes immediately following a ':' or the\n # first '='\". See the following for more details.\n # https://www.gnu.org/software/bash/manual/html_node/Tilde-Expansion.html\n pre, char, post = s.partition(\"=\")\n if char:\n s = expanduser(pre) + char\n s += os.pathsep.join(map(expanduser, post.split(os.pathsep)))\n else:\n s = expanduser(s)\n return s\n\n\ndef _expandpath(path):\n \"\"\"Performs environment variable / user expansion on a given path\n if EXPAND_ENV_VARS is set.\n \"\"\"\n session = getattr(builtins, \"__xonsh__\", None)\n env = os_environ if session is None else getattr(session, \"env\", os_environ)\n expand_user = env.get(\"EXPAND_ENV_VARS\", False)\n return expand_path(path, expand_user=expand_user)\n\n\ndef decode_bytes(b):\n \"\"\"Tries to decode the bytes using XONSH_ENCODING if available,\n otherwise using sys.getdefaultencoding().\n \"\"\"\n session = getattr(builtins, \"__xonsh__\", None)\n env = os_environ if session is None else getattr(session, \"env\", os_environ)\n enc = env.get(\"XONSH_ENCODING\") or DEFAULT_ENCODING\n err = env.get(\"XONSH_ENCODING_ERRORS\") or \"strict\"\n return b.decode(encoding=enc, errors=err)\n\n\ndef findfirst(s, substrs):\n \"\"\"Finds whichever of the given substrings occurs first in the given string\n and returns that substring, or returns None if no such strings occur.\n \"\"\"\n i = len(s)\n result = None\n for substr in substrs:\n pos = s.find(substr)\n if -1 < pos < i:\n i = pos\n result = substr\n return i, result\n\n\nclass EnvPath(cabc.MutableSequence):\n \"\"\"A class that implements an environment path, which is a list of\n strings. Provides a custom method that expands all paths if the\n relevant env variable has been set.\n \"\"\"\n\n def __init__(self, args=None):\n if not args:\n self._l = []\n else:\n if isinstance(args, str):\n self._l = args.split(os.pathsep)\n elif isinstance(args, pathlib.Path):\n self._l = [args]\n elif isinstance(args, bytes):\n # decode bytes to a string and then split based on\n # the default path separator\n self._l = decode_bytes(args).split(os.pathsep)\n elif isinstance(args, cabc.Iterable):\n # put everything in a list -before- performing the type check\n # in order to be able to retrieve it later, for cases such as\n # when a generator expression was passed as an argument\n args = list(args)\n if not all(isinstance(i, (str, bytes, pathlib.Path)) for i in args):\n # make TypeError's message as informative as possible\n # when given an invalid initialization sequence\n raise TypeError(\n \"EnvPath's initialization sequence should only \"\n \"contain str, bytes and pathlib.Path entries\"\n )\n self._l = args\n else:\n raise TypeError(\n \"EnvPath cannot be initialized with items \"\n \"of type %s\" % type(args)\n )\n\n def __getitem__(self, item):\n # handle slices separately\n if isinstance(item, slice):\n return [_expandpath(i) for i in self._l[item]]\n else:\n return _expandpath(self._l[item])\n\n def __setitem__(self, index, item):\n self._l.__setitem__(index, item)\n\n def __len__(self):\n return len(self._l)\n\n def __delitem__(self, key):\n self._l.__delitem__(key)\n\n def insert(self, index, value):\n self._l.insert(index, value)\n\n @property\n def paths(self):\n \"\"\"\n Returns the list of directories that this EnvPath contains.\n \"\"\"\n return list(self)\n\n def __repr__(self):\n return repr(self._l)\n\n def __eq__(self, other):\n if len(self) != len(other):\n return False\n return all(map(operator.eq, self, other))\n\n def _repr_pretty_(self, p, cycle):\n \"\"\" Pretty print path list \"\"\"\n if cycle:\n p.text(\"EnvPath(...)\")\n else:\n with p.group(1, \"EnvPath(\\n[\", \"]\\n)\"):\n for idx, item in enumerate(self):\n if idx:\n p.text(\",\")\n p.breakable()\n p.pretty(item)\n\n def __add__(self, other):\n if isinstance(other, EnvPath):\n other = other._l\n return EnvPath(self._l + other)\n\n def __radd__(self, other):\n if isinstance(other, EnvPath):\n other = other._l\n return EnvPath(other + self._l)\n\n def add(self, data, front=False, replace=False):\n \"\"\"Add a value to this EnvPath,\n\n path.add(data, front=bool, replace=bool) -> ensures that path contains data, with position determined by kwargs\n\n Parameters\n ----------\n data : string or bytes or pathlib.Path\n value to be added\n front : bool\n whether the value should be added to the front, will be\n ignored if the data already exists in this EnvPath and\n replace is False\n Default : False\n replace : bool\n If True, the value will be removed and added to the\n start or end(depending on the value of front)\n Default : False\n\n Returns\n -------\n None\n\n \"\"\"\n if data not in self._l:\n self._l.insert(0 if front else len(self._l), data)\n elif replace:\n self._l.remove(data)\n self._l.insert(0 if front else len(self._l), data)\n\n\n@lazyobject\ndef FORMATTER():\n return string.Formatter()\n\n\nclass DefaultNotGivenType(object):\n \"\"\"Singleton for representing when no default value is given.\"\"\"\n\n __inst = None\n\n def __new__(cls):\n if DefaultNotGivenType.__inst is None:\n DefaultNotGivenType.__inst = object.__new__(cls)\n return DefaultNotGivenType.__inst\n\n\nDefaultNotGiven = DefaultNotGivenType()\n\nBEG_TOK_SKIPS = LazyObject(\n lambda: frozenset([\"WS\", \"INDENT\", \"NOT\", \"LPAREN\"]), globals(), \"BEG_TOK_SKIPS\"\n)\nEND_TOK_TYPES = LazyObject(\n lambda: frozenset([\"SEMI\", \"AND\", \"OR\", \"RPAREN\"]), globals(), \"END_TOK_TYPES\"\n)\nRE_END_TOKS = LazyObject(\n lambda: re.compile(r\"(;|and|\\&\\&|or|\\|\\||\\))\"), globals(), \"RE_END_TOKS\"\n)\nLPARENS = LazyObject(\n lambda: frozenset(\n [\"LPAREN\", \"AT_LPAREN\", \"BANG_LPAREN\", \"DOLLAR_LPAREN\", \"ATDOLLAR_LPAREN\"]\n ),\n globals(),\n \"LPARENS\",\n)\n\n\ndef _is_not_lparen_and_rparen(lparens, rtok):\n \"\"\"Tests if an RPAREN token is matched with something other than a plain old\n LPAREN type.\n \"\"\"\n # note that any([]) is False, so this covers len(lparens) == 0\n return rtok.type == \"RPAREN\" and any(x != \"LPAREN\" for x in lparens)\n\n\ndef balanced_parens(line, mincol=0, maxcol=None, lexer=None):\n \"\"\"Determines if parentheses are balanced in an expression.\"\"\"\n line = line[mincol:maxcol]\n if lexer is None:\n lexer = builtins.__xonsh__.execer.parser.lexer\n if \"(\" not in line and \")\" not in line:\n return True\n cnt = 0\n lexer.input(line)\n for tok in lexer:\n if tok.type in LPARENS:\n cnt += 1\n elif tok.type == \"RPAREN\":\n cnt -= 1\n elif tok.type == \"ERRORTOKEN\" and \")\" in tok.value:\n cnt -= 1\n return cnt == 0\n\n\ndef find_next_break(line, mincol=0, lexer=None):\n \"\"\"Returns the column number of the next logical break in subproc mode.\n This function may be useful in finding the maxcol argument of\n subproc_toks().\n \"\"\"\n if mincol >= 1:\n line = line[mincol:]\n if lexer is None:\n lexer = builtins.__xonsh__.execer.parser.lexer\n if RE_END_TOKS.search(line) is None:\n return None\n maxcol = None\n lparens = []\n lexer.input(line)\n for tok in lexer:\n if tok.type in LPARENS:\n lparens.append(tok.type)\n elif tok.type in END_TOK_TYPES:\n if _is_not_lparen_and_rparen(lparens, tok):\n lparens.pop()\n else:\n maxcol = tok.lexpos + mincol + 1\n break\n elif tok.type == \"ERRORTOKEN\" and \")\" in tok.value:\n maxcol = tok.lexpos + mincol + 1\n break\n elif tok.type == \"BANG\":\n maxcol = mincol + len(line) + 1\n break\n return maxcol\n\n\ndef _offset_from_prev_lines(line, last):\n lines = line.splitlines(keepends=True)[:last]\n return sum(map(len, lines))\n\n\ndef subproc_toks(\n line, mincol=-1, maxcol=None, lexer=None, returnline=False, greedy=False\n):\n \"\"\"Encapsulates tokens in a source code line in a uncaptured\n subprocess ![] starting at a minimum column. If there are no tokens\n (ie in a comment line) this returns None. If greedy is True, it will encapsulate\n normal parentheses. Greedy is False by default.\n \"\"\"\n if lexer is None:\n lexer = builtins.__xonsh__.execer.parser.lexer\n if maxcol is None:\n maxcol = len(line) + 1\n lexer.reset()\n lexer.input(line)\n toks = []\n lparens = []\n saw_macro = False\n end_offset = 0\n for tok in lexer:\n pos = tok.lexpos\n if tok.type not in END_TOK_TYPES and pos >= maxcol:\n break\n if tok.type == \"BANG\":\n saw_macro = True\n if saw_macro and tok.type not in (\"NEWLINE\", \"DEDENT\"):\n toks.append(tok)\n continue\n if tok.type in LPARENS:\n lparens.append(tok.type)\n if greedy and len(lparens) > 0 and \"LPAREN\" in lparens:\n toks.append(tok)\n if tok.type == \"RPAREN\":\n lparens.pop()\n continue\n if len(toks) == 0 and tok.type in BEG_TOK_SKIPS:\n continue # handle indentation\n elif len(toks) > 0 and toks[-1].type in END_TOK_TYPES:\n if _is_not_lparen_and_rparen(lparens, toks[-1]):\n lparens.pop() # don't continue or break\n elif pos < maxcol and tok.type not in (\"NEWLINE\", \"DEDENT\", \"WS\"):\n if not greedy:\n toks.clear()\n if tok.type in BEG_TOK_SKIPS:\n continue\n else:\n break\n if pos < mincol:\n continue\n toks.append(tok)\n if tok.type == \"WS\" and tok.value == \"\\\\\":\n pass # line continuation\n elif tok.type == \"NEWLINE\":\n break\n elif tok.type == \"DEDENT\":\n # fake a newline when dedenting without a newline\n tok.type = \"NEWLINE\"\n tok.value = \"\\n\"\n tok.lineno -= 1\n if len(toks) >= 2:\n prev_tok_end = toks[-2].lexpos + len(toks[-2].value)\n else:\n prev_tok_end = len(line)\n if \"#\" in line[prev_tok_end:]:\n tok.lexpos = prev_tok_end # prevents wrapping comments\n else:\n tok.lexpos = len(line)\n break\n elif check_bad_str_token(tok):\n return\n else:\n if len(toks) > 0 and toks[-1].type in END_TOK_TYPES:\n if _is_not_lparen_and_rparen(lparens, toks[-1]):\n pass\n elif greedy and toks[-1].type == \"RPAREN\":\n pass\n else:\n toks.pop()\n if len(toks) == 0:\n return # handle comment lines\n tok = toks[-1]\n pos = tok.lexpos\n if isinstance(tok.value, str):\n end_offset = len(tok.value.rstrip())\n else:\n el = line[pos:].split(\"#\")[0].rstrip()\n end_offset = len(el)\n if len(toks) == 0:\n return # handle comment lines\n elif saw_macro or greedy:\n end_offset = len(toks[-1].value.rstrip()) + 1\n if toks[0].lineno != toks[-1].lineno:\n # handle multiline cases\n end_offset += _offset_from_prev_lines(line, toks[-1].lineno)\n beg, end = toks[0].lexpos, (toks[-1].lexpos + end_offset)\n end = len(line[:end].rstrip())\n rtn = \"![\" + line[beg:end] + \"]\"\n if returnline:\n rtn = line[:beg] + rtn + line[end:]\n return rtn\n\n\ndef check_bad_str_token(tok):\n \"\"\"Checks if a token is a bad string.\"\"\"\n if tok.type == \"ERRORTOKEN\" and tok.value == \"EOF in multi-line string\":\n return True\n elif isinstance(tok.value, str) and not check_quotes(tok.value):\n return True\n else:\n return False\n\n\ndef check_quotes(s):\n \"\"\"Checks a string to make sure that if it starts with quotes, it also\n ends with quotes.\n \"\"\"\n starts_as_str = RE_BEGIN_STRING.match(s) is not None\n ends_as_str = s.endswith('\"') or s.endswith(\"'\")\n if not starts_as_str and not ends_as_str:\n ok = True\n elif starts_as_str and not ends_as_str:\n ok = False\n elif not starts_as_str and ends_as_str:\n ok = False\n else:\n m = RE_COMPLETE_STRING.match(s)\n ok = m is not None\n return ok\n\n\ndef _have_open_triple_quotes(s):\n if s.count('\"\"\"') % 2 == 1:\n open_triple = '\"\"\"'\n elif s.count(\"'''\") % 2 == 1:\n open_triple = \"'''\"\n else:\n open_triple = False\n return open_triple\n\n\ndef get_line_continuation():\n \"\"\" The line continuation characters used in subproc mode. In interactive\n mode on Windows the backslash must be preceded by a space. This is because\n paths on Windows may end in a backslash.\n \"\"\"\n if (\n ON_WINDOWS\n and hasattr(builtins.__xonsh__, \"env\")\n and builtins.__xonsh__.env.get(\"XONSH_INTERACTIVE\", False)\n ):\n return \" \\\\\"\n else:\n return \"\\\\\"\n\n\ndef get_logical_line(lines, idx):\n \"\"\"Returns a single logical line (i.e. one without line continuations)\n from a list of lines. This line should begin at index idx. This also\n returns the number of physical lines the logical line spans. The lines\n should not contain newlines\n \"\"\"\n n = 1\n nlines = len(lines)\n linecont = get_line_continuation()\n while idx > 0 and lines[idx - 1].endswith(linecont):\n idx -= 1\n start = idx\n line = lines[idx]\n open_triple = _have_open_triple_quotes(line)\n while (line.endswith(linecont) or open_triple) and idx < nlines - 1:\n n += 1\n idx += 1\n if line.endswith(linecont):\n line = line[:-1] + lines[idx]\n else:\n line = line + \"\\n\" + lines[idx]\n open_triple = _have_open_triple_quotes(line)\n return line, n, start\n\n\ndef replace_logical_line(lines, logical, idx, n):\n \"\"\"Replaces lines at idx that may end in line continuation with a logical\n line that spans n lines.\n \"\"\"\n linecont = get_line_continuation()\n if n == 1:\n lines[idx] = logical\n return\n space = \" \"\n for i in range(idx, idx + n - 1):\n a = len(lines[i])\n b = logical.find(space, a - 1)\n if b < 0:\n # no space found\n lines[i] = logical\n logical = \"\"\n else:\n # found space to split on\n lines[i] = logical[:b] + linecont\n logical = logical[b:]\n lines[idx + n - 1] = logical\n\n\ndef is_balanced(expr, ltok, rtok):\n \"\"\"Determines whether an expression has unbalanced opening and closing tokens.\"\"\"\n lcnt = expr.count(ltok)\n if lcnt == 0:\n return True\n rcnt = expr.count(rtok)\n if lcnt == rcnt:\n return True\n else:\n return False\n\n\ndef subexpr_from_unbalanced(expr, ltok, rtok):\n \"\"\"Attempts to pull out a valid subexpression for unbalanced grouping,\n based on opening tokens, eg. '(', and closing tokens, eg. ')'. This\n does not do full tokenization, but should be good enough for tab\n completion.\n \"\"\"\n if is_balanced(expr, ltok, rtok):\n return expr\n subexpr = expr.rsplit(ltok, 1)[-1]\n subexpr = subexpr.rsplit(\",\", 1)[-1]\n subexpr = subexpr.rsplit(\":\", 1)[-1]\n return subexpr\n\n\ndef subexpr_before_unbalanced(expr, ltok, rtok):\n \"\"\"Obtains the expression prior to last unbalanced left token.\"\"\"\n subexpr, _, post = expr.rpartition(ltok)\n nrtoks_in_post = post.count(rtok)\n while nrtoks_in_post != 0:\n for i in range(nrtoks_in_post):\n subexpr, _, post = subexpr.rpartition(ltok)\n nrtoks_in_post = post.count(rtok)\n _, _, subexpr = subexpr.rpartition(rtok)\n _, _, subexpr = subexpr.rpartition(ltok)\n return subexpr\n\n\n@lazyobject\ndef STARTING_WHITESPACE_RE():\n return re.compile(r\"^(\\s*)\")\n\n\ndef starting_whitespace(s):\n \"\"\"Returns the whitespace at the start of a string\"\"\"\n return STARTING_WHITESPACE_RE.match(s).group(1)\n\n\ndef decode(s, encoding=None):\n encoding = encoding or DEFAULT_ENCODING\n return s.decode(encoding, \"replace\")\n\n\ndef encode(u, encoding=None):\n encoding = encoding or DEFAULT_ENCODING\n return u.encode(encoding, \"replace\")\n\n\ndef cast_unicode(s, encoding=None):\n if isinstance(s, bytes):\n return decode(s, encoding)\n return s\n\n\ndef safe_hasattr(obj, attr):\n \"\"\"In recent versions of Python, hasattr() only catches AttributeError.\n This catches all errors.\n \"\"\"\n try:\n getattr(obj, attr)\n return True\n except Exception: # pylint:disable=bare-except\n return False\n\n\ndef indent(instr, nspaces=4, ntabs=0, flatten=False):\n \"\"\"Indent a string a given number of spaces or tabstops.\n\n indent(str,nspaces=4,ntabs=0) -> indent str by ntabs+nspaces.\n\n Parameters\n ----------\n instr : basestring\n The string to be indented.\n nspaces : int (default: 4)\n The number of spaces to be indented.\n ntabs : int (default: 0)\n The number of tabs to be indented.\n flatten : bool (default: False)\n Whether to scrub existing indentation. If True, all lines will be\n aligned to the same indentation. If False, existing indentation will\n be strictly increased.\n\n Returns\n -------\n outstr : string indented by ntabs and nspaces.\n\n \"\"\"\n if instr is None:\n return\n ind = \"\\t\" * ntabs + \" \" * nspaces\n if flatten:\n pat = re.compile(r\"^\\s*\", re.MULTILINE)\n else:\n pat = re.compile(r\"^\", re.MULTILINE)\n outstr = re.sub(pat, ind, instr)\n if outstr.endswith(os.linesep + ind):\n return outstr[: -len(ind)]\n else:\n return outstr\n\n\ndef get_sep():\n \"\"\" Returns the appropriate filepath separator char depending on OS and\n xonsh options set\n \"\"\"\n if ON_WINDOWS and builtins.__xonsh__.env.get(\"FORCE_POSIX_PATHS\"):\n return os.altsep\n else:\n return os.sep\n\n\ndef fallback(cond, backup):\n \"\"\"Decorator for returning the object if cond is true and a backup if cond\n is false.\n \"\"\"\n\n def dec(obj):\n return obj if cond else backup\n\n return dec\n\n\n# The following redirect classes were taken directly from Python 3.5's source\n# code (from the contextlib module). This can be removed when 3.5 is released,\n# although redirect_stdout exists in 3.4, redirect_stderr does not.\n# See the Python software license: https://docs.python.org/3/license.html\n# Copyright (c) Python Software Foundation. All rights reserved.\nclass _RedirectStream:\n\n _stream = None\n\n def __init__(self, new_target):\n self._new_target = new_target\n # We use a list of old targets to make this CM re-entrant\n self._old_targets = []\n\n def __enter__(self):\n self._old_targets.append(getattr(sys, self._stream))\n setattr(sys, self._stream, self._new_target)\n return self._new_target\n\n def __exit__(self, exctype, excinst, exctb):\n setattr(sys, self._stream, self._old_targets.pop())\n\n\nclass redirect_stdout(_RedirectStream):\n \"\"\"Context manager for temporarily redirecting stdout to another file::\n\n # How to send help() to stderr\n with redirect_stdout(sys.stderr):\n help(dir)\n\n # How to write help() to a file\n with open('help.txt', 'w') as f:\n with redirect_stdout(f):\n help(pow)\n\n Mostly for backwards compatibility.\n \"\"\"\n\n _stream = \"stdout\"\n\n\nclass redirect_stderr(_RedirectStream):\n \"\"\"Context manager for temporarily redirecting stderr to another file.\"\"\"\n\n _stream = \"stderr\"\n\n\ndef _yield_accessible_unix_file_names(path):\n \"\"\"yield file names of executable files in path.\"\"\"\n if not os.path.exists(path):\n return\n for file_ in scandir(path):\n try:\n if file_.is_file() and os.access(file_.path, os.X_OK):\n yield file_.name\n except OSError:\n # broken Symlink are neither dir not files\n pass\n\n\ndef _executables_in_posix(path):\n if not os.path.exists(path):\n return\n elif PYTHON_VERSION_INFO < (3, 5, 0):\n for fname in os.listdir(path):\n fpath = os.path.join(path, fname)\n if (\n os.path.exists(fpath)\n and os.access(fpath, os.X_OK)\n and (not os.path.isdir(fpath))\n ):\n yield fname\n else:\n yield from _yield_accessible_unix_file_names(path)\n\n\ndef _executables_in_windows(path):\n if not os.path.isdir(path):\n return\n extensions = builtins.__xonsh__.env[\"PATHEXT\"]\n if PYTHON_VERSION_INFO < (3, 5, 0):\n for fname in os.listdir(path):\n fpath = os.path.join(path, fname)\n if os.path.exists(fpath) and not os.path.isdir(fpath):\n base_name, ext = os.path.splitext(fname)\n if ext.upper() in extensions:\n yield fname\n else:\n for x in scandir(path):\n try:\n is_file = x.is_file()\n except OSError:\n continue\n if is_file:\n fname = x.name\n else:\n continue\n base_name, ext = os.path.splitext(fname)\n if ext.upper() in extensions:\n yield fname\n\n\ndef executables_in(path):\n \"\"\"Returns a generator of files in path that the user could execute. \"\"\"\n if ON_WINDOWS:\n func = _executables_in_windows\n else:\n func = _executables_in_posix\n try:\n yield from func(path)\n except PermissionError:\n return\n\n\ndef command_not_found(cmd):\n \"\"\"Uses the debian/ubuntu command-not-found utility to suggest packages for a\n command that cannot currently be found.\n \"\"\"\n if not ON_LINUX:\n return \"\"\n elif not os.path.isfile(\"/usr/lib/command-not-found\"):\n # utility is not on PATH\n return \"\"\n c = \"/usr/lib/command-not-found {0}; exit 0\"\n s = subprocess.check_output(\n c.format(cmd), universal_newlines=True, stderr=subprocess.STDOUT, shell=True\n )\n s = \"\\n\".join(s.rstrip().splitlines()).strip()\n return s\n\n\ndef suggest_commands(cmd, env, aliases):\n \"\"\"Suggests alternative commands given an environment and aliases.\"\"\"\n if not env.get(\"SUGGEST_COMMANDS\"):\n return \"\"\n thresh = env.get(\"SUGGEST_THRESHOLD\")\n max_sugg = env.get(\"SUGGEST_MAX_NUM\")\n if max_sugg < 0:\n max_sugg = float(\"inf\")\n cmd = cmd.lower()\n suggested = {}\n\n for alias in builtins.aliases:\n if alias not in suggested:\n if levenshtein(alias.lower(), cmd, thresh) < thresh:\n suggested[alias] = \"Alias\"\n\n for path in filter(os.path.isdir, env.get(\"PATH\")):\n for _file in executables_in(path):\n if (\n _file not in suggested\n and levenshtein(_file.lower(), cmd, thresh) < thresh\n ):\n suggested[_file] = \"Command ({0})\".format(os.path.join(path, _file))\n\n suggested = collections.OrderedDict(\n sorted(\n suggested.items(), key=lambda x: suggestion_sort_helper(x[0].lower(), cmd)\n )\n )\n num = min(len(suggested), max_sugg)\n\n if num == 0:\n rtn = command_not_found(cmd)\n else:\n oneof = \"\" if num == 1 else \"one of \"\n tips = \"Did you mean {}the following?\".format(oneof)\n items = list(suggested.popitem(False) for _ in range(num))\n length = max(len(key) for key, _ in items) + 2\n alternatives = \"\\n\".join(\n \" {: <{}} {}\".format(key + \":\", length, val) for key, val in items\n )\n rtn = \"{}\\n{}\".format(tips, alternatives)\n c = command_not_found(cmd)\n rtn += (\"\\n\\n\" + c) if len(c) > 0 else \"\"\n return rtn\n\n\ndef print_exception(msg=None):\n \"\"\"Print exceptions with/without traceback.\"\"\"\n env = getattr(builtins.__xonsh__, \"env\", None)\n # flags indicating whether the traceback options have been manually set\n if env is None:\n env = os_environ\n manually_set_trace = \"XONSH_SHOW_TRACEBACK\" in env\n manually_set_logfile = \"XONSH_TRACEBACK_LOGFILE\" in env\n else:\n manually_set_trace = env.is_manually_set(\"XONSH_SHOW_TRACEBACK\")\n manually_set_logfile = env.is_manually_set(\"XONSH_TRACEBACK_LOGFILE\")\n if (not manually_set_trace) and (not manually_set_logfile):\n # Notify about the traceback output possibility if neither of\n # the two options have been manually set\n sys.stderr.write(\n \"xonsh: For full traceback set: \" \"$XONSH_SHOW_TRACEBACK = True\\n\"\n )\n # get env option for traceback and convert it if necessary\n show_trace = env.get(\"XONSH_SHOW_TRACEBACK\", False)\n if not is_bool(show_trace):\n show_trace = to_bool(show_trace)\n # if the trace option has been set, print all traceback info to stderr\n if show_trace:\n # notify user about XONSH_TRACEBACK_LOGFILE if it has\n # not been set manually\n if not manually_set_logfile:\n sys.stderr.write(\n \"xonsh: To log full traceback to a file set: \"\n \"$XONSH_TRACEBACK_LOGFILE = \\n\"\n )\n traceback.print_exc()\n # additionally, check if a file for traceback logging has been\n # specified and convert to a proper option if needed\n log_file = env.get(\"XONSH_TRACEBACK_LOGFILE\", None)\n log_file = to_logfile_opt(log_file)\n if log_file:\n # if log_file <> '' or log_file <> None, append\n # traceback log there as well\n with open(os.path.abspath(log_file), \"a\") as f:\n traceback.print_exc(file=f)\n\n if not show_trace:\n # if traceback output is disabled, print the exception's\n # error message on stderr.\n display_error_message()\n if msg:\n msg = msg if msg.endswith(\"\\n\") else msg + \"\\n\"\n sys.stderr.write(msg)\n\n\ndef display_error_message(strip_xonsh_error_types=True):\n \"\"\"\n Prints the error message of the current exception on stderr.\n \"\"\"\n exc_type, exc_value, exc_traceback = sys.exc_info()\n exception_only = traceback.format_exception_only(exc_type, exc_value)\n if exc_type is XonshError and strip_xonsh_error_types:\n exception_only[0] = exception_only[0].partition(\": \")[-1]\n sys.stderr.write(\"\".join(exception_only))\n\n\ndef is_writable_file(filepath):\n \"\"\"\n Checks if a filepath is valid for writing.\n \"\"\"\n filepath = expand_path(filepath)\n # convert to absolute path if needed\n if not os.path.isabs(filepath):\n filepath = os.path.abspath(filepath)\n # cannot write to directories\n if os.path.isdir(filepath):\n return False\n # if the file exists and is writable, we're fine\n if os.path.exists(filepath):\n return True if os.access(filepath, os.W_OK) else False\n # if the path doesn't exist, isolate its directory component\n # and ensure that directory is writable instead\n return os.access(os.path.dirname(filepath), os.W_OK)\n\n\n# Modified from Public Domain code, by Magnus Lie Hetland\n# from http://hetland.org/coding/python/levenshtein.py\ndef levenshtein(a, b, max_dist=float(\"inf\")):\n \"\"\"Calculates the Levenshtein distance between a and b.\"\"\"\n n, m = len(a), len(b)\n if abs(n - m) > max_dist:\n return float(\"inf\")\n if n > m:\n # Make sure n <= m, to use O(min(n,m)) space\n a, b = b, a\n n, m = m, n\n current = range(n + 1)\n for i in range(1, m + 1):\n previous, current = current, [i] + [0] * n\n for j in range(1, n + 1):\n add, delete = previous[j] + 1, current[j - 1] + 1\n change = previous[j - 1]\n if a[j - 1] != b[i - 1]:\n change = change + 1\n current[j] = min(add, delete, change)\n return current[n]\n\n\ndef suggestion_sort_helper(x, y):\n \"\"\"Returns a score (lower is better) for x based on how similar\n it is to y. Used to rank suggestions.\"\"\"\n x = x.lower()\n y = y.lower()\n lendiff = len(x) + len(y)\n inx = len([i for i in x if i not in y])\n iny = len([i for i in y if i not in x])\n return lendiff + inx + iny\n\n\ndef escape_windows_cmd_string(s):\n \"\"\"Returns a string that is usable by the Windows cmd.exe.\n The escaping is based on details here and empirical testing:\n http://www.robvanderwoude.com/escapechars.php\n \"\"\"\n for c in '^()%!<>&|\"':\n s = s.replace(c, \"^\" + c)\n return s\n\n\ndef argvquote(arg, force=False):\n \"\"\" Returns an argument quoted in such a way that that CommandLineToArgvW\n on Windows will return the argument string unchanged.\n This is the same thing Popen does when supplied with an list of arguments.\n Arguments in a command line should be separated by spaces; this\n function does not add these spaces. This implementation follows the\n suggestions outlined here:\n https://blogs.msdn.microsoft.com/twistylittlepassagesallalike/2011/04/23/everyone-quotes-command-line-arguments-the-wrong-way/\n \"\"\"\n if not force and len(arg) != 0 and not any([c in arg for c in ' \\t\\n\\v\"']):\n return arg\n else:\n n_backslashes = 0\n cmdline = '\"'\n for c in arg:\n if c == \"\\\\\":\n # first count the number of current backslashes\n n_backslashes += 1\n continue\n if c == '\"':\n # Escape all backslashes and the following double quotation mark\n cmdline += (n_backslashes * 2 + 1) * \"\\\\\"\n else:\n # backslashes are not special here\n cmdline += n_backslashes * \"\\\\\"\n n_backslashes = 0\n cmdline += c\n # Escape all backslashes, but let the terminating\n # double quotation mark we add below be interpreted\n # as a metacharacter\n cmdline += +n_backslashes * 2 * \"\\\\\" + '\"'\n return cmdline\n\n\ndef on_main_thread():\n \"\"\"Checks if we are on the main thread or not.\"\"\"\n return threading.current_thread() is threading.main_thread()\n\n\n_DEFAULT_SENTINEL = object()\n\n\n@contextlib.contextmanager\ndef swap(namespace, name, value, default=_DEFAULT_SENTINEL):\n \"\"\"Swaps a current variable name in a namespace for another value, and then\n replaces it when the context is exited.\n \"\"\"\n old = getattr(namespace, name, default)\n setattr(namespace, name, value)\n yield value\n if old is default:\n delattr(namespace, name)\n else:\n setattr(namespace, name, old)\n\n\n@contextlib.contextmanager\ndef swap_values(d, updates, default=_DEFAULT_SENTINEL):\n \"\"\"Updates a dictionary (or other mapping) with values from another mapping,\n and then restores the original mapping when the context is exited.\n \"\"\"\n old = {k: d.get(k, default) for k in updates}\n d.update(updates)\n yield\n for k, v in old.items():\n if v is default and k in d:\n del d[k]\n else:\n d[k] = v\n\n\n#\n# Validators and converters\n#\n\n\ndef detype(x):\n \"\"\"This assumes that the object has a detype method, and calls that.\"\"\"\n return x.detype()\n\n\ndef is_int(x):\n \"\"\"Tests if something is an integer\"\"\"\n return isinstance(x, int)\n\n\ndef is_float(x):\n \"\"\"Tests if something is a float\"\"\"\n return isinstance(x, float)\n\n\ndef is_string(x):\n \"\"\"Tests if something is a string\"\"\"\n return isinstance(x, str)\n\n\ndef is_slice(x):\n \"\"\"Tests if something is a slice\"\"\"\n return isinstance(x, slice)\n\n\ndef is_callable(x):\n \"\"\"Tests if something is callable\"\"\"\n return callable(x)\n\n\ndef is_string_or_callable(x):\n \"\"\"Tests if something is a string or callable\"\"\"\n return is_string(x) or is_callable(x)\n\n\ndef is_class(x):\n \"\"\"Tests if something is a class\"\"\"\n return isinstance(x, type)\n\n\ndef always_true(x):\n \"\"\"Returns True\"\"\"\n return True\n\n\ndef always_false(x):\n \"\"\"Returns False\"\"\"\n return False\n\n\ndef always_none(x):\n \"\"\"Returns None\"\"\"\n return None\n\n\ndef ensure_string(x):\n \"\"\"Returns a string if x is not a string, and x if it already is.\"\"\"\n return str(x)\n\n\ndef is_env_path(x):\n \"\"\"This tests if something is an environment path, ie a list of strings.\"\"\"\n return isinstance(x, EnvPath)\n\n\ndef str_to_env_path(x):\n \"\"\"Converts a string to an environment path, ie a list of strings,\n splitting on the OS separator.\n \"\"\"\n # splitting will be done implicitly in EnvPath's __init__\n return EnvPath(x)\n\n\ndef env_path_to_str(x):\n \"\"\"Converts an environment path to a string by joining on the OS\n separator.\n \"\"\"\n return os.pathsep.join(x)\n\n\ndef is_bool(x):\n \"\"\"Tests if something is a boolean.\"\"\"\n return isinstance(x, bool)\n\n\ndef is_logfile_opt(x):\n \"\"\"\n Checks if x is a valid $XONSH_TRACEBACK_LOGFILE option. Returns False\n if x is not a writable/creatable file or an empty string or None.\n \"\"\"\n if x is None:\n return True\n if not isinstance(x, str):\n return False\n else:\n return is_writable_file(x) or x == \"\"\n\n\ndef to_logfile_opt(x):\n \"\"\"\n Converts a $XONSH_TRACEBACK_LOGFILE option to either a str containing\n the filepath if it is a writable file or None if the filepath is not\n valid, informing the user on stderr about the invalid choice.\n \"\"\"\n superclass = pathlib.PurePath\n if PYTHON_VERSION_INFO >= (3, 6, 0):\n superclass = os.PathLike\n if isinstance(x, superclass):\n x = str(x)\n if is_logfile_opt(x):\n return x\n else:\n # if option is not valid, return a proper\n # option and inform the user on stderr\n sys.stderr.write(\n \"xonsh: $XONSH_TRACEBACK_LOGFILE must be a \"\n \"filepath pointing to a file that either exists \"\n \"and is writable or that can be created.\\n\"\n )\n return None\n\n\ndef logfile_opt_to_str(x):\n \"\"\"\n Detypes a $XONSH_TRACEBACK_LOGFILE option.\n \"\"\"\n if x is None:\n # None should not be detyped to 'None', as 'None' constitutes\n # a perfectly valid filename and retyping it would introduce\n # ambiguity. Detype to the empty string instead.\n return \"\"\n return str(x)\n\n\n_FALSES = LazyObject(\n lambda: frozenset([\"\", \"0\", \"n\", \"f\", \"no\", \"none\", \"false\", \"off\"]),\n globals(),\n \"_FALSES\",\n)\n\n\ndef to_bool(x):\n \"\"\"\"Converts to a boolean in a semantically meaningful way.\"\"\"\n if isinstance(x, bool):\n return x\n elif isinstance(x, str):\n return False if x.lower() in _FALSES else True\n else:\n return bool(x)\n\n\ndef to_itself(x):\n \"\"\"No conversion, returns itself.\"\"\"\n return x\n\n\ndef bool_to_str(x):\n \"\"\"Converts a bool to an empty string if False and the string '1' if\n True.\n \"\"\"\n return \"1\" if x else \"\"\n\n\n_BREAKS = LazyObject(\n lambda: frozenset([\"b\", \"break\", \"s\", \"skip\", \"q\", \"quit\"]), globals(), \"_BREAKS\"\n)\n\n\ndef to_bool_or_break(x):\n if isinstance(x, str) and x.lower() in _BREAKS:\n return \"break\"\n else:\n return to_bool(x)\n\n\ndef is_bool_or_int(x):\n \"\"\"Returns whether a value is a boolean or integer.\"\"\"\n return is_bool(x) or is_int(x)\n\n\ndef to_bool_or_int(x):\n \"\"\"Converts a value to a boolean or an integer.\"\"\"\n if isinstance(x, str):\n return int(x) if x.isdigit() else to_bool(x)\n elif is_int(x): # bools are ints too!\n return x\n else:\n return bool(x)\n\n\ndef bool_or_int_to_str(x):\n \"\"\"Converts a boolean or integer to a string.\"\"\"\n return bool_to_str(x) if is_bool(x) else str(x)\n\n\n@lazyobject\ndef SLICE_REG():\n return re.compile(\n r\"(?P(?:-\\d)?\\d*):(?P(?:-\\d)?\\d*):?(?P(?:-\\d)?\\d*)\"\n )\n\n\ndef ensure_slice(x):\n \"\"\"Try to convert an object into a slice, complain on failure\"\"\"\n if not x and x != 0:\n return slice(None)\n elif is_slice(x):\n return x\n try:\n x = int(x)\n if x != -1:\n s = slice(x, x + 1)\n else:\n s = slice(-1, None, None)\n except ValueError:\n x = x.strip(\"[]()\")\n m = SLICE_REG.fullmatch(x)\n if m:\n groups = (int(i) if i else None for i in m.groups())\n s = slice(*groups)\n else:\n raise ValueError(\"cannot convert {!r} to slice\".format(x))\n except TypeError:\n try:\n s = slice(*(int(i) for i in x))\n except (TypeError, ValueError):\n raise ValueError(\"cannot convert {!r} to slice\".format(x))\n return s\n\n\ndef get_portions(it, slices):\n \"\"\"Yield from portions of an iterable.\n\n Parameters\n ----------\n it: iterable\n slices: a slice or a list of slice objects\n \"\"\"\n if is_slice(slices):\n slices = [slices]\n if len(slices) == 1:\n s = slices[0]\n try:\n yield from itertools.islice(it, s.start, s.stop, s.step)\n return\n except ValueError: # islice failed\n pass\n it = list(it)\n for s in slices:\n yield from it[s]\n\n\ndef is_slice_as_str(x):\n \"\"\"\n Test if string x is a slice. If not a string return False.\n \"\"\"\n try:\n x = x.strip(\"[]()\")\n m = SLICE_REG.fullmatch(x)\n if m:\n return True\n except AttributeError:\n pass\n return False\n\n\ndef is_int_as_str(x):\n \"\"\"\n Test if string x is an integer. If not a string return False.\n \"\"\"\n try:\n return x.isdecimal()\n except AttributeError:\n return False\n\n\ndef is_string_set(x):\n \"\"\"Tests if something is a set of strings\"\"\"\n return isinstance(x, cabc.Set) and all(isinstance(a, str) for a in x)\n\n\ndef csv_to_set(x):\n \"\"\"Convert a comma-separated list of strings to a set of strings.\"\"\"\n if not x:\n return set()\n else:\n return set(x.split(\",\"))\n\n\ndef set_to_csv(x):\n \"\"\"Convert a set of strings to a comma-separated list of strings.\"\"\"\n return \",\".join(x)\n\n\ndef pathsep_to_set(x):\n \"\"\"Converts a os.pathsep separated string to a set of strings.\"\"\"\n if not x:\n return set()\n else:\n return set(x.split(os.pathsep))\n\n\ndef set_to_pathsep(x, sort=False):\n \"\"\"Converts a set to an os.pathsep separated string. The sort kwarg\n specifies whether to sort the set prior to str conversion.\n \"\"\"\n if sort:\n x = sorted(x)\n return os.pathsep.join(x)\n\n\ndef is_string_seq(x):\n \"\"\"Tests if something is a sequence of strings\"\"\"\n return isinstance(x, cabc.Sequence) and all(isinstance(a, str) for a in x)\n\n\ndef is_nonstring_seq_of_strings(x):\n \"\"\"Tests if something is a sequence of strings, where the top-level\n sequence is not a string itself.\n \"\"\"\n return (\n isinstance(x, cabc.Sequence)\n and not isinstance(x, str)\n and all(isinstance(a, str) for a in x)\n )\n\n\ndef pathsep_to_seq(x):\n \"\"\"Converts a os.pathsep separated string to a sequence of strings.\"\"\"\n if not x:\n return []\n else:\n return x.split(os.pathsep)\n\n\ndef seq_to_pathsep(x):\n \"\"\"Converts a sequence to an os.pathsep separated string.\"\"\"\n return os.pathsep.join(x)\n\n\ndef pathsep_to_upper_seq(x):\n \"\"\"Converts a os.pathsep separated string to a sequence of\n uppercase strings.\n \"\"\"\n if not x:\n return []\n else:\n return x.upper().split(os.pathsep)\n\n\ndef seq_to_upper_pathsep(x):\n \"\"\"Converts a sequence to an uppercase os.pathsep separated string.\"\"\"\n return os.pathsep.join(x).upper()\n\n\ndef is_bool_seq(x):\n \"\"\"Tests if an object is a sequence of bools.\"\"\"\n return isinstance(x, cabc.Sequence) and all(isinstance(y, bool) for y in x)\n\n\ndef csv_to_bool_seq(x):\n \"\"\"Takes a comma-separated string and converts it into a list of bools.\"\"\"\n return [to_bool(y) for y in csv_to_set(x)]\n\n\ndef bool_seq_to_csv(x):\n \"\"\"Converts a sequence of bools to a comma-separated string.\"\"\"\n return \",\".join(map(str, x))\n\n\ndef ptk2_color_depth_setter(x):\n \"\"\" Setter function for $PROMPT_TOOLKIT_COLOR_DEPTH. Also\n updates os.environ so prompt toolkit can pickup the value.\n \"\"\"\n x = str(x)\n if x in {\n \"DEPTH_1_BIT\",\n \"MONOCHROME\",\n \"DEPTH_4_BIT\",\n \"ANSI_COLORS_ONLY\",\n \"DEPTH_8_BIT\",\n \"DEFAULT\",\n \"DEPTH_24_BIT\",\n \"TRUE_COLOR\",\n }:\n pass\n elif x in {\"\", None}:\n x = \"\"\n else:\n msg = '\"{}\" is not a valid value for $PROMPT_TOOLKIT_COLOR_DEPTH. '.format(x)\n warnings.warn(msg, RuntimeWarning)\n x = \"\"\n if x == \"\" and \"PROMPT_TOOLKIT_COLOR_DEPTH\" in os_environ:\n del os_environ[\"PROMPT_TOOLKIT_COLOR_DEPTH\"]\n else:\n os_environ[\"PROMPT_TOOLKIT_COLOR_DEPTH\"] = x\n return x\n\n\ndef is_completions_display_value(x):\n return x in {\"none\", \"single\", \"multi\"}\n\n\ndef to_completions_display_value(x):\n x = str(x).lower()\n if x in {\"none\", \"false\"}:\n x = \"none\"\n elif x in {\"multi\", \"true\"}:\n x = \"multi\"\n elif x in {\"single\", \"readline\"}:\n pass\n else:\n msg = '\"{}\" is not a valid value for $COMPLETIONS_DISPLAY. '.format(x)\n msg += 'Using \"multi\".'\n warnings.warn(msg, RuntimeWarning)\n x = \"multi\"\n return x\n\n\ndef is_str_str_dict(x):\n \"\"\"Tests if something is a str:str dictionary\"\"\"\n return isinstance(x, dict) and all(\n isinstance(k, str) and isinstance(v, str) for k, v in x.items()\n )\n\n\ndef to_dict(x):\n \"\"\"Converts a string to a dictionary\"\"\"\n if isinstance(x, dict):\n return x\n try:\n x = ast.literal_eval(x)\n except (ValueError, SyntaxError):\n msg = '\"{}\" can not be converted to Python dictionary.'.format(x)\n warnings.warn(msg, RuntimeWarning)\n x = dict()\n return x\n\n\ndef to_str_str_dict(x):\n \"\"\"Converts a string to str:str dictionary\"\"\"\n if is_str_str_dict(x):\n return x\n x = to_dict(x)\n if not is_str_str_dict(x):\n msg = '\"{}\" can not be converted to str:str dictionary.'.format(x)\n warnings.warn(msg, RuntimeWarning)\n x = dict()\n return x\n\n\ndef dict_to_str(x):\n \"\"\"Converts a dictionary to a string\"\"\"\n if not x or len(x) == 0:\n return \"\"\n return str(x)\n\n\ndef setup_win_unicode_console(enable):\n \"\"\"\"Enables or disables unicode display on windows.\"\"\"\n try:\n import win_unicode_console\n except ImportError:\n win_unicode_console = False\n enable = to_bool(enable)\n if ON_WINDOWS and win_unicode_console:\n if enable:\n win_unicode_console.enable()\n else:\n win_unicode_console.disable()\n return enable\n\n\n# history validation\n\n_min_to_sec = lambda x: 60.0 * float(x)\n_hour_to_sec = lambda x: 60.0 * _min_to_sec(x)\n_day_to_sec = lambda x: 24.0 * _hour_to_sec(x)\n_month_to_sec = lambda x: 30.4375 * _day_to_sec(x)\n_year_to_sec = lambda x: 365.25 * _day_to_sec(x)\n_kb_to_b = lambda x: 1024 * int(x)\n_mb_to_b = lambda x: 1024 * _kb_to_b(x)\n_gb_to_b = lambda x: 1024 * _mb_to_b(x)\n_tb_to_b = lambda x: 1024 * _tb_to_b(x)\n\nCANON_HISTORY_UNITS = LazyObject(\n lambda: frozenset([\"commands\", \"files\", \"s\", \"b\"]), globals(), \"CANON_HISTORY_UNITS\"\n)\n\nHISTORY_UNITS = LazyObject(\n lambda: {\n \"\": (\"commands\", int),\n \"c\": (\"commands\", int),\n \"cmd\": (\"commands\", int),\n \"cmds\": (\"commands\", int),\n \"command\": (\"commands\", int),\n \"commands\": (\"commands\", int),\n \"f\": (\"files\", int),\n \"files\": (\"files\", int),\n \"s\": (\"s\", float),\n \"sec\": (\"s\", float),\n \"second\": (\"s\", float),\n \"seconds\": (\"s\", float),\n \"m\": (\"s\", _min_to_sec),\n \"min\": (\"s\", _min_to_sec),\n \"mins\": (\"s\", _min_to_sec),\n \"h\": (\"s\", _hour_to_sec),\n \"hr\": (\"s\", _hour_to_sec),\n \"hour\": (\"s\", _hour_to_sec),\n \"hours\": (\"s\", _hour_to_sec),\n \"d\": (\"s\", _day_to_sec),\n \"day\": (\"s\", _day_to_sec),\n \"days\": (\"s\", _day_to_sec),\n \"mon\": (\"s\", _month_to_sec),\n \"month\": (\"s\", _month_to_sec),\n \"months\": (\"s\", _month_to_sec),\n \"y\": (\"s\", _year_to_sec),\n \"yr\": (\"s\", _year_to_sec),\n \"yrs\": (\"s\", _year_to_sec),\n \"year\": (\"s\", _year_to_sec),\n \"years\": (\"s\", _year_to_sec),\n \"b\": (\"b\", int),\n \"byte\": (\"b\", int),\n \"bytes\": (\"b\", int),\n \"kb\": (\"b\", _kb_to_b),\n \"kilobyte\": (\"b\", _kb_to_b),\n \"kilobytes\": (\"b\", _kb_to_b),\n \"mb\": (\"b\", _mb_to_b),\n \"meg\": (\"b\", _mb_to_b),\n \"megs\": (\"b\", _mb_to_b),\n \"megabyte\": (\"b\", _mb_to_b),\n \"megabytes\": (\"b\", _mb_to_b),\n \"gb\": (\"b\", _gb_to_b),\n \"gig\": (\"b\", _gb_to_b),\n \"gigs\": (\"b\", _gb_to_b),\n \"gigabyte\": (\"b\", _gb_to_b),\n \"gigabytes\": (\"b\", _gb_to_b),\n \"tb\": (\"b\", _tb_to_b),\n \"terabyte\": (\"b\", _tb_to_b),\n \"terabytes\": (\"b\", _tb_to_b),\n },\n globals(),\n \"HISTORY_UNITS\",\n)\n\"\"\"Maps lowercase unit names to canonical name and conversion utilities.\"\"\"\n\n\ndef is_history_tuple(x):\n \"\"\"Tests if something is a proper history value, units tuple.\"\"\"\n if (\n isinstance(x, cabc.Sequence)\n and len(x) == 2\n and isinstance(x[0], (int, float))\n and x[1].lower() in CANON_HISTORY_UNITS\n ):\n return True\n return False\n\n\ndef is_history_backend(x):\n \"\"\"Tests if something is a valid history backend.\"\"\"\n return is_string(x) or is_class(x) or isinstance(x, object)\n\n\ndef is_dynamic_cwd_width(x):\n \"\"\" Determine if the input is a valid input for the DYNAMIC_CWD_WIDTH\n environment variable.\n \"\"\"\n return (\n isinstance(x, tuple)\n and len(x) == 2\n and isinstance(x[0], float)\n and x[1] in set(\"c%\")\n )\n\n\ndef to_dynamic_cwd_tuple(x):\n \"\"\"Convert to a canonical cwd_width tuple.\"\"\"\n unit = \"c\"\n if isinstance(x, str):\n if x[-1] == \"%\":\n x = x[:-1]\n unit = \"%\"\n else:\n unit = \"c\"\n return (float(x), unit)\n else:\n return (float(x[0]), x[1])\n\n\ndef dynamic_cwd_tuple_to_str(x):\n \"\"\"Convert a canonical cwd_width tuple to a string.\"\"\"\n if x[1] == \"%\":\n return str(x[0]) + \"%\"\n else:\n return str(x[0])\n\n\nRE_HISTORY_TUPLE = LazyObject(\n lambda: re.compile(r\"([-+]?[0-9]*\\.?[0-9]+([eE][-+]?[0-9]+)?)\\s*([A-Za-z]*)\"),\n globals(),\n \"RE_HISTORY_TUPLE\",\n)\n\n\ndef to_history_tuple(x):\n \"\"\"Converts to a canonical history tuple.\"\"\"\n if not isinstance(x, (cabc.Sequence, float, int)):\n raise ValueError(\"history size must be given as a sequence or number\")\n if isinstance(x, str):\n m = RE_HISTORY_TUPLE.match(x.strip().lower())\n return to_history_tuple((m.group(1), m.group(3)))\n elif isinstance(x, (float, int)):\n return to_history_tuple((x, \"commands\"))\n units, converter = HISTORY_UNITS[x[1]]\n value = converter(x[0])\n return (value, units)\n\n\ndef history_tuple_to_str(x):\n \"\"\"Converts a valid history tuple to a canonical string.\"\"\"\n return \"{0} {1}\".format(*x)\n\n\ndef format_color(string, **kwargs):\n \"\"\"Formats strings that may contain colors. This simply dispatches to the\n shell instances method of the same name. The results of this function should\n be directly usable by print_color().\n \"\"\"\n return builtins.__xonsh__.shell.shell.format_color(string, **kwargs)\n\n\ndef print_color(string, **kwargs):\n \"\"\"Prints a string that may contain colors. This dispatched to the shell\n method of the same name. Colors will be formatted if they have not already\n been.\n \"\"\"\n builtins.__xonsh__.shell.shell.print_color(string, **kwargs)\n\n\ndef color_style_names():\n \"\"\"Returns an iterable of all available style names.\"\"\"\n return builtins.__xonsh__.shell.shell.color_style_names()\n\n\ndef color_style():\n \"\"\"Returns the current color map.\"\"\"\n return builtins.__xonsh__.shell.shell.color_style()\n\n\ndef _token_attr_from_stylemap(stylemap):\n \"\"\"yields tokens attr, and index from a stylemap \"\"\"\n import prompt_toolkit as ptk\n\n if builtins.__xonsh__.shell.shell_type == \"prompt_toolkit1\":\n style = ptk.styles.style_from_dict(stylemap)\n for token in stylemap:\n yield token, style.token_to_attrs[token]\n else:\n style = ptk.styles.style_from_pygments_dict(stylemap)\n for token in stylemap:\n style_str = \"class:{}\".format(\n ptk.styles.pygments.pygments_token_to_classname(token)\n )\n yield (token, style.get_attrs_for_style_str(style_str))\n\n\ndef _get_color_lookup_table():\n \"\"\"Returns the prompt_toolkit win32 ColorLookupTable \"\"\"\n if builtins.__xonsh__.shell.shell_type == \"prompt_toolkit1\":\n from prompt_toolkit.terminal.win32_output import ColorLookupTable\n else:\n from prompt_toolkit.output.win32 import ColorLookupTable\n return ColorLookupTable()\n\n\ndef _get_color_indexes(style_map):\n \"\"\"Generates the color and windows color index for a style \"\"\"\n table = _get_color_lookup_table()\n for token, attr in _token_attr_from_stylemap(style_map):\n if attr.color:\n index = table.lookup_fg_color(attr.color)\n try:\n rgb = (\n int(attr.color[0:2], 16),\n int(attr.color[2:4], 16),\n int(attr.color[4:6], 16),\n )\n except Exception:\n rgb = None\n yield token, index, rgb\n\n\n# Map of new PTK2 color names to PTK1 variants\nPTK_NEW_OLD_COLOR_MAP = LazyObject(\n lambda: {\n \"black\": \"black\",\n \"red\": \"darkred\",\n \"green\": \"darkgreen\",\n \"yellow\": \"brown\",\n \"blue\": \"darkblue\",\n \"magenta\": \"purple\",\n \"cyan\": \"teal\",\n \"gray\": \"lightgray\",\n \"brightblack\": \"darkgray\",\n \"brightred\": \"red\",\n \"brightgreen\": \"green\",\n \"brightyellow\": \"yellow\",\n \"brightblue\": \"blue\",\n \"brightmagenta\": \"fuchsia\",\n \"brightcyan\": \"turquoise\",\n \"white\": \"white\",\n },\n globals(),\n \"PTK_NEW_OLD_COLOR_MAP\",\n)\n\n# Map of new ansicolor names to old PTK1 names\nANSICOLOR_NAMES_MAP = LazyObject(\n lambda: {\"ansi\" + k: \"#ansi\" + v for k, v in PTK_NEW_OLD_COLOR_MAP.items()},\n globals(),\n \"ANSICOLOR_NAMES_MAP\",\n)\n\n\ndef _win10_color_map():\n cmap = {\n \"ansiblack\": (12, 12, 12),\n \"ansiblue\": (0, 55, 218),\n \"ansigreen\": (19, 161, 14),\n \"ansicyan\": (58, 150, 221),\n \"ansired\": (197, 15, 31),\n \"ansimagenta\": (136, 23, 152),\n \"ansiyellow\": (193, 156, 0),\n \"ansigray\": (204, 204, 204),\n \"ansibrightblack\": (118, 118, 118),\n \"ansibrightblue\": (59, 120, 255),\n \"ansibrightgreen\": (22, 198, 12),\n \"ansibrightcyan\": (97, 214, 214),\n \"ansibrightred\": (231, 72, 86),\n \"ansibrightmagenta\": (180, 0, 158),\n \"ansibrightyellow\": (249, 241, 165),\n \"ansiwhite\": (242, 242, 242),\n }\n return {\n k: \"#{0:02x}{1:02x}{2:02x}\".format(r, g, b) for k, (r, g, b) in cmap.items()\n }\n\n\nWIN10_COLOR_MAP = LazyObject(_win10_color_map, globals(), \"WIN10_COLOR_MAP\")\n\n\ndef _win_bold_color_map():\n \"\"\" Map dark ansi colors to lighter version. \"\"\"\n return {\n \"ansiblack\": \"ansibrightblack\",\n \"ansiblue\": \"ansibrightblue\",\n \"ansigreen\": \"ansibrightgreen\",\n \"ansicyan\": \"ansibrightcyan\",\n \"ansired\": \"ansibrightred\",\n \"ansimagenta\": \"ansibrightmagenta\",\n \"ansiyellow\": \"ansibrightyellow\",\n \"ansigray\": \"ansiwhite\",\n }\n\n\nWIN_BOLD_COLOR_MAP = LazyObject(_win_bold_color_map, globals(), \"WIN_BOLD_COLOR_MAP\")\n\n\ndef hardcode_colors_for_win10(style_map):\n \"\"\"Replace all ansi colors with hardcoded colors to avoid unreadable defaults\n in conhost.exe\n \"\"\"\n modified_style = {}\n if not builtins.__xonsh__.env[\"PROMPT_TOOLKIT_COLOR_DEPTH\"]:\n builtins.__xonsh__.env[\"PROMPT_TOOLKIT_COLOR_DEPTH\"] = \"DEPTH_24_BIT\"\n # Replace all ansi colors with hardcoded colors to avoid unreadable defaults\n # in conhost.exe\n for token, style_str in style_map.items():\n for ansicolor in WIN10_COLOR_MAP:\n if ansicolor in style_str:\n if \"bold\" in style_str and \"nobold\" not in style_str:\n # Win10 doesn't yet handle bold colors. Instead dark\n # colors are mapped to their lighter version. We simulate\n # the same here.\n style_str.replace(\"bold\", \"\")\n hexcolor = WIN10_COLOR_MAP[\n WIN_BOLD_COLOR_MAP.get(ansicolor, ansicolor)\n ]\n else:\n hexcolor = WIN10_COLOR_MAP[ansicolor]\n style_str = style_str.replace(ansicolor, hexcolor)\n modified_style[token] = style_str\n return modified_style\n\n\ndef ansicolors_to_ptk1_names(stylemap):\n \"\"\"Converts ansicolor names in a stylemap to old PTK1 color names\n \"\"\"\n modified_stylemap = {}\n for token, style_str in stylemap.items():\n for color, ptk1_color in ANSICOLOR_NAMES_MAP.items():\n if \"#\" + color not in style_str:\n style_str = style_str.replace(color, ptk1_color)\n modified_stylemap[token] = style_str\n return modified_stylemap\n\n\ndef intensify_colors_for_cmd_exe(style_map):\n \"\"\"Returns a modified style to where colors that maps to dark\n colors are replaced with brighter versions.\n \"\"\"\n modified_style = {}\n replace_colors = {\n 1: \"ansibrightcyan\", # subst blue with bright cyan\n 2: \"ansibrightgreen\", # subst green with bright green\n 4: \"ansibrightred\", # subst red with bright red\n 5: \"ansibrightmagenta\", # subst magenta with bright magenta\n 6: \"ansibrightyellow\", # subst yellow with bright yellow\n 9: \"ansicyan\", # subst intense blue with dark cyan (more readable)\n }\n if builtins.__xonsh__.shell.shell_type == \"prompt_toolkit1\":\n replace_colors = ansicolors_to_ptk1_names(replace_colors)\n for token, idx, _ in _get_color_indexes(style_map):\n if idx in replace_colors:\n modified_style[token] = replace_colors[idx]\n return modified_style\n\n\ndef intensify_colors_on_win_setter(enable):\n \"\"\"Resets the style when setting the INTENSIFY_COLORS_ON_WIN\n environment variable.\n \"\"\"\n enable = to_bool(enable)\n if builtins.__xonsh__.shell is not None:\n if hasattr(builtins.__xonsh__.shell.shell.styler, \"style_name\"):\n delattr(builtins.__xonsh__.shell.shell.styler, \"style_name\")\n return enable\n\n\ndef format_std_prepost(template, env=None):\n \"\"\"Formats a template prefix/postfix string for a standard buffer.\n Returns a string suitable for prepending or appending.\n \"\"\"\n if not template:\n return \"\"\n env = builtins.__xonsh__.env if env is None else env\n shell = builtins.__xonsh__.shell.shell\n try:\n s = shell.prompt_formatter(template)\n except Exception:\n print_exception()\n # \\001\\002 is there to fool pygments into not returning an empty string\n # for potentially empty input. This happens when the template is just a\n # color code with no visible text.\n invis = \"\\001\\002\"\n s = shell.format_color(invis + s + invis, force_string=True)\n s = s.replace(invis, \"\")\n return s\n\n\n_RE_STRING_START = \"[bBprRuUf]*\"\n_RE_STRING_TRIPLE_DOUBLE = '\"\"\"'\n_RE_STRING_TRIPLE_SINGLE = \"'''\"\n_RE_STRING_DOUBLE = '\"'\n_RE_STRING_SINGLE = \"'\"\n_STRINGS = (\n _RE_STRING_TRIPLE_DOUBLE,\n _RE_STRING_TRIPLE_SINGLE,\n _RE_STRING_DOUBLE,\n _RE_STRING_SINGLE,\n)\nRE_BEGIN_STRING = LazyObject(\n lambda: re.compile(\"(\" + _RE_STRING_START + \"(\" + \"|\".join(_STRINGS) + \"))\"),\n globals(),\n \"RE_BEGIN_STRING\",\n)\n\"\"\"Regular expression matching the start of a string, including quotes and\nleading characters (r, b, or u)\"\"\"\n\nRE_STRING_START = LazyObject(\n lambda: re.compile(_RE_STRING_START), globals(), \"RE_STRING_START\"\n)\n\"\"\"Regular expression matching the characters before the quotes when starting a\nstring (r, b, or u, case insensitive)\"\"\"\n\nRE_STRING_CONT = LazyDict(\n {\n '\"': lambda: re.compile(r'((\\\\(.|\\n))|([^\"\\\\]))*'),\n \"'\": lambda: re.compile(r\"((\\\\(.|\\n))|([^'\\\\]))*\"),\n '\"\"\"': lambda: re.compile(r'((\\\\(.|\\n))|([^\"\\\\])|(\"(?!\"\"))|\\n)*'),\n \"'''\": lambda: re.compile(r\"((\\\\(.|\\n))|([^'\\\\])|('(?!''))|\\n)*\"),\n },\n globals(),\n \"RE_STRING_CONT\",\n)\n\"\"\"Dictionary mapping starting quote sequences to regular expressions that\nmatch the contents of a string beginning with those quotes (not including the\nterminating quotes)\"\"\"\n\n\n@lazyobject\ndef RE_COMPLETE_STRING():\n ptrn = (\n \"^\"\n + _RE_STRING_START\n + \"(?P\"\n + \"|\".join(_STRINGS)\n + \")\"\n + \".*?(?P=quote)$\"\n )\n return re.compile(ptrn, re.DOTALL)\n\n\ndef strip_simple_quotes(s):\n \"\"\"Gets rid of single quotes, double quotes, single triple quotes, and\n single double quotes from a string, if present front and back of a string.\n Otherwiswe, does nothing.\n \"\"\"\n starts_single = s.startswith(\"'\")\n starts_double = s.startswith('\"')\n if not starts_single and not starts_double:\n return s\n elif starts_single:\n ends_single = s.endswith(\"'\")\n if not ends_single:\n return s\n elif s.startswith(\"'''\") and s.endswith(\"'''\") and len(s) >= 6:\n return s[3:-3]\n elif len(s) >= 2:\n return s[1:-1]\n else:\n return s\n else:\n # starts double\n ends_double = s.endswith('\"')\n if not ends_double:\n return s\n elif s.startswith('\"\"\"') and s.endswith('\"\"\"') and len(s) >= 6:\n return s[3:-3]\n elif len(s) >= 2:\n return s[1:-1]\n else:\n return s\n\n\ndef check_for_partial_string(x):\n \"\"\"Returns the starting index (inclusive), ending index (exclusive), and\n starting quote string of the most recent Python string found in the input.\n\n check_for_partial_string(x) -> (startix, endix, quote)\n\n Parameters\n ----------\n x : str\n The string to be checked (representing a line of terminal input)\n\n Returns\n -------\n startix : int (or None)\n The index where the most recent Python string found started\n (inclusive), or None if no strings exist in the input\n\n endix : int (or None)\n The index where the most recent Python string found ended (exclusive),\n or None if no strings exist in the input OR if the input ended in the\n middle of a Python string\n\n quote : str (or None)\n A string containing the quote used to start the string (e.g., b\", \",\n '''), or None if no string was found.\n \"\"\"\n string_indices = []\n starting_quote = []\n current_index = 0\n match = re.search(RE_BEGIN_STRING, x)\n while match is not None:\n # add the start in\n start = match.start()\n quote = match.group(0)\n lenquote = len(quote)\n current_index += start\n # store the starting index of the string, as well as the\n # characters in the starting quotes (e.g., \", ', \"\"\", r\", etc)\n string_indices.append(current_index)\n starting_quote.append(quote)\n # determine the string that should terminate this string\n ender = re.sub(RE_STRING_START, \"\", quote)\n x = x[start + lenquote :]\n current_index += lenquote\n # figure out what is inside the string\n continuer = RE_STRING_CONT[ender]\n contents = re.match(continuer, x)\n inside = contents.group(0)\n leninside = len(inside)\n current_index += contents.start() + leninside + len(ender)\n # if we are not at the end of the input string, add the ending index of\n # the string to string_indices\n if contents.end() < len(x):\n string_indices.append(current_index)\n x = x[leninside + len(ender) :]\n # find the next match\n match = re.search(RE_BEGIN_STRING, x)\n numquotes = len(string_indices)\n if numquotes == 0:\n return (None, None, None)\n elif numquotes % 2:\n return (string_indices[-1], None, starting_quote[-1])\n else:\n return (string_indices[-2], string_indices[-1], starting_quote[-1])\n\n\n# regular expressions for matching environment variables\n# i.e $FOO, ${'FOO'}\n@lazyobject\ndef POSIX_ENVVAR_REGEX():\n pat = r\"\"\"\\$({(?P['\"])|)(?P\\w+)((?P=quote)}|(?:\\1\\b))\"\"\"\n return re.compile(pat)\n\n\ndef expandvars(path):\n \"\"\"Expand shell variables of the forms $var, ${var} and %var%.\n Unknown variables are left unchanged.\"\"\"\n env = builtins.__xonsh__.env\n if isinstance(path, bytes):\n path = path.decode(\n encoding=env.get(\"XONSH_ENCODING\"), errors=env.get(\"XONSH_ENCODING_ERRORS\")\n )\n elif isinstance(path, pathlib.Path):\n # get the path's string representation\n path = str(path)\n if \"$\" in path:\n for match in POSIX_ENVVAR_REGEX.finditer(path):\n name = match.group(\"envvar\")\n if name in env:\n ensurer = env.get_ensurer(name)\n val = env[name]\n value = str(val) if ensurer.detype is None else ensurer.detype(val)\n value = str(val) if value is None else value\n path = POSIX_ENVVAR_REGEX.sub(value, path, count=1)\n return path\n\n\n#\n# File handling tools\n#\n\n\ndef backup_file(fname):\n \"\"\"Moves an existing file to a new name that has the current time right\n before the extension.\n \"\"\"\n # lazy imports\n import shutil\n from datetime import datetime\n\n base, ext = os.path.splitext(fname)\n timestamp = datetime.now().strftime(\"%Y-%m-%d-%H-%M-%S-%f\")\n newfname = \"%s.%s%s\" % (base, timestamp, ext)\n shutil.move(fname, newfname)\n\n\ndef normabspath(p):\n \"\"\"Returns as normalized absolute path, namely, normcase(abspath(p))\"\"\"\n return os.path.normcase(os.path.abspath(p))\n\n\ndef expanduser_abs_path(inp):\n \"\"\" Provides user expanded absolute path \"\"\"\n return os.path.abspath(expanduser(inp))\n\n\nWINDOWS_DRIVE_MATCHER = LazyObject(\n lambda: re.compile(r\"^\\w:\"), globals(), \"WINDOWS_DRIVE_MATCHER\"\n)\n\n\ndef expand_case_matching(s):\n \"\"\"Expands a string to a case insensitive globable string.\"\"\"\n t = []\n openers = {\"[\", \"{\"}\n closers = {\"]\", \"}\"}\n nesting = 0\n\n drive_part = WINDOWS_DRIVE_MATCHER.match(s) if ON_WINDOWS else None\n\n if drive_part:\n drive_part = drive_part.group(0)\n t.append(drive_part)\n s = s[len(drive_part) :]\n\n for c in s:\n if c in openers:\n nesting += 1\n elif c in closers:\n nesting -= 1\n elif nesting > 0:\n pass\n elif c.isalpha():\n folded = c.casefold()\n if len(folded) == 1:\n c = \"[{0}{1}]\".format(c.upper(), c.lower())\n else:\n newc = [\"[{0}{1}]?\".format(f.upper(), f.lower()) for f in folded[:-1]]\n newc = \"\".join(newc)\n newc += \"[{0}{1}{2}]\".format(folded[-1].upper(), folded[-1].lower(), c)\n c = newc\n t.append(c)\n return \"\".join(t)\n\n\ndef globpath(\n s, ignore_case=False, return_empty=False, sort_result=None, include_dotfiles=None\n):\n \"\"\"Simple wrapper around glob that also expands home and env vars.\"\"\"\n o, s = _iglobpath(\n s,\n ignore_case=ignore_case,\n sort_result=sort_result,\n include_dotfiles=include_dotfiles,\n )\n o = list(o)\n no_match = [] if return_empty else [s]\n return o if len(o) != 0 else no_match\n\n\ndef _dotglobstr(s):\n modified = False\n dotted_s = s\n if \"/*\" in dotted_s:\n dotted_s = dotted_s.replace(\"/*\", \"/.*\")\n dotted_s = dotted_s.replace(\"/.**/.*\", \"/**/.*\")\n modified = True\n if dotted_s.startswith(\"*\") and not dotted_s.startswith(\"**\"):\n dotted_s = \".\" + dotted_s\n modified = True\n return dotted_s, modified\n\n\ndef _iglobpath(s, ignore_case=False, sort_result=None, include_dotfiles=None):\n s = builtins.__xonsh__.expand_path(s)\n if sort_result is None:\n sort_result = builtins.__xonsh__.env.get(\"GLOB_SORTED\")\n if include_dotfiles is None:\n include_dotfiles = builtins.__xonsh__.env.get(\"DOTGLOB\")\n if ignore_case:\n s = expand_case_matching(s)\n if sys.version_info > (3, 5):\n if \"**\" in s and \"**/*\" not in s:\n s = s.replace(\"**\", \"**/*\")\n if include_dotfiles:\n dotted_s, dotmodified = _dotglobstr(s)\n # `recursive` is only a 3.5+ kwarg.\n if sort_result:\n paths = glob.glob(s, recursive=True)\n if include_dotfiles and dotmodified:\n paths.extend(glob.iglob(dotted_s, recursive=True))\n paths.sort()\n paths = iter(paths)\n else:\n paths = glob.iglob(s, recursive=True)\n if include_dotfiles and dotmodified:\n paths = itertools.chain(glob.iglob(dotted_s, recursive=True), paths)\n return paths, s\n else:\n if include_dotfiles:\n dotted_s, dotmodified = _dotglobstr(s)\n if sort_result:\n paths = glob.glob(s)\n if include_dotfiles and dotmodified:\n paths.extend(glob.iglob(dotted_s))\n paths.sort()\n paths = iter(paths)\n else:\n paths = glob.iglob(s)\n if include_dotfiles and dotmodified:\n paths = itertools.chain(glob.iglob(dotted_s), paths)\n return paths, s\n\n\ndef iglobpath(s, ignore_case=False, sort_result=None, include_dotfiles=None):\n \"\"\"Simple wrapper around iglob that also expands home and env vars.\"\"\"\n try:\n return _iglobpath(\n s,\n ignore_case=ignore_case,\n sort_result=sort_result,\n include_dotfiles=include_dotfiles,\n )[0]\n except IndexError:\n # something went wrong in the actual iglob() call\n return iter(())\n\n\ndef ensure_timestamp(t, datetime_format=None):\n if isinstance(t, (int, float)):\n return t\n try:\n return float(t)\n except (ValueError, TypeError):\n pass\n if datetime_format is None:\n datetime_format = builtins.__xonsh__.env[\"XONSH_DATETIME_FORMAT\"]\n if isinstance(t, datetime.datetime):\n t = t.timestamp()\n else:\n t = datetime.datetime.strptime(t, datetime_format).timestamp()\n return t\n\n\ndef format_datetime(dt):\n \"\"\"Format datetime object to string base on $XONSH_DATETIME_FORMAT Env.\"\"\"\n format_ = builtins.__xonsh__.env[\"XONSH_DATETIME_FORMAT\"]\n return dt.strftime(format_)\n\n\ndef columnize(elems, width=80, newline=\"\\n\"):\n \"\"\"Takes an iterable of strings and returns a list of lines with the\n elements placed in columns. Each line will be at most *width* columns.\n The newline character will be appended to the end of each line.\n \"\"\"\n sizes = [len(e) + 1 for e in elems]\n total = sum(sizes)\n nelem = len(elems)\n if total - 1 <= width:\n ncols = len(sizes)\n nrows = 1\n columns = [sizes]\n last_longest_row = total\n enter_loop = False\n else:\n ncols = 1\n nrows = len(sizes)\n columns = [sizes]\n last_longest_row = max(sizes)\n enter_loop = True\n while enter_loop:\n longest_row = sum(map(max, columns))\n if longest_row - 1 <= width:\n # we might be able to fit another column.\n ncols += 1\n nrows = nelem // ncols\n columns = [sizes[i * nrows : (i + 1) * nrows] for i in range(ncols)]\n last_longest_row = longest_row\n else:\n # we can't fit another column\n ncols -= 1\n nrows = nelem // ncols\n break\n pad = (width - last_longest_row + ncols) // ncols\n pad = pad if pad > 1 else 1\n data = [elems[i * nrows : (i + 1) * nrows] for i in range(ncols)]\n colwidths = [max(map(len, d)) + pad for d in data]\n colwidths[-1] -= pad\n row_t = \"\".join([\"{{row[{i}]: <{{w[{i}]}}}}\".format(i=i) for i in range(ncols)])\n row_t += newline\n lines = [\n row_t.format(row=row, w=colwidths)\n for row in itertools.zip_longest(*data, fillvalue=\"\")\n ]\n return lines\n\n\nALIAS_KWARG_NAMES = frozenset([\"args\", \"stdin\", \"stdout\", \"stderr\", \"spec\", \"stack\"])\n\n\ndef unthreadable(f):\n \"\"\"Decorator that specifies that a callable alias should be run only\n on the main thread process. This is often needed for debuggers and\n profilers.\n \"\"\"\n f.__xonsh_threadable__ = False\n return f\n\n\ndef uncapturable(f):\n \"\"\"Decorator that specifies that a callable alias should not be run with\n any capturing. This is often needed if the alias call interactive\n subprocess, like pagers and text editors.\n \"\"\"\n f.__xonsh_capturable__ = False\n return f\n\n\ndef carriage_return():\n \"\"\"Writes a carriage return to stdout, and nothing else.\"\"\"\n print(\"\\r\", flush=True, end=\"\")\n\n\ndef deprecated(deprecated_in=None, removed_in=None):\n \"\"\"Parametrized decorator that deprecates a function in a graceful manner.\n\n Updates the decorated function's docstring to mention the version\n that deprecation occurred in and the version it will be removed\n in if both of these values are passed.\n\n When removed_in is not a release equal to or less than the current\n release, call ``warnings.warn`` with details, while raising\n ``DeprecationWarning``.\n\n When removed_in is a release equal to or less than the current release,\n raise an ``AssertionError``.\n\n Parameters\n ----------\n deprecated_in : str\n The version number that deprecated this function.\n removed_in : str\n The version number that this function will be removed in.\n \"\"\"\n message_suffix = _deprecated_message_suffix(deprecated_in, removed_in)\n if not message_suffix:\n message_suffix = \"\"\n\n def decorated(func):\n warning_message = \"{} has been deprecated\".format(func.__name__)\n warning_message += message_suffix\n\n @functools.wraps(func)\n def wrapped(*args, **kwargs):\n _deprecated_error_on_expiration(func.__name__, removed_in)\n func(*args, **kwargs)\n warnings.warn(warning_message, DeprecationWarning)\n\n wrapped.__doc__ = (\n \"{}\\n\\n{}\".format(wrapped.__doc__, warning_message)\n if wrapped.__doc__\n else warning_message\n )\n\n return wrapped\n\n return decorated\n\n\ndef _deprecated_message_suffix(deprecated_in, removed_in):\n if deprecated_in and removed_in:\n message_suffix = \" in version {} and will be removed in version {}\".format(\n deprecated_in, removed_in\n )\n elif deprecated_in and not removed_in:\n message_suffix = \" in version {}\".format(deprecated_in)\n elif not deprecated_in and removed_in:\n message_suffix = \" and will be removed in version {}\".format(removed_in)\n else:\n message_suffix = None\n\n return message_suffix\n\n\ndef _deprecated_error_on_expiration(name, removed_in):\n if not removed_in:\n return\n elif LooseVersion(__version__) >= LooseVersion(removed_in):\n raise AssertionError(\n \"{} has passed its version {} expiry date!\".format(name, removed_in)\n )\n" }, { "path": "xonsh/tracer.py", "content": "\"\"\"Implements a xonsh tracer.\"\"\"\nimport os\nimport re\nimport sys\nimport inspect\nimport argparse\nimport linecache\nimport importlib\nimport functools\n\nfrom xonsh.lazyasd import LazyObject\nfrom xonsh.platform import HAS_PYGMENTS\nfrom xonsh.tools import DefaultNotGiven, print_color, normabspath, to_bool\nfrom xonsh.inspectors import find_file, getouterframes\nfrom xonsh.lazyimps import pygments, pyghooks\nfrom xonsh.proc import STDOUT_CAPTURE_KINDS\nimport xonsh.prompt.cwd as prompt\n\nterminal = LazyObject(\n lambda: importlib.import_module(\"pygments.formatters.terminal\"),\n globals(),\n \"terminal\",\n)\n\n\nclass TracerType(object):\n \"\"\"Represents a xonsh tracer object, which keeps track of all tracing\n state. This is a singleton.\n \"\"\"\n\n _inst = None\n valid_events = frozenset([\"line\", \"call\"])\n\n def __new__(cls, *args, **kwargs):\n if cls._inst is None:\n cls._inst = super(TracerType, cls).__new__(cls, *args, **kwargs)\n return cls._inst\n\n def __init__(self):\n self.prev_tracer = DefaultNotGiven\n self.files = set()\n self.usecolor = True\n self.lexer = pyghooks.XonshLexer()\n self.formatter = terminal.TerminalFormatter()\n self._last = (\"\", -1) # filename, lineno tuple\n\n def __del__(self):\n for f in set(self.files):\n self.stop(f)\n\n def color_output(self, usecolor):\n \"\"\"Specify whether or not the tracer output should be colored.\"\"\"\n # we have to use a function to set usecolor because of the way that\n # lazyasd works. Namely, it cannot dispatch setattr to the target\n # object without being unable to access its own __dict__. This makes\n # setting an attr look like getting a function.\n self.usecolor = usecolor\n\n def start(self, filename):\n \"\"\"Starts tracing a file.\"\"\"\n files = self.files\n if len(files) == 0:\n self.prev_tracer = sys.gettrace()\n files.add(normabspath(filename))\n sys.settrace(self.trace)\n curr = inspect.currentframe()\n for frame, fname, *_ in getouterframes(curr, context=0):\n if normabspath(fname) in files:\n frame.f_trace = self.trace\n\n def stop(self, filename):\n \"\"\"Stops tracing a file.\"\"\"\n filename = normabspath(filename)\n self.files.discard(filename)\n if len(self.files) == 0:\n sys.settrace(self.prev_tracer)\n curr = inspect.currentframe()\n for frame, fname, *_ in getouterframes(curr, context=0):\n if normabspath(fname) == filename:\n frame.f_trace = self.prev_tracer\n self.prev_tracer = DefaultNotGiven\n\n def trace(self, frame, event, arg):\n \"\"\"Implements a line tracing function.\"\"\"\n if event not in self.valid_events:\n return self.trace\n fname = find_file(frame)\n if fname in self.files:\n lineno = frame.f_lineno\n curr = (fname, lineno)\n if curr != self._last:\n line = linecache.getline(fname, lineno).rstrip()\n s = tracer_format_line(\n fname,\n lineno,\n line,\n color=self.usecolor,\n lexer=self.lexer,\n formatter=self.formatter,\n )\n print_color(s)\n self._last = curr\n return self.trace\n\n\ntracer = LazyObject(TracerType, globals(), \"tracer\")\n\nCOLORLESS_LINE = \"{fname}:{lineno}:{line}\"\nCOLOR_LINE = \"{{PURPLE}}{fname}{{BLUE}}:\" \"{{GREEN}}{lineno}{{BLUE}}:\" \"{{NO_COLOR}}\"\n\n\ndef tracer_format_line(fname, lineno, line, color=True, lexer=None, formatter=None):\n \"\"\"Formats a trace line suitable for printing.\"\"\"\n fname = min(fname, prompt._replace_home(fname), os.path.relpath(fname), key=len)\n if not color:\n return COLORLESS_LINE.format(fname=fname, lineno=lineno, line=line)\n cline = COLOR_LINE.format(fname=fname, lineno=lineno)\n if not HAS_PYGMENTS:\n return cline + line\n # OK, so we have pygments\n tokens = pyghooks.partial_color_tokenize(cline)\n lexer = lexer or pyghooks.XonshLexer()\n tokens += pygments.lex(line, lexer=lexer)\n if tokens[-1][1] == \"\\n\":\n del tokens[-1]\n elif tokens[-1][1].endswith(\"\\n\"):\n tokens[-1] = (tokens[-1][0], tokens[-1][1].rstrip())\n return tokens\n\n\n#\n# Command line interface\n#\n\n\ndef _find_caller(args):\n \"\"\"Somewhat hacky method of finding the __file__ based on the line executed.\"\"\"\n re_line = re.compile(r\"[^;\\s|&<>]+\\s+\" + r\"\\s+\".join(args))\n curr = inspect.currentframe()\n for _, fname, lineno, _, lines, _ in getouterframes(curr, context=1)[3:]:\n if lines is not None and re_line.search(lines[0]) is not None:\n return fname\n elif (\n lineno == 1 and re_line.search(linecache.getline(fname, lineno)) is not None\n ):\n # There is a bug in CPython such that getouterframes(curr, context=1)\n # will actually return the 2nd line in the code_context field, even though\n # line number is itself correct. We manually fix that in this branch.\n return fname\n else:\n msg = (\n \"xonsh: warning: __file__ name could not be found. You may be \"\n \"trying to trace interactively. Please pass in the file names \"\n \"you want to trace explicitly.\"\n )\n print(msg, file=sys.stderr)\n\n\ndef _on(ns, args):\n \"\"\"Turns on tracing for files.\"\"\"\n for f in ns.files:\n if f == \"__file__\":\n f = _find_caller(args)\n if f is None:\n continue\n tracer.start(f)\n\n\ndef _off(ns, args):\n \"\"\"Turns off tracing for files.\"\"\"\n for f in ns.files:\n if f == \"__file__\":\n f = _find_caller(args)\n if f is None:\n continue\n tracer.stop(f)\n\n\ndef _color(ns, args):\n \"\"\"Manages color action for tracer CLI.\"\"\"\n tracer.color_output(ns.toggle)\n\n\n@functools.lru_cache(1)\ndef _tracer_create_parser():\n \"\"\"Creates tracer argument parser\"\"\"\n p = argparse.ArgumentParser(\n prog=\"trace\", description=\"tool for tracing xonsh code as it runs.\"\n )\n subp = p.add_subparsers(title=\"action\", dest=\"action\")\n onp = subp.add_parser(\n \"on\", aliases=[\"start\", \"add\"], help=\"begins tracing selected files.\"\n )\n onp.add_argument(\n \"files\",\n nargs=\"*\",\n default=[\"__file__\"],\n help=(\n 'file paths to watch, use \"__file__\" (default) to select '\n \"the current file.\"\n ),\n )\n off = subp.add_parser(\n \"off\", aliases=[\"stop\", \"del\", \"rm\"], help=\"removes selected files fom tracing.\"\n )\n off.add_argument(\n \"files\",\n nargs=\"*\",\n default=[\"__file__\"],\n help=(\n 'file paths to stop watching, use \"__file__\" (default) to '\n \"select the current file.\"\n ),\n )\n col = subp.add_parser(\"color\", help=\"output color management for tracer.\")\n col.add_argument(\n \"toggle\", type=to_bool, help=\"true/false, y/n, etc. to toggle color usage.\"\n )\n return p\n\n\n_TRACER_MAIN_ACTIONS = {\n \"on\": _on,\n \"add\": _on,\n \"start\": _on,\n \"rm\": _off,\n \"off\": _off,\n \"del\": _off,\n \"stop\": _off,\n \"color\": _color,\n}\n\n\ndef tracermain(args=None, stdin=None, stdout=None, stderr=None, spec=None):\n \"\"\"Main function for tracer command-line interface.\"\"\"\n parser = _tracer_create_parser()\n ns = parser.parse_args(args)\n usecolor = (spec.captured not in STDOUT_CAPTURE_KINDS) and sys.stdout.isatty()\n tracer.color_output(usecolor)\n return _TRACER_MAIN_ACTIONS[ns.action](ns, args)\n" }, { "path": "xonsh/winutils.py", "content": "\"\"\"\nThis file is based on the code from https://github.com/JustAMan/pyWinClobber/blob/master/win32elevate.py\n\nCopyright (c) 2013 by JustAMan at GitHub\n\nPermission is hereby granted, free of charge, to any person obtaining a copy of\nthis software and associated documentation files (the \"Software\"), to deal in\nthe Software without restriction, including without limitation the rights to\nuse, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of\nthe Software, and to permit persons to whom the Software is furnished to do so,\nsubject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS\nFOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR\nCOPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER\nIN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\nCONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n\"\"\"\nimport os\nimport ctypes\nimport subprocess\nfrom ctypes import c_ulong, c_char_p, c_int, c_void_p, POINTER, byref\nfrom ctypes.wintypes import (\n HANDLE,\n BOOL,\n DWORD,\n HWND,\n HINSTANCE,\n HKEY,\n LPDWORD,\n SHORT,\n LPCWSTR,\n WORD,\n SMALL_RECT,\n LPCSTR,\n)\n\nfrom xonsh.lazyasd import lazyobject\nfrom xonsh import lazyimps # we aren't amalgamated in this module.\nfrom xonsh import platform\n\n\n__all__ = (\"sudo\",)\n\n\n@lazyobject\ndef CloseHandle():\n ch = ctypes.windll.kernel32.CloseHandle\n ch.argtypes = (HANDLE,)\n ch.restype = BOOL\n return ch\n\n\n@lazyobject\ndef GetActiveWindow():\n gaw = ctypes.windll.user32.GetActiveWindow\n gaw.argtypes = ()\n gaw.restype = HANDLE\n return gaw\n\n\nTOKEN_READ = 0x20008\n\n\nclass ShellExecuteInfo(ctypes.Structure):\n _fields_ = [\n (\"cbSize\", DWORD),\n (\"fMask\", c_ulong),\n (\"hwnd\", HWND),\n (\"lpVerb\", c_char_p),\n (\"lpFile\", c_char_p),\n (\"lpParameters\", c_char_p),\n (\"lpDirectory\", c_char_p),\n (\"nShow\", c_int),\n (\"hInstApp\", HINSTANCE),\n (\"lpIDList\", c_void_p),\n (\"lpClass\", c_char_p),\n (\"hKeyClass\", HKEY),\n (\"dwHotKey\", DWORD),\n (\"hIcon\", HANDLE),\n (\"hProcess\", HANDLE),\n ]\n\n def __init__(self, **kw):\n ctypes.Structure.__init__(self)\n self.cbSize = ctypes.sizeof(self)\n for field_name, field_value in kw.items():\n setattr(self, field_name, field_value)\n\n\n@lazyobject\ndef ShellExecuteEx():\n see = ctypes.windll.Shell32.ShellExecuteExA\n PShellExecuteInfo = ctypes.POINTER(ShellExecuteInfo)\n see.argtypes = (PShellExecuteInfo,)\n see.restype = BOOL\n return see\n\n\n@lazyobject\ndef WaitForSingleObject():\n wfso = ctypes.windll.kernel32.WaitForSingleObject\n wfso.argtypes = (HANDLE, DWORD)\n wfso.restype = DWORD\n return wfso\n\n\n# SW_HIDE = 0\nSW_SHOW = 5\nSEE_MASK_NOCLOSEPROCESS = 0x00000040\nSEE_MASK_NO_CONSOLE = 0x00008000\nINFINITE = -1\n\n\ndef wait_and_close_handle(process_handle):\n \"\"\"\n Waits till spawned process finishes and closes the handle for it\n\n Parameters\n ----------\n process_handle : HANDLE\n The Windows handle for the process\n \"\"\"\n WaitForSingleObject(process_handle, INFINITE)\n CloseHandle(process_handle)\n\n\ndef sudo(executable, args=None):\n \"\"\"\n This will re-run current Python script requesting to elevate administrative rights.\n\n Parameters\n ----------\n param executable : str\n The path/name of the executable\n args : list of str\n The arguments to be passed to the executable\n \"\"\"\n if not args:\n args = []\n\n execute_info = ShellExecuteInfo(\n fMask=SEE_MASK_NOCLOSEPROCESS | SEE_MASK_NO_CONSOLE,\n hwnd=GetActiveWindow(),\n lpVerb=b\"runas\",\n lpFile=executable.encode(\"utf-8\"),\n lpParameters=subprocess.list2cmdline(args).encode(\"utf-8\"),\n lpDirectory=None,\n nShow=SW_SHOW,\n )\n\n if not ShellExecuteEx(byref(execute_info)):\n raise ctypes.WinError()\n\n wait_and_close_handle(execute_info.hProcess)\n\n\n#\n# The following has been refactored from\n# http://stackoverflow.com/a/37505496/2312428\n#\n\n# input flags\nENABLE_PROCESSED_INPUT = 0x0001\nENABLE_LINE_INPUT = 0x0002\nENABLE_ECHO_INPUT = 0x0004\nENABLE_WINDOW_INPUT = 0x0008\nENABLE_MOUSE_INPUT = 0x0010\nENABLE_INSERT_MODE = 0x0020\nENABLE_QUICK_EDIT_MODE = 0x0040\n\n# output flags\nENABLE_PROCESSED_OUTPUT = 0x0001\nENABLE_WRAP_AT_EOL_OUTPUT = 0x0002\nENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x0004 # VT100 (Win 10)\n\n\ndef check_zero(result, func, args):\n if not result:\n err = ctypes.get_last_error()\n if err:\n raise ctypes.WinError(err)\n return args\n\n\n@lazyobject\ndef GetStdHandle():\n return lazyimps._winapi.GetStdHandle\n\n\n@lazyobject\ndef STDHANDLES():\n \"\"\"Tuple of the Windows handles for (stdin, stdout, stderr).\"\"\"\n hs = [\n lazyimps._winapi.STD_INPUT_HANDLE,\n lazyimps._winapi.STD_OUTPUT_HANDLE,\n lazyimps._winapi.STD_ERROR_HANDLE,\n ]\n hcons = []\n for h in hs:\n hcon = GetStdHandle(int(h))\n hcons.append(hcon)\n return tuple(hcons)\n\n\n@lazyobject\ndef GetConsoleMode():\n gcm = ctypes.windll.kernel32.GetConsoleMode\n gcm.errcheck = check_zero\n gcm.argtypes = (HANDLE, LPDWORD) # _In_ hConsoleHandle # _Out_ lpMode\n return gcm\n\n\ndef get_console_mode(fd=1):\n \"\"\"Get the mode of the active console input, output, or error\n buffer. Note that if the process isn't attached to a\n console, this function raises an EBADF IOError.\n\n Parameters\n ----------\n fd : int\n Standard buffer file descriptor, 0 for stdin, 1 for stdout (default),\n and 2 for stderr\n \"\"\"\n mode = DWORD()\n hcon = STDHANDLES[fd]\n GetConsoleMode(hcon, byref(mode))\n return mode.value\n\n\n@lazyobject\ndef SetConsoleMode():\n scm = ctypes.windll.kernel32.SetConsoleMode\n scm.errcheck = check_zero\n scm.argtypes = (HANDLE, DWORD) # _In_ hConsoleHandle # _Out_ lpMode\n return scm\n\n\ndef set_console_mode(mode, fd=1):\n \"\"\"Set the mode of the active console input, output, or\n error buffer. Note that if the process isn't attached to a\n console, this function raises an EBADF IOError.\n\n Parameters\n ----------\n mode : int\n Mode flags to set on the handle.\n fd : int, optional\n Standard buffer file descriptor, 0 for stdin, 1 for stdout (default),\n and 2 for stderr\n \"\"\"\n hcon = STDHANDLES[fd]\n SetConsoleMode(hcon, mode)\n\n\ndef enable_virtual_terminal_processing():\n \"\"\"Enables virtual terminal processing on Windows.\n This includes ANSI escape sequence interpretation.\n See http://stackoverflow.com/a/36760881/2312428\n \"\"\"\n SetConsoleMode(GetStdHandle(-11), 7)\n\n\n@lazyobject\ndef COORD():\n if platform.has_prompt_toolkit():\n # turns out that PTK has a separate ctype wrapper\n # for this struct and also wraps similar function calls\n # we need to use the same struct to prevent clashes.\n import prompt_toolkit.win32_types\n\n return prompt_toolkit.win32_types.COORD\n\n class _COORD(ctypes.Structure):\n \"\"\"Struct from the winapi, representing coordinates in the console.\n\n Attributes\n ----------\n X : int\n Column position\n Y : int\n Row position\n \"\"\"\n\n _fields_ = [(\"X\", SHORT), (\"Y\", SHORT)]\n\n return _COORD\n\n\n@lazyobject\ndef ReadConsoleOutputCharacterA():\n rcoc = ctypes.windll.kernel32.ReadConsoleOutputCharacterA\n rcoc.errcheck = check_zero\n rcoc.argtypes = (\n HANDLE, # _In_ hConsoleOutput\n LPCSTR, # _Out_ LPTSTR lpMode\n DWORD, # _In_ nLength\n COORD, # _In_ dwReadCoord,\n LPDWORD,\n ) # _Out_ lpNumberOfCharsRead\n rcoc.restype = BOOL\n return rcoc\n\n\n@lazyobject\ndef ReadConsoleOutputCharacterW():\n rcoc = ctypes.windll.kernel32.ReadConsoleOutputCharacterW\n rcoc.errcheck = check_zero\n rcoc.argtypes = (\n HANDLE, # _In_ hConsoleOutput\n LPCWSTR, # _Out_ LPTSTR lpMode\n DWORD, # _In_ nLength\n COORD, # _In_ dwReadCoord,\n LPDWORD,\n ) # _Out_ lpNumberOfCharsRead\n rcoc.restype = BOOL\n return rcoc\n\n\ndef read_console_output_character(x=0, y=0, fd=1, buf=None, bufsize=1024, raw=False):\n \"\"\"Reads characters from the console buffer.\n\n Parameters\n ----------\n x : int, optional\n Starting column.\n y : int, optional\n Starting row.\n fd : int, optional\n Standard buffer file descriptor, 0 for stdin, 1 for stdout (default),\n and 2 for stderr.\n buf : ctypes.c_wchar_p if raw else ctypes.c_wchar_p, optional\n An existing buffer to (re-)use.\n bufsize : int, optional\n The maximum read size.\n raw : bool, optional\n Whether to read in and return as bytes (True) or as a\n unicode string (False, default).\n\n Returns\n -------\n value : str\n Result of what was read, may be shorter than bufsize.\n \"\"\"\n hcon = STDHANDLES[fd]\n if buf is None:\n if raw:\n buf = ctypes.c_char_p(b\" \" * bufsize)\n else:\n buf = ctypes.c_wchar_p(\" \" * bufsize)\n coord = COORD(x, y)\n n = DWORD()\n if raw:\n ReadConsoleOutputCharacterA(hcon, buf, bufsize, coord, byref(n))\n else:\n ReadConsoleOutputCharacterW(hcon, buf, bufsize, coord, byref(n))\n return buf.value[: n.value]\n\n\ndef pread_console(fd, buffersize, offset, buf=None):\n \"\"\"This is a console-based implementation of os.pread() for windows.\n that uses read_console_output_character().\n \"\"\"\n cols, rows = os.get_terminal_size(fd=fd)\n x = offset % cols\n y = offset // cols\n return read_console_output_character(\n x=x, y=y, fd=fd, buf=buf, bufsize=buffersize, raw=True\n )\n\n\n#\n# The following piece has been forked from colorama.win32\n# Copyright Jonathan Hartley 2013. BSD 3-Clause license, see LICENSE file.\n#\n\n\n@lazyobject\ndef CONSOLE_SCREEN_BUFFER_INFO():\n if platform.has_prompt_toolkit():\n # turns out that PTK has a separate ctype wrapper\n # for this struct and also wraps kernel32.GetConsoleScreenBufferInfo\n # we need to use the same struct to prevent clashes.\n import prompt_toolkit.win32_types\n\n return prompt_toolkit.win32_types.CONSOLE_SCREEN_BUFFER_INFO\n\n # Otherwise we should wrap it ourselves\n COORD() # force COORD to load\n\n class _CONSOLE_SCREEN_BUFFER_INFO(ctypes.Structure):\n \"\"\"Struct from in wincon.h. See Windows API docs\n for more details.\n\n Attributes\n ----------\n dwSize : COORD\n Size of\n dwCursorPosition : COORD\n Current cursor location.\n wAttributes : WORD\n Flags for screen buffer.\n srWindow : SMALL_RECT\n Actual size of screen\n dwMaximumWindowSize : COORD\n Maximum window scrollback size.\n \"\"\"\n\n _fields_ = [\n (\"dwSize\", COORD),\n (\"dwCursorPosition\", COORD),\n (\"wAttributes\", WORD),\n (\"srWindow\", SMALL_RECT),\n (\"dwMaximumWindowSize\", COORD),\n ]\n\n return _CONSOLE_SCREEN_BUFFER_INFO\n\n\n@lazyobject\ndef GetConsoleScreenBufferInfo():\n \"\"\"Returns the windows version of the get screen buffer.\"\"\"\n gcsbi = ctypes.windll.kernel32.GetConsoleScreenBufferInfo\n gcsbi.errcheck = check_zero\n gcsbi.argtypes = (HANDLE, POINTER(CONSOLE_SCREEN_BUFFER_INFO))\n gcsbi.restype = BOOL\n return gcsbi\n\n\ndef get_console_screen_buffer_info(fd=1):\n \"\"\"Returns an screen buffer info object for the relevant stdbuf.\n\n Parameters\n ----------\n fd : int, optional\n Standard buffer file descriptor, 0 for stdin, 1 for stdout (default),\n and 2 for stderr.\n\n Returns\n -------\n csbi : CONSOLE_SCREEN_BUFFER_INFO\n Information about the console screen buffer.\n \"\"\"\n hcon = STDHANDLES[fd]\n csbi = CONSOLE_SCREEN_BUFFER_INFO()\n GetConsoleScreenBufferInfo(hcon, byref(csbi))\n return csbi\n\n\n#\n# end colorama forked section\n#\n\n\ndef get_cursor_position(fd=1):\n \"\"\"Gets the current cursor position as an (x, y) tuple.\"\"\"\n csbi = get_console_screen_buffer_info(fd=fd)\n coord = csbi.dwCursorPosition\n return (coord.X, coord.Y)\n\n\ndef get_cursor_offset(fd=1):\n \"\"\"Gets the current cursor position as a total offset value.\"\"\"\n csbi = get_console_screen_buffer_info(fd=fd)\n pos = csbi.dwCursorPosition\n size = csbi.dwSize\n return (pos.Y * size.X) + pos.X\n\n\ndef get_position_size(fd=1):\n \"\"\"Gets the current cursor position and screen size tuple:\n (x, y, columns, lines).\n \"\"\"\n info = get_console_screen_buffer_info(fd)\n return (\n info.dwCursorPosition.X,\n info.dwCursorPosition.Y,\n info.dwSize.X,\n info.dwSize.Y,\n )\n\n\n@lazyobject\ndef SetConsoleScreenBufferSize():\n \"\"\"Set screen buffer dimensions.\"\"\"\n scsbs = ctypes.windll.kernel32.SetConsoleScreenBufferSize\n scsbs.errcheck = check_zero\n scsbs.argtypes = (HANDLE, COORD) # _In_ HANDLE hConsoleOutput # _In_ COORD dwSize\n scsbs.restype = BOOL\n return scsbs\n\n\ndef set_console_screen_buffer_size(x, y, fd=1):\n \"\"\"Sets the console size for a standard buffer.\n\n Parameters\n ----------\n x : int\n Number of columns.\n y : int\n Number of rows.\n fd : int, optional\n Standard buffer file descriptor, 0 for stdin, 1 for stdout (default),\n and 2 for stderr.\n \"\"\"\n coord = COORD()\n coord.X = x\n coord.Y = y\n hcon = STDHANDLES[fd]\n rtn = SetConsoleScreenBufferSize(hcon, coord)\n return rtn\n\n\n@lazyobject\ndef SetConsoleCursorPosition():\n \"\"\"Set cursor position in console.\"\"\"\n sccp = ctypes.windll.kernel32.SetConsoleCursorPosition\n sccp.errcheck = check_zero\n sccp.argtypes = (\n HANDLE, # _In_ HANDLE hConsoleOutput\n COORD, # _In_ COORD dwCursorPosition\n )\n sccp.restype = BOOL\n return sccp\n\n\ndef set_console_cursor_position(x, y, fd=1):\n \"\"\"Sets the console cursor position for a standard buffer.\n\n Parameters\n ----------\n x : int\n Number of columns.\n y : int\n Number of rows.\n fd : int, optional\n Standard buffer file descriptor, 0 for stdin, 1 for stdout (default),\n and 2 for stderr.\n \"\"\"\n coord = COORD()\n coord.X = x\n coord.Y = y\n hcon = STDHANDLES[fd]\n rtn = SetConsoleCursorPosition(hcon, coord)\n return rtn\n" }, { "path": "xonsh/wizard.py", "content": "\"\"\"Tools for creating command-line and web-based wizards from a tree of nodes.\n\"\"\"\nimport os\nimport re\nimport ast\nimport json\nimport pprint\nimport fnmatch\nimport builtins\nimport textwrap\nimport collections.abc as cabc\n\nfrom xonsh.tools import to_bool, to_bool_or_break, backup_file, print_color\nfrom xonsh.jsonutils import serialize_xonsh_json\n\n\n#\n# Nodes themselves\n#\nclass Node(object):\n \"\"\"Base type of all nodes.\"\"\"\n\n attrs = ()\n\n def __str__(self):\n return PrettyFormatter(self).visit()\n\n def __repr__(self):\n return str(self).replace(\"\\n\", \"\")\n\n\nclass Wizard(Node):\n \"\"\"Top-level node in the tree.\"\"\"\n\n attrs = (\"children\", \"path\")\n\n def __init__(self, children, path=None):\n self.children = children\n self.path = path\n\n\nclass Pass(Node):\n \"\"\"Simple do-nothing node\"\"\"\n\n\nclass Message(Node):\n \"\"\"Contains a simple message to report to the user.\"\"\"\n\n attrs = \"message\"\n\n def __init__(self, message):\n self.message = message\n\n\nclass Question(Node):\n \"\"\"Asks a question and then chooses the next node based on the response.\n \"\"\"\n\n attrs = (\"question\", \"responses\", \"converter\", \"path\")\n\n def __init__(self, question, responses, converter=None, path=None):\n \"\"\"\n Parameters\n ----------\n question : str\n The question itself.\n responses : dict with str keys and Node values\n Mapping from user-input responses to nodes.\n converter : callable, optional\n Converts the string the user typed into another object\n that serves as a key to the responses dict.\n path : str or sequence of str, optional\n A path within the storage object.\n \"\"\"\n self.question = question\n self.responses = responses\n self.converter = converter\n self.path = path\n\n\nclass Input(Node):\n \"\"\"Gets input from the user.\"\"\"\n\n attrs = (\"prompt\", \"converter\", \"show_conversion\", \"confirm\", \"path\")\n\n def __init__(\n self,\n prompt=\">>> \",\n converter=None,\n show_conversion=False,\n confirm=False,\n retry=False,\n path=None,\n ):\n \"\"\"\n Parameters\n ----------\n prompt : str, optional\n Prompt string prior to input\n converter : callable, optional\n Converts the string the user typed into another object\n prior to storage.\n show_conversion : bool, optional\n Flag for whether or not to show the results of the conversion\n function if the conversion function was meaningfully executed.\n Default False.\n confirm : bool, optional\n Whether the input should be confirmed until true or broken,\n default False.\n retry : bool, optional\n In the event that the conversion operation fails, should\n users be re-prompted until they provide valid input. Default False.\n path : str or sequence of str, optional\n A path within the storage object.\n \"\"\"\n self.prompt = prompt\n self.converter = converter\n self.show_conversion = show_conversion\n self.confirm = confirm\n self.retry = retry\n self.path = path\n\n\nclass While(Node):\n \"\"\"Computes a body while a condition function evaluates to true.\n\n The condition function has the form ``cond(visitor=None, node=None)`` and\n must return an object that responds to the Python magic method ``__bool__``.\n The beg attribute specifies the number to start the loop iteration at.\n \"\"\"\n\n attrs = (\"cond\", \"body\", \"idxname\", \"beg\", \"path\")\n\n def __init__(self, cond, body, idxname=\"idx\", beg=0, path=None):\n \"\"\"\n Parameters\n ----------\n cond : callable\n Function that determines if the next loop iteration should\n be executed.\n body : sequence of nodes\n A list of node to execute on each iteration. The condition function\n has the form ``cond(visitor=None, node=None)`` and must return an\n object that responds to the Python magic method ``__bool__``.\n idxname : str, optional\n The variable name for the index.\n beg : int, optional\n The first index value when evaluating path format strings.\n path : str or sequence of str, optional\n A path within the storage object.\n \"\"\"\n self.cond = cond\n self.body = body\n self.idxname = idxname\n self.beg = beg\n self.path = path\n\n\n#\n# Helper nodes\n#\n\n\nclass YesNo(Question):\n \"\"\"Represents a simple yes/no question.\"\"\"\n\n def __init__(self, question, yes, no, path=None):\n \"\"\"\n Parameters\n ----------\n question : str\n The question itself.\n yes : Node\n Node to execute if the response is True.\n no : Node\n Node to execute if the response is False.\n path : str or sequence of str, optional\n A path within the storage object.\n \"\"\"\n responses = {True: yes, False: no}\n super().__init__(question, responses, converter=to_bool, path=path)\n\n\nclass TrueFalse(Input):\n \"\"\"Input node the returns a True or False value.\"\"\"\n\n def __init__(self, prompt=\"yes or no [default: no]? \", path=None):\n super().__init__(\n prompt=prompt,\n converter=to_bool,\n show_conversion=False,\n confirm=False,\n path=path,\n )\n\n\nclass TrueFalseBreak(Input):\n \"\"\"Input node the returns a True, False, or 'break' value.\"\"\"\n\n def __init__(self, prompt=\"yes, no, or break [default: no]? \", path=None):\n super().__init__(\n prompt=prompt,\n converter=to_bool_or_break,\n show_conversion=False,\n confirm=False,\n path=path,\n )\n\n\nclass StoreNonEmpty(Input):\n \"\"\"Stores the user input only if the input was not an empty string.\n This works by wrapping the converter function.\n \"\"\"\n\n def __init__(\n self,\n prompt=\">>> \",\n converter=None,\n show_conversion=False,\n confirm=False,\n retry=False,\n path=None,\n store_raw=False,\n ):\n def nonempty_converter(x):\n \"\"\"Converts non-empty values and converts empty inputs to\n Unstorable.\n \"\"\"\n if len(x) == 0:\n x = Unstorable\n elif converter is None:\n pass\n elif store_raw:\n converter(x) # make sure str is valid, even if storing raw\n else:\n x = converter(x)\n return x\n\n super().__init__(\n prompt=prompt,\n converter=nonempty_converter,\n show_conversion=show_conversion,\n confirm=confirm,\n path=path,\n retry=retry,\n )\n\n\nclass StateFile(Input):\n \"\"\"Node for representing the state as a file under a default or user\n given file name. This node type is likely not useful on its own.\n \"\"\"\n\n attrs = (\"default_file\", \"check\", \"ask_filename\")\n\n def __init__(self, default_file=None, check=True, ask_filename=True):\n \"\"\"\n Parameters\n ----------\n default_file : str, optional\n The default filename to save the file as.\n check : bool, optional\n Whether to print the current state and ask if it should be\n saved/loaded prior to asking for the file name and saving the\n file, default=True.\n ask_filename : bool, optional\n Whether to ask for the filename (if ``False``, always use the\n default filename)\n \"\"\"\n self._df = None\n super().__init__(prompt=\"filename: \", converter=None, confirm=False, path=None)\n self.ask_filename = ask_filename\n self.default_file = default_file\n self.check = check\n\n @property\n def default_file(self):\n return self._df\n\n @default_file.setter\n def default_file(self, val):\n self._df = val\n if val is None:\n self.prompt = \"filename: \"\n else:\n self.prompt = \"filename [default={0!r}]: \".format(val)\n\n\nclass SaveJSON(StateFile):\n \"\"\"Node for saving the state as a JSON file under a default or user\n given file name.\n \"\"\"\n\n\nclass LoadJSON(StateFile):\n \"\"\"Node for loading the state as a JSON file under a default or user\n given file name.\n \"\"\"\n\n\nclass FileInserter(StateFile):\n \"\"\"Node for inserting the state into a file in between a prefix and suffix.\n The state is converted according to some dumper rules.\n \"\"\"\n\n attrs = (\"prefix\", \"suffix\", \"dump_rules\", \"default_file\", \"check\", \"ask_filename\")\n\n def __init__(\n self,\n prefix,\n suffix,\n dump_rules,\n default_file=None,\n check=True,\n ask_filename=True,\n ):\n \"\"\"\n Parameters\n ----------\n prefix : str\n Starting unique string in file to find and begin the insertion at,\n e.g. '# XONSH WIZARD START\\n'\n suffix : str\n Ending unique string to find in the file and end the replacement at,\n e.g. '\\n# XONSH WIZARD END'\n dump_rules : dict of strs to functions\n This is a dictionary that maps the path-like match strings to functions\n that take the flat path and the value as arguments and convert the state\n value at a path to a string. The keys here may use wildcards (as seen in\n the standard library fnmatch module). For example::\n\n dump_rules = {\n '/path/to/exact': lambda path, x: str(x),\n '/otherpath/*': lambda path, x: x,\n '*ending': lambda path x: repr(x),\n '/': None,\n }\n\n If a wildcard is not used in a path, then that rule will be used\n used on an exact match. If wildcards are used, the deepest and longest\n match is used. If None is given instead of a the function, it means to\n skip generating that key.\n default_file : str, optional\n The default filename to save the file as.\n check : bool, optional\n Whether to print the current state and ask if it should be\n saved/loaded prior to asking for the file name and saving the\n file, default=True.\n ask_filename : bool, optional\n Whether to ask for the filename (if ``False``, always use the\n default filename)\n \"\"\"\n self._dr = None\n super().__init__(\n default_file=default_file, check=check, ask_filename=ask_filename\n )\n self.prefix = prefix\n self.suffix = suffix\n self.dump_rules = self.string_rules = dump_rules\n\n @property\n def dump_rules(self):\n return self._dr\n\n @dump_rules.setter\n def dump_rules(self, value):\n dr = {}\n for key, func in value.items():\n key_trans = fnmatch.translate(key)\n r = re.compile(key_trans)\n dr[r] = func\n self._dr = dr\n\n @staticmethod\n def _find_rule_key(x):\n \"\"\"Key function for sorting regular expression rules\"\"\"\n return (x[0], len(x[1].pattern))\n\n def find_rule(self, path):\n \"\"\"For a path, find the key and conversion function that should be used to\n dump a value.\n \"\"\"\n if path in self.string_rules:\n return path, self.string_rules[path]\n len_funcs = []\n for rule, func in self.dump_rules.items():\n m = rule.match(path)\n if m is None:\n continue\n i, j = m.span()\n len_funcs.append((j - i, rule, func))\n if len(len_funcs) == 0:\n # No dump rule function for path\n return path, None\n len_funcs.sort(reverse=True, key=self._find_rule_key)\n _, rule, func = len_funcs[0]\n return rule, func\n\n def dumps(self, flat):\n \"\"\"Dumps a flat mapping of (string path keys, values) pairs and returns\n a formatted string.\n \"\"\"\n lines = [self.prefix]\n for path, value in sorted(flat.items()):\n rule, func = self.find_rule(path)\n if func is None:\n continue\n line = func(path, value)\n lines.append(line)\n lines.append(self.suffix)\n new = \"\\n\".join(lines) + \"\\n\"\n return new\n\n\ndef create_truefalse_cond(prompt=\"yes or no [default: no]? \", path=None):\n \"\"\"This creates a basic condition function for use with nodes like While\n or other conditions. The condition function creates and visits a TrueFalse\n node and returns the result. This TrueFalse node takes the prompt and\n path that is passed in here.\n \"\"\"\n\n def truefalse_cond(visitor, node=None):\n \"\"\"Prompts the user for a true/false condition.\"\"\"\n tf = TrueFalse(prompt=prompt, path=path)\n rtn = visitor.visit(tf)\n return rtn\n\n return truefalse_cond\n\n\n#\n# Tools for trees of nodes.\n#\n\n\ndef _lowername(cls):\n return cls.__name__.lower()\n\n\nclass Visitor(object):\n \"\"\"Super-class for all classes that should walk over a tree of nodes.\n This implements the visit() method.\n \"\"\"\n\n def __init__(self, tree=None):\n self.tree = tree\n\n def visit(self, node=None):\n \"\"\"Walks over a node. If no node is provided, the tree is used.\"\"\"\n if node is None:\n node = self.tree\n if node is None:\n raise RuntimeError(\"no node or tree given!\")\n for clsname in map(_lowername, type.mro(node.__class__)):\n meth = getattr(self, \"visit_\" + clsname, None)\n if callable(meth):\n rtn = meth(node)\n break\n else:\n msg = \"could not find valid visitor method for {0} on {1}\"\n nodename = node.__class__.__name__\n selfname = self.__class__.__name__\n raise AttributeError(msg.format(nodename, selfname))\n return rtn\n\n\nclass PrettyFormatter(Visitor):\n \"\"\"Formats a tree of nodes into a pretty string\"\"\"\n\n def __init__(self, tree=None, indent=\" \"):\n super().__init__(tree=tree)\n self.level = 0\n self.indent = indent\n\n def visit_node(self, node):\n s = node.__class__.__name__ + \"(\"\n if len(node.attrs) == 0:\n return s + \")\"\n s += \"\\n\"\n self.level += 1\n t = []\n for aname in node.attrs:\n a = getattr(node, aname)\n t.append(self.visit(a) if isinstance(a, Node) else pprint.pformat(a))\n t = [\"{0}={1}\".format(n, x) for n, x in zip(node.attrs, t)]\n s += textwrap.indent(\",\\n\".join(t), self.indent)\n self.level -= 1\n s += \"\\n)\"\n return s\n\n def visit_wizard(self, node):\n s = \"Wizard(children=[\"\n if len(node.children) == 0:\n if node.path is None:\n return s + \"])\"\n else:\n return s + \"], path={0!r})\".format(node.path)\n s += \"\\n\"\n self.level += 1\n s += textwrap.indent(\",\\n\".join(map(self.visit, node.children)), self.indent)\n self.level -= 1\n if node.path is None:\n s += \"\\n])\"\n else:\n s += \"{0}],\\n{0}path={1!r}\\n)\".format(self.indent, node.path)\n return s\n\n def visit_message(self, node):\n return \"Message({0!r})\".format(node.message)\n\n def visit_question(self, node):\n s = node.__class__.__name__ + \"(\\n\"\n self.level += 1\n s += self.indent + \"question={0!r},\\n\".format(node.question)\n s += self.indent + \"responses={\"\n if len(node.responses) == 0:\n s += \"}\"\n else:\n s += \"\\n\"\n t = sorted(node.responses.items())\n t = [\"{0!r}: {1}\".format(k, self.visit(v)) for k, v in t]\n s += textwrap.indent(\",\\n\".join(t), 2 * self.indent)\n s += \"\\n\" + self.indent + \"}\"\n if node.converter is not None:\n s += \",\\n\" + self.indent + \"converter={0!r}\".format(node.converter)\n if node.path is not None:\n s += \",\\n\" + self.indent + \"path={0!r}\".format(node.path)\n self.level -= 1\n s += \"\\n)\"\n return s\n\n def visit_input(self, node):\n s = \"{0}(prompt={1!r}\".format(node.__class__.__name__, node.prompt)\n if node.converter is None and node.path is None:\n return s + \"\\n)\"\n if node.converter is not None:\n s += \",\\n\" + self.indent + \"converter={0!r}\".format(node.converter)\n s += \",\\n\" + self.indent + \"show_conversion={0!r}\".format(node.show_conversion)\n s += \",\\n\" + self.indent + \"confirm={0!r}\".format(node.confirm)\n s += \",\\n\" + self.indent + \"retry={0!r}\".format(node.retry)\n if node.path is not None:\n s += \",\\n\" + self.indent + \"path={0!r}\".format(node.path)\n s += \"\\n)\"\n return s\n\n def visit_statefile(self, node):\n s = \"{0}(default_file={1!r}, check={2}, ask_filename={3})\"\n s = s.format(\n node.__class__.__name__, node.default_file, node.check, node.ask_filename\n )\n return s\n\n def visit_while(self, node):\n s = \"{0}(cond={1!r}\".format(node.__class__.__name__, node.cond)\n s += \",\\n\" + self.indent + \"body=[\"\n if len(node.body) > 0:\n s += \"\\n\"\n self.level += 1\n s += textwrap.indent(\",\\n\".join(map(self.visit, node.body)), self.indent)\n self.level -= 1\n s += \"\\n\" + self.indent\n s += \"]\"\n s += \",\\n\" + self.indent + \"idxname={0!r}\".format(node.idxname)\n s += \",\\n\" + self.indent + \"beg={0!r}\".format(node.beg)\n if node.path is not None:\n s += \",\\n\" + self.indent + \"path={0!r}\".format(node.path)\n s += \"\\n)\"\n return s\n\n\ndef ensure_str_or_int(x):\n \"\"\"Creates a string or int.\"\"\"\n if isinstance(x, int):\n return x\n x = x if isinstance(x, str) else str(x)\n try:\n x = ast.literal_eval(x)\n except (ValueError, SyntaxError):\n pass\n if not isinstance(x, (int, str)):\n msg = \"{0!r} could not be converted to int or str\".format(x)\n raise ValueError(msg)\n return x\n\n\ndef canon_path(path, indices=None):\n \"\"\"Returns the canonical form of a path, which is a tuple of str or ints.\n Indices may be optionally passed in.\n \"\"\"\n if not isinstance(path, str):\n return tuple(map(ensure_str_or_int, path))\n if indices is not None:\n path = path.format(**indices)\n path = path[1:] if path.startswith(\"/\") else path\n path = path[:-1] if path.endswith(\"/\") else path\n if len(path) == 0:\n return ()\n return tuple(map(ensure_str_or_int, path.split(\"/\")))\n\n\nclass UnstorableType(object):\n \"\"\"Represents an unstorable return value for when no input was given\n or such input was skipped. Typically represented by the Unstorable\n singleton.\n \"\"\"\n\n _inst = None\n\n def __new__(cls, *args, **kwargs):\n if cls._inst is None:\n cls._inst = super(UnstorableType, cls).__new__(cls, *args, **kwargs)\n return cls._inst\n\n\nUnstorable = UnstorableType()\n\n\nclass StateVisitor(Visitor):\n \"\"\"This class visits the nodes and stores the results in a top-level\n dict of data according to the state path of the node. The the node\n does not have a path or the path does not exist, the storage is skipped.\n This class can be optionally initialized with an existing state.\n \"\"\"\n\n def __init__(self, tree=None, state=None, indices=None):\n super().__init__(tree=tree)\n self.state = {} if state is None else state\n self.indices = {} if indices is None else indices\n\n def visit(self, node=None):\n if node is None:\n node = self.tree\n if node is None:\n raise RuntimeError(\"no node or tree given!\")\n rtn = super().visit(node)\n path = getattr(node, \"path\", None)\n if callable(path):\n path = path(visitor=self, node=node, val=rtn)\n if path is not None and rtn is not Unstorable:\n self.store(path, rtn, indices=self.indices)\n return rtn\n\n def store(self, path, val, indices=None):\n \"\"\"Stores a value at the path location.\"\"\"\n path = canon_path(path, indices=indices)\n loc = self.state\n for p, n in zip(path[:-1], path[1:]):\n if isinstance(p, str) and p not in loc:\n loc[p] = {} if isinstance(n, str) else []\n elif isinstance(p, int) and abs(p) + (p >= 0) > len(loc):\n i = abs(p) + (p >= 0) - len(loc)\n if isinstance(n, str):\n ex = [{} for _ in range(i)]\n else:\n ex = [[] for _ in range(i)]\n loc.extend(ex)\n loc = loc[p]\n p = path[-1]\n if isinstance(p, int) and abs(p) + (p >= 0) > len(loc):\n i = abs(p) + (p >= 0) - len(loc)\n ex = [None] * i\n loc.extend(ex)\n loc[p] = val\n\n def flatten(self, path=\"/\", value=None, flat=None):\n \"\"\"Returns a dict version of the store whose keys are paths.\n Note that list and dict entries will always end in '/', allowing\n disambiquation in dump_rules.\n \"\"\"\n value = self.state if value is None else value\n flat = {} if flat is None else flat\n if isinstance(value, cabc.Mapping):\n path = path if path.endswith(\"/\") else path + \"/\"\n flat[path] = value\n for k, v in value.items():\n p = path + k\n self.flatten(path=p, value=v, flat=flat)\n elif isinstance(value, (str, bytes)):\n flat[path] = value\n elif isinstance(value, cabc.Sequence):\n path = path if path.endswith(\"/\") else path + \"/\"\n flat[path] = value\n for i, v in enumerate(value):\n p = path + str(i)\n self.flatten(path=p, value=v, flat=flat)\n else:\n flat[path] = value\n return flat\n\n\nYN = \"{GREEN}yes{NO_COLOR} or {RED}no{NO_COLOR} [default: no]? \"\nYNB = (\n \"{GREEN}yes{NO_COLOR}, {RED}no{NO_COLOR}, or \"\n \"{YELLOW}break{NO_COLOR} [default: no]? \"\n)\n\n\nclass PromptVisitor(StateVisitor):\n \"\"\"Visits the nodes in the tree via the a command-line prompt.\"\"\"\n\n def __init__(self, tree=None, state=None, **kwargs):\n \"\"\"\n Parameters\n ----------\n tree : Node, optional\n Tree of nodes to start visitor with.\n state : dict, optional\n Initial state to begin with.\n kwargs : optional\n Options that are passed through to the prompt via the shell's\n singleline() method. See BaseShell for mor details.\n \"\"\"\n super().__init__(tree=tree, state=state)\n self.env = builtins.__xonsh__.env\n self.shell = builtins.__xonsh__.shell.shell\n self.shell_kwargs = kwargs\n\n def visit_wizard(self, node):\n for child in node.children:\n self.visit(child)\n\n def visit_pass(self, node):\n pass\n\n def visit_message(self, node):\n print_color(node.message)\n\n def visit_question(self, node):\n self.env[\"PROMPT\"] = node.question\n r = self.shell.singleline(**self.shell_kwargs)\n if callable(node.converter):\n r = node.converter(r)\n self.visit(node.responses[r])\n return r\n\n def visit_input(self, node):\n need_input = True\n while need_input:\n self.env[\"PROMPT\"] = node.prompt\n raw = self.shell.singleline(**self.shell_kwargs)\n if callable(node.converter):\n try:\n x = node.converter(raw)\n except KeyboardInterrupt:\n raise\n except Exception:\n if node.retry:\n msg = (\n \"{{BOLD_RED}}Invalid{{NO_COLOR}} input {0!r}, \"\n \"please retry.\"\n )\n print_color(msg.format(raw))\n continue\n else:\n raise\n if node.show_conversion and x is not Unstorable and str(x) != raw:\n msg = \"{{BOLD_PURPLE}}Converted{{NO_COLOR}} input {0!r} to {1!r}.\"\n print_color(msg.format(raw, x))\n else:\n x = raw\n if node.confirm:\n msg = \"Would you like to keep the input: {0}\"\n print(msg.format(pprint.pformat(x)))\n confirmer = TrueFalseBreak(prompt=YNB)\n status = self.visit(confirmer)\n if isinstance(status, str) and status == \"break\":\n x = Unstorable\n break\n else:\n need_input = not status\n else:\n need_input = False\n return x\n\n def visit_while(self, node):\n rtns = []\n origidx = self.indices.get(node.idxname, None)\n self.indices[node.idxname] = idx = node.beg\n while node.cond(visitor=self, node=node):\n rtn = list(map(self.visit, node.body))\n rtns.append(rtn)\n idx += 1\n self.indices[node.idxname] = idx\n if origidx is None:\n del self.indices[node.idxname]\n else:\n self.indices[node.idxname] = origidx\n return rtns\n\n def visit_savejson(self, node):\n jstate = json.dumps(\n self.state, indent=1, sort_keys=True, default=serialize_xonsh_json\n )\n if node.check:\n msg = \"The current state is:\\n\\n{0}\\n\"\n print(msg.format(textwrap.indent(jstate, \" \")))\n ap = \"Would you like to save this state, \" + YN\n asker = TrueFalse(prompt=ap)\n do_save = self.visit(asker)\n if not do_save:\n return Unstorable\n fname = None\n if node.ask_filename:\n fname = self.visit_input(node)\n if fname is None or len(fname) == 0:\n fname = node.default_file\n if os.path.isfile(fname):\n backup_file(fname)\n else:\n os.makedirs(os.path.dirname(fname), exist_ok=True)\n with open(fname, \"w\") as f:\n f.write(jstate)\n return fname\n\n def visit_loadjson(self, node):\n if node.check:\n ap = \"Would you like to load an existing file, \" + YN\n asker = TrueFalse(prompt=ap)\n do_load = self.visit(asker)\n if not do_load:\n return Unstorable\n fname = self.visit_input(node)\n if fname is None or len(fname) == 0:\n fname = node.default_file\n if os.path.isfile(fname):\n with open(fname, \"r\") as f:\n self.state = json.load(f)\n print_color(\"{{GREEN}}{0!r} loaded.{{NO_COLOR}}\".format(fname))\n else:\n print_color(\n (\"{{RED}}{0!r} could not be found, \" \"continuing.{{NO_COLOR}}\").format(\n fname\n )\n )\n return fname\n\n def visit_fileinserter(self, node):\n # perform the dumping operation.\n new = node.dumps(self.flatten())\n # check if we should write this out\n if node.check:\n msg = \"The current state to insert is:\\n\\n{0}\\n\"\n print(msg.format(textwrap.indent(new, \" \")))\n ap = \"Would you like to write out the current state, \" + YN\n asker = TrueFalse(prompt=ap)\n do_save = self.visit(asker)\n if not do_save:\n return Unstorable\n # get and backup the file.\n fname = None\n if node.ask_filename:\n fname = self.visit_input(node)\n if fname is None or len(fname) == 0:\n fname = node.default_file\n if os.path.isfile(fname):\n with open(fname, \"r\") as f:\n s = f.read()\n before, _, s = s.partition(node.prefix)\n _, _, after = s.partition(node.suffix)\n backup_file(fname)\n else:\n before = after = \"\"\n dname = os.path.dirname(fname)\n if dname:\n os.makedirs(dname, exist_ok=True)\n # write out the file\n with open(fname, \"w\") as f:\n f.write(before + new + after)\n return fname\n" }, { "path": "xonsh/xonfig.py", "content": "\"\"\"The xonsh configuration (xonfig) utility.\"\"\"\nimport os\nimport re\nimport ast\nimport json\nimport shutil\nimport random\nimport pprint\nimport textwrap\nimport builtins\nimport argparse\nimport functools\nimport itertools\nimport contextlib\nimport collections\n\nfrom xonsh.ply import ply\n\nimport xonsh.wizard as wiz\nfrom xonsh import __version__ as XONSH_VERSION\nfrom xonsh.prompt.base import is_template_string\nfrom xonsh.platform import (\n is_readline_available,\n ptk_version,\n PYTHON_VERSION_INFO,\n pygments_version,\n ON_POSIX,\n ON_LINUX,\n linux_distro,\n ON_DARWIN,\n ON_WINDOWS,\n ON_CYGWIN,\n DEFAULT_ENCODING,\n ON_MSYS,\n githash,\n)\nfrom xonsh.tools import (\n to_bool,\n is_string,\n print_exception,\n is_superuser,\n color_style_names,\n print_color,\n color_style,\n)\nfrom xonsh.foreign_shells import CANON_SHELL_NAMES\nfrom xonsh.xontribs import xontrib_metadata, find_xontrib\nfrom xonsh.lazyasd import lazyobject\n\nHR = \"'`-.,_,.-*'`-.,_,.-*'`-.,_,.-*'`-.,_,.-*'`-.,_,.-*'`-.,_,.-*'`-.,_,.-*'\"\nWIZARD_HEAD = \"\"\"\n {{BOLD_WHITE}}Welcome to the xonsh configuration wizard!{{NO_COLOR}}\n {{YELLOW}}------------------------------------------{{NO_COLOR}}\nThis will present a guided tour through setting up the xonsh static\nconfig file. Xonsh will automatically ask you if you want to run this\nwizard if the configuration file does not exist. However, you can\nalways rerun this wizard with the xonfig command:\n\n $ xonfig wizard\n\nThis wizard will load an existing configuration, if it is available.\nAlso never fear when this wizard saves its results! It will create\na backup of any existing configuration automatically.\n\nThis wizard has two main phases: foreign shell setup and environment\nvariable setup. Each phase may be skipped in its entirety.\n\nFor the configuration to take effect, you will need to restart xonsh.\n\n{hr}\n\"\"\".format(\n hr=HR\n)\n\nWIZARD_FS = \"\"\"\n{hr}\n\n {{BOLD_WHITE}}Foreign Shell Setup{{NO_COLOR}}\n {{YELLOW}}-------------------{{NO_COLOR}}\nThe xonsh shell has the ability to interface with foreign shells such\nas Bash, or zsh (fish not yet implemented).\n\nFor configuration, this means that xonsh can load the environment,\naliases, and functions specified in the config files of these shells.\nNaturally, these shells must be available on the system to work.\nBeing able to share configuration (and source) from foreign shells\nmakes it easier to transition to and from xonsh.\n\"\"\".format(\n hr=HR\n)\n\nWIZARD_ENV = \"\"\"\n{hr}\n\n {{BOLD_WHITE}}Environment Variable Setup{{NO_COLOR}}\n {{YELLOW}}--------------------------{{NO_COLOR}}\nThe xonsh shell also allows you to setup environment variables from\nthe static configuration file. Any variables set in this way are\nsuperseded by the definitions in the xonshrc or on the command line.\nStill, setting environment variables in this way can help define\noptions that are global to the system or user.\n\nThe following lists the environment variable name, its documentation,\nthe default value, and the current value. The default and current\nvalues are presented as pretty repr strings of their Python types.\n\n{{BOLD_GREEN}}Note:{{NO_COLOR}} Simply hitting enter for any environment variable\nwill accept the default value for that entry.\n\"\"\".format(\n hr=HR\n)\n\nWIZARD_ENV_QUESTION = \"Would you like to set env vars now, \" + wiz.YN\n\nWIZARD_XONTRIB = \"\"\"\n{hr}\n\n {{BOLD_WHITE}}Xontribs{{NO_COLOR}}\n {{YELLOW}}--------{{NO_COLOR}}\nNo shell is complete without extensions, and xonsh is no exception. Xonsh\nextensions are called {{BOLD_GREEN}}xontribs{{NO_COLOR}}, or xonsh contributions.\nXontribs are dynamically loadable, either by importing them directly or by\nusing the 'xontrib' command. However, you can also configure xonsh to load\nxontribs automatically on startup prior to loading the run control files.\nThis allows the xontrib to be used immediately in your xonshrc files.\n\nThe following describes all xontribs that have been registered with xonsh.\nThese come from users, 3rd party developers, or xonsh itself!\n\"\"\".format(\n hr=HR\n)\n\nWIZARD_XONTRIB_QUESTION = \"Would you like to enable xontribs now, \" + wiz.YN\n\nWIZARD_TAIL = \"\"\"\nThanks for using the xonsh configuration wizard!\"\"\"\n\n\n_XONFIG_SOURCE_FOREIGN_SHELL_COMMAND = collections.defaultdict(\n lambda: \"source-foreign\", bash=\"source-bash\", cmd=\"source-cmd\", zsh=\"source-zsh\"\n)\n\n\ndef _dump_xonfig_foreign_shell(path, value):\n shell = value[\"shell\"]\n shell = CANON_SHELL_NAMES.get(shell, shell)\n cmd = [_XONFIG_SOURCE_FOREIGN_SHELL_COMMAND[shell]]\n interactive = value.get(\"interactive\", None)\n if interactive is not None:\n cmd.extend([\"--interactive\", str(interactive)])\n login = value.get(\"login\", None)\n if login is not None:\n cmd.extend([\"--login\", str(login)])\n envcmd = value.get(\"envcmd\", None)\n if envcmd is not None:\n cmd.extend([\"--envcmd\", envcmd])\n aliascmd = value.get(\"aliasmd\", None)\n if aliascmd is not None:\n cmd.extend([\"--aliascmd\", aliascmd])\n extra_args = value.get(\"extra_args\", None)\n if extra_args:\n cmd.extend([\"--extra-args\", repr(\" \".join(extra_args))])\n safe = value.get(\"safe\", None)\n if safe is not None:\n cmd.extend([\"--safe\", str(safe)])\n prevcmd = value.get(\"prevcmd\", \"\")\n if prevcmd:\n cmd.extend([\"--prevcmd\", repr(prevcmd)])\n postcmd = value.get(\"postcmd\", \"\")\n if postcmd:\n cmd.extend([\"--postcmd\", repr(postcmd)])\n funcscmd = value.get(\"funcscmd\", None)\n if funcscmd:\n cmd.extend([\"--funcscmd\", repr(funcscmd)])\n sourcer = value.get(\"sourcer\", None)\n if sourcer:\n cmd.extend([\"--sourcer\", sourcer])\n if cmd[0] == \"source-foreign\":\n cmd.append(shell)\n cmd.append('\"echo loading xonsh foreign shell\"')\n return \" \".join(cmd)\n\n\ndef _dump_xonfig_env(path, value):\n name = os.path.basename(path.rstrip(\"/\"))\n ensurer = builtins.__xonsh__.env.get_ensurer(name)\n dval = str(value) if ensurer.detype is None else ensurer.detype(value)\n dval = str(value) if dval is None else dval\n return \"${name} = {val!r}\".format(name=name, val=dval)\n\n\ndef _dump_xonfig_xontribs(path, value):\n return \"xontrib load {0}\".format(\" \".join(value))\n\n\n@lazyobject\ndef XONFIG_DUMP_RULES():\n return {\n \"/\": None,\n \"/env/\": None,\n \"/foreign_shells/*/\": _dump_xonfig_foreign_shell,\n \"/env/*\": _dump_xonfig_env,\n \"/env/*/[0-9]*\": None,\n \"/xontribs/\": _dump_xonfig_xontribs,\n }\n\n\ndef make_fs_wiz():\n \"\"\"Makes the foreign shell part of the wizard.\"\"\"\n cond = wiz.create_truefalse_cond(prompt=\"Add a new foreign shell, \" + wiz.YN)\n fs = wiz.While(\n cond=cond,\n body=[\n wiz.Input(\"shell name (e.g. bash): \", path=\"/foreign_shells/{idx}/shell\"),\n wiz.StoreNonEmpty(\n \"interactive shell [bool, default=True]: \",\n converter=to_bool,\n show_conversion=True,\n path=\"/foreign_shells/{idx}/interactive\",\n ),\n wiz.StoreNonEmpty(\n \"login shell [bool, default=False]: \",\n converter=to_bool,\n show_conversion=True,\n path=\"/foreign_shells/{idx}/login\",\n ),\n wiz.StoreNonEmpty(\n \"env command [str, default='env']: \",\n path=\"/foreign_shells/{idx}/envcmd\",\n ),\n wiz.StoreNonEmpty(\n \"alias command [str, default='alias']: \",\n path=\"/foreign_shells/{idx}/aliascmd\",\n ),\n wiz.StoreNonEmpty(\n (\"extra command line arguments [list of str, \" \"default=[]]: \"),\n converter=ast.literal_eval,\n show_conversion=True,\n path=\"/foreign_shells/{idx}/extra_args\",\n ),\n wiz.StoreNonEmpty(\n \"safely handle exceptions [bool, default=True]: \",\n converter=to_bool,\n show_conversion=True,\n path=\"/foreign_shells/{idx}/safe\",\n ),\n wiz.StoreNonEmpty(\n \"pre-command [str, default='']: \", path=\"/foreign_shells/{idx}/prevcmd\"\n ),\n wiz.StoreNonEmpty(\n \"post-command [str, default='']: \", path=\"/foreign_shells/{idx}/postcmd\"\n ),\n wiz.StoreNonEmpty(\n \"foreign function command [str, default=None]: \",\n path=\"/foreign_shells/{idx}/funcscmd\",\n ),\n wiz.StoreNonEmpty(\n \"source command [str, default=None]: \",\n path=\"/foreign_shells/{idx}/sourcer\",\n ),\n wiz.Message(message=\"Foreign shell added.\\n\"),\n ],\n )\n return fs\n\n\ndef _wrap_paragraphs(text, width=70, **kwargs):\n \"\"\"Wraps paragraphs instead.\"\"\"\n pars = text.split(\"\\n\")\n pars = [\"\\n\".join(textwrap.wrap(p, width=width, **kwargs)) for p in pars]\n s = \"\\n\".join(pars)\n return s\n\n\nENVVAR_MESSAGE = \"\"\"\n{{BOLD_CYAN}}${name}{{NO_COLOR}}\n{docstr}\n{{RED}}default value:{{NO_COLOR}} {default}\n{{RED}}current value:{{NO_COLOR}} {current}\"\"\"\n\nENVVAR_PROMPT = \"{BOLD_GREEN}>>>{NO_COLOR} \"\n\n\ndef make_exit_message():\n \"\"\"Creates a message for how to exit the wizard.\"\"\"\n shell_type = builtins.__xonsh__.shell.shell_type\n keyseq = \"Ctrl-D\" if shell_type == \"readline\" else \"Ctrl-C\"\n msg = \"To exit the wizard at any time, press {BOLD_UNDERLINE_CYAN}\"\n msg += keyseq + \"{NO_COLOR}.\\n\"\n m = wiz.Message(message=msg)\n return m\n\n\ndef make_envvar(name):\n \"\"\"Makes a StoreNonEmpty node for an environment variable.\"\"\"\n env = builtins.__xonsh__.env\n vd = env.get_docs(name)\n if not vd.configurable:\n return\n default = vd.default\n if \"\\n\" in default:\n default = \"\\n\" + _wrap_paragraphs(default, width=69)\n curr = env.get(name)\n if is_string(curr) and is_template_string(curr):\n curr = curr.replace(\"{\", \"{{\").replace(\"}\", \"}}\")\n curr = pprint.pformat(curr, width=69)\n if \"\\n\" in curr:\n curr = \"\\n\" + curr\n msg = ENVVAR_MESSAGE.format(\n name=name,\n default=default,\n current=curr,\n docstr=_wrap_paragraphs(vd.docstr, width=69),\n )\n mnode = wiz.Message(message=msg)\n ens = env.get_ensurer(name)\n path = \"/env/\" + name\n pnode = wiz.StoreNonEmpty(\n ENVVAR_PROMPT,\n converter=ens.convert,\n show_conversion=True,\n path=path,\n retry=True,\n store_raw=vd.store_as_str,\n )\n return mnode, pnode\n\n\ndef _make_flat_wiz(kidfunc, *args):\n kids = map(kidfunc, *args)\n flatkids = []\n for k in kids:\n if k is None:\n continue\n flatkids.extend(k)\n wizard = wiz.Wizard(children=flatkids)\n return wizard\n\n\ndef make_env_wiz():\n \"\"\"Makes an environment variable wizard.\"\"\"\n w = _make_flat_wiz(make_envvar, sorted(builtins.__xonsh__.env._docs.keys()))\n return w\n\n\nXONTRIB_PROMPT = \"{BOLD_GREEN}Add this xontrib{NO_COLOR}, \" + wiz.YN\n\n\ndef _xontrib_path(visitor=None, node=None, val=None):\n # need this to append only based on user-selected size\n return (\"xontribs\", len(visitor.state.get(\"xontribs\", ())))\n\n\ndef make_xontrib(xontrib, package):\n \"\"\"Makes a message and StoreNonEmpty node for a xontrib.\"\"\"\n name = xontrib.get(\"name\", \"\")\n msg = \"\\n{BOLD_CYAN}\" + name + \"{NO_COLOR}\\n\"\n if \"url\" in xontrib:\n msg += \"{RED}url:{NO_COLOR} \" + xontrib[\"url\"] + \"\\n\"\n if \"package\" in xontrib:\n msg += \"{RED}package:{NO_COLOR} \" + xontrib[\"package\"] + \"\\n\"\n if \"url\" in package:\n if \"url\" in xontrib and package[\"url\"] != xontrib[\"url\"]:\n msg += \"{RED}package-url:{NO_COLOR} \" + package[\"url\"] + \"\\n\"\n if \"license\" in package:\n msg += \"{RED}license:{NO_COLOR} \" + package[\"license\"] + \"\\n\"\n msg += \"{PURPLE}installed?{NO_COLOR} \"\n msg += (\"no\" if find_xontrib(name) is None else \"yes\") + \"\\n\"\n desc = xontrib.get(\"description\", \"\")\n if not isinstance(desc, str):\n desc = \"\".join(desc)\n msg += _wrap_paragraphs(desc, width=69)\n if msg.endswith(\"\\n\"):\n msg = msg[:-1]\n mnode = wiz.Message(message=msg)\n convert = lambda x: name if to_bool(x) else wiz.Unstorable\n pnode = wiz.StoreNonEmpty(XONTRIB_PROMPT, converter=convert, path=_xontrib_path)\n return mnode, pnode\n\n\ndef make_xontribs_wiz():\n \"\"\"Makes a xontrib wizard.\"\"\"\n md = xontrib_metadata()\n pkgs = [md[\"packages\"].get(d.get(\"package\", None), {}) for d in md[\"xontribs\"]]\n w = _make_flat_wiz(make_xontrib, md[\"xontribs\"], pkgs)\n return w\n\n\ndef make_xonfig_wizard(default_file=None, confirm=False, no_wizard_file=None):\n \"\"\"Makes a configuration wizard for xonsh config file.\n\n Parameters\n ----------\n default_file : str, optional\n Default filename to save and load to. User will still be prompted.\n confirm : bool, optional\n Confirm that the main part of the wizard should be run.\n no_wizard_file : str, optional\n Filename for that will flag to future runs that the wizard should not be\n run again. If None (default), this defaults to default_file.\n \"\"\"\n w = wiz.Wizard(\n children=[\n wiz.Message(message=WIZARD_HEAD),\n make_exit_message(),\n wiz.Message(message=WIZARD_FS),\n make_fs_wiz(),\n wiz.Message(message=WIZARD_ENV),\n wiz.YesNo(question=WIZARD_ENV_QUESTION, yes=make_env_wiz(), no=wiz.Pass()),\n wiz.Message(message=WIZARD_XONTRIB),\n wiz.YesNo(\n question=WIZARD_XONTRIB_QUESTION, yes=make_xontribs_wiz(), no=wiz.Pass()\n ),\n wiz.Message(message=\"\\n\" + HR + \"\\n\"),\n wiz.FileInserter(\n prefix=\"# XONSH WIZARD START\",\n suffix=\"# XONSH WIZARD END\",\n dump_rules=XONFIG_DUMP_RULES,\n default_file=default_file,\n check=True,\n ),\n wiz.Message(message=WIZARD_TAIL),\n ]\n )\n if confirm:\n q = (\n \"Would you like to run the xonsh configuration wizard now?\\n\\n\"\n \"1. Yes (You can abort at any time)\\n\"\n \"2. No, but ask me next time.\\n\"\n \"3. No, and don't ask me again.\\n\\n\"\n \"1, 2, or 3 [default: 2]? \"\n )\n no_wizard_file = default_file if no_wizard_file is None else no_wizard_file\n passer = wiz.Pass()\n saver = wiz.SaveJSON(\n check=False, ask_filename=False, default_file=no_wizard_file\n )\n w = wiz.Question(\n q, {1: w, 2: passer, 3: saver}, converter=lambda x: int(x) if x != \"\" else 2\n )\n return w\n\n\ndef _wizard(ns):\n env = builtins.__xonsh__.env\n shell = builtins.__xonsh__.shell.shell\n fname = env.get(\"XONSHRC\")[-1] if ns.file is None else ns.file\n no_wiz = os.path.join(env.get(\"XONSH_CONFIG_DIR\"), \"no-wizard\")\n w = make_xonfig_wizard(\n default_file=fname, confirm=ns.confirm, no_wizard_file=no_wiz\n )\n tempenv = {\"PROMPT\": \"\", \"XONSH_STORE_STDOUT\": False}\n pv = wiz.PromptVisitor(w, store_in_history=False, multiline=False)\n\n @contextlib.contextmanager\n def force_hide():\n if env.get(\"XONSH_STORE_STDOUT\") and hasattr(shell, \"_force_hide\"):\n orig, shell._force_hide = shell._force_hide, False\n yield\n shell._force_hide = orig\n else:\n yield\n\n with force_hide(), env.swap(tempenv):\n try:\n pv.visit()\n except (KeyboardInterrupt, Exception):\n print()\n print_exception()\n\n\ndef _xonfig_format_human(data):\n wcol1 = wcol2 = 0\n for key, val in data:\n wcol1 = max(wcol1, len(key))\n wcol2 = max(wcol2, len(str(val)))\n hr = \"+\" + (\"-\" * (wcol1 + 2)) + \"+\" + (\"-\" * (wcol2 + 2)) + \"+\\n\"\n row = \"| {key!s:<{wcol1}} | {val!s:<{wcol2}} |\\n\"\n s = hr\n for key, val in data:\n s += row.format(key=key, wcol1=wcol1, val=val, wcol2=wcol2)\n s += hr\n return s\n\n\ndef _xonfig_format_json(data):\n data = {k.replace(\" \", \"_\"): v for k, v in data}\n s = json.dumps(data, sort_keys=True, indent=1) + \"\\n\"\n return s\n\n\ndef _info(ns):\n env = builtins.__xonsh__.env\n data = [(\"xonsh\", XONSH_VERSION)]\n hash_, date_ = githash()\n if hash_:\n data.append((\"Git SHA\", hash_))\n data.append((\"Commit Date\", date_))\n data.extend(\n [\n (\"Python\", \"{}.{}.{}\".format(*PYTHON_VERSION_INFO)),\n (\"PLY\", ply.__version__),\n (\"have readline\", is_readline_available()),\n (\"prompt toolkit\", ptk_version() or None),\n (\"shell type\", env.get(\"SHELL_TYPE\")),\n (\"pygments\", pygments_version()),\n (\"on posix\", bool(ON_POSIX)),\n (\"on linux\", bool(ON_LINUX)),\n ]\n )\n if ON_LINUX:\n data.append((\"distro\", linux_distro()))\n data.extend(\n [\n (\"on darwin\", ON_DARWIN),\n (\"on windows\", ON_WINDOWS),\n (\"on cygwin\", ON_CYGWIN),\n (\"on msys2\", ON_MSYS),\n (\"is superuser\", is_superuser()),\n (\"default encoding\", DEFAULT_ENCODING),\n (\"xonsh encoding\", env.get(\"XONSH_ENCODING\")),\n (\"encoding errors\", env.get(\"XONSH_ENCODING_ERRORS\")),\n ]\n )\n formatter = _xonfig_format_json if ns.json else _xonfig_format_human\n s = formatter(data)\n return s\n\n\ndef _styles(ns):\n env = builtins.__xonsh__.env\n curr = env.get(\"XONSH_COLOR_STYLE\")\n styles = sorted(color_style_names())\n if ns.json:\n s = json.dumps(styles, sort_keys=True, indent=1)\n print(s)\n return\n lines = []\n for style in styles:\n if style == curr:\n lines.append(\"* {GREEN}\" + style + \"{NO_COLOR}\")\n else:\n lines.append(\" \" + style)\n s = \"\\n\".join(lines)\n print_color(s)\n\n\ndef _str_colors(cmap, cols):\n color_names = sorted(cmap.keys(), key=(lambda s: (len(s), s)))\n grper = lambda s: min(cols // (len(s) + 1), 8)\n lines = []\n for n, group in itertools.groupby(color_names, key=grper):\n width = cols // n\n line = \"\"\n for i, name in enumerate(group):\n buf = \" \" * (width - len(name))\n line += \"{\" + name + \"}\" + name + \"{NO_COLOR}\" + buf\n if (i + 1) % n == 0:\n lines.append(line)\n line = \"\"\n if len(line) != 0:\n lines.append(line)\n return \"\\n\".join(lines)\n\n\ndef _tok_colors(cmap, cols):\n from xonsh.style_tools import Color\n\n nc = Color.NO_COLOR\n names_toks = {}\n for t in cmap.keys():\n name = str(t)\n if name.startswith(\"Token.Color.\"):\n _, _, name = name.rpartition(\".\")\n names_toks[name] = t\n color_names = sorted(names_toks.keys(), key=(lambda s: (len(s), s)))\n grper = lambda s: min(cols // (len(s) + 1), 8)\n toks = []\n for n, group in itertools.groupby(color_names, key=grper):\n width = cols // n\n for i, name in enumerate(group):\n toks.append((names_toks[name], name))\n buf = \" \" * (width - len(name))\n if (i + 1) % n == 0:\n buf += \"\\n\"\n toks.append((nc, buf))\n if not toks[-1][1].endswith(\"\\n\"):\n toks[-1] = (nc, toks[-1][1] + \"\\n\")\n return toks\n\n\ndef _colors(args):\n columns, _ = shutil.get_terminal_size()\n columns -= int(ON_WINDOWS)\n style_stash = builtins.__xonsh__.env[\"XONSH_COLOR_STYLE\"]\n\n if args.style is not None:\n if args.style not in color_style_names():\n print(\"Invalid style: {}\".format(args.style))\n return\n builtins.__xonsh__.env[\"XONSH_COLOR_STYLE\"] = args.style\n\n color_map = color_style()\n akey = next(iter(color_map))\n if isinstance(akey, str):\n s = _str_colors(color_map, columns)\n else:\n s = _tok_colors(color_map, columns)\n print_color(s)\n builtins.__xonsh__.env[\"XONSH_COLOR_STYLE\"] = style_stash\n\n\ndef _tutorial(args):\n import webbrowser\n\n webbrowser.open(\"http://xon.sh/tutorial.html\")\n\n\n@functools.lru_cache(1)\ndef _xonfig_create_parser():\n p = argparse.ArgumentParser(\n prog=\"xonfig\", description=\"Manages xonsh configuration.\"\n )\n subp = p.add_subparsers(title=\"action\", dest=\"action\")\n info = subp.add_parser(\n \"info\", help=(\"displays configuration information, \" \"default action\")\n )\n info.add_argument(\n \"--json\", action=\"store_true\", default=False, help=\"reports results as json\"\n )\n wiz = subp.add_parser(\"wizard\", help=\"displays configuration information\")\n wiz.add_argument(\n \"--file\", default=None, help=\"config file location, default=$XONSHRC\"\n )\n wiz.add_argument(\n \"--confirm\",\n action=\"store_true\",\n default=False,\n help=\"confirm that the wizard should be run.\",\n )\n sty = subp.add_parser(\"styles\", help=\"prints available xonsh color styles\")\n sty.add_argument(\n \"--json\", action=\"store_true\", default=False, help=\"reports results as json\"\n )\n colors = subp.add_parser(\"colors\", help=\"preview color style\")\n colors.add_argument(\n \"style\", nargs=\"?\", default=None, help=\"style to preview, default: \"\n )\n subp.add_parser(\"tutorial\", help=\"Launch tutorial in browser.\")\n return p\n\n\n_XONFIG_MAIN_ACTIONS = {\n \"info\": _info,\n \"wizard\": _wizard,\n \"styles\": _styles,\n \"colors\": _colors,\n \"tutorial\": _tutorial,\n}\n\n\ndef xonfig_main(args=None):\n \"\"\"Main xonfig entry point.\"\"\"\n if not args or (\n args[0] not in _XONFIG_MAIN_ACTIONS and args[0] not in {\"-h\", \"--help\"}\n ):\n args.insert(0, \"info\")\n parser = _xonfig_create_parser()\n ns = parser.parse_args(args)\n if ns.action is None: # apply default action\n ns = parser.parse_args([\"info\"] + args)\n return _XONFIG_MAIN_ACTIONS[ns.action](ns)\n\n\n@lazyobject\ndef STRIP_COLOR_RE():\n return re.compile(\"{.*?}\")\n\n\ndef _align_string(string, align=\"<\", fill=\" \", width=80):\n \"\"\" Align and pad a color formatted string \"\"\"\n linelen = len(STRIP_COLOR_RE.sub(\"\", string))\n padlen = max(width - linelen, 0)\n if align == \"^\":\n return fill * (padlen // 2) + string + fill * (padlen // 2 + padlen % 2)\n elif align == \">\":\n return fill * padlen + string\n elif align == \"<\":\n return string + fill * padlen\n else:\n return string\n\n\n@lazyobject\ndef TAGLINES():\n return [\n \"Exofrills in the shell\",\n \"No frills in the shell\",\n \"Become the Lord of the Files\",\n \"Break out of your shell\",\n \"The only shell that is also a shell\",\n \"All that is and all that shell be\",\n \"It cannot be that hard\",\n \"Pass the xonsh, Piggy\",\n \"Piggy glanced nervously into hell and cradled the xonsh\",\n \"The xonsh is a symbol\",\n \"It is pronounced conch\",\n \"The shell, bourne again\",\n \"Snailed it\",\n \"Starfish loves you\",\n \"Come snail away\",\n \"This is Major Tom to Ground Xonshtrol\",\n \"Sally sells csh and keeps xonsh to herself\",\n \"Nice indeed. Everything's accounted for, except your old shell.\",\n \"I wanna thank you for putting me back in my snail shell\",\n \"Crustaceanly Yours\",\n \"With great shell comes great reproducibility\",\n \"None shell pass\",\n \"You shell not pass!\",\n \"The x-on shell\",\n \"Ever wonder why there isn't a Taco Shell? Because it is a corny idea.\",\n \"The carcolh will catch you!\",\n \"People xonshtantly mispronounce these things\",\n \"WHAT...is your favorite shell?\",\n \"Conches for the xonsh god!\",\n \"Python-powered, cross-platform, Unix-gazing shell\",\n \"Tab completion in Alderaan places\",\n \"This fix was trickier than expected\",\n \"The unholy cross of Bash/Python\",\n ]\n\n\n# list of strings or tuples (string, align, fill)\nWELCOME_MSG = [\n \"\",\n (\"{{INTENSE_WHITE}}Welcome to the xonsh shell ({version}){{NO_COLOR}}\", \"^\", \" \"),\n \"\",\n (\"{{INTENSE_RED}}~{{NO_COLOR}} {tagline} {{INTENSE_RED}}~{{NO_COLOR}}\", \"^\", \" \"),\n \"\",\n (\"{{INTENSE_BLACK}}\", \"<\", \"-\"),\n \"{{GREEN}}xonfig{{NO_COLOR}} tutorial {{INTENSE_WHITE}}-> Launch the tutorial in \"\n \"the browser{{NO_COLOR}}\",\n \"{{GREEN}}xonfig{{NO_COLOR}} wizard {{INTENSE_WHITE}}-> Run the configuration \"\n \"wizard and claim your shell {{NO_COLOR}}\",\n \"{{INTENSE_BLACK}}(Note: Run the Wizard or create a {{RED}}~/.xonshrc{{INTENSE_BLACK}} file \"\n \"to suppress the welcome screen)\",\n \"\",\n]\n\n\ndef print_welcome_screen():\n subst = dict(tagline=random.choice(list(TAGLINES)), version=XONSH_VERSION)\n for elem in WELCOME_MSG:\n if isinstance(elem, str):\n elem = (elem, \"\", \"\")\n line = elem[0].format(**subst)\n termwidth = os.get_terminal_size().columns\n line = _align_string(line, elem[1], elem[2], width=termwidth)\n print_color(line)\n" }, { "path": "xonsh/xonshrc", "content": "# adjust some paths\n#$BASH_COMPLETIONS.append('/usr/local/etc/bash_completion.d/git-completion.bash')\n#$PATH.append('/some/path/bin')\n#$LD_LIBRARY_PATH = ['/some/path1/lib', '/some/path2/lib', '']\n\naliases['hi'] = echo hello world\n\n# some customization options\n#$MULTILINE_PROMPT = '`·.,¸,.·*¯`·.,¸,.·*¯'" }, { "path": "xonsh/xontribs.json", "content": "{\"xontribs\": [\n {\"name\": \"apt_tabcomplete\",\n \"package\": \"xonsh-apt-tabcomplete\",\n \"url\": \"https://github.com/DangerOnTheRanger/xonsh-apt-tabcomplete\",\n \"description\": [\"Adds tabcomplete functionality to apt-get/apt-cache inside of xonsh.\"]\n },\n {\"name\": \"autojump\",\n \"package\": \"xontrib-autojump\",\n \"url\": \"https://github.com/gsaga/autojump-xonsh\",\n \"description\": [\"autojump support for xonsh\"]\n },\n {\"name\": \"autoxsh\",\n \"package\": \"xonsh-autoxsh\",\n \"url\": \"https://github.com/Granitas/xonsh-autoxsh\",\n \"description\": [\"Adds automatic execution of xonsh script files called\",\n \"``.autoxsh`` when enterting a directory with ``cd`` function\"]\n },\n {\"name\": \"bashisms\",\n \"package\": \"xonsh\",\n \"url\": \"http://xon.sh\",\n \"description\": [\n \"Enables additional Bash-like syntax while at the command prompt. For \",\n \"example, the ``!!`` syntax for running the previous command is now usable.\",\n \"Note that these features are implemented as precommand events and these \",\n \"additions do not affect the xonsh language when run as script. That said, \",\n \"you might find them useful if you have strong muscle memory.\\n\\n\",\n \"**Warning:** This xontrib may modify user command line input to implement \",\n \"its behavior. To see the modifications as they are applied (in unified diff\",\n \"format), please set ``$XONSH_DEBUG`` to ``2`` or higher.\"]\n },\n {\"name\": \"base16_shell\",\n \"package\": \"xontrib-base16-shell\",\n \"url\": \"https://github.com/ErickTucto/xontrib-base16-shell\",\n \"description\": [\"Change base16 shell themes\"]\n },\n {\"name\": \"coreutils\",\n \"package\": \"xonsh\",\n \"url\": \"http://xon.sh\",\n \"description\": [\n \"Additional core utilities that are implemented in xonsh. The current list \",\n \"includes:\\n\",\n \"\\n\",\n \"* cat\\n\",\n \"* echo\\n\",\n \"* pwd\\n\",\n \"* tee\\n\",\n \"* tty\",\n \"* yes\\n\",\n \"\\n\",\n \"In many cases, these may have a lower performance overhead than the \",\n \"posix command line utility with the same name. This is because these \",\n \"tools avoid the need for a full subprocess call. Additionally, these \",\n \"tools are cross-platform.\"]\n },\n {\"name\": \"direnv\",\n \"package\": \"xonsh-direnv\",\n \"url\": \"https://github.com/74th/xonsh-direnv\",\n \"description\": [\"Supports direnv.\"]\n },\n {\"name\": \"distributed\",\n \"package\": \"xonsh\",\n \"url\": \"http://xon.sh\",\n \"description\": [\n \"The distributed parallel computing library hooks for xonsh. \",\n \"Importantly this provides a substitute 'dworker' command which enables \",\n \"distributed workers to have access to xonsh builtins.\\n\\n\",\n \"Furthermore, this xontrib adds a 'DSubmitter' context manager for \",\n \"executing a block remotely. Moreover, this also adds a convenience \",\n \"function 'dsubmit()' for creating DSubmitter and Executor instances \",\n \"at the same time. Thus users may submit distributed jobs with::\\n\\n\",\n \" with dsubmit('127.0.0.1:8786', rtn='x') as dsub:\\n\",\n \" x = $(echo I am elsewhere)\\n\\n\",\n \" res = dsub.future.result()\\n\",\n \" print(res)\\n\\n\",\n \"This is useful for long running or non-blocking jobs.\"]\n },\n {\"name\": \"docker_tabcomplete\",\n \"package\": \"xonsh-docker-tabcomplete\",\n \"url\": \"https://github.com/xsteadfastx/xonsh-docker-tabcomplete\",\n \"description\": [\"Adds tabcomplete functionality to docker inside of xonsh.\"]\n },\n {\"name\": \"jedi\",\n \"package\": \"xonsh\",\n \"url\": \"http://xon.sh\",\n \"description\": [\"Jedi tab completion hooks for xonsh.\"]\n },\n {\"name\": \"mpl\",\n \"package\": \"xonsh\",\n \"url\": \"http://xon.sh\",\n \"description\": [\"Matplotlib hooks for xonsh, including the new 'mpl' alias \",\n \"that displays the current figure on the screen.\"]\n },\n {\"name\": \"prompt_ret_code\",\n \"package\": \"xonsh\",\n \"url\": \"http://xon.sh\",\n \"description\": [\"Adds return code info to the prompt\"]\n },\n {\"name\": \"free_cwd\",\n \"package\": \"xonsh\",\n \"url\": \"http://xon.sh\",\n \"description\": [\n \"Windows only xontrib, to release the lock on the current directory \",\n \"whenever the prompt is shown. Enabling this will allow the other \",\n \"programs or Windows Explorer to delete or rename the current or parent \",\n \"directories. Internally, it is accomplished by temporarily resetting \",\n \"CWD to the root drive folder while waiting at the prompt. This only \",\n \"works with the prompt_toolkit backend and can cause cause issues \",\n \"if any extensions are enabled that hook the prompt and relies on \",\n \"``os.getcwd()``\"]\n },\n {\"name\": \"whole_word_jumping\",\n \"package\": \"xonsh\",\n \"url\": \"http://xon.sh\",\n \"description\": [\n \"Jumping across whole words (non-whitespace) with Ctrl+Left/Right.\",\n \"Alt+Left/Right remains unmodified to jump over smaller word segments.\"]\n },\n {\"name\": \"scrapy_tabcomplete\",\n \"package\": \"xonsh-scrapy-tabcomplete\",\n \"url\": \"https://github.com/Granitas/xonsh-scrapy-tabcomplete\",\n \"description\": [\"Adds tabcomplete functionality to scrapy inside of xonsh.\"]\n },\n {\"name\": \"vox\",\n \"package\": \"xonsh\",\n \"url\": \"http://xon.sh\",\n \"description\": [\"Python virtual environment manager for xonsh.\"]\n },\n {\"name\": \"vox_tabcomplete\",\n \"package\": \"xonsh-vox-tabcomplete\",\n \"url\": \"https://github.com/Granitosaurus/xonsh-vox-tabcomplete\",\n \"description\": [\"Adds tabcomplete functionality to vox inside of xonsh.\"]\n },\n {\"name\": \"xo\",\n \"package\": \"exofrills\",\n \"url\": \"https://github.com/scopatz/xo\",\n \"description\": [\"Adds an 'xo' alias to run the exofrills text editor in the \",\n \"current Python interpreter session. This shaves off a \",\n \"bit of the startup time when running your favorite, minimal \",\n \"text editor.\"]\n },\n {\"name\": \"xonda\",\n \"package\": \"xonda\",\n \"url\": \"https://github.com/gforsyth/xonda\",\n \"description\": [\"A thin wrapper around conda with tab completion\"]\n },\n {\"name\": \"avox\",\n \"package\": \"xontrib-avox\",\n \"url\": \"https://github.com/astronouth7303/xontrib-avox\",\n \"description\": [\"Automatic (de)activation of virtual environments as you cd around\"]\n },\n {\"name\": \"z\",\n \"package\": \"xontrib-z\",\n \"url\": \"https://github.com/astronouth7303/xontrib-z\",\n \"description\": [\"Tracks your most used directories, based on 'frecency'.\"]\n },\n {\"name\": \"powerline\",\n \"package\": \"xontrib-powerline\",\n \"url\": \"https://github.com/santagada/xontrib-powerline\",\n \"description\": [\"Powerline for Xonsh shell\"]\n },\n {\"name\": \"prompt_vi_mode\",\n \"package\": \"xontrib-prompt-vi-mode\",\n \"url\": \"https://github.com/t184256/xontrib-prompt-vi-mode\",\n \"description\": [\"vi-mode status formatter for xonsh prompt\"]\n },\n {\"name\": \"click_tabcomplete\",\n \"package\": \"xonsh-click-tabcomplete\",\n \"url\": \"https://github.com/Granitosaurus/xonsh-click-tabcomplete\",\n \"description\": [\"Adds tabcomplete functionality to click based python applications inside of xonsh.\"]\n },\n {\"name\": \"fzf-widgets\",\n \"package\": \"xontrib-fzf-widgets\",\n \"url\": \"https://github.com/shahinism/xontrib-fzf-widgets\",\n \"description\": [\"Adds some fzf widgets to your xonsh shell.\"]\n },\n {\"name\": \"schedule\",\n \"package\": \"xontrib-schedule\",\n \"url\": \"https://github.com/astronouth7303/xontrib-schedule\",\n \"description\": [\"Xonsh Task Scheduling\"]\n }\n ],\n \"packages\": {\n \"exofrills\": {\n \"license\": \"WTFPL\",\n \"url\": \"http://exofrills.org\",\n \"install\": {\n \"conda\": \"conda install -c conda-forge xo\",\n \"pip\": \"xpip install exofrills\"}\n },\n \"xonsh\": {\n \"license\": \"BSD 3-clause\",\n \"url\": \"http://xon.sh\",\n \"install\": {\n \"conda\": \"conda install -c conda-forge xonsh\",\n \"pip\": \"xpip install xonsh\",\n \"aura\": \"sudo aura -A xonsh\",\n \"yaourt\": \"yaourt -Sa xonsh\"}\n },\n \"xontrib-prompt-ret-code\": {\n \"license\": \"MIT\",\n \"url\": \"https://github.com/Siecje/xontrib-prompt-ret-code\",\n \"install\": {\n \"pip\": \"xpip install xontrib-prompt-ret-code\"\n }\n },\n \"xonsh-apt-tabcomplete\": {\n \"license\": \"BSD 2-clause\",\n \"url\": \"https://github.com/DangerOnTheRanger/xonsh-apt-tabcomplete\",\n \"install\": {\n \"pip\": \"xpip install xonsh-apt-tabcomplete\"\n }\n },\n \"xonsh-direnv\": {\n \"license\": \"MIT\",\n \"url\": \"https://github.com/74th/xonsh-direnv\",\n \"install\": {\n \"pip\": \"xpip install xonsh-direnv\"\n }\n },\n \"xonsh-docker-tabcomplete\": {\n \"license\": \"MIT\",\n \"url\": \"https://github.com/xsteadfastx/xonsh-docker-tabcomplete\",\n \"install\": {\n \"pip\": \"xpip install xonsh-docker-tabcomplete\"\n }\n },\n \"xonsh-scrapy-tabcomplete\": {\n \"license\": \"GPLv3\",\n \"url\": \"https://github.com/Granitas/xonsh-scrapy-tabcomplete\",\n \"install\": {\n \"pip\": \"xpip install xonsh-scrapy-tabcomplete\"\n }\n },\n \"xonsh-vox-tabcomplete\": {\n \"license\": \"GPLv3\",\n \"url\": \"https://github.com/Granitosaurus/xonsh-vox-tabcomplete\",\n \"install\": {\n \"pip\": \"xpip install xonsh-vox-tabcomplete\"\n }\n },\n \"xonsh-click-tabcomplete\": {\n \"license\": \"GPLv3\",\n \"url\": \"https://github.com/Granitosaurus/xonsh-click-tabcomplete\",\n \"install\": {\n \"pip\": \"xpip install xonsh-click-tabcomplete\"\n }\n },\n \"xonsh-autoxsh\": {\n \"license\": \"GPLv3\",\n \"url\": \"https://github.com/Granitas/xonsh-autoxsh\",\n \"install\": {\n \"pip\": \"xpip install xonsh-autoxsh\"\n }\n },\n \"xonda\": {\n \"license\": \"MIT\",\n \"url\": \"https://github.com/gforsyth/xonda\",\n \"install\": {\n \"pip\": \"xpip install xonda\"\n }\n },\n \"xontrib-avox\": {\n \"license\": \"GPLv3\",\n \"url\": \"https://github.com/astronouth7303/xontrib-avox\",\n \"install\": {\n \"pip\": \"xpip install xontrib-avox\"\n }\n },\n \"xontrib-z\": {\n \"license\": \"GPLv3\",\n \"url\": \"https://github.com/astronouth7303/xontrib-z\",\n \"install\": {\n \"pip\": \"xpip install xontrib-z\"\n }\n },\n \"xontrib-powerline\": {\n \"license\": \"MIT\",\n \"url\": \"https://github.com/santagada/xontrib-powerline\",\n \"install\": {\n \"pip\": \"xpip install xontrib-powerline\"\n }\n },\n \"xontrib-thefuck\": {\n \"license\": \"MIT\",\n \"url\": \"https://github.com/meatballs/xontrib-thefuck\",\n \"install\": {\n \"pip\": \"xpip install xontrib-thefuck\"\n }\n },\n \"xontrib-prompt-vi-mode\": {\n \"license\": \"MIT\",\n \"url\": \"https://github.com/t184256/xontrib-prompt-vi-mode\",\n \"install\": {\n \"pip\": \"xpip install xontrib-prompt-vi-mode\"\n }\n },\n \"xontrib-fzf-widgets\": {\n \"license\": \"GPLv3\",\n \"url\": \"https://github.com/shahinism/xontrib-fzf-widgets\",\n \"install\": {\n \"pip\": \"xpip install xontrib-fzf-widgets\"\n }\n },\n \"xontrib-schedule\": {\n \"license\": \"MIT\",\n \"url\": \"https://github.com/astronouth7303/xontrib-schedule\",\n \"install\": {\n \"pip\": \"xpip install xontrib-schedule\"\n }\n }\n }\n}\n" }, { "path": "xonsh/xontribs.py", "content": "\"\"\"Tools for helping manage xontributions.\"\"\"\nimport os\nimport sys\nimport json\nimport builtins\nimport argparse\nimport functools\nimport importlib\nimport importlib.util\n\nfrom xonsh.tools import print_color, unthreadable\n\n\n@functools.lru_cache(1)\ndef xontribs_json():\n return os.path.join(os.path.dirname(__file__), \"xontribs.json\")\n\n\ndef find_xontrib(name):\n \"\"\"Finds a xontribution from its name.\"\"\"\n if name.startswith(\".\"):\n spec = importlib.util.find_spec(name, package=\"xontrib\")\n else:\n spec = importlib.util.find_spec(\".\" + name, package=\"xontrib\")\n return spec or importlib.util.find_spec(name)\n\n\ndef xontrib_context(name):\n \"\"\"Return a context dictionary for a xontrib of a given name.\"\"\"\n spec = find_xontrib(name)\n if spec is None:\n return None\n m = importlib.import_module(spec.name)\n pubnames = getattr(m, \"__all__\", None)\n if pubnames is not None:\n ctx = {k: getattr(m, k) for k in pubnames}\n else:\n ctx = {k: getattr(m, k) for k in dir(m) if not k.startswith(\"_\")}\n return ctx\n\n\ndef prompt_xontrib_install(names):\n \"\"\"Returns a formatted string with name of xontrib package to prompt user\"\"\"\n md = xontrib_metadata()\n packages = []\n for name in names:\n for xontrib in md[\"xontribs\"]:\n if xontrib[\"name\"] == name:\n packages.append(xontrib[\"package\"])\n\n print(\n \"The following xontribs are enabled but not installed: \\n\"\n \" {xontribs}\\n\"\n \"To install them run \\n\"\n \" xpip install {packages}\".format(\n xontribs=\" \".join(names), packages=\" \".join(packages)\n )\n )\n\n\ndef update_context(name, ctx=None):\n \"\"\"Updates a context in place from a xontrib. If ctx is not provided,\n then __xonsh__.ctx is updated.\n \"\"\"\n if ctx is None:\n ctx = builtins.__xonsh__.ctx\n if not hasattr(update_context, \"bad_imports\"):\n update_context.bad_imports = []\n modctx = xontrib_context(name)\n if modctx is None:\n update_context.bad_imports.append(name)\n return ctx\n return ctx.update(modctx)\n\n\n@functools.lru_cache()\ndef xontrib_metadata():\n \"\"\"Loads and returns the xontribs.json file.\"\"\"\n with open(xontribs_json(), \"r\") as f:\n md = json.load(f)\n return md\n\n\ndef xontribs_load(names, verbose=False):\n \"\"\"Load xontribs from a list of names\"\"\"\n ctx = builtins.__xonsh__.ctx\n for name in names:\n if verbose:\n print(\"loading xontrib {0!r}\".format(name))\n update_context(name, ctx=ctx)\n if update_context.bad_imports:\n prompt_xontrib_install(update_context.bad_imports)\n del update_context.bad_imports\n\n\ndef _load(ns):\n \"\"\"load xontribs\"\"\"\n xontribs_load(ns.names, verbose=ns.verbose)\n\n\ndef _list(ns):\n \"\"\"Lists xontribs.\"\"\"\n meta = xontrib_metadata()\n data = []\n nname = 6 # ensures some buffer space.\n names = None if len(ns.names) == 0 else set(ns.names)\n for md in meta[\"xontribs\"]:\n name = md[\"name\"]\n if names is not None and md[\"name\"] not in names:\n continue\n nname = max(nname, len(name))\n spec = find_xontrib(name)\n if spec is None:\n installed = loaded = False\n else:\n installed = True\n loaded = spec.name in sys.modules\n d = {\"name\": name, \"installed\": installed, \"loaded\": loaded}\n data.append(d)\n if ns.json:\n jdata = {d.pop(\"name\"): d for d in data}\n s = json.dumps(jdata)\n print(s)\n else:\n s = \"\"\n for d in data:\n name = d[\"name\"]\n lname = len(name)\n s += \"{PURPLE}\" + name + \"{NO_COLOR} \" + \" \" * (nname - lname)\n if d[\"installed\"]:\n s += \"{GREEN}installed{NO_COLOR} \"\n else:\n s += \"{RED}not-installed{NO_COLOR} \"\n if d[\"loaded\"]:\n s += \"{GREEN}loaded{NO_COLOR}\"\n else:\n s += \"{RED}not-loaded{NO_COLOR}\"\n s += \"\\n\"\n print_color(s[:-1])\n\n\n@functools.lru_cache()\ndef _create_xontrib_parser():\n # parse command line args\n parser = argparse.ArgumentParser(\n prog=\"xontrib\", description=\"Manages xonsh extensions\"\n )\n subp = parser.add_subparsers(title=\"action\", dest=\"action\")\n load = subp.add_parser(\"load\", help=\"loads xontribs\")\n load.add_argument(\n \"-v\", \"--verbose\", action=\"store_true\", default=False, dest=\"verbose\"\n )\n load.add_argument(\"names\", nargs=\"+\", default=(), help=\"names of xontribs\")\n lyst = subp.add_parser(\n \"list\", help=(\"list xontribs, whether they are \" \"installed, and loaded.\")\n )\n lyst.add_argument(\n \"--json\", action=\"store_true\", default=False, help=\"reports results as json\"\n )\n lyst.add_argument(\"names\", nargs=\"*\", default=(), help=\"names of xontribs\")\n return parser\n\n\n_MAIN_XONTRIB_ACTIONS = {\"load\": _load, \"list\": _list}\n\n\n@unthreadable\ndef xontribs_main(args=None, stdin=None):\n \"\"\"Alias that loads xontribs\"\"\"\n if not args or (\n args[0] not in _MAIN_XONTRIB_ACTIONS and args[0] not in {\"-h\", \"--help\"}\n ):\n args.insert(0, \"load\")\n parser = _create_xontrib_parser()\n ns = parser.parse_args(args)\n if ns.action is None: # apply default action\n ns = parser.parse_args([\"load\"] + args)\n return _MAIN_XONTRIB_ACTIONS[ns.action](ns)\n" }, { "path": "xonsh/xoreutils/__init__.py", "content": "# amalgamate\n# amalgamate end\n" }, { "path": "xonsh/xoreutils/_which.py", "content": "#!/usr/bin/env python\n# Copyright (c) 2002-2007 ActiveState Software Inc.\n\n# Permission is hereby granted, free of charge, to any person obtaining a\n# copy of this software and associated documentation files (the\n# \"Software\"), to deal in the Software without restriction, including\n# without limitation the rights to use, copy, modify, merge, publish,\n# distribute, sublicense, and/or sell copies of the Software, and to\n# permit persons to whom the Software is furnished to do so, subject to\n# the following conditions:\n#\n# The above copyright notice and this permission notice shall be included\n# in all copies or substantial portions of the Software.\n#\n# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS\n# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\n# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.\n# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY\n# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,\n# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\n# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n#\n# Author:\n# Trent Mick (TrentM@ActiveState.com)\n# Home:\n# http://trentm.com/projects/which/\nimport os\nimport sys\nimport stat\nimport getopt\nimport builtins\nimport collections.abc as cabc\n\nr\"\"\"Find the full path to commands.\n\nwhich(command, path=None, verbose=0, exts=None)\n Return the full path to the first match of the given command on the\n path.\n\nwhichall(command, path=None, verbose=0, exts=None)\n Return a list of full paths to all matches of the given command on\n the path.\n\nwhichgen(command, path=None, verbose=0, exts=None)\n Return a generator which will yield full paths to all matches of the\n given command on the path.\n\nBy default the PATH environment variable is searched (as well as, on\nWindows, the AppPaths key in the registry), but a specific 'path' list\nto search may be specified as well. On Windows, the PATHEXT environment\nvariable is applied as appropriate.\n\nIf \"verbose\" is true then a tuple of the form\n (, )\nis returned for each match. The latter element is a textual description\nof where the match was found. For example:\n from PATH element 0\n from HKLM\\SOFTWARE\\...\\perl.exe\n\"\"\"\n\n_cmdlnUsage = \"\"\"\n Show the full path of commands.\n\n Usage:\n which [...] [...]\n\n Options:\n -h, --help Print this help and exit.\n -V, --version Print the version info and exit.\n\n -a, --all Print *all* matching paths.\n -v, --verbose Print out how matches were located and\n show near misses on stderr.\n -q, --quiet Just print out matches. I.e., do not print out\n near misses.\n\n -p , --path=\n An alternative path (list of directories) may\n be specified for searching.\n -e , --exts=\n Specify a list of extensions to consider instead\n of the usual list (';'-separate list, Windows\n only).\n\n Show the full path to the program that would be run for each given\n command name, if any. Which, like GNU's which, returns the number of\n failed arguments, or -1 when no was given.\n\n Near misses include duplicates, non-regular files and (on Un*x)\n files without executable access.\n\"\"\"\n\n__version_info__ = (1, 2, 0)\n__version__ = \".\".join(map(str, __version_info__))\n__all__ = [\"which\", \"whichall\", \"whichgen\", \"WhichError\"]\n\n\nclass WhichError(Exception):\n pass\n\n\n# internal support stuff\n\n\ndef _getRegisteredExecutable(exeName):\n \"\"\"Windows allow application paths to be registered in the registry.\"\"\"\n registered = None\n if sys.platform.startswith(\"win\"):\n if os.path.splitext(exeName)[1].lower() != \".exe\":\n exeName += \".exe\"\n try:\n import winreg as _winreg\n except ImportError:\n import _winreg\n try:\n key = \"SOFTWARE\\\\Microsoft\\\\Windows\\\\CurrentVersion\\\\App Paths\\\\\" + exeName\n value = _winreg.QueryValue(_winreg.HKEY_LOCAL_MACHINE, key)\n registered = (value, \"from HKLM\\\\\" + key)\n except _winreg.error:\n pass\n if registered and not os.path.exists(registered[0]):\n registered = None\n return registered\n\n\ndef _samefile(fname1, fname2):\n if sys.platform.startswith(\"win\"):\n return os.path.normpath(os.path.normcase(fname1)) == os.path.normpath(\n os.path.normcase(fname2)\n )\n else:\n return os.path.samefile(fname1, fname2)\n\n\ndef _cull(potential, matches, verbose=0):\n \"\"\"Cull inappropriate matches. Possible reasons:\n - a duplicate of a previous match\n - not a disk file\n - not executable (non-Windows)\n If 'potential' is approved it is returned and added to 'matches'.\n Otherwise, None is returned.\n \"\"\"\n for match in matches: # don't yield duplicates\n if _samefile(potential[0], match[0]):\n if verbose:\n sys.stderr.write(\"duplicate: %s (%s)\\n\" % potential)\n return None\n else:\n if not stat.S_ISREG(os.stat(potential[0]).st_mode):\n if verbose:\n sys.stderr.write(\"not a regular file: %s (%s)\\n\" % potential)\n elif sys.platform != \"win32\" and not os.access(potential[0], os.X_OK):\n if verbose:\n sys.stderr.write(\"no executable access: %s (%s)\\n\" % potential)\n else:\n matches.append(potential)\n return potential\n\n\n# module API\n\n\ndef whichgen(command, path=None, verbose=0, exts=None):\n \"\"\"Return a generator of full paths to the given command.\n\n \"command\" is a the name of the executable to search for.\n \"path\" is an optional alternate path list to search. The default it\n to use the PATH environment variable.\n \"verbose\", if true, will cause a 2-tuple to be returned for each\n match. The second element is a textual description of where the\n match was found.\n \"exts\" optionally allows one to specify a list of extensions to use\n instead of the standard list for this system. This can\n effectively be used as an optimization to, for example, avoid\n stat's of \"foo.vbs\" when searching for \"foo\" and you know it is\n not a VisualBasic script but \".vbs\" is on PATHEXT. This option\n is only supported on Windows.\n\n This method returns a generator which yields tuples of the form (, ).\n \"\"\"\n matches = []\n if path is None:\n usingGivenPath = 0\n path = os.environ.get(\"PATH\", \"\").split(os.pathsep)\n if sys.platform.startswith(\"win\"):\n path.insert(0, os.curdir) # implied by Windows shell\n else:\n usingGivenPath = 1\n\n # Windows has the concept of a list of extensions (PATHEXT env var).\n if sys.platform.startswith(\"win\"):\n if exts is None:\n exts = builtins.__xonsh__.env[\"PATHEXT\"]\n # If '.exe' is not in exts then obviously this is Win9x and\n # or a bogus PATHEXT, then use a reasonable default.\n for ext in exts:\n if ext.lower() == \".exe\":\n break\n else:\n exts = [\".COM\", \".EXE\", \".BAT\", \".CMD\"]\n elif not isinstance(exts, cabc.Sequence):\n raise TypeError(\"'exts' argument must be a sequence or None\")\n else:\n if exts is not None:\n raise WhichError(\n \"'exts' argument is not supported on \" \"platform '%s'\" % sys.platform\n )\n exts = []\n\n # File name cannot have path separators because PATH lookup does not\n # work that way.\n if os.sep in command or os.altsep and os.altsep in command:\n if os.path.exists(command):\n match = _cull((command, \"explicit path given\"), matches, verbose)\n yield match\n else:\n for i in range(len(path)):\n dirName = path[i]\n # On windows the dirName *could* be quoted, drop the quotes\n if (\n sys.platform.startswith(\"win\")\n and len(dirName) >= 2\n and dirName[0] == '\"'\n and dirName[-1] == '\"'\n ):\n dirName = dirName[1:-1]\n for ext in [\"\"] + exts:\n absName = os.path.abspath(\n os.path.normpath(os.path.join(dirName, command + ext))\n )\n if os.path.isfile(absName):\n if usingGivenPath:\n fromWhere = \"from given path element %d\" % i\n elif not sys.platform.startswith(\"win\"):\n fromWhere = \"from PATH element %d\" % i\n elif i == 0:\n fromWhere = \"from current directory\"\n else:\n fromWhere = \"from PATH element %d\" % (i - 1)\n match = _cull((absName, fromWhere), matches, verbose)\n if match:\n yield match\n match = _getRegisteredExecutable(command)\n if match is not None:\n match = _cull(match, matches, verbose)\n if match:\n yield match\n\n\ndef which(command, path=None, verbose=0, exts=None):\n \"\"\"Return the full path to the first match of the given command on\n the path.\n\n \"command\" is a the name of the executable to search for.\n \"path\" is an optional alternate path list to search. The default it\n to use the PATH environment variable.\n \"verbose\", if true, will cause a 2-tuple to be returned. The second\n element is a textual description of where the match was found.\n \"exts\" optionally allows one to specify a list of extensions to use\n instead of the standard list for this system. This can\n effectively be used as an optimization to, for example, avoid\n stat's of \"foo.vbs\" when searching for \"foo\" and you know it is\n not a VisualBasic script but \".vbs\" is on PATHEXT. This option\n is only supported on Windows.\n\n If no match is found for the command, a WhichError is raised.\n \"\"\"\n try:\n absName, fromWhere = next(whichgen(command, path, verbose, exts))\n except StopIteration:\n raise WhichError(\"Could not find '%s' on the path.\" % command)\n if verbose:\n return absName, fromWhere\n else:\n return absName\n\n\ndef whichall(command, path=None, verbose=0, exts=None):\n \"\"\"Return a list of full paths to all matches of the given command\n on the path.\n\n \"command\" is a the name of the executable to search for.\n \"path\" is an optional alternate path list to search. The default it\n to use the PATH environment variable.\n \"verbose\", if true, will cause a 2-tuple to be returned for each\n match. The second element is a textual description of where the\n match was found.\n \"exts\" optionally allows one to specify a list of extensions to use\n instead of the standard list for this system. This can\n effectively be used as an optimization to, for example, avoid\n stat's of \"foo.vbs\" when searching for \"foo\" and you know it is\n not a VisualBasic script but \".vbs\" is on PATHEXT. This option\n is only supported on Windows.\n \"\"\"\n if verbose:\n return list(whichgen(command, path, verbose, exts))\n else:\n return list(absName for absName, _ in whichgen(command, path, verbose, exts))\n\n\n# mainline\n\n\ndef main(argv):\n all = 0\n verbose = 0\n altpath = None\n exts = None\n try:\n optlist, args = getopt.getopt(\n argv[1:],\n \"haVvqp:e:\",\n [\"help\", \"all\", \"version\", \"verbose\", \"quiet\", \"path=\", \"exts=\"],\n )\n except getopt.GetoptErrsor as msg:\n sys.stderr.write(\"which: error: %s. Your invocation was: %s\\n\" % (msg, argv))\n sys.stderr.write(\"Try 'which --help'.\\n\")\n return 1\n for opt, optarg in optlist:\n if opt in (\"-h\", \"--help\"):\n print(_cmdlnUsage)\n return 0\n elif opt in (\"-V\", \"--version\"):\n print(\"which %s\" % __version__)\n return 0\n elif opt in (\"-a\", \"--all\"):\n all = 1\n elif opt in (\"-v\", \"--verbose\"):\n verbose = 1\n elif opt in (\"-q\", \"--quiet\"):\n verbose = 0\n elif opt in (\"-p\", \"--path\"):\n if optarg:\n altpath = optarg.split(os.pathsep)\n else:\n altpath = []\n elif opt in (\"-e\", \"--exts\"):\n if optarg:\n exts = optarg.split(os.pathsep)\n else:\n exts = []\n\n if len(args) == 0:\n return -1\n\n failures = 0\n for arg in args:\n # print \"debug: search for %r\" % arg\n nmatches = 0\n for absName, fromWhere in whichgen(\n arg, path=altpath, verbose=verbose, exts=exts\n ):\n if verbose:\n print(\"%s (%s)\" % (absName, fromWhere))\n else:\n print(absName)\n nmatches += 1\n if not all:\n break\n if not nmatches:\n failures += 1\n return failures\n\n\nif __name__ == \"__main__\":\n sys.exit(main(sys.argv))\n" }, { "path": "xonsh/xoreutils/cat.py", "content": "\"\"\"Implements a cat command for xonsh.\"\"\"\nimport os\nimport time\nimport builtins\n\nimport xonsh.proc as xproc\nfrom xonsh.xoreutils.util import arg_handler\n\n\ndef _cat_line(\n f, sep, last_was_blank, line_count, opts, out, enc, enc_errors, read_size\n):\n _r = r = f.readline(size=80)\n if isinstance(_r, str):\n _r = r = _r.encode(enc, enc_errors)\n if r == b\"\":\n last_was_blank, line_count, read_size, True\n if r.endswith(sep):\n _r = _r[: -len(sep)]\n this_one_blank = _r == b\"\"\n if last_was_blank and this_one_blank and opts[\"squeeze_blank\"]:\n last_was_blank, line_count, read_size, False\n last_was_blank = this_one_blank\n if opts[\"number_all\"] or (opts[\"number_nonblank\"] and not this_one_blank):\n start = (\"%6d \" % line_count).encode(enc, enc_errors)\n _r = start + _r\n line_count += 1\n if opts[\"show_ends\"]:\n _r = _r + b\"$\"\n out.buffer.write(_r)\n out.flush()\n read_size += len(r)\n return last_was_blank, line_count, read_size, False\n\n\ndef _cat_single_file(opts, fname, stdin, out, err, line_count=1):\n env = builtins.__xonsh__.env\n enc = env.get(\"XONSH_ENCODING\")\n enc_errors = env.get(\"XONSH_ENCODING_ERRORS\")\n read_size = 0\n file_size = fobj = None\n if fname == \"-\":\n f = stdin\n elif os.path.isdir(fname):\n print(\"cat: {}: Is a directory.\".format(fname), file=err)\n return True, line_count\n elif not os.path.exists(fname):\n print(\"cat: No such file or directory: {}\".format(fname), file=err)\n return True, line_count\n else:\n file_size = os.stat(fname).st_size\n if file_size == 0:\n file_size = None\n fobj = open(fname, \"rb\")\n f = xproc.NonBlockingFDReader(fobj.fileno(), timeout=0.1)\n sep = os.linesep.encode(enc, enc_errors)\n last_was_blank = False\n while file_size is None or read_size < file_size:\n try:\n last_was_blank, line_count, read_size, endnow = _cat_line(\n f,\n sep,\n last_was_blank,\n line_count,\n opts,\n out,\n enc,\n enc_errors,\n read_size,\n )\n if endnow:\n break\n if last_was_blank:\n time.sleep(1e-3)\n except KeyboardInterrupt:\n print(\"got except\", flush=True, file=out)\n break\n except Exception as e:\n print(\"xonsh:\", e, flush=True, file=out)\n pass\n if fobj is not None:\n fobj.close()\n return False, line_count\n\n\ndef cat(args, stdin, stdout, stderr):\n \"\"\"A cat command for xonsh.\"\"\"\n opts = _cat_parse_args(args)\n if opts is None:\n print(CAT_HELP_STR, file=stdout)\n return 0\n\n line_count = 1\n errors = False\n if len(args) == 0:\n args = [\"-\"]\n for i in args:\n o = _cat_single_file(opts, i, stdin, stdout, stderr, line_count)\n if o is None:\n return -1\n _e, line_count = o\n errors = _e or errors\n\n return int(errors)\n\n\ndef _cat_parse_args(args):\n out = {\n \"number_nonblank\": False,\n \"number_all\": False,\n \"squeeze_blank\": False,\n \"show_ends\": False,\n }\n if \"--help\" in args:\n return\n\n arg_handler(args, out, \"-b\", \"number_nonblank\", True, \"--number-nonblank\")\n arg_handler(args, out, \"-n\", \"number_all\", True, \"--number\")\n arg_handler(args, out, \"-E\", \"show_ends\", True, \"--show-ends\")\n arg_handler(args, out, \"-s\", \"squeeze_blank\", True, \"--squeeze-blank\")\n arg_handler(args, out, \"-T\", \"show_tabs\", True, \"--show-tabs\")\n\n return out\n\n\nCAT_HELP_STR = \"\"\"This version of cat was written in Python for the xonsh project: http://xon.sh\nBased on cat from GNU coreutils: http://www.gnu.org/software/coreutils/\n\nUsage: cat [OPTION]... [FILE]...\nConcatenate FILE(s), or standard input, to standard output.\n\n -b, --number-nonblank number nonempty output lines, overrides -n\n -E, --show-ends display $ at end of each line\n -n, --number number all output lines\n -s, --squeeze-blank suppress repeated empty output lines\n -T, --show-tabs display TAB characters as ^I\n -u (ignored)\n --help display this help and exit\n\nWith no FILE, or when FILE is -, read standard input.\n\nExamples:\n cat f - g Output f's contents, then standard input, then g's contents.\n cat Copy standard input to standard output.\"\"\"\n\n# NOT IMPLEMENTED:\n# -A, --show-all equivalent to -vET\n# -e equivalent to -vE\n# -t equivalent to -vT\n# -v, --show-nonprinting use ^ and M- notation, except for LFD and TAB\n# --version output version information and exit\"\"\"\n\n\ndef cat_main(args=None):\n import sys\n from xonsh.main import setup\n\n setup()\n args = sys.argv if args is None else args\n cat(args, sys.stdin, sys.stdout, sys.stderr)\n\n\nif __name__ == \"__main__\":\n cat_main()\n" }, { "path": "xonsh/xoreutils/echo.py", "content": "\"\"\"Implements a simple echo command for xonsh.\"\"\"\n\n\ndef echo(args, stdin, stdout, stderr):\n \"\"\"A simple echo command.\"\"\"\n opts = _echo_parse_args(args)\n if opts is None:\n return\n if opts[\"help\"]:\n print(ECHO_HELP, file=stdout)\n return 0\n ender = opts[\"end\"]\n args = map(str, args)\n if opts[\"escapes\"]:\n args = map(lambda x: x.encode().decode(\"unicode_escape\"), args)\n print(*args, end=ender, file=stdout)\n\n\ndef _echo_parse_args(args):\n out = {\"escapes\": False, \"end\": \"\\n\", \"help\": False}\n if \"-e\" in args:\n args.remove(\"-e\")\n out[\"escapes\"] = True\n if \"-E\" in args:\n args.remove(\"-E\")\n out[\"escapes\"] = False\n if \"-n\" in args:\n args.remove(\"-n\")\n out[\"end\"] = \"\"\n if \"-h\" in args or \"--help\" in args:\n out[\"help\"] = True\n return out\n\n\nECHO_HELP = \"\"\"Usage: echo [OPTIONS]... [STRING]...\nEcho the STRING(s) to standard output.\n\n -n do not include the trailing newline\n -e enable interpretation of backslash escapes\n -E disable interpretation of backslash escapes (default)\n -h --help display this message and exit\n\nThis version of echo was written in Python for the xonsh project: http://xon.sh\nBased on echo from GNU coreutils: http://www.gnu.org/software/coreutils/\"\"\"\n" }, { "path": "xonsh/xoreutils/pwd.py", "content": "\"\"\"A pwd implementation for xonsh.\"\"\"\nimport os\n\n\ndef pwd(args, stdin, stdout, stderr):\n \"\"\"A pwd implementation\"\"\"\n e = __xonsh__.env[\"PWD\"]\n if \"-h\" in args or \"--help\" in args:\n print(PWD_HELP, file=stdout)\n return 0\n if \"-P\" in args:\n e = os.path.realpath(e)\n print(e, file=stdout)\n return 0\n\n\nPWD_HELP = \"\"\"Usage: pwd [OPTION]...\nPrint the full filename of the current working directory.\n\n -P, --physical avoid all symlinks\n --help display this help and exit\n\nThis version of pwd was written in Python for the xonsh project: http://xon.sh\nBased on pwd from GNU coreutils: http://www.gnu.org/software/coreutils/\"\"\"\n\n\n# Not Implemented\n# -L, --logical use PWD from environment, even if it contains symlinks\n" }, { "path": "xonsh/xoreutils/tee.py", "content": "\"\"\"A tee implementation for xonsh.\"\"\"\n\n\ndef tee(args, stdin, stdout, stderr):\n \"\"\"A tee command for xonsh.\"\"\"\n mode = \"w\"\n if \"-a\" in args:\n args.remove(\"-a\")\n mode = \"a\"\n if \"--append\" in args:\n args.remove(\"--append\")\n mode = \"a\"\n if \"--help\" in args:\n print(TEE_HELP, file=stdout)\n return 0\n if stdin is None:\n msg = \"tee was not piped stdin, must have input stream to read from.\"\n print(msg, file=stderr)\n return 1\n\n errors = False\n files = []\n for i in args:\n if i == \"-\":\n files.append(stdout)\n else:\n try:\n files.append(open(i, mode))\n except:\n print(\"tee: failed to open {}\".format(i), file=stderr)\n errors = True\n files.append(stdout)\n\n while True:\n r = stdin.read(1024)\n if r == \"\":\n break\n for i in files:\n i.write(r)\n for i in files:\n if i != stdout:\n i.close()\n\n return int(errors)\n\n\nTEE_HELP = \"\"\"This version of tee was written in Python for the xonsh project: http://xon.sh\nBased on tee from GNU coreutils: http://www.gnu.org/software/coreutils/\n\nUsage: tee [OPTION]... [FILE]...\nCopy standard input to each FILE, and also to standard output.\n\n -a, --append append to the given FILEs, do not overwrite\n --help display this help and exit\n\nIf a FILE is -, copy again to standard output.\"\"\"\n\n# NOT IMPLEMENTED:\n# -i, --ignore-interrupts ignore interrupt signals\n" }, { "path": "xonsh/xoreutils/tty.py", "content": "\"\"\"A tty implementation for xonsh\"\"\"\nimport os\nimport sys\n\n\ndef tty(args, stdin, stdout, stderr):\n \"\"\"A tty command for xonsh.\"\"\"\n if \"--help\" in args:\n print(TTY_HELP, file=stdout)\n return 0\n silent = False\n for i in (\"-s\", \"--silent\", \"--quiet\"):\n if i in args:\n silent = True\n args.remove(i)\n if len(args) > 0:\n if not silent:\n for i in args:\n print(\"tty: Invalid option: {}\".format(i), file=stderr)\n print(\"Try 'tty --help' for more information\", file=stderr)\n return 2\n try:\n fd = stdin.fileno()\n except:\n fd = sys.stdin.fileno()\n if not os.isatty(fd):\n if not silent:\n print(\"not a tty\", file=stdout)\n return 1\n if not silent:\n try:\n print(os.ttyname(fd), file=stdout)\n except:\n return 3\n return 0\n\n\nTTY_HELP = \"\"\"Usage: tty [OPTION]...\nPrint the file name of the terminal connected to standard input.\n\n -s, --silent, --quiet print nothing, only return an exit status\n --help display this help and exit\n\nThis version of tty was written in Python for the xonsh project: http://xon.sh\nBased on tty from GNU coreutils: http://www.gnu.org/software/coreutils/\"\"\"\n" }, { "path": "xonsh/xoreutils/uptime.py", "content": "\"\"\"\nProvides a cross-platform way to figure out the system uptime.\n\nShould work on damned near any operating system you can realistically expect\nto be asked to write Python code for.\nIf this module is invoked as a stand-alone script, it will print the current\nuptime in a human-readable format, or display an error message if it can't,\nto standard output.\n\nThis file was forked from the uptime project: https://github.com/Cairnarvon/uptime\nCopyright (c) 2012, Koen Crolla, All rights reserved.\n\"\"\"\nimport os\nimport sys\nimport time\nimport ctypes\nimport struct\n\nimport xonsh.platform as xp\nimport xonsh.lazyimps as xlimps\nimport xonsh.lazyasd as xl\n\n\n_BOOTTIME = None\n\n\ndef _uptime_osx():\n \"\"\"Returns the uptime on mac / darwin.\"\"\"\n global _BOOTTIME\n bt = xlimps.macutils.sysctlbyname(b\"kern.boottime\", return_str=False)\n if len(bt) == 4:\n bt = struct.unpack_from(\"@hh\", bt)\n elif len(bt) == 8:\n bt = struct.unpack_from(\"@ii\", bt)\n elif len(bt) == 16:\n bt = struct.unpack_from(\"@qq\", bt)\n else:\n raise ValueError(\"length of boot time not understood: \" + repr(bt))\n bt = bt[0] + bt[1] * 1e-6\n if bt == 0.0:\n return None\n _BOOTTIME = bt\n return time.time() - bt\n\n\ndef _uptime_linux():\n \"\"\"Returns uptime in seconds or None, on Linux.\"\"\"\n # With procfs\n try:\n with open(\"/proc/uptime\", \"r\") as f:\n up = float(f.readline().split()[0])\n return up\n except (IOError, ValueError):\n pass\n buf = ctypes.create_string_buffer(128) # 64 suffices on 32-bit, whatever.\n if xp.LIBC.sysinfo(buf) < 0:\n return None\n up = struct.unpack_from(\"@l\", buf.raw)[0]\n if up < 0:\n up = None\n return up\n\n\ndef _boottime_linux():\n \"\"\"A way to figure out the boot time directly on Linux.\"\"\"\n global _BOOTTIME\n try:\n with open(\"/proc/stat\", \"r\") as f:\n for line in f:\n if line.startswith(\"btime\"):\n _BOOTTIME = float(line.split()[1])\n return _BOOTTIME\n except (IOError, IndexError):\n return None\n\n\ndef _uptime_amiga():\n \"\"\"Returns uptime in seconds or None, on AmigaOS.\"\"\"\n global _BOOTTIME\n try:\n _BOOTTIME = os.stat(\"RAM:\").st_ctime\n return time.time() - _BOOTTIME\n except (NameError, OSError):\n return None\n\n\ndef _uptime_beos():\n \"\"\"Returns uptime in seconds on None, on BeOS/Haiku.\"\"\"\n if not hasattr(xp.LIBC, \"system_time\"):\n return None\n xp.LIBC.system_time.restype = ctypes.c_int64\n return xp.LIBC.system_time() / 1000000.\n\n\ndef _uptime_bsd():\n \"\"\"Returns uptime in seconds or None, on BSD (including OS X).\"\"\"\n global _BOOTTIME\n if not hasattr(xp.LIBC, \"sysctlbyname\"):\n # Not BSD.\n return None\n # Determine how much space we need for the response.\n sz = ctypes.c_uint(0)\n xp.LIBC.sysctlbyname(\"kern.boottime\", None, ctypes.byref(sz), None, 0)\n if sz.value != struct.calcsize(\"@LL\"):\n # Unexpected, let's give up.\n return None\n # For real now.\n buf = ctypes.create_string_buffer(sz.value)\n xp.LIBC.sysctlbyname(\"kern.boottime\", buf, ctypes.byref(sz), None, 0)\n sec, usec = struct.unpack_from(\"@LL\", buf.raw)\n # OS X disagrees what that second value is.\n if usec > 1000000:\n usec = 0.\n _BOOTTIME = sec + usec / 1000000.\n up = time.time() - _BOOTTIME\n if up < 0:\n up = None\n return up\n\n\ndef _uptime_minix():\n \"\"\"Returns uptime in seconds or None, on MINIX.\"\"\"\n try:\n with open(\"/proc/uptime\", \"r\") as f:\n up = float(f.read())\n return up\n except (IOError, ValueError):\n return None\n\n\ndef _uptime_plan9():\n \"\"\"Returns uptime in seconds or None, on Plan 9.\"\"\"\n # Apparently Plan 9 only has Python 2.2, which I'm not prepared to\n # support. Maybe some Linuxes implement /dev/time, though, someone was\n # talking about it somewhere.\n try:\n # The time file holds one 32-bit number representing the sec-\n # onds since start of epoch and three 64-bit numbers, repre-\n # senting nanoseconds since start of epoch, clock ticks, and\n # clock frequency.\n # -- cons(3)\n with open(\"/dev/time\", \"r\") as f:\n s, ns, ct, cf = f.read().split()\n return float(ct) / float(cf)\n except (IOError, ValueError):\n return None\n\n\ndef _uptime_solaris():\n \"\"\"Returns uptime in seconds or None, on Solaris.\"\"\"\n global _BOOTTIME\n try:\n kstat = ctypes.CDLL(\"libkstat.so\")\n except (AttributeError, OSError):\n return None\n\n # kstat doesn't have uptime, but it does have boot time.\n # Unfortunately, getting at it isn't perfectly straightforward.\n # First, let's pretend to be kstat.h\n\n # Constant\n KSTAT_STRLEN = 31 # According to every kstat.h I could find.\n\n # Data structures\n class anon_union(ctypes.Union):\n # The ``value'' union in kstat_named_t actually has a bunch more\n # members, but we're only using it for boot_time, so we only need\n # the padding and the one we're actually using.\n _fields_ = [(\"c\", ctypes.c_char * 16), (\"time\", ctypes.c_int)]\n\n class kstat_named_t(ctypes.Structure):\n _fields_ = [\n (\"name\", ctypes.c_char * KSTAT_STRLEN),\n (\"data_type\", ctypes.c_char),\n (\"value\", anon_union),\n ]\n\n # Function signatures\n kstat.kstat_open.restype = ctypes.c_void_p\n kstat.kstat_lookup.restype = ctypes.c_void_p\n kstat.kstat_lookup.argtypes = [\n ctypes.c_void_p,\n ctypes.c_char_p,\n ctypes.c_int,\n ctypes.c_char_p,\n ]\n kstat.kstat_read.restype = ctypes.c_int\n kstat.kstat_read.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p]\n kstat.kstat_data_lookup.restype = ctypes.POINTER(kstat_named_t)\n kstat.kstat_data_lookup.argtypes = [ctypes.c_void_p, ctypes.c_char_p]\n\n # Now, let's do something useful.\n # Initialise kstat control structure.\n kc = kstat.kstat_open()\n if not kc:\n return None\n # We're looking for unix:0:system_misc:boot_time.\n ksp = kstat.kstat_lookup(kc, \"unix\", 0, \"system_misc\")\n if ksp and kstat.kstat_read(kc, ksp, None) != -1:\n data = kstat.kstat_data_lookup(ksp, \"boot_time\")\n if data:\n _BOOTTIME = data.contents.value.time\n # Clean-up.\n kstat.kstat_close(kc)\n if _BOOTTIME is not None:\n return time.time() - _BOOTTIME\n return None\n\n\ndef _uptime_syllable():\n \"\"\"Returns uptime in seconds or None, on Syllable.\"\"\"\n global _BOOTTIME\n try:\n _BOOTTIME = os.stat(\"/dev/pty/mst/pty0\").st_mtime\n return time.time() - _BOOTTIME\n except (NameError, OSError):\n return None\n\n\ndef _uptime_windows():\n \"\"\"\n Returns uptime in seconds or None, on Windows. Warning: may return\n incorrect answers after 49.7 days on versions older than Vista.\n \"\"\"\n if hasattr(xp.LIBC, \"GetTickCount64\"):\n # Vista/Server 2008 or later.\n xp.LIBC.GetTickCount64.restype = ctypes.c_uint64\n return xp.LIBC.GetTickCount64() / 1000.\n if hasattr(xp.LIBC, \"GetTickCount\"):\n # WinCE and Win2k or later; gives wrong answers after 49.7 days.\n xp.LIBC.GetTickCount.restype = ctypes.c_uint32\n return xp.LIBC.GetTickCount() / 1000.\n return None\n\n\n@xl.lazyobject\ndef _UPTIME_FUNCS():\n return {\n \"amiga\": _uptime_amiga,\n \"aros12\": _uptime_amiga,\n \"beos5\": _uptime_beos,\n \"cygwin\": _uptime_linux,\n \"darwin\": _uptime_osx,\n \"haiku1\": _uptime_beos,\n \"linux\": _uptime_linux,\n \"linux-armv71\": _uptime_linux,\n \"linux2\": _uptime_linux,\n \"minix3\": _uptime_minix,\n \"sunos5\": _uptime_solaris,\n \"syllable\": _uptime_syllable,\n \"win32\": _uptime_windows,\n \"wince\": _uptime_windows,\n }\n\n\ndef uptime():\n \"\"\"Returns uptime in seconds if even remotely possible, or None if not.\"\"\"\n if _BOOTTIME is not None:\n return time.time() - _BOOTTIME\n up = _UPTIME_FUNCS.get(sys.platform, _uptime_bsd)()\n if up is None:\n up = (\n _uptime_bsd()\n or _uptime_plan9()\n or _uptime_linux()\n or _uptime_windows()\n or _uptime_solaris()\n or _uptime_beos()\n or _uptime_amiga()\n or _uptime_syllable()\n or _uptime_osx()\n )\n return up\n\n\ndef boottime():\n \"\"\"Returns boot time if remotely possible, or None if not.\"\"\"\n global _BOOTTIME\n if _BOOTTIME is None:\n up = uptime()\n if up is None:\n return None\n _BOOTTIME = time.time() - up\n return _BOOTTIME\n" }, { "path": "xonsh/xoreutils/util.py", "content": "\"\"\"Assorted utilities for xonsh core utils.\"\"\"\n\n\ndef arg_handler(args, out, short, key, val, long=None):\n \"\"\"A simple argument handler for xoreutils.\"\"\"\n if short in args:\n args.remove(short)\n if isinstance(key, (list, tuple)):\n for k in key:\n out[k] = val\n else:\n out[key] = val\n if long is not None and long in args:\n args.remove(long)\n if isinstance(key, (list, tuple)):\n for k in key:\n out[k] = val\n else:\n out[key] = val\n" }, { "path": "xonsh/xoreutils/which.py", "content": "\"\"\"Implements the which xoreutil.\"\"\"\nimport os\nimport argparse\nimport builtins\nimport functools\n\nfrom xonsh.xoreutils import _which\nimport xonsh.platform as xp\nimport xonsh.proc as xproc\n\n\n@functools.lru_cache()\ndef _which_create_parser():\n desc = \"Parses arguments to which wrapper\"\n parser = argparse.ArgumentParser(\"which\", description=desc)\n parser.add_argument(\n \"args\", type=str, nargs=\"+\", help=\"The executables or aliases to search for\"\n )\n parser.add_argument(\n \"-a\",\n \"--all\",\n action=\"store_true\",\n dest=\"all\",\n help=\"Show all matches in globals, xonsh.aliases, $PATH\",\n )\n parser.add_argument(\n \"-s\",\n \"--skip-alias\",\n action=\"store_true\",\n help=\"Do not search inxonsh.aliases\",\n dest=\"skip\",\n )\n parser.add_argument(\n \"-V\",\n \"--version\",\n action=\"version\",\n version=\"{}\".format(_which.__version__),\n help=\"Display the version of the python which module \" \"used by xonsh\",\n )\n parser.add_argument(\n \"-v\",\n \"--verbose\",\n action=\"store_true\",\n dest=\"verbose\",\n help=\"Print out how matches were located and show \" \"near misses on stderr\",\n )\n parser.add_argument(\n \"-p\",\n \"--plain\",\n action=\"store_true\",\n dest=\"plain\",\n help=\"Do not display alias expansions or location of \"\n \"where binaries are found. This is the \"\n \"default behavior, but the option can be used to \"\n \"override the --verbose option\",\n )\n parser.add_argument(\"--very-small-rocks\", action=AWitchAWitch)\n if xp.ON_WINDOWS:\n parser.add_argument(\n \"-e\",\n \"--exts\",\n nargs=\"*\",\n type=str,\n help=\"Specify a list of extensions to use instead \"\n \"of the standard list for this system. This can \"\n \"effectively be used as an optimization to, for \"\n 'example, avoid stat\\'s of \"foo.vbs\" when '\n 'searching for \"foo\" and you know it is not a '\n 'VisualBasic script but \".vbs\" is on PATHEXT. '\n \"This option is only supported on Windows\",\n dest=\"exts\",\n )\n return parser\n\n\ndef print_global_object(arg, stdout):\n \"\"\"Print the object.\"\"\"\n obj = builtins.__xonsh__.ctx.get(arg)\n print(\"global object of {}\".format(type(obj)), file=stdout)\n\n\ndef print_path(abs_name, from_where, stdout, verbose=False, captured=False):\n \"\"\"Print the name and path of the command.\"\"\"\n if xp.ON_WINDOWS:\n # Use list dir to get correct case for the filename\n # i.e. windows is case insensitive but case preserving\n p, f = os.path.split(abs_name)\n f = next(s.name for s in xp.scandir(p) if s.name.lower() == f.lower())\n abs_name = os.path.join(p, f)\n if builtins.__xonsh__.env.get(\"FORCE_POSIX_PATHS\", False):\n abs_name.replace(os.sep, os.altsep)\n if verbose:\n print(\"{} ({})\".format(abs_name, from_where), file=stdout)\n else:\n end = \"\" if captured else \"\\n\"\n print(abs_name, end=end, file=stdout)\n\n\ndef print_alias(arg, stdout, verbose=False):\n \"\"\"Print the alias.\"\"\"\n if not verbose:\n if not callable(builtins.aliases[arg]):\n print(\" \".join(builtins.aliases[arg]), file=stdout)\n else:\n print(arg, file=stdout)\n else:\n print(\"aliases['{}'] = {}\".format(arg, builtins.aliases[arg]), file=stdout)\n if callable(builtins.aliases[arg]):\n builtins.__xonsh__.superhelp(builtins.aliases[arg])\n\n\ndef which(args, stdin=None, stdout=None, stderr=None, spec=None):\n \"\"\"\n Checks if each arguments is a xonsh aliases, then if it's an executable,\n then finally return an error code equal to the number of misses.\n If '-a' flag is passed, run both to return both `xonsh` match and\n `which` match.\n \"\"\"\n parser = _which_create_parser()\n if len(args) == 0:\n parser.print_usage(file=stderr)\n return -1\n\n pargs = parser.parse_args(args)\n verbose = pargs.verbose or pargs.all\n if spec is not None:\n captured = spec.captured in xproc.STDOUT_CAPTURE_KINDS\n else:\n captured = False\n if pargs.plain:\n verbose = False\n if xp.ON_WINDOWS:\n if pargs.exts:\n exts = pargs.exts\n else:\n exts = builtins.__xonsh__.env[\"PATHEXT\"]\n else:\n exts = None\n failures = []\n for arg in pargs.args:\n nmatches = 0\n if pargs.all and arg in builtins.__xonsh__.ctx:\n print_global_object(arg, stdout)\n nmatches += 1\n if arg in builtins.aliases and not pargs.skip:\n print_alias(arg, stdout, verbose)\n nmatches += 1\n if not pargs.all:\n continue\n # which.whichgen gives the nicest 'verbose' output if PATH is taken\n # from os.environ so we temporarily override it with\n # __xosnh_env__['PATH']\n original_os_path = xp.os_environ[\"PATH\"]\n xp.os_environ[\"PATH\"] = builtins.__xonsh__.env.detype()[\"PATH\"]\n matches = _which.whichgen(arg, exts=exts, verbose=verbose)\n for abs_name, from_where in matches:\n print_path(abs_name, from_where, stdout, verbose, captured)\n nmatches += 1\n if not pargs.all:\n break\n xp.os_environ[\"PATH\"] = original_os_path\n if not nmatches:\n failures.append(arg)\n if len(failures) == 0:\n return 0\n else:\n print(\"{} not in \".format(\", \".join(failures)), file=stderr, end=\"\")\n if pargs.all:\n print(\"globals or \", file=stderr, end=\"\")\n print(\"$PATH\", file=stderr, end=\"\")\n if not pargs.skip:\n print(\" or xonsh.builtins.aliases\", file=stderr, end=\"\")\n print(\"\", file=stderr, end=\"\\n\")\n return len(failures)\n\n\nclass AWitchAWitch(argparse.Action):\n \"\"\"The Ducstring, the mother of all ducs.\"\"\"\n\n SUPPRESS = \"==SUPPRESS==\"\n\n def __init__(\n self, option_strings, version=None, dest=SUPPRESS, default=SUPPRESS, **kwargs\n ):\n super().__init__(\n option_strings=option_strings, dest=dest, default=default, nargs=0, **kwargs\n )\n\n def __call__(self, parser, namespace, values, option_string=None):\n import webbrowser\n\n webbrowser.open(\"https://github.com/xonsh/xonsh/commit/f49b400\")\n parser.exit()\n" }, { "path": "xonsh/xoreutils/yes.py", "content": "\"\"\"An implementation of yes for xonsh.\"\"\"\n\n\ndef yes(args, stdin, stdout, stderr):\n \"\"\"A yes command.\"\"\"\n if \"--help\" in args:\n print(YES_HELP, file=stdout)\n return 0\n\n to_print = [\"y\"] if len(args) == 0 else [str(i) for i in args]\n\n while True:\n print(*to_print, file=stdout)\n\n return 0\n\n\nYES_HELP = \"\"\"Usage: yes [STRING]...\n or: yes OPTION\nRepeatedly output a line with all specified STRING(s), or 'y'.\n\n --help display this help and exit\n\nThis version of yes was written in Python for the xonsh project: http://xon.sh\nBased on yes from GNU coreutils: http://www.gnu.org/software/coreutils/\"\"\"\n" } ]