Repository: donnemartin/gitsome Branch: master Commit: d7c57abc7cb6 Files: 357 Total size: 2.3 MB Directory structure: gitextract_f42mvrxs/ ├── .gitignore ├── .gitsomeconfig ├── .gitsomeconfigurl ├── .travis.yml ├── CHANGELOG.md ├── CHANGELOG.rst ├── CHECKLIST.md ├── COMMANDS.md ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE.txt ├── PRIVACY.md ├── README.md ├── TOS.md ├── appveyor.yml ├── codecov.yml ├── gitsome/ │ ├── __init__.py │ ├── compat.py │ ├── completer.py │ ├── completions.py │ ├── completions_git.py │ ├── config.py │ ├── formatter.py │ ├── github.py │ ├── githubcli.py │ ├── main.py │ ├── main_cli.py │ ├── rss_feed.py │ ├── table.py │ ├── utils.py │ ├── view_entry.py │ └── web_viewer.py ├── requirements-dev.txt ├── scripts/ │ ├── aws_complete.sh │ ├── create_changelog.sh │ ├── create_readme_rst.sh │ ├── gh_complete.sh │ ├── gitsome │ ├── gitsome.bat │ ├── run_code_checks.sh │ ├── set_changelog_as_readme.sh │ ├── set_changelog_as_readme_undo.sh │ ├── update_docs.sh │ ├── upload_pypi.sh │ ├── xonsh │ └── xonsh.bat ├── setup.py ├── tests/ │ ├── __init__.py │ ├── compat.py │ ├── data/ │ │ ├── __init__.py │ │ ├── email.py │ │ ├── emoji.py │ │ ├── events.py │ │ ├── gitignores.py │ │ ├── issue.py │ │ ├── license.py │ │ ├── markdown.py │ │ ├── thread.py │ │ ├── trends.p │ │ ├── trends.py │ │ ├── user.py │ │ ├── user_feed.p │ │ ├── user_feed.py │ │ └── user_public_feed.p │ ├── mock_feed_parser.py │ ├── mock_github_api.py │ ├── mock_pretty_date_time.py │ ├── run_tests.py │ ├── test_completer.py │ ├── test_config.py │ ├── test_github.py │ ├── test_github_cli.py │ └── test_web_viewer.py ├── tox.ini └── xonsh/ ├── __init__.py ├── __main__.py ├── aliases.py ├── ansi_colors.py ├── ast.py ├── base_shell.py ├── built_ins.py ├── codecache.py ├── color_tools.py ├── commands_cache.py ├── completer.py ├── completers/ │ ├── __init__.py │ ├── _aliases.py │ ├── base.py │ ├── bash.py │ ├── bash_completion.py │ ├── commands.py │ ├── completer.py │ ├── dirs.py │ ├── init.py │ ├── man.py │ ├── path.py │ ├── pip.py │ ├── python.py │ ├── tools.py │ └── xompletions.py ├── contexts.py ├── data/ │ └── data.txt ├── diff_history.py ├── dirstack.py ├── dumb_shell.py ├── environ.py ├── events.py ├── execer.py ├── foreign_shells.py ├── fs.py ├── history/ │ ├── __init__.py │ ├── base.py │ ├── dummy.py │ ├── json.py │ ├── main.py │ └── sqlite.py ├── imphooks.py ├── inspectors.py ├── jobs.py ├── jsonutils.py ├── jupyter_kernel.py ├── jupyter_shell.py ├── lazyasd.py ├── lazyimps.py ├── lazyjson.py ├── lexer.py ├── macutils.py ├── main.py ├── openpy.py ├── parser.py ├── parsers/ │ ├── __init__.py │ ├── base.py │ ├── context_check.py │ ├── v34.py │ ├── v35.py │ └── v36.py ├── platform.py ├── ply/ │ ├── .gitignore │ ├── .travis.yml │ ├── CHANGES │ ├── CONTRIBUTING.md │ ├── Makefile │ ├── README.md │ ├── __init__.py │ ├── doc/ │ │ ├── internal.html │ │ ├── makedoc.py │ │ └── ply.html │ ├── example/ │ │ ├── BASIC/ │ │ │ ├── README │ │ │ ├── basic.py │ │ │ ├── basiclex.py │ │ │ ├── basiclog.py │ │ │ ├── basinterp.py │ │ │ ├── basparse.py │ │ │ ├── dim.bas │ │ │ ├── func.bas │ │ │ ├── gcd.bas │ │ │ ├── gosub.bas │ │ │ ├── hello.bas │ │ │ ├── linear.bas │ │ │ ├── maxsin.bas │ │ │ ├── powers.bas │ │ │ ├── rand.bas │ │ │ ├── sales.bas │ │ │ ├── sears.bas │ │ │ ├── sqrt1.bas │ │ │ └── sqrt2.bas │ │ ├── GardenSnake/ │ │ │ ├── GardenSnake.py │ │ │ └── README │ │ ├── README │ │ ├── ansic/ │ │ │ ├── README │ │ │ ├── clex.py │ │ │ └── cparse.py │ │ ├── calc/ │ │ │ └── calc.py │ │ ├── calcdebug/ │ │ │ └── calc.py │ │ ├── calceof/ │ │ │ └── calc.py │ │ ├── classcalc/ │ │ │ └── calc.py │ │ ├── cleanup.sh │ │ ├── closurecalc/ │ │ │ └── calc.py │ │ ├── hedit/ │ │ │ └── hedit.py │ │ ├── newclasscalc/ │ │ │ └── calc.py │ │ ├── optcalc/ │ │ │ ├── README │ │ │ └── calc.py │ │ ├── unicalc/ │ │ │ └── calc.py │ │ └── yply/ │ │ ├── README │ │ ├── ylex.py │ │ ├── yparse.py │ │ └── yply.py │ ├── ply/ │ │ ├── __init__.py │ │ ├── cpp.py │ │ ├── ctokens.py │ │ ├── lex.py │ │ ├── yacc.py │ │ └── ygen.py │ ├── setup.md │ └── test/ │ ├── README │ ├── calclex.py │ ├── cleanup.sh │ ├── lex_closure.py │ ├── lex_doc1.py │ ├── lex_dup1.py │ ├── lex_dup2.py │ ├── lex_dup3.py │ ├── lex_empty.py │ ├── lex_error1.py │ ├── lex_error2.py │ ├── lex_error3.py │ ├── lex_error4.py │ ├── lex_hedit.py │ ├── lex_ignore.py │ ├── lex_ignore2.py │ ├── lex_literal1.py │ ├── lex_literal2.py │ ├── lex_literal3.py │ ├── lex_many_tokens.py │ ├── lex_module.py │ ├── lex_module_import.py │ ├── lex_object.py │ ├── lex_opt_alias.py │ ├── lex_optimize.py │ ├── lex_optimize2.py │ ├── lex_optimize3.py │ ├── lex_optimize4.py │ ├── lex_re1.py │ ├── lex_re2.py │ ├── lex_re3.py │ ├── lex_rule1.py │ ├── lex_rule2.py │ ├── lex_rule3.py │ ├── lex_state1.py │ ├── lex_state2.py │ ├── lex_state3.py │ ├── lex_state4.py │ ├── lex_state5.py │ ├── lex_state_noerror.py │ ├── lex_state_norule.py │ ├── lex_state_try.py │ ├── lex_token1.py │ ├── lex_token2.py │ ├── lex_token3.py │ ├── lex_token4.py │ ├── lex_token5.py │ ├── lex_token_dup.py │ ├── pkg_test1/ │ │ ├── __init__.py │ │ └── parsing/ │ │ ├── __init__.py │ │ ├── calclex.py │ │ └── calcparse.py │ ├── pkg_test2/ │ │ ├── __init__.py │ │ └── parsing/ │ │ ├── __init__.py │ │ ├── calclex.py │ │ └── calcparse.py │ ├── pkg_test3/ │ │ ├── __init__.py │ │ ├── generated/ │ │ │ └── __init__.py │ │ └── parsing/ │ │ ├── __init__.py │ │ ├── calclex.py │ │ └── calcparse.py │ ├── pkg_test4/ │ │ ├── __init__.py │ │ └── parsing/ │ │ ├── __init__.py │ │ ├── calclex.py │ │ └── calcparse.py │ ├── pkg_test5/ │ │ ├── __init__.py │ │ └── parsing/ │ │ ├── __init__.py │ │ ├── calclex.py │ │ └── calcparse.py │ ├── pkg_test6/ │ │ ├── __init__.py │ │ └── parsing/ │ │ ├── __init__.py │ │ ├── calclex.py │ │ ├── calcparse.py │ │ ├── expression.py │ │ └── statement.py │ ├── test_cpp_nonascii.c │ ├── testcpp.py │ ├── testlex.py │ ├── testyacc.py │ ├── yacc_badargs.py │ ├── yacc_badid.py │ ├── yacc_badprec.py │ ├── yacc_badprec2.py │ ├── yacc_badprec3.py │ ├── yacc_badrule.py │ ├── yacc_badtok.py │ ├── yacc_dup.py │ ├── yacc_error1.py │ ├── yacc_error2.py │ ├── yacc_error3.py │ ├── yacc_error4.py │ ├── yacc_error5.py │ ├── yacc_error6.py │ ├── yacc_error7.py │ ├── yacc_inf.py │ ├── yacc_literal.py │ ├── yacc_misplaced.py │ ├── yacc_missing1.py │ ├── yacc_nested.py │ ├── yacc_nodoc.py │ ├── yacc_noerror.py │ ├── yacc_nop.py │ ├── yacc_notfunc.py │ ├── yacc_notok.py │ ├── yacc_prec1.py │ ├── yacc_rr.py │ ├── yacc_rr_unused.py │ ├── yacc_simple.py │ ├── yacc_sr.py │ ├── yacc_term1.py │ ├── yacc_unicode_literals.py │ ├── yacc_unused.py │ ├── yacc_unused_rule.py │ ├── yacc_uprec.py │ └── yacc_uprec2.py ├── pretty.py ├── proc.py ├── prompt/ │ ├── __init__.py │ ├── base.py │ ├── cwd.py │ ├── env.py │ ├── gitstatus.py │ ├── job.py │ └── vc.py ├── ptk/ │ ├── __init__.py │ ├── completer.py │ ├── history.py │ ├── key_bindings.py │ ├── shell.py │ └── shortcuts.py ├── ptk2/ │ ├── __init__.py │ ├── completer.py │ ├── history.py │ ├── key_bindings.py │ └── shell.py ├── pyghooks.py ├── pygments_cache.py ├── pytest_plugin.py ├── readline_shell.py ├── replay.py ├── shell.py ├── style_tools.py ├── timings.py ├── tokenize.py ├── tools.py ├── tracer.py ├── winutils.py ├── wizard.py ├── xonfig.py ├── xonshrc ├── xontribs.json ├── xontribs.py └── xoreutils/ ├── __init__.py ├── _which.py ├── cat.py ├── echo.py ├── pwd.py ├── tee.py ├── tty.py ├── uptime.py ├── util.py ├── which.py └── yes.py ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitignore ================================================ # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so # Distribution / packaging .Python env/ build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ *.egg-info/ .installed.cfg *.egg # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *,cover .hypothesis/ # Translations *.mo *.pot # Django stuff: *.log # Sphinx documentation docs/_build/ # PyBuilder target/ #Ipython Notebook .ipynb_checkpoints # temporary files from vim and emacs *~ *# .#* *.swp *.swo # Virtualenv pip-selfcheck.json bin/ include/ # Mac .DS_Store * Misc scratch/ *.egg-info/ *.pyc *.out *.xcf recipe/ .binstar.yml binstar.yml .landscape.yaml .cache/ build/ dist/ gitsome.egg-info/ docs/_build/ .tox/ lexer_table.py parser_table.py parser_test_table.py gitsome/lexer_table.py gitsome/parser_table.py tests/lexer_table.py tests/parser_table.py tests/lexer_test_table.py tests/parser_test_table.py ================================================ FILE: .gitsomeconfig ================================================ [github] user_login = None enable_avatar = True verify_ssl = True clr_primary = None clr_secondary = green clr_tertiary = cyan clr_quaternary = yellow clr_bold = cyan clr_code = red clr_error = red clr_header = yellow clr_link = green clr_list = cyan clr_message = None clr_num_comments = green clr_num_points = green clr_tag = cyan clr_time = yellow clr_title = None clr_tooltip = None clr_user = cyan clr_view_link = magenta clr_view_index = magenta ================================================ FILE: .gitsomeconfigurl ================================================ [url] url_list = ['https://github.com/octocat/spoon-knife'] ================================================ FILE: .travis.yml ================================================ language: python matrix: include: - python: 3.5 env: TOXENV=py35 - python: 3.6 env: TOXENV=py36 - python: 3.7 dist: xenial # required for Python >= 3.7 env: TOXENV=py37 install: - travis_retry pip install codecov tox script: - tox after_success: - codecov ================================================ FILE: CHANGELOG.md ================================================ ![](http://i.imgur.com/0SXZ90y.gif) gitsome ======= [![Build Status](https://travis-ci.org/donnemartin/gitsome.svg?branch=master)](https://travis-ci.org/donnemartin/gitsome) [![Codecov](https://img.shields.io/codecov/c/github/donnemartin/gitsome.svg)](https://codecov.io/github/donnemartin/gitsome) [![PyPI version](https://badge.fury.io/py/gitsome.svg)](http://badge.fury.io/py/gitsome) [![PyPI](https://img.shields.io/pypi/pyversions/gitsome.svg)](https://pypi.python.org/pypi/gitsome/) [![License](https://img.shields.io/:license-apache-blue.svg)](http://www.apache.org/licenses/LICENSE-2.0.html) To view the latest `README`, `docs`, and `code`, visit the GitHub repo: https://github.com/donnemartin/gitsome To submit bugs or feature requests, visit the issue tracker: https://github.com/donnemartin/gitsome/issues Changelog ========= 0.8.0 (2019-04-07) ------------------ This version adds support for Python 3.7. ### Updates * [#160](https://github.com/donnemartin/gitsome/pull/160) - Add Python 3.7 support. Fixes [#152](https://github.com/donnemartin/gitsome/pull/152), [#144](https://github.com/donnemartin/gitsome/pull/144), [#126](https://github.com/donnemartin/gitsome/pull/126), [#105](https://github.com/donnemartin/gitsome/pull/105) and several other related bugs. * [#147](https://github.com/donnemartin/gitsome/pull/148) - Gracefully ignore missing avatar image, by [kBite](https://github.com/kBite). * [#142](https://github.com/donnemartin/gitsome/pull/142) - Update release checklist. * [#134](https://github.com/donnemartin/gitsome/pull/134) - Update GitHub integrations link. * [#120](https://github.com/donnemartin/gitsome/pull/120) - Add license disclaimer. ### Bug Fixes * [#151](https://github.com/donnemartin/gitsome/pull/151) - Fix gh command typos in docs, by [cyblue9](https://github.com/cyblue9). * [#137](https://github.com/donnemartin/gitsome/pull/137) - Fix Running as a Docker Container anchor in README, by [kamontat](https://github.com/kamontat). * [#129](https://github.com/donnemartin/gitsome/pull/129) - Fix trending command to handle empty summaries, by [emres](https://github.com/emres). * [#123](https://github.com/donnemartin/gitsome/pull/123) - Remove buggy codecov badge. * [#117](https://github.com/donnemartin/gitsome/pull/117) - Fix 0.7.0 CHANGELOG date, by [dbaio](https://github.com/dbaio). 0.7.0 (2017-03-26) ------------------ ### Features * [#99](https://github.com/donnemartin/gitsome/pull/99) - Add Dockerfile to run gitsome in a Docker container, by [l0rd](https://github.com/l0rd) and [larson004](https://github.com/larson004). ### Bug Fixes * [#67](https://github.com/donnemartin/gitsome/pull/67) - Fix `gh_issues` typo in the `README`, by [srisankethu](https://github.com/srisankethu). * [#69](https://github.com/donnemartin/gitsome/pull/69) - Fix `--issue_filter` typo for `gh_issues` command in `COMMANDS.md`. * [#80](https://github.com/donnemartin/gitsome/pull/80) - Fix path for auto completions in `README`. * [#92](https://github.com/donnemartin/gitsome/pull/92) - Fix viewing HTML contents in the terminal for GitHub Enterprise users, by [dongweiming](https://github.com/dongweiming). * [#97](https://github.com/donnemartin/gitsome/pull/97) - Fix error hint from `gh gitignores` to `gh gitignore-templates`, by [zYeoman](https://github.com/zYeoman). * [#116](https://github.com/donnemartin/gitsome/pull/116) - Fix gh trending command resulting in an error. ### Updates * [#58](https://github.com/donnemartin/gitsome/pull/58) - Tweak `README` intro, add logo. * [#74](https://github.com/donnemartin/gitsome/pull/74) - Add link to official GitHub integration page in `README`. * [#79](https://github.com/donnemartin/gitsome/pull/79) - Only store password in config for GitHub Enterprise (due to Enterprise limitations), by [nttibbetts](https://github.com/nttibbetts). * [#86](https://github.com/donnemartin/gitsome/pull/86) - Update dependency info for `uritemplate`. * [#89](https://github.com/donnemartin/gitsome/pull/89) - Fix a bug listing info on repos without a desc field, by [SanketDG](https://github.com/SanketDG). * [#98](https://github.com/donnemartin/gitsome/pull/98) - Prefer GitHub Enterprise token before password. * [#104](https://github.com/donnemartin/gitsome/pull/104) - Update install instructions to use pip3. * [#111](https://github.com/donnemartin/gitsome/pull/111) - Add note about current Python 3.6 incompatibility. * [#115](https://github.com/donnemartin/gitsome/pull/115) - Set current Python support to 3.4 and 3.5. 0.6.0 (2016-05-29) ------------------ ### Features * [#3](https://github.com/donnemartin/gitsome/issues/3) - Add GitHub Enterprise support. * [#33](https://github.com/donnemartin/gitsome/issues/33) - Revamp the info shown with the `gh feed` command. ### Bug Fixes * [#30](https://github.com/donnemartin/gitsome/issues/30) - Fix a typo in the `pip3` install instructions. * [#39](https://github.com/donnemartin/gitsome/issues/39) - Fix `gh feed` `-pr/--private` flag in docs. * [#40](https://github.com/donnemartin/gitsome/issues/40) - Fix `create-issue` `NoneType` error if no `-b/--body` is specified. * [#46](https://github.com/donnemartin/gitsome/issues/46) - Fix `gh view` with the -b/--browser option only working for repos, not for issues or PRs. * [#48](https://github.com/donnemartin/gitsome/issues/48) - Fix `create-repo` `NoneType` error if no `-d/--description` is specified. * [#54](https://github.com/donnemartin/gitsome/pull/54) - Update to `prompt-toolkit` 1.0.0, which includes performance improvements (especially noticeable on Windows) and bug fixes. * Fix `Config` docstrings. ### Updates * [#26](https://github.com/donnemartin/gitsome/issues/26), [#32](https://github.com/donnemartin/gitsome/issues/32) - Add copyright notices for third party libraries. * [#44](https://github.com/donnemartin/gitsome/pull/44), [#53](https://github.com/donnemartin/gitsome/pull/53) - Update packaging dependencies based on semantic versioning. * Tweak `README` intro. 0.5.0 (2016-05-15) ------------------ ### Features * [#12](https://github.com/donnemartin/gitsome/issues/12) - Allow 2FA-enabled users to log in with a password + 2FA code. Previously 2FA-enabled users could only log in with a [personal access token](https://github.com/settings/tokens). Also includes an update of login prompts to improve clarity. ### Bug Fixes * [#16](https://github.com/donnemartin/gitsome/pull/16), [#28](https://github.com/donnemartin/gitsome/pull/28) - Fix typos in README. * [#18](https://github.com/donnemartin/gitsome/pull/18) - Fix dev install instructions in README. * [#24](https://github.com/donnemartin/gitsome/pull/24) - Fix style guide broken link in CONTRIBUTING. ### Updates * [#1](https://github.com/donnemartin/gitsome/issues/1) - Add Codecov coverage testing status to README. * [#2](https://github.com/donnemartin/gitsome/issues/2) - Add note about enabling Zsh completions to README. * [#4](https://github.com/donnemartin/gitsome/issues/4) - Add note about using `pip3` to README. * [#5](https://github.com/donnemartin/gitsome/issues/5) - Decrease speed of README gif. * [#6](https://github.com/donnemartin/gitsome/pull/6) - Update url for `click`. * [#20](https://github.com/donnemartin/gitsome/issues/20) - Add note about enabling more completions to README. * [#21](https://github.com/donnemartin/gitsome/issues/21) - Bump up `prompt-toolkit` version from `0.51` to `0.52`. * [#26](https://github.com/donnemartin/gitsome/issues/26) - Add `xonsh` copyright notice to LICENSE. * [#32](https://github.com/donnemartin/gitsome/pull/32) - Add `github3.py`, `html2text`, and `img2txt` copyright notices to LICENSE. * Update installation instructions in README. * Update color customization discussion in README. 0.4.0 (2016-05-09) ------------------ * Initial release. ================================================ FILE: CHANGELOG.rst ================================================ .. figure:: http://i.imgur.com/0SXZ90y.gif :alt: gitsome ======= |Build Status| |Codecov| |PyPI version| |PyPI| |License| To view the latest ``README``, ``docs``, and ``code``, visit the GitHub repo: https://github.com/donnemartin/gitsome To submit bugs or feature requests, visit the issue tracker: https://github.com/donnemartin/gitsome/issues Changelog ========= 0.8.0 (2019-04-07) ------------------ This version adds support for Python 3.7. Updates ~~~~~~~ - `#160 `__ - Add Python 3.7 support. Fixes `#152 `__, `#144 `__, `#126 `__, `#105 `__ and several other related bugs. - `#147 `__ - Gracefully ignore missing avatar image, by `kBite `__. - `#142 `__ - Update release checklist. - `#134 `__ - Update GitHub integrations link. - `#120 `__ - Add license disclaimer. Bug Fixes ~~~~~~~~~ - `#151 `__ - Fix gh command typos in docs, by `cyblue9 `__. - `#137 `__ - Fix Running as a Docker Container anchor in README, by `kamontat `__. - `#129 `__ - Fix trending command to handle empty summaries, by `emres `__. - `#123 `__ - Remove buggy codecov badge. - `#117 `__ - Fix 0.7.0 CHANGELOG date, by `dbaio `__. 0.7.0 (2017-03-26) ------------------ Features ~~~~~~~~ - `#99 `__ - Add Dockerfile to run gitsome in a Docker container, by `l0rd `__ and `larson004 `__. Bug Fixes ~~~~~~~~~ - `#67 `__ - Fix ``gh_issues`` typo in the ``README``, by `srisankethu `__. - `#69 `__ - Fix ``--issue_filter`` typo for ``gh_issues`` command in ``COMMANDS.md``. - `#80 `__ - Fix path for auto completions in ``README``. - `#92 `__ - Fix viewing HTML contents in the terminal for GitHub Enterprise users, by `dongweiming `__. - `#97 `__ - Fix error hint from ``gh gitignores`` to ``gh gitignore-templates``, by `zYeoman `__. - `#116 `__ - Fix gh trending command resulting in an error. Updates ~~~~~~~ - `#58 `__ - Tweak ``README`` intro, add logo. - `#74 `__ - Add link to official GitHub integration page in ``README``. - `#79 `__ - Only store password in config for GitHub Enterprise (due to Enterprise limitations), by `nttibbetts `__. - `#86 `__ - Update dependency info for ``uritemplate``. - `#89 `__ - Fix a bug listing info on repos without a desc field, by `SanketDG `__. - `#98 `__ - Prefer GitHub Enterprise token before password. - `#104 `__ - Update install instructions to use pip3. - `#111 `__ - Add note about current Python 3.6 incompatibility. - `#115 `__ - Set current Python support to 3.4 and 3.5. 0.6.0 (2016-05-29) ------------------ Features ~~~~~~~~ - `#3 `__ - Add GitHub Enterprise support. - `#33 `__ - Revamp the info shown with the ``gh feed`` command. Bug Fixes ~~~~~~~~~ - `#30 `__ - Fix a typo in the ``pip3`` install instructions. - `#39 `__ - Fix ``gh feed`` ``-pr/--private`` flag in docs. - `#40 `__ - Fix ``create-issue`` ``NoneType`` error if no ``-b/--body`` is specified. - `#46 `__ - Fix ``gh view`` with the -b/--browser option only working for repos, not for issues or PRs. - `#48 `__ - Fix ``create-repo`` ``NoneType`` error if no ``-d/--description`` is specified. - `#54 `__ - Update to ``prompt-toolkit`` 1.0.0, which includes performance improvements (especially noticeable on Windows) and bug fixes. - Fix ``Config`` docstrings. Updates ~~~~~~~ - `#26 `__, `#32 `__ - Add copyright notices for third party libraries. - `#44 `__, `#53 `__ - Update packaging dependencies based on semantic versioning. - Tweak ``README`` intro. 0.5.0 (2016-05-15) ------------------ Features ~~~~~~~~ - `#12 `__ - Allow 2FA-enabled users to log in with a password + 2FA code. Previously 2FA-enabled users could only log in with a `personal access token `__. Also includes an update of login prompts to improve clarity. Bug Fixes ~~~~~~~~~ - `#16 `__, `#28 `__ - Fix typos in README. - `#18 `__ - Fix dev install instructions in README. - `#24 `__ - Fix style guide broken link in CONTRIBUTING. Updates ~~~~~~~ - `#1 `__ - Add Codecov coverage testing status to README. - `#2 `__ - Add note about enabling Zsh completions to README. - `#4 `__ - Add note about using ``pip3`` to README. - `#5 `__ - Decrease speed of README gif. - `#6 `__ - Update url for ``click``. - `#20 `__ - Add note about enabling more completions to README. - `#21 `__ - Bump up ``prompt-toolkit`` version from ``0.51`` to ``0.52``. - `#26 `__ - Add ``xonsh`` copyright notice to LICENSE. - `#32 `__ - Add ``github3.py``, ``html2text``, and ``img2txt`` copyright notices to LICENSE. - Update installation instructions in README. - Update color customization discussion in README. 0.4.0 (2016-05-09) ------------------ - Initial release. .. |Build Status| image:: https://travis-ci.org/donnemartin/gitsome.svg?branch=master :target: https://travis-ci.org/donnemartin/gitsome .. |Codecov| image:: https://img.shields.io/codecov/c/github/donnemartin/gitsome.svg :target: https://codecov.io/github/donnemartin/gitsome .. |PyPI version| image:: https://badge.fury.io/py/gitsome.svg :target: http://badge.fury.io/py/gitsome .. |PyPI| image:: https://img.shields.io/pypi/pyversions/gitsome.svg :target: https://pypi.python.org/pypi/gitsome/ .. |License| image:: https://img.shields.io/:license-apache-blue.svg :target: http://www.apache.org/licenses/LICENSE-2.0.html ================================================ FILE: CHECKLIST.md ================================================ Release Checklist ================= A. Install in a new venv and run unit tests Note, you can't seem to script the virtualenv calls, see: https://bitbucket.org/dhellmann/virtualenvwrapper/issues/219/cant-deactivate-active-virtualenv-from $ deactivate $ rmvirtualenv gitsome $ mkvirtualenv gitsome $ pip install -e . $ pip install -r requirements-dev.txt $ rm -rf .tox && tox B. Run code checks $ scripts/run_code_checks.sh C. Run manual [smoke tests](#smoke-tests) on Mac, Ubuntu, Windows D. Update and review `README.rst` and `Sphinx` docs, then check gitsome/docs/build/html/index.html $ scripts/update_docs.sh E. Push changes F. Review Travis, Codecov, and Gemnasium G. Start a new release branch $ git flow release start x.y.z H. Increment the version number in `gitsome/__init__.py` I. Update and review `CHANGELOG` $ scripts/create_changelog.sh J. Commit the changes K. Finish the release branch $ git flow release finish 'x.y.z' L. Input a tag $ vx.y.z M. Push tagged release to develop and master $ git checkout master $ git push Might need to recreate develop branch. N. Set CHANGELOG.rst as `README.rst` $ scripts/set_changelog_as_readme.sh O. Register package with PyPi $ python setup.py register -r pypi P. Upload to PyPi $ python setup.py sdist upload -r pypi Q. Upload Sphinx docs to PyPi $ python setup.py upload_sphinx R. Review newly released package from PyPi S. Release on GitHub: https://github.com/donnemartin/gitsome/tags 1. Click "Add release notes" for latest release 2. Copy release notes from `CHANGELOG.md` 3. Click "Publish release" T. Install in a new venv and run manual [smoke tests](#smoke-tests) on Mac, Ubuntu, Windows ## Smoke Tests Run the following on Python 3.4: * Craete a new `virtualenv` * Pip install `gitsome` into new `virtualenv` * Run `gitsome` * Run targeted tests based on recent code changes ================================================ FILE: COMMANDS.md ================================================ # Commands ## GitHub Integration Commands Reference Check out the handy [autocompleter with interactive help](https://github.com/donnemartin/gitsome/blob/master/README.md#git-and-github-autocompleter-with-interactive-help) to guide you through each command. ### gh configure Configure `gitsome`. Attempts to authenticate the user and to set up the user's news feed. If `gitsome` has not yet been configured, calling a `gh` command that requires authentication will automatically invoke the `configure` command. Usage/Example(s): $ gh configure For GitHub Enterprise users, run with the `-e/--enterprise` flag: $ gh configure -e #### Authentication To properly integrate with GitHub, you will be asked to enter a user name and either a password or a [personal access token](https://github.com/settings/tokens). If you use two-factor authentication, you will also need to enter your 2FA code, or you can log in with a personal access token. Visit the following page to generate a token: [https://github.com/settings/tokens](https://github.com/settings/tokens) `gitsome` will need the 'repo' and 'user' permissions. ![Imgur](http://i.imgur.com/1C7gBHz.png) #### GitHub Enterprise GitHub Enterprise users will be asked to enter the GitHub Enterprise url and whether they want to verify SSL certificates. #### Authentication Source Code Curious what's going on behind the scenes with authentication? Check out the [authentication source code](https://github.com/donnemartin/gitsome/blob/master/gitsome/config.py#L177-L328). #### User Feed `gitsome` will need your news feed url to run the `gh feed` command with no arguments. ![Imgur](http://i.imgur.com/2LWcyS6.png) To integrate `gitsome` with your news feed, visit the following url while logged into GitHub: [https://github.com](https://github.com) You will be asked to enter the url found when clicking 'Subscribe to your news feed', which will look something like this: https://github.com/donnemartin.private.atom?token=TOKEN ![Imgur](http://i.imgur.com/f2zvdIm.png) ### gh create-comment Create a comment on the given issue. Usage: $ gh create-comment [user_repo_number] [-t/--text] Param(s): ``` :type user_repo_number: str :param user_repo_number: The user/repo/issue_number. ``` Option(s): ``` :type text: str :param text: The comment text. ``` Example(s): $ gh create-comment donnemartin/saws/1 -t "hello world" $ gh create-comment donnemartin/saws/1 --text "hello world" ### gh create-issue Create an issue. Usage: $ gh create-issue [user_repo] [-t/--issue_title] [-d/--issue_desc] Param(s): ``` :type user_repo: str :param user_repo: The user/repo. ``` Option(s): ``` :type issue_title: str :param issue_title: The issue title. :type issue_desc: str :param issue_desc: The issue body (optional). ``` Example(s): $ gh create-issue donnemartin/gitsome -t "title" $ gh create-issue donnemartin/gitsome -t "title" -d "desc" $ gh create-issue donnemartin/gitsome --issue_title "title" --issue_desc "desc" ### gh create-repo Create a repo. Usage: $ gh create-repo [repo_name] [-d/--repo_desc] [-pr/--private] Param(s): ``` :type repo_name: str :param repo_name: The repo name. ``` Option(s): ``` :type repo_desc: str :param repo_desc: The repo description (optional). :type private: bool :param private: Determines whether the repo is private. Default: False. ``` Example(s): $ gh create-repo repo_name $ gh create-repo repo_name -d "desc" $ gh create-repo repo_name --repo_desc "desc" $ gh create-repo repo_name -pr $ gh create-repo repo_name --repo_desc "desc" --private ### gh emails List all the user's registered emails. Usage/Example(s): $ gh emails ### gh emojis List all GitHub supported emojis. Usage: $ gh emojis [-p/--pager] Option(s): ``` :type pager: bool :param pager: Determines whether to show the output in a pager, if available. ``` Example(s): $ gh emojis $ gh emojis -p $ gh emojis --pager ### gh feed List all activity for the given user or repo. If `user_or_repo` is not provided, uses the logged in user's news feed seen while visiting https://github.com. If `user_or_repo` is provided, shows either the public or `[-pr/--private]` feed activity of the user or repo. Usage: $ gh feed [user_or_repo] [-pr/--private] [-p/--pager] Param(s): ``` :type user_or_repo: str :param user_or_repo: The user or repo to list events for (optional). If no entry, defaults to the logged in user's feed. ``` Option(s): ``` :type private: bool :param private: Determines whether to show the private events (True) or public events (False). :type pager: bool :param pager: Determines whether to show the output in a pager, if available. ``` Example(s): $ gh feed $ gh feed | grep foo $ gh feed donnemartin $ gh feed donnemartin -pr -p $ gh feed donnemartin --private --pager $ gh feed donnemartin/haxor-news -p #### News Feed ![Imgur](http://i.imgur.com/2LWcyS6.png) #### User Activity Feed ![Imgur](http://i.imgur.com/kryGLXz.png) #### Repo Activity Feed ![Imgur](http://i.imgur.com/d2kxDg9.png) ### gh following List all followed users and the total followed count. Usage: $ gh following [user] [-p/--pager] Param(s): ``` :type user: str :param user: The user login. If None, returns the followed users of the logged in user. ``` Option(s): ``` :type pager: bool :param pager: Determines whether to show the output in a pager, if available. ``` Example(s): $ gh following $ gh following -p $ gh following octocat --pager ![Imgur](http://i.imgur.com/bjUmbf3.png) Also check out the [`gh user`](#gh-user) command. ### gh followers List all followers and the total follower count. Usage: $ gh followers [user] [-p/--pager] Param(s): ``` :type user: str :param user: The user login (optional). If None, returns the followers of the logged in user. ``` Option(s): ``` :type pager: bool :param pager: Determines whether to show the output in a pager, if available. ``` Example(s): $ gh followers $ gh followers -p $ gh followers octocat --pager ### gh gitignore-template Output the gitignore template for the given language. Usage: $ gh gitignore-template [language] Param(s): ``` :type language: str :param language: The language. ``` Example(s): $ gh gitignore-template Python $ gh gitignore-template Python > .gitignore ![Imgur](http://i.imgur.com/S5m5ZcO.png) ### gh gitignore-templates Output all supported gitignore templates. Usage: $ gh gitignore-templates [-p/--pager] Option(s): ``` :type pager: bool :param pager: Determines whether to show the output in a pager, if available. ``` Example(s): $ gh gitignore-templates $ gh gitignore-templates -p $ gh gitignore-templates --pager ![Imgur](http://i.imgur.com/u8qYx1s.png) ### gh issue Output detailed information about the given issue. Usage: $ gh issue [user_repo_number] Param(s): ``` :type user_repo_number: str :param user_repo_number: The user/repo/issue_number. ``` Example(s): $ gh issue donnemartin/saws/1 ![Imgur](http://i.imgur.com/ZFv9MuV.png) ### gh issues List all issues matching the filter. Usage: $ gh issues [-f/--issue_filter] [-s/--issue_state] [-l/--limit] [-p/--pager] Option(s): ``` :type issue_filter: str :param issue_filter: assigned, created, mentioned, subscribed (default). :type issue_state: str :param issue_state: all, open (default), closed. :type limit: int :param limit: The number of items to display. :type pager: bool :param pager: Determines whether to show the output in a pager, if available. ``` Example(s): $ gh issues $ gh issues -f assigned $ gh issues --issue_filter created $ gh issues -s all -l 20 -p $ gh issues --issue_state closed --limit 20 --pager $ gh issues -f created -s all -p ![Imgur](http://i.imgur.com/AB5zxxo.png) ### gh license Output the license template for the given license. Usage: $ gh license [license_name] Param(s): ``` :type license_name: str :param license_name: The license name. ``` Example(s): $ gh license apache-2.0 $ gh license mit > LICENSE ![Imgur](http://i.imgur.com/zJHVxaA.png) ### gh licenses Output all supported license templates. Usage/Licenses: $ gh licenses ![Imgur](http://i.imgur.com/S9SbMLJ.png) ### gh me List information about the logged in user. Displaying the avatar will require [installing the optional `PIL` dependency](#installing-pil). Usage: $ gh me [-b/--browser] [-t/--text_avatar] [-l/--limit] [-p/--pager] Option(s): ``` :type browser: bool :param browser: Determines whether to view the profile in a browser, or in the terminal. :type text_avatar: bool :param text_avatar: Determines whether to view the profile avatar in plain text. On Windows this value is always set to True due to lack of support of `img2txt` on Windows. :type limit: int :param limit: The number of user repos to display. :type pager: bool :param pager: Determines whether to show the output in a pager, if available. ``` Example(s): $ gh me $ gh me -b $ gh me --browser $ gh me -t -l 20 -p $ gh me --text_avatar --limit 20 --pager ![Imgur](http://i.imgur.com/csk5j0S.png) ### gh notifications List all notifications. Usage: $ gh notifications [-l/--limit] [-p/--pager] Option(s): ``` :type limit: int :param limit: The number of items to display. :type pager: bool :param pager: Determines whether to show the output in a pager, if available. ``` Example(s): $ gh notifications $ gh notifications -l 20 -p $ gh notifications --limit 20 --pager ![Imgur](http://i.imgur.com/uwmwxsW.png) ### gh octo Output an Easter egg or the given message from Octocat. Usage: $ gh octo [say] Param(s): ``` :type say: str :param say: What Octocat should say. If say is None, octocat speaks an Easter egg. ``` Example(s): $ gh octo $ gh octo "foo bar" ![Imgur](http://i.imgur.com/bNzCa5p.png) ### gh pull-request Output detailed information about the given pull request. Usage: $ gh pull-request [user_repo_number] Param(s): ``` :type user_repo_number: str :param user_repo_number: The user/repo/pull_number. ``` Example(s): $ gh pull-request donnemartin/saws/80 ![Imgur](http://i.imgur.com/3MtKjKy.png) ### gh pull-requests List all pull requests. Usage: $ gh pull-requests [-l/--limit] [-p/--pager] Option(s): ``` :type limit: int :param limit: The number of items to display. :type pager: bool :param pager: Determines whether to show the output in a pager, if available. ``` Example(s): $ gh pull-requests $ gh pull-requests -l 20 -p $ gh pull-requests --limit 20 --pager ![Imgur](http://i.imgur.com/4A2eYM9.png) ### gh rate-limit Output the rate limit. Not available for GitHub Enterprise. Usage/Example(s): $ gh rate-limit ### gh repo Output detailed information about the given repo. Usage: $ gh repo [user_repo] Param(s): ``` :type user_repo: str :param user_repo: The user/repo. ``` Example(s): $ gh repo donnemartin/haxor-news ![Imgur](http://i.imgur.com/XFMpWCI.png) ### gh repos List all repos matching the given filter. Usage: $ gh repos [repo_filter] [-l/--limit] [-p/--pager] Param(s): ``` :type repo_filter: str :param repo_filter: The filter for repo names. Only repos matching the filter will be returned. If None, outputs all the logged in user's repos. ``` Option(s): ``` :type limit: int :param limit: The number of items to display. :type pager: bool :param pager: Determines whether to show the output in a pager, if available. ``` Example(s): $ gh repos $ gh repos aws $ gh repos aws -l 20 -p $ gh repos aws --limit 20 --pager ![Imgur](http://i.imgur.com/YXWPWma.png) ### gh search-issues Search for all issues matching the given query. For more information about the query qualifiers, visit the [searching issues reference](https://help.github.com/articles/searching-issues/). Usage: $ gh search-issues [query] [-l/--limit] [-p/--pager] Param(s): ``` :type query: str :param query: The search query. The query can contain any combination of the following supported qualifers: - `type` With this qualifier you can restrict the search to issues or pull request only. - `in` Qualifies which fields are searched. With this qualifier you can restrict the search to just the title, body, comments, or any combination of these. - `author` Finds issues created by a certain user. - `assignee` Finds issues that are assigned to a certain user. - `mentions` Finds issues that mention a certain user. - `commenter` Finds issues that a certain user commented on. - `involves` Finds issues that were either created by a certain user, assigned to that user, mention that user, or were commented on by that user. - `state` Filter issues based on whether they’re open or closed. - `labels` Filters issues based on their labels. - `language` Searches for issues within repositories that match a certain language. - `created` or `updated` Filters issues based on times of creation, or when they were last updated. - `comments` Filters issues based on the quantity of comments. - `user` or `repo` Limits searches to a specific user or repository. For more information about these qualifiers, see: http://git.io/d1oELA ``` Option(s): ``` :type limit: int :param limit: The number of items to display. :type pager: bool :param pager: Determines whether to show the output in a pager, if available. ``` Example(s): $ gh search-issues "foo type:pr author:donnemartin" -l 20 -p $ gh search-issues "foobarbaz in:title created:>=2015-01-01" --limit 20 --pager Additional Example(s): ``` Search issues that have your user name tagged @donnemartin: $ gh search-issues "is:issue donnemartin is:open" -p Search issues that have the most +1s: $ gh search-issues "is:open is:issue sort:reactions-+1-desc" -p Search issues that have the most comments: $ gh search-issues "is:open is:issue sort:comments-desc" -p Search issues with the "help wanted" tag: $ gh search-issues "is:open is:issue label:\"help wanted\"" -p Search all your open private issues: $ gh search-issues "is:open is:issue is:private" -p ``` ![Imgur](http://i.imgur.com/DXXxkBD.png) ### gh search-repos Search for all repos matching the given query. For more information about the query qualifiers, visit the [searching repos reference](https://help.github.com/articles/searching-repositories/). Usage: $ gh search-repos [query] [-s/--sort] [-l/--limit] [-p/--pager] Param(s): ``` :type query: str :param query: The search query. The query can contain any combination of the following supported qualifers: - `in` Qualifies which fields are searched. With this qualifier you can restrict the search to just the repository name, description, readme, or any combination of these. - `size` Finds repositories that match a certain size (in kilobytes). - `forks` Filters repositories based on the number of forks, and/or whether forked repositories should be included in the results at all. - `created` or `pushed` Filters repositories based on times of creation, or when they were last updated. Format: `YYYY-MM-DD`. Examples: `created:<2011`, `pushed:<2013-02`, `pushed:>=2013-03-06` - `user` or `repo` Limits searches to a specific user or repository. - `language` Searches repositories based on the language they're written in. - `stars` Searches repositories based on the number of stars. For more information about these qualifiers, see: http://git.io/4Z8AkA ``` Option(s): ``` :type sort: str :param sort: Optional: 'stars', 'forks', 'updated'. If not specified, sorting is done by query best match. :type limit: int :param limit: The number of items to display. :type pager: bool :param pager: Determines whether to show the output in a pager, if available. ``` Example(s): $ gh search-repos "maps language:python" -s stars -l 20 -p $ gh search-repos "created:>=2015-01-01 stars:>=1000 language:python" --sort stars --limit 20 --pager ![Imgur](http://i.imgur.com/kazXWWY.png) ### gh starred Output starred repos. Usage: $ gh starred [repo_filter] [-l/--limit] [-p/--pager] Param(s): ``` :type repo_filter: str :param repo_filter: The filter for repo names. Only repos matching the filter will be returned. If None, outputs all starred repos. ``` Option(s): ``` :type limit: int :param limit: The number of items to display. :type pager: bool :param pager: Determines whether to show the output in a pager, if available. ``` Example(s): $ gh starred $ gh starred foo -l 20 -p $ gh starred foo --limit 20 --pager ![Imgur](http://i.imgur.com/JB88Kw8.png) ### gh trending List trending repos for the given language. Usage: $ gh trending [language] [-w/--weekly] [-m/--monthly] [-D/--devs] [-b/--browser] [-p/--pager] Param(s): ``` :type language: str :param language: The language (optional). If blank, shows 'Overall'. ``` Option(s): ``` :type weekly: bool :param weekly: Determines whether to show the weekly rankings. Daily is the default. :type monthly: bool :param monthly: Determines whether to show the monthly rankings. Daily is the default. If both `monthly` and `weekly` are set, `monthly` takes precedence. :type devs: bool :param devs: determines whether to display the trending devs or repos. Only valid with the -b/--browser option. :type browser: bool :param browser: Determines whether to view the profile in a browser, or in the terminal. :type pager: bool :param pager: Determines whether to show the output in a pager, if available. ``` Example(s): $ gh trending $ gh trending Python -w -p $ gh trending Python --weekly --devs --browser $ gh trending --browser ![Imgur](http://i.imgur.com/Dx77HFW.png) ### gh user List information about the given user. Displaying the avatar will require [installing the optional `PIL` dependency](#installing-pil). Usage: $ gh user [user_id] [-b/--browser] [-t/--text_avatar] [-l/--limit] [-p/--pager] Param(s): ``` :type user_id: str :param user_id: The user id/login. If None, returns followers of the logged in user. ``` Option(s): ``` :type browser: bool :param browser: Determines whether to view the profile in a browser, or in the terminal. :type text_avatar: bool :param text_avatar: Determines whether to view the profile avatar in plain text instead of ansi (default). On Windows this value is always set to True due to lack of support of `img2txt` on Windows. :type limit: int :param limit: The number of items to display. :type pager: bool :param pager: Determines whether to show the output in a pager, if available. ``` Example(s): $ gh user octocat $ gh user octocat -b $ gh user octocat --browser $ gh user octocat -t -l 10 -p $ gh user octocat --text_avatar --limit 10 --pager ![Imgur](http://i.imgur.com/xVoVPVe.png) ### gh view View the given notification/repo/issue/pull_request/user index in the terminal or a browser. This method is meant to be called after one of the following commands which outputs a table of notifications/repos/issues/pull_requests/users: $ gh repos $ gh search-repos $ gh starred $ gh issues $ gh pull-requests $ gh search-issues $ gh notifications $ gh trending $ gh user $ gh me Usage: $ gh view [index] [-b/--browser] Param(s): ``` :type index: str :param index: Determines the index to view. ``` Option(s): ``` :type browser: bool :param browser: Determines whether to view the profile in a browser, or in the terminal. ``` Example(s): $ gh repos $ gh view 1 $ gh starred $ gh view 1 -b $ gh view 1 --browser ![Imgur](http://i.imgur.com/NVEwGbV.png) ### Option: View in a Pager Many `gh` commands support a `-p/--pager` option that displays results in a pager, where available. Usage: $ gh [param] [options] -p $ gh [param] [options] --pager ### Option: View in a Browser Many `gh` commands support a `-b/--browser` option that displays results in your default browser instead of your terminal. Usage: $ gh [param] [options] -b $ gh [param] [options] --browser ================================================ FILE: CONTRIBUTING.md ================================================ Contributing ============ Contributions are welcome! **Please carefully read this page to make the code review process go as smoothly as possible and to maximize the likelihood of your contribution being merged.** ## Bug Reports For bug reports or requests [submit an issue](https://github.com/donnemartin/gitsome/issues). ## Pull Requests The preferred way to contribute is to fork the [main repository](https://github.com/donnemartin/gitsome) on GitHub. 1. Fork the [main repository](https://github.com/donnemartin/gitsome). Click on the 'Fork' button near the top of the page. This creates a copy of the code under your account on the GitHub server. 2. Clone this copy to your local disk: $ git clone git@github.com:YourLogin/gitsome.git $ cd gitsome 3. Create a branch to hold your changes and start making changes. Don't work in the `master` branch! $ git checkout -b my-feature 4. Work on this copy on your computer using Git to do the version control. When you're done editing, run the following to record your changes in Git: $ git add modified_files $ git commit 5. Push your changes to GitHub with: $ git push -u origin my-feature 6. Finally, go to the web page of your fork of the `gitsome` repo and click 'Pull Request' to send your changes for review. ### GitHub Pull Requests Docs If you are not familiar with pull requests, review the [pull request docs](https://help.github.com/articles/using-pull-requests/). ### Code Quality Ensure your pull request satisfies all of the following, where applicable: * Is covered by [unit tests](https://github.com/donnemartin/gitsome#unit-tests-and-code-coverage) * Passes [continuous integration](https://github.com/donnemartin/gitsome#continuous-integration) * Is covered by [documentation](https://github.com/donnemartin/gitsome#documentation) Review the following [style guide](https://google.github.io/styleguide/pyguide.html). Run code checks and fix any issues: $ scripts/run_code_checks.sh ### Installation Refer to the [Installation](https://github.com/donnemartin/gitsome#installation) and [Developer Installation](https://github.com/donnemartin/gitsome#developer-installation) sections. ================================================ FILE: Dockerfile ================================================ ########################################################## # # # Build the image: # # docker build -t gitsome . # # # # Run the container: # # docker run -ti --rm -v $(pwd):/src/ \ # # -v ${HOME}/.gitsomeconfig:/root/.gitsomeconfig \ # # -v ${HOME}/.gitconfig:/root/.gitconfig \ # # gitsome # # # ########################################################## FROM python:3.5 RUN pip install Pillow COPY /requirements-dev.txt /gitsome/ WORKDIR /gitsome/ RUN pip install -r requirements-dev.txt COPY / /gitsome/ RUN pip install -e . RUN mkdir /src/ WORKDIR /src/ ENTRYPOINT ["gitsome"] ================================================ FILE: LICENSE.txt ================================================ gitsome is licensed under Apache 2.0: I am providing code and resources in this repository to you under an open source license. Because this is my personal repository, the license you receive to my code and resources is from me and not my employer (Facebook). Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "{}" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright 2016 Donne Martin. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. xonsh is licensed under BSD: Copyright 2015, the xonsh developers. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE XONSH DEVELOPERS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE XONSH DEVELOPERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. The views and conclusions contained in the software and documentation are those of the authors and should not be interpreted as representing official policies, either expressed or implied, of the stakeholders of the xonsh project or the employers of xonsh developers. github3.py is licensed under BSD: Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. The name of the author may not be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. img2txt is licensed under BSD: Copyright (c) 2013 - 2015, hit9 All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of img2txt nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. html2text is licensed under GNU: GNU GENERAL PUBLIC LICENSE Version 3, 29 June 2007 Copyright (C) 2007 Free Software Foundation, Inc. Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Preamble The GNU General Public License is a free, copyleft license for software and other kinds of works. The licenses for most software and other practical works are designed to take away your freedom to share and change the works. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change all versions of a program--to make sure it remains free software for all its users. We, the Free Software Foundation, use the GNU General Public License for most of our software; it applies also to any other work released this way by its authors. You can apply it to your programs, too. When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for them if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs, and that you know you can do these things. To protect your rights, we need to prevent others from denying you these rights or asking you to surrender the rights. Therefore, you have certain responsibilities if you distribute copies of the software, or if you modify it: responsibilities to respect the freedom of others. For example, if you distribute copies of such a program, whether gratis or for a fee, you must pass on to the recipients the same freedoms that you received. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights. Developers that use the GNU GPL protect your rights with two steps: (1) assert copyright on the software, and (2) offer you this License giving you legal permission to copy, distribute and/or modify it. For the developers' and authors' protection, the GPL clearly explains that there is no warranty for this free software. For both users' and authors' sake, the GPL requires that modified versions be marked as changed, so that their problems will not be attributed erroneously to authors of previous versions. Some devices are designed to deny users access to install or run modified versions of the software inside them, although the manufacturer can do so. This is fundamentally incompatible with the aim of protecting users' freedom to change the software. The systematic pattern of such abuse occurs in the area of products for individuals to use, which is precisely where it is most unacceptable. Therefore, we have designed this version of the GPL to prohibit the practice for those products. If such problems arise substantially in other domains, we stand ready to extend this provision to those domains in future versions of the GPL, as needed to protect the freedom of users. Finally, every program is threatened constantly by software patents. States should not allow patents to restrict development and use of software on general-purpose computers, but in those that do, we wish to avoid the special danger that patents applied to a free program could make it effectively proprietary. To prevent this, the GPL assures that patents cannot be used to render the program non-free. The precise terms and conditions for copying, distribution and modification follow. TERMS AND CONDITIONS 0. Definitions. "This License" refers to version 3 of the GNU General Public License. "Copyright" also means copyright-like laws that apply to other kinds of works, such as semiconductor masks. "The Program" refers to any copyrightable work licensed under this License. Each licensee is addressed as "you". "Licensees" and "recipients" may be individuals or organizations. To "modify" a work means to copy from or adapt all or part of the work in a fashion requiring copyright permission, other than the making of an exact copy. The resulting work is called a "modified version" of the earlier work or a work "based on" the earlier work. A "covered work" means either the unmodified Program or a work based on the Program. To "propagate" a work means to do anything with it that, without permission, would make you directly or secondarily liable for infringement under applicable copyright law, except executing it on a computer or modifying a private copy. Propagation includes copying, distribution (with or without modification), making available to the public, and in some countries other activities as well. To "convey" a work means any kind of propagation that enables other parties to make or receive copies. Mere interaction with a user through a computer network, with no transfer of a copy, is not conveying. An interactive user interface displays "Appropriate Legal Notices" to the extent that it includes a convenient and prominently visible feature that (1) displays an appropriate copyright notice, and (2) tells the user that there is no warranty for the work (except to the extent that warranties are provided), that licensees may convey the work under this License, and how to view a copy of this License. If the interface presents a list of user commands or options, such as a menu, a prominent item in the list meets this criterion. 1. Source Code. The "source code" for a work means the preferred form of the work for making modifications to it. "Object code" means any non-source form of a work. A "Standard Interface" means an interface that either is an official standard defined by a recognized standards body, or, in the case of interfaces specified for a particular programming language, one that is widely used among developers working in that language. The "System Libraries" of an executable work include anything, other than the work as a whole, that (a) is included in the normal form of packaging a Major Component, but which is not part of that Major Component, and (b) serves only to enable use of the work with that Major Component, or to implement a Standard Interface for which an implementation is available to the public in source code form. A "Major Component", in this context, means a major essential component (kernel, window system, and so on) of the specific operating system (if any) on which the executable work runs, or a compiler used to produce the work, or an object code interpreter used to run it. The "Corresponding Source" for a work in object code form means all the source code needed to generate, install, and (for an executable work) run the object code and to modify the work, including scripts to control those activities. However, it does not include the work's System Libraries, or general-purpose tools or generally available free programs which are used unmodified in performing those activities but which are not part of the work. For example, Corresponding Source includes interface definition files associated with source files for the work, and the source code for shared libraries and dynamically linked subprograms that the work is specifically designed to require, such as by intimate data communication or control flow between those subprograms and other parts of the work. The Corresponding Source need not include anything that users can regenerate automatically from other parts of the Corresponding Source. The Corresponding Source for a work in source code form is that same work. 2. Basic Permissions. All rights granted under this License are granted for the term of copyright on the Program, and are irrevocable provided the stated conditions are met. This License explicitly affirms your unlimited permission to run the unmodified Program. The output from running a covered work is covered by this License only if the output, given its content, constitutes a covered work. This License acknowledges your rights of fair use or other equivalent, as provided by copyright law. You may make, run and propagate covered works that you do not convey, without conditions so long as your license otherwise remains in force. You may convey covered works to others for the sole purpose of having them make modifications exclusively for you, or provide you with facilities for running those works, provided that you comply with the terms of this License in conveying all material for which you do not control copyright. Those thus making or running the covered works for you must do so exclusively on your behalf, under your direction and control, on terms that prohibit them from making any copies of your copyrighted material outside their relationship with you. Conveying under any other circumstances is permitted solely under the conditions stated below. Sublicensing is not allowed; section 10 makes it unnecessary. 3. Protecting Users' Legal Rights From Anti-Circumvention Law. No covered work shall be deemed part of an effective technological measure under any applicable law fulfilling obligations under article 11 of the WIPO copyright treaty adopted on 20 December 1996, or similar laws prohibiting or restricting circumvention of such measures. When you convey a covered work, you waive any legal power to forbid circumvention of technological measures to the extent such circumvention is effected by exercising rights under this License with respect to the covered work, and you disclaim any intention to limit operation or modification of the work as a means of enforcing, against the work's users, your or third parties' legal rights to forbid circumvention of technological measures. 4. Conveying Verbatim Copies. You may convey verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice; keep intact all notices stating that this License and any non-permissive terms added in accord with section 7 apply to the code; keep intact all notices of the absence of any warranty; and give all recipients a copy of this License along with the Program. You may charge any price or no price for each copy that you convey, and you may offer support or warranty protection for a fee. 5. Conveying Modified Source Versions. You may convey a work based on the Program, or the modifications to produce it from the Program, in the form of source code under the terms of section 4, provided that you also meet all of these conditions: a) The work must carry prominent notices stating that you modified it, and giving a relevant date. b) The work must carry prominent notices stating that it is released under this License and any conditions added under section 7. This requirement modifies the requirement in section 4 to "keep intact all notices". c) You must license the entire work, as a whole, under this License to anyone who comes into possession of a copy. This License will therefore apply, along with any applicable section 7 additional terms, to the whole of the work, and all its parts, regardless of how they are packaged. This License gives no permission to license the work in any other way, but it does not invalidate such permission if you have separately received it. d) If the work has interactive user interfaces, each must display Appropriate Legal Notices; however, if the Program has interactive interfaces that do not display Appropriate Legal Notices, your work need not make them do so. A compilation of a covered work with other separate and independent works, which are not by their nature extensions of the covered work, and which are not combined with it such as to form a larger program, in or on a volume of a storage or distribution medium, is called an "aggregate" if the compilation and its resulting copyright are not used to limit the access or legal rights of the compilation's users beyond what the individual works permit. Inclusion of a covered work in an aggregate does not cause this License to apply to the other parts of the aggregate. 6. Conveying Non-Source Forms. You may convey a covered work in object code form under the terms of sections 4 and 5, provided that you also convey the machine-readable Corresponding Source under the terms of this License, in one of these ways: a) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by the Corresponding Source fixed on a durable physical medium customarily used for software interchange. b) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by a written offer, valid for at least three years and valid for as long as you offer spare parts or customer support for that product model, to give anyone who possesses the object code either (1) a copy of the Corresponding Source for all the software in the product that is covered by this License, on a durable physical medium customarily used for software interchange, for a price no more than your reasonable cost of physically performing this conveying of source, or (2) access to copy the Corresponding Source from a network server at no charge. c) Convey individual copies of the object code with a copy of the written offer to provide the Corresponding Source. This alternative is allowed only occasionally and noncommercially, and only if you received the object code with such an offer, in accord with subsection 6b. d) Convey the object code by offering access from a designated place (gratis or for a charge), and offer equivalent access to the Corresponding Source in the same way through the same place at no further charge. You need not require recipients to copy the Corresponding Source along with the object code. If the place to copy the object code is a network server, the Corresponding Source may be on a different server (operated by you or a third party) that supports equivalent copying facilities, provided you maintain clear directions next to the object code saying where to find the Corresponding Source. Regardless of what server hosts the Corresponding Source, you remain obligated to ensure that it is available for as long as needed to satisfy these requirements. e) Convey the object code using peer-to-peer transmission, provided you inform other peers where the object code and Corresponding Source of the work are being offered to the general public at no charge under subsection 6d. A separable portion of the object code, whose source code is excluded from the Corresponding Source as a System Library, need not be included in conveying the object code work. A "User Product" is either (1) a "consumer product", which means any tangible personal property which is normally used for personal, family, or household purposes, or (2) anything designed or sold for incorporation into a dwelling. In determining whether a product is a consumer product, doubtful cases shall be resolved in favor of coverage. For a particular product received by a particular user, "normally used" refers to a typical or common use of that class of product, regardless of the status of the particular user or of the way in which the particular user actually uses, or expects or is expected to use, the product. A product is a consumer product regardless of whether the product has substantial commercial, industrial or non-consumer uses, unless such uses represent the only significant mode of use of the product. "Installation Information" for a User Product means any methods, procedures, authorization keys, or other information required to install and execute modified versions of a covered work in that User Product from a modified version of its Corresponding Source. The information must suffice to ensure that the continued functioning of the modified object code is in no case prevented or interfered with solely because modification has been made. If you convey an object code work under this section in, or with, or specifically for use in, a User Product, and the conveying occurs as part of a transaction in which the right of possession and use of the User Product is transferred to the recipient in perpetuity or for a fixed term (regardless of how the transaction is characterized), the Corresponding Source conveyed under this section must be accompanied by the Installation Information. But this requirement does not apply if neither you nor any third party retains the ability to install modified object code on the User Product (for example, the work has been installed in ROM). The requirement to provide Installation Information does not include a requirement to continue to provide support service, warranty, or updates for a work that has been modified or installed by the recipient, or for the User Product in which it has been modified or installed. Access to a network may be denied when the modification itself materially and adversely affects the operation of the network or violates the rules and protocols for communication across the network. Corresponding Source conveyed, and Installation Information provided, in accord with this section must be in a format that is publicly documented (and with an implementation available to the public in source code form), and must require no special password or key for unpacking, reading or copying. 7. Additional Terms. "Additional permissions" are terms that supplement the terms of this License by making exceptions from one or more of its conditions. Additional permissions that are applicable to the entire Program shall be treated as though they were included in this License, to the extent that they are valid under applicable law. If additional permissions apply only to part of the Program, that part may be used separately under those permissions, but the entire Program remains governed by this License without regard to the additional permissions. When you convey a copy of a covered work, you may at your option remove any additional permissions from that copy, or from any part of it. (Additional permissions may be written to require their own removal in certain cases when you modify the work.) You may place additional permissions on material, added by you to a covered work, for which you have or can give appropriate copyright permission. Notwithstanding any other provision of this License, for material you add to a covered work, you may (if authorized by the copyright holders of that material) supplement the terms of this License with terms: a) Disclaiming warranty or limiting liability differently from the terms of sections 15 and 16 of this License; or b) Requiring preservation of specified reasonable legal notices or author attributions in that material or in the Appropriate Legal Notices displayed by works containing it; or c) Prohibiting misrepresentation of the origin of that material, or requiring that modified versions of such material be marked in reasonable ways as different from the original version; or d) Limiting the use for publicity purposes of names of licensors or authors of the material; or e) Declining to grant rights under trademark law for use of some trade names, trademarks, or service marks; or f) Requiring indemnification of licensors and authors of that material by anyone who conveys the material (or modified versions of it) with contractual assumptions of liability to the recipient, for any liability that these contractual assumptions directly impose on those licensors and authors. All other non-permissive additional terms are considered "further restrictions" within the meaning of section 10. If the Program as you received it, or any part of it, contains a notice stating that it is governed by this License along with a term that is a further restriction, you may remove that term. If a license document contains a further restriction but permits relicensing or conveying under this License, you may add to a covered work material governed by the terms of that license document, provided that the further restriction does not survive such relicensing or conveying. If you add terms to a covered work in accord with this section, you must place, in the relevant source files, a statement of the additional terms that apply to those files, or a notice indicating where to find the applicable terms. Additional terms, permissive or non-permissive, may be stated in the form of a separately written license, or stated as exceptions; the above requirements apply either way. 8. Termination. You may not propagate or modify a covered work except as expressly provided under this License. Any attempt otherwise to propagate or modify it is void, and will automatically terminate your rights under this License (including any patent licenses granted under the third paragraph of section 11). However, if you cease all violation of this License, then your license from a particular copyright holder is reinstated (a) provisionally, unless and until the copyright holder explicitly and finally terminates your license, and (b) permanently, if the copyright holder fails to notify you of the violation by some reasonable means prior to 60 days after the cessation. Moreover, your license from a particular copyright holder is reinstated permanently if the copyright holder notifies you of the violation by some reasonable means, this is the first time you have received notice of violation of this License (for any work) from that copyright holder, and you cure the violation prior to 30 days after your receipt of the notice. Termination of your rights under this section does not terminate the licenses of parties who have received copies or rights from you under this License. If your rights have been terminated and not permanently reinstated, you do not qualify to receive new licenses for the same material under section 10. 9. Acceptance Not Required for Having Copies. You are not required to accept this License in order to receive or run a copy of the Program. Ancillary propagation of a covered work occurring solely as a consequence of using peer-to-peer transmission to receive a copy likewise does not require acceptance. However, nothing other than this License grants you permission to propagate or modify any covered work. These actions infringe copyright if you do not accept this License. Therefore, by modifying or propagating a covered work, you indicate your acceptance of this License to do so. 10. Automatic Licensing of Downstream Recipients. Each time you convey a covered work, the recipient automatically receives a license from the original licensors, to run, modify and propagate that work, subject to this License. You are not responsible for enforcing compliance by third parties with this License. An "entity transaction" is a transaction transferring control of an organization, or substantially all assets of one, or subdividing an organization, or merging organizations. If propagation of a covered work results from an entity transaction, each party to that transaction who receives a copy of the work also receives whatever licenses to the work the party's predecessor in interest had or could give under the previous paragraph, plus a right to possession of the Corresponding Source of the work from the predecessor in interest, if the predecessor has it or can get it with reasonable efforts. You may not impose any further restrictions on the exercise of the rights granted or affirmed under this License. For example, you may not impose a license fee, royalty, or other charge for exercise of rights granted under this License, and you may not initiate litigation (including a cross-claim or counterclaim in a lawsuit) alleging that any patent claim is infringed by making, using, selling, offering for sale, or importing the Program or any portion of it. 11. Patents. A "contributor" is a copyright holder who authorizes use under this License of the Program or a work on which the Program is based. The work thus licensed is called the contributor's "contributor version". A contributor's "essential patent claims" are all patent claims owned or controlled by the contributor, whether already acquired or hereafter acquired, that would be infringed by some manner, permitted by this License, of making, using, or selling its contributor version, but do not include claims that would be infringed only as a consequence of further modification of the contributor version. For purposes of this definition, "control" includes the right to grant patent sublicenses in a manner consistent with the requirements of this License. Each contributor grants you a non-exclusive, worldwide, royalty-free patent license under the contributor's essential patent claims, to make, use, sell, offer for sale, import and otherwise run, modify and propagate the contents of its contributor version. In the following three paragraphs, a "patent license" is any express agreement or commitment, however denominated, not to enforce a patent (such as an express permission to practice a patent or covenant not to sue for patent infringement). To "grant" such a patent license to a party means to make such an agreement or commitment not to enforce a patent against the party. If you convey a covered work, knowingly relying on a patent license, and the Corresponding Source of the work is not available for anyone to copy, free of charge and under the terms of this License, through a publicly available network server or other readily accessible means, then you must either (1) cause the Corresponding Source to be so available, or (2) arrange to deprive yourself of the benefit of the patent license for this particular work, or (3) arrange, in a manner consistent with the requirements of this License, to extend the patent license to downstream recipients. "Knowingly relying" means you have actual knowledge that, but for the patent license, your conveying the covered work in a country, or your recipient's use of the covered work in a country, would infringe one or more identifiable patents in that country that you have reason to believe are valid. If, pursuant to or in connection with a single transaction or arrangement, you convey, or propagate by procuring conveyance of, a covered work, and grant a patent license to some of the parties receiving the covered work authorizing them to use, propagate, modify or convey a specific copy of the covered work, then the patent license you grant is automatically extended to all recipients of the covered work and works based on it. A patent license is "discriminatory" if it does not include within the scope of its coverage, prohibits the exercise of, or is conditioned on the non-exercise of one or more of the rights that are specifically granted under this License. You may not convey a covered work if you are a party to an arrangement with a third party that is in the business of distributing software, under which you make payment to the third party based on the extent of your activity of conveying the work, and under which the third party grants, to any of the parties who would receive the covered work from you, a discriminatory patent license (a) in connection with copies of the covered work conveyed by you (or copies made from those copies), or (b) primarily for and in connection with specific products or compilations that contain the covered work, unless you entered into that arrangement, or that patent license was granted, prior to 28 March 2007. Nothing in this License shall be construed as excluding or limiting any implied license or other defenses to infringement that may otherwise be available to you under applicable patent law. 12. No Surrender of Others' Freedom. If conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot convey a covered work so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not convey it at all. For example, if you agree to terms that obligate you to collect a royalty for further conveying from those to whom you convey the Program, the only way you could satisfy both those terms and this License would be to refrain entirely from conveying the Program. 13. Use with the GNU Affero General Public License. Notwithstanding any other provision of this License, you have permission to link or combine any covered work with a work licensed under version 3 of the GNU Affero General Public License into a single combined work, and to convey the resulting work. The terms of this License will continue to apply to the part which is the covered work, but the special requirements of the GNU Affero General Public License, section 13, concerning interaction through a network will apply to the combination as such. 14. Revised Versions of this License. The Free Software Foundation may publish revised and/or new versions of the GNU General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Program specifies that a certain numbered version of the GNU General Public License "or any later version" applies to it, you have the option of following the terms and conditions either of that numbered version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of the GNU General Public License, you may choose any version ever published by the Free Software Foundation. If the Program specifies that a proxy can decide which future versions of the GNU General Public License can be used, that proxy's public statement of acceptance of a version permanently authorizes you to choose that version for the Program. Later license versions may give you additional or different permissions. However, no additional obligations are imposed on any author or copyright holder as a result of your choosing to follow a later version. 15. Disclaimer of Warranty. THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 16. Limitation of Liability. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. 17. Interpretation of Sections 15 and 16. If the disclaimer of warranty and limitation of liability provided above cannot be given local legal effect according to their terms, reviewing courts shall apply local law that most closely approximates an absolute waiver of all civil liability in connection with the Program, unless a warranty or assumption of liability accompanies a copy of the Program in return for a fee. END OF TERMS AND CONDITIONS How to Apply These Terms to Your New Programs If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms. To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively state the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. Copyright (C) This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . Also add information on how to contact you by electronic and paper mail. If the program does terminal interaction, make it output a short notice like this when it starts in an interactive mode: Copyright (C) This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details. The hypothetical commands `show w' and `show c' should show the appropriate parts of the General Public License. Of course, your program's commands might be different; for a GUI interface, you would use an "about box". You should also get your employer (if you work as a programmer) or school, if any, to sign a "copyright disclaimer" for the program, if necessary. For more information on this, and how to apply and follow the GNU GPL, see . The GNU General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Lesser General Public License instead of this License. But first, please read . ================================================ FILE: PRIVACY.md ================================================ # Privacy Policy ## What information do we collect? To properly integrate with GitHub, `gitsome` must authenticate with GitHub and/or GitHub Enterprise. When using `gitsome`, you may be asked to enter your: user name, password, two factor authentication token, user access token, news feed url, or GitHub Enterprise url. See the [configure](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-configure) section for more details. You may, however, use features of `gitsome` that do not require GitHub authentication anonymously. ## What do we use your information for? Any of the information we collect from you is used solely to interact with GitHub and/or GitHub Enterprise. Other than to provide integration with GitHub, we do not disclose or transfer your information to outside parties. ## How do we protect your information? All supplied sensitive information is encrypted during transmission to GitHub via Secure Socket Layer (SSL) technology. ## Terms and Conditions Please also visit our Terms and Conditions section establishing the use, disclaimers, and limitations of liability governing the use of `gitsome`. ## Your Consent By using `gitsome`, you consent to our privacy policy. ## Changes to our Privacy Policy If we decide to change our privacy policy, we will post those changes on this page. ## Contacting Us Any questions about this document should be addressed to [Donne Martin](donne.martin@gmail.com). ================================================ FILE: README.md ================================================

An Official Integration for GitHub and GitHub Enterprise.

gitsome ======= [![Build Status](https://travis-ci.org/donnemartin/gitsome.svg?branch=master)](https://travis-ci.org/donnemartin/gitsome) [![PyPI version](https://badge.fury.io/py/gitsome.svg)](http://badge.fury.io/py/gitsome) [![PyPI](https://img.shields.io/pypi/pyversions/gitsome.svg)](https://pypi.python.org/pypi/gitsome/) [![License](https://img.shields.io/:license-apache-blue.svg)](http://www.apache.org/licenses/LICENSE-2.0.html) ## Why `gitsome`? ### The Git Command Line Although the standard Git command line is a great tool to manage your Git-powered repos, it can be **tough to remember the usage** of: * 150+ porcelain and plumbing commands * Countless command-specific options * Resources such as tags and branches The Git command line **does not integrate with GitHub**, forcing you to toggle between command line and browser. ## `gitsome` - A Supercharged Git/GitHub CLI With Autocomplete

`gitsome` aims to supercharge your standard git/shell interface by focusing on: * **Improving ease-of-use** * **Increasing productivity** ### Deep GitHub Integration Not all GitHub workflows work well in a terminal; `gitsome` attempts to target those that do. `gitsome` includes 29 GitHub integrated commands that work with **[ALL](#enabling-gh-tab-completions-outside-of-gitsome)** shells: $ gh [param] [options] * [Quick reference](#github-integration-commands-quick-reference) * [General reference](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md) Run `gh` commands along with [Git-Extras](https://github.com/tj/git-extras/blob/master/Commands.md) and [hub](https://hub.github.com/) commands to unlock even more GitHub integrations! ![Imgur](http://i.imgur.com/sG09AJH.png) ### Git and GitHub Autocompleter With Interactive Help You can run the **optional** shell: $ gitsome to enable **autocompletion** and **interactive help** for the following: * Git commands * Git options * Git branches, tags, etc * [Git-Extras commands](https://github.com/tj/git-extras/blob/master/Commands.md) * [GitHub integration commands](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md) ![Imgur](http://i.imgur.com/08OMNjz.png) ![Imgur](http://i.imgur.com/fHjMwlh.png) ### General Autocompleter `gitsome` autocompletes the following: * Shell commands * Files and directories * Environment variables * Man pages * Python To enable additional autocompletions, check out the [Enabling Bash Completions](#enabling-bash-completions) section. ![Imgur](http://i.imgur.com/hg1dpk6.png) ## Fish-Style Auto-Suggestions `gitsome` supports Fish-style auto-suggestions. Use the `right arrow` key to complete a suggestion. ![Imgur](http://i.imgur.com/ZRaFGpY.png) ## Python REPL `gitsome` is powered by [`xonsh`](https://github.com/scopatz/xonsh), which supports a Python REPL. Run Python commands alongside shell commands: ![Imgur](http://i.imgur.com/NYk7WYO.png) Additional `xonsh` features can be found in the [`xonsh tutorial`](http://xon.sh/tutorial.html). ## Command History `gitsome` keeps track of commands you enter and stores them in `~/.xonsh_history.json`. Use the up and down arrow keys to cycle through the command history. ![Imgur](http://i.imgur.com/wq0caZu.png) ## Customizable Highlighting You can control the ansi colors used for highlighting by updating your `~/.gitsomeconfig` file. Color options include: ``` 'black', 'red', 'green', 'yellow', 'blue', 'magenta', 'cyan', 'white' ``` For no color, set the value(s) to `None`. `white` can appear as light gray on some terminals. ![Imgur](http://i.imgur.com/BN1lfEf.png) ## Available Platforms `gitsome` is available for Mac, Linux, Unix, [Windows](#windows-support), and [Docker](#running-as-docker-container). ## TODO >Not all GitHub workflows work well in a terminal; `gitsome` attempts to target those that do. * Add additional GitHub API integrations `gitsome` is just getting started. Feel free to [contribute!](#contributing) ## Index ### GitHub Integration Commands * [GitHub Integration Commands Syntax](#github-integration-commands-syntax) * [GitHub Integration Commands Listing](#github-integration-commands-listing) * [GitHub Integration Commands Quick Reference](#github-integration-commands-quick-reference) * [GitHub Integration Commands Reference in COMMANDS.md](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md) * [`gh configure`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-configure) * [`gh create-comment`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-create-comment) * [`gh create-issue`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-create-issue) * [`gh create-repo`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-create-repo) * [`gh emails`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-emails) * [`gh emojis`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-emojis) * [`gh feed`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-feed) * [`gh followers`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-followers) * [`gh following`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-following) * [`gh gitignore-template`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-gitignore-template) * [`gh gitignore-templates`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-gitignore-templates) * [`gh issue`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-issue) * [`gh issues`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-issues) * [`gh license`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-license) * [`gh licenses`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-licenses) * [`gh me`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-me) * [`gh notifications`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-notifications) * [`gh octo`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-octo) * [`gh pull-request`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-pull-request) * [`gh pull-requests`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-pull-requests) * [`gh rate-limit`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-rate-limit) * [`gh repo`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-repo) * [`gh repos`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-repos) * [`gh search-issues`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-search-issues) * [`gh search-repos`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-search-repos) * [`gh starred`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-starred) * [`gh trending`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-trending) * [`gh user`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-user) * [`gh view`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-view) * [Option: View in a Pager](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#option-view-in-a-pager) * [Option: View in a Browser](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#option-view-in-a-browser) ### Installation and Tests * [Installation](#installation) * [Pip Installation](#pip-installation) * [Virtual Environment Installation](#virtual-environment-installation) * [Running as a Docker Container](#running-as-a-docker-container) * [Running the `gh configure` Command](#running-the-gh-configure-command) * [For GitHub Enterprise Users](#for-github-enterprise-users) * [Enabling Bash Completions](#enabling-bash-completions) * [Enabling `gh` Tab Completions Outside of `gitsome`](#enabling-gh-tab-completions-outside-of-gitsome) * [For Zsh Users](#for-zsh-users) * [Optional: Installing `PIL` or `Pillow`](#optional-installing-pil-or-pillow) * [Supported Python Versions](#supported-python-versions) * [Supported Platforms](#supported-platforms) * [Windows Support](#windows-support) * [Developer Installation](#developer-installation) * [Continuous Integration](#continuous-integration) * [Unit Tests and Code Coverage](#unit-tests-and-code-coverage) * [Documentation](#documentation) ### Misc * [Contributing](#contributing) * [Credits](#credits) * [Contact Info](#contact-info) * [License](#license) ## GitHub Integration Commands Syntax Usage: $ gh [param] [options] ## GitHub Integration Commands Listing ``` configure Configure gitsome. create-comment Create a comment on the given issue. create-issue Create an issue. create-repo Create a repo. emails List all the user's registered emails. emojis List all GitHub supported emojis. feed List all activity for the given user or repo. followers List all followers and the total follower count. following List all followed users and the total followed count. gitignore-template Output the gitignore template for the given language. gitignore-templates Output all supported gitignore templates. issue Output detailed information about the given issue. issues List all issues matching the filter. license Output the license template for the given license. licenses Output all supported license templates. me List information about the logged in user. notifications List all notifications. octo Output an Easter egg or the given message from Octocat. pull-request Output detailed information about the given pull request. pull-requests List all pull requests. rate-limit Output the rate limit. Not available for Enterprise. repo Output detailed information about the given filter. repos List all repos matching the given filter. search-issues Search for all issues matching the given query. search-repos Search for all repos matching the given query. starred Output starred repos. trending List trending repos for the given language. user List information about the given user. view View the given index in the terminal or a browser. ``` ## GitHub Integration Commands Reference: COMMANDS.md See the [GitHub Integration Commands Reference in COMMANDS.md](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md) for a **detailed discussion** of all GitHub integration commands, parameters, options, and examples. Check out the next section for a **quick reference**. ## GitHub Integration Commands Quick Reference ### Configuring `gitsome` To properly integrate with GitHub, you must first configure `gitsome`: $ gh configure For GitHub Enterprise users, run with the `-e/--enterprise` flag: $ gh configure -e ### Listing Feeds #### Listing Your News Feed $ gh feed ![Imgur](http://i.imgur.com/2LWcyS6.png) #### Listing A User's Activity Feed View your activity feed or another user's activity feed, optionally through a pager with `-p/--pager`. The [pager option](#option-view-in-a-pager) is available for many commands. $ gh feed donnemartin -p ![Imgur](http://i.imgur.com/kryGLXz.png) #### Listing A Repo's Activity Feed $ gh feed donnemartin/gitsome -p ![Imgur](http://i.imgur.com/d2kxDg9.png) ### Listing Notifications $ gh notifications ![Imgur](http://i.imgur.com/uwmwxsW.png) ### Listing Pull Requests View all pull requests for your repos: $ gh pull-requests ![Imgur](http://i.imgur.com/4A2eYM9.png) ### Filtering Issues View all open issues where you have been mentioned: $ gh issues --issue_state open --issue_filter mentioned ![Imgur](http://i.imgur.com/AB5zxxo.png) View all issues, filtering for only those assigned to you, regardless of state (open, closed): $ gh issues --issue_state all --issue_filter assigned For more information about the filter and state qualifiers, visit the [`gh issues`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-issues) reference in [COMMANDS.md](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md). ### Filtering Starred Repos $ gh starred "repo filter" ![Imgur](http://i.imgur.com/JB88Kw8.png) ### Searching Issues and Repos #### Searching Issues Search issues that have the most +1s: $ gh search-issues "is:open is:issue sort:reactions-+1-desc" -p ![Imgur](http://i.imgur.com/DXXxkBD.png) Search issues that have the most comments: $ gh search-issues "is:open is:issue sort:comments-desc" -p Search issues with the "help wanted" tag: $ gh search-issues "is:open is:issue label:\"help wanted\"" -p Search issues that have your user name tagged **@donnemartin**: $ gh search-issues "is:issue donnemartin is:open" -p Search all your open private issues: $ gh search-issues "is:open is:issue is:private" -p For more information about the query qualifiers, visit the [searching issues reference](https://help.github.com/articles/searching-issues/). #### Searching Repos Search all Python repos created on or after 2015, with >= 1000 stars: $ gh search-repos "created:>=2015-01-01 stars:>=1000 language:python" --sort stars -p ![Imgur](http://i.imgur.com/kazXWWY.png) For more information about the query qualifiers, visit the [searching repos reference](https://help.github.com/articles/searching-repositories/). ### Listing Trending Repos and Devs View trending repos: $ gh trending [language] [-w/--weekly] [-m/--monthly] [-d/--devs] [-b/--browser] ![Imgur](http://i.imgur.com/aa1gOg7.png) View trending devs (devs are currently only supported in browser): $ gh trending [language] --devs --browser ### Viewing Content #### The `view` command View the previously listed notifications, pull requests, issues, repos, users etc, with HTML nicely formatted for your terminal, or optionally in your browser: $ gh view [#] [-b/--browser] ![Imgur](http://i.imgur.com/NVEwGbV.png) #### The `issue` command View an issue: $ gh issue donnemartin/saws/1 ![Imgur](http://i.imgur.com/ZFv9MuV.png) #### The `pull-request` command View a pull request: $ gh pull-request donnemartin/awesome-aws/2 ![Imgur](http://i.imgur.com/3MtKjKy.png) ### Setting Up `.gitignore` List all available `.gitignore` templates: $ gh gitignore-templates ![Imgur](http://i.imgur.com/u8qYx1s.png) Set up your `.gitignore`: $ gh gitignore-template Python > .gitignore ![Imgur](http://i.imgur.com/S5m5ZcO.png) ### Setting Up `LICENSE` List all available `LICENSE` templates: $ gh licenses ![Imgur](http://i.imgur.com/S9SbMLJ.png) Set up your or `LICENSE`: $ gh license MIT > LICENSE ![Imgur](http://i.imgur.com/zJHVxaA.png) ### Summoning Octocat Call on Octocat to say the given message or an Easter egg: $ gh octo [say] ![Imgur](http://i.imgur.com/bNzCa5p.png) ### Viewing Profiles #### Viewing A User's Profile $ gh user octocat ![Imgur](http://i.imgur.com/xVoVPVe.png) #### Viewing Your Profile View your profile with the `gh user [YOUR_USER_ID]` command or with the following shortcut: $ gh me ![Imgur](http://i.imgur.com/csk5j0S.png) ### Creating Comments, Issues, and Repos Create a comment: $ gh create-comment donnemartin/gitsome/1 -t "hello world" Create an issue: $ gh create-issue donnemartin/gitsome -t "title" -b "body" Create a repo: $ gh create-repo gitsome ### Option: View in a Pager Many `gh` commands support a `-p/--pager` option that displays results in a pager, where available. Usage: $ gh [param] [options] -p $ gh [param] [options] --pager ### Option: View in a Browser Many `gh` commands support a `-b/--browser` option that displays results in your default browser instead of your terminal. Usage: $ gh [param] [options] -b $ gh [param] [options] --browser See the [COMMANDS.md](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md) for a detailed listing of all GitHub integration commands, parameters, options, and examples. Having trouble remembering these commands? Check out the handy [autocompleter with interactive help](#git-and-github-autocompleter-with-interactive-help) to guide you through each command. *Note, you can combine `gitsome` with other utilities such as [Git-Extras](https://github.com/tj/git-extras/blob/master/Commands.md).* ## Installation ### Pip Installation [![PyPI version](https://badge.fury.io/py/gitsome.svg)](http://badge.fury.io/py/gitsome) [![PyPI](https://img.shields.io/pypi/pyversions/gitsome.svg)](https://pypi.python.org/pypi/gitsome/) `gitsome` is hosted on [PyPI](https://pypi.python.org/pypi/gitsome). The following command will install `gitsome`: $ pip3 install gitsome You can also install the latest `gitsome` from GitHub source which can contain changes not yet pushed to PyPI: $ pip3 install git+https://github.com/donnemartin/gitsome.git If you are not installing in a `virtualenv`, you might need to run with `sudo`: $ sudo pip3 install gitsome #### `pip3` Depending on your setup, you might also want to run `pip3` with the [`-H flag`](http://stackoverflow.com/a/28619739): $ sudo -H pip3 install gitsome For most linux users, `pip3` can be installed on your system using the `python3-pip` package. For example, Ubuntu users can run: $ sudo apt-get install python3-pip See this [ticket](https://github.com/donnemartin/gitsome/issues/4) for more details. ### Virtual Environment Installation You can install Python packages in a [`virtualenv`](http://docs.python-guide.org/en/latest/dev/virtualenvs/) to avoid potential issues with dependencies or permissions. If you are a Windows user or if you would like more details on `virtualenv`, check out this [guide](http://docs.python-guide.org/en/latest/dev/virtualenvs/). Install `virtualenv` and `virtualenvwrapper`: $ pip3 install virtualenv $ pip3 install virtualenvwrapper $ export WORKON_HOME=~/.virtualenvs $ source /usr/local/bin/virtualenvwrapper.sh Create a `gitsome` `virtualenv` and install `gitsome`: $ mkvirtualenv gitsome $ pip3 install gitsome If the `pip` install does not work, you might be running Python 2 by default. Check what version of Python you are running: $ python --version If the call above results in Python 2, find the path for Python 3: $ which python3 # Python 3 path for mkvirtualenv's --python option Install Python 3 if needed. Set the Python version when calling `mkvirtualenv`: $ mkvirtualenv --python [Python 3 path from above] gitsome $ pip3 install gitsome If you want to activate the `gitsome` `virtualenv` again later, run: $ workon gitsome To deactivate the `gitsome` `virtualenv`, run: $ deactivate ### Running as a Docker Container You can run gitsome in a Docker container to avoid installing Python and `pip3` locally. To install Docker check out the [official Docker documentation](https://docs.docker.com/engine/getstarted/step_one/#step-1-get-docker). Once you have docker installed you can run gitsome: $ docker run -ti --rm mariolet/gitsome You can use Docker volumes to let gitsome access your working directory, your local .gitsomeconfig and .gitconfig: $ docker run -ti --rm -v $(pwd):/src/ \ -v ${HOME}/.gitsomeconfig:/root/.gitsomeconfig \ -v ${HOME}/.gitconfig:/root/.gitconfig \ mariolet/gitsome If you are running this command often you will probably want to define an alias: $ alias gitsome="docker run -ti --rm -v $(pwd):/src/ \ -v ${HOME}/.gitsomeconfig:/root/.gitsomeconfig \ -v ${HOME}/.gitconfig:/root/.gitconfig \ mariolet/gitsome" To build the Docker image from sources: $ git clone https://github.com/donnemartin/gitsome.git $ cd gitsome $ docker build -t gitsome . ### Starting the `gitsome` Shell Once installed, run the optional `gitsome` autocompleter with interactive help: $ gitsome Running the optional `gitsome` shell will provide you with autocompletion, interactive help, fish-style suggestions, a Python REPL, etc. ### Running `gh` Commands Run GitHub-integrated commands: $ gh [param] [options] Note: Running the `gitsome` shell is not required to execute `gh` commands. After [installing](#installation) `gitsome` you can run `gh` commands from any shell. ### Running the `gh configure` Command To properly integrate with GitHub, `gitsome` must be properly configured: $ gh configure #### For GitHub Enterprise Users Run with the `-e/--enterprise` flag: $ gh configure -e View more details in the [gh configure](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-configure) section. ### Enabling Bash Completions By default, `gitsome` looks at the following [locations to enable bash completions](https://github.com/donnemartin/gitsome/blob/master/xonsh/environ.py#L123-L131). To add additional bash completions, update the `~/.xonshrc` file with the location of your bash completions. If `~/.xonshrc` does not exist, create it: $ touch ~/.xonshrc For example, if additional completions are found in `/usr/local/etc/my_bash_completion.d/completion.bash`, add the following line in `~/.xonshrc`: ``` $BASH_COMPLETIONS.append('/usr/local/etc/my_bash_completion.d/completion.bash') ``` You will need to restart `gitsome` for the changes to take effect. ### Enabling `gh` Tab Completions Outside of `gitsome` You can run `gh` commands outside of the `gitsome` shell completer. To enable `gh` tab completions for this workflow, copy the [`gh_complete.sh`](https://github.com/donnemartin/gitsome/blob/master/scripts/gh_complete.sh) file locally. Let bash know completion is available for the `gh` command within your current session: $ source /path/to/gh_complete.sh To enable tab completion for all terminal sessions, add the following to your `bashrc` file: source /path/to/gh_complete.sh Reload your `bashrc`: $ source ~/.bashrc Tip: `.` is the short form of `source`, so you can run this instead: $ . ~/.bashrc #### For Zsh Users `zsh` includes a module which is compatible with bash completions. Download the [`gh_complete.sh`](https://github.com/donnemartin/gitsome/blob/master/scripts/gh_complete.sh) file as above and append the following to your `.zshrc`: autoload bashcompinit bashcompinit source /path/to/gh_complete.sh Reload your `zshrc`: $ source ~/.zshrc ### Optional: Installing `PIL` or `Pillow` Displaying the avatar for the `gh me` and `gh user` commands will require installing the optional `PIL` or `Pillow` dependency. Windows* and Mac: $ pip3 install Pillow *See the [Windows Support](#windows-support) section for limitations on the avatar. Ubuntu users, check out these [instructions on askubuntu](http://askubuntu.com/a/272095) ### Supported Python Versions * Python 3.4 * Python 3.5 * Python 3.6 * Python 3.7 `gitsome` is powered by `xonsh` which does not currently support Python 2.x, as discussed in this [ticket](https://github.com/scopatz/xonsh/issues/66). ### Supported Platforms * Mac OS X * Tested on OS X 10.10 * Linux, Unix * Tested on Ubuntu 14.04 LTS * Windows * Tested on Windows 10 ### Windows Support `gitsome` has been tested on Windows 10 with `cmd` and `cmder`. Although you can use the standard Windows command prompt, you'll probably have a better experience with either [cmder](https://github.com/cmderdev/cmder) or [conemu](https://github.com/Maximus5/ConEmu). ![Imgur](http://i.imgur.com/A1VCsjV.png) #### Text Only Avatar The commands [`gh user`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-user) and [`gh me`](https://github.com/donnemartin/gitsome/blob/master/COMMANDS.md#gh-me) will always have the `-t/--text_avatar` flag enabled, since [`img2txt`](#credits) does not support the ansi avatar on Windows. #### Config File On Windows, the `.gitsomeconfig ` file can be found in `%userprofile%`. For example: C:\Users\dmartin\.gitsomeconfig ## Developer Installation If you're interested in contributing to `gitsome`, run the following commands: $ git clone https://github.com/donnemartin/gitsome.git $ cd gitsome $ pip3 install -e . $ pip3 install -r requirements-dev.txt $ gitsome $ gh [param] [options] #### `pip3` If you get an error while installing saying that you need Python 3.4+, it could be because your `pip` command is configured for an older version of Python. To fix this issue, it is recommended to install `pip3`: $ sudo apt-get install python3-pip See this [ticket](https://github.com/donnemartin/gitsome/issues/4) for more details. ### Continuous Integration [![Build Status](https://travis-ci.org/donnemartin/gitsome.svg?branch=master)](https://travis-ci.org/donnemartin/gitsome) Continuous integration details are available on [Travis CI](https://travis-ci.org/donnemartin/gitsome). ### Unit Tests and Code Coverage Run unit tests in your active Python environment: $ python tests/run_tests.py Run unit tests with [tox](https://pypi.python.org/pypi/tox) on multiple Python environments: $ tox ### Documentation Source code documentation will soon be available on [Readthedocs.org](https://readthedocs.org/). Check out the [source docstrings](https://github.com/donnemartin/gitsome/blob/master/gitsome/githubcli.py). Run the following to build the docs: $ scripts/update_docs.sh ## Contributing Contributions are welcome! Review the [Contributing Guidelines](https://github.com/donnemartin/gitsome/blob/master/CONTRIBUTING.md) for details on how to: * Submit issues * Submit pull requests ## Credits * [click](https://github.com/pallets/click) by [mitsuhiko](https://github.com/mitsuhiko) * [github_trends_rss](https://github.com/ryotarai/github_trends_rss) by [ryotarai](https://github.com/ryotarai) * [github3.py](https://github.com/sigmavirus24/github3.py) by [sigmavirus24](https://github.com/sigmavirus24) * [html2text](https://github.com/aaronsw/html2text) by [aaronsw](https://github.com/aaronsw) * [img2txt](https://github.com/hit9/img2txt) by [hit9](https://github.com/hit9) * [python-prompt-toolkit](https://github.com/jonathanslenders/python-prompt-toolkit) by [jonathanslenders](https://github.com/jonathanslenders) * [requests](https://github.com/kennethreitz/requests) by [kennethreitz](https://github.com/kennethreitz) * [xonsh](https://github.com/scopatz/xonsh) by [scopatz](https://github.com/scopatz) ## Contact Info Feel free to contact me to discuss any issues, questions, or comments. My contact info can be found on my [GitHub page](https://github.com/donnemartin). ## License *I am providing code and resources in this repository to you under an open source license. Because this is my personal repository, the license you receive to my code and resources is from me and not my employer (Facebook).* [![License](http://img.shields.io/:license-apache-blue.svg)](http://www.apache.org/licenses/LICENSE-2.0.html) Copyright 2016 Donne Martin Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: TOS.md ================================================ # Terms of Service By using `gitsome`, you are agreeing to be bound by the following terms and conditions ("Terms of Service"). If you are entering into this agreement on behalf of a company or other legal entity, you represent that you have the authority to bind such entity, its affiliates and all users who access `gitsome` through your account to these terms and conditions, in which case the terms "you" or "your" shall refer to such entity, its affiliates and users associated with it. If you do not have such authority, or if you do not agree with these terms and conditions, you must not accept this agreement and may not use `gitsome`. `gitsome` is maintained by [Donne Martin](donne.martin@gmail.com). Donne Martin reserves the right to update and change the Terms of Service from time to time without notice. Any new features that augment or enhance `gitsome`, including the release of new tools and resources, shall be subject to the Terms of Service. Continued use of `gitsome` after any such changes shall constitute your consent to such changes. ## Account Terms * You are responsible for maintaining the security of your GitHub account. * You are responsible for all activity that occurs by using `gitsome`. * You may not use the `gitsome` for any illegal or unauthorized purpose. * You must not, in the use of `gitsome`, violate any laws in your jurisdiction. * Your use of `gitsome` is at your sole risk. ## Copyright `gitsome` is copyright ©2016 Donne Martin. ## Questions Any questions about this document should be addressed to [Donne Martin](donne.martin@gmail.com). ================================================ FILE: appveyor.yml ================================================ # What Python version is installed where: # http://www.appveyor.com/docs/installed-software#python environment: matrix: - PYTHON: "C:\\Python27" TOX_ENV: "py27" - PYTHON: "C:\\Python33" TOX_ENV: "py33" - PYTHON: "C:\\Python34" TOX_ENV: "py34" - PYTHON: "C:\\Python35" TOX_ENV: "py35" init: - "%PYTHON%/python -V" - "%PYTHON%/python -c \"import struct;print( 8 * struct.calcsize(\'P\'))\"" install: - "%PYTHON%/Scripts/easy_install -U pip" - "%PYTHON%/Scripts/pip install tox" - "%PYTHON%/Scripts/pip install wheel" build: false # Not a C# project, build stuff at the test step instead. test_script: - "%PYTHON%/Scripts/tox -e %TOX_ENV%" after_test: - "%PYTHON%/python setup.py bdist_wheel" - ps: "ls dist" artifacts: - path: dist\* #on_success: # - TODO: upload the content of dist/*.whl to a public wheelhouse ================================================ FILE: codecov.yml ================================================ comment: layout: header, changes, diff coverage: ignore: - gitsome/lib/* - gitsome/compat.py ================================================ FILE: gitsome/__init__.py ================================================ # -*- coding: utf-8 -*- # Copyright 2015 Donne Martin. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of # the License is located at # # http://www.apache.org/licenses/LICENSE-2.0 # # or in the "license" file accompanying this file. This file is # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. __version__ = '0.8.4' ================================================ FILE: gitsome/compat.py ================================================ # -*- coding: utf-8 -*- # Copyright 2015 Donne Martin. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of # the License is located at # # http://www.apache.org/licenses/LICENSE-2.0 # # or in the "license" file accompanying this file. This file is # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. import sys import urllib try: # Python 3 import configparser from urllib.parse import urlparse from urllib.request import urlretrieve from urllib.error import URLError except ImportError: # Python 2 import ConfigParser as configparser from urlparse import urlparse from urllib import urlretrieve from urllib2 import URLError if sys.version_info < (3, 3): import HTMLParser else: import html as HTMLParser ================================================ FILE: gitsome/completer.py ================================================ # -*- coding: utf-8 -*- # Copyright 2015 Donne Martin. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of # the License is located at # # http://www.apache.org/licenses/LICENSE-2.0 # # or in the "license" file accompanying this file. This file is # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. from __future__ import unicode_literals from __future__ import print_function from prompt_toolkit.completion import Completer, Completion from .completions import COMPLETIONS_GH, SUBCOMMANDS from .completions_git import META_LOOKUP_GIT, META_LOOKUP_GIT_EXTRAS from .utils import TextUtils class CompleterGitsome(Completer): """Gitsome prompt toolkit completer. :type text_utils: :class:`utils.TextUtils` :param text_utils: An instance of `utils.TextUtils`. :type fuzzy_match: bool :param fuzzy_match: Determines whether to use fuzzy matching. Currently, this is always set to False but can be enabled. """ def __init__(self): self.fuzzy_match = False self.text_utils = TextUtils() def build_completions_with_meta(self, line, prefix, completions): """Build prompt_toolkit Completions with meta info. :type line: list :param line: The input text as a list of words. :type prefix: string :param prefix: The current word. :type completions: list :param completions: Completions to build meta info for. :rtype: list :return: Completions with meta info. """ completions_with_meta = [] tokens = line.split(' ') if tokens[0] != 'gh': for comp in completions: display = None display_meta = None if 'git' in line and comp.strip() in META_LOOKUP_GIT: display_meta = META_LOOKUP_GIT[comp.strip()] elif 'git' in line and comp.strip() in META_LOOKUP_GIT_EXTRAS: display_meta = META_LOOKUP_GIT_EXTRAS[comp.strip()] completions_with_meta.append( Completion(comp, -len(prefix), display=display, display_meta=display_meta)) return completions_with_meta def completing_command(self, words, word_before_cursor): """Determine if we are currently completing the gh command. :type words: list :param words: The input text broken into word tokens. :type word_before_cursor: str :param word_before_cursor: The current word before the cursor, which might be one or more blank spaces. :rtype: bool :return: Specifies whether we are currently completing the gh command. """ if len(words) == 1 and word_before_cursor != '': return True else: return False def completing_subcommand(self, words, word_before_cursor): """Determine if we are currently completing a subcommand. :type words: list :param words: The input text broken into word tokens. :type word_before_cursor: str :param word_before_cursor: The current word before the cursor, which might be one or more blank spaces. :rtype: bool :return: Specifies whether we are currently completing a subcommand. """ if (len(words) == 1 and word_before_cursor == '') \ or (len(words) == 2 and word_before_cursor != ''): return True else: return False def completing_arg(self, words, word_before_cursor): """Determine if we are currently completing an arg. :type words: list :param words: The input text broken into word tokens. :type word_before_cursor: str :param word_before_cursor: The current word before the cursor, which might be one or more blank spaces. :rtype: bool :return: Specifies whether we are currently completing an arg. """ if (len(words) == 2 and word_before_cursor == '') \ or (len(words) == 3 and word_before_cursor != ''): return True else: return False def completing_subcommand_option(self, words, word_before_cursor): """Determine if we are currently completing an option. :type words: list :param words: The input text broken into word tokens. :type word_before_cursor: str :param word_before_cursor: The current word before the cursor, which might be one or more blank spaces. :rtype: list :return: A list of options. """ options = [] for subcommand, args_opts in COMPLETIONS_GH.items(): if subcommand in words and \ (words[-2] == subcommand or self.completing_subcommand_option_util(subcommand, words)): options.extend(COMPLETIONS_GH[subcommand]['opts']) return options def completing_subcommand_option_util(self, option, words): """Determine if we are currently completing an option. Called by completing_subcommand_option as a utility method. :type words: list :param words: The input text broken into word tokens. :type word_before_cursor: str :param word_before_cursor: The current word before the cursor, which might be one or more blank spaces. :rtype: bool :return: Specifies whether we are currently completing an option. """ # Example: Return True for: gh view 1 --pag if len(words) > 3: if option in words: return True return False def arg_completions(self, words, word_before_cursor): """Generates arguments completions based on the input. :type words: list :param words: The input text broken into word tokens. :type word_before_cursor: str :param word_before_cursor: The current word before the cursor, which might be one or more blank spaces. :rtype: list :return: A list of completions. """ if 'gh' not in words: return [] for subcommand, args_opts in COMPLETIONS_GH.items(): if subcommand in words: args = list(COMPLETIONS_GH[subcommand]['args'].keys()) if not args: # Some commands don't have args, complete options instead. args = list(COMPLETIONS_GH[subcommand]['opts'].keys()) return args if args else [] return [] def get_completions(self, document, _): """Get completions for the current scope. :type document: :class:`prompt_toolkit.Document` :param document: An instance of `prompt_toolkit.Document`. :type _: :class:`prompt_toolkit.completion.Completion` :param _: (Unused). :rtype: generator :return: Yields an instance of `prompt_toolkit.completion.Completion`. """ word_before_cursor = document.get_word_before_cursor(WORD=True) words = self.text_utils.get_tokens(document.text) commands = [] if len(words) == 0: return commands if self.completing_command(words, word_before_cursor): commands = ['gh'] else: if 'gh' not in words: return commands if self.completing_subcommand(words, word_before_cursor): commands = list(SUBCOMMANDS.keys()) else: if self.completing_arg(words, word_before_cursor): commands = self.arg_completions(words, word_before_cursor) else: commands = self.completing_subcommand_option( words, word_before_cursor) completions = self.text_utils.find_matches( word_before_cursor, commands, fuzzy=self.fuzzy_match) return completions ================================================ FILE: gitsome/completions.py ================================================ # -*- coding: utf-8 -*- # Copyright 2015 Donne Martin. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of # the License is located at # # http://www.apache.org/licenses/LICENSE-2.0 # # or in the "license" file accompanying this file. This file is # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. COMPLETIONS_GH = { 'configure': { 'desc': "Configures gitsome.", 'args': {}, 'opts': { '-e': 'flag (opt) configure GitHub Enterprise.', '--enterprise': 'flag (opt) configure GitHub Enterprise.', }, }, 'create-comment': { 'desc': 'Creates a comment on the given issue.', 'args': { 'octocat/Spoon-Knife/1': 'str (req) user/repo/issue_number combo.', }, 'opts': { '-t': 'see associated -- option for details.', '--text': 'str (req) comment text.', }, }, 'create-issue': { 'desc': 'Creates an issue.', 'args': { 'octocat/Spoon-Knife': 'str (req) user/repository combo.', }, 'opts': { '-t': 'see associated -- option for details.', '--issue_title': 'str (req) issue title.', '-d': 'str (opt) issue description.', '--issue_desc': 'str (opt) issue description.', }, }, 'create-repo': { 'desc': 'Creates a repository.', 'args': { 'Spoon-Knife': 'str (req) repository name.', }, 'opts': { '-d': 'str (opt) repo description', '--repo_desc': 'str (opt) repo description.', '-pr': 'flag (opt) create a private repo', '--private': 'flag (opt) create a private repo', }, }, 'emails': { 'desc': "Lists all the user's registered emails.", 'args': {}, 'opts': {}, }, 'emojis': { 'desc': 'Lists all GitHub supported emojis.', 'args': {}, 'opts': { '-p': 'flag (req) show results in a pager.', '--pager': 'flag (req) show results in a pager.', }, }, 'feed': { 'desc': "Lists all activity for the given user or repo, if called with no arg, shows the logged in user's feed.", 'args': { 'octocat/Hello-World --pager': "str (opt) user or user/repository combo, if blank, shows the logged in user's feed.", }, 'opts': { '-pr': 'flag (req) also show private events.', '--private': 'flag (req) also show private events.', '-p': 'flag (req) show results in a pager.', '--pager': 'flag (req) show results in a pager.', }, }, 'followers': { 'desc': 'Lists all followers and the total follower count.', 'args': { 'octocat': "str (req) the user's login id, if blank, shows logged in user's info.", }, 'opts': { '-p': 'flag (opt) show results in a pager.', '--pager': 'flag (opt) show results in a pager.', }, }, 'following': { 'desc': 'Lists all followed users and the total followed count.', 'args': { 'octocat': "str (req) the user's login id, if blank, shows logged in user's info.", }, 'opts': { '-p': 'flag (opt) show results in a pager.', '--pager': 'flag (opt) show results in a pager.', }, }, 'gitignore-template': { 'desc': 'Outputs the gitignore template for the given language.', 'args': { 'Python': 'str (req) the language-specific .gitignore.', }, 'opts': {}, }, 'gitignore-templates': { 'desc': 'Outputs all supported gitignore templates.', 'args': {}, 'opts': { '-p': 'flag (opt) show results in a pager.', '--pager': 'flag (opt) show results in a pager.', }, }, 'issue': { 'desc': 'Outputs detailed information about the given issue.', 'args': { 'octocat/Spoon-Knife/1': 'str (req) user/repo/issue_number combo.', }, 'opts': {}, }, 'issues': { 'desc': 'Lists all issues matching the filter.', 'args': {}, 'opts': { '-f': 'str (opt) "assigned", "created", "mentioned", "subscribed" (default).', '--issue_filter': 'str (opt) "assigned", "created", "mentioned", "subscribed" (default).', '-s': 'str (opt) "all", "open" (default), "closed".', '--issue_state': 'str (opt) "all", "open" (default), "closed".', '-l': 'flag (opt) num items to show, defaults to 1000.', '--limit': 'flag (opt) num items to show, defaults to 1000.', '-p': 'flag (opt) show results in a pager.', '--pager': 'flag (opt) show results in a pager.', }, }, 'license': { 'desc': 'Outputs the license template for the given license.', 'args': { 'apache-2.0': 'str (req) the license name.', }, 'opts': {}, }, 'licenses': { 'desc': 'Outputs all supported license templates.', 'args': {}, 'opts': {}, }, 'me': { 'desc': 'Lists information about the logged in user.', 'args': {}, 'opts': { '-b': 'flag (opt) view profile in a browser instead of the terminal.', '--browser': 'flag (opt) view profile in a browser instead of the terminal.', '-t': 'see associated -- option for details.', '--text_avatar': 'flag (opt) view profile pic in plain text.', '-l': 'flag (opt) num items to show, defaults to 1000.', '--limit': 'flag (opt) num items to show, defaults to 1000.', '-p': 'flag (opt) show results in a pager.', '--pager': 'flag (opt) show results in a pager.', }, }, 'notifications': { 'desc': 'Lists all notifications.', 'args': {}, 'opts': { '-l': 'flag (opt) num items to show, defaults to 1000.', '--limit': 'flag (opt) num items to show, defaults to 1000.', '-p': 'flag (opt) show results in a pager.', '--pager': 'flag (opt) show results in a pager.', }, }, 'octo': { 'desc': 'Outputs an Easter egg or the given message from Octocat.', 'args': { '"Keep it logically awesome"': 'str (req) a message from Octocat, if empty, Octocat speaks an Easter egg.', }, 'opts': {}, }, 'pull-request': { 'desc': 'Outputs detailed information about the given pull request.', 'args': { 'octocat/Spoon-Knife/3': 'str (req) user/repo/pull_number combo.', }, 'opts': {}, }, 'pull-requests': { 'desc': 'Lists all pull requests.', 'args': {}, 'opts': { '-l': 'flag (opt) num items to show, defaults to 1000.', '--limit': 'flag (opt) num items to show, defaults to 1000.', '-p': 'flag (opt) show results in a pager.', '--pager': 'flag (opt) show results in a pager.', }, }, 'rate-limit': { 'desc': 'Outputs the rate limit.', 'args': {}, 'opts': {}, }, 'repo': { 'desc': 'Outputs detailed information about the given repo.', 'args': { 'octocat/Spoon-Knife': 'str (req) user/repository combo.', }, 'opts': {}, }, 'repos': { 'desc': 'Lists all repos matching the given filter.', 'args': { '"foo bar optional filter"': 'str (opt) filters repos by name.', }, 'opts': { '-l': 'flag (opt) num items to show, defaults to 1000.', '--limit': 'flag (opt) num items to show, defaults to 1000.', '-p': 'flag (opt) show results in a pager.', '--pager': 'flag (opt) show results in a pager.', }, }, 'search-issues': { 'desc': 'Searches for all issues matching the given query.', 'args': { '"foobarbaz in:title created:>=2015-01-01"': 'str (req) the search query.', }, 'opts': { '-s': 'str (opt) "stars", "forks", "updated", if blank, sorts by query best match.', '--sort': 'str (opt) "stars", "forks", "updated", if blank, sorts by query best match.', '-l': 'flag (opt) num items to show, defaults to 1000.', '--limit': 'flag (opt) num items to show, defaults to 1000.', '-p': 'flag (opt) show results in a pager.', '--pager': 'flag (opt) show results in a pager.', }, }, 'search-repos': { 'desc': 'Searches for all repos matching the given query.', 'args': { '"created:>=2015-01-01 stars:>=1500 language:python"': 'str (req) the search query.', }, 'opts': { '-s': 'str (opt) "stars", "forks", "updated", if blank, sorts by query best match.', '--sort': 'str (opt) "stars", "forks", "updated", if blank, sorts by query best match.', '-l': 'flag (opt) num items to show, defaults to 1000.', '--limit': 'flag (opt) num items to show, defaults to 1000.', '-p': 'flag (opt) show results in a pager.', '--pager': 'flag (opt) show results in a pager.', }, }, 'starred': { 'desc': 'Outputs starred repos.', 'args': { '"foo bar optional filter"': 'str (opt) filters repos by name.', }, 'opts': { '-l': 'flag (opt) num items to show, defaults to 1000.', '--limit': 'flag (opt) num items to show, defaults to 1000.', '-p': 'flag (opt) show results in a pager.', '--pager': 'flag (opt) show results in a pager.', }, }, 'trending': { 'desc': 'Lists trending repos for the given language.', 'args': { 'Overall': 'str (opt) the language filter for trending repos, if blank, the overall rankings are shown.', }, 'opts': { '-w': 'flag (opt) show the weekly trending repos.', '--weekly': 'flag (opt) show the weekly trending repos.', '-m': 'flag (opt) show the monthly trending repos.', '--monthly': 'flag (opt) show the monthly trending repos.', '-D': 'flag (opt) view trending devs. Only valid with -b/--browser.', '--devs': 'flag (opt) view trending devs. Only valid with -b/--browser.', '-b': 'flag (opt) view profile in a browser instead of the terminal.', '--browser': 'flag (opt) view profile in a browser instead of the terminal.', '-p': 'flag (opt) show results in a pager.', '--pager': 'flag (opt) show results in a pager.', }, }, 'user': { 'desc': 'Lists information about the given user.', 'args': { 'github --pager': "str (req) the user's login id.", }, 'opts': { '-b': 'flag (opt) view profile in a browser instead of the terminal.', '--browser': 'flag (opt) view profile in a browser instead of the terminal.', '-t': 'see associated -- option for details.', '--text_avatar': 'flag (opt) view profile pic in plain text.', '-l': 'flag (opt) num items to show, defaults to 1000.', '--limit': 'flag (opt) num items to show, defaults to 1000.', '-p': 'flag (opt) show results in a pager.', '--pager': 'flag (opt) show results in a pager.', }, }, 'view': { 'desc': 'Views the given repo or issue index in the terminal or a browser.', 'args': { '1': 'int (req) the 1-based index to view.', }, 'opts': { '-b': 'flag (opt) view profile in a browser instead of the terminal.', '--browser': 'flag (opt) view profile in a browser instead of the terminal.', }, }, } META_LOOKUP_GH = { '10': 'limit: int (opt) limits the posts displayed', '"(?i)(Python|Django)"': ('regex_query: string (opt) applies a regular ' 'expression comment filter'), '1': 'index: int (req) views the post index', '"user"': 'user:string (req) shows info on the specified user', 'gh': 'Git auto-completer with GitHub integration.', } SUBCOMMANDS = {} def build_meta_lookups(): for subcommand, args_opts in COMPLETIONS_GH.items(): META_LOOKUP_GH.update({subcommand: COMPLETIONS_GH[subcommand]['desc']}) SUBCOMMANDS.update({subcommand: COMPLETIONS_GH[subcommand]['desc']}) for opt, meta in args_opts['opts'].items(): META_LOOKUP_GH.update({opt: meta}) for arg, meta in args_opts['args'].items(): META_LOOKUP_GH.update({arg: meta}) build_meta_lookups() ================================================ FILE: gitsome/completions_git.py ================================================ # -*- coding: utf-8 -*- # Copyright 2015 Donne Martin. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of # the License is located at # # http://www.apache.org/licenses/LICENSE-2.0 # # or in the "license" file accompanying this file. This file is # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. META_LOOKUP_GIT = { 'git': 'The stupid content tracker.', '--version': 'Prints the Git suite version that the git program came from.', '--help': 'Prints the synopsis and a list of the most commonly used commands.', '-C': 'Run as if git was started in instead of the current working directory.', '-c': 'Pass a configuration parameter to the command.', '--exec-path': 'Path to wherever your core Git programs are installed.', '--html-path': "Print the path, without trailing slash, where Git's HTML documentation is installed and exit.", '--man-path': 'Print the manpath (see man(1)) for the man pages for this version of Git and exit.', '--info-path': 'Print the path where the Info files documenting this version of Git are installed and exit.', '-p': 'Pipe all output into less (or if set, $PAGER) if standard output is a terminal.', '--paginate': 'Pipe all output into less (or if set, $PAGER) if standard output is a terminal.', '--no-pager': 'Do not pipe Git output into a pager.', '--git-dir=': 'Set the path to the repository. This can also be controlled by setting the GIT_DIR environment variable.', '--work-tree=': 'Set the path to the working tree. It can be an absolute path or a path relative to the current working directory.', '--namespace=': 'Set the Git namespace. See gitnamespaces(7) for more details.', '--bare': 'Treat the repository as a bare repository. If GIT_DIR environment is not set, it is set to the current working directory.', '--no-replace-objects': 'Do not use replacement refs to replace Git objects.', '--literal-pathspecs': 'Treat pathspecs literally (i.e. no globbing, no pathspec magic).', '--glob-pathspecs': 'Add "glob" magic to all pathspec.', '--noglob-pathspecs': 'Add "literal" magic to all pathspec.', '--icase-pathspecs': 'Add "icase" magic to all pathspec.', 'add': 'Add file contents to the index.', 'am': 'Apply a series of patches from a mailbox.', 'amend': 'Amend the previous commit.', 'archive': 'Create an archive of files from a named tree.', 'bisect': 'Use binary search to find the commit that introduced a bug.', 'branch': 'List, create, or delete branches.', 'branches': 'List all branches, including remotes.', 'bundle': 'Move objects and refs by archive.', 'c': 'Alias for git clone.', 'ca': 'Alias for git add -A && git commit -av.', 'checkout': 'Switch branches or restore working tree files.', 'cherry-pick': 'Apply the changes introduced by some existing commits.', 'citool': 'Graphical alternative to git-commit.', 'clean': 'Remove untracked files from the working tree.', 'clone': 'Clone a repository into a new directory.', 'commit': 'Record changes to the repository.', 'contributors': 'List repo contributors.', 'describe': 'Describe a commit using the most recent tag reachable from it.', 'diff': 'Show changes between commits, commit and working tree, etc.', 'fetch': 'Download objects and refs from another repository.', 'flow': 'Git branching model extensions.', 'format-patch': 'Prepare patches for e-mail submission.', 'gc': 'Cleanup unnecessary files and optimize the local repository.', 'grep': 'Print lines matching a pattern.', 'gui': 'A portable graphical interface to Git.', 'init': 'Create an empty Git repository or reinitialize an existing one.', 'log': 'Show commit logs.', 'merge': 'Join two or more development histories together.', 'mv': 'Move or rename a file, a directory, or a symlink.', 'notes': 'Add or inspect object notes.', 'pull': 'Fetch from and integrate with another repository or a local branch.', 'push': 'Update remote refs along with associated objects.', 'rebase': 'Forward-port local commits to the updated upstream head.', 'reset': 'Reset current HEAD to the specified state.', 'revert': 'Revert some existing commits.', 'rm': 'Remove files from the working tree and from the index.', 'shortlog': 'Summarize git log output.', 'show': 'Show various types of objects.', 'stage': 'Add file contents to the staging area.', 'stash': 'Stash the changes in a dirty working directory away.', 'status': 'Show the working tree status.', 'submodule': 'Initialize, update or inspect submodules.', 'tag': 'Create, list, delete or verify a tag object signed with GPG.', 'tags': 'List all tags.', 'worktree': 'Manage multiple working trees.', 'gitk': 'The Git repository browser.', 'config': 'Get and set repository or global options.', 'fast-export': 'Git data exporter.', 'fast-import': 'Backend for fast Git data importers.', 'filter-branch': 'Rewrite branches.', 'mergetool': 'Run merge conflict resolution tools to resolve merge conflicts.', 'pack-refs': 'Pack heads and tags for efficient repository access.', 'prune': 'Prune all unreachable objects from the object database.', 'reflog': 'Manage reflog information.', 'relink': 'Hardlink common objects in local repositories.', 'remote': 'Manage set of tracked repositories.', 'remotes': 'List set of tracked repositories.', 'repack': 'Pack unpacked objects in a repository.', 'replace': 'Create, list, delete refs to replace objects.', 'annotate': 'Annotate file lines with commit information.', 'blame': 'Show what revision and author last modified each line of a file.', 'cherry': 'Find commits yet to be applied to upstream.', 'count-objects': 'Count unpacked number of objects and their disk consumption.', 'difftool': 'Show changes using common diff tools.', 'fsck': 'Verifies the connectivity and validity of the objects in the database.', 'get-tar-commit-id': 'Extract commit ID from an archive created using git-archive.', 'help': 'Display help information about Git.', 'instaweb': 'Instantly browse your working repository in gitweb.', 'merge-tree': 'Show three-way merge without touching index.', 'rerere': 'Reuse recorded resolution of conflicted merges.', 'rev-parse': 'Pick out and massage parameters.', 'show-branch': 'Show branches and their commits.', 'verify-commit': 'Check the GPG signature of commits.', 'verify-tag': 'Check the GPG signature of tags.', 'whatchanged': 'Show logs with difference each commit introduces.', 'gitweb': 'Git web interface (web frontend to Git repositories).', 'archimport': 'Import an Arch repository into Git.', 'cvsexportcommit': 'Export a single commit to a CVS checkout.', 'cvsimport': 'Salvage your data out of another SCM people love to hate.', 'cvsserver': 'A CVS server emulator for Git.', 'imap-send': 'Send a collection of patches from stdin to an IMAP folder.', 'p4': 'Import from and submit to Perforce repositories.', 'quiltimport': 'Applies a quilt patchset onto the current branch.', 'request-pull': 'Generates a summary of pending changes.', 'send-email': 'Send a collection of patches as emails.', 'svn': 'Bidirectional operation between a Subversion repository and Git.', 'apply': 'Apply a patch to files and/or to the index.', 'checkout-index': 'Copy files from the index to the working tree.', 'commit-tree': 'Create a new commit object.', 'hash-object': 'Compute object ID and optionally creates a blob from a file.', 'index-pack': 'Build pack index file for an existing packed archive.', 'lfs': 'Git extension for versioning large files.', 'merge-file': 'Run a three-way file merge.', 'merge-index': 'Run a merge for files needing merging.', 'mktag': 'Creates a tag object.', 'mktree': 'Build a tree-object from ls-tree formatted text.', 'pack-objects': 'Create a packed archive of objects.', 'prune-packed': 'Remove extra objects that are already in pack files.', 'read-tree': 'Reads tree information into the index.', 'symbolic-ref': 'Read, modify and delete symbolic refs.', 'unpack-objects': 'Unpack objects from a packed archive.', 'update-index': 'Register file contents in the working tree to the index.', 'update-ref': 'Update the object name stored in a ref safely.', 'write-tree': 'Create a tree object from the current index.', 'cat-file': 'Provide content or type and size information for repository objects.', 'diff-files': 'Compares files in the working tree and the index.', 'diff-index': 'Compare a tree to the working tree or index.', 'diff-tree': 'Compares the content and mode of blobs found via two tree objects.', 'for-each-ref': 'Output information on each ref.', 'ls-files': 'Show information about files in the index and the working tree.', 'ls-remote': 'List references in a remote repository.', 'ls-tree': 'List the contents of a tree object.', 'merge-base': 'Find as good common ancestors as possible for a merge.', 'name-rev': 'Find symbolic names for given revs.', 'pack-redundant': 'Find redundant pack files.', 'rev-list': 'Lists commit objects in reverse chronological order.', 'show-index': 'Show packed archive index.', 'show-ref': 'List references in a local repository.', 'unpack-file': "Creates a temporary file with a blob's contents.", 'var': 'Show a Git logical variable.', 'verify-pack': 'Validate packed Git archive files.', 'daemon': 'A really simple server for Git repositories.', 'fetch-pack': 'Receive missing objects from another repository.', 'http-backend': 'Server side implementation of Git over HTTP.', 'send-pack': 'Push objects over Git protocol to another repository.', 'update-server-info': 'Update auxiliary info file to help dumb servers.', 'http-fetch': 'Download from a remote Git repository via HTTP.', 'http-push': 'Push objects over HTTP/DAV to another repository.', 'parse-remote': 'Routines to help parsing remote repository access parameters.', 'receive-pack': 'Receive what is pushed into the repository.', 'shell': 'Restricted login shell for Git-only SSH access.', 'upload-archive': 'Send archive back to git-archive.', 'upload-pack': 'Send objects packed back to git-fetch-pack.', 'check-attr': 'Display gitattributes information.', 'check-ignore': 'Debug gitignore / exclude files.', 'check-mailmap': 'Show canonical names and email addresses of contacts.', 'check-ref-format': 'Ensures that a reference name is well formed.', 'column': 'Display data in columns.', 'credential': 'Retrieve and store user credentials.', 'credential-cache': 'Helper to temporarily store passwords in memory.', 'credential-store': 'Helper to store credentials on disk.', 'fmt-merge-msg': 'Produce a merge commit message.', 'interpret-trailers': 'help add structured information into commit messages.', 'mailinfo': 'Extracts patch and authorship from a single e-mail message.', 'mailsplit': 'Simple UNIX mbox splitter program.', 'merge-one-file': 'The standard helper program to use with git-merge-index.', 'patch-id': 'Compute unique ID for a patch.', 'sh-i18n': "Git's i18n setup code for shell scripts.", 'sh-setup': 'Common Git shell script setup code.', 'stripspace': 'Remove unnecessary whitespace.', 'subtree': 'Merge subtrees together and split repository into subtrees.', 'git-cvsserver': 'A CVS server emulator for Git.', 'git-credential-osxkeychain': 'Cache GitHub credentials in the OS X Keychain.', 'git-lfs': 'Git extension for versioning large files.', 'git-receive-pack': 'Receive what is pushed into the repository', 'git-shell': 'Restricted login shell for Git-only SSH access.', 'git-subtree': 'Merge subtrees together and split repository into subtrees.', 'git-upload-archive': 'Send archive back to git-archive.', 'git-upload-pack': 'Send objects packed back to git-fetch-pack.', 'git-flow': 'Git branching model extensions.', 'gitchangelog': 'Generates a changelog from git tags and commit messages.', } META_LOOKUP_GIT_EXTRAS = { 'git-extras': 'Various Git utilities.', 'git-alias': 'Define, search and show aliases.', 'git-archive-file': 'Export the current HEAD of the git repository to a archive.', 'git-authors': 'Generate authors report.', 'git-back': 'Undo and Stage latest commits.', 'git-bug': 'Create bug branch.', 'git-changelog': 'Generate a changelog report.', 'git-chore': 'Create chore branch.', 'git-clear': 'Rigorously clean up a repository.', 'git-commits-since': 'Show commit logs since some date.', 'git-contrib': "Show user's contributions", 'git-count': 'Show commit count.', 'git-create-branch': 'Create branches.', 'git-delete-branch': 'Delete branches.', 'git-delete-merged-branches': 'Delete merged branches.', 'git-delete-submodule': 'Delete submodules.', 'git-delete-tag': 'Delete tags.', 'git-delta': 'Lists changed files.', 'git-effort': 'Show effort statistics on file(s).', 'git-feature': 'Create/Merge feature branch.', 'git-fork': 'Fork a repo on github.', 'git-fresh-branch': 'Create fresh branches.', 'git-gh-pages': 'Create the GitHub Pages branch.', 'git-graft': 'Merge and destroy a given branch.', 'git-guilt': 'calculate change between two revisions.', 'git-ignore-io': 'Get sample gitignore file.', 'git-ignore': 'Add .gitignore patterns.', 'git-info': 'Returns information on current repository.', 'git-line-summary': 'Show repository summary by line.', 'git-local-commits': 'List local commits.', 'git-lock': 'Lock a file excluded from version control.', 'git-locked': 'ls files that have been locked.', 'git-merge-into': 'Merge one branch into another.', 'git-merge-repo': 'Merge two repo histories.', 'git-missing': 'Show commits missing from another branch.', 'git-obliterate': 'Completely remove a file from the repository, including past commits and tags.', 'git-pr': 'Checks out a pull request locally.', 'git-psykorebase': 'Rebase a branch with a merge commit.', 'git-pull-request': 'Checks out a pull request from GitHub.', 'git-rebase-patch': 'Rebases a patch.', 'git-refactor': 'Create refactor branch.', 'git-release': 'Commit, tag and push changes to the repository.', 'git-rename-tag': 'Rename a tag.', 'git-repl': 'git read-eval-print-loop.', 'git-reset-file': 'Reset one file.', 'git-root': 'show path of root.', 'git-rscp': 'Copies specific files from the working directory of a remote repository to the current working directory.', 'git-scp': 'Copy files to SSH compatible git-remote.', 'git-sed': 'replace patterns in git-controlled files.', 'git-setup': 'Set up a git repository.', 'git-show-merged-branches': 'Show merged branches.', 'git-show-tree': 'show branch tree of commit history.', 'git-show-unmerged-branches': 'Show unmerged branches.', 'git-squash': 'Import changes from a branch.', 'git-summary': 'Show repository summary.', 'git-touch': 'Touch and add file to the index.', 'git-undo': 'Remove latest commits.', 'git-unlock': 'Unlock a file excluded from version control.', 'extras': 'Various Git utilities.', 'alias': 'Define, search and show aliases.', 'archive-file': 'Export the current HEAD of the git repository to a archive.', 'authors': 'Generate authors report.', 'back': 'Undo and Stage latest commits.', 'bug': 'Create bug branch.', 'changelog': 'Generate a changelog report.', 'chore': 'Create chore branch.', 'clear': 'Rigorously clean up a repository.', 'commits-since': 'Show commit logs since some date.', 'contrib': "Show user's contributions", 'count': 'Show commit count.', 'create-branch': 'Create branches.', 'delete-branch': 'Delete branches.', 'delete-merged-branches': 'Delete merged branches.', 'delete-submodule': 'Delete submodules.', 'delete-tag': 'Delete tags.', 'delta': 'Lists changed files.', 'effort': 'Show effort statistics on file(s).', 'feature': 'Create/Merge feature branch.', 'fork': 'Fork a repo on github.', 'fresh-branch': 'Create fresh branches.', 'gh-pages': 'Create the GitHub Pages branch.', 'graft': 'Merge and destroy a given branch.', 'guilt': 'calculate change between two revisions.', 'ignore-io': 'Get sample gitignore file.', 'ignore': 'Add .gitignore patterns.', 'info': 'Returns information on current repository.', 'line-summary': 'Show repository summary by line.', 'local-commits': 'List local commits.', 'lock': 'Lock a file excluded from version control.', 'locked': 'ls files that have been locked.', 'merge-into': 'Merge one branch into another.', 'merge-repo': 'Merge two repo histories.', 'missing': 'Show commits missing from another branch.', 'obliterate': 'Completely remove a file from the repository, including past commits and tags.', 'pr': 'Checks out a pull request locally.', 'psykorebase': 'Rebase a branch with a merge commit.', 'pull-request': 'Checks out a pull request from GitHub.', 'rebase-patch': 'Rebases a patch.', 'refactor': 'Create refactor branch.', 'release': 'Commit, tag and push changes to the repository.', 'rename-tag': 'Rename a tag.', 'repl': 'git read-eval-print-loop.', 'reset-file': 'Reset one file.', 'root': 'show path of root.', 'rscp': 'Copies specific files from the working directory of a remote repository to the current working directory.', 'scp': 'Copy files to SSH compatible git-remote.', 'sed': 'replace patterns in git-controlled files.', 'setup': 'Set up a git repository.', 'show-merged-branches': 'Show merged branches.', 'show-tree': 'show branch tree of commit history.', 'show-unmerged-branches': 'Show unmerged branches.', 'squash': 'Import changes from a branch.', 'summary': 'Show repository summary.', 'touch': 'Touch and add file to the index.', 'undo': 'Remove latest commits.', 'unlock': 'Unlock a file excluded from version control.', } ================================================ FILE: gitsome/config.py ================================================ # -*- coding: utf-8 -*- # Copyright 2015 Donne Martin. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of # the License is located at # # http://www.apache.org/licenses/LICENSE-2.0 # # or in the "license" file accompanying this file. This file is # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. from __future__ import unicode_literals from __future__ import print_function import click from getpass import getpass import os import requests from requests.packages.urllib3.exceptions import InsecureRequestWarning from .compat import configparser from .lib.github3 import authorize, enterprise_login, login from .lib.github3.exceptions import AuthenticationFailed, UnprocessableEntity class Config(object): """Gitsome config. :type api: :class:`github3.github.Github` :param api: An instance of github3.github.Github. :type clr_x: str :param clr_x: Various ansi color config colors to use for highlights. :type CONFIG: str :param CONFIG: The config file name. :type CONFIG_SECTION: str :param CONFIG_SECTION: The main config file section label. :type CONFIG_CLR_X: str :param CONFIG_CLR_X: Various ansi color config labels to use for highlights. :type CONFIG_ENTERPRISE_URL: str :param CONFIG_ENTERPRISE_URL: The GitHub Enterprise url. :type CONFIG_USER_LOGIN: str :param CONFIG_USER_LOGIN: The user login. :type CONFIG_USER_PASS: str :param CONFIG_USER_PASS: The user password. :type CONFIG_USER_TOKEN: str :param CONFIG_USER_TOKEN: The user token. :type CONFIG_USER_FEED: str :param CONFIG_USER_FEED: The user feed config. This is the feed on https://github.com/ when logged in and requires the basic auth model, which doesn't work when logging in with tokens or 2FA. This config listed the pre-signed url to access the feed. :type CONFIG_URL: str :param CONFIG_URL: The config file name that contains urls used in the `gh view` command. :type CONFIG_URL_SECTION: str :param CONFIG_URL_SECTION: The config file section that contains urls used in the `gh view [url_index]` command. :type CONFIG_URL_LIST: str :param CONFIG_URL_LIST: The config containing a list of the last set of urls the user has seen, which allows the user to quickly access a repo url with the `gh view [url_index]` command. :type CONFIG_VERIFY_SSL: str :param CONFIG_VERIFY_SSL: Determines whether to verify SSL certs. :type enterprise_url: str :param enterprise_url: The GitHub Enterprise url. :type urls: list :param urls: The last set of urls the user has seen, which allows the user to quickly access a repo url with the gh view [url_index] command. :type user_login: str :param user_login: The user's login in ~/.gitsomeconfig. :type user_pass: str :param user_pass: The user's pass in ~/.gitsomeconfig. This is only stored for GitHub Enterprise users since using only a personal access token does not seem to be supported. :type user_token: str :param user_token: The user's token in ~/.gitsomeconfig. :type verify_ssl: bool :param verify_ssl: Determines whether to verify SSL certs. :type enable_avatar: bool :param enable_avatar: Determines whether to request avatar image. """ CONFIG = '.gitsomeconfig' CONFIG_CLR_PRIMARY = 'clr_primary' CONFIG_CLR_SECONDARY = 'clr_secondary' CONFIG_CLR_TERTIARY = 'clr_tertiary' CONFIG_CLR_QUATERNARY = 'clr_quaternary' CONFIG_CLR_BOLD = 'clr_bold' CONFIG_CLR_CODE = 'clr_code' CONFIG_CLR_ERROR = 'clr_error' CONFIG_CLR_HEADER = 'clr_header' CONFIG_CLR_LINK = 'clr_link' CONFIG_CLR_LIST = 'clr_list' CONFIG_CLR_MESSAGE = 'clr_message' CONFIG_CLR_NUM_COMMENTS = 'clr_num_comments' CONFIG_CLR_NUM_POINTS = 'clr_num_points' CONFIG_CLR_TAG = 'clr_tag' CONFIG_CLR_TIME = 'clr_time' CONFIG_CLR_TITLE = 'clr_title' CONFIG_CLR_TOOLTIP = 'clr_tooltip' CONFIG_CLR_USER = 'clr_user' CONFIG_CLR_VIEW_LINK = 'clr_view_link' CONFIG_CLR_VIEW_INDEX = 'clr_view_index' CONFIG_SECTION = 'github' CONFIG_USER_LOGIN = 'user_login' CONFIG_USER_PASS = 'user_pass' CONFIG_USER_TOKEN = 'user_token' CONFIG_USER_FEED = 'user_feed' CONFIG_ENTERPRISE_URL = 'enterprise_url' CONFIG_VERIFY_SSL = 'verify_ssl' CONFIG_URL = '.gitsomeconfigurl' CONFIG_URL_SECTION = 'url' CONFIG_URL_LIST = 'url_list' CONFIG_AVATAR = '.gitsomeconfigavatar.png' CONFIG_ENABLE_AVATAR = 'enable_avatar' def __init__(self): self.api = None self.user_login = None self.user_pass = None self.user_token = None self.user_feed = None self.enterprise_url = None self.verify_ssl = True self.urls = [] self.enable_avatar = True self._init_colors() self.load_configs([ self.load_config_colors, ]) self.login = login self.authorize = authorize self.getpass = getpass def _init_colors(self): """Initialize colors to their defaults.""" self.clr_primary = None self.clr_secondary = 'green' self.clr_tertiary = 'cyan' self.clr_quaternary = 'yellow' self.clr_bold = 'cyan' self.clr_code = 'cyan' self.clr_error = 'red' self.clr_header = 'yellow' self.clr_link = 'green' self.clr_list = 'cyan' self.clr_message = None self.clr_num_comments = 'green' self.clr_num_points = 'green' self.clr_tag = 'cyan' self.clr_time = 'yellow' self.clr_title = None self.clr_tooltip = None self.clr_user = 'cyan' self.clr_view_link = 'magenta' self.clr_view_index = 'magenta' def authenticate_cached_credentials(self, config, parser, enterprise_auth=enterprise_login): """Authenticate with the user's credentials in ~/.gitsomeconfig. :type config: str :param config: The config path. :type parser: :class:`ConfigParser.RawConfigParser` :param parser: An instance of `ConfigParser.RawConfigParser. """ with open(config) as config_file: try: parser.read_file(config_file) except AttributeError: parser.readfp(config_file) self.user_login = self.load_config( parser=parser, cfg_label=self.CONFIG_USER_LOGIN) self.user_pass = self.load_config( parser=parser, cfg_label=self.CONFIG_USER_PASS) self.user_token = self.load_config( parser=parser, cfg_label=self.CONFIG_USER_TOKEN) self.enterprise_url = self.load_config( parser=parser, cfg_label=self.CONFIG_ENTERPRISE_URL) self.verify_ssl = self.load_config( parser=parser, cfg_label=self.CONFIG_VERIFY_SSL, boolean_config=True) self.enable_avatar = self.load_config( parser=parser, cfg_label=self.CONFIG_ENABLE_AVATAR, boolean_config=True) self.user_feed = self.load_config( parser=parser, cfg_label=self.CONFIG_USER_FEED) if not self.verify_ssl: # The user has chosen not to verify SSL certs. # Disable warnings related to this option. requests.packages.urllib3.disable_warnings( InsecureRequestWarning) login_kwargs = { 'username': self.user_login, 'two_factor_callback': self.request_two_factor_code, } if self.enterprise_url is not None: self.login = enterprise_auth login_kwargs.update({ 'url': self.enterprise_url, 'verify': self.verify_ssl, }) if self.user_token is not None: login_kwargs.update({'token': self.user_token}) elif self.user_pass is not None: login_kwargs.update({'password': self.user_pass}) else: self.print_auth_error() return else: login_kwargs.update({'token': self.user_token}) self.api = self.login(**login_kwargs) def authenticate(self, enterprise=False, enterprise_auth=enterprise_login, overwrite=False): """Log into GitHub. Adapted from https://github.com/sigmavirus24/github-cli. :type enterprise: bool :param enterprise: Determines whether to configure GitHub Enterprise. Default: False. :type overwrite: bool :param overwrite: indicates whether we cant to overwrite the current set of credentials. Default: False. """ if self.api is not None and not overwrite: return # Get the full path to the configuration file. config = self.get_github_config_path(self.CONFIG) parser = configparser.RawConfigParser() # Check to make sure the file exists and we are allowed to read it. # Skip if we want to overwrite the auth settings. if os.path.isfile(config) and os.access(config, os.R_OK | os.W_OK) and \ not overwrite: with open(config) as config_file: try: parser.read_file(config_file) except AttributeError: parser.readfp(config_file) self.authenticate_cached_credentials(config, parser) else: # The file didn't exist or we don't have the correct permissions. login_kwargs = { 'two_factor_callback': self.request_two_factor_code, } if enterprise: self.login = enterprise_auth while not self.enterprise_url: self.enterprise_url = input('Enterprise URL: ') if click.confirm('Do you want to verify SSL certs?', default=True): self.verify_ssl = True else: self.verify_ssl = False login_kwargs.update({ 'url': self.enterprise_url, 'verify': self.verify_ssl, }) while not self.user_login: self.user_login = input('User Login: ') login_kwargs.update({'username': self.user_login}) if click.confirm(('Do you want to log in with a password [Y] or ' 'a personal access token [n]?'), default=True): user_pass = None while not user_pass: user_pass = self.getpass('Password: ') login_kwargs.update({'password': user_pass}) try: if not enterprise: # Trade the user password for a personal access token. # This does not seem to be available for Enterprise. auth = self.authorize( self.user_login, user_pass, scopes=['user', 'repo'], note='gitsome', note_url='https://github.com/donnemartin/gitsome', two_factor_callback=self.request_two_factor_code ) self.user_token = auth.token else: self.user_pass = user_pass except (UnprocessableEntity, AuthenticationFailed): click.secho('Error creating token.', fg=self.clr_error) click.secho(('Visit the following page and verify you do ' 'not have an existing token named "gitsome":\n' ' https://github.com/settings/tokens\n' 'If a token already exists, update your ' '~/.gitsomeconfig file with your token:\n' ' user_token = TOKEN\n' 'You can also generate a new token.'), fg=self.clr_message) self.print_auth_error() return else: # The user has chosen to authenticate with a token. while not self.user_token: self.user_token = input('Token: ') login_kwargs.update({'token': self.user_token}) self.api = self.login(**login_kwargs) if self.user_feed: parser.set(self.CONFIG_SECTION, self.CONFIG_USER_FEED, self.user_feed) def check_auth(self): """Check if the current authorization is valid. This method uses the ratelimit_remaining api to check whether the currently authenticated user's credentials are valid without deducting from the rate limit. The ratelimit_remaining api does not seem to be available for GitHub Enterprise. github3.py's method check_authorization seems to only work given an authorization created by a registered application. TODO: Determine a better way to check the authorization for GitHub Enterprise. :type enterprise: bool :param enterprise: Determines whether we are authenticating with GitHub Enterprise. """ if self.enterprise_url is not None: return True try: if self.api is not None: # Throws AuthenticationFailed if invalid credentials but # does not deduct from the rate limit. self.api.ratelimit_remaining return True else: self.print_auth_error() except AuthenticationFailed: self.print_auth_error() return False def get_github_config_path(self, config_file_name): """Attempt to find the github config file. Adapted from https://github.com/sigmavirus24/github-cli. :type config_file_name: str :param config_file_name: The config file name. :rtype: str :return: The github config file path. """ home = os.path.abspath(os.environ.get('HOME', '')) config_file_path = os.path.join(home, config_file_name) return config_file_path def load_config(self, parser, cfg_label, default=None, color_config=False, boolean_config=False): """Load the specified config from ~/.gitsomeconfig. :type parser: :class:`ConfigParser.RawConfigParser` :param parser: An instance of `ConfigParser.RawConfigParser`. :type cfg_label: str :param cfg_label: The config label to load. :type default: str :param default: The default color if no color config exists. Default: None. :type color_config: bool :param color_config: Determines whether this is a color config. Default: False. :type boolean_config: bool :param boolean_config: Determines whether to load a boolean config. Default: False. """ try: if boolean_config: cfg = parser.getboolean(self.CONFIG_SECTION, cfg_label) else: cfg = parser.get(self.CONFIG_SECTION, cfg_label) if color_config: if cfg == 'none': cfg = None # Check if the user input a valid color. # If invalid, this will throw a TypeError click.style('', fg=cfg) except (TypeError, configparser.NoOptionError): return default return cfg def load_configs(self, config_funcs): """Load the specified config from ~/.gitsomeconfig. :type foo: list :param foo: The config methods to run. """ config_file_path = self.get_github_config_path(self.CONFIG) parser = configparser.RawConfigParser() try: with open(config_file_path) as config_file: try: parser.read_file(config_file) except AttributeError: parser.readfp(config_file) for config_func in config_funcs: config_func(parser) except IOError: # There might not be a cache yet, just silently return. return None def load_config_colors(self, parser): """Load the color config from ~/.gitsomeconfig. :type parser: :class:`ConfigParser.RawConfigParser` :param parser: An instance of `ConfigParser.RawConfigParser`. """ self.load_colors(parser) def load_colors(self, parser): """Load all colors from ~/.gitsomeconfig. :type parser: :class:`ConfigParser.RawConfigParser` :param parser: An instance of `ConfigParser.RawConfigParser`. """ self.clr_primary = self.load_config( parser=parser, cfg_label=self.CONFIG_CLR_PRIMARY, default=self.clr_primary, color_config=True) self.clr_secondary = self.load_config( parser=parser, cfg_label=self.CONFIG_CLR_SECONDARY, default=self.clr_secondary, color_config=True) self.clr_tertiary = self.load_config( parser=parser, cfg_label=self.CONFIG_CLR_TERTIARY, default=self.clr_tertiary, color_config=True) self.clr_quaternary = self.load_config( parser=parser, cfg_label=self.CONFIG_CLR_QUATERNARY, default=self.clr_quaternary, color_config=True) self.clr_bold = self.load_config( parser=parser, cfg_label=self.CONFIG_CLR_BOLD, default=self.clr_bold, color_config=True) self.clr_code = self.load_config( parser=parser, cfg_label=self.CONFIG_CLR_CODE, default=self.clr_code, color_config=True) self.clr_code = self.load_config( parser=parser, cfg_label=self.CONFIG_CLR_ERROR, default=self.clr_code, color_config=True) self.clr_header = self.load_config( parser=parser, cfg_label=self.CONFIG_CLR_HEADER, default=self.clr_header, color_config=True) self.clr_link = self.load_config( parser=parser, cfg_label=self.CONFIG_CLR_LINK, default=self.clr_link, color_config=True) self.clr_list = self.load_config( parser=parser, cfg_label=self.CONFIG_CLR_LIST, default=self.clr_list, color_config=True) self.clr_message = self.load_config( parser=parser, cfg_label=self.CONFIG_CLR_MESSAGE, default=self.clr_message, color_config=True) self.clr_num_comments = self.load_config( parser=parser, cfg_label=self.CONFIG_CLR_NUM_COMMENTS, default=self.clr_num_comments, color_config=True) self.clr_num_points = self.load_config( parser=parser, cfg_label=self.CONFIG_CLR_NUM_POINTS, default=self.clr_num_points, color_config=True) self.clr_tag = self.load_config( parser=parser, cfg_label=self.CONFIG_CLR_TAG, default=self.clr_tag, color_config=True) self.clr_time = self.load_config( parser=parser, cfg_label=self.CONFIG_CLR_TIME, default=self.clr_time, color_config=True) self.clr_title = self.load_config( parser=parser, cfg_label=self.CONFIG_CLR_TITLE, default=self.clr_title, color_config=True) self.clr_tooltip = self.load_config( parser=parser, cfg_label=self.CONFIG_CLR_TOOLTIP, default=self.clr_tooltip, color_config=True) self.clr_user = self.load_config( parser=parser, cfg_label=self.CONFIG_CLR_USER, default=self.clr_user, color_config=True) self.clr_view_link = self.load_config( parser=parser, cfg_label=self.CONFIG_CLR_VIEW_LINK, default=self.clr_view_link, color_config=True) self.clr_view_index = self.load_config( parser=parser, cfg_label=self.CONFIG_CLR_VIEW_INDEX, default=self.clr_view_index, color_config=True) def load_urls(self, view_in_browser): """Load the current set of urls from ~/.gitsomeconfigurl. :type view_in_browser: bool :param view_in_browser: Determines whether to view the urls in a browser. :rtype: list :return: Collection of urls. """ config = self.get_github_config_path(self.CONFIG_URL) parser = configparser.RawConfigParser() with open(config) as config_file: try: parser.read_file(config_file) except AttributeError: parser.readfp(config_file) urls = parser.get(self.CONFIG_URL_SECTION, self.CONFIG_URL_LIST) urls = urls.strip() excludes = ['[', ']', "'"] for exclude in excludes: urls = urls.replace(exclude, '') if not view_in_browser: urls = urls.replace('https://github.com/', '') return urls.split(', ') def print_auth_error(self): """Print a message the authorization has failed.""" click.secho('Authentication error.', fg=self.clr_error) click.secho(('Update your credentials in ~/.gitsomeconfig ' 'or run:\n gh configure'), fg=self.clr_message) def prompt_news_feed(self): """Prompt the user to enter a news feed url.""" if click.confirm(('No feed url detected.\n Calling gh events without ' "an argument\n displays the logged in user's " 'news feed.\nDo you want gitsome to track your ' 'news feed?'), default=True): click.secho(('Visit the following url while logged into GitHub:\n' ' https://github.com\n' 'Enter the url found under "Subscribe to your ' 'news feed".'), fg=self.clr_message) self.user_feed = '' while not self.user_feed: self.user_feed = input('URL: ') def request_two_factor_code(self): """Request two factor authentication code. Callback if two factor authentication is requested. :rtype: str :return: The user input two factor authentication code. """ code = '' while not code: code = input('Enter 2FA code: ') return code def save_config(self): """Saves the config to ~/.gitsomeconfig.""" if self.check_auth(): config = self.get_github_config_path(self.CONFIG) parser = configparser.RawConfigParser() parser.add_section(self.CONFIG_SECTION) parser.set(self.CONFIG_SECTION, self.CONFIG_USER_LOGIN, self.user_login) if self.user_token is not None: parser.set(self.CONFIG_SECTION, self.CONFIG_USER_TOKEN, self.user_token) if self.user_feed is not None: parser.set(self.CONFIG_SECTION, self.CONFIG_USER_FEED, self.user_feed) if self.enable_avatar is not None: parser.set(self.CONFIG_SECTION, self.CONFIG_ENABLE_AVATAR, self.enable_avatar) if self.enterprise_url is not None: parser.set(self.CONFIG_SECTION, self.CONFIG_ENTERPRISE_URL, self.enterprise_url) if self.user_pass is not None: parser.set(self.CONFIG_SECTION, self.CONFIG_USER_PASS, self.user_pass) else: parser.remove_option(self.CONFIG_SECTION, self.CONFIG_USER_PASS) parser.set(self.CONFIG_SECTION, self.CONFIG_VERIFY_SSL, self.verify_ssl) parser.set(self.CONFIG_SECTION, self.CONFIG_CLR_PRIMARY, self.clr_primary) parser.set(self.CONFIG_SECTION, self.CONFIG_CLR_SECONDARY, self.clr_secondary) parser.set(self.CONFIG_SECTION, self.CONFIG_CLR_TERTIARY, self.clr_tertiary) parser.set(self.CONFIG_SECTION, self.CONFIG_CLR_QUATERNARY, self.clr_quaternary) parser.set(self.CONFIG_SECTION, self.CONFIG_CLR_BOLD, self.clr_bold) parser.set(self.CONFIG_SECTION, self.CONFIG_CLR_CODE, self.clr_code) parser.set(self.CONFIG_SECTION, self.CONFIG_CLR_ERROR, self.clr_error) parser.set(self.CONFIG_SECTION, self.CONFIG_CLR_HEADER, self.clr_header) parser.set(self.CONFIG_SECTION, self.CONFIG_CLR_LINK, self.clr_link) parser.set(self.CONFIG_SECTION, self.CONFIG_CLR_LIST, self.clr_list) parser.set(self.CONFIG_SECTION, self.CONFIG_CLR_MESSAGE, self.clr_message) parser.set(self.CONFIG_SECTION, self.CONFIG_CLR_NUM_COMMENTS, self.clr_num_comments) parser.set(self.CONFIG_SECTION, self.CONFIG_CLR_NUM_POINTS, self.clr_num_points) parser.set(self.CONFIG_SECTION, self.CONFIG_CLR_TAG, self.clr_tag) parser.set(self.CONFIG_SECTION, self.CONFIG_CLR_TIME, self.clr_time) parser.set(self.CONFIG_SECTION, self.CONFIG_CLR_TITLE, self.clr_title) parser.set(self.CONFIG_SECTION, self.CONFIG_CLR_TOOLTIP, self.clr_tooltip) parser.set(self.CONFIG_SECTION, self.CONFIG_CLR_USER, self.clr_user) parser.set(self.CONFIG_SECTION, self.CONFIG_CLR_VIEW_LINK, self.clr_view_link) parser.set(self.CONFIG_SECTION, self.CONFIG_CLR_VIEW_INDEX, self.clr_view_index) with open(config, 'w+') as config_file: parser.write(config_file) def save_urls(self): """Save the current set of urls to ~/.gitsomeconfigurl.""" config = self.get_github_config_path(self.CONFIG_URL) parser = configparser.RawConfigParser() try: parser.add_section(self.CONFIG_URL_SECTION) except configparser.DuplicateSectionError: pass parser.set(self.CONFIG_URL_SECTION, self.CONFIG_URL_LIST, self.urls) with open(config, 'w+') as config_file: parser.write(config_file) def show_bash_completions_info(self): """Show info on how to enable bash completions""" click.secho(('By default, gitsome looks at the following locations ' 'to enable bash completions:\n' ' https://github.com/donnemartin/gitsome/blob/master/xonsh/environ.py#L123-L130\n' # NOQA 'If bash completions are not working for you, check out ' 'the following link:\n' ' https://github.com/donnemartin/gitsome#enabling-bash-completions'), # NOQA fg=self.clr_message) ================================================ FILE: gitsome/formatter.py ================================================ # -*- coding: utf-8 -*- # Copyright 2015 Donne Martin. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of # the License is located at # # http://www.apache.org/licenses/LICENSE-2.0 # # or in the "license" file accompanying this file. This file is # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. from __future__ import unicode_literals from __future__ import print_function import re from .lib.pretty_date_time import pretty_date_time import click class Formatter(object): """Handle formatting of isssues, repos, threads, etc. :type config: :class:`config.Config` :param config: An instance of config.Config. :type event_handlers: dict :param event_handlers: A mapping of raw event types to format methods. :type event_type_mapping: dict :param event_type_mapping: A mapping of raw event types to more human readable text. :type pretty_dt: :class:`pretty_date_time` :param pretty_dt: An instance of pretty_date_time. """ def __init__(self, config): self.config = config self.event_type_mapping = { 'CommitCommentEvent': 'commented on commit', 'CreateEvent': 'created', 'DeleteEvent': 'deleted', 'FollowEvent': 'followed', 'ForkEvent': 'forked', 'GistEvent': 'created/updated gist', 'GollumEvent': 'created/updated wiki', 'IssueCommentEvent': 'commented on', 'IssuesEvent': '', 'MemberEvent': 'added collaborator', 'MembershipEvent': 'added/removed user', 'PublicEvent': 'open sourced', 'PullRequestEvent': '', 'PullRequestReviewCommentEvent': 'commented on pull request', 'PushEvent': 'pushed to', 'ReleaseEvent': 'released', 'RepositoryEvent': 'created repository', 'WatchEvent': 'starred', } self.event_handlers = { 'CommitCommentEvent': self._format_commit_comment_event, 'CreateEvent': self._format_create_delete_event, 'DeleteEvent': self._format_create_delete_event, 'FollowEvent': self._format_general_event, 'ForkEvent': self._format_fork_event, 'ForkApplyEvent': self._format_general_event, 'GistEvent': self._format_general_event, 'GollumEvent': self._format_general_event, 'IssueCommentEvent': self._format_issue_commment_event, 'IssuesEvent': self._format_issues_event, 'MemberEvent': self._format_general_event, 'MembershipEvent': self._format_general_event, 'PublicEvent': self._format_general_event, 'PullRequestEvent': self._format_pull_request_event, 'PullRequestReviewCommentEvent': self._format_commit_comment_event, 'PushEvent': self._format_push_event, 'ReleaseEvent': self._format_release_event, 'StatusEvent': self._format_general_event, 'TeamAddEvent': self._format_general_event, 'RepositoryEvent': self._format_general_event, 'WatchEvent': self._format_general_event, } self.pretty_dt = pretty_date_time def _format_time(self, event): """Format time. :type event: :class:`github3` Event. :param event: An instance of `github3` Event. """ item = click.style( ' (' + str(self.pretty_dt(event.created_at)) + ')', fg=self.config.clr_time) return item def _format_issue_comment(self, event, key): """Format an issue comment. :type event: :class:`github3` Event. :param event: An instance of `github3` Event. """ issue = '{repo[0]}/{repo[1]}#{num}'.format( repo=event.payload[key].repository, num=event.payload[key].number) return click.style(issue, fg=self.config.clr_tertiary) def _format_indented_message(self, message, newline=True, indent=' ', sha=''): """Format an indented message. :type message: str :param message: The commit comment. :type newline: bool :param newline: Determines whether to prepend a newline. :type indent: str :param indent: The indent, consisting of blank chars. TODO: Consider passing an int denoting # blank chars, or try to calculate the indent dynamically. :type sha: str :param sha: The commit hash. :rtype: str :return: The formattted commit comment. """ subsequent_indent = indent if sha != '': subsequent_indent += ' ' message = self.strip_line_breaks(message) formatted_message = click.wrap_text( text=click.style(sha, fg=self.config.clr_tertiary)+message, initial_indent=indent, subsequent_indent=subsequent_indent) if newline: formatted_message = click.style('\n' + formatted_message) return formatted_message def _format_sha(self, sha): """Format commit hash. :type sha: str :param sha: The commit hash. """ return sha[:7] def _format_commit_comment_event(self, event): """Format commit comment and commit hash. :type event: :class:`github3` Event. :param event: An instance of `github3` Event. """ item = click.style(self.event_type_mapping[event.type] + ' ', fg=self.config.clr_secondary) item += click.style( self._format_sha(event.payload['comment'].commit_id), fg=self.config.clr_tertiary) item += click.style(' at ', fg=self.config.clr_secondary) item += click.style(self.format_user_repo(event.repo), fg=self.config.clr_tertiary) try: item += click.style( '#' + str(event.payload['pull_request'].number) + ' ', fg=self.config.clr_tertiary) except KeyError: pass item += self._format_time(event) try: item += self._format_indented_message( event.payload['pull_request'].title) item += self._format_indented_message( event.payload['comment'].body, indent=' ') except KeyError: item += self._format_indented_message( event.payload['comment'].body) return item def _format_create_delete_event(self, event): """Format a create or delete event. :type event: :class:`github3` Event. :param event: An instance of `github3` Event. """ item = click.style(self.event_type_mapping[event.type], fg=self.config.clr_secondary) item += click.style(' ' + event.payload['ref_type'], fg=self.config.clr_secondary) if event.payload['ref']: item += click.style(' ' + event.payload['ref'], fg=self.config.clr_tertiary) item += click.style(' at ', fg=self.config.clr_secondary) item += click.style(self.format_user_repo(event.repo), fg=self.config.clr_tertiary) item += self._format_time(event) try: item += self._format_indented_message( ('' if event.payload['description'] is None else event.payload['description'])) except KeyError: pass return item def _format_fork_event(self, event): """Format a repo fork event. :type event: :class:`github3` Event. :param event: An instance of `github3` Event. """ item = click.style(self.event_type_mapping[event.type], fg=self.config.clr_secondary) item += click.style(' ' + self.format_user_repo(event.repo), fg=self.config.clr_tertiary) item += self._format_time(event) return item def _format_issue_commment_event(self, event): """Format a repo fork event. :type event: :class:`github3` Event. :param event: An instance of `github3` Event. """ item = click.style(self.event_type_mapping[event.type] + ' ', fg=self.config.clr_secondary) item += self._format_issue_comment(event, key='issue') item += self._format_time(event) item += self._format_indented_message( event.payload['issue'].title) item += self._format_indented_message( event.payload['comment'].body, indent=' ') return item def _format_issues_event(self, event): """Format an issue event. :type event: :class:`github3` Event. :param event: An instance of `github3` Event. """ item = click.style(event.payload['action'] + ' issue ', fg=self.config.clr_secondary) item += self._format_issue_comment(event, key='issue') item += self._format_time(event) item += self._format_indented_message( event.payload['issue'].title) return item def _format_pull_request_event(self, event): """Format a pull request event. :type event: :class:`github3` Event. :param event: An instance of `github3` Event. """ item = click.style(event.payload['action'] + ' pull request ', fg=self.config.clr_secondary) item += self._format_issue_comment(event, key='pull_request') item += self._format_time(event) item += self._format_indented_message( event.payload['pull_request'].title) return item def _format_push_event(self, event): """Format a push event. :type event: :class:`github3` Event. :param event: An instance of `github3` Event. """ item = click.style(self.event_type_mapping[event.type], fg=self.config.clr_secondary) branch = event.payload['ref'].split('/')[-1] item += click.style(' ' + branch, fg=self.config.clr_tertiary) item += click.style(' at ', fg=self.config.clr_secondary) item += click.style(self.format_user_repo(event.repo), fg=self.config.clr_tertiary) item += self._format_time(event) for commit in event.payload['commits']: sha = click.style(self._format_sha(commit['sha']) + ': ', fg=self.config.clr_message) item += self._format_indented_message( commit['message'], sha=sha) return item def _format_release_event(self, event): """Format a release event. :type event: :class:`github3` Event. :param event: An instance of `github3` Event. """ item = click.style(self.event_type_mapping[event.type] + ' ', fg=self.config.clr_secondary) item += click.style(event.payload['release'].tag_name + ' ', fg=self.config.clr_tertiary) item += click.style('at ', fg=self.config.clr_secondary) item += click.style(self.format_user_repo(event.repo), fg=self.config.clr_tertiary) item += self._format_time(event) return item def _format_general_event(self, event): """Format an event, general case used by various event types. :type event: :class:`github3` Event. :param event: An instance of `github3` Event. """ item = click.style(self.event_type_mapping[event.type] + ' ', fg=self.config.clr_secondary) item += click.style(self.format_user_repo(event.repo), fg=self.config.clr_tertiary) item += self._format_time(event) return item def format_email(self, view_entry): """Format an email. :type view_entry: :class:`github3` Email :param view_entry: An instance of `github3` Email. :rtype: str :return: The formattted email. """ email = view_entry.item item = self.format_index_title(view_entry.index, email.email) item += '\n' item += click.style((' ' + 'Primary: ' + str(email.primary).ljust(7) + ' '), fg=self.config.clr_secondary) item += click.style(('Verified: ' + str(email.verified).ljust(5) + ' '), fg=self.config.clr_tertiary) return item def format_emoji(self, view_entry): """Format an emoji. :type view_entry: str :param view_entry: The emoji name. :rtype: str :return: The formattted emoji. """ emoji = view_entry.item item = self.format_index_title(view_entry.index, emoji) return item def format_event(self, view_entry): """Format an event. :type view_entry: :class:`github3` Event :param view_entry: An instance of `github3` Event. :rtype: str :return: The formattted event. """ event = view_entry.item item = self.format_index_title(view_entry.index, str(event.actor)) item += self.event_handlers[event.type](event) return item def format_gitignore_template_name(self, view_entry): """Format a gitignore template name. :type view_entry: str :param view_entry: The gitignore template name. :rtype: str :return: The formattted gitignore template name. """ gitignore_template_name = view_entry.item item = self.format_index_title(view_entry.index, gitignore_template_name) return item def format_feed_entry(self, view_entry): """Format a feed entry. :type view_entry: dict :param view_entry: The URITemplates feed. :rtype: str :return: The formattted feed entry. """ feed_entry = view_entry.item item_parts = feed_entry.title.split(' ') title = item_parts[0] action = item_parts[1:-1] repo = item_parts[-1] item = self.format_index_title(view_entry.index, title) if action[0] == 'forked': item += click.style(action[0] + ' ', fg=self.config.clr_secondary) item += click.style(action[1] + ' ', fg=self.config.clr_tertiary) else: item += click.style(' '.join(action), fg=self.config.clr_secondary) item += click.style(' ' + repo + ' ', fg=self.config.clr_tertiary) item += click.style( '(' + str(self.pretty_dt(feed_entry.updated_parsed)) + ')', fg=self.config.clr_time) if action[0] == 'commented': comment_parts = feed_entry['summary'].split('blockquote') if len(comment_parts) > 2: comment = comment_parts[-2] parts_mention = comment.split('class="user-mention">') if len(parts_mention) > 1: comment = parts_mention[1] comment = self._format_indented_message(comment, newline=False) comment = re.sub(r'(*)', r'', comment) comment = re.sub(r'(

*)', r'', comment) comment = re.sub(r'(

*)', r'', comment) comment = re.sub(r'( *)', r'', comment) item += click.style('\n' + comment, fg=self.config.clr_message) return item def format_license_name(self, view_entry): """Format a license template name. :type view_entry: :class:`github3` License :param view_entry: An instance of `github3` License. :rtype: str :return: The formattted license template name. """ license_template_name = view_entry.item item = self.format_index_title(view_entry.index, license_template_name.key) item += click.style('(' + license_template_name.name + ')', fg=self.config.clr_secondary) return item def format_user(self, view_entry): """Format a user. :type view_entry: :class:`github3` User :param view_entry: An instance of `github3` User. :rtype: str :return: The formattted user. """ user = view_entry.item item = self.format_index_title(view_entry.index, user.login) return item def format_issues_url_from_issue(self, issue): """Format the issue url based on the given issue. :type issue: :class:`github3` Issue :param issue: An instance of `github3` Issue. :rtype: str :return: The formattted issues url. """ return self.format_user_repo(issue.repository) + '/' + \ 'issues/' + str(issue.number) def format_issues_url_from_thread(self, thread): """Format the issue url based on the given thread. :type issue: :class:`github3` Thread :param issue: An instance of `github3` Thread. :rtype: str :return: The formattted issues url. """ url_parts = thread.subject['url'].split('/') user = url_parts[4] repo = url_parts[5] issues_uri = 'issues' issue_id = url_parts[7] return '/'.join([user, repo, issues_uri, issue_id]) def format_index_title(self, index, title): """Format an item's index and title. :type index: str :param index: The index for the given item. :type title: str :param title: The item's title. :rtype: str :return: The formatted index and title. """ formatted_index_title = click.style(' ' + (str(index) + '.').ljust(5), fg=self.config.clr_view_index) formatted_index_title += click.style(title + ' ', fg=self.config.clr_primary) return formatted_index_title def format_issue(self, view_entry): """Format an issue. :type view_entry: :class:`github3` Issue :param view_entry: An instance of `github3` Issue. :rtype: str :return: The formatted issue. """ issue = view_entry.item item = self.format_index_title(view_entry.index, issue.title) item += click.style('@' + str(issue.user) + ' ', fg=self.config.clr_primary) item += click.style(('(' + self.format_issues_url_from_issue(issue) + ')'), fg=self.config.clr_view_link) item += '\n' indent = ' ' if len(item) == 8: item += click.style((' Score: ' + str(item[7]).ljust(10) + ' '), fg=self.config.clr_quaternary) indent = ' ' item += click.style((indent + 'State: ' + str(issue.state).ljust(10) + ' '), fg=self.config.clr_secondary) item += click.style(('Comments: ' + str(issue.comments_count).ljust(5) + ' '), fg=self.config.clr_tertiary) item += click.style(('Assignee: ' + str(issue.assignee).ljust(10) + ' '), fg=self.config.clr_quaternary) return item def format_repo(self, view_entry): """Format a repo. :type view_entry: :class:`github3` Repository :param view_entry: An instance of `github3` Repository. :rtype: str :return: The formatted repo. """ repo = view_entry.item item = self.format_index_title(view_entry.index, repo.full_name) language = repo.language if repo.language is not None else 'Unknown' item += click.style('(' + language + ')', fg=self.config.clr_message) item += '\n' item += click.style((' ' + 'Stars: ' + str(repo.stargazers_count).ljust(6) + ' '), fg=self.config.clr_secondary) item += click.style('Forks: ' + str(repo.forks_count).ljust(6) + ' ', fg=self.config.clr_tertiary) item += click.style(('Updated: ' + str(self.pretty_dt(repo.updated_at)) + ' '), fg=self.config.clr_time) return item def format_thread(self, view_entry): """Format a thread. :type view_entry: :class:`github3` Thread :param view_entry: An instance of `github3` Thread. :rtype: str :return: The formatted thread. """ thread = view_entry.item item = self.format_index_title(view_entry.index, thread.subject['title']) item += click.style('(' + view_entry.url + ')', fg=self.config.clr_view_link) item += '\n' item += click.style((' ' + 'Seen: ' + str(not thread.unread).ljust(7) + ' '), fg=self.config.clr_secondary) item += click.style(('Type: ' + str(thread.subject['type']).ljust(12) + ' '), fg=self.config.clr_tertiary) item += click.style(('Updated: ' + str(self.pretty_dt(thread.updated_at)) + ' '), fg=self.config.clr_time) return item def format_trending_entry(self, view_entry): """Formats a trending repo entry. :type view_entry: dict :param view_entry: The URITemplates feed. :rtype: str :return: The formattted trending entry. """ trending_entry = view_entry.item item_parts = trending_entry.title.split(' ') title = item_parts[0] item = self.format_index_title(view_entry.index, title) try: summary_parts = trending_entry.summary.split('\n') summary = summary_parts[0] if len(summary_parts) > 1 else '' summary = self.strip_line_breaks(summary) language = summary_parts[-1] except AttributeError: summary = '' language = '' if language == '()': language = '(Unknown)' language = re.sub(r'(\()', r'', language) language = re.sub(r'(\))', r'', language) if summary: item += '\n' summary = click.wrap_text( text=summary, initial_indent=' ', subsequent_indent=' ') item += click.style(summary, self.config.clr_message) item += '\n' item += click.style(' ' + language, fg=self.config.clr_message) return item def format_user_repo(self, user_repo_tuple): """Format a repo tuple for pretty print. Example: Input: ('donnemartin', 'gitsome') Output: donnemartin/gitsome Input: ('repos/donnemartin', 'gitsome') Output: donnemartin/gitsome :type user_repo_tuple: tuple :param user_repo_tuple: The user and repo. :rtype: str :return: A string of the form user/repo. """ result = '/'.join(user_repo_tuple) if result.startswith('repos/'): return result[len('repos/'):] return result def strip_line_breaks(self, text): """Strips \r and \n characters. These characters seem to cause issues with `click.wrap_text`. :type text: str :param text: The text to strip of line breaks. :rtype: str :return: The input text without line breaks. """ text = re.sub(r'\r', '', text) text = re.sub(r'\n', ' ', text) return text ================================================ FILE: gitsome/github.py ================================================ # -*- coding: utf-8 -*- # Copyright 2015 Donne Martin. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of # the License is located at # # http://www.apache.org/licenses/LICENSE-2.0 # # or in the "license" file accompanying this file. This file is # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. from __future__ import unicode_literals from __future__ import print_function import os import platform import sys import urllib import webbrowser from .lib.github3 import null from .lib.github3.exceptions import AuthenticationFailed, UnprocessableEntity from .lib.img2txt import img2txt import click import feedparser from requests.exceptions import MissingSchema, SSLError from .config import Config from .formatter import Formatter from .rss_feed import language_rss_map from .table import Table from .view_entry import ViewEntry from .web_viewer import WebViewer from .utils import TextUtils class GitHub(object): """Provide integration with the GitHub API. :type config: :class:`config.Config` :param config: An instance of `config.Config`. :type formatter: :class:`formatter.Formatter` :param formatter: An instance of `formatter.Formatter`. :type img2txt: callable :param img2txt: A callable fom img2txt. :type table: :class:`table.Table` :param table: An instance of `table.Table`. :type trend_parser: :class:`feedparser` :param trend_parser: An instance of `feedparser`. :type web_viewer: :class:`web_viewer.WebViewer` :param web_viewer: An instance of `web_viewer.WebViewer`. :type _base_url: str :param _base_url: The base GitHub or GitHub Enterprise url. """ def __init__(self): self.config = Config() self.formatter = Formatter(self.config) self.img2txt = img2txt.img2txt self.table = Table(self.config) self.web_viewer = WebViewer(self.config) self.trend_parser = feedparser self.text_utils = TextUtils() self._base_url = 'https://github.com/' @property def base_url(self): return self.config.enterprise_url or self._base_url def add_base_url(self, url): """Add the base url if it is not already part of the given url. :type url: str :param url: The url. :return: The url including the base url. """ return self.base_url + url if self.base_url not in url else url def authenticate(func): """Decorator that authenticates credentials. :type func: callable :param func: A method to execute if authorization passes. :return: The return value of `func` if authorization passes, or None if authorization fails. """ def auth_wrapper(self, *args, **kwargs): self.config.authenticate() self.config.save_config() if self.config.check_auth(): try: return func(self, *args, **kwargs) except SSLError: click.secho(('SSL cert verification failed.\n Try running ' 'gh configure --enterprise\n and type ' "'n' when asked whether to verify SSL certs."), fg=self.config.clr_error) except MissingSchema: click.secho('Invalid GitHub Enterprise url', fg=self.config.clr_error) except AuthenticationFailed: self.config.print_auth_error() return auth_wrapper def avatar(self, url, text_avatar): """Display the user's avatar from the specified url. This method requires PIL. :type url: str :param url: The user's avatar image. :type text_avatar: bool :param text_avatar: Determines whether to view the profile avatar in plain text (True) or in ansi (False). On Windows this value is always set to True due to lack of support of `img2txt` on Windows. :rtype: str :return: The avatar. """ if platform.system() == 'Windows': text_avatar = True avatar_enabled = self.config.enable_avatar avatar_text = '' if avatar_enabled: avatar = self.config.get_github_config_path( self.config.CONFIG_AVATAR) try: urllib.request.urlretrieve(url, avatar) except urllib.error.URLError: pass if os.path.exists(avatar): avatar_text = self.img2txt(avatar, ansi=(not text_avatar)) avatar_text += '\n' os.remove(avatar) return avatar_text def avatar_setup(self, url, text_avatar): """Prepare to display the user's avatar from the specified url. This method requires PIL. :type url: str :param url: The user's avatar image. :type text_avatar: bool :param text_avatar: Determines whether to view the profile avatar in plain text (True) or in ansi (False). On Windows this value is always set to True due to lack of support of `img2txt` on Windows. :rtype: str :return: The avatar. """ try: import PIL # NOQA return self.avatar(url, text_avatar) except ImportError: avatar_text = click.style(('To view the avatar in your terminal, ' 'install the Python Image Library.\n'), fg=self.config.clr_message) return avatar_text def configure(self, enterprise): """Configure gitsome. Attempts to authenticate the user and to set up the user's news feed. If `gitsome` has not yet been configured, calling a `gh` command that requires authentication will automatically invoke the `configure` command. :type enterprise: bool :param enterprise: Determines whether to configure GitHub Enterprise. """ self.config.authenticate(enterprise=enterprise, overwrite=True) self.config.prompt_news_feed() self.config.show_bash_completions_info() self.config.save_config() @authenticate def create_comment(self, user_repo_number, text): """Create a comment on the given issue. :type user_repo_number: str :param user_repo_number: The user/repo/issue_number. :type text: str :param text: The comment text. """ try: user, repo, number = user_repo_number.split('/') int(number) # Check for int except ValueError: click.secho(('Expected argument: user/repo/# and option -t ' '"comment".'), fg=self.config.clr_error) return issue = self.config.api.issue(user, repo, number) issue_comment = issue.create_comment(text) if type(issue_comment) is not null.NullObject: click.secho('Created comment: ' + issue_comment.body, fg=self.config.clr_message) else: click.secho('Error creating comment', fg=self.config.clr_error) @authenticate def create_issue(self, user_repo, issue_title, issue_desc=''): """Create an issue. :type user_repo: str :param user_repo: The user/repo. :type issue_title: str :param issue_title: The issue title. :type issue_desc: str :param issue_desc: The issue body (optional). """ try: user, repo_name = user_repo.split('/') except ValueError: click.secho('Expected argument: user/repo and option -t "title".', fg=self.config.clr_error) return issue = self.config.api.create_issue(user, repo_name, issue_title, issue_desc) if type(issue) is not null.NullObject: body = self.text_utils.sanitize_if_none(issue.body) click.secho('Created issue: ' + issue.title + '\n' + body, fg=self.config.clr_message) else: click.secho('Error creating issue.', fg=self.config.clr_error) @authenticate def create_repo(self, repo_name, repo_desc='', private=False): """Create a repo. :type repo_name: str :param repo_name: The repo name. :type repo_desc: str :param repo_desc: The repo description (optional). :type private: bool :param private: Determines whether the repo is private. Default: False. """ try: repo = self.config.api.create_repository(repo_name, repo_desc, private=private) desc = self.text_utils.sanitize_if_none(repo.description) click.secho(('Created repo: ' + repo.full_name + '\n' + desc), fg=self.config.clr_message) except UnprocessableEntity as e: click.secho('Error creating repo: ' + str(e.msg), fg=self.config.clr_error) @authenticate def emails(self): """List all the user's registered emails.""" self.table.build_table_setup(self.config.api.emails(), self.formatter.format_email, limit=sys.maxsize, pager=False, build_urls=False) @authenticate def emojis(self, pager=False): """List all GitHub supported emojis. :type pager: bool :param pager: Determines whether to show the output in a pager, if available. """ self.table.build_table_setup(self.config.api.emojis(), self.formatter.format_emoji, limit=sys.maxsize, pager=pager, build_urls=False) @authenticate def feed(self, user_or_repo='', private=False, pager=False): """List all activity for the given user or repo. If `user_or_repo` is not provided, uses the logged in user's news feed seen while visiting https://github.com. If `user_or_repo` is provided, shows either the public or `[-p/--private]` feed activity of the user or repo. :type user_or_repo: str :param user_or_repo: The user or repo to list events for (optional). If no entry, defaults to the logged in user's feed. :type private: bool :param private: Determines whether to show the private events (True) or public events (False). :type pager: bool :param pager: Determines whether to show the output in a pager, if available. """ click.secho('Listing events...', fg=self.config.clr_message) if user_or_repo == '': if self.config.user_feed is None: self.config.prompt_news_feed() self.config.save_config() if self.config.user_feed: items = self.trend_parser.parse(self.config.user_feed) self.table.build_table_setup_feed( items, self.formatter.format_feed_entry, pager) else: if '/' in user_or_repo: user, repo = user_or_repo.split('/') repo = self.config.api.repository(user, repo) items = repo.events() else: public = False if private else True items = self.config.api.user(user_or_repo).events(public=public) self.table.build_table_setup( items, self.formatter.format_event, limit=sys.maxsize, pager=pager, build_urls=False) @authenticate def followers(self, user, pager=False): """List all followers and the total follower count. :type user: str :param user: The user login (optional). If None, returns the followers of the logged in user. :type pager: bool :param pager: Determines whether to show the output in a pager, if available. """ if user is None: user = self.config.user_login self.table.build_table_setup_user( self.config.api.followers_of(user), self.formatter.format_user, limit=sys.maxsize, pager=pager) @authenticate def following(self, user, pager=False): """List all followed users and the total followed count. :type user: str :param user: The user login. If None, returns the followed users of the logged in user. :type pager: bool :param pager: Determines whether to show the output in a pager, if available. """ if user is None: user = self.config.user_login self.table.build_table_setup_user( self.config.api.followed_by(user), self.formatter.format_user, limit=sys.maxsize, pager=pager) @authenticate def gitignore_template(self, language): """Output the gitignore template for the given language. :type language: str :param language: The language. """ template = self.config.api.gitignore_template(language) if template: click.secho(template, fg=self.config.clr_message) else: click.secho(('Invalid case-sensitive template requested, run the ' 'following command to see available templates:\n' ' gh gitignore-templates'), fg=self.config.clr_error) @authenticate def gitignore_templates(self, pager=False): """Output all supported gitignore templates. :type pager: bool :param pager: Determines whether to show the output in a pager, if available. """ self.table.build_table_setup( self.config.api.gitignore_templates(), self.formatter.format_gitignore_template_name, limit=sys.maxsize, pager=pager, build_urls=False) click.secho((' Run the following command to view or download a ' '.gitignore file:\n' ' View: gh gitignore Python\n' ' Download: gh gitignore Python > .gitignore\n'), fg=self.config.clr_message) @authenticate def issue(self, user_repo_number): """Output detailed information about the given issue. :type user_repo_number: str :param user_repo_number: The user/repo/issue_number. """ try: user, repo, number = user_repo_number.split('/') int(number) # Check for int except ValueError: click.secho('Expected argument: user/repo/#.', fg=self.config.clr_error) return url = (self.base_url + user + '/' + repo + '/' + 'issues/' + number) self.web_viewer.view_url(url) @authenticate def issues(self, issues_list, limit=1000, pager=False, sort=True): """List all issues. :type issues_list: list :param issues_list: A list of `github3` Issues. :type limit: int :param limit: The number of items to display. :type pager: bool :param pager: Determines whether to show the output in a pager, if available. :type sort: bool :param sort: Determines whether to sort the issues by: state, repo, created_at. """ view_entries = [] for current_issue in issues_list: url = self.formatter.format_issues_url_from_issue(current_issue) view_entries.append( ViewEntry( current_issue, url=url, sort_key_primary=current_issue.state, sort_key_secondary=current_issue.repository, sort_key_tertiary=current_issue.created_at)) if sort: view_entries = sorted(view_entries, reverse=False) self.table.build_table(view_entries, limit, pager, self.formatter.format_issue) @authenticate def issues_setup(self, issue_filter='subscribed', issue_state='open', limit=1000, pager=False): """Prepare to list all issues matching the filter. :type issue_filter: str :param issue_filter: 'assigned', 'created', 'mentioned', 'subscribed' (default). :type issue_state: str :param issue_state: 'all', 'open' (default), 'closed'. :type limit: int :param limit: The number of items to display. :type pager: bool :param pager: Determines whether to show the output in a pager, if available. """ self.issues(self.config.api.issues(issue_filter, issue_state), limit, pager) @authenticate def license(self, license_name): """Output the gitignore template for the given language. :type license_name: str :param license_name: The license name. """ result = self.config.api.license(license_name) if type(result) is not null.NullObject: click.secho(result.body, fg=self.config.clr_message) else: click.secho((' Invalid case-sensitive license requested, run the ' 'following command to see available licenses:\n' ' gh licenses'), fg=self.config.clr_error) @authenticate def licenses(self): """Output the gitignore template for the given language.""" self.table.build_table_setup( self.config.api.licenses(), self.formatter.format_license_name, limit=sys.maxsize, pager=False, build_urls=False) click.secho((' Run the following command to view or download a ' 'LICENSE file:\n' ' gh license apache-2.0\n' ' gh license apache-2.0 > LICENSE\n'), fg=self.config.clr_message) @authenticate def notifications(self, limit=1000, pager=False): """List all notifications. :type limit: int :param limit: The number of items to display. :type pager: bool :param pager: Determines whether to show the output in a pager, if available. """ view_entries = [] for thread in self.config.api.notifications(all=True, participating=False): url = self.formatter.format_issues_url_from_thread(thread) view_entries.append(ViewEntry(thread, url=url)) self.table.build_table(view_entries, limit, pager, self.formatter.format_thread) @authenticate def octocat(self, say): """Output an Easter egg or the given message from Octocat. :type say: str :param say: What Octocat should say. If say is None, octocat speaks an Easter egg. """ output = str(self.config.api.octocat(say)) output = output.replace('\\n', '\n') click.secho(output, fg=self.config.clr_message) @authenticate def pull_requests(self, limit=1000, pager=False): """List all pull requests. :type limit: int :param limit: The number of items to display. :type pager: bool :param pager: Determines whether to show the output in a pager, if available. """ issues_list = [] repositories = self.config.api.repositories() for repository in repositories: repo_pulls = repository.pull_requests() for repo_pull in repo_pulls: url = self.formatter.format_issues_url_from_issue(repo_pull) user, repo, issues, number = url.split('/') repo_pull = self.config.api.pull_request(user, repo, number) issues_list.append(repo_pull) self.issues(issues_list, limit, pager) @authenticate def rate_limit(self): """Output the rate limit. Not available for GitHub Enterprise. Logged in users can make 5000 requests per hour. See: https://developer.github.com/v3/#rate-limiting """ click.secho('Rate limit: ' + str(self.config.api.ratelimit_remaining), fg=self.config.clr_message) @authenticate def repositories(self, repos, limit=1000, pager=False, repo_filter='', print_output=True, sort=True): """List all repos matching the given filter. :type repos: list :param repos: A list of `github3` Repository. :type limit: int :param limit: The number of items to display. :type pager: bool :param pager: Determines whether to show the output in a pager, if available. :type repo_filter: str :param repo_filter: The filter for repo names. Only repos matching the filter will be returned. If None, outputs all repos. :type print_output: bool :param print_output: Determines whether to print the output (True) or return the output as a string (False). :type sort: bool :param sort: Determines whether to sort the issues by: state, repo, created_at. :rtype: str :return: The output if print_output is True else, returns None. """ view_entries = [] for repo in repos: url = repo.clone_url if (repo.full_name is not None and repo_filter in repo.full_name.lower()) or \ (repo.description is not None and repo_filter in repo.description.lower()): view_entries.append( ViewEntry(repo, url=url, sort_key_primary=repo.stargazers_count)) if sort: view_entries = sorted(view_entries, reverse=True) return self.table.build_table(view_entries, limit, pager, self.formatter.format_repo, print_output=print_output) @authenticate def repositories_setup(self, repo_filter, limit=1000, pager=False): """Prepare to list all repos matching the given filter. :type repo_filter: str :param repo_filter: The filter for repo names. Only repos matching the filter will be returned. If None, outputs all repos retrieved by the GitHub API. :type limit: int :param limit: The number of items to display. :type pager: bool :param pager: Determines whether to show the output in a pager, if available. """ self.repositories(self.config.api.repositories(), limit, pager, repo_filter) @authenticate def repository(self, user_repo): """Output detailed information about the given repo. :type user_repo: str :param user_repo: The user/repo. """ try: user, repo = user_repo.split('/') except ValueError: click.secho('Expected argument: user/repo.', fg=self.config.clr_error) return self.web_viewer.view_url(self.base_url + user_repo) @authenticate def search_issues(self, query, limit=1000, pager=False): """Search for all issues matching the given query. :type query: str :param query: The search query. :type limit: int :param limit: The number of items to display. :type pager: bool :param pager: Determines whether to show the output in a pager, if available. """ click.secho('Searching for all matching issues on GitHub...', fg=self.config.clr_message) results = self.config.api.search_issues(query) issues_list = [] for result in results: issues_list.append(result.issue) self.issues(issues_list, limit, pager, sort=False) @authenticate def search_repositories(self, query, sort, limit=1000, pager=False): """Search for all repos matching the given query. :type query: str :param query: The search query. :type sort: str :param sort: Optional: 'stars', 'forks', 'updated'. If not specified, sorting is done by query best match. :type limit: int :param limit: The number of items to display. :type pager: bool :param pager: Determines whether to show the output in a pager, if available. """ click.secho('Searching for all matching repos on GitHub...', fg=self.config.clr_message) results = self.config.api.search_repositories(query, sort) repos = [] for result in results: repos.append(result.repository) self.repositories(repos, limit, pager, sort=False) @authenticate def starred(self, repo_filter, limit=1000, pager=False): """Output starred repos. :type repo_filter: str :param repo_filter: The filter for repo names. Only repos matching the filter will be returned. If None, outputs all starred repos. :type limit: int :param limit: The number of items to display. :type pager: bool :param pager: Determines whether to show the output in a pager, if available. """ self.repositories(self.config.api.starred(), limit, pager, repo_filter.lower()) def trending(self, language, weekly, monthly, devs=False, browser=False, pager=False): """List trending repos for the given language. :type language: str :param language: The language (optional). If blank, shows 'Overall'. :type weekly: bool :param weekly: Determines whether to show the weekly rankings. Daily is the default. :type monthly: bool :param monthly: Determines whether to show the monthly rankings. Daily is the default. If both `monthly` and `weekly` are set, `monthly` takes precedence. :type devs: bool :param devs: determines whether to display the trending devs or repos. Only valid with the -b/--browser option. :type browser: bool :param browser: Determines whether to view the profile in a browser, or in the terminal. :type pager: bool :param pager: Determines whether to show the output in a pager, if available. """ language = language.lower() if language in language_rss_map: language = language_rss_map[language] if monthly: period = 'monthly' url_param = '?since=monthly' elif weekly: period = 'weekly' url_param = '?since=weekly' else: period = 'daily' url_param = '' if browser: webbrowser.open( ('https://github.com/trending' + ('/developers' if devs else '') + ('/' + language if language is not 'overall' else '') + url_param)) else: click.secho( 'Listing {p} trending {l} repos...'.format(l=language, p=period), fg=self.config.clr_message) url = ('http://github-trends.ryotarai.info/rss/github_trends_' + language + '_') url += period + '.rss' items = self.trend_parser.parse(url) self.table.build_table_setup_trending( items.entries, self.formatter.format_trending_entry, limit=sys.maxsize, pager=pager) @authenticate def user(self, user_id, browser=False, text_avatar=False, limit=1000, pager=False): """List information about the logged in user. :type user_id: str :param user_id: The user id/login. If None, returns followers of the logged in user. :type browser: bool :param browser: Determines whether to view the profile in a browser, or in the terminal. :type text_avatar: bool :param text_avatar: Determines whether to view the profile avatar in plain text instead of ansi (default). On Windows this value is always set to True due to lack of support of `img2txt` on Windows. :type limit: int :param limit: The number of items to display. :type pager: bool :param pager: Determines whether to show the output in a pager, if available. """ if browser: webbrowser.open(self.base_url + user_id) else: user = self.config.api.user(user_id) if type(user) is null.NullObject: click.secho('Invalid user.', fg=self.config.clr_error) return output = '' output += click.style(self.avatar_setup(user.avatar_url, text_avatar)) output += click.style(user.login + '\n', fg=self.config.clr_primary) if user.company is not None: output += click.style(user.company + '\n', fg=self.config.clr_secondary) if user.location is not None: output += click.style(user.location + '\n', fg=self.config.clr_secondary) if user.email is not None: output += click.style(user.email + '\n', fg=self.config.clr_secondary) if user.type == 'Organization': output += click.style('Organization\n\n', fg=self.config.clr_tertiary) else: output += click.style( 'Followers: ' + str(user.followers_count) + ' | ', fg=self.config.clr_tertiary) output += click.style( 'Following: ' + str(user.following_count) + '\n\n', fg=self.config.clr_tertiary) output += self.repositories(self.config.api.repositories(user_id), limit, pager, print_output=False) if pager: color = None if platform.system() == 'Windows': color = True click.echo_via_pager(output, color) else: click.secho(output) @authenticate def user_me(self, browser, text_avatar, limit=1000, pager=False): """List information about the logged in user. :type browser: bool :param browser: Determines whether to view the profile in a browser, or in the terminal. :type text_avatar: bool :param text_avatar: Determines whether to view the profile avatar in plain text. On Windows this value is always set to True due to lack of support of `img2txt` on Windows. :type limit: int :param limit: The number of items to display. :type pager: bool :param pager: Determines whether to show the output in a pager, if available. """ self.user(self.config.user_login, browser, text_avatar, limit, pager) @authenticate def view(self, index, view_in_browser=False): """View the given index in a browser. Load urls from ~/.gitsomeconfigurl and stores them in self.config.urls. Open a browser with the url based on the given index. :type index: int :param index: Determines the index to view. :type browser: bool :param browser: Determines whether to view the profile in a browser, or in the terminal. """ self.config.urls = self.config.load_urls(view_in_browser) url = self.config.urls[index-1] click.secho('Viewing ' + url + '...', fg=self.config.clr_message) if view_in_browser: webbrowser.open(self.add_base_url(url)) else: if 'issues/' in url: url = url.replace('issues/', '') self.issue(url) elif len(url.split('/')) == 2: self.repository(url) else: self.web_viewer.view_url(self.add_base_url(url)) ================================================ FILE: gitsome/githubcli.py ================================================ # -*- coding: utf-8 -*- # Copyright 2015 Donne Martin. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of # the License is located at # # http://www.apache.org/licenses/LICENSE-2.0 # # or in the "license" file accompanying this file. This file is # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. from __future__ import unicode_literals from __future__ import print_function import click from .github import GitHub click.disable_unicode_literals_warning = True pass_github = click.make_pass_decorator(GitHub) class GitHubCli(object): """The GitHubCli, builds `click` commands and runs `GitHub` methods.""" @click.group() @click.pass_context def cli(ctx): """Main entry point for GitHubCli. :type ctx: :class:`click.core.Context` :param ctx: An instance of click.core.Context that stores an instance of `github.GitHub`. """ # Create a GitHub object and remember it as the context object. # From this point onwards other commands can refer to it by using the # @pass_github decorator. ctx.obj = GitHub() @cli.command() @click.option('-e', '--enterprise', is_flag=True) @pass_github def configure(github, enterprise): """Configure gitsome. Attempts to authenticate the user and to set up the user's news feed. Usage/Example(s): gh configure gh configure -e gh configure --enterprise :type github: :class:`github.GitHub` :param github: An instance of `github.GitHub`. :type enterprise: bool :param enterprise: Determines whether to configure GitHub Enterprise. Default: False. """ github.configure(enterprise) @cli.command('create-comment') @click.argument('user_repo_number') @click.option('-t', '--text') @pass_github def create_comment(github, user_repo_number, text): """Create a comment on the given issue. Usage: gh create-comment [user_repo_number] [-t/--text] Example(s): gh create-comment donnemartin/saws/1 -t "hello world" gh create-comment donnemartin/saws/1 --text "hello world" :type github: :class:`github.GitHub` :param github: An instance of `github.GitHub`. :type user_repo_number: str :param user_repo_number: The user/repo/issue_number. :type text: str :param text: The comment text. """ github.create_comment(user_repo_number, text) @cli.command('create-issue') @click.argument('user_repo') @click.option('-t', '--issue_title') @click.option('-d', '--issue_desc', required=False) @pass_github def create_issue(github, user_repo, issue_title, issue_desc): """Create an issue. Usage: gh create-issue [user_repo] [-t/--issue_title] [-d/--issue_desc] Example(s): gh create-issue donnemartin/gitsome -t "title" gh create-issue donnemartin/gitsome -t "title" -d "desc" gh create-issue donnemartin/gitsome --issue_title "title" --issue_desc "desc" # NOQA :type github: :class:`github.GitHub` :param github: An instance of `github.GitHub`. :type user_repo: str :param user_repo: The user/repo. :type issue_title: str :param issue_title: The issue title. :type issue_desc: str :param issue_desc: The issue body (optional). """ github.create_issue(user_repo, issue_title, issue_desc) @cli.command('create-repo') @click.argument('repo_name') @click.option('-d', '--repo_desc', required=False) @click.option('-pr', '--private', is_flag=True) @pass_github def create_repo(github, repo_name, repo_desc, private): """Create a repo. Usage: gh create-repo [repo_name] [-d/--repo_desc] [-pr/--private] Example(s): gh create-repo repo_name gh create-repo repo_name -d "desc" gh create-repo repo_name --repo_desc "desc" gh create-repo repo_name -pr gh create-repo repo_name --repo_desc "desc" --private :type github: :class:`github.GitHub` :param github: An instance of `github.GitHub`. :type repo_name: str :param repo_name: The repo name. :type repo_desc: str :param repo_desc: The repo description (optional). :type private: bool :param private: Determines whether the repo is private. Default: False. """ github.create_repo(repo_name, repo_desc, private) @cli.command() @pass_github def emails(github): """List all the user's registered emails. Usage/Example(s): gh emails :type github: :class:`github.GitHub` :param github: An instance of `github.GitHub`. """ github.emails() @cli.command() @click.option('-p', '--pager', is_flag=True) @pass_github def emojis(github, pager): """List all GitHub supported emojis. Usage: gh emojis [-p/--pager] Example(s): gh emojis | grep octo :type github: :class:`github.GitHub` :param github: An instance of `github.GitHub`. :type pager: bool :param pager: Determines whether to show the output in a pager, if available. """ github.emojis(pager) @cli.command() @click.argument('user_or_repo', required=False, default='') @click.option('-pr', '--private', is_flag=True, default=False) @click.option('-p', '--pager', is_flag=True) @pass_github def feed(github, user_or_repo, private, pager): """List all activity for the given user or repo. If `user_or_repo` is not provided, uses the logged in user's news feed seen while visiting https://github.com. If `user_or_repo` is provided, shows either the public or `[-pr/--private]` feed activity of the user or repo. Usage: gh feed [user_or_repo] [-pr/--private] [-p/--pager] Examples: gh feed gh feed | grep foo gh feed donnemartin gh feed donnemartin -pr -p gh feed donnemartin --private --pager gh feed donnemartin/haxor-news -p :type github: :class:`github.GitHub` :param github: An instance of `github.GitHub`. :type user_or_repo: str :param user_or_repo: The user or repo to list events for (optional). If no entry, defaults to the logged in user's feed. :type private: bool :param private: Determines whether to show the private events (True) or public events (False). :type pager: bool :param pager: Determines whether to show the output in a pager, if available. """ github.feed(user_or_repo, private, pager) @cli.command() @click.argument('user', required=False) @click.option('-p', '--pager', is_flag=True) @pass_github def followers(github, user, pager): """List all followers and the total follower count. Usage: gh followers [user] [-p/--pager] Example(s): gh followers gh followers -p gh followers octocat --pager :type github: :class:`github.GitHub` :param github: An instance of `github.GitHub`. :type user: str :param user: The user login (optional). If None, returns the followers of the logged in user. :type pager: bool :param pager: Determines whether to show the output in a pager, if available. """ github.followers(user, pager) @cli.command() @click.argument('user', required=False) @click.option('-p', '--pager', is_flag=True) @pass_github def following(github, user, pager): """List all followed users and the total followed count. Usage: gh following [user] [-p/--pager] Example(s): gh following gh following -p gh following octocat --pager :type github: :class:`github.GitHub` :param github: An instance of `github.GitHub`. :type user: str :param user: The user login. If None, returns the followed users of the logged in user. :type pager: bool :param pager: Determines whether to show the output in a pager, if available. """ github.following(user, pager) @cli.command('gitignore-template') @click.argument('language') @pass_github def gitignore_template(github, language): """Output the gitignore template for the given language. Usage: gh gitignore-template [language] Example(s): gh gitignore-template Python gh gitignore-template Python > .gitignore :type github: :class:`github.GitHub` :param github: An instance of `github.GitHub`. :type language: str :param language: The language. """ github.gitignore_template(language) @cli.command('gitignore-templates') @click.option('-p', '--pager', is_flag=True) @pass_github def gitignore_templates(github, pager): """Output all supported gitignore templates. Usage: gh gitignore-templates Example(s): gh gitignore-templates gh gitignore-templates -p gh gitignore-templates --pager :type github: :class:`github.GitHub` :param github: An instance of `github.GitHub`. :type pager: bool :param pager: Determines whether to show the output in a pager, if available. """ github.gitignore_templates(pager) @cli.command() @click.argument('user_repo_number') @pass_github def issue(github, user_repo_number): """Output detailed information about the given issue. Usage: gh issue [user_repo_number] Example(s): gh issue donnemartin/saws/1 :type github: :class:`github.GitHub` :param github: An instance of `github.GitHub`. :type user_repo_number: str :param user_repo_number: The user/repo/issue_number. """ github.issue(user_repo_number) @cli.command() @click.option('-f', '--issue_filter', required=False, default='subscribed') @click.option('-s', '--issue_state', required=False, default='open') @click.option('-l', '--limit', required=False, default=1000) @click.option('-p', '--pager', is_flag=True) @pass_github def issues(github, issue_filter, issue_state, limit, pager): """List all issues matching the filter. Usage: gh issues [-f/--issue_filter] [-s/--issue_state] [-l/--limit] [-p/--pager] # NOQA Example(s): gh issues gh issues -f assigned gh issues ---issue_filter created gh issues -s all -l 20 -p gh issues --issue_state closed --limit 20 --pager gh issues -f created -s all -p :type github: :class:`github.GitHub` :param github: An instance of `github.GitHub`. :type issue_filter: str :param issue_filter: assigned, created, mentioned, subscribed (default). :type issue_state: str :param issue_state: all, open (default), closed. :type limit: int :param limit: The number of items to display. :type pager: bool :param pager: Determines whether to show the output in a pager, if available. """ github.issues_setup(issue_filter, issue_state, limit, pager) @cli.command() @click.argument('license_name') @pass_github def license(github, license_name): """Output the license template for the given license. Usage: gh license [license_name] Example(s): gh license apache-2.0 gh license mit > LICENSE :type github: :class:`github.GitHub` :param github: An instance of `github.GitHub`. :type license_name: str :param license_name: The license name. """ github.license(license_name) @cli.command() @pass_github def licenses(github): """Output all supported license templates. Usage/Example(s): gh licenses :type github: :class:`github.GitHub` :param github: An instance of `github.GitHub`. """ github.licenses() @cli.command() @click.option('-b', '--browser', is_flag=True) @click.option('-t', '--text_avatar', is_flag=True) @click.option('-l', '--limit', required=False, default=1000) @click.option('-p', '--pager', is_flag=True) @pass_github def me(github, browser, text_avatar, limit, pager): """List information about the logged in user. Usage: gh me [-b/--browser] [-t/--text_avatar] [-l/--limit] [-p/--pager] Example(s): gh me gh me -b gh me --browser gh me -t -l 20 -p gh me --text_avatar --limit 20 --pager :type github: :class:`github.GitHub` :param github: An instance of `github.GitHub`. :type browser: bool :param browser: Determines whether to view the profile in a browser, or in the terminal. :type text_avatar: bool :param text_avatar: Determines whether to view the profile avatar in plain text instead of ansi (default). On Windows this value is always set to True due to lack of support of `img2txt` on Windows. :type limit: int :param limit: The number of user repos to display. :type pager: bool :param pager: Determines whether to show the output in a pager, if available. """ github.user_me(browser, text_avatar, limit, pager) @cli.command() @click.option('-l', '--limit', required=False, default=1000) @click.option('-p', '--pager', is_flag=True) @pass_github def notifications(github, limit, pager): """List all notifications. Usage: gh notifications [-l/--limit] [-p/--pager] Example(s): gh notifications gh notifications -l 20 -p gh notifications --limit 20 --pager :type github: :class:`github.GitHub` :param github: An instance of `github.GitHub`. :type limit: int :param limit: The number of items to display. :type pager: bool :param pager: Determines whether to show the output in a pager, if available. """ github.notifications(limit, pager) @cli.command('octo') @click.argument('say', required=False) @pass_github def octocat(github, say): """Output an Easter egg or the given message from Octocat. Usage: gh octo [say] Example(s): gh octo gh octo "foo bar" :type github: :class:`github.GitHub` :param github: An instance of `github.GitHub`. :type say: str :param say: What Octocat should say. If say is None, octocat speaks an Easter egg. """ github.octocat(say) @cli.command('pull-request') @click.argument('user_repo_number') @pass_github def pull_request(github, user_repo_number): """Output detailed information about the given pull request. Usage: gh pull-request [user_repo_number] Example(s): gh pull-request donnemartin/saws/80 :type github: :class:`github.GitHub` :param github: An instance of `github.GitHub`. :type user_repo_number: str :param user_repo_number: The user/repo/pull_number. """ github.issue(user_repo_number) @cli.command('pull-requests') @click.option('-l', '--limit', required=False, default=1000) @click.option('-p', '--pager', is_flag=True) @pass_github def pull_requests(github, limit, pager): """List all pull requests. Usage: gh pull-requests [-l/--limit] [-p/--pager] Example(s): gh pull-requests gh pull-requests -l 20 -p gh pull-requests --limit 20 --pager :type github: :class:`github.GitHub` :param github: An instance of `github.GitHub`. :type limit: int :param limit: The number of items to display. :type pager: bool :param pager: Determines whether to show the output in a pager, if available. """ github.pull_requests(limit, pager) @cli.command('rate-limit') @pass_github def rate_limit(github): """Output the rate limit. Not available for GitHub Enterprise. Logged in users can make 5000 requests per hour. See: https://developer.github.com/v3/#rate-limiting Usage/Example(s): gh rate-limit :type github: :class:`github.GitHub` :param github: An instance of `github.GitHub`. """ github.rate_limit() @cli.command('repos') @click.argument('repo_filter', required=False, default='') @click.option('-l', '--limit', required=False, default=1000) @click.option('-p', '--pager', is_flag=True) @pass_github def repositories(github, repo_filter, limit, pager): """List all repos matching the given filter. Usage: gh repos [repo_filter] [-l/--limit] [-p/--pager] Example(s): gh repos gh repos "data-science" gh repos "data-science" -l 20 -p gh repos "data-science" --limit 20 --pager :type github: :class:`github.GitHub` :param github: An instance of `github.GitHub`. :type repo_filter: str :param repo_filter: The filter for repo names. Only repos matching the filter will be returned. If None, outputs all the logged in user's repos. :type limit: int :param limit: The number of items to display. :type pager: bool :param pager: Determines whether to show the output in a pager, if available. """ github.repositories_setup(repo_filter, limit, pager) @cli.command('repo') @click.argument('user_repo') @pass_github def repository(github, user_repo): """Output detailed information about the given repo. Usage: gh repo [user_repo] Example(s): gh repo donnemartin/gitsome :type github: :class:`github.GitHub` :param github: An instance of `github.GitHub`. :type user_repo: str :param user_repo: The user/repo. """ github.repository(user_repo) @cli.command('search-issues') @click.argument('query') @click.option('-l', '--limit', required=False, default=1000) @click.option('-p', '--pager', is_flag=True) @pass_github def search_issues(github, query, limit, pager): """Search for all issues matching the given query. Usage: gh search-issues [query] [-l/--limit] [-p/--pager] Example(s): gh search-issues "foo type:pr author:donnemartin" -l 20 -p gh search-issues "foobarbaz in:title created:>=2015-01-01" --limit 20 --pager # NOQA Additional Example(s): Search issues that have your user name tagged @donnemartin: gh search-issues "is:issue donnemartin is:open" -p Search issues that have the most +1s: gh search-issues "is:open is:issue sort:reactions-+1-desc" -p Search issues that have the most comments: gh search-issues "is:open is:issue sort:comments-desc" -p Search issues with the "help wanted" tag: gh search-issues "is:open is:issue label:\"help wanted\"" -p Search all your open private issues: gh search-issues "is:open is:issue is:private" -p The query can contain any combination of the following supported qualifers: - `type` With this qualifier you can restrict the search to issues or pull request only. - `in` Qualifies which fields are searched. With this qualifier you can restrict the search to just the title, body, comments, or any combination of these. - `author` Finds issues created by a certain user. - `assignee` Finds issues that are assigned to a certain user. - `mentions` Finds issues that mention a certain user. - `commenter` Finds issues that a certain user commented on. - `involves` Finds issues that were either created by a certain user, assigned to that user, mention that user, or were commented on by that user. - `state` Filter issues based on whether they’re open or closed. - `labels` Filters issues based on their labels. - `language` Searches for issues within repositories that match a certain language. - `created` or `updated` Filters issues based on times of creation, or when they were last updated. - `comments` Filters issues based on the quantity of comments. - `user` or `repo` Limits searches to a specific user or repository. For more information about these qualifiers, see: http://git.io/d1oELA :type github: :class:`github.GitHub` :param github: An instance of `github.GitHub`. :type query: str :param query: The search query. :type limit: int :param limit: The number of items to display. :type pager: bool :param pager: Determines whether to show the output in a pager, if available. """ github.search_issues(query, limit, pager) @cli.command('search-repos') @click.argument('query') @click.option('-s', '--sort', required=False, default='') @click.option('-l', '--limit', required=False, default=1000) @click.option('-p', '--pager', is_flag=True) @pass_github def search_repositories(github, query, sort, limit, pager): """Search for all repos matching the given query. Usage: gh search-repos [query] [-s/--sort] [-l/--limit] [-p/--pager] Example(s): gh search-repos "maps language:python" -s stars -l 20 -p gh search-repos "created:>=2015-01-01 stars:>=1000 language:python" --sort stars --limit 20 --pager # NOQA The query can contain any combination of the following supported qualifers: - `in` Qualifies which fields are searched. With this qualifier you can restrict the search to just the repository name, description, readme, or any combination of these. - `size` Finds repositories that match a certain size (in kilobytes). - `forks` Filters repositories based on the number of forks, and/or whether forked repositories should be included in the results at all. - `created` or `pushed` Filters repositories based on times of creation, or when they were last updated. Format: `YYYY-MM-DD`. Examples: `created:<2011`, `pushed:<2013-02`, `pushed:>=2013-03-06` - `user` or `repo` Limits searches to a specific user or repository. - `language` Searches repositories based on the language they're written in. - `stars` Searches repositories based on the number of stars. For more information about these qualifiers, see: http://git.io/4Z8AkA :type github: :class:`github.GitHub` :param github: An instance of `github.GitHub`. :type query: str :param query: The search query. :type sort: str :param sort: Optional: 'stars', 'forks', 'updated'. If not specified, sorting is done by query best match. :type limit: int :param limit: The number of items to display. :type pager: bool :param pager: Determines whether to show the output in a pager, if available. """ github.search_repositories(query, sort, limit, pager) @cli.command() @click.argument('repo_filter', required=False, default='') @click.option('-l', '--limit', required=False, default=1000) @click.option('-p', '--pager', is_flag=True) @pass_github def starred(github, repo_filter, limit, pager): """Output starred repos. Usage: gh starred [repo_filter] [-l/--limit] [-p/--pager] Example(s): gh starred gh starred foo -l 20 -p gh starred foo --limit 20 --pager :type github: :class:`github.GitHub` :param github: An instance of `github.GitHub`. :type repo_filter: str :param repo_filter: The filter for repo names. Only repos matching the filter will be returned. If None, outputs all starred repos. :type limit: int :param limit: The number of items to display. :type pager: bool :param pager: Determines whether to show the output in a pager, if available. """ github.starred(repo_filter, limit, pager) @cli.command() @click.argument('language', required=False, default='Overall') @click.option('-w', '--weekly', is_flag=True) @click.option('-m', '--monthly', is_flag=True) @click.option('-D', '--devs', is_flag=True) @click.option('-b', '--browser', is_flag=True) @click.option('-p', '--pager', is_flag=True) @pass_github def trending(github, language, weekly, monthly, devs, browser, pager): """List trending repos for the given language. Usage: gh trending [language] [-w/--weekly] [-m/--monthly] [-D/--devs] [-b/--browser] [-p/--pager] # NOQA Example(s): gh trending gh trending Python -w -p gh trending Python --weekly --devs --browser gh trending --browser :type github: :class:`github.GitHub` :param github: An instance of `github.GitHub`. :type language: str :param language: The language (optional). If blank, shows 'Overall'. :type weekly: bool :param weekly: Determines whether to show the weekly rankings. Daily is the default. :type monthly: bool :param monthly: Determines whether to show the monthly rankings. Daily is the default. If both `monthly` and `weekly` are set, `monthly` takes precedence. :type devs: bool :param devs: determines whether to display the trending devs or repos. Only valid with the -b/--browser option. :type browser: bool :param browser: Determines whether to view the profile in a browser, or in the terminal. :type pager: bool :param pager: Determines whether to show the output in a pager, if available. """ github.trending(language, weekly, monthly, devs, browser, pager) @cli.command() @click.argument('user_id', required=True) @click.option('-b', '--browser', is_flag=True) @click.option('-t', '--text_avatar', is_flag=True) @click.option('-l', '--limit', required=False, default=1000) @click.option('-p', '--pager', is_flag=True) @pass_github def user(github, user_id, browser, text_avatar, limit, pager): """List information about the given user. Usage: gh user [user_id] [-b/--browser] [-t/--text_avatar] [-l/--limit] [-p/--pager] # NOQA Example(s): gh user octocat gh user octocat -b gh user octocat --browser gh user octocat -t -l 10 -p gh user octocat --text_avatar --limit 10 --pager :type github: :class:`github.GitHub` :param github: An instance of `github.GitHub`. :type user_id: str :param user_id: The user id/login. If None, returns followers of the logged in user. :type browser: bool :param browser: Determines whether to view the profile in a browser, or in the terminal. :type text_avatar: bool :param text_avatar: Determines whether to view the profile avatar in plain text instead of ansi (default). On Windows this value is always set to True due to lack of support of `img2txt` on Windows. :type limit: int :param limit: The number of items to display. :type pager: bool :param pager: Determines whether to show the output in a pager, if available. """ github.user(user_id, browser, text_avatar, limit, pager) @cli.command() @click.argument('index') @click.option('-b', '--browser', is_flag=True) @pass_github def view(github, index, browser): """View the given notification/repo/issue/pull_request/user index. This method is meant to be called after one of the following commands which outputs a table of notifications/repos/issues/pull_requests/users: gh repos gh search_repos gh starred gh issues gh pull_requests gh search_issues gh notifications gh trending gh user gh me Usage: gh view [index] [-b/--browser] Example(s): gh repos gh view 1 gh starred gh view 1 -b gh view 1 --browser :type github: :class:`github.GitHub` :param github: An instance of `github.GitHub`. :type index: str :param index: Determines the index to view. :type browser: bool :param browser: Determines whether to view the profile in a browser, or in the terminal. """ github.view(int(index), browser) ================================================ FILE: gitsome/main.py ================================================ #!/usr/bin/env python # -*- coding: utf-8 -*- # Copyright 2015 Donne Martin. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of # the License is located at # # http://www.apache.org/licenses/LICENSE-2.0 # # or in the "license" file accompanying this file. This file is # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. from __future__ import unicode_literals from __future__ import print_function from .gitsome import GitSome def cli(): """Create and call Gitsome.""" try: gitsome = GitSome() gitsome.run_cli() except (EOFError, KeyboardInterrupt): gitsome.cli.set_return_value(None) if __name__ == "__main__": cli() ================================================ FILE: gitsome/main_cli.py ================================================ #!/usr/bin/env python # -*- coding: utf-8 -*- # Copyright 2015 Donne Martin. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of # the License is located at # # http://www.apache.org/licenses/LICENSE-2.0 # # or in the "license" file accompanying this file. This file is # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. from __future__ import unicode_literals from __future__ import print_function from .githubcli import GitHubCli def cli(): """Create and call GitHubCli.""" github = GitHubCli() github.cli() if __name__ == "__main__": cli() ================================================ FILE: gitsome/rss_feed.py ================================================ # -*- coding: utf-8 -*- # Copyright 2015 Donne Martin. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of # the License is located at # # http://www.apache.org/licenses/LICENSE-2.0 # # or in the "license" file accompanying this file. This file is # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. language_rss_map = { 'c#': 'csharp', 'c++': 'cpp', 'overall': 'all', } ================================================ FILE: gitsome/table.py ================================================ # -*- coding: utf-8 -*- # Copyright 2015 Donne Martin. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of # the License is located at # # http://www.apache.org/licenses/LICENSE-2.0 # # or in the "license" file accompanying this file. This file is # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. from __future__ import unicode_literals from __future__ import print_function import platform import re import sys import click from .view_entry import ViewEntry class Table(object): """Display table information for repos, issues, prs, etc. :type config: :class:`config.Config` :param config: An instance of `config.Config`. """ def __init__(self, config): self.config = config def build_table(self, view_entries, limit, pager, format_method, build_urls=True, print_output=True): """Build the table used for the gh view command. :type view_entries: list :param view_entries: A list of `github3` items. :type limit: int :param limit: Determines the number of items to show. :type format_method: callable :param format_method: A method called to format each item in the table. :type build_urls: bool :param build_urls: Determines whether to build urls for the gh view # command. :type print_output: bool :param print_output: determines whether to print the output (True) or return the output as a string (False). :rtype: str :return: the output if print_output is True, else, return None. """ if build_urls: self.build_table_urls(view_entries) index = 0 output = '' for view_entry in view_entries: index += 1 view_entry.index = index output += format_method(view_entry) + '\n' if index >= limit: break if build_urls: if len(view_entries) > limit: output += click.style((' \n'), fg=self.config.clr_message) if index == 0: output += click.style('No results found', fg=self.config.clr_message) elif build_urls: output += click.style(self.create_tip(index)) else: output += click.style('') if print_output: if pager: color = None if platform.system() == 'Windows': color = True # Strip out Unicode, which seems to have issues on # Windows with click.echo_via_pager. output = re.sub(r'[^\x00-\x7F]+', '', output) click.echo_via_pager(output, color) else: click.secho(output) return None else: return output def build_table_setup(self, items, format_method, limit, pager, build_urls=True): """Convert items to a list of ViewEntry before calling `build_table`. :type items: list :param items: A list of `github3` items. :type format_method: callable :param format_method: A method called to format each item in the table. :type limit: int :param limit: Determines the number of items to show. :type pager: bool :param pager: Determines whether to show results in a pager, if available. :type build_urls: bool :param build_urls: determines whether to build urls for the `gh view` [#] command. """ view_entries = [] for item in items: view_entries.append(ViewEntry(item)) self.build_table(view_entries, limit, pager, format_method, build_urls) def build_table_setup_feed(self, items, format_method, pager): """Perform feed-specific processing before calling `build_table`. :type items: list :param items: A list of `github3` items. :type format_method: callable :param format_method: A method called to format each item in the table. :type pager: bool :param pager: Determines whether to show results in a pager, if available. """ self.build_table_setup(items.entries, format_method, limit=sys.maxsize, pager=pager, build_urls=False) def build_table_setup_user(self, items, format_method, limit, pager, build_urls=True): """Convert items to a list of ViewEntry before calling `build_table`. Specific to GitHub3.User.users. :type items: list :param items: A list of `github3` items. :type format_method: callable :param format_method: A method called to format each item in the table. :type limit: int :param limit: Determines the number of items to show. :type pager: bool :param pager: Determines whether to show results in a pager, if available. :type build_urls: bool :param build_urls: determines whether to build urls for the `gh view` [#] command. """ view_entries = [] for item in items: view_entries.append(ViewEntry(item=item, url=item.login)) self.build_table(view_entries, limit=sys.maxsize, pager=pager, format_method=format_method) def build_table_setup_trending(self, items, format_method, limit, pager, build_urls=True): """Convert items to a list of ViewEntry before calling `build_table`. Specific to feedparser entries. :type items: list :param items: A list of `github3` items. :type format_method: callable :param format_method: A method called to format each item in the table. :type limit: int :param limit: Determines the number of items to show. :type pager: bool :param pager: Determines whether to show results in a pager, if available. :type build_urls: bool :param build_urls: determines whether to build urls for the `gh view` [#] command. """ view_entries = [] for item in items: url = 'https://github.com/' + '/'.join(item['link'].split('/')[-2:]) view_entries.append(ViewEntry(item=item, url=url)) self.build_table(view_entries, limit=sys.maxsize, pager=pager, format_method=format_method) def build_table_urls(self, view_entries): """Build the GitHub urls for the specified view_entries. :type view_entries: list :param view_entries: A list of ViewEntry items. """ for view_entry in view_entries: self.config.urls.append(view_entry.url) self.config.save_urls() def create_tip(self, max_index): """Create the tip about the view command after showing a table. :type max_index: int :param max_index: The index upper bound. :rtype: str :return: The formatted tip. """ tip = click.style(' View the page for ', fg=self.config.clr_message) tip += click.style('1 through ', fg=self.config.clr_view_index) tip += click.style(str(max_index), fg=self.config.clr_view_index) tip += click.style(' with the following command:\n', fg=self.config.clr_message) tip += click.style(' gh view [#] ', fg=self.config.clr_view_index) tip += click.style('optional: [-b/--browser] [--help]\n', fg=self.config.clr_message) return tip ================================================ FILE: gitsome/utils.py ================================================ # -*- coding: utf-8 -*- # Copyright 2015 Donne Martin. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of # the License is located at # # http://www.apache.org/licenses/LICENSE-2.0 # # or in the "license" file accompanying this file. This file is # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. from __future__ import unicode_literals from __future__ import print_function import re import six import shlex from prompt_toolkit.completion import Completion from .completions import META_LOOKUP_GH class TextUtils(object): """Utilities for parsing and matching text.""" def find_matches(self, word, collection, fuzzy): """Find all matches in collection for word. :type word: str :param word: The word before the cursor. :type collection: iterable :param collection: A collection of words to match. :type fuzzy: bool :param fuzzy: Determines whether to use fuzzy matching. :rtype: generator :return: Yields an instance of `prompt_toolkit.completion.Completion`. """ word = self._last_token(word).lower() matches = [] for suggestion in self._find_collection_matches(word, collection, fuzzy): matches.append(suggestion) return matches def get_tokens(self, text): """Parse out all tokens. :type text: str :param text: A string to be split into tokens. :rtype: list :return: A list of strings for each word in the text. """ if text is not None: text = text.strip() words = self._safe_split(text) return words return [] def _last_token(self, text): """Find the last word in text. :type text: str :param text: A string to parse and obtain the last word. :rtype: str :return: The last word in the text. """ if text is not None: text = text.strip() if len(text) > 0: word = self._safe_split(text)[-1] word = word.strip() return word return '' def _fuzzy_finder(self, text, collection, case_sensitive=True): """Customized fuzzy finder with optional case-insensitive matching. Adapted from: https://github.com/amjith/fuzzyfinder. :type text: str :param text: Input string entered by user. :type collection: iterable :param collection: collection of strings which will be filtered based on the input `text`. :type case_sensitive: bool :param case_sensitive: Determines whether the find will be case sensitive. :rtype: generator :return: Yields a list of suggestions narrowed down from `collections` using the `text` input. """ suggestions = [] if case_sensitive: pat = '.*?'.join(map(re.escape, text)) else: pat = '.*?'.join(map(re.escape, text.lower())) regex = re.compile(pat) for item in collection: if case_sensitive: r = regex.search(item) else: r = regex.search(item.lower()) if r: suggestions.append((len(r.group()), r.start(), item)) return (z for _, _, z in sorted(suggestions)) def _find_collection_matches(self, word, collection, fuzzy): """Yield all matching names in list. :type word: str :param word: The word before the cursor. :type collection: iterable :param collection: A collection of words to match. :type fuzzy: bool :param fuzzy: Determines whether to use fuzzy matching. :rtype: generator :return: Yields an instance of `prompt_toolkit.completion.Completion`. """ word = word.lower() if fuzzy: for suggestion in self._fuzzy_finder(word, collection, case_sensitive=False): yield Completion(suggestion, -len(word), display_meta='display_meta') else: for name in sorted(collection): if name.lower().startswith(word) or not word: display = None display_meta = None # print(len(words)) if name in META_LOOKUP_GH: display_meta = META_LOOKUP_GH[name] yield Completion(name, -len(word), display=display, display_meta=display_meta) def _shlex_split(self, text): """Wrapper for shlex, because it does not seem to handle unicode in 2.6. :type text: str :param text: A string to split. :rtype: list :return: A list that contains words for each split element of text. """ if six.PY2: text = text.encode('utf-8') return shlex.split(text) def _safe_split(self, text): """Safely splits the input text. Shlex can't always split. For example, "\" crashes the completer. :type text: str :param text: A string to split. :rtype: list :return: A list that contains words for each split element of text. """ try: words = self._shlex_split(text) return words except: return text def sanitize_if_none(self, text): """Sanitize text to ensure it is not None :type text:str :param text: String to sanitize. :rtype: str :return: String which is empty if None. """ return text if text is not None else '' ================================================ FILE: gitsome/view_entry.py ================================================ # -*- coding: utf-8 -*- # Copyright 2015 Donne Martin. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of # the License is located at # # http://www.apache.org/licenses/LICENSE-2.0 # # or in the "license" file accompanying this file. This file is # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. from __future__ import unicode_literals from __future__ import print_function class ViewEntry(object): """A table entry used with the `gh view` command. :type index: int :param index: The row index. :type item: :class:`github3` repo, issue, thread, etc. :param item: An instance of a `github3` repo, issue, thread, etc. :param sort_key_primary: A class member representing the primary sort key. :param sort_key_secondary: A class member representing the secondary sort key. :param sort_key_tertiary: A class member representing the tertiary sort key. :type url: str :param url: The item's url. """ def __init__(self, item, url=None, index=-1, sort_key_primary=None, sort_key_secondary=None, sort_key_tertiary=None): self.item = item self.url = url self.index = index self.sort_key_primary = sort_key_primary self.sort_key_secondary = sort_key_secondary self.sort_key_tertiary = sort_key_tertiary def __lt__(self, other): """Implement 'less than' used for sorting. :type other: :class:`ViewEntry` :param other: An instance of `ViewEntry` used for comparison. :rtype: bool :return: Determines whether the current ViewEntry is less than the `other` view entry. """ if self.sort_key_primary != other.sort_key_primary or \ not self.sort_key_secondary: return self.sort_key_primary < other.sort_key_primary else: if self.sort_key_secondary != other.sort_key_secondary or \ not self.sort_key_tertiary: return self.sort_key_secondary < other.sort_key_secondary else: return self.sort_key_tertiary < other.sort_key_tertiary ================================================ FILE: gitsome/web_viewer.py ================================================ # -*- coding: utf-8 -*- # Copyright 2015 Donne Martin. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of # the License is located at # # http://www.apache.org/licenses/LICENSE-2.0 # # or in the "license" file accompanying this file. This file is # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. from __future__ import unicode_literals from __future__ import print_function import platform import re from .compat import HTMLParser import click from .lib.html2text.html2text import HTML2Text import requests class WebViewer(object): """Handle viewing of web content within the terminal. :type config: :class:`config.Config` :param config: An instance of `config.Config`. :type html: :class:`HTMLParser.HTMLParser` :param html: An instance of `HTMLParser.HTMLParser`. :type html_to_text: :class:`html2text.HTML2Text` :param html_to_text: An instance of `html2text.HTML2Text`. """ def __init__(self, config): self.config = config try: self.html = HTMLParser.HTMLParser() except: self.html = HTMLParser self.html_to_text = None self._init_html_to_text() def _init_html_to_text(self): """Initialize HTML2Text.""" self.html_to_text = HTML2Text() self.html_to_text.body_width = 0 self.html_to_text.ignore_images = False self.html_to_text.ignore_emphasis = False self.html_to_text.ignore_links = False self.html_to_text.skip_internal_links = False self.html_to_text.inline_links = False self.html_to_text.links_each_paragraph = False def format_markdown(self, text): """Add color to the input markdown using click.style. :type text: str :param text: The markdown text. :rtype: str :return: The input `text`, formatted. """ pattern_url_name = r'[^]]*' pattern_url_link = r'[^)]+' pattern_url = r'([!]*\[{0}]\(\s*{1}\s*\))'.format( pattern_url_name, pattern_url_link) regex_url = re.compile(pattern_url) text = regex_url.sub(click.style(r'\1', fg='green'), text) pattern_url_ref_name = r'[^]]*' pattern_url_ref_link = r'[^]]+' pattern_url_ref = r'([!]*\[{0}]\[\s*{1}\s*\])'.format( pattern_url_ref_name, pattern_url_ref_link) regex_url_ref = re.compile(pattern_url_ref) text = regex_url_ref.sub(click.style(r'\1', fg='green'), text) regex_list = re.compile(r'( \*.*)') text = regex_list.sub(click.style(r'\1', fg='cyan'), text) regex_header = re.compile(r'(#+) (.*)') text = regex_header.sub(click.style(r'\2', fg='yellow'), text) regex_bold = re.compile(r'(\*\*|__)(.*?)\1') text = regex_bold.sub(click.style(r'\2', fg='cyan'), text) regex_code = re.compile(r'(`)(.*?)\1') text = regex_code.sub(click.style(r'\1\2\1', fg='cyan'), text) text = re.sub(r'(\s*\r?\n\s*){2,}', r'\n\n', text) return text def generate_url_contents(self, url): """Generate the formatted contents of the given item's url. Converts the HTML to text using HTML2Text, colors it, then displays the output in a pager. :type url: str :param url: The url whose contents to fetch. :rtype: str :return: The string representation of the formatted url contents. """ try: headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'} # NOQA raw_response = requests.get(url, headers=headers, verify=self.config.verify_ssl) except (requests.exceptions.SSLError, requests.exceptions.ConnectionError) as e: contents = 'Error: ' + str(e) + '\n' contents += 'Try running gh view # with the --browser/-b flag\n' return contents contents = self.html_to_text.handle(raw_response.text) # Strip out Unicode, which seems to have issues when html2txt is # coupled with click.echo_via_pager. contents = re.sub(r'[^\x00-\x7F]+', '', contents) contents = self.format_markdown(contents) return contents def view_url(self, url): """View the given url. :type index: int :param index: The index for the given item, used with the gh view [index] commend. :type url: str :param url: The url to view """ contents = self.generate_url_contents(url) header = click.style('Viewing ' + url + '\n\n', fg=self.config.clr_message) contents = header + contents contents += click.style(('\nView this article in a browser with' ' the -b/--browser flag.\n'), fg=self.config.clr_message) contents += click.style(('\nPress q to quit viewing this ' 'article.\n'), fg=self.config.clr_message) if contents == '{"error":"Not Found"}\n': click.secho('Invalid user/repo combination.') return color = None if platform.system() == 'Windows': color = True click.echo_via_pager(contents, color) ================================================ FILE: requirements-dev.txt ================================================ codecov>=1.3.3 flake8>=2.4.1 gitchangelog>=2.2.1 mock>=1.0.1 pexpect>=3.3 pytest>=2.7.0 sphinx>=1.3.1 Sphinx-PyPI-upload>=0.2.1 tox>=1.9.2 ================================================ FILE: scripts/aws_complete.sh ================================================ complete -C aws_completer aws #complete -C /Users/donnemartin/.virtualenvs/gitsome/bin/aws_completer aws ================================================ FILE: scripts/create_changelog.sh ================================================ #!/usr/bin/env bash gitchangelog > CHANGELOG_DRAFT pandoc --from=markdown --to=rst --output=CHANGELOG.rst CHANGELOG.md ================================================ FILE: scripts/create_readme_rst.sh ================================================ #!/usr/bin/env bash pandoc --from=markdown --to=rst --output=docs/source/README.rst README.md ================================================ FILE: scripts/gh_complete.sh ================================================ _gh_completion() { COMPREPLY=( $( env COMP_WORDS="${COMP_WORDS[*]}" \ COMP_CWORD=$COMP_CWORD \ _GH_COMPLETE=complete $1 ) ) return 0 } complete -F _gh_completion -o default gh; ================================================ FILE: scripts/gitsome ================================================ #!/usr/bin/env python -u from xonsh.main import main main() ================================================ FILE: scripts/gitsome.bat ================================================ @echo off python -m xonsh.main ================================================ FILE: scripts/run_code_checks.sh ================================================ #!/usr/bin/env bash flake8 --max-line-length=80 --exclude=build,docs,scratch,docs,html2text.py,compat.py,xonsh,lib,parser.table.py,parser.out,parser_test_table.py,parser_table.py,completions.py,completions_git.py,tests/data/*,.tox/,.eggs/ . ================================================ FILE: scripts/set_changelog_as_readme.sh ================================================ mv README.md README_temp.md cp CHANGELOG.rst README.md ================================================ FILE: scripts/set_changelog_as_readme_undo.sh ================================================ mv README_temp.md README.md ================================================ FILE: scripts/update_docs.sh ================================================ #!/usr/bin/env bash scripts/create_readme_rst.sh python setup.py build_sphinx ================================================ FILE: scripts/upload_pypi.sh ================================================ #!/usr/bin/env bash # N. Set CHANGELOG as `README.md` scripts/set_changelog_as_readme.sh # O. Register package with PyPi python setup.py register -r pypi # P. Upload to PyPi python setup.py sdist upload -r pypi # Q. Upload Sphinx docs to PyPi python setup.py upload_sphinx # R. Restore `README.md` scripts/set_changelog_as_readme_undo.sh ================================================ FILE: scripts/xonsh ================================================ #!/usr/bin/env python -u from xonsh.main import main main() ================================================ FILE: scripts/xonsh.bat ================================================ @echo off python -m xonsh.main ================================================ FILE: setup.py ================================================ #!/usr/bin/env python # coding=utf-8 """The gitsome installer.""" from __future__ import print_function, unicode_literals import os import sys try: from setuptools import setup, find_packages from setuptools.command.sdist import sdist from setuptools.command.install import install from setuptools.command.develop import develop HAVE_SETUPTOOLS = True except ImportError: from distutils.core import setup from distutils.command.sdist import sdist as sdist from distutils.command.install import install as install HAVE_SETUPTOOLS = False from gitsome.__init__ import __version__ as VERSION TABLES = ['xonsh/lexer_table.py', 'xonsh/parser_table.py'] def clean_tables(): for f in TABLES: if os.path.isfile(f): os.remove(f) print('Remove ' + f) def build_tables(): print('Building lexer and parser tables.') sys.path.insert(0, os.path.dirname(__file__)) from xonsh.parser import Parser Parser(lexer_table='lexer_table', yacc_table='parser_table', outputdir='xonsh') sys.path.pop(0) class xinstall(install): def run(self): clean_tables() build_tables() install.run(self) class xsdist(sdist): def make_release_tree(self, basedir, files): clean_tables() build_tables() sdist.make_release_tree(self, basedir, files) if HAVE_SETUPTOOLS: class xdevelop(develop): def run(self): clean_tables() build_tables() develop.run(self) def main(): if sys.version_info < (3, 5): print('gitsome requires at least Python 3.5.') sys.exit(1) try: if '--name' not in sys.argv: print(logo) except UnicodeEncodeError: pass skw = dict( name='gitsome', description='A Supercharged Git/Shell Autocompleter with GitHub Integration.', # NOQA license='Apache License 2.0', version=VERSION, author='Donne Martin', maintainer='Donne Martin', author_email='donne.martin@gmail.com', url='https://github.com/donnemartin/gitsome', platforms='Cross Platform', classifiers=[ 'Intended Audience :: Developers', 'Intended Audience :: System Administrators', 'License :: OSI Approved :: Apache Software License', 'Natural Language :: English', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Topic :: Software Development', 'Topic :: Software Development :: Libraries :: Python Modules', ], packages=find_packages(), scripts=['scripts/xonsh', 'scripts/xonsh.bat', 'scripts/gitsome', 'scripts/gitsome.bat'], cmdclass={'install': xinstall, 'sdist': xsdist}, ) if HAVE_SETUPTOOLS: skw['setup_requires'] = ['ply'] skw['install_requires'] = [ 'ply>=3.4,<4.0', 'prompt_toolkit>=2.0.0,<2.1.0', 'requests>=2.8.1,<3.0.0', 'colorama>=0.3.3,<1.0.0', 'click>=5.1,<7.0', 'pygments>=2.0.2,<3.0.0', 'feedparser>=5.2.1,<6.0.0', 'pytz>=2016.3,<2017.0', 'docopt>=0.6.2,<1.0.0', 'uritemplate.py>=1.0.0,<4.0.0', ], skw['entry_points'] = { 'pygments.lexers': ['gitsome = xonsh.pyghooks:XonshLexer', 'gitsomecon = xonsh.pyghooks:XonshConsoleLexer', ], 'console_scripts': ['gh = gitsome.main_cli:cli', ], } skw['cmdclass']['develop'] = xdevelop setup(**skw) logo = '' if __name__ == '__main__': main() ================================================ FILE: tests/__init__.py ================================================ # -*- coding: utf-8 -*- # Copyright 2015 Donne Martin. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of # the License is located at # # http://www.apache.org/licenses/LICENSE-2.0 # # or in the "license" file accompanying this file. This file is # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. ================================================ FILE: tests/compat.py ================================================ # -*- coding: utf-8 -*- # Copyright 2015 Donne Martin. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of # the License is located at # # http://www.apache.org/licenses/LICENSE-2.0 # # or in the "license" file accompanying this file. This file is # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. import sys if sys.version_info < (2, 7): import unittest2 as unittest else: import unittest ================================================ FILE: tests/data/__init__.py ================================================ # -*- coding: utf-8 -*- # Copyright 2015 Donne Martin. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of # the License is located at # # http://www.apache.org/licenses/LICENSE-2.0 # # or in the "license" file accompanying this file. This file is # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. ================================================ FILE: tests/data/email.py ================================================ # -*- coding: utf-8 -*- # Copyright 2015 Donne Martin. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of # the License is located at # # http://www.apache.org/licenses/LICENSE-2.0 # # or in the "license" file accompanying this file. This file is # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. formatted_emails = u'\x1b[35m 1. \x1b[0mfoo@baz.com \x1b[0m\n\x1b[32m Primary: True \x1b[0m\x1b[36mVerified: False \x1b[0m\n\x1b[35m 2. \x1b[0mbar@baz.com \x1b[0m\n\x1b[32m Primary: False \x1b[0m\x1b[36mVerified: True \x1b[0m\n\x1b[0m' # NOQA ================================================ FILE: tests/data/emoji.py ================================================ # -*- coding: utf-8 -*- # Copyright 2015 Donne Martin. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of # the License is located at # # http://www.apache.org/licenses/LICENSE-2.0 # # or in the "license" file accompanying this file. This file is # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. formatted_emojis = u'\x1b[35m 1. \x1b[0mdolls \x1b[0m\n\x1b[35m 2. \x1b[0mpalm_tree \x1b[0m\n\x1b[35m 3. \x1b[0muk \x1b[0m\n\x1b[35m 4. \x1b[0m100 \x1b[0m\n\x1b[35m 5. \x1b[0mbaby_chick \x1b[0m\n\x1b[0m' # NOQA ================================================ FILE: tests/data/events.py ================================================ # -*- coding: utf-8 -*- # Copyright 2015 Donne Martin. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of # the License is located at # # http://www.apache.org/licenses/LICENSE-2.0 # # or in the "license" file accompanying this file. This file is # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. formatted_events = u'\x1b[35m 1. \x1b[0mdonnemartin \x1b[0m\x1b[32mcommented on commit \x1b[0m\x1b[36mAAA23e2\x1b[0m\x1b[32m at \x1b[0m\x1b[36muser1/repo1\x1b[0m\x1b[33m (just now)\x1b[0m\n \x1b[36m\x1b[0mfoo\x1b[0m\n\x1b[35m 2. \x1b[0mdonnemartin \x1b[0m\x1b[32mcreated\x1b[0m\x1b[32m branch\x1b[0m\x1b[36m master\x1b[0m\x1b[32m at \x1b[0m\x1b[36muser1/repo1\x1b[0m\x1b[33m (just now)\x1b[0m\n \x1b[36m\x1b[0m\x1b[0m\n\x1b[35m 3. \x1b[0mdonnemartin \x1b[0m\x1b[32mfollowed \x1b[0m\x1b[36muser1/repo1\x1b[0m\x1b[33m (just now)\x1b[0m\n\x1b[35m 4. \x1b[0mdonnemartin \x1b[0m\x1b[32mforked\x1b[0m\x1b[36m user1/repo1\x1b[0m\x1b[33m (just now)\x1b[0m\n\x1b[35m 5. \x1b[0mdonnemartin \x1b[0m\x1b[32mcommented on \x1b[0m\x1b[36muser1/repo1#1\x1b[0m\x1b[33m (just now)\x1b[0m\n \x1b[36m\x1b[0mfoo\x1b[0m\n \x1b[36m\x1b[0mfoo\x1b[0m\n\x1b[35m 6. \x1b[0mdonnemartin \x1b[0m\x1b[32mclosed issue \x1b[0m\x1b[36muser1/repo1#1\x1b[0m\x1b[33m (just now)\x1b[0m\n \x1b[36m\x1b[0mfoo\x1b[0m\n\x1b[35m 7. \x1b[0mdonnemartin \x1b[0m\x1b[32mclosed pull request \x1b[0m\x1b[36muser1/repo1#1\x1b[0m\x1b[33m (just now)\x1b[0m\n \x1b[36m\x1b[0mfoo\x1b[0m\n\x1b[35m 8. \x1b[0mdonnemartin \x1b[0m\x1b[32mpushed to\x1b[0m\x1b[36m master\x1b[0m\x1b[32m at \x1b[0m\x1b[36muser1/repo1\x1b[0m\x1b[33m (just now)\x1b[0m\n \x1b[36m5ee4d1b: \x1b[0m\x1b[0mFix GitHubCli class docstring\x1b[0m\n \x1b[36mfc2309b: \x1b[0m\x1b[0mUpdate gh configure docstring\x1b[0m\n \x1b[36mdde19b7: \x1b[0m\x1b[0mUpdate gh create-comment docstring\x1b[0m\n\x1b[35m 9. \x1b[0mdonnemartin \x1b[0m\x1b[32mreleased \x1b[0m\x1b[36m0.5.0 \x1b[0m\x1b[32mat \x1b[0m\x1b[36muser1/repo1\x1b[0m\x1b[33m (just now)\x1b[0m\n\x1b[0m' ================================================ FILE: tests/data/gitignores.py ================================================ # -*- coding: utf-8 -*- # Copyright 2015 Donne Martin. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of # the License is located at # # http://www.apache.org/licenses/LICENSE-2.0 # # or in the "license" file accompanying this file. This file is # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. formatted_gitignores = u'\x1b[35m 1. \x1b[0mActionscript \x1b[0m\n\x1b[35m 2. \x1b[0mAda \x1b[0m\n\x1b[35m 3. \x1b[0mAgda \x1b[0m\n\x1b[35m 4. \x1b[0mAndroid \x1b[0m\n\x1b[35m 5. \x1b[0mAppEngine \x1b[0m\n\x1b[0m' formatted_gitignores_tip = u' Run the following command to view or download a .gitignore file:\n View: gh gitignore Python\n Download: gh gitignore Python > .gitignore\n' ================================================ FILE: tests/data/issue.py ================================================ # -*- coding: utf-8 -*- # Copyright 2015 Donne Martin. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of # the License is located at # # http://www.apache.org/licenses/LICENSE-2.0 # # or in the "license" file accompanying this file. This file is # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. formatted_issues = u'\x1b[35m 1. \x1b[0mtitle1 \x1b[0m@user2 \x1b[0m\x1b[35m(user1/repo1/issues/1)\x1b[0m\n\x1b[32m State: open \x1b[0m\x1b[36mComments: 1 \x1b[0m\x1b[33mAssignee: user1 \x1b[0m\n\x1b[35m 2. \x1b[0mtitle2 \x1b[0m@user2 \x1b[0m\x1b[35m(user1/repo1/issues/2)\x1b[0m\n\x1b[32m State: open \x1b[0m\x1b[36mComments: 1 \x1b[0m\x1b[33mAssignee: user1 \x1b[0m\n\x1b[35m 3. \x1b[0mtitle3 \x1b[0m@user2 \x1b[0m\x1b[35m(user1/repo1/issues/3)\x1b[0m\n\x1b[32m State: open \x1b[0m\x1b[36mComments: 1 \x1b[0m\x1b[33mAssignee: user1 \x1b[0m\n View the page for \x1b[0m\x1b[35m1 through \x1b[0m\x1b[35m3\x1b[0m with the following command:\n\x1b[0m\x1b[35m gh view [#] \x1b[0moptional: [-b/--browser] [--help]\n\x1b[0m\x1b[0m' formatted_pull_requests = u'\x1b[35m 1. \x1b[0mtitle1 \x1b[0m@user2 \x1b[0m\x1b[35m(user1/repo1/issues/1)\x1b[0m\n\x1b[32m State: open \x1b[0m\x1b[36mComments: 1 \x1b[0m\x1b[33mAssignee: user1 \x1b[0m\n\x1b[35m 2. \x1b[0mtitle1 \x1b[0m@user2 \x1b[0m\x1b[35m(user1/repo1/issues/1)\x1b[0m\n\x1b[32m State: open \x1b[0m\x1b[36mComments: 1 \x1b[0m\x1b[33mAssignee: user1 \x1b[0m\n\x1b[35m 3. \x1b[0mtitle1 \x1b[0m@user2 \x1b[0m\x1b[35m(user1/repo1/issues/1)\x1b[0m\n\x1b[32m State: open \x1b[0m\x1b[36mComments: 1 \x1b[0m\x1b[33mAssignee: user1 \x1b[0m\n View the page for \x1b[0m\x1b[35m1 through \x1b[0m\x1b[35m3\x1b[0m with the following command:\n\x1b[0m\x1b[35m gh view [#] \x1b[0moptional: [-b/--browser] [--help]\n\x1b[0m\x1b[0m' ================================================ FILE: tests/data/license.py ================================================ # -*- coding: utf-8 -*- # Copyright 2015 Donne Martin. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of # the License is located at # # http://www.apache.org/licenses/LICENSE-2.0 # # or in the "license" file accompanying this file. This file is # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. formatted_licenses = u'\x1b[35m 1. \x1b[0mmit \x1b[0m\x1b[32m((MIT License))\x1b[0m\n\x1b[35m 2. \x1b[0mgpl-2.0 \x1b[0m\x1b[32m((GNU General Public License v2.0))\x1b[0m\n\x1b[35m 3. \x1b[0mbsd-2-clause \x1b[0m\x1b[32m((BSD 2-clause "Simplified" License))\x1b[0m\n\x1b[35m 4. \x1b[0misc \x1b[0m\x1b[32m((ISC License))\x1b[0m\n\x1b[35m 5. \x1b[0mepl-1.0 \x1b[0m\x1b[32m((Eclipse Public License 1.0))\x1b[0m\n\x1b[0m' formatted_licenses_tip = u' Run the following command to view or download a LICENSE file:\n gh license apache-2.0\n gh license apache-2.0 > LICENSE\n' ================================================ FILE: tests/data/markdown.py ================================================ # -*- coding: utf-8 -*- # Copyright 2015 Donne Martin. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of # the License is located at # # http://www.apache.org/licenses/LICENSE-2.0 # # or in the "license" file accompanying this file. This file is # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. raw_markdown = u""" [Skip to content](#start-of-content)\n\n[ ](https://github.com/)\n\n[Sign up](/join) [Sign in](/login?return_to=%2Fdonnemartin%2Fsaws)\n\nThis repository\n\n * [Explore](/explore)\n * [Features](/features)\n * [Enterprise](https://enterprise.github.com/)\n * [Pricing](/pricing)\n\n * [ Watch ](/login?return_to=%2Fdonnemartin%2Fsaws) [ 80 ](/donnemartin/saws/watchers)\n * [ Star ](/login?return_to=%2Fdonnemartin%2Fsaws) [ 2,428 ](/donnemartin/saws/stargazers)\n * [ Fork ](/login?return_to=%2Fdonnemartin%2Fsaws) [ 77 ](/donnemartin/saws/network)\n\n# [donnemartin](/donnemartin)/**[saws](/donnemartin/saws)** ![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\n\n * [ Code ![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) ](/donnemartin/saws)\n * [ Issues ![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) ](/donnemartin/saws/issues)\n * [ Pull requests ![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) ](/donnemartin/saws/pulls)\n\n * [ Pulse ![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) ](/donnemartin/saws/pulse)\n * [ Graphs ![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) ](/donnemartin/saws/graphs)\n\n### HTTPS clone URL\n\n### Subversion checkout URL\n\nYou can clone with \n\nHTTPS or \n\nSubversion. [ ](https://help.github.com/articles/which-remote-url-should-i-use)\n\n[ Download ZIP ](/donnemartin/saws/archive/master.zip "Download the contents of donnemartin/saws as a zip file" )\n\nA Supercharged AWS Command Line Interface (CLI). \n\n * [ 544 commits ](/donnemartin/saws/commits/master)\n * [ 2 branches ](/donnemartin/saws/branches)\n * [ 7 releases ](/donnemartin/saws/releases)\n * [ 5 contributors ](/donnemartin/saws/graphs/contributors)\n\n 1. [ Python 99.4% ](/donnemartin/saws/search?l=python)\n 2. [ Shell 0.6% ](/donnemartin/saws/search?l=bash)\n\nPython Shell\n\n[ ](/donnemartin/saws/find/master) [ ](/donnemartin/saws/compare)\n\n_Branch:_ master\n\nSwitch branches/tags\n\n * [Branches](#)\n * [Tags](#)\n\n[ develop ](/donnemartin/saws/tree/develop) [ master ](/donnemartin/saws/tree/master)\n\nNothing to show\n\n[0.3.2](/donnemartin/saws/tree/0.3.2 "0.3.2" )\n\n[0.3.1](/donnemartin/saws/tree/0.3.1 "0.3.1" )\n\n[0.3.0](/donnemartin/saws/tree/0.3.0 "0.3.0" )\n\n[0.2.1](/donnemartin/saws/tree/0.2.1 "0.2.1" )\n\n[0.2.0](/donnemartin/saws/tree/0.2.0 "0.2.0" )\n\n[0.1.1](/donnemartin/saws/tree/0.1.1 "0.1.1" )\n\n[0.1.0](/donnemartin/saws/tree/0.1.0 "0.1.0" )\n\nNothing to show\n\n[saws](/donnemartin/saws)/\n\nLatest commit [ 07c1d7d ](/donnemartin/saws/commit/07c1d7df22c9032b993d18f22162b91fb16e86d6) Nov 30, 2015 ![@donnemartin](https://avatars3.githubusercontent.com/u/5458997?v=3&s=40) [donnemartin](/donnemartin) [Fix](/donnemartin/saws/commit/07c1d7df22c9032b993d18f22162b91fb16e86d6 "Fix #66: Remove docs/build from the source repository." ) [#66](https://github.com/donnemartin/saws/issues/66 "Remove docs/build from the source repository." )[: Remove docs/build from the source repository.](/donnemartin/saws/commit/07c1d7df22c9032b993d18f22162b91fb16e86d6 "Fix #66: Remove docs/build from the source repository." )\n\n[Permalink](/donnemartin/saws/tree/07c1d7df22c9032b993d18f22162b91fb16e86d6) | Failed to load latest commit information. \n---|--- \n![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) | [docs](/donnemartin/saws/tree/master/docs "docs" ) | [Fix](/donnemartin/saws/commit/07c1d7df22c9032b993d18f22162b91fb16e86d6 "Fix #66: Remove docs/build from the source repository." ) [#66](https://github.com/donnemartin/saws/issues/66 "Remove docs/build from the source repository." )[: Remove docs/build from the source repository.](/donnemartin/saws/commit/07c1d7df22c9032b993d18f22162b91fb16e86d6 "Fix #66: Remove docs/build from the source repository." ) | Nov 30, 2015 \n![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) | [saws](/donnemartin/saws/tree/master/saws "saws" ) | [Merge branch \'master\' into develop](/donnemartin/saws/commit/4d174fe79f61eeeda363c0b03e9e53392da671ef "Merge branch \'master\' into develop\n\n* master:\n Removed mention of scripts/release_pypi.sh, which is no longer used.\n Updated commands." ) | Nov 30, 2015 \n![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) | [scripts](/donnemartin/saws/tree/master/scripts "scripts" ) | [Updated CHANGELOG to README script to use the new CHANGELOG.rst.](/donnemartin/saws/commit/5c03b2cc6882590aa567a98cc8dceae780d1428c "Updated CHANGELOG to README script to use the new CHANGELOG.rst." ) | Oct 16, 2015 \n![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) | [tests](/donnemartin/saws/tree/master/tests "tests" ) | [Fix linter line length issues.](/donnemartin/saws/commit/bf1afa4c73bf1a7dd462cb0c92f945d2b7d7cf5a "Fix linter line length issues." ) | Oct 15, 2015 \n![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) | [.gitchangelog.rc](/donnemartin/saws/blob/master/.gitchangelog.rc ".gitchangelog.rc" ) | [Added gitchangelog.](/donnemartin/saws/commit/7347e0cc55a9ac75368152b9683881d41982a943 "Added gitchangelog." ) | Sep 15, 2015 \n![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) | [.gitignore](/donnemartin/saws/blob/master/.gitignore ".gitignore" ) | [Fix](/donnemartin/saws/commit/07c1d7df22c9032b993d18f22162b91fb16e86d6 "Fix #66: Remove docs/build from the source repository." ) [#66](https://github.com/donnemartin/saws/issues/66 "Remove docs/build from the source repository." )[: Remove docs/build from the source repository.](/donnemartin/saws/commit/07c1d7df22c9032b993d18f22162b91fb16e86d6 "Fix #66: Remove docs/build from the source repository." ) | Nov 30, 2015 \n![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) | [.travis.yml](/donnemartin/saws/blob/master/.travis.yml ".travis.yml" ) | [Revert Python 3.5 build and test support due to:](/donnemartin/saws/commit/36e073de59b0fbe872fb30c80f3521c3b558b37f "Revert Python 3.5 build and test support due to: https://github.com/travis-ci/travis-ci/issues/4794." ) [travis-ci/travis-ci#\u2026](https://github.com/travis-ci/travis-ci/issues/4794 "Tox builds for Python 3.5 fail" ) | Sep 23, 2015 \n![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) | [CHANGELOG.md](/donnemartin/saws/blob/master/CHANGELOG.md "CHANGELOG.md" ) | [Prepped version 0.3.2.](/donnemartin/saws/commit/0408df93b135542c2388fe60444ba4651feec481 "Prepped version 0.3.2." ) | Oct 16, 2015 \n![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) | [CHANGELOG.rst](/donnemartin/saws/blob/master/CHANGELOG.rst "CHANGELOG.rst" ) | [Prepped version 0.3.2.](/donnemartin/saws/commit/0408df93b135542c2388fe60444ba4651feec481 "Prepped version 0.3.2." ) | Oct 16, 2015 \n![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) | [CHECKLIST.md](/donnemartin/saws/blob/master/CHECKLIST.md "CHECKLIST.md" ) | [Removed mention of scripts/release_pypi.sh, which is no longer used.](/donnemartin/saws/commit/ba73311f8dd04e9a1eb159ff4d8c741a22027e50 "Removed mention of scripts/release_pypi.sh, which is no longer used." ) | Nov 16, 2015 \n![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) | [CONTRIBUTING.md](/donnemartin/saws/blob/master/CONTRIBUTING.md "CONTRIBUTING.md" ) | [Add note about AWS collaboration.](/donnemartin/saws/commit/9fb37599835024ef7c99a7f00822aab54904fa5b "Add note about AWS collaboration." ) | Nov 30, 2015 \n![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) | [Dockerfile](/donnemartin/saws/blob/master/Dockerfile "Dockerfile" ) | [Add Dockerfile](/donnemartin/saws/commit/7e185a10918059f0c763b1437d540b78432ea1a9 "Add Dockerfile" ) | Sep 26, 2015 \n![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) | [INSTALLATION.md](/donnemartin/saws/blob/master/INSTALLATION.md "INSTALLATION.md" ) | [Added INSTALLATION.md.](/donnemartin/saws/commit/8317c4c92441f40ce20dad5ced97fe090f77e93e "Added INSTALLATION.md." ) | Oct 3, 2015 \n![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) | [LICENSE.txt](/donnemartin/saws/blob/master/LICENSE.txt "LICENSE.txt" ) | [Added LICENSE.](/donnemartin/saws/commit/1166ffddffffbd981868d700df1f8ecc2061847f "Added LICENSE." ) | Sep 19, 2015 \n![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) | [MANIFEST.in](/donnemartin/saws/blob/master/MANIFEST.in "MANIFEST.in" ) | [Updated MANIFEST.in.](/donnemartin/saws/commit/cba64a756e8dd193bb93083ee7aecd4316c2c9b4 "Updated MANIFEST.in." ) | Sep 19, 2015 \n![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) | [README.md](/donnemartin/saws/blob/master/README.md "README.md" ) | [Fix #todo-add-more-resources anchor link.](/donnemartin/saws/commit/8427a6432c58aa9f95bf4fe26ab76a13645a8261 "Fix #todo-add-more-resources anchor link." ) | Nov 30, 2015 \n![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) | [appveyor.yml](/donnemartin/saws/blob/master/appveyor.yml "appveyor.yml" ) | [Added Appveyor support, inspired by](/donnemartin/saws/commit/f756340ad4f9f4d47860644b0e677f49dac575d4 "Added Appveyor support, inspired by https://github.com/audreyr/cookiecutter/blob/master/appveyor.yml." ) [https://github.com/audreyr/cookie\u2026](https://github.com/audreyr/cookiecutter/blob/master/appveyor.yml) | Sep 28, 2015 \n![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) | [requirements-dev.txt](/donnemartin/saws/blob/master/requirements-dev.txt "requirements-dev.txt" ) | [Added Sphinx-PyPI-upload support.](/donnemartin/saws/commit/7068598aa117cc16d6e32000afa05963d3d76093 "Added Sphinx-PyPI-upload support." ) | Sep 19, 2015 \n![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) | [setup.cfg](/donnemartin/saws/blob/master/setup.cfg "setup.cfg" ) | [Added Sphinx-PyPI-upload support.](/donnemartin/saws/commit/7068598aa117cc16d6e32000afa05963d3d76093 "Added Sphinx-PyPI-upload support." ) | Sep 19, 2015 \n![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) | [setup.py](/donnemartin/saws/blob/master/setup.py "setup.py" ) | [Fixed](/donnemartin/saws/commit/1628d3efda27ba638833b54bdc3124db68a7fb30 "Fixed #53: Shortcuts should be placed separately in a .saws.shortcuts file." ) [#53](https://github.com/donnemartin/saws/issues/53 "Shortcuts should be placed separately in a .saws.shortcuts file" )[: Shortcuts should be placed separately in a .saws.shortcuts\u2026](/donnemartin/saws/commit/1628d3efda27ba638833b54bdc3124db68a7fb30 "Fixed #53: Shortcuts should be placed separately in a .saws.shortcuts file." ) | Oct 14, 2015 \n![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif) | [tox.ini](/donnemartin/saws/blob/master/tox.ini "tox.ini" ) | [Revert Python 3.5 build and test support due to:](/donnemartin/saws/commit/36e073de59b0fbe872fb30c80f3521c3b558b37f "Revert Python 3.5 build and test support due to: https://github.com/travis-ci/travis-ci/issues/4794." ) [travis-ci/travis-ci#\u2026](https://github.com/travis-ci/travis-ci/issues/4794 "Tox builds for Python 3.5 fail" ) | Sep 23, 2015 \n \n### README.md \n\n[![](https://camo.githubusercontent.com/2af72023269b0b320adf4ec55576435f5c8b79c4/687474703a2f2f692e696d6775722e636f6d2f767a43357a6d412e676966)](https://camo.githubusercontent.com/2af72023269b0b320adf4ec55576435f5c8b79c4/687474703a2f2f692e696d6775722e636f6d2f767a43357a6d412e676966)\n\n[![Build Status](https://camo.githubusercontent.com/c1d6d1e8f7cdac1e6a82bf900832bbbe52d61e95/68747470733a2f2f7472617669732d63692e6f72672f646f6e6e656d617274696e2f736177732e7376673f6272616e63683d6d6173746572)](https://travis-ci.org/donnemartin/saws) [![Documentation Status](https://camo.githubusercontent.com/41afe5f96e9f915ecd9706bdc7dfd19518eaf05a/68747470733a2f2f72656164746865646f63732e6f72672f70726f6a656374732f736177732f62616467652f3f76657273696f6e3d6c6174657374)](http://saws.readthedocs.org/en/latest/?badge=latest) [![Dependency Status](https://camo.githubusercontent.com/11e16ba1560f0232928945436bf86340615d85ae/68747470733a2f2f67656d6e617369756d2e636f6d2f646f6e6e656d617274696e2f736177732e737667)](https://gemnasium.com/donnemartin/saws) [![Codecov](https://camo.githubusercontent.com/e24a50e9dd63768d90d26a9f6be1cf3d0c3b0230/68747470733a2f2f696d672e736869656c64732e696f2f636f6465636f762f632f6769746875622f646f6e6e656d617274696e2f736177732e737667)](https://codecov.io/github/donnemartin/saws/saws)\n\n[![PyPI version](https://camo.githubusercontent.com/818edda38216a62a4bb36557372877febec35086/68747470733a2f2f62616467652e667572792e696f2f70792f736177732e737667)](http://badge.fury.io/py/saws) [![PyPI](https://camo.githubusercontent.com/04574f61c5dff98734bd6a013cc9217b9a428e47/68747470733a2f2f696d672e736869656c64732e696f2f707970692f707976657273696f6e732f736177732e737667)](https://pypi.python.org/pypi/saws/) [![License](https://camo.githubusercontent.com/8bf81d4b6a63eaf6f712e3092fe7238f8716e615/687474703a2f2f696d672e736869656c64732e696f2f3a6c6963656e73652d6170616368652d626c75652e737667)](http://www.apache.org/licenses/LICENSE-2.0.html)\n\n# [](#saws)SAWS\n\n## [](#motivation)Motivation\n\n### [](#aws-cli)AWS CLI\n\nAlthough the [AWS CLI](https://github.com/aws/aws-cli) is a great resource to manage your AWS-powered services, it\'s **tough to remember usage** of:\n\n * 50+ top-level commands\n * 1400+ subcommands\n * Countless command-specific options\n * Resources such as instance tags and buckets\n\n### [](#saws-a-supercharged-aws-cli)SAWS: A Supercharged AWS CLI\n\n`SAWS` aims to **supercharge** the AWS CLI with features focusing on:\n\n * **Improving ease-of-use**\n * **Increasing productivity**\n\nUnder the hood, `SAWS` is **powered by the AWS CLI** and supports the **same commands** and **command structure**.\n\n`SAWS` and `AWS CLI` Usage:\n \n \n aws [parameters] [options]\n \n\n`SAWS` features:\n\n * Auto-completion of: \n * Commands\n * Subcommands\n * Options\n * Auto-completion of resources: \n * Bucket names\n * Instance ids\n * Instance tags\n * [More coming soon!](#todo-add-more-resources)\n * Customizable shortcuts\n * Fuzzy completion of resources and shortcuts\n * Fish-style auto-suggestions\n * Syntax and output highlighting\n * Execution of shell commands\n * Command history\n * Contextual help\n * Toolbar options\n\n`SAWS` is available for Mac, Linux, Unix, and [Windows](#windows-support).\n\n[![](https://camo.githubusercontent.com/571a2d19a969447b477713d54d1d3ba46d7418ee/687474703a2f2f692e696d6775722e636f6d2f456f31327139542e706e67)](https://camo.githubusercontent.com/571a2d19a969447b477713d54d1d3ba46d7418ee/687474703a2f2f692e696d6775722e636f6d2f456f31327139542e706e67)\n\n## [](#index)Index\n\n### [](#features)Features\n\n * [Syntax and Output Highlighting](#syntax-and-output-highlighting)\n * [Auto-Completion of Commands, Subcommands, and Options](#auto-completion-of-commands-subcommands-and-options)\n * [Auto-Completion of AWS Resources](#auto-completion-of-aws-resources)\n * [S3 Buckets](#s3-buckets)\n * [EC2 Instance Ids](#ec2-instance-ids)\n * [EC2 Instance Tags](#ec2-instance-tags)\n * [TODO: Add More Resources](#todo-add-more-resources)\n * [Customizable Shortcuts](#customizable-shortcuts)\n * [Fuzzy Resource and Shortcut Completion](#fuzzy-resource-and-shortcut-completion)\n * [Fish-Style Auto-Suggestions](#fish-style-auto-suggestions)\n * [Executing Shell Commands](#executing-shell-commands)\n * [Command History](#command-history)\n * [Contextual Help](#contextual-help)\n * [Contextual Command Line Help](#contextual-command-line-help)\n * [Contextual Web Docs](#contextual-web-docs)\n * [Toolbar Options](#toolbar-options)\n * [Windows Support](#windows-support)\n\n### [](#installation-and-tests)Installation and Tests\n\n * [Installation](#installation)\n * [Pip Installation](#pip-installation)\n * [Virtual Environment and Docker Installation](#virtual-environment-and-docker-installation)\n * [AWS Credentials and Named Profiles](#aws-credentials-and-named-profiles)\n * [Supported Python Versions](#supported-python-versions)\n * [Supported Platforms](#supported-platforms)\n * [Developer Installation](#developer-installation)\n * [Continuous Integration](#continuous-integration)\n * [Dependencies Management](#dependencies-management)\n * [Unit Tests and Code Coverage](#unit-tests-and-code-coverage)\n * [Documentation](#documentation)\n\n### [](#misc)Misc\n\n * [Contributing](#contributing)\n * [Credits](#credits)\n * [Contact Info](#contact-info)\n * [License](#license)\n\n## [](#syntax-and-output-highlighting)Syntax and Output Highlighting\n\n[![](https://camo.githubusercontent.com/c527a4c00862bb1fe47c272a3c3563418f06866f/687474703a2f2f692e696d6775722e636f6d2f785144707737302e706e67)](https://camo.githubusercontent.com/c527a4c00862bb1fe47c272a3c3563418f06866f/687474703a2f2f692e696d6775722e636f6d2f785144707737302e706e67)\n\nYou can control which theme to load for syntax highlighting by updating your [~/.sawsrc](https://github.com/donnemartin/saws/blob/master/saws/sawsrc) file:\n \n \n # Visual theme. Possible values: manni, igor, xcode, vim, autumn, vs, rrt,\n # native, perldoc, borland, tango, emacs, friendly, monokai, paraiso-dark,\n # colorful, murphy, bw, pastie, paraiso-light, trac, default, fruity\n theme = vim\n \n\n## [](#auto-completion-of-commands-subcommands-and-options)Auto-Completion of Commands, Subcommands, and Options\n\n`SAWS` provides smart autocompletion as you type. Entering the following command will interactively list and auto-complete all subcommands **specific only** to `ec2`:\n \n \n aws ec2\n \n\n[![](https://camo.githubusercontent.com/df93a08e1371694a8086f3b45ba6e5ec955212e2/687474703a2f2f692e696d6775722e636f6d2f5032744c3976572e706e67)](https://camo.githubusercontent.com/df93a08e1371694a8086f3b45ba6e5ec955212e2/687474703a2f2f692e696d6775722e636f6d2f5032744c3976572e706e67)\n\n## [](#auto-completion-of-aws-resources)Auto-Completion of AWS Resources\n\nIn addition to the default commands, subcommands, and options the AWS CLI provides, `SAWS` supports auto-completion of your AWS resources. Currently, bucket names, instance ids, and instance tags are included, with additional support for more resources [under development](#todo-add-more-resources).\n\n### [](#s3-buckets)S3 Buckets\n\nOption for `s3api`:\n \n \n --bucket\n \n\nSample Usage:\n \n \n aws s3api get-bucket-acl --bucket\n \n\nSyntax for `s3`:\n \n \n s3://\n \n\nSample Usage:\n \n \n aws s3 ls s3://\n \n\nNote: The example below demonstrates the use of [fuzzy resource completion](/donnemartin/saws/blob/master/fuzzy-resource-and-shortcutcompletion):\n\n[![](https://camo.githubusercontent.com/756f14d2ed77a7a9a3da9b0d9501cab9b7433e36/687474703a2f2f692e696d6775722e636f6d2f333943415335542e706e67)](https://camo.githubusercontent.com/756f14d2ed77a7a9a3da9b0d9501cab9b7433e36/687474703a2f2f692e696d6775722e636f6d2f333943415335542e706e67)\n\n### [](#ec2-instance-ids)EC2 Instance Ids\n\nOption for `ec2`:\n \n \n --instance-ids\n \n\nSample Usage:\n \n \n aws ec2 describe-instances --instance-ids\n aws ec2 ls --instance-ids\n \n\nNote: The `ls` command demonstrates the use of [customizable shortcuts](#customizable-shortcuts):\n\n[![](https://camo.githubusercontent.com/d4b0298e0ecd0afc1dac0d62617a4be53a44f598/687474703a2f2f692e696d6775722e636f6d2f6a46794353586c2e706e67)](https://camo.githubusercontent.com/d4b0298e0ecd0afc1dac0d62617a4be53a44f598/687474703a2f2f692e696d6775722e636f6d2f6a46794353586c2e706e67)\n\n### [](#ec2-instance-tags)EC2 Instance Tags\n\nOption for `ec2`:\n \n \n --ec2-tag-key\n --ec2-tag-value\n \n\nSample Usage:\n \n \n aws ec2 ls --ec2-tag-key\n aws ec2 ls --ec2-tag-value\n \n\n**Tags support wildcards** with the `*` character.\n\nNote: `ls`, `--ec2-tag-value`, and `--ec2-tag-key` demonstrate the use of [customizable shortcuts](#customizable-shortcuts):\n\n[![](https://camo.githubusercontent.com/593e9fc80cc6e1fb8e0fd7324cb4cb3a1b51bf2f/687474703a2f2f692e696d6775722e636f6d2f56494b7747335a2e706e67)](https://camo.githubusercontent.com/593e9fc80cc6e1fb8e0fd7324cb4cb3a1b51bf2f/687474703a2f2f692e696d6775722e636f6d2f56494b7747335a2e706e67)\n\n### [](#todo-add-more-resources)TODO: Add More Resources\n\nFeel free to [submit an issue or a pull request](#contributions) if you\'d like support for additional resources.\n\n## [](#customizable-shortcuts)Customizable Shortcuts\n\nThe [~/.saws.shortcuts](https://github.com/donnemartin/saws/blob/master/saws/saws.shortcuts) file contains shortcuts that you can modify. It comes pre-populated with several [handy shortcuts](https://github.com/donnemartin/saws/blob/master/saws/saws.shortcuts) out of the box. You can combine shortcuts with [fuzzy completion](#fuzzy-resource-and-shortcut-completion) for even less keystrokes. Below are a few examples.\n\nList all EC2 instances:\n \n \n aws ec2 ls\n \n\nList all running EC2 instances:\n \n \n aws ec2 ls --ec2-state running # fuzzy shortcut: aws ecstate\n \n\n[![](https://camo.githubusercontent.com/7b56fd124f76cf2a9573de7b963bbb791f4cae16/687474703a2f2f692e696d6775722e636f6d2f6a594645736f4d2e706e67)](https://camo.githubusercontent.com/7b56fd124f76cf2a9573de7b963bbb791f4cae16/687474703a2f2f692e696d6775722e636f6d2f6a594645736f4d2e706e67)\n\nList all EC2 instances with a matching tag (supports wildcards `*`):\n \n \n aws ec2 ls --ec2-tag-key # fuzzy shortcut: aws ectagk\n aws ec2 ls --ec2-tag-value # fuzzy shortcut: aws ectagv\n \n\n[![](https://camo.githubusercontent.com/c45874a55d4ac6485ba78c6d4a91cc1d928703a4/687474703a2f2f692e696d6775722e636f6d2f505375775549772e706e67)](https://camo.githubusercontent.com/c45874a55d4ac6485ba78c6d4a91cc1d928703a4/687474703a2f2f692e696d6775722e636f6d2f505375775549772e706e67)\n\nList EC2 instance with matching id:\n \n \n aws ec2 ls --instance-ids # fuzzy shortcut: aws eclsi\n \n\n[![](https://camo.githubusercontent.com/48136027017398a300b9fe968e50e2e7c2a8b18b/687474703a2f2f692e696d6775722e636f6d2f774763554373612e706e67)](https://camo.githubusercontent.com/48136027017398a300b9fe968e50e2e7c2a8b18b/687474703a2f2f692e696d6775722e636f6d2f774763554373612e706e67)\n\nList all DynamoDB tables:\n \n \n aws dynamodb ls # fuzzy shortcut: aws dls\n \n\nList all EMR clusters:\n \n \n aws emr ls # fuzzy shortcut: aws emls\n \n\nAdd/remove/modify shortcuts in your [~/.saws.shortcuts](https://github.com/donnemartin/saws/blob/master/saws/shortcuts) file to suit your needs.\n\nFeel free to submit:\n\n * An issue to request additional shortcuts\n * A pull request if you\'d like to share your shortcuts (see [contributing guidelines](#contributions))\n\n### [](#fuzzy-resource-and-shortcut-completion)Fuzzy Resource and Shortcut Completion\n\nTo toggle fuzzy completion of AWS resources and shortcuts, use `F3` key.\n\nSample fuzzy shortcuts to start and stop EC2 instances:\n \n \n aws ecstop\n aws ecstart\n \n\nNote: Fuzzy completion currently only works with AWS [resources](#auto-completion-of-aws-resources) and [shortcuts](/donnemartin/saws/blob/master/customizable-shortcuts).\n\n[![](https://camo.githubusercontent.com/a918a4d70ca61189149746e052f5bb2ffec9ae65/687474703a2f2f692e696d6775722e636f6d2f374f76464843772e706e67)](https://camo.githubusercontent.com/a918a4d70ca61189149746e052f5bb2ffec9ae65/687474703a2f2f692e696d6775722e636f6d2f374f76464843772e706e67)\n\n### [](#fish-style-auto-suggestions)Fish-Style Auto-Suggestions\n\n`SAWS` supports Fish-style auto-suggestions. Use the `right arrow` key to complete a suggestion.\n\n[![](https://camo.githubusercontent.com/6671d93da542b2f2cdec589fcb2c174e3bbc25f6/687474703a2f2f692e696d6775722e636f6d2f743532303071312e706e67)](https://camo.githubusercontent.com/6671d93da542b2f2cdec589fcb2c174e3bbc25f6/687474703a2f2f692e696d6775722e636f6d2f743532303071312e706e67)\n\n### [](#executing-shell-commands)Executing Shell Commands\n\n`SAWS` allows you to execute shell commands from the `saws>` prompt.\n\n[![](https://camo.githubusercontent.com/4d26d484e5b94b7ca62607ec1557d847f5db96f3/687474703a2f2f692e696d6775722e636f6d2f4669536e3662322e706e67)](https://camo.githubusercontent.com/4d26d484e5b94b7ca62607ec1557d847f5db96f3/687474703a2f2f692e696d6775722e636f6d2f4669536e3662322e706e67)\n\n### [](#command-history)Command History\n\n`SAWS` keeps track of commands you enter and stores them in `~/.saws-history`. Use the up and down arrow keys to cycle through the command history.\n\n[![](https://camo.githubusercontent.com/a5ded83f813be928236c804a90d85fc0fd5e517d/687474703a2f2f692e696d6775722e636f6d2f7a3852724451422e706e67)](https://camo.githubusercontent.com/a5ded83f813be928236c804a90d85fc0fd5e517d/687474703a2f2f692e696d6775722e636f6d2f7a3852724451422e706e67)\n\n## [](#contextual-help)Contextual Help\n\n`SAWS` supports contextual command line `help` and contextual web `docs`.\n\n### [](#contextual-command-line-help)Contextual Command Line Help\n\nThe `help` command is powered by the AWS CLI and outputs help within the command line.\n\nUsage:\n \n \n aws help\n \n\n[![](https://camo.githubusercontent.com/e10dae70d7f9303bc5fae0e3f5dba453fa905df7/687474703a2f2f692e696d6775722e636f6d2f7a536b7a7436792e706e67)](https://camo.githubusercontent.com/e10dae70d7f9303bc5fae0e3f5dba453fa905df7/687474703a2f2f692e696d6775722e636f6d2f7a536b7a7436792e706e67)\n\n### [](#contextual-web-docs)Contextual Web Docs\n\nSometimes you\'re not quite sure what specific command/subcommand/option combination you need to use. In such cases, browsing through several combinations with the `help` command line is cumbersome versus browsing the online AWS CLI docs through a web browser.\n\n`SAWS` supports contextual web docs with the `docs` command or the `F9` key. `SAWS` will display the web docs specific to the currently entered command and subcommand.\n\nUsage:\n \n \n aws docs\n \n\n[![](https://camo.githubusercontent.com/e0a9e9bd21f2ff174f2e257f0c9966c6620d97d7/687474703a2f2f692e696d6775722e636f6d2f7a4b34494a59702e706e67)](https://camo.githubusercontent.com/e0a9e9bd21f2ff174f2e257f0c9966c6620d97d7/687474703a2f2f692e696d6775722e636f6d2f7a4b34494a59702e706e67)\n\n## [](#toolbar-options)Toolbar Options\n\n`SAWS` supports a number of toolbar options:\n\n * `F2` toggles [output syntax highlighting](#syntax-and-output-highlighting)\n * `F3` toggles [fuzzy completion of AWS resources and shortcuts](#fuzzy-resource-and-shortcut-completion)\n * `F4` toggles [completion of shortcuts](#customizable-shortcuts)\n * `F5` refreshes [resources for auto-completion](#auto-completion-of-aws-resources)\n * `F9` displays the [contextual web docs](#contextual-web-docs)\n * `F10` or `control d` exits `SAWS`\n\n[![](https://camo.githubusercontent.com/f1bb53eda629178af0d849a112193a91daca5628/687474703a2f2f692e696d6775722e636f6d2f37767a384f53632e706e67)](https://camo.githubusercontent.com/f1bb53eda629178af0d849a112193a91daca5628/687474703a2f2f692e696d6775722e636f6d2f37767a384f53632e706e67)\n\n### [](#windows-support)Windows Support\n\n`SAWS` has been tested on Windows 7 and Windows 10.\n\nOn Windows, the [.sawsrc](https://github.com/donnemartin/saws/blob/master/saws/sawsrc) file can be found in `%userprofile%`. For example:\n \n \n C:\\Users\\dmartin\\.sawsrc\n \n\nAlthough you can use the standard Windows command prompt, you\'ll probably have a better experience with either [cmder](https://github.com/cmderdev/cmder) or [conemu](https://github.com/Maximus5/ConEmu).\n\n[![](https://camo.githubusercontent.com/cfd9f911b9960308e0346d90ba4cf2338145fb52/687474703a2f2f692e696d6775722e636f6d2f7055774a57636b2e706e67)](https://camo.githubusercontent.com/cfd9f911b9960308e0346d90ba4cf2338145fb52/687474703a2f2f692e696d6775722e636f6d2f7055774a57636b2e706e67)\n\n## [](#installation)Installation\n\n### [](#pip-installation)Pip Installation\n\n[![PyPI version](https://camo.githubusercontent.com/818edda38216a62a4bb36557372877febec35086/68747470733a2f2f62616467652e667572792e696f2f70792f736177732e737667)](http://badge.fury.io/py/saws) [![PyPI](https://camo.githubusercontent.com/04574f61c5dff98734bd6a013cc9217b9a428e47/68747470733a2f2f696d672e736869656c64732e696f2f707970692f707976657273696f6e732f736177732e737667)](https://pypi.python.org/pypi/saws/)\n\n`SAWS` is hosted on [PyPI](https://pypi.python.org/pypi/saws). The following command will install `SAWS` along with dependencies such as the [AWS CLI](https://github.com/aws/aws-cli):\n \n \n $ pip install saws\n \n\nYou can also install the latest `SAWS` from GitHub source which can contain changes not yet pushed to PyPI:\n \n \n $ pip install git+https://github.com/donnemartin/saws.git\n \n\nIf you are not installing in a [virtualenv](#virtual-environment-and-docker-installation), run with `sudo`:\n \n \n $ sudo pip install saws\n \n\nOnce installed, start `SAWS`:\n \n \n $ saws\n \n\n### [](#virtual-environment-and-docker-installation)Virtual Environment and Docker Installation\n\nIt is recommended that you install Python packages in a [virtualenv](http://docs.python-guide.org/en/latest/dev/virtualenvs/) to avoid potential [issues with dependencies or permissions](https://github.com/donnemartin/saws/issues/15).\n\nTo view `SAWS` `virtualenv` and [Docker](https://www.docker.com/) installation instructions, click [here](https://github.com/donnemartin/saws/blob/master/INSTALLATION.md).\n\n### [](#aws-credentials-and-named-profiles)AWS Credentials and Named Profiles\n\n[Configure your credentials](https://github.com/aws/aws-cli#getting-started) with the AWS CLI:\n \n \n $ aws configure\n \n\nIf you\'d like to use a specific named profile with `SAWS`, run the following commands on OS X, Linux, or Unix:\n \n \n $ export AWS_DEFAULT_PROFILE=user1\n $ saws\n \n\nOr as a one-liner:\n \n \n $ AWS_DEFAULT_PROFILE=user1 saws\n \n\nWindows users can run the following commands:\n \n \n > set AWS_DEFAULT_PROFILE=user1\n > saws\n \n\nCommand line options for starting `SAWS` with a specific profile are [under development](https://github.com/donnemartin/saws/issues/16). For more details on how to install and configure the AWS CLI, refer to the following [documentation](http://docs.aws.amazon.com/cli/latest/userguide/installing.html).\n\n### [](#supported-python-versions)Supported Python Versions\n\n * Python 2.6\n * Python 2.7\n * Python 3.3\n * Python 3.4\n * Pypy\n\nLight testing indicates that `SAWS` also seems to be compatible with Python 3.5.\n\nPypy3 is not supported due to [lack of support](https://github.com/boto/botocore/issues/622) from [boto](https://github.com/boto/boto).\n\n### [](#supported-platforms)Supported Platforms\n\n * Mac OS X \n * Tested on OS X 10.10\n * Linux, Unix \n * Tested on Ubuntu 14.04 LTS\n * Windows \n * Tested on Windows 7 and 10\n\n## [](#developer-installation)Developer Installation\n\nIf you\'re interested in contributing to `SAWS`, run the following commands:\n \n \n $ git clone https://github.com/donnemartin/saws.git\n $ pip install -e .\n $ pip install -r requirements-dev.txt\n $ saws\n \n\n### [](#continuous-integration)Continuous Integration\n\n[![Build Status](https://camo.githubusercontent.com/c1d6d1e8f7cdac1e6a82bf900832bbbe52d61e95/68747470733a2f2f7472617669732d63692e6f72672f646f6e6e656d617274696e2f736177732e7376673f6272616e63683d6d6173746572)](https://travis-ci.org/donnemartin/saws)\n\nContinuous integration details are available on [Travis CI](https://travis-ci.org/donnemartin/saws).\n\n### [](#dependencies-management)Dependencies Management\n\n[![Dependency Status](https://camo.githubusercontent.com/11e16ba1560f0232928945436bf86340615d85ae/68747470733a2f2f67656d6e617369756d2e636f6d2f646f6e6e656d617274696e2f736177732e737667)](https://gemnasium.com/donnemartin/saws)\n\nDependencies management details are available on [Gemnasium](https://gemnasium.com/donnemartin/saws).\n\n### [](#unit-tests-and-code-coverage)Unit Tests and Code Coverage\n\n[![Codecov](https://camo.githubusercontent.com/e24a50e9dd63768d90d26a9f6be1cf3d0c3b0230/68747470733a2f2f696d672e736869656c64732e696f2f636f6465636f762f632f6769746875622f646f6e6e656d617274696e2f736177732e737667)](https://codecov.io/github/donnemartin/saws/saws)\n\n[![](https://camo.githubusercontent.com/a3588f226c5b2c9007fd9e49212ae8294c5bc023/687474703a2f2f636f6465636f762e696f2f6769746875622f646f6e6e656d617274696e2f736177732f6272616e63682e7376673f6272616e63683d6d6173746572)](https://camo.githubusercontent.com/a3588f226c5b2c9007fd9e49212ae8294c5bc023/687474703a2f2f636f6465636f762e696f2f6769746875622f646f6e6e656d617274696e2f736177732f6272616e63682e7376673f6272616e63683d6d6173746572)\n\nCode coverage details are available on [Codecov](https://codecov.io/github/donnemartin/saws/saws).\n\nRun unit tests in your active Python environment:\n \n \n $ python tests/run_tests.py\n \n\nRun unit tests with [tox](https://pypi.python.org/pypi/tox) on multiple Python environments:\n \n \n $ tox\n \n\n### [](#documentation)Documentation\n\n[![Documentation Status](https://camo.githubusercontent.com/41afe5f96e9f915ecd9706bdc7dfd19518eaf05a/68747470733a2f2f72656164746865646f63732e6f72672f70726f6a656374732f736177732f62616467652f3f76657273696f6e3d6c6174657374)](http://saws.readthedocs.org/en/latest/?badge=latest)\n\nSource code documentation is available on [Readthedocs.org](http://saws.readthedocs.org/en/latest/?badge=latest).\n\nRun the following to build the docs:\n \n \n $ scripts/update_docs.sh\n \n\n## [](#contributing)Contributing\n\nContributions are welcome!\n\nReview the [Contributing Guidelines](https://github.com/donnemartin/saws/blob/master/CONTRIBUTING.md) for details on how to:\n\n * Submit issues\n * Submit pull requests\n\n## [](#credits)Credits\n\n * [AWS CLI](https://github.com/aws/aws-cli) by [AWS](https://github.com/aws) for powering `SAWS` under the hood\n * [Python Prompt Toolkit](https://github.com/jonathanslenders/python-prompt-toolkit) by [jonathanslenders](https://github.com/jonathanslenders) for simplifying the creation of `SAWS`\n * [Wharfee](https://github.com/j-bennet/wharfee) by [j-bennet](https://github.com/j-bennet) for inspiring the creation of `SAWS` and for some handy utility functions\n\n## [](#contact-info)Contact Info\n\nFeel free to contact me to discuss any issues, questions, or comments.\n\n * Email: [donne.martin@gmail.com](mailto:donne.martin@gmail.com)\n * Twitter: [donne_martin](https://twitter.com/donne_martin)\n * GitHub: [donnemartin](https://github.com/donnemartin)\n * LinkedIn: [donnemartin](https://www.linkedin.com/in/donnemartin)\n * Website: [donnemartin.com](http://donnemartin.com)\n\n## [](#license)License\n \n \n Copyright 2015 Donne Martin\n \n Licensed under the Apache License, Version 2.0 (the "License");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n \n http://www.apache.org/licenses/LICENSE-2.0\n \n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an "AS IS" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n \n\n * [Status](https://status.github.com/)\n * [API](https://developer.github.com)\n * [Training](https://training.github.com)\n * [Shop](https://shop.github.com)\n * [Blog](https://github.com/blog)\n * [About](https://github.com/about)\n * [Pricing](https://github.com/pricing)\n[ ](https://github.com)\n\n * (C) 2015 GitHub, Inc.\n * [Terms](https://github.com/site/terms)\n * [Privacy](https://github.com/site/privacy)\n * [Security](https://github.com/security)\n * [Contact](https://github.com/contact)\n * [Help](https://help.github.com)\n\nSomething went wrong with that request. Please try again. \n\nYou signed in with another tab or window. [Reload]() to refresh your session. You signed out in another tab or window. [Reload]() to refresh your session.\n """ # NOQA formatted_markdown = u""" \x1b[32m[Skip to content](#start-of-content)\x1b[0m\n\n\x1b[32m[ ](https://github.com/)\x1b[0m\n\n\x1b[32m[Sign up](/join)\x1b[0m \x1b[32m[Sign in](/login?return_to=%2Fdonnemartin%2Fsaws)\x1b[0m\n\nThis repository\n\n\x1b[36m * \x1b[32m[Explore](/explore)\x1b[0m\x1b[0m\n\x1b[36m * \x1b[32m[Features](/features)\x1b[0m\x1b[0m\n\x1b[36m * \x1b[32m[Enterprise](https://enterprise.github.com/)\x1b[0m\x1b[0m\n\x1b[36m * \x1b[32m[Pricing](/pricing)\x1b[0m\x1b[0m\n\n\x1b[36m * \x1b[32m[ Watch ](/login?return_to=%2Fdonnemartin%2Fsaws)\x1b[0m \x1b[32m[ 80 ](/donnemartin/saws/watchers)\x1b[0m\x1b[0m\n\x1b[36m * \x1b[32m[ Star ](/login?return_to=%2Fdonnemartin%2Fsaws)\x1b[0m \x1b[32m[ 2,428 ](/donnemartin/saws/stargazers)\x1b[0m\x1b[0m\n\x1b[36m * \x1b[32m[ Fork ](/login?return_to=%2Fdonnemartin%2Fsaws)\x1b[0m \x1b[32m[ 77 ](/donnemartin/saws/network)\x1b[0m\x1b[0m\n\n\x1b[33m \x1b[32m[donnemartin](/donnemartin)\x1b[0m/\x1b[36m\x1b[32m[saws](/donnemartin/saws)\x1b[0m\x1b[0m \x1b[32m![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\x1b[0m\x1b[0m\n\n\x1b[36m * \x1b[32m[ Code ![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\x1b[0m ](/donnemartin/saws)\x1b[0m\n\x1b[36m * \x1b[32m[ Issues ![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\x1b[0m ](/donnemartin/saws/issues)\x1b[0m\n\x1b[36m * \x1b[32m[ Pull requests ![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\x1b[0m ](/donnemartin/saws/pulls)\x1b[0m\n\n\x1b[36m * \x1b[32m[ Pulse ![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\x1b[0m ](/donnemartin/saws/pulse)\x1b[0m\n\x1b[36m * \x1b[32m[ Graphs ![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\x1b[0m ](/donnemartin/saws/graphs)\x1b[0m\n\n\x1b[33mHTTPS clone URL\x1b[0m\n\n\x1b[33mSubversion checkout URL\x1b[0m\n\nYou can clone with\n\nHTTPS or\n\nSubversion. \x1b[32m[ ](https://help.github.com/articles/which-remote-url-should-i-use)\x1b[0m\n\n\x1b[32m[ Download ZIP ](/donnemartin/saws/archive/master.zip "Download the contents of donnemartin/saws as a zip file" )\x1b[0m\n\nA Supercharged AWS Command Line Interface (CLI). \n\n\x1b[36m * \x1b[32m[ 544 commits ](/donnemartin/saws/commits/master)\x1b[0m\x1b[0m\n\x1b[36m * \x1b[32m[ 2 branches ](/donnemartin/saws/branches)\x1b[0m\x1b[0m\n\x1b[36m * \x1b[32m[ 7 releases ](/donnemartin/saws/releases)\x1b[0m\x1b[0m\n\x1b[36m * \x1b[32m[ 5 contributors ](/donnemartin/saws/graphs/contributors)\x1b[0m\x1b[0m\n\n1. \x1b[32m[ Python 99.4% ](/donnemartin/saws/search?l=python)\x1b[0m\n 2. \x1b[32m[ Shell 0.6% ](/donnemartin/saws/search?l=bash)\x1b[0m\n\nPython Shell\n\n\x1b[32m[ ](/donnemartin/saws/find/master)\x1b[0m \x1b[32m[ ](/donnemartin/saws/compare)\x1b[0m\n\n_Branch:_ master\n\nSwitch branches/tags\n\n\x1b[36m * \x1b[32m[Branches](#)\x1b[0m\x1b[0m\n\x1b[36m * \x1b[32m[Tags](#)\x1b[0m\x1b[0m\n\n\x1b[32m[ develop ](/donnemartin/saws/tree/develop)\x1b[0m \x1b[32m[ master ](/donnemartin/saws/tree/master)\x1b[0m\n\nNothing to show\n\n\x1b[32m[0.3.2](/donnemartin/saws/tree/0.3.2 "0.3.2" )\x1b[0m\n\n\x1b[32m[0.3.1](/donnemartin/saws/tree/0.3.1 "0.3.1" )\x1b[0m\n\n\x1b[32m[0.3.0](/donnemartin/saws/tree/0.3.0 "0.3.0" )\x1b[0m\n\n\x1b[32m[0.2.1](/donnemartin/saws/tree/0.2.1 "0.2.1" )\x1b[0m\n\n\x1b[32m[0.2.0](/donnemartin/saws/tree/0.2.0 "0.2.0" )\x1b[0m\n\n\x1b[32m[0.1.1](/donnemartin/saws/tree/0.1.1 "0.1.1" )\x1b[0m\n\n\x1b[32m[0.1.0](/donnemartin/saws/tree/0.1.0 "0.1.0" )\x1b[0m\n\nNothing to show\n\n\x1b[32m[saws](/donnemartin/saws)\x1b[0m/\n\nLatest commit \x1b[32m[ 07c1d7d ](/donnemartin/saws/commit/07c1d7df22c9032b993d18f22162b91fb16e86d6)\x1b[0m Nov 30, 2015 \x1b[32m![@donnemartin](https://avatars3.githubusercontent.com/u/5458997?v=3&s=40)\x1b[0m \x1b[32m[donnemartin](/donnemartin)\x1b[0m \x1b[32m[Fix](/donnemartin/saws/commit/07c1d7df22c9032b993d18f22162b91fb16e86d6 "Fix #66: Remove docs/build from the source repository." )\x1b[0m \x1b[32m[#66](https://github.com/donnemartin/saws/issues/66 "Remove docs/build from the source repository." )\x1b[0m\x1b[32m[: Remove docs/build from the source repository.](/donnemartin/saws/commit/07c1d7df22c9032b993d18f22162b91fb16e86d6 "Fix #66: Remove docs/build from the source repository." )\x1b[0m\n\n\x1b[32m[Permalink](/donnemartin/saws/tree/07c1d7df22c9032b993d18f22162b91fb16e86d6)\x1b[0m | Failed to load latest commit information. \n---|--- \n\x1b[32m![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\x1b[0m | \x1b[32m[docs](/donnemartin/saws/tree/master/docs "docs" )\x1b[0m | \x1b[32m[Fix](/donnemartin/saws/commit/07c1d7df22c9032b993d18f22162b91fb16e86d6 "Fix #66: Remove docs/build from the source repository." )\x1b[0m \x1b[32m[#66](https://github.com/donnemartin/saws/issues/66 "Remove docs/build from the source repository." )\x1b[0m\x1b[32m[: Remove docs/build from the source repository.](/donnemartin/saws/commit/07c1d7df22c9032b993d18f22162b91fb16e86d6 "Fix #66: Remove docs/build from the source repository." )\x1b[0m | Nov 30, 2015 \n\x1b[32m![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\x1b[0m | \x1b[32m[saws](/donnemartin/saws/tree/master/saws "saws" )\x1b[0m | \x1b[32m[Merge branch \'master\' into develop](/donnemartin/saws/commit/4d174fe79f61eeeda363c0b03e9e53392da671ef "Merge branch \'master\' into develop\n\n* master:\n Removed mention of scripts/release_pypi.sh, which is no longer used.\n Updated commands." )\x1b[0m | Nov 30, 2015 \n\x1b[32m![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\x1b[0m | \x1b[32m[scripts](/donnemartin/saws/tree/master/scripts "scripts" )\x1b[0m | \x1b[32m[Updated CHANGELOG to README script to use the new CHANGELOG.rst.](/donnemartin/saws/commit/5c03b2cc6882590aa567a98cc8dceae780d1428c "Updated CHANGELOG to README script to use the new CHANGELOG.rst." )\x1b[0m | Oct 16, 2015 \n\x1b[32m![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\x1b[0m | \x1b[32m[tests](/donnemartin/saws/tree/master/tests "tests" )\x1b[0m | \x1b[32m[Fix linter line length issues.](/donnemartin/saws/commit/bf1afa4c73bf1a7dd462cb0c92f945d2b7d7cf5a "Fix linter line length issues." )\x1b[0m | Oct 15, 2015 \n\x1b[32m![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\x1b[0m | \x1b[32m[.gitchangelog.rc](/donnemartin/saws/blob/master/.gitchangelog.rc ".gitchangelog.rc" )\x1b[0m | \x1b[32m[Added gitchangelog.](/donnemartin/saws/commit/7347e0cc55a9ac75368152b9683881d41982a943 "Added gitchangelog." )\x1b[0m | Sep 15, 2015 \n\x1b[32m![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\x1b[0m | \x1b[32m[.gitignore](/donnemartin/saws/blob/master/.gitignore ".gitignore" )\x1b[0m | \x1b[32m[Fix](/donnemartin/saws/commit/07c1d7df22c9032b993d18f22162b91fb16e86d6 "Fix #66: Remove docs/build from the source repository." )\x1b[0m \x1b[32m[#66](https://github.com/donnemartin/saws/issues/66 "Remove docs/build from the source repository." )\x1b[0m\x1b[32m[: Remove docs/build from the source repository.](/donnemartin/saws/commit/07c1d7df22c9032b993d18f22162b91fb16e86d6 "Fix #66: Remove docs/build from the source repository." )\x1b[0m | Nov 30, 2015 \n\x1b[32m![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\x1b[0m | \x1b[32m[.travis.yml](/donnemartin/saws/blob/master/.travis.yml ".travis.yml" )\x1b[0m | \x1b[32m[Revert Python 3.5 build and test support due to:](/donnemartin/saws/commit/36e073de59b0fbe872fb30c80f3521c3b558b37f "Revert Python 3.5 build and test support due to: https://github.com/travis-ci/travis-ci/issues/4794." )\x1b[0m \x1b[32m[travis-ci/travis-ci#…](https://github.com/travis-ci/travis-ci/issues/4794 "Tox builds for Python 3.5 fail" )\x1b[0m | Sep 23, 2015 \n\x1b[32m![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\x1b[0m | \x1b[32m[CHANGELOG.md](/donnemartin/saws/blob/master/CHANGELOG.md "CHANGELOG.md" )\x1b[0m | \x1b[32m[Prepped version 0.3.2.](/donnemartin/saws/commit/0408df93b135542c2388fe60444ba4651feec481 "Prepped version 0.3.2." )\x1b[0m | Oct 16, 2015 \n\x1b[32m![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\x1b[0m | \x1b[32m[CHANGELOG.rst](/donnemartin/saws/blob/master/CHANGELOG.rst "CHANGELOG.rst" )\x1b[0m | \x1b[32m[Prepped version 0.3.2.](/donnemartin/saws/commit/0408df93b135542c2388fe60444ba4651feec481 "Prepped version 0.3.2." )\x1b[0m | Oct 16, 2015 \n\x1b[32m![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\x1b[0m | \x1b[32m[CHECKLIST.md](/donnemartin/saws/blob/master/CHECKLIST.md "CHECKLIST.md" )\x1b[0m | \x1b[32m[Removed mention of scripts/release_pypi.sh, which is no longer used.](/donnemartin/saws/commit/ba73311f8dd04e9a1eb159ff4d8c741a22027e50 "Removed mention of scripts/release_pypi.sh, which is no longer used." )\x1b[0m | Nov 16, 2015 \n\x1b[32m![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\x1b[0m | \x1b[32m[CONTRIBUTING.md](/donnemartin/saws/blob/master/CONTRIBUTING.md "CONTRIBUTING.md" )\x1b[0m | \x1b[32m[Add note about AWS collaboration.](/donnemartin/saws/commit/9fb37599835024ef7c99a7f00822aab54904fa5b "Add note about AWS collaboration." )\x1b[0m | Nov 30, 2015 \n\x1b[32m![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\x1b[0m | \x1b[32m[Dockerfile](/donnemartin/saws/blob/master/Dockerfile "Dockerfile" )\x1b[0m | \x1b[32m[Add Dockerfile](/donnemartin/saws/commit/7e185a10918059f0c763b1437d540b78432ea1a9 "Add Dockerfile" )\x1b[0m | Sep 26, 2015 \n\x1b[32m![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\x1b[0m | \x1b[32m[INSTALLATION.md](/donnemartin/saws/blob/master/INSTALLATION.md "INSTALLATION.md" )\x1b[0m | \x1b[32m[Added INSTALLATION.md.](/donnemartin/saws/commit/8317c4c92441f40ce20dad5ced97fe090f77e93e "Added INSTALLATION.md." )\x1b[0m | Oct 3, 2015 \n\x1b[32m![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\x1b[0m | \x1b[32m[LICENSE.txt](/donnemartin/saws/blob/master/LICENSE.txt "LICENSE.txt" )\x1b[0m | \x1b[32m[Added LICENSE.](/donnemartin/saws/commit/1166ffddffffbd981868d700df1f8ecc2061847f "Added LICENSE." )\x1b[0m | Sep 19, 2015 \n\x1b[32m![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\x1b[0m | \x1b[32m[MANIFEST.in](/donnemartin/saws/blob/master/MANIFEST.in "MANIFEST.in" )\x1b[0m | \x1b[32m[Updated MANIFEST.in.](/donnemartin/saws/commit/cba64a756e8dd193bb93083ee7aecd4316c2c9b4 "Updated MANIFEST.in." )\x1b[0m | Sep 19, 2015 \n\x1b[32m![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\x1b[0m | \x1b[32m[README.md](/donnemartin/saws/blob/master/README.md "README.md" )\x1b[0m | \x1b[32m[Fix #todo-add-more-resources anchor link.](/donnemartin/saws/commit/8427a6432c58aa9f95bf4fe26ab76a13645a8261 "Fix #todo-add-more-resources anchor link." )\x1b[0m | Nov 30, 2015 \n\x1b[32m![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\x1b[0m | \x1b[32m[appveyor.yml](/donnemartin/saws/blob/master/appveyor.yml "appveyor.yml" )\x1b[0m | \x1b[32m[Added Appveyor support, inspired by](/donnemartin/saws/commit/f756340ad4f9f4d47860644b0e677f49dac575d4 "Added Appveyor support, inspired by https://github.com/audreyr/cookiecutter/blob/master/appveyor.yml." )\x1b[0m \x1b[32m[https://github.com/audreyr/cookie…](https://github.com/audreyr/cookiecutter/blob/master/appveyor.yml)\x1b[0m | Sep 28, 2015 \n\x1b[32m![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\x1b[0m | \x1b[32m[requirements-dev.txt](/donnemartin/saws/blob/master/requirements-dev.txt "requirements-dev.txt" )\x1b[0m | \x1b[32m[Added Sphinx-PyPI-upload support.](/donnemartin/saws/commit/7068598aa117cc16d6e32000afa05963d3d76093 "Added Sphinx-PyPI-upload support." )\x1b[0m | Sep 19, 2015 \n\x1b[32m![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\x1b[0m | \x1b[32m[setup.cfg](/donnemartin/saws/blob/master/setup.cfg "setup.cfg" )\x1b[0m | \x1b[32m[Added Sphinx-PyPI-upload support.](/donnemartin/saws/commit/7068598aa117cc16d6e32000afa05963d3d76093 "Added Sphinx-PyPI-upload support." )\x1b[0m | Sep 19, 2015 \n\x1b[32m![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\x1b[0m | \x1b[32m[setup.py](/donnemartin/saws/blob/master/setup.py "setup.py" )\x1b[0m | \x1b[32m[Fixed](/donnemartin/saws/commit/1628d3efda27ba638833b54bdc3124db68a7fb30 "Fixed #53: Shortcuts should be placed separately in a .saws.shortcuts file." )\x1b[0m \x1b[32m[#53](https://github.com/donnemartin/saws/issues/53 "Shortcuts should be placed separately in a .saws.shortcuts file" )\x1b[0m\x1b[32m[: Shortcuts should be placed separately in a .saws.shortcuts…](/donnemartin/saws/commit/1628d3efda27ba638833b54bdc3124db68a7fb30 "Fixed #53: Shortcuts should be placed separately in a .saws.shortcuts file." )\x1b[0m | Oct 14, 2015 \n\x1b[32m![](https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif)\x1b[0m | \x1b[32m[tox.ini](/donnemartin/saws/blob/master/tox.ini "tox.ini" )\x1b[0m | \x1b[32m[Revert Python 3.5 build and test support due to:](/donnemartin/saws/commit/36e073de59b0fbe872fb30c80f3521c3b558b37f "Revert Python 3.5 build and test support due to: https://github.com/travis-ci/travis-ci/issues/4794." )\x1b[0m \x1b[32m[travis-ci/travis-ci#…](https://github.com/travis-ci/travis-ci/issues/4794 "Tox builds for Python 3.5 fail" )\x1b[0m | Sep 23, 2015\n\n\x1b[33m README.md \x1b[0m\n\n\x1b[32m[![](https://camo.githubusercontent.com/2af72023269b0b320adf4ec55576435f5c8b79c4/687474703a2f2f692e696d6775722e636f6d2f767a43357a6d412e676966)\x1b[0m](https://camo.githubusercontent.com/2af72023269b0b320adf4ec55576435f5c8b79c4/687474703a2f2f692e696d6775722e636f6d2f767a43357a6d412e676966)\n\n\x1b[32m[![Build Status](https://camo.githubusercontent.com/c1d6d1e8f7cdac1e6a82bf900832bbbe52d61e95/68747470733a2f2f7472617669732d63692e6f72672f646f6e6e656d617274696e2f736177732e7376673f6272616e63683d6d6173746572)\x1b[0m](https://travis-ci.org/donnemartin/saws) \x1b[32m[![Documentation Status](https://camo.githubusercontent.com/41afe5f96e9f915ecd9706bdc7dfd19518eaf05a/68747470733a2f2f72656164746865646f63732e6f72672f70726f6a656374732f736177732f62616467652f3f76657273696f6e3d6c6174657374)\x1b[0m](http://saws.readthedocs.org/en/latest/?badge=latest) \x1b[32m[![Dependency Status](https://camo.githubusercontent.com/11e16ba1560f0232928945436bf86340615d85ae/68747470733a2f2f67656d6e617369756d2e636f6d2f646f6e6e656d617274696e2f736177732e737667)\x1b[0m](https://gemnasium.com/donnemartin/saws) \x1b[32m[![Codecov](https://camo.githubusercontent.com/e24a50e9dd63768d90d26a9f6be1cf3d0c3b0230/68747470733a2f2f696d672e736869656c64732e696f2f636f6465636f762f632f6769746875622f646f6e6e656d617274696e2f736177732e737667)\x1b[0m](https://codecov.io/github/donnemartin/saws/saws)\n\n\x1b[32m[![PyPI version](https://camo.githubusercontent.com/818edda38216a62a4bb36557372877febec35086/68747470733a2f2f62616467652e667572792e696f2f70792f736177732e737667)\x1b[0m](http://badge.fury.io/py/saws) \x1b[32m[![PyPI](https://camo.githubusercontent.com/04574f61c5dff98734bd6a013cc9217b9a428e47/68747470733a2f2f696d672e736869656c64732e696f2f707970692f707976657273696f6e732f736177732e737667)\x1b[0m](https://pypi.python.org/pypi/saws/) \x1b[32m[![License](https://camo.githubusercontent.com/8bf81d4b6a63eaf6f712e3092fe7238f8716e615/687474703a2f2f696d672e736869656c64732e696f2f3a6c6963656e73652d6170616368652d626c75652e737667)\x1b[0m](http://www.apache.org/licenses/LICENSE-2.0.html)\n\n\x1b[33m\x1b[32m[](#saws)\x1b[0mSAWS\x1b[0m\n\n\x1b[33m\x1b[32m[](#motivation)\x1b[0mMotivation\x1b[0m\n\n\x1b[33m\x1b[32m[](#aws-cli)\x1b[0mAWS CLI\x1b[0m\n\nAlthough the \x1b[32m[AWS CLI](https://github.com/aws/aws-cli)\x1b[0m is a great resource to manage your AWS-powered services, it\'s \x1b[36mtough to remember usage\x1b[0m of:\n\n\x1b[36m * 50+ top-level commands\x1b[0m\n\x1b[36m * 1400+ subcommands\x1b[0m\n\x1b[36m * Countless command-specific options\x1b[0m\n\x1b[36m * Resources such as instance tags and buckets\x1b[0m\n\n\x1b[33m\x1b[32m[](#saws-a-supercharged-aws-cli)\x1b[0mSAWS: A Supercharged AWS CLI\x1b[0m\n\n\x1b[36m`SAWS`\x1b[0m aims to \x1b[36msupercharge\x1b[0m the AWS CLI with features focusing on:\n\n\x1b[36m * \x1b[36mImproving ease-of-use\x1b[0m\x1b[0m\n\x1b[36m * \x1b[36mIncreasing productivity\x1b[0m\x1b[0m\n\nUnder the hood, \x1b[36m`SAWS`\x1b[0m is \x1b[36mpowered by the AWS CLI\x1b[0m and supports the \x1b[36msame commands\x1b[0m and \x1b[36mcommand structure\x1b[0m.\n\n\x1b[36m`SAWS`\x1b[0m and \x1b[36m`AWS CLI`\x1b[0m Usage:\n\naws [parameters] [options]\n\n\x1b[36m`SAWS`\x1b[0m features:\n\n\x1b[36m * Auto-completion of: \x1b[0m\n \x1b[36m * Commands\x1b[0m\n \x1b[36m * Subcommands\x1b[0m\n \x1b[36m * Options\x1b[0m\n\x1b[36m * Auto-completion of resources: \x1b[0m\n \x1b[36m * Bucket names\x1b[0m\n \x1b[36m * Instance ids\x1b[0m\n \x1b[36m * Instance tags\x1b[0m\n \x1b[36m * \x1b[32m[More coming soon!](#todo-add-more-resources)\x1b[0m\x1b[0m\n\x1b[36m * Customizable shortcuts\x1b[0m\n\x1b[36m * Fuzzy completion of resources and shortcuts\x1b[0m\n\x1b[36m * Fish-style auto-suggestions\x1b[0m\n\x1b[36m * Syntax and output highlighting\x1b[0m\n\x1b[36m * Execution of shell commands\x1b[0m\n\x1b[36m * Command history\x1b[0m\n\x1b[36m * Contextual help\x1b[0m\n\x1b[36m * Toolbar options\x1b[0m\n\n\x1b[36m`SAWS`\x1b[0m is available for Mac, Linux, Unix, and \x1b[32m[Windows](#windows-support)\x1b[0m.\n\n\x1b[32m[![](https://camo.githubusercontent.com/571a2d19a969447b477713d54d1d3ba46d7418ee/687474703a2f2f692e696d6775722e636f6d2f456f31327139542e706e67)\x1b[0m](https://camo.githubusercontent.com/571a2d19a969447b477713d54d1d3ba46d7418ee/687474703a2f2f692e696d6775722e636f6d2f456f31327139542e706e67)\n\n\x1b[33m\x1b[32m[](#index)\x1b[0mIndex\x1b[0m\n\n\x1b[33m\x1b[32m[](#features)\x1b[0mFeatures\x1b[0m\n\n\x1b[36m * \x1b[32m[Syntax and Output Highlighting](#syntax-and-output-highlighting)\x1b[0m\x1b[0m\n\x1b[36m * \x1b[32m[Auto-Completion of Commands, Subcommands, and Options](#auto-completion-of-commands-subcommands-and-options)\x1b[0m\x1b[0m\n\x1b[36m * \x1b[32m[Auto-Completion of AWS Resources](#auto-completion-of-aws-resources)\x1b[0m\x1b[0m\n \x1b[36m * \x1b[32m[S3 Buckets](#s3-buckets)\x1b[0m\x1b[0m\n \x1b[36m * \x1b[32m[EC2 Instance Ids](#ec2-instance-ids)\x1b[0m\x1b[0m\n \x1b[36m * \x1b[32m[EC2 Instance Tags](#ec2-instance-tags)\x1b[0m\x1b[0m\n \x1b[36m * \x1b[32m[TODO: Add More Resources](#todo-add-more-resources)\x1b[0m\x1b[0m\n\x1b[36m * \x1b[32m[Customizable Shortcuts](#customizable-shortcuts)\x1b[0m\x1b[0m\n\x1b[36m * \x1b[32m[Fuzzy Resource and Shortcut Completion](#fuzzy-resource-and-shortcut-completion)\x1b[0m\x1b[0m\n\x1b[36m * \x1b[32m[Fish-Style Auto-Suggestions](#fish-style-auto-suggestions)\x1b[0m\x1b[0m\n\x1b[36m * \x1b[32m[Executing Shell Commands](#executing-shell-commands)\x1b[0m\x1b[0m\n\x1b[36m * \x1b[32m[Command History](#command-history)\x1b[0m\x1b[0m\n\x1b[36m * \x1b[32m[Contextual Help](#contextual-help)\x1b[0m\x1b[0m\n \x1b[36m * \x1b[32m[Contextual Command Line Help](#contextual-command-line-help)\x1b[0m\x1b[0m\n \x1b[36m * \x1b[32m[Contextual Web Docs](#contextual-web-docs)\x1b[0m\x1b[0m\n\x1b[36m * \x1b[32m[Toolbar Options](#toolbar-options)\x1b[0m\x1b[0m\n\x1b[36m * \x1b[32m[Windows Support](#windows-support)\x1b[0m\x1b[0m\n\n\x1b[33m\x1b[32m[](#installation-and-tests)\x1b[0mInstallation and Tests\x1b[0m\n\n\x1b[36m * \x1b[32m[Installation](#installation)\x1b[0m\x1b[0m\n \x1b[36m * \x1b[32m[Pip Installation](#pip-installation)\x1b[0m\x1b[0m\n \x1b[36m * \x1b[32m[Virtual Environment and Docker Installation](#virtual-environment-and-docker-installation)\x1b[0m\x1b[0m\n \x1b[36m * \x1b[32m[AWS Credentials and Named Profiles](#aws-credentials-and-named-profiles)\x1b[0m\x1b[0m\n \x1b[36m * \x1b[32m[Supported Python Versions](#supported-python-versions)\x1b[0m\x1b[0m\n \x1b[36m * \x1b[32m[Supported Platforms](#supported-platforms)\x1b[0m\x1b[0m\n\x1b[36m * \x1b[32m[Developer Installation](#developer-installation)\x1b[0m\x1b[0m\n \x1b[36m * \x1b[32m[Continuous Integration](#continuous-integration)\x1b[0m\x1b[0m\n \x1b[36m * \x1b[32m[Dependencies Management](#dependencies-management)\x1b[0m\x1b[0m\n \x1b[36m * \x1b[32m[Unit Tests and Code Coverage](#unit-tests-and-code-coverage)\x1b[0m\x1b[0m\n \x1b[36m * \x1b[32m[Documentation](#documentation)\x1b[0m\x1b[0m\n\n\x1b[33m\x1b[32m[](#misc)\x1b[0mMisc\x1b[0m\n\n\x1b[36m * \x1b[32m[Contributing](#contributing)\x1b[0m\x1b[0m\n\x1b[36m * \x1b[32m[Credits](#credits)\x1b[0m\x1b[0m\n\x1b[36m * \x1b[32m[Contact Info](#contact-info)\x1b[0m\x1b[0m\n\x1b[36m * \x1b[32m[License](#license)\x1b[0m\x1b[0m\n\n\x1b[33m\x1b[32m[](#syntax-and-output-highlighting)\x1b[0mSyntax and Output Highlighting\x1b[0m\n\n\x1b[32m[![](https://camo.githubusercontent.com/c527a4c00862bb1fe47c272a3c3563418f06866f/687474703a2f2f692e696d6775722e636f6d2f785144707737302e706e67)\x1b[0m](https://camo.githubusercontent.com/c527a4c00862bb1fe47c272a3c3563418f06866f/687474703a2f2f692e696d6775722e636f6d2f785144707737302e706e67)\n\nYou can control which theme to load for syntax highlighting by updating your \x1b[32m[~/.sawsrc](https://github.com/donnemartin/saws/blob/master/saws/sawsrc)\x1b[0m file:\n\n\x1b[33mVisual theme. Possible values: manni, igor, xcode, vim, autumn, vs, rrt,\x1b[0m\n \x1b[33mnative, perldoc, borland, tango, emacs, friendly, monokai, paraiso-dark,\x1b[0m\n \x1b[33mcolorful, murphy, bw, pastie, paraiso-light, trac, default, fruity\x1b[0m\n theme = vim\n\n\x1b[33m\x1b[32m[](#auto-completion-of-commands-subcommands-and-options)\x1b[0mAuto-Completion of Commands, Subcommands, and Options\x1b[0m\n\n\x1b[36m`SAWS`\x1b[0m provides smart autocompletion as you type. Entering the following command will interactively list and auto-complete all subcommands \x1b[36mspecific only\x1b[0m to \x1b[36m`ec2`\x1b[0m:\n\naws ec2\n\n\x1b[32m[![](https://camo.githubusercontent.com/df93a08e1371694a8086f3b45ba6e5ec955212e2/687474703a2f2f692e696d6775722e636f6d2f5032744c3976572e706e67)\x1b[0m](https://camo.githubusercontent.com/df93a08e1371694a8086f3b45ba6e5ec955212e2/687474703a2f2f692e696d6775722e636f6d2f5032744c3976572e706e67)\n\n\x1b[33m\x1b[32m[](#auto-completion-of-aws-resources)\x1b[0mAuto-Completion of AWS Resources\x1b[0m\n\nIn addition to the default commands, subcommands, and options the AWS CLI provides, \x1b[36m`SAWS`\x1b[0m supports auto-completion of your AWS resources. Currently, bucket names, instance ids, and instance tags are included, with additional support for more resources \x1b[32m[under development](#todo-add-more-resources)\x1b[0m.\n\n\x1b[33m\x1b[32m[](#s3-buckets)\x1b[0mS3 Buckets\x1b[0m\n\nOption for \x1b[36m`s3api`\x1b[0m:\n\n--bucket\n\nSample Usage:\n\naws s3api get-bucket-acl --bucket\n\nSyntax for \x1b[36m`s3`\x1b[0m:\n\ns3://\n\nSample Usage:\n\naws s3 ls s3://\n\nNote: The example below demonstrates the use of \x1b[32m[fuzzy resource completion](/donnemartin/saws/blob/master/fuzzy-resource-and-shortcutcompletion)\x1b[0m:\n\n\x1b[32m[![](https://camo.githubusercontent.com/756f14d2ed77a7a9a3da9b0d9501cab9b7433e36/687474703a2f2f692e696d6775722e636f6d2f333943415335542e706e67)\x1b[0m](https://camo.githubusercontent.com/756f14d2ed77a7a9a3da9b0d9501cab9b7433e36/687474703a2f2f692e696d6775722e636f6d2f333943415335542e706e67)\n\n\x1b[33m\x1b[32m[](#ec2-instance-ids)\x1b[0mEC2 Instance Ids\x1b[0m\n\nOption for \x1b[36m`ec2`\x1b[0m:\n\n--instance-ids\n\nSample Usage:\n\naws ec2 describe-instances --instance-ids\n aws ec2 ls --instance-ids\n\nNote: The \x1b[36m`ls`\x1b[0m command demonstrates the use of \x1b[32m[customizable shortcuts](#customizable-shortcuts)\x1b[0m:\n\n\x1b[32m[![](https://camo.githubusercontent.com/d4b0298e0ecd0afc1dac0d62617a4be53a44f598/687474703a2f2f692e696d6775722e636f6d2f6a46794353586c2e706e67)\x1b[0m](https://camo.githubusercontent.com/d4b0298e0ecd0afc1dac0d62617a4be53a44f598/687474703a2f2f692e696d6775722e636f6d2f6a46794353586c2e706e67)\n\n\x1b[33m\x1b[32m[](#ec2-instance-tags)\x1b[0mEC2 Instance Tags\x1b[0m\n\nOption for \x1b[36m`ec2`\x1b[0m:\n\n--ec2-tag-key\n --ec2-tag-value\n\nSample Usage:\n\naws ec2 ls --ec2-tag-key\n aws ec2 ls --ec2-tag-value\n\n\x1b[36mTags support wildcards\x1b[0m with the \x1b[36m`*`\x1b[0m character.\n\nNote: \x1b[36m`ls`\x1b[0m, \x1b[36m`--ec2-tag-value`\x1b[0m, and \x1b[36m`--ec2-tag-key`\x1b[0m demonstrate the use of \x1b[32m[customizable shortcuts](#customizable-shortcuts)\x1b[0m:\n\n\x1b[32m[![](https://camo.githubusercontent.com/593e9fc80cc6e1fb8e0fd7324cb4cb3a1b51bf2f/687474703a2f2f692e696d6775722e636f6d2f56494b7747335a2e706e67)\x1b[0m](https://camo.githubusercontent.com/593e9fc80cc6e1fb8e0fd7324cb4cb3a1b51bf2f/687474703a2f2f692e696d6775722e636f6d2f56494b7747335a2e706e67)\n\n\x1b[33m\x1b[32m[](#todo-add-more-resources)\x1b[0mTODO: Add More Resources\x1b[0m\n\nFeel free to \x1b[32m[submit an issue or a pull request](#contributions)\x1b[0m if you\'d like support for additional resources.\n\n\x1b[33m\x1b[32m[](#customizable-shortcuts)\x1b[0mCustomizable Shortcuts\x1b[0m\n\nThe \x1b[32m[~/.saws.shortcuts](https://github.com/donnemartin/saws/blob/master/saws/saws.shortcuts)\x1b[0m file contains shortcuts that you can modify. It comes pre-populated with several \x1b[32m[handy shortcuts](https://github.com/donnemartin/saws/blob/master/saws/saws.shortcuts)\x1b[0m out of the box. You can combine shortcuts with \x1b[32m[fuzzy completion](#fuzzy-resource-and-shortcut-completion)\x1b[0m for even less keystrokes. Below are a few examples.\n\nList all EC2 instances:\n\naws ec2 ls\n\nList all running EC2 instances:\n\naws ec2 ls --ec2-state running \x1b[33mfuzzy shortcut: aws ecstate\x1b[0m\n\n\x1b[32m[![](https://camo.githubusercontent.com/7b56fd124f76cf2a9573de7b963bbb791f4cae16/687474703a2f2f692e696d6775722e636f6d2f6a594645736f4d2e706e67)\x1b[0m](https://camo.githubusercontent.com/7b56fd124f76cf2a9573de7b963bbb791f4cae16/687474703a2f2f692e696d6775722e636f6d2f6a594645736f4d2e706e67)\n\nList all EC2 instances with a matching tag (supports wildcards \x1b[36m`*`\x1b[0m):\n\naws ec2 ls --ec2-tag-key \x1b[33mfuzzy shortcut: aws ectagk\x1b[0m\n aws ec2 ls --ec2-tag-value \x1b[33mfuzzy shortcut: aws ectagv\x1b[0m\n\n\x1b[32m[![](https://camo.githubusercontent.com/c45874a55d4ac6485ba78c6d4a91cc1d928703a4/687474703a2f2f692e696d6775722e636f6d2f505375775549772e706e67)\x1b[0m](https://camo.githubusercontent.com/c45874a55d4ac6485ba78c6d4a91cc1d928703a4/687474703a2f2f692e696d6775722e636f6d2f505375775549772e706e67)\n\nList EC2 instance with matching id:\n\naws ec2 ls --instance-ids \x1b[33mfuzzy shortcut: aws eclsi\x1b[0m\n\n\x1b[32m[![](https://camo.githubusercontent.com/48136027017398a300b9fe968e50e2e7c2a8b18b/687474703a2f2f692e696d6775722e636f6d2f774763554373612e706e67)\x1b[0m](https://camo.githubusercontent.com/48136027017398a300b9fe968e50e2e7c2a8b18b/687474703a2f2f692e696d6775722e636f6d2f774763554373612e706e67)\n\nList all DynamoDB tables:\n\naws dynamodb ls \x1b[33mfuzzy shortcut: aws dls\x1b[0m\n\nList all EMR clusters:\n\naws emr ls \x1b[33mfuzzy shortcut: aws emls\x1b[0m\n\nAdd/remove/modify shortcuts in your \x1b[32m[~/.saws.shortcuts](https://github.com/donnemartin/saws/blob/master/saws/shortcuts)\x1b[0m file to suit your needs.\n\nFeel free to submit:\n\n\x1b[36m * An issue to request additional shortcuts\x1b[0m\n\x1b[36m * A pull request if you\'d like to share your shortcuts (see \x1b[32m[contributing guidelines](#contributions)\x1b[0m)\x1b[0m\n\n\x1b[33m\x1b[32m[](#fuzzy-resource-and-shortcut-completion)\x1b[0mFuzzy Resource and Shortcut Completion\x1b[0m\n\nTo toggle fuzzy completion of AWS resources and shortcuts, use \x1b[36m`F3`\x1b[0m key.\n\nSample fuzzy shortcuts to start and stop EC2 instances:\n\naws ecstop\n aws ecstart\n\nNote: Fuzzy completion currently only works with AWS \x1b[32m[resources](#auto-completion-of-aws-resources)\x1b[0m and \x1b[32m[shortcuts](/donnemartin/saws/blob/master/customizable-shortcuts)\x1b[0m.\n\n\x1b[32m[![](https://camo.githubusercontent.com/a918a4d70ca61189149746e052f5bb2ffec9ae65/687474703a2f2f692e696d6775722e636f6d2f374f76464843772e706e67)\x1b[0m](https://camo.githubusercontent.com/a918a4d70ca61189149746e052f5bb2ffec9ae65/687474703a2f2f692e696d6775722e636f6d2f374f76464843772e706e67)\n\n\x1b[33m\x1b[32m[](#fish-style-auto-suggestions)\x1b[0mFish-Style Auto-Suggestions\x1b[0m\n\n\x1b[36m`SAWS`\x1b[0m supports Fish-style auto-suggestions. Use the \x1b[36m`right arrow`\x1b[0m key to complete a suggestion.\n\n\x1b[32m[![](https://camo.githubusercontent.com/6671d93da542b2f2cdec589fcb2c174e3bbc25f6/687474703a2f2f692e696d6775722e636f6d2f743532303071312e706e67)\x1b[0m](https://camo.githubusercontent.com/6671d93da542b2f2cdec589fcb2c174e3bbc25f6/687474703a2f2f692e696d6775722e636f6d2f743532303071312e706e67)\n\n\x1b[33m\x1b[32m[](#executing-shell-commands)\x1b[0mExecuting Shell Commands\x1b[0m\n\n\x1b[36m`SAWS`\x1b[0m allows you to execute shell commands from the \x1b[36m`saws>`\x1b[0m prompt.\n\n\x1b[32m[![](https://camo.githubusercontent.com/4d26d484e5b94b7ca62607ec1557d847f5db96f3/687474703a2f2f692e696d6775722e636f6d2f4669536e3662322e706e67)\x1b[0m](https://camo.githubusercontent.com/4d26d484e5b94b7ca62607ec1557d847f5db96f3/687474703a2f2f692e696d6775722e636f6d2f4669536e3662322e706e67)\n\n\x1b[33m\x1b[32m[](#command-history)\x1b[0mCommand History\x1b[0m\n\n\x1b[36m`SAWS`\x1b[0m keeps track of commands you enter and stores them in \x1b[36m`~/.saws-history`\x1b[0m. Use the up and down arrow keys to cycle through the command history.\n\n\x1b[32m[![](https://camo.githubusercontent.com/a5ded83f813be928236c804a90d85fc0fd5e517d/687474703a2f2f692e696d6775722e636f6d2f7a3852724451422e706e67)\x1b[0m](https://camo.githubusercontent.com/a5ded83f813be928236c804a90d85fc0fd5e517d/687474703a2f2f692e696d6775722e636f6d2f7a3852724451422e706e67)\n\n\x1b[33m\x1b[32m[](#contextual-help)\x1b[0mContextual Help\x1b[0m\n\n\x1b[36m`SAWS`\x1b[0m supports contextual command line \x1b[36m`help`\x1b[0m and contextual web \x1b[36m`docs`\x1b[0m.\n\n\x1b[33m\x1b[32m[](#contextual-command-line-help)\x1b[0mContextual Command Line Help\x1b[0m\n\nThe \x1b[36m`help`\x1b[0m command is powered by the AWS CLI and outputs help within the command line.\n\nUsage:\n\naws help\n\n\x1b[32m[![](https://camo.githubusercontent.com/e10dae70d7f9303bc5fae0e3f5dba453fa905df7/687474703a2f2f692e696d6775722e636f6d2f7a536b7a7436792e706e67)\x1b[0m](https://camo.githubusercontent.com/e10dae70d7f9303bc5fae0e3f5dba453fa905df7/687474703a2f2f692e696d6775722e636f6d2f7a536b7a7436792e706e67)\n\n\x1b[33m\x1b[32m[](#contextual-web-docs)\x1b[0mContextual Web Docs\x1b[0m\n\nSometimes you\'re not quite sure what specific command/subcommand/option combination you need to use. In such cases, browsing through several combinations with the \x1b[36m`help`\x1b[0m command line is cumbersome versus browsing the online AWS CLI docs through a web browser.\n\n\x1b[36m`SAWS`\x1b[0m supports contextual web docs with the \x1b[36m`docs`\x1b[0m command or the \x1b[36m`F9`\x1b[0m key. \x1b[36m`SAWS`\x1b[0m will display the web docs specific to the currently entered command and subcommand.\n\nUsage:\n\naws docs\n\n\x1b[32m[![](https://camo.githubusercontent.com/e0a9e9bd21f2ff174f2e257f0c9966c6620d97d7/687474703a2f2f692e696d6775722e636f6d2f7a4b34494a59702e706e67)\x1b[0m](https://camo.githubusercontent.com/e0a9e9bd21f2ff174f2e257f0c9966c6620d97d7/687474703a2f2f692e696d6775722e636f6d2f7a4b34494a59702e706e67)\n\n\x1b[33m\x1b[32m[](#toolbar-options)\x1b[0mToolbar Options\x1b[0m\n\n\x1b[36m`SAWS`\x1b[0m supports a number of toolbar options:\n\n\x1b[36m * \x1b[36m`F2`\x1b[0m toggles \x1b[32m[output syntax highlighting](#syntax-and-output-highlighting)\x1b[0m\x1b[0m\n\x1b[36m * \x1b[36m`F3`\x1b[0m toggles \x1b[32m[fuzzy completion of AWS resources and shortcuts](#fuzzy-resource-and-shortcut-completion)\x1b[0m\x1b[0m\n\x1b[36m * \x1b[36m`F4`\x1b[0m toggles \x1b[32m[completion of shortcuts](#customizable-shortcuts)\x1b[0m\x1b[0m\n\x1b[36m * \x1b[36m`F5`\x1b[0m refreshes \x1b[32m[resources for auto-completion](#auto-completion-of-aws-resources)\x1b[0m\x1b[0m\n\x1b[36m * \x1b[36m`F9`\x1b[0m displays the \x1b[32m[contextual web docs](#contextual-web-docs)\x1b[0m\x1b[0m\n\x1b[36m * \x1b[36m`F10`\x1b[0m or \x1b[36m`control d`\x1b[0m exits \x1b[36m`SAWS`\x1b[0m\x1b[0m\n\n\x1b[32m[![](https://camo.githubusercontent.com/f1bb53eda629178af0d849a112193a91daca5628/687474703a2f2f692e696d6775722e636f6d2f37767a384f53632e706e67)\x1b[0m](https://camo.githubusercontent.com/f1bb53eda629178af0d849a112193a91daca5628/687474703a2f2f692e696d6775722e636f6d2f37767a384f53632e706e67)\n\n\x1b[33m\x1b[32m[](#windows-support)\x1b[0mWindows Support\x1b[0m\n\n\x1b[36m`SAWS`\x1b[0m has been tested on Windows 7 and Windows 10.\n\nOn Windows, the \x1b[32m[.sawsrc](https://github.com/donnemartin/saws/blob/master/saws/sawsrc)\x1b[0m file can be found in \x1b[36m`%userprofile%`\x1b[0m. For example:\n\nC:\\Users\\dmartin\\.sawsrc\n\nAlthough you can use the standard Windows command prompt, you\'ll probably have a better experience with either \x1b[32m[cmder](https://github.com/cmderdev/cmder)\x1b[0m or \x1b[32m[conemu](https://github.com/Maximus5/ConEmu)\x1b[0m.\n\n\x1b[32m[![](https://camo.githubusercontent.com/cfd9f911b9960308e0346d90ba4cf2338145fb52/687474703a2f2f692e696d6775722e636f6d2f7055774a57636b2e706e67)\x1b[0m](https://camo.githubusercontent.com/cfd9f911b9960308e0346d90ba4cf2338145fb52/687474703a2f2f692e696d6775722e636f6d2f7055774a57636b2e706e67)\n\n\x1b[33m\x1b[32m[](#installation)\x1b[0mInstallation\x1b[0m\n\n\x1b[33m\x1b[32m[](#pip-installation)\x1b[0mPip Installation\x1b[0m\n\n\x1b[32m[![PyPI version](https://camo.githubusercontent.com/818edda38216a62a4bb36557372877febec35086/68747470733a2f2f62616467652e667572792e696f2f70792f736177732e737667)\x1b[0m](http://badge.fury.io/py/saws) \x1b[32m[![PyPI](https://camo.githubusercontent.com/04574f61c5dff98734bd6a013cc9217b9a428e47/68747470733a2f2f696d672e736869656c64732e696f2f707970692f707976657273696f6e732f736177732e737667)\x1b[0m](https://pypi.python.org/pypi/saws/)\n\n\x1b[36m`SAWS`\x1b[0m is hosted on \x1b[32m[PyPI](https://pypi.python.org/pypi/saws)\x1b[0m. The following command will install \x1b[36m`SAWS`\x1b[0m along with dependencies such as the \x1b[32m[AWS CLI](https://github.com/aws/aws-cli)\x1b[0m:\n\n$ pip install saws\n\nYou can also install the latest \x1b[36m`SAWS`\x1b[0m from GitHub source which can contain changes not yet pushed to PyPI:\n\n$ pip install git+https://github.com/donnemartin/saws.git\n\nIf you are not installing in a \x1b[32m[virtualenv](#virtual-environment-and-docker-installation)\x1b[0m, run with \x1b[36m`sudo`\x1b[0m:\n\n$ sudo pip install saws\n\nOnce installed, start \x1b[36m`SAWS`\x1b[0m:\n\n$ saws\n\n\x1b[33m\x1b[32m[](#virtual-environment-and-docker-installation)\x1b[0mVirtual Environment and Docker Installation\x1b[0m\n\nIt is recommended that you install Python packages in a \x1b[32m[virtualenv](http://docs.python-guide.org/en/latest/dev/virtualenvs/)\x1b[0m to avoid potential \x1b[32m[issues with dependencies or permissions](https://github.com/donnemartin/saws/issues/15)\x1b[0m.\n\nTo view \x1b[36m`SAWS`\x1b[0m \x1b[36m`virtualenv`\x1b[0m and \x1b[32m[Docker](https://www.docker.com/)\x1b[0m installation instructions, click \x1b[32m[here](https://github.com/donnemartin/saws/blob/master/INSTALLATION.md)\x1b[0m.\n\n\x1b[33m\x1b[32m[](#aws-credentials-and-named-profiles)\x1b[0mAWS Credentials and Named Profiles\x1b[0m\n\n\x1b[32m[Configure your credentials](https://github.com/aws/aws-cli#getting-started)\x1b[0m with the AWS CLI:\n\n$ aws configure\n\nIf you\'d like to use a specific named profile with \x1b[36m`SAWS`\x1b[0m, run the following commands on OS X, Linux, or Unix:\n\n$ export AWS_DEFAULT_PROFILE=user1\n $ saws\n\nOr as a one-liner:\n\n$ AWS_DEFAULT_PROFILE=user1 saws\n\nWindows users can run the following commands:\n\n> set AWS_DEFAULT_PROFILE=user1\n > saws\n\nCommand line options for starting \x1b[36m`SAWS`\x1b[0m with a specific profile are \x1b[32m[under development](https://github.com/donnemartin/saws/issues/16)\x1b[0m. For more details on how to install and configure the AWS CLI, refer to the following \x1b[32m[documentation](http://docs.aws.amazon.com/cli/latest/userguide/installing.html)\x1b[0m.\n\n\x1b[33m\x1b[32m[](#supported-python-versions)\x1b[0mSupported Python Versions\x1b[0m\n\n\x1b[36m * Python 2.6\x1b[0m\n\x1b[36m * Python 2.7\x1b[0m\n\x1b[36m * Python 3.3\x1b[0m\n\x1b[36m * Python 3.4\x1b[0m\n\x1b[36m * Pypy\x1b[0m\n\nLight testing indicates that \x1b[36m`SAWS`\x1b[0m also seems to be compatible with Python 3.5.\n\nPypy3 is not supported due to \x1b[32m[lack of support](https://github.com/boto/botocore/issues/622)\x1b[0m from \x1b[32m[boto](https://github.com/boto/boto)\x1b[0m.\n\n\x1b[33m\x1b[32m[](#supported-platforms)\x1b[0mSupported Platforms\x1b[0m\n\n\x1b[36m * Mac OS X \x1b[0m\n \x1b[36m * Tested on OS X 10.10\x1b[0m\n\x1b[36m * Linux, Unix \x1b[0m\n \x1b[36m * Tested on Ubuntu 14.04 LTS\x1b[0m\n\x1b[36m * Windows \x1b[0m\n \x1b[36m * Tested on Windows 7 and 10\x1b[0m\n\n\x1b[33m\x1b[32m[](#developer-installation)\x1b[0mDeveloper Installation\x1b[0m\n\nIf you\'re interested in contributing to \x1b[36m`SAWS`\x1b[0m, run the following commands:\n\n$ git clone https://github.com/donnemartin/saws.git\n $ pip install -e .\n $ pip install -r requirements-dev.txt\n $ saws\n\n\x1b[33m\x1b[32m[](#continuous-integration)\x1b[0mContinuous Integration\x1b[0m\n\n\x1b[32m[![Build Status](https://camo.githubusercontent.com/c1d6d1e8f7cdac1e6a82bf900832bbbe52d61e95/68747470733a2f2f7472617669732d63692e6f72672f646f6e6e656d617274696e2f736177732e7376673f6272616e63683d6d6173746572)\x1b[0m](https://travis-ci.org/donnemartin/saws)\n\nContinuous integration details are available on \x1b[32m[Travis CI](https://travis-ci.org/donnemartin/saws)\x1b[0m.\n\n\x1b[33m\x1b[32m[](#dependencies-management)\x1b[0mDependencies Management\x1b[0m\n\n\x1b[32m[![Dependency Status](https://camo.githubusercontent.com/11e16ba1560f0232928945436bf86340615d85ae/68747470733a2f2f67656d6e617369756d2e636f6d2f646f6e6e656d617274696e2f736177732e737667)\x1b[0m](https://gemnasium.com/donnemartin/saws)\n\nDependencies management details are available on \x1b[32m[Gemnasium](https://gemnasium.com/donnemartin/saws)\x1b[0m.\n\n\x1b[33m\x1b[32m[](#unit-tests-and-code-coverage)\x1b[0mUnit Tests and Code Coverage\x1b[0m\n\n\x1b[32m[![Codecov](https://camo.githubusercontent.com/e24a50e9dd63768d90d26a9f6be1cf3d0c3b0230/68747470733a2f2f696d672e736869656c64732e696f2f636f6465636f762f632f6769746875622f646f6e6e656d617274696e2f736177732e737667)\x1b[0m](https://codecov.io/github/donnemartin/saws/saws)\n\n\x1b[32m[![](https://camo.githubusercontent.com/a3588f226c5b2c9007fd9e49212ae8294c5bc023/687474703a2f2f636f6465636f762e696f2f6769746875622f646f6e6e656d617274696e2f736177732f6272616e63682e7376673f6272616e63683d6d6173746572)\x1b[0m](https://camo.githubusercontent.com/a3588f226c5b2c9007fd9e49212ae8294c5bc023/687474703a2f2f636f6465636f762e696f2f6769746875622f646f6e6e656d617274696e2f736177732f6272616e63682e7376673f6272616e63683d6d6173746572)\n\nCode coverage details are available on \x1b[32m[Codecov](https://codecov.io/github/donnemartin/saws/saws)\x1b[0m.\n\nRun unit tests in your active Python environment:\n\n$ python tests/run_tests.py\n\nRun unit tests with \x1b[32m[tox](https://pypi.python.org/pypi/tox)\x1b[0m on multiple Python environments:\n\n$ tox\n\n\x1b[33m\x1b[32m[](#documentation)\x1b[0mDocumentation\x1b[0m\n\n\x1b[32m[![Documentation Status](https://camo.githubusercontent.com/41afe5f96e9f915ecd9706bdc7dfd19518eaf05a/68747470733a2f2f72656164746865646f63732e6f72672f70726f6a656374732f736177732f62616467652f3f76657273696f6e3d6c6174657374)\x1b[0m](http://saws.readthedocs.org/en/latest/?badge=latest)\n\nSource code documentation is available on \x1b[32m[Readthedocs.org](http://saws.readthedocs.org/en/latest/?badge=latest)\x1b[0m.\n\nRun the following to build the docs:\n\n$ scripts/update_docs.sh\n\n\x1b[33m\x1b[32m[](#contributing)\x1b[0mContributing\x1b[0m\n\nContributions are welcome!\n\nReview the \x1b[32m[Contributing Guidelines](https://github.com/donnemartin/saws/blob/master/CONTRIBUTING.md)\x1b[0m for details on how to:\n\n\x1b[36m * Submit issues\x1b[0m\n\x1b[36m * Submit pull requests\x1b[0m\n\n\x1b[33m\x1b[32m[](#credits)\x1b[0mCredits\x1b[0m\n\n\x1b[36m * \x1b[32m[AWS CLI](https://github.com/aws/aws-cli)\x1b[0m by \x1b[32m[AWS](https://github.com/aws)\x1b[0m for powering \x1b[36m`SAWS`\x1b[0m under the hood\x1b[0m\n\x1b[36m * \x1b[32m[Python Prompt Toolkit](https://github.com/jonathanslenders/python-prompt-toolkit)\x1b[0m by \x1b[32m[jonathanslenders](https://github.com/jonathanslenders)\x1b[0m for simplifying the creation of \x1b[36m`SAWS`\x1b[0m\x1b[0m\n\x1b[36m * \x1b[32m[Wharfee](https://github.com/j-bennet/wharfee)\x1b[0m by \x1b[32m[j-bennet](https://github.com/j-bennet)\x1b[0m for inspiring the creation of \x1b[36m`SAWS`\x1b[0m and for some handy utility functions\x1b[0m\n\n\x1b[33m\x1b[32m[](#contact-info)\x1b[0mContact Info\x1b[0m\n\nFeel free to contact me to discuss any issues, questions, or comments.\n\n\x1b[36m * Email: \x1b[32m[donne.martin@gmail.com](mailto:donne.martin@gmail.com)\x1b[0m\x1b[0m\n\x1b[36m * Twitter: \x1b[32m[donne_martin](https://twitter.com/donne_martin)\x1b[0m\x1b[0m\n\x1b[36m * GitHub: \x1b[32m[donnemartin](https://github.com/donnemartin)\x1b[0m\x1b[0m\n\x1b[36m * LinkedIn: \x1b[32m[donnemartin](https://www.linkedin.com/in/donnemartin)\x1b[0m\x1b[0m\n\x1b[36m * Website: \x1b[32m[donnemartin.com](http://donnemartin.com)\x1b[0m\x1b[0m\n\n\x1b[33m\x1b[32m[](#license)\x1b[0mLicense\x1b[0m\n\nCopyright 2015 Donne Martin\n\nLicensed under the Apache License, Version 2.0 (the "License");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\nhttp://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an "AS IS" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n\x1b[36m * \x1b[32m[Status](https://status.github.com/)\x1b[0m\x1b[0m\n\x1b[36m * \x1b[32m[API](https://developer.github.com)\x1b[0m\x1b[0m\n\x1b[36m * \x1b[32m[Training](https://training.github.com)\x1b[0m\x1b[0m\n\x1b[36m * \x1b[32m[Shop](https://shop.github.com)\x1b[0m\x1b[0m\n\x1b[36m * \x1b[32m[Blog](https://github.com/blog)\x1b[0m\x1b[0m\n\x1b[36m * \x1b[32m[About](https://github.com/about)\x1b[0m\x1b[0m\n\x1b[36m * \x1b[32m[Pricing](https://github.com/pricing)\x1b[0m\x1b[0m\n\x1b[32m[ ](https://github.com)\x1b[0m\n\n\x1b[36m * (C) 2015 GitHub, Inc.\x1b[0m\n\x1b[36m * \x1b[32m[Terms](https://github.com/site/terms)\x1b[0m\x1b[0m\n\x1b[36m * \x1b[32m[Privacy](https://github.com/site/privacy)\x1b[0m\x1b[0m\n\x1b[36m * \x1b[32m[Security](https://github.com/security)\x1b[0m\x1b[0m\n\x1b[36m * \x1b[32m[Contact](https://github.com/contact)\x1b[0m\x1b[0m\n\x1b[36m * \x1b[32m[Help](https://help.github.com)\x1b[0m\x1b[0m\n\nSomething went wrong with that request. Please try again.\n\nYou signed in with another tab or window. [Reload]() to refresh your session. You signed out in another tab or window. [Reload]() to refresh your session.\n """ # NOQA ssl_error = u"Viewing https://requestb.in\n\n\x1b[0mError: hostname 'requestb.in' doesn't match either of '*.herokuapp.com', 'herokuapp.com'\nTry running gh view # with the --browser/-b flag\n\nView this article in a browser with the -b/--browser flag.\n\x1b[0m\nPress q to quit viewing this article.\n\x1b[0m" # NOQA ================================================ FILE: tests/data/thread.py ================================================ # -*- coding: utf-8 -*- # Copyright 2015 Donne Martin. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of # the License is located at # # http://www.apache.org/licenses/LICENSE-2.0 # # or in the "license" file accompanying this file. This file is # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. formatted_threads = u'\x1b[35m 1. \x1b[0mtitle1 \x1b[0m\x1b[35m(foo/bar/issues/1)\x1b[0m\n\x1b[32m Seen: False \x1b[0m\x1b[36mType: type1 \x1b[0m\x1b[33mUpdated: just now \x1b[0m\n\x1b[35m 2. \x1b[0mtitle2 \x1b[0m\x1b[35m(foo/bar/issues/1)\x1b[0m\n\x1b[32m Seen: True \x1b[0m\x1b[36mType: type2 \x1b[0m\x1b[33mUpdated: just now \x1b[0m\n\x1b[35m 3. \x1b[0mtitle3 \x1b[0m\x1b[35m(foo/bar/issues/1)\x1b[0m\n\x1b[32m Seen: False \x1b[0m\x1b[36mType: type3 \x1b[0m\x1b[33mUpdated: just now \x1b[0m\n View the page for \x1b[0m\x1b[35m1 through \x1b[0m\x1b[35m3\x1b[0m with the following command:\n\x1b[0m\x1b[35m gh view [#] \x1b[0moptional: [-b/--browser] [--help]\n\x1b[0m\x1b[0m' ================================================ FILE: tests/data/trends.py ================================================ # -*- coding: utf-8 -*- # Copyright 2015 Donne Martin. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of # the License is located at # # http://www.apache.org/licenses/LICENSE-2.0 # # or in the "license" file accompanying this file. This file is # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. formatted_trends = u'\x1b[35m 1. \x1b[0mpuikinsh/gentelella \x1b[0m\n Free Bootstrap 3 Admin Template\x1b[0m\n HTML\x1b[0m\n\x1b[35m 2. \x1b[0mrg3/youtube-dl \x1b[0m\n Small command-line program to download videos from YouTube.com and\n other video sites\x1b[0m\n Python\x1b[0m\n\x1b[35m 3. \x1b[0mFreeCodeCamp/FreeCodeCamp \x1b[0m\n The http://FreeCodeCamp.com open source codebase and curriculum.\n Learn to code and help nonprofits.\x1b[0m\n JavaScript\x1b[0m\n\x1b[35m 4. \x1b[0mmartanne/vis \x1b[0m\n a vim like text editor\x1b[0m\n C\x1b[0m\n\x1b[35m 5. \x1b[0mFezVrasta/popper.js \x1b[0m\n A ~3.5KB kickass library to manage your poppers\x1b[0m\n JavaScript\x1b[0m\n\x1b[35m 6. \x1b[0mfirehol/netdata \x1b[0m\n Real-time performance monitoring, done right!\x1b[0m\n C\x1b[0m\n\x1b[35m 7. \x1b[0marachnys/athenapdf \x1b[0m\n Drop-in replacement for wkhtmltopdf built on Go, Electron and Docker\x1b[0m\n Go\x1b[0m\n\x1b[35m 8. \x1b[0mmortenjust/cleartext-mac \x1b[0m\n A text editor that only allows the top 1,000 most common words in\n English\x1b[0m\n Swift\x1b[0m\n\x1b[35m 9. \x1b[0mfeross/webtorrent-desktop \x1b[0m\n WebTorrent, the streaming torrent client. For OS X, Windows, and\n Linux.\x1b[0m\n JavaScript\x1b[0m\n\x1b[35m 10. \x1b[0mfuture-architect/vuls \x1b[0m\n Vulnerability scanner for Linux, agentless, written in golang.\x1b[0m\n Go\x1b[0m\n\x1b[35m 11. \x1b[0mDevLight-Mobile-Agency/NavigationTabBar \x1b[0m\n Navigation tab bar with colorful interactions.\x1b[0m\n Java\x1b[0m\n\x1b[35m 12. \x1b[0mfeross/webtorrent \x1b[0m\n Streaming torrent client for the web\x1b[0m\n JavaScript\x1b[0m\n\x1b[35m 13. \x1b[0mfacebook/react \x1b[0m\n A declarative, efficient, and flexible JavaScript library for\n building user interfaces.\x1b[0m\n JavaScript\x1b[0m\n\x1b[35m 14. \x1b[0mtatsuhiro-t/aria2 \x1b[0m\n aria2 is a lightweight multi-protocol & multi-source, cross platform\n download utility operated in command-line. It supports HTTP/HTTPS,\n FTP, SFTP, BitTorrent and Metalink.\x1b[0m\n C++\x1b[0m\n\x1b[35m 15. \x1b[0mjessepollak/command \x1b[0m\n Making the web better with Slack-like slash commands.\x1b[0m\n JavaScript\x1b[0m\n\x1b[35m 16. \x1b[0msoimort/you-get \x1b[0m\n Dumb downloader that scrapes the web\x1b[0m\n Python\x1b[0m\n\x1b[35m 17. \x1b[0mpacketzoom/logzoom \x1b[0m\n A lightweight replacement for logstash indexer in Go\x1b[0m\n Go\x1b[0m\n\x1b[35m 18. \x1b[0mairbnb/caravel \x1b[0m\n Caravel is a data exploration platform designed to be visual,\n intuitive, and interactive\x1b[0m\n Python\x1b[0m\n\x1b[35m 19. \x1b[0mdylang/npm-check \x1b[0m\n Check for outdated, incorrect, and unused dependencies.\x1b[0m\n JavaScript\x1b[0m\n\x1b[35m 20. \x1b[0mkadirahq/react-storybook \x1b[0m\n Isolate your React UI Component development from the main app\x1b[0m\n JavaScript\x1b[0m\n\x1b[35m 21. \x1b[0mnnupoor/js_designpatterns \x1b[0m\x1b[0m\n JavaScript\x1b[0m\n\x1b[35m 22. \x1b[0mgooglesamples/android-architecture \x1b[0m\n A collection of samples to discuss and showcase different\n architectural tools and patterns for Android apps.\x1b[0m\n Unknown\x1b[0m\n\x1b[35m 23. \x1b[0mMrS0m30n3/youtube-dl-gui \x1b[0m\n A cross platform front-end GUI of the popular youtube-dl written in\n wxPython.\x1b[0m\n Python\x1b[0m\n\x1b[35m 24. \x1b[0mSkyscanner/SkyFloatingLabelTextField \x1b[0m\n A beautiful and flexible text field control implementation of "Float\n Label Pattern". Written in Swift. As used in the Skyscanner TravelPro\n iOS app.\x1b[0m\n Swift\x1b[0m\n\x1b[35m 25. \x1b[0mshutterstock/List-of-Dirty-Naughty-Obscene-and-Otherwise-Bad-Words \x1b[0m\n List of Dirty, Naughty, Obscene, and Otherwise Bad Words\x1b[0m\n Unknown\x1b[0m\n View the page for \x1b[0m\x1b[35m1 through \x1b[0m\x1b[35m25\x1b[0m with the following command:\n\x1b[0m\x1b[35m gh view [#] \x1b[0moptional: [-b/--browser] [--help]\n\x1b[0m\x1b[0m' ================================================ FILE: tests/data/user.py ================================================ # -*- coding: utf-8 -*- # Copyright 2015 Donne Martin. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of # the License is located at # # http://www.apache.org/licenses/LICENSE-2.0 # # or in the "license" file accompanying this file. This file is # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. formatted_org = u'To view the avatar in your terminal, install the Python Image Library.\n\x1b[0m\x1b[0muser2\n\x1b[0m\x1b[32mbigcorp\n\x1b[0m\x1b[32minterwebz\n\x1b[0m\x1b[32muser2@foo.com\n\x1b[0m\x1b[36mOrganization\n\n\x1b[0mNo results found\x1b[0m' formatted_user = u'To view the avatar in your terminal, install the Python Image Library.\n\x1b[0m\x1b[0muser1\n\x1b[0m\x1b[32mbigcorp\n\x1b[0m\x1b[32minterwebz\n\x1b[0m\x1b[32muser1@foo.com\n\x1b[0m\x1b[36mFollowers: 0 | \x1b[0m\x1b[36mFollowing: 0\n\n\x1b[0m\x1b[35m 1. \x1b[0mrepo1 \x1b[0m()\x1b[0m\n\x1b[32m Stars: 1 \x1b[0m\x1b[36mForks: 1 \x1b[0m\x1b[33mUpdated: just now \x1b[0m\n View the page for \x1b[0m\x1b[35m1 through \x1b[0m\x1b[35m1\x1b[0m with the following command:\n\x1b[0m\x1b[35m gh view [#] \x1b[0moptional: [-b/--browser] [--help]\n\x1b[0m\x1b[0m' formatted_users = u'\x1b[35m 1. \x1b[0mfoo1 \x1b[0m\n\x1b[35m 2. \x1b[0mfoo2 \x1b[0m\n\x1b[35m 3. \x1b[0mfoo3 \x1b[0m\n View the page for \x1b[0m\x1b[35m1 through \x1b[0m\x1b[35m3\x1b[0m with the following command:\n\x1b[0m\x1b[35m gh view [#] \x1b[0moptional: [-b/--browser] [--help]\n\x1b[0m\x1b[0m' ================================================ FILE: tests/data/user_feed.py ================================================ # -*- coding: utf-8 -*- # Copyright 2015 Donne Martin. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of # the License is located at # # http://www.apache.org/licenses/LICENSE-2.0 # # or in the "license" file accompanying this file. This file is # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. formatted_user_feed = u'\x1b[35m 1. \x1b[0mLeonardoGitHub \x1b[0m\x1b[32mforked \x1b[0m\x1b[36mdonnemartin/data-science-ipython-notebooks \x1b[0m\x1b[33m(just now)\x1b[0m\n\x1b[35m 2. \x1b[0mnavroopsingh \x1b[0m\x1b[32mstarred\x1b[0m\x1b[36m donnemartin/dev-setup \x1b[0m\x1b[33m(just now)\x1b[0m\n\x1b[35m 3. \x1b[0mprabaharanr \x1b[0m\x1b[32mforked \x1b[0m\x1b[36mdonnemartin/data-science-ipython-notebooks \x1b[0m\x1b[33m(just now)\x1b[0m\n\x1b[35m 4. \x1b[0mwdavis12 \x1b[0m\x1b[32mstarred\x1b[0m\x1b[36m donnemartin/data-science-ipython-notebooks \x1b[0m\x1b[33m(just now)\x1b[0m\n\x1b[35m 5. \x1b[0mvonec \x1b[0m\x1b[32mcommented on issue\x1b[0m\x1b[36m awslabs/aws-shell#115 \x1b[0m\x1b[33m(just now)\x1b[0m\n \x1b[36m\x1b[0mFixed \x1b[0m\n\x1b[35m 6. \x1b[0mbenc \x1b[0m\x1b[32mstarred\x1b[0m\x1b[36m donnemartin/dev-setup \x1b[0m\x1b[33m(just now)\x1b[0m\n\x1b[35m 7. \x1b[0mryusun \x1b[0m\x1b[32mstarred\x1b[0m\x1b[36m donnemartin/data-science-ipython-notebooks \x1b[0m\x1b[33m(just now)\x1b[0m\n\x1b[35m 8. \x1b[0mchirag200666 \x1b[0m\x1b[32mstarred\x1b[0m\x1b[36m donnemartin/data-science-ipython-notebooks \x1b[0m\x1b[33m(just now)\x1b[0m\n\x1b[35m 9. \x1b[0mravi-kothari \x1b[0m\x1b[32mstarred\x1b[0m\x1b[36m donnemartin/interactive-coding-challenges \x1b[0m\x1b[33m(just now)\x1b[0m\n\x1b[35m 10. \x1b[0mravi-kothari \x1b[0m\x1b[32mstarred\x1b[0m\x1b[36m donnemartin/data-science-ipython-notebooks \x1b[0m\x1b[33m(just now)\x1b[0m\n\x1b[35m 11. \x1b[0mmarnovo \x1b[0m\x1b[32mstarred\x1b[0m\x1b[36m donnemartin/data-science-ipython-notebooks \x1b[0m\x1b[33m(just now)\x1b[0m\n\x1b[35m 12. \x1b[0mandersy005 \x1b[0m\x1b[32mstarred\x1b[0m\x1b[36m donnemartin/interactive-coding-challenges \x1b[0m\x1b[33m(just now)\x1b[0m\n\x1b[35m 13. \x1b[0mandersy005 \x1b[0m\x1b[32mstarred\x1b[0m\x1b[36m donnemartin/viz \x1b[0m\x1b[33m(just now)\x1b[0m\n\x1b[35m 14. \x1b[0mt9md \x1b[0m\x1b[32mstarred\x1b[0m\x1b[36m donnemartin/interactive-coding-challenges \x1b[0m\x1b[33m(just now)\x1b[0m\n\x1b[35m 15. \x1b[0msilivizzle \x1b[0m\x1b[32mstarred\x1b[0m\x1b[36m donnemartin/dev-setup \x1b[0m\x1b[33m(just now)\x1b[0m\n\x1b[35m 16. \x1b[0mahna \x1b[0m\x1b[32mstarred\x1b[0m\x1b[36m donnemartin/data-science-ipython-notebooks \x1b[0m\x1b[33m(just now)\x1b[0m\n\x1b[35m 17. \x1b[0mziyasal \x1b[0m\x1b[32mstarred\x1b[0m\x1b[36m donnemartin/saws \x1b[0m\x1b[33m(just now)\x1b[0m\n\x1b[35m 18. \x1b[0mchefablz \x1b[0m\x1b[32mstarred\x1b[0m\x1b[36m donnemartin/data-science-ipython-notebooks \x1b[0m\x1b[33m(just now)\x1b[0m\n\x1b[35m 19. \x1b[0mkevinkn \x1b[0m\x1b[32mstarred\x1b[0m\x1b[36m donnemartin/dev-setup \x1b[0m\x1b[33m(just now)\x1b[0m\n\x1b[35m 20. \x1b[0mfrrp \x1b[0m\x1b[32mstarred\x1b[0m\x1b[36m donnemartin/interactive-coding-challenges \x1b[0m\x1b[33m(just now)\x1b[0m\n\x1b[35m 21. \x1b[0mfrrp \x1b[0m\x1b[32mstarred\x1b[0m\x1b[36m donnemartin/data-science-ipython-notebooks \x1b[0m\x1b[33m(just now)\x1b[0m\n\x1b[35m 22. \x1b[0mLiberTang0 \x1b[0m\x1b[32mforked \x1b[0m\x1b[36mdonnemartin/data-science-ipython-notebooks \x1b[0m\x1b[33m(just now)\x1b[0m\n\x1b[35m 23. \x1b[0mPhilippe-Caparroy \x1b[0m\x1b[32mstarred\x1b[0m\x1b[36m donnemartin/data-science-ipython-notebooks \x1b[0m\x1b[33m(just now)\x1b[0m\n\x1b[35m 24. \x1b[0mPhilippe-Caparroy \x1b[0m\x1b[32mforked \x1b[0m\x1b[36mdonnemartin/data-science-ipython-notebooks \x1b[0m\x1b[33m(just now)\x1b[0m\n\x1b[35m 25. \x1b[0mtiendan \x1b[0m\x1b[32mstarred\x1b[0m\x1b[36m donnemartin/data-science-ipython-notebooks \x1b[0m\x1b[33m(just now)\x1b[0m\n\x1b[35m 26. \x1b[0mSkyfold \x1b[0m\x1b[32mclosed issue\x1b[0m\x1b[36m awslabs/aws-shell#115 \x1b[0m\x1b[33m(just now)\x1b[0m\n\x1b[35m 27. \x1b[0mSkyfold \x1b[0m\x1b[32mcommented on issue\x1b[0m\x1b[36m awslabs/aws-shell#115 \x1b[0m\x1b[33m(just now)\x1b[0m\n \x1b[36m\x1b[0mFixed \x1b[0m\n\x1b[35m 28. \x1b[0msouadah \x1b[0m\x1b[32mforked \x1b[0m\x1b[36mdonnemartin/data-science-ipython-notebooks \x1b[0m\x1b[33m(just now)\x1b[0m\n\x1b[35m 29. \x1b[0mjoshuadsilva \x1b[0m\x1b[32mcommented on issue\x1b[0m\x1b[36m awslabs/aws-shell#115 \x1b[0m\x1b[33m(just now)\x1b[0m\n \x1b[36m\x1b[0mWorks for me too img class="emoji" title=":+1:"\n alt=":+1:" src="https://assets-\n cdn.github.com/images/icons/emoji/unicode/1f44d.png" height="20"\n width="20" align="absmiddle"\x1b[0m\n\x1b[35m 30. \x1b[0mwilsantos99 \x1b[0m\x1b[32mstarred\x1b[0m\x1b[36m donnemartin/awesome-aws \x1b[0m\x1b[33m(just now)\x1b[0m\n\x1b[0m' ================================================ FILE: tests/mock_feed_parser.py ================================================ # -*- coding: utf-8 -*- # Copyright 2015 Donne Martin. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of # the License is located at # # http://www.apache.org/licenses/LICENSE-2.0 # # or in the "license" file accompanying this file. This file is # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. from __future__ import unicode_literals from __future__ import print_function import pickle class MockFeedParser(object): def parse(self, url): if url == 'user_feed': with open('tests/data/user_feed.p', 'rb') as fp: return pickle.load(fp) else: with open('tests/data/trends.p', 'rb') as fp: return pickle.load(fp) ================================================ FILE: tests/mock_github_api.py ================================================ # -*- coding: utf-8 -*- # Copyright 2015 Donne Martin. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of # the License is located at # # http://www.apache.org/licenses/LICENSE-2.0 # # or in the "license" file accompanying this file. This file is # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. from __future__ import unicode_literals from __future__ import print_function from datetime import datetime import mock import pytz from gitsome.lib.github3 import null from gitsome.lib.github3.exceptions import UnprocessableEntity class MockEmail(object): def __init__(self, email, primary=False, verified=False): self.email = email self.primary = primary self.verified = verified class MockUser(object): def __init__(self, login, user_type='User'): self.login = login self.repositories = {} self.emails = [] self.avatar_url = 'https://www.github.com' self.company = 'bigcorp' self.location = 'interwebz' self.email = login + '@foo.com' self.type = user_type self.followers_count = 0 self.following_count = 0 self.feed_events = [] def events(self, public): feed_events = [] feed_events.append(MockEvent( event_type='CommitCommentEvent', payload={ 'comment': MockRepoComment(), })) feed_events.append(MockEvent( event_type='CreateEvent', payload={ 'ref_type': 'branch', 'ref': 'master', 'description': None, })) feed_events.append(MockEvent( event_type='FollowEvent')) feed_events.append(MockEvent( event_type='ForkEvent')) feed_events.append(MockEvent( event_type='IssueCommentEvent', payload={ 'comment': MockIssueComment('foo'), 'issue': MockIssue('1', MockRepo(self, 'repo1'), 'foo'), })) feed_events.append(MockEvent( event_type='IssuesEvent', payload={ 'action': 'closed', 'issue': MockIssue('1', MockRepo(self, 'repo1'), 'foo'), })) feed_events.append(MockEvent( event_type='PullRequestEvent', payload={ 'action': 'closed', 'pull_request': MockIssue('1', MockRepo(self, 'repo1'), 'foo'), })) feed_events.append(MockEvent( event_type='PushEvent', payload={ 'ref': 'refs/heads/master', 'commits': [{'url': 'https://api.github.com/repos/donnemartin/gitsome/commits/5ee4d1b20ee7cb16cd5be19b103301541a41003f', 'message': 'Fix GitHubCli class docstring', 'distinct': True, 'author': {'email': 'donne.martin@gmail.com', 'name': 'Donne Martin'}, 'sha': '5ee4d1b20ee7cb16cd5be19b103301541a41003f'}, {'url': 'https://api.github.com/repos/donnemartin/gitsome/commits/fc2309b645313646a3792eca9e0e9168cf25b267', 'message': 'Update gh configure docstring', 'distinct': True, 'author': {'email': 'donne.martin@gmail.com', 'name': 'Donne Martin'}, 'sha': 'fc2309b645313646a3792eca9e0e9168cf25b267'}, {'url': 'https://api.github.com/repos/donnemartin/gitsome/commits/dde19b7685ad7a07872fea1b4dc8019585322fdb', 'message': 'Update gh create-comment docstring', 'distinct': True, 'author': {'email': 'donne.martin@gmail.com', 'name': 'Donne Martin'}, 'sha': 'dde19b7685ad7a07872fea1b4dc8019585322fdb'}] # NOQA })) mock_release_tag = mock.Mock() mock_release_tag.tag_name = '0.5.0' feed_events.append(MockEvent( event_type='ReleaseEvent', payload={ 'release': mock_release_tag, })) return feed_events def raise_mock_unprocessableentity(self): response = mock.Mock() response.json = lambda: exec('raise(Exception())') response.content = 'foobar' raise UnprocessableEntity(response) def create_repo(self, name, desc='', private=False): if name in self.repositories: self.raise_mock_unprocessableentity() repo = MockRepo(self, name, desc, private) self.repositories.update({repo.full_name: repo}) return repo class MockRepo(object): def __init__(self, user, full_name, description='', private=False): self.user = user self.full_name = full_name self.description = description self.private = private self.issues = {} self.clone_url = 'https://github.com/octocat/spoon-knife' self.stargazers_count = 1 self.forks_count = 1 self.language = '' self.updated_at = '' self.repository = 'foobar' def __lt__(self, other): return self.full_name < other.full_name def gen_key(self): return len(self.issues) + 1 def create_issue(self, issue_title, issue_desc=''): number = self.gen_key() issue = MockIssue(number, self, issue_title, issue_desc) self.issues.update({number: issue}) return issue def pull_requests(self): return list(self.issues.values()) class MockIssue(object): def __init__(self, number, repository, title, body=''): self.number = number self.repository = (repository.user.login, repository.full_name) self.title = title self.body = body self.state = 'open' self.comments_count = 1 self.assignee = 'user1' self.user = 'user2' self.created_at = '' self.comments = [] self.issue = 'foobar' def create_comment(self, body): issue_comment = MockIssueComment(body) self.comments.append(issue_comment) return issue_comment class MockIssueComment(object): def __init__(self, body): self.body = body class MockRepoComment(object): def __init__(self): self.commit_id = 'AAA23e2c6cb6997d25cfe61673aea6d701e9bZZZ' self.body = 'foo' class MockLicense(object): def __init__(self, key, name): self.key = key self.name = name class MockThread(object): def __init__(self, thread_type, title, unread): self.subject = { 'title': title, 'type': thread_type, 'url': 'https://api.github.com/repos/foo/bar/pulls/1', } self.unread = unread self.updated_at = '' class MockEvent(object): def __init__(self, event_type, payload=''): self.id = 1 self.created_at = datetime.now(pytz.utc) self.actor = 'donnemartin' self.org = 'org' self.type = event_type self.payload = payload self.repo = ('user1', 'repo1') self.public = True class MockGitHubApi(object): def __init__(self): self.users = {} self.current_user = 'user1' self.ratelimit_remaining = 5000 self._generate_mock_data() def _generate_mock_data(self): user1 = MockUser(self.current_user, 'User') user1_repo1 = user1.create_repo('repo1') user1_repo1.create_issue('title1', 'body1') user1_repo1.create_issue('title2', 'body2') user1_repo1.create_issue('title3', 'body3') user1.emails.extend([ MockEmail('foo@baz.com', True, False), MockEmail('bar@baz.com', False, True), ]) user2 = MockUser('user2', 'Organization') self.users.update({ user1.login: user1, user2.login: user2, }) def create_issue(self, user_login, repo_name, issue_title, issue_desc): try: user = self.users[user_login] repo = user.repositories[repo_name] issue = repo.create_issue(issue_title, issue_desc) return issue except KeyError: return null.NullObject('Issue') def create_repository(self, repo_name, repo_desc='', private=False): user = self.users[self.current_user] return user.create_repo(repo_name, repo_desc, private) def emails(self): user = self.users[self.current_user] return user.emails def emojis(self, pager=False): return [ 'dolls', 'palm_tree', 'uk', '100', 'baby_chick', ] def followers_of(self, user_login): return [ MockUser('foo1'), MockUser('foo2'), MockUser('foo3'), ] def followed_by(self, user_login): return self.followers_of(user_login) def gitignore_template(self, language): if language == 'valid_language': return 'template' else: return '' def gitignore_templates(self): return [ 'Actionscript', 'Ada', 'Agda', 'Android', 'AppEngine', ] def issue(self, user_login, repo_name, number): try: user = self.users[user_login] repo = user.repositories[repo_name] return repo.issues[int(number)] except KeyError: return null.NullObject('Issue') def issues(self, issue_filter='subscribed', issue_state='open'): user = self.users[self.current_user] repo = user.repositories['repo1'] issues_dict = repo.issues issues = list(issues_dict.values()) return issues def license(self, license): if license == 'valid_license': template = mock.Mock() template.body = 'template' return template else: return null.NullObject('License') def licenses(self): return [ MockLicense('mit', '(MIT License)'), MockLicense('gpl-2.0', '(GNU General Public License v2.0)'), MockLicense('bsd-2-clause', '(BSD 2-clause "Simplified" License)'), MockLicense('isc', '(ISC License)'), MockLicense('epl-1.0', '(Eclipse Public License 1.0)'), ] def notifications(self, all=True, participating=False): return [ MockThread('type1', 'title1', True), MockThread('type2', 'title2', False), MockThread('type3', 'title3', True), ] def octocat(self, say): return say def pull_request(self, owner, repository, number): pull_requests = self.issues() return pull_requests[0] def search_issues(self, query): return self.issues() def search_repositories(self, query, sort): return self.repositories() def repositories(self, user_id=None): if user_id is None: user_id = self.current_user user = self.users[user_id] repos = list(user.repositories.values()) repos_sorted = sorted(repos) return repos_sorted def user(self, user_id): try: return self.users[user_id] except KeyError: return null.NullObject('User') ================================================ FILE: tests/mock_pretty_date_time.py ================================================ # -*- coding: utf-8 -*- # Copyright 2015 Donne Martin. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of # the License is located at # # http://www.apache.org/licenses/LICENSE-2.0 # # or in the "license" file accompanying this file. This file is # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. from __future__ import unicode_literals from __future__ import print_function def pretty_date_time(date_time): return "just now" ================================================ FILE: tests/run_tests.py ================================================ #!/usr/bin/env python # -*- coding: utf-8 -*- # Copyright 2015 Donne Martin. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of # the License is located at # # http://www.apache.org/licenses/LICENSE-2.0 # # or in the "license" file accompanying this file. This file is # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. from __future__ import unicode_literals from __future__ import print_function from compat import unittest from test_completer import CompleterTest # NOQA from test_config import ConfigTest # NOQA from test_github import GitHubTest # NOQA from test_github_cli import GitHubCliTest # NOQA from test_web_viewer import WebViewerTest # NOQA if __name__ == '__main__': unittest.main() ================================================ FILE: tests/test_completer.py ================================================ # -*- coding: utf-8 -*- # Copyright 2015 Donne Martin. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of # the License is located at # # http://www.apache.org/licenses/LICENSE-2.0 # # or in the "license" file accompanying this file. This file is # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. from __future__ import unicode_literals from __future__ import print_function import mock from tests.compat import unittest from prompt_toolkit.document import Document from gitsome.completer import CompleterGitsome class CompleterTest(unittest.TestCase): def setUp(self): self.completer = CompleterGitsome() self.completer_event = self.create_completer_event() def create_completer_event(self): return mock.Mock() def _get_completions(self, command): position = len(command) result = self.completer.get_completions( Document(text=command, cursor_position=position), self.completer_event) return result def verify_completions(self, commands, expected): result = [] for command in commands: # Call the AWS CLI autocompleter result.extend(self._get_completions(command)) result_texts = [] for item in result: # Each result item is a Completion object, # we are only interested in the text portion result_texts.append(item.text) assert result_texts if len(expected) == 1: assert expected[0] in result_texts else: for item in expected: assert item in result_texts def test_blank(self): text = '' expected = [] result = self._get_completions(text) assert result == expected def test_no_completions(self): text = 'foo' expected = [] result = self._get_completions(text) assert result == expected def test_command(self): text = ['g'] expected = ['gh'] self.verify_completions(text, expected) def test_subcommand(self): self.verify_completions(['gh c'], ['configure', 'create-comment', 'create-issue', 'create-repo']) self.verify_completions(['gh e'], ['emails', 'emojis']) self.verify_completions(['gh f'], ['feed', 'followers', 'following']) self.verify_completions(['gh g'], ['gitignore-template', 'gitignore-templates']) self.verify_completions(['gh i'], ['issue', 'issues']) self.verify_completions(['gh l'], ['license', 'licenses']) self.verify_completions(['gh m'], ['me']) self.verify_completions(['gh n'], ['notifications']) self.verify_completions(['gh o'], ['octo']) self.verify_completions(['gh p'], ['pull-request', 'pull-requests']) self.verify_completions(['gh r'], ['rate-limit', 'repo', 'repos']) self.verify_completions(['gh s'], ['search-issues', 'search-repos', 'starred']) self.verify_completions(['gh t'], ['trending']) self.verify_completions(['gh u'], ['user']) self.verify_completions(['gh v'], ['view']) def test_args(self): self.verify_completions(['gh octo '], ['"Keep it logically awesome"']) def test_no_args_with_options(self): self.verify_completions(['gh octo '], ['"Keep it logically awesome"']) def test_options(self): self.verify_completions(['gh emojis '], ['-p', '--pager']) def test_multiple_options(self): self.verify_completions(['gh feed -pr --pa'], ['--pager']) self.verify_completions(['gh feed foo -pr --pa'], ['--pager']) def test_fuzzy(self): self.completer.fuzzy_match = True self.verify_completions(['gh ot'], ['octo']) def test_build_completions_with_meta(self): result = self.completer.build_completions_with_meta('git ad', 'ad', ['add']) assert result[0].display_meta_text == 'Add file contents to the index.' result = self.completer.build_completions_with_meta('git-alia', 'git-alia', ['git-alias']) assert result[0].display_meta_text == 'Define, search and show aliases.' ================================================ FILE: tests/test_config.py ================================================ # -*- coding: utf-8 -*- # Copyright 2015 Donne Martin. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of # the License is located at # # http://www.apache.org/licenses/LICENSE-2.0 # # or in the "license" file accompanying this file. This file is # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. from __future__ import unicode_literals from __future__ import print_function import mock import os from gitsome.compat import configparser from tests.compat import unittest from gitsome.github import GitHub from tests.mock_github_api import MockGitHubApi class ConfigTest(unittest.TestCase): def setUp(self): self.github = GitHub() self.github.config.api = MockGitHubApi() self.github.config.login = mock.Mock() self.github.config.authorize = mock.Mock() self.github.config.getpass = mock.Mock() def verify_login_token(self, username=None, password=None, token=None, url=None, two_factor_callback=None, verify=True): assert username is not None assert token is not None assert two_factor_callback is not None assert verify def verify_login_pass(self, username=None, password=None, token=None, url=None, two_factor_callback=None, verify=True): assert username is not None assert password is not None assert password assert two_factor_callback is not None assert verify def verify_login_token_url_enterprise(self, username=None, password=None, token=None, url=None, two_factor_callback=None, verify=True): assert username is not None assert token is not None assert url is not None assert two_factor_callback is not None assert verify def verify_login_token_url_no_verify_enterprise(self, username=None, password=None, token=None, url=None, two_factor_callback=None, verify=True): assert username is not None assert token is not None assert url is not None assert two_factor_callback is not None assert not verify def verify_login_pass_url_enterprise(self, username=None, password=None, token=None, url=None, two_factor_callback=None, verify=True): assert username is not None assert password is not None assert url is not None assert two_factor_callback is not None assert verify def test_config(self): expected = os.path.join(os.path.abspath(os.environ.get('HOME', '')), self.github.config.CONFIG) assert self.github.config \ .get_github_config_path(self.github.config.CONFIG) == expected def test_authenticate_cached_credentials_token(self): self.github.config.user_login = 'foo' self.github.config.user_token = 'bar' self.github.config.save_config() self.github.config.user_login = '' self.github.config.user_token = '' self.github.config.api = None config = self.github.config.get_github_config_path( self.github.config.CONFIG) parser = configparser.RawConfigParser() self.github.config.login = self.verify_login_token self.github.config.authenticate_cached_credentials(config, parser) assert self.github.config.user_login == 'foo' assert self.github.config.user_token == 'bar' def test_authenticate_cached_credentials_pass(self): self.github.config.user_login = 'foo' self.github.config.user_pass = 'bar' self.github.config.save_config() self.github.config.user_login = '' self.github.config.user_pass = '' self.github.config.api = None config = self.github.config.get_github_config_path( self.github.config.CONFIG) parser = configparser.RawConfigParser() self.github.config.authenticate_cached_credentials(config, parser) assert self.github.config.user_login == 'foo' assert self.github.config.user_pass is None def test_authenticate_cached_credentials_token_enterprise(self): self.github.config.user_login = 'foo' self.github.config.user_token = 'bar' self.github.config.enterprise_url = 'baz' self.github.config.save_config() self.github.config.user_login = '' self.github.config.user_token = '' self.github.config.enterprise_url = '' self.github.config.api = None config = self.github.config.get_github_config_path( self.github.config.CONFIG) parser = configparser.RawConfigParser() self.github.config.authenticate_cached_credentials( config, parser, enterprise_auth=self.verify_login_token_url_enterprise) assert self.github.config.user_login == 'foo' assert self.github.config.user_token == 'bar' assert self.github.config.enterprise_url == 'baz' def test_authenticate_cached_credentials_pass_enterprise(self): self.github.config.user_login = 'foo' self.github.config.user_pass = 'bar' self.github.config.enterprise_url = 'baz' self.github.config.save_config() self.github.config.user_login = '' self.github.config.user_pass = '' self.github.config.enterprise_url = '' self.github.config.api = None config = self.github.config.get_github_config_path( self.github.config.CONFIG) parser = configparser.RawConfigParser() self.github.config.authenticate_cached_credentials( config, parser, enterprise_auth=self.verify_login_pass_url_enterprise) assert self.github.config.user_login == 'foo' assert self.github.config.user_pass == 'bar' assert self.github.config.enterprise_url == 'baz' @mock.patch('gitsome.github.click.secho') @mock.patch('gitsome.config.Config.authenticate_cached_credentials') def test_authenticate_token(self, mock_auth, mock_click_secho): with mock.patch('click.confirm', return_value=False): with mock.patch('builtins.input', return_value='foo'): self.github.config.login = self.verify_login_token self.github.config.user_login = 'foo' self.github.config.user_token = 'bar' self.github.config.authenticate( enterprise=False, overwrite=True) @mock.patch('gitsome.github.click.secho') @mock.patch('gitsome.config.Config.authenticate_cached_credentials') def test_authenticate_pass(self, mock_auth, mock_click_secho): self.github.config.getpass.return_value = 'bar' with mock.patch('click.confirm', return_value=True): with mock.patch('builtins.input', return_value='foo'): self.github.config.login = self.verify_login_pass self.github.config.user_login = 'foo' self.github.config.authenticate( enterprise=False, overwrite=True) assert self.github.config.user_pass is None @mock.patch('gitsome.github.click.secho') @mock.patch('gitsome.config.Config.authenticate_cached_credentials') def test_authenticate_enterprise_token(self, mock_auth, mock_click_secho): with mock.patch('click.confirm', return_value=False): with mock.patch('builtins.input', return_value='foo'): self.github.config.user_login = 'foo' self.github.config.user_token = 'bar' enterprise_auth = \ self.verify_login_token_url_no_verify_enterprise self.github.config.authenticate( enterprise=True, enterprise_auth=enterprise_auth, overwrite=True) @mock.patch('gitsome.github.click.secho') @mock.patch('gitsome.config.Config.authenticate_cached_credentials') def test_authenticate_enterprise_pass(self, mock_auth, mock_click_secho): self.github.config.getpass.return_value = 'bar' with mock.patch('click.confirm', return_value=True): with mock.patch('builtins.input', return_value='foo'): self.github.config.user_login = 'foo' self.github.config.authenticate( enterprise=True, enterprise_auth=self.verify_login_pass_url_enterprise, overwrite=True) assert self.github.config.user_pass is not None @mock.patch('gitsome.github.click.secho') def test_check_auth_error(self, mock_click_secho): self.github.config.api = None self.github.config.check_auth() mock_click_secho.assert_any_call('Authentication error.', fg='red') mock_click_secho.assert_any_call('Update your credentials in ~/.gitsomeconfig or run:\n gh configure', fg=None) # NOQA def test_load_urls(self): urls = self.github.config.load_urls(view_in_browser=False) assert urls == ['octocat/spoon-knife'] def test_request_two_factor_code(self): with mock.patch('builtins.input', return_value='code'): assert self.github.config.request_two_factor_code() == 'code' @mock.patch('gitsome.github.click.secho') def test_prompt_news_feed(self, mock_click_secho): with mock.patch('click.confirm', return_value='y'): with mock.patch('builtins.input', return_value='feed'): self.github.config.prompt_news_feed() assert self.github.config.user_feed == 'feed' ================================================ FILE: tests/test_github.py ================================================ # -*- coding: utf-8 -*- # Copyright 2015 Donne Martin. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of # the License is located at # # http://www.apache.org/licenses/LICENSE-2.0 # # or in the "license" file accompanying this file. This file is # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. from __future__ import unicode_literals from __future__ import print_function import mock from compat import unittest from gitsome.github import GitHub from tests.mock_feed_parser import MockFeedParser from tests.mock_github_api import MockGitHubApi from tests.mock_pretty_date_time import pretty_date_time from tests.data.email import formatted_emails from tests.data.emoji import formatted_emojis from tests.data.events import formatted_events from tests.data.user import formatted_org, formatted_user, formatted_users from tests.data.gitignores import formatted_gitignores, formatted_gitignores_tip from tests.data.issue import formatted_issues, formatted_pull_requests from tests.data.license import formatted_licenses, formatted_licenses_tip from tests.data.thread import formatted_threads from tests.data.trends import formatted_trends from tests.data.user_feed import formatted_user_feed class GitHubTest(unittest.TestCase): def setUp(self): self.github = GitHub() self.github.config.api = MockGitHubApi() self.github.formatter.pretty_dt = pretty_date_time self.github.trend_parser = MockFeedParser() def test_avatar_no_pil(self): avatar_text = self.github.avatar( 'https://avatars.githubusercontent.com/u/583231?v=3', False) assert avatar_text == 'PIL not found.\n' @mock.patch('gitsome.github.click.secho') def test_create_comment(self, mock_click_secho): self.github.create_comment('user1/repo1/1', 'text') mock_click_secho.assert_called_with( 'Created comment: text', fg=self.github.config.clr_message) @mock.patch('gitsome.github.click.secho') def test_create_comment_invalid_args(self, mock_click_secho): self.github.create_comment('invalid/repo1/1', 'text') mock_click_secho.assert_called_with( 'Error creating comment', fg=self.github.config.clr_error) self.github.create_comment('user1/repo1/foo', 'text') mock_click_secho.assert_called_with( 'Expected argument: user/repo/# and option -t "comment".', fg=self.github.config.clr_error) @mock.patch('gitsome.github.click.secho') def test_create_issue(self, mock_click_secho): self.github.create_issue('user1/repo1', 'title', 'desc') mock_click_secho.assert_called_with( 'Created issue: title\ndesc', fg=self.github.config.clr_message) @mock.patch('gitsome.github.click.secho') def test_create_issue_no_desc(self, mock_click_secho): self.github.create_issue('user1/repo1', 'title', issue_desc=None) mock_click_secho.assert_called_with( 'Created issue: title\n', fg=self.github.config.clr_message) @mock.patch('gitsome.github.click.secho') def test_create_issue_invalid_args(self, mock_click_secho): self.github.create_issue('invalid/repo1', 'title', 'desc') mock_click_secho.assert_called_with( 'Error creating issue.', fg=self.github.config.clr_error) self.github.create_issue('u', 'title', 'desc') mock_click_secho.assert_called_with( 'Expected argument: user/repo and option -t "title".', fg=self.github.config.clr_error) @mock.patch('gitsome.github.click.secho') def test_create_repo(self, mock_click_secho): self.github.create_repo('name', 'desc', True) mock_click_secho.assert_called_with( 'Created repo: name\ndesc', fg=self.github.config.clr_message) @mock.patch('gitsome.github.click.secho') def test_create_repo_no_desc(self, mock_click_secho): self.github.create_repo('name', repo_desc=None) mock_click_secho.assert_called_with( 'Created repo: name\n', fg=self.github.config.clr_message) @mock.patch('gitsome.github.click.secho') def test_create_repo_invalid_args(self, mock_click_secho): self.github.create_repo('repo1', 'desc', True) mock_click_secho.assert_called_with( 'Error creating repo: foobar', fg=self.github.config.clr_error) @mock.patch('gitsome.github.click.secho') def test_emails(self, mock_click_secho): self.github.emails() mock_click_secho.assert_called_with(formatted_emails) @mock.patch('gitsome.github.click.secho') @mock.patch('gitsome.config.Config.prompt_news_feed') def test_feed_config(self, mock_config_prompt_news_feed, mock_click_secho): self.github.feed() mock_config_prompt_news_feed.assert_called_with() @mock.patch('gitsome.github.click.secho') def test_feed(self, mock_click_secho): self.github.config.user_feed = 'user_feed' self.github.feed() mock_click_secho.assert_called_with(formatted_user_feed) @mock.patch('gitsome.github.click.secho') @mock.patch('gitsome.config.Config') def test_feed_user(self, mock_config, mock_click_secho): self.github.feed('user1') mock_click_secho.assert_called_with(formatted_events) @mock.patch('gitsome.github.click.secho') def test_emojis(self, mock_click_secho): self.github.emojis() mock_click_secho.assert_called_with(formatted_emojis) @mock.patch('gitsome.github.click.secho') def test_followers(self, mock_click_secho): self.github.followers('foo') mock_click_secho.assert_called_with(formatted_users) @mock.patch('gitsome.github.click.secho') def test_following(self, mock_click_secho): self.github.following('foo') mock_click_secho.assert_called_with(formatted_users) @mock.patch('gitsome.github.click.secho') def test_gitignore_template(self, mock_click_secho): self.github.gitignore_template('valid_language') mock_click_secho.assert_called_with( 'template', fg=self.github.config.clr_message) @mock.patch('gitsome.github.click.secho') def test_gitignore_template_invalid(self, mock_click_secho): self.github.gitignore_template('invalid_language') mock_click_secho.assert_called_with( ('Invalid case-sensitive template requested, run the ' 'following command to see available templates:\n' ' gh gitignore-templates'), fg=self.github.config.clr_error) @mock.patch('gitsome.github.click.secho') def test_gitignore_templates(self, mock_click_secho): self.github.gitignore_templates() mock_click_secho.assert_any_call(formatted_gitignores) mock_click_secho.assert_any_call(formatted_gitignores_tip, fg=self.github.config.clr_message) @mock.patch('gitsome.web_viewer.WebViewer.view_url') def test_issue(self, mock_view_url): self.github.issue('user1/repo1/1') mock_view_url.assert_called_with( 'https://github.com/user1/repo1/issues/1') @mock.patch('gitsome.github.click.secho') def test_issue_invalid_args(self, mock_click_secho): self.github.issue('user1/repo1/foo') mock_click_secho.assert_called_with( 'Expected argument: user/repo/#.', fg=self.github.config.clr_error) @mock.patch('gitsome.github.click.secho') def test_issues_setup(self, mock_click_secho): self.github.issues_setup() mock_click_secho.assert_called_with(formatted_issues) @mock.patch('gitsome.github.click.secho') def test_license(self, mock_click_secho): self.github.license('valid_license') mock_click_secho.assert_called_with( 'template', fg=self.github.config.clr_message) @mock.patch('gitsome.github.click.secho') def test_license_invalid(self, mock_click_secho): self.github.license('invalid_license') mock_click_secho.assert_called_with( (' Invalid case-sensitive license requested, run the ' 'following command to see available licenses:\n' ' gh licenses'), fg=self.github.config.clr_error) @mock.patch('gitsome.github.click.secho') def test_licenses(self, mock_click_secho): self.github.licenses() mock_click_secho.assert_any_call(formatted_licenses) mock_click_secho.assert_any_call(formatted_licenses_tip, fg=self.github.config.clr_message) @mock.patch('gitsome.github.click.secho') def test_notifications(self, mock_click_secho): self.github.notifications() mock_click_secho.assert_called_with(formatted_threads) @mock.patch('gitsome.github.click.secho') def test_octocat(self, mock_click_secho): self.github.octocat('foo\\nbar') mock_click_secho.assert_called_with( 'foo\nbar', fg=self.github.config.clr_message) @mock.patch('gitsome.github.click.secho') def test_pull_requests(self, mock_click_secho): self.github.pull_requests() mock_click_secho.assert_called_with(formatted_pull_requests) @mock.patch('gitsome.github.click.secho') def test_rate_limit(self, mock_click_secho): self.github.rate_limit() mock_click_secho.assert_called_with( 'Rate limit: 5000', fg=self.github.config.clr_message) @mock.patch('gitsome.web_viewer.WebViewer.view_url') def test_repository(self, mock_view_url): self.github.repository('user1/repo1') mock_view_url.assert_called_with( 'https://github.com/user1/repo1') @mock.patch('gitsome.github.click.secho') def test_repository_invalid(self, mock_click_secho): self.github.repository('user1/repo1/1') mock_click_secho.assert_called_with( 'Expected argument: user/repo.', fg=self.github.config.clr_error) @mock.patch('gitsome.github.click.secho') @mock.patch('gitsome.github.GitHub.issues') def test_search_issues(self, mock_github_issues, mock_click_secho): self.github.search_issues('foo') mock_github_issues.assert_called_with( ['foobar', 'foobar', 'foobar'], 1000, False, sort=False) @mock.patch('gitsome.github.click.secho') @mock.patch('gitsome.github.GitHub.repositories') def test_search_repos(self, mock_github_repositories, mock_click_secho): self.github.search_repositories('foo', 'stars') mock_github_repositories.assert_called_with( ['foobar'], 1000, False, sort=False) @mock.patch('gitsome.github.click.secho') def test_trending(self, mock_click_secho): self.github.trending('Python', False, False, False) mock_click_secho.assert_called_with(formatted_trends) @mock.patch('gitsome.github.click.secho') def test_user(self, mock_click_secho): self.github.user('user1') mock_click_secho.assert_called_with(formatted_user) self.github.user('user2') mock_click_secho.assert_called_with(formatted_org) @mock.patch('gitsome.github.click.secho') def test_user_invalid(self, mock_click_secho): self.github.user('invalid_user') mock_click_secho.assert_called_with( 'Invalid user.', fg=self.github.config.clr_error) @mock.patch('gitsome.github.click.secho') @mock.patch('gitsome.github.webbrowser.open') def test_user_browser(self, mock_webbrowser_open, mock_click_secho): self.github.user('invalid_user', browser=True) mock_webbrowser_open.assert_called_with( 'https://github.com/invalid_user') @mock.patch('gitsome.github.click.secho') @mock.patch('gitsome.github.webbrowser.open') def test_view_browser(self, mock_webbrowser_open, mock_click_secho): self.github.config.load_urls = lambda x: ['user1/foo'] self.github.view(1, view_in_browser=True) mock_webbrowser_open.assert_called_with( 'https://github.com/user1/foo') @mock.patch('gitsome.github.click.secho') @mock.patch('gitsome.github.GitHub.issue') def test_view_issue(self, mock_github_issue, mock_click_secho): self.github.config.load_urls = lambda x: ['user1/foo/issues/1'] self.github.view(0) mock_github_issue.assert_called_with('user1/foo/1') @mock.patch('gitsome.github.click.secho') @mock.patch('gitsome.github.GitHub.repository') def test_view_repo(self, mock_github_repository, mock_click_secho): self.github.config.load_urls = lambda x: ['user1/foo'] self.github.view(0) mock_github_repository.assert_called_with('user1/foo') @mock.patch('gitsome.github.click.secho') @mock.patch('gitsome.web_viewer.WebViewer.view_url') def test_view_user(self, mock_view_url, mock_click_secho): self.github.config.load_urls = lambda x: ['user1'] self.github.view(0) mock_view_url.assert_called_with('https://github.com/user1') def test_base_url(self): self.github.config.enterprise_url = 'https://github.intra.example.com' assert self.github.base_url == 'https://github.intra.example.com' self.github.config.enterprise_url = None assert self.github.base_url == self.github._base_url def test_add_base_url(self): expected = self.github.base_url + 'foo.html' assert self.github.add_base_url('foo.html') == expected assert self.github.add_base_url(expected) == expected ================================================ FILE: tests/test_github_cli.py ================================================ # -*- coding: utf-8 -*- # Copyright 2015 Donne Martin. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of # the License is located at # # http://www.apache.org/licenses/LICENSE-2.0 # # or in the "license" file accompanying this file. This file is # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. from __future__ import unicode_literals from __future__ import print_function import mock from compat import unittest from click.testing import CliRunner from gitsome.githubcli import GitHubCli class GitHubCliTest(unittest.TestCase): def setUp(self): self.runner = CliRunner() self.github_cli = GitHubCli() self.limit = 1000 def test_cli(self): result = self.runner.invoke(self.github_cli.cli) assert result.exit_code == 0 @mock.patch('gitsome.githubcli.GitHub.configure') def test_configure(self, mock_gh_call): result = self.runner.invoke(self.github_cli.cli, ['configure']) mock_gh_call.assert_called_with(False) assert result.exit_code == 0 @mock.patch('gitsome.githubcli.GitHub.create_comment') def test_create_comment(self, mock_gh_call): result = self.runner.invoke(self.github_cli.cli, ['create-comment', 'u/r/n', '--text', 'foo']) mock_gh_call.assert_called_with('u/r/n', 'foo') assert result.exit_code == 0 @mock.patch('gitsome.githubcli.GitHub.create_issue') def test_create_issue(self, mock_gh_call): result = self.runner.invoke(self.github_cli.cli, ['create-issue', 'u/r', '--issue_title', 'foo', '--issue_desc', 'bar']) mock_gh_call.assert_called_with('u/r', 'foo', 'bar') assert result.exit_code == 0 @mock.patch('gitsome.githubcli.GitHub.create_repo') def test_create_repo(self, mock_gh_call): result = self.runner.invoke(self.github_cli.cli, ['create-repo', 'r', '--repo_desc', 'foo', '--private']) mock_gh_call.assert_called_with('r', 'foo', True) assert result.exit_code == 0 @mock.patch('gitsome.githubcli.GitHub.emails') def test_emails(self, mock_gh_call): result = self.runner.invoke(self.github_cli.cli, ['emails']) mock_gh_call.assert_called_with() assert result.exit_code == 0 @mock.patch('gitsome.githubcli.GitHub.emojis') def test_emojis(self, mock_gh_call): result = self.runner.invoke(self.github_cli.cli, ['emojis', '--pager']) mock_gh_call.assert_called_with(True) assert result.exit_code == 0 @mock.patch('gitsome.githubcli.GitHub.feed') def test_feed(self, mock_gh_call): result = self.runner.invoke(self.github_cli.cli, ['feed', 'u', '--private', '--pager']) mock_gh_call.assert_called_with('u', True, True) assert result.exit_code == 0 @mock.patch('gitsome.githubcli.GitHub.followers') def test_followers(self, mock_gh_call): result = self.runner.invoke(self.github_cli.cli, ['followers', 'u', '--pager']) mock_gh_call.assert_called_with('u', True) assert result.exit_code == 0 @mock.patch('gitsome.githubcli.GitHub.following') def test_following(self, mock_gh_call): result = self.runner.invoke(self.github_cli.cli, ['following', 'u', '--pager']) mock_gh_call.assert_called_with('u', True) assert result.exit_code == 0 @mock.patch('gitsome.githubcli.GitHub.gitignore_template') def test_gitignore_template(self, mock_gh_call): result = self.runner.invoke(self.github_cli.cli, ['gitignore-template', 'l']) mock_gh_call.assert_called_with('l') assert result.exit_code == 0 @mock.patch('gitsome.githubcli.GitHub.gitignore_templates') def test_gitignore_templates(self, mock_gh_call): result = self.runner.invoke(self.github_cli.cli, ['gitignore-templates', '--pager']) mock_gh_call.assert_called_with(True) assert result.exit_code == 0 @mock.patch('gitsome.githubcli.GitHub.issue') def test_issue(self, mock_gh_call): result = self.runner.invoke(self.github_cli.cli, ['issue', 'u/r/n']) mock_gh_call.assert_called_with('u/r/n') assert result.exit_code == 0 @mock.patch('gitsome.githubcli.GitHub.issues_setup') def test_issues(self, mock_gh_call): result = self.runner.invoke(self.github_cli.cli, ['issues', '--issue_filter', 'mentioned', '--issue_state', 'closed', '--limit', '10', '--pager']) mock_gh_call.assert_called_with('mentioned', 'closed', 10, True) assert result.exit_code == 0 @mock.patch('gitsome.githubcli.GitHub.license') def test_license(self, mock_gh_call): result = self.runner.invoke(self.github_cli.cli, ['license', 'l']) mock_gh_call.assert_called_with('l') assert result.exit_code == 0 @mock.patch('gitsome.githubcli.GitHub.licenses') def test_licenses(self, mock_gh_call): result = self.runner.invoke(self.github_cli.cli, ['licenses']) mock_gh_call.assert_called_with() assert result.exit_code == 0 @mock.patch('gitsome.githubcli.GitHub.user_me') def test_me(self, mock_gh_call): result = self.runner.invoke(self.github_cli.cli, ['me', '--browser', '--text_avatar', '--limit', 10, '--pager']) mock_gh_call.assert_called_with(True, True, 10, True) assert result.exit_code == 0 @mock.patch('gitsome.githubcli.GitHub.notifications') def test_notifications(self, mock_gh_call): result = self.runner.invoke(self.github_cli.cli, ['notifications', '--limit', 10, '--pager']) mock_gh_call.assert_called_with(10, True) assert result.exit_code == 0 @mock.patch('gitsome.githubcli.GitHub.octocat') def test_octocat(self, mock_gh_call): result = self.runner.invoke(self.github_cli.cli, ['octo', 'foo']) mock_gh_call.assert_called_with('foo') assert result.exit_code == 0 @mock.patch('gitsome.githubcli.GitHub.issue') def test_pull_request(self, mock_gh_call): result = self.runner.invoke(self.github_cli.cli, ['pull-request', 'u/r/n']) mock_gh_call.assert_called_with('u/r/n') assert result.exit_code == 0 @mock.patch('gitsome.githubcli.GitHub.pull_requests') def test_pull_requests(self, mock_gh_call): result = self.runner.invoke(self.github_cli.cli, ['pull-requests', '--limit', 10, '--pager']) mock_gh_call.assert_called_with(10, True) assert result.exit_code == 0 @mock.patch('gitsome.githubcli.GitHub.rate_limit') def test_rate_limit(self, mock_gh_call): result = self.runner.invoke(self.github_cli.cli, ['rate-limit']) mock_gh_call.assert_called_with() assert result.exit_code == 0 @mock.patch('gitsome.githubcli.GitHub.repository') def test_repository(self, mock_gh_call): result = self.runner.invoke(self.github_cli.cli, ['repo', 'u/r']) mock_gh_call.assert_called_with('u/r') assert result.exit_code == 0 @mock.patch('gitsome.githubcli.GitHub.repositories_setup') def test_repositories(self, mock_gh_call): result = self.runner.invoke(self.github_cli.cli, ['repos', 'foo', '--limit', 10, '--pager']) mock_gh_call.assert_called_with('foo', 10, True) assert result.exit_code == 0 @mock.patch('gitsome.githubcli.GitHub.search_issues') def test_search_issues(self, mock_gh_call): result = self.runner.invoke(self.github_cli.cli, ['search-issues', 'foo', '--limit', 10, '--pager']) mock_gh_call.assert_called_with('foo', 10, True) assert result.exit_code == 0 @mock.patch('gitsome.githubcli.GitHub.search_repositories') def test_search_repositories(self, mock_gh_call): result = self.runner.invoke(self.github_cli.cli, ['search-repos', 'foo', '--sort', 'stars', '--limit', 10, '--pager']) mock_gh_call.assert_called_with('foo', 'stars', 10, True) assert result.exit_code == 0 @mock.patch('gitsome.githubcli.GitHub.starred') def test_starred(self, mock_gh_call): result = self.runner.invoke(self.github_cli.cli, ['starred', 'foo', '--limit', 10, '--pager']) mock_gh_call.assert_called_with('foo', 10, True) assert result.exit_code == 0 @mock.patch('gitsome.githubcli.GitHub.trending') def test_trending(self, mock_gh_call): result = self.runner.invoke(self.github_cli.cli, ['trending', 'l', '--pager']) mock_gh_call.assert_called_with('l', False, False, False, False, True) result = self.runner.invoke(self.github_cli.cli, ['trending', 'l', '--weekly', '--pager']) mock_gh_call.assert_called_with('l', True, False, False, False, True) result = self.runner.invoke(self.github_cli.cli, ['trending', 'l', '--monthly', '--devs', '--browser', '--pager']) mock_gh_call.assert_called_with('l', False, True, True, True, True) assert result.exit_code == 0 @mock.patch('gitsome.githubcli.GitHub.user') def test_user(self, mock_gh_call): result = self.runner.invoke(self.github_cli.cli, ['user', 'foo', '--browser', '--text_avatar', '--limit', 10, '--pager']) mock_gh_call.assert_called_with('foo', True, True, 10, True) assert result.exit_code == 0 @mock.patch('gitsome.githubcli.GitHub.view') def test_view(self, mock_gh_call): result = self.runner.invoke(self.github_cli.cli, ['view', '1', '--browser']) mock_gh_call.assert_called_with(1, True) assert result.exit_code == 0 ================================================ FILE: tests/test_web_viewer.py ================================================ # -*- coding: utf-8 -*- # Copyright 2015 Donne Martin. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of # the License is located at # # http://www.apache.org/licenses/LICENSE-2.0 # # or in the "license" file accompanying this file. This file is # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. from __future__ import unicode_literals from __future__ import print_function import mock from tests.compat import unittest from gitsome.github import GitHub from tests.data.markdown import formatted_markdown, raw_markdown, ssl_error class WebViewerTest(unittest.TestCase): def setUp(self): self.github = GitHub() def test_format_markdown(self): result = self.github.web_viewer.format_markdown(raw_markdown) assert result == formatted_markdown @mock.patch('gitsome.github.click.echo_via_pager') def test_view_url(self, mock_click_echo_via_pager): url = 'https://www.github.com/donnemartin/gitsome' self.github.web_viewer.view_url(url) assert mock_click_echo_via_pager.mock_calls @unittest.skip('Skipping test_view_url_ssl_error') @mock.patch('gitsome.github.click.echo_via_pager') def test_view_url_ssl_error(self, mock_click_echo_via_pager): """Temp skipping this test due to a change [undocumented?] in the way the requests ssl error sample website is handled: http://docs.python-requests.org/en/master/user/advanced/#ssl-cert-verification # NOQA See https://github.com/donnemartin/gitsome/pull/64 for more details. """ url = 'https://requestb.in' self.github.web_viewer.view_url(url) mock_click_echo_via_pager.assert_called_with(ssl_error, None) ================================================ FILE: tox.ini ================================================ # Tox (http://tox.testrun.org/) is a tool for running tests # in multiple virtualenvs. This configuration file will run the # test suite on all supported python versions. To use it, "pip install tox" # and then run "tox" from this directory. [tox] envlist = py35, py36, py37 [testenv] passenv = TRAVIS TRAVIS_JOB_ID TRAVIS_BRANCH deps = coverage mock unittest2 commands = coverage erase coverage run {toxinidir}/tests/run_tests.py coverage report --include={toxinidir}/*gitsome/* --omit={toxinidir}/*gitsome/lib/*,*compat.py -m ================================================ FILE: xonsh/__init__.py ================================================ __version__ = "0.8.10" # amalgamate exclude jupyter_kernel parser_table parser_test_table pyghooks # amalgamate exclude winutils wizard pytest_plugin fs macutils pygments_cache # amalgamate exclude jupyter_shell import os as _os if _os.getenv("XONSH_DEBUG", ""): pass else: import sys as _sys try: from xonsh import __amalgam__ completer = __amalgam__ _sys.modules["xonsh.completer"] = __amalgam__ contexts = __amalgam__ _sys.modules["xonsh.contexts"] = __amalgam__ lazyasd = __amalgam__ _sys.modules["xonsh.lazyasd"] = __amalgam__ lazyjson = __amalgam__ _sys.modules["xonsh.lazyjson"] = __amalgam__ color_tools = __amalgam__ _sys.modules["xonsh.color_tools"] = __amalgam__ platform = __amalgam__ _sys.modules["xonsh.platform"] = __amalgam__ pretty = __amalgam__ _sys.modules["xonsh.pretty"] = __amalgam__ codecache = __amalgam__ _sys.modules["xonsh.codecache"] = __amalgam__ lazyimps = __amalgam__ _sys.modules["xonsh.lazyimps"] = __amalgam__ parser = __amalgam__ _sys.modules["xonsh.parser"] = __amalgam__ tokenize = __amalgam__ _sys.modules["xonsh.tokenize"] = __amalgam__ tools = __amalgam__ _sys.modules["xonsh.tools"] = __amalgam__ ansi_colors = __amalgam__ _sys.modules["xonsh.ansi_colors"] = __amalgam__ ast = __amalgam__ _sys.modules["xonsh.ast"] = __amalgam__ commands_cache = __amalgam__ _sys.modules["xonsh.commands_cache"] = __amalgam__ diff_history = __amalgam__ _sys.modules["xonsh.diff_history"] = __amalgam__ events = __amalgam__ _sys.modules["xonsh.events"] = __amalgam__ foreign_shells = __amalgam__ _sys.modules["xonsh.foreign_shells"] = __amalgam__ jobs = __amalgam__ _sys.modules["xonsh.jobs"] = __amalgam__ jsonutils = __amalgam__ _sys.modules["xonsh.jsonutils"] = __amalgam__ lexer = __amalgam__ _sys.modules["xonsh.lexer"] = __amalgam__ openpy = __amalgam__ _sys.modules["xonsh.openpy"] = __amalgam__ style_tools = __amalgam__ _sys.modules["xonsh.style_tools"] = __amalgam__ xontribs = __amalgam__ _sys.modules["xonsh.xontribs"] = __amalgam__ dirstack = __amalgam__ _sys.modules["xonsh.dirstack"] = __amalgam__ inspectors = __amalgam__ _sys.modules["xonsh.inspectors"] = __amalgam__ proc = __amalgam__ _sys.modules["xonsh.proc"] = __amalgam__ shell = __amalgam__ _sys.modules["xonsh.shell"] = __amalgam__ timings = __amalgam__ _sys.modules["xonsh.timings"] = __amalgam__ xonfig = __amalgam__ _sys.modules["xonsh.xonfig"] = __amalgam__ base_shell = __amalgam__ _sys.modules["xonsh.base_shell"] = __amalgam__ environ = __amalgam__ _sys.modules["xonsh.environ"] = __amalgam__ tracer = __amalgam__ _sys.modules["xonsh.tracer"] = __amalgam__ readline_shell = __amalgam__ _sys.modules["xonsh.readline_shell"] = __amalgam__ replay = __amalgam__ _sys.modules["xonsh.replay"] = __amalgam__ aliases = __amalgam__ _sys.modules["xonsh.aliases"] = __amalgam__ dumb_shell = __amalgam__ _sys.modules["xonsh.dumb_shell"] = __amalgam__ built_ins = __amalgam__ _sys.modules["xonsh.built_ins"] = __amalgam__ execer = __amalgam__ _sys.modules["xonsh.execer"] = __amalgam__ imphooks = __amalgam__ _sys.modules["xonsh.imphooks"] = __amalgam__ main = __amalgam__ _sys.modules["xonsh.main"] = __amalgam__ del __amalgam__ except ImportError: pass del _sys del _os # amalgamate end ================================================ FILE: xonsh/__main__.py ================================================ from xonsh.main import main main() ================================================ FILE: xonsh/aliases.py ================================================ # -*- coding: utf-8 -*- """Aliases for the xonsh shell.""" import os import re import sys import inspect import argparse import builtins import collections.abc as cabc from xonsh.lazyasd import lazyobject from xonsh.dirstack import cd, pushd, popd, dirs, _get_cwd from xonsh.environ import locate_binary, make_args_env from xonsh.foreign_shells import foreign_shell_data from xonsh.jobs import jobs, fg, bg, clean_jobs from xonsh.platform import ( ON_ANACONDA, ON_DARWIN, ON_WINDOWS, ON_FREEBSD, ON_NETBSD, ON_DRAGONFLY, ) from xonsh.tools import ( XonshError, argvquote, escape_windows_cmd_string, to_bool, swap_values, strip_simple_quotes, ALIAS_KWARG_NAMES, unthreadable, print_color, ) from xonsh.replay import replay_main from xonsh.timings import timeit_alias from xonsh.xontribs import xontribs_main from xonsh.ast import isexpression import xonsh.completers._aliases as xca import xonsh.history.main as xhm import xonsh.xoreutils.which as xxw @lazyobject def SUB_EXEC_ALIAS_RE(): return re.compile(r"@\(|\$\(|!\(|\$\[|!\[") class Aliases(cabc.MutableMapping): """Represents a location to hold and look up aliases.""" def __init__(self, *args, **kwargs): self._raw = {} self.update(*args, **kwargs) def get(self, key, default=None): """Returns the (possibly modified) value. If the key is not present, then `default` is returned. If the value is callable, it is returned without modification. If it is an iterable of strings it will be evaluated recursively to expand other aliases, resulting in a new list or a "partially applied" callable. """ val = self._raw.get(key) if val is None: return default elif isinstance(val, cabc.Iterable) or callable(val): return self.eval_alias(val, seen_tokens={key}) else: msg = "alias of {!r} has an inappropriate type: {!r}" raise TypeError(msg.format(key, val)) def eval_alias(self, value, seen_tokens=frozenset(), acc_args=()): """ "Evaluates" the alias ``value``, by recursively looking up the leftmost token and "expanding" if it's also an alias. A value like ``["cmd", "arg"]`` might transform like this: ``> ["cmd", "arg"] -> ["ls", "-al", "arg"] -> callable()`` where ``cmd=ls -al`` and ``ls`` is an alias with its value being a callable. The resulting callable will be "partially applied" with ``["-al", "arg"]``. """ # Beware of mutability: default values for keyword args are evaluated # only once. if callable(value): return partial_eval_alias(value, acc_args=acc_args) else: expand_path = builtins.__xonsh__.expand_path token, *rest = map(expand_path, value) if token in seen_tokens or token not in self._raw: # ^ Making sure things like `egrep=egrep --color=auto` works, # and that `l` evals to `ls --color=auto -CF` if `l=ls -CF` # and `ls=ls --color=auto` rtn = [token] rtn.extend(rest) rtn.extend(acc_args) return rtn else: seen_tokens = seen_tokens | {token} acc_args = rest + list(acc_args) return self.eval_alias(self._raw[token], seen_tokens, acc_args) def expand_alias(self, line): """Expands any aliases present in line if alias does not point to a builtin function and if alias is only a single command. """ word = line.split(" ", 1)[0] if word in builtins.aliases and isinstance(self.get(word), cabc.Sequence): word_idx = line.find(word) expansion = " ".join(self.get(word)) line = line[:word_idx] + expansion + line[word_idx + len(word) :] return line # # Mutable mapping interface # def __getitem__(self, key): return self._raw[key] def __setitem__(self, key, val): if isinstance(val, str): f = "" if SUB_EXEC_ALIAS_RE.search(val) is not None: # We have a sub-command, e.g. $(cmd), to evaluate self._raw[key] = ExecAlias(val, filename=f) elif isexpression(val): # expansion substitution lexer = builtins.__xonsh__.execer.parser.lexer self._raw[key] = list(map(strip_simple_quotes, lexer.split(val))) else: # need to exec alias self._raw[key] = ExecAlias(val, filename=f) else: self._raw[key] = val def __delitem__(self, key): del self._raw[key] def update(self, *args, **kwargs): for key, val in dict(*args, **kwargs).items(): self[key] = val def __iter__(self): yield from self._raw def __len__(self): return len(self._raw) def __str__(self): return str(self._raw) def __repr__(self): return "{0}.{1}({2})".format( self.__class__.__module__, self.__class__.__name__, self._raw ) def _repr_pretty_(self, p, cycle): name = "{0}.{1}".format(self.__class__.__module__, self.__class__.__name__) with p.group(0, name + "(", ")"): if cycle: p.text("...") elif len(self): p.break_() p.pretty(dict(self)) class ExecAlias: """Provides a callable alias for xonsh source code.""" def __init__(self, src, filename=""): """ Parameters ---------- src : str Source code that will be """ self.src = src if src.endswith("\n") else src + "\n" self.filename = filename def __call__( self, args, stdin=None, stdout=None, stderr=None, spec=None, stack=None ): execer = builtins.__xonsh__.execer frame = stack[0][0] # execute as though we are at the call site execer.exec( self.src, glbs=frame.f_globals, locs=frame.f_locals, filename=self.filename ) def __repr__(self): return "ExecAlias({0!r}, filename={1!r})".format(self.src, self.filename) class PartialEvalAliasBase: """Partially evaluated alias.""" def __init__(self, f, acc_args=()): """ Parameters ---------- f : callable A function to dispatch to. acc_args : sequence of strings, optional Additional arguments to prepent to the argument list passed in when the alias is called. """ self.f = f self.acc_args = acc_args self.__name__ = getattr(f, "__name__", self.__class__.__name__) def __call__( self, args, stdin=None, stdout=None, stderr=None, spec=None, stack=None ): args = list(self.acc_args) + args return self.f(args, stdin, stdout, stderr, spec, stack) def __repr__(self): return "{name}({f!r}, acc_args={acc_args!r})".format( name=self.__class__.__name__, f=self.f, acc_args=self.acc_args ) class PartialEvalAlias0(PartialEvalAliasBase): def __call__( self, args, stdin=None, stdout=None, stderr=None, spec=None, stack=None ): args = list(self.acc_args) + args if args: msg = "callable alias {f!r} takes no arguments, but {args!f} provided. " msg += "Of these {acc_args!r} were partially applied." raise XonshError(msg.format(f=self.f, args=args, acc_args=self.acc_args)) return self.f() class PartialEvalAlias1(PartialEvalAliasBase): def __call__( self, args, stdin=None, stdout=None, stderr=None, spec=None, stack=None ): args = list(self.acc_args) + args return self.f(args) class PartialEvalAlias2(PartialEvalAliasBase): def __call__( self, args, stdin=None, stdout=None, stderr=None, spec=None, stack=None ): args = list(self.acc_args) + args return self.f(args, stdin) class PartialEvalAlias3(PartialEvalAliasBase): def __call__( self, args, stdin=None, stdout=None, stderr=None, spec=None, stack=None ): args = list(self.acc_args) + args return self.f(args, stdin, stdout) class PartialEvalAlias4(PartialEvalAliasBase): def __call__( self, args, stdin=None, stdout=None, stderr=None, spec=None, stack=None ): args = list(self.acc_args) + args return self.f(args, stdin, stdout, stderr) class PartialEvalAlias5(PartialEvalAliasBase): def __call__( self, args, stdin=None, stdout=None, stderr=None, spec=None, stack=None ): args = list(self.acc_args) + args return self.f(args, stdin, stdout, stderr, spec) class PartialEvalAlias6(PartialEvalAliasBase): def __call__( self, args, stdin=None, stdout=None, stderr=None, spec=None, stack=None ): args = list(self.acc_args) + args return self.f(args, stdin, stdout, stderr, spec, stack) PARTIAL_EVAL_ALIASES = ( PartialEvalAlias0, PartialEvalAlias1, PartialEvalAlias2, PartialEvalAlias3, PartialEvalAlias4, PartialEvalAlias5, PartialEvalAlias6, ) def partial_eval_alias(f, acc_args=()): """Dispatches the appropriate eval alias based on the number of args to the original callable alias and how many arguments to apply. """ # no partial needed if no extra args if not acc_args: return f # need to dispatch numargs = 0 for name, param in inspect.signature(f).parameters.items(): if ( param.kind == param.POSITIONAL_ONLY or param.kind == param.POSITIONAL_OR_KEYWORD ): numargs += 1 elif name in ALIAS_KWARG_NAMES and param.kind == param.KEYWORD_ONLY: numargs += 1 if numargs < 7: return PARTIAL_EVAL_ALIASES[numargs](f, acc_args=acc_args) else: e = "Expected proxy with 6 or fewer arguments for {}, not {}" raise XonshError(e.format(", ".join(ALIAS_KWARG_NAMES), numargs)) # # Actual aliases below # def xonsh_exit(args, stdin=None): """Sends signal to exit shell.""" if not clean_jobs(): # Do not exit if jobs not cleaned up return None, None builtins.__xonsh__.exit = True print() # gimme a newline return None, None def xonsh_reset(args, stdin=None): """ Clears __xonsh__.ctx""" builtins.__xonsh__.ctx.clear() @lazyobject def _SOURCE_FOREIGN_PARSER(): desc = "Sources a file written in a foreign shell language." parser = argparse.ArgumentParser("source-foreign", description=desc) parser.add_argument("shell", help="Name or path to the foreign shell") parser.add_argument( "files_or_code", nargs="+", help="file paths to source or code in the target " "language.", ) parser.add_argument( "-i", "--interactive", type=to_bool, default=True, help="whether the sourced shell should be interactive", dest="interactive", ) parser.add_argument( "-l", "--login", type=to_bool, default=False, help="whether the sourced shell should be login", dest="login", ) parser.add_argument( "--envcmd", default=None, dest="envcmd", help="command to print environment" ) parser.add_argument( "--aliascmd", default=None, dest="aliascmd", help="command to print aliases" ) parser.add_argument( "--extra-args", default=(), dest="extra_args", type=(lambda s: tuple(s.split())), help="extra arguments needed to run the shell", ) parser.add_argument( "-s", "--safe", type=to_bool, default=True, help="whether the source shell should be run safely, " "and not raise any errors, even if they occur.", dest="safe", ) parser.add_argument( "-p", "--prevcmd", default=None, dest="prevcmd", help="command(s) to run before any other commands, " "replaces traditional source.", ) parser.add_argument( "--postcmd", default="", dest="postcmd", help="command(s) to run after all other commands", ) parser.add_argument( "--funcscmd", default=None, dest="funcscmd", help="code to find locations of all native functions " "in the shell language.", ) parser.add_argument( "--sourcer", default=None, dest="sourcer", help="the source command in the target shell " "language, default: source.", ) parser.add_argument( "--use-tmpfile", type=to_bool, default=False, help="whether the commands for source shell should be " "written to a temporary file.", dest="use_tmpfile", ) parser.add_argument( "--seterrprevcmd", default=None, dest="seterrprevcmd", help="command(s) to set exit-on-error before any" "other commands.", ) parser.add_argument( "--seterrpostcmd", default=None, dest="seterrpostcmd", help="command(s) to set exit-on-error after all" "other commands.", ) parser.add_argument( "--overwrite-aliases", default=False, action="store_true", dest="overwrite_aliases", help="flag for whether or not sourced aliases should " "replace the current xonsh aliases.", ) parser.add_argument( "--suppress-skip-message", default=None, action="store_true", dest="suppress_skip_message", help="flag for whether or not skip messages should be suppressed.", ) parser.add_argument( "--show", default=False, action="store_true", dest="show", help="Will show the script output.", ) parser.add_argument( "-d", "--dry-run", default=False, action="store_true", dest="dryrun", help="Will not actually source the file.", ) return parser def source_foreign(args, stdin=None, stdout=None, stderr=None): """Sources a file written in a foreign shell language.""" env = builtins.__xonsh__.env ns = _SOURCE_FOREIGN_PARSER.parse_args(args) ns.suppress_skip_message = ( env.get("FOREIGN_ALIASES_SUPPRESS_SKIP_MESSAGE") if ns.suppress_skip_message is None else ns.suppress_skip_message ) if ns.prevcmd is not None: pass # don't change prevcmd if given explicitly elif os.path.isfile(ns.files_or_code[0]): # we have filename to source ns.prevcmd = '{} "{}"'.format(ns.sourcer, '" "'.join(ns.files_or_code)) elif ns.prevcmd is None: ns.prevcmd = " ".join(ns.files_or_code) # code to run, no files foreign_shell_data.cache_clear() # make sure that we don't get prev src fsenv, fsaliases = foreign_shell_data( shell=ns.shell, login=ns.login, interactive=ns.interactive, envcmd=ns.envcmd, aliascmd=ns.aliascmd, extra_args=ns.extra_args, safe=ns.safe, prevcmd=ns.prevcmd, postcmd=ns.postcmd, funcscmd=ns.funcscmd, sourcer=ns.sourcer, use_tmpfile=ns.use_tmpfile, seterrprevcmd=ns.seterrprevcmd, seterrpostcmd=ns.seterrpostcmd, show=ns.show, dryrun=ns.dryrun, ) if fsenv is None: if ns.dryrun: return else: msg = "xonsh: error: Source failed: {0!r}\n".format(ns.prevcmd) msg += "xonsh: error: Possible reasons: File not found or syntax error\n" return (None, msg, 1) # apply results denv = env.detype() for k, v in fsenv.items(): if k in denv and v == denv[k]: continue # no change from original env[k] = v # Remove any env-vars that were unset by the script. for k in denv: if k not in fsenv: env.pop(k, None) # Update aliases baliases = builtins.aliases for k, v in fsaliases.items(): if k in baliases and v == baliases[k]: continue # no change from original elif ns.overwrite_aliases or k not in baliases: baliases[k] = v elif ns.suppress_skip_message: pass else: msg = ( "Skipping application of {0!r} alias from {1!r} " "since it shares a name with an existing xonsh alias. " 'Use "--overwrite-alias" option to apply it anyway.' 'You may prevent this message with "--suppress-skip-message" or ' '"$FOREIGN_ALIASES_SUPPRESS_SKIP_MESSAGE = True".' ) print(msg.format(k, ns.shell), file=stderr) def source_alias(args, stdin=None): """Executes the contents of the provided files in the current context. If sourced file isn't found in cwd, search for file along $PATH to source instead. """ env = builtins.__xonsh__.env encoding = env.get("XONSH_ENCODING") errors = env.get("XONSH_ENCODING_ERRORS") for i, fname in enumerate(args): fpath = fname if not os.path.isfile(fpath): fpath = locate_binary(fname) if fpath is None: if env.get("XONSH_DEBUG"): print("source: {}: No such file".format(fname), file=sys.stderr) if i == 0: raise RuntimeError( "must source at least one file, " + fname + "does not exist." ) break _, fext = os.path.splitext(fpath) if fext and fext != ".xsh" and fext != ".py": raise RuntimeError( "attempting to source non-xonsh file! If you are " "trying to source a file in another language, " "then please use the appropriate source command. " "For example, source-bash script.sh" ) with open(fpath, "r", encoding=encoding, errors=errors) as fp: src = fp.read() if not src.endswith("\n"): src += "\n" ctx = builtins.__xonsh__.ctx updates = {"__file__": fpath, "__name__": os.path.abspath(fpath)} with env.swap(**make_args_env(args[i + 1 :])), swap_values(ctx, updates): try: builtins.execx(src, "exec", ctx, filename=fpath) except Exception: print_color( "{RED}You may be attempting to source non-xonsh file! " "{NO_COLOR}If you are trying to source a file in " "another language, then please use the appropriate " "source command. For example, {GREEN}source-bash " "script.sh{NO_COLOR}", file=sys.stderr, ) raise def source_cmd(args, stdin=None): """Simple cmd.exe-specific wrapper around source-foreign.""" args = list(args) fpath = locate_binary(args[0]) args[0] = fpath if fpath else args[0] if not os.path.isfile(args[0]): return (None, "xonsh: error: File not found: {}\n".format(args[0]), 1) prevcmd = "call " prevcmd += " ".join([argvquote(arg, force=True) for arg in args]) prevcmd = escape_windows_cmd_string(prevcmd) args.append("--prevcmd={}".format(prevcmd)) args.insert(0, "cmd") args.append("--interactive=0") args.append("--sourcer=call") args.append("--envcmd=set") args.append("--seterrpostcmd=if errorlevel 1 exit 1") args.append("--use-tmpfile=1") with builtins.__xonsh__.env.swap(PROMPT="$P$G"): return source_foreign(args, stdin=stdin) def xexec(args, stdin=None): """exec [-h|--help] command [args...] exec (also aliased as xexec) uses the os.execvpe() function to replace the xonsh process with the specified program. This provides the functionality of the bash 'exec' builtin:: >>> exec bash -l -i bash $ The '-h' and '--help' options print this message and exit. Notes ----- This command **is not** the same as the Python builtin function exec(). That function is for running Python code. This command, which shares the same name as the sh-lang statement, is for launching a command directly in the same process. In the event of a name conflict, please use the xexec command directly or dive into subprocess mode explicitly with ![exec command]. For more details, please see http://xon.sh/faq.html#exec. """ if len(args) == 0: return (None, "xonsh: exec: no args specified\n", 1) elif args[0] == "-h" or args[0] == "--help": return inspect.getdoc(xexec) else: denv = builtins.__xonsh__.env.detype() try: os.execvpe(args[0], args, denv) except FileNotFoundError as e: return ( None, "xonsh: exec: file not found: {}: {}" "\n".format(e.args[1], args[0]), 1, ) class AWitchAWitch(argparse.Action): SUPPRESS = "==SUPPRESS==" def __init__( self, option_strings, version=None, dest=SUPPRESS, default=SUPPRESS, **kwargs ): super().__init__( option_strings=option_strings, dest=dest, default=default, nargs=0, **kwargs ) def __call__(self, parser, namespace, values, option_string=None): import webbrowser webbrowser.open("https://github.com/xonsh/xonsh/commit/f49b400") parser.exit() def xonfig(args, stdin=None): """Runs the xonsh configuration utility.""" from xonsh.xonfig import xonfig_main # lazy import return xonfig_main(args) @unthreadable def trace(args, stdin=None, stdout=None, stderr=None, spec=None): """Runs the xonsh tracer utility.""" from xonsh.tracer import tracermain # lazy import try: return tracermain(args, stdin=stdin, stdout=stdout, stderr=stderr, spec=spec) except SystemExit: pass def showcmd(args, stdin=None): """usage: showcmd [-h|--help|cmd args] Displays the command and arguments as a list of strings that xonsh would run in subprocess mode. This is useful for determining how xonsh evaluates your commands and arguments prior to running these commands. optional arguments: -h, --help show this help message and exit example: >>> showcmd echo $USER can't hear "the sea" ['echo', 'I', "can't", 'hear', 'the sea'] """ if len(args) == 0 or (len(args) == 1 and args[0] in {"-h", "--help"}): print(showcmd.__doc__.rstrip().replace("\n ", "\n")) else: sys.displayhook(args) def detect_xpip_alias(): """ Determines the correct invocation to get xonsh's pip """ if not getattr(sys, "executable", None): return lambda args, stdin=None: ( "", "Sorry, unable to run pip on your system (missing sys.executable)", 1, ) basecmd = [sys.executable, "-m", "pip"] try: if ON_WINDOWS: # XXX: Does windows have an installation mode that requires UAC? return basecmd elif not os.access(os.path.dirname(sys.executable), os.W_OK): return ["sudo"] + basecmd else: return basecmd except Exception: # Something freaky happened, return something that'll probably work return basecmd def make_default_aliases(): """Creates a new default aliases dictionary.""" default_aliases = { "cd": cd, "pushd": pushd, "popd": popd, "dirs": dirs, "jobs": jobs, "fg": fg, "bg": bg, "EOF": xonsh_exit, "exit": xonsh_exit, "quit": xonsh_exit, "exec": xexec, "xexec": xexec, "source": source_alias, "source-zsh": ["source-foreign", "zsh", "--sourcer=source"], "source-bash": ["source-foreign", "bash", "--sourcer=source"], "source-cmd": source_cmd, "source-foreign": source_foreign, "history": xhm.history_main, "replay": replay_main, "trace": trace, "timeit": timeit_alias, "xonfig": xonfig, "scp-resume": ["rsync", "--partial", "-h", "--progress", "--rsh=ssh"], "showcmd": showcmd, "ipynb": ["jupyter", "notebook", "--no-browser"], "which": xxw.which, "xontrib": xontribs_main, "completer": xca.completer_alias, "xpip": detect_xpip_alias(), "xonsh-reset": xonsh_reset, } if ON_WINDOWS: # Borrow builtin commands from cmd.exe. windows_cmd_aliases = { "cls", "copy", "del", "dir", "echo", "erase", "md", "mkdir", "mklink", "move", "rd", "ren", "rename", "rmdir", "time", "type", "vol", } for alias in windows_cmd_aliases: default_aliases[alias] = ["cmd", "/c", alias] default_aliases["call"] = ["source-cmd"] default_aliases["source-bat"] = ["source-cmd"] default_aliases["clear"] = "cls" if ON_ANACONDA: # Add aliases specific to the Anaconda python distribution. default_aliases["activate"] = ["source-cmd", "activate.bat"] default_aliases["deactivate"] = ["source-cmd", "deactivate.bat"] if not locate_binary("sudo"): import xonsh.winutils as winutils def sudo(args): if len(args) < 1: print( "You need to provide an executable to run as " "Administrator." ) return cmd = args[0] if locate_binary(cmd): return winutils.sudo(cmd, args[1:]) elif cmd.lower() in windows_cmd_aliases: args = ["/D", "/C", "CD", _get_cwd(), "&&"] + args return winutils.sudo("cmd", args) else: msg = 'Cannot find the path for executable "{0}".' print(msg.format(cmd)) default_aliases["sudo"] = sudo elif ON_DARWIN: default_aliases["ls"] = ["ls", "-G"] elif ON_FREEBSD or ON_DRAGONFLY: default_aliases["grep"] = ["grep", "--color=auto"] default_aliases["egrep"] = ["egrep", "--color=auto"] default_aliases["fgrep"] = ["fgrep", "--color=auto"] default_aliases["ls"] = ["ls", "-G"] elif ON_NETBSD: default_aliases["grep"] = ["grep", "--color=auto"] default_aliases["egrep"] = ["egrep", "--color=auto"] default_aliases["fgrep"] = ["fgrep", "--color=auto"] else: default_aliases["grep"] = ["grep", "--color=auto"] default_aliases["egrep"] = ["egrep", "--color=auto"] default_aliases["fgrep"] = ["fgrep", "--color=auto"] default_aliases["ls"] = ["ls", "--color=auto", "-v"] return default_aliases ================================================ FILE: xonsh/ansi_colors.py ================================================ """Tools for helping with ANSI color codes.""" import re import sys import warnings import builtins from xonsh.platform import HAS_PYGMENTS from xonsh.lazyasd import LazyDict, lazyobject from xonsh.color_tools import ( RE_BACKGROUND, BASE_XONSH_COLORS, make_palette, find_closest_color, rgb2short, rgb_to_256, short_to_ints, ) from xonsh.tools import FORMATTER def ansi_partial_color_format(template, style="default", cmap=None, hide=False): """Formats a template string but only with respect to the colors. Another template string is returned, with the color values filled in. Parameters ---------- template : str The template string, potentially with color names. style : str, optional Style name to look up color map from. cmap : dict, optional A color map to use, this will prevent the color map from being looked up via the style name. hide : bool, optional Whether to wrap the color codes in the \\001 and \\002 escape codes, so that the color codes are not counted against line length. Returns ------- A template string with the color values filled in. """ try: return _ansi_partial_color_format_main( template, style=style, cmap=cmap, hide=hide ) except Exception: return template def _ansi_partial_color_format_main(template, style="default", cmap=None, hide=False): if cmap is not None: pass elif style in ANSI_STYLES: cmap = ANSI_STYLES[style] else: try: # dynamically loading the style cmap = ansi_style_by_name(style) except Exception: msg = "Could not find color style {0!r}, using default." print(msg.format(style), file=sys.stderr) builtins.__xonsh__.env["XONSH_COLOR_STYLE"] = "default" cmap = ANSI_STYLES["default"] esc = ("\001" if hide else "") + "\033[" m = "m" + ("\002" if hide else "") bopen = "{" bclose = "}" colon = ":" expl = "!" toks = [] for literal, field, spec, conv in FORMATTER.parse(template): toks.append(literal) if field is None: pass elif field in cmap: toks.extend([esc, cmap[field], m]) elif "#" in field: field = field.lower() pre, _, post = field.partition("#") f_or_b = "38" if RE_BACKGROUND.search(pre) is None else "48" rgb, _, post = post.partition("_") c256, _ = rgb_to_256(rgb) color = f_or_b + ";5;" + c256 mods = pre + "_" + post if "underline" in mods: color = "4;" + color if "bold" in mods: color = "1;" + color toks.extend([esc, color, m]) elif field is not None: toks.append(bopen) toks.append(field) if conv is not None and len(conv) > 0: toks.append(expl) toks.append(conv) if spec is not None and len(spec) > 0: toks.append(colon) toks.append(spec) toks.append(bclose) return "".join(toks) def ansi_color_style_names(): """Returns an iterable of all ANSI color style names.""" return ANSI_STYLES.keys() def ansi_color_style(style="default"): """Returns the current color map.""" if style in ANSI_STYLES: cmap = ANSI_STYLES[style] else: msg = "Could not find color style {0!r}, using default.".format(style) warnings.warn(msg, RuntimeWarning) cmap = ANSI_STYLES["default"] return cmap def ansi_reverse_style(style="default", return_style=False): """Reverses an ANSI color style mapping so that escape codes map to colors. Style may either be string or mapping. May also return the style it looked up. """ style = ansi_style_by_name(style) if isinstance(style, str) else style reversed_style = {v: k for k, v in style.items()} # add keys to make this more useful updates = { "1": "BOLD_", "2": "FAINT_", "4": "UNDERLINE_", "5": "SLOWBLINK_", "1;4": "BOLD_UNDERLINE_", "4;1": "BOLD_UNDERLINE_", "38": "SET_FOREGROUND_", "48": "SET_BACKGROUND_", "38;2": "SET_FOREGROUND_3INTS_", "48;2": "SET_BACKGROUND_3INTS_", "38;5": "SET_FOREGROUND_SHORT_", "48;5": "SET_BACKGROUND_SHORT_", } for ec, name in reversed_style.items(): no_left_zero = ec.lstrip("0") if no_left_zero.startswith(";"): updates[no_left_zero[1:]] = name elif no_left_zero != ec: updates[no_left_zero] = name reversed_style.update(updates) # return results if return_style: return style, reversed_style else: return reversed_style @lazyobject def ANSI_ESCAPE_CODE_RE(): return re.compile(r"\001?(\033\[)?([0-9;]+)m?\002?") @lazyobject def ANSI_REVERSE_COLOR_NAME_TRANSLATIONS(): base = { "SET_FOREGROUND_FAINT_": "SET_FOREGROUND_3INTS_", "SET_BACKGROUND_FAINT_": "SET_BACKGROUND_3INTS_", "SET_FOREGROUND_SLOWBLINK_": "SET_FOREGROUND_SHORT_", "SET_BACKGROUND_SLOWBLINK_": "SET_BACKGROUND_SHORT_", } data = {"UNDERLINE_BOLD_": "BOLD_UNDERLINE_"} data.update(base) data.update({"BOLD_" + k: "BOLD_" + v for k, v in base.items()}) data.update({"UNDERLINE_" + k: "UNDERLINE_" + v for k, v in base.items()}) data.update({"BOLD_UNDERLINE_" + k: "BOLD_UNDERLINE_" + v for k, v in base.items()}) data.update({"UNDERLINE_BOLD_" + k: "BOLD_UNDERLINE_" + v for k, v in base.items()}) return data @lazyobject def ANSI_COLOR_NAME_SET_3INTS_RE(): return re.compile(r"(\w+_)?SET_(FORE|BACK)GROUND_3INTS_(\d+)_(\d+)_(\d+)") @lazyobject def ANSI_COLOR_NAME_SET_SHORT_RE(): return re.compile(r"(\w+_)?SET_(FORE|BACK)GROUND_SHORT_(\d+)") def _color_name_from_ints(ints, background=False, prefix=None): name = find_closest_color(ints, BASE_XONSH_COLORS) if background: name = "BACKGROUND_" + name name = name if prefix is None else prefix + name return name _ANSI_COLOR_ESCAPE_CODE_TO_NAME_CACHE = {} def ansi_color_escape_code_to_name(escape_code, style, reversed_style=None): """Converts an ASNI color code escape sequence to a tuple of color names in the provided style ('default' should almost be the style). For example, '0' becomes ('NO_COLOR',) and '32;41' becomes ('GREEN', 'BACKGROUND_RED'). The style keyword may either be a string, in which the style is looked up, or an actual style dict. You can also provide a reversed style mapping, too, which is just the keys/values of the style dict swapped. If reversed style is not provided, it is computed. """ key = (escape_code, style) if key in _ANSI_COLOR_ESCAPE_CODE_TO_NAME_CACHE: return _ANSI_COLOR_ESCAPE_CODE_TO_NAME_CACHE[key] if reversed_style is None: style, reversed_style = ansi_reverse_style(style, return_style=True) # strip some actual escape codes, if needed. ec = ANSI_ESCAPE_CODE_RE.match(escape_code).group(2) names = [] n_ints = 0 seen_set_foreback = False for e in ec.split(";"): no_left_zero = e.lstrip("0") if len(e) > 1 else e if seen_set_foreback and n_ints > 0: names.append(e) n_ints -= 1 if n_ints == 0: seen_set_foreback = False continue else: names.append(reversed_style.get(no_left_zero, no_left_zero)) # set the flags for next time if "38" == e or "48" == e: seen_set_foreback = True elif "2" == e: n_ints = 3 elif "5" == e: n_ints = 1 # normalize names n = "" norm_names = [] colors = set(reversed_style.values()) for name in names: if name == "NO_COLOR": # skip most '0' entries continue n = n + name if n else name n = ANSI_REVERSE_COLOR_NAME_TRANSLATIONS.get(n, n) if n.endswith("_"): continue elif ANSI_COLOR_NAME_SET_SHORT_RE.match(n) is not None: pre, fore_back, short = ANSI_COLOR_NAME_SET_SHORT_RE.match(n).groups() n = _color_name_from_ints( short_to_ints(short), background=(fore_back == "BACK"), prefix=pre ) elif ANSI_COLOR_NAME_SET_3INTS_RE.match(n) is not None: pre, fore_back, r, g, b = ANSI_COLOR_NAME_SET_3INTS_RE.match(n).groups() n = _color_name_from_ints( (int(r), int(g), int(b)), background=(fore_back == "BACK"), prefix=pre ) elif "GROUND_3INTS_" in n: # have 1 or 2, but not 3 ints n += "_" continue # error check if n not in colors: msg = ( "Could not translate ANSI color code {escape_code!r} " "into a known color in the palette. Specifically, the {n!r} " "portion of {name!r} in {names!r} seems to missing." ) raise ValueError( msg.format(escape_code=escape_code, names=names, name=name, n=n) ) norm_names.append(n) n = "" # return if len(norm_names) == 0: return ("NO_COLOR",) else: return tuple(norm_names) def _ansi_expand_style(cmap): """Expands a style in order to more quickly make color map changes.""" for key, val in list(cmap.items()): if key == "NO_COLOR": continue elif len(val) == 0: cmap["BOLD_" + key] = "1" cmap["UNDERLINE_" + key] = "4" cmap["BOLD_UNDERLINE_" + key] = "1;4" cmap["BACKGROUND_" + key] = val else: cmap["BOLD_" + key] = "1;" + val cmap["UNDERLINE_" + key] = "4;" + val cmap["BOLD_UNDERLINE_" + key] = "1;4;" + val cmap["BACKGROUND_" + key] = val.replace("38", "48", 1) def _bw_style(): style = { "BLACK": "0;30", "BLUE": "0;37", "CYAN": "0;37", "GREEN": "0;37", "INTENSE_BLACK": "0;90", "INTENSE_BLUE": "0;97", "INTENSE_CYAN": "0;97", "INTENSE_GREEN": "0;97", "INTENSE_PURPLE": "0;97", "INTENSE_RED": "0;97", "INTENSE_WHITE": "0;97", "INTENSE_YELLOW": "0;97", "NO_COLOR": "0", "PURPLE": "0;37", "RED": "0;37", "WHITE": "0;37", "YELLOW": "0;37", } _ansi_expand_style(style) return style def _default_style(): style = { # Reset "NO_COLOR": "0", # Text Reset # Regular Colors "BLACK": "0;30", # BLACK "RED": "0;31", # RED "GREEN": "0;32", # GREEN "YELLOW": "0;33", # YELLOW "BLUE": "0;34", # BLUE "PURPLE": "0;35", # PURPLE "CYAN": "0;36", # CYAN "WHITE": "0;37", # WHITE # Bold "BOLD_BLACK": "1;30", # BLACK "BOLD_RED": "1;31", # RED "BOLD_GREEN": "1;32", # GREEN "BOLD_YELLOW": "1;33", # YELLOW "BOLD_BLUE": "1;34", # BLUE "BOLD_PURPLE": "1;35", # PURPLE "BOLD_CYAN": "1;36", # CYAN "BOLD_WHITE": "1;37", # WHITE # Underline "UNDERLINE_BLACK": "4;30", # BLACK "UNDERLINE_RED": "4;31", # RED "UNDERLINE_GREEN": "4;32", # GREEN "UNDERLINE_YELLOW": "4;33", # YELLOW "UNDERLINE_BLUE": "4;34", # BLUE "UNDERLINE_PURPLE": "4;35", # PURPLE "UNDERLINE_CYAN": "4;36", # CYAN "UNDERLINE_WHITE": "4;37", # WHITE # Bold, Underline "BOLD_UNDERLINE_BLACK": "1;4;30", # BLACK "BOLD_UNDERLINE_RED": "1;4;31", # RED "BOLD_UNDERLINE_GREEN": "1;4;32", # GREEN "BOLD_UNDERLINE_YELLOW": "1;4;33", # YELLOW "BOLD_UNDERLINE_BLUE": "1;4;34", # BLUE "BOLD_UNDERLINE_PURPLE": "1;4;35", # PURPLE "BOLD_UNDERLINE_CYAN": "1;4;36", # CYAN "BOLD_UNDERLINE_WHITE": "1;4;37", # WHITE # Background "BACKGROUND_BLACK": "40", # BLACK "BACKGROUND_RED": "41", # RED "BACKGROUND_GREEN": "42", # GREEN "BACKGROUND_YELLOW": "43", # YELLOW "BACKGROUND_BLUE": "44", # BLUE "BACKGROUND_PURPLE": "45", # PURPLE "BACKGROUND_CYAN": "46", # CYAN "BACKGROUND_WHITE": "47", # WHITE # High Intensity "INTENSE_BLACK": "0;90", # BLACK "INTENSE_RED": "0;91", # RED "INTENSE_GREEN": "0;92", # GREEN "INTENSE_YELLOW": "0;93", # YELLOW "INTENSE_BLUE": "0;94", # BLUE "INTENSE_PURPLE": "0;95", # PURPLE "INTENSE_CYAN": "0;96", # CYAN "INTENSE_WHITE": "0;97", # WHITE # Bold High Intensity "BOLD_INTENSE_BLACK": "1;90", # BLACK "BOLD_INTENSE_RED": "1;91", # RED "BOLD_INTENSE_GREEN": "1;92", # GREEN "BOLD_INTENSE_YELLOW": "1;93", # YELLOW "BOLD_INTENSE_BLUE": "1;94", # BLUE "BOLD_INTENSE_PURPLE": "1;95", # PURPLE "BOLD_INTENSE_CYAN": "1;96", # CYAN "BOLD_INTENSE_WHITE": "1;97", # WHITE # Underline High Intensity "UNDERLINE_INTENSE_BLACK": "4;90", # BLACK "UNDERLINE_INTENSE_RED": "4;91", # RED "UNDERLINE_INTENSE_GREEN": "4;92", # GREEN "UNDERLINE_INTENSE_YELLOW": "4;93", # YELLOW "UNDERLINE_INTENSE_BLUE": "4;94", # BLUE "UNDERLINE_INTENSE_PURPLE": "4;95", # PURPLE "UNDERLINE_INTENSE_CYAN": "4;96", # CYAN "UNDERLINE_INTENSE_WHITE": "4;97", # WHITE # Bold Underline High Intensity "BOLD_UNDERLINE_INTENSE_BLACK": "1;4;90", # BLACK "BOLD_UNDERLINE_INTENSE_RED": "1;4;91", # RED "BOLD_UNDERLINE_INTENSE_GREEN": "1;4;92", # GREEN "BOLD_UNDERLINE_INTENSE_YELLOW": "1;4;93", # YELLOW "BOLD_UNDERLINE_INTENSE_BLUE": "1;4;94", # BLUE "BOLD_UNDERLINE_INTENSE_PURPLE": "1;4;95", # PURPLE "BOLD_UNDERLINE_INTENSE_CYAN": "1;4;96", # CYAN "BOLD_UNDERLINE_INTENSE_WHITE": "1;4;97", # WHITE # High Intensity backgrounds "BACKGROUND_INTENSE_BLACK": "0;100", # BLACK "BACKGROUND_INTENSE_RED": "0;101", # RED "BACKGROUND_INTENSE_GREEN": "0;102", # GREEN "BACKGROUND_INTENSE_YELLOW": "0;103", # YELLOW "BACKGROUND_INTENSE_BLUE": "0;104", # BLUE "BACKGROUND_INTENSE_PURPLE": "0;105", # PURPLE "BACKGROUND_INTENSE_CYAN": "0;106", # CYAN "BACKGROUND_INTENSE_WHITE": "0;107", # WHITE } return style def _monokai_style(): style = { "NO_COLOR": "0", "BLACK": "38;5;16", "BLUE": "38;5;63", "CYAN": "38;5;81", "GREEN": "38;5;40", "PURPLE": "38;5;89", "RED": "38;5;124", "WHITE": "38;5;188", "YELLOW": "38;5;184", "INTENSE_BLACK": "38;5;59", "INTENSE_BLUE": "38;5;20", "INTENSE_CYAN": "38;5;44", "INTENSE_GREEN": "38;5;148", "INTENSE_PURPLE": "38;5;141", "INTENSE_RED": "38;5;197", "INTENSE_WHITE": "38;5;15", "INTENSE_YELLOW": "38;5;186", } _ansi_expand_style(style) return style #################################### # Auto-generated below this line # #################################### def _algol_style(): style = { "BLACK": "38;5;59", "BLUE": "38;5;59", "CYAN": "38;5;59", "GREEN": "38;5;59", "INTENSE_BLACK": "38;5;59", "INTENSE_BLUE": "38;5;102", "INTENSE_CYAN": "38;5;102", "INTENSE_GREEN": "38;5;102", "INTENSE_PURPLE": "38;5;102", "INTENSE_RED": "38;5;09", "INTENSE_WHITE": "38;5;102", "INTENSE_YELLOW": "38;5;102", "NO_COLOR": "0", "PURPLE": "38;5;59", "RED": "38;5;09", "WHITE": "38;5;102", "YELLOW": "38;5;09", } _ansi_expand_style(style) return style def _algol_nu_style(): style = { "BLACK": "38;5;59", "BLUE": "38;5;59", "CYAN": "38;5;59", "GREEN": "38;5;59", "INTENSE_BLACK": "38;5;59", "INTENSE_BLUE": "38;5;102", "INTENSE_CYAN": "38;5;102", "INTENSE_GREEN": "38;5;102", "INTENSE_PURPLE": "38;5;102", "INTENSE_RED": "38;5;09", "INTENSE_WHITE": "38;5;102", "INTENSE_YELLOW": "38;5;102", "NO_COLOR": "0", "PURPLE": "38;5;59", "RED": "38;5;09", "WHITE": "38;5;102", "YELLOW": "38;5;09", } _ansi_expand_style(style) return style def _autumn_style(): style = { "BLACK": "38;5;18", "BLUE": "38;5;19", "CYAN": "38;5;37", "GREEN": "38;5;34", "INTENSE_BLACK": "38;5;59", "INTENSE_BLUE": "38;5;33", "INTENSE_CYAN": "38;5;33", "INTENSE_GREEN": "38;5;64", "INTENSE_PURPLE": "38;5;217", "INTENSE_RED": "38;5;130", "INTENSE_WHITE": "38;5;145", "INTENSE_YELLOW": "38;5;217", "NO_COLOR": "0", "PURPLE": "38;5;90", "RED": "38;5;124", "WHITE": "38;5;145", "YELLOW": "38;5;130", } _ansi_expand_style(style) return style def _borland_style(): style = { "BLACK": "38;5;16", "BLUE": "38;5;18", "CYAN": "38;5;30", "GREEN": "38;5;28", "INTENSE_BLACK": "38;5;59", "INTENSE_BLUE": "38;5;21", "INTENSE_CYAN": "38;5;194", "INTENSE_GREEN": "38;5;102", "INTENSE_PURPLE": "38;5;188", "INTENSE_RED": "38;5;09", "INTENSE_WHITE": "38;5;224", "INTENSE_YELLOW": "38;5;188", "NO_COLOR": "0", "PURPLE": "38;5;90", "RED": "38;5;124", "WHITE": "38;5;145", "YELLOW": "38;5;124", } _ansi_expand_style(style) return style def _colorful_style(): style = { "BLACK": "38;5;16", "BLUE": "38;5;20", "CYAN": "38;5;31", "GREEN": "38;5;34", "INTENSE_BLACK": "38;5;59", "INTENSE_BLUE": "38;5;61", "INTENSE_CYAN": "38;5;145", "INTENSE_GREEN": "38;5;102", "INTENSE_PURPLE": "38;5;217", "INTENSE_RED": "38;5;166", "INTENSE_WHITE": "38;5;15", "INTENSE_YELLOW": "38;5;217", "NO_COLOR": "0", "PURPLE": "38;5;90", "RED": "38;5;124", "WHITE": "38;5;145", "YELLOW": "38;5;130", } _ansi_expand_style(style) return style def _emacs_style(): style = { "BLACK": "38;5;28", "BLUE": "38;5;18", "CYAN": "38;5;26", "GREEN": "38;5;34", "INTENSE_BLACK": "38;5;59", "INTENSE_BLUE": "38;5;26", "INTENSE_CYAN": "38;5;145", "INTENSE_GREEN": "38;5;34", "INTENSE_PURPLE": "38;5;129", "INTENSE_RED": "38;5;167", "INTENSE_WHITE": "38;5;145", "INTENSE_YELLOW": "38;5;145", "NO_COLOR": "0", "PURPLE": "38;5;90", "RED": "38;5;124", "WHITE": "38;5;145", "YELLOW": "38;5;130", } _ansi_expand_style(style) return style def _friendly_style(): style = { "BLACK": "38;5;22", "BLUE": "38;5;18", "CYAN": "38;5;31", "GREEN": "38;5;34", "INTENSE_BLACK": "38;5;59", "INTENSE_BLUE": "38;5;74", "INTENSE_CYAN": "38;5;74", "INTENSE_GREEN": "38;5;71", "INTENSE_PURPLE": "38;5;134", "INTENSE_RED": "38;5;167", "INTENSE_WHITE": "38;5;15", "INTENSE_YELLOW": "38;5;145", "NO_COLOR": "0", "PURPLE": "38;5;90", "RED": "38;5;124", "WHITE": "38;5;145", "YELLOW": "38;5;166", } _ansi_expand_style(style) return style def _fruity_style(): style = { "BLACK": "38;5;16", "BLUE": "38;5;32", "CYAN": "38;5;32", "GREEN": "38;5;28", "INTENSE_BLACK": "38;5;59", "INTENSE_BLUE": "38;5;33", "INTENSE_CYAN": "38;5;33", "INTENSE_GREEN": "38;5;102", "INTENSE_PURPLE": "38;5;198", "INTENSE_RED": "38;5;202", "INTENSE_WHITE": "38;5;15", "INTENSE_YELLOW": "38;5;187", "NO_COLOR": "0", "PURPLE": "38;5;198", "RED": "38;5;09", "WHITE": "38;5;187", "YELLOW": "38;5;202", } _ansi_expand_style(style) return style def _igor_style(): style = { "BLACK": "38;5;34", "BLUE": "38;5;21", "CYAN": "38;5;30", "GREEN": "38;5;34", "INTENSE_BLACK": "38;5;30", "INTENSE_BLUE": "38;5;21", "INTENSE_CYAN": "38;5;30", "INTENSE_GREEN": "38;5;34", "INTENSE_PURPLE": "38;5;163", "INTENSE_RED": "38;5;166", "INTENSE_WHITE": "38;5;163", "INTENSE_YELLOW": "38;5;166", "NO_COLOR": "0", "PURPLE": "38;5;163", "RED": "38;5;166", "WHITE": "38;5;163", "YELLOW": "38;5;166", } _ansi_expand_style(style) return style def _lovelace_style(): style = { "BLACK": "38;5;59", "BLUE": "38;5;25", "CYAN": "38;5;29", "GREEN": "38;5;65", "INTENSE_BLACK": "38;5;59", "INTENSE_BLUE": "38;5;25", "INTENSE_CYAN": "38;5;102", "INTENSE_GREEN": "38;5;29", "INTENSE_PURPLE": "38;5;133", "INTENSE_RED": "38;5;131", "INTENSE_WHITE": "38;5;102", "INTENSE_YELLOW": "38;5;136", "NO_COLOR": "0", "PURPLE": "38;5;133", "RED": "38;5;124", "WHITE": "38;5;102", "YELLOW": "38;5;130", } _ansi_expand_style(style) return style def _manni_style(): style = { "BLACK": "38;5;16", "BLUE": "38;5;18", "CYAN": "38;5;30", "GREEN": "38;5;40", "INTENSE_BLACK": "38;5;59", "INTENSE_BLUE": "38;5;105", "INTENSE_CYAN": "38;5;45", "INTENSE_GREEN": "38;5;113", "INTENSE_PURPLE": "38;5;165", "INTENSE_RED": "38;5;202", "INTENSE_WHITE": "38;5;224", "INTENSE_YELLOW": "38;5;221", "NO_COLOR": "0", "PURPLE": "38;5;165", "RED": "38;5;124", "WHITE": "38;5;145", "YELLOW": "38;5;166", } _ansi_expand_style(style) return style def _murphy_style(): style = { "BLACK": "38;5;16", "BLUE": "38;5;18", "CYAN": "38;5;31", "GREEN": "38;5;34", "INTENSE_BLACK": "38;5;59", "INTENSE_BLUE": "38;5;63", "INTENSE_CYAN": "38;5;86", "INTENSE_GREEN": "38;5;86", "INTENSE_PURPLE": "38;5;213", "INTENSE_RED": "38;5;209", "INTENSE_WHITE": "38;5;15", "INTENSE_YELLOW": "38;5;222", "NO_COLOR": "0", "PURPLE": "38;5;90", "RED": "38;5;124", "WHITE": "38;5;145", "YELLOW": "38;5;166", } _ansi_expand_style(style) return style def _native_style(): style = { "BLACK": "38;5;52", "BLUE": "38;5;67", "CYAN": "38;5;31", "GREEN": "38;5;64", "INTENSE_BLACK": "38;5;59", "INTENSE_BLUE": "38;5;68", "INTENSE_CYAN": "38;5;87", "INTENSE_GREEN": "38;5;70", "INTENSE_PURPLE": "38;5;188", "INTENSE_RED": "38;5;160", "INTENSE_WHITE": "38;5;15", "INTENSE_YELLOW": "38;5;214", "NO_COLOR": "0", "PURPLE": "38;5;59", "RED": "38;5;124", "WHITE": "38;5;145", "YELLOW": "38;5;124", } _ansi_expand_style(style) return style def _paraiso_dark_style(): style = { "BLACK": "38;5;95", "BLUE": "38;5;97", "CYAN": "38;5;39", "GREEN": "38;5;72", "INTENSE_BLACK": "38;5;95", "INTENSE_BLUE": "38;5;97", "INTENSE_CYAN": "38;5;79", "INTENSE_GREEN": "38;5;72", "INTENSE_PURPLE": "38;5;188", "INTENSE_RED": "38;5;203", "INTENSE_WHITE": "38;5;188", "INTENSE_YELLOW": "38;5;220", "NO_COLOR": "0", "PURPLE": "38;5;97", "RED": "38;5;203", "WHITE": "38;5;79", "YELLOW": "38;5;214", } _ansi_expand_style(style) return style def _paraiso_light_style(): style = { "BLACK": "38;5;16", "BLUE": "38;5;16", "CYAN": "38;5;39", "GREEN": "38;5;72", "INTENSE_BLACK": "38;5;16", "INTENSE_BLUE": "38;5;97", "INTENSE_CYAN": "38;5;79", "INTENSE_GREEN": "38;5;72", "INTENSE_PURPLE": "38;5;97", "INTENSE_RED": "38;5;203", "INTENSE_WHITE": "38;5;79", "INTENSE_YELLOW": "38;5;220", "NO_COLOR": "0", "PURPLE": "38;5;97", "RED": "38;5;16", "WHITE": "38;5;102", "YELLOW": "38;5;214", } _ansi_expand_style(style) return style def _pastie_style(): style = { "BLACK": "38;5;16", "BLUE": "38;5;20", "CYAN": "38;5;25", "GREEN": "38;5;28", "INTENSE_BLACK": "38;5;59", "INTENSE_BLUE": "38;5;61", "INTENSE_CYAN": "38;5;194", "INTENSE_GREEN": "38;5;34", "INTENSE_PURPLE": "38;5;188", "INTENSE_RED": "38;5;172", "INTENSE_WHITE": "38;5;15", "INTENSE_YELLOW": "38;5;188", "NO_COLOR": "0", "PURPLE": "38;5;125", "RED": "38;5;124", "WHITE": "38;5;145", "YELLOW": "38;5;130", } _ansi_expand_style(style) return style def _perldoc_style(): style = { "BLACK": "38;5;18", "BLUE": "38;5;18", "CYAN": "38;5;31", "GREEN": "38;5;34", "INTENSE_BLACK": "38;5;59", "INTENSE_BLUE": "38;5;134", "INTENSE_CYAN": "38;5;145", "INTENSE_GREEN": "38;5;28", "INTENSE_PURPLE": "38;5;134", "INTENSE_RED": "38;5;167", "INTENSE_WHITE": "38;5;188", "INTENSE_YELLOW": "38;5;188", "NO_COLOR": "0", "PURPLE": "38;5;90", "RED": "38;5;124", "WHITE": "38;5;145", "YELLOW": "38;5;166", } _ansi_expand_style(style) return style def _rrt_style(): style = { "BLACK": "38;5;09", "BLUE": "38;5;117", "CYAN": "38;5;117", "GREEN": "38;5;46", "INTENSE_BLACK": "38;5;117", "INTENSE_BLUE": "38;5;117", "INTENSE_CYAN": "38;5;122", "INTENSE_GREEN": "38;5;46", "INTENSE_PURPLE": "38;5;213", "INTENSE_RED": "38;5;09", "INTENSE_WHITE": "38;5;188", "INTENSE_YELLOW": "38;5;222", "NO_COLOR": "0", "PURPLE": "38;5;213", "RED": "38;5;09", "WHITE": "38;5;117", "YELLOW": "38;5;09", } _ansi_expand_style(style) return style def _tango_style(): style = { "BLACK": "38;5;16", "BLUE": "38;5;20", "CYAN": "38;5;61", "GREEN": "38;5;34", "INTENSE_BLACK": "38;5;24", "INTENSE_BLUE": "38;5;62", "INTENSE_CYAN": "38;5;15", "INTENSE_GREEN": "38;5;64", "INTENSE_PURPLE": "38;5;15", "INTENSE_RED": "38;5;09", "INTENSE_WHITE": "38;5;15", "INTENSE_YELLOW": "38;5;178", "NO_COLOR": "0", "PURPLE": "38;5;90", "RED": "38;5;124", "WHITE": "38;5;15", "YELLOW": "38;5;94", } _ansi_expand_style(style) return style def _trac_style(): style = { "BLACK": "38;5;16", "BLUE": "38;5;18", "CYAN": "38;5;30", "GREEN": "38;5;100", "INTENSE_BLACK": "38;5;59", "INTENSE_BLUE": "38;5;60", "INTENSE_CYAN": "38;5;194", "INTENSE_GREEN": "38;5;102", "INTENSE_PURPLE": "38;5;188", "INTENSE_RED": "38;5;137", "INTENSE_WHITE": "38;5;224", "INTENSE_YELLOW": "38;5;188", "NO_COLOR": "0", "PURPLE": "38;5;90", "RED": "38;5;124", "WHITE": "38;5;145", "YELLOW": "38;5;100", } _ansi_expand_style(style) return style def _vim_style(): style = { "BLACK": "38;5;18", "BLUE": "38;5;18", "CYAN": "38;5;44", "GREEN": "38;5;40", "INTENSE_BLACK": "38;5;60", "INTENSE_BLUE": "38;5;68", "INTENSE_CYAN": "38;5;44", "INTENSE_GREEN": "38;5;40", "INTENSE_PURPLE": "38;5;164", "INTENSE_RED": "38;5;09", "INTENSE_WHITE": "38;5;188", "INTENSE_YELLOW": "38;5;184", "NO_COLOR": "0", "PURPLE": "38;5;164", "RED": "38;5;160", "WHITE": "38;5;188", "YELLOW": "38;5;160", } _ansi_expand_style(style) return style def _vs_style(): style = { "BLACK": "38;5;28", "BLUE": "38;5;21", "CYAN": "38;5;31", "GREEN": "38;5;28", "INTENSE_BLACK": "38;5;31", "INTENSE_BLUE": "38;5;31", "INTENSE_CYAN": "38;5;31", "INTENSE_GREEN": "38;5;31", "INTENSE_PURPLE": "38;5;31", "INTENSE_RED": "38;5;09", "INTENSE_WHITE": "38;5;31", "INTENSE_YELLOW": "38;5;31", "NO_COLOR": "0", "PURPLE": "38;5;124", "RED": "38;5;124", "WHITE": "38;5;31", "YELLOW": "38;5;124", } _ansi_expand_style(style) return style def _xcode_style(): style = { "BLACK": "38;5;16", "BLUE": "38;5;20", "CYAN": "38;5;60", "GREEN": "38;5;28", "INTENSE_BLACK": "38;5;60", "INTENSE_BLUE": "38;5;20", "INTENSE_CYAN": "38;5;60", "INTENSE_GREEN": "38;5;60", "INTENSE_PURPLE": "38;5;126", "INTENSE_RED": "38;5;160", "INTENSE_WHITE": "38;5;60", "INTENSE_YELLOW": "38;5;94", "NO_COLOR": "0", "PURPLE": "38;5;126", "RED": "38;5;160", "WHITE": "38;5;60", "YELLOW": "38;5;94", } _ansi_expand_style(style) return style ANSI_STYLES = LazyDict( { "algol": _algol_style, "algol_nu": _algol_nu_style, "autumn": _autumn_style, "borland": _borland_style, "bw": _bw_style, "colorful": _colorful_style, "default": _default_style, "emacs": _emacs_style, "friendly": _friendly_style, "fruity": _fruity_style, "igor": _igor_style, "lovelace": _lovelace_style, "manni": _manni_style, "monokai": _monokai_style, "murphy": _murphy_style, "native": _native_style, "paraiso-dark": _paraiso_dark_style, "paraiso-light": _paraiso_light_style, "pastie": _pastie_style, "perldoc": _perldoc_style, "rrt": _rrt_style, "tango": _tango_style, "trac": _trac_style, "vim": _vim_style, "vs": _vs_style, "xcode": _xcode_style, }, globals(), "ANSI_STYLES", ) del ( _algol_style, _algol_nu_style, _autumn_style, _borland_style, _bw_style, _colorful_style, _default_style, _emacs_style, _friendly_style, _fruity_style, _igor_style, _lovelace_style, _manni_style, _monokai_style, _murphy_style, _native_style, _paraiso_dark_style, _paraiso_light_style, _pastie_style, _perldoc_style, _rrt_style, _tango_style, _trac_style, _vim_style, _vs_style, _xcode_style, ) # # Dynamically generated styles # def make_ansi_style(palette): """Makes an ANSI color style from a color palette""" style = {"NO_COLOR": "0"} for name, t in BASE_XONSH_COLORS.items(): closest = find_closest_color(t, palette) if len(closest) == 3: closest = "".join([a * 2 for a in closest]) short = rgb2short(closest)[0] style[name] = "38;5;" + short style["BOLD_" + name] = "1;38;5;" + short style["UNDERLINE_" + name] = "4;38;5;" + short style["BOLD_UNDERLINE_" + name] = "1;4;38;5;" + short style["BACKGROUND_" + name] = "48;5;" + short return style def ansi_style_by_name(name): """Gets or makes an ANSI color style by name. If the styles does not exist, it will look for a style using the pygments name. """ if name in ANSI_STYLES: return ANSI_STYLES[name] elif not HAS_PYGMENTS: raise KeyError("could not find style {0!r}".format(name)) from xonsh.pygments_cache import get_style_by_name pstyle = get_style_by_name(name) palette = make_palette(pstyle.styles.values()) astyle = make_ansi_style(palette) ANSI_STYLES[name] = astyle return astyle ================================================ FILE: xonsh/ast.py ================================================ # -*- coding: utf-8 -*- """The xonsh abstract syntax tree node.""" # These are imported into our module namespace for the benefit of parser.py. # pylint: disable=unused-import import sys import builtins from ast import ( Module, Num, Expr, Str, Bytes, UnaryOp, UAdd, USub, Invert, BinOp, Add, Sub, Mult, Div, FloorDiv, Mod, Pow, Compare, Lt, Gt, LtE, GtE, Eq, NotEq, In, NotIn, Is, IsNot, Not, BoolOp, Or, And, Subscript, Load, Slice, ExtSlice, List, Tuple, Set, Dict, AST, NameConstant, Name, GeneratorExp, Store, comprehension, ListComp, SetComp, DictComp, Assign, AugAssign, BitXor, BitAnd, BitOr, LShift, RShift, Assert, Delete, Del, Pass, Raise, Import, alias, ImportFrom, Continue, Break, Yield, YieldFrom, Return, IfExp, Lambda, arguments, arg, Call, keyword, Attribute, Global, Nonlocal, If, While, For, withitem, With, Try, ExceptHandler, FunctionDef, ClassDef, Starred, NodeTransformer, Interactive, Expression, Index, literal_eval, dump, walk, increment_lineno, ) from ast import Ellipsis as EllipsisNode # pylint: enable=unused-import import textwrap import itertools from xonsh.tools import subproc_toks, find_next_break, get_logical_line from xonsh.platform import PYTHON_VERSION_INFO if PYTHON_VERSION_INFO >= (3, 5, 0): # pylint: disable=unused-import # pylint: disable=no-name-in-module from ast import MatMult, AsyncFunctionDef, AsyncWith, AsyncFor, Await else: MatMult = AsyncFunctionDef = AsyncWith = AsyncFor = Await = None if PYTHON_VERSION_INFO >= (3, 6, 0): # pylint: disable=unused-import # pylint: disable=no-name-in-module from ast import JoinedStr, FormattedValue, AnnAssign else: JoinedStr = FormattedValue = AnnAssign = None STATEMENTS = ( FunctionDef, ClassDef, Return, Delete, Assign, AugAssign, For, While, If, With, Raise, Try, Assert, Import, ImportFrom, Global, Nonlocal, Expr, Pass, Break, Continue, ) if PYTHON_VERSION_INFO >= (3, 6, 0): STATEMENTS += (AnnAssign,) def leftmostname(node): """Attempts to find the first name in the tree.""" if isinstance(node, Name): rtn = node.id elif isinstance(node, (BinOp, Compare)): rtn = leftmostname(node.left) elif isinstance(node, (Attribute, Subscript, Starred, Expr)): rtn = leftmostname(node.value) elif isinstance(node, Call): rtn = leftmostname(node.func) elif isinstance(node, UnaryOp): rtn = leftmostname(node.operand) elif isinstance(node, BoolOp): rtn = leftmostname(node.values[0]) elif isinstance(node, (Assign, AnnAssign)): rtn = leftmostname(node.targets[0]) elif isinstance(node, (Str, Bytes, JoinedStr)): # handles case of "./my executable" rtn = leftmostname(node.s) elif isinstance(node, Tuple) and len(node.elts) > 0: # handles case of echo ,1,2,3 rtn = leftmostname(node.elts[0]) else: rtn = None return rtn def get_lineno(node, default=0): """Gets the lineno of a node or returns the default.""" return getattr(node, "lineno", default) def min_line(node): """Computes the minimum lineno.""" node_line = get_lineno(node) return min(map(get_lineno, walk(node), itertools.repeat(node_line))) def max_line(node): """Computes the maximum lineno.""" return max(map(get_lineno, walk(node))) def get_col(node, default=-1): """Gets the col_offset of a node, or returns the default""" return getattr(node, "col_offset", default) def min_col(node): """Computes the minimum col_offset.""" return min(map(get_col, walk(node), itertools.repeat(node.col_offset))) def max_col(node): """Returns the maximum col_offset of the node and all sub-nodes.""" col = getattr(node, "max_col", None) if col is not None: return col highest = max(walk(node), key=get_col) col = highest.col_offset + node_len(highest) return col def node_len(node): """The length of a node as a string""" val = 0 for n in walk(node): if isinstance(n, Name): val += len(n.id) elif isinstance(n, Attribute): val += 1 + (len(n.attr) if isinstance(n.attr, str) else 0) # this may need to be added to for more nodes as more cases are found return val def get_id(node, default=None): """Gets the id attribute of a node, or returns a default.""" return getattr(node, "id", default) def gather_names(node): """Returns the set of all names present in the node's tree.""" rtn = set(map(get_id, walk(node))) rtn.discard(None) return rtn def get_id_ctx(node): """Gets the id and attribute of a node, or returns a default.""" nid = getattr(node, "id", None) if nid is None: return (None, None) return (nid, node.ctx) def gather_load_store_names(node): """Returns the names present in the node's tree in a set of load nodes and a set of store nodes. """ load = set() store = set() for nid, ctx in map(get_id_ctx, walk(node)): if nid is None: continue elif isinstance(ctx, Load): load.add(nid) else: store.add(nid) return (load, store) def has_elts(x): """Tests if x is an AST node with elements.""" return isinstance(x, AST) and hasattr(x, "elts") def load_attribute_chain(name, lineno=None, col=None): """Creates an AST that loads variable name that may (or may not) have attribute chains. For example, "a.b.c" """ names = name.split(".") node = Name(id=names.pop(0), ctx=Load(), lineno=lineno, col_offset=col) for attr in names: node = Attribute( value=node, attr=attr, ctx=Load(), lineno=lineno, col_offset=col ) return node def xonsh_call(name, args, lineno=None, col=None): """Creates the AST node for calling a function of a given name. Functions names may contain attribute access, e.g. __xonsh__.env. """ return Call( func=load_attribute_chain(name, lineno=lineno, col=col), args=args, keywords=[], starargs=None, kwargs=None, lineno=lineno, col_offset=col, ) def isdescendable(node): """Determines whether or not a node is worth visiting. Currently only UnaryOp and BoolOp nodes are visited. """ return isinstance(node, (UnaryOp, BoolOp)) def isexpression(node, ctx=None, *args, **kwargs): """Determines whether a node (or code string) is an expression, and does not contain any statements. The execution context (ctx) and other args and kwargs are passed down to the parser, as needed. """ # parse string to AST if isinstance(node, str): node = node if node.endswith("\n") else node + "\n" ctx = builtins.__xonsh__.ctx if ctx is None else ctx node = builtins.__xonsh__.execer.parse(node, ctx, *args, **kwargs) # determin if expresission-like enough if isinstance(node, (Expr, Expression)): isexpr = True elif isinstance(node, Module) and len(node.body) == 1: isexpr = isinstance(node.body[0], (Expr, Expression)) else: isexpr = False return isexpr class CtxAwareTransformer(NodeTransformer): """Transforms a xonsh AST based to use subprocess calls when the first name in an expression statement is not known in the context. This assumes that the expression statement is instead parseable as a subprocess. """ def __init__(self, parser): """Parameters ---------- parser : xonsh.Parser A parse instance to try to parse subprocess statements with. """ super(CtxAwareTransformer, self).__init__() self.parser = parser self.input = None self.contexts = [] self.lines = None self.mode = None self._nwith = 0 self.filename = "" self.debug_level = 0 def ctxvisit(self, node, inp, ctx, mode="exec", filename=None, debug_level=0): """Transforms the node in a context-dependent way. Parameters ---------- node : ast.AST A syntax tree to transform. input : str The input code in string format. ctx : dict The root context to use. filename : str, optional File we are to transform. debug_level : int, optional Debugging level to use in lexing and parsing. Returns ------- node : ast.AST The transformed node. """ self.filename = self.filename if filename is None else filename self.debug_level = debug_level self.lines = inp.splitlines() self.contexts = [ctx, set()] self.mode = mode self._nwith = 0 node = self.visit(node) del self.lines, self.contexts, self.mode self._nwith = 0 return node def ctxupdate(self, iterable): """Updated the most recent context.""" self.contexts[-1].update(iterable) def ctxadd(self, value): """Adds a value the most recent context.""" self.contexts[-1].add(value) def ctxremove(self, value): """Removes a value the most recent context.""" for ctx in reversed(self.contexts): if value in ctx: ctx.remove(value) break def try_subproc_toks(self, node, strip_expr=False): """Tries to parse the line of the node as a subprocess.""" line, nlogical, idx = get_logical_line(self.lines, node.lineno - 1) if self.mode == "eval": mincol = len(line) - len(line.lstrip()) maxcol = None else: mincol = max(min_col(node) - 1, 0) maxcol = max_col(node) if mincol == maxcol: maxcol = find_next_break(line, mincol=mincol, lexer=self.parser.lexer) elif nlogical > 1: maxcol = None elif maxcol < len(line) and line[maxcol] == ";": pass else: maxcol += 1 spline = subproc_toks( line, mincol=mincol, maxcol=maxcol, returnline=False, lexer=self.parser.lexer, ) if spline is None or spline != "![{}]".format(line[mincol:maxcol].strip()): # failed to get something consistent, try greedy wrap spline = subproc_toks( line, mincol=mincol, maxcol=maxcol, returnline=False, lexer=self.parser.lexer, greedy=True, ) if spline is None: return node try: newnode = self.parser.parse( spline, mode=self.mode, filename=self.filename, debug_level=(self.debug_level > 2), ) newnode = newnode.body if not isinstance(newnode, AST): # take the first (and only) Expr newnode = newnode[0] increment_lineno(newnode, n=node.lineno - 1) newnode.col_offset = node.col_offset if self.debug_level > 1: msg = "{0}:{1}:{2}{3} - {4}\n" "{0}:{1}:{2}{3} + {5}" mstr = "" if maxcol is None else ":" + str(maxcol) msg = msg.format(self.filename, node.lineno, mincol, mstr, line, spline) print(msg, file=sys.stderr) except SyntaxError: newnode = node if strip_expr and isinstance(newnode, Expr): newnode = newnode.value return newnode def is_in_scope(self, node): """Determines whether or not the current node is in scope.""" names, store = gather_load_store_names(node) names -= store if not names: return True inscope = False for ctx in reversed(self.contexts): names -= ctx if not names: inscope = True break return inscope # # Replacement visitors # def visit_Expression(self, node): """Handle visiting an expression body.""" if isdescendable(node.body): node.body = self.visit(node.body) body = node.body inscope = self.is_in_scope(body) if not inscope: node.body = self.try_subproc_toks(body) return node def visit_Expr(self, node): """Handle visiting an expression.""" if isdescendable(node.value): node.value = self.visit(node.value) # this allows diving into BoolOps if self.is_in_scope(node) or isinstance(node.value, Lambda): return node else: newnode = self.try_subproc_toks(node) if not isinstance(newnode, Expr): newnode = Expr( value=newnode, lineno=node.lineno, col_offset=node.col_offset ) if hasattr(node, "max_lineno"): newnode.max_lineno = node.max_lineno newnode.max_col = node.max_col return newnode def visit_UnaryOp(self, node): """Handle visiting an unary operands, like not.""" if isdescendable(node.operand): node.operand = self.visit(node.operand) operand = node.operand inscope = self.is_in_scope(operand) if not inscope: node.operand = self.try_subproc_toks(operand, strip_expr=True) return node def visit_BoolOp(self, node): """Handle visiting an boolean operands, like and/or.""" for i in range(len(node.values)): val = node.values[i] if isdescendable(val): val = node.values[i] = self.visit(val) inscope = self.is_in_scope(val) if not inscope: node.values[i] = self.try_subproc_toks(val, strip_expr=True) return node # # Context aggregator visitors # def visit_Assign(self, node): """Handle visiting an assignment statement.""" ups = set() for targ in node.targets: if isinstance(targ, (Tuple, List)): ups.update(leftmostname(elt) for elt in targ.elts) elif isinstance(targ, BinOp): newnode = self.try_subproc_toks(node) if newnode is node: ups.add(leftmostname(targ)) else: return newnode else: ups.add(leftmostname(targ)) self.ctxupdate(ups) return node visit_AnnAssign = visit_Assign def visit_Import(self, node): """Handle visiting a import statement.""" for name in node.names: if name.asname is None: self.ctxadd(name.name) else: self.ctxadd(name.asname) return node def visit_ImportFrom(self, node): """Handle visiting a "from ... import ..." statement.""" for name in node.names: if name.asname is None: self.ctxadd(name.name) else: self.ctxadd(name.asname) return node def visit_With(self, node): """Handle visiting a with statement.""" for item in node.items: if item.optional_vars is not None: self.ctxupdate(gather_names(item.optional_vars)) self._nwith += 1 self.generic_visit(node) self._nwith -= 1 return node def visit_For(self, node): """Handle visiting a for statement.""" targ = node.target self.ctxupdate(gather_names(targ)) self.generic_visit(node) return node def visit_FunctionDef(self, node): """Handle visiting a function definition.""" self.ctxadd(node.name) self.contexts.append(set()) args = node.args argchain = [args.args, args.kwonlyargs] if args.vararg is not None: argchain.append((args.vararg,)) if args.kwarg is not None: argchain.append((args.kwarg,)) self.ctxupdate(a.arg for a in itertools.chain.from_iterable(argchain)) self.generic_visit(node) self.contexts.pop() return node def visit_ClassDef(self, node): """Handle visiting a class definition.""" self.ctxadd(node.name) self.contexts.append(set()) self.generic_visit(node) self.contexts.pop() return node def visit_Delete(self, node): """Handle visiting a del statement.""" for targ in node.targets: if isinstance(targ, Name): self.ctxremove(targ.id) self.generic_visit(node) return node def visit_Try(self, node): """Handle visiting a try statement.""" for handler in node.handlers: if handler.name is not None: self.ctxadd(handler.name) self.generic_visit(node) return node def visit_Global(self, node): """Handle visiting a global statement.""" self.contexts[1].update(node.names) # contexts[1] is the global ctx self.generic_visit(node) return node def pdump(s, **kwargs): """performs a pretty dump of an AST node.""" if isinstance(s, AST): s = dump(s, **kwargs).replace(",", ",\n") openers = "([{" closers = ")]}" lens = len(s) + 1 if lens == 1: return s i = min([s.find(o) % lens for o in openers]) if i == lens - 1: return s closer = closers[openers.find(s[i])] j = s.rfind(closer) if j == -1 or j <= i: return s[: i + 1] + "\n" + textwrap.indent(pdump(s[i + 1 :]), " ") pre = s[: i + 1] + "\n" mid = s[i + 1 : j] post = "\n" + s[j:] mid = textwrap.indent(pdump(mid), " ") if "(" in post or "[" in post or "{" in post: post = pdump(post) return pre + mid + post def pprint_ast(s, *, sep=None, end=None, file=None, flush=False, **kwargs): """Performs a pretty print of the AST nodes.""" print(pdump(s, **kwargs), sep=sep, end=end, file=file, flush=flush) # # Private helpers # def _getblockattr(name, lineno, col): """calls getattr(name, '__xonsh_block__', False).""" return xonsh_call( "getattr", args=[ Name(id=name, ctx=Load(), lineno=lineno, col_offset=col), Str(s="__xonsh_block__", lineno=lineno, col_offset=col), NameConstant(value=False, lineno=lineno, col_offset=col), ], lineno=lineno, col=col, ) ================================================ FILE: xonsh/base_shell.py ================================================ # -*- coding: utf-8 -*- """The base class for xonsh shell""" import io import os import sys import time import builtins from xonsh.tools import ( XonshError, print_exception, DefaultNotGiven, check_for_partial_string, format_std_prepost, get_line_continuation, ) from xonsh.platform import HAS_PYGMENTS, ON_WINDOWS from xonsh.codecache import ( should_use_cache, code_cache_name, code_cache_check, get_cache_filename, update_cache, run_compiled_code, ) from xonsh.completer import Completer from xonsh.prompt.base import multiline_prompt, PromptFormatter from xonsh.events import events from xonsh.shell import transform_command from xonsh.lazyimps import pygments, pyghooks from xonsh.ansi_colors import ansi_partial_color_format if ON_WINDOWS: import ctypes kernel32 = ctypes.windll.kernel32 kernel32.SetConsoleTitleW.argtypes = [ctypes.c_wchar_p] class _TeeStdBuf(io.RawIOBase): """A dispatcher for bytes to two buffers, as std stream buffer and an in memory buffer. """ def __init__( self, stdbuf, membuf, encoding=None, errors=None, prestd=b"", poststd=b"" ): """ Parameters ---------- stdbuf : BytesIO-like or StringIO-like The std stream buffer. membuf : BytesIO-like The in memory stream buffer. encoding : str or None, optional The encoding of the stream. Only used if stdbuf is a text stream, rather than a binary one. Defaults to $XONSH_ENCODING if None. errors : str or None, optional The error form for the encoding of the stream. Only used if stdbuf is a text stream, rather than a binary one. Deafults to $XONSH_ENCODING_ERRORS if None. prestd : bytes, optional The prefix to prepend to the standard buffer. poststd : bytes, optional The postfix to append to the standard buffer. """ self.stdbuf = stdbuf self.membuf = membuf env = builtins.__xonsh__.env self.encoding = env.get("XONSH_ENCODING") if encoding is None else encoding self.errors = env.get("XONSH_ENCODING_ERRORS") if errors is None else errors self.prestd = prestd self.poststd = poststd self._std_is_binary = not hasattr(stdbuf, "encoding") def fileno(self): """Returns the file descriptor of the std buffer.""" return self.stdbuf.fileno() def seek(self, offset, whence=io.SEEK_SET): """Sets the location in both the stdbuf and the membuf.""" self.stdbuf.seek(offset, whence) self.membuf.seek(offset, whence) def truncate(self, size=None): """Truncate both buffers.""" self.stdbuf.truncate(size) self.membuf.truncate(size) def readinto(self, b): """Read bytes into buffer from both streams.""" if self._std_is_binary: self.stdbuf.readinto(b) return self.membuf.readinto(b) def write(self, b): """Write bytes into both buffers.""" std_b = b if self.prestd: std_b = self.prestd + b if self.poststd: std_b += self.poststd # write to stdbuf if self._std_is_binary: self.stdbuf.write(std_b) else: self.stdbuf.write(std_b.decode(encoding=self.encoding, errors=self.errors)) return self.membuf.write(b) class _TeeStd(io.TextIOBase): """Tees a std stream into an in-memory container and the original stream.""" def __init__(self, name, mem, prestd="", poststd=""): """ Parameters ---------- name : str The name of the buffer in the sys module, e.g. 'stdout'. mem : io.TextIOBase-like The in-memory text-based representation. prestd : str, optional The prefix to prepend to the standard stream. poststd : str, optional The postfix to append to the standard stream. """ self._name = name self.std = std = getattr(sys, name) self.mem = mem self.prestd = prestd self.poststd = poststd preb = prestd.encode(encoding=mem.encoding, errors=mem.errors) postb = poststd.encode(encoding=mem.encoding, errors=mem.errors) if hasattr(std, "buffer"): buffer = _TeeStdBuf(std.buffer, mem.buffer, prestd=preb, poststd=postb) else: # TextIO does not have buffer as part of the API, so std streams # may not either. buffer = _TeeStdBuf( std, mem.buffer, encoding=mem.encoding, errors=mem.errors, prestd=preb, poststd=postb, ) self.buffer = buffer setattr(sys, name, self) @property def encoding(self): """The encoding of the in-memory buffer.""" return self.mem.encoding @property def errors(self): """The errors of the in-memory buffer.""" return self.mem.errors @property def newlines(self): """The newlines of the in-memory buffer.""" return self.mem.newlines def _replace_std(self): std = self.std if std is None: return setattr(sys, self._name, std) self.std = self._name = None def __del__(self): self._replace_std() def close(self): """Restores the original std stream.""" self._replace_std() def write(self, s): """Writes data to the original std stream and the in-memory object.""" self.mem.write(s) if self.std is None: return std_s = s if self.prestd: std_s = self.prestd + std_s if self.poststd: std_s += self.poststd self.std.write(std_s) def flush(self): """Flushes both the original stdout and the buffer.""" self.std.flush() self.mem.flush() def fileno(self): """Tunnel fileno() calls to the std stream.""" return self.std.fileno() def seek(self, offset, whence=io.SEEK_SET): """Seek to a location in both streams.""" self.std.seek(offset, whence) self.mem.seek(offset, whence) def truncate(self, size=None): """Seek to a location in both streams.""" self.std.truncate(size) self.mem.truncate(size) def detach(self): """This operation is not supported.""" raise io.UnsupportedOperation def read(self, size=None): """Read from the in-memory stream and seek to a new location in the std stream. """ s = self.mem.read(size) loc = self.std.tell() self.std.seek(loc + len(s)) return s def readline(self, size=-1): """Read a line from the in-memory stream and seek to a new location in the std stream. """ s = self.mem.readline(size) loc = self.std.tell() self.std.seek(loc + len(s)) return s class Tee: """Class that merges tee'd stdout and stderr into a single stream. This represents what a user would actually see on the command line. This class has the same interface as io.TextIOWrapper, except that the buffer is optional. """ # pylint is a stupid about counting public methods when using inheritance. # pylint: disable=too-few-public-methods def __init__( self, buffer=None, encoding=None, errors=None, newline=None, line_buffering=False, write_through=False, ): self.buffer = io.BytesIO() if buffer is None else buffer self.memory = io.TextIOWrapper( self.buffer, encoding=encoding, errors=errors, newline=newline, line_buffering=line_buffering, write_through=write_through, ) self.stdout = _TeeStd("stdout", self.memory) env = builtins.__xonsh__.env prestderr = format_std_prepost(env.get("XONSH_STDERR_PREFIX")) poststderr = format_std_prepost(env.get("XONSH_STDERR_POSTFIX")) self.stderr = _TeeStd( "stderr", self.memory, prestd=prestderr, poststd=poststderr ) @property def line_buffering(self): return self.memory.line_buffering def __del__(self): del self.stdout, self.stderr self.stdout = self.stderr = None def close(self): """Closes the buffer as well as the stdout and stderr tees.""" self.stdout.close() self.stderr.close() self.memory.close() def getvalue(self): """Gets the current contents of the in-memory buffer.""" m = self.memory loc = m.tell() m.seek(0) s = m.read() m.seek(loc) return s class BaseShell(object): """The xonsh shell.""" def __init__(self, execer, ctx, **kwargs): super().__init__() self.execer = execer self.ctx = ctx self.completer = Completer() if kwargs.get("completer", True) else None self.buffer = [] self.need_more_lines = False self.mlprompt = None self._styler = DefaultNotGiven self.prompt_formatter = PromptFormatter() self.accumulated_inputs = "" @property def styler(self): if self._styler is DefaultNotGiven: if HAS_PYGMENTS: from xonsh.pyghooks import XonshStyle env = builtins.__xonsh__.env self._styler = XonshStyle(env.get("XONSH_COLOR_STYLE")) else: self._styler = None return self._styler @styler.setter def styler(self, value): self._styler = value @styler.deleter def styler(self): self._styler = DefaultNotGiven def emptyline(self): """Called when an empty line has been entered.""" self.need_more_lines = False self.default("") def singleline(self, **kwargs): """Reads a single line of input from the shell.""" msg = "{0} has not implemented singleline()." raise RuntimeError(msg.format(self.__class__.__name__)) def precmd(self, line): """Called just before execution of line.""" return line if self.need_more_lines else line.lstrip() def default(self, line): """Implements code execution.""" line = line if line.endswith("\n") else line + "\n" src, code = self.push(line) if code is None: return events.on_precommand.fire(cmd=src) env = builtins.__xonsh__.env hist = builtins.__xonsh__.history # pylint: disable=no-member ts1 = None enc = env.get("XONSH_ENCODING") err = env.get("XONSH_ENCODING_ERRORS") tee = Tee(encoding=enc, errors=err) try: ts0 = time.time() run_compiled_code(code, self.ctx, None, "single") ts1 = time.time() if hist is not None and hist.last_cmd_rtn is None: hist.last_cmd_rtn = 0 # returncode for success except XonshError as e: print(e.args[0], file=sys.stderr) if hist is not None and hist.last_cmd_rtn is None: hist.last_cmd_rtn = 1 # return code for failure except Exception: # pylint: disable=broad-except print_exception() if hist is not None and hist.last_cmd_rtn is None: hist.last_cmd_rtn = 1 # return code for failure finally: ts1 = ts1 or time.time() tee_out = tee.getvalue() self._append_history(inp=src, ts=[ts0, ts1], tee_out=tee_out) self.accumulated_inputs += src if ( tee_out and env.get("XONSH_APPEND_NEWLINE") and not tee_out.endswith(os.linesep) ): print(os.linesep, end="") tee.close() self._fix_cwd() if builtins.__xonsh__.exit: # pylint: disable=no-member return True def _append_history(self, tee_out=None, **info): """Append information about the command to the history. This also handles on_postcommand because this is the place where all the information is available. """ hist = builtins.__xonsh__.history # pylint: disable=no-member info["rtn"] = hist.last_cmd_rtn if hist is not None else None tee_out = tee_out or None last_out = hist.last_cmd_out if hist is not None else None if last_out is None and tee_out is None: pass elif last_out is None and tee_out is not None: info["out"] = tee_out elif last_out is not None and tee_out is None: info["out"] = last_out else: info["out"] = tee_out + "\n" + last_out events.on_postcommand.fire( cmd=info["inp"], rtn=info["rtn"], out=info.get("out", None), ts=info["ts"] ) if hist is not None: hist.append(info) hist.last_cmd_rtn = hist.last_cmd_out = None def _fix_cwd(self): """Check if the cwd changed out from under us.""" env = builtins.__xonsh__.env try: cwd = os.getcwd() except (FileNotFoundError, OSError): cwd = None if cwd is None: # directory has been deleted out from under us, most likely pwd = env.get("PWD", None) if pwd is None: # we have no idea where we are env["PWD"] = "" elif os.path.isdir(pwd): # unclear why os.getcwd() failed. do nothing. pass else: # OK PWD is really gone. msg = "{UNDERLINE_INTENSE_WHITE}{BACKGROUND_INTENSE_BLACK}" msg += "xonsh: working directory does not exist: " + pwd msg += "{NO_COLOR}" self.print_color(msg, file=sys.stderr) elif "PWD" not in env: # $PWD is missing from env, recreate it env["PWD"] = cwd elif os.path.realpath(cwd) != os.path.realpath(env["PWD"]): # The working directory has changed without updating $PWD, fix this old = env["PWD"] env["PWD"] = cwd env["OLDPWD"] = old events.on_chdir.fire(olddir=old, newdir=cwd) def push(self, line): """Pushes a line onto the buffer and compiles the code in a way that enables multiline input. """ self.buffer.append(line) if self.need_more_lines: return None, None src = "".join(self.buffer) src = transform_command(src) return self.compile(src) def compile(self, src): """Compiles source code and returns the (possibly modified) source and a valid code object. """ _cache = should_use_cache(self.execer, "single") if _cache: codefname = code_cache_name(src) cachefname = get_cache_filename(codefname, code=True) usecache, code = code_cache_check(cachefname) if usecache: self.reset_buffer() return src, code lincont = get_line_continuation() if src.endswith(lincont + "\n"): self.need_more_lines = True return src, None try: code = self.execer.compile(src, mode="single", glbs=self.ctx, locs=None) if _cache: update_cache(code, cachefname) self.reset_buffer() except SyntaxError: partial_string_info = check_for_partial_string(src) in_partial_string = ( partial_string_info[0] is not None and partial_string_info[1] is None ) if (src == "\n" or src.endswith("\n\n")) and not in_partial_string: self.reset_buffer() print_exception() return src, None self.need_more_lines = True code = None except Exception: # pylint: disable=broad-except self.reset_buffer() print_exception() code = None return src, code def reset_buffer(self): """Resets the line buffer.""" self.buffer.clear() self.need_more_lines = False self.mlprompt = None def settitle(self): """Sets terminal title.""" env = builtins.__xonsh__.env # pylint: disable=no-member term = env.get("TERM", None) # Shells running in emacs sets TERM to "dumb" or "eterm-color". # Do not set title for these to avoid garbled prompt. if (term is None and not ON_WINDOWS) or term in [ "dumb", "eterm-color", "linux", ]: return t = 'gitsome' if t is None: return t = self.prompt_formatter(t) if ON_WINDOWS and "ANSICON" not in env: kernel32.SetConsoleTitleW(t) else: with open(1, "wb", closefd=False) as f: # prevent xonsh from answering interactive questions # on the next command by writing the title f.write("\x1b]0;{0}\x07".format(t).encode()) f.flush() @property def prompt(self): """Obtains the current prompt string.""" if self.need_more_lines: if self.mlprompt is None: try: self.mlprompt = multiline_prompt() except Exception: # pylint: disable=broad-except print_exception() self.mlprompt = " " return self.mlprompt env = builtins.__xonsh__.env # pylint: disable=no-member p = env.get("PROMPT") try: p = self.prompt_formatter(p) except Exception: # pylint: disable=broad-except print_exception() self.settitle() return p def format_color(self, string, hide=False, force_string=False, **kwargs): """Formats the colors in a string. ``BaseShell``'s default implementation of this method uses colors based on ANSI color codes. """ style = builtins.__xonsh__.env.get("XONSH_COLOR_STYLE") return ansi_partial_color_format(string, hide=hide, style=style) def print_color(self, string, hide=False, **kwargs): """Prints a string in color. This base implementation's colors are based on ANSI color codes if a string was given as input. If a list of token pairs is given, it will color based on pygments, if available. If pygments is not available, it will print a colorless string. """ if isinstance(string, str): s = self.format_color(string, hide=hide) elif HAS_PYGMENTS: # assume this is a list of (Token, str) tuples and format it env = builtins.__xonsh__.env self.styler.style_name = env.get("XONSH_COLOR_STYLE") style_proxy = pyghooks.xonsh_style_proxy(self.styler) formatter = pyghooks.XonshTerminal256Formatter(style=style_proxy) s = pygments.format(string, formatter).rstrip() else: # assume this is a list of (Token, str) tuples and remove color s = "".join([x for _, x in string]) print(s, **kwargs) def color_style_names(self): """Returns an iterable of all available style names.""" return () def color_style(self): """Returns the current color map.""" return {} def restore_tty_sanity(self): """An interface for resetting the TTY stdin mode. This is highly dependent on the shell backend. Also it is mostly optional since it only affects ^Z backgrounding behaviour. """ pass ================================================ FILE: xonsh/built_ins.py ================================================ # -*- coding: utf-8 -*- """The xonsh built-ins. Note that this module is named 'built_ins' so as not to be confused with the special Python builtins module. """ import io import os import re import sys import types import shlex import signal import atexit import pathlib import inspect import warnings import builtins import itertools import subprocess import contextlib import collections.abc as cabc from xonsh.ast import AST from xonsh.lazyasd import LazyObject, lazyobject from xonsh.inspectors import Inspector from xonsh.aliases import Aliases, make_default_aliases from xonsh.environ import Env, default_env, locate_binary from xonsh.jobs import add_job from xonsh.platform import ON_POSIX, ON_WINDOWS, ON_WSL from xonsh.proc import ( PopenThread, ProcProxyThread, ProcProxy, ConsoleParallelReader, pause_call_resume, CommandPipeline, HiddenCommandPipeline, STDOUT_CAPTURE_KINDS, ) from xonsh.tools import ( suggest_commands, expand_path, globpath, XonshError, XonshCalledProcessError, ) from xonsh.lazyimps import pty, termios from xonsh.commands_cache import CommandsCache from xonsh.events import events import xonsh.completers.init BUILTINS_LOADED = False INSPECTOR = LazyObject(Inspector, globals(), "INSPECTOR") warnings.filterwarnings("once", category=DeprecationWarning) @lazyobject def AT_EXIT_SIGNALS(): sigs = ( signal.SIGABRT, signal.SIGFPE, signal.SIGILL, signal.SIGSEGV, signal.SIGTERM, ) if ON_POSIX: sigs += (signal.SIGTSTP, signal.SIGQUIT, signal.SIGHUP) return sigs def resetting_signal_handle(sig, f): """Sets a new signal handle that will automatically restore the old value once the new handle is finished. """ oldh = signal.getsignal(sig) def newh(s=None, frame=None): f(s, frame) signal.signal(sig, oldh) if sig != 0: sys.exit(sig) signal.signal(sig, newh) def helper(x, name=""): """Prints help about, and then returns that variable.""" INSPECTOR.pinfo(x, oname=name, detail_level=0) return x def superhelper(x, name=""): """Prints help about, and then returns that variable.""" INSPECTOR.pinfo(x, oname=name, detail_level=1) return x def reglob(path, parts=None, i=None): """Regular expression-based globbing.""" if parts is None: path = os.path.normpath(path) drive, tail = os.path.splitdrive(path) parts = tail.split(os.sep) d = os.sep if os.path.isabs(path) else "." d = os.path.join(drive, d) return reglob(d, parts, i=0) base = subdir = path if i == 0: if not os.path.isabs(base): base = "" elif len(parts) > 1: i += 1 regex = os.path.join(base, parts[i]) if ON_WINDOWS: # currently unable to access regex backslash sequences # on Windows due to paths using \. regex = regex.replace("\\", "\\\\") regex = re.compile(regex) files = os.listdir(subdir) files.sort() paths = [] i1 = i + 1 if i1 == len(parts): for f in files: p = os.path.join(base, f) if regex.fullmatch(p) is not None: paths.append(p) else: for f in files: p = os.path.join(base, f) if regex.fullmatch(p) is None or not os.path.isdir(p): continue paths += reglob(p, parts=parts, i=i1) return paths def path_literal(s): s = expand_path(s) return pathlib.Path(s) def regexsearch(s): s = expand_path(s) return reglob(s) def globsearch(s): csc = builtins.__xonsh__.env.get("CASE_SENSITIVE_COMPLETIONS") glob_sorted = builtins.__xonsh__.env.get("GLOB_SORTED") dotglob = builtins.__xonsh__.env.get("DOTGLOB") return globpath( s, ignore_case=(not csc), return_empty=True, sort_result=glob_sorted, include_dotfiles=dotglob, ) def pathsearch(func, s, pymode=False, pathobj=False): """ Takes a string and returns a list of file paths that match (regex, glob, or arbitrary search function). If pathobj=True, the return is a list of pathlib.Path objects instead of strings. """ if not callable(func) or len(inspect.signature(func).parameters) != 1: error = "%r is not a known path search function" raise XonshError(error % func) o = func(s) if pathobj and pymode: o = list(map(pathlib.Path, o)) no_match = [] if pymode else [s] return o if len(o) != 0 else no_match RE_SHEBANG = LazyObject(lambda: re.compile(r"#![ \t]*(.+?)$"), globals(), "RE_SHEBANG") def _is_binary(fname, limit=80): with open(fname, "rb") as f: for i in range(limit): char = f.read(1) if char == b"\0": return True if char == b"\n": return False if char == b"": return False return False def _un_shebang(x): if x == "/usr/bin/env": return [] elif any(x.startswith(i) for i in ["/usr/bin", "/usr/local/bin", "/bin"]): x = os.path.basename(x) elif x.endswith("python") or x.endswith("python.exe"): x = "python" if x == "xonsh": return ["python", "-m", "xonsh.main"] return [x] def get_script_subproc_command(fname, args): """Given the name of a script outside the path, returns a list representing an appropriate subprocess command to execute the script. Raises PermissionError if the script is not executable. """ # make sure file is executable if not os.access(fname, os.X_OK): raise PermissionError if ON_POSIX and not os.access(fname, os.R_OK): # on some systems, some important programs (e.g. sudo) will have # execute permissions but not read/write permissions. This enables # things with the SUID set to be run. Needs to come before _is_binary() # is called, because that function tries to read the file. return [fname] + args elif _is_binary(fname): # if the file is a binary, we should call it directly return [fname] + args if ON_WINDOWS: # Windows can execute various filetypes directly # as given in PATHEXT _, ext = os.path.splitext(fname) if ext.upper() in builtins.__xonsh__.env.get("PATHEXT"): return [fname] + args # find interpreter with open(fname, "rb") as f: first_line = f.readline().decode().strip() m = RE_SHEBANG.match(first_line) # xonsh is the default interpreter if m is None: interp = ["xonsh"] else: interp = m.group(1).strip() if len(interp) > 0: interp = shlex.split(interp) else: interp = ["xonsh"] if ON_WINDOWS: o = [] for i in interp: o.extend(_un_shebang(i)) interp = o return interp + [fname] + args @lazyobject def _REDIR_REGEX(): name = r"(o(?:ut)?|e(?:rr)?|a(?:ll)?|&?\d?)" return re.compile("{r}(>?>|<){r}$".format(r=name)) _MODES = LazyObject(lambda: {">>": "a", ">": "w", "<": "r"}, globals(), "_MODES") _WRITE_MODES = LazyObject(lambda: frozenset({"w", "a"}), globals(), "_WRITE_MODES") _REDIR_ALL = LazyObject(lambda: frozenset({"&", "a", "all"}), globals(), "_REDIR_ALL") _REDIR_ERR = LazyObject(lambda: frozenset({"2", "e", "err"}), globals(), "_REDIR_ERR") _REDIR_OUT = LazyObject( lambda: frozenset({"", "1", "o", "out"}), globals(), "_REDIR_OUT" ) _E2O_MAP = LazyObject( lambda: frozenset( {"{}>{}".format(e, o) for e in _REDIR_ERR for o in _REDIR_OUT if o != ""} ), globals(), "_E2O_MAP", ) _O2E_MAP = LazyObject( lambda: frozenset( {"{}>{}".format(o, e) for e in _REDIR_ERR for o in _REDIR_OUT if o != ""} ), globals(), "_O2E_MAP", ) def _is_redirect(x): return isinstance(x, str) and _REDIR_REGEX.match(x) def safe_open(fname, mode, buffering=-1): """Safely attempts to open a file in for xonsh subprocs.""" # file descriptors try: return io.open(fname, mode, buffering=buffering) except PermissionError: raise XonshError("xonsh: {0}: permission denied".format(fname)) except FileNotFoundError: raise XonshError("xonsh: {0}: no such file or directory".format(fname)) except Exception: raise XonshError("xonsh: {0}: unable to open file".format(fname)) def safe_close(x): """Safely attempts to close an object.""" if not isinstance(x, io.IOBase): return if x.closed: return try: x.close() except Exception: pass def _parse_redirects(r, loc=None): """returns origin, mode, destination tuple""" orig, mode, dest = _REDIR_REGEX.match(r).groups() # redirect to fd if dest.startswith("&"): try: dest = int(dest[1:]) if loc is None: loc, dest = dest, "" # NOQA else: e = "Unrecognized redirection command: {}".format(r) raise XonshError(e) except (ValueError, XonshError): raise except Exception: pass mode = _MODES.get(mode, None) if mode == "r" and (len(orig) > 0 or len(dest) > 0): raise XonshError("Unrecognized redirection command: {}".format(r)) elif mode in _WRITE_MODES and len(dest) > 0: raise XonshError("Unrecognized redirection command: {}".format(r)) return orig, mode, dest def _redirect_streams(r, loc=None): """Returns stdin, stdout, stderr tuple of redirections.""" stdin = stdout = stderr = None no_ampersand = r.replace("&", "") # special case of redirecting stderr to stdout if no_ampersand in _E2O_MAP: stderr = subprocess.STDOUT return stdin, stdout, stderr elif no_ampersand in _O2E_MAP: stdout = 2 # using 2 as a flag, rather than using a file object return stdin, stdout, stderr # get streams orig, mode, dest = _parse_redirects(r) if mode == "r": stdin = safe_open(loc, mode) elif mode in _WRITE_MODES: if orig in _REDIR_ALL: stdout = stderr = safe_open(loc, mode) elif orig in _REDIR_OUT: stdout = safe_open(loc, mode) elif orig in _REDIR_ERR: stderr = safe_open(loc, mode) else: raise XonshError("Unrecognized redirection command: {}".format(r)) else: raise XonshError("Unrecognized redirection command: {}".format(r)) return stdin, stdout, stderr def default_signal_pauser(n, f): """Pauses a signal, as needed.""" signal.pause() def no_pg_xonsh_preexec_fn(): """Default subprocess preexec function for when there is no existing pipeline group. """ os.setpgrp() signal.signal(signal.SIGTSTP, default_signal_pauser) class SubprocSpec: """A container for specifying how a subprocess command should be executed. """ kwnames = ("stdin", "stdout", "stderr", "universal_newlines", "close_fds") def __init__( self, cmd, cls=subprocess.Popen, stdin=None, stdout=None, stderr=None, universal_newlines=False, close_fds=False, captured=False, ): """ Parameters ---------- cmd : list of str Command to be run. cls : Popen-like Class to run the subprocess with. stdin : file-like Popen file descriptor or flag for stdin. stdout : file-like Popen file descriptor or flag for stdout. stderr : file-like Popen file descriptor or flag for stderr. universal_newlines : bool Whether or not to use universal newlines. close_fds : bool Whether or not to close the file descriptiors when the process exits. captured : bool or str, optional The flag for if the subprocess is captured, may be one of: False for $[], 'stdout' for $(), 'hiddenobject' for ![], or 'object' for !(). Attributes ---------- args : list of str Arguments as originally supplied. alias : list of str, callable, or None The alias that was resolved for this command, if any. binary_loc : str or None Path to binary to execute. is_proxy : bool Whether or not the subprocess is or should be run as a proxy. background : bool Whether or not the subprocess should be started in the background. threadable : bool Whether or not the subprocess is able to be run in a background thread, rather than the main thread. pipeline_index : int or None The index number of this sepc into the pipeline that is being setup. last_in_pipeline : bool Whether the subprocess is the last in the execution pipeline. captured_stdout : file-like Handle to captured stdin captured_stderr : file-like Handle to captured stderr stack : list of FrameInfo namedtuples or None The stack of the call-site of alias, if the alias requires it. None otherwise. """ self._stdin = self._stdout = self._stderr = None # args self.cmd = list(cmd) self.cls = cls self.stdin = stdin self.stdout = stdout self.stderr = stderr self.universal_newlines = universal_newlines self.close_fds = close_fds self.captured = captured # pure attrs self.args = list(cmd) self.alias = None self.binary_loc = None self.is_proxy = False self.background = False self.threadable = True self.pipeline_index = None self.last_in_pipeline = False self.captured_stdout = None self.captured_stderr = None self.stack = None def __str__(self): s = self.__class__.__name__ + "(" + str(self.cmd) + ", " s += self.cls.__name__ + ", " kws = [n + "=" + str(getattr(self, n)) for n in self.kwnames] s += ", ".join(kws) + ")" return s def __repr__(self): s = self.__class__.__name__ + "(" + repr(self.cmd) + ", " s += self.cls.__name__ + ", " kws = [n + "=" + repr(getattr(self, n)) for n in self.kwnames] s += ", ".join(kws) + ")" return s # # Properties # @property def stdin(self): return self._stdin @stdin.setter def stdin(self, value): if self._stdin is None: self._stdin = value elif value is None: pass else: safe_close(value) msg = "Multiple inputs for stdin for {0!r}" msg = msg.format(" ".join(self.args)) raise XonshError(msg) @property def stdout(self): return self._stdout @stdout.setter def stdout(self, value): if self._stdout is None: self._stdout = value elif value is None: pass else: safe_close(value) msg = "Multiple redirections for stdout for {0!r}" msg = msg.format(" ".join(self.args)) raise XonshError(msg) @property def stderr(self): return self._stderr @stderr.setter def stderr(self, value): if self._stderr is None: self._stderr = value elif value is None: pass else: safe_close(value) msg = "Multiple redirections for stderr for {0!r}" msg = msg.format(" ".join(self.args)) raise XonshError(msg) # # Execution methods # def run(self, *, pipeline_group=None): """Launches the subprocess and returns the object.""" event_name = self._cmd_event_name() self._pre_run_event_fire(event_name) kwargs = {n: getattr(self, n) for n in self.kwnames} self.prep_env(kwargs) self.prep_preexec_fn(kwargs, pipeline_group=pipeline_group) if callable(self.alias): if "preexec_fn" in kwargs: kwargs.pop("preexec_fn") p = self.cls(self.alias, self.cmd, **kwargs) else: self._fix_null_cmd_bytes() p = self._run_binary(kwargs) p.spec = self p.last_in_pipeline = self.last_in_pipeline p.captured_stdout = self.captured_stdout p.captured_stderr = self.captured_stderr self._post_run_event_fire(event_name, p) return p def _run_binary(self, kwargs): try: bufsize = 1 p = self.cls(self.cmd, bufsize=bufsize, **kwargs) except PermissionError: e = "xonsh: subprocess mode: permission denied: {0}" raise XonshError(e.format(self.cmd[0])) except FileNotFoundError: cmd0 = self.cmd[0] e = "xonsh: subprocess mode: command not found: {0}".format(cmd0) env = builtins.__xonsh__.env sug = suggest_commands(cmd0, env, builtins.aliases) if len(sug.strip()) > 0: e += "\n" + suggest_commands(cmd0, env, builtins.aliases) raise XonshError(e) return p def prep_env(self, kwargs): """Prepares the environment to use in the subprocess.""" denv = builtins.__xonsh__.env.detype() if ON_WINDOWS: # Over write prompt variable as xonsh's $PROMPT does # not make much sense for other subprocs denv["PROMPT"] = "$P$G" kwargs["env"] = denv def prep_preexec_fn(self, kwargs, pipeline_group=None): """Prepares the 'preexec_fn' keyword argument""" if not ON_POSIX: return if not builtins.__xonsh__.env.get("XONSH_INTERACTIVE"): return if pipeline_group is None or ON_WSL: # If there is no pipeline group # or the platform is windows subsystem for linux (WSL) xonsh_preexec_fn = no_pg_xonsh_preexec_fn else: def xonsh_preexec_fn(): """Preexec function bound to a pipeline group.""" os.setpgid(0, pipeline_group) signal.signal(signal.SIGTSTP, default_signal_pauser) kwargs["preexec_fn"] = xonsh_preexec_fn def _fix_null_cmd_bytes(self): # Popen does not accept null bytes in its input commands. # That doesn't stop some subprocesses from using them. Here we # escape them just in case. cmd = self.cmd for i in range(len(cmd)): cmd[i] = cmd[i].replace("\0", "\\0") def _cmd_event_name(self): if callable(self.alias): return self.alias.__name__ elif self.binary_loc is None: return "" else: return os.path.basename(self.binary_loc) def _pre_run_event_fire(self, name): event_name = "on_pre_spec_run_" + name if events.exists(event_name): event = getattr(events, event_name) event.fire(spec=self) def _post_run_event_fire(self, name, proc): event_name = "on_post_spec_run_" + name if events.exists(event_name): event = getattr(events, event_name) event.fire(spec=self, proc=proc) # # Building methods # @classmethod def build(kls, cmd, *, cls=subprocess.Popen, **kwargs): """Creates an instance of the subprocess command, with any modifications and adjustments based on the actual cmd that was received. """ # modifications that do not alter cmds may come before creating instance spec = kls(cmd, cls=cls, **kwargs) # modifications that alter cmds must come after creating instance # perform initial redirects spec.redirect_leading() spec.redirect_trailing() # apply aliases spec.resolve_alias() spec.resolve_binary_loc() spec.resolve_auto_cd() spec.resolve_executable_commands() spec.resolve_alias_cls() spec.resolve_stack() return spec def redirect_leading(self): """Manage leading redirects such as with '< input.txt COMMAND'. """ while len(self.cmd) >= 3 and self.cmd[0] == "<": self.stdin = safe_open(self.cmd[1], "r") self.cmd = self.cmd[2:] def redirect_trailing(self): """Manages trailing redirects.""" while True: cmd = self.cmd if len(cmd) >= 3 and _is_redirect(cmd[-2]): streams = _redirect_streams(cmd[-2], cmd[-1]) self.stdin, self.stdout, self.stderr = streams self.cmd = cmd[:-2] elif len(cmd) >= 2 and _is_redirect(cmd[-1]): streams = _redirect_streams(cmd[-1]) self.stdin, self.stdout, self.stderr = streams self.cmd = cmd[:-1] else: break def resolve_alias(self): """Sets alias in command, if applicable.""" cmd0 = self.cmd[0] if callable(cmd0): alias = cmd0 else: alias = builtins.aliases.get(cmd0, None) self.alias = alias def resolve_binary_loc(self): """Sets the binary location""" alias = self.alias if alias is None: binary_loc = locate_binary(self.cmd[0]) elif callable(alias): binary_loc = None else: binary_loc = locate_binary(alias[0]) self.binary_loc = binary_loc def resolve_auto_cd(self): """Implements AUTO_CD functionality.""" if not ( self.alias is None and self.binary_loc is None and len(self.cmd) == 1 and builtins.__xonsh__.env.get("AUTO_CD") and os.path.isdir(self.cmd[0]) ): return self.cmd.insert(0, "cd") self.alias = builtins.aliases.get("cd", None) def resolve_executable_commands(self): """Resolve command executables, if applicable.""" alias = self.alias if alias is None: pass elif callable(alias): self.cmd.pop(0) return else: self.cmd = alias + self.cmd[1:] # resolve any redirects the aliases may have applied self.redirect_leading() self.redirect_trailing() if self.binary_loc is None: return try: self.cmd = get_script_subproc_command(self.binary_loc, self.cmd[1:]) except PermissionError: e = "xonsh: subprocess mode: permission denied: {0}" raise XonshError(e.format(self.cmd[0])) def resolve_alias_cls(self): """Determine which proxy class to run an alias with.""" alias = self.alias if not callable(alias): return self.is_proxy = True thable = getattr(alias, "__xonsh_threadable__", True) cls = ProcProxyThread if thable else ProcProxy self.cls = cls self.threadable = thable # also check capturability, while we are here cpable = getattr(alias, "__xonsh_capturable__", self.captured) self.captured = cpable def resolve_stack(self): """Computes the stack for a callable alias's call-site, if needed.""" if not callable(self.alias): return # check that we actual need the stack sig = inspect.signature(self.alias) if len(sig.parameters) <= 5 and "stack" not in sig.parameters: return # compute the stack, and filter out these build methods # run_subproc() is the 4th command in the stack # we want to filter out one up, e.g. subproc_captured_hiddenobject() # after that the stack from the call site starts. stack = inspect.stack(context=0) assert stack[3][3] == "run_subproc", "xonsh stack has changed!" del stack[:5] self.stack = stack def _safe_pipe_properties(fd, use_tty=False): """Makes sure that a pipe file descriptor properties are sane.""" if not use_tty: return # due to some weird, long standing issue in Python, PTYs come out # replacing newline \n with \r\n. This causes issues for raw unix # protocols, like git and ssh, which expect unix line endings. # see https://mail.python.org/pipermail/python-list/2013-June/650460.html # for more details and the following solution. props = termios.tcgetattr(fd) props[1] = props[1] & (~termios.ONLCR) | termios.ONLRET termios.tcsetattr(fd, termios.TCSANOW, props) def _update_last_spec(last): captured = last.captured last.last_in_pipeline = True if not captured: return callable_alias = callable(last.alias) if callable_alias: pass else: cmds_cache = builtins.__xonsh__.commands_cache thable = cmds_cache.predict_threadable( last.args ) and cmds_cache.predict_threadable(last.cmd) if captured and thable: last.cls = PopenThread elif not thable: # foreground processes should use Popen last.threadable = False if captured == "object" or captured == "hiddenobject": # CommandPipeline objects should not pipe stdout, stderr return # cannot used PTY pipes for aliases, for some dark reason, # and must use normal pipes instead. use_tty = ON_POSIX and not callable_alias # Do not set standard in! Popen is not a fan of redirections here # set standard out if last.stdout is not None: last.universal_newlines = True elif captured in STDOUT_CAPTURE_KINDS: last.universal_newlines = False r, w = os.pipe() last.stdout = safe_open(w, "wb") last.captured_stdout = safe_open(r, "rb") elif builtins.__xonsh__.stdout_uncaptured is not None: last.universal_newlines = True last.stdout = builtins.__xonsh__.stdout_uncaptured last.captured_stdout = last.stdout elif ON_WINDOWS and not callable_alias: last.universal_newlines = True last.stdout = None # must truly stream on windows last.captured_stdout = ConsoleParallelReader(1) else: last.universal_newlines = True r, w = pty.openpty() if use_tty else os.pipe() _safe_pipe_properties(w, use_tty=use_tty) last.stdout = safe_open(w, "w") _safe_pipe_properties(r, use_tty=use_tty) last.captured_stdout = safe_open(r, "r") # set standard error if last.stderr is not None: pass elif captured == "object": r, w = os.pipe() last.stderr = safe_open(w, "w") last.captured_stderr = safe_open(r, "r") elif builtins.__xonsh__.stderr_uncaptured is not None: last.stderr = builtins.__xonsh__.stderr_uncaptured last.captured_stderr = last.stderr elif ON_WINDOWS and not callable_alias: last.universal_newlines = True last.stderr = None # must truly stream on windows else: r, w = pty.openpty() if use_tty else os.pipe() _safe_pipe_properties(w, use_tty=use_tty) last.stderr = safe_open(w, "w") _safe_pipe_properties(r, use_tty=use_tty) last.captured_stderr = safe_open(r, "r") # redirect stdout to stderr, if we should if isinstance(last.stdout, int) and last.stdout == 2: # need to use private interface to avoid duplication. last._stdout = last.stderr # redirect stderr to stdout, if we should if callable_alias and last.stderr == subprocess.STDOUT: last._stderr = last.stdout last.captured_stderr = last.captured_stdout def cmds_to_specs(cmds, captured=False): """Converts a list of cmds to a list of SubprocSpec objects that are ready to be executed. """ # first build the subprocs independently and separate from the redirects i = 0 specs = [] redirects = [] for cmd in cmds: if isinstance(cmd, str): redirects.append(cmd) else: if cmd[-1] == "&": cmd = cmd[:-1] redirects.append("&") spec = SubprocSpec.build(cmd, captured=captured) spec.pipeline_index = i specs.append(spec) i += 1 # now modify the subprocs based on the redirects. for i, redirect in enumerate(redirects): if redirect == "|": # these should remain integer file descriptors, and not Python # file objects since they connect processes. r, w = os.pipe() specs[i].stdout = w specs[i + 1].stdin = r elif redirect == "&" and i == len(redirects) - 1: specs[-1].background = True else: raise XonshError("unrecognized redirect {0!r}".format(redirect)) # Apply boundary conditions _update_last_spec(specs[-1]) return specs def _should_set_title(captured=False): env = builtins.__xonsh__.env return ( env.get("XONSH_INTERACTIVE") and not env.get("XONSH_STORE_STDOUT") and captured not in STDOUT_CAPTURE_KINDS and builtins.__xonsh__.shell is not None ) def run_subproc(cmds, captured=False): """Runs a subprocess, in its many forms. This takes a list of 'commands,' which may be a list of command line arguments or a string, representing a special connecting character. For example:: $ ls | grep wakka is represented by the following cmds:: [['ls'], '|', ['grep', 'wakka']] Lastly, the captured argument affects only the last real command. """ specs = cmds_to_specs(cmds, captured=captured) captured = specs[-1].captured if captured == "hiddenobject": command = HiddenCommandPipeline(specs) else: command = CommandPipeline(specs) proc = command.proc background = command.spec.background if not all(x.is_proxy for x in specs): add_job( { "cmds": cmds, "pids": [i.pid for i in command.procs], "obj": proc, "bg": background, "pipeline": command, "pgrp": command.term_pgid, } ) if _should_set_title(captured=captured): # set title here to get currently executing command pause_call_resume(proc, builtins.__xonsh__.shell.settitle) else: # for some reason, some programs are in a stopped state when the flow # reaches this point, hence a SIGCONT should be sent to `proc` to make # sure that the shell doesn't hang. This `pause_call_resume` invocation # does this pause_call_resume(proc, int) # create command or return if backgrounding. if background: return # now figure out what we should return. if captured == "stdout": command.end() return command.output elif captured == "object": return command elif captured == "hiddenobject": command.end() return command else: command.end() return def subproc_captured_stdout(*cmds): """Runs a subprocess, capturing the output. Returns the stdout that was produced as a str. """ return run_subproc(cmds, captured="stdout") def subproc_captured_inject(*cmds): """Runs a subprocess, capturing the output. Returns a list of whitespace-separated strings of the stdout that was produced. The string is split using xonsh's lexer, rather than Python's str.split() or shlex.split(). """ s = run_subproc(cmds, captured="stdout") toks = builtins.__xonsh__.execer.parser.lexer.split(s.strip()) return toks def subproc_captured_object(*cmds): """ Runs a subprocess, capturing the output. Returns an instance of CommandPipeline representing the completed command. """ return run_subproc(cmds, captured="object") def subproc_captured_hiddenobject(*cmds): """Runs a subprocess, capturing the output. Returns an instance of HiddenCommandPipeline representing the completed command. """ return run_subproc(cmds, captured="hiddenobject") def subproc_uncaptured(*cmds): """Runs a subprocess, without capturing the output. Returns the stdout that was produced as a str. """ return run_subproc(cmds, captured=False) def ensure_list_of_strs(x): """Ensures that x is a list of strings.""" if isinstance(x, str): rtn = [x] elif isinstance(x, cabc.Sequence): rtn = [i if isinstance(i, str) else str(i) for i in x] else: rtn = [str(x)] return rtn def list_of_strs_or_callables(x): """Ensures that x is a list of strings or functions""" if isinstance(x, str) or callable(x): rtn = [x] elif isinstance(x, cabc.Iterable): rtn = [i if isinstance(i, str) or callable(i) else str(i) for i in x] else: rtn = [str(x)] return rtn def list_of_list_of_strs_outer_product(x): """Takes an outer product of a list of strings""" lolos = map(ensure_list_of_strs, x) rtn = [] for los in itertools.product(*lolos): s = "".join(los) if "*" in s: rtn.extend(builtins.__xonsh__.glob(s)) else: rtn.append(builtins.__xonsh__.expand_path(s)) return rtn @lazyobject def MACRO_FLAG_KINDS(): return { "s": str, "str": str, "string": str, "a": AST, "ast": AST, "c": types.CodeType, "code": types.CodeType, "compile": types.CodeType, "v": eval, "eval": eval, "x": exec, "exec": exec, "t": type, "type": type, } def _convert_kind_flag(x): """Puts a kind flag (string) a canonical form.""" x = x.lower() kind = MACRO_FLAG_KINDS.get(x, None) if kind is None: raise TypeError("{0!r} not a recognized macro type.".format(x)) return kind def convert_macro_arg(raw_arg, kind, glbs, locs, *, name="", macroname=""): """Converts a string macro argument based on the requested kind. Parameters ---------- raw_arg : str The str representation of the macro argument. kind : object A flag or type representing how to convert the argument. glbs : Mapping The globals from the call site. locs : Mapping or None The locals from the call site. name : str, optional The macro argument name. macroname : str, optional The name of the macro itself. Returns ------- The converted argument. """ # munge kind and mode to start mode = None if isinstance(kind, cabc.Sequence) and not isinstance(kind, str): # have (kind, mode) tuple kind, mode = kind if isinstance(kind, str): kind = _convert_kind_flag(kind) if kind is str or kind is None: return raw_arg # short circuit since there is nothing else to do # select from kind and convert execer = builtins.__xonsh__.execer filename = macroname + "(" + name + ")" if kind is AST: ctx = set(dir(builtins)) | set(glbs.keys()) if locs is not None: ctx |= set(locs.keys()) mode = mode or "eval" if mode != "eval" and not raw_arg.endswith("\n"): raw_arg += "\n" arg = execer.parse(raw_arg, ctx, mode=mode, filename=filename) elif kind is types.CodeType or kind is compile: # NOQA mode = mode or "eval" arg = execer.compile( raw_arg, mode=mode, glbs=glbs, locs=locs, filename=filename ) elif kind is eval: arg = execer.eval(raw_arg, glbs=glbs, locs=locs, filename=filename) elif kind is exec: mode = mode or "exec" if not raw_arg.endswith("\n"): raw_arg += "\n" arg = execer.exec(raw_arg, mode=mode, glbs=glbs, locs=locs, filename=filename) elif kind is type: arg = type(execer.eval(raw_arg, glbs=glbs, locs=locs, filename=filename)) else: msg = "kind={0!r} and mode={1!r} was not recognized for macro " "argument {2!r}" raise TypeError(msg.format(kind, mode, name)) return arg @contextlib.contextmanager def in_macro_call(f, glbs, locs): """Attaches macro globals and locals temporarily to function as a context manager. Parameters ---------- f : callable object The function that is called as ``f(*args)``. glbs : Mapping The globals from the call site. locs : Mapping or None The locals from the call site. """ prev_glbs = getattr(f, "macro_globals", None) prev_locs = getattr(f, "macro_locals", None) f.macro_globals = glbs f.macro_locals = locs yield if prev_glbs is None: del f.macro_globals else: f.macro_globals = prev_glbs if prev_locs is None: del f.macro_locals else: f.macro_locals = prev_locs def call_macro(f, raw_args, glbs, locs): """Calls a function as a macro, returning its result. Parameters ---------- f : callable object The function that is called as ``f(*args)``. raw_args : tuple of str The str representation of arguments of that were passed into the macro. These strings will be parsed, compiled, evaled, or left as a string depending on the annotations of f. glbs : Mapping The globals from the call site. locs : Mapping or None The locals from the call site. """ sig = inspect.signature(f) empty = inspect.Parameter.empty macroname = f.__name__ i = 0 args = [] for (key, param), raw_arg in zip(sig.parameters.items(), raw_args): i += 1 if raw_arg == "*": break kind = param.annotation if kind is empty or kind is None: kind = str arg = convert_macro_arg( raw_arg, kind, glbs, locs, name=key, macroname=macroname ) args.append(arg) reg_args, kwargs = _eval_regular_args(raw_args[i:], glbs, locs) args += reg_args with in_macro_call(f, glbs, locs): rtn = f(*args, **kwargs) return rtn @lazyobject def KWARG_RE(): return re.compile(r"([A-Za-z_]\w*=|\*\*)") def _starts_as_arg(s): """Tests if a string starts as a non-kwarg string would.""" return KWARG_RE.match(s) is None def _eval_regular_args(raw_args, glbs, locs): if not raw_args: return [], {} arglist = list(itertools.takewhile(_starts_as_arg, raw_args)) kwarglist = raw_args[len(arglist) :] execer = builtins.__xonsh__.execer if not arglist: args = arglist kwargstr = "dict({})".format(", ".join(kwarglist)) kwargs = execer.eval(kwargstr, glbs=glbs, locs=locs) elif not kwarglist: argstr = "({},)".format(", ".join(arglist)) args = execer.eval(argstr, glbs=glbs, locs=locs) kwargs = {} else: argstr = "({},)".format(", ".join(arglist)) kwargstr = "dict({})".format(", ".join(kwarglist)) both = "({}, {})".format(argstr, kwargstr) args, kwargs = execer.eval(both, glbs=glbs, locs=locs) return args, kwargs def enter_macro(obj, raw_block, glbs, locs): """Prepares to enter a context manager macro by attaching the contents of the macro block, globals, and locals to the object. These modifications are made in-place and the original object is returned. Parameters ---------- obj : context manager The object that is about to be entered via a with-statement. raw_block : str The str of the block that is the context body. This string will be parsed, compiled, evaled, or left as a string depending on the return annotation of obj.__enter__. glbs : Mapping The globals from the context site. locs : Mapping or None The locals from the context site. Returns ------- obj : context manager The same context manager but with the new macro information applied. """ # recurse down sequences if isinstance(obj, cabc.Sequence): for x in obj: enter_macro(x, raw_block, glbs, locs) return obj # convert block as needed kind = getattr(obj, "__xonsh_block__", str) macroname = getattr(obj, "__name__", "") block = convert_macro_arg( raw_block, kind, glbs, locs, name="", macroname=macroname ) # attach attrs obj.macro_globals = glbs obj.macro_locals = locs obj.macro_block = block return obj def load_builtins(execer=None, ctx=None): """Loads the xonsh builtins into the Python builtins. Sets the BUILTINS_LOADED variable to True. """ global BUILTINS_LOADED if not hasattr(builtins, "__xonsh__"): builtins.__xonsh__ = XonshSession(execer=execer, ctx=ctx) builtins.__xonsh__.load(execer=execer, ctx=ctx) builtins.__xonsh__.link_builtins(execer=execer) BUILTINS_LOADED = True def _lastflush(s=None, f=None): if hasattr(builtins, "__xonsh__"): if builtins.__xonsh__.history is not None: builtins.__xonsh__.history.flush(at_exit=True) def unload_builtins(): """Removes the xonsh builtins from the Python builtins, if the BUILTINS_LOADED is True, sets BUILTINS_LOADED to False, and returns. """ global BUILTINS_LOADED if not hasattr(builtins, "__xonsh__"): BUILTINS_LOADED = False return env = getattr(builtins.__xonsh__, "env", None) if isinstance(env, Env): env.undo_replace_env() if hasattr(builtins.__xonsh__, "pyexit"): builtins.exit = builtins.__xonsh__.pyexit if hasattr(builtins.__xonsh__, "pyquit"): builtins.quit = builtins.__xonsh__.pyquit if not BUILTINS_LOADED: return builtins.__xonsh__.unlink_builtins() delattr(builtins, "__xonsh__") BUILTINS_LOADED = False @contextlib.contextmanager def xonsh_builtins(execer=None): """A context manager for using the xonsh builtins only in a limited scope. Likely useful in testing. """ load_builtins(execer=execer) # temporary shims for old __xonsh_*__ builtins load_proxies() yield # temporary shims for old __xonsh_*__ builtins unload_proxies() unload_builtins() class XonshSession: """All components defining a xonsh session. """ def __init__(self, execer=None, ctx=None): """ Parameters --------- execer : Execer, optional Xonsh execution object, may be None to start ctx : Mapping, optional Context to start xonsh session with. """ self.execer = execer self.ctx = {} if ctx is None else ctx def load(self, execer=None, ctx=None): """Loads the session with default values. Parameters --------- execer : Execer, optional Xonsh execution object, may be None to start ctx : Mapping, optional Context to start xonsh session with. """ if ctx is not None: self.ctx = ctx self.env = Env(default_env()) self.help = helper self.superhelp = superhelper self.pathsearch = pathsearch self.globsearch = globsearch self.regexsearch = regexsearch self.glob = globpath self.expand_path = expand_path self.exit = False self.stdout_uncaptured = None self.stderr_uncaptured = None if hasattr(builtins, "exit"): self.pyexit = builtins.exit del builtins.exit if hasattr(builtins, "quit"): self.pyquit = builtins.quit del builtins.quit self.subproc_captured_stdout = subproc_captured_stdout self.subproc_captured_inject = subproc_captured_inject self.subproc_captured_object = subproc_captured_object self.subproc_captured_hiddenobject = subproc_captured_hiddenobject self.subproc_uncaptured = subproc_uncaptured self.execer = execer self.commands_cache = CommandsCache() self.all_jobs = {} self.ensure_list_of_strs = ensure_list_of_strs self.list_of_strs_or_callables = list_of_strs_or_callables self.list_of_list_of_strs_outer_product = list_of_list_of_strs_outer_product self.completers = xonsh.completers.init.default_completers() self.call_macro = call_macro self.enter_macro = enter_macro self.path_literal = path_literal self.builtins = _BuiltIns(execer) self.history = None self.shell = None def link_builtins(self, execer=None): # public built-ins builtins.XonshError = self.builtins.XonshError builtins.XonshCalledProcessError = self.builtins.XonshCalledProcessError builtins.evalx = None if execer is None else execer.eval builtins.execx = None if execer is None else execer.exec builtins.compilex = None if execer is None else execer.compile builtins.events = self.builtins.events # sneak the path search functions into the aliases # Need this inline/lazy import here since we use locate_binary that # relies on __xonsh__.env in default aliases builtins.default_aliases = builtins.aliases = Aliases(make_default_aliases()) atexit.register(_lastflush) for sig in AT_EXIT_SIGNALS: resetting_signal_handle(sig, _lastflush) def unlink_builtins(self): names = [ "XonshError", "XonshCalledProcessError", "evalx", "execx", "compilex", "default_aliases", ] for name in names: if hasattr(builtins, name): delattr(builtins, name) class _BuiltIns: def __init__(self, execer=None): # public built-ins self.XonshError = XonshError self.XonshCalledProcessError = XonshCalledProcessError self.evalx = None if execer is None else execer.eval self.execx = None if execer is None else execer.exec self.compilex = None if execer is None else execer.compile self.events = events class DynamicAccessProxy: """Proxies access dynamically.""" def __init__(self, refname, objname): """ Parameters ---------- refname : str '.'-separated string that represents the new, reference name that the user will access. objname : str '.'-separated string that represents the name where the target object actually lives that refname points to. """ super().__setattr__("refname", refname) super().__setattr__("objname", objname) @property def obj(self): """Dynamically grabs object""" names = self.objname.split(".") obj = builtins for name in names: obj = getattr(obj, name) return obj def __getattr__(self, name): return getattr(self.obj, name) def __setattr__(self, name, value): return super().__setattr__(self.obj, name, value) def __delattr__(self, name): return delattr(self.obj, name) def __getitem__(self, item): return self.obj.__getitem__(item) def __setitem__(self, item, value): return self.obj.__setitem__(item, value) def __delitem__(self, item): del self.obj[item] def __call__(self, *args, **kwargs): return self.obj.__call__(*args, **kwargs) class DeprecationWarningProxy: """Proxies access, but warns in the process.""" def __init__(self, oldname, newname): super().__setattr__("oldname", oldname) super().__setattr__("newname", newname) @property def obj(self): """Dynamically grabs object""" names = self.newname.split(".") obj = builtins for name in names: obj = getattr(obj, name) return obj def warn(self): """Issues deprecation warning.""" warnings.warn( "{} has been deprecated, please use {} instead.".format( self.oldname, self.newname ), DeprecationWarning, stacklevel=3, ) def __getattr__(self, name): self.warn() return getattr(self.obj, name) def __setattr__(self, name, value): self.warn() return super().__setattr__(self.obj, name, value) def __delattr__(self, name): self.warn() return delattr(self.obj, name) def __getitem__(self, item): self.warn() return self.obj.__getitem__(item) def __setitem__(self, item, value): self.warn() return self.obj.__setitem__(item, value) def __delitem__(self, item): self.warn() del self.obj[item] def __call__(self, *args, **kwargs): self.warn() return self.obj.__call__(*args, **kwargs) def load_proxies(): """Loads builtin dynamic access proxies. Also puts temporary shims in place for `__xonsh_*__` builtins. """ proxy_mapping = { "XonshError": "__xonsh__.builtins.XonshError", "XonshCalledProcessError": "__xonsh__.builtins.XonshCalledProcessError", "evalx": "__xonsh__.builtins.evalx", "execx": "__xonsh__.builtins.execx", "compilex": "__xonsh__.builtins.compilex", "events": "__xonsh__.builtins.events", } for refname, objname in proxy_mapping.items(): proxy = DynamicAccessProxy(refname, objname) setattr(builtins, refname, proxy) deprecated_mapping = { "__xonsh_env__": "__xonsh__.env", "__xonsh_history__": "__xonsh__.history", "__xonsh_ctx__": "__xonsh__.ctx", "__xonsh_help__": "__xonsh__.help", "__xonsh_superhelp__": "__xonsh__.superhelp", "__xonsh_pathsearch__": "__xonsh__.pathsearch", "__xonsh_globsearch__": "__xonsh__.globsearch", "__xonsh_regexsearch__": "__xonsh__.regexsearch", "__xonsh_glob__": "__xonsh__.glob", "__xonsh_expand_path__": "__xonsh__.expand_path", "__xonsh_exit__": "__xonsh__.exit", "__xonsh_stdout_uncaptured__": "__xonsh__.stdout_uncaptured", "__xonsh_stderr_uncaptured__": "__xonsh__.stderr_uncaptured", "__xonsh_subproc_captured_stdout__": "__xonsh__.subproc_captured_stdout", "__xonsh_subproc_captured_inject__": "__xonsh__.subproc_captured_inject", "__xonsh_subproc_captured_object__": "__xonsh__.subproc_captured_object", "__xonsh_subproc_captured_hiddenobject__": "__xonsh__.subproc_captured_hiddenobject", "__xonsh_subproc_uncaptured__": "__xonsh__.subproc_uncaptured", "__xonsh_execer__": "__xonsh__.execer", "__xonsh_commands_cache__": "__xonsh__.commands_cache", "__xonsh_all_jobs__": "__xonsh__.all_jobs", "__xonsh_ensure_list_of_strs__": "__xonsh__.ensure_list_of_strs", "__xonsh_list_of_strs_or_callables__": "__xonsh__.list_of_strs_or_callables", "__xonsh_list_of_list_of_strs_outer_product__": "__xonsh__.list_of_list_of_strs_outer_product", "__xonsh_completers__": "__xonsh__.completers", "__xonsh_call_macro__": "__xonsh__.call_macro", "__xonsh_enter_macro__": "__xonsh__.enter_macro", "__xonsh_path_literal__": "__xonsh__.path_literal", } for badname, goodname in deprecated_mapping.items(): proxy = DeprecationWarningProxy(badname, goodname) setattr(builtins, badname, proxy) if hasattr(builtins.__xonsh__, "pyexit"): builtins.__xonsh_pyexit__ = DeprecationWarningProxy( "builtins.__xonsh_pyexit__", "builtins.__xonsh__.pyexit" ) if hasattr(builtins.__xonsh__, "quit"): builtins.__xonsh_pyquit__ = DeprecationWarningProxy( "builtins.__xonsh_pyquit__", "builtins.__xonsh__.pyquit" ) def unload_proxies(): """Removes the xonsh builtins (proxies) from the Python builtins. """ if hasattr(builtins, "__xonsh_pyexit__"): builtins.exit = builtins.__xonsh_pyexit__ if hasattr(builtins, "__xonsh_pyquit__"): builtins.quit = builtins.__xonsh_pyquit__ names = [ "__xonsh_env__", "__xonsh_ctx__", "__xonsh_help__", "__xonsh_superhelp__", "__xonsh_pathsearch__", "__xonsh_globsearch__", "__xonsh_regexsearch__", "__xonsh_glob__", "__xonsh_expand_path__", "__xonsh_exit__", "__xonsh_stdout_uncaptured__", "__xonsh_stderr_uncaptured__", "__xonsh_pyexit__", "__xonsh_pyquit__", "__xonsh_subproc_captured_stdout__", "__xonsh_subproc_captured_inject__", "__xonsh_subproc_captured_object__", "__xonsh_subproc_captured_hiddenobject__", "__xonsh_subproc_uncaptured__", "__xonsh_execer__", "__xonsh_commands_cache__", "__xonsh_completers__", "__xonsh_call_macro__", "__xonsh_enter_macro__", "__xonsh_path_literal__", "XonshError", "XonshCalledProcessError", "evalx", "execx", "compilex", "default_aliases", "__xonsh_all_jobs__", "__xonsh_ensure_list_of_strs__", "__xonsh_list_of_strs_or_callables__", "__xonsh_list_of_list_of_strs_outer_product__", "__xonsh_history__", ] for name in names: if hasattr(builtins, name): delattr(builtins, name) ================================================ FILE: xonsh/codecache.py ================================================ """Tools for caching xonsh code.""" import os import sys import hashlib import marshal import builtins from xonsh import __version__ as XONSH_VERSION from xonsh.lazyasd import lazyobject from xonsh.platform import PYTHON_VERSION_INFO_BYTES def _splitpath(path, sofar=[]): folder, path = os.path.split(path) if path == "": return sofar[::-1] elif folder == "": return (sofar + [path])[::-1] else: return _splitpath(folder, sofar + [path]) @lazyobject def _CHARACTER_MAP(): cmap = {chr(o): "_%s" % chr(o + 32) for o in range(65, 91)} cmap.update({".": "_.", "_": "__"}) return cmap def _cache_renamer(path, code=False): if not code: path = os.path.realpath(path) o = ["".join(_CHARACTER_MAP.get(i, i) for i in w) for w in _splitpath(path)] o[-1] = "{}.{}".format(o[-1], sys.implementation.cache_tag) return o def _make_if_not_exists(dirname): if not os.path.isdir(dirname): os.makedirs(dirname) def should_use_cache(execer, mode): """ Return ``True`` if caching has been enabled for this mode (through command line flags or environment variables) """ if mode == "exec": return (execer.scriptcache or execer.cacheall) and ( builtins.__xonsh__.env["XONSH_CACHE_SCRIPTS"] or builtins.__xonsh__.env["XONSH_CACHE_EVERYTHING"] ) else: return execer.cacheall or builtins.__xonsh__.env["XONSH_CACHE_EVERYTHING"] def run_compiled_code(code, glb, loc, mode): """ Helper to run code in a given mode and context """ if code is None: return if mode in {"exec", "single"}: func = exec else: func = eval func(code, glb, loc) def get_cache_filename(fname, code=True): """ Return the filename of the cache for the given filename. Cache filenames are similar to those used by the Mercurial DVCS for its internal store. The ``code`` switch should be true if we should use the code store rather than the script store. """ datadir = builtins.__xonsh__.env["XONSH_DATA_DIR"] cachedir = os.path.join( datadir, "xonsh_code_cache" if code else "xonsh_script_cache" ) cachefname = os.path.join(cachedir, *_cache_renamer(fname, code=code)) return cachefname def update_cache(ccode, cache_file_name): """ Update the cache at ``cache_file_name`` to contain the compiled code represented by ``ccode``. """ if cache_file_name is not None: _make_if_not_exists(os.path.dirname(cache_file_name)) with open(cache_file_name, "wb") as cfile: cfile.write(XONSH_VERSION.encode() + b"\n") cfile.write(bytes(PYTHON_VERSION_INFO_BYTES) + b"\n") marshal.dump(ccode, cfile) def _check_cache_versions(cfile): # version data should be < 1 kb ver = cfile.readline(1024).strip() if ver != XONSH_VERSION.encode(): return False ver = cfile.readline(1024).strip() return ver == PYTHON_VERSION_INFO_BYTES def compile_code(filename, code, execer, glb, loc, mode): """ Wrapper for ``execer.compile`` to compile the given code """ try: if not code.endswith("\n"): code += "\n" old_filename = execer.filename execer.filename = filename ccode = execer.compile(code, glbs=glb, locs=loc, mode=mode, filename=filename) except Exception: raise finally: execer.filename = old_filename return ccode def script_cache_check(filename, cachefname): """ Check whether the script cache for a particular file is valid. Returns a tuple containing: a boolean representing whether the cached code should be used, and the cached code (or ``None`` if the cache should not be used). """ ccode = None run_cached = False if os.path.isfile(cachefname): if os.stat(cachefname).st_mtime >= os.stat(filename).st_mtime: with open(cachefname, "rb") as cfile: if not _check_cache_versions(cfile): return False, None ccode = marshal.load(cfile) run_cached = True return run_cached, ccode def run_script_with_cache(filename, execer, glb=None, loc=None, mode="exec"): """ Run a script, using a cached version if it exists (and the source has not changed), and updating the cache as necessary. """ run_cached = False use_cache = should_use_cache(execer, mode) cachefname = get_cache_filename(filename, code=False) if use_cache: run_cached, ccode = script_cache_check(filename, cachefname) if not run_cached: with open(filename, "r") as f: code = f.read() ccode = compile_code(filename, code, execer, glb, loc, mode) update_cache(ccode, cachefname) run_compiled_code(ccode, glb, loc, mode) def code_cache_name(code): """ Return an appropriate spoofed filename for the given code. """ if isinstance(code, str): _code = code.encode() else: _code = code return hashlib.md5(_code).hexdigest() def code_cache_check(cachefname): """ Check whether the code cache for a particular piece of code is valid. Returns a tuple containing: a boolean representing whether the cached code should be used, and the cached code (or ``None`` if the cache should not be used). """ ccode = None run_cached = False if os.path.isfile(cachefname): with open(cachefname, "rb") as cfile: if not _check_cache_versions(cfile): return False, None ccode = marshal.load(cfile) run_cached = True return run_cached, ccode def run_code_with_cache(code, execer, glb=None, loc=None, mode="exec"): """ Run a piece of code, using a cached version if it exists, and updating the cache as necessary. """ use_cache = should_use_cache(execer, mode) filename = code_cache_name(code) cachefname = get_cache_filename(filename, code=True) run_cached = False if use_cache: run_cached, ccode = code_cache_check(cachefname) if not run_cached: ccode = compile_code(filename, code, execer, glb, loc, mode) update_cache(ccode, cachefname) run_compiled_code(ccode, glb, loc, mode) ================================================ FILE: xonsh/color_tools.py ================================================ """Tools for color handling in xonsh. This includes Convert values between RGB hex codes and xterm-256 color codes. Parts of this file were originally forked from Micah Elliott http://MicahElliott.com Copyright (C) 2011 Micah Elliott. All rights reserved. WTFPL http://sam.zoy.org/wtfpl/ """ import re import math from xonsh.lazyasd import lazyobject, LazyObject RE_BACKGROUND = LazyObject( lambda: re.compile("(BG#|BGHEX|BACKGROUND)"), globals(), "RE_BACKGROUND" ) @lazyobject def BASE_XONSH_COLORS(): return { "BLACK": (0, 0, 0), "RED": (170, 0, 0), "GREEN": (0, 170, 0), "YELLOW": (170, 85, 0), "BLUE": (0, 0, 170), "PURPLE": (170, 0, 170), "CYAN": (0, 170, 170), "WHITE": (170, 170, 170), "INTENSE_BLACK": (85, 85, 85), "INTENSE_RED": (255, 85, 85), "INTENSE_GREEN": (85, 255, 85), "INTENSE_YELLOW": (255, 255, 85), "INTENSE_BLUE": (85, 85, 255), "INTENSE_PURPLE": (255, 85, 255), "INTENSE_CYAN": (85, 255, 255), "INTENSE_WHITE": (255, 255, 255), } @lazyobject def CLUT(): """color look-up table""" return [ # 8-bit, RGB hex # Primary 3-bit (8 colors). Unique representation! ("00", "000000"), ("01", "800000"), ("02", "008000"), ("03", "808000"), ("04", "000080"), ("05", "800080"), ("06", "008080"), ("07", "c0c0c0"), # Equivalent "bright" versions of original 8 colors. ("08", "808080"), ("09", "ff0000"), ("10", "00ff00"), ("11", "ffff00"), ("12", "0000ff"), ("13", "ff00ff"), ("14", "00ffff"), ("15", "ffffff"), # Strictly ascending. ("16", "000000"), ("17", "00005f"), ("18", "000087"), ("19", "0000af"), ("20", "0000d7"), ("21", "0000ff"), ("22", "005f00"), ("23", "005f5f"), ("24", "005f87"), ("25", "005faf"), ("26", "005fd7"), ("27", "005fff"), ("28", "008700"), ("29", "00875f"), ("30", "008787"), ("31", "0087af"), ("32", "0087d7"), ("33", "0087ff"), ("34", "00af00"), ("35", "00af5f"), ("36", "00af87"), ("37", "00afaf"), ("38", "00afd7"), ("39", "00afff"), ("40", "00d700"), ("41", "00d75f"), ("42", "00d787"), ("43", "00d7af"), ("44", "00d7d7"), ("45", "00d7ff"), ("46", "00ff00"), ("47", "00ff5f"), ("48", "00ff87"), ("49", "00ffaf"), ("50", "00ffd7"), ("51", "00ffff"), ("52", "5f0000"), ("53", "5f005f"), ("54", "5f0087"), ("55", "5f00af"), ("56", "5f00d7"), ("57", "5f00ff"), ("58", "5f5f00"), ("59", "5f5f5f"), ("60", "5f5f87"), ("61", "5f5faf"), ("62", "5f5fd7"), ("63", "5f5fff"), ("64", "5f8700"), ("65", "5f875f"), ("66", "5f8787"), ("67", "5f87af"), ("68", "5f87d7"), ("69", "5f87ff"), ("70", "5faf00"), ("71", "5faf5f"), ("72", "5faf87"), ("73", "5fafaf"), ("74", "5fafd7"), ("75", "5fafff"), ("76", "5fd700"), ("77", "5fd75f"), ("78", "5fd787"), ("79", "5fd7af"), ("80", "5fd7d7"), ("81", "5fd7ff"), ("82", "5fff00"), ("83", "5fff5f"), ("84", "5fff87"), ("85", "5fffaf"), ("86", "5fffd7"), ("87", "5fffff"), ("88", "870000"), ("89", "87005f"), ("90", "870087"), ("91", "8700af"), ("92", "8700d7"), ("93", "8700ff"), ("94", "875f00"), ("95", "875f5f"), ("96", "875f87"), ("97", "875faf"), ("98", "875fd7"), ("99", "875fff"), ("100", "878700"), ("101", "87875f"), ("102", "878787"), ("103", "8787af"), ("104", "8787d7"), ("105", "8787ff"), ("106", "87af00"), ("107", "87af5f"), ("108", "87af87"), ("109", "87afaf"), ("110", "87afd7"), ("111", "87afff"), ("112", "87d700"), ("113", "87d75f"), ("114", "87d787"), ("115", "87d7af"), ("116", "87d7d7"), ("117", "87d7ff"), ("118", "87ff00"), ("119", "87ff5f"), ("120", "87ff87"), ("121", "87ffaf"), ("122", "87ffd7"), ("123", "87ffff"), ("124", "af0000"), ("125", "af005f"), ("126", "af0087"), ("127", "af00af"), ("128", "af00d7"), ("129", "af00ff"), ("130", "af5f00"), ("131", "af5f5f"), ("132", "af5f87"), ("133", "af5faf"), ("134", "af5fd7"), ("135", "af5fff"), ("136", "af8700"), ("137", "af875f"), ("138", "af8787"), ("139", "af87af"), ("140", "af87d7"), ("141", "af87ff"), ("142", "afaf00"), ("143", "afaf5f"), ("144", "afaf87"), ("145", "afafaf"), ("146", "afafd7"), ("147", "afafff"), ("148", "afd700"), ("149", "afd75f"), ("150", "afd787"), ("151", "afd7af"), ("152", "afd7d7"), ("153", "afd7ff"), ("154", "afff00"), ("155", "afff5f"), ("156", "afff87"), ("157", "afffaf"), ("158", "afffd7"), ("159", "afffff"), ("160", "d70000"), ("161", "d7005f"), ("162", "d70087"), ("163", "d700af"), ("164", "d700d7"), ("165", "d700ff"), ("166", "d75f00"), ("167", "d75f5f"), ("168", "d75f87"), ("169", "d75faf"), ("170", "d75fd7"), ("171", "d75fff"), ("172", "d78700"), ("173", "d7875f"), ("174", "d78787"), ("175", "d787af"), ("176", "d787d7"), ("177", "d787ff"), ("178", "d7af00"), ("179", "d7af5f"), ("180", "d7af87"), ("181", "d7afaf"), ("182", "d7afd7"), ("183", "d7afff"), ("184", "d7d700"), ("185", "d7d75f"), ("186", "d7d787"), ("187", "d7d7af"), ("188", "d7d7d7"), ("189", "d7d7ff"), ("190", "d7ff00"), ("191", "d7ff5f"), ("192", "d7ff87"), ("193", "d7ffaf"), ("194", "d7ffd7"), ("195", "d7ffff"), ("196", "ff0000"), ("197", "ff005f"), ("198", "ff0087"), ("199", "ff00af"), ("200", "ff00d7"), ("201", "ff00ff"), ("202", "ff5f00"), ("203", "ff5f5f"), ("204", "ff5f87"), ("205", "ff5faf"), ("206", "ff5fd7"), ("207", "ff5fff"), ("208", "ff8700"), ("209", "ff875f"), ("210", "ff8787"), ("211", "ff87af"), ("212", "ff87d7"), ("213", "ff87ff"), ("214", "ffaf00"), ("215", "ffaf5f"), ("216", "ffaf87"), ("217", "ffafaf"), ("218", "ffafd7"), ("219", "ffafff"), ("220", "ffd700"), ("221", "ffd75f"), ("222", "ffd787"), ("223", "ffd7af"), ("224", "ffd7d7"), ("225", "ffd7ff"), ("226", "ffff00"), ("227", "ffff5f"), ("228", "ffff87"), ("229", "ffffaf"), ("230", "ffffd7"), ("231", "ffffff"), # Gray-scale range. ("232", "080808"), ("233", "121212"), ("234", "1c1c1c"), ("235", "262626"), ("236", "303030"), ("237", "3a3a3a"), ("238", "444444"), ("239", "4e4e4e"), ("240", "585858"), ("241", "626262"), ("242", "6c6c6c"), ("243", "767676"), ("244", "808080"), ("245", "8a8a8a"), ("246", "949494"), ("247", "9e9e9e"), ("248", "a8a8a8"), ("249", "b2b2b2"), ("250", "bcbcbc"), ("251", "c6c6c6"), ("252", "d0d0d0"), ("253", "dadada"), ("254", "e4e4e4"), ("255", "eeeeee"), ] def _str2hex(hexstr): return int(hexstr, 16) def _strip_hash(rgb): # Strip leading `#` if exists. if rgb.startswith("#"): rgb = rgb.lstrip("#") return rgb @lazyobject def SHORT_TO_RGB(): return dict(CLUT) @lazyobject def RGB_TO_SHORT(): return {v: k for k, v in SHORT_TO_RGB.items()} def short2rgb(short): return SHORT_TO_RGB[short] def rgb_to_256(rgb): """Find the closest ANSI 256 approximation to the given RGB value. >>> rgb2short('123456') ('23', '005f5f') >>> rgb2short('ffffff') ('231', 'ffffff') >>> rgb2short('0DADD6') # vimeo logo ('38', '00afd7') Parameters ---------- rgb : Hex code representing an RGB value, eg, 'abcdef' Returns ------- String between 0 and 255, compatible with xterm. """ rgb = rgb.lstrip("#") if len(rgb) == 0: return "0", "000000" incs = (0x00, 0x5F, 0x87, 0xAF, 0xD7, 0xFF) # Break 6-char RGB code into 3 integer vals. parts = rgb_to_ints(rgb) res = [] for part in parts: i = 0 while i < len(incs) - 1: s, b = incs[i], incs[i + 1] # smaller, bigger if s <= part <= b: s1 = abs(s - part) b1 = abs(b - part) if s1 < b1: closest = s else: closest = b res.append(closest) break i += 1 res = "".join([("%02.x" % i) for i in res]) equiv = RGB_TO_SHORT[res] return equiv, res rgb2short = rgb_to_256 @lazyobject def RE_RGB3(): return re.compile(r"(.)(.)(.)") @lazyobject def RE_RGB6(): return re.compile(r"(..)(..)(..)") def rgb_to_ints(rgb): if len(rgb) == 6: return tuple([int(h, 16) for h in RE_RGB6.split(rgb)[1:4]]) else: return tuple([int(h * 2, 16) for h in RE_RGB3.split(rgb)[1:4]]) def short_to_ints(short): """Coverts a short (256) color to a 3-tuple of ints.""" return rgb_to_ints(short2rgb(short)) def color_dist(x, y): return math.sqrt((x[0] - y[0]) ** 2 + (x[1] - y[1]) ** 2 + (x[2] - y[2]) ** 2) def find_closest_color(x, palette): return min(sorted(palette.keys())[::-1], key=lambda k: color_dist(x, palette[k])) def make_palette(strings): """Makes a color palette from a collection of strings.""" palette = {} for s in strings: while "#" in s: _, t = s.split("#", 1) t, _, s = t.partition(" ") palette[t] = rgb_to_ints(t) return palette ================================================ FILE: xonsh/commands_cache.py ================================================ # -*- coding: utf-8 -*- """Module for caching command & alias names as well as for predicting whether a command will be able to be run in the background. A background predictor is a function that accepts a single argument list and returns whether or not the process can be run in the background (returns True) or must be run the foreground (returns False). """ import os import time import builtins import argparse import collections.abc as cabc from xonsh.platform import ON_WINDOWS, ON_POSIX, pathbasename from xonsh.tools import executables_in from xonsh.lazyasd import lazyobject class CommandsCache(cabc.Mapping): """A lazy cache representing the commands available on the file system. The keys are the command names and the values a tuple of (loc, has_alias) where loc is either a str pointing to the executable on the file system or None (if no executable exists) and has_alias is a boolean flag for whether the command has an alias. """ def __init__(self): self._cmds_cache = {} self._path_checksum = None self._alias_checksum = None self._path_mtime = -1 self.threadable_predictors = default_threadable_predictors() def __contains__(self, key): _ = self.all_commands return self.lazyin(key) def __iter__(self): for cmd, (path, is_alias) in self.all_commands.items(): if ON_WINDOWS and path is not None: # All command keys are stored in uppercase on Windows. # This ensures the original command name is returned. cmd = pathbasename(path) yield cmd def __len__(self): return len(self.all_commands) def __getitem__(self, key): _ = self.all_commands return self.lazyget(key) def is_empty(self): """Returns whether the cache is populated or not.""" return len(self._cmds_cache) == 0 @staticmethod def get_possible_names(name): """Generates the possible `PATHEXT` extension variants of a given executable name on Windows as a list, conserving the ordering in `PATHEXT`. Returns a list as `name` being the only item in it on other platforms.""" if ON_WINDOWS: pathext = builtins.__xonsh__.env.get("PATHEXT", []) name = name.upper() return [name + ext for ext in ([""] + pathext)] else: return [name] @staticmethod def remove_dups(p): ret = list() for e in p: if e not in ret: ret.append(e) return ret @property def all_commands(self): paths = builtins.__xonsh__.env.get("PATH", []) paths = CommandsCache.remove_dups(paths) path_immut = tuple(x for x in paths if os.path.isdir(x)) # did PATH change? path_hash = hash(path_immut) cache_valid = path_hash == self._path_checksum self._path_checksum = path_hash # did aliases change? alss = getattr(builtins, "aliases", dict()) al_hash = hash(frozenset(alss)) cache_valid = cache_valid and al_hash == self._alias_checksum self._alias_checksum = al_hash # did the contents of any directory in PATH change? max_mtime = 0 for path in path_immut: mtime = os.stat(path).st_mtime if mtime > max_mtime: max_mtime = mtime cache_valid = cache_valid and (max_mtime <= self._path_mtime) self._path_mtime = max_mtime if cache_valid: return self._cmds_cache allcmds = {} for path in reversed(path_immut): # iterate backwards so that entries at the front of PATH overwrite # entries at the back. for cmd in executables_in(path): key = cmd.upper() if ON_WINDOWS else cmd allcmds[key] = (os.path.join(path, cmd), alss.get(key, None)) for cmd in alss: if cmd not in allcmds: key = cmd.upper() if ON_WINDOWS else cmd allcmds[key] = (cmd, True) self._cmds_cache = allcmds return allcmds def cached_name(self, name): """Returns the name that would appear in the cache, if it exists.""" if name is None: return None cached = pathbasename(name) if ON_WINDOWS: keys = self.get_possible_names(cached) cached = next((k for k in keys if k in self._cmds_cache), None) return cached def lazyin(self, key): """Checks if the value is in the current cache without the potential to update the cache. It just says whether the value is known *now*. This may not reflect precisely what is on the $PATH. """ return self.cached_name(key) in self._cmds_cache def lazyiter(self): """Returns an iterator over the current cache contents without the potential to update the cache. This may not reflect what is on the $PATH. """ return iter(self._cmds_cache) def lazylen(self): """Returns the length of the current cache contents without the potential to update the cache. This may not reflect precisely what is on the $PATH. """ return len(self._cmds_cache) def lazyget(self, key, default=None): """A lazy value getter.""" return self._cmds_cache.get(self.cached_name(key), default) def locate_binary(self, name, ignore_alias=False): """Locates an executable on the file system using the cache. Arguments --------- name : str name of binary to search for ignore_alias : bool, optional Force return of binary path even if alias of ``name`` exists (default ``False``) """ # make sure the cache is up to date by accessing the property _ = self.all_commands return self.lazy_locate_binary(name, ignore_alias) def lazy_locate_binary(self, name, ignore_alias=False): """Locates an executable in the cache, without checking its validity. Arguments --------- name : str name of binary to search for ignore_alias : bool, optional Force return of binary path even if alias of ``name`` exists (default ``False``) """ possibilities = self.get_possible_names(name) if ON_WINDOWS: # Windows users expect to be able to execute files in the same # directory without `./` local_bin = next((fn for fn in possibilities if os.path.isfile(fn)), None) if local_bin: return os.path.abspath(local_bin) cached = next((cmd for cmd in possibilities if cmd in self._cmds_cache), None) if cached: (path, alias) = self._cmds_cache[cached] ispure = path == pathbasename(path) if alias and ignore_alias and ispure: # pure alias, which we are ignoring return None else: return path elif os.path.isfile(name) and name != pathbasename(name): return name def is_only_functional_alias(self, name): """Returns whether or not a command is only a functional alias, and has no underlying executable. For example, the "cd" command is only available as a functional alias. """ _ = self.all_commands return self.lazy_is_only_functional_alias(name) def lazy_is_only_functional_alias(self, name): """Returns whether or not a command is only a functional alias, and has no underlying executable. For example, the "cd" command is only available as a functional alias. This search is performed lazily. """ val = self._cmds_cache.get(name, None) if val is None: return False return ( val == (name, True) and self.locate_binary(name, ignore_alias=True) is None ) def predict_threadable(self, cmd): """Predicts whether a command list is able to be run on a background thread, rather than the main thread. """ name = self.cached_name(cmd[0]) predictors = self.threadable_predictors if ON_WINDOWS: # On all names (keys) are stored in upper case so instead # we get the original cmd or alias name path, _ = self.lazyget(name, (None, None)) if path is None: return True else: name = pathbasename(path) if name not in predictors: pre, ext = os.path.splitext(name) if pre in predictors: predictors[name] = predictors[pre] if name not in predictors: predictors[name] = self.default_predictor(name, cmd[0]) predictor = predictors[name] return predictor(cmd[1:]) # # Background Predictors (as methods) # def default_predictor(self, name, cmd0): if ON_POSIX: return self.default_predictor_readbin( name, cmd0, timeout=0.1, failure=predict_true ) else: return predict_true def default_predictor_readbin(self, name, cmd0, timeout, failure): """Make a default predictor by analyzing the content of the binary. Should only works on POSIX. Return failure if the analysis fails. """ fname = cmd0 if os.path.isabs(cmd0) else None fname = cmd0 if fname is None and os.sep in cmd0 else fname fname = self.lazy_locate_binary(name) if fname is None else fname if fname is None: return failure if not os.path.isfile(fname): return failure try: fd = os.open(fname, os.O_RDONLY | os.O_NONBLOCK) except Exception: return failure # opening error search_for = { (b"ncurses",): [False], (b"libgpm",): [False], (b"isatty", b"tcgetattr", b"tcsetattr"): [False, False, False], } tstart = time.time() block = b"" while time.time() < tstart + timeout: previous_block = block try: block = os.read(fd, 2048) except Exception: # should not occur, except e.g. if a file is deleted a a dir is # created with the same name between os.path.isfile and os.open os.close(fd) return failure if len(block) == 0: os.close(fd) return predict_true # no keys of search_for found analyzed_block = previous_block + block for k, v in search_for.items(): for i in range(len(k)): if v[i]: continue if k[i] in analyzed_block: v[i] = True if all(v): os.close(fd) return predict_false # use one key of search_for os.close(fd) return failure # timeout # # Background Predictors # def predict_true(args): """Always say the process is threadable.""" return True def predict_false(args): """Never say the process is threadable.""" return False @lazyobject def SHELL_PREDICTOR_PARSER(): p = argparse.ArgumentParser("shell", add_help=False) p.add_argument("-c", nargs="?", default=None) p.add_argument("filename", nargs="?", default=None) return p def predict_shell(args): """Predict the backgroundability of the normal shell interface, which comes down to whether it is being run in subproc mode. """ ns, _ = SHELL_PREDICTOR_PARSER.parse_known_args(args) if ns.c is None and ns.filename is None: pred = False else: pred = True return pred @lazyobject def HELP_VER_PREDICTOR_PARSER(): p = argparse.ArgumentParser("cmd", add_help=False) p.add_argument("-h", "--help", dest="help", action="store_true", default=None) p.add_argument( "-v", "-V", "--version", dest="version", action="store_true", default=None ) return p def predict_help_ver(args): """Predict the backgroundability of commands that have help & version switches: -h, --help, -v, -V, --version. If either of these options is present, the command is assumed to print to stdout normally and is therefore threadable. Otherwise, the command is assumed to not be threadable. This is useful for commands, like top, that normally enter alternate mode but may not in certain circumstances. """ ns, _ = HELP_VER_PREDICTOR_PARSER.parse_known_args(args) pred = ns.help is not None or ns.version is not None return pred @lazyobject def HG_PREDICTOR_PARSER(): p = argparse.ArgumentParser("hg", add_help=False) p.add_argument("command") p.add_argument( "-i", "--interactive", action="store_true", default=False, dest="interactive" ) return p def predict_hg(args): """Predict if mercurial is about to be run in interactive mode. If it is interactive, predict False. If it isn't, predict True. Also predict False for certain commands, such as split. """ ns, _ = HG_PREDICTOR_PARSER.parse_known_args(args) if ns.command == "split": return False else: return not ns.interactive def default_threadable_predictors(): """Generates a new defaultdict for known threadable predictors. The default is to predict true. """ # alphabetical, for what it is worth. predictors = { "aurman": predict_false, "bash": predict_shell, "csh": predict_shell, "clear": predict_false, "cls": predict_false, "cmd": predict_shell, "cryptop": predict_false, "curl": predict_true, "ex": predict_false, "emacsclient": predict_false, "fish": predict_shell, "gvim": predict_help_ver, "hg": predict_hg, "htop": predict_help_ver, "ipython": predict_shell, "ksh": predict_shell, "less": predict_help_ver, "ls": predict_true, "man": predict_help_ver, "more": predict_help_ver, "mvim": predict_help_ver, "mutt": predict_help_ver, "nano": predict_help_ver, "nvim": predict_false, "ponysay": predict_help_ver, "psql": predict_false, "python": predict_shell, "python2": predict_shell, "python3": predict_shell, "repo": predict_help_ver, "ranger": predict_help_ver, "rview": predict_false, "rvim": predict_false, "scp": predict_false, "sh": predict_shell, "ssh": predict_false, "startx": predict_false, "sudo": predict_help_ver, "tcsh": predict_shell, "telnet": predict_false, "top": predict_help_ver, "vi": predict_false, "view": predict_false, "vim": predict_false, "vimpager": predict_help_ver, "weechat": predict_help_ver, "xclip": predict_help_ver, "xo": predict_help_ver, "xonsh": predict_shell, "xon.sh": predict_shell, "zsh": predict_shell, } return predictors ================================================ FILE: xonsh/completer.py ================================================ # -*- coding: utf-8 -*- """A (tab-)completer for xonsh.""" import builtins import collections.abc as cabc class Completer(object): """This provides a list of optional completions for the xonsh shell.""" def complete(self, prefix, line, begidx, endidx, ctx=None): """Complete the string, given a possible execution context. Parameters ---------- prefix : str The string to match line : str The line that prefix appears on. begidx : int The index in line that prefix starts on. endidx : int The index in line that prefix ends on. ctx : Iterable of str (ie dict, set, etc), optional Names in the current execution context. Returns ------- rtn : list of str Possible completions of prefix, sorted alphabetically. lprefix : int Length of the prefix to be replaced in the completion. """ ctx = ctx or {} for func in builtins.__xonsh__.completers.values(): try: out = func(prefix, line, begidx, endidx, ctx) except StopIteration: return set(), len(prefix) if isinstance(out, cabc.Sequence): res, lprefix = out else: res = out lprefix = len(prefix) if res is not None and len(res) != 0: def sortkey(s): return s.lstrip(''''"''').lower() return tuple(sorted(res, key=sortkey)), lprefix return set(), lprefix ================================================ FILE: xonsh/completers/__init__.py ================================================ # amalgamate exclude import os as _os if _os.getenv("XONSH_DEBUG", ""): pass else: import sys as _sys try: from xonsh.completers import __amalgam__ bash_completion = __amalgam__ _sys.modules["xonsh.completers.bash_completion"] = __amalgam__ completer = __amalgam__ _sys.modules["xonsh.completers.completer"] = __amalgam__ pip = __amalgam__ _sys.modules["xonsh.completers.pip"] = __amalgam__ tools = __amalgam__ _sys.modules["xonsh.completers.tools"] = __amalgam__ xompletions = __amalgam__ _sys.modules["xonsh.completers.xompletions"] = __amalgam__ _aliases = __amalgam__ _sys.modules["xonsh.completers._aliases"] = __amalgam__ commands = __amalgam__ _sys.modules["xonsh.completers.commands"] = __amalgam__ man = __amalgam__ _sys.modules["xonsh.completers.man"] = __amalgam__ path = __amalgam__ _sys.modules["xonsh.completers.path"] = __amalgam__ python = __amalgam__ _sys.modules["xonsh.completers.python"] = __amalgam__ base = __amalgam__ _sys.modules["xonsh.completers.base"] = __amalgam__ bash = __amalgam__ _sys.modules["xonsh.completers.bash"] = __amalgam__ dirs = __amalgam__ _sys.modules["xonsh.completers.dirs"] = __amalgam__ init = __amalgam__ _sys.modules["xonsh.completers.init"] = __amalgam__ del __amalgam__ except ImportError: pass del _sys del _os # amalgamate end ================================================ FILE: xonsh/completers/_aliases.py ================================================ import builtins import collections import xonsh.lazyasd as xl from xonsh.completers.tools import justify VALID_ACTIONS = xl.LazyObject( lambda: frozenset({"add", "remove", "list"}), globals(), "VALID_ACTIONS" ) def _add_one_completer(name, func, loc="end"): new = collections.OrderedDict() if loc == "start": new[name] = func for (k, v) in builtins.__xonsh__.completers.items(): new[k] = v elif loc == "end": for (k, v) in builtins.__xonsh__.completers.items(): new[k] = v new[name] = func else: direction, rel = loc[0], loc[1:] found = False for (k, v) in builtins.__xonsh__.completers.items(): if rel == k and direction == "<": new[name] = func found = True new[k] = v if rel == k and direction == ">": new[name] = func found = True if not found: new[name] = func builtins.__xonsh__.completers.clear() builtins.__xonsh__.completers.update(new) def _list_completers(args, stdin=None, stack=None): o = "Registered Completer Functions: \n" _comp = builtins.__xonsh__.completers ml = max((len(i) for i in _comp), default=0) _strs = [] for c in _comp: if _comp[c].__doc__ is None: doc = "No description provided" else: doc = " ".join(_comp[c].__doc__.split()) doc = justify(doc, 80, ml + 3) _strs.append("{: >{}} : {}".format(c, ml, doc)) return o + "\n".join(_strs) + "\n" def _remove_completer(args, stdin=None, stack=None): err = None if len(args) != 1: err = "completer remove takes exactly 1 argument." else: name = args[0] if name not in builtins.__xonsh__.completers: err = ("The name %s is not a registered " "completer function.") % name if err is None: del builtins.__xonsh__.completers[name] return else: return None, err + "\n", 1 def _register_completer(args, stdin=None, stack=None): err = None if len(args) not in {2, 3}: err = ( "completer add takes either 2 or 3 arguments.\n" "For help, run: completer help add" ) else: name = args[0] func_name = args[1] if name in builtins.__xonsh__.completers: err = ("The name %s is already a registered " "completer function.") % name else: if func_name in builtins.__xonsh__.ctx: func = builtins.__xonsh__.ctx[func_name] if not callable(func): err = "%s is not callable" % func_name else: for frame_info in stack: frame = frame_info[0] if func_name in frame.f_locals: func = frame.f_locals[func_name] break elif func_name in frame.f_globals: func = frame.f_globals[func_name] break else: err = "No such function: %s" % func_name if err is None: position = "start" if len(args) == 2 else args[2] _add_one_completer(name, func, position) else: return None, err + "\n", 1 def completer_alias(args, stdin=None, stdout=None, stderr=None, spec=None, stack=None): err = None if len(args) == 0 or args[0] not in (VALID_ACTIONS | {"help"}): err = ( "Please specify an action. Valid actions are: " '"add", "remove", "list", or "help".' ) elif args[0] == "help": if len(args) == 1 or args[1] not in VALID_ACTIONS: return ( "Valid actions are: add, remove, list. For help with a " "specific action, run: completer help ACTION\n" ) elif args[1] == "add": return COMPLETER_ADD_HELP_STR elif args[1] == "remove": return COMPLETER_REMOVE_HELP_STR elif args[1] == "list": return COMPLETER_LIST_HELP_STR if err is not None: return None, err + "\n", 1 if args[0] == "add": func = _register_completer elif args[0] == "remove": func = _remove_completer elif args[0] == "list": func = _list_completers return func(args[1:], stdin=stdin, stack=stack) COMPLETER_LIST_HELP_STR = """completer list: ordered list the active completers Usage: completer remove """ COMPLETER_REMOVE_HELP_STR = """completer remove: removes a completer from xonsh Usage: completer remove NAME NAME is a unique name of a completer (run "completer list" to see the current completers in order) """ COMPLETER_ADD_HELP_STR = """completer add: adds a new completer to xonsh Usage: completer add NAME FUNC [POS] NAME is a unique name to use in the listing (run "completer list" to see the current completers in order) FUNC is the name of a completer function to use. This should be a function of the following arguments, and should return a set of valid completions for the given prefix. If this completer should not be used in a given context, it should return an empty set or None. Arguments to FUNC: * prefix: the string to be matched * line: a string representing the whole current line, for context * begidx: the index at which prefix starts in line * endidx: the index at which prefix ends in line * ctx: the current Python environment If the completer expands the prefix in any way, it should return a tuple of two elements: the first should be the set of completions, and the second should be the length of the modified prefix (for an example, see xonsh.completers.path.complete_path). POS (optional) is a position into the list of completers at which the new completer should be added. It can be one of the following values: * "start" indicates that the completer should be added to the start of the list of completers (it should be run before all others) * "end" indicates that the completer should be added to the end of the list of completers (it should be run after all others) * ">KEY", where KEY is a pre-existing name, indicates that this should be added after the completer named KEY * " /dev/null || echo "-F _minimal" }} _complete_stmt=$(_get_complete_statement) if echo "$_complete_stmt" | grep --quiet -e "_minimal" then declare -f _completion_loader > /dev/null && _completion_loader {cmd} _complete_stmt=$(_get_complete_statement) fi _func=$(echo "$_complete_stmt" | grep -o -e '-F \w\+' | cut -d ' ' -f 2) declare -f "$_func" > /dev/null || exit 1 echo "$_complete_stmt" COMP_WORDS=({line}) COMP_LINE={comp_line} COMP_POINT=${{#COMP_LINE}} COMP_COUNT={end} COMP_CWORD={n} $_func {cmd} {prefix} {prev} # print out completions, right-stripped if they contain no internal spaces shopt -s extglob for ((i=0;i<${{#COMPREPLY[*]}};i++)) do no_spaces="${{COMPREPLY[i]//[[:space:]]}}" no_trailing_spaces="${{COMPREPLY[i]%%+([[:space:]])}}" if [[ "$no_spaces" == "$no_trailing_spaces" ]]; then echo "$no_trailing_spaces" else echo "${{COMPREPLY[i]}}" fi done """ def bash_completions( prefix, line, begidx, endidx, env=None, paths=None, command=None, quote_paths=_bash_quote_paths, **kwargs ): """Completes based on results from BASH completion. Parameters ---------- prefix : str The string to match line : str The line that prefix appears on. begidx : int The index in line that prefix starts on. endidx : int The index in line that prefix ends on. env : Mapping, optional The environment dict to execute the Bash subprocess in. paths : list or tuple of str or None, optional This is a list (or tuple) of strings that specifies where the ``bash_completion`` script may be found. The first valid path will be used. For better performance, bash-completion v2.x is recommended since it lazy-loads individual completion scripts. For both bash-completion v1.x and v2.x, paths of individual completion scripts (like ``.../completes/ssh``) do not need to be included here. The default values are platform dependent, but sane. command : str or None, optional The /path/to/bash to use. If None, it will be selected based on the from the environment and platform. quote_paths : callable, optional A functions that quotes file system paths. You shouldn't normally need this as the default is acceptable 99+% of the time. This function should return a set of the new paths and a boolean for whether the paths were quoted. Returns ------- rtn : set of str Possible completions of prefix lprefix : int Length of the prefix to be replaced in the completion. """ source = _get_bash_completions_source(paths) or "" if prefix.startswith("$"): # do not complete env variables return set(), 0 splt = line.split() cmd = splt[0] idx = n = 0 prev = "" for n, tok in enumerate(splt): if tok == prefix: idx = line.find(prefix, idx) if idx >= begidx: break prev = tok if len(prefix) == 0: prefix_quoted = '""' n += 1 else: prefix_quoted = shlex.quote(prefix) script = BASH_COMPLETE_SCRIPT.format( source=source, line=" ".join(shlex.quote(p) for p in splt), comp_line=shlex.quote(line), n=n, cmd=shlex.quote(cmd), end=endidx + 1, prefix=prefix_quoted, prev=shlex.quote(prev), ) if command is None: command = _bash_command(env=env) try: out = subprocess.check_output( [command, "-c", script], universal_newlines=True, stderr=subprocess.PIPE, env=env, ) if not out: raise ValueError except ( subprocess.CalledProcessError, FileNotFoundError, UnicodeDecodeError, ValueError, ): return set(), 0 out = out.splitlines() complete_stmt = out[0] out = set(out[1:]) # From GNU Bash document: The results of the expansion are prefix-matched # against the word being completed # Ensure input to `commonprefix` is a list (now required by Python 3.6) commprefix = os.path.commonprefix(list(out)) strip_len = 0 strip_prefix = prefix.strip("\"'") while strip_len < len(strip_prefix) and strip_len < len(commprefix): if commprefix[strip_len] == strip_prefix[strip_len]: break strip_len += 1 if "-o noquote" not in complete_stmt: out, need_quotes = quote_paths(out, "", "") if "-o nospace" in complete_stmt: out = set([x.rstrip() for x in out]) return out, max(len(prefix) - strip_len, 0) def bash_complete_line(line, return_line=True, **kwargs): """Provides the completion from the end of the line. Parameters ---------- line : str Line to complete return_line : bool, optional If true (default), will return the entire line, with the completion added. If false, this will instead return the strings to append to the original line. kwargs : optional All other keyword arguments are passed to the bash_completions() function. Returns ------- rtn : set of str Possible completions of prefix """ # set up for completing from the end of the line split = line.split() if len(split) > 1 and not line.endswith(" "): prefix = split[-1] begidx = len(line.rsplit(prefix)[0]) else: prefix = "" begidx = len(line) endidx = len(line) # get completions out, lprefix = bash_completions(prefix, line, begidx, endidx, **kwargs) # reformat output if return_line: preline = line[:-lprefix] rtn = {preline + o for o in out} else: rtn = {o[lprefix:] for o in out} return rtn def _bc_main(args=None): """Runs complete_line() and prints the output.""" from argparse import ArgumentParser p = ArgumentParser("bash_completions") p.add_argument( "--return-line", action="store_true", dest="return_line", default=True, help="will return the entire line, with the completion added", ) p.add_argument( "--no-return-line", action="store_false", dest="return_line", help="will instead return the strings to append to the original line", ) p.add_argument("line", help="line to complete") ns = p.parse_args(args=args) out = bash_complete_line(ns.line, return_line=ns.return_line) for o in sorted(out): print(o) if __name__ == "__main__": _bc_main() ================================================ FILE: xonsh/completers/commands.py ================================================ import os import builtins import xonsh.tools as xt import xonsh.platform as xp from xonsh.completers.tools import get_filter_function SKIP_TOKENS = {"sudo", "time", "timeit", "which", "showcmd", "man"} END_PROC_TOKENS = {"|", "||", "&&", "and", "or"} def complete_command(cmd, line, start, end, ctx): """ Returns a list of valid commands starting with the first argument """ space = " " out = { s + space for s in builtins.__xonsh__.commands_cache if get_filter_function()(s, cmd) } if xp.ON_WINDOWS: out |= {i for i in xt.executables_in(".") if i.startswith(cmd)} base = os.path.basename(cmd) if os.path.isdir(base): out |= { os.path.join(base, i) for i in xt.executables_in(base) if i.startswith(cmd) } return out def complete_skipper(cmd, line, start, end, ctx): """ Skip over several tokens (e.g., sudo) and complete based on the rest of the line. """ parts = line.split(" ") skip_part_num = 0 for i, s in enumerate(parts): if s in END_PROC_TOKENS: skip_part_num = i + 1 while len(parts) > skip_part_num: if parts[skip_part_num] not in SKIP_TOKENS: break skip_part_num += 1 if skip_part_num == 0: return set() # If there's no space following an END_PROC_TOKEN, insert one if parts[-1] in END_PROC_TOKENS: return(set(" "), 0) if len(parts) == skip_part_num + 1: comp_func = complete_command else: comp = builtins.__xonsh__.shell.shell.completer comp_func = comp.complete skip_len = len(" ".join(line[:skip_part_num])) + 1 return comp_func( cmd, " ".join(parts[skip_part_num:]), start - skip_len, end - skip_len, ctx ) ================================================ FILE: xonsh/completers/completer.py ================================================ import builtins def complete_completer(prefix, line, start, end, ctx): """ Completion for "completer" """ args = line.split(" ") if len(args) == 0 or args[0] != "completer": return None curix = args.index(prefix) compnames = set(builtins.__xonsh__.completers.keys()) if curix == 1: possible = {"list", "help", "add", "remove"} elif curix == 2: if args[1] == "help": possible = {"list", "add", "remove"} elif args[1] == "remove": possible = compnames else: raise StopIteration else: if args[1] != "add": raise StopIteration if curix == 3: possible = {i for i, j in builtins.__xonsh__.ctx.items() if callable(j)} elif curix == 4: possible = ( {"start", "end"} | {">" + n for n in compnames} | {"<" + n for n in compnames} ) else: raise StopIteration return {i for i in possible if i.startswith(prefix)} ================================================ FILE: xonsh/completers/dirs.py ================================================ from xonsh.completers.man import complete_from_man from xonsh.completers.path import complete_dir def complete_cd(prefix, line, start, end, ctx): """ Completion for "cd", includes only valid directory names. """ if start != 0 and line.split(" ")[0] == "cd": return complete_dir(prefix, line, start, end, ctx, True) return set() def complete_rmdir(prefix, line, start, end, ctx): """ Completion for "rmdir", includes only valid directory names. """ if start != 0 and line.split(" ")[0] == "rmdir": opts = { i for i in complete_from_man("-", "rmdir -", 6, 7, ctx) if i.startswith(prefix) } comps, lp = complete_dir(prefix, line, start, end, ctx, True) return comps | opts, lp return set() ================================================ FILE: xonsh/completers/init.py ================================================ """Constructor for xonsh completer objects.""" import collections from xonsh.completers.pip import complete_pip from xonsh.completers.man import complete_from_man from xonsh.completers.bash import complete_from_bash from xonsh.completers.base import complete_base from xonsh.completers.path import complete_path from xonsh.completers.dirs import complete_cd, complete_rmdir from xonsh.completers.python import ( complete_python, complete_import, complete_python_mode, ) from xonsh.completers.commands import complete_skipper from xonsh.completers.completer import complete_completer from xonsh.completers.xompletions import complete_xonfig, complete_xontrib def default_completers(): """Creates a copy of the default completers.""" return collections.OrderedDict( [ ("python_mode", complete_python_mode), ("base", complete_base), ("completer", complete_completer), ("skip", complete_skipper), ("pip", complete_pip), ("cd", complete_cd), ("rmdir", complete_rmdir), ("xonfig", complete_xonfig), ("xontrib", complete_xontrib), ("bash", complete_from_bash), ("man", complete_from_man), ("import", complete_import), ("python", complete_python), ("path", complete_path), ] ) ================================================ FILE: xonsh/completers/man.py ================================================ import os import re import pickle import builtins import subprocess import xonsh.lazyasd as xl from xonsh.completers.tools import get_filter_function OPTIONS = None OPTIONS_PATH = None @xl.lazyobject def SCRAPE_RE(): return re.compile(r"^(?:\s*(?:-\w|--[a-z0-9-]+)[\s,])+", re.M) @xl.lazyobject def INNER_OPTIONS_RE(): return re.compile(r"-\w|--[a-z0-9-]+") def complete_from_man(prefix, line, start, end, ctx): """ Completes an option name, based on the contents of the associated man page. """ global OPTIONS, OPTIONS_PATH if OPTIONS is None: datadir = builtins.__xonsh__.env["XONSH_DATA_DIR"] OPTIONS_PATH = os.path.join(datadir, "man_completions_cache") try: with open(OPTIONS_PATH, "rb") as f: OPTIONS = pickle.load(f) except Exception: OPTIONS = {} if not prefix.startswith("-"): return set() cmd = line.split()[0] if cmd not in OPTIONS: try: manpage = subprocess.Popen( ["man", cmd], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL ) # This is a trick to get rid of reverse line feeds text = subprocess.check_output(["col", "-b"], stdin=manpage.stdout) text = text.decode("utf-8") scraped_text = " ".join(SCRAPE_RE.findall(text)) matches = INNER_OPTIONS_RE.findall(scraped_text) OPTIONS[cmd] = matches with open(OPTIONS_PATH, "wb") as f: pickle.dump(OPTIONS, f) except Exception: return set() return {s for s in OPTIONS[cmd] if get_filter_function()(s, prefix)} ================================================ FILE: xonsh/completers/path.py ================================================ import os import re import ast import glob import builtins import xonsh.tools as xt import xonsh.platform as xp import xonsh.lazyasd as xl from xonsh.completers.tools import get_filter_function @xl.lazyobject def PATTERN_NEED_QUOTES(): pattern = r'\s`\$\{\}\,\*\(\)"\'\?&#' if xp.ON_WINDOWS: pattern += "%" pattern = "[" + pattern + "]" + r"|\band\b|\bor\b" return re.compile(pattern) def cd_in_command(line): """Returns True if "cd" is a token in the line, False otherwise.""" lexer = builtins.__xonsh__.execer.parser.lexer lexer.reset() lexer.input(line) have_cd = False for tok in lexer: if tok.type == "NAME" and tok.value == "cd": have_cd = True break return have_cd def _path_from_partial_string(inp, pos=None): if pos is None: pos = len(inp) partial = inp[:pos] startix, endix, quote = xt.check_for_partial_string(partial) _post = "" if startix is None: return None elif endix is None: string = partial[startix:] else: if endix != pos: _test = partial[endix:pos] if not any(i == " " for i in _test): _post = _test else: return None string = partial[startix:endix] end = xt.RE_STRING_START.sub("", quote) _string = string if not _string.endswith(end): _string = _string + end try: val = ast.literal_eval(_string) except (SyntaxError, ValueError): return None if isinstance(val, bytes): env = builtins.__xonsh__.env val = val.decode( encoding=env.get("XONSH_ENCODING"), errors=env.get("XONSH_ENCODING_ERRORS") ) return string + _post, val + _post, quote, end def _normpath(p): """ Wraps os.normpath() to avoid removing './' at the beginning and '/' at the end. On windows it does the same with backslashes """ initial_dotslash = p.startswith(os.curdir + os.sep) initial_dotslash |= xp.ON_WINDOWS and p.startswith(os.curdir + os.altsep) p = p.rstrip() trailing_slash = p.endswith(os.sep) trailing_slash |= xp.ON_WINDOWS and p.endswith(os.altsep) p = os.path.normpath(p) if initial_dotslash and p != ".": p = os.path.join(os.curdir, p) if trailing_slash: p = os.path.join(p, "") if xp.ON_WINDOWS and builtins.__xonsh__.env.get("FORCE_POSIX_PATHS"): p = p.replace(os.sep, os.altsep) return p def _startswithlow(x, start, startlow=None): if startlow is None: startlow = start.lower() return x.startswith(start) or x.lower().startswith(startlow) def _startswithnorm(x, start, startlow=None): return x.startswith(start) def _env(prefix): if prefix.startswith("$"): key = prefix[1:] return { "$" + k for k in builtins.__xonsh__.env if get_filter_function()(k, key) } return () def _dots(prefix): slash = xt.get_sep() if slash == "\\": slash = "" if prefix in {"", "."}: return ("." + slash, ".." + slash) elif prefix == "..": return (".." + slash,) else: return () def _add_cdpaths(paths, prefix): """Completes current prefix using CDPATH""" env = builtins.__xonsh__.env csc = env.get("CASE_SENSITIVE_COMPLETIONS") glob_sorted = env.get("GLOB_SORTED") for cdp in env.get("CDPATH"): test_glob = os.path.join(cdp, prefix) + "*" for s in xt.iglobpath( test_glob, ignore_case=(not csc), sort_result=glob_sorted ): if os.path.isdir(s): paths.add(os.path.basename(s)) def _quote_to_use(x): single = "'" double = '"' if single in x and double not in x: return double else: return single def _quote_paths(paths, start, end, append_end=True): expand_path = builtins.__xonsh__.expand_path out = set() space = " " backslash = "\\" double_backslash = "\\\\" slash = xt.get_sep() orig_start = start orig_end = end # quote on all or none, to make readline completes to max prefix need_quotes = any( re.search(PATTERN_NEED_QUOTES, x) or (backslash in x and slash != backslash) for x in paths ) for s in paths: start = orig_start end = orig_end if start == "" and need_quotes: start = end = _quote_to_use(s) if os.path.isdir(expand_path(s)): _tail = slash elif end == "": _tail = space else: _tail = "" if start != "" and "r" not in start and backslash in s: start = "r%s" % start s = s + _tail if end != "": if "r" not in start.lower(): s = s.replace(backslash, double_backslash) if s.endswith(backslash) and not s.endswith(double_backslash): s += backslash if end in s: s = s.replace(end, "".join("\\%s" % i for i in end)) s = start + s + end if append_end else start + s out.add(s) return out, need_quotes def _joinpath(path): # convert our tuple representation back into a string representing a path if path is None: return "" elif len(path) == 0: return "" elif path == ("",): return xt.get_sep() elif path[0] == "": return xt.get_sep() + _normpath(os.path.join(*path)) else: return _normpath(os.path.join(*path)) def _splitpath(path): # convert a path into an intermediate tuple representation # if this tuple starts with '', it means that the path was an absolute path path = _normpath(path) if path.startswith(xt.get_sep()): pre = ("",) else: pre = () return pre + _splitpath_helper(path, ()) def _splitpath_helper(path, sofar=()): folder, path = os.path.split(path) if path: sofar = sofar + (path,) if not folder or folder == xt.get_sep(): return sofar[::-1] elif xp.ON_WINDOWS and not path: return os.path.splitdrive(folder)[:1] + sofar[::-1] elif xp.ON_WINDOWS and os.path.splitdrive(path)[0]: return sofar[::-1] return _splitpath_helper(folder, sofar) def subsequence_match(ref, typed, csc): """ Detects whether typed is a subsequence of ref. Returns ``True`` if the characters in ``typed`` appear (in order) in ``ref``, regardless of exactly where in ``ref`` they occur. If ``csc`` is ``False``, ignore the case of ``ref`` and ``typed``. Used in "subsequence" path completion (e.g., ``~/u/ro`` expands to ``~/lou/carcohl``) """ if csc: return _subsequence_match_iter(ref, typed) else: return _subsequence_match_iter(ref.lower(), typed.lower()) def _subsequence_match_iter(ref, typed): if len(typed) == 0: return True elif len(ref) == 0: return False elif ref[0] == typed[0]: return _subsequence_match_iter(ref[1:], typed[1:]) else: return _subsequence_match_iter(ref[1:], typed) def _expand_one(sofar, nextone, csc): out = set() glob_sorted = builtins.__xonsh__.env.get("GLOB_SORTED") for i in sofar: _glob = os.path.join(_joinpath(i), "*") if i is not None else "*" for j in xt.iglobpath(_glob, sort_result=glob_sorted): j = os.path.basename(j) if subsequence_match(j, nextone, csc): out.add((i or ()) + (j,)) return out def complete_path(prefix, line, start, end, ctx, cdpath=True, filtfunc=None): """Completes based on a path name.""" # string stuff for automatic quoting path_str_start = "" path_str_end = "" append_end = True p = _path_from_partial_string(line, end) lprefix = len(prefix) if p is not None: lprefix = len(p[0]) prefix = p[1] path_str_start = p[2] path_str_end = p[3] if len(line) >= end + 1 and line[end] == path_str_end: append_end = False tilde = "~" paths = set() env = builtins.__xonsh__.env csc = env.get("CASE_SENSITIVE_COMPLETIONS") glob_sorted = env.get("GLOB_SORTED") prefix = glob.escape(prefix) for s in xt.iglobpath(prefix + "*", ignore_case=(not csc), sort_result=glob_sorted): paths.add(s) if len(paths) == 0 and env.get("SUBSEQUENCE_PATH_COMPLETION"): # this block implements 'subsequence' matching, similar to fish and zsh. # matches are based on subsequences, not substrings. # e.g., ~/u/ro completes to ~/lou/carcolh # see above functions for details. p = _splitpath(os.path.expanduser(prefix)) if len(p) != 0: if p[0] == "": basedir = ("",) p = p[1:] else: basedir = None matches_so_far = {basedir} for i in p: matches_so_far = _expand_one(matches_so_far, i, csc) paths |= {_joinpath(i) for i in matches_so_far} if len(paths) == 0 and env.get("FUZZY_PATH_COMPLETION"): threshold = env.get("SUGGEST_THRESHOLD") for s in xt.iglobpath( os.path.dirname(prefix) + "*", ignore_case=(not csc), sort_result=glob_sorted, ): if xt.levenshtein(prefix, s, threshold) < threshold: paths.add(s) if tilde in prefix: home = os.path.expanduser(tilde) paths = {s.replace(home, tilde) for s in paths} if cdpath and cd_in_command(line): _add_cdpaths(paths, prefix) paths = set(filter(filtfunc, paths)) paths, _ = _quote_paths( {_normpath(s) for s in paths}, path_str_start, path_str_end, append_end ) paths.update(filter(filtfunc, _dots(prefix))) paths.update(filter(filtfunc, _env(prefix))) return paths, lprefix def complete_dir(prefix, line, start, end, ctx, cdpath=False): return complete_path(prefix, line, start, end, cdpath, filtfunc=os.path.isdir) ================================================ FILE: xonsh/completers/pip.py ================================================ """Completers for pip.""" # pylint: disable=invalid-name, missing-docstring, unsupported-membership-test # pylint: disable=unused-argument, not-an-iterable import re import subprocess import xonsh.lazyasd as xl @xl.lazyobject def PIP_RE(): return re.compile(r"pip(?:\d|\.)*") @xl.lazyobject def PIP_LIST_RE(): return re.compile(r"pip(?:\d|\.)* (?:uninstall|show)") @xl.lazyobject def ALL_COMMANDS(): try: help_text = str( subprocess.check_output(["pip", "--help"], stderr=subprocess.DEVNULL) ) except FileNotFoundError: return [] commands = re.findall(r" (\w+) ", help_text) return [c for c in commands if c not in ["completion", "help"]] def complete_pip(prefix, line, begidx, endidx, ctx): """Completes python's package manager pip""" line_len = len(line.split()) if ( (line_len > 3) or (line_len > 2 and line.endswith(" ")) or (not PIP_RE.search(line)) ): return if PIP_LIST_RE.search(line): try: items = subprocess.check_output(["pip", "list"], stderr=subprocess.DEVNULL) except FileNotFoundError: return set() items = items.decode("utf-8").splitlines() return set(i.split()[0] for i in items if i.split()[0].startswith(prefix)) if (line_len > 1 and line.endswith(" ")) or line_len > 2: # "pip show " -> no complete (note space) return if prefix not in ALL_COMMANDS: suggestions = [c for c in ALL_COMMANDS if c.startswith(prefix)] if suggestions: return suggestions, len(prefix) return ALL_COMMANDS, len(prefix) ================================================ FILE: xonsh/completers/python.py ================================================ """Completers for Python code""" import re import sys import inspect import builtins import importlib import collections.abc as cabc import xonsh.tools as xt import xonsh.lazyasd as xl from xonsh.completers.tools import get_filter_function @xl.lazyobject def RE_ATTR(): return re.compile(r"([^\s\(\)]+(\.[^\s\(\)]+)*)\.(\w*)$") @xl.lazyobject def XONSH_EXPR_TOKENS(): return { "and ", "else", "for ", "if ", "in ", "is ", "lambda ", "not ", "or ", "+", "-", "/", "//", "%", "**", "|", "&", "~", "^", ">>", "<<", "<", "<=", ">", ">=", "==", "!=", ",", "?", "??", "$(", "${", "$[", "...", "![", "!(", "@(", "@$(", "@", } @xl.lazyobject def XONSH_STMT_TOKENS(): return { "as ", "assert ", "break", "class ", "continue", "def ", "del ", "elif ", "except ", "finally:", "from ", "global ", "import ", "nonlocal ", "pass", "raise ", "return ", "try:", "while ", "with ", "yield ", "-", "/", "//", "%", "**", "|", "&", "~", "^", ">>", "<<", "<", "<=", "->", "=", "+=", "-=", "*=", "/=", "%=", "**=", ">>=", "<<=", "&=", "^=", "|=", "//=", ";", ":", "..", } @xl.lazyobject def XONSH_TOKENS(): return set(XONSH_EXPR_TOKENS) | set(XONSH_STMT_TOKENS) def complete_python(prefix, line, start, end, ctx): """ Completes based on the contents of the current Python environment, the Python built-ins, and xonsh operators. If there are no matches, split on common delimiters and try again. """ rtn = _complete_python(prefix, line, start, end, ctx) if not rtn: prefix = ( re.split(r"\(|=|{|\[|,", prefix)[-1] if not prefix.startswith(",") else prefix ) start = line.find(prefix) rtn = _complete_python(prefix, line, start, end, ctx) return rtn, len(prefix) return rtn def _complete_python(prefix, line, start, end, ctx): """ Completes based on the contents of the current Python environment, the Python built-ins, and xonsh operators. """ if line != "": first = line.split()[0] if first in builtins.__xonsh__.commands_cache and first not in ctx: return set() filt = get_filter_function() rtn = set() if ctx is not None: if "." in prefix: rtn |= attr_complete(prefix, ctx, filt) args = python_signature_complete(prefix, line, end, ctx, filt) rtn |= args rtn |= {s for s in ctx if filt(s, prefix)} else: args = () if len(args) == 0: # not in a function call, so we can add non-expression tokens rtn |= {s for s in XONSH_TOKENS if filt(s, prefix)} else: rtn |= {s for s in XONSH_EXPR_TOKENS if filt(s, prefix)} rtn |= {s for s in dir(builtins) if filt(s, prefix)} return rtn def complete_python_mode(prefix, line, start, end, ctx): """ Python-mode completions for @( and ${ """ if not (prefix.startswith("@(") or prefix.startswith("${")): return set() prefix_start = prefix[:2] python_matches = complete_python(prefix[2:], line, start - 2, end - 2, ctx) if isinstance(python_matches, cabc.Sequence): python_matches = python_matches[0] return set(prefix_start + i for i in python_matches) def _safe_eval(expr, ctx): """Safely tries to evaluate an expression. If this fails, it will return a (None, None) tuple. """ _ctx = None xonsh_safe_eval = builtins.__xonsh__.execer.eval try: val = xonsh_safe_eval(expr, ctx, ctx, transform=False) _ctx = ctx except: # pylint:disable=bare-except try: val = xonsh_safe_eval(expr, builtins.__dict__, transform=False) _ctx = builtins.__dict__ except: # pylint:disable=bare-except val = _ctx = None return val, _ctx def attr_complete(prefix, ctx, filter_func): """Complete attributes of an object.""" attrs = set() m = RE_ATTR.match(prefix) if m is None: return attrs expr, attr = m.group(1, 3) expr = xt.subexpr_from_unbalanced(expr, "(", ")") expr = xt.subexpr_from_unbalanced(expr, "[", "]") expr = xt.subexpr_from_unbalanced(expr, "{", "}") val, _ctx = _safe_eval(expr, ctx) if val is None and _ctx is None: return attrs if len(attr) == 0: opts = [o for o in dir(val) if not o.startswith("_")] else: opts = [o for o in dir(val) if filter_func(o, attr)] prelen = len(prefix) for opt in opts: # check whether these options actually work (e.g., disallow 7.imag) _expr = "{0}.{1}".format(expr, opt) _val_, _ctx_ = _safe_eval(_expr, _ctx) if _val_ is None and _ctx_ is None: continue a = getattr(val, opt) if builtins.__xonsh__.env["COMPLETIONS_BRACKETS"]: if callable(a): rpl = opt + "(" elif isinstance(a, (cabc.Sequence, cabc.Mapping)): rpl = opt + "[" else: rpl = opt else: rpl = opt # note that prefix[:prelen-len(attr)] != prefix[:-len(attr)] # when len(attr) == 0. comp = prefix[: prelen - len(attr)] + rpl attrs.add(comp) return attrs def python_signature_complete(prefix, line, end, ctx, filter_func): """Completes a python function (or other callable) call by completing argument and keyword argument names. """ front = line[:end] if xt.is_balanced(front, "(", ")"): return set() funcname = xt.subexpr_before_unbalanced(front, "(", ")") val, _ctx = _safe_eval(funcname, ctx) if val is None: return set() try: sig = inspect.signature(val) except ValueError: return set() args = {p + "=" for p in sig.parameters if filter_func(p, prefix)} return args def complete_import(prefix, line, start, end, ctx): """ Completes module names and contents for "import ..." and "from ... import ..." """ ltoks = line.split() ntoks = len(ltoks) if ntoks == 2 and ltoks[0] == "from": # completing module to import return {"{} ".format(i) for i in complete_module(prefix)} if ntoks > 1 and ltoks[0] == "import" and start == len("import "): # completing module to import return complete_module(prefix) if ntoks > 2 and ltoks[0] == "from" and ltoks[2] == "import": # complete thing inside a module try: mod = importlib.import_module(ltoks[1]) except ImportError: return set() out = {i[0] for i in inspect.getmembers(mod) if i[0].startswith(prefix)} return out return set() def complete_module(prefix): return {s for s in sys.modules if get_filter_function()(s, prefix)} ================================================ FILE: xonsh/completers/tools.py ================================================ """Xonsh completer tools.""" import builtins import textwrap def _filter_normal(s, x): return s.startswith(x) def _filter_ignorecase(s, x): return s.lower().startswith(x.lower()) def get_filter_function(): """ Return an appropriate filtering function for completions, given the valid of $CASE_SENSITIVE_COMPLETIONS """ csc = builtins.__xonsh__.env.get("CASE_SENSITIVE_COMPLETIONS") if csc: return _filter_normal else: return _filter_ignorecase def justify(s, max_length, left_pad=0): """ Re-wrap the string s so that each line is no more than max_length characters long, padding all lines but the first on the left with the string left_pad. """ txt = textwrap.wrap(s, width=max_length, subsequent_indent=" " * left_pad) return "\n".join(txt) ================================================ FILE: xonsh/completers/xompletions.py ================================================ """Provides completions for xonsh internal utilities""" import xonsh.xontribs as xx import xonsh.tools as xt def complete_xonfig(prefix, line, start, end, ctx): """Completion for ``xonfig``""" args = line.split(" ") if len(args) == 0 or args[0] != "xonfig": return None curix = args.index(prefix) if curix == 1: possible = {"info", "wizard", "styles", "colors", "-h"} elif curix == 2 and args[1] == "colors": possible = set(xt.color_style_names()) else: raise StopIteration return {i for i in possible if i.startswith(prefix)} def _list_installed_xontribs(): meta = xx.xontrib_metadata() installed = [] for md in meta["xontribs"]: name = md["name"] spec = xx.find_xontrib(name) if spec is not None: installed.append(spec.name.rsplit(".")[-1]) return installed def complete_xontrib(prefix, line, start, end, ctx): """Completion for ``xontrib``""" args = line.split(" ") if len(args) == 0 or args[0] != "xontrib": return None curix = args.index(prefix) if curix == 1: possible = {"list", "load"} elif curix == 2: if args[1] == "load": possible = _list_installed_xontribs() else: raise StopIteration return {i for i in possible if i.startswith(prefix)} ================================================ FILE: xonsh/contexts.py ================================================ """Context management tools for xonsh.""" import sys import textwrap import builtins from collections.abc import Mapping class Block(object): """This is a context manager for obtaining a block of lines without actually executing the block. The lines are accessible as the 'lines' attribute. This must be used as a macro. """ __xonsh_block__ = str def __init__(self): """ Attributes ---------- lines : list of str or None Block lines as if split by str.splitlines(), if available. glbs : Mapping or None Global execution context, ie globals(). locs : Mapping or None Local execution context, ie locals(). """ self.lines = self.glbs = self.locs = None def __enter__(self): if not hasattr(self, "macro_block"): raise XonshError(self.__class__.__name__ + " must be entered as a macro!") self.lines = self.macro_block.splitlines() self.glbs = self.macro_globals if self.macro_locals is not self.macro_globals: # leave locals as None when it is the same as globals self.locs = self.macro_locals return self def __exit__(self, exc_type, exc_value, traceback): pass class Functor(Block): """This is a context manager that turns the block into a callable object, bound to the execution context it was created in. """ def __init__(self, args=(), kwargs=None, rtn=""): """ Parameters ---------- args : Sequence of str, optional A tuple of argument names for the functor. kwargs : Mapping of str to values or list of item tuples, optional Keyword argument names and values, if available. rtn : str, optional Name of object to return, if available. Attributes ---------- func : function The underlying function object. This defaults to none and is set after the the block is exited. """ super().__init__() self.func = None self.args = args if kwargs is None: self.kwargs = [] elif isinstance(kwargs, Mapping): self.kwargs = sorted(kwargs.items()) else: self.kwargs = kwargs self.rtn = rtn def __enter__(self): super().__enter__() body = textwrap.indent(self.macro_block, " ") uid = hash(body) + sys.maxsize # should always be a positive int name = "__xonsh_functor_{uid}__".format(uid=uid) # construct signature string sig = rtn = "" sig = ", ".join(self.args) kwstr = ", ".join([k + "=None" for k, _ in self.kwargs]) if len(kwstr) > 0: sig = kwstr if len(sig) == 0 else sig + ", " + kwstr # construct return string rtn = str(self.rtn) if len(rtn) > 0: rtn = " return " + rtn + "\n" # construct function string fstr = "def {name}({sig}):\n{body}\n{rtn}" fstr = fstr.format(name=name, sig=sig, body=body, rtn=rtn) glbs = self.glbs locs = self.locs execer = builtins.__xonsh__.execer execer.exec(fstr, glbs=glbs, locs=locs) if locs is not None and name in locs: func = locs[name] elif name in glbs: func = glbs[name] else: raise ValueError("Functor block could not be found in context.") if len(self.kwargs) > 0: func.__defaults__ = tuple(v for _, v in self.kwargs) self.func = func return self def __exit__(self, exc_type, exc_value, traceback): pass def __call__(self, *args, **kwargs): """Dispatches to func.""" if self.func is None: msg = "{} block with 'None' func not callable" raise AttributeError(msg.formst(self.__class__.__name__)) return self.func(*args, **kwargs) ================================================ FILE: xonsh/data/data.txt ================================================ (u'vnpy', 363) (u'thoppe', 104) (u'edc', 114) (u'projectatomic', 110) (u'jrosebr1', 289) (u'compose', 305) (u'4148', 163) (u'Dobiasd', 179) (u'blaze', 374) (u'Bekt', 133) (u'josauder', 251) (u'nylas', 101) (u'facebook', 2664) (u'izacus', 139) (u'HelloZeroNet', 1117) (u'michael-lazar', 527) (u'simonschellaert', 132) (u'black-perl', 272) (u'andersbll', 351) (u'wecite', 274) (u'stasinopoulos', 463) (u'vmware', 669) (u'waditu', 607) (u'Leviathan1995', 372) (u'TrackMaven', 146) (u'tokland', 114) (u'dellis23', 144) (u'amueller', 143) (u'NVIDIA', 733) (u'trevorstephens', 207) (u'rantonels', 312) (u'klen', 293) (u'vsergeev', 125) (u'madisonmay', 911) (u'DanMcInerney', 456) (u'realpython', 181) (u'graphific', 1418) (u'shoopio', 163) (u'conorpp', 123) (u'sdiehl', 139) (u'ajinabraham', 338) (u'prashanthellina', 170) (u'laike9m', 110) (u'spyder-ide', 570) (u'GedRap', 156) (u'diafygi', 161) (u'maxpumperla', 105) (u'ClusterHQ', 282) (u'dhilipsiva', 110) (u'entropy1337', 133) (u'dbpedia', 225) (u'kelvinxu', 127) (u'jsvine', 172) (u'python', 147) (u'OffensivePython', 305) (u'jkbrzt', 476) (u'quodlibet', 116) (u'ulope', 200) (u'andrea-cuttone', 157) (u'alehander42', 193) (u'KirkHadley', 139) (u'revsys', 112) (u'aigamedev', 583) (u'sphinx-doc', 389) (u'rietveld-codereview', 123) (u'quantopian', 319) (u'Deimos', 285) (u'dgilland', 107) (u'nccgroup', 146) (u'nickjj', 138) (u'nbs-system', 107) (u'elceef', 272) (u'compjour', 630) (u'jrnewell', 207) (u'bndr', 510) (u'EricssonResearch', 188) (u'raelgc', 701) (u'avinassh', 2659) (u'vitruvianscience', 152) (u'achiku', 811) (u'eugene-eeo', 144) (u'paul-nechifor', 465) (u'antiboredom', 582) (u'datalib', 301) (u'brandon-rhodes', 110) (u'philpep', 101) (u'python-cn', 151) (u'jakevdp', 275) (u'CiscoCloud', 1635) (u'yandex', 174) (u'jbornschein', 201) (u'j-bennet', 307) (u'Skycrab', 124) (u'spotify', 376) (u'pyparallel', 345) (u'JakeCooper', 130) (u'jorgebastida', 201) (u'rouseguy', 253) (u'pfalcon', 108) (u'geerlingguy', 155) (u'aliyun', 113) (u'dorneanu', 124) (u'cloudera', 500) (u'drzoidberg33', 127) (u'brennerm', 1154) (u'TomAnthony', 336) (u'clarkduvall', 310) (u'matthewearl', 322) (u'guidepointsecurity', 167) (u'samim23', 260) (u'rhiever', 1848) (u'warner', 345) (u'gojhonny', 304) (u'joshenders', 210) (u'itdxer', 108) (u'fraser-hemp', 1366) (u'omriher', 269) (u'motet', 265) (u'waf-project', 158) (u'AlexEne', 135) (u'jmcarp', 159) (u'scrapinghub', 141) (u'puniaze', 114) (u'orf', 207) (u'rasguanabana', 892) (u'yasoob', 899) (u'biicode', 205) (u'pybee', 237) (u'apprenticeharper', 625) (u'angr', 279) (u'Hypsurus', 214) (u'PaulSec', 228) (u'xybu', 111) (u'ryankiros', 343) (u'wapiflapi', 171) (u'trustedsec', 337) (u'jfalken', 135) (u'ncrocfer', 337) (u'mila-udem', 398) (u'mardix', 119) (u'automl', 227) (u'ylovern', 171) (u'jacebrowning', 135) (u'Eyepea', 328) (u'csvoss', 468) (u'dittos', 135) (u'icgood', 292) (u'dciabrin', 114) (u'minimaxir', 8920) (u'brandonshin', 341) (u'jonathanslenders', 1531) (u'donnemartin', 7955) (u'KeyboardFire', 3324) (u'offapi', 142) (u'no13bus', 367) (u'tatanus', 108) (u'Vector35', 281) (u'micheloosterhof', 189) (u'gamechanger', 220) (u'isislab', 233) (u'joxeankoret', 263) (u'certsocietegenerale', 189) (u'inconvergent', 283) (u'rpp0', 235) (u'yukuku', 225) (u'lijiejie', 174) (u'stitchfix', 928) (u'LionSec', 108) (u'n0tr00t', 108) (u'ssteuteville', 150) (u'getsentry', 661) (u'agermanidis', 731) (u'78', 354) (u'piskvorky', 308) (u'sogisha', 172) (u'mre', 113) (u'rbgirshick', 435) (u'rtluckie', 112) (u'dbcli', 2072) (u'eternnoir', 170) (u'recipy', 164) (u'rasbt', 350) (u'ant4g0nist', 125) (u'm57', 598) (u'TailorDev', 119) (u'cosven', 169) (u'zachriggle', 130) (u'knownsec', 151) (u'awslabs', 200) (u'n1nj4sec', 827) (u'armadaplatform', 158) (u'DoctorTeeth', 129) (u'atmb4u', 155) (u'mmin18', 1063) (u'divmain', 1206) (u'fuzzing', 101) (u'urbenlegend', 181) (u'regebro', 245) (u'QuantumFractal', 222) (u'kootenpv', 243) (u'Alexis-benoist', 120) (u'sontek', 108) (u'harshasrinivas', 147) (u'rabidgremlin', 187) (u'Slava', 171) (u'10se1ucgo', 1884) (u'fugue', 266) (u'AlessandroZ', 1423) (u'unixy', 107) (u'pindexis', 582) (u'Yinzo', 115) (u'joelpx', 351) (u'ryanss', 864) (u'Dynetics', 110) (u'ExPHAT', 385) (u'hephaest0s', 1246) (u'armbues', 112) (u'pytest-dev', 360) (u'moha99sa', 105) (u'nvbn', 14856) (u'scopatz', 511) (u'andrewgodwin', 205) (u'architv', 422) (u'owais', 122) (u'lorin', 107) (u'takluyver', 170) (u'garabik', 104) (u'kevinlawler', 183) (u'krockode', 455) (u'IndicoDataSolutions', 384) (u'x43x61x69', 107) (u'semirook', 103) (u'DIYgod', 195) (u'fffonion', 101) (u'snoack', 429) (u'rodricios', 197) (u'RealHacker', 132) (u'krisfields', 246) (u'nils-werner', 202) (u'mementum', 124) (u'uaca', 220) (u'tburmeister', 226) (u'CoreSecurity', 294) (u'lukasschwab', 298) (u'jcjohnson', 275) (u'timothycrosley', 903) (u'linkedin', 142) (u'rushter', 2155) (u'WatchPeopleCode', 209) (u'hellerve', 426) (u'JacobPlaster', 198) (u'thampiman', 1101) (u'timlib', 116) (u'pfnet', 832) (u'mitsuhiko', 231) (u'ziggear', 153) (u'datawire', 152) (u'jrfonseca', 199) (u'Lasagne', 151) (u'ssut', 130) (u'ChrisTruncer', 199) (u'projectcalico', 167) (u'lehui99', 219) (u'ring04h', 390) (u'amoffat', 1225) (u'317070', 134) (u'p-e-w', 734) (u'srsudar', 735) (u'aromanovich', 102) (u'twoscoops', 244) (u'fchollet', 2958) (u'minrk', 113) (u'chrissimpkins', 2211) (u'qiwsir', 673) (u'smarr', 612) (u'anishathalye', 796) (u'onyxfish', 120) (u'hexahedria', 602) (u'deepmind', 344) (u'wooey', 484) (u'manugarri', 129) (u'joschu', 387) (u'parkouss', 134) (u'ShawnDEvans', 165) (u'Max00355', 507) (u'uptimejp', 156) (u'google', 5720) (u'yosinski', 352) (u'buckyroberts', 211) (u'borgbackup', 142) (u'chriscannon', 233) (u'benanne', 175) (u'jandre', 257) (u'zeruniverse', 232) (u'asciimoo', 167) (u'fengsp', 208) (u'Koed00', 201) (u'duerrp', 193) (u'byt3bl33d3r', 464) (u'amjith', 154) (u'DrkSephy', 404) (u'andelf', 832) (u'vertical-knowledge', 109) (u'Ivaylo-Popov', 248) (u'lhartikk', 422) (u'lmco', 313) (u'paylogic', 161) (u'karan', 2672) (u'zulip', 3108) (u'billpmurphy', 375) (u'saelo', 163) (u'rdnetto', 173) (u'unlimitedlabs', 191) (u'lonetwin', 129) (u'closeio', 304) (u'billryan', 178) (u'larsenwork', 2720) (u'Dionach', 150) (u'ottogroup', 308) (u'paypal', 221) (u'install-logos', 116) (u'Maratyszcza', 610) (u'avehtari', 292) (u'AtnNn', 125) (u'Neo23x0', 208) (u'justmarkham', 135) (u'vladimarius', 168) (u'graphql-python', 128) (u'mattya', 115) (u'coodict', 537) (u'sourcelair', 268) (u'walkr', 222) (u'bittorrent', 363) (u'hugsy', 201) (u'ayoungprogrammer', 292) (u'MorganZhang100', 249) (u'ChrisBeaumont', 245) (u'Netflix', 346) (u'yahoo', 302) (u'mhallsmoore', 136) (u'XX-net', 2731) (u'leandrotoledo', 175) (u'Brobin', 154) (u'pyskell', 732) (u'NathanEpstein', 119) (u'yadayada', 245) (u'dxa4481', 216) (u'lyst', 408) ================================================ FILE: xonsh/diff_history.py ================================================ # -*- coding: utf-8 -*- """Tools for diff'ing two xonsh history files in a meaningful fashion.""" import difflib import datetime import itertools import argparse from xonsh.lazyjson import LazyJSON from xonsh.tools import print_color NO_COLOR_S = "{NO_COLOR}" RED_S = "{RED}" GREEN_S = "{GREEN}" BOLD_RED_S = "{BOLD_RED}" BOLD_GREEN_S = "{BOLD_GREEN}" # intern some strings REPLACE_S = "replace" DELETE_S = "delete" INSERT_S = "insert" EQUAL_S = "equal" def bold_str_diff(a, b, sm=None): if sm is None: sm = difflib.SequenceMatcher() aline = RED_S + "- " bline = GREEN_S + "+ " sm.set_seqs(a, b) for tag, i1, i2, j1, j2 in sm.get_opcodes(): if tag == REPLACE_S: aline += BOLD_RED_S + a[i1:i2] + RED_S bline += BOLD_GREEN_S + b[j1:j2] + GREEN_S elif tag == DELETE_S: aline += BOLD_RED_S + a[i1:i2] + RED_S elif tag == INSERT_S: bline += BOLD_GREEN_S + b[j1:j2] + GREEN_S elif tag == EQUAL_S: aline += a[i1:i2] bline += b[j1:j2] else: raise RuntimeError("tag not understood") return aline + NO_COLOR_S + "\n" + bline + NO_COLOR_S + "\n" def redline(line): return "{red}- {line}{no_color}\n".format(red=RED_S, line=line, no_color=NO_COLOR_S) def greenline(line): return "{green}+ {line}{no_color}\n".format( green=GREEN_S, line=line, no_color=NO_COLOR_S ) def highlighted_ndiff(a, b): """Returns a highlighted string, with bold characters where different.""" s = "" sm = difflib.SequenceMatcher() sm.set_seqs(a, b) linesm = difflib.SequenceMatcher() for tag, i1, i2, j1, j2 in sm.get_opcodes(): if tag == REPLACE_S: for aline, bline in itertools.zip_longest(a[i1:i2], b[j1:j2]): if bline is None: s += redline(aline) elif aline is None: s += greenline(bline) else: s += bold_str_diff(aline, bline, sm=linesm) elif tag == DELETE_S: for aline in a[i1:i2]: s += redline(aline) elif tag == INSERT_S: for bline in b[j1:j2]: s += greenline(bline) elif tag == EQUAL_S: for aline in a[i1:i2]: s += " " + aline + "\n" else: raise RuntimeError("tag not understood") return s class HistoryDiffer(object): """This class helps diff two xonsh history files.""" def __init__(self, afile, bfile, reopen=False, verbose=False): """ Parameters ---------- afile : file handle or str The first file to diff bfile : file handle or str The second file to diff reopen : bool, optional Whether or not to reopen the file handles each time. The default here is opposite from the LazyJSON default because we know that we will be doing a lot of reading so it is best to keep the handles open. verbose : bool, optional Whether to print a verbose amount of information. """ self.a = LazyJSON(afile, reopen=reopen) self.b = LazyJSON(bfile, reopen=reopen) self.verbose = verbose self.sm = difflib.SequenceMatcher(autojunk=False) def __del__(self): self.a.close() self.b.close() def __str__(self): return self.format() def _header_line(self, lj): s = lj._f.name if hasattr(lj._f, "name") else "" s += " (" + lj["sessionid"] + ")" s += " [locked]" if lj["locked"] else " [unlocked]" ts = lj["ts"].load() ts0 = datetime.datetime.fromtimestamp(ts[0]) s += " started: " + ts0.isoformat(" ") if ts[1] is not None: ts1 = datetime.datetime.fromtimestamp(ts[1]) s += " stopped: " + ts1.isoformat(" ") + " runtime: " + str(ts1 - ts0) return s def header(self): """Computes a header string difference.""" s = "{red}--- {aline}{no_color}\n" "{green}+++ {bline}{no_color}" s = s.format( aline=self._header_line(self.a), bline=self._header_line(self.b), red=RED_S, green=GREEN_S, no_color=NO_COLOR_S, ) return s def _env_both_diff(self, in_both, aenv, benv): sm = self.sm s = "" for key in sorted(in_both): aval = aenv[key] bval = benv[key] if aval == bval: continue s += "{0!r} is in both, but differs\n".format(key) s += bold_str_diff(aval, bval, sm=sm) + "\n" return s def _env_in_one_diff(self, x, y, color, xid, xenv): only_x = sorted(x - y) if len(only_x) == 0: return "" if self.verbose: xstr = ",\n".join( [" {0!r}: {1!r}".format(key, xenv[key]) for key in only_x] ) xstr = "\n" + xstr else: xstr = ", ".join(["{0!r}".format(key) for key in only_x]) in_x = "These vars are only in {color}{xid}{no_color}: {{{xstr}}}\n\n" return in_x.format(xid=xid, color=color, no_color=NO_COLOR_S, xstr=xstr) def envdiff(self): """Computes the difference between the environments.""" aenv = self.a["env"].load() benv = self.b["env"].load() akeys = frozenset(aenv) bkeys = frozenset(benv) in_both = akeys & bkeys if len(in_both) == len(akeys) == len(bkeys): keydiff = self._env_both_diff(in_both, aenv, benv) if len(keydiff) == 0: return "" in_a = in_b = "" else: keydiff = self._env_both_diff(in_both, aenv, benv) in_a = self._env_in_one_diff(akeys, bkeys, RED_S, self.a["sessionid"], aenv) in_b = self._env_in_one_diff( bkeys, akeys, GREEN_S, self.b["sessionid"], benv ) s = "Environment\n-----------\n" + in_a + keydiff + in_b return s def _cmd_in_one_diff(self, inp, i, xlj, xid, color): s = "cmd #{i} only in {color}{xid}{no_color}:\n" s = s.format(i=i, color=color, xid=xid, no_color=NO_COLOR_S) lines = inp.splitlines() lt = "{color}{pre}{no_color} {line}\n" s += lt.format(color=color, no_color=NO_COLOR_S, line=lines[0], pre=">>>") for line in lines[1:]: s += lt.format(color=color, no_color=NO_COLOR_S, line=line, pre="...") if not self.verbose: return s + "\n" out = xlj["cmds"][0].get("out", "Note: no output stored") s += out.rstrip() + "\n\n" return s def _cmd_out_and_rtn_diff(self, i, j): s = "" aout = self.a["cmds"][i].get("out", None) bout = self.b["cmds"][j].get("out", None) if aout is None and bout is None: # s += 'Note: neither output stored\n' pass elif bout is None: aid = self.a["sessionid"] s += "Note: only {red}{aid}{no_color} output stored\n".format( red=RED_S, aid=aid, no_color=NO_COLOR_S ) elif aout is None: bid = self.b["sessionid"] s += "Note: only {green}{bid}{no_color} output stored\n".format( green=GREEN_S, bid=bid, no_color=NO_COLOR_S ) elif aout != bout: s += "Outputs differ\n" s += highlighted_ndiff(aout.splitlines(), bout.splitlines()) else: pass artn = self.a["cmds"][i]["rtn"] brtn = self.b["cmds"][j]["rtn"] if artn != brtn: s += ( "Return vals {red}{artn}{no_color} & {green}{brtn}{no_color} differ\n" ).format( red=RED_S, green=GREEN_S, no_color=NO_COLOR_S, artn=artn, brtn=brtn ) return s def _cmd_replace_diff(self, i, ainp, aid, j, binp, bid): s = ( "cmd #{i} in {red}{aid}{no_color} is replaced by \n" "cmd #{j} in {green}{bid}{no_color}:\n" ) s = s.format( i=i, aid=aid, j=j, bid=bid, red=RED_S, green=GREEN_S, no_color=NO_COLOR_S ) s += highlighted_ndiff(ainp.splitlines(), binp.splitlines()) if not self.verbose: return s + "\n" s += self._cmd_out_and_rtn_diff(i, j) return s + "\n" def cmdsdiff(self): """Computes the difference of the commands themselves.""" aid = self.a["sessionid"] bid = self.b["sessionid"] ainps = [c["inp"] for c in self.a["cmds"]] binps = [c["inp"] for c in self.b["cmds"]] sm = self.sm sm.set_seqs(ainps, binps) s = "" for tag, i1, i2, j1, j2 in sm.get_opcodes(): if tag == REPLACE_S: zipper = itertools.zip_longest for i, ainp, j, binp in zipper( range(i1, i2), ainps[i1:i2], range(j1, j2), binps[j1:j2] ): if j is None: s += self._cmd_in_one_diff(ainp, i, self.a, aid, RED_S) elif i is None: s += self._cmd_in_one_diff(binp, j, self.b, bid, GREEN_S) else: self._cmd_replace_diff(i, ainp, aid, j, binp, bid) elif tag == DELETE_S: for i, inp in enumerate(ainps[i1:i2], i1): s += self._cmd_in_one_diff(inp, i, self.a, aid, RED_S) elif tag == INSERT_S: for j, inp in enumerate(binps[j1:j2], j1): s += self._cmd_in_one_diff(inp, j, self.b, bid, GREEN_S) elif tag == EQUAL_S: for i, j in zip(range(i1, i2), range(j1, j2)): odiff = self._cmd_out_and_rtn_diff(i, j) if len(odiff) > 0: h = ( "cmd #{i} in {red}{aid}{no_color} input is the same as \n" "cmd #{j} in {green}{bid}{no_color}, but output differs:\n" ) s += h.format( i=i, aid=aid, j=j, bid=bid, red=RED_S, green=GREEN_S, no_color=NO_COLOR_S, ) s += odiff + "\n" else: raise RuntimeError("tag not understood") if len(s) == 0: return s return "Commands\n--------\n" + s def format(self): """Formats the difference between the two history files.""" s = self.header() ed = self.envdiff() if len(ed) > 0: s += "\n\n" + ed cd = self.cmdsdiff() if len(cd) > 0: s += "\n\n" + cd return s.rstrip() _HD_PARSER = None def dh_create_parser(p=None): global _HD_PARSER p_was_none = p is None if _HD_PARSER is not None and p_was_none: return _HD_PARSER if p_was_none: p = argparse.ArgumentParser( "diff-history", description="diffs two xonsh history files" ) p.add_argument( "--reopen", dest="reopen", default=False, action="store_true", help="make lazy file loading reopen files each time", ) p.add_argument( "-v", "--verbose", dest="verbose", default=False, action="store_true", help="whether to print even more information", ) p.add_argument("a", help="first file in diff") p.add_argument("b", help="second file in diff") if p_was_none: _HD_PARSER = p return p def dh_main_action(ns, hist=None, stdout=None, stderr=None): hd = HistoryDiffer(ns.a, ns.b, reopen=ns.reopen, verbose=ns.verbose) print_color(hd.format(), file=stdout) ================================================ FILE: xonsh/dirstack.py ================================================ # -*- coding: utf-8 -*- """Directory stack and associated utilities for the xonsh shell.""" import os import glob import argparse import builtins import subprocess from xonsh.lazyasd import lazyobject from xonsh.tools import get_sep from xonsh.events import events from xonsh.platform import ON_WINDOWS DIRSTACK = [] """A list containing the currently remembered directories.""" _unc_tempDrives = {} """ drive: sharePath for temp drive letters we create for UNC mapping""" def _unc_check_enabled() -> bool: r"""Check whether CMD.EXE is enforcing no-UNC-as-working-directory check. Check can be disabled by setting {HKCU, HKLM}/SOFTWARE\Microsoft\Command Processor\DisableUNCCheck:REG_DWORD=1 Returns: True if `CMD.EXE` is enforcing the check (default Windows situation) False if check is explicitly disabled. """ if not ON_WINDOWS: return import winreg wval = None try: key = winreg.OpenKey( winreg.HKEY_CURRENT_USER, r"software\microsoft\command processor" ) wval, wtype = winreg.QueryValueEx(key, "DisableUNCCheck") winreg.CloseKey(key) except OSError: pass if wval is None: try: key2 = winreg.OpenKey( winreg.HKEY_LOCAL_MACHINE, r"software\microsoft\command processor" ) wval, wtype = winreg.QueryValueEx(key2, "DisableUNCCheck") winreg.CloseKey(key2) except OSError as e: # NOQA pass return False if wval else True def _is_unc_path(some_path) -> bool: """True if path starts with 2 backward (or forward, due to python path hacking) slashes.""" return ( len(some_path) > 1 and some_path[0] == some_path[1] and some_path[0] in (os.sep, os.altsep) ) def _unc_map_temp_drive(unc_path) -> str: r"""Map a new temporary drive letter for each distinct share, unless `CMD.EXE` is not insisting on non-UNC working directory. Emulating behavior of `CMD.EXE` `pushd`, create a new mapped drive (starting from Z: towards A:, skipping existing drive letters) for each new UNC path user selects. Args: unc_path: the path specified by user. Assumed to be a UNC path of form \\\share... Returns: a replacement for `unc_path` to be used as the actual new working directory. Note that the drive letter may be a the same as one already mapped if the server and share portion of `unc_path` is the same as one still active on the stack. """ global _unc_tempDrives assert unc_path[1] in (os.sep, os.altsep), "unc_path is UNC form of path" if not _unc_check_enabled(): return unc_path else: unc_share, rem_path = os.path.splitdrive(unc_path) unc_share = unc_share.casefold() for d in _unc_tempDrives: if _unc_tempDrives[d] == unc_share: return os.path.join(d, rem_path) for dord in range(ord("z"), ord("a"), -1): d = chr(dord) + ":" if not os.path.isdir(d): # find unused drive letter starting from z: subprocess.check_output( ["NET", "USE", d, unc_share], universal_newlines=True ) _unc_tempDrives[d] = unc_share return os.path.join(d, rem_path) def _unc_unmap_temp_drive(left_drive, cwd): """Unmap a temporary drive letter if it is no longer needed. Called after popping `DIRSTACK` and changing to new working directory, so we need stack *and* new current working directory to be sure drive letter no longer needed. Args: left_drive: driveletter (and colon) of working directory we just left cwd: full path of new current working directory """ global _unc_tempDrives if left_drive not in _unc_tempDrives: # if not one we've mapped, don't unmap it return for p in DIRSTACK + [cwd]: # if still in use , don't unmap it. if p.casefold().startswith(left_drive): return _unc_tempDrives.pop(left_drive) subprocess.check_output( ["NET", "USE", left_drive, "/delete"], universal_newlines=True ) events.doc( "on_chdir", """ on_chdir(olddir: str, newdir: str) -> None Fires when the current directory is changed for any reason. """, ) def _get_cwd(): try: return os.getcwd() except (OSError, FileNotFoundError): return None def _change_working_directory(newdir, follow_symlinks=False): env = builtins.__xonsh__.env old = env["PWD"] new = os.path.join(old, newdir) absnew = os.path.abspath(new) if follow_symlinks: absnew = os.path.realpath(absnew) try: os.chdir(absnew) except (OSError, FileNotFoundError): if new.endswith(get_sep()): new = new[:-1] if os.path.basename(new) == "..": env["PWD"] = new else: if old is not None: env["OLDPWD"] = old if new is not None: env["PWD"] = absnew # Fire event if the path actually changed if old != env["PWD"]: events.on_chdir.fire(olddir=old, newdir=env["PWD"]) def _try_cdpath(apath): # NOTE: this CDPATH implementation differs from the bash one. # In bash if a CDPATH is set, an unqualified local folder # is considered after all CDPATHs, example: # CDPATH=$HOME/src (with src/xonsh/ inside) # $ cd xonsh -> src/xonsh (with xonsh/xonsh) # a second $ cd xonsh has no effects, to move in the nested xonsh # in bash a full $ cd ./xonsh is needed. # In xonsh a relative folder is always preferred. env = builtins.__xonsh__.env cdpaths = env.get("CDPATH") for cdp in cdpaths: globber = builtins.__xonsh__.expand_path(os.path.join(cdp, apath)) for cdpath_prefixed_path in glob.iglob(globber): return cdpath_prefixed_path return apath def cd(args, stdin=None): """Changes the directory. If no directory is specified (i.e. if `args` is None) then this changes to the current user's home directory. """ env = builtins.__xonsh__.env oldpwd = env.get("OLDPWD", None) cwd = env["PWD"] follow_symlinks = False if len(args) > 0 and args[0] == "-P": follow_symlinks = True del args[0] if len(args) == 0: d = os.path.expanduser("~") elif len(args) == 1: d = os.path.expanduser(args[0]) if not os.path.isdir(d): if d == "-": if oldpwd is not None: d = oldpwd else: return "", "cd: no previous directory stored\n", 1 elif d.startswith("-"): try: num = int(d[1:]) except ValueError: return "", "cd: Invalid destination: {0}\n".format(d), 1 if num == 0: return None, None, 0 elif num < 0: return "", "cd: Invalid destination: {0}\n".format(d), 1 elif num > len(DIRSTACK): e = "cd: Too few elements in dirstack ({0} elements)\n" return "", e.format(len(DIRSTACK)), 1 else: d = DIRSTACK[num - 1] else: d = _try_cdpath(d) else: return ( "", ( "cd takes 0 or 1 arguments, not {0}. An additional `-P` " "flag can be passed in first position to follow symlinks." "\n".format(len(args)) ), 1, ) if not os.path.exists(d): return "", "cd: no such file or directory: {0}\n".format(d), 1 if not os.path.isdir(d): return "", "cd: {0} is not a directory\n".format(d), 1 if not os.access(d, os.X_OK): return "", "cd: permission denied: {0}\n".format(d), 1 if ( ON_WINDOWS and _is_unc_path(d) and _unc_check_enabled() and (not env.get("AUTO_PUSHD")) ): return ( "", "cd: can't cd to UNC path on Windows, unless $AUTO_PUSHD set or reg entry " + r"HKCU\SOFTWARE\MICROSOFT\Command Processor\DisableUNCCheck:DWORD = 1" + "\n", 1, ) # now, push the directory onto the dirstack if AUTO_PUSHD is set if cwd is not None and env.get("AUTO_PUSHD"): pushd(["-n", "-q", cwd]) if ON_WINDOWS and _is_unc_path(d): d = _unc_map_temp_drive(d) _change_working_directory(d, follow_symlinks) return None, None, 0 @lazyobject def pushd_parser(): parser = argparse.ArgumentParser(prog="pushd") parser.add_argument("dir", nargs="?") parser.add_argument( "-n", dest="cd", help="Suppresses the normal change of directory when" " adding directories to the stack, so that only the" " stack is manipulated.", action="store_false", ) parser.add_argument( "-q", dest="quiet", help="Do not call dirs, regardless of $PUSHD_SILENT", action="store_true", ) return parser def pushd(args, stdin=None): r"""xonsh command: pushd Adds a directory to the top of the directory stack, or rotates the stack, making the new top of the stack the current working directory. On Windows, if the path is a UNC path (begins with `\\\`) and if the `DisableUNCCheck` registry value is not enabled, creates a temporary mapped drive letter and sets the working directory there, emulating behavior of `PUSHD` in `CMD.EXE` """ global DIRSTACK try: args = pushd_parser.parse_args(args) except SystemExit: return None, None, 1 env = builtins.__xonsh__.env pwd = env["PWD"] if env.get("PUSHD_MINUS", False): BACKWARD = "-" FORWARD = "+" else: BACKWARD = "+" FORWARD = "-" if args.dir is None: try: new_pwd = DIRSTACK.pop(0) except IndexError: e = "pushd: Directory stack is empty\n" return None, e, 1 elif os.path.isdir(args.dir): new_pwd = args.dir else: try: num = int(args.dir[1:]) except ValueError: e = "Invalid argument to pushd: {0}\n" return None, e.format(args.dir), 1 if num < 0: e = "Invalid argument to pushd: {0}\n" return None, e.format(args.dir), 1 if num > len(DIRSTACK): e = "Too few elements in dirstack ({0} elements)\n" return None, e.format(len(DIRSTACK)), 1 elif args.dir.startswith(FORWARD): if num == len(DIRSTACK): new_pwd = None else: new_pwd = DIRSTACK.pop(len(DIRSTACK) - 1 - num) elif args.dir.startswith(BACKWARD): if num == 0: new_pwd = None else: new_pwd = DIRSTACK.pop(num - 1) else: e = "Invalid argument to pushd: {0}\n" return None, e.format(args.dir), 1 if new_pwd is not None: if ON_WINDOWS and _is_unc_path(new_pwd): new_pwd = _unc_map_temp_drive(new_pwd) if args.cd: DIRSTACK.insert(0, os.path.expanduser(pwd)) _change_working_directory(new_pwd) else: DIRSTACK.insert(0, os.path.expanduser(new_pwd)) maxsize = env.get("DIRSTACK_SIZE") if len(DIRSTACK) > maxsize: DIRSTACK = DIRSTACK[:maxsize] if not args.quiet and not env.get("PUSHD_SILENT"): return dirs([], None) return None, None, 0 @lazyobject def popd_parser(): parser = argparse.ArgumentParser(prog="popd") parser.add_argument("dir", nargs="?") parser.add_argument( "-n", dest="cd", help="Suppresses the normal change of directory when" " adding directories to the stack, so that only the" " stack is manipulated.", action="store_false", ) parser.add_argument( "-q", dest="quiet", help="Do not call dirs, regardless of $PUSHD_SILENT", action="store_true", ) return parser def popd(args, stdin=None): """ xonsh command: popd Removes entries from the directory stack. """ global DIRSTACK try: args = pushd_parser.parse_args(args) except SystemExit: return None, None, 1 env = builtins.__xonsh__.env if env.get("PUSHD_MINUS"): BACKWARD = "-" FORWARD = "+" else: BACKWARD = "-" FORWARD = "+" if args.dir is None: try: new_pwd = DIRSTACK.pop(0) except IndexError: e = "popd: Directory stack is empty\n" return None, e, 1 else: try: num = int(args.dir[1:]) except ValueError: e = "Invalid argument to popd: {0}\n" return None, e.format(args.dir), 1 if num < 0: e = "Invalid argument to popd: {0}\n" return None, e.format(args.dir), 1 if num > len(DIRSTACK): e = "Too few elements in dirstack ({0} elements)\n" return None, e.format(len(DIRSTACK)), 1 elif args.dir.startswith(FORWARD): if num == len(DIRSTACK): new_pwd = DIRSTACK.pop(0) else: new_pwd = None DIRSTACK.pop(len(DIRSTACK) - 1 - num) elif args.dir.startswith(BACKWARD): if num == 0: new_pwd = DIRSTACK.pop(0) else: new_pwd = None DIRSTACK.pop(num - 1) else: e = "Invalid argument to popd: {0}\n" return None, e.format(args.dir), 1 if new_pwd is not None: e = None if args.cd: env = builtins.__xonsh__.env pwd = env["PWD"] _change_working_directory(new_pwd) if ON_WINDOWS: drive, rem_path = os.path.splitdrive(pwd) _unc_unmap_temp_drive(drive.casefold(), new_pwd) if not args.quiet and not env.get("PUSHD_SILENT"): return dirs([], None) return None, None, 0 @lazyobject def dirs_parser(): parser = argparse.ArgumentParser(prog="dirs") parser.add_argument("N", nargs="?") parser.add_argument( "-c", dest="clear", help="Clears the directory stack by deleting all of" " the entries.", action="store_true", ) parser.add_argument( "-p", dest="print_long", help="Print the directory stack with one entry per" " line.", action="store_true", ) parser.add_argument( "-v", dest="verbose", help="Print the directory stack with one entry per" " line, prefixing each entry with its index in the" " stack.", action="store_true", ) parser.add_argument( "-l", dest="long", help="Produces a longer listing; the default listing" " format uses a tilde to denote the home directory.", action="store_true", ) return parser def dirs(args, stdin=None): """xonsh command: dirs Displays the list of currently remembered directories. Can also be used to clear the directory stack. """ global DIRSTACK try: args = dirs_parser.parse_args(args) except SystemExit: return None, None env = builtins.__xonsh__.env dirstack = [os.path.expanduser(env["PWD"])] + DIRSTACK if env.get("PUSHD_MINUS"): BACKWARD = "-" FORWARD = "+" else: BACKWARD = "-" FORWARD = "+" if args.clear: DIRSTACK = [] return None, None, 0 if args.long: o = dirstack else: d = os.path.expanduser("~") o = [i.replace(d, "~") for i in dirstack] if args.verbose: out = "" pad = len(str(len(o) - 1)) for (ix, e) in enumerate(o): blanks = " " * (pad - len(str(ix))) out += "\n{0}{1} {2}".format(blanks, ix, e) out = out[1:] elif args.print_long: out = "\n".join(o) else: out = " ".join(o) N = args.N if N is not None: try: num = int(N[1:]) except ValueError: e = "Invalid argument to dirs: {0}\n" return None, e.format(N), 1 if num < 0: e = "Invalid argument to dirs: {0}\n" return None, e.format(len(o)), 1 if num >= len(o): e = "Too few elements in dirstack ({0} elements)\n" return None, e.format(len(o)), 1 if N.startswith(BACKWARD): idx = num elif N.startswith(FORWARD): idx = len(o) - 1 - num else: e = "Invalid argument to dirs: {0}\n" return None, e.format(N), 1 out = o[idx] return out + "\n", None, 0 ================================================ FILE: xonsh/dumb_shell.py ================================================ """A dumb shell for when $TERM == 'dumb', which usually happens in emacs.""" import builtins from xonsh.readline_shell import ReadlineShell class DumbShell(ReadlineShell): """A dumb shell for when $TERM == 'dumb', which usually happens in emacs.""" def __init__(self, *args, **kwargs): builtins.__xonsh__.env["XONSH_COLOR_STYLE"] = "emacs" super().__init__(*args, **kwargs) ================================================ FILE: xonsh/environ.py ================================================ # -*- coding: utf-8 -*- """Environment for the xonsh shell.""" import os import re import sys import pprint import textwrap import locale import builtins import warnings import contextlib import collections import collections.abc as cabc import subprocess from xonsh import __version__ as XONSH_VERSION from xonsh.lazyasd import LazyObject, lazyobject from xonsh.codecache import run_script_with_cache from xonsh.dirstack import _get_cwd from xonsh.events import events from xonsh.platform import ( BASH_COMPLETIONS_DEFAULT, DEFAULT_ENCODING, PATH_DEFAULT, ON_WINDOWS, ON_LINUX, os_environ, ) from xonsh.style_tools import PTK2_STYLE from xonsh.tools import ( always_true, always_false, detype, ensure_string, is_env_path, str_to_env_path, env_path_to_str, is_bool, to_bool, bool_to_str, is_history_tuple, to_history_tuple, history_tuple_to_str, is_float, is_string, is_string_or_callable, is_completions_display_value, to_completions_display_value, is_string_set, csv_to_set, set_to_csv, is_int, is_bool_seq, to_bool_or_int, bool_or_int_to_str, csv_to_bool_seq, bool_seq_to_csv, DefaultNotGiven, print_exception, setup_win_unicode_console, intensify_colors_on_win_setter, is_dynamic_cwd_width, to_dynamic_cwd_tuple, dynamic_cwd_tuple_to_str, is_logfile_opt, to_logfile_opt, logfile_opt_to_str, executables_in, is_nonstring_seq_of_strings, pathsep_to_upper_seq, seq_to_upper_pathsep, print_color, is_history_backend, to_itself, swap_values, ptk2_color_depth_setter, is_str_str_dict, to_str_str_dict, dict_to_str, ) from xonsh.ansi_colors import ( ansi_color_escape_code_to_name, ansi_reverse_style, ansi_style_by_name, ) import xonsh.prompt.base as prompt events.doc( "on_envvar_new", """ on_envvar_new(name: str, value: Any) -> None Fires after a new environment variable is created. Note: Setting envvars inside the handler might cause a recursion until the limit. """, ) events.doc( "on_envvar_change", """ on_envvar_change(name: str, oldvalue: Any, newvalue: Any) -> None Fires after an environment variable is changed. Note: Setting envvars inside the handler might cause a recursion until the limit. """, ) events.doc( "on_pre_spec_run_ls", """ on_pre_spec_run_ls(spec: xonsh.built_ins.SubprocSpec) -> None Fires right before a SubprocSpec.run() is called for the ls command. """, ) @lazyobject def HELP_TEMPLATE(): return ( "{{INTENSE_RED}}{envvar}{{NO_COLOR}}:\n\n" "{{INTENSE_YELLOW}}{docstr}{{NO_COLOR}}\n\n" "default: {{CYAN}}{default}{{NO_COLOR}}\n" "configurable: {{CYAN}}{configurable}{{NO_COLOR}}" ) @lazyobject def LOCALE_CATS(): lc = { "LC_CTYPE": locale.LC_CTYPE, "LC_COLLATE": locale.LC_COLLATE, "LC_NUMERIC": locale.LC_NUMERIC, "LC_MONETARY": locale.LC_MONETARY, "LC_TIME": locale.LC_TIME, } if hasattr(locale, "LC_MESSAGES"): lc["LC_MESSAGES"] = locale.LC_MESSAGES return lc def locale_convert(key): """Creates a converter for a locale key.""" def lc_converter(val): try: locale.setlocale(LOCALE_CATS[key], val) val = locale.setlocale(LOCALE_CATS[key]) except (locale.Error, KeyError): msg = "Failed to set locale {0!r} to {1!r}".format(key, val) warnings.warn(msg, RuntimeWarning) return val return lc_converter def to_debug(x): """Converts value using to_bool_or_int() and sets this value on as the execer's debug level. """ val = to_bool_or_int(x) if ( hasattr(builtins, "__xonsh__") and hasattr(builtins.__xonsh__, "execer") and builtins.__xonsh__.execer is not None ): builtins.__xonsh__.execer.debug_level = val return val # # $LS_COLORS tools # class LsColors(cabc.MutableMapping): """Helps convert to/from $LS_COLORS format, respecting the xonsh color style. This accepts the same inputs as dict(). """ default_settings = { "*.7z": ("BOLD_RED",), "*.Z": ("BOLD_RED",), "*.aac": ("CYAN",), "*.ace": ("BOLD_RED",), "*.alz": ("BOLD_RED",), "*.arc": ("BOLD_RED",), "*.arj": ("BOLD_RED",), "*.asf": ("BOLD_PURPLE",), "*.au": ("CYAN",), "*.avi": ("BOLD_PURPLE",), "*.bmp": ("BOLD_PURPLE",), "*.bz": ("BOLD_RED",), "*.bz2": ("BOLD_RED",), "*.cab": ("BOLD_RED",), "*.cgm": ("BOLD_PURPLE",), "*.cpio": ("BOLD_RED",), "*.deb": ("BOLD_RED",), "*.dl": ("BOLD_PURPLE",), "*.dwm": ("BOLD_RED",), "*.dz": ("BOLD_RED",), "*.ear": ("BOLD_RED",), "*.emf": ("BOLD_PURPLE",), "*.esd": ("BOLD_RED",), "*.flac": ("CYAN",), "*.flc": ("BOLD_PURPLE",), "*.fli": ("BOLD_PURPLE",), "*.flv": ("BOLD_PURPLE",), "*.gif": ("BOLD_PURPLE",), "*.gl": ("BOLD_PURPLE",), "*.gz": ("BOLD_RED",), "*.jar": ("BOLD_RED",), "*.jpeg": ("BOLD_PURPLE",), "*.jpg": ("BOLD_PURPLE",), "*.lha": ("BOLD_RED",), "*.lrz": ("BOLD_RED",), "*.lz": ("BOLD_RED",), "*.lz4": ("BOLD_RED",), "*.lzh": ("BOLD_RED",), "*.lzma": ("BOLD_RED",), "*.lzo": ("BOLD_RED",), "*.m2v": ("BOLD_PURPLE",), "*.m4a": ("CYAN",), "*.m4v": ("BOLD_PURPLE",), "*.mid": ("CYAN",), "*.midi": ("CYAN",), "*.mjpeg": ("BOLD_PURPLE",), "*.mjpg": ("BOLD_PURPLE",), "*.mka": ("CYAN",), "*.mkv": ("BOLD_PURPLE",), "*.mng": ("BOLD_PURPLE",), "*.mov": ("BOLD_PURPLE",), "*.mp3": ("CYAN",), "*.mp4": ("BOLD_PURPLE",), "*.mp4v": ("BOLD_PURPLE",), "*.mpc": ("CYAN",), "*.mpeg": ("BOLD_PURPLE",), "*.mpg": ("BOLD_PURPLE",), "*.nuv": ("BOLD_PURPLE",), "*.oga": ("CYAN",), "*.ogg": ("CYAN",), "*.ogm": ("BOLD_PURPLE",), "*.ogv": ("BOLD_PURPLE",), "*.ogx": ("BOLD_PURPLE",), "*.opus": ("CYAN",), "*.pbm": ("BOLD_PURPLE",), "*.pcx": ("BOLD_PURPLE",), "*.pgm": ("BOLD_PURPLE",), "*.png": ("BOLD_PURPLE",), "*.ppm": ("BOLD_PURPLE",), "*.qt": ("BOLD_PURPLE",), "*.ra": ("CYAN",), "*.rar": ("BOLD_RED",), "*.rm": ("BOLD_PURPLE",), "*.rmvb": ("BOLD_PURPLE",), "*.rpm": ("BOLD_RED",), "*.rz": ("BOLD_RED",), "*.sar": ("BOLD_RED",), "*.spx": ("CYAN",), "*.svg": ("BOLD_PURPLE",), "*.svgz": ("BOLD_PURPLE",), "*.swm": ("BOLD_RED",), "*.t7z": ("BOLD_RED",), "*.tar": ("BOLD_RED",), "*.taz": ("BOLD_RED",), "*.tbz": ("BOLD_RED",), "*.tbz2": ("BOLD_RED",), "*.tga": ("BOLD_PURPLE",), "*.tgz": ("BOLD_RED",), "*.tif": ("BOLD_PURPLE",), "*.tiff": ("BOLD_PURPLE",), "*.tlz": ("BOLD_RED",), "*.txz": ("BOLD_RED",), "*.tz": ("BOLD_RED",), "*.tzo": ("BOLD_RED",), "*.tzst": ("BOLD_RED",), "*.vob": ("BOLD_PURPLE",), "*.war": ("BOLD_RED",), "*.wav": ("CYAN",), "*.webm": ("BOLD_PURPLE",), "*.wim": ("BOLD_RED",), "*.wmv": ("BOLD_PURPLE",), "*.xbm": ("BOLD_PURPLE",), "*.xcf": ("BOLD_PURPLE",), "*.xpm": ("BOLD_PURPLE",), "*.xspf": ("CYAN",), "*.xwd": ("BOLD_PURPLE",), "*.xz": ("BOLD_RED",), "*.yuv": ("BOLD_PURPLE",), "*.z": ("BOLD_RED",), "*.zip": ("BOLD_RED",), "*.zoo": ("BOLD_RED",), "*.zst": ("BOLD_RED",), "bd": ("BACKGROUND_BLACK", "YELLOW"), "ca": ("BLACK", "BACKGROUND_RED"), "cd": ("BACKGROUND_BLACK", "YELLOW"), "di": ("BOLD_BLUE",), "do": ("BOLD_PURPLE",), "ex": ("BOLD_GREEN",), "ln": ("BOLD_CYAN",), "mh": ("NO_COLOR",), "mi": ("NO_COLOR",), "or": ("BACKGROUND_BLACK", "RED"), "ow": ("BLUE", "BACKGROUND_GREEN"), "pi": ("BACKGROUND_BLACK", "YELLOW"), "rs": ("NO_COLOR",), "sg": ("BLACK", "BACKGROUND_YELLOW"), "so": ("BOLD_PURPLE",), "st": ("WHITE", "BACKGROUND_BLUE"), "su": ("WHITE", "BACKGROUND_RED"), "tw": ("BLACK", "BACKGROUND_GREEN"), } def __init__(self, *args, **kwargs): self._d = dict(*args, **kwargs) self._style = self._style_name = None self._detyped = None def __getitem__(self, key): return self._d[key] def __setitem__(self, key, value): self._detyped = None self._d[key] = value def __delitem__(self, key): self._detyped = None del self._d[key] def __len__(self): return len(self._d) def __iter__(self): yield from self._d def __str__(self): return str(self._d) def __repr__(self): return "{0}.{1}(...)".format( self.__class__.__module__, self.__class__.__name__, self._d ) def _repr_pretty_(self, p, cycle): name = "{0}.{1}".format(self.__class__.__module__, self.__class__.__name__) with p.group(0, name + "(", ")"): if cycle: p.text("...") elif len(self): p.break_() p.pretty(dict(self)) def detype(self): """De-types the instance, allowing it to be exported to the environment.""" style = self.style if self._detyped is None: self._detyped = ":".join( [ key + "=" + ";".join([style[v] or "0" for v in val]) for key, val in sorted(self._d.items()) ] ) return self._detyped @property def style_name(self): """Current XONSH_COLOR_STYLE value""" env = builtins.__xonsh__.env env_style_name = env.get("XONSH_COLOR_STYLE") if self._style_name is None or self._style_name != env_style_name: self._style_name = env_style_name self._style = self._dtyped = None return self._style_name @property def style(self): """The ANSI color style for the current XONSH_COLOR_STYLE""" style_name = self.style_name if self._style is None: self._style = ansi_style_by_name(style_name) self._detyped = None return self._style @classmethod def fromstring(cls, s): """Creates a new instance of the LsColors class from a colon-separated string of dircolor-valid keys to ANSI color escape sequences. """ obj = cls() # string inputs always use default codes, so translating into # xonsh names should be done from defaults reversed_default = ansi_reverse_style(style="default") data = {} for item in s.split(":"): key, eq, esc = item.partition("=") if not eq: # not a valid item continue data[key] = ansi_color_escape_code_to_name( esc, "default", reversed_style=reversed_default ) obj._d = data return obj @classmethod def fromdircolors(cls, filename=None): """Constructs an LsColors instance by running dircolors. If a filename is provided, it is passed down to the dircolors command. """ # assemble command cmd = ["dircolors", "-b"] if filename is not None: cmd.append(filename) # get env if hasattr(builtins, "__xonsh__") and hasattr(builtins.__xonsh__, "env"): denv = builtins.__xonsh__.env.detype() else: denv = None # run dircolors try: out = subprocess.check_output( cmd, env=denv, universal_newlines=True, stderr=subprocess.DEVNULL ) except (subprocess.CalledProcessError, FileNotFoundError): return cls(cls.default_settings) s = out.splitlines()[0] _, _, s = s.partition("'") s, _, _ = s.rpartition("'") return cls.fromstring(s) @classmethod def convert(cls, x): """Converts an object to LsColors, if needed.""" if isinstance(x, cls): return x elif isinstance(x, str): return cls.fromstring(x) elif isinstance(x, bytes): return cls.fromstring(x.decode()) else: return cls(x) def is_lscolors(x): """Checks if an object is an instance of LsColors""" return isinstance(x, LsColors) @events.on_pre_spec_run_ls def ensure_ls_colors_in_env(spec=None, **kwargs): """This ensures that the $LS_COLORS environment variable is in the environment. This fires exactly once upon the first time the ls command is called. """ env = builtins.__xonsh__.env if "LS_COLORS" not in env._d: # this adds it to the env too default_lscolors(env) events.on_pre_spec_run_ls.discard(ensure_ls_colors_in_env) # # Ensurerers # Ensurer = collections.namedtuple("Ensurer", ["validate", "convert", "detype"]) Ensurer.__doc__ = """Named tuples whose elements are functions that represent environment variable validation, conversion, detyping. """ @lazyobject def DEFAULT_ENSURERS(): return { "AUTO_CD": (is_bool, to_bool, bool_to_str), "AUTO_PUSHD": (is_bool, to_bool, bool_to_str), "AUTO_SUGGEST": (is_bool, to_bool, bool_to_str), "AUTO_SUGGEST_IN_COMPLETIONS": (is_bool, to_bool, bool_to_str), "BASH_COMPLETIONS": (is_env_path, str_to_env_path, env_path_to_str), "CASE_SENSITIVE_COMPLETIONS": (is_bool, to_bool, bool_to_str), re.compile(r"\w*DIRS$"): (is_env_path, str_to_env_path, env_path_to_str), "COLOR_INPUT": (is_bool, to_bool, bool_to_str), "COLOR_RESULTS": (is_bool, to_bool, bool_to_str), "COMPLETIONS_BRACKETS": (is_bool, to_bool, bool_to_str), "COMPLETIONS_CONFIRM": (is_bool, to_bool, bool_to_str), "COMPLETIONS_DISPLAY": ( is_completions_display_value, to_completions_display_value, str, ), "COMPLETIONS_MENU_ROWS": (is_int, int, str), "COMPLETION_QUERY_LIMIT": (is_int, int, str), "DIRSTACK_SIZE": (is_int, int, str), "DOTGLOB": (is_bool, to_bool, bool_to_str), "DYNAMIC_CWD_WIDTH": ( is_dynamic_cwd_width, to_dynamic_cwd_tuple, dynamic_cwd_tuple_to_str, ), "DYNAMIC_CWD_ELISION_CHAR": (is_string, ensure_string, ensure_string), "EXPAND_ENV_VARS": (is_bool, to_bool, bool_to_str), "FORCE_POSIX_PATHS": (is_bool, to_bool, bool_to_str), "FOREIGN_ALIASES_SUPPRESS_SKIP_MESSAGE": (is_bool, to_bool, bool_to_str), "FOREIGN_ALIASES_OVERRIDE": (is_bool, to_bool, bool_to_str), "FUZZY_PATH_COMPLETION": (is_bool, to_bool, bool_to_str), "GLOB_SORTED": (is_bool, to_bool, bool_to_str), "HISTCONTROL": (is_string_set, csv_to_set, set_to_csv), "IGNOREEOF": (is_bool, to_bool, bool_to_str), "INTENSIFY_COLORS_ON_WIN": ( always_false, intensify_colors_on_win_setter, bool_to_str, ), "LANG": (is_string, ensure_string, ensure_string), "LC_COLLATE": (always_false, locale_convert("LC_COLLATE"), ensure_string), "LC_CTYPE": (always_false, locale_convert("LC_CTYPE"), ensure_string), "LC_MESSAGES": (always_false, locale_convert("LC_MESSAGES"), ensure_string), "LC_MONETARY": (always_false, locale_convert("LC_MONETARY"), ensure_string), "LC_NUMERIC": (always_false, locale_convert("LC_NUMERIC"), ensure_string), "LC_TIME": (always_false, locale_convert("LC_TIME"), ensure_string), "LS_COLORS": (is_lscolors, LsColors.convert, detype), "LOADED_RC_FILES": (is_bool_seq, csv_to_bool_seq, bool_seq_to_csv), "MOUSE_SUPPORT": (is_bool, to_bool, bool_to_str), "MULTILINE_PROMPT": (is_string_or_callable, ensure_string, ensure_string), re.compile(r"\w*PATH$"): (is_env_path, str_to_env_path, env_path_to_str), "PATHEXT": ( is_nonstring_seq_of_strings, pathsep_to_upper_seq, seq_to_upper_pathsep, ), "PRETTY_PRINT_RESULTS": (is_bool, to_bool, bool_to_str), "PROMPT": (is_string_or_callable, ensure_string, ensure_string), "PROMPT_FIELDS": (always_true, None, None), "PROMPT_TOOLKIT_COLOR_DEPTH": ( always_false, ptk2_color_depth_setter, ensure_string, ), "PUSHD_MINUS": (is_bool, to_bool, bool_to_str), "PUSHD_SILENT": (is_bool, to_bool, bool_to_str), "PTK_STYLE_OVERRIDES": (is_str_str_dict, to_str_str_dict, dict_to_str), "RAISE_SUBPROC_ERROR": (is_bool, to_bool, bool_to_str), "RIGHT_PROMPT": (is_string_or_callable, ensure_string, ensure_string), "BOTTOM_TOOLBAR": (is_string_or_callable, ensure_string, ensure_string), "SUBSEQUENCE_PATH_COMPLETION": (is_bool, to_bool, bool_to_str), "SUGGEST_COMMANDS": (is_bool, to_bool, bool_to_str), "SUGGEST_MAX_NUM": (is_int, int, str), "SUGGEST_THRESHOLD": (is_int, int, str), "SUPPRESS_BRANCH_TIMEOUT_MESSAGE": (is_bool, to_bool, bool_to_str), "UPDATE_COMPLETIONS_ON_KEYPRESS": (is_bool, to_bool, bool_to_str), "UPDATE_OS_ENVIRON": (is_bool, to_bool, bool_to_str), "UPDATE_PROMPT_ON_KEYPRESS": (is_bool, to_bool, bool_to_str), "VC_BRANCH_TIMEOUT": (is_float, float, str), "VC_HG_SHOW_BRANCH": (is_bool, to_bool, bool_to_str), "VI_MODE": (is_bool, to_bool, bool_to_str), "VIRTUAL_ENV": (is_string, ensure_string, ensure_string), "WIN_UNICODE_CONSOLE": (always_false, setup_win_unicode_console, bool_to_str), "XONSHRC": (is_env_path, str_to_env_path, env_path_to_str), "XONSH_APPEND_NEWLINE": (is_bool, to_bool, bool_to_str), "XONSH_AUTOPAIR": (is_bool, to_bool, bool_to_str), "XONSH_CACHE_SCRIPTS": (is_bool, to_bool, bool_to_str), "XONSH_CACHE_EVERYTHING": (is_bool, to_bool, bool_to_str), "XONSH_COLOR_STYLE": (is_string, ensure_string, ensure_string), "XONSH_DEBUG": (always_false, to_debug, bool_or_int_to_str), "XONSH_ENCODING": (is_string, ensure_string, ensure_string), "XONSH_ENCODING_ERRORS": (is_string, ensure_string, ensure_string), "XONSH_HISTORY_BACKEND": (is_history_backend, to_itself, ensure_string), "XONSH_HISTORY_FILE": (is_string, ensure_string, ensure_string), "XONSH_HISTORY_MATCH_ANYWHERE": (is_bool, to_bool, bool_to_str), "XONSH_HISTORY_SIZE": ( is_history_tuple, to_history_tuple, history_tuple_to_str, ), "XONSH_LOGIN": (is_bool, to_bool, bool_to_str), "XONSH_PROC_FREQUENCY": (is_float, float, str), "XONSH_SHOW_TRACEBACK": (is_bool, to_bool, bool_to_str), "XONSH_STDERR_PREFIX": (is_string, ensure_string, ensure_string), "XONSH_STDERR_POSTFIX": (is_string, ensure_string, ensure_string), "XONSH_STORE_STDOUT": (is_bool, to_bool, bool_to_str), "XONSH_STORE_STDIN": (is_bool, to_bool, bool_to_str), "XONSH_TRACEBACK_LOGFILE": (is_logfile_opt, to_logfile_opt, logfile_opt_to_str), "XONSH_DATETIME_FORMAT": (is_string, ensure_string, ensure_string), } # # Defaults # def default_value(f): """Decorator for making callable default values.""" f._xonsh_callable_default = True return f def is_callable_default(x): """Checks if a value is a callable default.""" return callable(x) and getattr(x, "_xonsh_callable_default", False) DEFAULT_TITLE = "{current_job:{} | }{user}@{hostname}: {cwd} | xonsh" @default_value def xonsh_data_dir(env): """Ensures and returns the $XONSH_DATA_DIR""" xdd = os.path.expanduser(os.path.join(env.get("XDG_DATA_HOME"), "xonsh")) os.makedirs(xdd, exist_ok=True) return xdd @default_value def xonsh_config_dir(env): """Ensures and returns the $XONSH_CONFIG_DIR""" xcd = os.path.expanduser(os.path.join(env.get("XDG_CONFIG_HOME"), "xonsh")) os.makedirs(xcd, exist_ok=True) return xcd def xonshconfig(env): """Ensures and returns the $XONSHCONFIG""" xcd = env.get("XONSH_CONFIG_DIR") xc = os.path.join(xcd, "config.json") return xc @default_value def default_xonshrc(env): """Creates a new instance of the default xonshrc tuple.""" xcdrc = os.path.join(xonsh_config_dir(env), "rc.xsh") if ON_WINDOWS: dxrc = ( os.path.join(os_environ["ALLUSERSPROFILE"], "xonsh", "xonshrc"), xcdrc, os.path.expanduser("~/.xonshrc"), ) else: dxrc = ("/etc/xonshrc", xcdrc, os.path.expanduser("~/.xonshrc")) # Check if old config file exists and issue warning old_config_filename = xonshconfig(env) if os.path.isfile(old_config_filename): print( "WARNING! old style configuration (" + old_config_filename + ") is no longer supported. " + "Please migrate to xonshrc." ) return dxrc @default_value def xonsh_append_newline(env): """Appends a newline if we are in interactive mode""" return env.get("XONSH_INTERACTIVE", False) @default_value def default_lscolors(env): """Gets a default instanse of LsColors""" inherited_lscolors = os_environ.get("LS_COLORS", None) if inherited_lscolors is None: lsc = LsColors.fromdircolors() else: lsc = LsColors.fromstring(inherited_lscolors) # have to place this in the env, so it is applied env["LS_COLORS"] = lsc return lsc # Default values should generally be immutable, that way if a user wants # to set them they have to do a copy and write them to the environment. # try to keep this sorted. @lazyobject def DEFAULT_VALUES(): dv = { "AUTO_CD": False, "AUTO_PUSHD": False, "AUTO_SUGGEST": True, "AUTO_SUGGEST_IN_COMPLETIONS": False, "BASH_COMPLETIONS": BASH_COMPLETIONS_DEFAULT, "CASE_SENSITIVE_COMPLETIONS": ON_LINUX, "CDPATH": (), "COLOR_INPUT": True, "COLOR_RESULTS": False, "COMPLETIONS_BRACKETS": True, "COMPLETIONS_CONFIRM": False, "COMPLETIONS_DISPLAY": "single", "COMPLETIONS_MENU_ROWS": 5, "COMPLETION_QUERY_LIMIT": 100, "DIRSTACK_SIZE": 20, "DOTGLOB": False, "DYNAMIC_CWD_WIDTH": (float("inf"), "c"), "DYNAMIC_CWD_ELISION_CHAR": "", "EXPAND_ENV_VARS": True, "FORCE_POSIX_PATHS": False, "FOREIGN_ALIASES_SUPPRESS_SKIP_MESSAGE": False, "FOREIGN_ALIASES_OVERRIDE": False, "PROMPT_FIELDS": dict(prompt.PROMPT_FIELDS), "FUZZY_PATH_COMPLETION": True, "GLOB_SORTED": True, "HISTCONTROL": set(), "IGNOREEOF": False, "INDENT": " ", "INTENSIFY_COLORS_ON_WIN": True, "LANG": "C.UTF-8", "LC_CTYPE": locale.setlocale(locale.LC_CTYPE), "LC_COLLATE": locale.setlocale(locale.LC_COLLATE), "LC_TIME": locale.setlocale(locale.LC_TIME), "LC_MONETARY": locale.setlocale(locale.LC_MONETARY), "LC_NUMERIC": locale.setlocale(locale.LC_NUMERIC), "LS_COLORS": default_lscolors, "LOADED_RC_FILES": (), "MOUSE_SUPPORT": False, "MULTILINE_PROMPT": ".", "PATH": PATH_DEFAULT, "PATHEXT": [".COM", ".EXE", ".BAT", ".CMD"] if ON_WINDOWS else [], "PRETTY_PRINT_RESULTS": True, "PROMPT": prompt.default_prompt(), "PROMPT_TOOLKIT_COLOR_DEPTH": "", "PTK_STYLE_OVERRIDES": dict(PTK2_STYLE), "PUSHD_MINUS": False, "PUSHD_SILENT": False, "RAISE_SUBPROC_ERROR": False, "RIGHT_PROMPT": "", "BOTTOM_TOOLBAR": "", "SHELL_TYPE": "best", "SUBSEQUENCE_PATH_COMPLETION": True, "SUPPRESS_BRANCH_TIMEOUT_MESSAGE": False, "SUGGEST_COMMANDS": True, "SUGGEST_MAX_NUM": 5, "SUGGEST_THRESHOLD": 3, "TITLE": DEFAULT_TITLE, "UPDATE_COMPLETIONS_ON_KEYPRESS": True, "UPDATE_OS_ENVIRON": False, "UPDATE_PROMPT_ON_KEYPRESS": False, "VC_BRANCH_TIMEOUT": 0.2 if ON_WINDOWS else 0.1, "VC_HG_SHOW_BRANCH": True, "VI_MODE": False, "WIN_UNICODE_CONSOLE": True, "XDG_CONFIG_HOME": os.path.expanduser(os.path.join("~", ".config")), "XDG_DATA_HOME": os.path.expanduser(os.path.join("~", ".local", "share")), "XONSHRC": default_xonshrc, "XONSH_APPEND_NEWLINE": xonsh_append_newline, "XONSH_AUTOPAIR": False, "XONSH_CACHE_SCRIPTS": True, "XONSH_CACHE_EVERYTHING": False, "XONSH_COLOR_STYLE": "default", "XONSH_CONFIG_DIR": xonsh_config_dir, "XONSH_DATA_DIR": xonsh_data_dir, "XONSH_DEBUG": 0, "XONSH_ENCODING": DEFAULT_ENCODING, "XONSH_ENCODING_ERRORS": "surrogateescape", "XONSH_HISTORY_BACKEND": "json", "XONSH_HISTORY_FILE": os.path.expanduser("~/.xonsh_history.json"), "XONSH_HISTORY_MATCH_ANYWHERE": False, "XONSH_HISTORY_SIZE": (8128, "commands"), "XONSH_LOGIN": False, "XONSH_PROC_FREQUENCY": 1e-4, "XONSH_SHOW_TRACEBACK": False, "XONSH_STDERR_PREFIX": "", "XONSH_STDERR_POSTFIX": "", "XONSH_STORE_STDIN": False, "XONSH_STORE_STDOUT": False, "XONSH_TRACEBACK_LOGFILE": None, "XONSH_DATETIME_FORMAT": "%Y-%m-%d %H:%M", } if hasattr(locale, "LC_MESSAGES"): dv["LC_MESSAGES"] = locale.setlocale(locale.LC_MESSAGES) return dv VarDocs = collections.namedtuple( "VarDocs", ["docstr", "configurable", "default", "store_as_str"] ) VarDocs.__doc__ = """Named tuple for environment variable documentation Parameters ---------- docstr : str The environment variable docstring. configurable : bool, optional Flag for whether the environment variable is configurable or not. default : str, optional Custom docstring for the default value for complex defaults. Is this is DefaultNotGiven, then the default will be looked up from DEFAULT_VALUES and converted to a str. store_as_str : bool, optional Flag for whether the environment variable should be stored as a string. This is used when persisting a variable that is not JSON serializable to the config file. For example, sets, frozensets, and potentially other non-trivial data types. default, False. """ # iterates from back VarDocs.__new__.__defaults__ = (True, DefaultNotGiven, False) # Please keep the following in alphabetic order - scopatz @lazyobject def DEFAULT_DOCS(): return { "ANSICON": VarDocs( "This is used on Windows to set the title, " "if available.", configurable=False, ), "AUTO_CD": VarDocs( "Flag to enable changing to a directory by entering the dirname or " "full path only (without the cd command)." ), "AUTO_PUSHD": VarDocs( "Flag for automatically pushing directories onto the directory stack." ), "AUTO_SUGGEST": VarDocs( "Enable automatic command suggestions based on history, like in the fish " "shell.\n\nPressing the right arrow key inserts the currently " "displayed suggestion. Only usable with ``$SHELL_TYPE=prompt_toolkit.``" ), "AUTO_SUGGEST_IN_COMPLETIONS": VarDocs( "Places the auto-suggest result as the first option in the completions. " "This enables you to tab complete the auto-suggestion." ), "BASH_COMPLETIONS": VarDocs( "This is a list (or tuple) of strings that specifies where the " "``bash_completion`` script may be found. " "The first valid path will be used. For better performance, " "bash-completion v2.x is recommended since it lazy-loads individual " "completion scripts. " "For both bash-completion v1.x and v2.x, paths of individual completion " "scripts (like ``.../completes/ssh``) do not need to be included here. " "The default values are platform " "dependent, but sane. To specify an alternate list, do so in the run " "control file.", default=( "Normally this is:\n\n" " ``('/usr/share/bash-completion/bash_completion', )``\n\n" "But, on Mac it is:\n\n" " ``('/usr/local/share/bash-completion/bash_completion', " "'/usr/local/etc/bash_completion')``\n\n" "Other OS-specific defaults may be added in the future." ), ), "CASE_SENSITIVE_COMPLETIONS": VarDocs( "Sets whether completions should be case sensitive or case " "insensitive.", default="True on Linux, False otherwise.", ), "CDPATH": VarDocs( "A list of paths to be used as roots for a cd, breaking compatibility " "with Bash, xonsh always prefer an existing relative path." ), "COLOR_INPUT": VarDocs("Flag for syntax highlighting interactive input."), "COLOR_RESULTS": VarDocs("Flag for syntax highlighting return values."), "COMPLETIONS_BRACKETS": VarDocs( "Flag to enable/disable inclusion of square brackets and parentheses " "in Python attribute completions.", default="True", ), "COMPLETIONS_DISPLAY": VarDocs( "Configure if and how Python completions are displayed by the " "``prompt_toolkit`` shell.\n\nThis option does not affect Bash " "completions, auto-suggestions, etc.\n\nChanging it at runtime will " "take immediate effect, so you can quickly disable and enable " "completions during shell sessions.\n\n" "- If ``$COMPLETIONS_DISPLAY`` is ``none`` or ``false``, do not display\n" " those completions.\n" "- If ``$COMPLETIONS_DISPLAY`` is ``single``, display completions in a\n" " single column while typing.\n" "- If ``$COMPLETIONS_DISPLAY`` is ``multi`` or ``true``, display completions\n" " in multiple columns while typing.\n\n" "- If ``$COMPLETIONS_DISPLAY`` is ``readline``, display completions\n" " will emulate the behavior of readline.\n\n" "These option values are not case- or type-sensitive, so e.g." "writing ``$COMPLETIONS_DISPLAY = None`` " "and ``$COMPLETIONS_DISPLAY = 'none'`` are equivalent. Only usable with " "``$SHELL_TYPE=prompt_toolkit``" ), "COMPLETIONS_CONFIRM": VarDocs( "While tab-completions menu is displayed, press to confirm " "completion instead of running command. This only affects the " "prompt-toolkit shell." ), "COMPLETIONS_MENU_ROWS": VarDocs( "Number of rows to reserve for tab-completions menu if " "``$COMPLETIONS_DISPLAY`` is ``single`` or ``multi``. This only affects the " "prompt-toolkit shell." ), "COMPLETION_QUERY_LIMIT": VarDocs( "The number of completions to display before the user is asked " "for confirmation." ), "DIRSTACK_SIZE": VarDocs("Maximum size of the directory stack."), "DOTGLOB": VarDocs( 'Globbing files with "*" or "**" will also match ' "dotfiles, or those 'hidden' files whose names " "begin with a literal '.'. Such files are filtered " "out by default." ), "DYNAMIC_CWD_WIDTH": VarDocs( "Maximum length in number of characters " "or as a percentage for the ``cwd`` prompt variable. For example, " '"20" is a twenty character width and "10%" is ten percent of the ' "number of columns available." ), "DYNAMIC_CWD_ELISION_CHAR": VarDocs( "The string used to show a shortened directory in a shortened cwd, " "e.g. ``'…'``." ), "EXPAND_ENV_VARS": VarDocs( "Toggles whether environment variables are expanded inside of strings " "in subprocess mode." ), "FORCE_POSIX_PATHS": VarDocs( "Forces forward slashes (``/``) on Windows systems when using auto " "completion if set to anything truthy.", configurable=ON_WINDOWS, ), "FOREIGN_ALIASES_SUPPRESS_SKIP_MESSAGE": VarDocs( "Whether or not foreign aliases should suppress the message " "that informs the user when a foreign alias has been skipped " "because it already exists in xonsh.", configurable=True, ), "FOREIGN_ALIASES_OVERRIDE": VarDocs( "Whether or not foreign aliases should override xonsh aliases " "with the same name. Note that setting of this must happen in the " "environment that xonsh was started from. " "It cannot be set in the ``.xonshrc`` as loading of foreign aliases happens before" "``.xonshrc`` is parsed", configurable=True, ), "PROMPT_FIELDS": VarDocs( "Dictionary containing variables to be used when formatting $PROMPT " "and $TITLE. See 'Customizing the Prompt' " "http://xon.sh/tutorial.html#customizing-the-prompt", configurable=False, default="``xonsh.prompt.PROMPT_FIELDS``", ), "FUZZY_PATH_COMPLETION": VarDocs( "Toggles 'fuzzy' matching of paths for tab completion, which is only " "used as a fallback if no other completions succeed but can be used " "as a way to adjust for typographical errors. If ``True``, then, e.g.," " ``xonhs`` will match ``xonsh``." ), "GLOB_SORTED": VarDocs( "Toggles whether globbing results are manually sorted. If ``False``, " "the results are returned in arbitrary order." ), "HISTCONTROL": VarDocs( "A set of strings (comma-separated list in string form) of options " "that determine what commands are saved to the history list. By " "default all commands are saved. The option ``ignoredups`` will not " "save the command if it matches the previous command. The option " "'ignoreerr' will cause any commands that fail (i.e. return non-zero " "exit status) to not be added to the history list.", store_as_str=True, ), "IGNOREEOF": VarDocs("Prevents Ctrl-D from exiting the shell."), "INDENT": VarDocs("Indentation string for multiline input"), "INTENSIFY_COLORS_ON_WIN": VarDocs( "Enhance style colors for readability " "when using the default terminal (``cmd.exe``) on Windows. Blue colors, " "which are hard to read, are replaced with cyan. Other colors are " "generally replaced by their bright counter parts.", configurable=ON_WINDOWS, ), "LANG": VarDocs("Fallback locale setting for systems where it matters"), "LS_COLORS": VarDocs("Color settings for ``ls`` command line utility"), "LOADED_RC_FILES": VarDocs( "Whether or not any of the xonsh run control files were loaded at " "startup. This is a sequence of bools in Python that is converted " "to a CSV list in string form, ie ``[True, False]`` becomes " "``'True,False'``.", configurable=False, ), "MOUSE_SUPPORT": VarDocs( "Enable mouse support in the ``prompt_toolkit`` shell. This allows " "clicking for positioning the cursor or selecting a completion. In " "some terminals however, this disables the ability to scroll back " "through the history of the terminal. Only usable with " "``$SHELL_TYPE=prompt_toolkit``" ), "MULTILINE_PROMPT": VarDocs( "Prompt text for 2nd+ lines of input, may be str or function which " "returns a str." ), "OLDPWD": VarDocs( "Used to represent a previous present working directory.", configurable=False, ), "PATH": VarDocs("List of strings representing where to look for executables."), "PATHEXT": VarDocs( "Sequence of extension strings (eg, ``.EXE``) for " "filtering valid executables by. Each element must be " "uppercase." ), "PRETTY_PRINT_RESULTS": VarDocs('Flag for "pretty printing" return values.'), "PROMPT": VarDocs( "The prompt text. May contain keyword arguments which are " "auto-formatted, see 'Customizing the Prompt' at " "http://xon.sh/tutorial.html#customizing-the-prompt. " "This value is never inherited from parent processes.", default="``xonsh.environ.DEFAULT_PROMPT``", ), "PROMPT_TOOLKIT_COLOR_DEPTH": VarDocs( "The color depth used by prompt toolkit 2. Possible values are: " "``DEPTH_1_BIT``, ``DEPTH_4_BIT``, ``DEPTH_8_BIT``, ``DEPTH_24_BIT`` " "colors. Default is an empty string which means that prompt toolkit decide." ), "PTK_STYLE_OVERRIDES": VarDocs( "A dictionary containing custom prompt_toolkit style definitions." ), "PUSHD_MINUS": VarDocs( "Flag for directory pushing functionality. False is the normal " "behavior." ), "PUSHD_SILENT": VarDocs( "Whether or not to suppress directory stack manipulation output." ), "RAISE_SUBPROC_ERROR": VarDocs( "Whether or not to raise an error if a subprocess (captured or " "uncaptured) returns a non-zero exit status, which indicates failure. " "This is most useful in xonsh scripts or modules where failures " "should cause an end to execution. This is less useful at a terminal. " "The error that is raised is a ``subprocess.CalledProcessError``." ), "RIGHT_PROMPT": VarDocs( "Template string for right-aligned text " "at the prompt. This may be parametrized in the same way as " "the ``$PROMPT`` variable. Currently, this is only available in the " "prompt-toolkit shell." ), "BOTTOM_TOOLBAR": VarDocs( "Template string for the bottom toolbar. " "This may be parametrized in the same way as " "the ``$PROMPT`` variable. Currently, this is only available in the " "prompt-toolkit shell." ), "SHELL_TYPE": VarDocs( "Which shell is used. Currently two base shell types are supported:\n\n" " - ``readline`` that is backed by Python's readline module\n" " - ``prompt_toolkit`` that uses external library of the same name\n" " - ``random`` selects a random shell from the above on startup\n" " - ``best`` selects the most feature-rich shell available on the\n" " user's system\n\n" "To use the ``prompt_toolkit`` shell you need to have the " "`prompt_toolkit `_" " library installed. To specify which shell should be used, do so in " "the run control file.", default="``best``", ), "SUBSEQUENCE_PATH_COMPLETION": VarDocs( "Toggles subsequence matching of paths for tab completion. " "If ``True``, then, e.g., ``~/u/ro`` can match ``~/lou/carcolh``." ), "SUGGEST_COMMANDS": VarDocs( "When a user types an invalid command, xonsh will try to offer " "suggestions of similar valid commands if this is True." ), "SUGGEST_MAX_NUM": VarDocs( "xonsh will show at most this many suggestions in response to an " "invalid command. If negative, there is no limit to how many " "suggestions are shown." ), "SUGGEST_THRESHOLD": VarDocs( "An error threshold. If the Levenshtein distance between the entered " "command and a valid command is less than this value, the valid " 'command will be offered as a suggestion. Also used for "fuzzy" ' "tab completion of paths." ), "SUPPRESS_BRANCH_TIMEOUT_MESSAGE": VarDocs( "Whether or not to suppress branch timeout warning messages." ), "TERM": VarDocs( "TERM is sometimes set by the terminal emulator. This is used (when " "valid) to determine whether or not to set the title. Users shouldn't " "need to set this themselves. Note that this variable should be set as " "early as possible in order to ensure it is effective. Here are a few " "options:\n\n" "* Set this from the program that launches xonsh. On POSIX systems, \n" " this can be performed by using env, e.g. \n" " ``/usr/bin/env TERM=xterm-color xonsh`` or similar.\n" "* From the xonsh command line, namely ``xonsh -DTERM=xterm-color``.\n" '* In the config file with ``{"env": {"TERM": "xterm-color"}}``.\n' "* Lastly, in xonshrc with ``$TERM``\n\n" "Ideally, your terminal emulator will set this correctly but that does " "not always happen.", configurable=False, ), "TITLE": VarDocs( "The title text for the window in which xonsh is running. Formatted " "in the same manner as ``$PROMPT``, see 'Customizing the Prompt' " "http://xon.sh/tutorial.html#customizing-the-prompt.", default="``xonsh.environ.DEFAULT_TITLE``", ), "UPDATE_COMPLETIONS_ON_KEYPRESS": VarDocs( "Completions display is evaluated and presented whenever a key is " "pressed. This avoids the need to press TAB, except to cycle through " "the possibilities. This currently only affects the prompt-toolkit shell." ), "UPDATE_OS_ENVIRON": VarDocs( "If True ``os_environ`` will always be updated " "when the xonsh environment changes. The environment can be reset to " "the default value by calling ``__xonsh__.env.undo_replace_env()``" ), "UPDATE_PROMPT_ON_KEYPRESS": VarDocs( "Disables caching the prompt between commands, " "so that it would be reevaluated on each keypress. " "Disabled by default because of the incurred performance penalty." ), "VC_BRANCH_TIMEOUT": VarDocs( "The timeout (in seconds) for version control " "branch computations. This is a timeout per subprocess call, so the " "total time to compute will be larger than this in many cases." ), "VC_HG_SHOW_BRANCH": VarDocs( "Whether or not to show the Mercurial branch in the prompt." ), "VI_MODE": VarDocs( "Flag to enable ``vi_mode`` in the ``prompt_toolkit`` shell." ), "VIRTUAL_ENV": VarDocs( "Path to the currently active Python environment.", configurable=False ), "WIN_UNICODE_CONSOLE": VarDocs( "Enables unicode support in windows terminals. Requires the external " "library ``win_unicode_console``.", configurable=ON_WINDOWS, ), "XDG_CONFIG_HOME": VarDocs( "Open desktop standard configuration home dir. This is the same " "default as used in the standard.", configurable=False, default="``~/.config``", ), "XDG_DATA_HOME": VarDocs( "Open desktop standard data home dir. This is the same default as " "used in the standard.", default="``~/.local/share``", ), "XONSHRC": VarDocs( "A list of the locations of run control files, if they exist. User " "defined run control file will supersede values set in system-wide " "control file if there is a naming collision.", default=( "On Linux & Mac OSX: ``['/etc/xonshrc', '~/.config/xonsh/rc.xsh', '~/.xonshrc']``\n" "\nOn Windows: " "``['%ALLUSERSPROFILE%\\\\xonsh\\\\xonshrc', '~/.config/xonsh/rc.xsh', '~/.xonshrc']``" ), ), "XONSH_APPEND_NEWLINE": VarDocs( "Append new line when a partial line is preserved in output." ), "XONSH_AUTOPAIR": VarDocs( "Whether Xonsh will auto-insert matching parentheses, brackets, and " "quotes. Only available under the prompt-toolkit shell." ), "XONSH_CACHE_SCRIPTS": VarDocs( "Controls whether the code for scripts run from xonsh will be cached" " (``True``) or re-compiled each time (``False``)." ), "XONSH_CACHE_EVERYTHING": VarDocs( "Controls whether all code (including code entered at the interactive" " prompt) will be cached." ), "XONSH_COLOR_STYLE": VarDocs( "Sets the color style for xonsh colors. This is a style name, not " "a color map. Run ``xonfig styles`` to see the available styles." ), "XONSH_CONFIG_DIR": VarDocs( "This is the location where xonsh configuration information is stored.", configurable=False, default="``$XDG_CONFIG_HOME/xonsh``", ), "XONSH_DEBUG": VarDocs( "Sets the xonsh debugging level. This may be an integer or a boolean. " "Setting this variable prior to stating xonsh to ``1`` or ``True`` " "will suppress amalgamated imports. Setting it to ``2`` will get some " "basic information like input transformation, command replacement. " "With ``3`` or a higher number will make more debugging information " "presented, like PLY parsing messages.", configurable=False, ), "XONSH_DATA_DIR": VarDocs( "This is the location where xonsh data files are stored, such as " "history.", default="``$XDG_DATA_HOME/xonsh``", ), "XONSH_ENCODING": VarDocs( "This is the encoding that xonsh should use for subprocess operations.", default="``sys.getdefaultencoding()``", ), "XONSH_ENCODING_ERRORS": VarDocs( "The flag for how to handle encoding errors should they happen. " "Any string flag that has been previously registered with Python " "is allowed. See the 'Python codecs documentation' " "(https://docs.python.org/3/library/codecs.html#error-handlers) " "for more information and available options.", default="``surrogateescape``", ), "XONSH_GITSTATUS_*": VarDocs( "Symbols for gitstatus prompt. Default values are: \n\n" "* ``XONSH_GITSTATUS_HASH``: ``:``\n" "* ``XONSH_GITSTATUS_BRANCH``: ``{CYAN}``\n" "* ``XONSH_GITSTATUS_OPERATION``: ``{CYAN}``\n" "* ``XONSH_GITSTATUS_STAGED``: ``{RED}●``\n" "* ``XONSH_GITSTATUS_CONFLICTS``: ``{RED}×``\n" "* ``XONSH_GITSTATUS_CHANGED``: ``{BLUE}+``\n" "* ``XONSH_GITSTATUS_UNTRACKED``: ``…``\n" "* ``XONSH_GITSTATUS_STASHED``: ``⚑``\n" "* ``XONSH_GITSTATUS_CLEAN``: ``{BOLD_GREEN}✓``\n" "* ``XONSH_GITSTATUS_AHEAD``: ``↑·``\n" "* ``XONSH_GITSTATUS_BEHIND``: ``↓·``\n" ), "XONSH_HISTORY_BACKEND": VarDocs( "Set which history backend to use. Options are: 'json', " "'sqlite', and 'dummy'. The default is 'json'. " "``XONSH_HISTORY_BACKEND`` also accepts a class type that inherits " "from ``xonsh.history.base.History``, or its instance." ), "XONSH_HISTORY_FILE": VarDocs( "Location of history file (deprecated).", configurable=False, default="``~/.xonsh_history``", ), "XONSH_HISTORY_MATCH_ANYWHERE": VarDocs( "When searching history from a partial string (by pressing up arrow), " "match command history anywhere in a given line (not just the start)", default="False", ), "XONSH_HISTORY_SIZE": VarDocs( "Value and units tuple that sets the size of history after garbage " "collection. Canonical units are:\n\n" "- ``commands`` for the number of past commands executed,\n" "- ``files`` for the number of history files to keep,\n" "- ``s`` for the number of seconds in the past that are allowed, and\n" "- ``b`` for the number of bytes that history may consume.\n\n" "Common abbreviations, such as '6 months' or '1 GB' are also allowed.", default="``(8128, 'commands')`` or ``'8128 commands'``", ), "XONSH_INTERACTIVE": VarDocs( "``True`` if xonsh is running interactively, and ``False`` otherwise.", configurable=False, ), "XONSH_LOGIN": VarDocs( "``True`` if xonsh is running as a login shell, and ``False`` otherwise.", configurable=False, ), "XONSH_PROC_FREQUENCY": VarDocs( "The process frequency is the time that " "xonsh process threads sleep for while running command pipelines. " "The value has units of seconds [s]." ), "XONSH_SHOW_TRACEBACK": VarDocs( "Controls if a traceback is shown if exceptions occur in the shell. " "Set to ``True`` to always show traceback or ``False`` to always hide. " "If undefined then the traceback is hidden but a notice is shown on how " "to enable the full traceback." ), "XONSH_SOURCE": VarDocs( "When running a xonsh script, this variable contains the absolute path " "to the currently executing script's file.", configurable=False, ), "XONSH_STDERR_PREFIX": VarDocs( "A format string, using the same keys and colors as ``$PROMPT``, that " "is prepended whenever stderr is displayed. This may be used in " "conjunction with ``$XONSH_STDERR_POSTFIX`` to close out the block." "For example, to have stderr appear on a red background, the " 'prefix & postfix pair would be "{BACKGROUND_RED}" & "{NO_COLOR}".' ), "XONSH_STDERR_POSTFIX": VarDocs( "A format string, using the same keys and colors as ``$PROMPT``, that " "is appended whenever stderr is displayed. This may be used in " "conjunction with ``$XONSH_STDERR_PREFIX`` to start the block." "For example, to have stderr appear on a red background, the " 'prefix & postfix pair would be "{BACKGROUND_RED}" & "{NO_COLOR}".' ), "XONSH_STORE_STDIN": VarDocs( "Whether or not to store the stdin that is supplied to the " "``!()`` and ``![]`` operators." ), "XONSH_STORE_STDOUT": VarDocs( "Whether or not to store the ``stdout`` and ``stderr`` streams in the " "history files." ), "XONSH_TRACEBACK_LOGFILE": VarDocs( "Specifies a file to store the traceback log to, regardless of whether " "``XONSH_SHOW_TRACEBACK`` has been set. Its value must be a writable file " "or None / the empty string if traceback logging is not desired. " "Logging to a file is not enabled by default." ), "XONSH_DATETIME_FORMAT": VarDocs( "The format that is used for ``datetime.strptime()`` in various places" "i.e the history timestamp option" ), } # # actual environment # class Env(cabc.MutableMapping): """A xonsh environment, whose variables have limited typing (unlike BASH). Most variables are, by default, strings (like BASH). However, the following rules also apply based on variable-name: * PATH: any variable whose name ends in PATH is a list of strings. * XONSH_HISTORY_SIZE: this variable is an (int | float, str) tuple. * LC_* (locale categories): locale category names get/set the Python locale via locale.getlocale() and locale.setlocale() functions. An Env instance may be converted to an untyped version suitable for use in a subprocess. """ _arg_regex = None def __init__(self, *args, **kwargs): """If no initial environment is given, os_environ is used.""" self._d = {} # sentinel value for non existing envvars self._no_value = object() self._orig_env = None self._ensurers = {k: Ensurer(*v) for k, v in DEFAULT_ENSURERS.items()} self._defaults = DEFAULT_VALUES self._docs = DEFAULT_DOCS if len(args) == 0 and len(kwargs) == 0: args = (os_environ,) for key, val in dict(*args, **kwargs).items(): self[key] = val if ON_WINDOWS: path_key = next((k for k in self._d if k.upper() == "PATH"), None) if path_key: self["PATH"] = self._d.pop(path_key) if "PATH" not in self._d: # this is here so the PATH is accessible to subprocs and so that # it can be modified in-place in the xonshrc file self._d["PATH"] = list(PATH_DEFAULT) self._detyped = None def detype(self): if self._detyped is not None: return self._detyped ctx = {} for key, val in self._d.items(): if not isinstance(key, str): key = str(key) ensurer = self.get_ensurer(key) if ensurer.detype is None: # cannot be detyped continue deval = ensurer.detype(val) if deval is None: # cannot be detyped continue ctx[key] = deval self._detyped = ctx return ctx def replace_env(self): """Replaces the contents of os_environ with a detyped version of the xonsh environment. """ if self._orig_env is None: self._orig_env = dict(os_environ) os_environ.clear() os_environ.update(self.detype()) def undo_replace_env(self): """Replaces the contents of os_environ with a detyped version of the xonsh environment. """ if self._orig_env is not None: os_environ.clear() os_environ.update(self._orig_env) self._orig_env = None def _get_default_ensurer(self, default=None): if default is not None: return default else: default = Ensurer(always_true, None, ensure_string) return default def get_ensurer(self, key, default=None): """Gets an ensurer for the given key.""" if key in self._ensurers: return self._ensurers[key] for k, ensurer in self._ensurers.items(): if isinstance(k, str): continue if k.match(key) is not None: break else: ensurer = self._get_default_ensurer(default=default) self._ensurers[key] = ensurer return ensurer def set_ensurer(self, key, value): """Sets an ensurer.""" self._detyped = None self._ensurers[key] = value def get_docs(self, key, default=VarDocs("")): """Gets the documentation for the environment variable.""" vd = self._docs.get(key, None) if vd is None: return default if vd.default is DefaultNotGiven: dval = pprint.pformat(self._defaults.get(key, "")) vd = vd._replace(default=dval) self._docs[key] = vd return vd def help(self, key): """Get information about a specific environment variable.""" vardocs = self.get_docs(key) width = min(79, os.get_terminal_size()[0]) docstr = "\n".join(textwrap.wrap(vardocs.docstr, width=width)) template = HELP_TEMPLATE.format( envvar=key, docstr=docstr, default=vardocs.default, configurable=vardocs.configurable, ) print_color(template) def is_manually_set(self, varname): """ Checks if an environment variable has been manually set. """ return varname in self._d @contextlib.contextmanager def swap(self, other=None, **kwargs): """Provides a context manager for temporarily swapping out certain environment variables with other values. On exit from the context manager, the original values are restored. """ old = {} # single positional argument should be a dict-like object if other is not None: for k, v in other.items(): old[k] = self.get(k, NotImplemented) self[k] = v # kwargs could also have been sent in for k, v in kwargs.items(): old[k] = self.get(k, NotImplemented) self[k] = v exception = None try: yield self except Exception as e: exception = e finally: # restore the values for k, v in old.items(): if v is NotImplemented: del self[k] else: self[k] = v if exception is not None: raise exception from None # # Mutable mapping interface # def __getitem__(self, key): # remove this block on next release if key is Ellipsis: return self elif key in self._d: val = self._d[key] elif key in self._defaults: val = self._defaults[key] if is_callable_default(val): val = val(self) else: e = "Unknown environment variable: ${}" raise KeyError(e.format(key)) if isinstance( val, (cabc.MutableSet, cabc.MutableSequence, cabc.MutableMapping) ): self._detyped = None return val def __setitem__(self, key, val): ensurer = self.get_ensurer(key) if not ensurer.validate(val): val = ensurer.convert(val) # existing envvars can have any value including None old_value = self._d[key] if key in self._d else self._no_value self._d[key] = val self._detyped = None if self.get("UPDATE_OS_ENVIRON"): if self._orig_env is None: self.replace_env() elif ensurer.detype is None: pass else: deval = ensurer.detype(val) if deval is not None: os_environ[key] = deval if old_value is self._no_value: events.on_envvar_new.fire(name=key, value=val) elif old_value != val: events.on_envvar_change.fire(name=key, oldvalue=old_value, newvalue=val) def __delitem__(self, key): del self._d[key] self._detyped = None if self.get("UPDATE_OS_ENVIRON") and key in os_environ: del os_environ[key] def get(self, key, default=None): """The environment will look up default values from its own defaults if a default is not given here. """ try: return self[key] except KeyError: return default def __iter__(self): yield from (set(self._d) | set(self._defaults)) def __contains__(self, item): return item in self._d or item in self._defaults def __len__(self): return len(self._d) def __str__(self): return str(self._d) def __repr__(self): return "{0}.{1}(...)".format( self.__class__.__module__, self.__class__.__name__, self._d ) def _repr_pretty_(self, p, cycle): name = "{0}.{1}".format(self.__class__.__module__, self.__class__.__name__) with p.group(0, name + "(", ")"): if cycle: p.text("...") elif len(self): p.break_() p.pretty(dict(self)) def _yield_executables(directory, name): if ON_WINDOWS: base_name, ext = os.path.splitext(name.lower()) for fname in executables_in(directory): fbase, fext = os.path.splitext(fname.lower()) if base_name == fbase and (len(ext) == 0 or ext == fext): yield os.path.join(directory, fname) else: for x in executables_in(directory): if x == name: yield os.path.join(directory, name) return def locate_binary(name): """Locates an executable on the file system.""" return builtins.__xonsh__.commands_cache.locate_binary(name) BASE_ENV = LazyObject( lambda: { "BASH_COMPLETIONS": list(DEFAULT_VALUES["BASH_COMPLETIONS"]), "PROMPT_FIELDS": dict(DEFAULT_VALUES["PROMPT_FIELDS"]), "XONSH_VERSION": XONSH_VERSION, }, globals(), "BASE_ENV", ) def xonshrc_context(rcfiles=None, execer=None, ctx=None, env=None, login=True): """Attempts to read in all xonshrc files and return the context.""" loaded = env["LOADED_RC_FILES"] = [] ctx = {} if ctx is None else ctx if rcfiles is None: return env env["XONSHRC"] = tuple(rcfiles) for rcfile in rcfiles: if not os.path.isfile(rcfile): loaded.append(False) continue _, ext = os.path.splitext(rcfile) status = xonsh_script_run_control(rcfile, ctx, env, execer=execer, login=login) loaded.append(status) return ctx def windows_foreign_env_fixes(ctx): """Environment fixes for Windows. Operates in-place.""" # remove these bash variables which only cause problems. for ev in ["HOME", "OLDPWD"]: if ev in ctx: del ctx[ev] # Override path-related bash variables; on Windows bash uses # /c/Windows/System32 syntax instead of C:\\Windows\\System32 # which messes up these environment variables for xonsh. for ev in ["PATH", "TEMP", "TMP"]: if ev in os_environ: ctx[ev] = os_environ[ev] elif ev in ctx: del ctx[ev] ctx["PWD"] = _get_cwd() or "" def foreign_env_fixes(ctx): """Environment fixes for all operating systems""" if "PROMPT" in ctx: del ctx["PROMPT"] def xonsh_script_run_control(filename, ctx, env, execer=None, login=True): """Loads a xonsh file and applies it as a run control.""" if execer is None: return False updates = {"__file__": filename, "__name__": os.path.abspath(filename)} try: with swap_values(ctx, updates): run_script_with_cache(filename, execer, ctx) loaded = True except SyntaxError as err: msg = "syntax error in xonsh run control file {0!r}: {1!s}" print_exception(msg.format(filename, err)) loaded = False except Exception as err: msg = "error running xonsh run control file {0!r}: {1!s}" print_exception(msg.format(filename, err)) loaded = False return loaded def default_env(env=None): """Constructs a default xonsh environment.""" # in order of increasing precedence ctx = dict(BASE_ENV) ctx.update(os_environ) ctx["PWD"] = _get_cwd() or "" # These can cause problems for programs (#2543) ctx.pop("LINES", None) ctx.pop("COLUMNS", None) # other shells' PROMPT definitions generally don't work in XONSH: try: del ctx["PROMPT"] except KeyError: pass # finalize env if env is not None: ctx.update(env) return ctx def make_args_env(args=None): """Makes a dictionary containing the $ARGS and $ARG environment variables. If the supplied ARGS is None, then sys.argv is used. """ if args is None: args = sys.argv env = {"ARG" + str(i): arg for i, arg in enumerate(args)} env["ARGS"] = list(args) # make a copy so we don't interfere with original variable return env ================================================ FILE: xonsh/events.py ================================================ """ Events for xonsh. In all likelihood, you want builtins.events The best way to "declare" an event is something like:: events.doc('on_spam', "Comes with eggs") """ import abc import builtins import collections.abc import inspect from xonsh.tools import print_exception def has_kwargs(func): return any( p.kind == p.VAR_KEYWORD for p in inspect.signature(func).parameters.values() ) def debug_level(): if hasattr(builtins, "__xonsh__") and hasattr(builtins.__xonsh__, "env"): return builtins.__xonsh__.env.get("XONSH_DEBUG") # FIXME: Under py.test, return 1(?) else: return 0 # Optimize for speed, not guaranteed correctness class AbstractEvent(collections.abc.MutableSet, abc.ABC): """ A given event that handlers can register against. Acts as a ``MutableSet`` for registered handlers. Note that ordering is never guaranteed. """ @property def species(self): """ The species (basically, class) of the event """ return type(self).__bases__[ 0 ] # events.on_chdir -> -> def __call__(self, handler): """ Registers a handler. It's suggested to use this as a decorator. A decorator method is added to the handler, validator(). If a validator function is added, it can filter if the handler will be considered. The validator takes the same arguments as the handler. If it returns False, the handler will not called or considered, as if it was not registered at all. Parameters ---------- handler : callable The handler to register Returns ------- rtn : callable The handler """ # Using Python's "private" munging to minimize hypothetical collisions handler.__validator = None if debug_level(): if not has_kwargs(handler): raise ValueError("Event handlers need a **kwargs for future proofing") self.add(handler) def validator(vfunc): """ Adds a validator function to a handler to limit when it is considered. """ if debug_level(): if not has_kwargs(handler): raise ValueError( "Event validators need a **kwargs for future proofing" ) handler.__validator = vfunc handler.validator = validator return handler def _filterhandlers(self, handlers, **kwargs): """ Helper method for implementing classes. Generates the handlers that pass validation. """ for handler in handlers: if handler.__validator is not None and not handler.__validator(**kwargs): continue yield handler @abc.abstractmethod def fire(self, **kwargs): """ Fires an event, calling registered handlers with the given arguments. Parameters ---------- **kwargs : Keyword arguments to pass to each handler """ class Event(AbstractEvent): """ An event species for notify and scatter-gather events. """ # Wish I could just pull from set... def __init__(self): self._handlers = set() self._firing = False self._delayed_adds = None self._delayed_discards = None def __len__(self): return len(self._handlers) def __contains__(self, item): return item in self._handlers def __iter__(self): yield from self._handlers def add(self, item): """ Add an element to a set. This has no effect if the element is already present. """ if self._firing: if self._delayed_adds is None: self._delayed_adds = set() self._delayed_adds.add(item) else: self._handlers.add(item) def discard(self, item): """ Remove an element from a set if it is a member. If the element is not a member, do nothing. """ if self._firing: if self._delayed_discards is None: self._delayed_discards = set() self._delayed_discards.add(item) else: self._handlers.discard(item) def fire(self, **kwargs): """ Fires an event, calling registered handlers with the given arguments. A non-unique iterable of the results is returned. Each handler is called immediately. Exceptions are turned in to warnings. Parameters ---------- **kwargs : Keyword arguments to pass to each handler Returns ------- vals : iterable Return values of each handler. If multiple handlers return the same value, it will appear multiple times. """ vals = [] self._firing = True for handler in self._filterhandlers(self._handlers, **kwargs): try: rv = handler(**kwargs) except Exception: print_exception("Exception raised in event handler; ignored.") else: vals.append(rv) # clean up self._firing = False if self._delayed_adds is not None: self._handlers.update(self._delayed_adds) self._delayed_adds = None if self._delayed_discards is not None: self._handlers.difference_update(self._delayed_discards) self._delayed_discards = None return vals class LoadEvent(AbstractEvent): """ An event species where each handler is called exactly once, shortly after either the event is fired or the handler is registered (whichever is later). Additional firings are ignored. Note: Does not support scatter/gather, due to never knowing when we have all the handlers. Note: Maintains a strong reference to pargs/kwargs in case of the addition of future handlers. Note: This is currently NOT thread safe. """ def __init__(self): self._fired = set() self._unfired = set() self._hasfired = False def __len__(self): return len(self._fired) + len(self._unfired) def __contains__(self, item): return item in self._fired or item in self._unfired def __iter__(self): yield from self._fired yield from self._unfired def add(self, item): """ Add an element to a set. This has no effect if the element is already present. """ if self._hasfired: self._call(item) self._fired.add(item) else: self._unfired.add(item) def discard(self, item): """ Remove an element from a set if it is a member. If the element is not a member, do nothing. """ self._fired.discard(item) self._unfired.discard(item) def _call(self, handler): try: handler(**self._kwargs) except Exception: print_exception("Exception raised in event handler; ignored.") def fire(self, **kwargs): if self._hasfired: return self._kwargs = kwargs while self._unfired: handler = self._unfired.pop() self._call(handler) self._hasfired = True return () # Entirely for API compatibility class EventManager: """ Container for all events in a system. Meant to be a singleton, but doesn't enforce that itself. Each event is just an attribute. They're created dynamically on first use. """ def doc(self, name, docstring): """ Applies a docstring to an event. Parameters ---------- name : str The name of the event, eg "on_precommand" docstring : str The docstring to apply to the event """ type(getattr(self, name)).__doc__ = docstring @staticmethod def _mkevent(name, species=Event, doc=None): # NOTE: Also used in `xonsh_events` test fixture # (A little bit of magic to enable docstrings to work right) return type( name, (species,), { "__doc__": doc, "__module__": "xonsh.events", "__qualname__": "events." + name, }, )() def transmogrify(self, name, species): """ Converts an event from one species to another, preserving handlers and docstring. Please note: Some species maintain specialized state. This is lost on transmogrification. Parameters ---------- name : str The name of the event, eg "on_precommand" species : subclass of AbstractEvent The type to turn the event in to. """ if isinstance(species, str): species = globals()[species] if not issubclass(species, AbstractEvent): raise ValueError("Invalid event class; must be a subclass of AbstractEvent") oldevent = getattr(self, name) newevent = self._mkevent(name, species, type(oldevent).__doc__) setattr(self, name, newevent) for handler in oldevent: newevent.add(handler) def exists(self, name): """Checks if an event with a given name exist. If it does not exist, it will not be created. That is what makes this different than ``hasattr(events, name)``, which will create the event. """ return name in self.__dict__ def __getattr__(self, name): """Get an event, if it doesn't already exist.""" if name.startswith("_"): raise AttributeError # This is only called if the attribute doesn't exist, so create the Event... e = self._mkevent(name) # ... and save it. setattr(self, name, e) # Now it exists, and we won't be called again. return e # Not lazy because: # 1. Initialization of EventManager can't be much cheaper # 2. It's expected to be used at load time, negating any benefits of using lazy object events = EventManager() ================================================ FILE: xonsh/execer.py ================================================ # -*- coding: utf-8 -*- """Implements the xonsh executer.""" import sys import types import inspect import builtins import collections.abc as cabc from xonsh.ast import CtxAwareTransformer from xonsh.parser import Parser from xonsh.tools import ( subproc_toks, find_next_break, get_logical_line, replace_logical_line, balanced_parens, starting_whitespace, ) from xonsh.built_ins import load_builtins, unload_builtins, load_proxies, unload_proxies class Execer(object): """Executes xonsh code in a context.""" def __init__( self, filename="", debug_level=0, parser_args=None, unload=True, xonsh_ctx=None, scriptcache=True, cacheall=False, ): """Parameters ---------- filename : str, optional File we are to execute. debug_level : int, optional Debugging level to use in lexing and parsing. parser_args : dict, optional Arguments to pass down to the parser. unload : bool, optional Whether or not to unload xonsh builtins upon deletion. xonsh_ctx : dict or None, optional Xonsh xontext to load as builtins.__xonsh__.ctx scriptcache : bool, optional Whether or not to use a precompiled bytecode cache when execing code, default: True. cacheall : bool, optional Whether or not to cache all xonsh code, and not just files. If this is set to true, it will cache command line input too, default: False. """ parser_args = parser_args or {} self.parser = Parser(**parser_args) self.filename = filename self.debug_level = debug_level self.unload = unload self.scriptcache = scriptcache self.cacheall = cacheall self.ctxtransformer = CtxAwareTransformer(self.parser) load_builtins(execer=self, ctx=xonsh_ctx) load_proxies() def __del__(self): if self.unload: unload_proxies() unload_builtins() def parse(self, input, ctx, mode="exec", filename=None, transform=True): """Parses xonsh code in a context-aware fashion. For context-free parsing, please use the Parser class directly or pass in transform=False. """ if filename is None: filename = self.filename if not transform: return self.parser.parse( input, filename=filename, mode=mode, debug_level=(self.debug_level > 2) ) # Parsing actually happens in a couple of phases. The first is a # shortcut for a context-free parser. Normally, all subprocess # lines should be wrapped in $(), to indicate that they are a # subproc. But that would be super annoying. Unfortunately, Python # mode - after indentation - is whitespace agnostic while, using # the Python token, subproc mode is whitespace aware. That is to say, # in Python mode "ls -l", "ls-l", and "ls - l" all parse to the # same AST because whitespace doesn't matter to the minus binary op. # However, these phases all have very different meaning in subproc # mode. The 'right' way to deal with this is to make the entire # grammar whitespace aware, and then ignore all of the whitespace # tokens for all of the Python rules. The lazy way implemented here # is to parse a line a second time with a $() wrapper if it fails # the first time. This is a context-free phase. tree, input = self._parse_ctx_free(input, mode=mode, filename=filename) if tree is None: return None # Now we need to perform context-aware AST transformation. This is # because the "ls -l" is valid Python. The only way that we know # it is not actually Python is by checking to see if the first token # (ls) is part of the execution context. If it isn't, then we will # assume that this line is supposed to be a subprocess line, assuming # it also is valid as a subprocess line. if ctx is None: ctx = set() elif isinstance(ctx, cabc.Mapping): ctx = set(ctx.keys()) tree = self.ctxtransformer.ctxvisit( tree, input, ctx, mode=mode, debug_level=self.debug_level ) return tree def compile( self, input, mode="exec", glbs=None, locs=None, stacklevel=2, filename=None, transform=True, ): """Compiles xonsh code into a Python code object, which may then be execed or evaled. """ if filename is None: filename = self.filename if glbs is None or locs is None: frame = inspect.stack()[stacklevel][0] glbs = frame.f_globals if glbs is None else glbs locs = frame.f_locals if locs is None else locs ctx = set(dir(builtins)) | set(glbs.keys()) | set(locs.keys()) tree = self.parse(input, ctx, mode=mode, filename=filename, transform=transform) if tree is None: return None # handles comment only input code = compile(tree, filename, mode) return code def eval( self, input, glbs=None, locs=None, stacklevel=2, filename=None, transform=True ): """Evaluates (and returns) xonsh code.""" if isinstance(input, types.CodeType): code = input else: if filename is None: filename = self.filename code = self.compile( input=input, glbs=glbs, locs=locs, mode="eval", stacklevel=stacklevel, filename=filename, transform=transform, ) if code is None: return None # handles comment only input return eval(code, glbs, locs) def exec( self, input, mode="exec", glbs=None, locs=None, stacklevel=2, filename=None, transform=True, ): """Execute xonsh code.""" if isinstance(input, types.CodeType): code = input else: if filename is None: filename = self.filename code = self.compile( input=input, glbs=glbs, locs=locs, mode=mode, stacklevel=stacklevel, filename=filename, transform=transform, ) if code is None: return None # handles comment only input return exec(code, glbs, locs) def _print_debug_wrapping( self, line, sbpline, last_error_line, last_error_col, maxcol=None ): """print some debugging info if asked for.""" if self.debug_level > 1: msg = "{0}:{1}:{2}{3} - {4}\n" "{0}:{1}:{2}{3} + {5}" mstr = "" if maxcol is None else ":" + str(maxcol) msg = msg.format( self.filename, last_error_line, last_error_col, mstr, line, sbpline ) print(msg, file=sys.stderr) def _parse_ctx_free(self, input, mode="exec", filename=None, logical_input=False): last_error_line = last_error_col = -1 parsed = False original_error = None greedy = False if filename is None: filename = self.filename if logical_input: beg_spaces = starting_whitespace(input) input = input[len(beg_spaces) :] while not parsed: try: tree = self.parser.parse( input, filename=filename, mode=mode, debug_level=(self.debug_level > 2), ) parsed = True except IndentationError as e: if original_error is None: raise e else: raise original_error except SyntaxError as e: if original_error is None: original_error = e if (e.loc is None) or ( last_error_line == e.loc.lineno and last_error_col in (e.loc.column + 1, e.loc.column) ): raise original_error from None elif last_error_line != e.loc.lineno: original_error = e last_error_col = e.loc.column last_error_line = e.loc.lineno idx = last_error_line - 1 lines = input.splitlines() line, nlogical, idx = get_logical_line(lines, idx) if nlogical > 1 and not logical_input: _, sbpline = self._parse_ctx_free( line, mode=mode, filename=filename, logical_input=True ) self._print_debug_wrapping( line, sbpline, last_error_line, last_error_col, maxcol=None ) replace_logical_line(lines, sbpline, idx, nlogical) last_error_col += 3 input = "\n".join(lines) continue if input.endswith("\n"): lines.append("") if len(line.strip()) == 0: # whitespace only lines are not valid syntax in Python's # interactive mode='single', who knew?! Just ignore them. # this might cause actual syntax errors to have bad line # numbers reported, but should only affect interactive mode del lines[idx] last_error_line = last_error_col = -1 input = "\n".join(lines) continue if last_error_line > 1 and lines[idx - 1].rstrip()[-1:] == ":": # catch non-indented blocks and raise error. prev_indent = len(lines[idx - 1]) - len(lines[idx - 1].lstrip()) curr_indent = len(lines[idx]) - len(lines[idx].lstrip()) if prev_indent == curr_indent: raise original_error lexer = self.parser.lexer maxcol = ( None if greedy else find_next_break(line, mincol=last_error_col, lexer=lexer) ) if not greedy and maxcol in (e.loc.column + 1, e.loc.column): # go greedy the first time if the syntax error was because # we hit an end token out of place. This usually indicates # a subshell or maybe a macro. if not balanced_parens(line, maxcol=maxcol): greedy = True maxcol = None sbpline = subproc_toks( line, returnline=True, greedy=greedy, maxcol=maxcol, lexer=lexer ) if sbpline is None: # subprocess line had no valid tokens, if len(line.partition("#")[0].strip()) == 0: # likely because it only contained a comment. del lines[idx] last_error_line = last_error_col = -1 input = "\n".join(lines) continue elif not greedy: greedy = True continue else: # or for some other syntax error raise original_error elif sbpline[last_error_col:].startswith( "![![" ) or sbpline.lstrip().startswith("![!["): # if we have already wrapped this in subproc tokens # and it still doesn't work, adding more won't help # anything if not greedy: greedy = True continue else: raise original_error # replace the line self._print_debug_wrapping( line, sbpline, last_error_line, last_error_col, maxcol=maxcol ) replace_logical_line(lines, sbpline, idx, nlogical) last_error_col += 3 input = "\n".join(lines) if logical_input: input = beg_spaces + input return tree, input ================================================ FILE: xonsh/foreign_shells.py ================================================ # -*- coding: utf-8 -*- """Tools to help interface with foreign shells, such as Bash.""" import os import re import json import shlex import sys import tempfile import builtins import subprocess import warnings import functools import collections.abc as cabc from xonsh.lazyasd import lazyobject from xonsh.tools import to_bool, ensure_string from xonsh.platform import ON_WINDOWS, ON_CYGWIN, ON_MSYS COMMAND = """{seterrprevcmd} {prevcmd} echo __XONSH_ENV_BEG__ {envcmd} echo __XONSH_ENV_END__ echo __XONSH_ALIAS_BEG__ {aliascmd} echo __XONSH_ALIAS_END__ echo __XONSH_FUNCS_BEG__ {funcscmd} echo __XONSH_FUNCS_END__ {postcmd} {seterrpostcmd}""" DEFAULT_BASH_FUNCSCMD = r"""# get function names from declare declstr=$(declare -F) read -r -a decls <<< $declstr funcnames="" for((n=0;n<${#decls[@]};n++)); do if (( $(($n % 3 )) == 2 )); then # get every 3rd entry funcnames="$funcnames ${decls[$n]}" fi done # get functions locations: funcname lineno filename shopt -s extdebug namelocfilestr=$(declare -F $funcnames) shopt -u extdebug # print just names and files as JSON object read -r -a namelocfile <<< $namelocfilestr sep=" " namefile="{" while IFS='' read -r line || [[ -n "$line" ]]; do name=${line%%"$sep"*} locfile=${line#*"$sep"} loc=${locfile%%"$sep"*} file=${locfile#*"$sep"} namefile="${namefile}\"${name}\":\"${file//\\/\\\\}\"," done <<< "$namelocfilestr" if [[ "{" == "${namefile}" ]]; then namefile="${namefile}}" else namefile="${namefile%?}}" fi echo $namefile""" DEFAULT_ZSH_FUNCSCMD = """# get function names autoload -U is-at-least # We'll need to version check zsh namefile="{" for name in ${(ok)functions}; do # force zsh to load the func in order to get the filename, # but use +X so that it isn't executed. autoload +X $name || continue loc=$(whence -v $name) loc=${(z)loc} if is-at-least 5.2; then file=${loc[-1]} else file=${loc[7,-1]} fi namefile="${namefile}\\"${name}\\":\\"${(Q)file:A}\\"," done if [[ "{" == "${namefile}" ]]; then namefile="${namefile}}" else namefile="${namefile%?}}" fi echo ${namefile}""" # mapping of shell name aliases to keys in other lookup dictionaries. @lazyobject def CANON_SHELL_NAMES(): return { "bash": "bash", "/bin/bash": "bash", "zsh": "zsh", "/bin/zsh": "zsh", "/usr/bin/zsh": "zsh", "cmd": "cmd", "cmd.exe": "cmd", } @lazyobject def DEFAULT_ENVCMDS(): return {"bash": "env", "zsh": "env", "cmd": "set"} @lazyobject def DEFAULT_ALIASCMDS(): return {"bash": "alias", "zsh": "alias -L", "cmd": ""} @lazyobject def DEFAULT_FUNCSCMDS(): return {"bash": DEFAULT_BASH_FUNCSCMD, "zsh": DEFAULT_ZSH_FUNCSCMD, "cmd": ""} @lazyobject def DEFAULT_SOURCERS(): return {"bash": "source", "zsh": "source", "cmd": "call"} @lazyobject def DEFAULT_TMPFILE_EXT(): return {"bash": ".sh", "zsh": ".zsh", "cmd": ".bat"} @lazyobject def DEFAULT_RUNCMD(): return {"bash": "-c", "zsh": "-c", "cmd": "/C"} @lazyobject def DEFAULT_SETERRPREVCMD(): return {"bash": "set -e", "zsh": "set -e", "cmd": "@echo off"} @lazyobject def DEFAULT_SETERRPOSTCMD(): return {"bash": "", "zsh": "", "cmd": "if errorlevel 1 exit 1"} @functools.lru_cache() def foreign_shell_data( shell, interactive=True, login=False, envcmd=None, aliascmd=None, extra_args=(), currenv=None, safe=True, prevcmd="", postcmd="", funcscmd=None, sourcer=None, use_tmpfile=False, tmpfile_ext=None, runcmd=None, seterrprevcmd=None, seterrpostcmd=None, show=False, dryrun=False, ): """Extracts data from a foreign (non-xonsh) shells. Currently this gets the environment, aliases, and functions but may be extended in the future. Parameters ---------- shell : str The name of the shell, such as 'bash' or '/bin/sh'. interactive : bool, optional Whether the shell should be run in interactive mode. login : bool, optional Whether the shell should be a login shell. envcmd : str or None, optional The command to generate environment output with. aliascmd : str or None, optional The command to generate alias output with. extra_args : tuple of str, optional Additional command line options to pass into the shell. currenv : tuple of items or None, optional Manual override for the current environment. safe : bool, optional Flag for whether or not to safely handle exceptions and other errors. prevcmd : str, optional A command to run in the shell before anything else, useful for sourcing and other commands that may require environment recovery. postcmd : str, optional A command to run after everything else, useful for cleaning up any damage that the prevcmd may have caused. funcscmd : str or None, optional This is a command or script that can be used to determine the names and locations of any functions that are native to the foreign shell. This command should print *only* a JSON object that maps function names to the filenames where the functions are defined. If this is None, then a default script will attempted to be looked up based on the shell name. Callable wrappers for these functions will be returned in the aliases dictionary. sourcer : str or None, optional How to source a foreign shell file for purposes of calling functions in that shell. If this is None, a default value will attempt to be looked up based on the shell name. use_tmpfile : bool, optional This specifies if the commands are written to a tmp file or just parsed directly to the shell tmpfile_ext : str or None, optional If tmpfile is True this sets specifies the extension used. runcmd : str or None, optional Command line switches to use when running the script, such as -c for Bash and /C for cmd.exe. seterrprevcmd : str or None, optional Command that enables exit-on-error for the shell that is run at the start of the script. For example, this is "set -e" in Bash. To disable exit-on-error behavior, simply pass in an empty string. seterrpostcmd : str or None, optional Command that enables exit-on-error for the shell that is run at the end of the script. For example, this is "if errorlevel 1 exit 1" in cmd.exe. To disable exit-on-error behavior, simply pass in an empty string. show : bool, optional Whether or not to display the script that will be run. dryrun : bool, optional Whether or not to actually run and process the command. Returns ------- env : dict Dictionary of shell's environment. (None if the subproc command fails) aliases : dict Dictionary of shell's aliases, this includes foreign function wrappers.(None if the subproc command fails) """ cmd = [shell] cmd.extend(extra_args) # needs to come here for GNU long options if interactive: cmd.append("-i") if login: cmd.append("-l") shkey = CANON_SHELL_NAMES[shell] envcmd = DEFAULT_ENVCMDS.get(shkey, "env") if envcmd is None else envcmd aliascmd = DEFAULT_ALIASCMDS.get(shkey, "alias") if aliascmd is None else aliascmd funcscmd = DEFAULT_FUNCSCMDS.get(shkey, "echo {}") if funcscmd is None else funcscmd tmpfile_ext = ( DEFAULT_TMPFILE_EXT.get(shkey, "sh") if tmpfile_ext is None else tmpfile_ext ) runcmd = DEFAULT_RUNCMD.get(shkey, "-c") if runcmd is None else runcmd seterrprevcmd = ( DEFAULT_SETERRPREVCMD.get(shkey, "") if seterrprevcmd is None else seterrprevcmd ) seterrpostcmd = ( DEFAULT_SETERRPOSTCMD.get(shkey, "") if seterrpostcmd is None else seterrpostcmd ) command = COMMAND.format( envcmd=envcmd, aliascmd=aliascmd, prevcmd=prevcmd, postcmd=postcmd, funcscmd=funcscmd, seterrprevcmd=seterrprevcmd, seterrpostcmd=seterrpostcmd, ).strip() if show: print(command) if dryrun: return None, None cmd.append(runcmd) if not use_tmpfile: cmd.append(command) else: tmpfile = tempfile.NamedTemporaryFile(suffix=tmpfile_ext, delete=False) tmpfile.write(command.encode("utf8")) tmpfile.close() cmd.append(tmpfile.name) if currenv is None and hasattr(builtins.__xonsh__, "env"): currenv = builtins.__xonsh__.env.detype() elif currenv is not None: currenv = dict(currenv) try: s = subprocess.check_output( cmd, stderr=subprocess.PIPE, env=currenv, # start new session to avoid hangs # (doesn't work on Cygwin though) start_new_session=((not ON_CYGWIN) and (not ON_MSYS)), universal_newlines=True, ) except (subprocess.CalledProcessError, FileNotFoundError): if not safe: raise return None, None finally: if use_tmpfile: os.remove(tmpfile.name) env = parse_env(s) aliases = parse_aliases(s, shell=shell, sourcer=sourcer, extra_args=extra_args) funcs = parse_funcs(s, shell=shell, sourcer=sourcer, extra_args=extra_args) aliases.update(funcs) return env, aliases @lazyobject def ENV_RE(): return re.compile("__XONSH_ENV_BEG__\n(.*)" "__XONSH_ENV_END__", flags=re.DOTALL) @lazyobject def ENV_SPLIT_RE(): return re.compile("^([^=]+)=([^=]*|[^\n]*)$", flags=re.DOTALL | re.MULTILINE) def parse_env(s): """Parses the environment portion of string into a dict.""" m = ENV_RE.search(s) if m is None: return {} g1 = m.group(1) g1 = g1[:-1] if g1.endswith("\n") else g1 env = dict(ENV_SPLIT_RE.findall(g1)) return env @lazyobject def ALIAS_RE(): return re.compile( "__XONSH_ALIAS_BEG__\n(.*)" "__XONSH_ALIAS_END__", flags=re.DOTALL ) @lazyobject def FS_EXEC_ALIAS_RE(): return re.compile(r";|`|\$\(") def parse_aliases(s, shell, sourcer=None, extra_args=()): """Parses the aliases portion of string into a dict.""" m = ALIAS_RE.search(s) if m is None: return {} g1 = m.group(1) items = [ line.split("=", 1) for line in g1.splitlines() if line.startswith("alias ") and "=" in line ] aliases = {} for key, value in items: try: key = key[6:] # lstrip 'alias ' # undo bash's weird quoting of single quotes (sh_single_quote) value = value.replace("'\\''", "'") # strip one single quote at the start and end of value if value[0] == "'" and value[-1] == "'": value = value[1:-1] # now compute actual alias if FS_EXEC_ALIAS_RE.search(value) is None: # simple list of args alias value = shlex.split(value) else: # alias is more complex, use ExecAlias, but via shell filename = "" value = ForeignShellExecAlias( src=value, shell=shell, filename=filename, sourcer=sourcer, extra_args=extra_args, ) except ValueError as exc: warnings.warn( 'could not parse alias "{0}": {1!r}'.format(key, exc), RuntimeWarning ) continue aliases[key] = value return aliases @lazyobject def FUNCS_RE(): return re.compile( "__XONSH_FUNCS_BEG__\n(.+)\n" "__XONSH_FUNCS_END__", flags=re.DOTALL ) def parse_funcs(s, shell, sourcer=None, extra_args=()): """Parses the funcs portion of a string into a dict of callable foreign function wrappers. """ m = FUNCS_RE.search(s) if m is None: return {} g1 = m.group(1) if ON_WINDOWS: g1 = g1.replace(os.sep, os.altsep) try: namefiles = json.loads(g1.strip()) except json.decoder.JSONDecodeError as exc: msg = ( "{0!r}\n\ncould not parse {1} functions:\n" " s = {2!r}\n" " g1 = {3!r}\n\n" "Note: you may be seeing this error if you use zsh with " "prezto. Prezto overwrites GNU coreutils functions (like echo) " "with its own zsh functions. Please try disabling prezto." ) warnings.warn(msg.format(exc, shell, s, g1), RuntimeWarning) return {} sourcer = DEFAULT_SOURCERS.get(shell, "source") if sourcer is None else sourcer funcs = {} for funcname, filename in namefiles.items(): if funcname.startswith("_") or not filename: continue # skip private functions and invalid files if not os.path.isabs(filename): filename = os.path.abspath(filename) wrapper = ForeignShellFunctionAlias( funcname=funcname, shell=shell, sourcer=sourcer, filename=filename, extra_args=extra_args, ) funcs[funcname] = wrapper return funcs class ForeignShellBaseAlias(object): """This class is responsible for calling foreign shell functions as if they were aliases. This does not currently support taking stdin. """ INPUT = "echo ForeignShellBaseAlias {shell} {filename} {args}\n" def __init__(self, shell, filename, sourcer=None, extra_args=()): """ Parameters ---------- shell : str Name or path to shell filename : str Where the function is defined, path to source. sourcer : str or None, optional Command to source foreign files with. extra_args : tuple of str, optional Additional command line options to pass into the shell. """ sourcer = DEFAULT_SOURCERS.get(shell, "source") if sourcer is None else sourcer self.shell = shell self.filename = filename self.sourcer = sourcer self.extra_args = extra_args def _input_kwargs(self): return { "shell": self.shell, "filename": self.filename, "sourcer": self.sourcer, "extra_args": self.extra_args, } def __eq__(self, other): if not hasattr(other, "_input_kwargs") or not callable(other._input_kwargs): return NotImplemented return self._input_kwargs() == other._input_kwargs() def __call__( self, args, stdin=None, stdout=None, stderr=None, spec=None, stack=None ): args, streaming = self._is_streaming(args) input = self.INPUT.format(args=" ".join(args), **self._input_kwargs()) cmd = [self.shell] + list(self.extra_args) + ["-c", input] env = builtins.__xonsh__.env denv = env.detype() if streaming: subprocess.check_call(cmd, env=denv) out = None else: out = subprocess.check_output(cmd, env=denv, stderr=subprocess.STDOUT) out = out.decode( encoding=env.get("XONSH_ENCODING"), errors=env.get("XONSH_ENCODING_ERRORS"), ) out = out.replace("\r\n", "\n") return out def __repr__(self): return ( self.__class__.__name__ + "(" + ", ".join( [ "{k}={v!r}".format(k=k, v=v) for k, v in sorted(self._input_kwargs().items()) ] ) + ")" ) @staticmethod def _is_streaming(args): """Test and modify args if --xonsh-stream is present.""" if "--xonsh-stream" not in args: return args, False args = list(args) args.remove("--xonsh-stream") return args, True class ForeignShellFunctionAlias(ForeignShellBaseAlias): """This class is responsible for calling foreign shell functions as if they were aliases. This does not currently support taking stdin. """ INPUT = '{sourcer} "{filename}"\n' "{funcname} {args}\n" def __init__(self, funcname, shell, filename, sourcer=None, extra_args=()): """ Parameters ---------- funcname : str function name shell : str Name or path to shell filename : str Where the function is defined, path to source. sourcer : str or None, optional Command to source foreign files with. extra_args : tuple of str, optional Additional command line options to pass into the shell. """ super().__init__( shell=shell, filename=filename, sourcer=sourcer, extra_args=extra_args ) self.funcname = funcname def _input_kwargs(self): inp = super()._input_kwargs() inp["funcname"] = self.funcname return inp class ForeignShellExecAlias(ForeignShellBaseAlias): """Provides a callable alias for source code in a foreign shell.""" INPUT = "{src} {args}\n" def __init__( self, src, shell, filename="", sourcer=None, extra_args=(), ): """ Parameters ---------- src : str Source code in the shell language shell : str Name or path to shell filename : str Where the function is defined, path to source. sourcer : str or None, optional Command to source foreign files with. extra_args : tuple of str, optional Additional command line options to pass into the shell. """ super().__init__( shell=shell, filename=filename, sourcer=sourcer, extra_args=extra_args ) self.src = src.strip() def _input_kwargs(self): inp = super()._input_kwargs() inp["src"] = self.src return inp @lazyobject def VALID_SHELL_PARAMS(): return frozenset( [ "shell", "interactive", "login", "envcmd", "aliascmd", "extra_args", "currenv", "safe", "prevcmd", "postcmd", "funcscmd", "sourcer", ] ) def ensure_shell(shell): """Ensures that a mapping follows the shell specification.""" if not isinstance(shell, cabc.MutableMapping): shell = dict(shell) shell_keys = set(shell.keys()) if not (shell_keys <= VALID_SHELL_PARAMS): msg = "unknown shell keys: {0}" raise KeyError(msg.format(shell_keys - VALID_SHELL_PARAMS)) shell["shell"] = ensure_string(shell["shell"]).lower() if "interactive" in shell_keys: shell["interactive"] = to_bool(shell["interactive"]) if "login" in shell_keys: shell["login"] = to_bool(shell["login"]) if "envcmd" in shell_keys: shell["envcmd"] = ( None if shell["envcmd"] is None else ensure_string(shell["envcmd"]) ) if "aliascmd" in shell_keys: shell["aliascmd"] = ( None if shell["aliascmd"] is None else ensure_string(shell["aliascmd"]) ) if "extra_args" in shell_keys and not isinstance(shell["extra_args"], tuple): shell["extra_args"] = tuple(map(ensure_string, shell["extra_args"])) if "currenv" in shell_keys and not isinstance(shell["currenv"], tuple): ce = shell["currenv"] if isinstance(ce, cabc.Mapping): ce = tuple([(ensure_string(k), v) for k, v in ce.items()]) elif isinstance(ce, cabc.Sequence): ce = tuple([(ensure_string(k), v) for k, v in ce]) else: raise RuntimeError("unrecognized type for currenv") shell["currenv"] = ce if "safe" in shell_keys: shell["safe"] = to_bool(shell["safe"]) if "prevcmd" in shell_keys: shell["prevcmd"] = ensure_string(shell["prevcmd"]) if "postcmd" in shell_keys: shell["postcmd"] = ensure_string(shell["postcmd"]) if "funcscmd" in shell_keys: shell["funcscmd"] = ( None if shell["funcscmd"] is None else ensure_string(shell["funcscmd"]) ) if "sourcer" in shell_keys: shell["sourcer"] = ( None if shell["sourcer"] is None else ensure_string(shell["sourcer"]) ) if "seterrprevcmd" in shell_keys: shell["seterrprevcmd"] = ( None if shell["seterrprevcmd"] is None else ensure_string(shell["seterrprevcmd"]) ) if "seterrpostcmd" in shell_keys: shell["seterrpostcmd"] = ( None if shell["seterrpostcmd"] is None else ensure_string(shell["seterrpostcmd"]) ) return shell def load_foreign_envs(shells): """Loads environments from foreign shells. Parameters ---------- shells : sequence of dicts An iterable of dicts that can be passed into foreign_shell_data() as keyword arguments. Returns ------- env : dict A dictionary of the merged environments. """ env = {} for shell in shells: shell = ensure_shell(shell) shenv, _ = foreign_shell_data(**shell) if shenv: env.update(shenv) return env def load_foreign_aliases(shells): """Loads aliases from foreign shells. Parameters ---------- shells : sequence of dicts An iterable of dicts that can be passed into foreign_shell_data() as keyword arguments. Returns ------- aliases : dict A dictionary of the merged aliases. """ aliases = {} xonsh_aliases = builtins.aliases for shell in shells: shell = ensure_shell(shell) _, shaliases = foreign_shell_data(**shell) if not builtins.__xonsh__.env.get("FOREIGN_ALIASES_OVERRIDE"): shaliases = {} if shaliases is None else shaliases for alias in set(shaliases) & set(xonsh_aliases): del shaliases[alias] if builtins.__xonsh__.env.get("XONSH_DEBUG") > 1: print( "aliases: ignoring alias {!r} of shell {!r} " "which tries to override xonsh alias." "".format(alias, shell["shell"]), file=sys.stderr, ) aliases.update(shaliases) return aliases ================================================ FILE: xonsh/fs.py ================================================ """ Backported functions to implement the PEP 519 (Adding a file system path protocol) API. """ import abc import sys import io import pathlib try: from os import PathLike, fspath, fsencode, fsdecode except ImportError: class PathLike(abc.ABC): """Abstract base class for implementing the file system path protocol.""" @abc.abstractmethod def __fspath__(self): """Return the file system path representation of the object.""" raise NotImplementedError PathLike.register(pathlib.Path) def fspath(path): """Return the string representation of the path. If str or bytes is passed in, it is returned unchanged. If __fspath__() returns something other than str or bytes then TypeError is raised. If this function is given something that is not str, bytes, or os.PathLike then TypeError is raised. """ if isinstance(path, (str, bytes)): return path if isinstance(path, pathlib.Path): return str(path) # Work from the object's type to match method resolution of other magic # methods. path_type = type(path) try: path = path_type.__fspath__(path) except AttributeError: if hasattr(path_type, "__fspath__"): raise else: if isinstance(path, (str, bytes)): return path else: raise TypeError( "expected __fspath__() to return str or bytes, " "not " + type(path).__name__ ) raise TypeError( "expected str, bytes or os.PathLike object, not " + path_type.__name__ ) def _fscodec(): encoding = sys.getfilesystemencoding() if encoding == "mbcs": errors = "strict" else: errors = "surrogateescape" def fsencode(filename): """Encode filename (an os.PathLike, bytes, or str) to the filesystem encoding with 'surrogateescape' error handler, return bytes unchanged. On Windows, use 'strict' error handler if the file system encoding is 'mbcs' (which is the default encoding). """ filename = fspath(filename) # Does type-checking of `filename`. if isinstance(filename, str): return filename.encode(encoding, errors) else: return filename def fsdecode(filename): """Decode filename (an os.PathLike, bytes, or str) from the filesystem encoding with 'surrogateescape' error handler, return str unchanged. On Windows, use 'strict' error handler if the file system encoding is 'mbcs' (which is the default encoding). """ filename = fspath(filename) # Does type-checking of `filename`. if isinstance(filename, bytes): return filename.decode(encoding, errors) else: return filename return fsencode, fsdecode fsencode, fsdecode = _fscodec() del _fscodec def open(file, *pargs, **kwargs): if isinstance(file, PathLike): file = fspath(file) return io.open(file, *pargs, **kwargs) ================================================ FILE: xonsh/history/__init__.py ================================================ # amalgamate exclude import os as _os if _os.getenv("XONSH_DEBUG", ""): pass else: import sys as _sys try: from xonsh.history import __amalgam__ base = __amalgam__ _sys.modules["xonsh.history.base"] = __amalgam__ dummy = __amalgam__ _sys.modules["xonsh.history.dummy"] = __amalgam__ json = __amalgam__ _sys.modules["xonsh.history.json"] = __amalgam__ sqlite = __amalgam__ _sys.modules["xonsh.history.sqlite"] = __amalgam__ main = __amalgam__ _sys.modules["xonsh.history.main"] = __amalgam__ del __amalgam__ except ImportError: pass del _sys del _os # amalgamate end ================================================ FILE: xonsh/history/base.py ================================================ # -*- coding: utf-8 -*- """Base class of Xonsh History backends.""" import types import uuid class HistoryEntry(types.SimpleNamespace): """Represent a command in history. Attributes ---------- cmd: str The command as typed by the user, including newlines out: str The output of the command, if xonsh is configured to save it rtn: int The return of the command (ie, 0 on success) ts: two-tuple of floats The timestamps of when the command started and finished, including fractions. """ class History: """Xonsh history backend base class. History objects should be created via a subclass of History. Indexing -------- History acts like a sequence that can be indexed to return ``HistoryEntry`` objects. Note that the most recent command is the last item in history. Attributes ---------- rtns : sequence of ints The return of the command (ie, 0 on success) inps : sequence of strings The command as typed by the user, including newlines tss : sequence of two-tuples of floats The timestamps of when the command started and finished, including fractions outs : sequence of strings The output of the command, if xonsh is configured to save it gc : A garbage collector or None The garbage collector In all of these sequences, index 0 is the oldest and -1 (the last item) is the newest. """ def __init__(self, sessionid=None, **kwargs): """Represents a xonsh session's history. Parameters ---------- sessionid : int, uuid, str, optional Current session identifier, will generate a new sessionid if not set. """ self.sessionid = uuid.uuid4() if sessionid is None else sessionid self.gc = None self.buffer = None self.filename = None self.inps = None self.rtns = None self.tss = None self.outs = None self.last_cmd_rtn = None self.last_cmd_out = None def __len__(self): """Return the number of items in current session.""" return len(list(self.items())) def __getitem__(self, item): """Retrieve history entries, see ``History`` docs for more info.""" if isinstance(item, int): if item >= len(self): raise IndexError("history index out of range") return HistoryEntry( cmd=self.inps[item], out=self.outs[item], rtn=self.rtns[item], ts=self.tss[item], ) elif isinstance(item, slice): cmds = self.inps[item] outs = self.outs[item] rtns = self.rtns[item] tss = self.tss[item] return [ HistoryEntry(cmd=c, out=o, rtn=r, ts=t) for c, o, r, t in zip(cmds, outs, rtns, tss) ] else: raise TypeError( "history indices must be integers " "or slices, not {}".format(type(item)) ) def __setitem__(self, *args): raise PermissionError( "You cannot change history! " "you can create new though." ) def append(self, cmd): """Append a command item into history. Parameters ---------- cmd: dict This dict contains information about the command that is to be added to the history list. It should contain the keys ``inp``, ``rtn`` and ``ts``. These key names mirror the same names defined as instance variables in the ``HistoryEntry`` class. """ pass def flush(self, **kwargs): """Flush the history items to disk from a buffer.""" pass def items(self, newest_first=False): """Get history items of current session.""" raise NotImplementedError def all_items(self, newest_first=False): """Get all history items.""" raise NotImplementedError def info(self): """A collection of information about the shell history. Returns ------- dict or collections.OrderedDict Contains history information as str key pairs. """ raise NotImplementedError def run_gc(self, size=None, blocking=True): """Run the garbage collector. Parameters ---------- size: None or tuple of a int and a string Determines the size and units of what would be allowed to remain. blocking: bool If set blocking, then wait until gc action finished. """ pass ================================================ FILE: xonsh/history/dummy.py ================================================ # -*- coding: utf-8 -*- """Implements the xonsh history backend.""" import collections from xonsh.history.base import History class DummyHistory(History): """A dummy implement of history backend.""" def append(self, cmd): pass def items(self, newest_first=False): yield {"inp": "dummy in action", "ts": 1464652800, "ind": 0} def all_items(self, newest_first=False): return self.items(newest_first=newest_first) def info(self): data = collections.OrderedDict() data["backend"] = "dummy" data["sessionid"] = str(self.sessionid) return data ================================================ FILE: xonsh/history/json.py ================================================ # -*- coding: utf-8 -*- """Implements JSON version of xonsh history backend.""" import os import sys import time import json import builtins import collections import threading import collections.abc as cabc from xonsh.history.base import History import xonsh.tools as xt import xonsh.lazyjson as xlj import xonsh.xoreutils.uptime as uptime def _xhj_gc_commands_to_rmfiles(hsize, files): """Return the history files to remove to get under the command limit.""" rmfiles = [] n = 0 ncmds = 0 for ts, fcmds, f in files[::-1]: if fcmds == 0: # we need to make sure that 'empty' history files don't hang around rmfiles.append((ts, fcmds, f)) if ncmds + fcmds > hsize: break ncmds += fcmds n += 1 rmfiles += files[:-n] return rmfiles def _xhj_gc_files_to_rmfiles(hsize, files): """Return the history files to remove to get under the file limit.""" rmfiles = files[:-hsize] if len(files) > hsize else [] return rmfiles def _xhj_gc_seconds_to_rmfiles(hsize, files): """Return the history files to remove to get under the age limit.""" rmfiles = [] now = time.time() for ts, _, f in files: if (now - ts) < hsize: break rmfiles.append((None, None, f)) return rmfiles def _xhj_gc_bytes_to_rmfiles(hsize, files): """Return the history files to remove to get under the byte limit.""" rmfiles = [] n = 0 nbytes = 0 for _, _, f in files[::-1]: fsize = os.stat(f).st_size if nbytes + fsize > hsize: break nbytes += fsize n += 1 rmfiles = files[:-n] return rmfiles def _xhj_get_history_files(sort=True, newest_first=False): """Find and return the history files. Optionally sort files by modify time. """ data_dir = builtins.__xonsh__.env.get("XONSH_DATA_DIR") data_dir = xt.expanduser_abs_path(data_dir) try: files = [ os.path.join(data_dir, f) for f in os.listdir(data_dir) if f.startswith("xonsh-") and f.endswith(".json") ] except OSError: files = [] if builtins.__xonsh__.env.get("XONSH_DEBUG"): xt.print_exception("Could not collect xonsh history files.") if sort: files.sort(key=lambda x: os.path.getmtime(x), reverse=newest_first) return files class JsonHistoryGC(threading.Thread): """Shell history garbage collection.""" def __init__(self, wait_for_shell=True, size=None, *args, **kwargs): """Thread responsible for garbage collecting old history. May wait for shell (and for xonshrc to have been loaded) to start work. """ super().__init__(*args, **kwargs) self.daemon = True self.size = size self.wait_for_shell = wait_for_shell self.start() self.gc_units_to_rmfiles = { "commands": _xhj_gc_commands_to_rmfiles, "files": _xhj_gc_files_to_rmfiles, "s": _xhj_gc_seconds_to_rmfiles, "b": _xhj_gc_bytes_to_rmfiles, } def run(self): while self.wait_for_shell: time.sleep(0.01) env = builtins.__xonsh__.env # pylint: disable=no-member if self.size is None: hsize, units = env.get("XONSH_HISTORY_SIZE") else: hsize, units = xt.to_history_tuple(self.size) files = self.files(only_unlocked=True) rmfiles_fn = self.gc_units_to_rmfiles.get(units) if rmfiles_fn is None: raise ValueError("Units type {0!r} not understood".format(units)) for _, _, f in rmfiles_fn(hsize, files): try: os.remove(f) except OSError: pass def files(self, only_unlocked=False): """Find and return the history files. Optionally locked files may be excluded. This is sorted by the last closed time. Returns a list of (timestamp, number of cmds, file name) tuples. """ # pylint: disable=no-member env = getattr(builtins, "__xonsh__.env", None) if env is None: return [] boot = uptime.boottime() fs = _xhj_get_history_files(sort=False) files = [] for f in fs: try: if os.path.getsize(f) == 0: # collect empty files (for gc) files.append((time.time(), 0, f)) continue lj = xlj.LazyJSON(f, reopen=False) if lj["locked"] and lj["ts"][0] < boot: # computer was rebooted between when this history was created # and now and so this history should be unlocked. hist = lj.load() lj.close() hist["locked"] = False with open(f, "w", newline="\n") as fp: xlj.ljdump(hist, fp, sort_keys=True) lj = xlj.LazyJSON(f, reopen=False) if only_unlocked and lj["locked"]: continue # info: closing timestamp, number of commands, filename files.append((lj["ts"][1] or lj["ts"][0], len(lj.sizes["cmds"]) - 1, f)) lj.close() except (IOError, OSError, ValueError): continue files.sort() return files class JsonHistoryFlusher(threading.Thread): """Flush shell history to disk periodically.""" def __init__(self, filename, buffer, queue, cond, at_exit=False, *args, **kwargs): """Thread for flushing history.""" super(JsonHistoryFlusher, self).__init__(*args, **kwargs) self.filename = filename self.buffer = buffer self.queue = queue queue.append(self) self.cond = cond self.at_exit = at_exit if at_exit: self.dump() queue.popleft() else: self.start() def run(self): with self.cond: self.cond.wait_for(self.i_am_at_the_front) self.dump() self.queue.popleft() def i_am_at_the_front(self): """Tests if the flusher is at the front of the queue.""" return self is self.queue[0] def dump(self): """Write the cached history to external storage.""" opts = builtins.__xonsh__.env.get("HISTCONTROL") last_inp = None cmds = [] for cmd in self.buffer: if "ignoredups" in opts and cmd["inp"] == last_inp: # Skipping dup cmd continue if "ignoreerr" in opts and cmd["rtn"] != 0: # Skipping failed cmd continue cmds.append(cmd) last_inp = cmd["inp"] with open(self.filename, "r", newline="\n") as f: hist = xlj.LazyJSON(f).load() load_hist_len = len(hist["cmds"]) hist["cmds"].extend(cmds) if self.at_exit: hist["ts"][1] = time.time() # apply end time hist["locked"] = False if not builtins.__xonsh__.env.get("XONSH_STORE_STDOUT", False): [cmd.pop("out") for cmd in hist["cmds"][load_hist_len:] if "out" in cmd] with open(self.filename, "w", newline="\n") as f: xlj.ljdump(hist, f, sort_keys=True) class JsonCommandField(cabc.Sequence): """A field in the 'cmds' portion of history.""" def __init__(self, field, hist, default=None): """Represents a field in the 'cmds' portion of history. Will query the buffer for the relevant data, if possible. Otherwise it will lazily acquire data from the file. Parameters ---------- field : str The name of the field to query. hist : History object The history object to query. default : optional The default value to return if key is not present. """ self.field = field self.hist = hist self.default = default def __len__(self): return len(self.hist) def __getitem__(self, key): size = len(self) if isinstance(key, slice): return [self[i] for i in range(*key.indices(size))] elif not isinstance(key, int): raise IndexError("JsonCommandField may only be indexed by int or slice.") elif size == 0: raise IndexError("JsonCommandField is empty.") # now we know we have an int key = size + key if key < 0 else key # ensure key is non-negative bufsize = len(self.hist.buffer) if size - bufsize <= key: # key is in buffer return self.hist.buffer[key + bufsize - size].get(self.field, self.default) # now we know we have to go into the file queue = self.hist._queue queue.append(self) with self.hist._cond: self.hist._cond.wait_for(self.i_am_at_the_front) with open(self.hist.filename, "r", newline="\n") as f: lj = xlj.LazyJSON(f, reopen=False) rtn = lj["cmds"][key].get(self.field, self.default) if isinstance(rtn, xlj.LJNode): rtn = rtn.load() queue.popleft() return rtn def i_am_at_the_front(self): """Tests if the command field is at the front of the queue.""" return self is self.hist._queue[0] class JsonHistory(History): """Xonsh history backend implemented with JSON files. JsonHistory implements two extra actions: ``diff``, and ``replay``. """ def __init__(self, filename=None, sessionid=None, buffersize=100, gc=True, **meta): """Represents a xonsh session's history as an in-memory buffer that is periodically flushed to disk. Parameters ---------- filename : str, optional Location of history file, defaults to ``$XONSH_DATA_DIR/xonsh-{sessionid}.json``. sessionid : int, uuid, str, optional Current session identifier, will generate a new sessionid if not set. buffersize : int, optional Maximum buffersize in memory. meta : optional Top-level metadata to store along with the history. The kwargs 'cmds' and 'sessionid' are not allowed and will be overwritten. gc : bool, optional Run garbage collector flag. """ super().__init__(sessionid=sessionid, **meta) if filename is None: # pylint: disable=no-member data_dir = builtins.__xonsh__.env.get("XONSH_DATA_DIR") data_dir = os.path.expanduser(data_dir) self.filename = os.path.join( data_dir, "xonsh-{0}.json".format(self.sessionid) ) else: self.filename = filename self.buffer = [] self.buffersize = buffersize self._queue = collections.deque() self._cond = threading.Condition() self._len = 0 self.last_cmd_out = None self.last_cmd_rtn = None meta["cmds"] = [] meta["sessionid"] = str(self.sessionid) with open(self.filename, "w", newline="\n") as f: xlj.ljdump(meta, f, sort_keys=True) self.gc = JsonHistoryGC() if gc else None # command fields that are known self.tss = JsonCommandField("ts", self) self.inps = JsonCommandField("inp", self) self.outs = JsonCommandField("out", self) self.rtns = JsonCommandField("rtn", self) def __len__(self): return self._len def append(self, cmd): """Appends command to history. Will periodically flush the history to file. Parameters ---------- cmd : dict This dict contains information about the command that is to be added to the history list. It should contain the keys ``inp``, ``rtn`` and ``ts``. These key names mirror the same names defined as instance variables in the ``HistoryEntry`` class. Returns ------- hf : JsonHistoryFlusher or None The thread that was spawned to flush history """ self.buffer.append(cmd) self._len += 1 # must come before flushing if len(self.buffer) >= self.buffersize: hf = self.flush() else: hf = None return hf def flush(self, at_exit=False): """Flushes the current command buffer to disk. Parameters ---------- at_exit : bool, optional Whether the JsonHistoryFlusher should act as a thread in the background, or execute immediately and block. Returns ------- hf : JsonHistoryFlusher or None The thread that was spawned to flush history """ if len(self.buffer) == 0: return hf = JsonHistoryFlusher( self.filename, tuple(self.buffer), self._queue, self._cond, at_exit=at_exit ) self.buffer.clear() return hf def items(self, newest_first=False): """Display history items of current session.""" if newest_first: items = zip(reversed(self.inps), reversed(self.tss)) else: items = zip(self.inps, self.tss) for item, tss in items: yield {"inp": item.rstrip(), "ts": tss[0]} def all_items(self, newest_first=False, **kwargs): """ Returns all history as found in XONSH_DATA_DIR. yield format: {'inp': cmd, 'rtn': 0, ...} """ while self.gc and self.gc.is_alive(): time.sleep(0.011) # gc sleeps for 0.01 secs, sleep a beat longer for f in _xhj_get_history_files(newest_first=newest_first): try: json_file = xlj.LazyJSON(f, reopen=False) except ValueError: # Invalid json file continue try: commands = json_file.load()["cmds"] except json.decoder.JSONDecodeError: # file is corrupted somehow if builtins.__xonsh__.env.get("XONSH_DEBUG") > 0: msg = "xonsh history file {0!r} is not valid JSON" print(msg.format(f), file=sys.stderr) continue if newest_first: commands = reversed(commands) for c in commands: yield {"inp": c["inp"].rstrip(), "ts": c["ts"][0]} # all items should also include session items yield from self.items() def info(self): data = collections.OrderedDict() data["backend"] = "json" data["sessionid"] = str(self.sessionid) data["filename"] = self.filename data["length"] = len(self) data["buffersize"] = self.buffersize data["bufferlength"] = len(self.buffer) envs = builtins.__xonsh__.env data["gc options"] = envs.get("XONSH_HISTORY_SIZE") return data def run_gc(self, size=None, blocking=True): self.gc = JsonHistoryGC(wait_for_shell=False, size=size) if blocking: while self.gc.is_alive(): continue ================================================ FILE: xonsh/history/main.py ================================================ # -*- coding: utf-8 -*- """Main entry points of the xonsh history.""" import argparse import builtins import datetime import functools import json import os import sys from xonsh.history.base import History from xonsh.history.dummy import DummyHistory from xonsh.history.json import JsonHistory from xonsh.history.sqlite import SqliteHistory import xonsh.diff_history as xdh import xonsh.lazyasd as xla import xonsh.tools as xt HISTORY_BACKENDS = {"dummy": DummyHistory, "json": JsonHistory, "sqlite": SqliteHistory} def construct_history(**kwargs): """Construct the history backend object.""" env = builtins.__xonsh__.env backend = env.get("XONSH_HISTORY_BACKEND") if isinstance(backend, str) and backend in HISTORY_BACKENDS: kls_history = HISTORY_BACKENDS[backend] elif xt.is_class(backend): kls_history = backend elif isinstance(backend, History): return backend else: print( "Unknown history backend: {}. Using JSON version".format(backend), file=sys.stderr, ) kls_history = JsonHistory return kls_history(**kwargs) def _xh_session_parser(hist=None, newest_first=False, **kwargs): """Returns history items of current session.""" if hist is None: hist = builtins.__xonsh__.history return hist.items() def _xh_all_parser(hist=None, newest_first=False, **kwargs): """Returns all history items.""" if hist is None: hist = builtins.__xonsh__.history return hist.all_items(newest_first=newest_first) def _xh_find_histfile_var(file_list, default=None): """Return the path of the history file from the value of the envvar HISTFILE. """ for f in file_list: f = xt.expanduser_abs_path(f) if not os.path.isfile(f): continue with open(f, "r") as rc_file: for line in rc_file: if line.startswith("HISTFILE="): hist_file = line.split("=", 1)[1].strip("'\"\n") hist_file = xt.expanduser_abs_path(hist_file) if os.path.isfile(hist_file): return hist_file else: if default: default = xt.expanduser_abs_path(default) if os.path.isfile(default): return default def _xh_bash_hist_parser(location=None, **kwargs): """Yield commands from bash history file""" if location is None: location = _xh_find_histfile_var( [os.path.join("~", ".bashrc"), os.path.join("~", ".bash_profile")], os.path.join("~", ".bash_history"), ) if location: with open(location, "r", errors="backslashreplace") as bash_hist: for ind, line in enumerate(bash_hist): yield {"inp": line.rstrip(), "ts": 0.0, "ind": ind} else: print("No bash history file", file=sys.stderr) def _xh_zsh_hist_parser(location=None, **kwargs): """Yield commands from zsh history file""" if location is None: location = _xh_find_histfile_var( [os.path.join("~", ".zshrc"), os.path.join("~", ".zprofile")], os.path.join("~", ".zsh_history"), ) if location: with open(location, "r", errors="backslashreplace") as zsh_hist: for ind, line in enumerate(zsh_hist): if line.startswith(":"): try: start_time, command = line.split(";", 1) except ValueError: # Invalid history entry continue try: start_time = float(start_time.split(":")[1]) except ValueError: start_time = 0.0 yield {"inp": command.rstrip(), "ts": start_time, "ind": ind} else: yield {"inp": line.rstrip(), "ts": 0.0, "ind": ind} else: print("No zsh history file found", file=sys.stderr) def _xh_filter_ts(commands, start_time, end_time): """Yield only the commands between start and end time.""" for cmd in commands: if start_time <= cmd["ts"] < end_time: yield cmd def _xh_get_history( session="session", *, slices=None, datetime_format=None, start_time=None, end_time=None, location=None ): """Get the requested portion of shell history. Parameters ---------- session: {'session', 'all', 'xonsh', 'bash', 'zsh'} The history session to get. slices : list of slice-like objects, optional Get only portions of history. start_time, end_time: float, optional Filter commands by timestamp. location: string, optional The history file location (bash or zsh) Returns ------- generator A filtered list of commands """ cmds = [] for i, item in enumerate(_XH_HISTORY_SESSIONS[session](location=location)): item["ind"] = i cmds.append(item) if slices: # transform/check all slices slices = [xt.ensure_slice(s) for s in slices] cmds = xt.get_portions(cmds, slices) if start_time or end_time: if start_time is None: start_time = 0.0 else: start_time = xt.ensure_timestamp(start_time, datetime_format) if end_time is None: end_time = float("inf") else: end_time = xt.ensure_timestamp(end_time, datetime_format) cmds = _xh_filter_ts(cmds, start_time, end_time) return cmds def _xh_show_history(hist, ns, stdout=None, stderr=None): """Show the requested portion of shell history. Accepts same parameters with `_xh_get_history`. """ try: commands = _xh_get_history( ns.session, slices=ns.slices, start_time=ns.start_time, end_time=ns.end_time, datetime_format=ns.datetime_format, ) except Exception as err: print("history: error: {}".format(err), file=stderr) return if ns.reverse: commands = reversed(list(commands)) end = "\0" if ns.null_byte else "\n" if ns.numerate and ns.timestamp: for c in commands: dt = datetime.datetime.fromtimestamp(c["ts"]) print( "{}:({}) {}".format(c["ind"], xt.format_datetime(dt), c["inp"]), file=stdout, end=end, ) elif ns.numerate: for c in commands: print("{}: {}".format(c["ind"], c["inp"]), file=stdout, end=end) elif ns.timestamp: for c in commands: dt = datetime.datetime.fromtimestamp(c["ts"]) print( "({}) {}".format(xt.format_datetime(dt), c["inp"]), file=stdout, end=end ) else: for c in commands: print(c["inp"], file=stdout, end=end) @xla.lazyobject def _XH_HISTORY_SESSIONS(): return { "session": _xh_session_parser, "xonsh": _xh_all_parser, "all": _xh_all_parser, "zsh": _xh_zsh_hist_parser, "bash": _xh_bash_hist_parser, } _XH_MAIN_ACTIONS = {"show", "id", "file", "info", "diff", "gc"} @functools.lru_cache() def _xh_create_parser(): """Create a parser for the "history" command.""" p = argparse.ArgumentParser( prog="history", description="try 'history --help' " "for more info" ) subp = p.add_subparsers(title="commands", dest="action") # session action show = subp.add_parser( "show", prefix_chars="-+", help="display history of a session, default command" ) show.add_argument( "-r", dest="reverse", default=False, action="store_true", help="reverses the direction", ) show.add_argument( "-n", dest="numerate", default=False, action="store_true", help="numerate each command", ) show.add_argument( "-t", dest="timestamp", default=False, action="store_true", help="show command timestamps", ) show.add_argument( "-T", dest="end_time", default=None, help="show only commands before timestamp" ) show.add_argument( "+T", dest="start_time", default=None, help="show only commands after timestamp" ) show.add_argument( "-f", dest="datetime_format", default=None, help="the datetime format to be used for" "filtering and printing", ) show.add_argument( "-0", dest="null_byte", default=False, action="store_true", help="separate commands by the null character for piping " "history to external filters", ) show.add_argument( "session", nargs="?", choices=_XH_HISTORY_SESSIONS.keys(), default="session", metavar="session", help="{} (default: current session, all is an alias for xonsh)" "".format(", ".join(map(repr, _XH_HISTORY_SESSIONS.keys()))), ) show.add_argument( "slices", nargs="*", default=None, metavar="slice", help="integer or slice notation", ) # 'id' subcommand subp.add_parser("id", help="display the current session id") # 'file' subcommand subp.add_parser("file", help="display the current history filename") # 'info' subcommand info = subp.add_parser( "info", help=("display information about the " "current history") ) info.add_argument( "--json", dest="json", default=False, action="store_true", help="print in JSON format", ) # gc gcp = subp.add_parser("gc", help="launches a new history garbage collector") gcp.add_argument( "--size", nargs=2, dest="size", default=None, help=( "next two arguments represent the history size and " 'units; e.g. "--size 8128 commands"' ), ) bgcp = gcp.add_mutually_exclusive_group() bgcp.add_argument( "--blocking", dest="blocking", default=True, action="store_true", help=("ensures that the gc blocks the main thread, " "default True"), ) bgcp.add_argument( "--non-blocking", dest="blocking", action="store_false", help="makes the gc non-blocking, and thus return sooner", ) hist = builtins.__xonsh__.history if isinstance(hist, JsonHistory): # add actions belong only to JsonHistory diff = subp.add_parser("diff", help="diff two xonsh history files") xdh.dh_create_parser(p=diff) import xonsh.replay as xrp replay = subp.add_parser("replay", help="replay a xonsh history file") xrp.replay_create_parser(p=replay) _XH_MAIN_ACTIONS.add("replay") return p def _xh_parse_args(args): """Prepare and parse arguments for the history command. Add default action for ``history`` and default session for ``history show``. """ parser = _xh_create_parser() if not args: args = ["show", "session"] elif args[0] not in _XH_MAIN_ACTIONS and args[0] not in ("-h", "--help"): args = ["show", "session"] + args if args[0] == "show": if not any(a in _XH_HISTORY_SESSIONS for a in args): args.insert(1, "session") ns, slices = parser.parse_known_args(args) if slices: if not ns.slices: ns.slices = slices else: ns.slices.extend(slices) else: ns = parser.parse_args(args) return ns def history_main( args=None, stdin=None, stdout=None, stderr=None, spec=None, stack=None ): """This is the history command entry point.""" hist = builtins.__xonsh__.history ns = _xh_parse_args(args) if not ns or not ns.action: return if ns.action == "show": _xh_show_history(hist, ns, stdout=stdout, stderr=stderr) elif ns.action == "info": data = hist.info() if ns.json: s = json.dumps(data) print(s, file=stdout) else: lines = ["{0}: {1}".format(k, v) for k, v in data.items()] print("\n".join(lines), file=stdout) elif ns.action == "id": if not hist.sessionid: return print(str(hist.sessionid), file=stdout) elif ns.action == "file": if not hist.filename: return print(str(hist.filename), file=stdout) elif ns.action == "gc": hist.run_gc(size=ns.size, blocking=ns.blocking) elif ns.action == "diff": if isinstance(hist, JsonHistory): xdh.dh_main_action(ns) elif ns.action == "replay": if isinstance(hist, JsonHistory): import xonsh.replay as xrp xrp.replay_main_action(hist, ns, stdout=stdout, stderr=stderr) else: print("Unknown history action {}".format(ns.action), file=sys.stderr) ================================================ FILE: xonsh/history/sqlite.py ================================================ # -*- coding: utf-8 -*- """Implements the xonsh history backend via sqlite3.""" import builtins import collections import json import os import sqlite3 import sys import threading import time from xonsh.history.base import History import xonsh.tools as xt def _xh_sqlite_get_file_name(): envs = builtins.__xonsh__.env file_name = envs.get("XONSH_HISTORY_SQLITE_FILE") if not file_name: data_dir = envs.get("XONSH_DATA_DIR") file_name = os.path.join(data_dir, "xonsh-history.sqlite") return xt.expanduser_abs_path(file_name) def _xh_sqlite_get_conn(filename=None): if filename is None: filename = _xh_sqlite_get_file_name() return sqlite3.connect(filename) def _xh_sqlite_create_history_table(cursor): """Create Table for history items. Columns: info - JSON formatted, reserved for future extension. """ cursor.execute( """ CREATE TABLE IF NOT EXISTS xonsh_history (inp TEXT, rtn INTEGER, tsb REAL, tse REAL, sessionid TEXT, out TEXT, info TEXT ) """ ) def _xh_sqlite_insert_command(cursor, cmd, sessionid, store_stdout): sql = "INSERT INTO xonsh_history (inp, rtn, tsb, tse, sessionid" tss = cmd.get("ts", [None, None]) params = [cmd["inp"].rstrip(), cmd["rtn"], tss[0], tss[1], sessionid] if store_stdout and "out" in cmd: sql += ", out" params.append(cmd["out"]) if "info" in cmd: sql += ", info" info = json.dumps(cmd["info"]) params.append(info) sql += ") VALUES (" + ("?, " * len(params)).rstrip(", ") + ")" cursor.execute(sql, tuple(params)) def _xh_sqlite_get_count(cursor, sessionid=None): sql = "SELECT count(*) FROM xonsh_history " params = [] if sessionid is not None: sql += "WHERE sessionid = ? " params.append(str(sessionid)) cursor.execute(sql, tuple(params)) return cursor.fetchone()[0] def _xh_sqlite_get_records(cursor, sessionid=None, limit=None, newest_first=False): sql = "SELECT inp, tsb, rtn FROM xonsh_history " params = [] if sessionid is not None: sql += "WHERE sessionid = ? " params.append(sessionid) sql += "ORDER BY tsb " if newest_first: sql += "DESC " if limit is not None: sql += "LIMIT %d " % limit cursor.execute(sql, tuple(params)) return cursor.fetchall() def _xh_sqlite_delete_records(cursor, size_to_keep): sql = "SELECT min(tsb) FROM (" sql += "SELECT tsb FROM xonsh_history ORDER BY tsb DESC " sql += "LIMIT %d)" % size_to_keep cursor.execute(sql) result = cursor.fetchone() if not result: return max_tsb = result[0] sql = "DELETE FROM xonsh_history WHERE tsb < ?" result = cursor.execute(sql, (max_tsb,)) return result.rowcount def xh_sqlite_append_history(cmd, sessionid, store_stdout, filename=None): with _xh_sqlite_get_conn(filename=filename) as conn: c = conn.cursor() _xh_sqlite_create_history_table(c) _xh_sqlite_insert_command(c, cmd, sessionid, store_stdout) conn.commit() def xh_sqlite_get_count(sessionid=None, filename=None): with _xh_sqlite_get_conn(filename=filename) as conn: c = conn.cursor() return _xh_sqlite_get_count(c, sessionid=sessionid) def xh_sqlite_items(sessionid=None, filename=None, newest_first=False): with _xh_sqlite_get_conn(filename=filename) as conn: c = conn.cursor() _xh_sqlite_create_history_table(c) return _xh_sqlite_get_records(c, sessionid=sessionid, newest_first=newest_first) def xh_sqlite_delete_items(size_to_keep, filename=None): with _xh_sqlite_get_conn(filename=filename) as conn: c = conn.cursor() _xh_sqlite_create_history_table(c) return _xh_sqlite_delete_records(c, size_to_keep) class SqliteHistoryGC(threading.Thread): """Shell history garbage collection.""" def __init__(self, wait_for_shell=True, size=None, filename=None, *args, **kwargs): """Thread responsible for garbage collecting old history. May wait for shell (and for xonshrc to have been loaded) to start work. """ super().__init__(*args, **kwargs) self.daemon = True self.filename = filename self.size = size self.wait_for_shell = wait_for_shell self.start() def run(self): while self.wait_for_shell: time.sleep(0.01) if self.size is not None: hsize, units = xt.to_history_tuple(self.size) else: envs = builtins.__xonsh__.env hsize, units = envs.get("XONSH_HISTORY_SIZE") if units != "commands": print( "sqlite backed history gc currently only supports " '"commands" as units', file=sys.stderr, ) return if hsize < 0: return xh_sqlite_delete_items(hsize, filename=self.filename) class SqliteHistory(History): """Xonsh history backend implemented with sqlite3.""" def __init__(self, gc=True, filename=None, **kwargs): super().__init__(**kwargs) if filename is None: filename = _xh_sqlite_get_file_name() self.filename = filename self.gc = SqliteHistoryGC() if gc else None self._last_hist_inp = None self.inps = [] self.rtns = [] self.outs = [] self.tss = [] def append(self, cmd): envs = builtins.__xonsh__.env opts = envs.get("HISTCONTROL") inp = cmd["inp"].rstrip() self.inps.append(inp) store_stdout = envs.get("XONSH_STORE_STDOUT", False) if store_stdout: self.outs.append(cmd.get("out")) else: self.outs.append(None) self.rtns.append(cmd["rtn"]) self.tss.append(cmd.get("ts", (None, None))) opts = envs.get("HISTCONTROL") if "ignoredups" in opts and inp == self._last_hist_inp: # Skipping dup cmd return if "ignoreerr" in opts and cmd["rtn"] != 0: # Skipping failed cmd return self._last_hist_inp = inp xh_sqlite_append_history( cmd, str(self.sessionid), store_stdout, filename=self.filename ) def all_items(self, newest_first=False): """Display all history items.""" for item in xh_sqlite_items(filename=self.filename, newest_first=newest_first): yield {"inp": item[0], "ts": item[1], "rtn": item[2]} def items(self, newest_first=False): """Display history items of current session.""" for item in xh_sqlite_items( sessionid=str(self.sessionid), filename=self.filename, newest_first=newest_first, ): yield {"inp": item[0], "ts": item[1], "rtn": item[2]} def info(self): data = collections.OrderedDict() data["backend"] = "sqlite" data["sessionid"] = str(self.sessionid) data["filename"] = self.filename data["session items"] = xh_sqlite_get_count( sessionid=self.sessionid, filename=self.filename ) data["all items"] = xh_sqlite_get_count(filename=self.filename) envs = builtins.__xonsh__.env data["gc options"] = envs.get("XONSH_HISTORY_SIZE") return data def run_gc(self, size=None, blocking=True): self.gc = SqliteHistoryGC(wait_for_shell=False, size=size) if blocking: while self.gc.is_alive(): continue ================================================ FILE: xonsh/imphooks.py ================================================ # -*- coding: utf-8 -*- """Import hooks for importing xonsh source files. This module registers the hooks it defines when it is imported. """ import os import re import sys import types import builtins import contextlib import importlib from importlib.machinery import ModuleSpec from importlib.abc import MetaPathFinder, SourceLoader, Loader from xonsh.events import events from xonsh.execer import Execer from xonsh.platform import scandir from xonsh.lazyasd import lazyobject @lazyobject def ENCODING_LINE(): # this regex comes from PEP 263 # https://www.python.org/dev/peps/pep-0263/#defining-the-encoding return re.compile(b"^[ tv]*#.*?coding[:=][ t]*([-_.a-zA-Z0-9]+)") def find_source_encoding(src): """Finds the source encoding given bytes representing a file. If no encoding is found, UTF-8 will be returned as per the docs https://docs.python.org/3/howto/unicode.html#unicode-literals-in-python-source-code """ utf8 = "UTF-8" first, _, rest = src.partition(b"\n") m = ENCODING_LINE.match(first) if m is not None: return m.group(1).decode(utf8) second, _, _ = rest.partition(b"\n") m = ENCODING_LINE.match(second) if m is not None: return m.group(1).decode(utf8) return utf8 class XonshImportHook(MetaPathFinder, SourceLoader): """Implements the import hook for xonsh source files.""" def __init__(self, *args, **kwargs): super(XonshImportHook, self).__init__(*args, **kwargs) self._filenames = {} self._execer = None @property def execer(self): if ( hasattr(builtins, "__xonsh__") and hasattr(builtins.__xonsh__, "execer") and builtins.__xonsh__.execer is not None ): execer = builtins.__xonsh__.execer if self._execer is not None: self._execer = None elif self._execer is None: self._execer = execer = Execer(unload=False) else: execer = self._execer return execer # # MetaPathFinder methods # def find_spec(self, fullname, path, target=None): """Finds the spec for a xonsh module if it exists.""" dot = "." spec = None path = sys.path if path is None else path if dot not in fullname and dot not in path: path = [dot] + path name = fullname.rsplit(dot, 1)[-1] fname = name + ".xsh" for p in path: if not isinstance(p, str): continue if not os.path.isdir(p) or not os.access(p, os.R_OK): continue if fname not in {x.name for x in scandir(p)}: continue spec = ModuleSpec(fullname, self) self._filenames[fullname] = os.path.join(p, fname) break return spec # # SourceLoader methods # def create_module(self, spec): """Create a xonsh module with the appropriate attributes.""" mod = types.ModuleType(spec.name) mod.__file__ = self.get_filename(spec.name) mod.__loader__ = self mod.__package__ = spec.parent or "" return mod def get_filename(self, fullname): """Returns the filename for a module's fullname.""" return self._filenames[fullname] def get_data(self, path): """Gets the bytes for a path.""" raise NotImplementedError def get_code(self, fullname): """Gets the code object for a xonsh file.""" filename = self.get_filename(fullname) if filename is None: msg = "xonsh file {0!r} could not be found".format(fullname) raise ImportError(msg) with open(filename, "rb") as f: src = f.read() enc = find_source_encoding(src) src = src.decode(encoding=enc) src = src if src.endswith("\n") else src + "\n" execer = self.execer execer.filename = filename ctx = {} # dummy for modules code = execer.compile(src, glbs=ctx, locs=ctx) return code # # Import events # events.doc( "on_import_pre_find_spec", """ on_import_pre_find_spec(fullname: str, path: str, target: module or None) -> None Fires before any import find_spec() calls have been executed. The parameters here are the same as importlib.abc.MetaPathFinder.find_spec(). Namely, :``fullname``: The full name of the module to import. :``path``: None if a top-level import, otherwise the ``__path__`` of the parent package. :``target``: Target module used to make a better guess about the package spec. """, ) events.doc( "on_import_post_find_spec", """ on_import_post_find_spec(spec, fullname, path, target) -> None Fires after all import find_spec() calls have been executed. The parameters here the spec and the arguments importlib.abc.MetaPathFinder.find_spec(). Namely, :``spec``: A ModuleSpec object if the spec was found, or None if it was not. :``fullname``: The full name of the module to import. :``path``: None if a top-level import, otherwise the ``__path__`` of the parent package. :``target``: Target module used to make a better guess about the package spec. """, ) events.doc( "on_import_pre_create_module", """ on_import_pre_create_module(spec: ModuleSpec) -> None Fires right before a module is created by its loader. The only parameter is the spec object. See importlib for more details. """, ) events.doc( "on_import_post_create_module", """ on_import_post_create_module(module: Module, spec: ModuleSpec) -> None Fires after a module is created by its loader but before the loader returns it. The parameters here are the module object itself and the spec object. See importlib for more details. """, ) events.doc( "on_import_pre_exec_module", """ on_import_pre_exec_module(module: Module) -> None Fires right before a module is executed by its loader. The only parameter is the module itself. See importlib for more details. """, ) events.doc( "on_import_post_exec_module", """ on_import_post_create_module(module: Module) -> None Fires after a module is executed by its loader but before the loader returns it. The only parameter is the module itself. See importlib for more details. """, ) def _should_dispatch_xonsh_import_event_loader(): """Figures out if we should dispatch to a load event""" return ( len(events.on_import_pre_create_module) > 0 or len(events.on_import_post_create_module) > 0 or len(events.on_import_pre_exec_module) > 0 or len(events.on_import_post_exec_module) > 0 ) class XonshImportEventHook(MetaPathFinder): """Implements the import hook for firing xonsh events on import.""" def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._fullname_stack = [] @contextlib.contextmanager def append_stack(self, fullname): """A context manager for appending and then removing a name from the fullname stack. """ self._fullname_stack.append(fullname) yield del self._fullname_stack[-1] # # MetaPathFinder methods # def find_spec(self, fullname, path, target=None): """Finds the spec for a xonsh module if it exists.""" if fullname in reversed(self._fullname_stack): # don't execute if we are already in the stack. return None npre = len(events.on_import_pre_find_spec) npost = len(events.on_import_post_find_spec) dispatch_load = _should_dispatch_xonsh_import_event_loader() if npre > 0: events.on_import_pre_find_spec.fire( fullname=fullname, path=path, target=target ) elif npost == 0 and not dispatch_load: # no events to fire, proceed normally and prevent recursion return None # now find the spec with self.append_stack(fullname): spec = importlib.util.find_spec(fullname) # fire post event if npost > 0: events.on_import_post_find_spec.fire( spec=spec, fullname=fullname, path=path, target=target ) if dispatch_load and spec is not None and hasattr(spec.loader, "create_module"): spec.loader = XonshImportEventLoader(spec.loader) return spec class XonshImportEventLoader(Loader): """A class that dispatches loader calls to another loader and fires relevant xonsh events. """ def __init__(self, loader): self.loader = loader # # Loader methods # def create_module(self, spec): """Creates and returns the module object.""" events.on_import_pre_create_module.fire(spec=spec) mod = self.loader.create_module(spec) events.on_import_post_create_module.fire(module=mod, spec=spec) return mod def exec_module(self, module): """Executes the module in its own namespace.""" events.on_import_pre_exec_module.fire(module=module) rtn = self.loader.exec_module(module) events.on_import_post_exec_module.fire(module=module) return rtn def load_module(self, fullname): """Legacy module loading, provided for backwards compatibility.""" return self.loader.load_module(fullname) def module_repr(self, module): """Legacy module repr, provided for backwards compatibility.""" return self.loader.module_repr(module) def install_import_hooks(): """ Install Xonsh import hooks in ``sys.meta_path`` in order for ``.xsh`` files to be importable and import events to be fired. Can safely be called many times, will be no-op if xonsh import hooks are already present. """ found_imp = found_event = False for hook in sys.meta_path: if isinstance(hook, XonshImportHook): found_imp = True elif isinstance(hook, XonshImportEventHook): found_event = True if not found_imp: sys.meta_path.append(XonshImportHook()) if not found_event: sys.meta_path.insert(0, XonshImportEventHook()) # alias to deprecated name install_hook = install_import_hooks ================================================ FILE: xonsh/inspectors.py ================================================ # -*- coding: utf-8 -*- """Tools for inspecting Python objects. This file was forked from the IPython project: * Copyright (c) 2008-2014, IPython Development Team * Copyright (C) 2001-2007 Fernando Perez * Copyright (c) 2001, Janko Hauser * Copyright (c) 2001, Nathaniel Gray """ import os import io import sys import types import inspect import itertools import linecache import collections from xonsh.lazyasd import LazyObject from xonsh.tokenize import detect_encoding from xonsh.openpy import read_py_file from xonsh.tools import cast_unicode, safe_hasattr, indent, print_color, format_color from xonsh.platform import HAS_PYGMENTS, PYTHON_VERSION_INFO from xonsh.lazyimps import pygments, pyghooks from xonsh.style_tools import partial_color_tokenize # builtin docstrings to ignore _func_call_docstring = LazyObject( lambda: types.FunctionType.__call__.__doc__, globals(), "_func_call_docstring" ) _object_init_docstring = LazyObject( lambda: object.__init__.__doc__, globals(), "_object_init_docstring" ) _builtin_type_docstrings = LazyObject( lambda: { t.__doc__ for t in (types.ModuleType, types.MethodType, types.FunctionType) }, globals(), "_builtin_type_docstrings", ) _builtin_func_type = LazyObject(lambda: type(all), globals(), "_builtin_func_type") # Bound methods have the same type as builtin functions _builtin_meth_type = LazyObject( lambda: type(str.upper), globals(), "_builtin_meth_type" ) info_fields = LazyObject( lambda: [ "type_name", "base_class", "string_form", "namespace", "length", "file", "definition", "docstring", "source", "init_definition", "class_docstring", "init_docstring", "call_def", "call_docstring", # These won't be printed but will be used to determine how to # format the object "ismagic", "isalias", "isclass", "argspec", "found", "name", ], globals(), "info_fields", ) def object_info(**kw): """Make an object info dict with all fields present.""" infodict = dict(itertools.zip_longest(info_fields, [None])) infodict.update(kw) return infodict def get_encoding(obj): """Get encoding for python source file defining obj Returns None if obj is not defined in a sourcefile. """ ofile = find_file(obj) # run contents of file through pager starting at line where the object # is defined, as long as the file isn't binary and is actually on the # filesystem. if ofile is None: return None elif ofile.endswith((".so", ".dll", ".pyd")): return None elif not os.path.isfile(ofile): return None else: # Print only text files, not extension binaries. Note that # getsourcelines returns lineno with 1-offset and page() uses # 0-offset, so we must adjust. with io.open(ofile, "rb") as buf: # Tweaked to use io.open for Python 2 encoding, _ = detect_encoding(buf.readline) return encoding def getdoc(obj): """Stable wrapper around inspect.getdoc. This can't crash because of attribute problems. It also attempts to call a getdoc() method on the given object. This allows objects which provide their docstrings via non-standard mechanisms (like Pyro proxies) to still be inspected by ipython's ? system.""" # Allow objects to offer customized documentation via a getdoc method: try: ds = obj.getdoc() except Exception: # pylint:disable=broad-except pass else: # if we get extra info, we add it to the normal docstring. if isinstance(ds, str): return inspect.cleandoc(ds) try: docstr = inspect.getdoc(obj) encoding = get_encoding(obj) return cast_unicode(docstr, encoding=encoding) except Exception: # pylint:disable=broad-except # Harden against an inspect failure, which can occur with # SWIG-wrapped extensions. raise def getsource(obj, is_binary=False): """Wrapper around inspect.getsource. This can be modified by other projects to provide customized source extraction. Inputs: - obj: an object whose source code we will attempt to extract. Optional inputs: - is_binary: whether the object is known to come from a binary source. This implementation will skip returning any output for binary objects, but custom extractors may know how to meaningfully process them.""" if is_binary: return None else: # get source if obj was decorated with @decorator if hasattr(obj, "__wrapped__"): obj = obj.__wrapped__ try: src = inspect.getsource(obj) except TypeError: if hasattr(obj, "__class__"): src = inspect.getsource(obj.__class__) encoding = get_encoding(obj) return cast_unicode(src, encoding=encoding) def is_simple_callable(obj): """True if obj is a function ()""" return ( inspect.isfunction(obj) or inspect.ismethod(obj) or isinstance(obj, _builtin_func_type) or isinstance(obj, _builtin_meth_type) ) def getargspec(obj): """Wrapper around :func:`inspect.getfullargspec` on Python 3, and :func:inspect.getargspec` on Python 2. In addition to functions and methods, this can also handle objects with a ``__call__`` attribute. """ if safe_hasattr(obj, "__call__") and not is_simple_callable(obj): obj = obj.__call__ return inspect.getfullargspec(obj) def format_argspec(argspec): """Format argspect, convenience wrapper around inspect's. This takes a dict instead of ordered arguments and calls inspect.format_argspec with the arguments in the necessary order. """ return inspect.formatargspec( argspec["args"], argspec["varargs"], argspec["varkw"], argspec["defaults"] ) def call_tip(oinfo, format_call=True): """Extract call tip data from an oinfo dict. Parameters ---------- oinfo : dict format_call : bool, optional If True, the call line is formatted and returned as a string. If not, a tuple of (name, argspec) is returned. Returns ------- call_info : None, str or (str, dict) tuple. When format_call is True, the whole call information is formatted as a single string. Otherwise, the object's name and its argspec dict are returned. If no call information is available, None is returned. docstring : str or None The most relevant docstring for calling purposes is returned, if available. The priority is: call docstring for callable instances, then constructor docstring for classes, then main object's docstring otherwise (regular functions). """ # Get call definition argspec = oinfo.get("argspec") if argspec is None: call_line = None else: # Callable objects will have 'self' as their first argument, prune # it out if it's there for clarity (since users do *not* pass an # extra first argument explicitly). try: has_self = argspec["args"][0] == "self" except (KeyError, IndexError): pass else: if has_self: argspec["args"] = argspec["args"][1:] call_line = oinfo["name"] + format_argspec(argspec) # Now get docstring. # The priority is: call docstring, constructor docstring, main one. doc = oinfo.get("call_docstring") if doc is None: doc = oinfo.get("init_docstring") if doc is None: doc = oinfo.get("docstring", "") return call_line, doc def find_file(obj): """Find the absolute path to the file where an object was defined. This is essentially a robust wrapper around `inspect.getabsfile`. Returns None if no file can be found. Parameters ---------- obj : any Python object Returns ------- fname : str The absolute path to the file where the object was defined. """ # get source if obj was decorated with @decorator if safe_hasattr(obj, "__wrapped__"): obj = obj.__wrapped__ fname = None try: fname = inspect.getabsfile(obj) except TypeError: # For an instance, the file that matters is where its class was # declared. if hasattr(obj, "__class__"): try: fname = inspect.getabsfile(obj.__class__) except TypeError: # Can happen for builtins pass except: # pylint:disable=bare-except pass return cast_unicode(fname) def find_source_lines(obj): """Find the line number in a file where an object was defined. This is essentially a robust wrapper around `inspect.getsourcelines`. Returns None if no file can be found. Parameters ---------- obj : any Python object Returns ------- lineno : int The line number where the object definition starts. """ # get source if obj was decorated with @decorator if safe_hasattr(obj, "__wrapped__"): obj = obj.__wrapped__ try: try: lineno = inspect.getsourcelines(obj)[1] except TypeError: # For instances, try the class object like getsource() does if hasattr(obj, "__class__"): lineno = inspect.getsourcelines(obj.__class__)[1] else: lineno = None except: # pylint:disable=bare-except return None return lineno if PYTHON_VERSION_INFO < (3, 5, 0): FrameInfo = collections.namedtuple( "FrameInfo", ["frame", "filename", "lineno", "function", "code_context", "index"], ) def getouterframes(frame, context=1): """Wrapper for getouterframes so that it acts like the Python v3.5 version.""" return [FrameInfo(*f) for f in inspect.getouterframes(frame, context=context)] else: getouterframes = inspect.getouterframes class Inspector(object): """Inspects objects.""" def __init__(self, str_detail_level=0): self.str_detail_level = str_detail_level def _getdef(self, obj, oname=""): """Return the call signature for any callable object. If any exception is generated, None is returned instead and the exception is suppressed. """ try: hdef = oname + inspect.signature(*getargspec(obj)) return cast_unicode(hdef) except: # pylint:disable=bare-except return None def noinfo(self, msg, oname): """Generic message when no information is found.""" print("No %s found" % msg, end=" ") if oname: print("for %s" % oname) else: print() def pdef(self, obj, oname=""): """Print the call signature for any callable object. If the object is a class, print the constructor information. """ if not callable(obj): print("Object is not callable.") return header = "" if inspect.isclass(obj): header = self.__head("Class constructor information:\n") obj = obj.__init__ output = self._getdef(obj, oname) if output is None: self.noinfo("definition header", oname) else: print(header, output, end=" ", file=sys.stdout) def pdoc(self, obj, oname=""): """Print the docstring for any object. Optional -formatter: a function to run the docstring through for specially formatted docstrings. """ head = self.__head # For convenience lines = [] ds = getdoc(obj) if ds: lines.append(head("Class docstring:")) lines.append(indent(ds)) if inspect.isclass(obj) and hasattr(obj, "__init__"): init_ds = getdoc(obj.__init__) if init_ds is not None: lines.append(head("Init docstring:")) lines.append(indent(init_ds)) elif hasattr(obj, "__call__"): call_ds = getdoc(obj.__call__) if call_ds: lines.append(head("Call docstring:")) lines.append(indent(call_ds)) if not lines: self.noinfo("documentation", oname) else: print("\n".join(lines)) def psource(self, obj, oname=""): """Print the source code for an object.""" # Flush the source cache because inspect can return out-of-date source linecache.checkcache() try: src = getsource(obj) except: # pylint:disable=bare-except self.noinfo("source", oname) else: print(src) def pfile(self, obj, oname=""): """Show the whole file where an object was defined.""" lineno = find_source_lines(obj) if lineno is None: self.noinfo("file", oname) return ofile = find_file(obj) # run contents of file through pager starting at line where the object # is defined, as long as the file isn't binary and is actually on the # filesystem. if ofile.endswith((".so", ".dll", ".pyd")): print("File %r is binary, not printing." % ofile) elif not os.path.isfile(ofile): print("File %r does not exist, not printing." % ofile) else: # Print only text files, not extension binaries. Note that # getsourcelines returns lineno with 1-offset and page() uses # 0-offset, so we must adjust. o = read_py_file(ofile, skip_encoding_cookie=False) print(o, lineno - 1) def _format_fields_str(self, fields, title_width=0): """Formats a list of fields for display using color strings. Parameters ---------- fields : list A list of 2-tuples: (field_title, field_content) title_width : int How many characters to pad titles to. Default to longest title. """ out = [] if title_width == 0: title_width = max(len(title) + 2 for title, _ in fields) for title, content in fields: title_len = len(title) title = "{BOLD_RED}" + title + ":{NO_COLOR}" if len(content.splitlines()) > 1: title += "\n" else: title += " ".ljust(title_width - title_len) out.append(cast_unicode(title) + cast_unicode(content)) return format_color("\n".join(out) + "\n") def _format_fields_tokens(self, fields, title_width=0): """Formats a list of fields for display using color tokens from pygments. Parameters ---------- fields : list A list of 2-tuples: (field_title, field_content) title_width : int How many characters to pad titles to. Default to longest title. """ out = [] if title_width == 0: title_width = max(len(title) + 2 for title, _ in fields) for title, content in fields: title_len = len(title) title = "{BOLD_RED}" + title + ":{NO_COLOR}" if not isinstance(content, str) or len(content.splitlines()) > 1: title += "\n" else: title += " ".ljust(title_width - title_len) out += partial_color_tokenize(title) if isinstance(content, str): out[-1] = (out[-1][0], out[-1][1] + content + "\n") else: out += content out[-1] = (out[-1][0], out[-1][1] + "\n") out[-1] = (out[-1][0], out[-1][1] + "\n") return out def _format_fields(self, fields, title_width=0): """Formats a list of fields for display using color tokens from pygments. Parameters ---------- fields : list A list of 2-tuples: (field_title, field_content) title_width : int How many characters to pad titles to. Default to longest title. """ if HAS_PYGMENTS: rtn = self._format_fields_tokens(fields, title_width=title_width) else: rtn = self._format_fields_str(fields, title_width=title_width) return rtn # The fields to be displayed by pinfo: (fancy_name, key_in_info_dict) pinfo_fields1 = [("Type", "type_name")] pinfo_fields2 = [("String form", "string_form")] pinfo_fields3 = [ ("Length", "length"), ("File", "file"), ("Definition", "definition"), ] pinfo_fields_obj = [ ("Class docstring", "class_docstring"), ("Init docstring", "init_docstring"), ("Call def", "call_def"), ("Call docstring", "call_docstring"), ] def pinfo(self, obj, oname="", info=None, detail_level=0): """Show detailed information about an object. Parameters ---------- obj : object oname : str, optional name of the variable pointing to the object. info : dict, optional a structure with some information fields which may have been precomputed already. detail_level : int, optional if set to 1, more information is given. """ info = self.info(obj, oname=oname, info=info, detail_level=detail_level) displayfields = [] def add_fields(fields): for title, key in fields: field = info[key] if field is not None: displayfields.append((title, field.rstrip())) add_fields(self.pinfo_fields1) add_fields(self.pinfo_fields2) # Namespace if info["namespace"] is not None and info["namespace"] != "Interactive": displayfields.append(("Namespace", info["namespace"].rstrip())) add_fields(self.pinfo_fields3) if info["isclass"] and info["init_definition"]: displayfields.append(("Init definition", info["init_definition"].rstrip())) # Source or docstring, depending on detail level and whether # source found. if detail_level > 0 and info["source"] is not None: displayfields.append(("Source", cast_unicode(info["source"]))) elif info["docstring"] is not None: displayfields.append(("Docstring", info["docstring"])) # Constructor info for classes if info["isclass"]: if info["init_docstring"] is not None: displayfields.append(("Init docstring", info["init_docstring"])) # Info for objects: else: add_fields(self.pinfo_fields_obj) # Finally send to printer/pager: if displayfields: print_color(self._format_fields(displayfields)) def info(self, obj, oname="", info=None, detail_level=0): """Compute a dict with detailed information about an object. Optional arguments: - oname: name of the variable pointing to the object. - info: a structure with some information fields which may have been precomputed already. - detail_level: if set to 1, more information is given. """ obj_type = type(obj) if info is None: ismagic = 0 isalias = 0 ospace = "" else: ismagic = info.ismagic isalias = info.isalias ospace = info.namespace # Get docstring, special-casing aliases: if isalias: if not callable(obj): if len(obj) >= 2 and isinstance(obj[1], str): ds = "Alias to the system command:\n {0}".format(obj[1]) else: # pylint:disable=bare-except ds = "Alias: " + str(obj) else: ds = "Alias to " + str(obj) if obj.__doc__: ds += "\nDocstring:\n" + obj.__doc__ else: ds = getdoc(obj) if ds is None: ds = "" # store output in a dict, we initialize it here and fill it as we go out = dict(name=oname, found=True, isalias=isalias, ismagic=ismagic) string_max = 200 # max size of strings to show (snipped if longer) shalf = int((string_max - 5) / 2) if ismagic: obj_type_name = "Magic function" elif isalias: obj_type_name = "System alias" else: obj_type_name = obj_type.__name__ out["type_name"] = obj_type_name try: bclass = obj.__class__ out["base_class"] = str(bclass) except: # pylint:disable=bare-except pass # String form, but snip if too long in ? form (full in ??) if detail_level >= self.str_detail_level: try: ostr = str(obj) str_head = "string_form" if not detail_level and len(ostr) > string_max: ostr = ostr[:shalf] + " <...> " + ostr[-shalf:] ostr = ("\n" + " " * len(str_head.expandtabs())).join( q.strip() for q in ostr.split("\n") ) out[str_head] = ostr except: # pylint:disable=bare-except pass if ospace: out["namespace"] = ospace # Length (for strings and lists) try: out["length"] = str(len(obj)) except: # pylint:disable=bare-except pass # Filename where object was defined binary_file = False fname = find_file(obj) if fname is None: # if anything goes wrong, we don't want to show source, so it's as # if the file was binary binary_file = True else: if fname.endswith((".so", ".dll", ".pyd")): binary_file = True elif fname.endswith(""): fname = "Dynamically generated function. " "No source code available." out["file"] = fname # Docstrings only in detail 0 mode, since source contains them (we # avoid repetitions). If source fails, we add them back, see below. if ds and detail_level == 0: out["docstring"] = ds # Original source code for any callable if detail_level: # Flush the source cache because inspect can return out-of-date # source linecache.checkcache() source = None try: try: source = getsource(obj, binary_file) except TypeError: if hasattr(obj, "__class__"): source = getsource(obj.__class__, binary_file) if source is not None: source = source.rstrip() if HAS_PYGMENTS: lexer = pyghooks.XonshLexer() source = list(pygments.lex(source, lexer=lexer)) out["source"] = source except Exception: # pylint:disable=broad-except pass if ds and source is None: out["docstring"] = ds # Constructor docstring for classes if inspect.isclass(obj): out["isclass"] = True # reconstruct the function definition and print it: try: obj_init = obj.__init__ except AttributeError: init_def = init_ds = None else: init_def = self._getdef(obj_init, oname) init_ds = getdoc(obj_init) # Skip Python's auto-generated docstrings if init_ds == _object_init_docstring: init_ds = None if init_def or init_ds: if init_def: out["init_definition"] = init_def if init_ds: out["init_docstring"] = init_ds # and class docstring for instances: else: # reconstruct the function definition and print it: defln = self._getdef(obj, oname) if defln: out["definition"] = defln # First, check whether the instance docstring is identical to the # class one, and print it separately if they don't coincide. In # most cases they will, but it's nice to print all the info for # objects which use instance-customized docstrings. if ds: try: cls = getattr(obj, "__class__") except: # pylint:disable=bare-except class_ds = None else: class_ds = getdoc(cls) # Skip Python's auto-generated docstrings if class_ds in _builtin_type_docstrings: class_ds = None if class_ds and ds != class_ds: out["class_docstring"] = class_ds # Next, try to show constructor docstrings try: init_ds = getdoc(obj.__init__) # Skip Python's auto-generated docstrings if init_ds == _object_init_docstring: init_ds = None except AttributeError: init_ds = None if init_ds: out["init_docstring"] = init_ds # Call form docstring for callable instances if safe_hasattr(obj, "__call__") and not is_simple_callable(obj): call_def = self._getdef(obj.__call__, oname) if call_def: call_def = call_def # it may never be the case that call def and definition # differ, but don't include the same signature twice if call_def != out.get("definition"): out["call_def"] = call_def call_ds = getdoc(obj.__call__) # Skip Python's auto-generated docstrings if call_ds == _func_call_docstring: call_ds = None if call_ds: out["call_docstring"] = call_ds # Compute the object's argspec as a callable. The key is to decide # whether to pull it from the object itself, from its __init__ or # from its __call__ method. if inspect.isclass(obj): # Old-style classes need not have an __init__ callable_obj = getattr(obj, "__init__", None) elif callable(obj): callable_obj = obj else: callable_obj = None if callable_obj: try: argspec = getargspec(callable_obj) except (TypeError, AttributeError): # For extensions/builtins we can't retrieve the argspec pass else: # named tuples' _asdict() method returns an OrderedDict, but we # we want a normal out["argspec"] = argspec_dict = dict(argspec._asdict()) # We called this varkw before argspec became a named tuple. # With getfullargspec it's also called varkw. if "varkw" not in argspec_dict: argspec_dict["varkw"] = argspec_dict.pop("keywords") return object_info(**out) ================================================ FILE: xonsh/jobs.py ================================================ # -*- coding: utf-8 -*- """Job control for the xonsh shell.""" import os import sys import time import ctypes import signal import builtins import subprocess import collections from xonsh.lazyasd import LazyObject from xonsh.platform import FD_STDERR, ON_DARWIN, ON_WINDOWS, ON_CYGWIN, ON_MSYS, LIBC from xonsh.tools import unthreadable tasks = LazyObject(collections.deque, globals(), "tasks") # Track time stamp of last exit command, so that two consecutive attempts to # exit can kill all jobs and exit. _last_exit_time = None if ON_DARWIN: def _send_signal(job, signal): # On OS X, os.killpg() may cause PermissionError when there are # any zombie processes in the process group. # See github issue #1012 for details for pid in job["pids"]: if pid is None: # the pid of an aliased proc is None continue try: os.kill(pid, signal) except ProcessLookupError: pass elif ON_WINDOWS: pass elif ON_CYGWIN or ON_MSYS: # Similar to what happened on OSX, more issues on Cygwin # (see Github issue #514). def _send_signal(job, signal): try: os.killpg(job["pgrp"], signal) except Exception: for pid in job["pids"]: try: os.kill(pid, signal) except Exception: pass else: def _send_signal(job, signal): pgrp = job["pgrp"] if pgrp is None: for pid in job["pids"]: try: os.kill(pid, signal) except Exception: pass else: os.killpg(job["pgrp"], signal) if ON_WINDOWS: def _continue(job): job["status"] = "running" def _kill(job): subprocess.check_output(["taskkill", "/F", "/T", "/PID", str(job["obj"].pid)]) def ignore_sigtstp(): pass def give_terminal_to(pgid): pass def wait_for_active_job(last_task=None, backgrounded=False): """ Wait for the active job to finish, to be killed by SIGINT, or to be suspended by ctrl-z. """ _clear_dead_jobs() active_task = get_next_task() # Return when there are no foreground active task if active_task is None: return last_task obj = active_task["obj"] _continue(active_task) while obj.returncode is None: try: obj.wait(0.01) except subprocess.TimeoutExpired: pass except KeyboardInterrupt: _kill(active_task) return wait_for_active_job(last_task=active_task) else: def _continue(job): _send_signal(job, signal.SIGCONT) def _kill(job): _send_signal(job, signal.SIGKILL) def ignore_sigtstp(): signal.signal(signal.SIGTSTP, signal.SIG_IGN) _shell_pgrp = os.getpgrp() _block_when_giving = LazyObject( lambda: (signal.SIGTTOU, signal.SIGTTIN, signal.SIGTSTP, signal.SIGCHLD), globals(), "_block_when_giving", ) if ON_CYGWIN or ON_MSYS: # on cygwin, signal.pthread_sigmask does not exist in Python, even # though pthread_sigmask is defined in the kernel. thus, we use # ctypes to mimic the calls in the "normal" version below. LIBC.pthread_sigmask.restype = ctypes.c_int LIBC.pthread_sigmask.argtypes = [ ctypes.c_int, ctypes.POINTER(ctypes.c_ulong), ctypes.POINTER(ctypes.c_ulong), ] def _pthread_sigmask(how, signals): mask = 0 for sig in signals: mask |= 1 << sig oldmask = ctypes.c_ulong() mask = ctypes.c_ulong(mask) result = LIBC.pthread_sigmask( how, ctypes.byref(mask), ctypes.byref(oldmask) ) if result: raise OSError(result, "Sigmask error.") return { sig for sig in getattr(signal, "Signals", range(0, 65)) if (oldmask.value >> sig) & 1 } else: _pthread_sigmask = signal.pthread_sigmask # give_terminal_to is a simplified version of: # give_terminal_to from bash 4.3 source, jobs.c, line 4030 # this will give the terminal to the process group pgid def give_terminal_to(pgid): if pgid is None: return False oldmask = _pthread_sigmask(signal.SIG_BLOCK, _block_when_giving) try: os.tcsetpgrp(FD_STDERR, pgid) return True except ProcessLookupError: # when the process finished before giving terminal to it, # see issue #2288 return False except OSError as e: if e.errno == 22: # [Errno 22] Invalid argument # there are cases that all the processes of pgid have # finished, then we don't need to do anything here, see # issue #2220 return False elif e.errno == 25: # [Errno 25] Inappropriate ioctl for device # There are also cases where we are not connected to a # real TTY, even though we may be run in interactive # mode. See issue #2267 for an example with emacs return False else: raise finally: if oldmask: # only reset the mask if it is non-empty! See #2989 _pthread_sigmask(signal.SIG_SETMASK, oldmask) def wait_for_active_job(last_task=None, backgrounded=False, _nochild=False): """ Wait for the active job to finish, to be killed by SIGINT, or to be suspended by ctrl-z. """ _clear_dead_jobs() active_task = get_next_task() # Return when there are no foreground active task if active_task is None: return last_task obj = active_task["obj"] backgrounded = False try: _, wcode = os.waitpid(obj.pid, os.WUNTRACED) except ChildProcessError: # No child processes if _nochild: return active_task else: return wait_for_active_job( last_task=active_task, backgrounded=backgrounded, _nochild=True ) if os.WIFSTOPPED(wcode): print("^Z") active_task["status"] = "stopped" backgrounded = True elif os.WIFSIGNALED(wcode): print() # get a newline because ^C will have been printed obj.signal = (os.WTERMSIG(wcode), os.WCOREDUMP(wcode)) obj.returncode = None else: obj.returncode = os.WEXITSTATUS(wcode) obj.signal = None return wait_for_active_job(last_task=active_task, backgrounded=backgrounded) def get_next_task(): """ Get the next active task and put it on top of the queue""" selected_task = None for tid in tasks: task = get_task(tid) if not task["bg"] and task["status"] == "running": selected_task = tid break if selected_task is None: return tasks.remove(selected_task) tasks.appendleft(selected_task) return get_task(selected_task) def get_task(tid): return builtins.__xonsh__.all_jobs[tid] def _clear_dead_jobs(): to_remove = set() for tid in tasks: obj = get_task(tid)["obj"] if obj is None or obj.poll() is not None: to_remove.add(tid) for job in to_remove: tasks.remove(job) del builtins.__xonsh__.all_jobs[job] def print_one_job(num, outfile=sys.stdout): """Print a line describing job number ``num``.""" try: job = builtins.__xonsh__.all_jobs[num] except KeyError: return pos = "+" if tasks[0] == num else "-" if tasks[1] == num else " " status = job["status"] cmd = [" ".join(i) if isinstance(i, list) else i for i in job["cmds"]] cmd = " ".join(cmd) pid = job["pids"][-1] bg = " &" if job["bg"] else "" print("[{}]{} {}: {}{} ({})".format(num, pos, status, cmd, bg, pid), file=outfile) def get_next_job_number(): """Get the lowest available unique job number (for the next job created). """ _clear_dead_jobs() i = 1 while i in builtins.__xonsh__.all_jobs: i += 1 return i def add_job(info): """Add a new job to the jobs dictionary.""" num = get_next_job_number() info["started"] = time.time() info["status"] = "running" tasks.appendleft(num) builtins.__xonsh__.all_jobs[num] = info if info["bg"] and builtins.__xonsh__.env.get("XONSH_INTERACTIVE"): print_one_job(num) def clean_jobs(): """Clean up jobs for exiting shell In non-interactive mode, kill all jobs. In interactive mode, check for suspended or background jobs, print a warning if any exist, and return False. Otherwise, return True. """ jobs_clean = True if builtins.__xonsh__.env["XONSH_INTERACTIVE"]: _clear_dead_jobs() if builtins.__xonsh__.all_jobs: global _last_exit_time hist = builtins.__xonsh__.history if hist is not None and len(hist.tss) > 0: last_cmd_start = hist.tss[-1][0] else: last_cmd_start = None if _last_exit_time and last_cmd_start and _last_exit_time > last_cmd_start: # Exit occurred after last command started, so it was called as # part of the last command and is now being called again # immediately. Kill jobs and exit without reminder about # unfinished jobs in this case. kill_all_jobs() else: if len(builtins.__xonsh__.all_jobs) > 1: msg = "there are unfinished jobs" else: msg = "there is an unfinished job" if builtins.__xonsh__.env["SHELL_TYPE"] != "prompt_toolkit": # The Ctrl+D binding for prompt_toolkit already inserts a # newline print() print("xonsh: {}".format(msg), file=sys.stderr) print("-" * 5, file=sys.stderr) jobs([], stdout=sys.stderr) print("-" * 5, file=sys.stderr) print( 'Type "exit" or press "ctrl-d" again to force quit.', file=sys.stderr, ) jobs_clean = False _last_exit_time = time.time() else: kill_all_jobs() return jobs_clean def kill_all_jobs(): """ Send SIGKILL to all child processes (called when exiting xonsh). """ _clear_dead_jobs() for job in builtins.__xonsh__.all_jobs.values(): _kill(job) def jobs(args, stdin=None, stdout=sys.stdout, stderr=None): """ xonsh command: jobs Display a list of all current jobs. """ _clear_dead_jobs() for j in tasks: print_one_job(j, outfile=stdout) return None, None def resume_job(args, wording): """ used by fg and bg to resume a job either in the foreground or in the background. """ _clear_dead_jobs() if len(tasks) == 0: return "", "There are currently no suspended jobs" if len(args) == 0: tid = tasks[0] # take the last manipulated task by default elif len(args) == 1: try: if args[0] == "+": # take the last manipulated task tid = tasks[0] elif args[0] == "-": # take the second to last manipulated task tid = tasks[1] else: tid = int(args[0]) except (ValueError, IndexError): return "", "Invalid job: {}\n".format(args[0]) if tid not in builtins.__xonsh__.all_jobs: return "", "Invalid job: {}\n".format(args[0]) else: return "", "{} expects 0 or 1 arguments, not {}\n".format(wording, len(args)) # Put this one on top of the queue tasks.remove(tid) tasks.appendleft(tid) job = get_task(tid) job["bg"] = False job["status"] = "running" if builtins.__xonsh__.env.get("XONSH_INTERACTIVE"): print_one_job(tid) pipeline = job["pipeline"] pipeline.resume(job) @unthreadable def fg(args, stdin=None): """ xonsh command: fg Bring the currently active job to the foreground, or, if a single number is given as an argument, bring that job to the foreground. Additionally, specify "+" for the most recent job and "-" for the second most recent job. """ return resume_job(args, wording="fg") def bg(args, stdin=None): """xonsh command: bg Resume execution of the currently active job in the background, or, if a single number is given as an argument, resume that job in the background. """ res = resume_job(args, wording="bg") if res is None: curtask = get_task(tasks[0]) curtask["bg"] = True _continue(curtask) else: return res ================================================ FILE: xonsh/jsonutils.py ================================================ """Custom tools for managing JSON serialization / deserialization of xonsh objects. """ import functools from xonsh.tools import EnvPath @functools.singledispatch def serialize_xonsh_json(val): """JSON serializer for xonsh custom data structures. This is only called when another normal JSON types are not found. """ return str(val) @serialize_xonsh_json.register(EnvPath) def _serialize_xonsh_json_env_path(val): return val.paths ================================================ FILE: xonsh/jupyter_kernel.py ================================================ # -*- coding: utf-8 -*- """Hooks for Jupyter Xonsh Kernel.""" import sys import json import hmac import uuid import errno import hashlib import datetime import builtins import threading from pprint import pformat from argparse import ArgumentParser from collections.abc import Set import zmq from zmq.eventloop import ioloop, zmqstream from zmq.error import ZMQError from xonsh import __version__ as version from xonsh.main import setup from xonsh.completer import Completer from xonsh.commands_cache import predict_true MAX_SIZE = 8388608 # 8 Mb DELIM = b"" def dump_bytes(*args, **kwargs): """Converts an object to JSON and returns the bytes.""" return json.dumps(*args, **kwargs).encode("ascii") def load_bytes(b): """Converts bytes of JSON to an object.""" return json.loads(b.decode("ascii")) def bind(socket, connection, port): """Binds a socket to a port, or a random port if needed. Returns the port.""" if port <= 0: return socket.bind_to_random_port(connection) else: socket.bind("{}:{}".format(connection, port)) return port class XonshKernel: """Xonsh xernal for Jupyter""" implementation = "Xonsh " + version implementation_version = version language = "xonsh" language_version = version.split(".")[:3] banner = "Xonsh - Python-powered, cross-platform shell" language_info = { "name": "xonsh", "version": version, "pygments_lexer": "xonsh", "codemirror_mode": "shell", "mimetype": "text/x-sh", "file_extension": ".xsh", } signature_schemes = {"hmac-sha256": hashlib.sha256} def __init__(self, debug_level=0, session_id=None, config=None, **kwargs): """ Parameters ---------- debug_level : int, optional Integer from 0 (no debugging) to 3 (all debugging), default: 0. session_id : str or None, optional Unique string id representing the kernel session. If None, this will be replaced with a random UUID. config : dict or None, optional Configuration dictionary to start server with. BY default will search the command line for options (if given) or use default configuration. """ self.debug_level = debug_level self.session_id = str(uuid.uuid4()) if session_id is None else session_id self._parser = None self.config = self.make_default_config() if config is None else config self.exiting = False self.execution_count = 1 self.completer = Completer() @property def parser(self): if self._parser is None: p = ArgumentParser("jupyter_kerenel") p.add_argument("-f", dest="config_file", default=None) self._parser = p return self._parser def make_default_config(self): """Provides default configuration""" ns, unknown = self.parser.parse_known_args(sys.argv) if ns.config_file is None: self.dprint(1, "Starting xonsh kernel with default args...") config = { "control_port": 0, "hb_port": 0, "iopub_port": 0, "ip": "127.0.0.1", "key": str(uuid.uuid4()), "shell_port": 0, "signature_scheme": "hmac-sha256", "stdin_port": 0, "transport": "tcp", } else: self.dprint(1, "Loading simple_kernel with args:", sys.argv) self.dprint(1, "Reading config file {!r}...".format(ns.config_file)) with open(ns.config_file) as f: config = json.load(f) return config def iopub_handler(self, message): """Handles iopub requests.""" self.dprint(2, "iopub received:", message) def control_handler(self, wire_message): """Handles control requests""" self.dprint(1, "control received:", wire_message) identities, msg = self.deserialize_wire_message(wire_message) if msg["header"]["msg_type"] == "shutdown_request": self.shutdown() def stdin_handler(self, message): self.dprint(2, "stdin received:", message) def start(self): """Starts the server""" ioloop.install() connection = self.config["transport"] + "://" + self.config["ip"] secure_key = self.config["key"].encode() digestmod = self.signature_schemes[self.config["signature_scheme"]] self.auth = hmac.HMAC(secure_key, digestmod=digestmod) # Heartbeat ctx = zmq.Context() self.heartbeat_socket = ctx.socket(zmq.REP) self.config["hb_port"] = bind( self.heartbeat_socket, connection, self.config["hb_port"] ) # IOPub/Sub, aslo called SubSocketChannel in IPython sources self.iopub_socket = ctx.socket(zmq.PUB) self.config["iopub_port"] = bind( self.iopub_socket, connection, self.config["iopub_port"] ) self.iopub_stream = zmqstream.ZMQStream(self.iopub_socket) self.iopub_stream.on_recv(self.iopub_handler) # Control self.control_socket = ctx.socket(zmq.ROUTER) self.config["control_port"] = bind( self.control_socket, connection, self.config["control_port"] ) self.control_stream = zmqstream.ZMQStream(self.control_socket) self.control_stream.on_recv(self.control_handler) # Stdin: self.stdin_socket = ctx.socket(zmq.ROUTER) self.config["stdin_port"] = bind( self.stdin_socket, connection, self.config["stdin_port"] ) self.stdin_stream = zmqstream.ZMQStream(self.stdin_socket) self.stdin_stream.on_recv(self.stdin_handler) # Shell self.shell_socket = ctx.socket(zmq.ROUTER) self.config["shell_port"] = bind( self.shell_socket, connection, self.config["shell_port"] ) self.shell_stream = zmqstream.ZMQStream(self.shell_socket) self.shell_stream.on_recv(self.shell_handler) # start up configurtation self.dprint(2, "Config:", json.dumps(self.config)) self.dprint(1, "Starting loops...") self.hb_thread = threading.Thread(target=self.heartbeat_loop) self.hb_thread.daemon = True self.hb_thread.start() self.dprint(1, "Ready! Listening...") ioloop.IOLoop.instance().start() def shutdown(self): """Shutsdown the kernel""" self.exiting = True ioloop.IOLoop.instance().stop() def dprint(self, level, *args, **kwargs): """Print but with debug information.""" if level <= self.debug_level: print("DEBUG" + str(level) + ":", file=sys.__stdout__, *args, **kwargs) sys.__stdout__.flush() def sign(self, messages): """Sign a message list with a secure signature.""" h = self.auth.copy() for m in messages: h.update(m) return h.hexdigest().encode("ascii") def new_header(self, message_type): """Make a new header""" return { "date": datetime.datetime.now().isoformat(), "msg_id": str(uuid.uuid4()), "username": "kernel", "session": self.session_id, "msg_type": message_type, "version": "5.0", } def send( self, stream, message_type, content=None, parent_header=None, metadata=None, identities=None, ): """Send data to the client via a stream""" header = self.new_header(message_type) if content is None: content = {} if parent_header is None: parent_header = {} if metadata is None: metadata = {} messages = list(map(dump_bytes, [header, parent_header, metadata, content])) signature = self.sign(messages) parts = [DELIM, signature] + messages if identities: parts = identities + parts self.dprint(3, "send parts:", parts) stream.send_multipart(parts) if isinstance(stream, zmqstream.ZMQStream): stream.flush() def deserialize_wire_message(self, wire_message): """Split the routing prefix and message frames from a message on the wire""" delim_idx = wire_message.index(DELIM) identities = wire_message[:delim_idx] m_signature = wire_message[delim_idx + 1] msg_frames = wire_message[delim_idx + 2 :] keys = ("header", "parent_header", "metadata", "content") m = {k: load_bytes(v) for k, v in zip(keys, msg_frames)} check_sig = self.sign(msg_frames) if check_sig != m_signature: raise ValueError("Signatures do not match") return identities, m def run_thread(self, loop, name): """Run main thread""" self.dprint(2, "Starting loop for {name!r}...".format(name=name)) while not self.exiting: self.dprint(2, "{} Loop!".format(name)) try: loop.start() except ZMQError as e: self.dprint(1, "{} ZMQError!\n {}".format(name, e)) if e.errno == errno.EINTR: continue else: raise except Exception: self.dprint(2, "{} Exception!".format(name)) if self.exiting: break else: raise else: self.dprint(2, "{} Break!".format(name)) break def heartbeat_loop(self): """Run heartbeat""" self.dprint(2, "Starting heartbeat loop...") while not self.exiting: self.dprint(3, ".", end="") try: zmq.device(zmq.FORWARDER, self.heartbeat_socket, self.heartbeat_socket) except zmq.ZMQError as e: if e.errno == errno.EINTR: continue else: raise else: break def shell_handler(self, message): """Dispatch shell messages to their handlers""" self.dprint(1, "received:", message) identities, msg = self.deserialize_wire_message(message) handler = getattr(self, "handle_" + msg["header"]["msg_type"], None) if handler is None: self.dprint(0, "unknown message type:", msg["header"]["msg_type"]) return handler(msg, identities) def handle_execute_request(self, message, identities): """Handle execute request messages.""" self.dprint(2, "Xonsh Kernel Executing:", pformat(message["content"]["code"])) # Start by sending busy signal content = {"execution_state": "busy"} self.send(self.iopub_stream, "status", content, parent_header=message["header"]) # confirm the input that we are executing content = { "execution_count": self.execution_count, "code": message["content"]["code"], } self.send( self.iopub_stream, "execute_input", content, parent_header=message["header"] ) # execute the code metadata = { "dependencies_met": True, "engine": self.session_id, "status": "ok", "started": datetime.datetime.now().isoformat(), } content = self.do_execute(parent_header=message["header"], **message["content"]) self.send( self.shell_stream, "execute_reply", content, metadata=metadata, parent_header=message["header"], identities=identities, ) self.execution_count += 1 # once we are done, send a signal that we are idle content = {"execution_state": "idle"} self.send(self.iopub_stream, "status", content, parent_header=message["header"]) def do_execute( self, code="", silent=False, store_history=True, user_expressions=None, allow_stdin=False, parent_header=None, **kwargs ): """Execute user code.""" if len(code.strip()) == 0: return { "status": "ok", "execution_count": self.execution_count, "payload": [], "user_expressions": {}, } shell = builtins.__xonsh__.shell hist = builtins.__xonsh__.history try: shell.default(code, self, parent_header) interrupted = False except KeyboardInterrupt: interrupted = True if interrupted: return {"status": "abort", "execution_count": self.execution_count} rtn = 0 if (hist is None or len(hist) == 0) else hist.rtns[-1] if 0 < rtn: message = { "status": "error", "execution_count": self.execution_count, "ename": "", "evalue": str(rtn), "traceback": [], } else: message = { "status": "ok", "execution_count": self.execution_count, "payload": [], "user_expressions": {}, } return message def _respond_in_chunks(self, name, s, chunksize=1024, parent_header=None): if s is None: return n = len(s) if n == 0: return lower = range(0, n, chunksize) upper = range(chunksize, n + chunksize, chunksize) for l, u in zip(lower, upper): response = {"name": name, "text": s[l:u]} self.send( self.iopub_socket, "stream", response, parent_header=parent_header ) def handle_complete_request(self, message, identities): """Handles kernel info requests.""" content = self.do_complete( message["content"]["code"], message["content"]["cursor_pos"] ) self.send( self.shell_stream, "complete_reply", content, parent_header=message["header"], identities=identities, ) def do_complete(self, code, pos): """Get completions.""" shell = builtins.__xonsh__.shell line = code.split("\n")[-1] line = builtins.aliases.expand_alias(line) prefix = line.split(" ")[-1] endidx = pos begidx = pos - len(prefix) rtn, _ = self.completer.complete(prefix, line, begidx, endidx, shell.ctx) if isinstance(rtn, Set): rtn = list(rtn) message = { "matches": rtn, "cursor_start": begidx, "cursor_end": endidx, "metadata": {}, "status": "ok", } return message def handle_kernel_info_request(self, message, identities): """Handles kernel info requests.""" content = { "protocol_version": "5.0", "ipython_version": [1, 1, 0, ""], "language": self.language, "language_version": self.language_version, "implementation": self.implementation, "implementation_version": self.implementation_version, "language_info": self.language_info, "banner": self.banner, } self.send( self.shell_stream, "kernel_info_reply", content, parent_header=message["header"], identities=identities, ) if __name__ == "__main__": setup( shell_type="jupyter", env={"PAGER": "cat"}, aliases={"less": "cat"}, xontribs=["coreutils"], threadable_predictors={"git": predict_true, "man": predict_true}, ) if builtins.__xonsh__.commands_cache.is_only_functional_alias("cat"): # this is needed if the underlying system doesn't have cat # we supply our own, because we can builtins.aliases["cat"] = "xonsh-cat" builtins.__xonsh__.env["PAGER"] = "xonsh-cat" shell = builtins.__xonsh__.shell kernel = shell.kernel = XonshKernel() kernel.start() ================================================ FILE: xonsh/jupyter_shell.py ================================================ """An interactive shell for the Jupyter kernel.""" import io import sys import builtins from xonsh.base_shell import BaseShell class StdJupyterRedirectBuf(io.RawIOBase): """Redirects standard I/O buffers to the Jupyter kernel.""" def __init__(self, redirect): self.redirect = redirect self.encoding = redirect.encoding self.errors = redirect.errors def fileno(self): """Returns the file descriptor of the std buffer.""" return self.redirect.fileno() def seek(self, offset, whence=io.SEEK_SET): """Sets the location in both the stdbuf and the membuf.""" raise io.UnsupportedOperation("cannot seek Jupyter redirect") def truncate(self, size=None): """Truncate both buffers.""" raise io.UnsupportedOperation("cannot truncate Jupyter redirect") def readinto(self, b): """Read bytes into buffer from both streams.""" raise io.UnsupportedOperation("cannot read into Jupyter redirect") def write(self, b): """Write bytes to kernel.""" s = b if isinstance(b, str) else b.decode(self.encoding, self.errors) self.redirect.write(s) class StdJupyterRedirect(io.TextIOBase): """Redirects a standard I/O stream to the Jupyter kernel.""" def __init__(self, name, kernel, parent_header=None): """ Parameters ---------- name : str The name of the buffer in the sys module, e.g. 'stdout'. kernel : XonshKernel Instance of a Jupyter kernel parent_header : dict or None, optional parent header information to pass along with the kernel """ self._name = name self.kernel = kernel self.parent_header = parent_header self.std = getattr(sys, name) self.buffer = StdJupyterRedirectBuf(self) setattr(sys, name, self) @property def encoding(self): """The encoding of the stream""" env = builtins.__xonsh__.env return getattr(self.std, "encoding", env.get("XONSH_ENCODING")) @property def errors(self): """The encoding errors of the stream""" env = builtins.__xonsh__.env return getattr(self.std, "errors", env.get("XONSH_ENCODING_ERRORS")) @property def newlines(self): """The newlines of the standard buffer.""" return self.std.newlines def _replace_std(self): std = self.std if std is None: return setattr(sys, self._name, std) self.std = None def __del__(self): self._replace_std() def close(self): """Restores the original std stream.""" self._replace_std() def __enter__(self): return self def __exit__(self, *args, **kwargs): self.close() def write(self, s): """Writes data to the original kernel stream.""" self.kernel._respond_in_chunks(self._name, s, parent_header=self.parent_header) def flush(self): """Flushes kernel iopub_stream.""" self.kernel.iopub_stream.flush() def fileno(self): """Tunnel fileno() calls to the std stream.""" return self.std.fileno() def seek(self, offset, whence=io.SEEK_SET): """Seek to a location.""" raise io.UnsupportedOperation("cannot seek Jupyter redirect") def truncate(self, size=None): """Truncate the streams.""" raise io.UnsupportedOperation("cannot truncate Jupyter redirect") def detach(self): """This operation is not supported.""" raise io.UnsupportedOperation("cannot detach a Jupyter redirect") def read(self, size=None): """Read from the stream""" raise io.UnsupportedOperation("cannot read a Jupyter redirect") def readline(self, size=-1): """Read a line.""" raise io.UnsupportedOperation("cannot read a line from a Jupyter redirect") class JupyterShell(BaseShell): """A shell for the Jupyter kernel.""" def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.kernel = None def default(self, line, kernel, parent_header=None): """Executes code, but redirects output to Jupyter client""" stdout = StdJupyterRedirect("stdout", kernel, parent_header) stderr = StdJupyterRedirect("stderr", kernel, parent_header) with stdout, stderr: rtn = super().default(line) return rtn ================================================ FILE: xonsh/lazyasd.py ================================================ """Lazy and self destructive containers for speeding up module import.""" # Copyright 2015-2016, the xonsh developers. All rights reserved. import os import sys import time import types import builtins import threading import importlib import importlib.util import collections.abc as cabc __version__ = "0.1.3" class LazyObject(object): def __init__(self, load, ctx, name): """Lazily loads an object via the load function the first time an attribute is accessed. Once loaded it will replace itself in the provided context (typically the globals of the call site) with the given name. For example, you can prevent the compilation of a regular expression until it is actually used:: DOT = LazyObject((lambda: re.compile('.')), globals(), 'DOT') Parameters ---------- load : function with no arguments A loader function that performs the actual object construction. ctx : Mapping Context to replace the LazyObject instance in with the object returned by load(). name : str Name in the context to give the loaded object. This *should* be the name on the LHS of the assignment. """ self._lasdo = {"loaded": False, "load": load, "ctx": ctx, "name": name} def _lazy_obj(self): d = self._lasdo if d["loaded"]: obj = d["obj"] else: obj = d["load"]() d["ctx"][d["name"]] = d["obj"] = obj d["loaded"] = True return obj def __getattribute__(self, name): if name == "_lasdo" or name == "_lazy_obj": return super().__getattribute__(name) obj = self._lazy_obj() return getattr(obj, name) def __bool__(self): obj = self._lazy_obj() return bool(obj) def __iter__(self): obj = self._lazy_obj() yield from obj def __getitem__(self, item): obj = self._lazy_obj() return obj[item] def __setitem__(self, key, value): obj = self._lazy_obj() obj[key] = value def __delitem__(self, item): obj = self._lazy_obj() del obj[item] def __call__(self, *args, **kwargs): obj = self._lazy_obj() return obj(*args, **kwargs) def __lt__(self, other): obj = self._lazy_obj() return obj < other def __le__(self, other): obj = self._lazy_obj() return obj <= other def __eq__(self, other): obj = self._lazy_obj() return obj == other def __ne__(self, other): obj = self._lazy_obj() return obj != other def __gt__(self, other): obj = self._lazy_obj() return obj > other def __ge__(self, other): obj = self._lazy_obj() return obj >= other def __hash__(self): obj = self._lazy_obj() return hash(obj) def __or__(self, other): obj = self._lazy_obj() return obj | other def __str__(self): return str(self._lazy_obj()) def __repr__(self): return repr(self._lazy_obj()) def lazyobject(f): """Decorator for constructing lazy objects from a function.""" return LazyObject(f, f.__globals__, f.__name__) class LazyDict(cabc.MutableMapping): def __init__(self, loaders, ctx, name): """Dictionary like object that lazily loads its values from an initial dict of key-loader function pairs. Each key is loaded when its value is first accessed. Once fully loaded, this object will replace itself in the provided context (typically the globals of the call site) with the given name. For example, you can prevent the compilation of a bunch of regular expressions until they are actually used:: RES = LazyDict({ 'dot': lambda: re.compile('.'), 'all': lambda: re.compile('.*'), 'two': lambda: re.compile('..'), }, globals(), 'RES') Parameters ---------- loaders : Mapping of keys to functions with no arguments A mapping of loader function that performs the actual value construction upon access. ctx : Mapping Context to replace the LazyDict instance in with the the fully loaded mapping. name : str Name in the context to give the loaded mapping. This *should* be the name on the LHS of the assignment. """ self._loaders = loaders self._ctx = ctx self._name = name self._d = type(loaders)() # make sure to return the same type def _destruct(self): if len(self._loaders) == 0: self._ctx[self._name] = self._d def __getitem__(self, key): d = self._d if key in d: val = d[key] else: # pop will raise a key error for us loader = self._loaders.pop(key) d[key] = val = loader() self._destruct() return val def __setitem__(self, key, value): self._d[key] = value if key in self._loaders: del self._loaders[key] self._destruct() def __delitem__(self, key): if key in self._d: del self._d[key] else: del self._loaders[key] self._destruct() def __iter__(self): yield from (set(self._d.keys()) | set(self._loaders.keys())) def __len__(self): return len(self._d) + len(self._loaders) def lazydict(f): """Decorator for constructing lazy dicts from a function.""" return LazyDict(f, f.__globals__, f.__name__) class LazyBool(object): def __init__(self, load, ctx, name): """Boolean like object that lazily computes it boolean value when it is first asked. Once loaded, this result will replace itself in the provided context (typically the globals of the call site) with the given name. For example, you can prevent the complex boolean until it is actually used:: ALIVE = LazyDict(lambda: not DEAD, globals(), 'ALIVE') Parameters ---------- load : function with no arguments A loader function that performs the actual boolean evaluation. ctx : Mapping Context to replace the LazyBool instance in with the the fully loaded mapping. name : str Name in the context to give the loaded mapping. This *should* be the name on the LHS of the assignment. """ self._load = load self._ctx = ctx self._name = name self._result = None def __bool__(self): if self._result is None: res = self._ctx[self._name] = self._result = self._load() else: res = self._result return res def lazybool(f): """Decorator for constructing lazy booleans from a function.""" return LazyBool(f, f.__globals__, f.__name__) # # Background module loaders # class BackgroundModuleProxy(types.ModuleType): """Proxy object for modules loaded in the background that block attribute access until the module is loaded.. """ def __init__(self, modname): self.__dct__ = {"loaded": False, "modname": modname} def __getattribute__(self, name): passthrough = frozenset({"__dct__", "__class__", "__spec__"}) if name in passthrough: return super().__getattribute__(name) dct = self.__dct__ modname = dct["modname"] if dct["loaded"]: mod = sys.modules[modname] else: delay_types = (BackgroundModuleProxy, type(None)) while isinstance(sys.modules.get(modname, None), delay_types): time.sleep(0.001) mod = sys.modules[modname] dct["loaded"] = True # some modules may do construction after import, give them a second stall = 0 while not hasattr(mod, name) and stall < 1000: stall += 1 time.sleep(0.001) return getattr(mod, name) class BackgroundModuleLoader(threading.Thread): """Thread to load modules in the background.""" def __init__(self, name, package, replacements, *args, **kwargs): super().__init__(*args, **kwargs) self.daemon = True self.name = name self.package = package self.replacements = replacements self.start() def run(self): # wait for other modules to stop being imported # We assume that module loading is finished when sys.modules doesn't # get longer in 5 consecutive 1ms waiting steps counter = 0 last = -1 while counter < 5: new = len(sys.modules) if new == last: counter += 1 else: last = new counter = 0 time.sleep(0.001) # now import module properly modname = importlib.util.resolve_name(self.name, self.package) if isinstance(sys.modules[modname], BackgroundModuleProxy): del sys.modules[modname] mod = importlib.import_module(self.name, package=self.package) for targname, varname in self.replacements.items(): if targname in sys.modules: targmod = sys.modules[targname] setattr(targmod, varname, mod) def load_module_in_background( name, package=None, debug="DEBUG", env=None, replacements=None ): """Entry point for loading modules in background thread. Parameters ---------- name : str Module name to load in background thread. package : str or None, optional Package name, has the same meaning as in importlib.import_module(). debug : str, optional Debugging symbol name to look up in the environment. env : Mapping or None, optional Environment this will default to __xonsh__.env, if available, and os.environ otherwise. replacements : Mapping or None, optional Dictionary mapping fully qualified module names (eg foo.bar.baz) that import the lazily loaded module, with the variable name in that module. For example, suppose that foo.bar imports module a as b, this dict is then {'foo.bar': 'b'}. Returns ------- module : ModuleType This is either the original module that is found in sys.modules or a proxy module that will block until delay attribute access until the module is fully loaded. """ modname = importlib.util.resolve_name(name, package) if modname in sys.modules: return sys.modules[modname] if env is None: xonsh_obj = getattr(builtins, "__xonsh__", None) env = os.environ if xonsh_obj is None else getattr(xonsh_obj, "env", os.environ) if env.get(debug, None): mod = importlib.import_module(name, package=package) return mod proxy = sys.modules[modname] = BackgroundModuleProxy(modname) BackgroundModuleLoader(name, package, replacements or {}) return proxy ================================================ FILE: xonsh/lazyimps.py ================================================ """Lazy imports that may apply across the xonsh package.""" import importlib from xonsh.platform import ON_WINDOWS, ON_DARWIN from xonsh.lazyasd import LazyObject, lazyobject pygments = LazyObject( lambda: importlib.import_module("pygments"), globals(), "pygments" ) pyghooks = LazyObject( lambda: importlib.import_module("xonsh.pyghooks"), globals(), "pyghooks" ) @lazyobject def pty(): if ON_WINDOWS: return else: return importlib.import_module("pty") @lazyobject def termios(): if ON_WINDOWS: return else: return importlib.import_module("termios") @lazyobject def fcntl(): if ON_WINDOWS: return else: return importlib.import_module("fcntl") @lazyobject def tty(): if ON_WINDOWS: return else: return importlib.import_module("tty") @lazyobject def _winapi(): if ON_WINDOWS: import _winapi as m else: m = None return m @lazyobject def msvcrt(): if ON_WINDOWS: import msvcrt as m else: m = None return m @lazyobject def winutils(): if ON_WINDOWS: import xonsh.winutils as m else: m = None return m @lazyobject def macutils(): if ON_DARWIN: import xonsh.macutils as m else: m = None return m @lazyobject def terminal256(): return importlib.import_module("pygments.formatters.terminal256") ================================================ FILE: xonsh/lazyjson.py ================================================ # -*- coding: utf-8 -*- """Implements a lazy JSON file class that wraps around json data.""" import io import json import weakref import contextlib import collections.abc as cabc def _to_json_with_size(obj, offset=0, sort_keys=False): if isinstance(obj, str): s = json.dumps(obj) o = offset n = size = len(s.encode()) # size in bytes elif isinstance(obj, cabc.Mapping): s = "{" j = offset + 1 o = {} size = {} items = sorted(obj.items()) if sort_keys else obj.items() for key, val in items: s_k, o_k, n_k, size_k = _to_json_with_size( key, offset=j, sort_keys=sort_keys ) s += s_k + ": " j += n_k + 2 s_v, o_v, n_v, size_v = _to_json_with_size( val, offset=j, sort_keys=sort_keys ) o[key] = o_v size[key] = size_v s += s_v + ", " j += n_v + 2 if s.endswith(", "): s = s[:-2] s += "}\n" n = len(s) o["__total__"] = offset size["__total__"] = n elif isinstance(obj, cabc.Sequence): s = "[" j = offset + 1 o = [] size = [] for x in obj: s_x, o_x, n_x, size_x = _to_json_with_size(x, offset=j, sort_keys=sort_keys) o.append(o_x) size.append(size_x) s += s_x + ", " j += n_x + 2 if s.endswith(", "): s = s[:-2] s += "]\n" n = len(s) o.append(offset) size.append(n) else: s = json.dumps(obj, sort_keys=sort_keys) o = offset n = size = len(s) return s, o, n, size def index(obj, sort_keys=False): """Creates an index for a JSON file.""" idx = {} json_obj = _to_json_with_size(obj, sort_keys=sort_keys) s, idx["offsets"], _, idx["sizes"] = json_obj return s, idx JSON_FORMAT = """{{"locs": [{iloc:>10}, {ilen:>10}, {dloc:>10}, {dlen:>10}], "index": {index}, "data": {data} }} """ def dumps(obj, sort_keys=False): """Dumps an object to JSON with an index.""" data, idx = index(obj, sort_keys=sort_keys) jdx = json.dumps(idx, sort_keys=sort_keys) iloc = 69 ilen = len(jdx) dloc = iloc + ilen + 11 dlen = len(data) s = JSON_FORMAT.format( index=jdx, data=data, iloc=iloc, ilen=ilen, dloc=dloc, dlen=dlen ) return s def ljdump(obj, fp, sort_keys=False): """Dumps an object to JSON file.""" s = dumps(obj, sort_keys=sort_keys) fp.write(s) class LJNode(cabc.Mapping, cabc.Sequence): """A proxy node for JSON nodes. Acts as both sequence and mapping.""" def __init__(self, offsets, sizes, root): """Parameters ---------- offsets : dict, list, or int offsets of corresponding data structure, in bytes sizes : dict, list, or int sizes of corresponding data structure, in bytes root : weakref.proxy of LazyJSON weakref back to root node, which should be a LazyJSON object. """ self.offsets = offsets self.sizes = sizes self.root = root self.is_mapping = isinstance(self.offsets, cabc.Mapping) self.is_sequence = isinstance(self.offsets, cabc.Sequence) def __len__(self): # recall that for maps, the '__total__' key is added and for # sequences the last element represents the total size/offset. return len(self.sizes) - 1 def load(self): """Returns the Python data structure represented by the node.""" if self.is_mapping: offset = self.offsets["__total__"] size = self.sizes["__total__"] elif self.is_sequence: offset = self.offsets[-1] size = self.sizes[-1] elif isinstance(self.offsets, int): offset = self.offsets size = self.sizes return self._load_or_node(offset, size) def _load_or_node(self, offset, size): if isinstance(offset, int): with self.root._open(newline="\n") as f: f.seek(self.root.dloc + offset) s = f.read(size) val = json.loads(s) elif isinstance(offset, (cabc.Mapping, cabc.Sequence)): val = LJNode(offset, size, self.root) else: raise TypeError("incorrect types for offset node") return val def _getitem_mapping(self, key): if key == "__total__": raise KeyError('"__total__" is a special LazyJSON key!') offset = self.offsets[key] size = self.sizes[key] return self._load_or_node(offset, size) def _getitem_sequence(self, key): if isinstance(key, int): rtn = self._load_or_node(self.offsets[key], self.sizes[key]) elif isinstance(key, slice): key = slice(*key.indices(len(self))) rtn = list(map(self._load_or_node, self.offsets[key], self.sizes[key])) else: raise TypeError("only integer indexing available") return rtn def __getitem__(self, key): if self.is_mapping: rtn = self._getitem_mapping(key) elif self.is_sequence: rtn = self._getitem_sequence(key) else: raise NotImplementedError return rtn def __iter__(self): if self.is_mapping: keys = set(self.offsets.keys()) keys.discard("__total__") yield from iter(keys) elif self.is_sequence: i = 0 n = len(self) while i < n: yield self._load_or_node(self.offsets[i], self.sizes[i]) i += 1 else: raise NotImplementedError class LazyJSON(LJNode): """Represents a lazy json file. Can be used like a normal Python dict or list. """ def __init__(self, f, reopen=True): """Parameters ---------- f : file handle or str JSON file to open. reopen : bool, optional Whether new file handle should be opened for each load. """ self._f = f self.reopen = reopen if not reopen and isinstance(f, str): self._f = open(f, "r", newline="\n") self._load_index() self.root = weakref.proxy(self) self.is_mapping = isinstance(self.offsets, cabc.Mapping) self.is_sequence = isinstance(self.offsets, cabc.Sequence) def __del__(self): self.close() def close(self): """Close the file handle, if appropriate.""" if not self.reopen and isinstance(self._f, io.IOBase): try: self._f.close() except OSError: pass @contextlib.contextmanager def _open(self, *args, **kwargs): if self.reopen and isinstance(self._f, str): f = open(self._f, *args, **kwargs) yield f f.close() else: yield self._f def _load_index(self): """Loads the index from the start of the file.""" with self._open(newline="\n") as f: # read in the location data f.seek(9) locs = f.read(48) locs = json.loads(locs) self.iloc, self.ilen, self.dloc, self.dlen = locs # read in the index f.seek(self.iloc) idx = f.read(self.ilen) idx = json.loads(idx) self.offsets = idx["offsets"] self.sizes = idx["sizes"] def __enter__(self): return self def __exit__(self, exc_type, exc_value, traceback): self.close() ================================================ FILE: xonsh/lexer.py ================================================ # -*- coding: utf-8 -*- """Lexer for xonsh code. Written using a hybrid of ``tokenize`` and PLY. """ import io import re # 'keyword' interferes with ast.keyword import keyword as kwmod from xonsh.ply.ply.lex import LexToken from xonsh.lazyasd import lazyobject from xonsh.platform import PYTHON_VERSION_INFO from xonsh.tokenize import ( OP, IOREDIRECT, STRING, DOLLARNAME, NUMBER, SEARCHPATH, NEWLINE, INDENT, DEDENT, NL, COMMENT, ENCODING, ENDMARKER, NAME, ERRORTOKEN, GREATER, LESS, RIGHTSHIFT, tokenize, TokenError, ) @lazyobject def token_map(): """Mapping from ``tokenize`` tokens (or token types) to PLY token types. If a simple one-to-one mapping from ``tokenize`` to PLY exists, the lexer will look it up here and generate a single PLY token of the given type. Otherwise, it will fall back to handling that token using one of the handlers in``special_handlers``. """ tm = {} # operators _op_map = { # punctuation ",": "COMMA", ".": "PERIOD", ";": "SEMI", ":": "COLON", "...": "ELLIPSIS", # basic operators "+": "PLUS", "-": "MINUS", "*": "TIMES", "@": "AT", "/": "DIVIDE", "//": "DOUBLEDIV", "%": "MOD", "**": "POW", "|": "PIPE", "~": "TILDE", "^": "XOR", "<<": "LSHIFT", ">>": "RSHIFT", "<": "LT", "<=": "LE", ">": "GT", ">=": "GE", "==": "EQ", "!=": "NE", "->": "RARROW", # assignment operators "=": "EQUALS", "+=": "PLUSEQUAL", "-=": "MINUSEQUAL", "*=": "TIMESEQUAL", "@=": "ATEQUAL", "/=": "DIVEQUAL", "%=": "MODEQUAL", "**=": "POWEQUAL", "<<=": "LSHIFTEQUAL", ">>=": "RSHIFTEQUAL", "&=": "AMPERSANDEQUAL", "^=": "XOREQUAL", "|=": "PIPEEQUAL", "//=": "DOUBLEDIVEQUAL", # extra xonsh operators "?": "QUESTION", "??": "DOUBLE_QUESTION", "@$": "ATDOLLAR", "&": "AMPERSAND", } for (op, typ) in _op_map.items(): tm[(OP, op)] = typ tm[IOREDIRECT] = "IOREDIRECT" tm[STRING] = "STRING" tm[DOLLARNAME] = "DOLLAR_NAME" tm[NUMBER] = "NUMBER" tm[SEARCHPATH] = "SEARCHPATH" tm[NEWLINE] = "NEWLINE" tm[INDENT] = "INDENT" tm[DEDENT] = "DEDENT" if (3, 5, 0) <= PYTHON_VERSION_INFO < (3, 7, 0): from xonsh.tokenize import ASYNC, AWAIT tm[ASYNC] = "ASYNC" tm[AWAIT] = "AWAIT" return tm NEED_WHITESPACE = frozenset(["and", "or"]) @lazyobject def RE_NEED_WHITESPACE(): pattern = r"\s?(" + "|".join(NEED_WHITESPACE) + r")(\s|[\\]$)" return re.compile(pattern) def handle_name(state, token): """Function for handling name tokens""" typ = "NAME" state["last"] = token needs_whitespace = token.string in NEED_WHITESPACE has_whitespace = needs_whitespace and RE_NEED_WHITESPACE.match( token.line[max(0, token.start[1] - 1) :] ) if state["pymode"][-1][0]: if needs_whitespace and not has_whitespace: pass elif token.string in kwmod.kwlist: typ = token.string.upper() yield _new_token(typ, token.string, token.start) else: if has_whitespace and token.string == "and": yield _new_token("AND", token.string, token.start) elif has_whitespace and token.string == "or": yield _new_token("OR", token.string, token.start) else: yield _new_token("NAME", token.string, token.start) def _end_delimiter(state, token): py = state["pymode"] s = token.string l, c = token.start if len(py) > 1: mode, orig, match, pos = py.pop() if s != match: e = '"{}" at {} ends "{}" at {} (expected "{}")' return e.format(s, (l, c), orig, pos, match) else: return 'Unmatched "{}" at line {}, column {}'.format(s, l, c) def handle_rparen(state, token): """ Function for handling ``)`` """ e = _end_delimiter(state, token) if e is None: state["last"] = token yield _new_token("RPAREN", ")", token.start) else: yield _new_token("ERRORTOKEN", e, token.start) def handle_rbrace(state, token): """Function for handling ``}``""" e = _end_delimiter(state, token) if e is None: state["last"] = token yield _new_token("RBRACE", "}", token.start) else: yield _new_token("ERRORTOKEN", e, token.start) def handle_rbracket(state, token): """ Function for handling ``]`` """ e = _end_delimiter(state, token) if e is None: state["last"] = token yield _new_token("RBRACKET", "]", token.start) else: yield _new_token("ERRORTOKEN", e, token.start) def handle_error_space(state, token): """ Function for handling special whitespace characters in subprocess mode """ if not state["pymode"][-1][0]: state["last"] = token yield _new_token("WS", token.string, token.start) else: yield from [] def handle_error_linecont(state, token): """Function for handling special line continuations as whitespace characters in subprocess mode. """ if state["pymode"][-1][0]: return prev = state["last"] if prev.end != token.start: return # previous token is separated by whitespace state["last"] = token yield _new_token("WS", "\\", token.start) def handle_error_token(state, token): """ Function for handling error tokens """ state["last"] = token if token.string == "!": typ = "BANG" elif not state["pymode"][-1][0]: typ = "NAME" else: typ = "ERRORTOKEN" yield _new_token(typ, token.string, token.start) def handle_ignore(state, token): """Function for handling tokens that should be ignored""" yield from [] def handle_double_amps(state, token): yield _new_token("AND", "and", token.start) def handle_double_pipe(state, token): yield _new_token("OR", "or", token.start) def handle_redirect(state, token): # The parser expects whitespace after a redirection in subproc mode. # If whitespace does not exist, we'll issue an empty whitespace # token before proceeding. state["last"] = token typ = token.type st = token.string key = (typ, st) if (typ, st) in token_map else typ yield _new_token(token_map[key], st, token.start) if state["pymode"][-1][0]: return # add a whitespace token after a redirection, if we need to next_tok = next(state["stream"]) if next_tok.start == token.end: yield _new_token("WS", "", token.end) yield from handle_token(state, next_tok) def _make_matcher_handler(tok, typ, pymode, ender, handlers): matcher = ( ")" if tok.endswith("(") else "}" if tok.endswith("{") else "]" if tok.endswith("[") else None ) def _inner_handler(state, token): state["pymode"].append((pymode, tok, matcher, token.start)) state["last"] = token yield _new_token(typ, tok, token.start) handlers[(OP, tok)] = _inner_handler @lazyobject def special_handlers(): """Mapping from ``tokenize`` tokens (or token types) to the proper function for generating PLY tokens from them. In addition to yielding PLY tokens, these functions may manipulate the Lexer's state. """ sh = { NL: handle_ignore, COMMENT: handle_ignore, ENCODING: handle_ignore, ENDMARKER: handle_ignore, NAME: handle_name, ERRORTOKEN: handle_error_token, LESS: handle_redirect, GREATER: handle_redirect, RIGHTSHIFT: handle_redirect, IOREDIRECT: handle_redirect, (OP, "<"): handle_redirect, (OP, ">"): handle_redirect, (OP, ">>"): handle_redirect, (OP, ")"): handle_rparen, (OP, "}"): handle_rbrace, (OP, "]"): handle_rbracket, (OP, "&&"): handle_double_amps, (OP, "||"): handle_double_pipe, (ERRORTOKEN, " "): handle_error_space, (ERRORTOKEN, "\\\n"): handle_error_linecont, (ERRORTOKEN, "\\\r\n"): handle_error_linecont, } _make_matcher_handler("(", "LPAREN", True, ")", sh) _make_matcher_handler("[", "LBRACKET", True, "]", sh) _make_matcher_handler("{", "LBRACE", True, "}", sh) _make_matcher_handler("$(", "DOLLAR_LPAREN", False, ")", sh) _make_matcher_handler("$[", "DOLLAR_LBRACKET", False, "]", sh) _make_matcher_handler("${", "DOLLAR_LBRACE", True, "}", sh) _make_matcher_handler("!(", "BANG_LPAREN", False, ")", sh) _make_matcher_handler("![", "BANG_LBRACKET", False, "]", sh) _make_matcher_handler("@(", "AT_LPAREN", True, ")", sh) _make_matcher_handler("@$(", "ATDOLLAR_LPAREN", False, ")", sh) return sh def handle_token(state, token): """ General-purpose token handler. Makes use of ``token_map`` or ``special_map`` to yield one or more PLY tokens from the given input. Parameters ---------- state : The current state of the lexer, including information about whether we are in Python mode or subprocess mode, which changes the lexer's behavior. Also includes the stream of tokens yet to be considered. token : The token (from ``tokenize``) currently under consideration """ typ = token.type st = token.string pymode = state["pymode"][-1][0] if not pymode: if state["last"] is not None and state["last"].end != token.start: cur = token.start old = state["last"].end if cur[0] == old[0] and cur[1] > old[1]: yield _new_token("WS", token.line[old[1] : cur[1]], old) if (typ, st) in special_handlers: yield from special_handlers[(typ, st)](state, token) elif (typ, st) in token_map: state["last"] = token yield _new_token(token_map[(typ, st)], st, token.start) elif typ in special_handlers: yield from special_handlers[typ](state, token) elif typ in token_map: state["last"] = token yield _new_token(token_map[typ], st, token.start) else: m = "Unexpected token: {0}".format(token) yield _new_token("ERRORTOKEN", m, token.start) def get_tokens(s): """ Given a string containing xonsh code, generates a stream of relevant PLY tokens using ``handle_token``. """ state = { "indents": [0], "last": None, "pymode": [(True, "", "", (0, 0))], "stream": tokenize(io.BytesIO(s.encode("utf-8")).readline), } while True: try: token = next(state["stream"]) yield from handle_token(state, token) except StopIteration: if len(state["pymode"]) > 1: pm, o, m, p = state["pymode"][-1] l, c = p e = 'Unmatched "{}" at line {}, column {}' yield _new_token("ERRORTOKEN", e.format(o, l, c), (0, 0)) break except TokenError as e: # this is recoverable in single-line mode (from the shell) # (e.g., EOF while scanning string literal) yield _new_token("ERRORTOKEN", e.args[0], (0, 0)) break except IndentationError as e: # this is never recoverable yield _new_token("ERRORTOKEN", e, (0, 0)) break # synthesize a new PLY token def _new_token(type, value, pos): o = LexToken() o.type = type o.value = value o.lineno, o.lexpos = pos return o class Lexer(object): """Implements a lexer for the xonsh language.""" _tokens = None def __init__(self): """ Attributes ---------- fname : str Filename last : token The last token seen. lineno : int The last line number seen. """ self.fname = "" self.last = None self.beforelast = None def build(self, **kwargs): """Part of the PLY lexer API.""" pass def reset(self): pass def input(self, s): """Calls the lexer on the string s.""" self.token_stream = get_tokens(s) def token(self): """Retrieves the next token.""" self.beforelast = self.last self.last = next(self.token_stream, None) return self.last def __iter__(self): t = self.token() while t is not None: yield t t = self.token() def split(self, s): """Splits a string into a list of strings which are whitespace-separated tokens. """ vals = [] self.input(s) l = c = -1 ws = "WS" nl = "\n" for t in self: if t.type == ws: continue elif l < t.lineno: vals.append(t.value) elif len(vals) > 0 and c == t.lexpos: vals[-1] = vals[-1] + t.value else: vals.append(t.value) nnl = t.value.count(nl) if nnl == 0: l = t.lineno c = t.lexpos + len(t.value) else: l = t.lineno + nnl c = len(t.value.rpartition(nl)[-1]) return vals # # All the tokens recognized by the lexer # @property def tokens(self): if self._tokens is None: t = ( tuple(token_map.values()) + ( "NAME", # name tokens "BANG", # ! tokens "WS", # whitespace in subprocess mode "LPAREN", "RPAREN", # ( ) "LBRACKET", "RBRACKET", # [ ] "LBRACE", "RBRACE", # { } "AT_LPAREN", # @( "BANG_LPAREN", # !( "BANG_LBRACKET", # ![ "DOLLAR_LPAREN", # $( "DOLLAR_LBRACE", # ${ "DOLLAR_LBRACKET", # $[ "ATDOLLAR_LPAREN", # @$( "ERRORTOKEN", # whoops! ) + tuple(i.upper() for i in kwmod.kwlist) ) self._tokens = t return self._tokens ================================================ FILE: xonsh/macutils.py ================================================ """Provides some Mac / Darwin based utility functions for xonsh.""" from ctypes import c_uint, byref, create_string_buffer from xonsh.platform import LIBC def sysctlbyname(name, return_str=True): """Gets a sysctl value by name. If return_str is true, this will return a string representation, else it will return the raw value. """ # forked from https://gist.github.com/pudquick/581a71425439f2cf8f09 size = c_uint(0) # Find out how big our buffer will be LIBC.sysctlbyname(name, None, byref(size), None, 0) # Make the buffer buf = create_string_buffer(size.value) # Re-run, but provide the buffer LIBC.sysctlbyname(name, buf, byref(size), None, 0) if return_str: return buf.value else: return buf.raw ================================================ FILE: xonsh/main.py ================================================ # -*- coding: utf-8 -*- """The main xonsh script.""" import os import sys import enum import argparse import builtins import contextlib import signal import traceback from xonsh import __version__ from xonsh.timings import setup_timings from xonsh.lazyasd import lazyobject from xonsh.shell import Shell from xonsh.pretty import pretty from xonsh.execer import Execer from xonsh.proc import HiddenCommandPipeline from xonsh.jobs import ignore_sigtstp from xonsh.tools import setup_win_unicode_console, print_color, to_bool_or_int from xonsh.platform import HAS_PYGMENTS, ON_WINDOWS from xonsh.codecache import run_script_with_cache, run_code_with_cache from xonsh.xonfig import print_welcome_screen from xonsh.xontribs import xontribs_load from xonsh.lazyimps import pygments, pyghooks from xonsh.imphooks import install_import_hooks from xonsh.events import events from xonsh.environ import xonshrc_context, make_args_env from xonsh.built_ins import XonshSession, load_builtins, load_proxies from gitsome import __version__ as gitsome_version events.transmogrify("on_post_init", "LoadEvent") events.doc( "on_post_init", """ on_post_init() -> None Fired after all initialization is finished and we're ready to do work. NOTE: This is fired before the wizard is automatically started. """, ) events.transmogrify("on_exit", "LoadEvent") events.doc( "on_exit", """ on_exit() -> None Fired after all commands have been executed, before tear-down occurs. NOTE: All the caveats of the ``atexit`` module also apply to this event. """, ) events.transmogrify("on_pre_cmdloop", "LoadEvent") events.doc( "on_pre_cmdloop", """ on_pre_cmdloop() -> None Fired just before the command loop is started, if it is. """, ) events.transmogrify("on_post_cmdloop", "LoadEvent") events.doc( "on_post_cmdloop", """ on_post_cmdloop() -> None Fired just after the command loop finishes, if it is. NOTE: All the caveats of the ``atexit`` module also apply to this event. """, ) events.transmogrify("on_pre_rc", "LoadEvent") events.doc( "on_pre_rc", """ on_pre_rc() -> None Fired just before rc files are loaded, if they are. """, ) events.transmogrify("on_post_rc", "LoadEvent") events.doc( "on_post_rc", """ on_post_rc() -> None Fired just after rc files are loaded, if they are. """, ) def get_setproctitle(): """Proxy function for loading process title""" try: from setproctitle import setproctitle as spt except ImportError: return return spt def path_argument(s): """Return a path only if the path is actually legal This is very similar to argparse.FileType, except that it doesn't return an open file handle, but rather simply validates the path.""" s = os.path.abspath(os.path.expanduser(s)) if not os.path.isfile(s): msg = "{0!r} must be a valid path to a file".format(s) raise argparse.ArgumentTypeError(msg) return s @lazyobject def parser(): p = argparse.ArgumentParser(description="xonsh", add_help=False) p.add_argument( "-h", "--help", dest="help", action="store_true", default=False, help="show help and exit", ) p.add_argument( "-V", "--version", dest="version", action="store_true", default=False, help="show version information and exit", ) p.add_argument( "-c", help="Run a single command and exit", dest="command", required=False, default=None, ) p.add_argument( "-i", "--interactive", help="force running in interactive mode", dest="force_interactive", action="store_true", default=False, ) p.add_argument( "-l", "--login", help="run as a login shell", dest="login", action="store_true", default=False, ) p.add_argument( "--config-path", help="DEPRECATED: static configuration files may now be used " "in the XONSHRC file list, see the --rc option.", dest="config_path", default=None, type=path_argument, ) p.add_argument( "--rc", help="The xonshrc files to load, these may be either xonsh " "files or JSON-based static configuration files.", dest="rc", nargs="+", type=path_argument, default=None, ) p.add_argument( "--no-rc", help="Do not load the .xonshrc files", dest="norc", action="store_true", default=False, ) p.add_argument( "--no-script-cache", help="Do not cache scripts as they are run", dest="scriptcache", action="store_false", default=True, ) p.add_argument( "--cache-everything", help="Use a cache, even for interactive commands", dest="cacheall", action="store_true", default=False, ) p.add_argument( "-D", dest="defines", help="define an environment variable, in the form of " "-DNAME=VAL. May be used many times.", metavar="ITEM", action="append", default=None, ) p.add_argument( "--shell-type", help="What kind of shell should be used. " "Possible options: readline, prompt_toolkit, random. " "Warning! If set this overrides $SHELL_TYPE variable.", dest="shell_type", choices=tuple(Shell.shell_type_aliases.keys()), default=None, ) p.add_argument( "--timings", help="Prints timing information before the prompt is shown. " "This is useful while tracking down performance issues " "and investigating startup times.", dest="timings", action="store_true", default=None, ) p.add_argument( "file", metavar="script-file", help="If present, execute the script in script-file" " and exit", nargs="?", default=None, ) p.add_argument( "args", metavar="args", help="Additional arguments to the script specified " "by script-file", nargs=argparse.REMAINDER, default=[], ) return p def _pprint_displayhook(value): if value is None: return builtins._ = None # Set '_' to None to avoid recursion if isinstance(value, HiddenCommandPipeline): builtins._ = value return env = builtins.__xonsh__.env if env.get("PRETTY_PRINT_RESULTS"): printed_val = pretty(value) else: printed_val = repr(value) if HAS_PYGMENTS and env.get("COLOR_RESULTS"): tokens = list(pygments.lex(printed_val, lexer=pyghooks.XonshLexer())) end = "" if env.get("SHELL_TYPE") == "prompt_toolkit2" else "\n" print_color(tokens, end=end) else: print(printed_val) # black & white case builtins._ = value class XonshMode(enum.Enum): single_command = 0 script_from_file = 1 script_from_stdin = 2 interactive = 3 def start_services(shell_kwargs, args): """Starts up the essential services in the proper order. This returns the environment instance as a convenience. """ install_import_hooks() # create execer, which loads builtins ctx = shell_kwargs.get("ctx", {}) debug = to_bool_or_int(os.getenv("XONSH_DEBUG", "0")) events.on_timingprobe.fire(name="pre_execer_init") execer = Execer( xonsh_ctx=ctx, debug_level=debug, scriptcache=shell_kwargs.get("scriptcache", True), cacheall=shell_kwargs.get("cacheall", False), ) events.on_timingprobe.fire(name="post_execer_init") # load rc files login = shell_kwargs.get("login", True) env = builtins.__xonsh__.env rc = shell_kwargs.get("rc", None) rc = env.get("XONSHRC") if rc is None else rc if args.mode != XonshMode.interactive and not args.force_interactive: # Don't load xonshrc if not interactive shell rc = None events.on_pre_rc.fire() xonshrc_context(rcfiles=rc, execer=execer, ctx=ctx, env=env, login=login) events.on_post_rc.fire() # create shell builtins.__xonsh__.shell = Shell(execer=execer, **shell_kwargs) ctx["__name__"] = "__main__" return env def premain(argv=None): """Setup for main xonsh entry point. Returns parsed arguments.""" if argv is None: argv = sys.argv[1:] builtins.__xonsh__ = XonshSession() setup_timings(argv) setproctitle = get_setproctitle() if setproctitle is not None: setproctitle(" ".join(["xonsh"] + argv)) args = parser.parse_args(argv) if args.help: parser.print_help() parser.exit() if args.version: version = "/".join(("xonsh", __version__)) print(version) parser.exit() shell_kwargs = { "shell_type": args.shell_type, "completer": False, "login": False, "scriptcache": args.scriptcache, "cacheall": args.cacheall, "ctx": builtins.__xonsh__.ctx, } if args.login: shell_kwargs["login"] = True if args.norc: shell_kwargs["rc"] = () elif args.rc: shell_kwargs["rc"] = args.rc setattr(sys, "displayhook", _pprint_displayhook) print('Gitsome Version: ' + gitsome_version) if args.command is not None: args.mode = XonshMode.single_command shell_kwargs["shell_type"] = "none" elif args.file is not None: args.mode = XonshMode.script_from_file shell_kwargs["shell_type"] = "none" elif not sys.stdin.isatty() and not args.force_interactive: args.mode = XonshMode.script_from_stdin shell_kwargs["shell_type"] = "none" else: args.mode = XonshMode.interactive shell_kwargs["completer"] = True shell_kwargs["login"] = True env = start_services(shell_kwargs, args) env["XONSH_LOGIN"] = shell_kwargs["login"] if args.defines is not None: env.update([x.split("=", 1) for x in args.defines]) env["XONSH_INTERACTIVE"] = args.force_interactive or ( args.mode == XonshMode.interactive ) if ON_WINDOWS: setup_win_unicode_console(env.get("WIN_UNICODE_CONSOLE", True)) return args def _failback_to_other_shells(args, err): # only failback for interactive shell; if we cannot tell, treat it # as an interactive one for safe. if hasattr(args, "mode") and args.mode != XonshMode.interactive: raise err foreign_shell = None shells_file = "/etc/shells" if not os.path.exists(shells_file): # right now, it will always break here on Windows raise err excluded_list = ["xonsh", "screen"] with open(shells_file) as f: for line in f: line = line.strip() if not line or line.startswith("#"): continue if "/" not in line: continue _, shell = line.rsplit("/", 1) if shell in excluded_list: continue if not os.path.exists(line): continue foreign_shell = line break if foreign_shell: traceback.print_exc() print("Xonsh encountered an issue during launch", file=sys.stderr) print("Failback to {}".format(foreign_shell), file=sys.stderr) os.execlp(foreign_shell, foreign_shell) else: raise err def main(argv=None): args = None try: args = premain(argv) return main_xonsh(args) except Exception as err: _failback_to_other_shells(args, err) def main_xonsh(args): """Main entry point for xonsh cli.""" if not ON_WINDOWS: def func_sig_ttin_ttou(n, f): pass signal.signal(signal.SIGTTIN, func_sig_ttin_ttou) signal.signal(signal.SIGTTOU, func_sig_ttin_ttou) events.on_post_init.fire() env = builtins.__xonsh__.env shell = builtins.__xonsh__.shell try: if args.mode == XonshMode.interactive: # enter the shell env["XONSH_INTERACTIVE"] = True ignore_sigtstp() if env["XONSH_INTERACTIVE"] and not any( os.path.isfile(i) for i in env["XONSHRC"] ): print_welcome_screen() events.on_pre_cmdloop.fire() try: shell.shell.cmdloop() finally: events.on_post_cmdloop.fire() elif args.mode == XonshMode.single_command: # run a single command and exit run_code_with_cache(args.command.lstrip(), shell.execer, mode="single") elif args.mode == XonshMode.script_from_file: # run a script contained in a file path = os.path.abspath(os.path.expanduser(args.file)) if os.path.isfile(path): sys.argv = [args.file] + args.args env.update(make_args_env()) # $ARGS is not sys.argv env["XONSH_SOURCE"] = path shell.ctx.update({"__file__": args.file, "__name__": "__main__"}) run_script_with_cache( args.file, shell.execer, glb=shell.ctx, loc=None, mode="exec" ) else: print("xonsh: {0}: No such file or directory.".format(args.file)) elif args.mode == XonshMode.script_from_stdin: # run a script given on stdin code = sys.stdin.read() run_code_with_cache( code, shell.execer, glb=shell.ctx, loc=None, mode="exec" ) finally: events.on_exit.fire() postmain(args) def postmain(args=None): """Teardown for main xonsh entry point, accepts parsed arguments.""" if ON_WINDOWS: setup_win_unicode_console(enable=False) builtins.__xonsh__.shell = None @contextlib.contextmanager def main_context(argv=None): """Generator that runs pre- and post-main() functions. This has two iterations. The first yields the shell. The second returns None but cleans up the shell. """ args = premain(argv) yield builtins.__xonsh__.shell postmain(args) def setup( ctx=None, shell_type="none", env=(("RAISE_SUBPROC_ERROR", True),), aliases=(), xontribs=(), threadable_predictors=(), ): """Starts up a new xonsh shell. Calling this in function in another packages __init__.py will allow xonsh to be fully used in the package in headless or headed mode. This function is primarily indended to make starting up xonsh for 3rd party packages easier. Parameters ---------- ctx : dict-like or None, optional The xonsh context to start with. If None, an empty dictionary is provided. shell_type : str, optional The type of shell to start. By default this is 'none', indicating we should start in headless mode. env : dict-like, optional Environment to update the current environment with after the shell has been initialized. aliases : dict-like, optional Aliases to add after the shell has been initialized. xontribs : iterable of str, optional Xontrib names to load. threadable_predictors : dict-like, optional Threadable predictors to start up with. These overide the defaults. """ ctx = {} if ctx is None else ctx # setup xonsh ctx and execer if not hasattr(builtins, "__xonsh__"): execer = Execer(xonsh_ctx=ctx) builtins.__xonsh__ = XonshSession(ctx=ctx, execer=execer) load_builtins(ctx=ctx, execer=execer) load_proxies() builtins.__xonsh__.shell = Shell(execer, ctx=ctx, shell_type=shell_type) builtins.__xonsh__.env.update(env) install_import_hooks() builtins.aliases.update(aliases) if xontribs: xontribs_load(xontribs) tp = builtins.__xonsh__.commands_cache.threadable_predictors tp.update(threadable_predictors) ================================================ FILE: xonsh/openpy.py ================================================ # -*- coding: utf-8 -*- """Tools to open ``*.py`` files as Unicode. Uses the encoding specified within the file, as per PEP 263. Much of the code is taken from the tokenize module in Python 3.2. This file was forked from the IPython project: * Copyright (c) 2008-2014, IPython Development Team * Copyright (C) 2001-2007 Fernando Perez * Copyright (c) 2001, Janko Hauser * Copyright (c) 2001, Nathaniel Gray """ import io import re from xonsh.lazyasd import LazyObject from xonsh.tokenize import detect_encoding, tokopen cookie_comment_re = LazyObject( lambda: re.compile(r"^\s*#.*coding[:=]\s*([-\w.]+)", re.UNICODE), globals(), "cookie_comment_re", ) def source_to_unicode(txt, errors="replace", skip_encoding_cookie=True): """Converts a bytes string with python source code to unicode. Unicode strings are passed through unchanged. Byte strings are checked for the python source file encoding cookie to determine encoding. txt can be either a bytes buffer or a string containing the source code. """ if isinstance(txt, str): return txt if isinstance(txt, bytes): buf = io.BytesIO(txt) else: buf = txt try: encoding, _ = detect_encoding(buf.readline) except SyntaxError: encoding = "ascii" buf.seek(0) text = io.TextIOWrapper(buf, encoding, errors=errors, line_buffering=True) text.mode = "r" if skip_encoding_cookie: return u"".join(strip_encoding_cookie(text)) else: return text.read() def strip_encoding_cookie(filelike): """Generator to pull lines from a text-mode file, skipping the encoding cookie if it is found in the first two lines. """ it = iter(filelike) try: first = next(it) if not cookie_comment_re.match(first): yield first second = next(it) if not cookie_comment_re.match(second): yield second except StopIteration: return for line in it: yield line def read_py_file(filename, skip_encoding_cookie=True): """Read a Python file, using the encoding declared inside the file. Parameters ---------- filename : str The path to the file to read. skip_encoding_cookie : bool If True (the default), and the encoding declaration is found in the first two lines, that line will be excluded from the output - compiling a unicode string with an encoding declaration is a SyntaxError in Python 2. Returns ------- A unicode string containing the contents of the file. """ with tokopen(filename) as f: # the open function defined in this module. if skip_encoding_cookie: return "".join(strip_encoding_cookie(f)) else: return f.read() def read_py_url(url, errors="replace", skip_encoding_cookie=True): """Read a Python file from a URL, using the encoding declared inside the file. Parameters ---------- url : str The URL from which to fetch the file. errors : str How to handle decoding errors in the file. Options are the same as for bytes.decode(), but here 'replace' is the default. skip_encoding_cookie : bool If True (the default), and the encoding declaration is found in the first two lines, that line will be excluded from the output - compiling a unicode string with an encoding declaration is a SyntaxError in Python 2. Returns ------- A unicode string containing the contents of the file. """ # Deferred import for faster start try: from urllib.request import urlopen # Py 3 except ImportError: from urllib import urlopen response = urlopen(url) buf = io.BytesIO(response.read()) return source_to_unicode(buf, errors, skip_encoding_cookie) def _list_readline(x): """Given a list, returns a readline() function that returns the next element with each call. """ x = iter(x) def readline(): return next(x) return readline ================================================ FILE: xonsh/parser.py ================================================ # -*- coding: utf-8 -*- """Implements the xonsh parser.""" from xonsh.lazyasd import lazyobject from xonsh.platform import PYTHON_VERSION_INFO @lazyobject def Parser(): if PYTHON_VERSION_INFO > (3, 6): from xonsh.parsers.v36 import Parser as p elif PYTHON_VERSION_INFO > (3, 5): from xonsh.parsers.v35 import Parser as p else: from xonsh.parsers.v34 import Parser as p return p ================================================ FILE: xonsh/parsers/__init__.py ================================================ ================================================ FILE: xonsh/parsers/base.py ================================================ # -*- coding: utf-8 -*- """Implements the base xonsh parser.""" import os import re import time import textwrap from threading import Thread from ast import parse as pyparse from collections.abc import Iterable, Sequence, Mapping from xonsh.ply.ply import yacc from xonsh.tools import FORMATTER from xonsh import ast from xonsh.ast import has_elts, xonsh_call, load_attribute_chain from xonsh.lexer import Lexer, LexToken from xonsh.platform import PYTHON_VERSION_INFO from xonsh.tokenize import SearchPath, StringPrefix from xonsh.lazyasd import LazyObject, lazyobject from xonsh.parsers.context_check import check_contexts RE_SEARCHPATH = LazyObject(lambda: re.compile(SearchPath), globals(), "RE_SEARCHPATH") RE_STRINGPREFIX = LazyObject( lambda: re.compile(StringPrefix), globals(), "RE_STRINGPREFIX" ) @lazyobject def RE_FSTR_EVAL_CHARS(): return re.compile(".*?[!@$`]") class Location(object): """Location in a file.""" def __init__(self, fname, lineno, column=None): """Takes a filename, line number, and optionally a column number.""" self.fname = fname self.lineno = lineno self.column = column def __str__(self): s = "{0}:{1}".format(self.fname, self.lineno) if self.column is not None: s += ":{0}".format(self.column) return s def ensure_has_elts(x, lineno=None, col_offset=None): """Ensures that x is an AST node with elements.""" if not has_elts(x): if not isinstance(x, Iterable): x = [x] lineno = x[0].lineno if lineno is None else lineno col_offset = x[0].col_offset if col_offset is None else col_offset x = ast.Tuple(elts=x, ctx=ast.Load(), lineno=lineno, col_offset=col_offset) return x def empty_list(lineno=None, col=None): """Creates the AST node for an empty list.""" return ast.List(elts=[], ctx=ast.Load(), lineno=lineno, col_offset=col) def binop(x, op, y, lineno=None, col=None): """Creates the AST node for a binary operation.""" lineno = x.lineno if lineno is None else lineno col = x.col_offset if col is None else col return ast.BinOp(left=x, op=op, right=y, lineno=lineno, col_offset=col) def call_split_lines(x, lineno=None, col=None): """Creates the AST node for calling the 'splitlines' attribute of an object, nominally a string. """ return ast.Call( func=ast.Attribute( value=x, attr="splitlines", ctx=ast.Load(), lineno=lineno, col_offset=col ), args=[], keywords=[], starargs=None, kwargs=None, lineno=lineno, col_offset=col, ) def ensure_list_from_str_or_list(x, lineno=None, col=None): """Creates the AST node for the following expression:: [x] if isinstance(x, str) else x Somewhat useful. """ return ast.IfExp( test=ast.Call( func=ast.Name( id="isinstance", ctx=ast.Load(), lineno=lineno, col_offset=col ), args=[x, ast.Name(id="str", ctx=ast.Load(), lineno=lineno, col_offset=col)], keywords=[], starargs=None, kwargs=None, lineno=lineno, col_offset=col, ), body=ast.List(elts=[x], ctx=ast.Load(), lineno=lineno, col_offset=col), orelse=x, lineno=lineno, col_offset=col, ) def xonsh_help(x, lineno=None, col=None): """Creates the AST node for calling the __xonsh__.help() function.""" return xonsh_call("__xonsh__.help", [x], lineno=lineno, col=col) def xonsh_superhelp(x, lineno=None, col=None): """Creates the AST node for calling the __xonsh__.superhelp() function.""" return xonsh_call("__xonsh__.superhelp", [x], lineno=lineno, col=col) def xonsh_pathsearch(pattern, pymode=False, lineno=None, col=None): """Creates the AST node for calling the __xonsh__.pathsearch() function. The pymode argument indicate if it is called from subproc or python mode""" pymode = ast.NameConstant(value=pymode, lineno=lineno, col_offset=col) searchfunc, pattern = RE_SEARCHPATH.match(pattern).groups() pattern = ast.Str(s=pattern, lineno=lineno, col_offset=col) pathobj = False if searchfunc.startswith("@"): func = searchfunc[1:] elif "g" in searchfunc: func = "__xonsh__.globsearch" pathobj = "p" in searchfunc else: func = "__xonsh__.regexsearch" pathobj = "p" in searchfunc func = load_attribute_chain(func, lineno=lineno, col=col) pathobj = ast.NameConstant(value=pathobj, lineno=lineno, col_offset=col) return xonsh_call( "__xonsh__.pathsearch", args=[func, pattern, pymode, pathobj], lineno=lineno, col=col, ) def load_ctx(x): """Recursively sets ctx to ast.Load()""" if not hasattr(x, "ctx"): return x.ctx = ast.Load() if isinstance(x, (ast.Tuple, ast.List)): for e in x.elts: load_ctx(e) elif isinstance(x, ast.Starred): load_ctx(x.value) def store_ctx(x): """Recursively sets ctx to ast.Store()""" if not hasattr(x, "ctx"): return x.ctx = ast.Store() if isinstance(x, (ast.Tuple, ast.List)): for e in x.elts: store_ctx(e) elif isinstance(x, ast.Starred): store_ctx(x.value) def del_ctx(x): """Recursively sets ctx to ast.Del()""" if not hasattr(x, "ctx"): return x.ctx = ast.Del() if isinstance(x, (ast.Tuple, ast.List)): for e in x.elts: del_ctx(e) elif isinstance(x, ast.Starred): del_ctx(x.value) def empty_list_if_newline(x): return [] if x == "\n" else x def lopen_loc(x): """Extracts the line and column number for a node that may have an opening parenthesis, brace, or bracket. """ lineno = x._lopen_lineno if hasattr(x, "_lopen_lineno") else x.lineno col = x._lopen_col if hasattr(x, "_lopen_col") else x.col_offset return lineno, col def hasglobstar(x): """Returns True if a node has literal '*' for globbing.""" if isinstance(x, ast.Str): return "*" in x.s elif isinstance(x, list): for e in x: if hasglobstar(e): return True else: return False else: return False def _wrap_fstr_field(field, spec, conv): rtn = "{" + field if conv: rtn += "!" + conv if spec: rtn += ":" + spec rtn += "}" return rtn def eval_fstr_fields(fstring, prefix, filename=None): """Takes an fstring (and its prefix, ie f") that may contain xonsh expressions as its field values and substitues them for a xonsh eval() call as needed. Roughly, for example, this will take f"{$HOME}" and transform it to be f"{__xonsh__.execer.eval(r'$HOME')}". """ last = fstring[-1] q, r = ("'", r"\'") if last == '"' else ('"', r"\"") prelen = len(prefix) postlen = len(fstring) - len(fstring.rstrip(last)) template = fstring[prelen:-postlen] repl = prefix for literal, field, spec, conv in FORMATTER.parse(template): repl += literal if field is None: continue elif RE_FSTR_EVAL_CHARS.match(field) is None: # just a normal python field, simply reconstruct. repl += _wrap_fstr_field(field, spec, conv) else: # the field has a special xonsh character, so we must eval it eval_field = "__xonsh__.execer.eval(r" + q eval_field += field.lstrip().replace(q, r) eval_field += q + ", glbs=globals(), locs=locals()" if filename is not None: eval_field += ", filename=" + q + filename + q eval_field += ")" repl += _wrap_fstr_field(eval_field, spec, conv) repl += last * postlen return repl class YaccLoader(Thread): """Thread to load (but not shave) the yacc parser.""" def __init__(self, parser, yacc_kwargs, *args, **kwargs): super().__init__(*args, **kwargs) self.daemon = True self.parser = parser self.yacc_kwargs = yacc_kwargs self.start() def run(self): self.parser.parser = yacc.yacc(**self.yacc_kwargs) class BaseParser(object): """A base class that parses the xonsh language.""" def __init__( self, lexer_optimize=True, lexer_table="xonsh.lexer_table", yacc_optimize=True, yacc_table="xonsh.parser_table", yacc_debug=False, outputdir=None, ): """Parameters ---------- lexer_optimize : bool, optional Set to false when unstable and true when lexer is stable. lexer_table : str, optional Lexer module used when optimized. yacc_optimize : bool, optional Set to false when unstable and true when parser is stable. yacc_table : str, optional Parser module used when optimized. yacc_debug : debug, optional Dumps extra debug info. outputdir : str or None, optional The directory to place generated tables within. Defaults to the root xonsh dir. """ self.lexer = lexer = Lexer() self.tokens = lexer.tokens self._lines = None self.xonsh_code = None self._attach_nocomma_tok_rules() self._attach_nocloser_base_rules() self._attach_nodedent_base_rules() self._attach_nonewline_base_rules() self._attach_subproc_arg_part_rules() opt_rules = [ "newlines", "arglist", "func_call", "rarrow_test", "typedargslist", "equals_test", "colon_test", "tfpdef", "comma_tfpdef_list", "comma_pow_tfpdef", "vfpdef", "comma_vfpdef_list", "comma_pow_vfpdef", "equals_yield_expr_or_testlist_list", "testlist", "as_name", "period_or_ellipsis_list", "comma_import_as_name_list", "comma_dotted_as_name_list", "comma_name_list", "comma_test", "elif_part_list", "finally_part", "varargslist", "or_and_test_list", "and_not_test_list", "comp_op_expr_list", "xor_and_expr_list", "ampersand_shift_expr_list", "shift_arith_expr_list", "op_factor_list", "trailer_list", "testlist_comp", "yield_expr_or_testlist_comp", "dictorsetmaker", "comma_subscript_list", "test", "sliceop", "comp_iter", "yield_arg", "test_comma_list", "macroarglist", "any_raw_toks", ] for rule in opt_rules: self._opt_rule(rule) list_rules = [ "comma_tfpdef", "comma_vfpdef", "semi_small_stmt", "comma_test_or_star_expr", "period_or_ellipsis", "comma_import_as_name", "comma_dotted_as_name", "period_name", "comma_name", "elif_part", "except_part", "comma_with_item", "or_and_test", "and_not_test", "comp_op_expr", "pipe_xor_expr", "xor_and_expr", "ampersand_shift_expr", "shift_arith_expr", "pm_term", "op_factor", "trailer", "comma_subscript", "comma_expr_or_star_expr", "comma_test", "comma_argument", "comma_item", "attr_period_name", "test_comma", "equals_yield_expr_or_testlist", "comma_nocomma", ] for rule in list_rules: self._list_rule(rule) tok_rules = [ "def", "class", "return", "number", "name", "bang", "none", "true", "false", "ellipsis", "if", "del", "assert", "lparen", "lbrace", "lbracket", "string", "times", "plus", "minus", "divide", "doublediv", "mod", "at", "lshift", "rshift", "pipe", "xor", "ampersand", "for", "colon", "import", "except", "nonlocal", "global", "yield", "from", "raise", "with", "dollar_lparen", "dollar_lbrace", "dollar_lbracket", "try", "bang_lparen", "bang_lbracket", "comma", "rparen", "rbracket", "at_lparen", "atdollar_lparen", "indent", "dedent", "newline", "lambda", "ampersandequal", "as", "atdollar", "atequal", "break", "continue", "divequal", "dollar_name", "double_question", "doubledivequal", "elif", "else", "eq", "equals", "errortoken", "finally", "ge", "in", "is", "le", "lshiftequal", "minusequal", "modequal", "ne", "pass", "period", "pipeequal", "plusequal", "pow", "powequal", "question", "rarrow", "rshiftequal", "semi", "tilde", "timesequal", "while", "xorequal", ] for rule in tok_rules: self._tok_rule(rule) yacc_kwargs = dict( module=self, debug=yacc_debug, start="start_symbols", optimize=yacc_optimize, tabmodule=yacc_table, ) if not yacc_debug: yacc_kwargs["errorlog"] = yacc.NullLogger() if outputdir is None: outputdir = os.path.dirname(os.path.dirname(__file__)) yacc_kwargs["outputdir"] = outputdir if yacc_debug: # create parser on main thread self.parser = yacc.yacc(**yacc_kwargs) else: self.parser = None YaccLoader(self, yacc_kwargs) # Keeps track of the last token given to yacc (the lookahead token) self._last_yielded_token = None def reset(self): """Resets for clean parsing.""" self.lexer.reset() self._last_yielded_token = None self._lines = None self.xonsh_code = None def parse(self, s, filename="", mode="exec", debug_level=0): """Returns an abstract syntax tree of xonsh code. Parameters ---------- s : str The xonsh code. filename : str, optional Name of the file. mode : str, optional Execution mode, one of: exec, eval, or single. debug_level : str, optional Debugging level passed down to yacc. Returns ------- tree : AST """ self.reset() self.xonsh_code = s self.lexer.fname = filename while self.parser is None: time.sleep(0.01) # block until the parser is ready tree = self.parser.parse(input=s, lexer=self.lexer, debug=debug_level) if tree is not None: check_contexts(tree) # hack for getting modes right if mode == "single": if isinstance(tree, ast.Expression): tree = ast.Interactive(body=[self.expr(tree.body)]) elif isinstance(tree, ast.Module): tree = ast.Interactive(body=tree.body) return tree def _lexer_errfunc(self, msg, line, column): self._parse_error(msg, self.currloc(line, column)) def _yacc_lookahead_token(self): """Gets the next-to-last and last token seen by the lexer.""" return self.lexer.beforelast, self.lexer.last def _opt_rule(self, rulename): """For a rule name, creates an associated optional rule. '_opt' is appended to the rule name. """ def optfunc(self, p): p[0] = p[1] optfunc.__doc__ = ("{0}_opt : empty\n" " | {0}").format(rulename) optfunc.__name__ = "p_" + rulename + "_opt" setattr(self.__class__, optfunc.__name__, optfunc) def _list_rule(self, rulename): """For a rule name, creates an associated list rule. '_list' is appended to the rule name. """ def listfunc(self, p): p[0] = p[1] if len(p) == 2 else p[1] + p[2] listfunc.__doc__ = ("{0}_list : {0}\n" " | {0}_list {0}").format( rulename ) listfunc.__name__ = "p_" + rulename + "_list" setattr(self.__class__, listfunc.__name__, listfunc) def _tok_rule(self, rulename): """For a rule name, creates a rule that returns the corresponding token. '_tok' is appended to the rule name. """ def tokfunc(self, p): s, t = self._yacc_lookahead_token() uprule = rulename.upper() if s is not None and s.type == uprule: p[0] = s elif t is not None and t.type == uprule: p[0] = t else: raise TypeError("token for {0!r} not found.".format(rulename)) tokfunc.__doc__ = "{0}_tok : {1}".format(rulename, rulename.upper()) tokfunc.__name__ = "p_" + rulename + "_tok" setattr(self.__class__, tokfunc.__name__, tokfunc) def currloc(self, lineno, column=None): """Returns the current location.""" return Location(fname=self.lexer.fname, lineno=lineno, column=column) def expr(self, p): """Creates an expression for a token.""" expr = ast.Expr(value=p, lineno=p.lineno, col_offset=p.col_offset) expr.max_lineno = self.lineno expr.max_col = self.col return expr def token_col(self, t): """Gets ths token column""" return t.lexpos @property def lineno(self): if self.lexer.last is None: return 1 else: return self.lexer.last.lineno @property def col(self): s, t = self._yacc_lookahead_token() if t is not None: if t.type == "NEWLINE": t = s return self.token_col(t) return 0 @property def lines(self): if self._lines is None and self.xonsh_code is not None: self._lines = self.xonsh_code.splitlines(keepends=True) return self._lines def source_slice(self, start, stop): """Gets the original source code from two (line, col) tuples in source-space (i.e. lineno start at 1). """ bline, bcol = start eline, ecol = stop bline -= 1 lines = self.lines[bline:eline] if ecol == 0: explen = eline - bline if explen == len(lines) and explen > 1: lines[-1] = "" else: lines[-1] = lines[-1][:ecol] lines[0] = lines[0][bcol:] return "".join(lines) def _parse_error(self, msg, loc): if self.xonsh_code is None or loc is None: err_line_pointer = "" else: col = loc.column + 1 lines = self.lines if loc.lineno == 0: loc.lineno = len(lines) i = loc.lineno - 1 if 0 <= i < len(lines): err_line = lines[i].rstrip() err_line_pointer = "\n{}\n{: >{}}".format(err_line, "^", col) else: err_line_pointer = "" err = SyntaxError("{0}: {1}{2}".format(loc, msg, err_line_pointer)) err.loc = loc raise err # # Precedence of operators # precedence = ( ("left", "PIPE"), ("left", "XOR"), ("left", "AMPERSAND"), ("left", "EQ", "NE"), ("left", "GT", "GE", "LT", "LE"), ("left", "RSHIFT", "LSHIFT"), ("left", "PLUS", "MINUS"), ("left", "TIMES", "DIVIDE", "DOUBLEDIV", "MOD"), ("left", "POW"), ) # # Grammar as defined by BNF # def p_start_symbols(self, p): """start_symbols : single_input | file_input | eval_input | empty """ p[0] = p[1] def p_single_input(self, p): """single_input : compound_stmt NEWLINE """ p1 = empty_list_if_newline(p[1]) p0 = ast.Interactive(body=p1) p[0] = p0 def p_file_input(self, p): """file_input : file_stmts""" p[0] = ast.Module(body=p[1]) def p_file_stmts_nl(self, p): """file_stmts : newline_or_stmt""" # newline_or_stmt ENDMARKER p[0] = empty_list_if_newline(p[1]) def p_file_stmts_files(self, p): """file_stmts : file_stmts newline_or_stmt""" # file_input newline_or_stmt ENDMARKER p2 = empty_list_if_newline(p[2]) p[0] = p[1] + p2 def p_newline_or_stmt(self, p): """newline_or_stmt : NEWLINE | stmt """ p[0] = p[1] def p_newlines(self, p): """newlines : NEWLINE | newlines NEWLINE """ p[0] = p[1] if len(p) == 2 else p[1] + p[2] def p_eval_input(self, p): """eval_input : testlist newlines_opt """ p1 = p[1] p[0] = ast.Expression(body=p1, lineno=p1.lineno, col_offset=p1.col_offset) def p_func_call(self, p): """func_call : LPAREN arglist_opt RPAREN""" p[0] = p[2] def p_attr_period_name(self, p): """attr_period_name : PERIOD NAME""" p[0] = [p[2]] def p_attr_name_alone(self, p): """attr_name : name_tok""" p1 = p[1] p[0] = ast.Name( id=p1.value, ctx=ast.Load(), lineno=p1.lineno, col_offset=p1.lexpos ) def p_attr_name_with(self, p): """attr_name : name_tok attr_period_name_list""" p1 = p[1] name = ast.Name( id=p1.value, ctx=ast.Load(), lineno=p1.lineno, col_offset=p1.lexpos ) p2 = p[2] p0 = ast.Attribute( value=name, attr=p2[0], ctx=ast.Load(), lineno=p1.lineno, col_offset=p1.lexpos, ) for a in p2[1:]: p0 = ast.Attribute( value=p0, attr=a, ctx=ast.Load(), lineno=p0.lineno, col_offset=p0.col_offset, ) p[0] = p0 def p_decorator_no_call(self, p): """decorator : at_tok attr_name NEWLINE""" p[0] = p[2] def p_decorator_call(self, p): """decorator : at_tok attr_name func_call NEWLINE""" p1, name, p3 = p[1], p[2], p[3] if isinstance(name, ast.Attribute) or (p3 is not None): lineno, col = name.lineno, name.col_offset else: lineno, col = p1.lineno, p1.lexpos if p3 is None: p0 = ast.Call( func=name, args=[], keywords=[], starargs=None, kwargs=None, lineno=lineno, col_offset=col, ) else: p0 = ast.Call(func=name, lineno=lineno, col_offset=col, **p3) p[0] = p0 def p_decorators(self, p): """decorators : decorator | decorators decorator """ p[0] = [p[1]] if len(p) == 2 else p[1] + [p[2]] def p_decorated(self, p): """decorated : decorators classdef_or_funcdef""" p1, p2 = p[1], p[2] targ = p2[0] targ.decorator_list = p1 # this is silly, CPython. This claims a func or class starts on # the line of the first decorator, rather than the 'def' or 'class' # line. However, it retains the original col_offset. targ.lineno = p1[0].lineno # async functions take the col number of the 'def', unless they are # decorated, in which case they have the col of the 'async'. WAT? if hasattr(targ, "_async_tok"): targ.col_offset = targ._async_tok.lexpos del targ._async_tok p[0] = p2 def p_rarrow_test(self, p): """rarrow_test : RARROW test""" p[0] = p[2] def p_funcdef(self, p): """funcdef : def_tok NAME parameters rarrow_test_opt COLON suite""" f = ast.FunctionDef( name=p[2], args=p[3], returns=p[4], body=p[6], decorator_list=[], lineno=p[1].lineno, col_offset=p[1].lexpos, ) p[0] = [f] def p_parameters(self, p): """parameters : LPAREN typedargslist_opt RPAREN""" p2 = p[2] if p2 is None: p2 = ast.arguments( args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[], ) p[0] = p2 def p_equals_test(self, p): """equals_test : EQUALS test""" p[0] = p[2] def p_typedargslist_kwarg(self, p): """typedargslist : POW tfpdef""" p[0] = ast.arguments( args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=p[2], defaults=[] ) def p_typedargslist_times4_tfpdef(self, p): """typedargslist : TIMES tfpdef comma_pow_tfpdef_opt""" # *args, **kwargs p0 = ast.arguments( args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=p[3], defaults=[] ) self._set_var_args(p0, p[2], None) p[0] = p0 def p_typedargslist_times4_comma(self, p): """typedargslist : TIMES comma_pow_tfpdef""" # *, **kwargs p0 = ast.arguments( args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=p[2], defaults=[] ) p[0] = p0 def p_typedargslist_times5_tdpdef(self, p): """typedargslist : TIMES tfpdef comma_tfpdef_list comma_pow_tfpdef_opt""" # *args, x, **kwargs p0 = ast.arguments( args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=p[4], defaults=[] ) self._set_var_args(p0, p[2], p[3]) # *args p[0] = p0 def p_typedargslist_times5_comma(self, p): """typedargslist : TIMES comma_tfpdef_list comma_pow_tfpdef_opt""" # *, x, **kwargs p0 = ast.arguments( args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=p[3], defaults=[] ) self._set_var_args(p0, None, p[2]) # *args p[0] = p0 def p_typedargslist_t5(self, p): """typedargslist : tfpdef equals_test_opt comma_tfpdef_list_opt comma_opt""" # x p0 = ast.arguments( args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[] ) self._set_regular_args(p0, p[1], p[2], p[3], p[4]) p[0] = p0 def p_typedargslist_t7(self, p): """typedargslist : tfpdef equals_test_opt comma_tfpdef_list_opt comma_opt POW tfpdef""" # x, **kwargs p0 = ast.arguments( args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=p[6], defaults=[] ) self._set_regular_args(p0, p[1], p[2], p[3], p[4]) p[0] = p0 def p_typedargslist_t8(self, p): """typedargslist : tfpdef equals_test_opt comma_tfpdef_list_opt comma_opt TIMES tfpdef_opt comma_tfpdef_list_opt""" p0 = ast.arguments( args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[] ) self._set_regular_args(p0, p[1], p[2], p[3], p[4]) self._set_var_args(p0, p[6], p[7]) p[0] = p0 def p_typedargslist_t10(self, p): """typedargslist : tfpdef equals_test_opt comma_tfpdef_list_opt comma_opt TIMES tfpdef_opt COMMA POW vfpdef""" # x, *args, **kwargs p0 = ast.arguments( args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=p[9], defaults=[] ) self._set_regular_args(p0, p[1], p[2], p[3], p[4]) self._set_var_args(p0, p[6], None) p[0] = p0 def p_typedargslist_t11(self, p): """typedargslist : tfpdef equals_test_opt comma_tfpdef_list_opt comma_opt TIMES tfpdef_opt comma_tfpdef_list COMMA POW tfpdef""" # x, *args, **kwargs p0 = ast.arguments( args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=p[10], defaults=[], ) self._set_regular_args(p0, p[1], p[2], p[3], p[4]) self._set_var_args(p0, p[6], p[7]) p[0] = p0 def p_colon_test(self, p): """colon_test : COLON test""" p[0] = p[2] def p_tfpdef(self, p): """tfpdef : name_tok colon_test_opt""" p1 = p[1] kwargs = {"arg": p1.value, "annotation": p[2]} if PYTHON_VERSION_INFO >= (3, 5, 1): kwargs.update({"lineno": p1.lineno, "col_offset": p1.lexpos}) p[0] = ast.arg(**kwargs) def p_comma_tfpdef_empty(self, p): """comma_tfpdef : COMMA""" p[0] = [] def p_comma_tfpdef_args(self, p): """comma_tfpdef : COMMA tfpdef equals_test_opt""" p[0] = [{"arg": p[2], "default": p[3]}] def p_comma_pow_tfpdef(self, p): """comma_pow_tfpdef : COMMA POW tfpdef""" p[0] = p[3] def _set_args_def(self, argmts, vals, kwargs=False): args, defs = ( (argmts.kwonlyargs, argmts.kw_defaults) if kwargs else (argmts.args, argmts.defaults) ) if vals is None and kwargs: loc = self.currloc(self.lineno, self.col) self._parse_error("named arguments must follow bare *", loc) for v in vals: args.append(v["arg"]) d = v["default"] if kwargs or (d is not None): defs.append(d) def _set_regular_args(self, p0, p1, p2, p3, p4): if p2 is None and p3 is None: # x p0.args.append(p1) elif p2 is not None and p3 is None: # x=42 p0.args.append(p1) p0.defaults.append(p2) elif p2 is None and p3 is not None: # x, y and x, y=42 p0.args.append(p1) self._set_args_def(p0, p3) else: # x=42, y=42 p0.args.append(p1) p0.defaults.append(p2) self._set_args_def(p0, p3) def _set_var_args(self, p0, vararg, kwargs): if vararg is None and kwargs is not None: self._set_args_def(p0, kwargs, kwargs=True) elif vararg is not None and kwargs is None: # *args p0.vararg = vararg elif vararg is not None and kwargs is not None: # *args, x and *args, x, y and *args, x=10 and *args, x=10, y # and *args, x, y=10, and *args, x=42, y=65 p0.vararg = vararg self._set_args_def(p0, kwargs, kwargs=True) else: assert False def p_varargslist_kwargs(self, p): """varargslist : POW vfpdef""" p[0] = ast.arguments( args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=p[2], defaults=[] ) def p_varargslist_times4(self, p): """varargslist : TIMES vfpdef_opt comma_pow_vfpdef_opt""" p0 = ast.arguments( args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=p[3], defaults=[] ) self._set_var_args(p0, p[2], None) p[0] = p0 def p_varargslist_times5(self, p): """varargslist : TIMES vfpdef_opt comma_vfpdef_list comma_pow_vfpdef_opt""" # *args, x, **kwargs p0 = ast.arguments( args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=p[4], defaults=[] ) self._set_var_args(p0, p[2], p[3]) # *args p[0] = p0 def p_varargslist_v5(self, p): """varargslist : vfpdef equals_test_opt comma_vfpdef_list_opt comma_opt""" # x p0 = ast.arguments( args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[] ) self._set_regular_args(p0, p[1], p[2], p[3], p[4]) p[0] = p0 def p_varargslist_v7(self, p): """varargslist : vfpdef equals_test_opt comma_vfpdef_list_opt comma_opt POW vfpdef""" # x, **kwargs p0 = ast.arguments( args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=p[6], defaults=[] ) self._set_regular_args(p0, p[1], p[2], p[3], p[4]) p[0] = p0 def p_varargslist_v8(self, p): """varargslist : vfpdef equals_test_opt comma_vfpdef_list_opt comma_opt TIMES vfpdef_opt comma_vfpdef_list_opt""" # x, *args p0 = ast.arguments( args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[] ) self._set_regular_args(p0, p[1], p[2], p[3], p[4]) self._set_var_args(p0, p[6], p[7]) p[0] = p0 def p_varargslist_v10(self, p): """varargslist : vfpdef equals_test_opt comma_vfpdef_list_opt comma_opt TIMES vfpdef_opt COMMA POW vfpdef""" # x, *args, **kwargs p0 = ast.arguments( args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=p[9], defaults=[] ) self._set_regular_args(p0, p[1], p[2], p[3], p[4]) self._set_var_args(p0, p[6], None) p[0] = p0 def p_varargslist_v11(self, p): """varargslist : vfpdef equals_test_opt comma_vfpdef_list_opt comma_opt TIMES vfpdef_opt comma_vfpdef_list COMMA POW vfpdef""" p0 = ast.arguments( args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=p[10], defaults=[], ) self._set_regular_args(p0, p[1], p[2], p[3], p[4]) self._set_var_args(p0, p[6], p[7]) p[0] = p0 def p_vfpdef(self, p): """vfpdef : name_tok""" p1 = p[1] kwargs = {"arg": p1.value, "annotation": None} if PYTHON_VERSION_INFO >= (3, 5, 1): kwargs.update({"lineno": p1.lineno, "col_offset": p1.lexpos}) p[0] = ast.arg(**kwargs) def p_comma_vfpdef_empty(self, p): """comma_vfpdef : COMMA""" p[0] = [] def p_comma_vfpdef_value(self, p): """comma_vfpdef : COMMA vfpdef equals_test_opt""" p[0] = [{"arg": p[2], "default": p[3]}] def p_comma_pow_vfpdef(self, p): """comma_pow_vfpdef : COMMA POW vfpdef""" p[0] = p[3] def p_stmt(self, p): """stmt : simple_stmt | compound_stmt """ p[0] = p[1] def p_stmt_list(self, p): """stmt_list : stmt | stmt_list stmt """ if len(p) == 2: p[0] = p[1] else: p[0] = p[1] + p[2] def p_semi_opt(self, p): """semi_opt : SEMI | empty """ if len(p) == 2: p[0] = p[1] def p_semi_small_stmt(self, p): """semi_small_stmt : SEMI small_stmt""" p[0] = [p[2]] def p_simple_stmt_single(self, p): """simple_stmt : small_stmt semi_opt NEWLINE""" p[0] = [p[1]] def p_simple_stmt_many(self, p): """simple_stmt : small_stmt semi_small_stmt_list semi_opt NEWLINE""" p[0] = [p[1]] + p[2] def p_small_stmt(self, p): """small_stmt : expr_stmt | del_stmt | pass_stmt | flow_stmt | import_stmt | global_stmt | nonlocal_stmt | assert_stmt """ p[0] = p[1] _augassign_op = { "+=": ast.Add, "-=": ast.Sub, "*=": ast.Mult, "@=": ast.MatMult, "/=": ast.Div, "%=": ast.Mod, "//=": ast.FloorDiv, "**=": ast.Pow, "^=": ast.BitXor, "&=": ast.BitAnd, "|=": ast.BitOr, "<<=": ast.LShift, ">>=": ast.RShift, } def p_expr_stmt_testlist_assign(self, p): """expr_stmt : testlist_star_expr equals_yield_expr_or_testlist_list_opt | testlist equals_yield_expr_or_testlist_list_opt """ p1, p2 = p[1], p[2] if isinstance(p1, ast.Tuple): p1 = [p1] if p2 is None and len(p1) == 1: p[0] = self.expr(p1[0]) elif p2 is None: assert False else: for targ in p1: store_ctx(targ) list(map(store_ctx, p2[:-1])) lineno, col = lopen_loc(p1[0]) p[0] = ast.Assign( targets=p1 + p2[:-1], value=p2[-1], lineno=lineno, col_offset=col ) def p_expr_stmt_augassign(self, p): """expr_stmt : testlist_star_expr augassign yield_expr_or_testlist""" p1, p2 = p[1], p[2] if not isinstance(p1, ast.Tuple): p1 = p1[0] store_ctx(p1) op = self._augassign_op[p2] if op is None: self._parse_error( "operation {0!r} not supported".format(p2), self.currloc(lineno=p.lineno, column=p.lexpos), ) p[0] = ast.AugAssign( target=p1, op=op(), value=p[3], lineno=p1.lineno, col_offset=p1.col_offset ) def store_star_expr(self, p1, p2, targs, rhs): """Stores complex unpacking statements that target *x variables.""" p1 = [] if p1 is None else p1 if isinstance(p1, ast.Tuple): p1 = [p1] for targ in p1: store_ctx(targ) store_ctx(p2) for targ in targs: store_ctx(targ) p1.append(p2) p1.extend(targs) p1 = [ ast.Tuple( elts=p1, ctx=ast.Store(), lineno=p1[0].lineno, col_offset=p1[0].col_offset, ) ] p0 = ast.Assign( targets=p1, value=rhs, lineno=p1[0].lineno, col_offset=p1[0].col_offset ) return p0 def p_expr_stmt_star5(self, p): """expr_stmt : test_comma_list_opt star_expr comma_test_list equals_yield_expr_or_testlist""" targs, rhs = p[3], p[4][0] p[0] = self.store_star_expr(p[1], p[2], targs, rhs) def p_expr_stmt_star6(self, p): """expr_stmt : test_comma_list_opt star_expr comma_opt test_comma_list_opt equals_yield_expr_or_testlist""" targs, rhs = (p[4] or []), p[5][0] p[0] = self.store_star_expr(p[1], p[2], targs, rhs) def p_test_comma(self, p): """test_comma : test COMMA""" p[0] = [p[1]] def p_comma_opt(self, p): """comma_opt : COMMA | empty """ if len(p) == 2: p[0] = p[1] def p_test_or_star_expr(self, p): """test_or_star_expr : test | star_expr """ p[0] = p[1] def p_comma_test_or_star_expr(self, p): """comma_test_or_star_expr : COMMA test_or_star_expr""" p[0] = [p[2]] def p_testlist_star_expr(self, p): """testlist_star_expr : test_or_star_expr comma_test_or_star_expr_list comma_opt | test_or_star_expr comma_opt """ p1, p2 = p[1], p[2] if p2 is None: p0 = [p1] elif p2 == ",": p0 = [ ast.Tuple( elts=[p1], ctx=ast.Load(), lineno=p1.lineno, col_offset=p1.col_offset, ) ] else: p0 = [ ast.Tuple( elts=[p1] + p2, ctx=ast.Load(), lineno=p1.lineno, col_offset=p1.col_offset, ) ] p[0] = p0 def p_augassign(self, p): """augassign : PLUSEQUAL | MINUSEQUAL | TIMESEQUAL | ATEQUAL | DIVEQUAL | MODEQUAL | AMPERSANDEQUAL | PIPEEQUAL | XOREQUAL | LSHIFTEQUAL | RSHIFTEQUAL | POWEQUAL | DOUBLEDIVEQUAL """ p[0] = p[1] def p_yield_expr_or_testlist(self, p): """yield_expr_or_testlist : yield_expr | testlist """ p[0] = p[1] def p_equals_yield_expr_or_testlist(self, p): """equals_yield_expr_or_testlist : EQUALS yield_expr_or_testlist""" p[0] = [p[2]] # # For normal assignments, additional restrictions enforced # by the interpreter # def p_del_stmt(self, p): """del_stmt : del_tok exprlist""" p1 = p[1] p2 = p[2] for targ in p2: del_ctx(targ) p0 = ast.Delete( targets=p2, ctx=ast.Del(), lineno=p1.lineno, col_offset=p1.lexpos ) p[0] = p0 def p_pass_stmt(self, p): """pass_stmt : PASS""" p[0] = ast.Pass(lineno=self.lineno, col_offset=self.col) def p_flow_stmt(self, p): """flow_stmt : break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt """ p[0] = p[1] def p_break_stmt(self, p): """break_stmt : BREAK""" p[0] = ast.Break(lineno=self.lineno, col_offset=self.col) def p_continue_stmt(self, p): """continue_stmt : CONTINUE""" p[0] = ast.Continue(lineno=self.lineno, col_offset=self.col) def p_return_stmt(self, p): """return_stmt : return_tok testlist_opt""" p1 = p[1] p[0] = ast.Return(value=p[2], lineno=p1.lineno, col_offset=p1.lexpos) def p_yield_stmt(self, p): """yield_stmt : yield_expr""" p[0] = self.expr(p[1]) def p_raise_stmt_r1(self, p): """raise_stmt : raise_tok""" p1 = p[1] p[0] = ast.Raise(exc=None, cause=None, lineno=p1.lineno, col_offset=p1.lexpos) def p_raise_stmt_r2(self, p): """raise_stmt : raise_tok test""" p1 = p[1] p[0] = ast.Raise(exc=p[2], cause=None, lineno=p1.lineno, col_offset=p1.lexpos) def p_raise_stmt_r3(self, p): """raise_stmt : raise_tok test FROM test""" p1 = p[1] p[0] = ast.Raise(exc=p[2], cause=p[4], lineno=p1.lineno, col_offset=p1.lexpos) def p_import_stmt(self, p): """import_stmt : import_name | import_from """ p[0] = p[1] def p_import_name(self, p): """import_name : import_tok dotted_as_names """ p1 = p[1] p[0] = ast.Import(names=p[2], lineno=p1.lineno, col_offset=p1.lexpos) def p_import_from_pre_f3(self, p): """import_from_pre : from_tok period_or_ellipsis_list""" p1 = p[1] p[0] = (p[2], p1.lineno, p1.lexpos) def p_import_from_pre_f4(self, p): """import_from_pre : from_tok period_or_ellipsis_list_opt dotted_name""" p1, p2, p3 = p[1], p[2], p[3] p0 = p3 if p2 is None else p2 + p3 p[0] = (p0, p1.lineno, p1.lexpos) def p_import_from_post_times(self, p): """import_from_post : TIMES""" p[0] = [ast.alias(name="*", asname=None)] def p_import_from_post_as(self, p): """import_from_post : import_as_names""" p[0] = p[1] def p_import_from_post_paren(self, p): """import_from_post : LPAREN import_as_names""" p[0] = p[2] def p_import_from(self, p): """import_from : import_from_pre IMPORT import_from_post""" # note below: the ('.' | '...') is necessary because '...' is # tokenized as ELLIPSIS p1, lineno, col = p[1] mod = p1.lstrip(".") lvl = len(p1) - len(mod) mod = mod or None p[0] = ast.ImportFrom( module=mod, names=p[3], level=lvl, lineno=lineno, col_offset=col ) def p_period_or_ellipsis(self, p): """period_or_ellipsis : PERIOD | ELLIPSIS """ p[0] = p[1] def p_as_name(self, p): """as_name : AS NAME""" p[0] = p[2] def p_import_as_name(self, p): """import_as_name : NAME as_name_opt""" p[0] = ast.alias(name=p[1], asname=p[2]) def p_comma_import_as_name(self, p): """comma_import_as_name : COMMA import_as_name """ p[0] = [p[2]] def p_comma_import_as_name_tail(self, p): """comma_import_as_name : comma_opt RPAREN""" p[0] = list() def p_dotted_as_name(self, p): """dotted_as_name : dotted_name as_name_opt""" p0 = ast.alias(name=p[1], asname=p[2]) p[0] = p0 def p_comma_dotted_as_name(self, p): """comma_dotted_as_name : COMMA dotted_as_name""" p[0] = [p[2]] def p_import_as_names(self, p): """import_as_names : import_as_name comma_import_as_name_list_opt""" p1, p2 = p[1], p[2] p0 = [p1] if p2 is not None: p0.extend(p2) p[0] = p0 def p_dotted_as_names(self, p): """dotted_as_names : dotted_as_name comma_dotted_as_name_list_opt""" p1, p2 = p[1], p[2] p0 = [p1] if p2 is not None: p0.extend(p2) p[0] = p0 def p_period_name(self, p): """period_name : PERIOD NAME""" p[0] = p[1] + p[2] def p_dotted_name(self, p): """dotted_name : NAME | NAME period_name_list """ p[0] = p[1] if len(p) == 2 else p[1] + p[2] def p_comma_name(self, p): """comma_name : COMMA NAME""" p[0] = [p[2]] def p_global_stmt(self, p): """global_stmt : global_tok NAME comma_name_list_opt""" p1, p2, p3 = p[1], p[2], p[3] names = [p2] if p3 is not None: names += p3 p[0] = ast.Global(names=names, lineno=p1.lineno, col_offset=p1.lexpos) def p_nonlocal_stmt(self, p): """nonlocal_stmt : nonlocal_tok NAME comma_name_list_opt""" p1, p2, p3 = p[1], p[2], p[3] names = [p2] if p3 is not None: names += p3 p[0] = ast.Nonlocal(names=names, lineno=p1.lineno, col_offset=p1.lexpos) def p_comma_test(self, p): """comma_test : COMMA test""" p[0] = [p[2]] def p_assert_stmt(self, p): """assert_stmt : assert_tok test comma_test_opt""" p1, p2, p3 = p[1], p[2], p[3] if p3 is not None: if len(p3) != 1: assert False p3 = p3[0] p[0] = ast.Assert(test=p2, msg=p3, lineno=p1.lineno, col_offset=p1.lexpos) def p_compound_stmt(self, p): """compound_stmt : if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated """ p[0] = p[1] def p_elif_part(self, p): """elif_part : ELIF test COLON suite""" p2 = p[2] p[0] = [ ast.If( test=p2, body=p[4], orelse=[], lineno=p2.lineno, col_offset=p2.col_offset, ) ] def p_else_part(self, p): """else_part : ELSE COLON suite""" p[0] = p[3] def p_if_stmt(self, p): """if_stmt : if_tok test COLON suite elif_part_list_opt | if_tok test COLON suite elif_part_list_opt else_part """ p1 = p[1] lastif = ast.If( test=p[2], body=p[4], orelse=[], lineno=p1.lineno, col_offset=p1.lexpos ) p0 = [lastif] p5 = p[5] p6 = p[6] if len(p) > 6 else [] if p5 is not None: for elseif in p5: lastif.orelse.append(elseif) lastif = elseif lastif.orelse = p6 p[0] = p0 def p_while_stmt(self, p): """while_stmt : WHILE test COLON suite | WHILE test COLON suite else_part """ p5 = p[5] if len(p) > 5 else [] p[0] = [ ast.While( test=p[2], body=p[4], orelse=p5, lineno=self.lineno, col_offset=self.col ) ] def p_for_stmt(self, p): """for_stmt : for_tok exprlist IN testlist COLON suite | for_tok exprlist IN testlist COLON suite else_part """ p1, p2 = p[1], p[2] p7 = p[7] if len(p) > 7 else [] if len(p2) == 1: p2 = p2[0] store_ctx(p2) else: for x in p2: store_ctx(x) p2 = ast.Tuple( elts=p2, ctx=ast.Store(), lineno=p2[0].lineno, col_offset=p2[0].col_offset, ) p[0] = [ ast.For( target=p2, iter=p[4], body=p[6], orelse=p7, lineno=p1.lineno, col_offset=p1.lexpos, ) ] def p_except_part(self, p): """except_part : except_clause COLON suite""" p0 = p[1] p0.body = p[3] p[0] = [p0] def p_finally_part(self, p): """finally_part : FINALLY COLON suite""" p[0] = p[3] def p_try_stmt_t5(self, p): """try_stmt : try_tok COLON suite finally_part""" p1 = p[1] p[0] = [ ast.Try( body=p[3], handlers=[], orelse=[], finalbody=p[4], lineno=p1.lineno, col_offset=p1.lexpos, ) ] def p_try_stmt_t6(self, p): """try_stmt : try_tok COLON suite except_part_list finally_part_opt""" p1 = p[1] p[0] = [ ast.Try( body=p[3], handlers=p[4], orelse=[], finalbody=([] if p[5] is None else p[5]), lineno=p1.lineno, col_offset=p1.lexpos, ) ] def p_try_stmt_t7(self, p): """try_stmt : try_tok COLON suite except_part_list else_part finally_part_opt""" p1 = p[1] p[0] = [ ast.Try( body=p[3], handlers=p[4], orelse=([] if p[5] is None else p[5]), finalbody=([] if p[6] is None else p[6]), lineno=p1.lineno, col_offset=p1.lexpos, ) ] def p_with_stmt_w5(self, p): """with_stmt : with_tok with_item COLON suite""" p1 = p[1] p[0] = [ ast.With(items=[p[2]], body=p[4], lineno=p1.lineno, col_offset=p1.lexpos) ] def p_with_stmt_p6(self, p): """with_stmt : with_tok with_item comma_with_item_list COLON suite""" p1 = p[1] p[0] = [ ast.With( items=[p[2]] + p[3], body=p[5], lineno=p1.lineno, col_offset=p1.lexpos ) ] def p_with_bang_stmt_single_suite(self, p): """with_stmt : with_tok BANG with_item rawsuite""" p1, p3, p4 = p[1], p[3], p[4] expr = p3.context_expr l, c = expr.lineno, expr.col_offset gblcall = xonsh_call("globals", [], lineno=l, col=c) loccall = xonsh_call("locals", [], lineno=l, col=c) margs = [expr, p4, gblcall, loccall] p3.context_expr = xonsh_call("__xonsh__.enter_macro", margs, lineno=l, col=c) body = [ast.Pass(lineno=p4.lineno, col_offset=p4.col_offset)] p[0] = [ast.With(items=[p3], body=body, lineno=p1.lineno, col_offset=p1.lexpos)] def p_with_bang_stmt_many_suite(self, p): """with_stmt : with_tok BANG with_item comma_with_item_list rawsuite""" p1, p3, p4, p5 = p[1], p[3], p[4], p[5] items = [p3] + p4 for item in items: expr = item.context_expr l, c = expr.lineno, expr.col_offset gblcall = xonsh_call("globals", [], lineno=l, col=c) loccall = xonsh_call("locals", [], lineno=l, col=c) margs = [expr, p5, gblcall, loccall] item.context_expr = xonsh_call( "__xonsh__.enter_macro", margs, lineno=l, col=c ) body = [ast.Pass(lineno=p5.lineno, col_offset=p5.col_offset)] p[0] = [ ast.With(items=items, body=body, lineno=p1.lineno, col_offset=p1.lexpos) ] def p_as_expr(self, p): """as_expr : AS expr""" p2 = p[2] store_ctx(p2) p[0] = p2 def p_with_item(self, p): """with_item : test | test as_expr """ p2 = p[2] if len(p) > 2 else None p[0] = ast.withitem(context_expr=p[1], optional_vars=p2) def p_comma_with_item(self, p): """comma_with_item : COMMA with_item""" p[0] = [p[2]] def p_except_clause_e2(self, p): """except_clause : except_tok""" p1 = p[1] p[0] = ast.ExceptHandler( type=None, name=None, lineno=p1.lineno, col_offset=p1.lexpos ) def p_except_clause(self, p): """except_clause : except_tok test as_name_opt""" p1 = p[1] p[0] = ast.ExceptHandler( type=p[2], name=p[3], lineno=p1.lineno, col_offset=p1.lexpos ) def p_suite(self, p): """suite : simple_stmt | NEWLINE INDENT stmt_list DEDENT """ p[0] = p[1] if len(p) == 2 else p[3] def p_rawsuite_indent(self, p): """rawsuite : COLON NEWLINE indent_tok nodedent dedent_tok""" p3, p5 = p[3], p[5] beg = (p3.lineno, p3.lexpos) end = (p5.lineno, p5.lexpos) s = self.source_slice(beg, end) s = textwrap.dedent(s) p[0] = ast.Str(s=s, lineno=beg[0], col_offset=beg[1]) def p_rawsuite_simple_stmt(self, p): """rawsuite : colon_tok nonewline newline_tok""" p1, p3 = p[1], p[3] beg = (p1.lineno, p1.lexpos + 1) end = (p3.lineno, p3.lexpos) s = self.source_slice(beg, end).strip() p[0] = ast.Str(s=s, lineno=beg[0], col_offset=beg[1]) def _attach_nodedent_base_rules(self): toks = set(self.tokens) toks.remove("DEDENT") ts = "\n | ".join(sorted(toks)) doc = "nodedent : " + ts + "\n" self.p_nodedent_base.__func__.__doc__ = doc def p_nodedent_base(self, p): # see above attachment function pass def p_nodedent_any(self, p): """nodedent : INDENT any_dedent_toks DEDENT""" pass def p_nodedent_many(self, p): """nodedent : nodedent nodedent""" pass def p_any_dedent_tok(self, p): """any_dedent_tok : nodedent | DEDENT """ pass def p_any_dedent_toks(self, p): """any_dedent_toks : any_dedent_tok | any_dedent_toks any_dedent_tok """ pass def _attach_nonewline_base_rules(self): toks = set(self.tokens) toks -= { "NEWLINE", "LPAREN", "RPAREN", "LBRACE", "RBRACE", "LBRACKET", "RBRACKET", "AT_LPAREN", "BANG_LPAREN", "BANG_LBRACKET", "DOLLAR_LPAREN", "DOLLAR_LBRACE", "DOLLAR_LBRACKET", "ATDOLLAR_LPAREN", } ts = "\n | ".join(sorted(toks)) doc = "nonewline : " + ts + "\n" self.p_nonewline_base.__func__.__doc__ = doc def p_nonewline_base(self, p): # see above attachment function pass def p_nonewline_any(self, p): """nonewline : any_nested_raw""" pass def p_nonewline_many(self, p): """nonewline : nonewline nonewline""" pass def p_test_ol(self, p): """test : or_test | lambdef """ p[0] = p[1] def p_test_o5(self, p): """test : or_test IF or_test ELSE test""" p[0] = ast.IfExp( test=p[3], body=p[1], orelse=p[5], lineno=self.lineno, col_offset=self.col ) def p_test_nocond(self, p): """test_nocond : or_test | lambdef_nocond """ p[0] = p[1] def p_lambdef(self, p): """lambdef : lambda_tok varargslist_opt COLON test""" p1, p2, p4 = p[1], p[2], p[4] if p2 is None: args = ast.arguments( args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[], ) else: args = p2 p0 = ast.Lambda(args=args, body=p4, lineno=p1.lineno, col_offset=p1.lexpos) p[0] = p0 def p_lambdef_nocond(self, p): """lambdef_nocond : LAMBDA varargslist_opt COLON test_nocond""" assert False def p_or_test(self, p): """or_test : and_test or_and_test_list_opt""" p1, p2 = p[1], p[2] if p2 is None: p0 = p1 elif len(p2) == 2: lineno, col = lopen_loc(p1) p0 = ast.BoolOp(op=p2[0], values=[p1, p2[1]], lineno=lineno, col_offset=col) else: lineno, col = lopen_loc(p1) p0 = ast.BoolOp( op=p2[0], values=[p[1]] + p2[1::2], lineno=lineno, col_offset=col ) p[0] = p0 def p_or_and_test(self, p): """or_and_test : OR and_test""" p[0] = [ast.Or(), p[2]] def p_and_test(self, p): """and_test : not_test and_not_test_list_opt""" p1, p2 = p[1], p[2] if p2 is None: p0 = p1 elif len(p2) == 2: lineno, col = lopen_loc(p1) p0 = ast.BoolOp(op=p2[0], values=[p1, p2[1]], lineno=lineno, col_offset=col) else: lineno, col = lopen_loc(p1) p0 = ast.BoolOp( op=p2[0], values=[p1] + p2[1::2], lineno=lineno, col_offset=col ) p[0] = p0 def p_and_not_test(self, p): """and_not_test : AND not_test""" p[0] = [ast.And(), p[2]] def p_not_test_not(self, p): """not_test : NOT not_test""" p[0] = ast.UnaryOp( op=ast.Not(), operand=p[2], lineno=self.lineno, col_offset=self.col ) def p_not_test(self, p): """not_test : comparison""" p[0] = p[1] def p_comparison(self, p): """comparison : expr comp_op_expr_list_opt""" p1, p2 = p[1], p[2] if p2 is None: p0 = p1 else: p0 = ast.Compare( left=p1, ops=p2[::2], comparators=p2[1::2], lineno=p1.lineno, col_offset=p1.col_offset, ) p[0] = p0 def p_comp_op_expr(self, p): """comp_op_expr : comp_op expr""" p[0] = [p[1], p[2]] _comp_ops = { "<": ast.Lt, ">": ast.Gt, "==": ast.Eq, ">=": ast.GtE, "<=": ast.LtE, "!=": ast.NotEq, "in": ast.In, ("not", "in"): ast.NotIn, "is": ast.Is, ("is", "not"): ast.IsNot, } def p_comp_op_monograph(self, p): """comp_op : LT | GT | EQ | GE | LE | NE | IN | IS """ p[0] = self._comp_ops[p[1]]() def p_comp_op_digraph(self, p): """comp_op : NOT IN | IS NOT """ p[0] = self._comp_ops[(p[1], p[2])]() def p_star_expr(self, p): """star_expr : times_tok expr""" p1 = p[1] p[0] = ast.Starred( value=p[2], ctx=ast.Load(), lineno=p1.lineno, col_offset=p1.lexpos ) def _binop_combine(self, p1, p2): """Combines binary operations""" if p2 is None: p0 = p1 elif isinstance(p2, ast.BinOp): p2.left = p1 p0 = p2 elif isinstance(p2, Sequence) and isinstance(p2[0], ast.BinOp): p0 = p2[0] p0.left = p1 p0.lineno, p0.col_offset = lopen_loc(p1) for bop in p2[1:]: locer = p1 if p0.left is p1 else p0 bop.left = p0 p0.lineno, p0.col_offset = lopen_loc(locer) p0 = bop else: p0 = p1 + p2 return p0 def p_expr(self, p): """expr : xor_expr | xor_expr pipe_xor_expr_list """ p[0] = self._binop_combine(p[1], p[2] if len(p) > 2 else None) def p_pipe_xor_expr(self, p): """pipe_xor_expr : pipe_tok xor_expr""" p1 = p[1] p[0] = [ ast.BinOp( left=None, op=ast.BitOr(), right=p[2], lineno=p1.lineno, col_offset=p1.lexpos, ) ] def p_xor_expr(self, p): """xor_expr : and_expr xor_and_expr_list_opt""" p[0] = self._binop_combine(p[1], p[2]) def p_xor_and_expr(self, p): """xor_and_expr : xor_tok and_expr""" p1 = p[1] p[0] = [ ast.BinOp( left=None, op=ast.BitXor(), right=p[2], lineno=p1.lineno, col_offset=p1.lexpos, ) ] def p_and_expr(self, p): """and_expr : shift_expr ampersand_shift_expr_list_opt""" p[0] = self._binop_combine(p[1], p[2]) def p_ampersand_shift_expr(self, p): """ampersand_shift_expr : ampersand_tok shift_expr""" p1 = p[1] p[0] = [ ast.BinOp( left=None, op=ast.BitAnd(), right=p[2], lineno=p1.lineno, col_offset=p1.lexpos, ) ] def p_shift_expr(self, p): """shift_expr : arith_expr shift_arith_expr_list_opt""" p[0] = self._binop_combine(p[1], p[2]) def p_shift_arith_expr(self, p): """shift_arith_expr : lshift_tok arith_expr | rshift_tok arith_expr """ p1 = p[1] op = ast.LShift() if p1.value == "<<" else ast.RShift() p[0] = [ ast.BinOp( left=None, op=op, right=p[2], lineno=p1.lineno, col_offset=p1.lexpos ) ] def p_arith_expr_single(self, p): """arith_expr : term""" p[0] = p[1] def p_arith_expr_many(self, p): """arith_expr : term pm_term_list""" p1, p2 = p[1], p[2] if len(p2) == 2: lineno, col = lopen_loc(p1) p0 = ast.BinOp( left=p1, op=p2[0], right=p2[1], lineno=lineno, col_offset=col ) else: left = p1 for op, right in zip(p2[::2], p2[1::2]): locer = left if left is p1 else op lineno, col = lopen_loc(locer) left = ast.BinOp( left=left, op=op, right=right, lineno=lineno, col_offset=col ) p0 = left p[0] = p0 _term_binops = { "+": ast.Add, "-": ast.Sub, "*": ast.Mult, "@": ast.MatMult, "/": ast.Div, "%": ast.Mod, "//": ast.FloorDiv, } def p_pm_term(self, p): """pm_term : plus_tok term | minus_tok term """ p1 = p[1] op = self._term_binops[p1.value](lineno=p1.lineno, col_offset=p1.lexpos) p[0] = [op, p[2]] def p_term(self, p): """term : factor op_factor_list_opt""" p1, p2 = p[1], p[2] if p2 is None: p0 = p1 elif len(p2) == 2: lineno, col = lopen_loc(p1) p0 = ast.BinOp( left=p1, op=p2[0], right=p2[1], lineno=lineno, col_offset=col ) else: left = p1 for op, right in zip(p2[::2], p2[1::2]): locer = left if left is p1 else op lineno, col = lopen_loc(locer) left = ast.BinOp( left=left, op=op, right=right, lineno=lineno, col_offset=col ) p0 = left p[0] = p0 def p_op_factor(self, p): """op_factor : times_tok factor | at_tok factor | divide_tok factor | mod_tok factor | doublediv_tok factor """ p1 = p[1] op = self._term_binops[p1.value] if op is None: self._parse_error( "operation {0!r} not supported".format(p1), self.currloc(lineno=p.lineno, column=p.lexpos), ) p[0] = [op(lineno=p1.lineno, col_offset=p1.lexpos), p[2]] _factor_ops = {"+": ast.UAdd, "-": ast.USub, "~": ast.Invert} def p_factor_power(self, p): """factor : power""" p[0] = p[1] def p_factor_unary(self, p): """factor : PLUS factor | MINUS factor | TILDE factor """ op = self._factor_ops[p[1]]() p[0] = ast.UnaryOp(op=op, operand=p[2], lineno=self.lineno, col_offset=self.col) def p_power_atom(self, p): """power : atom_expr""" p[0] = p[1] def p_power(self, p): """power : atom_expr POW factor""" p1 = p[1] p[0] = ast.BinOp( left=p1, op=ast.Pow(), right=p[3], lineno=p1.lineno, col_offset=p1.col_offset, ) def p_yield_expr_or_testlist_comp(self, p): """yield_expr_or_testlist_comp : yield_expr | testlist_comp """ p[0] = p[1] def _list_or_elts_if_not_real_tuple(self, x): if isinstance(x, ast.Tuple) and not ( hasattr(x, "_real_tuple") and x._real_tuple ): rtn = x.elts else: rtn = [x] return rtn def apply_trailers(self, leader, trailers): """Helper function for atom expr.""" if trailers is None: return leader p0 = leader for trailer in trailers: if isinstance(trailer, (ast.Index, ast.Slice, ast.ExtSlice)): p0 = ast.Subscript( value=leader, slice=trailer, ctx=ast.Load(), lineno=leader.lineno, col_offset=leader.col_offset, ) elif isinstance(trailer, Mapping): # call normal functions p0 = ast.Call( func=leader, lineno=leader.lineno, col_offset=leader.col_offset, **trailer ) elif isinstance(trailer, (ast.Tuple, tuple)): # call macro functions l, c = leader.lineno, leader.col_offset gblcall = xonsh_call("globals", [], lineno=l, col=c) loccall = xonsh_call("locals", [], lineno=l, col=c) if isinstance(trailer, tuple): trailer, arglist = trailer margs = [leader, trailer, gblcall, loccall] p0 = xonsh_call("__xonsh__.call_macro", margs, lineno=l, col=c) elif isinstance(trailer, str): if trailer == "?": p0 = xonsh_help(leader, lineno=leader.lineno, col=leader.col_offset) elif trailer == "??": p0 = xonsh_superhelp( leader, lineno=leader.lineno, col=leader.col_offset ) else: p0 = ast.Attribute( value=leader, attr=trailer, ctx=ast.Load(), lineno=leader.lineno, col_offset=leader.col_offset, ) else: assert False leader = p0 return p0 def p_atom_expr(self, p): """atom_expr : atom trailer_list_opt""" p[0] = self.apply_trailers(p[1], p[2]) # # Atom rules! (So does Adam!) # def p_atom_lparen(self, p): """atom : lparen_tok yield_expr_or_testlist_comp_opt RPAREN""" p1, p2 = p[1], p[2] p1, p1_tok = p1.value, p1 if p2 is None: # empty container atom p0 = ast.Tuple( elts=[], ctx=ast.Load(), lineno=self.lineno, col_offset=self.col ) p0._real_tuple = True elif isinstance(p2, ast.AST): p0 = p2 p0._lopen_lineno, p0._lopen_col = p1_tok.lineno, p1_tok.lexpos p0._real_tuple = True elif len(p2) == 1 and isinstance(p2[0], ast.AST): p0 = p2[0] p0._lopen_lineno, p0._lopen_col = p1_tok.lineno, p1_tok.lexpos else: self.p_error(p) p[0] = p0 def p_atom_lbraket(self, p): """atom : lbracket_tok testlist_comp_opt RBRACKET""" p1, p2 = p[1], p[2] p1, p1_tok = p1.value, p1 if p2 is None: p0 = ast.List( elts=[], ctx=ast.Load(), lineno=self.lineno, col_offset=self.col ) elif isinstance(p2, ast.GeneratorExp): p0 = ast.ListComp( elt=p2.elt, generators=p2.generators, lineno=p2.lineno, col_offset=p2.col_offset, ) else: if isinstance(p2, ast.Tuple): if hasattr(p2, "_real_tuple") and p2._real_tuple: elts = [p2] else: elts = p2.elts else: elts = [p2] p0 = ast.List( elts=elts, ctx=ast.Load(), lineno=p1_tok.lineno, col_offset=p1_tok.lexpos, ) p[0] = p0 def p_atom_lbrace(self, p): """atom : lbrace_tok dictorsetmaker_opt RBRACE""" p1, p2 = p[1], p[2] p1, p1_tok = p1.value, p1 if p2 is None: p0 = ast.Dict( keys=[], values=[], ctx=ast.Load(), lineno=self.lineno, col_offset=self.col, ) else: p0 = p2 p0.lineno, p0.col_offset = p1_tok.lineno, p1_tok.lexpos p[0] = p0 def p_atom_ns(self, p): """atom : number | string_literal_list """ p[0] = p[1] def p_atom_name(self, p): """atom : name_tok""" p1 = p[1] p[0] = ast.Name( id=p1.value, ctx=ast.Load(), lineno=p1.lineno, col_offset=p1.lexpos ) def p_atom_ellip(self, p): """atom : ellipsis_tok""" p1 = p[1] p[0] = ast.EllipsisNode(lineno=p1.lineno, col_offset=p1.lexpos) def p_atom_none(self, p): """atom : none_tok""" p1 = p[1] p[0] = ast.NameConstant(value=None, lineno=p1.lineno, col_offset=p1.lexpos) def p_atom_true(self, p): """atom : true_tok""" p1 = p[1] p[0] = ast.NameConstant(value=True, lineno=p1.lineno, col_offset=p1.lexpos) def p_atom_false(self, p): """atom : false_tok""" p1 = p[1] p[0] = ast.NameConstant(value=False, lineno=p1.lineno, col_offset=p1.lexpos) def p_atom_pathsearch(self, p): """atom : SEARCHPATH""" p[0] = xonsh_pathsearch(p[1], pymode=True, lineno=self.lineno, col=self.col) def p_atom_dname(self, p): """atom : DOLLAR_NAME""" p[0] = self._envvar_by_name(p[1][1:], lineno=self.lineno, col=self.col) def p_atom_fistful_of_dollars(self, p): """atom : dollar_lbrace_tok test RBRACE | bang_lparen_tok subproc RPAREN | dollar_lparen_tok subproc RPAREN | bang_lbracket_tok subproc RBRACKET | dollar_lbracket_tok subproc RBRACKET """ p[0] = self._dollar_rules(p) def p_atom_bang_empty_fistful_of_dollars(self, p): """atom : bang_lparen_tok subproc bang_tok RPAREN | dollar_lparen_tok subproc bang_tok RPAREN | bang_lbracket_tok subproc bang_tok RBRACKET | dollar_lbracket_tok subproc bang_tok RBRACKET """ self._append_subproc_bang_empty(p) p[0] = self._dollar_rules(p) def p_atom_bang_fistful_of_dollars(self, p): """atom : bang_lparen_tok subproc bang_tok nocloser rparen_tok | dollar_lparen_tok subproc bang_tok nocloser rparen_tok | bang_lbracket_tok subproc bang_tok nocloser rbracket_tok | dollar_lbracket_tok subproc bang_tok nocloser rbracket_tok """ self._append_subproc_bang(p) p[0] = self._dollar_rules(p) def _attach_nocloser_base_rules(self): toks = set(self.tokens) toks -= { "LPAREN", "RPAREN", "LBRACE", "RBRACE", "LBRACKET", "RBRACKET", "AT_LPAREN", "BANG_LPAREN", "BANG_LBRACKET", "DOLLAR_LPAREN", "DOLLAR_LBRACE", "DOLLAR_LBRACKET", "ATDOLLAR_LPAREN", } ts = "\n | ".join(sorted(toks)) doc = "nocloser : " + ts + "\n" self.p_nocloser_base.__func__.__doc__ = doc def p_nocloser_base(self, p): # see above attachment function pass def p_nocloser_any(self, p): """nocloser : any_nested_raw""" pass def p_nocloser_many(self, p): """nocloser : nocloser nocloser""" pass def p_string_literal(self, p): """string_literal : string_tok""" p1 = p[1] prefix = RE_STRINGPREFIX.match(p1.value).group().lower() if "p" in prefix and "f" in prefix: new_pref = prefix.replace("p", "") value_without_p = new_pref + p1.value[len(prefix):] s = eval_fstr_fields(value_without_p, new_pref, filename=self.lexer.fname) s = pyparse(s).body[0].value s = ast.increment_lineno(s, p1.lineno - 1) p[0] = xonsh_call( "__xonsh__.path_literal", [s], lineno=p1.lineno, col=p1.lexpos ) elif "p" in prefix: value_without_p = prefix.replace("p", "") + p1.value[len(prefix) :] s = ast.Str( s=ast.literal_eval(value_without_p), lineno=p1.lineno, col_offset=p1.lexpos, ) p[0] = xonsh_call( "__xonsh__.path_literal", [s], lineno=p1.lineno, col=p1.lexpos ) elif "f" in prefix: s = eval_fstr_fields(p1.value, prefix, filename=self.lexer.fname) s = pyparse(s).body[0].value s = ast.increment_lineno(s, p1.lineno - 1) p[0] = s else: s = ast.literal_eval(p1.value) is_bytes = "b" in prefix cls = ast.Bytes if is_bytes else ast.Str p[0] = cls(s=s, lineno=p1.lineno, col_offset=p1.lexpos) def p_string_literal_list(self, p): """string_literal_list : string_literal | string_literal_list string_literal """ if len(p) == 3: p[1].s += p[2].s p[0] = p[1] def p_number(self, p): """number : number_tok""" p1 = p[1] p[0] = ast.Num( n=ast.literal_eval(p1.value.replace("_", "")), lineno=p1.lineno, col_offset=p1.lexpos, ) def p_testlist_comp_comp(self, p): """testlist_comp : test_or_star_expr comp_for""" p1, p2 = p[1], p[2] p[0] = ast.GeneratorExp( elt=p1, generators=p2["comps"], lineno=p1.lineno, col_offset=p1.col_offset ) def p_testlist_comp_comma(self, p): """testlist_comp : test_or_star_expr comma_opt""" p1, p2 = p[1], p[2] if p2 is None: # split out grouping parentheses. p[0] = p1 else: p[0] = ast.Tuple( elts=[p1], ctx=ast.Load(), lineno=p1.lineno, col_offset=p1.col_offset ) def p_testlist_comp_many(self, p): """testlist_comp : test_or_star_expr comma_test_or_star_expr_list comma_opt""" p1, p2 = p[1], p[2] p[0] = ast.Tuple( elts=[p1] + p2, ctx=ast.Load(), lineno=p1.lineno, col_offset=p1.col_offset ) def p_trailer_lparen(self, p): """trailer : LPAREN arglist_opt RPAREN""" p[0] = [p[2] or dict(args=[], keywords=[], starargs=None, kwargs=None)] def p_trailer_bang_lparen(self, p): """trailer : bang_lparen_tok macroarglist_opt rparen_tok | bang_lparen_tok nocomma comma_tok rparen_tok | bang_lparen_tok nocomma comma_tok WS rparen_tok | bang_lparen_tok macroarglist comma_tok rparen_tok | bang_lparen_tok macroarglist comma_tok WS rparen_tok """ p1, p2, p3 = p[1], p[2], p[3] begins = [(p1.lineno, p1.lexpos + 2)] ends = [(p3.lineno, p3.lexpos)] if p2: begins.extend([(x[0], x[1] + 1) for x in p2]) ends = p2 + ends elts = [] for beg, end in zip(begins, ends): s = self.source_slice(beg, end).strip() if not s: if len(begins) == 1: break else: msg = "empty macro arguments not allowed" self._parse_error(msg, self.currloc(*beg)) node = ast.Str(s=s, lineno=beg[0], col_offset=beg[1]) elts.append(node) p0 = ast.Tuple( elts=elts, ctx=ast.Load(), lineno=p1.lineno, col_offset=p1.lexpos ) p[0] = [p0] def p_trailer_p3(self, p): """trailer : LBRACKET subscriptlist RBRACKET | PERIOD NAME """ p[0] = [p[2]] def p_trailer_quest(self, p): """trailer : DOUBLE_QUESTION | QUESTION """ p[0] = [p[1]] def _attach_nocomma_tok_rules(self): toks = set(self.tokens) toks -= { "COMMA", "LPAREN", "RPAREN", "LBRACE", "RBRACE", "LBRACKET", "RBRACKET", "AT_LPAREN", "BANG_LPAREN", "BANG_LBRACKET", "DOLLAR_LPAREN", "DOLLAR_LBRACE", "DOLLAR_LBRACKET", "ATDOLLAR_LPAREN", } ts = "\n | ".join(sorted(toks)) doc = "nocomma_tok : " + ts + "\n" self.p_nocomma_tok.__func__.__doc__ = doc # The following grammar rules are no-ops because we don't need to glue the # source code back together piece-by-piece. Instead, we simply look for # top-level commas and record their positions. With these positions and the # respective positions of the bounding parentheses, we can use the # source_slice() method. This does a much better job of capturing exactly # the source code that was provided. The tokenizer & lexer can be a little # lossy, especially with respect to whitespace. def p_nocomma_tok(self, p): # see attachment function above for docstring pass def p_any_raw_tok(self, p): """any_raw_tok : nocomma | COMMA """ pass def p_any_raw_toks_one(self, p): """any_raw_toks : any_raw_tok""" pass def p_any_raw_toks_many(self, p): """any_raw_toks : any_raw_toks any_raw_tok""" pass def p_nocomma_part_tok(self, p): """nocomma_part : nocomma_tok""" pass def p_any_nested_raw(self, p): """any_nested_raw : LPAREN any_raw_toks_opt RPAREN | LBRACE any_raw_toks_opt RBRACE | LBRACKET any_raw_toks_opt RBRACKET | AT_LPAREN any_raw_toks_opt RPAREN | BANG_LPAREN any_raw_toks_opt RPAREN | BANG_LBRACKET any_raw_toks_opt RBRACKET | DOLLAR_LPAREN any_raw_toks_opt RPAREN | DOLLAR_LBRACE any_raw_toks_opt RBRACE | DOLLAR_LBRACKET any_raw_toks_opt RBRACKET | ATDOLLAR_LPAREN any_raw_toks_opt RPAREN """ pass def p_nocomma_part_any(self, p): """nocomma_part : any_nested_raw""" pass def p_nocomma_base(self, p): """nocomma : nocomma_part""" pass def p_nocomma_append(self, p): """nocomma : nocomma nocomma_part""" pass def p_comma_nocomma(self, p): """comma_nocomma : comma_tok nocomma""" p1 = p[1] p[0] = [(p1.lineno, p1.lexpos)] def p_macroarglist_single(self, p): """macroarglist : nocomma""" p[0] = [] def p_macroarglist_many(self, p): """macroarglist : nocomma comma_nocomma_list""" p[0] = p[2] def p_subscriptlist(self, p): """subscriptlist : subscript comma_subscript_list_opt comma_opt""" p1, p2 = p[1], p[2] if p2 is None: pass elif isinstance(p1, ast.Slice) or any([isinstance(x, ast.Slice) for x in p2]): p1 = ast.ExtSlice(dims=[p1] + p2) else: p1.value = ast.Tuple( elts=[p1.value] + [x.value for x in p2], ctx=ast.Load(), lineno=p1.lineno, col_offset=p1.col_offset, ) p[0] = p1 def p_comma_subscript(self, p): """comma_subscript : COMMA subscript""" p[0] = [p[2]] def p_subscript_test(self, p): """subscript : test""" p1 = p[1] p[0] = ast.Index(value=p1, lineno=p1.lineno, col_offset=p1.col_offset) def p_subscript_tok(self, p): """subscript : test_opt colon_tok test_opt sliceop_opt""" p1 = p[1] if p1 is None: p2 = p[2] lineno, col = p2.lineno, p2.lexpos else: lineno, col = p1.lineno, p1.col_offset p[0] = ast.Slice(lower=p1, upper=p[3], step=p[4], lineno=lineno, col_offset=col) def p_sliceop(self, p): """sliceop : COLON test_opt""" p[0] = p[2] def p_expr_or_star_expr(self, p): """expr_or_star_expr : expr | star_expr """ p[0] = p[1] def p_comma_expr_or_star_expr(self, p): """comma_expr_or_star_expr : COMMA expr_or_star_expr""" p[0] = [p[2]] def p_exprlist_e3(self, p): """exprlist : expr_or_star_expr comma_opt""" p[0] = [p[1]] def p_exprlist_many(self, p): """exprlist : expr_or_star_expr comma_expr_or_star_expr_list comma_opt""" p2 = p[2] p2.insert(0, p[1]) p[0] = p2 def p_testlist_test(self, p): """testlist : test""" p1 = p[1] if isinstance(p1, ast.Tuple) and ( hasattr(p1, "_real_tuple") and p1._real_tuple and p1.elts ): p1.lineno, p1.col_offset = lopen_loc(p1.elts[0]) p[0] = p1 def p_testlist_single(self, p): """testlist : test COMMA""" p1 = p[1] if isinstance(p1, ast.List) or ( isinstance(p1, ast.Tuple) and hasattr(p1, "_real_tuple") and p1._real_tuple ): lineno, col = lopen_loc(p1) p[0] = ast.Tuple( elts=[p1], ctx=ast.Load(), lineno=p1.lineno, col_offset=p1.col_offset ) else: p[0] = ensure_has_elts(p[1]) def p_testlist_many(self, p): """testlist : test comma_test_list COMMA | test comma_test_list """ p1 = p[1] if isinstance(p1, ast.List) or ( isinstance(p1, ast.Tuple) and hasattr(p1, "_real_tuple") and p1._real_tuple ): lineno, col = lopen_loc(p1) p1 = ast.Tuple( elts=[p1], ctx=ast.Load(), lineno=p1.lineno, col_offset=p1.col_offset ) else: p1 = ensure_has_elts(p1) p1.elts += p[2] p[0] = p1 def p_comma_item(self, p): """comma_item : COMMA item""" p[0] = p[2] # # Dict or set maker # def p_dictorsetmaker_t6(self, p): """dictorsetmaker : test COLON test comma_item_list comma_opt""" p1, p4 = p[1], p[4] keys = [p1] vals = [p[3]] for k, v in zip(p4[::2], p4[1::2]): keys.append(k) vals.append(v) lineno, col = lopen_loc(p1) p[0] = ast.Dict( keys=keys, values=vals, ctx=ast.Load(), lineno=lineno, col_offset=col ) def p_dictorsetmaker_i4(self, p): """dictorsetmaker : item comma_item_list comma_opt""" p1, p2 = p[1], p[2] keys = [p1[0]] vals = [p1[1]] for k, v in zip(p2[::2], p2[1::2]): keys.append(k) vals.append(v) lineno, col = lopen_loc(p1[0] or p2[0]) p[0] = ast.Dict( keys=keys, values=vals, ctx=ast.Load(), lineno=lineno, col_offset=col ) def p_dictorsetmaker_t4_dict(self, p): """dictorsetmaker : test COLON testlist""" keys = [p[1]] vals = self._list_or_elts_if_not_real_tuple(p[3]) lineno, col = lopen_loc(p[1]) p[0] = ast.Dict( keys=keys, values=vals, ctx=ast.Load(), lineno=lineno, col_offset=col ) def p_dictorsetmaker_t4_set(self, p): """dictorsetmaker : test_or_star_expr comma_test_or_star_expr_list comma_opt""" p[0] = ast.Set( elts=[p[1]] + p[2], ctx=ast.Load(), lineno=self.lineno, col_offset=self.col ) def p_dictorsetmaker_test_comma(self, p): """dictorsetmaker : test_or_star_expr comma_opt""" elts = self._list_or_elts_if_not_real_tuple(p[1]) p[0] = ast.Set( elts=elts, ctx=ast.Load(), lineno=self.lineno, col_offset=self.col ) def p_dictorsetmaker_testlist(self, p): """dictorsetmaker : testlist""" elts = self._list_or_elts_if_not_real_tuple(p[1]) p[0] = ast.Set( elts=elts, ctx=ast.Load(), lineno=self.lineno, col_offset=self.col ) def p_dictorsetmaker_comp(self, p): """dictorsetmaker : item comp_for | test_or_star_expr comp_for """ p1 = p[1] comps = p[2].get("comps", []) if isinstance(p1, list) and len(p1) == 2: p[0] = ast.DictComp( key=p1[0], value=p1[1], generators=comps, lineno=self.lineno, col_offset=self.col, ) else: p[0] = ast.SetComp( elt=p1, generators=comps, lineno=self.lineno, col_offset=self.col ) def p_classdef(self, p): """classdef : class_tok NAME func_call_opt COLON suite""" p1, p3 = p[1], p[3] b, kw = ([], []) if p3 is None else (p3["args"], p3["keywords"]) c = ast.ClassDef( name=p[2], bases=b, keywords=kw, starargs=None, kwargs=None, body=p[5], decorator_list=[], lineno=p1.lineno, col_offset=p1.lexpos, ) p[0] = [c] def p_comma_argument(self, p): """comma_argument : COMMA argument""" p[0] = [p[2]] def p_comp_iter(self, p): """comp_iter : comp_for | comp_if """ p[0] = p[1] def p_comp_for(self, p): """comp_for : FOR exprlist IN or_test comp_iter_opt""" targs, it, p5 = p[2], p[4], p[5] if len(targs) == 1: targ = targs[0] else: targ = ensure_has_elts(targs) store_ctx(targ) comp = ast.comprehension(target=targ, iter=it, ifs=[]) comps = [comp] p0 = {"comps": comps} if p5 is not None: comps += p5.get("comps", []) comp.ifs += p5.get("if", []) p[0] = p0 def p_comp_if(self, p): """comp_if : IF test_nocond comp_iter_opt""" p2, p3 = p[2], p[3] p0 = {"if": [p2]} if p3 is not None: p0["comps"] = p3.get("comps", []) p[0] = p0 def p_yield_expr(self, p): """yield_expr : yield_tok yield_arg_opt""" p1, p2 = p[1], p[2] if p2 is None: p0 = ast.Yield(value=p2, lineno=p1.lineno, col_offset=p1.lexpos) elif p2["from"]: p0 = ast.YieldFrom(value=p2["val"], lineno=p1.lineno, col_offset=p1.lexpos) else: p0 = ast.Yield(value=p2["val"], lineno=p1.lineno, col_offset=p1.lexpos) p[0] = p0 def p_yield_arg_from(self, p): """yield_arg : FROM test""" p[0] = {"from": True, "val": p[2]} def p_yield_arg_testlist(self, p): """yield_arg : testlist""" p[0] = {"from": False, "val": p[1]} # # subprocess # def _dollar_rules(self, p): """These handle the special xonsh $ shell atoms by looking up in a special __xonsh__.env dictionary injected in the __builtin__. """ lenp = len(p) p1, p2 = p[1], p[2] if isinstance(p1, LexToken): p1, p1_tok = p1.value, p1 lineno, col = p1_tok.lineno, p1_tok.lexpos else: lineno, col = self.lineno, self.col if lenp == 3: # $NAME p0 = self._envvar_by_name(p2, lineno=lineno, col=col) elif p1 == "${": xenv = load_attribute_chain("__xonsh__.env", lineno=lineno, col=col) idx = ast.Index(value=p2) p0 = ast.Subscript( value=xenv, slice=idx, ctx=ast.Load(), lineno=lineno, col_offset=col ) elif p1 == "$(": p0 = xonsh_call( "__xonsh__.subproc_captured_stdout", p2, lineno=lineno, col=col ) elif p1 == "!(": p0 = xonsh_call( "__xonsh__.subproc_captured_object", p2, lineno=lineno, col=col ) elif p1 == "![": p0 = xonsh_call( "__xonsh__.subproc_captured_hiddenobject", p2, lineno=lineno, col=col ) elif p1 == "$[": p0 = xonsh_call("__xonsh__.subproc_uncaptured", p2, lineno=lineno, col=col) else: assert False return p0 def _envvar_getter_by_name(self, var, lineno=None, col=None): xenv = load_attribute_chain("__xonsh__.env", lineno=lineno, col=col) func = ast.Attribute( value=xenv, attr="get", ctx=ast.Load(), lineno=lineno, col_offset=col ) return ast.Call( func=func, args=[ ast.Str(s=var, lineno=lineno, col_offset=col), ast.Str(s="", lineno=lineno, col_offset=col), ], keywords=[], starargs=None, kwargs=None, lineno=lineno, col_offset=col, ) def _envvar_by_name(self, var, lineno=None, col=None): """Looks up a xonsh variable by name.""" xenv = load_attribute_chain("__xonsh__.env", lineno=lineno, col=col) idx = ast.Index(value=ast.Str(s=var, lineno=lineno, col_offset=col)) return ast.Subscript( value=xenv, slice=idx, ctx=ast.Load(), lineno=lineno, col_offset=col ) def _subproc_cliargs(self, args, lineno=None, col=None): """Creates an expression for subprocess CLI arguments.""" cliargs = currlist = empty_list(lineno=lineno, col=col) for arg in args: action = arg._cliarg_action if action == "append": if currlist is None: currlist = empty_list(lineno=lineno, col=col) cliargs = binop( cliargs, ast.Add(), currlist, lineno=lineno, col=col ) currlist.elts.append(arg) elif action == "extend": cliargs = binop(cliargs, ast.Add(), arg, lineno=lineno, col=col) currlist = None elif action == "splitlines": sl = call_split_lines(arg, lineno=lineno, col=col) cliargs = binop(cliargs, ast.Add(), sl, lineno=lineno, col=col) currlist = None elif action == "ensure_list": x = ensure_list_from_str_or_list(arg, lineno=lineno, col=col) cliargs = binop(cliargs, ast.Add(), x, lineno=lineno, col=col) currlist = None else: raise ValueError("action not understood: " + action) del arg._cliarg_action return cliargs def p_pipe(self, p): """pipe : PIPE | WS PIPE | PIPE WS | WS PIPE WS """ p[0] = ast.Str(s="|", lineno=self.lineno, col_offset=self.col) def p_subproc_s2(self, p): """subproc : subproc_atoms | subproc_atoms WS """ p1 = p[1] p[0] = [self._subproc_cliargs(p1, lineno=self.lineno, col=self.col)] def p_subproc_amp(self, p): """subproc : subproc AMPERSAND""" p1 = p[1] p[0] = p1 + [ast.Str(s=p[2], lineno=self.lineno, col_offset=self.col)] def p_subproc_pipe(self, p): """subproc : subproc pipe subproc_atoms | subproc pipe subproc_atoms WS """ p1 = p[1] if len(p1) > 1 and hasattr(p1[-2], "s") and p1[-2].s != "|": msg = "additional redirect following non-pipe redirect" self._parse_error(msg, self.currloc(lineno=self.lineno, column=self.col)) cliargs = self._subproc_cliargs(p[3], lineno=self.lineno, col=self.col) p[0] = p1 + [p[2], cliargs] def p_subproc_atoms_single(self, p): """subproc_atoms : subproc_atom""" p[0] = [p[1]] def p_subproc_atoms_many(self, p): """subproc_atoms : subproc_atoms WS subproc_atom""" p1 = p[1] p1.append(p[3]) p[0] = p1 def p_subproc_atoms_subshell(self, p): """subproc_atoms : lparen_tok any_raw_tok rparen_tok | lparen_tok any_raw_toks rparen_tok """ p1 = p[1] p3 = p[3] l = p1.lineno c = p1.lexpos + 1 subcmd = self.source_slice((l, c), (p3.lineno, p3.lexpos)) subcmd = subcmd.strip() + "\n" p0 = [ ast.Str(s="xonsh", lineno=l, col_offset=c), ast.Str(s="-c", lineno=l, col_offset=c), ast.Str(s=subcmd, lineno=l, col_offset=c), ] for arg in p0: arg._cliarg_action = "append" p[0] = p0 # # Subproc atom rules # def _append_subproc_bang_empty(self, p): """Appends an empty string in subprocess mode to the argument list.""" p3 = p[3] node = ast.Str(s="", lineno=p3.lineno, col_offset=p3.lexpos + 1) p[2][-1].elts.append(node) def _append_subproc_bang(self, p): """Appends the part between ! and the ) or ] in subprocess mode to the argument list. """ p3, p5 = p[3], p[5] beg = (p3.lineno, p3.lexpos + 1) end = (p5.lineno, p5.lexpos) s = self.source_slice(beg, end).strip() node = ast.Str(s=s, lineno=beg[0], col_offset=beg[1]) p[2][-1].elts.append(node) def p_subproc_atom_uncaptured(self, p): """subproc_atom : dollar_lbracket_tok subproc RBRACKET""" p1 = p[1] p0 = xonsh_call( "__xonsh__.subproc_uncaptured", args=p[2], lineno=p1.lineno, col=p1.lexpos ) p0._cliarg_action = "splitlines" p[0] = p0 def p_subproc_atom_uncaptured_bang_empty(self, p): """subproc_atom : dollar_lbracket_tok subproc bang_tok RBRACKET""" self._append_subproc_bang_empty(p) self.p_subproc_atom_uncaptured(p) def p_subproc_atom_uncaptured_bang(self, p): """subproc_atom : dollar_lbracket_tok subproc bang_tok nocloser rbracket_tok""" self._append_subproc_bang(p) self.p_subproc_atom_uncaptured(p) def p_subproc_atom_captured_stdout(self, p): """subproc_atom : dollar_lparen_tok subproc RPAREN""" p1 = p[1] p0 = xonsh_call( "__xonsh__.subproc_captured_stdout", args=p[2], lineno=p1.lineno, col=p1.lexpos, ) p0._cliarg_action = "append" p[0] = p0 def p_subproc_atom_captured_stdout_bang_empty(self, p): """subproc_atom : dollar_lparen_tok subproc bang_tok RPAREN""" self._append_subproc_bang_empty(p) self.p_subproc_atom_captured_stdout(p) def p_subproc_atom_captured_stdout_bang(self, p): """subproc_atom : dollar_lparen_tok subproc bang_tok nocloser rparen_tok""" self._append_subproc_bang(p) self.p_subproc_atom_captured_stdout(p) def p_subproc_atom_pyenv_lookup(self, p): """subproc_atom : dollar_lbrace_tok test RBRACE""" p1 = p[1] lineno, col = p1.lineno, p1.lexpos xenv = load_attribute_chain("__xonsh__.env", lineno=lineno, col=col) func = ast.Attribute( value=xenv, attr="get", ctx=ast.Load(), lineno=lineno, col_offset=col ) p0 = ast.Call( func=func, args=[p[2], ast.Str(s="", lineno=lineno, col_offset=col)], keywords=[], starargs=None, kwargs=None, lineno=lineno, col_offset=col, ) p0._cliarg_action = "append" p[0] = p0 def p_subproc_atom_pyeval(self, p): """subproc_atom : at_lparen_tok testlist_comp RPAREN subproc_arg_part : at_lparen_tok testlist_comp RPAREN """ p1 = p[1] p0 = xonsh_call( "__xonsh__.list_of_strs_or_callables", [p[2]], lineno=p1.lineno, col=p1.lexpos, ) p0._cliarg_action = "extend" p[0] = p0 def p_subproc_atom_subproc_inject(self, p): """subproc_atom : atdollar_lparen_tok subproc RPAREN""" p1 = p[1] p0 = xonsh_call( "__xonsh__.subproc_captured_inject", p[2], lineno=p1.lineno, col=p1.lexpos ) p0._cliarg_action = "extend" p[0] = p0 def p_subproc_atom_subproc_inject_bang_empty(self, p): """subproc_atom : atdollar_lparen_tok subproc bang_tok RPAREN""" self._append_subproc_bang_empty(p) self.p_subproc_atom_subproc_inject(p) def p_subproc_atom_subproc_inject_bang(self, p): """subproc_atom : atdollar_lparen_tok subproc bang_tok nocloser rparen_tok""" self._append_subproc_bang(p) self.p_subproc_atom_subproc_inject(p) def p_subproc_atom_redirect(self, p): """subproc_atom : GT | LT | RSHIFT | IOREDIRECT """ p0 = ast.Str(s=p[1], lineno=self.lineno, col_offset=self.col) p0._cliarg_action = "append" p[0] = p0 def p_subproc_atom_re(self, p): """subproc_atom : SEARCHPATH""" p0 = xonsh_pathsearch(p[1], pymode=False, lineno=self.lineno, col=self.col) p0._cliarg_action = "extend" p[0] = p0 def p_subproc_atom_str(self, p): """subproc_atom : string_literal""" p0 = xonsh_call( "__xonsh__.expand_path", args=[p[1]], lineno=self.lineno, col=self.col ) p0._cliarg_action = "append" p[0] = p0 def p_subproc_atom_arg(self, p): """subproc_atom : subproc_arg""" p1 = p[1] if isinstance(p1, list): # has an expanding function call, such as @(x) p0 = xonsh_call( "__xonsh__.list_of_list_of_strs_outer_product", args=[ensure_has_elts(p1)], lineno=p1[0].lineno, col=p1[0].col_offset, ) p0._cliarg_action = "extend" elif hasglobstar(p1): # globbed literal argument p0 = xonsh_call( "__xonsh__.glob", args=[p1], lineno=p1.lineno, col=p1.col_offset ) p0._cliarg_action = "extend" else: # literal str argument p0 = xonsh_call( "__xonsh__.expand_path", args=[p1], lineno=p1.lineno, col=p1.col_offset ) p0._cliarg_action = "append" p[0] = p0 def p_subproc_arg_single(self, p): """subproc_arg : subproc_arg_part""" p[0] = p[1] def p_subproc_arg_many(self, p): """subproc_arg : subproc_arg subproc_arg_part""" # This glues the string together after parsing p1 = p[1] p2 = p[2] if isinstance(p1, ast.Str) and isinstance(p2, ast.Str): p0 = ast.Str(p1.s + p2.s, lineno=p1.lineno, col_offset=p1.col_offset) elif isinstance(p1, list): if isinstance(p2, list): p1.extend(p2) else: p1.append(p2) p0 = p1 elif isinstance(p2, list): p2.insert(0, p1) p0 = p2 else: p0 = [p1, p2] p[0] = p0 def _attach_subproc_arg_part_rules(self): toks = set(self.tokens) toks -= { "AND", "OR", "NOT", "BANG", "PIPE", "WS", "GT", "LT", "LSHIFT", "RSHIFT", "IOREDIRECT", "SEARCHPATH", "INDENT", "DEDENT", "LPAREN", "RPAREN", "LBRACE", "RBRACE", "LBRACKET", "RBRACKET", "AT_LPAREN", "BANG_LPAREN", "BANG_LBRACKET", "DOLLAR_LPAREN", "DOLLAR_LBRACE", "DOLLAR_LBRACKET", "ATDOLLAR_LPAREN", } ts = "\n | ".join(sorted([t.lower() + "_tok" for t in toks])) doc = "subproc_arg_part : " + ts + "\n" self.p_subproc_arg_part.__func__.__doc__ = doc def p_subproc_arg_part(self, p): # Many tokens cannot be part of this rule, such as $, ', ", () # Use a string atom instead. See above attachment functions p1 = p[1] p[0] = ast.Str(s=p1.value, lineno=p1.lineno, col_offset=p1.lexpos) # # Helpers # def p_test_comma_combine(self, p): """test_comma_list : test comma_test_list | test comma_test_list COMMA """ p2 = p[2] p2.insert(0, p[1]) p[0] = p2 def p_empty(self, p): "empty : " p[0] = None def p_error(self, p): if p is None: self._parse_error("no further code", None) elif p.type == "ERRORTOKEN": if isinstance(p.value, BaseException): raise p.value else: self._parse_error( p.value, self.currloc(lineno=p.lineno, column=p.lexpos) ) else: msg = ("code: {0}".format(p.value),) self._parse_error(msg, self.currloc(lineno=p.lineno, column=p.lexpos)) ================================================ FILE: xonsh/parsers/context_check.py ================================================ import ast import keyword import collections _all_keywords = frozenset(keyword.kwlist) def _not_assignable(x, augassign=False): """ If ``x`` represents a value that can be assigned to, return ``None``. Otherwise, return a string describing the object. For use in generating meaningful syntax errors. """ if augassign and isinstance(x, (ast.Tuple, ast.List)): return "literal" elif isinstance(x, (ast.Tuple, ast.List)): if len(x.elts) == 0: return "()" for i in x.elts: res = _not_assignable(i) if res is not None: return res elif isinstance(x, (ast.Set, ast.Dict, ast.Num, ast.Str, ast.Bytes)): return "literal" elif isinstance(x, ast.Call): return "function call" elif isinstance(x, ast.Lambda): return "lambda" elif isinstance(x, (ast.BoolOp, ast.BinOp, ast.UnaryOp)): return "operator" elif isinstance(x, ast.IfExp): return "conditional expression" elif isinstance(x, ast.ListComp): return "list comprehension" elif isinstance(x, ast.DictComp): return "dictionary comprehension" elif isinstance(x, ast.SetComp): return "set comprehension" elif isinstance(x, ast.GeneratorExp): return "generator expression" elif isinstance(x, ast.Compare): return "comparison" elif isinstance(x, ast.Name) and x.id in _all_keywords: return "keyword" elif isinstance(x, ast.NameConstant): return "keyword" _loc = collections.namedtuple("_loc", ["lineno", "column"]) def check_contexts(tree): c = ContextCheckingVisitor() c.visit(tree) if c.error is not None: e = SyntaxError(c.error[0]) e.loc = _loc(c.error[1], c.error[2]) raise e class ContextCheckingVisitor(ast.NodeVisitor): def __init__(self): self.error = None def visit_Delete(self, node): for i in node.targets: err = _not_assignable(i) if err is not None: msg = "can't delete {}".format(err) self.error = msg, i.lineno, i.col_offset break def visit_Assign(self, node): for i in node.targets: err = _not_assignable(i) if err is not None: msg = "can't assign to {}".format(err) self.error = msg, i.lineno, i.col_offset break def visit_AugAssign(self, node): err = _not_assignable(node.target, True) if err is not None: msg = "illegal target for augmented assignment: {}".format(err) self.error = msg, node.target.lineno, node.target.col_offset ================================================ FILE: xonsh/parsers/v34.py ================================================ # -*- coding: utf-8 -*- """Implements the xonsh parser for Python v3.4.""" import xonsh.ast as ast from xonsh.parsers.base import BaseParser class Parser(BaseParser): """A Python v3.4 compliant parser for the xonsh language.""" def __init__( self, lexer_optimize=True, lexer_table="xonsh.lexer_table", yacc_optimize=True, yacc_table="xonsh.parser_table", yacc_debug=False, outputdir=None, ): """Parameters ---------- lexer_optimize : bool, optional Set to false when unstable and true when lexer is stable. lexer_table : str, optional Lexer module used when optimized. yacc_optimize : bool, optional Set to false when unstable and true when parser is stable. yacc_table : str, optional Parser module used when optimized. yacc_debug : debug, optional Dumps extra debug info. outputdir : str or None, optional The directory to place generated tables within. """ # Rule creation and modification *must* take place before super() opt_rules = ["argument_comma_list", "comma_argument_list"] for rule in opt_rules: self._opt_rule(rule) list_rules = ["argument_comma"] for rule in list_rules: self._list_rule(rule) super().__init__( lexer_optimize=lexer_optimize, lexer_table=lexer_table, yacc_optimize=yacc_optimize, yacc_table=yacc_table, yacc_debug=yacc_debug, outputdir=outputdir, ) def p_classdef_or_funcdef(self, p): """classdef_or_funcdef : classdef | funcdef """ p[0] = p[1] def p_item(self, p): """item : test COLON test""" lenp = len(p) if lenp == 4: p0 = [p[1], p[3]] elif lenp == 3: p0 = [None, p[2]] else: assert False p[0] = p0 def _set_arg(self, args, arg, ensure_kw=False): if isinstance(arg, ast.keyword): args["keywords"].append(arg) elif ensure_kw: args["kwargs"] = arg else: args["args"].append(arg) def p_arglist(self, p): """arglist : argument comma_opt | argument_comma_list argument comma_opt | argument_comma_list_opt TIMES test comma_argument_list_opt | argument_comma_list_opt TIMES test COMMA POW test | argument_comma_list_opt TIMES test comma_argument_list COMMA POW test | argument_comma_list_opt POW test """ lenp = len(p) p1, p2 = p[1], p[2] p0 = {"args": [], "keywords": [], "starargs": None, "kwargs": None} if lenp == 3: self._set_arg(p0, p1) elif lenp == 4 and p2 != "**": for arg in p1: self._set_arg(p0, arg) self._set_arg(p0, p2) elif lenp == 4 and p2 == "**": if p1 is not None: for arg in p1: self._set_arg(p0, arg) self._set_arg(p0, p[3], ensure_kw=True) elif lenp == 5: p0["starargs"], p4 = p[3], p[4] if p1 is not None: for arg in p1: self._set_arg(p0, arg) if p4 is not None: for arg in p4: self._set_arg(p0, arg, ensure_kw=True) elif lenp == 7: p0["starargs"] = p[3] if p1 is not None: for arg in p1: self._set_arg(p0, arg) self._set_arg(p0, p[6], ensure_kw=True) elif lenp == 8: p0["starargs"], p4 = p[3], p[4] if p1 is not None: for arg in p1: self._set_arg(p0, arg) for arg in p4: self._set_arg(p0, arg, ensure_kw=True) self._set_arg(p0, p[7], ensure_kw=True) else: assert False p[0] = p0 def p_argument_comma(self, p): """argument_comma : argument COMMA""" p[0] = [p[1]] def p_argument(self, p): """argument : test | test comp_for | test EQUALS test """ # Really [keyword '='] test # The reason that keywords are test nodes instead of NAME is that using # NAME results in an ambiguity. p1 = p[1] lenp = len(p) if lenp == 2: p0 = p1 elif lenp == 3: if p1 == "**": p0 = ast.keyword(arg=None, value=p[2]) elif p1 == "*": p0 = ast.Starred(value=p[2]) else: p0 = ast.GeneratorExp( elt=p1, generators=p[2]["comps"], lineno=p1.lineno, col_offset=p1.col_offset, ) elif lenp == 4: p0 = ast.keyword(arg=p1.id, value=p[3]) else: assert False p[0] = p0 ================================================ FILE: xonsh/parsers/v35.py ================================================ # -*- coding: utf-8 -*- """Implements the xonsh parser for Python v3.5.""" import xonsh.ast as ast from xonsh.parsers.base import BaseParser class Parser(BaseParser): """A Python v3.5 compliant parser for the xonsh language.""" def __init__( self, lexer_optimize=True, lexer_table="xonsh.lexer_table", yacc_optimize=True, yacc_table="xonsh.parser_table", yacc_debug=False, outputdir=None, ): """Parameters ---------- lexer_optimize : bool, optional Set to false when unstable and true when lexer is stable. lexer_table : str, optional Lexer module used when optimized. yacc_optimize : bool, optional Set to false when unstable and true when parser is stable. yacc_table : str, optional Parser module used when optimized. yacc_debug : debug, optional Dumps extra debug info. outputdir : str or None, optional The directory to place generated tables within. """ # Rule creation and modification *must* take place before super() tok_rules = ["await", "async"] for rule in tok_rules: self._tok_rule(rule) super().__init__( lexer_optimize=lexer_optimize, lexer_table=lexer_table, yacc_optimize=yacc_optimize, yacc_table=yacc_table, yacc_debug=yacc_debug, outputdir=outputdir, ) def p_classdef_or_funcdef(self, p): """classdef_or_funcdef : classdef | funcdef | async_funcdef """ p[0] = p[1] def p_async_funcdef(self, p): """async_funcdef : async_tok funcdef""" p1, f = p[1], p[2][0] p[0] = [ast.AsyncFunctionDef(**f.__dict__)] p[0][0]._async_tok = p1 def p_async_compound_stmt(self, p): """compound_stmt : async_stmt""" p[0] = p[1] def p_async_for_stmt(self, p): """async_for_stmt : ASYNC for_stmt""" f = p[2][0] p[0] = [ast.AsyncFor(**f.__dict__)] def p_async_with_stmt(self, p): """async_with_stmt : ASYNC with_stmt""" w = p[2][0] p[0] = [ast.AsyncWith(**w.__dict__)] def p_atom_expr_await(self, p): """atom_expr : await_tok atom trailer_list_opt""" p0 = self.apply_trailers(p[2], p[3]) p1 = p[1] p0 = ast.Await(value=p0, ctx=ast.Load(), lineno=p1.lineno, col_offset=p1.lexpos) p[0] = p0 def p_async_stmt(self, p): """async_stmt : async_funcdef | async_with_stmt | async_for_stmt """ p[0] = p[1] def p_item_test(self, p): """item : test COLON test""" p[0] = [p[1], p[3]] def p_item_pow(self, p): """item : POW expr""" p[0] = [None, p[2]] def _set_arg(self, args, arg, ensure_kw=False): if isinstance(arg, ast.keyword): args["keywords"].append(arg) elif ensure_kw: args["keywords"].append(ast.keyword(arg=None, value=arg)) else: args["args"].append(arg) def p_arglist_single(self, p): """arglist : argument comma_opt""" p0 = {"args": [], "keywords": []} self._set_arg(p0, p[1]) p[0] = p0 def p_arglist_many(self, p): """arglist : argument comma_argument_list comma_opt """ p0 = {"args": [], "keywords": []} self._set_arg(p0, p[1]) for arg in p[2]: self._set_arg(p0, arg) p[0] = p0 # Argument rules # "test '=' test" is really "keyword '=' test", but we have no such token. # These need to be in a single rule to avoid grammar that is ambiguous # to our LL(1) parser. Even though 'test' includes '*expr' in star_expr, # we explicitly match '*' here, too, to give it proper precedence. # Illegal combinations and orderings are blocked in ast.c: # multiple (test comp_for) arguments are blocked; keyword unpackings # that precede iterable unpackings are blocked; etc. def p_argument_test_or_star(self, p): """argument : test_or_star_expr""" p[0] = p[1] def p_argument_kwargs(self, p): """argument : POW test""" p[0] = ast.keyword(arg=None, value=p[2]) def p_argument_args(self, p): """argument : TIMES test""" p[0] = ast.Starred(value=p[2]) def p_argument(self, p): """argument : test comp_for""" p1 = p[1] p[0] = ast.GeneratorExp( elt=p1, generators=p[2]["comps"], lineno=p1.lineno, col_offset=p1.col_offset ) def p_argument_eq(self, p): """argument : test EQUALS test""" p[0] = ast.keyword(arg=p[1].id, value=p[3]) ================================================ FILE: xonsh/parsers/v36.py ================================================ # -*- coding: utf-8 -*- """Implements the xonsh parser for Python v3.6.""" import xonsh.ast as ast from xonsh.parsers.v35 import Parser as ThreeFiveParser from xonsh.parsers.base import store_ctx, ensure_has_elts, lopen_loc class Parser(ThreeFiveParser): """A Python v3.6 compliant parser for the xonsh language.""" def p_comp_for(self, p): """comp_for : FOR exprlist IN or_test comp_iter_opt""" targs, it, p5 = p[2], p[4], p[5] if len(targs) == 1: targ = targs[0] else: targ = ensure_has_elts(targs) store_ctx(targ) # only difference with base should be the is_async=0 comp = ast.comprehension(target=targ, iter=it, ifs=[], is_async=0) comps = [comp] p0 = {"comps": comps} if p5 is not None: comps += p5.get("comps", []) comp.ifs += p5.get("if", []) p[0] = p0 def p_expr_stmt_annassign(self, p): """expr_stmt : testlist_star_expr COLON test EQUALS test""" p1 = p[1][0] lineno, col = lopen_loc(p1) if len(p[1]) > 1 or not isinstance(p1, ast.Name): loc = self.currloc(lineno, col) self._parse_error("only single target can be annotated", loc) store_ctx(p1) p[0] = ast.AnnAssign( target=p1, annotation=p[3], value=p[5], simple=1, lineno=lineno, col_offset=col, ) ================================================ FILE: xonsh/platform.py ================================================ """Module for platform-specific constants and implementations, as well as compatibility layers to make use of the 'best' implementation available on a platform. """ import os import sys import ctypes import signal import pathlib import builtins import platform import functools import subprocess import collections import collections.abc as cabc import importlib.util from xonsh.lazyasd import LazyBool, lazyobject, lazybool # do not import any xonsh-modules here to avoid circular dependencies FD_STDIN = 0 FD_STDOUT = 1 FD_STDERR = 2 @lazyobject def distro(): try: import distro as d except ImportError: d = None except Exception: raise return d # # OS # ON_DARWIN = LazyBool(lambda: platform.system() == "Darwin", globals(), "ON_DARWIN") """``True`` if executed on a Darwin platform, else ``False``. """ ON_LINUX = LazyBool(lambda: platform.system() == "Linux", globals(), "ON_LINUX") """``True`` if executed on a Linux platform, else ``False``. """ ON_WINDOWS = LazyBool(lambda: platform.system() == "Windows", globals(), "ON_WINDOWS") """``True`` if executed on a native Windows platform, else ``False``. """ ON_CYGWIN = LazyBool(lambda: sys.platform == "cygwin", globals(), "ON_CYGWIN") """``True`` if executed on a Cygwin Windows platform, else ``False``. """ ON_MSYS = LazyBool(lambda: sys.platform == "msys", globals(), "ON_MSYS") """``True`` if executed on a MSYS Windows platform, else ``False``. """ ON_POSIX = LazyBool(lambda: (os.name == "posix"), globals(), "ON_POSIX") """``True`` if executed on a POSIX-compliant platform, else ``False``. """ ON_FREEBSD = LazyBool( lambda: (sys.platform.startswith("freebsd")), globals(), "ON_FREEBSD" ) """``True`` if on a FreeBSD operating system, else ``False``.""" ON_DRAGONFLY = LazyBool( lambda: (sys.platform.startswith("dragonfly")), globals(), "ON_DRAGONFLY" ) """``True`` if on a DragonFly BSD operating system, else ``False``.""" ON_NETBSD = LazyBool( lambda: (sys.platform.startswith("netbsd")), globals(), "ON_NETBSD" ) """``True`` if on a NetBSD operating system, else ``False``.""" @lazybool def ON_BSD(): """``True`` if on a BSD operating system, else ``False``.""" return bool(ON_FREEBSD) or bool(ON_NETBSD) or bool(ON_DRAGONFLY) @lazybool def ON_BEOS(): """True if we are on BeOS or Haiku.""" return sys.platform == "beos5" or sys.platform == "haiku1" @lazybool def ON_WSL(): """True if we are on Windows Subsystem for Linux (WSL)""" return "Microsoft" in platform.release() # # Python & packages # PYTHON_VERSION_INFO = sys.version_info[:3] """ Version of Python interpreter as three-value tuple. """ @lazyobject def PYTHON_VERSION_INFO_BYTES(): """The python version info tuple in a canonical bytes form.""" return ".".join(map(str, sys.version_info)).encode() ON_ANACONDA = LazyBool( lambda: pathlib.Path(sys.prefix).joinpath("conda-meta").exists(), globals(), "ON_ANACONDA", ) """ ``True`` if executed in an Anaconda instance, else ``False``. """ CAN_RESIZE_WINDOW = LazyBool( lambda: hasattr(signal, "SIGWINCH"), globals(), "CAN_RESIZE_WINDOW" ) """``True`` if we can resize terminal window, as provided by the presense of signal.SIGWINCH, else ``False``. """ @lazybool def HAS_PYGMENTS(): """``True`` if `pygments` is available, else ``False``.""" spec = importlib.util.find_spec("pygments") return spec is not None @functools.lru_cache(1) def pygments_version(): """pygments.__version__ version if available, else None.""" if HAS_PYGMENTS: import pygments v = pygments.__version__ else: v = None return v @functools.lru_cache(1) def pygments_version_info(): """ Returns `pygments`'s version as tuple of integers. """ if HAS_PYGMENTS: return tuple(int(x) for x in pygments_version().strip("<>+-=.").split(".")) else: return None @functools.lru_cache(1) def has_prompt_toolkit(): """Tests if the `prompt_toolkit` is available.""" spec = importlib.util.find_spec("prompt_toolkit") return spec is not None @functools.lru_cache(1) def ptk_version(): """Returns `prompt_toolkit.__version__` if available, else ``None``.""" if has_prompt_toolkit(): import prompt_toolkit return getattr(prompt_toolkit, "__version__", "<0.57") else: return None @functools.lru_cache(1) def ptk_version_info(): """ Returns `prompt_toolkit`'s version as tuple of integers. """ if has_prompt_toolkit(): return tuple(int(x) for x in ptk_version().strip("<>+-=.").split(".")) else: return None @functools.lru_cache(1) def ptk_above_min_supported(): minimum_required_ptk_version = (1, 0) return ptk_version_info()[:2] >= minimum_required_ptk_version @functools.lru_cache(1) def ptk_shell_type(): """Returns the prompt_toolkit shell type based on the installed version.""" if ptk_version_info()[:2] < (2, 0): return "prompt_toolkit1" else: return "prompt_toolkit2" @functools.lru_cache(1) def win_ansi_support(): if ON_WINDOWS: try: from prompt_toolkit.utils import is_windows_vt100_supported, is_conemu_ansi except ImportError: return False return is_conemu_ansi() or is_windows_vt100_supported() else: return False @functools.lru_cache(1) def ptk_below_max_supported(): ptk_max_version_cutoff = (2, 0) return ptk_version_info()[:2] < ptk_max_version_cutoff @functools.lru_cache(1) def best_shell_type(): if builtins.__xonsh__.env.get("TERM", "") == "dumb": return "dumb" elif ON_WINDOWS or has_prompt_toolkit(): return "prompt_toolkit" else: return "readline" @functools.lru_cache(1) def is_readline_available(): """Checks if readline is available to import.""" spec = importlib.util.find_spec("readline") return spec is not None @lazyobject def seps(): """String of all path separators.""" s = os.path.sep if os.path.altsep is not None: s += os.path.altsep return s def pathsplit(p): """This is a safe version of os.path.split(), which does not work on input without a drive. """ n = len(p) while n and p[n - 1] not in seps: n -= 1 pre = p[:n] pre = pre.rstrip(seps) or pre post = p[n:] return pre, post def pathbasename(p): """This is a safe version of os.path.basename(), which does not work on input without a drive. This version does. """ return pathsplit(p)[-1] @lazyobject def expanduser(): """Dispatches to the correct platform-dependent expanduser() function.""" if ON_WINDOWS: return windows_expanduser else: return os.path.expanduser def windows_expanduser(path): """A Windows-specific expanduser() function for xonsh. This is needed since os.path.expanduser() does not check on Windows if the user actually exists. This restricts expanding the '~' if it is not followed by a separator. That is only '~/' and '~\' are expanded. """ if not path.startswith("~"): return path elif len(path) < 2 or path[1] in seps: return os.path.expanduser(path) else: return path # termios tc(get|set)attr indexes. IFLAG = 0 OFLAG = 1 CFLAG = 2 LFLAG = 3 ISPEED = 4 OSPEED = 5 CC = 6 # # Dev release info # @functools.lru_cache(1) def githash(): """Returns a tuple contains two strings: the hash and the date.""" install_base = os.path.dirname(__file__) githash_file = "{}/dev.githash".format(install_base) if not os.path.exists(githash_file): return None, None sha = None date_ = None try: with open(githash_file) as f: sha, date_ = f.read().strip().split("|") except ValueError: pass return sha, date_ # # Encoding # DEFAULT_ENCODING = sys.getdefaultencoding() """ Default string encoding. """ if PYTHON_VERSION_INFO < (3, 5, 0): class DirEntry: def __init__(self, directory, name): self.__path__ = pathlib.Path(directory) / name self.name = name self.path = str(self.__path__) self.is_symlink = self.__path__.is_symlink def inode(self): return os.stat(self.path, follow_symlinks=False).st_ino def is_dir(self, *, follow_symlinks=True): if follow_symlinks: return self.__path__.is_dir() else: return not self.__path__.is_symlink() and self.__path__.is_dir() def is_file(self, *, follow_symlinks=True): if follow_symlinks: return self.__path__.is_file() else: return not self.__path__.is_symlink() and self.__path__.is_file() def stat(self, *, follow_symlinks=True): return os.stat(self.path, follow_symlinks=follow_symlinks) def scandir(path): """ Compatibility layer for `os.scandir` from Python 3.5+. """ return (DirEntry(path, x) for x in os.listdir(path)) else: scandir = os.scandir # # Linux distro # @functools.lru_cache(1) def linux_distro(): """The id of the Linux distribution running on, possibly 'unknown'. None on non-Linux platforms. """ if ON_LINUX: if distro: ld = distro.id() elif PYTHON_VERSION_INFO < (3, 6, 6): ld = platform.linux_distribution()[0] or "unknown" elif "-ARCH-" in platform.platform(): ld = "arch" # that's the only one we need to know for now else: ld = "unknown" else: ld = None return ld # # Windows # @functools.lru_cache(1) def git_for_windows_path(): """Returns the path to git for windows, if available and None otherwise.""" import winreg try: key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, "SOFTWARE\\GitForWindows") gfwp, _ = winreg.QueryValueEx(key, "InstallPath") except FileNotFoundError: gfwp = None return gfwp @functools.lru_cache(1) def windows_bash_command(): """Determines the command for Bash on windows.""" # Check that bash is on path otherwise try the default directory # used by Git for windows wbc = "bash" cmd_cache = builtins.__xonsh__.commands_cache bash_on_path = cmd_cache.lazy_locate_binary("bash", ignore_alias=True) if bash_on_path: try: out = subprocess.check_output( [bash_on_path, "--version"], stderr=subprocess.PIPE, universal_newlines=True, ) except subprocess.CalledProcessError: bash_works = False else: # Check if Bash is from the "Windows Subsystem for Linux" (WSL) # which can't be used by xonsh foreign-shell/completer bash_works = out and "pc-linux-gnu" not in out.splitlines()[0] if bash_works: wbc = bash_on_path else: gfwp = git_for_windows_path() if gfwp: bashcmd = os.path.join(gfwp, "bin\\bash.exe") if os.path.isfile(bashcmd): wbc = bashcmd return wbc # # Environment variables defaults # if ON_WINDOWS: class OSEnvironCasePreserving(cabc.MutableMapping): """ Case-preserving wrapper for os.environ on Windows. It uses nt.environ to get the correct cased keys on initialization. It also preserves the case of any variables add after initialization. """ def __init__(self): import nt self._upperkeys = dict((k.upper(), k) for k in nt.environ) def _sync(self): """ Ensure that the case sensitive map of the keys are in sync with os.environ """ envkeys = set(os.environ.keys()) for key in envkeys.difference(self._upperkeys): self._upperkeys[key] = key.upper() for key in set(self._upperkeys).difference(envkeys): del self._upperkeys[key] def __contains__(self, k): self._sync() return k.upper() in self._upperkeys def __len__(self): self._sync() return len(self._upperkeys) def __iter__(self): self._sync() return iter(self._upperkeys.values()) def __getitem__(self, k): self._sync() return os.environ[k] def __setitem__(self, k, v): self._sync() self._upperkeys[k.upper()] = k os.environ[k] = v def __delitem__(self, k): self._sync() if k.upper() in self._upperkeys: del self._upperkeys[k.upper()] del os.environ[k] def getkey_actual_case(self, k): self._sync() return self._upperkeys.get(k.upper()) @lazyobject def os_environ(): """This dispatches to the correct, case-sensitive version of os.environ. This is mainly a problem for Windows. See #2024 for more details. This can probably go away once support for Python v3.5 or v3.6 is dropped. """ if ON_WINDOWS: return OSEnvironCasePreserving() else: return os.environ @functools.lru_cache(1) def bash_command(): """Determines the command for Bash on the current platform.""" if ON_WINDOWS: bc = windows_bash_command() else: bc = "bash" return bc @lazyobject def BASH_COMPLETIONS_DEFAULT(): """A possibly empty tuple with default paths to Bash completions known for the current platform. """ if ON_LINUX or ON_CYGWIN or ON_MSYS: bcd = ("/usr/share/bash-completion/bash_completion",) elif ON_DARWIN: bcd = ( "/usr/local/share/bash-completion/bash_completion", # v2.x "/usr/local/etc/bash_completion", "/opt/local/etc/profile.d/bash_completion.sh", "/usr/local/etc/bash_completion.d/git-completion.bash", ) # v1.x elif ON_WINDOWS and git_for_windows_path(): bcd = ( os.path.join( git_for_windows_path(), "usr\\share\\bash-completion\\bash_completion" ), os.path.join( git_for_windows_path(), "mingw64\\share\\git\\completion\\" "git-completion.bash", ), ) else: bcd = () return bcd @lazyobject def PATH_DEFAULT(): if ON_LINUX or ON_CYGWIN or ON_MSYS: if linux_distro() == "arch": pd = ( "/usr/local/sbin", "/usr/local/bin", "/usr/bin", "/usr/bin/site_perl", "/usr/bin/vendor_perl", "/usr/bin/core_perl", ) else: pd = ( os.path.expanduser("~/bin"), "/usr/local/sbin", "/usr/local/bin", "/usr/sbin", "/usr/bin", "/sbin", "/bin", "/usr/games", "/usr/local/games", ) elif ON_DARWIN: pd = ("/usr/local/bin", "/usr/bin", "/bin", "/usr/sbin", "/sbin") elif ON_WINDOWS: import winreg key = winreg.OpenKey( winreg.HKEY_LOCAL_MACHINE, r"SYSTEM\CurrentControlSet\Control\Session Manager\Environment", ) pd = tuple(winreg.QueryValueEx(key, "Path")[0].split(os.pathsep)) else: pd = () return pd # # libc # @lazyobject def LIBC(): """The platform dependent libc implementation.""" global ctypes if ON_DARWIN: import ctypes.util libc = ctypes.CDLL(ctypes.util.find_library("c")) elif ON_CYGWIN: libc = ctypes.CDLL("cygwin1.dll") elif ON_MSYS: libc = ctypes.CDLL("msys-2.0.dll") elif ON_BSD: try: libc = ctypes.CDLL("libc.so") except AttributeError: libc = None except OSError: # OS X; can't use ctypes.util.find_library because that creates # a new process on Linux, which is undesirable. try: libc = ctypes.CDLL("libc.dylib") except OSError: libc = None elif ON_POSIX: try: libc = ctypes.CDLL("libc.so") except AttributeError: libc = None except OSError: # Debian and derivatives do the wrong thing because /usr/lib/libc.so # is a GNU ld script rather than an ELF object. To get around this, we # have to be more specific. # We don't want to use ctypes.util.find_library because that creates a # new process on Linux. We also don't want to try too hard because at # this point we're already pretty sure this isn't Linux. try: libc = ctypes.CDLL("libc.so.6") except OSError: libc = None if not hasattr(libc, "sysinfo"): # Not Linux. libc = None elif ON_WINDOWS: if hasattr(ctypes, "windll") and hasattr(ctypes.windll, "kernel32"): libc = ctypes.windll.kernel32 else: try: # Windows CE uses the cdecl calling convention. libc = ctypes.CDLL("coredll.lib") except (AttributeError, OSError): libc = None elif ON_BEOS: libc = ctypes.CDLL("libroot.so") else: libc = None return libc ================================================ FILE: xonsh/ply/.gitignore ================================================ *.pyc *.pyo __pycache__ *.out *.dif *~ /dist /build /*.egg-info ================================================ FILE: xonsh/ply/.travis.yml ================================================ language: python python: - "2.6" - "2.7" - "3.3" - "3.4" - "3.5" - "3.6" install: true script: "cd test && python testlex.py && python testyacc.py" ================================================ FILE: xonsh/ply/CHANGES ================================================ Current Version --------------- IMPORTANT NOTE (2018-12-22): PLY is no longer be released in any package-installable format. If you want to use the latest version, you need to COPY the contents of the ply/ directory into your own project and use it. Although PLY is no longer distributed as a package, it is maintained as a mature library. No new features are planned, but issues and pull requests for bugs are still welcome. Any changes to the software will be noted here. 01/19/19 Some improvements to the preprocessor module contributed by Rob Reilink. Issue #195 fixes the evaluation of expressions such as #if a != b. Issue #196 fixes some some issues related to the lack of a specified Unicode encoding on reading text files. 12/22/18 Incorporated some minor changes to eliminate warning messages related to open() calls. Version 3.11 --------------------- 02/15/18 beazley Fixed some minor bugs related to re flags and token order. Github pull requests #151 and #153. 02/15/18 beazley Added a set_lexpos() method to grammar symbols. Github issue #148. 04/13/17 beazley Mostly minor bug fixes and small code cleanups. Version 3.10 --------------------- 01/31/17: beazley Changed grammar signature computation to not involve hashing functions. Parts are just combined into a big string. 10/07/16: beazley Fixed Issue #101: Incorrect shift-reduce conflict resolution with precedence specifier. PLY was incorrectly resolving shift-reduce conflicts in certain cases. For example, in the example/calc/calc.py example, you could trigger it doing this: calc > -3 - 4 1 (correct answer should be -7) calc > Issue and suggested patch contributed by https://github.com/RomaVis Version 3.9 --------------------- 08/30/16: beazley Exposed the parser state number as the parser.state attribute in productions and error functions. For example: def p_somerule(p): ''' rule : A B C ''' print('State:', p.parser.state) May address issue #65 (publish current state in error callback). 08/30/16: beazley Fixed Issue #88. Python3 compatibility with ply/cpp. 08/30/16: beazley Fixed Issue #93. Ply can crash if SyntaxError is raised inside a production. Not actually sure if the original implementation worked as documented at all. Yacc has been modified to follow the spec as outlined in the CHANGES noted for 11/27/07 below. 08/30/16: beazley Fixed Issue #97. Failure with code validation when the original source files aren't present. Validation step now ignores the missing file. 08/30/16: beazley Minor fixes to version numbers. Version 3.8 --------------------- 10/02/15: beazley Fixed issues related to Python 3.5. Patch contributed by Barry Warsaw. Version 3.7 --------------------- 08/25/15: beazley Fixed problems when reading table files from pickled data. 05/07/15: beazley Fixed regression in handling of table modules if specified as module objects. See https://github.com/dabeaz/ply/issues/63 Version 3.6 --------------------- 04/25/15: beazley If PLY is unable to create the 'parser.out' or 'parsetab.py' files due to permission issues, it now just issues a warning message and continues to operate. This could happen if a module using PLY is installed in a funny way where tables have to be regenerated, but for whatever reason, the user doesn't have write permission on the directory where PLY wants to put them. 04/24/15: beazley Fixed some issues related to use of packages and table file modules. Just to emphasize, PLY now generates its special files such as 'parsetab.py' and 'lextab.py' in the *SAME* directory as the source file that uses lex() and yacc(). If for some reason, you want to change the name of the table module, use the tabmodule and lextab options: lexer = lex.lex(lextab='spamlextab') parser = yacc.yacc(tabmodule='spamparsetab') If you specify a simple name as shown, the module will still be created in the same directory as the file invoking lex() or yacc(). If you want the table files to be placed into a different package, then give a fully qualified package name. For example: lexer = lex.lex(lextab='pkgname.files.lextab') parser = yacc.yacc(tabmodule='pkgname.files.parsetab') For this to work, 'pkgname.files' must already exist as a valid Python package (i.e., the directories must already exist and be set up with the proper __init__.py files, etc.). Version 3.5 --------------------- 04/21/15: beazley Added support for defaulted_states in the parser. A defaulted_state is a state where the only legal action is a reduction of a single grammar rule across all valid input tokens. For such states, the rule is reduced and the reading of the next lookahead token is delayed until it is actually needed at a later point in time. This delay in consuming the next lookahead token is a potentially important feature in advanced parsing applications that require tight interaction between the lexer and the parser. For example, a grammar rule change modify the lexer state upon reduction and have such changes take effect before the next input token is read. *** POTENTIAL INCOMPATIBILITY *** One potential danger of defaulted_states is that syntax errors might be deferred to a a later point of processing than where they were detected in past versions of PLY. Thus, it's possible that your error handling could change slightly on the same inputs. defaulted_states do not change the overall parsing of the input (i.e., the same grammar is accepted). If for some reason, you need to disable defaulted states, you can do this: parser = yacc.yacc() parser.defaulted_states = {} 04/21/15: beazley Fixed debug logging in the parser. It wasn't properly reporting goto states on grammar rule reductions. 04/20/15: beazley Added actions to be defined to character literals (Issue #32). For example: literals = [ '{', '}' ] def t_lbrace(t): r'\{' # Some action t.type = '{' return t def t_rbrace(t): r'\}' # Some action t.type = '}' return t 04/19/15: beazley Import of the 'parsetab.py' file is now constrained to only consider the directory specified by the outputdir argument to yacc(). If not supplied, the import will only consider the directory in which the grammar is defined. This should greatly reduce problems with the wrong parsetab.py file being imported by mistake. For example, if it's found somewhere else on the path by accident. *** POTENTIAL INCOMPATIBILITY *** It's possible that this might break some packaging/deployment setup if PLY was instructed to place its parsetab.py in a different location. You'll have to specify a proper outputdir= argument to yacc() to fix this if needed. 04/19/15: beazley Changed default output directory to be the same as that in which the yacc grammar is defined. If your grammar is in a file 'calc.py', then the parsetab.py and parser.out files should be generated in the same directory as that file. The destination directory can be changed using the outputdir= argument to yacc(). 04/19/15: beazley Changed the parsetab.py file signature slightly so that the parsetab won't regenerate if created on a different major version of Python (ie., a parsetab created on Python 2 will work with Python 3). 04/16/15: beazley Fixed Issue #44 call_errorfunc() should return the result of errorfunc() 04/16/15: beazley Support for versions of Python <2.7 is officially dropped. PLY may work, but the unit tests requires Python 2.7 or newer. 04/16/15: beazley Fixed bug related to calling yacc(start=...). PLY wasn't regenerating the table file correctly for this case. 04/16/15: beazley Added skipped tests for PyPy and Java. Related to use of Python's -O option. 05/29/13: beazley Added filter to make unit tests pass under 'python -3'. Reported by Neil Muller. 05/29/13: beazley Fixed CPP_INTEGER regex in ply/cpp.py (Issue 21). Reported by @vbraun. 05/29/13: beazley Fixed yacc validation bugs when from __future__ import unicode_literals is being used. Reported by Kenn Knowles. 05/29/13: beazley Added support for Travis-CI. Contributed by Kenn Knowles. 05/29/13: beazley Added a .gitignore file. Suggested by Kenn Knowles. 05/29/13: beazley Fixed validation problems for source files that include a different source code encoding specifier. Fix relies on the inspect module. Should work on Python 2.6 and newer. Not sure about older versions of Python. Contributed by Michael Droettboom 05/21/13: beazley Fixed unit tests for yacc to eliminate random failures due to dict hash value randomization in Python 3.3 Reported by Arfrever 10/15/12: beazley Fixed comment whitespace processing bugs in ply/cpp.py. Reported by Alexei Pososin. 10/15/12: beazley Fixed token names in ply/ctokens.py to match rule names. Reported by Alexei Pososin. 04/26/12: beazley Changes to functions available in panic mode error recover. In previous versions of PLY, the following global functions were available for use in the p_error() rule: yacc.errok() # Reset error state yacc.token() # Get the next token yacc.restart() # Reset the parsing stack The use of global variables was problematic for code involving multiple parsers and frankly was a poor design overall. These functions have been moved to methods of the parser instance created by the yacc() function. You should write code like this: def p_error(p): ... parser.errok() parser = yacc.yacc() *** POTENTIAL INCOMPATIBILITY *** The original global functions now issue a DeprecationWarning. 04/19/12: beazley Fixed some problems with line and position tracking and the use of error symbols. If you have a grammar rule involving an error rule like this: def p_assignment_bad(p): '''assignment : location EQUALS error SEMI''' ... You can now do line and position tracking on the error token. For example: def p_assignment_bad(p): '''assignment : location EQUALS error SEMI''' start_line = p.lineno(3) start_pos = p.lexpos(3) If the trackng=True option is supplied to parse(), you can additionally get spans: def p_assignment_bad(p): '''assignment : location EQUALS error SEMI''' start_line, end_line = p.linespan(3) start_pos, end_pos = p.lexspan(3) Note that error handling is still a hairy thing in PLY. This won't work unless your lexer is providing accurate information. Please report bugs. Suggested by a bug reported by Davis Herring. 04/18/12: beazley Change to doc string handling in lex module. Regex patterns are now first pulled from a function's .regex attribute. If that doesn't exist, then .doc is checked as a fallback. The @TOKEN decorator now sets the .regex attribute of a function instead of its doc string. Changed suggested by Kristoffer Ellersgaard Koch. 04/18/12: beazley Fixed issue #1: Fixed _tabversion. It should use __tabversion__ instead of __version__ Reported by Daniele Tricoli 04/18/12: beazley Fixed issue #8: Literals empty list causes IndexError Reported by Walter Nissen. 04/18/12: beazley Fixed issue #12: Typo in code snippet in documentation Reported by florianschanda. 04/18/12: beazley Fixed issue #10: Correctly escape t_XOREQUAL pattern. Reported by Andy Kittner. Version 3.4 --------------------- 02/17/11: beazley Minor patch to make cpp.py compatible with Python 3. Note: This is an experimental file not currently used by the rest of PLY. 02/17/11: beazley Fixed setup.py trove classifiers to properly list PLY as Python 3 compatible. 01/02/11: beazley Migration of repository to github. Version 3.3 ----------------------------- 08/25/09: beazley Fixed issue 15 related to the set_lineno() method in yacc. Reported by mdsherry. 08/25/09: beazley Fixed a bug related to regular expression compilation flags not being properly stored in lextab.py files created by the lexer when running in optimize mode. Reported by Bruce Frederiksen. Version 3.2 ----------------------------- 03/24/09: beazley Added an extra check to not print duplicated warning messages about reduce/reduce conflicts. 03/24/09: beazley Switched PLY over to a BSD-license. 03/23/09: beazley Performance optimization. Discovered a few places to make speedups in LR table generation. 03/23/09: beazley New warning message. PLY now warns about rules never reduced due to reduce/reduce conflicts. Suggested by Bruce Frederiksen. 03/23/09: beazley Some clean-up of warning messages related to reduce/reduce errors. 03/23/09: beazley Added a new picklefile option to yacc() to write the parsing tables to a filename using the pickle module. Here is how it works: yacc(picklefile="parsetab.p") This option can be used if the normal parsetab.py file is extremely large. For example, on jython, it is impossible to read parsing tables if the parsetab.py exceeds a certain threshold. The filename supplied to the picklefile option is opened relative to the current working directory of the Python interpreter. If you need to refer to the file elsewhere, you will need to supply an absolute or relative path. For maximum portability, the pickle file is written using protocol 0. 03/13/09: beazley Fixed a bug in parser.out generation where the rule numbers where off by one. 03/13/09: beazley Fixed a string formatting bug with one of the error messages. Reported by Richard Reitmeyer Version 3.1 ----------------------------- 02/28/09: beazley Fixed broken start argument to yacc(). PLY-3.0 broke this feature by accident. 02/28/09: beazley Fixed debugging output. yacc() no longer reports shift/reduce or reduce/reduce conflicts if debugging is turned off. This restores similar behavior in PLY-2.5. Reported by Andrew Waters. Version 3.0 ----------------------------- 02/03/09: beazley Fixed missing lexer attribute on certain tokens when invoking the parser p_error() function. Reported by Bart Whiteley. 02/02/09: beazley The lex() command now does all error-reporting and diagonistics using the logging module interface. Pass in a Logger object using the errorlog parameter to specify a different logger. 02/02/09: beazley Refactored ply.lex to use a more object-oriented and organized approach to collecting lexer information. 02/01/09: beazley Removed the nowarn option from lex(). All output is controlled by passing in a logger object. Just pass in a logger with a high level setting to suppress output. This argument was never documented to begin with so hopefully no one was relying upon it. 02/01/09: beazley Discovered and removed a dead if-statement in the lexer. This resulted in a 6-7% speedup in lexing when I tested it. 01/13/09: beazley Minor change to the procedure for signalling a syntax error in a production rule. A normal SyntaxError exception should be raised instead of yacc.SyntaxError. 01/13/09: beazley Added a new method p.set_lineno(n,lineno) that can be used to set the line number of symbol n in grammar rules. This simplifies manual tracking of line numbers. 01/11/09: beazley Vastly improved debugging support for yacc.parse(). Instead of passing debug as an integer, you can supply a Logging object (see the logging module). Messages will be generated at the ERROR, INFO, and DEBUG logging levels, each level providing progressively more information. The debugging trace also shows states, grammar rule, values passed into grammar rules, and the result of each reduction. 01/09/09: beazley The yacc() command now does all error-reporting and diagnostics using the interface of the logging module. Use the errorlog parameter to specify a logging object for error messages. Use the debuglog parameter to specify a logging object for the 'parser.out' output. 01/09/09: beazley *HUGE* refactoring of the the ply.yacc() implementation. The high-level user interface is backwards compatible, but the internals are completely reorganized into classes. No more global variables. The internals are also more extensible. For example, you can use the classes to construct a LALR(1) parser in an entirely different manner than what is currently the case. Documentation is forthcoming. 01/07/09: beazley Various cleanup and refactoring of yacc internals. 01/06/09: beazley Fixed a bug with precedence assignment. yacc was assigning the precedence each rule based on the left-most token, when in fact, it should have been using the right-most token. Reported by Bruce Frederiksen. 11/27/08: beazley Numerous changes to support Python 3.0 including removal of deprecated statements (e.g., has_key) and the additional of compatibility code to emulate features from Python 2 that have been removed, but which are needed. Fixed the unit testing suite to work with Python 3.0. The code should be backwards compatible with Python 2. 11/26/08: beazley Loosened the rules on what kind of objects can be passed in as the "module" parameter to lex() and yacc(). Previously, you could only use a module or an instance. Now, PLY just uses dir() to get a list of symbols on whatever the object is without regard for its type. 11/26/08: beazley Changed all except: statements to be compatible with Python2.x/3.x syntax. 11/26/08: beazley Changed all raise Exception, value statements to raise Exception(value) for forward compatibility. 11/26/08: beazley Removed all print statements from lex and yacc, using sys.stdout and sys.stderr directly. Preparation for Python 3.0 support. 11/04/08: beazley Fixed a bug with referring to symbols on the the parsing stack using negative indices. 05/29/08: beazley Completely revamped the testing system to use the unittest module for everything. Added additional tests to cover new errors/warnings. Version 2.5 ----------------------------- 05/28/08: beazley Fixed a bug with writing lex-tables in optimized mode and start states. Reported by Kevin Henry. Version 2.4 ----------------------------- 05/04/08: beazley A version number is now embedded in the table file signature so that yacc can more gracefully accomodate changes to the output format in the future. 05/04/08: beazley Removed undocumented .pushback() method on grammar productions. I'm not sure this ever worked and can't recall ever using it. Might have been an abandoned idea that never really got fleshed out. This feature was never described or tested so removing it is hopefully harmless. 05/04/08: beazley Added extra error checking to yacc() to detect precedence rules defined for undefined terminal symbols. This allows yacc() to detect a potential problem that can be really tricky to debug if no warning message or error message is generated about it. 05/04/08: beazley lex() now has an outputdir that can specify the output directory for tables when running in optimize mode. For example: lexer = lex.lex(optimize=True, lextab="ltab", outputdir="foo/bar") The behavior of specifying a table module and output directory are more aligned with the behavior of yacc(). 05/04/08: beazley [Issue 9] Fixed filename bug in when specifying the modulename in lex() and yacc(). If you specified options such as the following: parser = yacc.yacc(tabmodule="foo.bar.parsetab",outputdir="foo/bar") yacc would create a file "foo.bar.parsetab.py" in the given directory. Now, it simply generates a file "parsetab.py" in that directory. Bug reported by cptbinho. 05/04/08: beazley Slight modification to lex() and yacc() to allow their table files to be loaded from a previously loaded module. This might make it easier to load the parsing tables from a complicated package structure. For example: import foo.bar.spam.parsetab as parsetab parser = yacc.yacc(tabmodule=parsetab) Note: lex and yacc will never regenerate the table file if used in the form---you will get a warning message instead. This idea suggested by Brian Clapper. 04/28/08: beazley Fixed a big with p_error() functions being picked up correctly when running in yacc(optimize=1) mode. Patch contributed by Bart Whiteley. 02/28/08: beazley Fixed a bug with 'nonassoc' precedence rules. Basically the non-precedence was being ignored and not producing the correct run-time behavior in the parser. 02/16/08: beazley Slight relaxation of what the input() method to a lexer will accept as a string. Instead of testing the input to see if the input is a string or unicode string, it checks to see if the input object looks like it contains string data. This change makes it possible to pass string-like objects in as input. For example, the object returned by mmap. import mmap, os data = mmap.mmap(os.open(filename,os.O_RDONLY), os.path.getsize(filename), access=mmap.ACCESS_READ) lexer.input(data) 11/29/07: beazley Modification of ply.lex to allow token functions to aliased. This is subtle, but it makes it easier to create libraries and to reuse token specifications. For example, suppose you defined a function like this: def number(t): r'\d+' t.value = int(t.value) return t This change would allow you to define a token rule as follows: t_NUMBER = number In this case, the token type will be set to 'NUMBER' and use the associated number() function to process tokens. 11/28/07: beazley Slight modification to lex and yacc to grab symbols from both the local and global dictionaries of the caller. This modification allows lexers and parsers to be defined using inner functions and closures. 11/28/07: beazley Performance optimization: The lexer.lexmatch and t.lexer attributes are no longer set for lexer tokens that are not defined by functions. The only normal use of these attributes would be in lexer rules that need to perform some kind of special processing. Thus, it doesn't make any sense to set them on every token. *** POTENTIAL INCOMPATIBILITY *** This might break code that is mucking around with internal lexer state in some sort of magical way. 11/27/07: beazley Added the ability to put the parser into error-handling mode from within a normal production. To do this, simply raise a yacc.SyntaxError exception like this: def p_some_production(p): 'some_production : prod1 prod2' ... raise yacc.SyntaxError # Signal an error A number of things happen after this occurs: - The last symbol shifted onto the symbol stack is discarded and parser state backed up to what it was before the the rule reduction. - The current lookahead symbol is saved and replaced by the 'error' symbol. - The parser enters error recovery mode where it tries to either reduce the 'error' rule or it starts discarding items off of the stack until the parser resets. When an error is manually set, the parser does *not* call the p_error() function (if any is defined). *** NEW FEATURE *** Suggested on the mailing list 11/27/07: beazley Fixed structure bug in examples/ansic. Reported by Dion Blazakis. 11/27/07: beazley Fixed a bug in the lexer related to start conditions and ignored token rules. If a rule was defined that changed state, but returned no token, the lexer could be left in an inconsistent state. Reported by 11/27/07: beazley Modified setup.py to support Python Eggs. Patch contributed by Simon Cross. 11/09/07: beazely Fixed a bug in error handling in yacc. If a syntax error occurred and the parser rolled the entire parse stack back, the parser would be left in in inconsistent state that would cause it to trigger incorrect actions on subsequent input. Reported by Ton Biegstraaten, Justin King, and others. 11/09/07: beazley Fixed a bug when passing empty input strings to yacc.parse(). This would result in an error message about "No input given". Reported by Andrew Dalke. Version 2.3 ----------------------------- 02/20/07: beazley Fixed a bug with character literals if the literal '.' appeared as the last symbol of a grammar rule. Reported by Ales Smrcka. 02/19/07: beazley Warning messages are now redirected to stderr instead of being printed to standard output. 02/19/07: beazley Added a warning message to lex.py if it detects a literal backslash character inside the t_ignore declaration. This is to help problems that might occur if someone accidentally defines t_ignore as a Python raw string. For example: t_ignore = r' \t' The idea for this is from an email I received from David Cimimi who reported bizarre behavior in lexing as a result of defining t_ignore as a raw string by accident. 02/18/07: beazley Performance improvements. Made some changes to the internal table organization and LR parser to improve parsing performance. 02/18/07: beazley Automatic tracking of line number and position information must now be enabled by a special flag to parse(). For example: yacc.parse(data,tracking=True) In many applications, it's just not that important to have the parser automatically track all line numbers. By making this an optional feature, it allows the parser to run significantly faster (more than a 20% speed increase in many cases). Note: positional information is always available for raw tokens---this change only applies to positional information associated with nonterminal grammar symbols. *** POTENTIAL INCOMPATIBILITY *** 02/18/07: beazley Yacc no longer supports extended slices of grammar productions. However, it does support regular slices. For example: def p_foo(p): '''foo: a b c d e''' p[0] = p[1:3] This change is a performance improvement to the parser--it streamlines normal access to the grammar values since slices are now handled in a __getslice__() method as opposed to __getitem__(). 02/12/07: beazley Fixed a bug in the handling of token names when combined with start conditions. Bug reported by Todd O'Bryan. Version 2.2 ------------------------------ 11/01/06: beazley Added lexpos() and lexspan() methods to grammar symbols. These mirror the same functionality of lineno() and linespan(). For example: def p_expr(p): 'expr : expr PLUS expr' p.lexpos(1) # Lexing position of left-hand-expression p.lexpos(1) # Lexing position of PLUS start,end = p.lexspan(3) # Lexing range of right hand expression 11/01/06: beazley Minor change to error handling. The recommended way to skip characters in the input is to use t.lexer.skip() as shown here: def t_error(t): print "Illegal character '%s'" % t.value[0] t.lexer.skip(1) The old approach of just using t.skip(1) will still work, but won't be documented. 10/31/06: beazley Discarded tokens can now be specified as simple strings instead of functions. To do this, simply include the text "ignore_" in the token declaration. For example: t_ignore_cppcomment = r'//.*' Previously, this had to be done with a function. For example: def t_ignore_cppcomment(t): r'//.*' pass If start conditions/states are being used, state names should appear before the "ignore_" text. 10/19/06: beazley The Lex module now provides support for flex-style start conditions as described at http://www.gnu.org/software/flex/manual/html_chapter/flex_11.html. Please refer to this document to understand this change note. Refer to the PLY documentation for PLY-specific explanation of how this works. To use start conditions, you first need to declare a set of states in your lexer file: states = ( ('foo','exclusive'), ('bar','inclusive') ) This serves the same role as the %s and %x specifiers in flex. One a state has been declared, tokens for that state can be declared by defining rules of the form t_state_TOK. For example: t_PLUS = '\+' # Rule defined in INITIAL state t_foo_NUM = '\d+' # Rule defined in foo state t_bar_NUM = '\d+' # Rule defined in bar state t_foo_bar_NUM = '\d+' # Rule defined in both foo and bar t_ANY_NUM = '\d+' # Rule defined in all states In addition to defining tokens for each state, the t_ignore and t_error specifications can be customized for specific states. For example: t_foo_ignore = " " # Ignored characters for foo state def t_bar_error(t): # Handle errors in bar state With token rules, the following methods can be used to change states def t_TOKNAME(t): t.lexer.begin('foo') # Begin state 'foo' t.lexer.push_state('foo') # Begin state 'foo', push old state # onto a stack t.lexer.pop_state() # Restore previous state t.lexer.current_state() # Returns name of current state These methods mirror the BEGIN(), yy_push_state(), yy_pop_state(), and yy_top_state() functions in flex. The use of start states can be used as one way to write sub-lexers. For example, the lexer or parser might instruct the lexer to start generating a different set of tokens depending on the context. example/yply/ylex.py shows the use of start states to grab C/C++ code fragments out of traditional yacc specification files. *** NEW FEATURE *** Suggested by Daniel Larraz with whom I also discussed various aspects of the design. 10/19/06: beazley Minor change to the way in which yacc.py was reporting shift/reduce conflicts. Although the underlying LALR(1) algorithm was correct, PLY was under-reporting the number of conflicts compared to yacc/bison when precedence rules were in effect. This change should make PLY report the same number of conflicts as yacc. 10/19/06: beazley Modified yacc so that grammar rules could also include the '-' character. For example: def p_expr_list(p): 'expression-list : expression-list expression' Suggested by Oldrich Jedlicka. 10/18/06: beazley Attribute lexer.lexmatch added so that token rules can access the re match object that was generated. For example: def t_FOO(t): r'some regex' m = t.lexer.lexmatch # Do something with m This may be useful if you want to access named groups specified within the regex for a specific token. Suggested by Oldrich Jedlicka. 10/16/06: beazley Changed the error message that results if an illegal character is encountered and no default error function is defined in lex. The exception is now more informative about the actual cause of the error. Version 2.1 ------------------------------ 10/02/06: beazley The last Lexer object built by lex() can be found in lex.lexer. The last Parser object built by yacc() can be found in yacc.parser. 10/02/06: beazley New example added: examples/yply This example uses PLY to convert Unix-yacc specification files to PLY programs with the same grammar. This may be useful if you want to convert a grammar from bison/yacc to use with PLY. 10/02/06: beazley Added support for a start symbol to be specified in the yacc input file itself. Just do this: start = 'name' where 'name' matches some grammar rule. For example: def p_name(p): 'name : A B C' ... This mirrors the functionality of the yacc %start specifier. 09/30/06: beazley Some new examples added.: examples/GardenSnake : A simple indentation based language similar to Python. Shows how you might handle whitespace. Contributed by Andrew Dalke. examples/BASIC : An implementation of 1964 Dartmouth BASIC. Contributed by Dave against his better judgement. 09/28/06: beazley Minor patch to allow named groups to be used in lex regular expression rules. For example: t_QSTRING = r'''(?P['"]).*?(?P=quote)''' Patch submitted by Adam Ring. 09/28/06: beazley LALR(1) is now the default parsing method. To use SLR, use yacc.yacc(method="SLR"). Note: there is no performance impact on parsing when using LALR(1) instead of SLR. However, constructing the parsing tables will take a little longer. 09/26/06: beazley Change to line number tracking. To modify line numbers, modify the line number of the lexer itself. For example: def t_NEWLINE(t): r'\n' t.lexer.lineno += 1 This modification is both cleanup and a performance optimization. In past versions, lex was monitoring every token for changes in the line number. This extra processing is unnecessary for a vast majority of tokens. Thus, this new approach cleans it up a bit. *** POTENTIAL INCOMPATIBILITY *** You will need to change code in your lexer that updates the line number. For example, "t.lineno += 1" becomes "t.lexer.lineno += 1" 09/26/06: beazley Added the lexing position to tokens as an attribute lexpos. This is the raw index into the input text at which a token appears. This information can be used to compute column numbers and other details (e.g., scan backwards from lexpos to the first newline to get a column position). 09/25/06: beazley Changed the name of the __copy__() method on the Lexer class to clone(). This is used to clone a Lexer object (e.g., if you're running different lexers at the same time). 09/21/06: beazley Limitations related to the use of the re module have been eliminated. Several users reported problems with regular expressions exceeding more than 100 named groups. To solve this, lex.py is now capable of automatically splitting its master regular regular expression into smaller expressions as needed. This should, in theory, make it possible to specify an arbitrarily large number of tokens. 09/21/06: beazley Improved error checking in lex.py. Rules that match the empty string are now rejected (otherwise they cause the lexer to enter an infinite loop). An extra check for rules containing '#' has also been added. Since lex compiles regular expressions in verbose mode, '#' is interpreted as a regex comment, it is critical to use '\#' instead. 09/18/06: beazley Added a @TOKEN decorator function to lex.py that can be used to define token rules where the documentation string might be computed in some way. digit = r'([0-9])' nondigit = r'([_A-Za-z])' identifier = r'(' + nondigit + r'(' + digit + r'|' + nondigit + r')*)' from ply.lex import TOKEN @TOKEN(identifier) def t_ID(t): # Do whatever The @TOKEN decorator merely sets the documentation string of the associated token function as needed for lex to work. Note: An alternative solution is the following: def t_ID(t): # Do whatever t_ID.__doc__ = identifier Note: Decorators require the use of Python 2.4 or later. If compatibility with old versions is needed, use the latter solution. The need for this feature was suggested by Cem Karan. 09/14/06: beazley Support for single-character literal tokens has been added to yacc. These literals must be enclosed in quotes. For example: def p_expr(p): "expr : expr '+' expr" ... def p_expr(p): 'expr : expr "-" expr' ... In addition to this, it is necessary to tell the lexer module about literal characters. This is done by defining the variable 'literals' as a list of characters. This should be defined in the module that invokes the lex.lex() function. For example: literals = ['+','-','*','/','(',')','='] or simply literals = '+=*/()=' It is important to note that literals can only be a single character. When the lexer fails to match a token using its normal regular expression rules, it will check the current character against the literal list. If found, it will be returned with a token type set to match the literal character. Otherwise, an illegal character will be signalled. 09/14/06: beazley Modified PLY to install itself as a proper Python package called 'ply'. This will make it a little more friendly to other modules. This changes the usage of PLY only slightly. Just do this to import the modules import ply.lex as lex import ply.yacc as yacc Alternatively, you can do this: from ply import * Which imports both the lex and yacc modules. Change suggested by Lee June. 09/13/06: beazley Changed the handling of negative indices when used in production rules. A negative production index now accesses already parsed symbols on the parsing stack. For example, def p_foo(p): "foo: A B C D" print p[1] # Value of 'A' symbol print p[2] # Value of 'B' symbol print p[-1] # Value of whatever symbol appears before A # on the parsing stack. p[0] = some_val # Sets the value of the 'foo' grammer symbol This behavior makes it easier to work with embedded actions within the parsing rules. For example, in C-yacc, it is possible to write code like this: bar: A { printf("seen an A = %d\n", $1); } B { do_stuff; } In this example, the printf() code executes immediately after A has been parsed. Within the embedded action code, $1 refers to the A symbol on the stack. To perform this equivalent action in PLY, you need to write a pair of rules like this: def p_bar(p): "bar : A seen_A B" do_stuff def p_seen_A(p): "seen_A :" print "seen an A =", p[-1] The second rule "seen_A" is merely a empty production which should be reduced as soon as A is parsed in the "bar" rule above. The use of the negative index p[-1] is used to access whatever symbol appeared before the seen_A symbol. This feature also makes it possible to support inherited attributes. For example: def p_decl(p): "decl : scope name" def p_scope(p): """scope : GLOBAL | LOCAL""" p[0] = p[1] def p_name(p): "name : ID" if p[-1] == "GLOBAL": # ... else if p[-1] == "LOCAL": #... In this case, the name rule is inheriting an attribute from the scope declaration that precedes it. *** POTENTIAL INCOMPATIBILITY *** If you are currently using negative indices within existing grammar rules, your code will break. This should be extremely rare if non-existent in most cases. The argument to various grammar rules is not usually not processed in the same way as a list of items. Version 2.0 ------------------------------ 09/07/06: beazley Major cleanup and refactoring of the LR table generation code. Both SLR and LALR(1) table generation is now performed by the same code base with only minor extensions for extra LALR(1) processing. 09/07/06: beazley Completely reimplemented the entire LALR(1) parsing engine to use the DeRemer and Pennello algorithm for calculating lookahead sets. This significantly improves the performance of generating LALR(1) tables and has the added feature of actually working correctly! If you experienced weird behavior with LALR(1) in prior releases, this should hopefully resolve all of those problems. Many thanks to Andrew Waters and Markus Schoepflin for submitting bug reports and helping me test out the revised LALR(1) support. Version 1.8 ------------------------------ 08/02/06: beazley Fixed a problem related to the handling of default actions in LALR(1) parsing. If you experienced subtle and/or bizarre behavior when trying to use the LALR(1) engine, this may correct those problems. Patch contributed by Russ Cox. Note: This patch has been superceded by revisions for LALR(1) parsing in Ply-2.0. 08/02/06: beazley Added support for slicing of productions in yacc. Patch contributed by Patrick Mezard. Version 1.7 ------------------------------ 03/02/06: beazley Fixed infinite recursion problem ReduceToTerminals() function that would sometimes come up in LALR(1) table generation. Reported by Markus Schoepflin. 03/01/06: beazley Added "reflags" argument to lex(). For example: lex.lex(reflags=re.UNICODE) This can be used to specify optional flags to the re.compile() function used inside the lexer. This may be necessary for special situations such as processing Unicode (e.g., if you want escapes like \w and \b to consult the Unicode character property database). The need for this suggested by Andreas Jung. 03/01/06: beazley Fixed a bug with an uninitialized variable on repeated instantiations of parser objects when the write_tables=0 argument was used. Reported by Michael Brown. 03/01/06: beazley Modified lex.py to accept Unicode strings both as the regular expressions for tokens and as input. Hopefully this is the only change needed for Unicode support. Patch contributed by Johan Dahl. 03/01/06: beazley Modified the class-based interface to work with new-style or old-style classes. Patch contributed by Michael Brown (although I tweaked it slightly so it would work with older versions of Python). Version 1.6 ------------------------------ 05/27/05: beazley Incorporated patch contributed by Christopher Stawarz to fix an extremely devious bug in LALR(1) parser generation. This patch should fix problems numerous people reported with LALR parsing. 05/27/05: beazley Fixed problem with lex.py copy constructor. Reported by Dave Aitel, Aaron Lav, and Thad Austin. 05/27/05: beazley Added outputdir option to yacc() to control output directory. Contributed by Christopher Stawarz. 05/27/05: beazley Added rununit.py test script to run tests using the Python unittest module. Contributed by Miki Tebeka. Version 1.5 ------------------------------ 05/26/04: beazley Major enhancement. LALR(1) parsing support is now working. This feature was implemented by Elias Ioup (ezioup@alumni.uchicago.edu) and optimized by David Beazley. To use LALR(1) parsing do the following: yacc.yacc(method="LALR") Computing LALR(1) parsing tables takes about twice as long as the default SLR method. However, LALR(1) allows you to handle more complex grammars. For example, the ANSI C grammar (in example/ansic) has 13 shift-reduce conflicts with SLR, but only has 1 shift-reduce conflict with LALR(1). 05/20/04: beazley Added a __len__ method to parser production lists. Can be used in parser rules like this: def p_somerule(p): """a : B C D | E F" if (len(p) == 3): # Must have been first rule elif (len(p) == 2): # Must be second rule Suggested by Joshua Gerth and others. Version 1.4 ------------------------------ 04/23/04: beazley Incorporated a variety of patches contributed by Eric Raymond. These include: 0. Cleans up some comments so they don't wrap on an 80-column display. 1. Directs compiler errors to stderr where they belong. 2. Implements and documents automatic line counting when \n is ignored. 3. Changes the way progress messages are dumped when debugging is on. The new format is both less verbose and conveys more information than the old, including shift and reduce actions. 04/23/04: beazley Added a Python setup.py file to simply installation. Contributed by Adam Kerrison. 04/23/04: beazley Added patches contributed by Adam Kerrison. - Some output is now only shown when debugging is enabled. This means that PLY will be completely silent when not in debugging mode. - An optional parameter "write_tables" can be passed to yacc() to control whether or not parsing tables are written. By default, it is true, but it can be turned off if you don't want the yacc table file. Note: disabling this will cause yacc() to regenerate the parsing table each time. 04/23/04: beazley Added patches contributed by David McNab. This patch addes two features: - The parser can be supplied as a class instead of a module. For an example of this, see the example/classcalc directory. - Debugging output can be directed to a filename of the user's choice. Use yacc(debugfile="somefile.out") Version 1.3 ------------------------------ 12/10/02: jmdyck Various minor adjustments to the code that Dave checked in today. Updated test/yacc_{inf,unused}.exp to reflect today's changes. 12/10/02: beazley Incorporated a variety of minor bug fixes to empty production handling and infinite recursion checking. Contributed by Michael Dyck. 12/10/02: beazley Removed bogus recover() method call in yacc.restart() Version 1.2 ------------------------------ 11/27/02: beazley Lexer and parser objects are now available as an attribute of tokens and slices respectively. For example: def t_NUMBER(t): r'\d+' print t.lexer def p_expr_plus(t): 'expr: expr PLUS expr' print t.lexer print t.parser This can be used for state management (if needed). 10/31/02: beazley Modified yacc.py to work with Python optimize mode. To make this work, you need to use yacc.yacc(optimize=1) Furthermore, you need to first run Python in normal mode to generate the necessary parsetab.py files. After that, you can use python -O or python -OO. Note: optimized mode turns off a lot of error checking. Only use when you are sure that your grammar is working. Make sure parsetab.py is up to date! 10/30/02: beazley Added cloning of Lexer objects. For example: import copy l = lex.lex() lc = copy.copy(l) l.input("Some text") lc.input("Some other text") ... This might be useful if the same "lexer" is meant to be used in different contexts---or if multiple lexers are running concurrently. 10/30/02: beazley Fixed subtle bug with first set computation and empty productions. Patch submitted by Michael Dyck. 10/30/02: beazley Fixed error messages to use "filename:line: message" instead of "filename:line. message". This makes error reporting more friendly to emacs. Patch submitted by Franois Pinard. 10/30/02: beazley Improvements to parser.out file. Terminals and nonterminals are sorted instead of being printed in random order. Patch submitted by Franois Pinard. 10/30/02: beazley Improvements to parser.out file output. Rules are now printed in a way that's easier to understand. Contributed by Russ Cox. 10/30/02: beazley Added 'nonassoc' associativity support. This can be used to disable the chaining of operators like a < b < c. To use, simply specify 'nonassoc' in the precedence table precedence = ( ('nonassoc', 'LESSTHAN', 'GREATERTHAN'), # Nonassociative operators ('left', 'PLUS', 'MINUS'), ('left', 'TIMES', 'DIVIDE'), ('right', 'UMINUS'), # Unary minus operator ) Patch contributed by Russ Cox. 10/30/02: beazley Modified the lexer to provide optional support for Python -O and -OO modes. To make this work, Python *first* needs to be run in unoptimized mode. This reads the lexing information and creates a file "lextab.py". Then, run lex like this: # module foo.py ... ... lex.lex(optimize=1) Once the lextab file has been created, subsequent calls to lex.lex() will read data from the lextab file instead of using introspection. In optimized mode (-O, -OO) everything should work normally despite the loss of doc strings. To change the name of the file 'lextab.py' use the following: lex.lex(lextab="footab") (this creates a file footab.py) Version 1.1 October 25, 2001 ------------------------------ 10/25/01: beazley Modified the table generator to produce much more compact data. This should greatly reduce the size of the parsetab.py[c] file. Caveat: the tables still need to be constructed so a little more work is done in parsetab on import. 10/25/01: beazley There may be a possible bug in the cycle detector that reports errors about infinite recursion. I'm having a little trouble tracking it down, but if you get this problem, you can disable the cycle detector as follows: yacc.yacc(check_recursion = 0) 10/25/01: beazley Fixed a bug in lex.py that sometimes caused illegal characters to be reported incorrectly. Reported by Sverre Jrgensen. 7/8/01 : beazley Added a reference to the underlying lexer object when tokens are handled by functions. The lexer is available as the 'lexer' attribute. This was added to provide better lexing support for languages such as Fortran where certain types of tokens can't be conveniently expressed as regular expressions (and where the tokenizing function may want to perform a little backtracking). Suggested by Pearu Peterson. 6/20/01 : beazley Modified yacc() function so that an optional starting symbol can be specified. For example: yacc.yacc(start="statement") Normally yacc always treats the first production rule as the starting symbol. However, if you are debugging your grammar it may be useful to specify an alternative starting symbol. Idea suggested by Rich Salz. Version 1.0 June 18, 2001 -------------------------- Initial public offering ================================================ FILE: xonsh/ply/CONTRIBUTING.md ================================================ Contributing to PLY =================== PLY is a mature project that no longer makes releases. New features are no longer being added to it. However, if you feel that you have found a bug in PLY or its documentation, please submit an issue or a pull request. Important note: The Github repo for PLY always contains the most up-to-date version of the software. If you want to use the current version, you should COPY the contents of the `ply/` directory into your own project and use it. There will be no future package-installable releases of PLY. ================================================ FILE: xonsh/ply/Makefile ================================================ PYTHON ?= python test: cd test && $(PYTHON) testlex.py cd test && $(PYTHON) testyacc.py wheel: $(PYTHON) setup.py bdist_wheel sdist: $(PYTHON) setup.py sdist upload: wheel sdist $(PYTHON) setup.py bdist_wheel upload $(PYTHON) setup.py sdist upload .PHONY: test wheel sdist upload ================================================ FILE: xonsh/ply/README.md ================================================ # PLY (Python Lex-Yacc) [![Build Status](https://travis-ci.org/dabeaz/ply.svg?branch=master)](https://travis-ci.org/dabeaz/ply) Copyright (C) 2001-2019 David M. Beazley (Dabeaz LLC) All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of the David Beazley or Dabeaz LLC may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. Introduction ============ PLY is a 100% Python implementation of the common parsing tools lex and yacc. Here are a few highlights: - PLY is very closely modeled after traditional lex/yacc. If you know how to use these tools in C, you will find PLY to be similar. - PLY provides *very* extensive error reporting and diagnostic information to assist in parser construction. The original implementation was developed for instructional purposes. As a result, the system tries to identify the most common types of errors made by novice users. - PLY provides full support for empty productions, error recovery, precedence specifiers, and moderately ambiguous grammars. - Parsing is based on LR-parsing which is fast, memory efficient, better suited to large grammars, and which has a number of nice properties when dealing with syntax errors and other parsing problems. Currently, PLY builds its parsing tables using the LALR(1) algorithm used in yacc. - PLY uses Python introspection features to build lexers and parsers. This greatly simplifies the task of parser construction since it reduces the number of files and eliminates the need to run a separate lex/yacc tool before running your program. - PLY can be used to build parsers for "real" programming languages. Although it is not ultra-fast due to its Python implementation, PLY can be used to parse grammars consisting of several hundred rules (as might be found for a language like C). The lexer and LR parser are also reasonably efficient when parsing typically sized programs. People have used PLY to build parsers for C, C++, ADA, and other real programming languages. How to Use ========== PLY consists of two files : lex.py and yacc.py. These are contained within the 'ply' directory which may also be used as a Python package. To use PLY, simply copy the 'ply' directory to your project and import lex and yacc from the associated 'ply' package. For example: from .ply import lex from .ply import yacc Alternatively, you can copy just the files lex.py and yacc.py individually and use them as modules however you see fit. For example: import lex import yacc PLY has no third-party dependencies. The file doc/ply.html contains complete documentation on how to use the system. The example directory contains several different examples including a PLY specification for ANSI C as given in K&R 2nd Ed. A simple example is found at the end of this document Requirements ============ PLY requires the use of Python 2.6 or greater. However, you should use the latest Python release if possible. It should work on just about any platform. PLY has been tested with both CPython and Jython. It also seems to work with IronPython. Resources ========= More information about PLY can be obtained on the PLY webpage at: http://www.dabeaz.com/ply For a detailed overview of parsing theory, consult the excellent book "Compilers : Principles, Techniques, and Tools" by Aho, Sethi, and Ullman. The topics found in "Lex & Yacc" by Levine, Mason, and Brown may also be useful. The GitHub page for PLY can be found at: https://github.com/dabeaz/ply An old and inactive discussion group for PLY is found at: http://groups.google.com/group/ply-hack Acknowledgments =============== A special thanks is in order for all of the students in CS326 who suffered through about 25 different versions of these tools :-). The CHANGES file acknowledges those who have contributed patches. Elias Ioup did the first implementation of LALR(1) parsing in PLY-1.x. Andrew Waters and Markus Schoepflin were instrumental in reporting bugs and testing a revised LALR(1) implementation for PLY-2.0. Special Note for PLY-3.0 ======================== PLY-3.0 the first PLY release to support Python 3. However, backwards compatibility with Python 2.6 is still preserved. PLY provides dual Python 2/3 compatibility by restricting its implementation to a common subset of basic language features. You should not convert PLY using 2to3--it is not necessary and may in fact break the implementation. Example ======= Here is a simple example showing a PLY implementation of a calculator with variables. # ----------------------------------------------------------------------------- # calc.py # # A simple calculator with variables. # ----------------------------------------------------------------------------- tokens = ( 'NAME','NUMBER', 'PLUS','MINUS','TIMES','DIVIDE','EQUALS', 'LPAREN','RPAREN', ) # Tokens t_PLUS = r'\+' t_MINUS = r'-' t_TIMES = r'\*' t_DIVIDE = r'/' t_EQUALS = r'=' t_LPAREN = r'\(' t_RPAREN = r'\)' t_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*' def t_NUMBER(t): r'\d+' t.value = int(t.value) return t # Ignored characters t_ignore = " \t" def t_newline(t): r'\n+' t.lexer.lineno += t.value.count("\n") def t_error(t): print("Illegal character '%s'" % t.value[0]) t.lexer.skip(1) # Build the lexer import ply.lex as lex lex.lex() # Precedence rules for the arithmetic operators precedence = ( ('left','PLUS','MINUS'), ('left','TIMES','DIVIDE'), ('right','UMINUS'), ) # dictionary of names (for storing variables) names = { } def p_statement_assign(p): 'statement : NAME EQUALS expression' names[p[1]] = p[3] def p_statement_expr(p): 'statement : expression' print(p[1]) def p_expression_binop(p): '''expression : expression PLUS expression | expression MINUS expression | expression TIMES expression | expression DIVIDE expression''' if p[2] == '+' : p[0] = p[1] + p[3] elif p[2] == '-': p[0] = p[1] - p[3] elif p[2] == '*': p[0] = p[1] * p[3] elif p[2] == '/': p[0] = p[1] / p[3] def p_expression_uminus(p): 'expression : MINUS expression %prec UMINUS' p[0] = -p[2] def p_expression_group(p): 'expression : LPAREN expression RPAREN' p[0] = p[2] def p_expression_number(p): 'expression : NUMBER' p[0] = p[1] def p_expression_name(p): 'expression : NAME' try: p[0] = names[p[1]] except LookupError: print("Undefined name '%s'" % p[1]) p[0] = 0 def p_error(p): print("Syntax error at '%s'" % p.value) import ply.yacc as yacc yacc.yacc() while True: try: s = raw_input('calc > ') # use input() on Python 3 except EOFError: break yacc.parse(s) Bug Reports and Patches ======================= My goal with PLY is to simply have a decent lex/yacc implementation for Python. As a general rule, I don't spend huge amounts of time working on it unless I receive very specific bug reports and/or patches to fix problems. At this time, PLY is mature software and new features are no longer being added. If you think you have found a bug, please visit the PLY Github page at https://github.com/dabeaz/ply to report an issue. -- Dave ================================================ FILE: xonsh/ply/__init__.py ================================================ ================================================ FILE: xonsh/ply/doc/internal.html ================================================ PLY Internals

PLY Internals

David M. Beazley
dave@dabeaz.com

PLY Version: 3.11

1. Introduction

This document describes classes and functions that make up the internal operation of PLY. Using this programming interface, it is possible to manually build an parser using a different interface specification than what PLY normally uses. For example, you could build a gramar from information parsed in a completely different input format. Some of these objects may be useful for building more advanced parsing engines such as GLR.

It should be stressed that using PLY at this level is not for the faint of heart. Generally, it's assumed that you know a bit of the underlying compiler theory and how an LR parser is put together.

2. Grammar Class

The file ply.yacc defines a class Grammar that is used to hold and manipulate information about a grammar specification. It encapsulates the same basic information about a grammar that is put into a YACC file including the list of tokens, precedence rules, and grammar rules. Various operations are provided to perform different validations on the grammar. In addition, there are operations to compute the first and follow sets that are needed by the various table generation algorithms.

Grammar(terminals)

Creates a new grammar object. terminals is a list of strings specifying the terminals for the grammar. An instance g of Grammar has the following methods:

g.set_precedence(term,assoc,level)

Sets the precedence level and associativity for a given terminal term. assoc is one of 'right', 'left', or 'nonassoc' and level is a positive integer. The higher the value of level, the higher the precedence. Here is an example of typical precedence settings: 
g.set_precedence('PLUS',  'left',1)
g.set_precedence('MINUS', 'left',1)
g.set_precedence('TIMES', 'left',2)
g.set_precedence('DIVIDE','left',2)
g.set_precedence('UMINUS','left',3)
This method must be called prior to adding any productions to the grammar with g.add_production(). The precedence of individual grammar rules is determined by the precedence of the right-most terminal.

g.add_production(name,syms,func=None,file='',line=0)

Adds a new grammar rule. name is the name of the rule, syms is a list of symbols making up the right hand side of the rule, func is the function to call when reducing the rule. file and line specify the filename and line number of the rule and are used for generating error messages.

The list of symbols in syms may include character literals and %prec specifiers. Here are some examples: 

g.add_production('expr',['expr','PLUS','term'],func,file,line)
g.add_production('expr',['expr','"+"','term'],func,file,line)
g.add_production('expr',['MINUS','expr','%prec','UMINUS'],func,file,line)

If any kind of error is detected, a GrammarError exception is raised with a message indicating the reason for the failure.

g.set_start(start=None)

Sets the starting rule for the grammar. start is a string specifying the name of the start rule. If start is omitted, the first grammar rule added with add_production() is taken to be the starting rule. This method must always be called after all productions have been added.

g.find_unreachable()

Diagnostic function. Returns a list of all unreachable non-terminals defined in the grammar. This is used to identify inactive parts of the grammar specification.

g.infinite_cycle()

Diagnostic function. Returns a list of all non-terminals in the grammar that result in an infinite cycle. This condition occurs if there is no way for a grammar rule to expand to a string containing only terminal symbols.

g.undefined_symbols()

Diagnostic function. Returns a list of tuples (name, prod) corresponding to undefined symbols in the grammar. name is the name of the undefined symbol and prod is an instance of Production which has information about the production rule where the undefined symbol was used.

g.unused_terminals()

Diagnostic function. Returns a list of terminals that were defined, but never used in the grammar.

g.unused_rules()

Diagnostic function. Returns a list of Production instances corresponding to production rules that were defined in the grammar, but never used anywhere. This is slightly different than find_unreachable().

g.unused_precedence()

Diagnostic function. Returns a list of tuples (term, assoc) corresponding to precedence rules that were set, but never used the grammar. term is the terminal name and assoc is the precedence associativity (e.g., 'left', 'right', or 'nonassoc'.

g.compute_first()

Compute all of the first sets for all symbols in the grammar. Returns a dictionary mapping symbol names to a list of all first symbols.

g.compute_follow()

Compute all of the follow sets for all non-terminals in the grammar. The follow set is the set of all possible symbols that might follow a given non-terminal. Returns a dictionary mapping non-terminal names to a list of symbols.

g.build_lritems()

Calculates all of the LR items for all productions in the grammar. This step is required before using the grammar for any kind of table generation. See the section on LR items below.

The following attributes are set by the above methods and may be useful in code that works with the grammar. All of these attributes should be assumed to be read-only. Changing their values directly will likely break the grammar.

g.Productions

A list of all productions added. The first entry is reserved for a production representing the starting rule. The objects in this list are instances of the Production class, described shortly.

g.Prodnames

A dictionary mapping the names of nonterminals to a list of all productions of that nonterminal.

g.Terminals

A dictionary mapping the names of terminals to a list of the production numbers where they are used.

g.Nonterminals

A dictionary mapping the names of nonterminals to a list of the production numbers where they are used.

g.First

A dictionary representing the first sets for all grammar symbols. This is computed and returned by the compute_first() method.

g.Follow

A dictionary representing the follow sets for all grammar rules. This is computed and returned by the compute_follow() method.

g.Start

Starting symbol for the grammar. Set by the set_start() method.
For the purposes of debugging, a Grammar object supports the __len__() and __getitem__() special methods. Accessing g[n] returns the nth production from the grammar.

3. Productions

Grammar objects store grammar rules as instances of a Production class. This class has no public constructor--you should only create productions by calling Grammar.add_production(). The following attributes are available on a Production instance p.

p.name

The name of the production. For a grammar rule such as A : B C D, this is 'A'.

p.prod

A tuple of symbols making up the right-hand side of the production. For a grammar rule such as A : B C D, this is ('B','C','D').

p.number

Production number. An integer containing the index of the production in the grammar's Productions list.

p.func

The name of the reduction function associated with the production. This is the function that will execute when reducing the entire grammar rule during parsing.

p.callable

The callable object associated with the name in p.func. This is None unless the production has been bound using bind().

p.file

Filename associated with the production. Typically this is the file where the production was defined. Used for error messages.

p.lineno

Line number associated with the production. Typically this is the line number in p.file where the production was defined. Used for error messages.

p.prec

Precedence and associativity associated with the production. This is a tuple (assoc,level) where assoc is one of 'left','right', or 'nonassoc' and level is an integer. This value is determined by the precedence of the right-most terminal symbol in the production or by use of the %prec specifier when adding the production.

p.usyms

A list of all unique symbols found in the production.

p.lr_items

A list of all LR items for this production. This attribute only has a meaningful value if the Grammar.build_lritems() method has been called. The items in this list are instances of LRItem described below.

p.lr_next

The head of a linked-list representation of the LR items in p.lr_items. This attribute only has a meaningful value if the Grammar.build_lritems() method has been called. Each LRItem instance has a lr_next attribute to move to the next item. The list is terminated by None.

p.bind(dict)

Binds the production function name in p.func to a callable object in dict. This operation is typically carried out in the last step prior to running the parsing engine and is needed since parsing tables are typically read from files which only include the function names, not the functions themselves.

Production objects support the __len__(), __getitem__(), and __str__() special methods. len(p) returns the number of symbols in p.prod and p[n] is the same as p.prod[n].

4. LRItems

The construction of parsing tables in an LR-based parser generator is primarily done over a set of "LR Items". An LR item represents a stage of parsing one of the grammar rules. To compute the LR items, it is first necessary to call Grammar.build_lritems(). Once this step, all of the productions in the grammar will have their LR items attached to them.

Here is an interactive example that shows what LR items look like if you interactively experiment. In this example, g is a Grammar object. 

>>> g.build_lritems()
>>> p = g[1]
>>> p
Production(statement -> ID = expr)
>>>
In the above code, p represents the first grammar rule. In this case, a rule 'statement -> ID = expr'.

Now, let's look at the LR items for p. 

>>> p.lr_items
[LRItem(statement -> . ID = expr), 
 LRItem(statement -> ID . = expr), 
 LRItem(statement -> ID = . expr), 
 LRItem(statement -> ID = expr .)]
>>>
In each LR item, the dot (.) represents a specific stage of parsing. In each LR item, the dot is advanced by one symbol. It is only when the dot reaches the very end that a production is successfully parsed.

An instance lr of LRItem has the following attributes that hold information related to that specific stage of parsing.

lr.name

The name of the grammar rule. For example, 'statement' in the above example.

lr.prod

A tuple of symbols representing the right-hand side of the production, including the special '.' character. For example, ('ID','.','=','expr').

lr.number

An integer representing the production number in the grammar.

lr.usyms

A set of unique symbols in the production. Inherited from the original Production instance.

lr.lr_index

An integer representing the position of the dot (.). You should never use lr.prod.index() to search for it--the result will be wrong if the grammar happens to also use (.) as a character literal.

lr.lr_after

A list of all productions that can legally appear immediately to the right of the dot (.). This list contains Production instances. This attribute represents all of the possible branches a parse can take from the current position. For example, suppose that lr represents a stage immediately before an expression like this: 
>>> lr
LRItem(statement -> ID = . expr)
>>>
Then, the value of lr.lr_after might look like this, showing all productions that can legally appear next: 
>>> lr.lr_after
[Production(expr -> expr PLUS expr), 
 Production(expr -> expr MINUS expr), 
 Production(expr -> expr TIMES expr), 
 Production(expr -> expr DIVIDE expr), 
 Production(expr -> MINUS expr), 
 Production(expr -> LPAREN expr RPAREN), 
 Production(expr -> NUMBER), 
 Production(expr -> ID)]
>>>

lr.lr_before

The grammar symbol that appears immediately before the dot (.) or None if at the beginning of the parse.

lr.lr_next

A link to the next LR item, representing the next stage of the parse. None if lr is the last LR item.
LRItem instances also support the __len__() and __getitem__() special methods. len(lr) returns the number of items in lr.prod including the dot (.). lr[n] returns lr.prod[n].

It goes without saying that all of the attributes associated with LR items should be assumed to be read-only. Modifications will very likely create a small black-hole that will consume you and your code.

5. LRTable

The LRTable class is used to represent LR parsing table data. This minimally includes the production list, action table, and goto table.

LRTable()

Create an empty LRTable object. This object contains only the information needed to run an LR parser.
An instance lrtab of LRTable has the following methods:

lrtab.read_table(module)

Populates the LR table with information from the module specified in module. module is either a module object already loaded with import or the name of a Python module. If it's a string containing a module name, it is loaded and parsing data is extracted. Returns the signature value that was used when initially writing the tables. Raises a VersionError exception if the module was created using an incompatible version of PLY.

lrtab.bind_callables(dict)

This binds all of the function names used in productions to callable objects found in the dictionary dict. During table generation and when reading LR tables from files, PLY only uses the names of action functions such as 'p_expr', 'p_statement', etc. In order to actually run the parser, these names have to be bound to callable objects. This method is always called prior to running a parser.
After lrtab has been populated, the following attributes are defined.

lrtab.lr_method

The LR parsing method used (e.g., 'LALR')

lrtab.lr_productions

The production list. If the parsing tables have been newly constructed, this will be a list of Production instances. If the parsing tables have been read from a file, it's a list of MiniProduction instances. This, together with lr_action and lr_goto contain all of the information needed by the LR parsing engine.

lrtab.lr_action

The LR action dictionary that implements the underlying state machine. The keys of this dictionary are the LR states.

lrtab.lr_goto

The LR goto table that contains information about grammar rule reductions.

6. LRGeneratedTable

The LRGeneratedTable class represents constructed LR parsing tables on a grammar. It is a subclass of LRTable.

LRGeneratedTable(grammar, method='LALR',log=None)

Create the LR parsing tables on a grammar. grammar is an instance of Grammar, method is a string with the parsing method ('SLR' or 'LALR'), and log is a logger object used to write debugging information. The debugging information written to log is the same as what appears in the parser.out file created by yacc. By supplying a custom logger with a different message format, it is possible to get more information (e.g., the line number in yacc.py used for issuing each line of output in the log). The result is an instance of LRGeneratedTable.

An instance lr of LRGeneratedTable has the following attributes.

lr.grammar

A link to the Grammar object used to construct the parsing tables.

lr.lr_method

The LR parsing method used (e.g., 'LALR')

lr.lr_productions

A reference to grammar.Productions. This, together with lr_action and lr_goto contain all of the information needed by the LR parsing engine.

lr.lr_action

The LR action dictionary that implements the underlying state machine. The keys of this dictionary are the LR states.

lr.lr_goto

The LR goto table that contains information about grammar rule reductions.

lr.sr_conflicts

A list of tuples (state,token,resolution) identifying all shift/reduce conflicts. state is the LR state number where the conflict occurred, token is the token causing the conflict, and resolution is a string describing the resolution taken. resolution is either 'shift' or 'reduce'.

lr.rr_conflicts

A list of tuples (state,rule,rejected) identifying all reduce/reduce conflicts. state is the LR state number where the conflict occurred, rule is the production rule that was selected and rejected is the production rule that was rejected. Both rule and rejected are instances of Production. They can be inspected to provide the user with more information.

There are two public methods of LRGeneratedTable.

lr.write_table(modulename,outputdir="",signature="")

Writes the LR parsing table information to a Python module. modulename is a string specifying the name of a module such as "parsetab". outputdir is the name of a directory where the module should be created. signature is a string representing a grammar signature that's written into the output file. This can be used to detect when the data stored in a module file is out-of-sync with the the grammar specification (and that the tables need to be regenerated). If modulename is a string "parsetab", this function creates a file called parsetab.py. If the module name represents a package such as "foo.bar.parsetab", then only the last component, "parsetab" is used.

7. LRParser

The LRParser class implements the low-level LR parsing engine.

LRParser(lrtab, error_func)

Create an LRParser. lrtab is an instance of LRTable containing the LR production and state tables. error_func is the error function to invoke in the event of a parsing error.
An instance p of LRParser has the following methods:

p.parse(input=None,lexer=None,debug=0,tracking=0,tokenfunc=None)

Run the parser. input is a string, which if supplied is fed into the lexer using its input() method. lexer is an instance of the Lexer class to use for tokenizing. If not supplied, the last lexer created with the lex module is used. debug is a boolean flag that enables debugging. tracking is a boolean flag that tells the parser to perform additional line number tracking. tokenfunc is a callable function that returns the next token. If supplied, the parser will use it to get all tokens.

p.restart()

Resets the parser state for a parse already in progress.

8. ParserReflect

The ParserReflect class is used to collect parser specification data from a Python module or object. This class is what collects all of the p_rule() functions in a PLY file, performs basic error checking, and collects all of the needed information to build a grammar. Most of the high-level PLY interface as used by the yacc() function is actually implemented by this class.

ParserReflect(pdict, log=None)

Creates a ParserReflect instance. pdict is a dictionary containing parser specification data. This dictionary typically corresponds to the module or class dictionary of code that implements a PLY parser. log is a logger instance that will be used to report error messages.
An instance p of ParserReflect has the following methods:

p.get_all()

Collect and store all required parsing information.

p.validate_all()

Validate all of the collected parsing information. This is a seprate step from p.get_all() as a performance optimization. In order to increase parser start-up time, a parser can elect to only validate the parsing data when regenerating the parsing tables. The validation step tries to collect as much information as possible rather than raising an exception at the first sign of trouble. The attribute p.error is set if there are any validation errors. The value of this attribute is also returned.

p.signature()

Compute a signature representing the contents of the collected parsing data. The signature value should change if anything in the parser specification has changed in a way that would justify parser table regeneration. This method can be called after p.get_all(), but before p.validate_all().
The following attributes are set in the process of collecting data:

p.start

The grammar start symbol, if any. Taken from pdict['start'].

p.error_func

The error handling function or None. Taken from pdict['p_error'].

p.tokens

The token list. Taken from pdict['tokens'].

p.prec

The precedence specifier. Taken from pdict['precedence'].

p.preclist

A parsed version of the precedence specified. A list of tuples of the form (token,assoc,level) where token is the terminal symbol, assoc is the associativity (e.g., 'left') and level is a numeric precedence level.

p.grammar

A list of tuples (name, rules) representing the grammar rules. name is the name of a Python function or method in pdict that starts with "p_". rules is a list of tuples (filename,line,prodname,syms) representing the grammar rules found in the documentation string of that function. filename and line contain location information that can be used for debugging. prodname is the name of the production. syms is the right-hand side of the production. If you have a function like this 
def p_expr(p):
    '''expr : expr PLUS expr
            | expr MINUS expr
            | expr TIMES expr
            | expr DIVIDE expr'''
then the corresponding entry in p.grammar might look like this: 
('p_expr', [ ('calc.py',10,'expr', ['expr','PLUS','expr']),
             ('calc.py',11,'expr', ['expr','MINUS','expr']),
             ('calc.py',12,'expr', ['expr','TIMES','expr']),
             ('calc.py',13,'expr', ['expr','DIVIDE','expr'])
           ])

p.pfuncs

A sorted list of tuples (line, file, name, doc) representing all of the p_ functions found. line and file give location information. name is the name of the function. doc is the documentation string. This list is sorted in ascending order by line number.

p.files

A dictionary holding all of the source filenames that were encountered while collecting parser information. Only the keys of this dictionary have any meaning.

p.error

An attribute that indicates whether or not any critical errors occurred in validation. If this is set, it means that that some kind of problem was detected and that no further processing should be performed.

9. High-level operation

Using all of the above classes requires some attention to detail. The yacc() function carries out a very specific sequence of operations to create a grammar. This same sequence should be emulated if you build an alternative PLY interface.
  1. A ParserReflect object is created and raw grammar specification data is collected.
  2. A Grammar object is created and populated with information from the specification data.
  3. A LRGenerator object is created to run the LALR algorithm over the Grammar object.
  4. Productions in the LRGenerator and bound to callables using the bind_callables() method.
  5. A LRParser object is created from from the information in the LRGenerator object.
================================================ FILE: xonsh/ply/doc/makedoc.py ================================================ #!/usr/local/bin/python ############################################################################### # Takes a chapter as input and adds internal links and numbering to all # of the H1, H2, H3, H4 and H5 sections. # # Every heading HTML tag (H1, H2 etc) is given an autogenerated name to link # to. However, if the name is not an autogenerated name from a previous run, # it will be kept. If it is autogenerated, it might change on subsequent runs # of this program. Thus if you want to create links to one of the headings, # then change the heading link name to something that does not look like an # autogenerated link name. ############################################################################### import sys import re import string ############################################################################### # Functions ############################################################################### # Regexs for alink = re.compile(r"", re.IGNORECASE) heading = re.compile(r"(_nn\d)", re.IGNORECASE) def getheadingname(m): autogeneratedheading = True if m.group(1) != None: amatch = alink.match(m.group(1)) if amatch: # A non-autogenerated heading - keep it headingname = amatch.group(1) autogeneratedheading = heading.match(headingname) if autogeneratedheading: # The heading name was either non-existent or autogenerated, # We can create a new heading / change the existing heading headingname = "%s_nn%d" % (filenamebase, nameindex) return headingname ############################################################################### # Main program ############################################################################### if len(sys.argv) != 2: print "usage: makedoc.py filename" sys.exit(1) filename = sys.argv[1] filenamebase = string.split(filename,".")[0] section = 0 subsection = 0 subsubsection = 0 subsubsubsection = 0 nameindex = 0 name = "" # Regexs for

,...

sections h1 = re.compile(r".*?

()*[\d\.\s]*(.*?)

", re.IGNORECASE) h2 = re.compile(r".*?

()*[\d\.\s]*(.*?)

", re.IGNORECASE) h3 = re.compile(r".*?

()*[\d\.\s]*(.*?)

", re.IGNORECASE) h4 = re.compile(r".*?

()*[\d\.\s]*(.*?)

", re.IGNORECASE) h5 = re.compile(r".*?
()*[\d\.\s]*(.*?)
", re.IGNORECASE) # Make backup with open(filename) as src, open(filename+".bak","w") as dst: dst.write(src.read()) lines = data.splitlines() result = [ ] # This is the result of postprocessing the file index = "\n
\n" # index contains the index for adding at the top of the file. Also printed to stdout. skip = 0 skipspace = 0 for s in lines: if s == "": if not skip: result.append("@INDEX@") skip = 1 else: skip = 0 continue if skip: continue if not s and skipspace: continue if skipspace: result.append("") result.append("") skipspace = 0 m = h2.match(s) if m: prevheadingtext = m.group(2) nameindex += 1 section += 1 headingname = getheadingname(m) result.append("""

%d. %s

""" % (headingname,section, prevheadingtext)) if subsubsubsection: index += "\n" if subsubsection: index += "\n" if subsection: index += "\n" if section == 1: index += "
    \n" index += """
  • %s\n""" % (headingname,prevheadingtext) subsection = 0 subsubsection = 0 subsubsubsection = 0 skipspace = 1 continue m = h3.match(s) if m: prevheadingtext = m.group(2) nameindex += 1 subsection += 1 headingname = getheadingname(m) result.append("""

    %d.%d %s

    """ % (headingname,section, subsection, prevheadingtext)) if subsubsubsection: index += "
\n" if subsubsection: index += "\n" if subsection == 1: index += "
    \n" index += """
  • %s\n""" % (headingname,prevheadingtext) subsubsection = 0 skipspace = 1 continue m = h4.match(s) if m: prevheadingtext = m.group(2) nameindex += 1 subsubsection += 1 subsubsubsection = 0 headingname = getheadingname(m) result.append("""

    %d.%d.%d %s

    """ % (headingname,section, subsection, subsubsection, prevheadingtext)) if subsubsubsection: index += "
\n" if subsubsection == 1: index += "
    \n" index += """
  • %s\n""" % (headingname,prevheadingtext) skipspace = 1 continue m = h5.match(s) if m: prevheadingtext = m.group(2) nameindex += 1 subsubsubsection += 1 headingname = getheadingname(m) result.append("""
    %d.%d.%d.%d %s
    """ % (headingname,section, subsection, subsubsection, subsubsubsection, prevheadingtext)) if subsubsubsection == 1: index += "
      \n" index += """
    • %s\n""" % (headingname,prevheadingtext) skipspace = 1 continue result.append(s) if subsubsubsection: index += "
    \n" if subsubsection: index += "
\n" if subsection: index += "\n" if section: index += "\n" index += "
\n\n" data = "\n".join(result) data = data.replace("@INDEX@",index) + "\n" # Write the file back out with open(filename,"w") as f: f.write(data) ================================================ FILE: xonsh/ply/doc/ply.html ================================================ PLY (Python Lex-Yacc)

PLY (Python Lex-Yacc)

David M. Beazley
dave@dabeaz.com

PLY Version: 3.11

1. Preface and Requirements

This document provides an overview of lexing and parsing with PLY. Given the intrinsic complexity of parsing, I would strongly advise that you read (or at least skim) this entire document before jumping into a big development project with PLY.

PLY-3.5 is compatible with both Python 2 and Python 3. If you are using Python 2, you have to use Python 2.6 or newer.

2. Introduction

PLY is a pure-Python implementation of the popular compiler construction tools lex and yacc. The main goal of PLY is to stay fairly faithful to the way in which traditional lex/yacc tools work. This includes supporting LALR(1) parsing as well as providing extensive input validation, error reporting, and diagnostics. Thus, if you've used yacc in another programming language, it should be relatively straightforward to use PLY.

Early versions of PLY were developed to support an Introduction to Compilers Course I taught in 2001 at the University of Chicago. Since PLY was primarily developed as an instructional tool, you will find it to be fairly picky about token and grammar rule specification. In part, this added formality is meant to catch common programming mistakes made by novice users. However, advanced users will also find such features to be useful when building complicated grammars for real programming languages. It should also be noted that PLY does not provide much in the way of bells and whistles (e.g., automatic construction of abstract syntax trees, tree traversal, etc.). Nor would I consider it to be a parsing framework. Instead, you will find a bare-bones, yet fully capable lex/yacc implementation written entirely in Python.

The rest of this document assumes that you are somewhat familiar with parsing theory, syntax directed translation, and the use of compiler construction tools such as lex and yacc in other programming languages. If you are unfamiliar with these topics, you will probably want to consult an introductory text such as "Compilers: Principles, Techniques, and Tools", by Aho, Sethi, and Ullman. O'Reilly's "Lex and Yacc" by John Levine may also be handy. In fact, the O'Reilly book can be used as a reference for PLY as the concepts are virtually identical.

3. PLY Overview

PLY consists of two separate modules; lex.py and yacc.py, both of which are found in a Python package called ply. The lex.py module is used to break input text into a collection of tokens specified by a collection of regular expression rules. yacc.py is used to recognize language syntax that has been specified in the form of a context free grammar.

The two tools are meant to work together. Specifically, lex.py provides an external interface in the form of a token() function that returns the next valid token on the input stream. yacc.py calls this repeatedly to retrieve tokens and invoke grammar rules. The output of yacc.py is often an Abstract Syntax Tree (AST). However, this is entirely up to the user. If desired, yacc.py can also be used to implement simple one-pass compilers.

Like its Unix counterpart, yacc.py provides most of the features you expect including extensive error checking, grammar validation, support for empty productions, error tokens, and ambiguity resolution via precedence rules. In fact, almost everything that is possible in traditional yacc should be supported in PLY.

The primary difference between yacc.py and Unix yacc is that yacc.py doesn't involve a separate code-generation process. Instead, PLY relies on reflection (introspection) to build its lexers and parsers. Unlike traditional lex/yacc which require a special input file that is converted into a separate source file, the specifications given to PLY are valid Python programs. This means that there are no extra source files nor is there a special compiler construction step (e.g., running yacc to generate Python code for the compiler). Since the generation of the parsing tables is relatively expensive, PLY caches the results and saves them to a file. If no changes are detected in the input source, the tables are read from the cache. Otherwise, they are regenerated.

4. Lex

lex.py is used to tokenize an input string. For example, suppose you're writing a programming language and a user supplied the following input string: 
x = 3 + 42 * (s - t)
A tokenizer splits the string into individual tokens 
'x','=', '3', '+', '42', '*', '(', 's', '-', 't', ')'
Tokens are usually given names to indicate what they are. For example: 
'ID','EQUALS','NUMBER','PLUS','NUMBER','TIMES',
'LPAREN','ID','MINUS','ID','RPAREN'
More specifically, the input is broken into pairs of token types and values. For example: 
('ID','x'), ('EQUALS','='), ('NUMBER','3'), 
('PLUS','+'), ('NUMBER','42), ('TIMES','*'),
('LPAREN','('), ('ID','s'), ('MINUS','-'),
('ID','t'), ('RPAREN',')'
The identification of tokens is typically done by writing a series of regular expression rules. The next section shows how this is done using lex.py.

4.1 Lex Example

The following example shows how lex.py is used to write a simple tokenizer. 
# ------------------------------------------------------------
# calclex.py
#
# tokenizer for a simple expression evaluator for
# numbers and +,-,*,/
# ------------------------------------------------------------
import ply.lex as lex

# List of token names.   This is always required
tokens = (
   'NUMBER',
   'PLUS',
   'MINUS',
   'TIMES',
   'DIVIDE',
   'LPAREN',
   'RPAREN',
)

# Regular expression rules for simple tokens
t_PLUS    = r'\+'
t_MINUS   = r'-'
t_TIMES   = r'\*'
t_DIVIDE  = r'/'
t_LPAREN  = r'\('
t_RPAREN  = r'\)'

# A regular expression rule with some action code
def t_NUMBER(t):
    r'\d+'
    t.value = int(t.value)    
    return t

# Define a rule so we can track line numbers
def t_newline(t):
    r'\n+'
    t.lexer.lineno += len(t.value)

# A string containing ignored characters (spaces and tabs)
t_ignore  = ' \t'

# Error handling rule
def t_error(t):
    print("Illegal character '%s'" % t.value[0])
    t.lexer.skip(1)

# Build the lexer
lexer = lex.lex()
To use the lexer, you first need to feed it some input text using its input() method. After that, repeated calls to token() produce tokens. The following code shows how this works: 

# Test it out
data = '''
3 + 4 * 10
  + -20 *2
'''

# Give the lexer some input
lexer.input(data)

# Tokenize
while True:
    tok = lexer.token()
    if not tok: 
        break      # No more input
    print(tok)
When executed, the example will produce the following output: 
$ python example.py
LexToken(NUMBER,3,2,1)
LexToken(PLUS,'+',2,3)
LexToken(NUMBER,4,2,5)
LexToken(TIMES,'*',2,7)
LexToken(NUMBER,10,2,10)
LexToken(PLUS,'+',3,14)
LexToken(MINUS,'-',3,16)
LexToken(NUMBER,20,3,18)
LexToken(TIMES,'*',3,20)
LexToken(NUMBER,2,3,21)
Lexers also support the iteration protocol. So, you can write the above loop as follows: 
for tok in lexer:
    print(tok)
The tokens returned by lexer.token() are instances of LexToken. This object has attributes tok.type, tok.value, tok.lineno, and tok.lexpos. The following code shows an example of accessing these attributes: 
# Tokenize
while True:
    tok = lexer.token()
    if not tok: 
        break      # No more input
    print(tok.type, tok.value, tok.lineno, tok.lexpos)
The tok.type and tok.value attributes contain the type and value of the token itself. tok.lineno and tok.lexpos contain information about the location of the token. tok.lexpos is the index of the token relative to the start of the input text.

4.2 The tokens list

All lexers must provide a list tokens that defines all of the possible token names that can be produced by the lexer. This list is always required and is used to perform a variety of validation checks. The tokens list is also used by the yacc.py module to identify terminals.

In the example, the following code specified the token names: 

tokens = (
   'NUMBER',
   'PLUS',
   'MINUS',
   'TIMES',
   'DIVIDE',
   'LPAREN',
   'RPAREN',
)

4.3 Specification of tokens

Each token is specified by writing a regular expression rule compatible with Python's re module. Each of these rules are defined by making declarations with a special prefix t_ to indicate that it defines a token. For simple tokens, the regular expression can be specified as strings such as this (note: Python raw strings are used since they are the most convenient way to write regular expression strings): 
t_PLUS = r'\+'
In this case, the name following the t_ must exactly match one of the names supplied in tokens. If some kind of action needs to be performed, a token rule can be specified as a function. For example, this rule matches numbers and converts the string into a Python integer. 
def t_NUMBER(t):
    r'\d+'
    t.value = int(t.value)
    return t
When a function is used, the regular expression rule is specified in the function documentation string. The function always takes a single argument which is an instance of LexToken. This object has attributes of t.type which is the token type (as a string), t.value which is the lexeme (the actual text matched), t.lineno which is the current line number, and t.lexpos which is the position of the token relative to the beginning of the input text. By default, t.type is set to the name following the t_ prefix. The action function can modify the contents of the LexToken object as appropriate. However, when it is done, the resulting token should be returned. If no value is returned by the action function, the token is simply discarded and the next token read.

Internally, lex.py uses the re module to do its pattern matching. Patterns are compiled using the re.VERBOSE flag which can be used to help readability. However, be aware that unescaped whitespace is ignored and comments are allowed in this mode. If your pattern involves whitespace, make sure you use \s. If you need to match the # character, use [#].

When building the master regular expression, rules are added in the following order:

  1. All tokens defined by functions are added in the same order as they appear in the lexer file.
  2. Tokens defined by strings are added next by sorting them in order of decreasing regular expression length (longer expressions are added first).

Without this ordering, it can be difficult to correctly match certain types of tokens. For example, if you wanted to have separate tokens for "=" and "==", you need to make sure that "==" is checked first. By sorting regular expressions in order of decreasing length, this problem is solved for rules defined as strings. For functions, the order can be explicitly controlled since rules appearing first are checked first.

To handle reserved words, you should write a single rule to match an identifier and do a special name lookup in a function like this: 

reserved = {
   'if' : 'IF',
   'then' : 'THEN',
   'else' : 'ELSE',
   'while' : 'WHILE',
   ...
}

tokens = ['LPAREN','RPAREN',...,'ID'] + list(reserved.values())

def t_ID(t):
    r'[a-zA-Z_][a-zA-Z_0-9]*'
    t.type = reserved.get(t.value,'ID')    # Check for reserved words
    return t
This approach greatly reduces the number of regular expression rules and is likely to make things a little faster.

Note: You should avoid writing individual rules for reserved words. For example, if you write rules like this, 

t_FOR   = r'for'
t_PRINT = r'print'
those rules will be triggered for identifiers that include those words as a prefix such as "forget" or "printed". This is probably not what you want.

4.4 Token values

When tokens are returned by lex, they have a value that is stored in the value attribute. Normally, the value is the text that was matched. However, the value can be assigned to any Python object. For instance, when lexing identifiers, you may want to return both the identifier name and information from some sort of symbol table. To do this, you might write a rule like this: 
def t_ID(t):
    ...
    # Look up symbol table information and return a tuple
    t.value = (t.value, symbol_lookup(t.value))
    ...
    return t
It is important to note that storing data in other attribute names is not recommended. The yacc.py module only exposes the contents of the value attribute. Thus, accessing other attributes may be unnecessarily awkward. If you need to store multiple values on a token, assign a tuple, dictionary, or instance to value.

4.5 Discarded tokens

To discard a token, such as a comment, simply define a token rule that returns no value. For example: 
def t_COMMENT(t):
    r'\#.*'
    pass
    # No return value. Token discarded
Alternatively, you can include the prefix "ignore_" in the token declaration to force a token to be ignored. For example: 
t_ignore_COMMENT = r'\#.*'
Be advised that if you are ignoring many different kinds of text, you may still want to use functions since these provide more precise control over the order in which regular expressions are matched (i.e., functions are matched in order of specification whereas strings are sorted by regular expression length).

4.6 Line numbers and positional information

By default, lex.py knows nothing about line numbers. This is because lex.py doesn't know anything about what constitutes a "line" of input (e.g., the newline character or even if the input is textual data). To update this information, you need to write a special rule. In the example, the t_newline() rule shows how to do this. 

# Define a rule so we can track line numbers
def t_newline(t):
    r'\n+'
    t.lexer.lineno += len(t.value)
Within the rule, the lineno attribute of the underlying lexer t.lexer is updated. After the line number is updated, the token is simply discarded since nothing is returned.

lex.py does not perform any kind of automatic column tracking. However, it does record positional information related to each token in the lexpos attribute. Using this, it is usually possible to compute column information as a separate step. For instance, just count backwards until you reach a newline. 

# Compute column.
#     input is the input text string
#     token is a token instance
def find_column(input, token):
    line_start = input.rfind('\n', 0, token.lexpos) + 1
    return (token.lexpos - line_start) + 1
Since column information is often only useful in the context of error handling, calculating the column position can be performed when needed as opposed to doing it for each token.

4.7 Ignored characters

The special t_ignore rule is reserved by lex.py for characters that should be completely ignored in the input stream. Usually this is used to skip over whitespace and other non-essential characters. Although it is possible to define a regular expression rule for whitespace in a manner similar to t_newline(), the use of t_ignore provides substantially better lexing performance because it is handled as a special case and is checked in a much more efficient manner than the normal regular expression rules.

The characters given in t_ignore are not ignored when such characters are part of other regular expression patterns. For example, if you had a rule to capture quoted text, that pattern can include the ignored characters (which will be captured in the normal way). The main purpose of t_ignore is to ignore whitespace and other padding between the tokens that you actually want to parse.

4.8 Literal characters

Literal characters can be specified by defining a variable literals in your lexing module. For example: 

literals = [ '+','-','*','/' ]
or alternatively 
literals = "+-*/"
A literal character is simply a single character that is returned "as is" when encountered by the lexer. Literals are checked after all of the defined regular expression rules. Thus, if a rule starts with one of the literal characters, it will always take precedence.

When a literal token is returned, both its type and value attributes are set to the character itself. For example, '+'.

It's possible to write token functions that perform additional actions when literals are matched. However, you'll need to set the token type appropriately. For example: 

literals = [ '{', '}' ]

def t_lbrace(t):
    r'\{'
    t.type = '{'      # Set token type to the expected literal
    return t

def t_rbrace(t):
    r'\}'
    t.type = '}'      # Set token type to the expected literal
    return t

4.9 Error handling

The t_error() function is used to handle lexing errors that occur when illegal characters are detected. In this case, the t.value attribute contains the rest of the input string that has not been tokenized. In the example, the error function was defined as follows: 

# Error handling rule
def t_error(t):
    print("Illegal character '%s'" % t.value[0])
    t.lexer.skip(1)
In this case, we simply print the offending character and skip ahead one character by calling t.lexer.skip(1).

4.10 EOF Handling

The t_eof() function is used to handle an end-of-file (EOF) condition in the input. As input, it receives a token type 'eof' with the lineno and lexpos attributes set appropriately. The main use of this function is provide more input to the lexer so that it can continue to parse. Here is an example of how this works: 

# EOF handling rule
def t_eof(t):
    # Get more input (Example)
    more = raw_input('... ')
    if more:
        self.lexer.input(more)
        return self.lexer.token()
    return None

The EOF function should return the next available token (by calling self.lexer.token()) or None to indicate no more data. Be aware that setting more input with the self.lexer.input() method does NOT reset the lexer state or the lineno attribute used for position tracking. The lexpos attribute is reset so be aware of that if you're using it in error reporting.

4.11 Building and using the lexer

To build the lexer, the function lex.lex() is used. For example:

lexer = lex.lex()

This function uses Python reflection (or introspection) to read the regular expression rules out of the calling context and build the lexer. Once the lexer has been built, two methods can be used to control the lexer.

  • lexer.input(data). Reset the lexer and store a new input string.
  • lexer.token(). Return the next token. Returns a special LexToken instance on success or None if the end of the input text has been reached.

4.12 The @TOKEN decorator

In some applications, you may want to define build tokens from as a series of more complex regular expression rules. For example: 
digit            = r'([0-9])'
nondigit         = r'([_A-Za-z])'
identifier       = r'(' + nondigit + r'(' + digit + r'|' + nondigit + r')*)'        

def t_ID(t):
    # want docstring to be identifier above. ?????
    ...
In this case, we want the regular expression rule for ID to be one of the variables above. However, there is no way to directly specify this using a normal documentation string. To solve this problem, you can use the @TOKEN decorator. For example: 
from ply.lex import TOKEN

@TOKEN(identifier)
def t_ID(t):
    ...

This will attach identifier to the docstring for t_ID() allowing lex.py to work normally.

4.13 Optimized mode

For improved performance, it may be desirable to use Python's optimized mode (e.g., running Python with the -O option). However, doing so causes Python to ignore documentation strings. This presents special problems for lex.py. To handle this case, you can create your lexer using the optimize option as follows: 
lexer = lex.lex(optimize=1)
Next, run Python in its normal operating mode. When you do this, lex.py will write a file called lextab.py in the same directory as the module containing the lexer specification. This file contains all of the regular expression rules and tables used during lexing. On subsequent executions, lextab.py will simply be imported to build the lexer. This approach substantially improves the startup time of the lexer and it works in Python's optimized mode.

To change the name of the lexer-generated module, use the lextab keyword argument. For example: 

lexer = lex.lex(optimize=1,lextab="footab")
When running in optimized mode, it is important to note that lex disables most error checking. Thus, this is really only recommended if you're sure everything is working correctly and you're ready to start releasing production code.

4.14 Debugging

For the purpose of debugging, you can run lex() in a debugging mode as follows: 
lexer = lex.lex(debug=1)

This will produce various sorts of debugging information including all of the added rules, the master regular expressions used by the lexer, and tokens generating during lexing.

In addition, lex.py comes with a simple main function which will either tokenize input read from standard input or from a file specified on the command line. To use it, simply put this in your lexer: 

if __name__ == '__main__':
     lex.runmain()
Please refer to the "Debugging" section near the end for some more advanced details of debugging.

4.15 Alternative specification of lexers

As shown in the example, lexers are specified all within one Python module. If you want to put token rules in a different module from the one in which you invoke lex(), use the module keyword argument.

For example, you might have a dedicated module that just contains the token rules: 

# module: tokrules.py
# This module just contains the lexing rules

# List of token names.   This is always required
tokens = (
   'NUMBER',
   'PLUS',
   'MINUS',
   'TIMES',
   'DIVIDE',
   'LPAREN',
   'RPAREN',
)

# Regular expression rules for simple tokens
t_PLUS    = r'\+'
t_MINUS   = r'-'
t_TIMES   = r'\*'
t_DIVIDE  = r'/'
t_LPAREN  = r'\('
t_RPAREN  = r'\)'

# A regular expression rule with some action code
def t_NUMBER(t):
    r'\d+'
    t.value = int(t.value)    
    return t

# Define a rule so we can track line numbers
def t_newline(t):
    r'\n+'
    t.lexer.lineno += len(t.value)

# A string containing ignored characters (spaces and tabs)
t_ignore  = ' \t'

# Error handling rule
def t_error(t):
    print("Illegal character '%s'" % t.value[0])
    t.lexer.skip(1)
Now, if you wanted to build a tokenizer from these rules from within a different module, you would do the following (shown for Python interactive mode): 
>>> import tokrules
>>> lexer = lex.lex(module=tokrules)
>>> lexer.input("3 + 4")
>>> lexer.token()
LexToken(NUMBER,3,1,1,0)
>>> lexer.token()
LexToken(PLUS,'+',1,2)
>>> lexer.token()
LexToken(NUMBER,4,1,4)
>>> lexer.token()
None
>>>
The module option can also be used to define lexers from instances of a class. For example: 
import ply.lex as lex

class MyLexer(object):
    # List of token names.   This is always required
    tokens = (
       'NUMBER',
       'PLUS',
       'MINUS',
       'TIMES',
       'DIVIDE',
       'LPAREN',
       'RPAREN',
    )

    # Regular expression rules for simple tokens
    t_PLUS    = r'\+'
    t_MINUS   = r'-'
    t_TIMES   = r'\*'
    t_DIVIDE  = r'/'
    t_LPAREN  = r'\('
    t_RPAREN  = r'\)'

    # A regular expression rule with some action code
    # Note addition of self parameter since we're in a class
    def t_NUMBER(self,t):
        r'\d+'
        t.value = int(t.value)    
        return t

    # Define a rule so we can track line numbers
    def t_newline(self,t):
        r'\n+'
        t.lexer.lineno += len(t.value)

    # A string containing ignored characters (spaces and tabs)
    t_ignore  = ' \t'

    # Error handling rule
    def t_error(self,t):
        print("Illegal character '%s'" % t.value[0])
        t.lexer.skip(1)

    # Build the lexer
    def build(self,**kwargs):
        self.lexer = lex.lex(module=self, **kwargs)
    
    # Test it output
    def test(self,data):
        self.lexer.input(data)
        while True:
             tok = self.lexer.token()
             if not tok: 
                 break
             print(tok)

# Build the lexer and try it out
m = MyLexer()
m.build()           # Build the lexer
m.test("3 + 4")     # Test it
When building a lexer from class, you should construct the lexer from an instance of the class, not the class object itself. This is because PLY only works properly if the lexer actions are defined by bound-methods.

When using the module option to lex(), PLY collects symbols from the underlying object using the dir() function. There is no direct access to the __dict__ attribute of the object supplied as a module value.

Finally, if you want to keep things nicely encapsulated, but don't want to use a full-fledged class definition, lexers can be defined using closures. For example: 

import ply.lex as lex

# List of token names.   This is always required
tokens = (
  'NUMBER',
  'PLUS',
  'MINUS',
  'TIMES',
  'DIVIDE',
  'LPAREN',
  'RPAREN',
)

def MyLexer():
    # Regular expression rules for simple tokens
    t_PLUS    = r'\+'
    t_MINUS   = r'-'
    t_TIMES   = r'\*'
    t_DIVIDE  = r'/'
    t_LPAREN  = r'\('
    t_RPAREN  = r'\)'

    # A regular expression rule with some action code
    def t_NUMBER(t):
        r'\d+'
        t.value = int(t.value)    
        return t

    # Define a rule so we can track line numbers
    def t_newline(t):
        r'\n+'
        t.lexer.lineno += len(t.value)

    # A string containing ignored characters (spaces and tabs)
    t_ignore  = ' \t'

    # Error handling rule
    def t_error(t):
        print("Illegal character '%s'" % t.value[0])
        t.lexer.skip(1)

    # Build the lexer from my environment and return it    
    return lex.lex()

Important note: If you are defining a lexer using a class or closure, be aware that PLY still requires you to only define a single lexer per module (source file). There are extensive validation/error checking parts of the PLY that may falsely report error messages if you don't follow this rule.

4.16 Maintaining state

In your lexer, you may want to maintain a variety of state information. This might include mode settings, symbol tables, and other details. As an example, suppose that you wanted to keep track of how many NUMBER tokens had been encountered.

One way to do this is to keep a set of global variables in the module where you created the lexer. For example: 

num_count = 0
def t_NUMBER(t):
    r'\d+'
    global num_count
    num_count += 1
    t.value = int(t.value)    
    return t
If you don't like the use of a global variable, another place to store information is inside the Lexer object created by lex(). To this, you can use the lexer attribute of tokens passed to the various rules. For example: 
def t_NUMBER(t):
    r'\d+'
    t.lexer.num_count += 1     # Note use of lexer attribute
    t.value = int(t.value)    
    return t

lexer = lex.lex()
lexer.num_count = 0            # Set the initial count
This latter approach has the advantage of being simple and working correctly in applications where multiple instantiations of a given lexer exist in the same application. However, this might also feel like a gross violation of encapsulation to OO purists. Just to put your mind at some ease, all internal attributes of the lexer (with the exception of lineno) have names that are prefixed by lex (e.g., lexdata,lexpos, etc.). Thus, it is perfectly safe to store attributes in the lexer that don't have names starting with that prefix or a name that conflicts with one of the predefined methods (e.g., input(), token(), etc.).

If you don't like assigning values on the lexer object, you can define your lexer as a class as shown in the previous section: 

class MyLexer:
    ...
    def t_NUMBER(self,t):
        r'\d+'
        self.num_count += 1
        t.value = int(t.value)    
        return t

    def build(self, **kwargs):
        self.lexer = lex.lex(object=self,**kwargs)

    def __init__(self):
        self.num_count = 0
The class approach may be the easiest to manage if your application is going to be creating multiple instances of the same lexer and you need to manage a lot of state.

State can also be managed through closures. For example, in Python 3: 

def MyLexer():
    num_count = 0
    ...
    def t_NUMBER(t):
        r'\d+'
        nonlocal num_count
        num_count += 1
        t.value = int(t.value)    
        return t
    ...

4.17 Lexer cloning

If necessary, a lexer object can be duplicated by invoking its clone() method. For example: 

lexer = lex.lex()
...
newlexer = lexer.clone()
When a lexer is cloned, the copy is exactly identical to the original lexer including any input text and internal state. However, the clone allows a different set of input text to be supplied which may be processed separately. This may be useful in situations when you are writing a parser/compiler that involves recursive or reentrant processing. For instance, if you needed to scan ahead in the input for some reason, you could create a clone and use it to look ahead. Or, if you were implementing some kind of preprocessor, cloned lexers could be used to handle different input files.

Creating a clone is different than calling lex.lex() in that PLY doesn't regenerate any of the internal tables or regular expressions.

Special considerations need to be made when cloning lexers that also maintain their own internal state using classes or closures. Namely, you need to be aware that the newly created lexers will share all of this state with the original lexer. For example, if you defined a lexer as a class and did this: 

m = MyLexer()
a = lex.lex(object=m)      # Create a lexer

b = a.clone()              # Clone the lexer
Then both a and b are going to be bound to the same object m and any changes to m will be reflected in both lexers. It's important to emphasize that clone() is only meant to create a new lexer that reuses the regular expressions and environment of another lexer. If you need to make a totally new copy of a lexer, then call lex() again.

4.18 Internal lexer state

A Lexer object lexer has a number of internal attributes that may be useful in certain situations.

lexer.lexpos

This attribute is an integer that contains the current position within the input text. If you modify the value, it will change the result of the next call to token(). Within token rule functions, this points to the first character after the matched text. If the value is modified within a rule, the next returned token will be matched at the new position.

lexer.lineno

The current value of the line number attribute stored in the lexer. PLY only specifies that the attribute exists---it never sets, updates, or performs any processing with it. If you want to track line numbers, you will need to add code yourself (see the section on line numbers and positional information).

lexer.lexdata

The current input text stored in the lexer. This is the string passed with the input() method. It would probably be a bad idea to modify this unless you really know what you're doing.

lexer.lexmatch

This is the raw Match object returned by the Python re.match() function (used internally by PLY) for the current token. If you have written a regular expression that contains named groups, you can use this to retrieve those values. Note: This attribute is only updated when tokens are defined and processed by functions.

4.19 Conditional lexing and start conditions

In advanced parsing applications, it may be useful to have different lexing states. For instance, you may want the occurrence of a certain token or syntactic construct to trigger a different kind of lexing. PLY supports a feature that allows the underlying lexer to be put into a series of different states. Each state can have its own tokens, lexing rules, and so forth. The implementation is based largely on the "start condition" feature of GNU flex. Details of this can be found at http://flex.sourceforge.net/manual/Start-Conditions.html.

To define a new lexing state, it must first be declared. This is done by including a "states" declaration in your lex file. For example: 

states = (
   ('foo','exclusive'),
   ('bar','inclusive'),
)
This declaration declares two states, 'foo' and 'bar'. States may be of two types; 'exclusive' and 'inclusive'. An exclusive state completely overrides the default behavior of the lexer. That is, lex will only return tokens and apply rules defined specifically for that state. An inclusive state adds additional tokens and rules to the default set of rules. Thus, lex will return both the tokens defined by default in addition to those defined for the inclusive state.

Once a state has been declared, tokens and rules are declared by including the state name in token/rule declaration. For example: 

t_foo_NUMBER = r'\d+'                      # Token 'NUMBER' in state 'foo'        
t_bar_ID     = r'[a-zA-Z_][a-zA-Z0-9_]*'   # Token 'ID' in state 'bar'

def t_foo_newline(t):
    r'\n'
    t.lexer.lineno += 1
A token can be declared in multiple states by including multiple state names in the declaration. For example: 
t_foo_bar_NUMBER = r'\d+'         # Defines token 'NUMBER' in both state 'foo' and 'bar'
Alternative, a token can be declared in all states using the 'ANY' in the name. 
t_ANY_NUMBER = r'\d+'         # Defines a token 'NUMBER' in all states
If no state name is supplied, as is normally the case, the token is associated with a special state 'INITIAL'. For example, these two declarations are identical: 
t_NUMBER = r'\d+'
t_INITIAL_NUMBER = r'\d+'

States are also associated with the special t_ignore, t_error(), and t_eof() declarations. For example, if a state treats these differently, you can declare:

t_foo_ignore = " \t\n"       # Ignored characters for state 'foo'

def t_bar_error(t):          # Special error handler for state 'bar'
    pass
By default, lexing operates in the 'INITIAL' state. This state includes all of the normally defined tokens. For users who aren't using different states, this fact is completely transparent. If, during lexing or parsing, you want to change the lexing state, use the begin() method. For example: 
def t_begin_foo(t):
    r'start_foo'
    t.lexer.begin('foo')             # Starts 'foo' state
To get out of a state, you use begin() to switch back to the initial state. For example: 
def t_foo_end(t):
    r'end_foo'
    t.lexer.begin('INITIAL')        # Back to the initial state
The management of states can also be done with a stack. For example: 
def t_begin_foo(t):
    r'start_foo'
    t.lexer.push_state('foo')             # Starts 'foo' state

def t_foo_end(t):
    r'end_foo'
    t.lexer.pop_state()                   # Back to the previous state

The use of a stack would be useful in situations where there are many ways of entering a new lexing state and you merely want to go back to the previous state afterwards.

An example might help clarify. Suppose you were writing a parser and you wanted to grab sections of arbitrary C code enclosed by curly braces. That is, whenever you encounter a starting brace '{', you want to read all of the enclosed code up to the ending brace '}' and return it as a string. Doing this with a normal regular expression rule is nearly (if not actually) impossible. This is because braces can be nested and can be included in comments and strings. Thus, simply matching up to the first matching '}' character isn't good enough. Here is how you might use lexer states to do this: 

# Declare the state
states = (
  ('ccode','exclusive'),
)

# Match the first {. Enter ccode state.
def t_ccode(t):
    r'\{'
    t.lexer.code_start = t.lexer.lexpos        # Record the starting position
    t.lexer.level = 1                          # Initial brace level
    t.lexer.begin('ccode')                     # Enter 'ccode' state

# Rules for the ccode state
def t_ccode_lbrace(t):     
    r'\{'
    t.lexer.level +=1                

def t_ccode_rbrace(t):
    r'\}'
    t.lexer.level -=1

    # If closing brace, return the code fragment
    if t.lexer.level == 0:
         t.value = t.lexer.lexdata[t.lexer.code_start:t.lexer.lexpos+1]
         t.type = "CCODE"
         t.lexer.lineno += t.value.count('\n')
         t.lexer.begin('INITIAL')           
         return t

# C or C++ comment (ignore)    
def t_ccode_comment(t):
    r'(/\*(.|\n)*?\*/)|(//.*)'
    pass

# C string
def t_ccode_string(t):
   r'\"([^\\\n]|(\\.))*?\"'

# C character literal
def t_ccode_char(t):
   r'\'([^\\\n]|(\\.))*?\''

# Any sequence of non-whitespace characters (not braces, strings)
def t_ccode_nonspace(t):
   r'[^\s\{\}\'\"]+'

# Ignored characters (whitespace)
t_ccode_ignore = " \t\n"

# For bad characters, we just skip over it
def t_ccode_error(t):
    t.lexer.skip(1)
In this example, the occurrence of the first '{' causes the lexer to record the starting position and enter a new state 'ccode'. A collection of rules then match various parts of the input that follow (comments, strings, etc.). All of these rules merely discard the token (by not returning a value). However, if the closing right brace is encountered, the rule t_ccode_rbrace collects all of the code (using the earlier recorded starting position), stores it, and returns a token 'CCODE' containing all of that text. When returning the token, the lexing state is restored back to its initial state.

4.20 Miscellaneous Issues

  • The lexer requires input to be supplied as a single input string. Since most machines have more than enough memory, this rarely presents a performance concern. However, it means that the lexer currently can't be used with streaming data such as open files or sockets. This limitation is primarily a side-effect of using the re module. You might be able to work around this by implementing an appropriate def t_eof() end-of-file handling rule. The main complication here is that you'll probably need to ensure that data is fed to the lexer in a way so that it doesn't split in in the middle of a token.

  • The lexer should work properly with both Unicode strings given as token and pattern matching rules as well as for input text.

  • If you need to supply optional flags to the re.compile() function, use the reflags option to lex. For example: 
    lex.lex(reflags=re.UNICODE | re.VERBOSE)
    Note: by default, reflags is set to re.VERBOSE. If you provide your own flags, you may need to include this for PLY to preserve its normal behavior.

  • Since the lexer is written entirely in Python, its performance is largely determined by that of the Python re module. Although the lexer has been written to be as efficient as possible, it's not blazingly fast when used on very large input files. If performance is concern, you might consider upgrading to the most recent version of Python, creating a hand-written lexer, or offloading the lexer into a C extension module.

    If you are going to create a hand-written lexer and you plan to use it with yacc.py, it only needs to conform to the following requirements:

    • It must provide a token() method that returns the next token or None if no more tokens are available.
    • The token() method must return an object tok that has type and value attributes. If line number tracking is being used, then the token should also define a lineno attribute.

    5. Parsing basics

    yacc.py is used to parse language syntax. Before showing an example, there are a few important bits of background that must be mentioned. First, syntax is usually specified in terms of a BNF grammar. For example, if you wanted to parse simple arithmetic expressions, you might first write an unambiguous grammar specification like this: 
     
expression : expression + term
           | expression - term
           | term

term       : term * factor
           | term / factor
           | factor

factor     : NUMBER
           | ( expression )
    In the grammar, symbols such as NUMBER, +, -, *, and / are known as terminals and correspond to raw input tokens. Identifiers such as term and factor refer to grammar rules comprised of a collection of terminals and other rules. These identifiers are known as non-terminals.

    The semantic behavior of a language is often specified using a technique known as syntax directed translation. In syntax directed translation, attributes are attached to each symbol in a given grammar rule along with an action. Whenever a particular grammar rule is recognized, the action describes what to do. For example, given the expression grammar above, you might write the specification for a simple calculator like this: 

     
Grammar                             Action
--------------------------------    -------------------------------------------- 
expression0 : expression1 + term    expression0.val = expression1.val + term.val
            | expression1 - term    expression0.val = expression1.val - term.val
            | term                  expression0.val = term.val

term0       : term1 * factor        term0.val = term1.val * factor.val
            | term1 / factor        term0.val = term1.val / factor.val
            | factor                term0.val = factor.val

factor      : NUMBER                factor.val = int(NUMBER.lexval)
            | ( expression )        factor.val = expression.val
    A good way to think about syntax directed translation is to view each symbol in the grammar as a kind of object. Associated with each symbol is a value representing its "state" (for example, the val attribute above). Semantic actions are then expressed as a collection of functions or methods that operate on the symbols and associated values.

    Yacc uses a parsing technique known as LR-parsing or shift-reduce parsing. LR parsing is a bottom up technique that tries to recognize the right-hand-side of various grammar rules. Whenever a valid right-hand-side is found in the input, the appropriate action code is triggered and the grammar symbols are replaced by the grammar symbol on the left-hand-side.

    LR parsing is commonly implemented by shifting grammar symbols onto a stack and looking at the stack and the next input token for patterns that match one of the grammar rules. The details of the algorithm can be found in a compiler textbook, but the following example illustrates the steps that are performed if you wanted to parse the expression 3 + 5 * (10 - 20) using the grammar defined above. In the example, the special symbol $ represents the end of input. 

    Step Symbol Stack           Input Tokens            Action
---- ---------------------  ---------------------   -------------------------------
1                           3 + 5 * ( 10 - 20 )$    Shift 3
2    3                        + 5 * ( 10 - 20 )$    Reduce factor : NUMBER
3    factor                   + 5 * ( 10 - 20 )$    Reduce term   : factor
4    term                     + 5 * ( 10 - 20 )$    Reduce expr : term
5    expr                     + 5 * ( 10 - 20 )$    Shift +
6    expr +                     5 * ( 10 - 20 )$    Shift 5
7    expr + 5                     * ( 10 - 20 )$    Reduce factor : NUMBER
8    expr + factor                * ( 10 - 20 )$    Reduce term   : factor
9    expr + term                  * ( 10 - 20 )$    Shift *
10   expr + term *                  ( 10 - 20 )$    Shift (
11   expr + term * (                  10 - 20 )$    Shift 10
12   expr + term * ( 10                  - 20 )$    Reduce factor : NUMBER
13   expr + term * ( factor              - 20 )$    Reduce term : factor
14   expr + term * ( term                - 20 )$    Reduce expr : term
15   expr + term * ( expr                - 20 )$    Shift -
16   expr + term * ( expr -                20 )$    Shift 20
17   expr + term * ( expr - 20                )$    Reduce factor : NUMBER
18   expr + term * ( expr - factor            )$    Reduce term : factor
19   expr + term * ( expr - term              )$    Reduce expr : expr - term
20   expr + term * ( expr                     )$    Shift )
21   expr + term * ( expr )                    $    Reduce factor : (expr)
22   expr + term * factor                      $    Reduce term : term * factor
23   expr + term                               $    Reduce expr : expr + term
24   expr                                      $    Reduce expr
25                                             $    Success!
    When parsing the expression, an underlying state machine and the current input token determine what happens next. If the next token looks like part of a valid grammar rule (based on other items on the stack), it is generally shifted onto the stack. If the top of the stack contains a valid right-hand-side of a grammar rule, it is usually "reduced" and the symbols replaced with the symbol on the left-hand-side. When this reduction occurs, the appropriate action is triggered (if defined). If the input token can't be shifted and the top of stack doesn't match any grammar rules, a syntax error has occurred and the parser must take some kind of recovery step (or bail out). A parse is only successful if the parser reaches a state where the symbol stack is empty and there are no more input tokens.

    It is important to note that the underlying implementation is built around a large finite-state machine that is encoded in a collection of tables. The construction of these tables is non-trivial and beyond the scope of this discussion. However, subtle details of this process explain why, in the example above, the parser chooses to shift a token onto the stack in step 9 rather than reducing the rule expr : expr + term.

    6. Yacc

    The ply.yacc module implements the parsing component of PLY. The name "yacc" stands for "Yet Another Compiler Compiler" and is borrowed from the Unix tool of the same name.

    6.1 An example

    Suppose you wanted to make a grammar for simple arithmetic expressions as previously described. Here is how you would do it with yacc.py: 
    # Yacc example

import ply.yacc as yacc

# Get the token map from the lexer.  This is required.
from calclex import tokens

def p_expression_plus(p):
    'expression : expression PLUS term'
    p[0] = p[1] + p[3]

def p_expression_minus(p):
    'expression : expression MINUS term'
    p[0] = p[1] - p[3]

def p_expression_term(p):
    'expression : term'
    p[0] = p[1]

def p_term_times(p):
    'term : term TIMES factor'
    p[0] = p[1] * p[3]

def p_term_div(p):
    'term : term DIVIDE factor'
    p[0] = p[1] / p[3]

def p_term_factor(p):
    'term : factor'
    p[0] = p[1]

def p_factor_num(p):
    'factor : NUMBER'
    p[0] = p[1]

def p_factor_expr(p):
    'factor : LPAREN expression RPAREN'
    p[0] = p[2]

# Error rule for syntax errors
def p_error(p):
    print("Syntax error in input!")

# Build the parser
parser = yacc.yacc()

while True:
   try:
       s = raw_input('calc > ')
   except EOFError:
       break
   if not s: continue
   result = parser.parse(s)
   print(result)
    In this example, each grammar rule is defined by a Python function where the docstring to that function contains the appropriate context-free grammar specification. The statements that make up the function body implement the semantic actions of the rule. Each function accepts a single argument p that is a sequence containing the values of each grammar symbol in the corresponding rule. The values of p[i] are mapped to grammar symbols as shown here: 
    def p_expression_plus(p):
    'expression : expression PLUS term'
    #   ^            ^        ^    ^
    #  p[0]         p[1]     p[2] p[3]

    p[0] = p[1] + p[3]

    For tokens, the "value" of the corresponding p[i] is the same as the p.value attribute assigned in the lexer module. For non-terminals, the value is determined by whatever is placed in p[0] when rules are reduced. This value can be anything at all. However, it probably most common for the value to be a simple Python type, a tuple, or an instance. In this example, we are relying on the fact that the NUMBER token stores an integer value in its value field. All of the other rules simply perform various types of integer operations and propagate the result.

    Note: The use of negative indices have a special meaning in yacc---specially p[-1] does not have the same value as p[3] in this example. Please see the section on "Embedded Actions" for further details.

    The first rule defined in the yacc specification determines the starting grammar symbol (in this case, a rule for expression appears first). Whenever the starting rule is reduced by the parser and no more input is available, parsing stops and the final value is returned (this value will be whatever the top-most rule placed in p[0]). Note: an alternative starting symbol can be specified using the start keyword argument to yacc().

    The p_error(p) rule is defined to catch syntax errors. See the error handling section below for more detail.

    To build the parser, call the yacc.yacc() function. This function looks at the module and attempts to construct all of the LR parsing tables for the grammar you have specified. The first time yacc.yacc() is invoked, you will get a message such as this: 

    $ python calcparse.py
Generating LALR tables
calc >

    Since table construction is relatively expensive (especially for large grammars), the resulting parsing table is written to a file called parsetab.py. In addition, a debugging file called parser.out is created. On subsequent executions, yacc will reload the table from parsetab.py unless it has detected a change in the underlying grammar (in which case the tables and parsetab.py file are regenerated). Both of these files are written to the same directory as the module in which the parser is specified. The name of the parsetab module can be changed using the tabmodule keyword argument to yacc(). For example: 

    parser = yacc.yacc(tabmodule='fooparsetab')

    If any errors are detected in your grammar specification, yacc.py will produce diagnostic messages and possibly raise an exception. Some of the errors that can be detected include:

    • Duplicated function names (if more than one rule function have the same name in the grammar file).
    • Shift/reduce and reduce/reduce conflicts generated by ambiguous grammars.
    • Badly specified grammar rules.
    • Infinite recursion (rules that can never terminate).
    • Unused rules and tokens
    • Undefined rules and tokens
    The next few sections discuss grammar specification in more detail.

    The final part of the example shows how to actually run the parser created by yacc(). To run the parser, you simply have to call the parse() with a string of input text. This will run all of the grammar rules and return the result of the entire parse. This result return is the value assigned to p[0] in the starting grammar rule.

    6.2 Combining Grammar Rule Functions

    When grammar rules are similar, they can be combined into a single function. For example, consider the two rules in our earlier example: 
    def p_expression_plus(p):
    'expression : expression PLUS term'
    p[0] = p[1] + p[3]

def p_expression_minus(t):
    'expression : expression MINUS term'
    p[0] = p[1] - p[3]
    Instead of writing two functions, you might write a single function like this: 
    def p_expression(p):
    '''expression : expression PLUS term
                  | expression MINUS term'''
    if p[2] == '+':
        p[0] = p[1] + p[3]
    elif p[2] == '-':
        p[0] = p[1] - p[3]
    In general, the doc string for any given function can contain multiple grammar rules. So, it would have also been legal (although possibly confusing) to write this: 
    def p_binary_operators(p):
    '''expression : expression PLUS term
                  | expression MINUS term
       term       : term TIMES factor
                  | term DIVIDE factor'''
    if p[2] == '+':
        p[0] = p[1] + p[3]
    elif p[2] == '-':
        p[0] = p[1] - p[3]
    elif p[2] == '*':
        p[0] = p[1] * p[3]
    elif p[2] == '/':
        p[0] = p[1] / p[3]
    When combining grammar rules into a single function, it is usually a good idea for all of the rules to have a similar structure (e.g., the same number of terms). Otherwise, the corresponding action code may be more complicated than necessary. However, it is possible to handle simple cases using len(). For example: 
    def p_expressions(p):
    '''expression : expression MINUS expression
                  | MINUS expression'''
    if (len(p) == 4):
        p[0] = p[1] - p[3]
    elif (len(p) == 3):
        p[0] = -p[2]
    If parsing performance is a concern, you should resist the urge to put too much conditional processing into a single grammar rule as shown in these examples. When you add checks to see which grammar rule is being handled, you are actually duplicating the work that the parser has already performed (i.e., the parser already knows exactly what rule it matched). You can eliminate this overhead by using a separate p_rule() function for each grammar rule.

    6.3 Character Literals

    If desired, a grammar may contain tokens defined as single character literals. For example: 
    def p_binary_operators(p):
    '''expression : expression '+' term
                  | expression '-' term
       term       : term '*' factor
                  | term '/' factor'''
    if p[2] == '+':
        p[0] = p[1] + p[3]
    elif p[2] == '-':
        p[0] = p[1] - p[3]
    elif p[2] == '*':
        p[0] = p[1] * p[3]
    elif p[2] == '/':
        p[0] = p[1] / p[3]
    A character literal must be enclosed in quotes such as '+'. In addition, if literals are used, they must be declared in the corresponding lex file through the use of a special literals declaration. 
    # Literals.  Should be placed in module given to lex()
literals = ['+','-','*','/' ]
    Character literals are limited to a single character. Thus, it is not legal to specify literals such as '<=' or '=='. For this, use the normal lexing rules (e.g., define a rule such as t_EQ = r'==').

    6.4 Empty Productions

    yacc.py can handle empty productions by defining a rule like this: 
    def p_empty(p):
    'empty :'
    pass
    Now to use the empty production, simply use 'empty' as a symbol. For example: 
    def p_optitem(p):
    'optitem : item'
    '        | empty'
    ...
    Note: You can write empty rules anywhere by simply specifying an empty right hand side. However, I personally find that writing an "empty" rule and using "empty" to denote an empty production is easier to read and more clearly states your intentions.

    6.5 Changing the starting symbol

    Normally, the first rule found in a yacc specification defines the starting grammar rule (top level rule). To change this, simply supply a start specifier in your file. For example: 
    start = 'foo'

def p_bar(p):
    'bar : A B'

# This is the starting rule due to the start specifier above
def p_foo(p):
    'foo : bar X'
...
    The use of a start specifier may be useful during debugging since you can use it to have yacc build a subset of a larger grammar. For this purpose, it is also possible to specify a starting symbol as an argument to yacc(). For example: 
    parser = yacc.yacc(start='foo')

    6.6 Dealing With Ambiguous Grammars

    The expression grammar given in the earlier example has been written in a special format to eliminate ambiguity. However, in many situations, it is extremely difficult or awkward to write grammars in this format. A much more natural way to express the grammar is in a more compact form like this: 
    expression : expression PLUS expression
           | expression MINUS expression
           | expression TIMES expression
           | expression DIVIDE expression
           | LPAREN expression RPAREN
           | NUMBER
    Unfortunately, this grammar specification is ambiguous. For example, if you are parsing the string "3 * 4 + 5", there is no way to tell how the operators are supposed to be grouped. For example, does the expression mean "(3 * 4) + 5" or is it "3 * (4+5)"?

    When an ambiguous grammar is given to yacc.py it will print messages about "shift/reduce conflicts" or "reduce/reduce conflicts". A shift/reduce conflict is caused when the parser generator can't decide whether or not to reduce a rule or shift a symbol on the parsing stack. For example, consider the string "3 * 4 + 5" and the internal parsing stack: 

    Step Symbol Stack           Input Tokens            Action
---- ---------------------  ---------------------   -------------------------------
1    $                                3 * 4 + 5$    Shift 3
2    $ 3                                * 4 + 5$    Reduce : expression : NUMBER
3    $ expr                             * 4 + 5$    Shift *
4    $ expr *                             4 + 5$    Shift 4
5    $ expr * 4                             + 5$    Reduce: expression : NUMBER
6    $ expr * expr                          + 5$    SHIFT/REDUCE CONFLICT ????
    In this case, when the parser reaches step 6, it has two options. One is to reduce the rule expr : expr * expr on the stack. The other option is to shift the token + on the stack. Both options are perfectly legal from the rules of the context-free-grammar.

    By default, all shift/reduce conflicts are resolved in favor of shifting. Therefore, in the above example, the parser will always shift the + instead of reducing. Although this strategy works in many cases (for example, the case of "if-then" versus "if-then-else"), it is not enough for arithmetic expressions. In fact, in the above example, the decision to shift + is completely wrong---we should have reduced expr * expr since multiplication has higher mathematical precedence than addition.

    To resolve ambiguity, especially in expression grammars, yacc.py allows individual tokens to be assigned a precedence level and associativity. This is done by adding a variable precedence to the grammar file like this: 

    precedence = (
    ('left', 'PLUS', 'MINUS'),
    ('left', 'TIMES', 'DIVIDE'),
)
    This declaration specifies that PLUS/MINUS have the same precedence level and are left-associative and that TIMES/DIVIDE have the same precedence and are left-associative. Within the precedence declaration, tokens are ordered from lowest to highest precedence. Thus, this declaration specifies that TIMES/DIVIDE have higher precedence than PLUS/MINUS (since they appear later in the precedence specification).

    The precedence specification works by associating a numerical precedence level value and associativity direction to the listed tokens. For example, in the above example you get: 

    PLUS      : level = 1,  assoc = 'left'
MINUS     : level = 1,  assoc = 'left'
TIMES     : level = 2,  assoc = 'left'
DIVIDE    : level = 2,  assoc = 'left'
    These values are then used to attach a numerical precedence value and associativity direction to each grammar rule. This is always determined by looking at the precedence of the right-most terminal symbol. For example: 
    expression : expression PLUS expression                 # level = 1, left
           | expression MINUS expression                # level = 1, left
           | expression TIMES expression                # level = 2, left
           | expression DIVIDE expression               # level = 2, left
           | LPAREN expression RPAREN                   # level = None (not specified)
           | NUMBER                                     # level = None (not specified)
    When shift/reduce conflicts are encountered, the parser generator resolves the conflict by looking at the precedence rules and associativity specifiers.

    1. If the current token has higher precedence than the rule on the stack, it is shifted.
    2. If the grammar rule on the stack has higher precedence, the rule is reduced.
    3. If the current token and the grammar rule have the same precedence, the rule is reduced for left associativity, whereas the token is shifted for right associativity.
    4. If nothing is known about the precedence, shift/reduce conflicts are resolved in favor of shifting (the default).
    For example, if "expression PLUS expression" has been parsed and the next token is "TIMES", the action is going to be a shift because "TIMES" has a higher precedence level than "PLUS". On the other hand, if "expression TIMES expression" has been parsed and the next token is "PLUS", the action is going to be reduce because "PLUS" has a lower precedence than "TIMES."

    When shift/reduce conflicts are resolved using the first three techniques (with the help of precedence rules), yacc.py will report no errors or conflicts in the grammar (although it will print some information in the parser.out debugging file).

    One problem with the precedence specifier technique is that it is sometimes necessary to change the precedence of an operator in certain contexts. For example, consider a unary-minus operator in "3 + 4 * -5". Mathematically, the unary minus is normally given a very high precedence--being evaluated before the multiply. However, in our precedence specifier, MINUS has a lower precedence than TIMES. To deal with this, precedence rules can be given for so-called "fictitious tokens" like this: 

    precedence = (
    ('left', 'PLUS', 'MINUS'),
    ('left', 'TIMES', 'DIVIDE'),
    ('right', 'UMINUS'),            # Unary minus operator
)
    Now, in the grammar file, we can write our unary minus rule like this: 
    def p_expr_uminus(p):
    'expression : MINUS expression %prec UMINUS'
    p[0] = -p[2]
    In this case, %prec UMINUS overrides the default rule precedence--setting it to that of UMINUS in the precedence specifier.

    At first, the use of UMINUS in this example may appear very confusing. UMINUS is not an input token or a grammar rule. Instead, you should think of it as the name of a special marker in the precedence table. When you use the %prec qualifier, you're simply telling yacc that you want the precedence of the expression to be the same as for this special marker instead of the usual precedence.

    It is also possible to specify non-associativity in the precedence table. This would be used when you don't want operations to chain together. For example, suppose you wanted to support comparison operators like < and > but you didn't want to allow combinations like a < b < c. To do this, simply specify a rule like this: 

    precedence = (
    ('nonassoc', 'LESSTHAN', 'GREATERTHAN'),  # Nonassociative operators
    ('left', 'PLUS', 'MINUS'),
    ('left', 'TIMES', 'DIVIDE'),
    ('right', 'UMINUS'),            # Unary minus operator
)

    If you do this, the occurrence of input text such as a < b < c will result in a syntax error. However, simple expressions such as a < b will still be fine.

    Reduce/reduce conflicts are caused when there are multiple grammar rules that can be applied to a given set of symbols. This kind of conflict is almost always bad and is always resolved by picking the rule that appears first in the grammar file. Reduce/reduce conflicts are almost always caused when different sets of grammar rules somehow generate the same set of symbols. For example: 

    assignment :  ID EQUALS NUMBER
           |  ID EQUALS expression
           
expression : expression PLUS expression
           | expression MINUS expression
           | expression TIMES expression
           | expression DIVIDE expression
           | LPAREN expression RPAREN
           | NUMBER
    In this case, a reduce/reduce conflict exists between these two rules: 
    assignment  : ID EQUALS NUMBER
expression  : NUMBER
    For example, if you wrote "a = 5", the parser can't figure out if this is supposed to be reduced as assignment : ID EQUALS NUMBER or whether it's supposed to reduce the 5 as an expression and then reduce the rule assignment : ID EQUALS expression.

    It should be noted that reduce/reduce conflicts are notoriously difficult to spot simply looking at the input grammar. When a reduce/reduce conflict occurs, yacc() will try to help by printing a warning message such as this: 

    WARNING: 1 reduce/reduce conflict
WARNING: reduce/reduce conflict in state 15 resolved using rule (assignment -> ID EQUALS NUMBER)
WARNING: rejected rule (expression -> NUMBER)
    This message identifies the two rules that are in conflict. However, it may not tell you how the parser arrived at such a state. To try and figure it out, you'll probably have to look at your grammar and the contents of the parser.out debugging file with an appropriately high level of caffeination.

    6.7 The parser.out file

    Tracking down shift/reduce and reduce/reduce conflicts is one of the finer pleasures of using an LR parsing algorithm. To assist in debugging, yacc.py creates a debugging file called 'parser.out' when it generates the parsing table. The contents of this file look like the following: 
    Unused terminals:


Grammar

Rule 1     expression -> expression PLUS expression
Rule 2     expression -> expression MINUS expression
Rule 3     expression -> expression TIMES expression
Rule 4     expression -> expression DIVIDE expression
Rule 5     expression -> NUMBER
Rule 6     expression -> LPAREN expression RPAREN

Terminals, with rules where they appear

TIMES                : 3
error                : 
MINUS                : 2
RPAREN               : 6
LPAREN               : 6
DIVIDE               : 4
PLUS                 : 1
NUMBER               : 5

Nonterminals, with rules where they appear

expression           : 1 1 2 2 3 3 4 4 6 0


Parsing method: LALR


state 0

    S' -> . expression
    expression -> . expression PLUS expression
    expression -> . expression MINUS expression
    expression -> . expression TIMES expression
    expression -> . expression DIVIDE expression
    expression -> . NUMBER
    expression -> . LPAREN expression RPAREN

    NUMBER          shift and go to state 3
    LPAREN          shift and go to state 2


state 1

    S' -> expression .
    expression -> expression . PLUS expression
    expression -> expression . MINUS expression
    expression -> expression . TIMES expression
    expression -> expression . DIVIDE expression

    PLUS            shift and go to state 6
    MINUS           shift and go to state 5
    TIMES           shift and go to state 4
    DIVIDE          shift and go to state 7


state 2

    expression -> LPAREN . expression RPAREN
    expression -> . expression PLUS expression
    expression -> . expression MINUS expression
    expression -> . expression TIMES expression
    expression -> . expression DIVIDE expression
    expression -> . NUMBER
    expression -> . LPAREN expression RPAREN

    NUMBER          shift and go to state 3
    LPAREN          shift and go to state 2


state 3

    expression -> NUMBER .

    $               reduce using rule 5
    PLUS            reduce using rule 5
    MINUS           reduce using rule 5
    TIMES           reduce using rule 5
    DIVIDE          reduce using rule 5
    RPAREN          reduce using rule 5


state 4

    expression -> expression TIMES . expression
    expression -> . expression PLUS expression
    expression -> . expression MINUS expression
    expression -> . expression TIMES expression
    expression -> . expression DIVIDE expression
    expression -> . NUMBER
    expression -> . LPAREN expression RPAREN

    NUMBER          shift and go to state 3
    LPAREN          shift and go to state 2


state 5

    expression -> expression MINUS . expression
    expression -> . expression PLUS expression
    expression -> . expression MINUS expression
    expression -> . expression TIMES expression
    expression -> . expression DIVIDE expression
    expression -> . NUMBER
    expression -> . LPAREN expression RPAREN

    NUMBER          shift and go to state 3
    LPAREN          shift and go to state 2


state 6

    expression -> expression PLUS . expression
    expression -> . expression PLUS expression
    expression -> . expression MINUS expression
    expression -> . expression TIMES expression
    expression -> . expression DIVIDE expression
    expression -> . NUMBER
    expression -> . LPAREN expression RPAREN

    NUMBER          shift and go to state 3
    LPAREN          shift and go to state 2


state 7

    expression -> expression DIVIDE . expression
    expression -> . expression PLUS expression
    expression -> . expression MINUS expression
    expression -> . expression TIMES expression
    expression -> . expression DIVIDE expression
    expression -> . NUMBER
    expression -> . LPAREN expression RPAREN

    NUMBER          shift and go to state 3
    LPAREN          shift and go to state 2


state 8

    expression -> LPAREN expression . RPAREN
    expression -> expression . PLUS expression
    expression -> expression . MINUS expression
    expression -> expression . TIMES expression
    expression -> expression . DIVIDE expression

    RPAREN          shift and go to state 13
    PLUS            shift and go to state 6
    MINUS           shift and go to state 5
    TIMES           shift and go to state 4
    DIVIDE          shift and go to state 7


state 9

    expression -> expression TIMES expression .
    expression -> expression . PLUS expression
    expression -> expression . MINUS expression
    expression -> expression . TIMES expression
    expression -> expression . DIVIDE expression

    $               reduce using rule 3
    PLUS            reduce using rule 3
    MINUS           reduce using rule 3
    TIMES           reduce using rule 3
    DIVIDE          reduce using rule 3
    RPAREN          reduce using rule 3

  ! PLUS            [ shift and go to state 6 ]
  ! MINUS           [ shift and go to state 5 ]
  ! TIMES           [ shift and go to state 4 ]
  ! DIVIDE          [ shift and go to state 7 ]

state 10

    expression -> expression MINUS expression .
    expression -> expression . PLUS expression
    expression -> expression . MINUS expression
    expression -> expression . TIMES expression
    expression -> expression . DIVIDE expression

    $               reduce using rule 2
    PLUS            reduce using rule 2
    MINUS           reduce using rule 2
    RPAREN          reduce using rule 2
    TIMES           shift and go to state 4
    DIVIDE          shift and go to state 7

  ! TIMES           [ reduce using rule 2 ]
  ! DIVIDE          [ reduce using rule 2 ]
  ! PLUS            [ shift and go to state 6 ]
  ! MINUS           [ shift and go to state 5 ]

state 11

    expression -> expression PLUS expression .
    expression -> expression . PLUS expression
    expression -> expression . MINUS expression
    expression -> expression . TIMES expression
    expression -> expression . DIVIDE expression

    $               reduce using rule 1
    PLUS            reduce using rule 1
    MINUS           reduce using rule 1
    RPAREN          reduce using rule 1
    TIMES           shift and go to state 4
    DIVIDE          shift and go to state 7

  ! TIMES           [ reduce using rule 1 ]
  ! DIVIDE          [ reduce using rule 1 ]
  ! PLUS            [ shift and go to state 6 ]
  ! MINUS           [ shift and go to state 5 ]

state 12

    expression -> expression DIVIDE expression .
    expression -> expression . PLUS expression
    expression -> expression . MINUS expression
    expression -> expression . TIMES expression
    expression -> expression . DIVIDE expression

    $               reduce using rule 4
    PLUS            reduce using rule 4
    MINUS           reduce using rule 4
    TIMES           reduce using rule 4
    DIVIDE          reduce using rule 4
    RPAREN          reduce using rule 4

  ! PLUS            [ shift and go to state 6 ]
  ! MINUS           [ shift and go to state 5 ]
  ! TIMES           [ shift and go to state 4 ]
  ! DIVIDE          [ shift and go to state 7 ]

state 13

    expression -> LPAREN expression RPAREN .

    $               reduce using rule 6
    PLUS            reduce using rule 6
    MINUS           reduce using rule 6
    TIMES           reduce using rule 6
    DIVIDE          reduce using rule 6
    RPAREN          reduce using rule 6
    The different states that appear in this file are a representation of every possible sequence of valid input tokens allowed by the grammar. When receiving input tokens, the parser is building up a stack and looking for matching rules. Each state keeps track of the grammar rules that might be in the process of being matched at that point. Within each rule, the "." character indicates the current location of the parse within that rule. In addition, the actions for each valid input token are listed. When a shift/reduce or reduce/reduce conflict arises, rules not selected are prefixed with an !. For example: 
      ! TIMES           [ reduce using rule 2 ]
  ! DIVIDE          [ reduce using rule 2 ]
  ! PLUS            [ shift and go to state 6 ]
  ! MINUS           [ shift and go to state 5 ]
    By looking at these rules (and with a little practice), you can usually track down the source of most parsing conflicts. It should also be stressed that not all shift-reduce conflicts are bad. However, the only way to be sure that they are resolved correctly is to look at parser.out.

    6.8 Syntax Error Handling

    If you are creating a parser for production use, the handling of syntax errors is important. As a general rule, you don't want a parser to simply throw up its hands and stop at the first sign of trouble. Instead, you want it to report the error, recover if possible, and continue parsing so that all of the errors in the input get reported to the user at once. This is the standard behavior found in compilers for languages such as C, C++, and Java. In PLY, when a syntax error occurs during parsing, the error is immediately detected (i.e., the parser does not read any more tokens beyond the source of the error). However, at this point, the parser enters a recovery mode that can be used to try and continue further parsing. As a general rule, error recovery in LR parsers is a delicate topic that involves ancient rituals and black-magic. The recovery mechanism provided by yacc.py is comparable to Unix yacc so you may want consult a book like O'Reilly's "Lex and Yacc" for some of the finer details.

    When a syntax error occurs, yacc.py performs the following steps:

    1. On the first occurrence of an error, the user-defined p_error() function is called with the offending token as an argument. However, if the syntax error is due to reaching the end-of-file, p_error() is called with an argument of None. Afterwards, the parser enters an "error-recovery" mode in which it will not make future calls to p_error() until it has successfully shifted at least 3 tokens onto the parsing stack.

    2. If no recovery action is taken in p_error(), the offending lookahead token is replaced with a special error token.

    3. If the offending lookahead token is already set to error, the top item of the parsing stack is deleted.

    4. If the entire parsing stack is unwound, the parser enters a restart state and attempts to start parsing from its initial state.

    5. If a grammar rule accepts error as a token, it will be shifted onto the parsing stack.

    6. If the top item of the parsing stack is error, lookahead tokens will be discarded until the parser can successfully shift a new symbol or reduce a rule involving error.

    6.8.1 Recovery and resynchronization with error rules

    The most well-behaved approach for handling syntax errors is to write grammar rules that include the error token. For example, suppose your language had a grammar rule for a print statement like this: 
    def p_statement_print(p):
     'statement : PRINT expr SEMI'
     ...
    To account for the possibility of a bad expression, you might write an additional grammar rule like this: 
    def p_statement_print_error(p):
     'statement : PRINT error SEMI'
     print("Syntax error in print statement. Bad expression")
    In this case, the error token will match any sequence of tokens that might appear up to the first semicolon that is encountered. Once the semicolon is reached, the rule will be invoked and the error token will go away.

    This type of recovery is sometimes known as parser resynchronization. The error token acts as a wildcard for any bad input text and the token immediately following error acts as a synchronization token.

    It is important to note that the error token usually does not appear as the last token on the right in an error rule. For example: 

    def p_statement_print_error(p):
    'statement : PRINT error'
    print("Syntax error in print statement. Bad expression")
    This is because the first bad token encountered will cause the rule to be reduced--which may make it difficult to recover if more bad tokens immediately follow.

    6.8.2 Panic mode recovery

    An alternative error recovery scheme is to enter a panic mode recovery in which tokens are discarded to a point where the parser might be able to recover in some sensible manner.

    Panic mode recovery is implemented entirely in the p_error() function. For example, this function starts discarding tokens until it reaches a closing '}'. Then, it restarts the parser in its initial state. 

    def p_error(p):
    print("Whoa. You are seriously hosed.")
    if not p:
        print("End of File!")
        return

    # Read ahead looking for a closing '}'
    while True:
        tok = parser.token()             # Get the next token
        if not tok or tok.type == 'RBRACE': 
            break
    parser.restart()

    This function simply discards the bad token and tells the parser that the error was ok. 

    def p_error(p):
    if p:
         print("Syntax error at token", p.type)
         # Just discard the token and tell the parser it's okay.
         parser.errok()
    else:
         print("Syntax error at EOF")

    More information on these methods is as follows:

    • parser.errok(). This resets the parser state so it doesn't think it's in error-recovery mode. This will prevent an error token from being generated and will reset the internal error counters so that the next syntax error will call p_error() again.

    • parser.token(). This returns the next token on the input stream.

    • parser.restart(). This discards the entire parsing stack and resets the parser to its initial state.

    To supply the next lookahead token to the parser, p_error() can return a token. This might be useful if trying to synchronize on special characters. For example: 

    def p_error(p):
    # Read ahead looking for a terminating ";"
    while True:
        tok = parser.token()             # Get the next token
        if not tok or tok.type == 'SEMI': break
    parser.errok()

    # Return SEMI to the parser as the next lookahead token
    return tok

    Keep in mind in that the above error handling functions, parser is an instance of the parser created by yacc(). You'll need to save this instance someplace in your code so that you can refer to it during error handling.

    6.8.3 Signalling an error from a production

    If necessary, a production rule can manually force the parser to enter error recovery. This is done by raising the SyntaxError exception like this: 
    def p_production(p):
    'production : some production ...'
    raise SyntaxError
    The effect of raising SyntaxError is the same as if the last symbol shifted onto the parsing stack was actually a syntax error. Thus, when you do this, the last symbol shifted is popped off of the parsing stack and the current lookahead token is set to an error token. The parser then enters error-recovery mode where it tries to reduce rules that can accept error tokens. The steps that follow from this point are exactly the same as if a syntax error were detected and p_error() were called.

    One important aspect of manually setting an error is that the p_error() function will NOT be called in this case. If you need to issue an error message, make sure you do it in the production that raises SyntaxError.

    Note: This feature of PLY is meant to mimic the behavior of the YYERROR macro in yacc.

    6.8.4 When Do Syntax Errors Get Reported

    In most cases, yacc will handle errors as soon as a bad input token is detected on the input. However, be aware that yacc may choose to delay error handling until after it has reduced one or more grammar rules first. This behavior might be unexpected, but it's related to special states in the underlying parsing table known as "defaulted states." A defaulted state is parsing condition where the same grammar rule will be reduced regardless of what valid token comes next on the input. For such states, yacc chooses to go ahead and reduce the grammar rule without reading the next input token. If the next token is bad, yacc will eventually get around to reading it and report a syntax error. It's just a little unusual in that you might see some of your grammar rules firing immediately prior to the syntax error.

    Usually, the delayed error reporting with defaulted states is harmless (and there are other reasons for wanting PLY to behave in this way). However, if you need to turn this behavior off for some reason. You can clear the defaulted states table like this: 

    parser = yacc.yacc()
parser.defaulted_states = {}

    Disabling defaulted states is not recommended if your grammar makes use of embedded actions as described in Section 6.11.

    6.8.5 General comments on error handling

    For normal types of languages, error recovery with error rules and resynchronization characters is probably the most reliable technique. This is because you can instrument the grammar to catch errors at selected places where it is relatively easy to recover and continue parsing. Panic mode recovery is really only useful in certain specialized applications where you might want to discard huge portions of the input text to find a valid restart point.

    6.9 Line Number and Position Tracking

    Position tracking is often a tricky problem when writing compilers. By default, PLY tracks the line number and position of all tokens. This information is available using the following functions:
    • p.lineno(num). Return the line number for symbol num
    • p.lexpos(num). Return the lexing position for symbol num
    For example: 
    def p_expression(p):
    'expression : expression PLUS expression'
    line   = p.lineno(2)        # line number of the PLUS token
    index  = p.lexpos(2)        # Position of the PLUS token
    As an optional feature, yacc.py can automatically track line numbers and positions for all of the grammar symbols as well. However, this extra tracking requires extra processing and can significantly slow down parsing. Therefore, it must be enabled by passing the tracking=True option to yacc.parse(). For example: 
    yacc.parse(data,tracking=True)
    Once enabled, the lineno() and lexpos() methods work for all grammar symbols. In addition, two additional methods can be used:
    • p.linespan(num). Return a tuple (startline,endline) with the starting and ending line number for symbol num.
    • p.lexspan(num). Return a tuple (start,end) with the starting and ending positions for symbol num.
    For example: 
    def p_expression(p):
    'expression : expression PLUS expression'
    p.lineno(1)        # Line number of the left expression
    p.lineno(2)        # line number of the PLUS operator
    p.lineno(3)        # line number of the right expression
    ...
    start,end = p.linespan(3)    # Start,end lines of the right expression
    starti,endi = p.lexspan(3)   # Start,end positions of right expression
    Note: The lexspan() function only returns the range of values up to the start of the last grammar symbol.

    Although it may be convenient for PLY to track position information on all grammar symbols, this is often unnecessary. For example, if you are merely using line number information in an error message, you can often just key off of a specific token in the grammar rule. For example: 

    def p_bad_func(p):
    'funccall : fname LPAREN error RPAREN'
    # Line number reported from LPAREN token
    print("Bad function call at line", p.lineno(2))

    Similarly, you may get better parsing performance if you only selectively propagate line number information where it's needed using the p.set_lineno() method. For example: 

    def p_fname(p):
    'fname : ID'
    p[0] = p[1]
    p.set_lineno(0,p.lineno(1))
    PLY doesn't retain line number information from rules that have already been parsed. If you are building an abstract syntax tree and need to have line numbers, you should make sure that the line numbers appear in the tree itself.

    6.10 AST Construction

    yacc.py provides no special functions for constructing an abstract syntax tree. However, such construction is easy enough to do on your own.

    A minimal way to construct a tree is to simply create and propagate a tuple or list in each grammar rule function. There are many possible ways to do this, but one example would be something like this: 

    def p_expression_binop(p):
    '''expression : expression PLUS expression
                  | expression MINUS expression
                  | expression TIMES expression
                  | expression DIVIDE expression'''

    p[0] = ('binary-expression',p[2],p[1],p[3])

def p_expression_group(p):
    'expression : LPAREN expression RPAREN'
    p[0] = ('group-expression',p[2])

def p_expression_number(p):
    'expression : NUMBER'
    p[0] = ('number-expression',p[1])

    Another approach is to create a set of data structure for different kinds of abstract syntax tree nodes and assign nodes to p[0] in each rule. For example: 

    class Expr: pass

class BinOp(Expr):
    def __init__(self,left,op,right):
        self.type = "binop"
        self.left = left
        self.right = right
        self.op = op

class Number(Expr):
    def __init__(self,value):
        self.type = "number"
        self.value = value

def p_expression_binop(p):
    '''expression : expression PLUS expression
                  | expression MINUS expression
                  | expression TIMES expression
                  | expression DIVIDE expression'''

    p[0] = BinOp(p[1],p[2],p[3])

def p_expression_group(p):
    'expression : LPAREN expression RPAREN'
    p[0] = p[2]

def p_expression_number(p):
    'expression : NUMBER'
    p[0] = Number(p[1])
    The advantage to this approach is that it may make it easier to attach more complicated semantics, type checking, code generation, and other features to the node classes.

    To simplify tree traversal, it may make sense to pick a very generic tree structure for your parse tree nodes. For example: 

    class Node:
    def __init__(self,type,children=None,leaf=None):
         self.type = type
         if children:
              self.children = children
         else:
              self.children = [ ]
         self.leaf = leaf
	 
def p_expression_binop(p):
    '''expression : expression PLUS expression
                  | expression MINUS expression
                  | expression TIMES expression
                  | expression DIVIDE expression'''

    p[0] = Node("binop", [p[1],p[3]], p[2])

    6.11 Embedded Actions

    The parsing technique used by yacc only allows actions to be executed at the end of a rule. For example, suppose you have a rule like this: 
    def p_foo(p):
    "foo : A B C D"
    print("Parsed a foo", p[1],p[2],p[3],p[4])

    In this case, the supplied action code only executes after all of the symbols A, B, C, and D have been parsed. Sometimes, however, it is useful to execute small code fragments during intermediate stages of parsing. For example, suppose you wanted to perform some action immediately after A has been parsed. To do this, write an empty rule like this: 

    def p_foo(p):
    "foo : A seen_A B C D"
    print("Parsed a foo", p[1],p[3],p[4],p[5])
    print("seen_A returned", p[2])

def p_seen_A(p):
    "seen_A :"
    print("Saw an A = ", p[-1])   # Access grammar symbol to left
    p[0] = some_value            # Assign value to seen_A

    In this example, the empty seen_A rule executes immediately after A is shifted onto the parsing stack. Within this rule, p[-1] refers to the symbol on the stack that appears immediately to the left of the seen_A symbol. In this case, it would be the value of A in the foo rule immediately above. Like other rules, a value can be returned from an embedded action by simply assigning it to p[0]

    The use of embedded actions can sometimes introduce extra shift/reduce conflicts. For example, this grammar has no conflicts: 

    def p_foo(p):
    """foo : abcd
           | abcx"""

def p_abcd(p):
    "abcd : A B C D"

def p_abcx(p):
    "abcx : A B C X"
    However, if you insert an embedded action into one of the rules like this, 
    def p_foo(p):
    """foo : abcd
           | abcx"""

def p_abcd(p):
    "abcd : A B C D"

def p_abcx(p):
    "abcx : A B seen_AB C X"

def p_seen_AB(p):
    "seen_AB :"
    an extra shift-reduce conflict will be introduced. This conflict is caused by the fact that the same symbol C appears next in both the abcd and abcx rules. The parser can either shift the symbol (abcd rule) or reduce the empty rule seen_AB (abcx rule).

    A common use of embedded rules is to control other aspects of parsing such as scoping of local variables. For example, if you were parsing C code, you might write code like this: 

    def p_statements_block(p):
    "statements: LBRACE new_scope statements RBRACE"""
    # Action code
    ...
    pop_scope()        # Return to previous scope

def p_new_scope(p):
    "new_scope :"
    # Create a new scope for local variables
    s = new_scope()
    push_scope(s)
    ...
    In this case, the embedded action new_scope executes immediately after a LBRACE ({) symbol is parsed. This might adjust internal symbol tables and other aspects of the parser. Upon completion of the rule statements_block, code might undo the operations performed in the embedded action (e.g., pop_scope()).

    6.12 Miscellaneous Yacc Notes

    • By default, yacc.py relies on lex.py for tokenizing. However, an alternative tokenizer can be supplied as follows: 
      parser = yacc.parse(lexer=x)
      in this case, x must be a Lexer object that minimally has a x.token() method for retrieving the next token. If an input string is given to yacc.parse(), the lexer must also have an x.input() method.

    • By default, the yacc generates tables in debugging mode (which produces the parser.out file and other output). To disable this, use 
      parser = yacc.yacc(debug=False)

    • To change the name of the parsetab.py file, use: 
      parser = yacc.yacc(tabmodule="foo")

      Normally, the parsetab.py file is placed into the same directory as the module where the parser is defined. If you want it to go somewhere else, you can given an absolute package name for tabmodule instead. In that case, the tables will be written there.

    • To change the directory in which the parsetab.py file (and other output files) are written, use: 
      parser = yacc.yacc(tabmodule="foo",outputdir="somedirectory")

      Note: Be aware that unless the directory specified is also on Python's path (sys.path), subsequent imports of the table file will fail. As a general rule, it's better to specify a destination using the tabmodule argument instead of directly specifying a directory using the outputdir argument.

    • To prevent yacc from generating any kind of parser table file, use: 
      parser = yacc.yacc(write_tables=False)
      Note: If you disable table generation, yacc() will regenerate the parsing tables each time it runs (which may take awhile depending on how large your grammar is).

    • To print copious amounts of debugging during parsing, use: 
      parser.parse(input_text, debug=True)

    • Since the generation of the LALR tables is relatively expensive, previously generated tables are cached and reused if possible. The decision to regenerate the tables is determined by taking an MD5 checksum of all grammar rules and precedence rules. Only in the event of a mismatch are the tables regenerated.

      It should be noted that table generation is reasonably efficient, even for grammars that involve around a 100 rules and several hundred states.

    • Since LR parsing is driven by tables, the performance of the parser is largely independent of the size of the grammar. The biggest bottlenecks will be the lexer and the complexity of the code in your grammar rules.

    • yacc() also allows parsers to be defined as classes and as closures (see the section on alternative specification of lexers). However, be aware that only one parser may be defined in a single module (source file). There are various error checks and validation steps that may issue confusing error messages if you try to define multiple parsers in the same source file.

    • Decorators of production rules have to update the wrapped function's line number. wrapper.co_firstlineno = func.__code__.co_firstlineno: 
      from functools import wraps
from nodes import Collection


def strict(*types):
    def decorate(func):
        @wraps(func)
        def wrapper(p):
            func(p)
            if not isinstance(p[0], types):
                raise TypeError

        wrapper.co_firstlineno = func.__code__.co_firstlineno
        return wrapper

    return decorate

@strict(Collection)
def p_collection(p):
    """
    collection  : sequence
                | map
    """
    p[0] = p[1]

    7. Multiple Parsers and Lexers

    In advanced parsing applications, you may want to have multiple parsers and lexers.

    As a general rules this isn't a problem. However, to make it work, you need to carefully make sure everything gets hooked up correctly. First, make sure you save the objects returned by lex() and yacc(). For example: 

    lexer  = lex.lex()       # Return lexer object
parser = yacc.yacc()     # Return parser object
    Next, when parsing, make sure you give the parse() function a reference to the lexer it should be using. For example: 
    parser.parse(text,lexer=lexer)
    If you forget to do this, the parser will use the last lexer created--which is not always what you want.

    Within lexer and parser rule functions, these objects are also available. In the lexer, the "lexer" attribute of a token refers to the lexer object that triggered the rule. For example: 

    def t_NUMBER(t):
   r'\d+'
   ...
   print(t.lexer)           # Show lexer object
    In the parser, the "lexer" and "parser" attributes refer to the lexer and parser objects respectively. 
    def p_expr_plus(p):
   'expr : expr PLUS expr'
   ...
   print(p.parser)          # Show parser object
   print(p.lexer)           # Show lexer object
    If necessary, arbitrary attributes can be attached to the lexer or parser object. For example, if you wanted to have different parsing modes, you could attach a mode attribute to the parser object and look at it later.

    8. Using Python's Optimized Mode

    Because PLY uses information from doc-strings, parsing and lexing information must be gathered while running the Python interpreter in normal mode (i.e., not with the -O or -OO options). However, if you specify optimized mode like this: 
    lex.lex(optimize=1)
yacc.yacc(optimize=1)
    then PLY can later be used when Python runs in optimized mode. To make this work, make sure you first run Python in normal mode. Once the lexing and parsing tables have been generated the first time, run Python in optimized mode. PLY will use the tables without the need for doc strings.

    Beware: running PLY in optimized mode disables a lot of error checking. You should only do this when your project has stabilized and you don't need to do any debugging. One of the purposes of optimized mode is to substantially decrease the startup time of your compiler (by assuming that everything is already properly specified and works).

    9. Advanced Debugging

    Debugging a compiler is typically not an easy task. PLY provides some advanced diagostic capabilities through the use of Python's logging module. The next two sections describe this:

    9.1 Debugging the lex() and yacc() commands

    Both the lex() and yacc() commands have a debugging mode that can be enabled using the debug flag. For example: 

    lex.lex(debug=True)
yacc.yacc(debug=True)
    Normally, the output produced by debugging is routed to either standard error or, in the case of yacc(), to a file parser.out. This output can be more carefully controlled by supplying a logging object. Here is an example that adds information about where different debugging messages are coming from: 
    # Set up a logging object
import logging
logging.basicConfig(
    level = logging.DEBUG,
    filename = "parselog.txt",
    filemode = "w",
    format = "%(filename)10s:%(lineno)4d:%(message)s"
)
log = logging.getLogger()

lex.lex(debug=True,debuglog=log)
yacc.yacc(debug=True,debuglog=log)
    If you supply a custom logger, the amount of debugging information produced can be controlled by setting the logging level. Typically, debugging messages are either issued at the DEBUG, INFO, or WARNING levels.

    PLY's error messages and warnings are also produced using the logging interface. This can be controlled by passing a logging object using the errorlog parameter. 

    lex.lex(errorlog=log)
yacc.yacc(errorlog=log)
    If you want to completely silence warnings, you can either pass in a logging object with an appropriate filter level or use the NullLogger object defined in either lex or yacc. For example: 
    yacc.yacc(errorlog=yacc.NullLogger())

    9.2 Run-time Debugging

    To enable run-time debugging of a parser, use the debug option to parse. This option can either be an integer (which simply turns debugging on or off) or an instance of a logger object. For example: 

    log = logging.getLogger()
parser.parse(input,debug=log)
    If a logging object is passed, you can use its filtering level to control how much output gets generated. The INFO level is used to produce information about rule reductions. The DEBUG level will show information about the parsing stack, token shifts, and other details. The ERROR level shows information related to parsing errors.

    For very complicated problems, you should pass in a logging object that redirects to a file where you can more easily inspect the output after execution.

    10. Packaging Advice

    If you are distributing a package that makes use of PLY, you should spend a few moments thinking about how you want to handle the files that are automatically generated. For example, the parsetab.py file generated by the yacc() function.

    Starting in PLY-3.6, the table files are created in the same directory as the file where a parser is defined. This means that the parsetab.py file will live side-by-side with your parser specification. In terms of packaging, this is probably the easiest and most sane approach to manage. You don't need to give yacc() any extra arguments and it should just "work."

    One concern is the management of the parsetab.py file itself. For example, should you have this file checked into version control (e.g., GitHub), should it be included in a package distribution as a normal file, or should you just let PLY generate it automatically for the user when they install your package?

    As of PLY-3.6, the parsetab.py file should be compatible across all versions of Python including Python 2 and 3. Thus, a table file generated in Python 2 should work fine if it's used on Python 3. Because of this, it should be relatively harmless to distribute the parsetab.py file yourself if you need to. However, be aware that older/newer versions of PLY may try to regenerate the file if there are future enhancements or changes to its format.

    To make the generation of table files easier for the purposes of installation, you might way to make your parser files executable using the -m option or similar. For example: 

    # calc.py
...
...
def make_parser():
    parser = yacc.yacc()
    return parser

if __name__ == '__main__':
    make_parser()

    You can then use a command such as python -m calc.py to generate the tables. Alternatively, a setup.py script, can import the module and use make_parser() to create the parsing tables.

    If you're willing to sacrifice a little startup time, you can also instruct PLY to never write the tables using yacc.yacc(write_tables=False, debug=False). In this mode, PLY will regenerate the parsing tables from scratch each time. For a small grammar, you probably won't notice. For a large grammar, you should probably reconsider--the parsing tables are meant to dramatically speed up this process.

    During operation, it is normal for PLY to produce diagnostic error messages (usually printed to standard error). These are generated entirely using the logging module. If you want to redirect these messages or silence them, you can provide your own logging object to yacc(). For example: 

    import logging
log = logging.getLogger('ply')
...
parser = yacc.yacc(errorlog=log)

    11. Where to go from here?

    The examples directory of the PLY distribution contains several simple examples. Please consult a compilers textbook for the theory and underlying implementation details or LR parsing. ================================================ FILE: xonsh/ply/example/BASIC/README ================================================ Inspired by a September 14, 2006 Salon article "Why Johnny Can't Code" by David Brin (http://www.salon.com/tech/feature/2006/09/14/basic/index.html), I thought that a fully working BASIC interpreter might be an interesting, if not questionable, PLY example. Uh, okay, so maybe it's just a bad idea, but in any case, here it is. In this example, you'll find a rough implementation of 1964 Dartmouth BASIC as described in the manual at: http://www.bitsavers.org/pdf/dartmouth/BASIC_Oct64.pdf See also: http://en.wikipedia.org/wiki/Dartmouth_BASIC This dialect is downright primitive---there are no string variables and no facilities for interactive input. Moreover, subroutines and functions are brain-dead even more than they usually are for BASIC. Of course, the GOTO statement is provided. Nevertheless, there are a few interesting aspects of this example: - It illustrates a fully working interpreter including lexing, parsing, and interpretation of instructions. - The parser shows how to catch and report various kinds of parsing errors in a more graceful way. - The example both parses files (supplied on command line) and interactive input entered line by line. - It shows how you might represent parsed information. In this case, each BASIC statement is encoded into a Python tuple containing the statement type and parameters. These tuples are then stored in a dictionary indexed by program line numbers. - Even though it's just BASIC, the parser contains more than 80 rules and 150 parsing states. Thus, it's a little more meaty than the calculator example. To use the example, run it as follows: % python basic.py hello.bas HELLO WORLD % or use it interactively: % python basic.py [BASIC] 10 PRINT "HELLO WORLD" [BASIC] 20 END [BASIC] RUN HELLO WORLD [BASIC] The following files are defined: basic.py - High level script that controls everything basiclex.py - BASIC tokenizer basparse.py - BASIC parser basinterp.py - BASIC interpreter that runs parsed programs. In addition, a number of sample BASIC programs (.bas suffix) are provided. These were taken out of the Dartmouth manual. Disclaimer: I haven't spent a ton of time testing this and it's likely that I've skimped here and there on a few finer details (e.g., strictly enforcing variable naming rules). However, the interpreter seems to be able to run the examples in the BASIC manual. Have fun! -Dave ================================================ FILE: xonsh/ply/example/BASIC/basic.py ================================================ # An implementation of Dartmouth BASIC (1964) # import sys sys.path.insert(0, "../..") if sys.version_info[0] >= 3: raw_input = input import basiclex import basparse import basinterp # If a filename has been specified, we try to run it. # If a runtime error occurs, we bail out and enter # interactive mode below if len(sys.argv) == 2: with open(sys.argv[1]) as f: data = f.read() prog = basparse.parse(data) if not prog: raise SystemExit b = basinterp.BasicInterpreter(prog) try: b.run() raise SystemExit except RuntimeError: pass else: b = basinterp.BasicInterpreter({}) # Interactive mode. This incrementally adds/deletes statements # from the program stored in the BasicInterpreter object. In # addition, special commands 'NEW','LIST',and 'RUN' are added. # Specifying a line number with no code deletes that line from # the program. while 1: try: line = raw_input("[BASIC] ") except EOFError: raise SystemExit if not line: continue line += "\n" prog = basparse.parse(line) if not prog: continue keys = list(prog) if keys[0] > 0: b.add_statements(prog) else: stat = prog[keys[0]] if stat[0] == 'RUN': try: b.run() except RuntimeError: pass elif stat[0] == 'LIST': b.list() elif stat[0] == 'BLANK': b.del_line(stat[1]) elif stat[0] == 'NEW': b.new() ================================================ FILE: xonsh/ply/example/BASIC/basiclex.py ================================================ # An implementation of Dartmouth BASIC (1964) from ply import * keywords = ( 'LET', 'READ', 'DATA', 'PRINT', 'GOTO', 'IF', 'THEN', 'FOR', 'NEXT', 'TO', 'STEP', 'END', 'STOP', 'DEF', 'GOSUB', 'DIM', 'REM', 'RETURN', 'RUN', 'LIST', 'NEW', ) tokens = keywords + ( 'EQUALS', 'PLUS', 'MINUS', 'TIMES', 'DIVIDE', 'POWER', 'LPAREN', 'RPAREN', 'LT', 'LE', 'GT', 'GE', 'NE', 'COMMA', 'SEMI', 'INTEGER', 'FLOAT', 'STRING', 'ID', 'NEWLINE' ) t_ignore = ' \t' def t_REM(t): r'REM .*' return t def t_ID(t): r'[A-Z][A-Z0-9]*' if t.value in keywords: t.type = t.value return t t_EQUALS = r'=' t_PLUS = r'\+' t_MINUS = r'-' t_TIMES = r'\*' t_POWER = r'\^' t_DIVIDE = r'/' t_LPAREN = r'\(' t_RPAREN = r'\)' t_LT = r'<' t_LE = r'<=' t_GT = r'>' t_GE = r'>=' t_NE = r'<>' t_COMMA = r'\,' t_SEMI = r';' t_INTEGER = r'\d+' t_FLOAT = r'((\d*\.\d+)(E[\+-]?\d+)?|([1-9]\d*E[\+-]?\d+))' t_STRING = r'\".*?\"' def t_NEWLINE(t): r'\n' t.lexer.lineno += 1 return t def t_error(t): print("Illegal character %s" % t.value[0]) t.lexer.skip(1) lex.lex(debug=0) ================================================ FILE: xonsh/ply/example/BASIC/basiclog.py ================================================ # An implementation of Dartmouth BASIC (1964) # import sys sys.path.insert(0, "../..") if sys.version_info[0] >= 3: raw_input = input import logging logging.basicConfig( level=logging.INFO, filename="parselog.txt", filemode="w" ) log = logging.getLogger() import basiclex import basparse import basinterp # If a filename has been specified, we try to run it. # If a runtime error occurs, we bail out and enter # interactive mode below if len(sys.argv) == 2: with open(sys.argv[1]) as f: data = f.read() prog = basparse.parse(data, debug=log) if not prog: raise SystemExit b = basinterp.BasicInterpreter(prog) try: b.run() raise SystemExit except RuntimeError: pass else: b = basinterp.BasicInterpreter({}) # Interactive mode. This incrementally adds/deletes statements # from the program stored in the BasicInterpreter object. In # addition, special commands 'NEW','LIST',and 'RUN' are added. # Specifying a line number with no code deletes that line from # the program. while 1: try: line = raw_input("[BASIC] ") except EOFError: raise SystemExit if not line: continue line += "\n" prog = basparse.parse(line, debug=log) if not prog: continue keys = list(prog) if keys[0] > 0: b.add_statements(prog) else: stat = prog[keys[0]] if stat[0] == 'RUN': try: b.run() except RuntimeError: pass elif stat[0] == 'LIST': b.list() elif stat[0] == 'BLANK': b.del_line(stat[1]) elif stat[0] == 'NEW': b.new() ================================================ FILE: xonsh/ply/example/BASIC/basinterp.py ================================================ # This file provides the runtime support for running a basic program # Assumes the program has been parsed using basparse.py import sys import math import random class BasicInterpreter: # Initialize the interpreter. prog is a dictionary # containing (line,statement) mappings def __init__(self, prog): self.prog = prog self.functions = { # Built-in function table 'SIN': lambda z: math.sin(self.eval(z)), 'COS': lambda z: math.cos(self.eval(z)), 'TAN': lambda z: math.tan(self.eval(z)), 'ATN': lambda z: math.atan(self.eval(z)), 'EXP': lambda z: math.exp(self.eval(z)), 'ABS': lambda z: abs(self.eval(z)), 'LOG': lambda z: math.log(self.eval(z)), 'SQR': lambda z: math.sqrt(self.eval(z)), 'INT': lambda z: int(self.eval(z)), 'RND': lambda z: random.random() } # Collect all data statements def collect_data(self): self.data = [] for lineno in self.stat: if self.prog[lineno][0] == 'DATA': self.data = self.data + self.prog[lineno][1] self.dc = 0 # Initialize the data counter # Check for end statements def check_end(self): has_end = 0 for lineno in self.stat: if self.prog[lineno][0] == 'END' and not has_end: has_end = lineno if not has_end: print("NO END INSTRUCTION") self.error = 1 return if has_end != lineno: print("END IS NOT LAST") self.error = 1 # Check loops def check_loops(self): for pc in range(len(self.stat)): lineno = self.stat[pc] if self.prog[lineno][0] == 'FOR': forinst = self.prog[lineno] loopvar = forinst[1] for i in range(pc + 1, len(self.stat)): if self.prog[self.stat[i]][0] == 'NEXT': nextvar = self.prog[self.stat[i]][1] if nextvar != loopvar: continue self.loopend[pc] = i break else: print("FOR WITHOUT NEXT AT LINE %s" % self.stat[pc]) self.error = 1 # Evaluate an expression def eval(self, expr): etype = expr[0] if etype == 'NUM': return expr[1] elif etype == 'GROUP': return self.eval(expr[1]) elif etype == 'UNARY': if expr[1] == '-': return -self.eval(expr[2]) elif etype == 'BINOP': if expr[1] == '+': return self.eval(expr[2]) + self.eval(expr[3]) elif expr[1] == '-': return self.eval(expr[2]) - self.eval(expr[3]) elif expr[1] == '*': return self.eval(expr[2]) * self.eval(expr[3]) elif expr[1] == '/': return float(self.eval(expr[2])) / self.eval(expr[3]) elif expr[1] == '^': return abs(self.eval(expr[2]))**self.eval(expr[3]) elif etype == 'VAR': var, dim1, dim2 = expr[1] if not dim1 and not dim2: if var in self.vars: return self.vars[var] else: print("UNDEFINED VARIABLE %s AT LINE %s" % (var, self.stat[self.pc])) raise RuntimeError # May be a list lookup or a function evaluation if dim1 and not dim2: if var in self.functions: # A function return self.functions[var](dim1) else: # A list evaluation if var in self.lists: dim1val = self.eval(dim1) if dim1val < 1 or dim1val > len(self.lists[var]): print("LIST INDEX OUT OF BOUNDS AT LINE %s" % self.stat[self.pc]) raise RuntimeError return self.lists[var][dim1val - 1] if dim1 and dim2: if var in self.tables: dim1val = self.eval(dim1) dim2val = self.eval(dim2) if dim1val < 1 or dim1val > len(self.tables[var]) or dim2val < 1 or dim2val > len(self.tables[var][0]): print("TABLE INDEX OUT OUT BOUNDS AT LINE %s" % self.stat[self.pc]) raise RuntimeError return self.tables[var][dim1val - 1][dim2val - 1] print("UNDEFINED VARIABLE %s AT LINE %s" % (var, self.stat[self.pc])) raise RuntimeError # Evaluate a relational expression def releval(self, expr): etype = expr[1] lhs = self.eval(expr[2]) rhs = self.eval(expr[3]) if etype == '<': if lhs < rhs: return 1 else: return 0 elif etype == '<=': if lhs <= rhs: return 1 else: return 0 elif etype == '>': if lhs > rhs: return 1 else: return 0 elif etype == '>=': if lhs >= rhs: return 1 else: return 0 elif etype == '=': if lhs == rhs: return 1 else: return 0 elif etype == '<>': if lhs != rhs: return 1 else: return 0 # Assignment def assign(self, target, value): var, dim1, dim2 = target if not dim1 and not dim2: self.vars[var] = self.eval(value) elif dim1 and not dim2: # List assignment dim1val = self.eval(dim1) if not var in self.lists: self.lists[var] = [0] * 10 if dim1val > len(self.lists[var]): print ("DIMENSION TOO LARGE AT LINE %s" % self.stat[self.pc]) raise RuntimeError self.lists[var][dim1val - 1] = self.eval(value) elif dim1 and dim2: dim1val = self.eval(dim1) dim2val = self.eval(dim2) if not var in self.tables: temp = [0] * 10 v = [] for i in range(10): v.append(temp[:]) self.tables[var] = v # Variable already exists if dim1val > len(self.tables[var]) or dim2val > len(self.tables[var][0]): print("DIMENSION TOO LARGE AT LINE %s" % self.stat[self.pc]) raise RuntimeError self.tables[var][dim1val - 1][dim2val - 1] = self.eval(value) # Change the current line number def goto(self, linenum): if not linenum in self.prog: print("UNDEFINED LINE NUMBER %d AT LINE %d" % (linenum, self.stat[self.pc])) raise RuntimeError self.pc = self.stat.index(linenum) # Run it def run(self): self.vars = {} # All variables self.lists = {} # List variables self.tables = {} # Tables self.loops = [] # Currently active loops self.loopend = {} # Mapping saying where loops end self.gosub = None # Gosub return point (if any) self.error = 0 # Indicates program error self.stat = list(self.prog) # Ordered list of all line numbers self.stat.sort() self.pc = 0 # Current program counter # Processing prior to running self.collect_data() # Collect all of the data statements self.check_end() self.check_loops() if self.error: raise RuntimeError while 1: line = self.stat[self.pc] instr = self.prog[line] op = instr[0] # END and STOP statements if op == 'END' or op == 'STOP': break # We're done # GOTO statement elif op == 'GOTO': newline = instr[1] self.goto(newline) continue # PRINT statement elif op == 'PRINT': plist = instr[1] out = "" for label, val in plist: if out: out += ' ' * (15 - (len(out) % 15)) out += label if val: if label: out += " " eval = self.eval(val) out += str(eval) sys.stdout.write(out) end = instr[2] if not (end == ',' or end == ';'): sys.stdout.write("\n") if end == ',': sys.stdout.write(" " * (15 - (len(out) % 15))) if end == ';': sys.stdout.write(" " * (3 - (len(out) % 3))) # LET statement elif op == 'LET': target = instr[1] value = instr[2] self.assign(target, value) # READ statement elif op == 'READ': for target in instr[1]: if self.dc < len(self.data): value = ('NUM', self.data[self.dc]) self.assign(target, value) self.dc += 1 else: # No more data. Program ends return elif op == 'IF': relop = instr[1] newline = instr[2] if (self.releval(relop)): self.goto(newline) continue elif op == 'FOR': loopvar = instr[1] initval = instr[2] finval = instr[3] stepval = instr[4] # Check to see if this is a new loop if not self.loops or self.loops[-1][0] != self.pc: # Looks like a new loop. Make the initial assignment newvalue = initval self.assign((loopvar, None, None), initval) if not stepval: stepval = ('NUM', 1) stepval = self.eval(stepval) # Evaluate step here self.loops.append((self.pc, stepval)) else: # It's a repeat of the previous loop # Update the value of the loop variable according to the # step stepval = ('NUM', self.loops[-1][1]) newvalue = ( 'BINOP', '+', ('VAR', (loopvar, None, None)), stepval) if self.loops[-1][1] < 0: relop = '>=' else: relop = '<=' if not self.releval(('RELOP', relop, newvalue, finval)): # Loop is done. Jump to the NEXT self.pc = self.loopend[self.pc] self.loops.pop() else: self.assign((loopvar, None, None), newvalue) elif op == 'NEXT': if not self.loops: print("NEXT WITHOUT FOR AT LINE %s" % line) return nextvar = instr[1] self.pc = self.loops[-1][0] loopinst = self.prog[self.stat[self.pc]] forvar = loopinst[1] if nextvar != forvar: print("NEXT DOESN'T MATCH FOR AT LINE %s" % line) return continue elif op == 'GOSUB': newline = instr[1] if self.gosub: print("ALREADY IN A SUBROUTINE AT LINE %s" % line) return self.gosub = self.stat[self.pc] self.goto(newline) continue elif op == 'RETURN': if not self.gosub: print("RETURN WITHOUT A GOSUB AT LINE %s" % line) return self.goto(self.gosub) self.gosub = None elif op == 'FUNC': fname = instr[1] pname = instr[2] expr = instr[3] def eval_func(pvalue, name=pname, self=self, expr=expr): self.assign((pname, None, None), pvalue) return self.eval(expr) self.functions[fname] = eval_func elif op == 'DIM': for vname, x, y in instr[1]: if y == 0: # Single dimension variable self.lists[vname] = [0] * x else: # Double dimension variable temp = [0] * y v = [] for i in range(x): v.append(temp[:]) self.tables[vname] = v self.pc += 1 # Utility functions for program listing def expr_str(self, expr): etype = expr[0] if etype == 'NUM': return str(expr[1]) elif etype == 'GROUP': return "(%s)" % self.expr_str(expr[1]) elif etype == 'UNARY': if expr[1] == '-': return "-" + str(expr[2]) elif etype == 'BINOP': return "%s %s %s" % (self.expr_str(expr[2]), expr[1], self.expr_str(expr[3])) elif etype == 'VAR': return self.var_str(expr[1]) def relexpr_str(self, expr): return "%s %s %s" % (self.expr_str(expr[2]), expr[1], self.expr_str(expr[3])) def var_str(self, var): varname, dim1, dim2 = var if not dim1 and not dim2: return varname if dim1 and not dim2: return "%s(%s)" % (varname, self.expr_str(dim1)) return "%s(%s,%s)" % (varname, self.expr_str(dim1), self.expr_str(dim2)) # Create a program listing def list(self): stat = list(self.prog) # Ordered list of all line numbers stat.sort() for line in stat: instr = self.prog[line] op = instr[0] if op in ['END', 'STOP', 'RETURN']: print("%s %s" % (line, op)) continue elif op == 'REM': print("%s %s" % (line, instr[1])) elif op == 'PRINT': _out = "%s %s " % (line, op) first = 1 for p in instr[1]: if not first: _out += ", " if p[0] and p[1]: _out += '"%s"%s' % (p[0], self.expr_str(p[1])) elif p[1]: _out += self.expr_str(p[1]) else: _out += '"%s"' % (p[0],) first = 0 if instr[2]: _out += instr[2] print(_out) elif op == 'LET': print("%s LET %s = %s" % (line, self.var_str(instr[1]), self.expr_str(instr[2]))) elif op == 'READ': _out = "%s READ " % line first = 1 for r in instr[1]: if not first: _out += "," _out += self.var_str(r) first = 0 print(_out) elif op == 'IF': print("%s IF %s THEN %d" % (line, self.relexpr_str(instr[1]), instr[2])) elif op == 'GOTO' or op == 'GOSUB': print("%s %s %s" % (line, op, instr[1])) elif op == 'FOR': _out = "%s FOR %s = %s TO %s" % ( line, instr[1], self.expr_str(instr[2]), self.expr_str(instr[3])) if instr[4]: _out += " STEP %s" % (self.expr_str(instr[4])) print(_out) elif op == 'NEXT': print("%s NEXT %s" % (line, instr[1])) elif op == 'FUNC': print("%s DEF %s(%s) = %s" % (line, instr[1], instr[2], self.expr_str(instr[3]))) elif op == 'DIM': _out = "%s DIM " % line first = 1 for vname, x, y in instr[1]: if not first: _out += "," first = 0 if y == 0: _out += "%s(%d)" % (vname, x) else: _out += "%s(%d,%d)" % (vname, x, y) print(_out) elif op == 'DATA': _out = "%s DATA " % line first = 1 for v in instr[1]: if not first: _out += "," first = 0 _out += v print(_out) # Erase the current program def new(self): self.prog = {} # Insert statements def add_statements(self, prog): for line, stat in prog.items(): self.prog[line] = stat # Delete a statement def del_line(self, lineno): try: del self.prog[lineno] except KeyError: pass ================================================ FILE: xonsh/ply/example/BASIC/basparse.py ================================================ # An implementation of Dartmouth BASIC (1964) # from ply import * import basiclex tokens = basiclex.tokens precedence = ( ('left', 'PLUS', 'MINUS'), ('left', 'TIMES', 'DIVIDE'), ('left', 'POWER'), ('right', 'UMINUS') ) # A BASIC program is a series of statements. We represent the program as a # dictionary of tuples indexed by line number. def p_program(p): '''program : program statement | statement''' if len(p) == 2 and p[1]: p[0] = {} line, stat = p[1] p[0][line] = stat elif len(p) == 3: p[0] = p[1] if not p[0]: p[0] = {} if p[2]: line, stat = p[2] p[0][line] = stat # This catch-all rule is used for any catastrophic errors. In this case, # we simply return nothing def p_program_error(p): '''program : error''' p[0] = None p.parser.error = 1 # Format of all BASIC statements. def p_statement(p): '''statement : INTEGER command NEWLINE''' if isinstance(p[2], str): print("%s %s %s" % (p[2], "AT LINE", p[1])) p[0] = None p.parser.error = 1 else: lineno = int(p[1]) p[0] = (lineno, p[2]) # Interactive statements. def p_statement_interactive(p): '''statement : RUN NEWLINE | LIST NEWLINE | NEW NEWLINE''' p[0] = (0, (p[1], 0)) # Blank line number def p_statement_blank(p): '''statement : INTEGER NEWLINE''' p[0] = (0, ('BLANK', int(p[1]))) # Error handling for malformed statements def p_statement_bad(p): '''statement : INTEGER error NEWLINE''' print("MALFORMED STATEMENT AT LINE %s" % p[1]) p[0] = None p.parser.error = 1 # Blank line def p_statement_newline(p): '''statement : NEWLINE''' p[0] = None # LET statement def p_command_let(p): '''command : LET variable EQUALS expr''' p[0] = ('LET', p[2], p[4]) def p_command_let_bad(p): '''command : LET variable EQUALS error''' p[0] = "BAD EXPRESSION IN LET" # READ statement def p_command_read(p): '''command : READ varlist''' p[0] = ('READ', p[2]) def p_command_read_bad(p): '''command : READ error''' p[0] = "MALFORMED VARIABLE LIST IN READ" # DATA statement def p_command_data(p): '''command : DATA numlist''' p[0] = ('DATA', p[2]) def p_command_data_bad(p): '''command : DATA error''' p[0] = "MALFORMED NUMBER LIST IN DATA" # PRINT statement def p_command_print(p): '''command : PRINT plist optend''' p[0] = ('PRINT', p[2], p[3]) def p_command_print_bad(p): '''command : PRINT error''' p[0] = "MALFORMED PRINT STATEMENT" # Optional ending on PRINT. Either a comma (,) or semicolon (;) def p_optend(p): '''optend : COMMA | SEMI |''' if len(p) == 2: p[0] = p[1] else: p[0] = None # PRINT statement with no arguments def p_command_print_empty(p): '''command : PRINT''' p[0] = ('PRINT', [], None) # GOTO statement def p_command_goto(p): '''command : GOTO INTEGER''' p[0] = ('GOTO', int(p[2])) def p_command_goto_bad(p): '''command : GOTO error''' p[0] = "INVALID LINE NUMBER IN GOTO" # IF-THEN statement def p_command_if(p): '''command : IF relexpr THEN INTEGER''' p[0] = ('IF', p[2], int(p[4])) def p_command_if_bad(p): '''command : IF error THEN INTEGER''' p[0] = "BAD RELATIONAL EXPRESSION" def p_command_if_bad2(p): '''command : IF relexpr THEN error''' p[0] = "INVALID LINE NUMBER IN THEN" # FOR statement def p_command_for(p): '''command : FOR ID EQUALS expr TO expr optstep''' p[0] = ('FOR', p[2], p[4], p[6], p[7]) def p_command_for_bad_initial(p): '''command : FOR ID EQUALS error TO expr optstep''' p[0] = "BAD INITIAL VALUE IN FOR STATEMENT" def p_command_for_bad_final(p): '''command : FOR ID EQUALS expr TO error optstep''' p[0] = "BAD FINAL VALUE IN FOR STATEMENT" def p_command_for_bad_step(p): '''command : FOR ID EQUALS expr TO expr STEP error''' p[0] = "MALFORMED STEP IN FOR STATEMENT" # Optional STEP qualifier on FOR statement def p_optstep(p): '''optstep : STEP expr | empty''' if len(p) == 3: p[0] = p[2] else: p[0] = None # NEXT statement def p_command_next(p): '''command : NEXT ID''' p[0] = ('NEXT', p[2]) def p_command_next_bad(p): '''command : NEXT error''' p[0] = "MALFORMED NEXT" # END statement def p_command_end(p): '''command : END''' p[0] = ('END',) # REM statement def p_command_rem(p): '''command : REM''' p[0] = ('REM', p[1]) # STOP statement def p_command_stop(p): '''command : STOP''' p[0] = ('STOP',) # DEF statement def p_command_def(p): '''command : DEF ID LPAREN ID RPAREN EQUALS expr''' p[0] = ('FUNC', p[2], p[4], p[7]) def p_command_def_bad_rhs(p): '''command : DEF ID LPAREN ID RPAREN EQUALS error''' p[0] = "BAD EXPRESSION IN DEF STATEMENT" def p_command_def_bad_arg(p): '''command : DEF ID LPAREN error RPAREN EQUALS expr''' p[0] = "BAD ARGUMENT IN DEF STATEMENT" # GOSUB statement def p_command_gosub(p): '''command : GOSUB INTEGER''' p[0] = ('GOSUB', int(p[2])) def p_command_gosub_bad(p): '''command : GOSUB error''' p[0] = "INVALID LINE NUMBER IN GOSUB" # RETURN statement def p_command_return(p): '''command : RETURN''' p[0] = ('RETURN',) # DIM statement def p_command_dim(p): '''command : DIM dimlist''' p[0] = ('DIM', p[2]) def p_command_dim_bad(p): '''command : DIM error''' p[0] = "MALFORMED VARIABLE LIST IN DIM" # List of variables supplied to DIM statement def p_dimlist(p): '''dimlist : dimlist COMMA dimitem | dimitem''' if len(p) == 4: p[0] = p[1] p[0].append(p[3]) else: p[0] = [p[1]] # DIM items def p_dimitem_single(p): '''dimitem : ID LPAREN INTEGER RPAREN''' p[0] = (p[1], eval(p[3]), 0) def p_dimitem_double(p): '''dimitem : ID LPAREN INTEGER COMMA INTEGER RPAREN''' p[0] = (p[1], eval(p[3]), eval(p[5])) # Arithmetic expressions def p_expr_binary(p): '''expr : expr PLUS expr | expr MINUS expr | expr TIMES expr | expr DIVIDE expr | expr POWER expr''' p[0] = ('BINOP', p[2], p[1], p[3]) def p_expr_number(p): '''expr : INTEGER | FLOAT''' p[0] = ('NUM', eval(p[1])) def p_expr_variable(p): '''expr : variable''' p[0] = ('VAR', p[1]) def p_expr_group(p): '''expr : LPAREN expr RPAREN''' p[0] = ('GROUP', p[2]) def p_expr_unary(p): '''expr : MINUS expr %prec UMINUS''' p[0] = ('UNARY', '-', p[2]) # Relational expressions def p_relexpr(p): '''relexpr : expr LT expr | expr LE expr | expr GT expr | expr GE expr | expr EQUALS expr | expr NE expr''' p[0] = ('RELOP', p[2], p[1], p[3]) # Variables def p_variable(p): '''variable : ID | ID LPAREN expr RPAREN | ID LPAREN expr COMMA expr RPAREN''' if len(p) == 2: p[0] = (p[1], None, None) elif len(p) == 5: p[0] = (p[1], p[3], None) else: p[0] = (p[1], p[3], p[5]) # Builds a list of variable targets as a Python list def p_varlist(p): '''varlist : varlist COMMA variable | variable''' if len(p) > 2: p[0] = p[1] p[0].append(p[3]) else: p[0] = [p[1]] # Builds a list of numbers as a Python list def p_numlist(p): '''numlist : numlist COMMA number | number''' if len(p) > 2: p[0] = p[1] p[0].append(p[3]) else: p[0] = [p[1]] # A number. May be an integer or a float def p_number(p): '''number : INTEGER | FLOAT''' p[0] = eval(p[1]) # A signed number. def p_number_signed(p): '''number : MINUS INTEGER | MINUS FLOAT''' p[0] = eval("-" + p[2]) # List of targets for a print statement # Returns a list of tuples (label,expr) def p_plist(p): '''plist : plist COMMA pitem | pitem''' if len(p) > 3: p[0] = p[1] p[0].append(p[3]) else: p[0] = [p[1]] def p_item_string(p): '''pitem : STRING''' p[0] = (p[1][1:-1], None) def p_item_string_expr(p): '''pitem : STRING expr''' p[0] = (p[1][1:-1], p[2]) def p_item_expr(p): '''pitem : expr''' p[0] = ("", p[1]) # Empty def p_empty(p): '''empty : ''' # Catastrophic error handler def p_error(p): if not p: print("SYNTAX ERROR AT EOF") bparser = yacc.yacc() def parse(data, debug=0): bparser.error = 0 p = bparser.parse(data, debug=debug) if bparser.error: return None return p ================================================ FILE: xonsh/ply/example/BASIC/dim.bas ================================================ 5 DIM A(50,15) 10 FOR I = 1 TO 50 20 FOR J = 1 TO 15 30 LET A(I,J) = I + J 35 REM PRINT I,J, A(I,J) 40 NEXT J 50 NEXT I 100 FOR I = 1 TO 50 110 FOR J = 1 TO 15 120 PRINT A(I,J), 130 NEXT J 140 PRINT 150 NEXT I 999 END ================================================ FILE: xonsh/ply/example/BASIC/func.bas ================================================ 10 DEF FDX(X) = 2*X 20 FOR I = 0 TO 100 30 PRINT FDX(I) 40 NEXT I 50 END ================================================ FILE: xonsh/ply/example/BASIC/gcd.bas ================================================ 10 PRINT "A","B","C","GCD" 20 READ A,B,C 30 LET X = A 40 LET Y = B 50 GOSUB 200 60 LET X = G 70 LET Y = C 80 GOSUB 200 90 PRINT A, B, C, G 100 GOTO 20 110 DATA 60, 90, 120 120 DATA 38456, 64872, 98765 130 DATA 32, 384, 72 200 LET Q = INT(X/Y) 210 LET R = X - Q*Y 220 IF R = 0 THEN 300 230 LET X = Y 240 LET Y = R 250 GOTO 200 300 LET G = Y 310 RETURN 999 END ================================================ FILE: xonsh/ply/example/BASIC/gosub.bas ================================================ 100 LET X = 3 110 GOSUB 400 120 PRINT U, V, W 200 LET X = 5 210 GOSUB 400 220 LET Z = U + 2*V + 3*W 230 PRINT Z 240 GOTO 999 400 LET U = X*X 410 LET V = X*X*X 420 LET W = X*X*X*X + X*X*X + X*X + X 430 RETURN 999 END ================================================ FILE: xonsh/ply/example/BASIC/hello.bas ================================================ 5 REM HELLO WORLD PROGAM 10 PRINT "HELLO WORLD" 99 END ================================================ FILE: xonsh/ply/example/BASIC/linear.bas ================================================ 1 REM ::: SOLVE A SYSTEM OF LINEAR EQUATIONS 2 REM ::: A1*X1 + A2*X2 = B1 3 REM ::: A3*X1 + A4*X2 = B2 4 REM -------------------------------------- 10 READ A1, A2, A3, A4 15 LET D = A1 * A4 - A3 * A2 20 IF D = 0 THEN 65 30 READ B1, B2 37 LET X1 = (B1*A4 - B2*A2) / D 42 LET X2 = (A1*B2 - A3*B1) / D 55 PRINT X1, X2 60 GOTO 30 65 PRINT "NO UNIQUE SOLUTION" 70 DATA 1, 2, 4 80 DATA 2, -7, 5 85 DATA 1, 3, 4, -7 90 END ================================================ FILE: xonsh/ply/example/BASIC/maxsin.bas ================================================ 5 PRINT "X VALUE", "SINE", "RESOLUTION" 10 READ D 20 LET M = -1 30 FOR X = 0 TO 3 STEP D 40 IF SIN(X) <= M THEN 80 50 LET X0 = X 60 LET M = SIN(X) 80 NEXT X 85 PRINT X0, M, D 90 GOTO 10 100 DATA .1, .01, .001 110 END ================================================ FILE: xonsh/ply/example/BASIC/powers.bas ================================================ 5 PRINT "THIS PROGRAM COMPUTES AND PRINTS THE NTH POWERS" 6 PRINT "OF THE NUMBERS LESS THAN OR EQUAL TO N FOR VARIOUS" 7 PRINT "N FROM 1 THROUGH 7" 8 PRINT 10 FOR N = 1 TO 7 15 PRINT "N = "N 20 FOR I = 1 TO N 30 PRINT I^N, 40 NEXT I 50 PRINT 60 PRINT 70 NEXT N 80 END ================================================ FILE: xonsh/ply/example/BASIC/rand.bas ================================================ 10 FOR I = 1 TO 20 20 PRINT INT(10*RND(0)) 30 NEXT I 40 END ================================================ FILE: xonsh/ply/example/BASIC/sales.bas ================================================ 10 FOR I = 1 TO 3 20 READ P(I) 30 NEXT I 40 FOR I = 1 TO 3 50 FOR J = 1 TO 5 60 READ S(I,J) 70 NEXT J 80 NEXT I 90 FOR J = 1 TO 5 100 LET S = 0 110 FOR I = 1 TO 3 120 LET S = S + P(I) * S(I,J) 130 NEXT I 140 PRINT "TOTAL SALES FOR SALESMAN"J, "$"S 150 NEXT J 200 DATA 1.25, 4.30, 2.50 210 DATA 40, 20, 37, 29, 42 220 DATA 10, 16, 3, 21, 8 230 DATA 35, 47, 29, 16, 33 300 END ================================================ FILE: xonsh/ply/example/BASIC/sears.bas ================================================ 1 REM :: THIS PROGRAM COMPUTES HOW MANY TIMES YOU HAVE TO FOLD 2 REM :: A PIECE OF PAPER SO THAT IT IS TALLER THAN THE 3 REM :: SEARS TOWER. 4 REM :: S = HEIGHT OF TOWER (METERS) 5 REM :: T = THICKNESS OF PAPER (MILLIMETERS) 10 LET S = 442 20 LET T = 0.1 30 REM CONVERT T TO METERS 40 LET T = T * .001 50 LET F = 1 60 LET H = T 100 IF H > S THEN 200 120 LET H = 2 * H 125 LET F = F + 1 130 GOTO 100 200 PRINT "NUMBER OF FOLDS ="F 220 PRINT "FINAL HEIGHT ="H 999 END ================================================ FILE: xonsh/ply/example/BASIC/sqrt1.bas ================================================ 10 LET X = 0 20 LET X = X + 1 30 PRINT X, SQR(X) 40 IF X < 100 THEN 20 50 END ================================================ FILE: xonsh/ply/example/BASIC/sqrt2.bas ================================================ 10 FOR X = 1 TO 100 20 PRINT X, SQR(X) 30 NEXT X 40 END ================================================ FILE: xonsh/ply/example/GardenSnake/GardenSnake.py ================================================ # GardenSnake - a parser generator demonstration program # # This implements a modified version of a subset of Python: # - only 'def', 'return' and 'if' statements # - 'if' only has 'then' clause (no elif nor else) # - single-quoted strings only, content in raw format # - numbers are decimal.Decimal instances (not integers or floats) # - no print statment; use the built-in 'print' function # - only < > == + - / * implemented (and unary + -) # - assignment and tuple assignment work # - no generators of any sort # - no ... well, no quite a lot # Why? I'm thinking about a new indentation-based configuration # language for a project and wanted to figure out how to do it. Once # I got that working I needed a way to test it out. My original AST # was dumb so I decided to target Python's AST and compile it into # Python code. Plus, it's pretty cool that it only took a day or so # from sitting down with Ply to having working code. # This uses David Beazley's Ply from http://www.dabeaz.com/ply/ # This work is hereby released into the Public Domain. To view a copy of # the public domain dedication, visit # http://creativecommons.org/licenses/publicdomain/ or send a letter to # Creative Commons, 543 Howard Street, 5th Floor, San Francisco, # California, 94105, USA. # # Portions of this work are derived from Python's Grammar definition # and may be covered under the Python copyright and license # # Andrew Dalke / Dalke Scientific Software, LLC # 30 August 2006 / Cape Town, South Africa # Changelog: # 30 August - added link to CC license; removed the "swapcase" encoding # Modifications for inclusion in PLY distribution import sys sys.path.insert(0, "../..") from ply import * ##### Lexer ###### #import lex import decimal tokens = ( 'DEF', 'IF', 'NAME', 'NUMBER', # Python decimals 'STRING', # single quoted strings only; syntax of raw strings 'LPAR', 'RPAR', 'COLON', 'EQ', 'ASSIGN', 'LT', 'GT', 'PLUS', 'MINUS', 'MULT', 'DIV', 'RETURN', 'WS', 'NEWLINE', 'COMMA', 'SEMICOLON', 'INDENT', 'DEDENT', 'ENDMARKER', ) #t_NUMBER = r'\d+' # taken from decmial.py but without the leading sign def t_NUMBER(t): r"""(\d+(\.\d*)?|\.\d+)([eE][-+]? \d+)?""" t.value = decimal.Decimal(t.value) return t def t_STRING(t): r"'([^\\']+|\\'|\\\\)*'" # I think this is right ... t.value = t.value[1:-1].decode("string-escape") # .swapcase() # for fun return t t_COLON = r':' t_EQ = r'==' t_ASSIGN = r'=' t_LT = r'<' t_GT = r'>' t_PLUS = r'\+' t_MINUS = r'-' t_MULT = r'\*' t_DIV = r'/' t_COMMA = r',' t_SEMICOLON = r';' # Ply nicely documented how to do this. RESERVED = { "def": "DEF", "if": "IF", "return": "RETURN", } def t_NAME(t): r'[a-zA-Z_][a-zA-Z0-9_]*' t.type = RESERVED.get(t.value, "NAME") return t # Putting this before t_WS let it consume lines with only comments in # them so the latter code never sees the WS part. Not consuming the # newline. Needed for "if 1: #comment" def t_comment(t): r"[ ]*\043[^\n]*" # \043 is '#' pass # Whitespace def t_WS(t): r' [ ]+ ' if t.lexer.at_line_start and t.lexer.paren_count == 0: return t # Don't generate newline tokens when inside of parenthesis, eg # a = (1, # 2, 3) def t_newline(t): r'\n+' t.lexer.lineno += len(t.value) t.type = "NEWLINE" if t.lexer.paren_count == 0: return t def t_LPAR(t): r'\(' t.lexer.paren_count += 1 return t def t_RPAR(t): r'\)' # check for underflow? should be the job of the parser t.lexer.paren_count -= 1 return t def t_error(t): raise SyntaxError("Unknown symbol %r" % (t.value[0],)) print "Skipping", repr(t.value[0]) t.lexer.skip(1) # I implemented INDENT / DEDENT generation as a post-processing filter # The original lex token stream contains WS and NEWLINE characters. # WS will only occur before any other tokens on a line. # I have three filters. One tags tokens by adding two attributes. # "must_indent" is True if the token must be indented from the # previous code. The other is "at_line_start" which is True for WS # and the first non-WS/non-NEWLINE on a line. It flags the check so # see if the new line has changed indication level. # Python's syntax has three INDENT states # 0) no colon hence no need to indent # 1) "if 1: go()" - simple statements have a COLON but no need for an indent # 2) "if 1:\n go()" - complex statements have a COLON NEWLINE and must indent NO_INDENT = 0 MAY_INDENT = 1 MUST_INDENT = 2 # only care about whitespace at the start of a line def track_tokens_filter(lexer, tokens): lexer.at_line_start = at_line_start = True indent = NO_INDENT saw_colon = False for token in tokens: token.at_line_start = at_line_start if token.type == "COLON": at_line_start = False indent = MAY_INDENT token.must_indent = False elif token.type == "NEWLINE": at_line_start = True if indent == MAY_INDENT: indent = MUST_INDENT token.must_indent = False elif token.type == "WS": assert token.at_line_start == True at_line_start = True token.must_indent = False else: # A real token; only indent after COLON NEWLINE if indent == MUST_INDENT: token.must_indent = True else: token.must_indent = False at_line_start = False indent = NO_INDENT yield token lexer.at_line_start = at_line_start def _new_token(type, lineno): tok = lex.LexToken() tok.type = type tok.value = None tok.lineno = lineno return tok # Synthesize a DEDENT tag def DEDENT(lineno): return _new_token("DEDENT", lineno) # Synthesize an INDENT tag def INDENT(lineno): return _new_token("INDENT", lineno) # Track the indentation level and emit the right INDENT / DEDENT events. def indentation_filter(tokens): # A stack of indentation levels; will never pop item 0 levels = [0] token = None depth = 0 prev_was_ws = False for token in tokens: # if 1: # print "Process", token, # if token.at_line_start: # print "at_line_start", # if token.must_indent: # print "must_indent", # print # WS only occurs at the start of the line # There may be WS followed by NEWLINE so # only track the depth here. Don't indent/dedent # until there's something real. if token.type == "WS": assert depth == 0 depth = len(token.value) prev_was_ws = True # WS tokens are never passed to the parser continue if token.type == "NEWLINE": depth = 0 if prev_was_ws or token.at_line_start: # ignore blank lines continue # pass the other cases on through yield token continue # then it must be a real token (not WS, not NEWLINE) # which can affect the indentation level prev_was_ws = False if token.must_indent: # The current depth must be larger than the previous level if not (depth > levels[-1]): raise IndentationError("expected an indented block") levels.append(depth) yield INDENT(token.lineno) elif token.at_line_start: # Must be on the same level or one of the previous levels if depth == levels[-1]: # At the same level pass elif depth > levels[-1]: raise IndentationError( "indentation increase but not in new block") else: # Back up; but only if it matches a previous level try: i = levels.index(depth) except ValueError: raise IndentationError("inconsistent indentation") for _ in range(i + 1, len(levels)): yield DEDENT(token.lineno) levels.pop() yield token ### Finished processing ### # Must dedent any remaining levels if len(levels) > 1: assert token is not None for _ in range(1, len(levels)): yield DEDENT(token.lineno) # The top-level filter adds an ENDMARKER, if requested. # Python's grammar uses it. def filter(lexer, add_endmarker=True): token = None tokens = iter(lexer.token, None) tokens = track_tokens_filter(lexer, tokens) for token in indentation_filter(tokens): yield token if add_endmarker: lineno = 1 if token is not None: lineno = token.lineno yield _new_token("ENDMARKER", lineno) # Combine Ply and my filters into a new lexer class IndentLexer(object): def __init__(self, debug=0, optimize=0, lextab='lextab', reflags=0): self.lexer = lex.lex(debug=debug, optimize=optimize, lextab=lextab, reflags=reflags) self.token_stream = None def input(self, s, add_endmarker=True): self.lexer.paren_count = 0 self.lexer.input(s) self.token_stream = filter(self.lexer, add_endmarker) def token(self): try: return self.token_stream.next() except StopIteration: return None ########## Parser (tokens -> AST) ###### # also part of Ply #import yacc # I use the Python AST from compiler import ast # Helper function def Assign(left, right): names = [] if isinstance(left, ast.Name): # Single assignment on left return ast.Assign([ast.AssName(left.name, 'OP_ASSIGN')], right) elif isinstance(left, ast.Tuple): # List of things - make sure they are Name nodes names = [] for child in left.getChildren(): if not isinstance(child, ast.Name): raise SyntaxError("that assignment not supported") names.append(child.name) ass_list = [ast.AssName(name, 'OP_ASSIGN') for name in names] return ast.Assign([ast.AssTuple(ass_list)], right) else: raise SyntaxError("Can't do that yet") # The grammar comments come from Python's Grammar/Grammar file # NB: compound_stmt in single_input is followed by extra NEWLINE! # file_input: (NEWLINE | stmt)* ENDMARKER def p_file_input_end(p): """file_input_end : file_input ENDMARKER""" p[0] = ast.Stmt(p[1]) def p_file_input(p): """file_input : file_input NEWLINE | file_input stmt | NEWLINE | stmt""" if isinstance(p[len(p) - 1], basestring): if len(p) == 3: p[0] = p[1] else: p[0] = [] # p == 2 --> only a blank line else: if len(p) == 3: p[0] = p[1] + p[2] else: p[0] = p[1] # funcdef: [decorators] 'def' NAME parameters ':' suite # ignoring decorators def p_funcdef(p): "funcdef : DEF NAME parameters COLON suite" p[0] = ast.Function(None, p[2], tuple(p[3]), (), 0, None, p[5]) # parameters: '(' [varargslist] ')' def p_parameters(p): """parameters : LPAR RPAR | LPAR varargslist RPAR""" if len(p) == 3: p[0] = [] else: p[0] = p[2] # varargslist: (fpdef ['=' test] ',')* ('*' NAME [',' '**' NAME] | '**' NAME) | # highly simplified def p_varargslist(p): """varargslist : varargslist COMMA NAME | NAME""" if len(p) == 4: p[0] = p[1] + p[3] else: p[0] = [p[1]] # stmt: simple_stmt | compound_stmt def p_stmt_simple(p): """stmt : simple_stmt""" # simple_stmt is a list p[0] = p[1] def p_stmt_compound(p): """stmt : compound_stmt""" p[0] = [p[1]] # simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE def p_simple_stmt(p): """simple_stmt : small_stmts NEWLINE | small_stmts SEMICOLON NEWLINE""" p[0] = p[1] def p_small_stmts(p): """small_stmts : small_stmts SEMICOLON small_stmt | small_stmt""" if len(p) == 4: p[0] = p[1] + [p[3]] else: p[0] = [p[1]] # small_stmt: expr_stmt | print_stmt | del_stmt | pass_stmt | flow_stmt | # import_stmt | global_stmt | exec_stmt | assert_stmt def p_small_stmt(p): """small_stmt : flow_stmt | expr_stmt""" p[0] = p[1] # expr_stmt: testlist (augassign (yield_expr|testlist) | # ('=' (yield_expr|testlist))*) # augassign: ('+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' | # '<<=' | '>>=' | '**=' | '//=') def p_expr_stmt(p): """expr_stmt : testlist ASSIGN testlist | testlist """ if len(p) == 2: # a list of expressions p[0] = ast.Discard(p[1]) else: p[0] = Assign(p[1], p[3]) def p_flow_stmt(p): "flow_stmt : return_stmt" p[0] = p[1] # return_stmt: 'return' [testlist] def p_return_stmt(p): "return_stmt : RETURN testlist" p[0] = ast.Return(p[2]) def p_compound_stmt(p): """compound_stmt : if_stmt | funcdef""" p[0] = p[1] def p_if_stmt(p): 'if_stmt : IF test COLON suite' p[0] = ast.If([(p[2], p[4])], None) def p_suite(p): """suite : simple_stmt | NEWLINE INDENT stmts DEDENT""" if len(p) == 2: p[0] = ast.Stmt(p[1]) else: p[0] = ast.Stmt(p[3]) def p_stmts(p): """stmts : stmts stmt | stmt""" if len(p) == 3: p[0] = p[1] + p[2] else: p[0] = p[1] # No using Python's approach because Ply supports precedence # comparison: expr (comp_op expr)* # arith_expr: term (('+'|'-') term)* # term: factor (('*'|'/'|'%'|'//') factor)* # factor: ('+'|'-'|'~') factor | power # comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not' def make_lt_compare((left, right)): return ast.Compare(left, [('<', right), ]) def make_gt_compare((left, right)): return ast.Compare(left, [('>', right), ]) def make_eq_compare((left, right)): return ast.Compare(left, [('==', right), ]) binary_ops = { "+": ast.Add, "-": ast.Sub, "*": ast.Mul, "/": ast.Div, "<": make_lt_compare, ">": make_gt_compare, "==": make_eq_compare, } unary_ops = { "+": ast.UnaryAdd, "-": ast.UnarySub, } precedence = ( ("left", "EQ", "GT", "LT"), ("left", "PLUS", "MINUS"), ("left", "MULT", "DIV"), ) def p_comparison(p): """comparison : comparison PLUS comparison | comparison MINUS comparison | comparison MULT comparison | comparison DIV comparison | comparison LT comparison | comparison EQ comparison | comparison GT comparison | PLUS comparison | MINUS comparison | power""" if len(p) == 4: p[0] = binary_ops[p[2]]((p[1], p[3])) elif len(p) == 3: p[0] = unary_ops[p[1]](p[2]) else: p[0] = p[1] # power: atom trailer* ['**' factor] # trailers enables function calls. I only allow one level of calls # so this is 'trailer' def p_power(p): """power : atom | atom trailer""" if len(p) == 2: p[0] = p[1] else: if p[2][0] == "CALL": p[0] = ast.CallFunc(p[1], p[2][1], None, None) else: raise AssertionError("not implemented") def p_atom_name(p): """atom : NAME""" p[0] = ast.Name(p[1]) def p_atom_number(p): """atom : NUMBER | STRING""" p[0] = ast.Const(p[1]) def p_atom_tuple(p): """atom : LPAR testlist RPAR""" p[0] = p[2] # trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME def p_trailer(p): "trailer : LPAR arglist RPAR" p[0] = ("CALL", p[2]) # testlist: test (',' test)* [','] # Contains shift/reduce error def p_testlist(p): """testlist : testlist_multi COMMA | testlist_multi """ if len(p) == 2: p[0] = p[1] else: # May need to promote singleton to tuple if isinstance(p[1], list): p[0] = p[1] else: p[0] = [p[1]] # Convert into a tuple? if isinstance(p[0], list): p[0] = ast.Tuple(p[0]) def p_testlist_multi(p): """testlist_multi : testlist_multi COMMA test | test""" if len(p) == 2: # singleton p[0] = p[1] else: if isinstance(p[1], list): p[0] = p[1] + [p[3]] else: # singleton -> tuple p[0] = [p[1], p[3]] # test: or_test ['if' or_test 'else' test] | lambdef # as I don't support 'and', 'or', and 'not' this works down to 'comparison' def p_test(p): "test : comparison" p[0] = p[1] # arglist: (argument ',')* (argument [',']| '*' test [',' '**' test] | '**' test) # XXX INCOMPLETE: this doesn't allow the trailing comma def p_arglist(p): """arglist : arglist COMMA argument | argument""" if len(p) == 4: p[0] = p[1] + [p[3]] else: p[0] = [p[1]] # argument: test [gen_for] | test '=' test # Really [keyword '='] test def p_argument(p): "argument : test" p[0] = p[1] def p_error(p): # print "Error!", repr(p) raise SyntaxError(p) class GardenSnakeParser(object): def __init__(self, lexer=None): if lexer is None: lexer = IndentLexer() self.lexer = lexer self.parser = yacc.yacc(start="file_input_end") def parse(self, code): self.lexer.input(code) result = self.parser.parse(lexer=self.lexer) return ast.Module(None, result) ###### Code generation ###### from compiler import misc, syntax, pycodegen class GardenSnakeCompiler(object): def __init__(self): self.parser = GardenSnakeParser() def compile(self, code, filename=""): tree = self.parser.parse(code) # print tree misc.set_filename(filename, tree) syntax.check(tree) gen = pycodegen.ModuleCodeGenerator(tree) code = gen.getCode() return code ####### Test code ####### compile = GardenSnakeCompiler().compile code = r""" print('LET\'S TRY THIS \\OUT') #Comment here def x(a): print('called with',a) if a == 1: return 2 if a*2 > 10: return 999 / 4 # Another comment here return a+2*3 ints = (1, 2, 3, 4, 5) print('mutiline-expression', ints) t = 4+1/3*2+6*(9-5+1) print('predence test; should be 34+2/3:', t, t==(34+2/3)) print('numbers', 1,2,3,4,5) if 1: 8 a=9 print(x(a)) print(x(1)) print(x(2)) print(x(8),'3') print('this is decimal', 1/5) print('BIG DECIMAL', 1.234567891234567e12345) """ # Set up the GardenSnake run-time environment def print_(*args): print "-->", " ".join(map(str, args)) globals()["print"] = print_ compiled_code = compile(code) exec compiled_code in globals() print "Done" ================================================ FILE: xonsh/ply/example/GardenSnake/README ================================================ This example is Andrew Dalke's GardenSnake language. It shows how to process an indentation-like language like Python. Further details can be found here: http://dalkescientific.com/writings/diary/archive/2006/08/30/gardensnake_language.html ================================================ FILE: xonsh/ply/example/README ================================================ Simple examples: calc - Simple calculator classcalc - Simple calculate defined as a class Complex examples ansic - ANSI C grammar from K&R BASIC - A small BASIC interpreter GardenSnake - A simple python-like language yply - Converts Unix yacc files to PLY programs. ================================================ FILE: xonsh/ply/example/ansic/README ================================================ This example is incomplete. Was going to specify an ANSI C parser. This is part of it. ================================================ FILE: xonsh/ply/example/ansic/clex.py ================================================ # ---------------------------------------------------------------------- # clex.py # # A lexer for ANSI C. # ---------------------------------------------------------------------- import sys sys.path.insert(0, "../..") import ply.lex as lex # Reserved words reserved = ( 'AUTO', 'BREAK', 'CASE', 'CHAR', 'CONST', 'CONTINUE', 'DEFAULT', 'DO', 'DOUBLE', 'ELSE', 'ENUM', 'EXTERN', 'FLOAT', 'FOR', 'GOTO', 'IF', 'INT', 'LONG', 'REGISTER', 'RETURN', 'SHORT', 'SIGNED', 'SIZEOF', 'STATIC', 'STRUCT', 'SWITCH', 'TYPEDEF', 'UNION', 'UNSIGNED', 'VOID', 'VOLATILE', 'WHILE', ) tokens = reserved + ( # Literals (identifier, integer constant, float constant, string constant, # char const) 'ID', 'TYPEID', 'ICONST', 'FCONST', 'SCONST', 'CCONST', # Operators (+,-,*,/,%,|,&,~,^,<<,>>, ||, &&, !, <, <=, >, >=, ==, !=) 'PLUS', 'MINUS', 'TIMES', 'DIVIDE', 'MOD', 'OR', 'AND', 'NOT', 'XOR', 'LSHIFT', 'RSHIFT', 'LOR', 'LAND', 'LNOT', 'LT', 'LE', 'GT', 'GE', 'EQ', 'NE', # Assignment (=, *=, /=, %=, +=, -=, <<=, >>=, &=, ^=, |=) 'EQUALS', 'TIMESEQUAL', 'DIVEQUAL', 'MODEQUAL', 'PLUSEQUAL', 'MINUSEQUAL', 'LSHIFTEQUAL', 'RSHIFTEQUAL', 'ANDEQUAL', 'XOREQUAL', 'OREQUAL', # Increment/decrement (++,--) 'PLUSPLUS', 'MINUSMINUS', # Structure dereference (->) 'ARROW', # Conditional operator (?) 'CONDOP', # Delimeters ( ) [ ] { } , . ; : 'LPAREN', 'RPAREN', 'LBRACKET', 'RBRACKET', 'LBRACE', 'RBRACE', 'COMMA', 'PERIOD', 'SEMI', 'COLON', # Ellipsis (...) 'ELLIPSIS', ) # Completely ignored characters t_ignore = ' \t\x0c' # Newlines def t_NEWLINE(t): r'\n+' t.lexer.lineno += t.value.count("\n") # Operators t_PLUS = r'\+' t_MINUS = r'-' t_TIMES = r'\*' t_DIVIDE = r'/' t_MOD = r'%' t_OR = r'\|' t_AND = r'&' t_NOT = r'~' t_XOR = r'\^' t_LSHIFT = r'<<' t_RSHIFT = r'>>' t_LOR = r'\|\|' t_LAND = r'&&' t_LNOT = r'!' t_LT = r'<' t_GT = r'>' t_LE = r'<=' t_GE = r'>=' t_EQ = r'==' t_NE = r'!=' # Assignment operators t_EQUALS = r'=' t_TIMESEQUAL = r'\*=' t_DIVEQUAL = r'/=' t_MODEQUAL = r'%=' t_PLUSEQUAL = r'\+=' t_MINUSEQUAL = r'-=' t_LSHIFTEQUAL = r'<<=' t_RSHIFTEQUAL = r'>>=' t_ANDEQUAL = r'&=' t_OREQUAL = r'\|=' t_XOREQUAL = r'\^=' # Increment/decrement t_PLUSPLUS = r'\+\+' t_MINUSMINUS = r'--' # -> t_ARROW = r'->' # ? t_CONDOP = r'\?' # Delimeters t_LPAREN = r'\(' t_RPAREN = r'\)' t_LBRACKET = r'\[' t_RBRACKET = r'\]' t_LBRACE = r'\{' t_RBRACE = r'\}' t_COMMA = r',' t_PERIOD = r'\.' t_SEMI = r';' t_COLON = r':' t_ELLIPSIS = r'\.\.\.' # Identifiers and reserved words reserved_map = {} for r in reserved: reserved_map[r.lower()] = r def t_ID(t): r'[A-Za-z_][\w_]*' t.type = reserved_map.get(t.value, "ID") return t # Integer literal t_ICONST = r'\d+([uU]|[lL]|[uU][lL]|[lL][uU])?' # Floating literal t_FCONST = r'((\d+)(\.\d+)(e(\+|-)?(\d+))? | (\d+)e(\+|-)?(\d+))([lL]|[fF])?' # String literal t_SCONST = r'\"([^\\\n]|(\\.))*?\"' # Character constant 'c' or L'c' t_CCONST = r'(L)?\'([^\\\n]|(\\.))*?\'' # Comments def t_comment(t): r'/\*(.|\n)*?\*/' t.lexer.lineno += t.value.count('\n') # Preprocessor directive (ignored) def t_preprocessor(t): r'\#(.)*?\n' t.lexer.lineno += 1 def t_error(t): print("Illegal character %s" % repr(t.value[0])) t.lexer.skip(1) lexer = lex.lex() if __name__ == "__main__": lex.runmain(lexer) ================================================ FILE: xonsh/ply/example/ansic/cparse.py ================================================ # ----------------------------------------------------------------------------- # cparse.py # # Simple parser for ANSI C. Based on the grammar in K&R, 2nd Ed. # ----------------------------------------------------------------------------- import sys import clex import ply.yacc as yacc # Get the token map tokens = clex.tokens # translation-unit: def p_translation_unit_1(t): 'translation_unit : external_declaration' pass def p_translation_unit_2(t): 'translation_unit : translation_unit external_declaration' pass # external-declaration: def p_external_declaration_1(t): 'external_declaration : function_definition' pass def p_external_declaration_2(t): 'external_declaration : declaration' pass # function-definition: def p_function_definition_1(t): 'function_definition : declaration_specifiers declarator declaration_list compound_statement' pass def p_function_definition_2(t): 'function_definition : declarator declaration_list compound_statement' pass def p_function_definition_3(t): 'function_definition : declarator compound_statement' pass def p_function_definition_4(t): 'function_definition : declaration_specifiers declarator compound_statement' pass # declaration: def p_declaration_1(t): 'declaration : declaration_specifiers init_declarator_list SEMI' pass def p_declaration_2(t): 'declaration : declaration_specifiers SEMI' pass # declaration-list: def p_declaration_list_1(t): 'declaration_list : declaration' pass def p_declaration_list_2(t): 'declaration_list : declaration_list declaration ' pass # declaration-specifiers def p_declaration_specifiers_1(t): 'declaration_specifiers : storage_class_specifier declaration_specifiers' pass def p_declaration_specifiers_2(t): 'declaration_specifiers : type_specifier declaration_specifiers' pass def p_declaration_specifiers_3(t): 'declaration_specifiers : type_qualifier declaration_specifiers' pass def p_declaration_specifiers_4(t): 'declaration_specifiers : storage_class_specifier' pass def p_declaration_specifiers_5(t): 'declaration_specifiers : type_specifier' pass def p_declaration_specifiers_6(t): 'declaration_specifiers : type_qualifier' pass # storage-class-specifier def p_storage_class_specifier(t): '''storage_class_specifier : AUTO | REGISTER | STATIC | EXTERN | TYPEDEF ''' pass # type-specifier: def p_type_specifier(t): '''type_specifier : VOID | CHAR | SHORT | INT | LONG | FLOAT | DOUBLE | SIGNED | UNSIGNED | struct_or_union_specifier | enum_specifier | TYPEID ''' pass # type-qualifier: def p_type_qualifier(t): '''type_qualifier : CONST | VOLATILE''' pass # struct-or-union-specifier def p_struct_or_union_specifier_1(t): 'struct_or_union_specifier : struct_or_union ID LBRACE struct_declaration_list RBRACE' pass def p_struct_or_union_specifier_2(t): 'struct_or_union_specifier : struct_or_union LBRACE struct_declaration_list RBRACE' pass def p_struct_or_union_specifier_3(t): 'struct_or_union_specifier : struct_or_union ID' pass # struct-or-union: def p_struct_or_union(t): '''struct_or_union : STRUCT | UNION ''' pass # struct-declaration-list: def p_struct_declaration_list_1(t): 'struct_declaration_list : struct_declaration' pass def p_struct_declaration_list_2(t): 'struct_declaration_list : struct_declaration_list struct_declaration' pass # init-declarator-list: def p_init_declarator_list_1(t): 'init_declarator_list : init_declarator' pass def p_init_declarator_list_2(t): 'init_declarator_list : init_declarator_list COMMA init_declarator' pass # init-declarator def p_init_declarator_1(t): 'init_declarator : declarator' pass def p_init_declarator_2(t): 'init_declarator : declarator EQUALS initializer' pass # struct-declaration: def p_struct_declaration(t): 'struct_declaration : specifier_qualifier_list struct_declarator_list SEMI' pass # specifier-qualifier-list: def p_specifier_qualifier_list_1(t): 'specifier_qualifier_list : type_specifier specifier_qualifier_list' pass def p_specifier_qualifier_list_2(t): 'specifier_qualifier_list : type_specifier' pass def p_specifier_qualifier_list_3(t): 'specifier_qualifier_list : type_qualifier specifier_qualifier_list' pass def p_specifier_qualifier_list_4(t): 'specifier_qualifier_list : type_qualifier' pass # struct-declarator-list: def p_struct_declarator_list_1(t): 'struct_declarator_list : struct_declarator' pass def p_struct_declarator_list_2(t): 'struct_declarator_list : struct_declarator_list COMMA struct_declarator' pass # struct-declarator: def p_struct_declarator_1(t): 'struct_declarator : declarator' pass def p_struct_declarator_2(t): 'struct_declarator : declarator COLON constant_expression' pass def p_struct_declarator_3(t): 'struct_declarator : COLON constant_expression' pass # enum-specifier: def p_enum_specifier_1(t): 'enum_specifier : ENUM ID LBRACE enumerator_list RBRACE' pass def p_enum_specifier_2(t): 'enum_specifier : ENUM LBRACE enumerator_list RBRACE' pass def p_enum_specifier_3(t): 'enum_specifier : ENUM ID' pass # enumerator_list: def p_enumerator_list_1(t): 'enumerator_list : enumerator' pass def p_enumerator_list_2(t): 'enumerator_list : enumerator_list COMMA enumerator' pass # enumerator: def p_enumerator_1(t): 'enumerator : ID' pass def p_enumerator_2(t): 'enumerator : ID EQUALS constant_expression' pass # declarator: def p_declarator_1(t): 'declarator : pointer direct_declarator' pass def p_declarator_2(t): 'declarator : direct_declarator' pass # direct-declarator: def p_direct_declarator_1(t): 'direct_declarator : ID' pass def p_direct_declarator_2(t): 'direct_declarator : LPAREN declarator RPAREN' pass def p_direct_declarator_3(t): 'direct_declarator : direct_declarator LBRACKET constant_expression_opt RBRACKET' pass def p_direct_declarator_4(t): 'direct_declarator : direct_declarator LPAREN parameter_type_list RPAREN ' pass def p_direct_declarator_5(t): 'direct_declarator : direct_declarator LPAREN identifier_list RPAREN ' pass def p_direct_declarator_6(t): 'direct_declarator : direct_declarator LPAREN RPAREN ' pass # pointer: def p_pointer_1(t): 'pointer : TIMES type_qualifier_list' pass def p_pointer_2(t): 'pointer : TIMES' pass def p_pointer_3(t): 'pointer : TIMES type_qualifier_list pointer' pass def p_pointer_4(t): 'pointer : TIMES pointer' pass # type-qualifier-list: def p_type_qualifier_list_1(t): 'type_qualifier_list : type_qualifier' pass def p_type_qualifier_list_2(t): 'type_qualifier_list : type_qualifier_list type_qualifier' pass # parameter-type-list: def p_parameter_type_list_1(t): 'parameter_type_list : parameter_list' pass def p_parameter_type_list_2(t): 'parameter_type_list : parameter_list COMMA ELLIPSIS' pass # parameter-list: def p_parameter_list_1(t): 'parameter_list : parameter_declaration' pass def p_parameter_list_2(t): 'parameter_list : parameter_list COMMA parameter_declaration' pass # parameter-declaration: def p_parameter_declaration_1(t): 'parameter_declaration : declaration_specifiers declarator' pass def p_parameter_declaration_2(t): 'parameter_declaration : declaration_specifiers abstract_declarator_opt' pass # identifier-list: def p_identifier_list_1(t): 'identifier_list : ID' pass def p_identifier_list_2(t): 'identifier_list : identifier_list COMMA ID' pass # initializer: def p_initializer_1(t): 'initializer : assignment_expression' pass def p_initializer_2(t): '''initializer : LBRACE initializer_list RBRACE | LBRACE initializer_list COMMA RBRACE''' pass # initializer-list: def p_initializer_list_1(t): 'initializer_list : initializer' pass def p_initializer_list_2(t): 'initializer_list : initializer_list COMMA initializer' pass # type-name: def p_type_name(t): 'type_name : specifier_qualifier_list abstract_declarator_opt' pass def p_abstract_declarator_opt_1(t): 'abstract_declarator_opt : empty' pass def p_abstract_declarator_opt_2(t): 'abstract_declarator_opt : abstract_declarator' pass # abstract-declarator: def p_abstract_declarator_1(t): 'abstract_declarator : pointer ' pass def p_abstract_declarator_2(t): 'abstract_declarator : pointer direct_abstract_declarator' pass def p_abstract_declarator_3(t): 'abstract_declarator : direct_abstract_declarator' pass # direct-abstract-declarator: def p_direct_abstract_declarator_1(t): 'direct_abstract_declarator : LPAREN abstract_declarator RPAREN' pass def p_direct_abstract_declarator_2(t): 'direct_abstract_declarator : direct_abstract_declarator LBRACKET constant_expression_opt RBRACKET' pass def p_direct_abstract_declarator_3(t): 'direct_abstract_declarator : LBRACKET constant_expression_opt RBRACKET' pass def p_direct_abstract_declarator_4(t): 'direct_abstract_declarator : direct_abstract_declarator LPAREN parameter_type_list_opt RPAREN' pass def p_direct_abstract_declarator_5(t): 'direct_abstract_declarator : LPAREN parameter_type_list_opt RPAREN' pass # Optional fields in abstract declarators def p_constant_expression_opt_1(t): 'constant_expression_opt : empty' pass def p_constant_expression_opt_2(t): 'constant_expression_opt : constant_expression' pass def p_parameter_type_list_opt_1(t): 'parameter_type_list_opt : empty' pass def p_parameter_type_list_opt_2(t): 'parameter_type_list_opt : parameter_type_list' pass # statement: def p_statement(t): ''' statement : labeled_statement | expression_statement | compound_statement | selection_statement | iteration_statement | jump_statement ''' pass # labeled-statement: def p_labeled_statement_1(t): 'labeled_statement : ID COLON statement' pass def p_labeled_statement_2(t): 'labeled_statement : CASE constant_expression COLON statement' pass def p_labeled_statement_3(t): 'labeled_statement : DEFAULT COLON statement' pass # expression-statement: def p_expression_statement(t): 'expression_statement : expression_opt SEMI' pass # compound-statement: def p_compound_statement_1(t): 'compound_statement : LBRACE declaration_list statement_list RBRACE' pass def p_compound_statement_2(t): 'compound_statement : LBRACE statement_list RBRACE' pass def p_compound_statement_3(t): 'compound_statement : LBRACE declaration_list RBRACE' pass def p_compound_statement_4(t): 'compound_statement : LBRACE RBRACE' pass # statement-list: def p_statement_list_1(t): 'statement_list : statement' pass def p_statement_list_2(t): 'statement_list : statement_list statement' pass # selection-statement def p_selection_statement_1(t): 'selection_statement : IF LPAREN expression RPAREN statement' pass def p_selection_statement_2(t): 'selection_statement : IF LPAREN expression RPAREN statement ELSE statement ' pass def p_selection_statement_3(t): 'selection_statement : SWITCH LPAREN expression RPAREN statement ' pass # iteration_statement: def p_iteration_statement_1(t): 'iteration_statement : WHILE LPAREN expression RPAREN statement' pass def p_iteration_statement_2(t): 'iteration_statement : FOR LPAREN expression_opt SEMI expression_opt SEMI expression_opt RPAREN statement ' pass def p_iteration_statement_3(t): 'iteration_statement : DO statement WHILE LPAREN expression RPAREN SEMI' pass # jump_statement: def p_jump_statement_1(t): 'jump_statement : GOTO ID SEMI' pass def p_jump_statement_2(t): 'jump_statement : CONTINUE SEMI' pass def p_jump_statement_3(t): 'jump_statement : BREAK SEMI' pass def p_jump_statement_4(t): 'jump_statement : RETURN expression_opt SEMI' pass def p_expression_opt_1(t): 'expression_opt : empty' pass def p_expression_opt_2(t): 'expression_opt : expression' pass # expression: def p_expression_1(t): 'expression : assignment_expression' pass def p_expression_2(t): 'expression : expression COMMA assignment_expression' pass # assigment_expression: def p_assignment_expression_1(t): 'assignment_expression : conditional_expression' pass def p_assignment_expression_2(t): 'assignment_expression : unary_expression assignment_operator assignment_expression' pass # assignment_operator: def p_assignment_operator(t): ''' assignment_operator : EQUALS | TIMESEQUAL | DIVEQUAL | MODEQUAL | PLUSEQUAL | MINUSEQUAL | LSHIFTEQUAL | RSHIFTEQUAL | ANDEQUAL | OREQUAL | XOREQUAL ''' pass # conditional-expression def p_conditional_expression_1(t): 'conditional_expression : logical_or_expression' pass def p_conditional_expression_2(t): 'conditional_expression : logical_or_expression CONDOP expression COLON conditional_expression ' pass # constant-expression def p_constant_expression(t): 'constant_expression : conditional_expression' pass # logical-or-expression def p_logical_or_expression_1(t): 'logical_or_expression : logical_and_expression' pass def p_logical_or_expression_2(t): 'logical_or_expression : logical_or_expression LOR logical_and_expression' pass # logical-and-expression def p_logical_and_expression_1(t): 'logical_and_expression : inclusive_or_expression' pass def p_logical_and_expression_2(t): 'logical_and_expression : logical_and_expression LAND inclusive_or_expression' pass # inclusive-or-expression: def p_inclusive_or_expression_1(t): 'inclusive_or_expression : exclusive_or_expression' pass def p_inclusive_or_expression_2(t): 'inclusive_or_expression : inclusive_or_expression OR exclusive_or_expression' pass # exclusive-or-expression: def p_exclusive_or_expression_1(t): 'exclusive_or_expression : and_expression' pass def p_exclusive_or_expression_2(t): 'exclusive_or_expression : exclusive_or_expression XOR and_expression' pass # AND-expression def p_and_expression_1(t): 'and_expression : equality_expression' pass def p_and_expression_2(t): 'and_expression : and_expression AND equality_expression' pass # equality-expression: def p_equality_expression_1(t): 'equality_expression : relational_expression' pass def p_equality_expression_2(t): 'equality_expression : equality_expression EQ relational_expression' pass def p_equality_expression_3(t): 'equality_expression : equality_expression NE relational_expression' pass # relational-expression: def p_relational_expression_1(t): 'relational_expression : shift_expression' pass def p_relational_expression_2(t): 'relational_expression : relational_expression LT shift_expression' pass def p_relational_expression_3(t): 'relational_expression : relational_expression GT shift_expression' pass def p_relational_expression_4(t): 'relational_expression : relational_expression LE shift_expression' pass def p_relational_expression_5(t): 'relational_expression : relational_expression GE shift_expression' pass # shift-expression def p_shift_expression_1(t): 'shift_expression : additive_expression' pass def p_shift_expression_2(t): 'shift_expression : shift_expression LSHIFT additive_expression' pass def p_shift_expression_3(t): 'shift_expression : shift_expression RSHIFT additive_expression' pass # additive-expression def p_additive_expression_1(t): 'additive_expression : multiplicative_expression' pass def p_additive_expression_2(t): 'additive_expression : additive_expression PLUS multiplicative_expression' pass def p_additive_expression_3(t): 'additive_expression : additive_expression MINUS multiplicative_expression' pass # multiplicative-expression def p_multiplicative_expression_1(t): 'multiplicative_expression : cast_expression' pass def p_multiplicative_expression_2(t): 'multiplicative_expression : multiplicative_expression TIMES cast_expression' pass def p_multiplicative_expression_3(t): 'multiplicative_expression : multiplicative_expression DIVIDE cast_expression' pass def p_multiplicative_expression_4(t): 'multiplicative_expression : multiplicative_expression MOD cast_expression' pass # cast-expression: def p_cast_expression_1(t): 'cast_expression : unary_expression' pass def p_cast_expression_2(t): 'cast_expression : LPAREN type_name RPAREN cast_expression' pass # unary-expression: def p_unary_expression_1(t): 'unary_expression : postfix_expression' pass def p_unary_expression_2(t): 'unary_expression : PLUSPLUS unary_expression' pass def p_unary_expression_3(t): 'unary_expression : MINUSMINUS unary_expression' pass def p_unary_expression_4(t): 'unary_expression : unary_operator cast_expression' pass def p_unary_expression_5(t): 'unary_expression : SIZEOF unary_expression' pass def p_unary_expression_6(t): 'unary_expression : SIZEOF LPAREN type_name RPAREN' pass # unary-operator def p_unary_operator(t): '''unary_operator : AND | TIMES | PLUS | MINUS | NOT | LNOT ''' pass # postfix-expression: def p_postfix_expression_1(t): 'postfix_expression : primary_expression' pass def p_postfix_expression_2(t): 'postfix_expression : postfix_expression LBRACKET expression RBRACKET' pass def p_postfix_expression_3(t): 'postfix_expression : postfix_expression LPAREN argument_expression_list RPAREN' pass def p_postfix_expression_4(t): 'postfix_expression : postfix_expression LPAREN RPAREN' pass def p_postfix_expression_5(t): 'postfix_expression : postfix_expression PERIOD ID' pass def p_postfix_expression_6(t): 'postfix_expression : postfix_expression ARROW ID' pass def p_postfix_expression_7(t): 'postfix_expression : postfix_expression PLUSPLUS' pass def p_postfix_expression_8(t): 'postfix_expression : postfix_expression MINUSMINUS' pass # primary-expression: def p_primary_expression(t): '''primary_expression : ID | constant | SCONST | LPAREN expression RPAREN''' pass # argument-expression-list: def p_argument_expression_list(t): '''argument_expression_list : assignment_expression | argument_expression_list COMMA assignment_expression''' pass # constant: def p_constant(t): '''constant : ICONST | FCONST | CCONST''' pass def p_empty(t): 'empty : ' pass def p_error(t): print("Whoa. We're hosed") import profile # Build the grammar yacc.yacc() #yacc.yacc(method='LALR',write_tables=False,debug=False) #profile.run("yacc.yacc(method='LALR')") ================================================ FILE: xonsh/ply/example/calc/calc.py ================================================ # ----------------------------------------------------------------------------- # calc.py # # A simple calculator with variables. This is from O'Reilly's # "Lex and Yacc", p. 63. # ----------------------------------------------------------------------------- import sys sys.path.insert(0, "../..") if sys.version_info[0] >= 3: raw_input = input tokens = ( 'NAME', 'NUMBER', ) literals = ['=', '+', '-', '*', '/', '(', ')'] # Tokens t_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*' def t_NUMBER(t): r'\d+' t.value = int(t.value) return t t_ignore = " \t" def t_newline(t): r'\n+' t.lexer.lineno += t.value.count("\n") def t_error(t): print("Illegal character '%s'" % t.value[0]) t.lexer.skip(1) # Build the lexer import ply.lex as lex lex.lex() # Parsing rules precedence = ( ('left', '+', '-'), ('left', '*', '/'), ('right', 'UMINUS'), ) # dictionary of names names = {} def p_statement_assign(p): 'statement : NAME "=" expression' names[p[1]] = p[3] def p_statement_expr(p): 'statement : expression' print(p[1]) def p_expression_binop(p): '''expression : expression '+' expression | expression '-' expression | expression '*' expression | expression '/' expression''' if p[2] == '+': p[0] = p[1] + p[3] elif p[2] == '-': p[0] = p[1] - p[3] elif p[2] == '*': p[0] = p[1] * p[3] elif p[2] == '/': p[0] = p[1] / p[3] def p_expression_uminus(p): "expression : '-' expression %prec UMINUS" p[0] = -p[2] def p_expression_group(p): "expression : '(' expression ')'" p[0] = p[2] def p_expression_number(p): "expression : NUMBER" p[0] = p[1] def p_expression_name(p): "expression : NAME" try: p[0] = names[p[1]] except LookupError: print("Undefined name '%s'" % p[1]) p[0] = 0 def p_error(p): if p: print("Syntax error at '%s'" % p.value) else: print("Syntax error at EOF") import ply.yacc as yacc yacc.yacc() while 1: try: s = raw_input('calc > ') except EOFError: break if not s: continue yacc.parse(s) ================================================ FILE: xonsh/ply/example/calcdebug/calc.py ================================================ # ----------------------------------------------------------------------------- # calc.py # # This example shows how to run the parser in a debugging mode # with output routed to a logging object. # ----------------------------------------------------------------------------- import sys sys.path.insert(0, "../..") if sys.version_info[0] >= 3: raw_input = input tokens = ( 'NAME', 'NUMBER', ) literals = ['=', '+', '-', '*', '/', '(', ')'] # Tokens t_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*' def t_NUMBER(t): r'\d+' t.value = int(t.value) return t t_ignore = " \t" def t_newline(t): r'\n+' t.lexer.lineno += t.value.count("\n") def t_error(t): print("Illegal character '%s'" % t.value[0]) t.lexer.skip(1) # Build the lexer import ply.lex as lex lex.lex() # Parsing rules precedence = ( ('left', '+', '-'), ('left', '*', '/'), ('right', 'UMINUS'), ) # dictionary of names names = {} def p_statement_assign(p): 'statement : NAME "=" expression' names[p[1]] = p[3] def p_statement_expr(p): 'statement : expression' print(p[1]) def p_expression_binop(p): '''expression : expression '+' expression | expression '-' expression | expression '*' expression | expression '/' expression''' if p[2] == '+': p[0] = p[1] + p[3] elif p[2] == '-': p[0] = p[1] - p[3] elif p[2] == '*': p[0] = p[1] * p[3] elif p[2] == '/': p[0] = p[1] / p[3] def p_expression_uminus(p): "expression : '-' expression %prec UMINUS" p[0] = -p[2] def p_expression_group(p): "expression : '(' expression ')'" p[0] = p[2] def p_expression_number(p): "expression : NUMBER" p[0] = p[1] def p_expression_name(p): "expression : NAME" try: p[0] = names[p[1]] except LookupError: print("Undefined name '%s'" % p[1]) p[0] = 0 def p_error(p): if p: print("Syntax error at '%s'" % p.value) else: print("Syntax error at EOF") import ply.yacc as yacc yacc.yacc() import logging logging.basicConfig( level=logging.INFO, filename="parselog.txt" ) while 1: try: s = raw_input('calc > ') except EOFError: break if not s: continue yacc.parse(s, debug=logging.getLogger()) ================================================ FILE: xonsh/ply/example/calceof/calc.py ================================================ # ----------------------------------------------------------------------------- # calc.py # # A simple calculator with variables. Asks the user for more input and # demonstrates the use of the t_eof() rule. # ----------------------------------------------------------------------------- import sys sys.path.insert(0, "../..") if sys.version_info[0] >= 3: raw_input = input tokens = ( 'NAME', 'NUMBER', ) literals = ['=', '+', '-', '*', '/', '(', ')'] # Tokens t_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*' def t_NUMBER(t): r'\d+' t.value = int(t.value) return t t_ignore = " \t" def t_newline(t): r'\n+' t.lexer.lineno += t.value.count("\n") def t_eof(t): more = raw_input('... ') if more: t.lexer.input(more + '\n') return t.lexer.token() else: return None def t_error(t): print("Illegal character '%s'" % t.value[0]) t.lexer.skip(1) # Build the lexer import ply.lex as lex lex.lex() # Parsing rules precedence = ( ('left', '+', '-'), ('left', '*', '/'), ('right', 'UMINUS'), ) # dictionary of names names = {} def p_statement_assign(p): 'statement : NAME "=" expression' names[p[1]] = p[3] def p_statement_expr(p): 'statement : expression' print(p[1]) def p_expression_binop(p): '''expression : expression '+' expression | expression '-' expression | expression '*' expression | expression '/' expression''' if p[2] == '+': p[0] = p[1] + p[3] elif p[2] == '-': p[0] = p[1] - p[3] elif p[2] == '*': p[0] = p[1] * p[3] elif p[2] == '/': p[0] = p[1] / p[3] def p_expression_uminus(p): "expression : '-' expression %prec UMINUS" p[0] = -p[2] def p_expression_group(p): "expression : '(' expression ')'" p[0] = p[2] def p_expression_number(p): "expression : NUMBER" p[0] = p[1] def p_expression_name(p): "expression : NAME" try: p[0] = names[p[1]] except LookupError: print("Undefined name '%s'" % p[1]) p[0] = 0 def p_error(p): if p: print("Syntax error at '%s'" % p.value) else: print("Syntax error at EOF") import ply.yacc as yacc yacc.yacc() while 1: try: s = raw_input('calc > ') except EOFError: break if not s: continue yacc.parse(s + '\n') ================================================ FILE: xonsh/ply/example/classcalc/calc.py ================================================ #!/usr/bin/env python # ----------------------------------------------------------------------------- # calc.py # # A simple calculator with variables. This is from O'Reilly's # "Lex and Yacc", p. 63. # # Class-based example contributed to PLY by David McNab # ----------------------------------------------------------------------------- import sys sys.path.insert(0, "../..") if sys.version_info[0] >= 3: raw_input = input import ply.lex as lex import ply.yacc as yacc import os class Parser: """ Base class for a lexer/parser that has the rules defined as methods """ tokens = () precedence = () def __init__(self, **kw): self.debug = kw.get('debug', 0) self.names = {} try: modname = os.path.split(os.path.splitext(__file__)[0])[ 1] + "_" + self.__class__.__name__ except: modname = "parser" + "_" + self.__class__.__name__ self.debugfile = modname + ".dbg" self.tabmodule = modname + "_" + "parsetab" # print self.debugfile, self.tabmodule # Build the lexer and parser lex.lex(module=self, debug=self.debug) yacc.yacc(module=self, debug=self.debug, debugfile=self.debugfile, tabmodule=self.tabmodule) def run(self): while 1: try: s = raw_input('calc > ') except EOFError: break if not s: continue yacc.parse(s) class Calc(Parser): tokens = ( 'NAME', 'NUMBER', 'PLUS', 'MINUS', 'EXP', 'TIMES', 'DIVIDE', 'EQUALS', 'LPAREN', 'RPAREN', ) # Tokens t_PLUS = r'\+' t_MINUS = r'-' t_EXP = r'\*\*' t_TIMES = r'\*' t_DIVIDE = r'/' t_EQUALS = r'=' t_LPAREN = r'\(' t_RPAREN = r'\)' t_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*' def t_NUMBER(self, t): r'\d+' try: t.value = int(t.value) except ValueError: print("Integer value too large %s" % t.value) t.value = 0 # print "parsed number %s" % repr(t.value) return t t_ignore = " \t" def t_newline(self, t): r'\n+' t.lexer.lineno += t.value.count("\n") def t_error(self, t): print("Illegal character '%s'" % t.value[0]) t.lexer.skip(1) # Parsing rules precedence = ( ('left', 'PLUS', 'MINUS'), ('left', 'TIMES', 'DIVIDE'), ('left', 'EXP'), ('right', 'UMINUS'), ) def p_statement_assign(self, p): 'statement : NAME EQUALS expression' self.names[p[1]] = p[3] def p_statement_expr(self, p): 'statement : expression' print(p[1]) def p_expression_binop(self, p): """ expression : expression PLUS expression | expression MINUS expression | expression TIMES expression | expression DIVIDE expression | expression EXP expression """ # print [repr(p[i]) for i in range(0,4)] if p[2] == '+': p[0] = p[1] + p[3] elif p[2] == '-': p[0] = p[1] - p[3] elif p[2] == '*': p[0] = p[1] * p[3] elif p[2] == '/': p[0] = p[1] / p[3] elif p[2] == '**': p[0] = p[1] ** p[3] def p_expression_uminus(self, p): 'expression : MINUS expression %prec UMINUS' p[0] = -p[2] def p_expression_group(self, p): 'expression : LPAREN expression RPAREN' p[0] = p[2] def p_expression_number(self, p): 'expression : NUMBER' p[0] = p[1] def p_expression_name(self, p): 'expression : NAME' try: p[0] = self.names[p[1]] except LookupError: print("Undefined name '%s'" % p[1]) p[0] = 0 def p_error(self, p): if p: print("Syntax error at '%s'" % p.value) else: print("Syntax error at EOF") if __name__ == '__main__': calc = Calc() calc.run() ================================================ FILE: xonsh/ply/example/cleanup.sh ================================================ #!/bin/sh rm -f */*.pyc */parsetab.py */parser.out */*~ */*.class ================================================ FILE: xonsh/ply/example/closurecalc/calc.py ================================================ # ----------------------------------------------------------------------------- # calc.py # # A calculator parser that makes use of closures. The function make_calculator() # returns a function that accepts an input string and returns a result. All # lexing rules, parsing rules, and internal state are held inside the function. # ----------------------------------------------------------------------------- import sys sys.path.insert(0, "../..") if sys.version_info[0] >= 3: raw_input = input # Make a calculator function def make_calculator(): import ply.lex as lex import ply.yacc as yacc # ------- Internal calculator state variables = {} # Dictionary of stored variables # ------- Calculator tokenizing rules tokens = ( 'NAME', 'NUMBER', ) literals = ['=', '+', '-', '*', '/', '(', ')'] t_ignore = " \t" t_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*' def t_NUMBER(t): r'\d+' t.value = int(t.value) return t def t_newline(t): r'\n+' t.lexer.lineno += t.value.count("\n") def t_error(t): print("Illegal character '%s'" % t.value[0]) t.lexer.skip(1) # Build the lexer lexer = lex.lex() # ------- Calculator parsing rules precedence = ( ('left', '+', '-'), ('left', '*', '/'), ('right', 'UMINUS'), ) def p_statement_assign(p): 'statement : NAME "=" expression' variables[p[1]] = p[3] p[0] = None def p_statement_expr(p): 'statement : expression' p[0] = p[1] def p_expression_binop(p): '''expression : expression '+' expression | expression '-' expression | expression '*' expression | expression '/' expression''' if p[2] == '+': p[0] = p[1] + p[3] elif p[2] == '-': p[0] = p[1] - p[3] elif p[2] == '*': p[0] = p[1] * p[3] elif p[2] == '/': p[0] = p[1] / p[3] def p_expression_uminus(p): "expression : '-' expression %prec UMINUS" p[0] = -p[2] def p_expression_group(p): "expression : '(' expression ')'" p[0] = p[2] def p_expression_number(p): "expression : NUMBER" p[0] = p[1] def p_expression_name(p): "expression : NAME" try: p[0] = variables[p[1]] except LookupError: print("Undefined name '%s'" % p[1]) p[0] = 0 def p_error(p): if p: print("Syntax error at '%s'" % p.value) else: print("Syntax error at EOF") # Build the parser parser = yacc.yacc() # ------- Input function def input(text): result = parser.parse(text, lexer=lexer) return result return input # Make a calculator object and use it calc = make_calculator() while True: try: s = raw_input("calc > ") except EOFError: break r = calc(s) if r: print(r) ================================================ FILE: xonsh/ply/example/hedit/hedit.py ================================================ # ----------------------------------------------------------------------------- # hedit.py # # Paring of Fortran H Edit descriptions (Contributed by Pearu Peterson) # # These tokens can't be easily tokenized because they are of the following # form: # # nHc1...cn # # where n is a positive integer and c1 ... cn are characters. # # This example shows how to modify the state of the lexer to parse # such tokens # ----------------------------------------------------------------------------- import sys sys.path.insert(0, "../..") tokens = ( 'H_EDIT_DESCRIPTOR', ) # Tokens t_ignore = " \t\n" def t_H_EDIT_DESCRIPTOR(t): r"\d+H.*" # This grabs all of the remaining text i = t.value.index('H') n = eval(t.value[:i]) # Adjust the tokenizing position t.lexer.lexpos -= len(t.value) - (i + 1 + n) t.value = t.value[i + 1:i + 1 + n] return t def t_error(t): print("Illegal character '%s'" % t.value[0]) t.lexer.skip(1) # Build the lexer import ply.lex as lex lex.lex() lex.runmain() ================================================ FILE: xonsh/ply/example/newclasscalc/calc.py ================================================ #!/usr/bin/env python # ----------------------------------------------------------------------------- # calc.py # # A simple calculator with variables. This is from O'Reilly's # "Lex and Yacc", p. 63. # # Class-based example contributed to PLY by David McNab. # # Modified to use new-style classes. Test case. # ----------------------------------------------------------------------------- import sys sys.path.insert(0, "../..") if sys.version_info[0] >= 3: raw_input = input import ply.lex as lex import ply.yacc as yacc import os class Parser(object): """ Base class for a lexer/parser that has the rules defined as methods """ tokens = () precedence = () def __init__(self, **kw): self.debug = kw.get('debug', 0) self.names = {} try: modname = os.path.split(os.path.splitext(__file__)[0])[ 1] + "_" + self.__class__.__name__ except: modname = "parser" + "_" + self.__class__.__name__ self.debugfile = modname + ".dbg" self.tabmodule = modname + "_" + "parsetab" # print self.debugfile, self.tabmodule # Build the lexer and parser lex.lex(module=self, debug=self.debug) yacc.yacc(module=self, debug=self.debug, debugfile=self.debugfile, tabmodule=self.tabmodule) def run(self): while 1: try: s = raw_input('calc > ') except EOFError: break if not s: continue yacc.parse(s) class Calc(Parser): tokens = ( 'NAME', 'NUMBER', 'PLUS', 'MINUS', 'EXP', 'TIMES', 'DIVIDE', 'EQUALS', 'LPAREN', 'RPAREN', ) # Tokens t_PLUS = r'\+' t_MINUS = r'-' t_EXP = r'\*\*' t_TIMES = r'\*' t_DIVIDE = r'/' t_EQUALS = r'=' t_LPAREN = r'\(' t_RPAREN = r'\)' t_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*' def t_NUMBER(self, t): r'\d+' try: t.value = int(t.value) except ValueError: print("Integer value too large %s" % t.value) t.value = 0 # print "parsed number %s" % repr(t.value) return t t_ignore = " \t" def t_newline(self, t): r'\n+' t.lexer.lineno += t.value.count("\n") def t_error(self, t): print("Illegal character '%s'" % t.value[0]) t.lexer.skip(1) # Parsing rules precedence = ( ('left', 'PLUS', 'MINUS'), ('left', 'TIMES', 'DIVIDE'), ('left', 'EXP'), ('right', 'UMINUS'), ) def p_statement_assign(self, p): 'statement : NAME EQUALS expression' self.names[p[1]] = p[3] def p_statement_expr(self, p): 'statement : expression' print(p[1]) def p_expression_binop(self, p): """ expression : expression PLUS expression | expression MINUS expression | expression TIMES expression | expression DIVIDE expression | expression EXP expression """ # print [repr(p[i]) for i in range(0,4)] if p[2] == '+': p[0] = p[1] + p[3] elif p[2] == '-': p[0] = p[1] - p[3] elif p[2] == '*': p[0] = p[1] * p[3] elif p[2] == '/': p[0] = p[1] / p[3] elif p[2] == '**': p[0] = p[1] ** p[3] def p_expression_uminus(self, p): 'expression : MINUS expression %prec UMINUS' p[0] = -p[2] def p_expression_group(self, p): 'expression : LPAREN expression RPAREN' p[0] = p[2] def p_expression_number(self, p): 'expression : NUMBER' p[0] = p[1] def p_expression_name(self, p): 'expression : NAME' try: p[0] = self.names[p[1]] except LookupError: print("Undefined name '%s'" % p[1]) p[0] = 0 def p_error(self, p): if p: print("Syntax error at '%s'" % p.value) else: print("Syntax error at EOF") if __name__ == '__main__': calc = Calc() calc.run() ================================================ FILE: xonsh/ply/example/optcalc/README ================================================ An example showing how to use Python optimized mode. To run: - First run 'python calc.py' - Then run 'python -OO calc.py' If working correctly, the second version should run the same way. ================================================ FILE: xonsh/ply/example/optcalc/calc.py ================================================ # ----------------------------------------------------------------------------- # calc.py # # A simple calculator with variables. This is from O'Reilly's # "Lex and Yacc", p. 63. # ----------------------------------------------------------------------------- import sys sys.path.insert(0, "../..") if sys.version_info[0] >= 3: raw_input = input tokens = ( 'NAME', 'NUMBER', 'PLUS', 'MINUS', 'TIMES', 'DIVIDE', 'EQUALS', 'LPAREN', 'RPAREN', ) # Tokens t_PLUS = r'\+' t_MINUS = r'-' t_TIMES = r'\*' t_DIVIDE = r'/' t_EQUALS = r'=' t_LPAREN = r'\(' t_RPAREN = r'\)' t_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*' def t_NUMBER(t): r'\d+' try: t.value = int(t.value) except ValueError: print("Integer value too large %s" % t.value) t.value = 0 return t t_ignore = " \t" def t_newline(t): r'\n+' t.lexer.lineno += t.value.count("\n") def t_error(t): print("Illegal character '%s'" % t.value[0]) t.lexer.skip(1) # Build the lexer import ply.lex as lex lex.lex(optimize=1) # Parsing rules precedence = ( ('left', 'PLUS', 'MINUS'), ('left', 'TIMES', 'DIVIDE'), ('right', 'UMINUS'), ) # dictionary of names names = {} def p_statement_assign(t): 'statement : NAME EQUALS expression' names[t[1]] = t[3] def p_statement_expr(t): 'statement : expression' print(t[1]) def p_expression_binop(t): '''expression : expression PLUS expression | expression MINUS expression | expression TIMES expression | expression DIVIDE expression''' if t[2] == '+': t[0] = t[1] + t[3] elif t[2] == '-': t[0] = t[1] - t[3] elif t[2] == '*': t[0] = t[1] * t[3] elif t[2] == '/': t[0] = t[1] / t[3] elif t[2] == '<': t[0] = t[1] < t[3] def p_expression_uminus(t): 'expression : MINUS expression %prec UMINUS' t[0] = -t[2] def p_expression_group(t): 'expression : LPAREN expression RPAREN' t[0] = t[2] def p_expression_number(t): 'expression : NUMBER' t[0] = t[1] def p_expression_name(t): 'expression : NAME' try: t[0] = names[t[1]] except LookupError: print("Undefined name '%s'" % t[1]) t[0] = 0 def p_error(t): if t: print("Syntax error at '%s'" % t.value) else: print("Syntax error at EOF") import ply.yacc as yacc yacc.yacc(optimize=1) while 1: try: s = raw_input('calc > ') except EOFError: break yacc.parse(s) ================================================ FILE: xonsh/ply/example/unicalc/calc.py ================================================ # ----------------------------------------------------------------------------- # calc.py # # A simple calculator with variables. This is from O'Reilly's # "Lex and Yacc", p. 63. # # This example uses unicode strings for tokens, docstrings, and input. # ----------------------------------------------------------------------------- import sys sys.path.insert(0, "../..") tokens = ( 'NAME', 'NUMBER', 'PLUS', 'MINUS', 'TIMES', 'DIVIDE', 'EQUALS', 'LPAREN', 'RPAREN', ) # Tokens t_PLUS = ur'\+' t_MINUS = ur'-' t_TIMES = ur'\*' t_DIVIDE = ur'/' t_EQUALS = ur'=' t_LPAREN = ur'\(' t_RPAREN = ur'\)' t_NAME = ur'[a-zA-Z_][a-zA-Z0-9_]*' def t_NUMBER(t): ur'\d+' try: t.value = int(t.value) except ValueError: print "Integer value too large", t.value t.value = 0 return t t_ignore = u" \t" def t_newline(t): ur'\n+' t.lexer.lineno += t.value.count("\n") def t_error(t): print "Illegal character '%s'" % t.value[0] t.lexer.skip(1) # Build the lexer import ply.lex as lex lex.lex() # Parsing rules precedence = ( ('left', 'PLUS', 'MINUS'), ('left', 'TIMES', 'DIVIDE'), ('right', 'UMINUS'), ) # dictionary of names names = {} def p_statement_assign(p): 'statement : NAME EQUALS expression' names[p[1]] = p[3] def p_statement_expr(p): 'statement : expression' print p[1] def p_expression_binop(p): '''expression : expression PLUS expression | expression MINUS expression | expression TIMES expression | expression DIVIDE expression''' if p[2] == u'+': p[0] = p[1] + p[3] elif p[2] == u'-': p[0] = p[1] - p[3] elif p[2] == u'*': p[0] = p[1] * p[3] elif p[2] == u'/': p[0] = p[1] / p[3] def p_expression_uminus(p): 'expression : MINUS expression %prec UMINUS' p[0] = -p[2] def p_expression_group(p): 'expression : LPAREN expression RPAREN' p[0] = p[2] def p_expression_number(p): 'expression : NUMBER' p[0] = p[1] def p_expression_name(p): 'expression : NAME' try: p[0] = names[p[1]] except LookupError: print "Undefined name '%s'" % p[1] p[0] = 0 def p_error(p): if p: print "Syntax error at '%s'" % p.value else: print "Syntax error at EOF" import ply.yacc as yacc yacc.yacc() while 1: try: s = raw_input('calc > ') except EOFError: break if not s: continue yacc.parse(unicode(s)) ================================================ FILE: xonsh/ply/example/yply/README ================================================ yply.py This example implements a program yply.py that converts a UNIX-yacc specification file into a PLY-compatible program. To use, simply run it like this: % python yply.py [-nocode] inputfile.y >myparser.py The output of this program is Python code. In the output, any C code in the original file is included, but is commented out. If you use the -nocode option, then all of the C code in the original file is just discarded. To use the resulting grammer with PLY, you'll need to edit the myparser.py file. Within this file, some stub code is included that can be used to test the construction of the parsing tables. However, you'll need to do more editing to make a workable parser. Disclaimer: This just an example I threw together in an afternoon. It might have some bugs. However, it worked when I tried it on a yacc-specified C++ parser containing 442 rules and 855 parsing states. Comments: 1. This example does not parse specification files meant for lex/flex. You'll need to specify the tokenizer on your own. 2. This example shows a number of interesting PLY features including - Parsing of literal text delimited by nested parentheses - Some interaction between the parser and the lexer. - Use of literals in the grammar specification - One pass compilation. The program just emits the result, there is no intermediate parse tree. 3. This program could probably be cleaned up and enhanced a lot. It would be great if someone wanted to work on this (hint). -Dave ================================================ FILE: xonsh/ply/example/yply/ylex.py ================================================ # lexer for yacc-grammars # # Author: David Beazley (dave@dabeaz.com) # Date : October 2, 2006 import sys sys.path.append("../..") from ply import * tokens = ( 'LITERAL', 'SECTION', 'TOKEN', 'LEFT', 'RIGHT', 'PREC', 'START', 'TYPE', 'NONASSOC', 'UNION', 'CODE', 'ID', 'QLITERAL', 'NUMBER', ) states = (('code', 'exclusive'),) literals = [';', ',', '<', '>', '|', ':'] t_ignore = ' \t' t_TOKEN = r'%token' t_LEFT = r'%left' t_RIGHT = r'%right' t_NONASSOC = r'%nonassoc' t_PREC = r'%prec' t_START = r'%start' t_TYPE = r'%type' t_UNION = r'%union' t_ID = r'[a-zA-Z_][a-zA-Z_0-9]*' t_QLITERAL = r'''(?P['"]).*?(?P=quote)''' t_NUMBER = r'\d+' def t_SECTION(t): r'%%' if getattr(t.lexer, "lastsection", 0): t.value = t.lexer.lexdata[t.lexpos + 2:] t.lexer.lexpos = len(t.lexer.lexdata) else: t.lexer.lastsection = 0 return t # Comments def t_ccomment(t): r'/\*(.|\n)*?\*/' t.lexer.lineno += t.value.count('\n') t_ignore_cppcomment = r'//.*' def t_LITERAL(t): r'%\{(.|\n)*?%\}' t.lexer.lineno += t.value.count("\n") return t def t_NEWLINE(t): r'\n' t.lexer.lineno += 1 def t_code(t): r'\{' t.lexer.codestart = t.lexpos t.lexer.level = 1 t.lexer.begin('code') def t_code_ignore_string(t): r'\"([^\\\n]|(\\.))*?\"' def t_code_ignore_char(t): r'\'([^\\\n]|(\\.))*?\'' def t_code_ignore_comment(t): r'/\*(.|\n)*?\*/' def t_code_ignore_cppcom(t): r'//.*' def t_code_lbrace(t): r'\{' t.lexer.level += 1 def t_code_rbrace(t): r'\}' t.lexer.level -= 1 if t.lexer.level == 0: t.type = 'CODE' t.value = t.lexer.lexdata[t.lexer.codestart:t.lexpos + 1] t.lexer.begin('INITIAL') t.lexer.lineno += t.value.count('\n') return t t_code_ignore_nonspace = r'[^\s\}\'\"\{]+' t_code_ignore_whitespace = r'\s+' t_code_ignore = "" def t_code_error(t): raise RuntimeError def t_error(t): print("%d: Illegal character '%s'" % (t.lexer.lineno, t.value[0])) print(t.value) t.lexer.skip(1) lex.lex() if __name__ == '__main__': lex.runmain() ================================================ FILE: xonsh/ply/example/yply/yparse.py ================================================ # parser for Unix yacc-based grammars # # Author: David Beazley (dave@dabeaz.com) # Date : October 2, 2006 import ylex tokens = ylex.tokens from ply import * tokenlist = [] preclist = [] emit_code = 1 def p_yacc(p): '''yacc : defsection rulesection''' def p_defsection(p): '''defsection : definitions SECTION | SECTION''' p.lexer.lastsection = 1 print("tokens = ", repr(tokenlist)) print() print("precedence = ", repr(preclist)) print() print("# -------------- RULES ----------------") print() def p_rulesection(p): '''rulesection : rules SECTION''' print("# -------------- RULES END ----------------") print_code(p[2], 0) def p_definitions(p): '''definitions : definitions definition | definition''' def p_definition_literal(p): '''definition : LITERAL''' print_code(p[1], 0) def p_definition_start(p): '''definition : START ID''' print("start = '%s'" % p[2]) def p_definition_token(p): '''definition : toktype opttype idlist optsemi ''' for i in p[3]: if i[0] not in "'\"": tokenlist.append(i) if p[1] == '%left': preclist.append(('left',) + tuple(p[3])) elif p[1] == '%right': preclist.append(('right',) + tuple(p[3])) elif p[1] == '%nonassoc': preclist.append(('nonassoc',) + tuple(p[3])) def p_toktype(p): '''toktype : TOKEN | LEFT | RIGHT | NONASSOC''' p[0] = p[1] def p_opttype(p): '''opttype : '<' ID '>' | empty''' def p_idlist(p): '''idlist : idlist optcomma tokenid | tokenid''' if len(p) == 2: p[0] = [p[1]] else: p[0] = p[1] p[1].append(p[3]) def p_tokenid(p): '''tokenid : ID | ID NUMBER | QLITERAL | QLITERAL NUMBER''' p[0] = p[1] def p_optsemi(p): '''optsemi : ';' | empty''' def p_optcomma(p): '''optcomma : ',' | empty''' def p_definition_type(p): '''definition : TYPE '<' ID '>' namelist optsemi''' # type declarations are ignored def p_namelist(p): '''namelist : namelist optcomma ID | ID''' def p_definition_union(p): '''definition : UNION CODE optsemi''' # Union declarations are ignored def p_rules(p): '''rules : rules rule | rule''' if len(p) == 2: rule = p[1] else: rule = p[2] # Print out a Python equivalent of this rule embedded = [] # Embedded actions (a mess) embed_count = 0 rulename = rule[0] rulecount = 1 for r in rule[1]: # r contains one of the rule possibilities print("def p_%s_%d(p):" % (rulename, rulecount)) prod = [] prodcode = "" for i in range(len(r)): item = r[i] if item[0] == '{': # A code block if i == len(r) - 1: prodcode = item break else: # an embedded action embed_name = "_embed%d_%s" % (embed_count, rulename) prod.append(embed_name) embedded.append((embed_name, item)) embed_count += 1 else: prod.append(item) print(" '''%s : %s'''" % (rulename, " ".join(prod))) # Emit code print_code(prodcode, 4) print() rulecount += 1 for e, code in embedded: print("def p_%s(p):" % e) print(" '''%s : '''" % e) print_code(code, 4) print() def p_rule(p): '''rule : ID ':' rulelist ';' ''' p[0] = (p[1], [p[3]]) def p_rule2(p): '''rule : ID ':' rulelist morerules ';' ''' p[4].insert(0, p[3]) p[0] = (p[1], p[4]) def p_rule_empty(p): '''rule : ID ':' ';' ''' p[0] = (p[1], [[]]) def p_rule_empty2(p): '''rule : ID ':' morerules ';' ''' p[3].insert(0, []) p[0] = (p[1], p[3]) def p_morerules(p): '''morerules : morerules '|' rulelist | '|' rulelist | '|' ''' if len(p) == 2: p[0] = [[]] elif len(p) == 3: p[0] = [p[2]] else: p[0] = p[1] p[0].append(p[3]) # print("morerules", len(p), p[0]) def p_rulelist(p): '''rulelist : rulelist ruleitem | ruleitem''' if len(p) == 2: p[0] = [p[1]] else: p[0] = p[1] p[1].append(p[2]) def p_ruleitem(p): '''ruleitem : ID | QLITERAL | CODE | PREC''' p[0] = p[1] def p_empty(p): '''empty : ''' def p_error(p): pass yacc.yacc(debug=0) def print_code(code, indent): if not emit_code: return codelines = code.splitlines() for c in codelines: print("%s# %s" % (" " * indent, c)) ================================================ FILE: xonsh/ply/example/yply/yply.py ================================================ #!/usr/local/bin/python # yply.py # # Author: David Beazley (dave@dabeaz.com) # Date : October 2, 2006 # # Converts a UNIX-yacc specification file into a PLY-compatible # specification. To use, simply do this: # # % python yply.py [-nocode] inputfile.y >myparser.py # # The output of this program is Python code. In the output, # any C code in the original file is included, but is commented. # If you use the -nocode option, then all of the C code in the # original file is discarded. # # Disclaimer: This just an example I threw together in an afternoon. # It might have some bugs. However, it worked when I tried it on # a yacc-specified C++ parser containing 442 rules and 855 parsing # states. # import sys sys.path.insert(0, "../..") import ylex import yparse from ply import * if len(sys.argv) == 1: print("usage : yply.py [-nocode] inputfile") raise SystemExit if len(sys.argv) == 3: if sys.argv[1] == '-nocode': yparse.emit_code = 0 else: print("Unknown option '%s'" % sys.argv[1]) raise SystemExit filename = sys.argv[2] else: filename = sys.argv[1] yacc.parse(open(filename).read()) print(""" if __name__ == '__main__': from ply import * yacc.yacc() """) ================================================ FILE: xonsh/ply/ply/__init__.py ================================================ # PLY package # Author: David Beazley (dave@dabeaz.com) __version__ = '3.11' __all__ = ['lex','yacc'] ================================================ FILE: xonsh/ply/ply/cpp.py ================================================ # ----------------------------------------------------------------------------- # ply: cpp.py # # Copyright (C) 2001-2019 # David M. Beazley (Dabeaz LLC) # All rights reserved. # # Latest version: https://github.com/dabeaz/ply # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # * Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright notice, # this list of conditions and the following disclaimer in the documentation # and/or other materials provided with the distribution. # * Neither the name of David Beazley or Dabeaz LLC may be used to # endorse or promote products derived from this software without # specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # ----------------------------------------------------------------------------- # This module implements an ANSI-C style lexical preprocessor for PLY. # ----------------------------------------------------------------------------- from __future__ import generators import sys # Some Python 3 compatibility shims if sys.version_info.major < 3: STRING_TYPES = (str, unicode) else: STRING_TYPES = str xrange = range # ----------------------------------------------------------------------------- # Default preprocessor lexer definitions. These tokens are enough to get # a basic preprocessor working. Other modules may import these if they want # ----------------------------------------------------------------------------- tokens = ( 'CPP_ID','CPP_INTEGER', 'CPP_FLOAT', 'CPP_STRING', 'CPP_CHAR', 'CPP_WS', 'CPP_COMMENT1', 'CPP_COMMENT2', 'CPP_POUND','CPP_DPOUND' ) literals = "+-*/%|&~^<>=!?()[]{}.,;:\\\'\"" # Whitespace def t_CPP_WS(t): r'\s+' t.lexer.lineno += t.value.count("\n") return t t_CPP_POUND = r'\#' t_CPP_DPOUND = r'\#\#' # Identifier t_CPP_ID = r'[A-Za-z_][\w_]*' # Integer literal def CPP_INTEGER(t): r'(((((0x)|(0X))[0-9a-fA-F]+)|(\d+))([uU][lL]|[lL][uU]|[uU]|[lL])?)' return t t_CPP_INTEGER = CPP_INTEGER # Floating literal t_CPP_FLOAT = r'((\d+)(\.\d+)(e(\+|-)?(\d+))? | (\d+)e(\+|-)?(\d+))([lL]|[fF])?' # String literal def t_CPP_STRING(t): r'\"([^\\\n]|(\\(.|\n)))*?\"' t.lexer.lineno += t.value.count("\n") return t # Character constant 'c' or L'c' def t_CPP_CHAR(t): r'(L)?\'([^\\\n]|(\\(.|\n)))*?\'' t.lexer.lineno += t.value.count("\n") return t # Comment def t_CPP_COMMENT1(t): r'(/\*(.|\n)*?\*/)' ncr = t.value.count("\n") t.lexer.lineno += ncr # replace with one space or a number of '\n' t.type = 'CPP_WS'; t.value = '\n' * ncr if ncr else ' ' return t # Line comment def t_CPP_COMMENT2(t): r'(//.*?(\n|$))' # replace with '/n' t.type = 'CPP_WS'; t.value = '\n' return t def t_error(t): t.type = t.value[0] t.value = t.value[0] t.lexer.skip(1) return t import re import copy import time import os.path # ----------------------------------------------------------------------------- # trigraph() # # Given an input string, this function replaces all trigraph sequences. # The following mapping is used: # # ??= # # ??/ \ # ??' ^ # ??( [ # ??) ] # ??! | # ??< { # ??> } # ??- ~ # ----------------------------------------------------------------------------- _trigraph_pat = re.compile(r'''\?\?[=/\'\(\)\!<>\-]''') _trigraph_rep = { '=':'#', '/':'\\', "'":'^', '(':'[', ')':']', '!':'|', '<':'{', '>':'}', '-':'~' } def trigraph(input): return _trigraph_pat.sub(lambda g: _trigraph_rep[g.group()[-1]],input) # ------------------------------------------------------------------ # Macro object # # This object holds information about preprocessor macros # # .name - Macro name (string) # .value - Macro value (a list of tokens) # .arglist - List of argument names # .variadic - Boolean indicating whether or not variadic macro # .vararg - Name of the variadic parameter # # When a macro is created, the macro replacement token sequence is # pre-scanned and used to create patch lists that are later used # during macro expansion # ------------------------------------------------------------------ class Macro(object): def __init__(self,name,value,arglist=None,variadic=False): self.name = name self.value = value self.arglist = arglist self.variadic = variadic if variadic: self.vararg = arglist[-1] self.source = None # ------------------------------------------------------------------ # Preprocessor object # # Object representing a preprocessor. Contains macro definitions, # include directories, and other information # ------------------------------------------------------------------ class Preprocessor(object): def __init__(self,lexer=None): if lexer is None: lexer = lex.lexer self.lexer = lexer self.macros = { } self.path = [] self.temp_path = [] # Probe the lexer for selected tokens self.lexprobe() tm = time.localtime() self.define("__DATE__ \"%s\"" % time.strftime("%b %d %Y",tm)) self.define("__TIME__ \"%s\"" % time.strftime("%H:%M:%S",tm)) self.parser = None # ----------------------------------------------------------------------------- # tokenize() # # Utility function. Given a string of text, tokenize into a list of tokens # ----------------------------------------------------------------------------- def tokenize(self,text): tokens = [] self.lexer.input(text) while True: tok = self.lexer.token() if not tok: break tokens.append(tok) return tokens # --------------------------------------------------------------------- # error() # # Report a preprocessor error/warning of some kind # ---------------------------------------------------------------------- def error(self,file,line,msg): print("%s:%d %s" % (file,line,msg)) # ---------------------------------------------------------------------- # lexprobe() # # This method probes the preprocessor lexer object to discover # the token types of symbols that are important to the preprocessor. # If this works right, the preprocessor will simply "work" # with any suitable lexer regardless of how tokens have been named. # ---------------------------------------------------------------------- def lexprobe(self): # Determine the token type for identifiers self.lexer.input("identifier") tok = self.lexer.token() if not tok or tok.value != "identifier": print("Couldn't determine identifier type") else: self.t_ID = tok.type # Determine the token type for integers self.lexer.input("12345") tok = self.lexer.token() if not tok or int(tok.value) != 12345: print("Couldn't determine integer type") else: self.t_INTEGER = tok.type self.t_INTEGER_TYPE = type(tok.value) # Determine the token type for strings enclosed in double quotes self.lexer.input("\"filename\"") tok = self.lexer.token() if not tok or tok.value != "\"filename\"": print("Couldn't determine string type") else: self.t_STRING = tok.type # Determine the token type for whitespace--if any self.lexer.input(" ") tok = self.lexer.token() if not tok or tok.value != " ": self.t_SPACE = None else: self.t_SPACE = tok.type # Determine the token type for newlines self.lexer.input("\n") tok = self.lexer.token() if not tok or tok.value != "\n": self.t_NEWLINE = None print("Couldn't determine token for newlines") else: self.t_NEWLINE = tok.type self.t_WS = (self.t_SPACE, self.t_NEWLINE) # Check for other characters used by the preprocessor chars = [ '<','>','#','##','\\','(',')',',','.'] for c in chars: self.lexer.input(c) tok = self.lexer.token() if not tok or tok.value != c: print("Unable to lex '%s' required for preprocessor" % c) # ---------------------------------------------------------------------- # add_path() # # Adds a search path to the preprocessor. # ---------------------------------------------------------------------- def add_path(self,path): self.path.append(path) # ---------------------------------------------------------------------- # group_lines() # # Given an input string, this function splits it into lines. Trailing whitespace # is removed. Any line ending with \ is grouped with the next line. This # function forms the lowest level of the preprocessor---grouping into text into # a line-by-line format. # ---------------------------------------------------------------------- def group_lines(self,input): lex = self.lexer.clone() lines = [x.rstrip() for x in input.splitlines()] for i in xrange(len(lines)): j = i+1 while lines[i].endswith('\\') and (j < len(lines)): lines[i] = lines[i][:-1]+lines[j] lines[j] = "" j += 1 input = "\n".join(lines) lex.input(input) lex.lineno = 1 current_line = [] while True: tok = lex.token() if not tok: break current_line.append(tok) if tok.type in self.t_WS and '\n' in tok.value: yield current_line current_line = [] if current_line: yield current_line # ---------------------------------------------------------------------- # tokenstrip() # # Remove leading/trailing whitespace tokens from a token list # ---------------------------------------------------------------------- def tokenstrip(self,tokens): i = 0 while i < len(tokens) and tokens[i].type in self.t_WS: i += 1 del tokens[:i] i = len(tokens)-1 while i >= 0 and tokens[i].type in self.t_WS: i -= 1 del tokens[i+1:] return tokens # ---------------------------------------------------------------------- # collect_args() # # Collects comma separated arguments from a list of tokens. The arguments # must be enclosed in parenthesis. Returns a tuple (tokencount,args,positions) # where tokencount is the number of tokens consumed, args is a list of arguments, # and positions is a list of integers containing the starting index of each # argument. Each argument is represented by a list of tokens. # # When collecting arguments, leading and trailing whitespace is removed # from each argument. # # This function properly handles nested parenthesis and commas---these do not # define new arguments. # ---------------------------------------------------------------------- def collect_args(self,tokenlist): args = [] positions = [] current_arg = [] nesting = 1 tokenlen = len(tokenlist) # Search for the opening '('. i = 0 while (i < tokenlen) and (tokenlist[i].type in self.t_WS): i += 1 if (i < tokenlen) and (tokenlist[i].value == '('): positions.append(i+1) else: self.error(self.source,tokenlist[0].lineno,"Missing '(' in macro arguments") return 0, [], [] i += 1 while i < tokenlen: t = tokenlist[i] if t.value == '(': current_arg.append(t) nesting += 1 elif t.value == ')': nesting -= 1 if nesting == 0: if current_arg: args.append(self.tokenstrip(current_arg)) positions.append(i) return i+1,args,positions current_arg.append(t) elif t.value == ',' and nesting == 1: args.append(self.tokenstrip(current_arg)) positions.append(i+1) current_arg = [] else: current_arg.append(t) i += 1 # Missing end argument self.error(self.source,tokenlist[-1].lineno,"Missing ')' in macro arguments") return 0, [],[] # ---------------------------------------------------------------------- # macro_prescan() # # Examine the macro value (token sequence) and identify patch points # This is used to speed up macro expansion later on---we'll know # right away where to apply patches to the value to form the expansion # ---------------------------------------------------------------------- def macro_prescan(self,macro): macro.patch = [] # Standard macro arguments macro.str_patch = [] # String conversion expansion macro.var_comma_patch = [] # Variadic macro comma patch i = 0 while i < len(macro.value): if macro.value[i].type == self.t_ID and macro.value[i].value in macro.arglist: argnum = macro.arglist.index(macro.value[i].value) # Conversion of argument to a string if i > 0 and macro.value[i-1].value == '#': macro.value[i] = copy.copy(macro.value[i]) macro.value[i].type = self.t_STRING del macro.value[i-1] macro.str_patch.append((argnum,i-1)) continue # Concatenation elif (i > 0 and macro.value[i-1].value == '##'): macro.patch.append(('c',argnum,i-1)) del macro.value[i-1] i -= 1 continue elif ((i+1) < len(macro.value) and macro.value[i+1].value == '##'): macro.patch.append(('c',argnum,i)) del macro.value[i + 1] continue # Standard expansion else: macro.patch.append(('e',argnum,i)) elif macro.value[i].value == '##': if macro.variadic and (i > 0) and (macro.value[i-1].value == ',') and \ ((i+1) < len(macro.value)) and (macro.value[i+1].type == self.t_ID) and \ (macro.value[i+1].value == macro.vararg): macro.var_comma_patch.append(i-1) i += 1 macro.patch.sort(key=lambda x: x[2],reverse=True) # ---------------------------------------------------------------------- # macro_expand_args() # # Given a Macro and list of arguments (each a token list), this method # returns an expanded version of a macro. The return value is a token sequence # representing the replacement macro tokens # ---------------------------------------------------------------------- def macro_expand_args(self,macro,args,expanded): # Make a copy of the macro token sequence rep = [copy.copy(_x) for _x in macro.value] # Make string expansion patches. These do not alter the length of the replacement sequence str_expansion = {} for argnum, i in macro.str_patch: if argnum not in str_expansion: str_expansion[argnum] = ('"%s"' % "".join([x.value for x in args[argnum]])).replace("\\","\\\\") rep[i] = copy.copy(rep[i]) rep[i].value = str_expansion[argnum] # Make the variadic macro comma patch. If the variadic macro argument is empty, we get rid comma_patch = False if macro.variadic and not args[-1]: for i in macro.var_comma_patch: rep[i] = None comma_patch = True # Make all other patches. The order of these matters. It is assumed that the patch list # has been sorted in reverse order of patch location since replacements will cause the # size of the replacement sequence to expand from the patch point. expanded_args = { } for ptype, argnum, i in macro.patch: # Concatenation. Argument is left unexpanded if ptype == 'c': rep[i:i+1] = args[argnum] # Normal expansion. Argument is macro expanded first elif ptype == 'e': if argnum not in expanded_args: expanded_args[argnum] = self.expand_macros(args[argnum],expanded) rep[i:i+1] = expanded_args[argnum] # Get rid of removed comma if necessary if comma_patch: rep = [_i for _i in rep if _i] return rep # ---------------------------------------------------------------------- # expand_macros() # # Given a list of tokens, this function performs macro expansion. # The expanded argument is a dictionary that contains macros already # expanded. This is used to prevent infinite recursion. # ---------------------------------------------------------------------- def expand_macros(self,tokens,expanded=None): if expanded is None: expanded = {} i = 0 while i < len(tokens): t = tokens[i] if t.type == self.t_ID: if t.value in self.macros and t.value not in expanded: # Yes, we found a macro match expanded[t.value] = True m = self.macros[t.value] if not m.arglist: # A simple macro ex = self.expand_macros([copy.copy(_x) for _x in m.value],expanded) for e in ex: e.lineno = t.lineno tokens[i:i+1] = ex i += len(ex) else: # A macro with arguments j = i + 1 while j < len(tokens) and tokens[j].type in self.t_WS: j += 1 if j < len(tokens) and tokens[j].value == '(': tokcount,args,positions = self.collect_args(tokens[j:]) if not m.variadic and len(args) != len(m.arglist): self.error(self.source,t.lineno,"Macro %s requires %d arguments" % (t.value,len(m.arglist))) i = j + tokcount elif m.variadic and len(args) < len(m.arglist)-1: if len(m.arglist) > 2: self.error(self.source,t.lineno,"Macro %s must have at least %d arguments" % (t.value, len(m.arglist)-1)) else: self.error(self.source,t.lineno,"Macro %s must have at least %d argument" % (t.value, len(m.arglist)-1)) i = j + tokcount else: if m.variadic: if len(args) == len(m.arglist)-1: args.append([]) else: args[len(m.arglist)-1] = tokens[j+positions[len(m.arglist)-1]:j+tokcount-1] del args[len(m.arglist):] # Get macro replacement text rep = self.macro_expand_args(m,args,expanded) rep = self.expand_macros(rep,expanded) for r in rep: r.lineno = t.lineno tokens[i:j+tokcount] = rep i += len(rep) else: # This is not a macro. It is just a word which # equals to name of the macro. Hence, go to the # next token. i += 1 del expanded[t.value] continue elif t.value == '__LINE__': t.type = self.t_INTEGER t.value = self.t_INTEGER_TYPE(t.lineno) i += 1 return tokens # ---------------------------------------------------------------------- # evalexpr() # # Evaluate an expression token sequence for the purposes of evaluating # integral expressions. # ---------------------------------------------------------------------- def evalexpr(self,tokens): # tokens = tokenize(line) # Search for defined macros i = 0 while i < len(tokens): if tokens[i].type == self.t_ID and tokens[i].value == 'defined': j = i + 1 needparen = False result = "0L" while j < len(tokens): if tokens[j].type in self.t_WS: j += 1 continue elif tokens[j].type == self.t_ID: if tokens[j].value in self.macros: result = "1L" else: result = "0L" if not needparen: break elif tokens[j].value == '(': needparen = True elif tokens[j].value == ')': break else: self.error(self.source,tokens[i].lineno,"Malformed defined()") j += 1 tokens[i].type = self.t_INTEGER tokens[i].value = self.t_INTEGER_TYPE(result) del tokens[i+1:j+1] i += 1 tokens = self.expand_macros(tokens) return self.evalexpr_expanded(tokens) # ---------------------------------------------------------------------- # evalexpr_expanded() # # Helper for evalexpr that evaluates the expression that had its macros # and defined(...) expressions expanded by evalexpr # ---------------------------------------------------------------------- def evalexpr_expanded(self, tokens): for i,t in enumerate(tokens): if t.type == self.t_ID: tokens[i] = copy.copy(t) tokens[i].type = self.t_INTEGER tokens[i].value = self.t_INTEGER_TYPE("0") elif t.type == self.t_INTEGER: tokens[i] = copy.copy(t) # Strip off any trailing suffixes tokens[i].value = str(tokens[i].value) while tokens[i].value[-1] not in "0123456789abcdefABCDEF": tokens[i].value = tokens[i].value[:-1] return self.evalexpr_string("".join([str(x.value) for x in tokens])) # ---------------------------------------------------------------------- # evalexpr_string() # # Helper for evalexpr that evaluates a string expression # This implementation does basic C->python conversion and then uses eval() # ---------------------------------------------------------------------- def evalexpr_string(self, expr): expr = expr.replace("&&"," and ") expr = expr.replace("||"," or ") expr = expr.replace("!"," not ") expr = expr.replace(" not ="," !=") try: result = eval(expr) except Exception: self.error(self.source,tokens[0].lineno,"Couldn't evaluate expression") result = 0 return result # ---------------------------------------------------------------------- # parsegen() # # Parse an input string/ # ---------------------------------------------------------------------- def parsegen(self,input,source=None): # Replace trigraph sequences t = trigraph(input) lines = self.group_lines(t) if not source: source = "" self.define("__FILE__ \"%s\"" % source) self.source = source chunk = [] enable = True iftrigger = False ifstack = [] for x in lines: for i,tok in enumerate(x): if tok.type not in self.t_WS: break if tok.value == '#': # Preprocessor directive # insert necessary whitespace instead of eaten tokens for tok in x: if tok.type in self.t_WS and '\n' in tok.value: chunk.append(tok) dirtokens = self.tokenstrip(x[i+1:]) if dirtokens: name = dirtokens[0].value args = self.tokenstrip(dirtokens[1:]) else: name = "" args = [] if name == 'define': if enable: for tok in self.expand_macros(chunk): yield tok chunk = [] self.define(args) elif name == 'include': if enable: for tok in self.expand_macros(chunk): yield tok chunk = [] oldfile = self.macros['__FILE__'] for tok in self.include(args): yield tok self.macros['__FILE__'] = oldfile self.source = source elif name == 'undef': if enable: for tok in self.expand_macros(chunk): yield tok chunk = [] self.undef(args) elif name == 'ifdef': ifstack.append((enable,iftrigger)) if enable: if not args[0].value in self.macros: enable = False iftrigger = False else: iftrigger = True elif name == 'ifndef': ifstack.append((enable,iftrigger)) if enable: if args[0].value in self.macros: enable = False iftrigger = False else: iftrigger = True elif name == 'if': ifstack.append((enable,iftrigger)) if enable: result = self.evalexpr(args) if not result: enable = False iftrigger = False else: iftrigger = True elif name == 'elif': if ifstack: if ifstack[-1][0]: # We only pay attention if outer "if" allows this if enable: # If already true, we flip enable False enable = False elif not iftrigger: # If False, but not triggered yet, we'll check expression result = self.evalexpr(args) if result: enable = True iftrigger = True else: self.error(self.source,dirtokens[0].lineno,"Misplaced #elif") elif name == 'else': if ifstack: if ifstack[-1][0]: if enable: enable = False elif not iftrigger: enable = True iftrigger = True else: self.error(self.source,dirtokens[0].lineno,"Misplaced #else") elif name == 'endif': if ifstack: enable,iftrigger = ifstack.pop() else: self.error(self.source,dirtokens[0].lineno,"Misplaced #endif") else: # Unknown preprocessor directive pass else: # Normal text if enable: chunk.extend(x) for tok in self.expand_macros(chunk): yield tok chunk = [] # ---------------------------------------------------------------------- # include() # # Implementation of file-inclusion # ---------------------------------------------------------------------- def include(self,tokens): # Try to extract the filename and then process an include file if not tokens: return if tokens: if tokens[0].value != '<' and tokens[0].type != self.t_STRING: tokens = self.expand_macros(tokens) if tokens[0].value == '<': # Include <...> i = 1 while i < len(tokens): if tokens[i].value == '>': break i += 1 else: print("Malformed #include <...>") return filename = "".join([x.value for x in tokens[1:i]]) path = self.path + [""] + self.temp_path elif tokens[0].type == self.t_STRING: filename = tokens[0].value[1:-1] path = self.temp_path + [""] + self.path else: print("Malformed #include statement") return for p in path: iname = os.path.join(p,filename) try: data = self.read_include_file(iname) dname = os.path.dirname(iname) if dname: self.temp_path.insert(0,dname) for tok in self.parsegen(data,filename): yield tok if dname: del self.temp_path[0] break except IOError: pass else: print("Couldn't find '%s'" % filename) # ---------------------------------------------------------------------- # read_include_file() # # Reads a source file for inclusion using #include # Could be overridden to e.g. customize encoding, limit access to # certain paths on the filesystem, or provide the contents of system # include files # ---------------------------------------------------------------------- def read_include_file(self, filepath): with open(filepath, 'r', encoding='utf-8', errors='surrogateescape') as file: return file.read() # ---------------------------------------------------------------------- # define() # # Define a new macro # ---------------------------------------------------------------------- def define(self,tokens): if isinstance(tokens,STRING_TYPES): tokens = self.tokenize(tokens) linetok = tokens try: name = linetok[0] if len(linetok) > 1: mtype = linetok[1] else: mtype = None if not mtype: m = Macro(name.value,[]) self.macros[name.value] = m elif mtype.type in self.t_WS: # A normal macro m = Macro(name.value,self.tokenstrip(linetok[2:])) self.macros[name.value] = m elif mtype.value == '(': # A macro with arguments tokcount, args, positions = self.collect_args(linetok[1:]) variadic = False for a in args: if variadic: print("No more arguments may follow a variadic argument") break astr = "".join([str(_i.value) for _i in a]) if astr == "...": variadic = True a[0].type = self.t_ID a[0].value = '__VA_ARGS__' variadic = True del a[1:] continue elif astr[-3:] == "..." and a[0].type == self.t_ID: variadic = True del a[1:] # If, for some reason, "." is part of the identifier, strip off the name for the purposes # of macro expansion if a[0].value[-3:] == '...': a[0].value = a[0].value[:-3] continue if len(a) > 1 or a[0].type != self.t_ID: print("Invalid macro argument") break else: mvalue = self.tokenstrip(linetok[1+tokcount:]) i = 0 while i < len(mvalue): if i+1 < len(mvalue): if mvalue[i].type in self.t_WS and mvalue[i+1].value == '##': del mvalue[i] continue elif mvalue[i].value == '##' and mvalue[i+1].type in self.t_WS: del mvalue[i+1] i += 1 m = Macro(name.value,mvalue,[x[0].value for x in args],variadic) self.macro_prescan(m) self.macros[name.value] = m else: print("Bad macro definition") except LookupError: print("Bad macro definition") # ---------------------------------------------------------------------- # undef() # # Undefine a macro # ---------------------------------------------------------------------- def undef(self,tokens): id = tokens[0].value try: del self.macros[id] except LookupError: pass # ---------------------------------------------------------------------- # parse() # # Parse input text. # ---------------------------------------------------------------------- def parse(self,input,source=None,ignore={}): self.ignore = ignore self.parser = self.parsegen(input,source) # ---------------------------------------------------------------------- # token() # # Method to return individual tokens # ---------------------------------------------------------------------- def token(self): try: while True: tok = next(self.parser) if tok.type not in self.ignore: return tok except StopIteration: self.parser = None return None if __name__ == '__main__': import ply.lex as lex lexer = lex.lex() # Run a preprocessor import sys with open(sys.argv[1]) as f: input = f.read() p = Preprocessor(lexer) p.parse(input,sys.argv[1]) while True: tok = p.token() if not tok: break print(p.source, tok) ================================================ FILE: xonsh/ply/ply/ctokens.py ================================================ # ---------------------------------------------------------------------- # ctokens.py # # Token specifications for symbols in ANSI C and C++. This file is # meant to be used as a library in other tokenizers. # ---------------------------------------------------------------------- # Reserved words tokens = [ # Literals (identifier, integer constant, float constant, string constant, char const) 'ID', 'TYPEID', 'INTEGER', 'FLOAT', 'STRING', 'CHARACTER', # Operators (+,-,*,/,%,|,&,~,^,<<,>>, ||, &&, !, <, <=, >, >=, ==, !=) 'PLUS', 'MINUS', 'TIMES', 'DIVIDE', 'MODULO', 'OR', 'AND', 'NOT', 'XOR', 'LSHIFT', 'RSHIFT', 'LOR', 'LAND', 'LNOT', 'LT', 'LE', 'GT', 'GE', 'EQ', 'NE', # Assignment (=, *=, /=, %=, +=, -=, <<=, >>=, &=, ^=, |=) 'EQUALS', 'TIMESEQUAL', 'DIVEQUAL', 'MODEQUAL', 'PLUSEQUAL', 'MINUSEQUAL', 'LSHIFTEQUAL','RSHIFTEQUAL', 'ANDEQUAL', 'XOREQUAL', 'OREQUAL', # Increment/decrement (++,--) 'INCREMENT', 'DECREMENT', # Structure dereference (->) 'ARROW', # Ternary operator (?) 'TERNARY', # Delimeters ( ) [ ] { } , . ; : 'LPAREN', 'RPAREN', 'LBRACKET', 'RBRACKET', 'LBRACE', 'RBRACE', 'COMMA', 'PERIOD', 'SEMI', 'COLON', # Ellipsis (...) 'ELLIPSIS', ] # Operators t_PLUS = r'\+' t_MINUS = r'-' t_TIMES = r'\*' t_DIVIDE = r'/' t_MODULO = r'%' t_OR = r'\|' t_AND = r'&' t_NOT = r'~' t_XOR = r'\^' t_LSHIFT = r'<<' t_RSHIFT = r'>>' t_LOR = r'\|\|' t_LAND = r'&&' t_LNOT = r'!' t_LT = r'<' t_GT = r'>' t_LE = r'<=' t_GE = r'>=' t_EQ = r'==' t_NE = r'!=' # Assignment operators t_EQUALS = r'=' t_TIMESEQUAL = r'\*=' t_DIVEQUAL = r'/=' t_MODEQUAL = r'%=' t_PLUSEQUAL = r'\+=' t_MINUSEQUAL = r'-=' t_LSHIFTEQUAL = r'<<=' t_RSHIFTEQUAL = r'>>=' t_ANDEQUAL = r'&=' t_OREQUAL = r'\|=' t_XOREQUAL = r'\^=' # Increment/decrement t_INCREMENT = r'\+\+' t_DECREMENT = r'--' # -> t_ARROW = r'->' # ? t_TERNARY = r'\?' # Delimeters t_LPAREN = r'\(' t_RPAREN = r'\)' t_LBRACKET = r'\[' t_RBRACKET = r'\]' t_LBRACE = r'\{' t_RBRACE = r'\}' t_COMMA = r',' t_PERIOD = r'\.' t_SEMI = r';' t_COLON = r':' t_ELLIPSIS = r'\.\.\.' # Identifiers t_ID = r'[A-Za-z_][A-Za-z0-9_]*' # Integer literal t_INTEGER = r'\d+([uU]|[lL]|[uU][lL]|[lL][uU])?' # Floating literal t_FLOAT = r'((\d+)(\.\d+)(e(\+|-)?(\d+))? | (\d+)e(\+|-)?(\d+))([lL]|[fF])?' # String literal t_STRING = r'\"([^\\\n]|(\\.))*?\"' # Character constant 'c' or L'c' t_CHARACTER = r'(L)?\'([^\\\n]|(\\.))*?\'' # Comment (C-Style) def t_COMMENT(t): r'/\*(.|\n)*?\*/' t.lexer.lineno += t.value.count('\n') return t # Comment (C++-Style) def t_CPPCOMMENT(t): r'//.*\n' t.lexer.lineno += 1 return t ================================================ FILE: xonsh/ply/ply/lex.py ================================================ # ----------------------------------------------------------------------------- # ply: lex.py # # Copyright (C) 2001-2019 # David M. Beazley (Dabeaz LLC) # All rights reserved. # # Latest version: https://github.com/dabeaz/ply # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # * Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright notice, # this list of conditions and the following disclaimer in the documentation # and/or other materials provided with the distribution. # * Neither the name of David Beazley or Dabeaz LLC may be used to # endorse or promote products derived from this software without # specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # ----------------------------------------------------------------------------- __version__ = '3.11' __tabversion__ = '3.10' import re import sys import types import copy import os import inspect # This tuple contains known string types try: # Python 2.6 StringTypes = (types.StringType, types.UnicodeType) except AttributeError: # Python 3.0 StringTypes = (str, bytes) # This regular expression is used to match valid token names _is_identifier = re.compile(r'^[a-zA-Z0-9_]+$') # Exception thrown when invalid token encountered and no default error # handler is defined. class LexError(Exception): def __init__(self, message, s): self.args = (message,) self.text = s # Token class. This class is used to represent the tokens produced. class LexToken(object): def __str__(self): return 'LexToken(%s,%r,%d,%d)' % (self.type, self.value, self.lineno, self.lexpos) def __repr__(self): return str(self) # This object is a stand-in for a logging object created by the # logging module. class PlyLogger(object): def __init__(self, f): self.f = f def critical(self, msg, *args, **kwargs): self.f.write((msg % args) + '\n') def warning(self, msg, *args, **kwargs): self.f.write('WARNING: ' + (msg % args) + '\n') def error(self, msg, *args, **kwargs): self.f.write('ERROR: ' + (msg % args) + '\n') info = critical debug = critical # Null logger is used when no output is generated. Does nothing. class NullLogger(object): def __getattribute__(self, name): return self def __call__(self, *args, **kwargs): return self # ----------------------------------------------------------------------------- # === Lexing Engine === # # The following Lexer class implements the lexer runtime. There are only # a few public methods and attributes: # # input() - Store a new string in the lexer # token() - Get the next token # clone() - Clone the lexer # # lineno - Current line number # lexpos - Current position in the input string # ----------------------------------------------------------------------------- class Lexer: def __init__(self): self.lexre = None # Master regular expression. This is a list of # tuples (re, findex) where re is a compiled # regular expression and findex is a list # mapping regex group numbers to rules self.lexretext = None # Current regular expression strings self.lexstatere = {} # Dictionary mapping lexer states to master regexs self.lexstateretext = {} # Dictionary mapping lexer states to regex strings self.lexstaterenames = {} # Dictionary mapping lexer states to symbol names self.lexstate = 'INITIAL' # Current lexer state self.lexstatestack = [] # Stack of lexer states self.lexstateinfo = None # State information self.lexstateignore = {} # Dictionary of ignored characters for each state self.lexstateerrorf = {} # Dictionary of error functions for each state self.lexstateeoff = {} # Dictionary of eof functions for each state self.lexreflags = 0 # Optional re compile flags self.lexdata = None # Actual input data (as a string) self.lexpos = 0 # Current position in input text self.lexlen = 0 # Length of the input text self.lexerrorf = None # Error rule (if any) self.lexeoff = None # EOF rule (if any) self.lextokens = None # List of valid tokens self.lexignore = '' # Ignored characters self.lexliterals = '' # Literal characters that can be passed through self.lexmodule = None # Module self.lineno = 1 # Current line number self.lexoptimize = False # Optimized mode def clone(self, object=None): c = copy.copy(self) # If the object parameter has been supplied, it means we are attaching the # lexer to a new object. In this case, we have to rebind all methods in # the lexstatere and lexstateerrorf tables. if object: newtab = {} for key, ritem in self.lexstatere.items(): newre = [] for cre, findex in ritem: newfindex = [] for f in findex: if not f or not f[0]: newfindex.append(f) continue newfindex.append((getattr(object, f[0].__name__), f[1])) newre.append((cre, newfindex)) newtab[key] = newre c.lexstatere = newtab c.lexstateerrorf = {} for key, ef in self.lexstateerrorf.items(): c.lexstateerrorf[key] = getattr(object, ef.__name__) c.lexmodule = object return c # ------------------------------------------------------------ # writetab() - Write lexer information to a table file # ------------------------------------------------------------ def writetab(self, lextab, outputdir=''): if isinstance(lextab, types.ModuleType): raise IOError("Won't overwrite existing lextab module") basetabmodule = lextab.split('.')[-1] filename = os.path.join(outputdir, basetabmodule) + '.py' with open(filename, 'w') as tf: tf.write('# %s.py. This file automatically created by PLY (version %s). Don\'t edit!\n' % (basetabmodule, __version__)) tf.write('_tabversion = %s\n' % repr(__tabversion__)) tf.write('_lextokens = set(%s)\n' % repr(tuple(sorted(self.lextokens)))) tf.write('_lexreflags = %s\n' % repr(int(self.lexreflags))) tf.write('_lexliterals = %s\n' % repr(self.lexliterals)) tf.write('_lexstateinfo = %s\n' % repr(self.lexstateinfo)) # Rewrite the lexstatere table, replacing function objects with function names tabre = {} for statename, lre in self.lexstatere.items(): titem = [] for (pat, func), retext, renames in zip(lre, self.lexstateretext[statename], self.lexstaterenames[statename]): titem.append((retext, _funcs_to_names(func, renames))) tabre[statename] = titem tf.write('_lexstatere = %s\n' % repr(tabre)) tf.write('_lexstateignore = %s\n' % repr(self.lexstateignore)) taberr = {} for statename, ef in self.lexstateerrorf.items(): taberr[statename] = ef.__name__ if ef else None tf.write('_lexstateerrorf = %s\n' % repr(taberr)) tabeof = {} for statename, ef in self.lexstateeoff.items(): tabeof[statename] = ef.__name__ if ef else None tf.write('_lexstateeoff = %s\n' % repr(tabeof)) # ------------------------------------------------------------ # readtab() - Read lexer information from a tab file # ------------------------------------------------------------ def readtab(self, tabfile, fdict): if isinstance(tabfile, types.ModuleType): lextab = tabfile else: exec('import %s' % tabfile) lextab = sys.modules[tabfile] if getattr(lextab, '_tabversion', '0.0') != __tabversion__: raise ImportError('Inconsistent PLY version') self.lextokens = lextab._lextokens self.lexreflags = lextab._lexreflags self.lexliterals = lextab._lexliterals self.lextokens_all = self.lextokens | set(self.lexliterals) self.lexstateinfo = lextab._lexstateinfo self.lexstateignore = lextab._lexstateignore self.lexstatere = {} self.lexstateretext = {} for statename, lre in lextab._lexstatere.items(): titem = [] txtitem = [] for pat, func_name in lre: titem.append((re.compile(pat, lextab._lexreflags), _names_to_funcs(func_name, fdict))) self.lexstatere[statename] = titem self.lexstateretext[statename] = txtitem self.lexstateerrorf = {} for statename, ef in lextab._lexstateerrorf.items(): self.lexstateerrorf[statename] = fdict[ef] self.lexstateeoff = {} for statename, ef in lextab._lexstateeoff.items(): self.lexstateeoff[statename] = fdict[ef] self.begin('INITIAL') # ------------------------------------------------------------ # input() - Push a new string into the lexer # ------------------------------------------------------------ def input(self, s): # Pull off the first character to see if s looks like a string c = s[:1] if not isinstance(c, StringTypes): raise ValueError('Expected a string') self.lexdata = s self.lexpos = 0 self.lexlen = len(s) # ------------------------------------------------------------ # begin() - Changes the lexing state # ------------------------------------------------------------ def begin(self, state): if state not in self.lexstatere: raise ValueError('Undefined state') self.lexre = self.lexstatere[state] self.lexretext = self.lexstateretext[state] self.lexignore = self.lexstateignore.get(state, '') self.lexerrorf = self.lexstateerrorf.get(state, None) self.lexeoff = self.lexstateeoff.get(state, None) self.lexstate = state # ------------------------------------------------------------ # push_state() - Changes the lexing state and saves old on stack # ------------------------------------------------------------ def push_state(self, state): self.lexstatestack.append(self.lexstate) self.begin(state) # ------------------------------------------------------------ # pop_state() - Restores the previous state # ------------------------------------------------------------ def pop_state(self): self.begin(self.lexstatestack.pop()) # ------------------------------------------------------------ # current_state() - Returns the current lexing state # ------------------------------------------------------------ def current_state(self): return self.lexstate # ------------------------------------------------------------ # skip() - Skip ahead n characters # ------------------------------------------------------------ def skip(self, n): self.lexpos += n # ------------------------------------------------------------ # opttoken() - Return the next token from the Lexer # # Note: This function has been carefully implemented to be as fast # as possible. Don't make changes unless you really know what # you are doing # ------------------------------------------------------------ def token(self): # Make local copies of frequently referenced attributes lexpos = self.lexpos lexlen = self.lexlen lexignore = self.lexignore lexdata = self.lexdata while lexpos < lexlen: # This code provides some short-circuit code for whitespace, tabs, and other ignored characters if lexdata[lexpos] in lexignore: lexpos += 1 continue # Look for a regular expression match for lexre, lexindexfunc in self.lexre: m = lexre.match(lexdata, lexpos) if not m: continue # Create a token for return tok = LexToken() tok.value = m.group() tok.lineno = self.lineno tok.lexpos = lexpos i = m.lastindex func, tok.type = lexindexfunc[i] if not func: # If no token type was set, it's an ignored token if tok.type: self.lexpos = m.end() return tok else: lexpos = m.end() break lexpos = m.end() # If token is processed by a function, call it tok.lexer = self # Set additional attributes useful in token rules self.lexmatch = m self.lexpos = lexpos newtok = func(tok) # Every function must return a token, if nothing, we just move to next token if not newtok: lexpos = self.lexpos # This is here in case user has updated lexpos. lexignore = self.lexignore # This is here in case there was a state change break # Verify type of the token. If not in the token map, raise an error if not self.lexoptimize: if newtok.type not in self.lextokens_all: raise LexError("%s:%d: Rule '%s' returned an unknown token type '%s'" % ( func.__code__.co_filename, func.__code__.co_firstlineno, func.__name__, newtok.type), lexdata[lexpos:]) return newtok else: # No match, see if in literals if lexdata[lexpos] in self.lexliterals: tok = LexToken() tok.value = lexdata[lexpos] tok.lineno = self.lineno tok.type = tok.value tok.lexpos = lexpos self.lexpos = lexpos + 1 return tok # No match. Call t_error() if defined. if self.lexerrorf: tok = LexToken() tok.value = self.lexdata[lexpos:] tok.lineno = self.lineno tok.type = 'error' tok.lexer = self tok.lexpos = lexpos self.lexpos = lexpos newtok = self.lexerrorf(tok) if lexpos == self.lexpos: # Error method didn't change text position at all. This is an error. raise LexError("Scanning error. Illegal character '%s'" % (lexdata[lexpos]), lexdata[lexpos:]) lexpos = self.lexpos if not newtok: continue return newtok self.lexpos = lexpos raise LexError("Illegal character '%s' at index %d" % (lexdata[lexpos], lexpos), lexdata[lexpos:]) if self.lexeoff: tok = LexToken() tok.type = 'eof' tok.value = '' tok.lineno = self.lineno tok.lexpos = lexpos tok.lexer = self self.lexpos = lexpos newtok = self.lexeoff(tok) return newtok self.lexpos = lexpos + 1 if self.lexdata is None: raise RuntimeError('No input string given with input()') return None # Iterator interface def __iter__(self): return self def next(self): t = self.token() if t is None: raise StopIteration return t __next__ = next # ----------------------------------------------------------------------------- # ==== Lex Builder === # # The functions and classes below are used to collect lexing information # and build a Lexer object from it. # ----------------------------------------------------------------------------- # ----------------------------------------------------------------------------- # _get_regex(func) # # Returns the regular expression assigned to a function either as a doc string # or as a .regex attribute attached by the @TOKEN decorator. # ----------------------------------------------------------------------------- def _get_regex(func): return getattr(func, 'regex', func.__doc__) # ----------------------------------------------------------------------------- # get_caller_module_dict() # # This function returns a dictionary containing all of the symbols defined within # a caller further down the call stack. This is used to get the environment # associated with the yacc() call if none was provided. # ----------------------------------------------------------------------------- def get_caller_module_dict(levels): f = sys._getframe(levels) ldict = f.f_globals.copy() if f.f_globals != f.f_locals: ldict.update(f.f_locals) return ldict # ----------------------------------------------------------------------------- # _funcs_to_names() # # Given a list of regular expression functions, this converts it to a list # suitable for output to a table file # ----------------------------------------------------------------------------- def _funcs_to_names(funclist, namelist): result = [] for f, name in zip(funclist, namelist): if f and f[0]: result.append((name, f[1])) else: result.append(f) return result # ----------------------------------------------------------------------------- # _names_to_funcs() # # Given a list of regular expression function names, this converts it back to # functions. # ----------------------------------------------------------------------------- def _names_to_funcs(namelist, fdict): result = [] for n in namelist: if n and n[0]: result.append((fdict[n[0]], n[1])) else: result.append(n) return result # ----------------------------------------------------------------------------- # _form_master_re() # # This function takes a list of all of the regex components and attempts to # form the master regular expression. Given limitations in the Python re # module, it may be necessary to break the master regex into separate expressions. # ----------------------------------------------------------------------------- def _form_master_re(relist, reflags, ldict, toknames): if not relist: return [] regex = '|'.join(relist) try: lexre = re.compile(regex, reflags) # Build the index to function map for the matching engine lexindexfunc = [None] * (max(lexre.groupindex.values()) + 1) lexindexnames = lexindexfunc[:] for f, i in lexre.groupindex.items(): handle = ldict.get(f, None) if type(handle) in (types.FunctionType, types.MethodType): lexindexfunc[i] = (handle, toknames[f]) lexindexnames[i] = f elif handle is not None: lexindexnames[i] = f if f.find('ignore_') > 0: lexindexfunc[i] = (None, None) else: lexindexfunc[i] = (None, toknames[f]) return [(lexre, lexindexfunc)], [regex], [lexindexnames] except Exception: m = int(len(relist)/2) if m == 0: m = 1 llist, lre, lnames = _form_master_re(relist[:m], reflags, ldict, toknames) rlist, rre, rnames = _form_master_re(relist[m:], reflags, ldict, toknames) return (llist+rlist), (lre+rre), (lnames+rnames) # ----------------------------------------------------------------------------- # def _statetoken(s,names) # # Given a declaration name s of the form "t_" and a dictionary whose keys are # state names, this function returns a tuple (states,tokenname) where states # is a tuple of state names and tokenname is the name of the token. For example, # calling this with s = "t_foo_bar_SPAM" might return (('foo','bar'),'SPAM') # ----------------------------------------------------------------------------- def _statetoken(s, names): parts = s.split('_') for i, part in enumerate(parts[1:], 1): if part not in names and part != 'ANY': break if i > 1: states = tuple(parts[1:i]) else: states = ('INITIAL',) if 'ANY' in states: states = tuple(names) tokenname = '_'.join(parts[i:]) return (states, tokenname) # ----------------------------------------------------------------------------- # LexerReflect() # # This class represents information needed to build a lexer as extracted from a # user's input file. # ----------------------------------------------------------------------------- class LexerReflect(object): def __init__(self, ldict, log=None, reflags=0): self.ldict = ldict self.error_func = None self.tokens = [] self.reflags = reflags self.stateinfo = {'INITIAL': 'inclusive'} self.modules = set() self.error = False self.log = PlyLogger(sys.stderr) if log is None else log # Get all of the basic information def get_all(self): self.get_tokens() self.get_literals() self.get_states() self.get_rules() # Validate all of the information def validate_all(self): self.validate_tokens() self.validate_literals() self.validate_rules() return self.error # Get the tokens map def get_tokens(self): tokens = self.ldict.get('tokens', None) if not tokens: self.log.error('No token list is defined') self.error = True return if not isinstance(tokens, (list, tuple)): self.log.error('tokens must be a list or tuple') self.error = True return if not tokens: self.log.error('tokens is empty') self.error = True return self.tokens = tokens # Validate the tokens def validate_tokens(self): terminals = {} for n in self.tokens: if not _is_identifier.match(n): self.log.error("Bad token name '%s'", n) self.error = True if n in terminals: self.log.warning("Token '%s' multiply defined", n) terminals[n] = 1 # Get the literals specifier def get_literals(self): self.literals = self.ldict.get('literals', '') if not self.literals: self.literals = '' # Validate literals def validate_literals(self): try: for c in self.literals: if not isinstance(c, StringTypes) or len(c) > 1: self.log.error('Invalid literal %s. Must be a single character', repr(c)) self.error = True except TypeError: self.log.error('Invalid literals specification. literals must be a sequence of characters') self.error = True def get_states(self): self.states = self.ldict.get('states', None) # Build statemap if self.states: if not isinstance(self.states, (tuple, list)): self.log.error('states must be defined as a tuple or list') self.error = True else: for s in self.states: if not isinstance(s, tuple) or len(s) != 2: self.log.error("Invalid state specifier %s. Must be a tuple (statename,'exclusive|inclusive')", repr(s)) self.error = True continue name, statetype = s if not isinstance(name, StringTypes): self.log.error('State name %s must be a string', repr(name)) self.error = True continue if not (statetype == 'inclusive' or statetype == 'exclusive'): self.log.error("State type for state %s must be 'inclusive' or 'exclusive'", name) self.error = True continue if name in self.stateinfo: self.log.error("State '%s' already defined", name) self.error = True continue self.stateinfo[name] = statetype # Get all of the symbols with a t_ prefix and sort them into various # categories (functions, strings, error functions, and ignore characters) def get_rules(self): tsymbols = [f for f in self.ldict if f[:2] == 't_'] # Now build up a list of functions and a list of strings self.toknames = {} # Mapping of symbols to token names self.funcsym = {} # Symbols defined as functions self.strsym = {} # Symbols defined as strings self.ignore = {} # Ignore strings by state self.errorf = {} # Error functions by state self.eoff = {} # EOF functions by state for s in self.stateinfo: self.funcsym[s] = [] self.strsym[s] = [] if len(tsymbols) == 0: self.log.error('No rules of the form t_rulename are defined') self.error = True return for f in tsymbols: t = self.ldict[f] states, tokname = _statetoken(f, self.stateinfo) self.toknames[f] = tokname if hasattr(t, '__call__'): if tokname == 'error': for s in states: self.errorf[s] = t elif tokname == 'eof': for s in states: self.eoff[s] = t elif tokname == 'ignore': line = t.__code__.co_firstlineno file = t.__code__.co_filename self.log.error("%s:%d: Rule '%s' must be defined as a string", file, line, t.__name__) self.error = True else: for s in states: self.funcsym[s].append((f, t)) elif isinstance(t, StringTypes): if tokname == 'ignore': for s in states: self.ignore[s] = t if '\\' in t: self.log.warning("%s contains a literal backslash '\\'", f) elif tokname == 'error': self.log.error("Rule '%s' must be defined as a function", f) self.error = True else: for s in states: self.strsym[s].append((f, t)) else: self.log.error('%s not defined as a function or string', f) self.error = True # Sort the functions by line number for f in self.funcsym.values(): f.sort(key=lambda x: x[1].__code__.co_firstlineno) # Sort the strings by regular expression length for s in self.strsym.values(): s.sort(key=lambda x: len(x[1]), reverse=True) # Validate all of the t_rules collected def validate_rules(self): for state in self.stateinfo: # Validate all rules defined by functions for fname, f in self.funcsym[state]: line = f.__code__.co_firstlineno file = f.__code__.co_filename module = inspect.getmodule(f) self.modules.add(module) tokname = self.toknames[fname] if isinstance(f, types.MethodType): reqargs = 2 else: reqargs = 1 nargs = f.__code__.co_argcount if nargs > reqargs: self.log.error("%s:%d: Rule '%s' has too many arguments", file, line, f.__name__) self.error = True continue if nargs < reqargs: self.log.error("%s:%d: Rule '%s' requires an argument", file, line, f.__name__) self.error = True continue if not _get_regex(f): self.log.error("%s:%d: No regular expression defined for rule '%s'", file, line, f.__name__) self.error = True continue try: c = re.compile('(?P<%s>%s)' % (fname, _get_regex(f)), self.reflags) if c.match(''): self.log.error("%s:%d: Regular expression for rule '%s' matches empty string", file, line, f.__name__) self.error = True except re.error as e: self.log.error("%s:%d: Invalid regular expression for rule '%s'. %s", file, line, f.__name__, e) if '#' in _get_regex(f): self.log.error("%s:%d. Make sure '#' in rule '%s' is escaped with '\\#'", file, line, f.__name__) self.error = True # Validate all rules defined by strings for name, r in self.strsym[state]: tokname = self.toknames[name] if tokname == 'error': self.log.error("Rule '%s' must be defined as a function", name) self.error = True continue if tokname not in self.tokens and tokname.find('ignore_') < 0: self.log.error("Rule '%s' defined for an unspecified token %s", name, tokname) self.error = True continue try: c = re.compile('(?P<%s>%s)' % (name, r), self.reflags) if (c.match('')): self.log.error("Regular expression for rule '%s' matches empty string", name) self.error = True except re.error as e: self.log.error("Invalid regular expression for rule '%s'. %s", name, e) if '#' in r: self.log.error("Make sure '#' in rule '%s' is escaped with '\\#'", name) self.error = True if not self.funcsym[state] and not self.strsym[state]: self.log.error("No rules defined for state '%s'", state) self.error = True # Validate the error function efunc = self.errorf.get(state, None) if efunc: f = efunc line = f.__code__.co_firstlineno file = f.__code__.co_filename module = inspect.getmodule(f) self.modules.add(module) if isinstance(f, types.MethodType): reqargs = 2 else: reqargs = 1 nargs = f.__code__.co_argcount if nargs > reqargs: self.log.error("%s:%d: Rule '%s' has too many arguments", file, line, f.__name__) self.error = True if nargs < reqargs: self.log.error("%s:%d: Rule '%s' requires an argument", file, line, f.__name__) self.error = True for module in self.modules: self.validate_module(module) # ----------------------------------------------------------------------------- # validate_module() # # This checks to see if there are duplicated t_rulename() functions or strings # in the parser input file. This is done using a simple regular expression # match on each line in the source code of the given module. # ----------------------------------------------------------------------------- def validate_module(self, module): try: lines, linen = inspect.getsourcelines(module) except IOError: return fre = re.compile(r'\s*def\s+(t_[a-zA-Z_0-9]*)\(') sre = re.compile(r'\s*(t_[a-zA-Z_0-9]*)\s*=') counthash = {} linen += 1 for line in lines: m = fre.match(line) if not m: m = sre.match(line) if m: name = m.group(1) prev = counthash.get(name) if not prev: counthash[name] = linen else: filename = inspect.getsourcefile(module) self.log.error('%s:%d: Rule %s redefined. Previously defined on line %d', filename, linen, name, prev) self.error = True linen += 1 # ----------------------------------------------------------------------------- # lex(module) # # Build all of the regular expression rules from definitions in the supplied module # ----------------------------------------------------------------------------- def lex(module=None, object=None, debug=False, optimize=False, lextab='lextab', reflags=int(re.VERBOSE), nowarn=False, outputdir=None, debuglog=None, errorlog=None): if lextab is None: lextab = 'lextab' global lexer ldict = None stateinfo = {'INITIAL': 'inclusive'} lexobj = Lexer() lexobj.lexoptimize = optimize global token, input if errorlog is None: errorlog = PlyLogger(sys.stderr) if debug: if debuglog is None: debuglog = PlyLogger(sys.stderr) # Get the module dictionary used for the lexer if object: module = object # Get the module dictionary used for the parser if module: _items = [(k, getattr(module, k)) for k in dir(module)] ldict = dict(_items) # If no __file__ attribute is available, try to obtain it from the __module__ instead if '__file__' not in ldict: ldict['__file__'] = sys.modules[ldict['__module__']].__file__ else: ldict = get_caller_module_dict(2) # Determine if the module is package of a package or not. # If so, fix the tabmodule setting so that tables load correctly pkg = ldict.get('__package__') if pkg and isinstance(lextab, str): if '.' not in lextab: lextab = pkg + '.' + lextab # Collect parser information from the dictionary linfo = LexerReflect(ldict, log=errorlog, reflags=reflags) linfo.get_all() if not optimize: if linfo.validate_all(): raise SyntaxError("Can't build lexer") if optimize and lextab: try: lexobj.readtab(lextab, ldict) token = lexobj.token input = lexobj.input lexer = lexobj return lexobj except ImportError: pass # Dump some basic debugging information if debug: debuglog.info('lex: tokens = %r', linfo.tokens) debuglog.info('lex: literals = %r', linfo.literals) debuglog.info('lex: states = %r', linfo.stateinfo) # Build a dictionary of valid token names lexobj.lextokens = set() for n in linfo.tokens: lexobj.lextokens.add(n) # Get literals specification if isinstance(linfo.literals, (list, tuple)): lexobj.lexliterals = type(linfo.literals[0])().join(linfo.literals) else: lexobj.lexliterals = linfo.literals lexobj.lextokens_all = lexobj.lextokens | set(lexobj.lexliterals) # Get the stateinfo dictionary stateinfo = linfo.stateinfo regexs = {} # Build the master regular expressions for state in stateinfo: regex_list = [] # Add rules defined by functions first for fname, f in linfo.funcsym[state]: regex_list.append('(?P<%s>%s)' % (fname, _get_regex(f))) if debug: debuglog.info("lex: Adding rule %s -> '%s' (state '%s')", fname, _get_regex(f), state) # Now add all of the simple rules for name, r in linfo.strsym[state]: regex_list.append('(?P<%s>%s)' % (name, r)) if debug: debuglog.info("lex: Adding rule %s -> '%s' (state '%s')", name, r, state) regexs[state] = regex_list # Build the master regular expressions if debug: debuglog.info('lex: ==== MASTER REGEXS FOLLOW ====') for state in regexs: lexre, re_text, re_names = _form_master_re(regexs[state], reflags, ldict, linfo.toknames) lexobj.lexstatere[state] = lexre lexobj.lexstateretext[state] = re_text lexobj.lexstaterenames[state] = re_names if debug: for i, text in enumerate(re_text): debuglog.info("lex: state '%s' : regex[%d] = '%s'", state, i, text) # For inclusive states, we need to add the regular expressions from the INITIAL state for state, stype in stateinfo.items(): if state != 'INITIAL' and stype == 'inclusive': lexobj.lexstatere[state].extend(lexobj.lexstatere['INITIAL']) lexobj.lexstateretext[state].extend(lexobj.lexstateretext['INITIAL']) lexobj.lexstaterenames[state].extend(lexobj.lexstaterenames['INITIAL']) lexobj.lexstateinfo = stateinfo lexobj.lexre = lexobj.lexstatere['INITIAL'] lexobj.lexretext = lexobj.lexstateretext['INITIAL'] lexobj.lexreflags = reflags # Set up ignore variables lexobj.lexstateignore = linfo.ignore lexobj.lexignore = lexobj.lexstateignore.get('INITIAL', '') # Set up error functions lexobj.lexstateerrorf = linfo.errorf lexobj.lexerrorf = linfo.errorf.get('INITIAL', None) if not lexobj.lexerrorf: errorlog.warning('No t_error rule is defined') # Set up eof functions lexobj.lexstateeoff = linfo.eoff lexobj.lexeoff = linfo.eoff.get('INITIAL', None) # Check state information for ignore and error rules for s, stype in stateinfo.items(): if stype == 'exclusive': if s not in linfo.errorf: errorlog.warning("No error rule is defined for exclusive state '%s'", s) if s not in linfo.ignore and lexobj.lexignore: errorlog.warning("No ignore rule is defined for exclusive state '%s'", s) elif stype == 'inclusive': if s not in linfo.errorf: linfo.errorf[s] = linfo.errorf.get('INITIAL', None) if s not in linfo.ignore: linfo.ignore[s] = linfo.ignore.get('INITIAL', '') # Create global versions of the token() and input() functions token = lexobj.token input = lexobj.input lexer = lexobj # If in optimize mode, we write the lextab if lextab and optimize: if outputdir is None: # If no output directory is set, the location of the output files # is determined according to the following rules: # - If lextab specifies a package, files go into that package directory # - Otherwise, files go in the same directory as the specifying module if isinstance(lextab, types.ModuleType): srcfile = lextab.__file__ else: if '.' not in lextab: srcfile = ldict['__file__'] else: parts = lextab.split('.') pkgname = '.'.join(parts[:-1]) exec('import %s' % pkgname) srcfile = getattr(sys.modules[pkgname], '__file__', '') outputdir = os.path.dirname(srcfile) try: lexobj.writetab(lextab, outputdir) if lextab in sys.modules: del sys.modules[lextab] except IOError as e: errorlog.warning("Couldn't write lextab module %r. %s" % (lextab, e)) return lexobj # ----------------------------------------------------------------------------- # runmain() # # This runs the lexer as a main program # ----------------------------------------------------------------------------- def runmain(lexer=None, data=None): if not data: try: filename = sys.argv[1] with open(filename) as f: data = f.read() except IndexError: sys.stdout.write('Reading from standard input (type EOF to end):\n') data = sys.stdin.read() if lexer: _input = lexer.input else: _input = input _input(data) if lexer: _token = lexer.token else: _token = token while True: tok = _token() if not tok: break sys.stdout.write('(%s,%r,%d,%d)\n' % (tok.type, tok.value, tok.lineno, tok.lexpos)) # ----------------------------------------------------------------------------- # @TOKEN(regex) # # This decorator function can be used to set the regex expression on a function # when its docstring might need to be set in an alternative way # ----------------------------------------------------------------------------- def TOKEN(r): def set_regex(f): if hasattr(r, '__call__'): f.regex = _get_regex(r) else: f.regex = r return f return set_regex # Alternative spelling of the TOKEN decorator Token = TOKEN ================================================ FILE: xonsh/ply/ply/yacc.py ================================================ # ----------------------------------------------------------------------------- # ply: yacc.py # # Copyright (C) 2001-2019 # David M. Beazley (Dabeaz LLC) # All rights reserved. # # Latest version: https://github.com/dabeaz/ply # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # * Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright notice, # this list of conditions and the following disclaimer in the documentation # and/or other materials provided with the distribution. # * Neither the name of David Beazley or Dabeaz LLC may be used to # endorse or promote products derived from this software without # specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # ----------------------------------------------------------------------------- # # This implements an LR parser that is constructed from grammar rules defined # as Python functions. The grammar is specified by supplying the BNF inside # Python documentation strings. The inspiration for this technique was borrowed # from John Aycock's Spark parsing system. PLY might be viewed as cross between # Spark and the GNU bison utility. # # The current implementation is only somewhat object-oriented. The # LR parser itself is defined in terms of an object (which allows multiple # parsers to co-exist). However, most of the variables used during table # construction are defined in terms of global variables. Users shouldn't # notice unless they are trying to define multiple parsers at the same # time using threads (in which case they should have their head examined). # # This implementation supports both SLR and LALR(1) parsing. LALR(1) # support was originally implemented by Elias Ioup (ezioup@alumni.uchicago.edu), # using the algorithm found in Aho, Sethi, and Ullman "Compilers: Principles, # Techniques, and Tools" (The Dragon Book). LALR(1) has since been replaced # by the more efficient DeRemer and Pennello algorithm. # # :::::::: WARNING ::::::: # # Construction of LR parsing tables is fairly complicated and expensive. # To make this module run fast, a *LOT* of work has been put into # optimization---often at the expensive of readability and what might # consider to be good Python "coding style." Modify the code at your # own risk! # ---------------------------------------------------------------------------- import re import types import sys import os.path import inspect import warnings __version__ = '3.11' __tabversion__ = '3.10' #----------------------------------------------------------------------------- # === User configurable parameters === # # Change these to modify the default behavior of yacc (if you wish) #----------------------------------------------------------------------------- yaccdebug = True # Debugging mode. If set, yacc generates a # a 'parser.out' file in the current directory debug_file = 'parser.out' # Default name of the debugging file tab_module = 'parsetab' # Default name of the table module default_lr = 'LALR' # Default LR table generation method error_count = 3 # Number of symbols that must be shifted to leave recovery mode yaccdevel = False # Set to True if developing yacc. This turns off optimized # implementations of certain functions. resultlimit = 40 # Size limit of results when running in debug mode. pickle_protocol = 0 # Protocol to use when writing pickle files # String type-checking compatibility if sys.version_info[0] < 3: string_types = basestring else: string_types = str MAXINT = sys.maxsize # This object is a stand-in for a logging object created by the # logging module. PLY will use this by default to create things # such as the parser.out file. If a user wants more detailed # information, they can create their own logging object and pass # it into PLY. class PlyLogger(object): def __init__(self, f): self.f = f def debug(self, msg, *args, **kwargs): self.f.write((msg % args) + '\n') info = debug def warning(self, msg, *args, **kwargs): self.f.write('WARNING: ' + (msg % args) + '\n') def error(self, msg, *args, **kwargs): self.f.write('ERROR: ' + (msg % args) + '\n') critical = debug # Null logger is used when no output is generated. Does nothing. class NullLogger(object): def __getattribute__(self, name): return self def __call__(self, *args, **kwargs): return self # Exception raised for yacc-related errors class YaccError(Exception): pass # Format the result message that the parser produces when running in debug mode. def format_result(r): repr_str = repr(r) if '\n' in repr_str: repr_str = repr(repr_str) if len(repr_str) > resultlimit: repr_str = repr_str[:resultlimit] + ' ...' result = '<%s @ 0x%x> (%s)' % (type(r).__name__, id(r), repr_str) return result # Format stack entries when the parser is running in debug mode def format_stack_entry(r): repr_str = repr(r) if '\n' in repr_str: repr_str = repr(repr_str) if len(repr_str) < 16: return repr_str else: return '<%s @ 0x%x>' % (type(r).__name__, id(r)) # Panic mode error recovery support. This feature is being reworked--much of the # code here is to offer a deprecation/backwards compatible transition _errok = None _token = None _restart = None _warnmsg = '''PLY: Don't use global functions errok(), token(), and restart() in p_error(). Instead, invoke the methods on the associated parser instance: def p_error(p): ... # Use parser.errok(), parser.token(), parser.restart() ... parser = yacc.yacc() ''' def errok(): warnings.warn(_warnmsg) return _errok() def restart(): warnings.warn(_warnmsg) return _restart() def token(): warnings.warn(_warnmsg) return _token() # Utility function to call the p_error() function with some deprecation hacks def call_errorfunc(errorfunc, token, parser): global _errok, _token, _restart _errok = parser.errok _token = parser.token _restart = parser.restart r = errorfunc(token) try: del _errok, _token, _restart except NameError: pass return r #----------------------------------------------------------------------------- # === LR Parsing Engine === # # The following classes are used for the LR parser itself. These are not # used during table construction and are independent of the actual LR # table generation algorithm #----------------------------------------------------------------------------- # This class is used to hold non-terminal grammar symbols during parsing. # It normally has the following attributes set: # .type = Grammar symbol type # .value = Symbol value # .lineno = Starting line number # .endlineno = Ending line number (optional, set automatically) # .lexpos = Starting lex position # .endlexpos = Ending lex position (optional, set automatically) class YaccSymbol: def __str__(self): return self.type def __repr__(self): return str(self) # This class is a wrapper around the objects actually passed to each # grammar rule. Index lookup and assignment actually assign the # .value attribute of the underlying YaccSymbol object. # The lineno() method returns the line number of a given # item (or 0 if not defined). The linespan() method returns # a tuple of (startline,endline) representing the range of lines # for a symbol. The lexspan() method returns a tuple (lexpos,endlexpos) # representing the range of positional information for a symbol. class YaccProduction: def __init__(self, s, stack=None): self.slice = s self.stack = stack self.lexer = None self.parser = None def __getitem__(self, n): if isinstance(n, slice): return [s.value for s in self.slice[n]] elif n >= 0: return self.slice[n].value else: return self.stack[n].value def __setitem__(self, n, v): self.slice[n].value = v def __getslice__(self, i, j): return [s.value for s in self.slice[i:j]] def __len__(self): return len(self.slice) def lineno(self, n): return getattr(self.slice[n], 'lineno', 0) def set_lineno(self, n, lineno): self.slice[n].lineno = lineno def linespan(self, n): startline = getattr(self.slice[n], 'lineno', 0) endline = getattr(self.slice[n], 'endlineno', startline) return startline, endline def lexpos(self, n): return getattr(self.slice[n], 'lexpos', 0) def set_lexpos(self, n, lexpos): self.slice[n].lexpos = lexpos def lexspan(self, n): startpos = getattr(self.slice[n], 'lexpos', 0) endpos = getattr(self.slice[n], 'endlexpos', startpos) return startpos, endpos def error(self): raise SyntaxError # ----------------------------------------------------------------------------- # == LRParser == # # The LR Parsing engine. # ----------------------------------------------------------------------------- class LRParser: def __init__(self, lrtab, errorf): self.productions = lrtab.lr_productions self.action = lrtab.lr_action self.goto = lrtab.lr_goto self.errorfunc = errorf self.set_defaulted_states() self.errorok = True def errok(self): self.errorok = True def restart(self): del self.statestack[:] del self.symstack[:] sym = YaccSymbol() sym.type = '$end' self.symstack.append(sym) self.statestack.append(0) # Defaulted state support. # This method identifies parser states where there is only one possible reduction action. # For such states, the parser can make a choose to make a rule reduction without consuming # the next look-ahead token. This delayed invocation of the tokenizer can be useful in # certain kinds of advanced parsing situations where the lexer and parser interact with # each other or change states (i.e., manipulation of scope, lexer states, etc.). # # See: http://www.gnu.org/software/bison/manual/html_node/Default-Reductions.html#Default-Reductions def set_defaulted_states(self): self.defaulted_states = {} for state, actions in self.action.items(): rules = list(actions.values()) if len(rules) == 1 and rules[0] < 0: self.defaulted_states[state] = rules[0] def disable_defaulted_states(self): self.defaulted_states = {} def parse(self, input=None, lexer=None, debug=False, tracking=False, tokenfunc=None): if debug or yaccdevel: if isinstance(debug, int): debug = PlyLogger(sys.stderr) return self.parsedebug(input, lexer, debug, tracking, tokenfunc) elif tracking: return self.parseopt(input, lexer, debug, tracking, tokenfunc) else: return self.parseopt_notrack(input, lexer, debug, tracking, tokenfunc) # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! # parsedebug(). # # This is the debugging enabled version of parse(). All changes made to the # parsing engine should be made here. Optimized versions of this function # are automatically created by the ply/ygen.py script. This script cuts out # sections enclosed in markers such as this: # # #--! DEBUG # statements # #--! DEBUG # # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! def parsedebug(self, input=None, lexer=None, debug=False, tracking=False, tokenfunc=None): #--! parsedebug-start lookahead = None # Current lookahead symbol lookaheadstack = [] # Stack of lookahead symbols actions = self.action # Local reference to action table (to avoid lookup on self.) goto = self.goto # Local reference to goto table (to avoid lookup on self.) prod = self.productions # Local reference to production list (to avoid lookup on self.) defaulted_states = self.defaulted_states # Local reference to defaulted states pslice = YaccProduction(None) # Production object passed to grammar rules errorcount = 0 # Used during error recovery #--! DEBUG debug.info('PLY: PARSE DEBUG START') #--! DEBUG # If no lexer was given, we will try to use the lex module if not lexer: from . import lex lexer = lex.lexer # Set up the lexer and parser objects on pslice pslice.lexer = lexer pslice.parser = self # If input was supplied, pass to lexer if input is not None: lexer.input(input) if tokenfunc is None: # Tokenize function get_token = lexer.token else: get_token = tokenfunc # Set the parser() token method (sometimes used in error recovery) self.token = get_token # Set up the state and symbol stacks statestack = [] # Stack of parsing states self.statestack = statestack symstack = [] # Stack of grammar symbols self.symstack = symstack pslice.stack = symstack # Put in the production errtoken = None # Err token # The start state is assumed to be (0,$end) statestack.append(0) sym = YaccSymbol() sym.type = '$end' symstack.append(sym) state = 0 while True: # Get the next symbol on the input. If a lookahead symbol # is already set, we just use that. Otherwise, we'll pull # the next token off of the lookaheadstack or from the lexer #--! DEBUG debug.debug('') debug.debug('State : %s', state) #--! DEBUG if state not in defaulted_states: if not lookahead: if not lookaheadstack: lookahead = get_token() # Get the next token else: lookahead = lookaheadstack.pop() if not lookahead: lookahead = YaccSymbol() lookahead.type = '$end' # Check the action table ltype = lookahead.type t = actions[state].get(ltype) else: t = defaulted_states[state] #--! DEBUG debug.debug('Defaulted state %s: Reduce using %d', state, -t) #--! DEBUG #--! DEBUG debug.debug('Stack : %s', ('%s . %s' % (' '.join([xx.type for xx in symstack][1:]), str(lookahead))).lstrip()) #--! DEBUG if t is not None: if t > 0: # shift a symbol on the stack statestack.append(t) state = t #--! DEBUG debug.debug('Action : Shift and goto state %s', t) #--! DEBUG symstack.append(lookahead) lookahead = None # Decrease error count on successful shift if errorcount: errorcount -= 1 continue if t < 0: # reduce a symbol on the stack, emit a production p = prod[-t] pname = p.name plen = p.len # Get production function sym = YaccSymbol() sym.type = pname # Production name sym.value = None #--! DEBUG if plen: debug.info('Action : Reduce rule [%s] with %s and goto state %d', p.str, '['+','.join([format_stack_entry(_v.value) for _v in symstack[-plen:]])+']', goto[statestack[-1-plen]][pname]) else: debug.info('Action : Reduce rule [%s] with %s and goto state %d', p.str, [], goto[statestack[-1]][pname]) #--! DEBUG if plen: targ = symstack[-plen-1:] targ[0] = sym #--! TRACKING if tracking: t1 = targ[1] sym.lineno = t1.lineno sym.lexpos = t1.lexpos t1 = targ[-1] sym.endlineno = getattr(t1, 'endlineno', t1.lineno) sym.endlexpos = getattr(t1, 'endlexpos', t1.lexpos) #--! TRACKING # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! # The code enclosed in this section is duplicated # below as a performance optimization. Make sure # changes get made in both locations. pslice.slice = targ try: # Call the grammar rule with our special slice object del symstack[-plen:] self.state = state p.callable(pslice) del statestack[-plen:] #--! DEBUG debug.info('Result : %s', format_result(pslice[0])) #--! DEBUG symstack.append(sym) state = goto[statestack[-1]][pname] statestack.append(state) except SyntaxError: # If an error was set. Enter error recovery state lookaheadstack.append(lookahead) # Save the current lookahead token symstack.extend(targ[1:-1]) # Put the production slice back on the stack statestack.pop() # Pop back one state (before the reduce) state = statestack[-1] sym.type = 'error' sym.value = 'error' lookahead = sym errorcount = error_count self.errorok = False continue # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! else: #--! TRACKING if tracking: sym.lineno = lexer.lineno sym.lexpos = lexer.lexpos #--! TRACKING targ = [sym] # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! # The code enclosed in this section is duplicated # above as a performance optimization. Make sure # changes get made in both locations. pslice.slice = targ try: # Call the grammar rule with our special slice object self.state = state p.callable(pslice) #--! DEBUG debug.info('Result : %s', format_result(pslice[0])) #--! DEBUG symstack.append(sym) state = goto[statestack[-1]][pname] statestack.append(state) except SyntaxError: # If an error was set. Enter error recovery state lookaheadstack.append(lookahead) # Save the current lookahead token statestack.pop() # Pop back one state (before the reduce) state = statestack[-1] sym.type = 'error' sym.value = 'error' lookahead = sym errorcount = error_count self.errorok = False continue # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! if t == 0: n = symstack[-1] result = getattr(n, 'value', None) #--! DEBUG debug.info('Done : Returning %s', format_result(result)) debug.info('PLY: PARSE DEBUG END') #--! DEBUG return result if t is None: #--! DEBUG debug.error('Error : %s', ('%s . %s' % (' '.join([xx.type for xx in symstack][1:]), str(lookahead))).lstrip()) #--! DEBUG # We have some kind of parsing error here. To handle # this, we are going to push the current token onto # the tokenstack and replace it with an 'error' token. # If there are any synchronization rules, they may # catch it. # # In addition to pushing the error token, we call call # the user defined p_error() function if this is the # first syntax error. This function is only called if # errorcount == 0. if errorcount == 0 or self.errorok: errorcount = error_count self.errorok = False errtoken = lookahead if errtoken.type == '$end': errtoken = None # End of file! if self.errorfunc: if errtoken and not hasattr(errtoken, 'lexer'): errtoken.lexer = lexer self.state = state tok = call_errorfunc(self.errorfunc, errtoken, self) if self.errorok: # User must have done some kind of panic # mode recovery on their own. The # returned token is the next lookahead lookahead = tok errtoken = None continue else: if errtoken: if hasattr(errtoken, 'lineno'): lineno = lookahead.lineno else: lineno = 0 if lineno: sys.stderr.write('yacc: Syntax error at line %d, token=%s\n' % (lineno, errtoken.type)) else: sys.stderr.write('yacc: Syntax error, token=%s' % errtoken.type) else: sys.stderr.write('yacc: Parse error in input. EOF\n') return else: errorcount = error_count # case 1: the statestack only has 1 entry on it. If we're in this state, the # entire parse has been rolled back and we're completely hosed. The token is # discarded and we just keep going. if len(statestack) <= 1 and lookahead.type != '$end': lookahead = None errtoken = None state = 0 # Nuke the pushback stack del lookaheadstack[:] continue # case 2: the statestack has a couple of entries on it, but we're # at the end of the file. nuke the top entry and generate an error token # Start nuking entries on the stack if lookahead.type == '$end': # Whoa. We're really hosed here. Bail out return if lookahead.type != 'error': sym = symstack[-1] if sym.type == 'error': # Hmmm. Error is on top of stack, we'll just nuke input # symbol and continue #--! TRACKING if tracking: sym.endlineno = getattr(lookahead, 'lineno', sym.lineno) sym.endlexpos = getattr(lookahead, 'lexpos', sym.lexpos) #--! TRACKING lookahead = None continue # Create the error symbol for the first time and make it the new lookahead symbol t = YaccSymbol() t.type = 'error' if hasattr(lookahead, 'lineno'): t.lineno = t.endlineno = lookahead.lineno if hasattr(lookahead, 'lexpos'): t.lexpos = t.endlexpos = lookahead.lexpos t.value = lookahead lookaheadstack.append(lookahead) lookahead = t else: sym = symstack.pop() #--! TRACKING if tracking: lookahead.lineno = sym.lineno lookahead.lexpos = sym.lexpos #--! TRACKING statestack.pop() state = statestack[-1] continue # Call an error function here raise RuntimeError('yacc: internal parser error!!!\n') #--! parsedebug-end # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! # parseopt(). # # Optimized version of parse() method. DO NOT EDIT THIS CODE DIRECTLY! # This code is automatically generated by the ply/ygen.py script. Make # changes to the parsedebug() method instead. # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! def parseopt(self, input=None, lexer=None, debug=False, tracking=False, tokenfunc=None): #--! parseopt-start lookahead = None # Current lookahead symbol lookaheadstack = [] # Stack of lookahead symbols actions = self.action # Local reference to action table (to avoid lookup on self.) goto = self.goto # Local reference to goto table (to avoid lookup on self.) prod = self.productions # Local reference to production list (to avoid lookup on self.) defaulted_states = self.defaulted_states # Local reference to defaulted states pslice = YaccProduction(None) # Production object passed to grammar rules errorcount = 0 # Used during error recovery # If no lexer was given, we will try to use the lex module if not lexer: from . import lex lexer = lex.lexer # Set up the lexer and parser objects on pslice pslice.lexer = lexer pslice.parser = self # If input was supplied, pass to lexer if input is not None: lexer.input(input) if tokenfunc is None: # Tokenize function get_token = lexer.token else: get_token = tokenfunc # Set the parser() token method (sometimes used in error recovery) self.token = get_token # Set up the state and symbol stacks statestack = [] # Stack of parsing states self.statestack = statestack symstack = [] # Stack of grammar symbols self.symstack = symstack pslice.stack = symstack # Put in the production errtoken = None # Err token # The start state is assumed to be (0,$end) statestack.append(0) sym = YaccSymbol() sym.type = '$end' symstack.append(sym) state = 0 while True: # Get the next symbol on the input. If a lookahead symbol # is already set, we just use that. Otherwise, we'll pull # the next token off of the lookaheadstack or from the lexer if state not in defaulted_states: if not lookahead: if not lookaheadstack: lookahead = get_token() # Get the next token else: lookahead = lookaheadstack.pop() if not lookahead: lookahead = YaccSymbol() lookahead.type = '$end' # Check the action table ltype = lookahead.type t = actions[state].get(ltype) else: t = defaulted_states[state] if t is not None: if t > 0: # shift a symbol on the stack statestack.append(t) state = t symstack.append(lookahead) lookahead = None # Decrease error count on successful shift if errorcount: errorcount -= 1 continue if t < 0: # reduce a symbol on the stack, emit a production p = prod[-t] pname = p.name plen = p.len # Get production function sym = YaccSymbol() sym.type = pname # Production name sym.value = None if plen: targ = symstack[-plen-1:] targ[0] = sym #--! TRACKING if tracking: t1 = targ[1] sym.lineno = t1.lineno sym.lexpos = t1.lexpos t1 = targ[-1] sym.endlineno = getattr(t1, 'endlineno', t1.lineno) sym.endlexpos = getattr(t1, 'endlexpos', t1.lexpos) #--! TRACKING # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! # The code enclosed in this section is duplicated # below as a performance optimization. Make sure # changes get made in both locations. pslice.slice = targ try: # Call the grammar rule with our special slice object del symstack[-plen:] self.state = state p.callable(pslice) del statestack[-plen:] symstack.append(sym) state = goto[statestack[-1]][pname] statestack.append(state) except SyntaxError: # If an error was set. Enter error recovery state lookaheadstack.append(lookahead) # Save the current lookahead token symstack.extend(targ[1:-1]) # Put the production slice back on the stack statestack.pop() # Pop back one state (before the reduce) state = statestack[-1] sym.type = 'error' sym.value = 'error' lookahead = sym errorcount = error_count self.errorok = False continue # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! else: #--! TRACKING if tracking: sym.lineno = lexer.lineno sym.lexpos = lexer.lexpos #--! TRACKING targ = [sym] # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! # The code enclosed in this section is duplicated # above as a performance optimization. Make sure # changes get made in both locations. pslice.slice = targ try: # Call the grammar rule with our special slice object self.state = state p.callable(pslice) symstack.append(sym) state = goto[statestack[-1]][pname] statestack.append(state) except SyntaxError: # If an error was set. Enter error recovery state lookaheadstack.append(lookahead) # Save the current lookahead token statestack.pop() # Pop back one state (before the reduce) state = statestack[-1] sym.type = 'error' sym.value = 'error' lookahead = sym errorcount = error_count self.errorok = False continue # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! if t == 0: n = symstack[-1] result = getattr(n, 'value', None) return result if t is None: # We have some kind of parsing error here. To handle # this, we are going to push the current token onto # the tokenstack and replace it with an 'error' token. # If there are any synchronization rules, they may # catch it. # # In addition to pushing the error token, we call call # the user defined p_error() function if this is the # first syntax error. This function is only called if # errorcount == 0. if errorcount == 0 or self.errorok: errorcount = error_count self.errorok = False errtoken = lookahead if errtoken.type == '$end': errtoken = None # End of file! if self.errorfunc: if errtoken and not hasattr(errtoken, 'lexer'): errtoken.lexer = lexer self.state = state tok = call_errorfunc(self.errorfunc, errtoken, self) if self.errorok: # User must have done some kind of panic # mode recovery on their own. The # returned token is the next lookahead lookahead = tok errtoken = None continue else: if errtoken: if hasattr(errtoken, 'lineno'): lineno = lookahead.lineno else: lineno = 0 if lineno: sys.stderr.write('yacc: Syntax error at line %d, token=%s\n' % (lineno, errtoken.type)) else: sys.stderr.write('yacc: Syntax error, token=%s' % errtoken.type) else: sys.stderr.write('yacc: Parse error in input. EOF\n') return else: errorcount = error_count # case 1: the statestack only has 1 entry on it. If we're in this state, the # entire parse has been rolled back and we're completely hosed. The token is # discarded and we just keep going. if len(statestack) <= 1 and lookahead.type != '$end': lookahead = None errtoken = None state = 0 # Nuke the pushback stack del lookaheadstack[:] continue # case 2: the statestack has a couple of entries on it, but we're # at the end of the file. nuke the top entry and generate an error token # Start nuking entries on the stack if lookahead.type == '$end': # Whoa. We're really hosed here. Bail out return if lookahead.type != 'error': sym = symstack[-1] if sym.type == 'error': # Hmmm. Error is on top of stack, we'll just nuke input # symbol and continue #--! TRACKING if tracking: sym.endlineno = getattr(lookahead, 'lineno', sym.lineno) sym.endlexpos = getattr(lookahead, 'lexpos', sym.lexpos) #--! TRACKING lookahead = None continue # Create the error symbol for the first time and make it the new lookahead symbol t = YaccSymbol() t.type = 'error' if hasattr(lookahead, 'lineno'): t.lineno = t.endlineno = lookahead.lineno if hasattr(lookahead, 'lexpos'): t.lexpos = t.endlexpos = lookahead.lexpos t.value = lookahead lookaheadstack.append(lookahead) lookahead = t else: sym = symstack.pop() #--! TRACKING if tracking: lookahead.lineno = sym.lineno lookahead.lexpos = sym.lexpos #--! TRACKING statestack.pop() state = statestack[-1] continue # Call an error function here raise RuntimeError('yacc: internal parser error!!!\n') #--! parseopt-end # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! # parseopt_notrack(). # # Optimized version of parseopt() with line number tracking removed. # DO NOT EDIT THIS CODE DIRECTLY. This code is automatically generated # by the ply/ygen.py script. Make changes to the parsedebug() method instead. # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! def parseopt_notrack(self, input=None, lexer=None, debug=False, tracking=False, tokenfunc=None): #--! parseopt-notrack-start lookahead = None # Current lookahead symbol lookaheadstack = [] # Stack of lookahead symbols actions = self.action # Local reference to action table (to avoid lookup on self.) goto = self.goto # Local reference to goto table (to avoid lookup on self.) prod = self.productions # Local reference to production list (to avoid lookup on self.) defaulted_states = self.defaulted_states # Local reference to defaulted states pslice = YaccProduction(None) # Production object passed to grammar rules errorcount = 0 # Used during error recovery # If no lexer was given, we will try to use the lex module if not lexer: from . import lex lexer = lex.lexer # Set up the lexer and parser objects on pslice pslice.lexer = lexer pslice.parser = self # If input was supplied, pass to lexer if input is not None: lexer.input(input) if tokenfunc is None: # Tokenize function get_token = lexer.token else: get_token = tokenfunc # Set the parser() token method (sometimes used in error recovery) self.token = get_token # Set up the state and symbol stacks statestack = [] # Stack of parsing states self.statestack = statestack symstack = [] # Stack of grammar symbols self.symstack = symstack pslice.stack = symstack # Put in the production errtoken = None # Err token # The start state is assumed to be (0,$end) statestack.append(0) sym = YaccSymbol() sym.type = '$end' symstack.append(sym) state = 0 while True: # Get the next symbol on the input. If a lookahead symbol # is already set, we just use that. Otherwise, we'll pull # the next token off of the lookaheadstack or from the lexer if state not in defaulted_states: if not lookahead: if not lookaheadstack: lookahead = get_token() # Get the next token else: lookahead = lookaheadstack.pop() if not lookahead: lookahead = YaccSymbol() lookahead.type = '$end' # Check the action table ltype = lookahead.type t = actions[state].get(ltype) else: t = defaulted_states[state] if t is not None: if t > 0: # shift a symbol on the stack statestack.append(t) state = t symstack.append(lookahead) lookahead = None # Decrease error count on successful shift if errorcount: errorcount -= 1 continue if t < 0: # reduce a symbol on the stack, emit a production p = prod[-t] pname = p.name plen = p.len # Get production function sym = YaccSymbol() sym.type = pname # Production name sym.value = None if plen: targ = symstack[-plen-1:] targ[0] = sym # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! # The code enclosed in this section is duplicated # below as a performance optimization. Make sure # changes get made in both locations. pslice.slice = targ try: # Call the grammar rule with our special slice object del symstack[-plen:] self.state = state p.callable(pslice) del statestack[-plen:] symstack.append(sym) state = goto[statestack[-1]][pname] statestack.append(state) except SyntaxError: # If an error was set. Enter error recovery state lookaheadstack.append(lookahead) # Save the current lookahead token symstack.extend(targ[1:-1]) # Put the production slice back on the stack statestack.pop() # Pop back one state (before the reduce) state = statestack[-1] sym.type = 'error' sym.value = 'error' lookahead = sym errorcount = error_count self.errorok = False continue # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! else: targ = [sym] # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! # The code enclosed in this section is duplicated # above as a performance optimization. Make sure # changes get made in both locations. pslice.slice = targ try: # Call the grammar rule with our special slice object self.state = state p.callable(pslice) symstack.append(sym) state = goto[statestack[-1]][pname] statestack.append(state) except SyntaxError: # If an error was set. Enter error recovery state lookaheadstack.append(lookahead) # Save the current lookahead token statestack.pop() # Pop back one state (before the reduce) state = statestack[-1] sym.type = 'error' sym.value = 'error' lookahead = sym errorcount = error_count self.errorok = False continue # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! if t == 0: n = symstack[-1] result = getattr(n, 'value', None) return result if t is None: # We have some kind of parsing error here. To handle # this, we are going to push the current token onto # the tokenstack and replace it with an 'error' token. # If there are any synchronization rules, they may # catch it. # # In addition to pushing the error token, we call call # the user defined p_error() function if this is the # first syntax error. This function is only called if # errorcount == 0. if errorcount == 0 or self.errorok: errorcount = error_count self.errorok = False errtoken = lookahead if errtoken.type == '$end': errtoken = None # End of file! if self.errorfunc: if errtoken and not hasattr(errtoken, 'lexer'): errtoken.lexer = lexer self.state = state tok = call_errorfunc(self.errorfunc, errtoken, self) if self.errorok: # User must have done some kind of panic # mode recovery on their own. The # returned token is the next lookahead lookahead = tok errtoken = None continue else: if errtoken: if hasattr(errtoken, 'lineno'): lineno = lookahead.lineno else: lineno = 0 if lineno: sys.stderr.write('yacc: Syntax error at line %d, token=%s\n' % (lineno, errtoken.type)) else: sys.stderr.write('yacc: Syntax error, token=%s' % errtoken.type) else: sys.stderr.write('yacc: Parse error in input. EOF\n') return else: errorcount = error_count # case 1: the statestack only has 1 entry on it. If we're in this state, the # entire parse has been rolled back and we're completely hosed. The token is # discarded and we just keep going. if len(statestack) <= 1 and lookahead.type != '$end': lookahead = None errtoken = None state = 0 # Nuke the pushback stack del lookaheadstack[:] continue # case 2: the statestack has a couple of entries on it, but we're # at the end of the file. nuke the top entry and generate an error token # Start nuking entries on the stack if lookahead.type == '$end': # Whoa. We're really hosed here. Bail out return if lookahead.type != 'error': sym = symstack[-1] if sym.type == 'error': # Hmmm. Error is on top of stack, we'll just nuke input # symbol and continue lookahead = None continue # Create the error symbol for the first time and make it the new lookahead symbol t = YaccSymbol() t.type = 'error' if hasattr(lookahead, 'lineno'): t.lineno = t.endlineno = lookahead.lineno if hasattr(lookahead, 'lexpos'): t.lexpos = t.endlexpos = lookahead.lexpos t.value = lookahead lookaheadstack.append(lookahead) lookahead = t else: sym = symstack.pop() statestack.pop() state = statestack[-1] continue # Call an error function here raise RuntimeError('yacc: internal parser error!!!\n') #--! parseopt-notrack-end # ----------------------------------------------------------------------------- # === Grammar Representation === # # The following functions, classes, and variables are used to represent and # manipulate the rules that make up a grammar. # ----------------------------------------------------------------------------- # regex matching identifiers _is_identifier = re.compile(r'^[a-zA-Z0-9_-]+$') # ----------------------------------------------------------------------------- # class Production: # # This class stores the raw information about a single production or grammar rule. # A grammar rule refers to a specification such as this: # # expr : expr PLUS term # # Here are the basic attributes defined on all productions # # name - Name of the production. For example 'expr' # prod - A list of symbols on the right side ['expr','PLUS','term'] # prec - Production precedence level # number - Production number. # func - Function that executes on reduce # file - File where production function is defined # lineno - Line number where production function is defined # # The following attributes are defined or optional. # # len - Length of the production (number of symbols on right hand side) # usyms - Set of unique symbols found in the production # ----------------------------------------------------------------------------- class Production(object): reduced = 0 def __init__(self, number, name, prod, precedence=('right', 0), func=None, file='', line=0): self.name = name self.prod = tuple(prod) self.number = number self.func = func self.callable = None self.file = file self.line = line self.prec = precedence # Internal settings used during table construction self.len = len(self.prod) # Length of the production # Create a list of unique production symbols used in the production self.usyms = [] for s in self.prod: if s not in self.usyms: self.usyms.append(s) # List of all LR items for the production self.lr_items = [] self.lr_next = None # Create a string representation if self.prod: self.str = '%s -> %s' % (self.name, ' '.join(self.prod)) else: self.str = '%s -> ' % self.name def __str__(self): return self.str def __repr__(self): return 'Production(' + str(self) + ')' def __len__(self): return len(self.prod) def __nonzero__(self): return 1 def __getitem__(self, index): return self.prod[index] # Return the nth lr_item from the production (or None if at the end) def lr_item(self, n): if n > len(self.prod): return None p = LRItem(self, n) # Precompute the list of productions immediately following. try: p.lr_after = self.Prodnames[p.prod[n+1]] except (IndexError, KeyError): p.lr_after = [] try: p.lr_before = p.prod[n-1] except IndexError: p.lr_before = None return p # Bind the production function name to a callable def bind(self, pdict): if self.func: self.callable = pdict[self.func] # This class serves as a minimal standin for Production objects when # reading table data from files. It only contains information # actually used by the LR parsing engine, plus some additional # debugging information. class MiniProduction(object): def __init__(self, str, name, len, func, file, line): self.name = name self.len = len self.func = func self.callable = None self.file = file self.line = line self.str = str def __str__(self): return self.str def __repr__(self): return 'MiniProduction(%s)' % self.str # Bind the production function name to a callable def bind(self, pdict): if self.func: self.callable = pdict[self.func] # ----------------------------------------------------------------------------- # class LRItem # # This class represents a specific stage of parsing a production rule. For # example: # # expr : expr . PLUS term # # In the above, the "." represents the current location of the parse. Here # basic attributes: # # name - Name of the production. For example 'expr' # prod - A list of symbols on the right side ['expr','.', 'PLUS','term'] # number - Production number. # # lr_next Next LR item. Example, if we are ' expr -> expr . PLUS term' # then lr_next refers to 'expr -> expr PLUS . term' # lr_index - LR item index (location of the ".") in the prod list. # lookaheads - LALR lookahead symbols for this item # len - Length of the production (number of symbols on right hand side) # lr_after - List of all productions that immediately follow # lr_before - Grammar symbol immediately before # ----------------------------------------------------------------------------- class LRItem(object): def __init__(self, p, n): self.name = p.name self.prod = list(p.prod) self.number = p.number self.lr_index = n self.lookaheads = {} self.prod.insert(n, '.') self.prod = tuple(self.prod) self.len = len(self.prod) self.usyms = p.usyms def __str__(self): if self.prod: s = '%s -> %s' % (self.name, ' '.join(self.prod)) else: s = '%s -> ' % self.name return s def __repr__(self): return 'LRItem(' + str(self) + ')' # ----------------------------------------------------------------------------- # rightmost_terminal() # # Return the rightmost terminal from a list of symbols. Used in add_production() # ----------------------------------------------------------------------------- def rightmost_terminal(symbols, terminals): i = len(symbols) - 1 while i >= 0: if symbols[i] in terminals: return symbols[i] i -= 1 return None # ----------------------------------------------------------------------------- # === GRAMMAR CLASS === # # The following class represents the contents of the specified grammar along # with various computed properties such as first sets, follow sets, LR items, etc. # This data is used for critical parts of the table generation process later. # ----------------------------------------------------------------------------- class GrammarError(YaccError): pass class Grammar(object): def __init__(self, terminals): self.Productions = [None] # A list of all of the productions. The first # entry is always reserved for the purpose of # building an augmented grammar self.Prodnames = {} # A dictionary mapping the names of nonterminals to a list of all # productions of that nonterminal. self.Prodmap = {} # A dictionary that is only used to detect duplicate # productions. self.Terminals = {} # A dictionary mapping the names of terminal symbols to a # list of the rules where they are used. for term in terminals: self.Terminals[term] = [] self.Terminals['error'] = [] self.Nonterminals = {} # A dictionary mapping names of nonterminals to a list # of rule numbers where they are used. self.First = {} # A dictionary of precomputed FIRST(x) symbols self.Follow = {} # A dictionary of precomputed FOLLOW(x) symbols self.Precedence = {} # Precedence rules for each terminal. Contains tuples of the # form ('right',level) or ('nonassoc', level) or ('left',level) self.UsedPrecedence = set() # Precedence rules that were actually used by the grammer. # This is only used to provide error checking and to generate # a warning about unused precedence rules. self.Start = None # Starting symbol for the grammar def __len__(self): return len(self.Productions) def __getitem__(self, index): return self.Productions[index] # ----------------------------------------------------------------------------- # set_precedence() # # Sets the precedence for a given terminal. assoc is the associativity such as # 'left','right', or 'nonassoc'. level is a numeric level. # # ----------------------------------------------------------------------------- def set_precedence(self, term, assoc, level): assert self.Productions == [None], 'Must call set_precedence() before add_production()' if term in self.Precedence: raise GrammarError('Precedence already specified for terminal %r' % term) if assoc not in ['left', 'right', 'nonassoc']: raise GrammarError("Associativity must be one of 'left','right', or 'nonassoc'") self.Precedence[term] = (assoc, level) # ----------------------------------------------------------------------------- # add_production() # # Given an action function, this function assembles a production rule and # computes its precedence level. # # The production rule is supplied as a list of symbols. For example, # a rule such as 'expr : expr PLUS term' has a production name of 'expr' and # symbols ['expr','PLUS','term']. # # Precedence is determined by the precedence of the right-most non-terminal # or the precedence of a terminal specified by %prec. # # A variety of error checks are performed to make sure production symbols # are valid and that %prec is used correctly. # ----------------------------------------------------------------------------- def add_production(self, prodname, syms, func=None, file='', line=0): if prodname in self.Terminals: raise GrammarError('%s:%d: Illegal rule name %r. Already defined as a token' % (file, line, prodname)) if prodname == 'error': raise GrammarError('%s:%d: Illegal rule name %r. error is a reserved word' % (file, line, prodname)) if not _is_identifier.match(prodname): raise GrammarError('%s:%d: Illegal rule name %r' % (file, line, prodname)) # Look for literal tokens for n, s in enumerate(syms): if s[0] in "'\"": try: c = eval(s) if (len(c) > 1): raise GrammarError('%s:%d: Literal token %s in rule %r may only be a single character' % (file, line, s, prodname)) if c not in self.Terminals: self.Terminals[c] = [] syms[n] = c continue except SyntaxError: pass if not _is_identifier.match(s) and s != '%prec': raise GrammarError('%s:%d: Illegal name %r in rule %r' % (file, line, s, prodname)) # Determine the precedence level if '%prec' in syms: if syms[-1] == '%prec': raise GrammarError('%s:%d: Syntax error. Nothing follows %%prec' % (file, line)) if syms[-2] != '%prec': raise GrammarError('%s:%d: Syntax error. %%prec can only appear at the end of a grammar rule' % (file, line)) precname = syms[-1] prodprec = self.Precedence.get(precname) if not prodprec: raise GrammarError('%s:%d: Nothing known about the precedence of %r' % (file, line, precname)) else: self.UsedPrecedence.add(precname) del syms[-2:] # Drop %prec from the rule else: # If no %prec, precedence is determined by the rightmost terminal symbol precname = rightmost_terminal(syms, self.Terminals) prodprec = self.Precedence.get(precname, ('right', 0)) # See if the rule is already in the rulemap map = '%s -> %s' % (prodname, syms) if map in self.Prodmap: m = self.Prodmap[map] raise GrammarError('%s:%d: Duplicate rule %s. ' % (file, line, m) + 'Previous definition at %s:%d' % (m.file, m.line)) # From this point on, everything is valid. Create a new Production instance pnumber = len(self.Productions) if prodname not in self.Nonterminals: self.Nonterminals[prodname] = [] # Add the production number to Terminals and Nonterminals for t in syms: if t in self.Terminals: self.Terminals[t].append(pnumber) else: if t not in self.Nonterminals: self.Nonterminals[t] = [] self.Nonterminals[t].append(pnumber) # Create a production and add it to the list of productions p = Production(pnumber, prodname, syms, prodprec, func, file, line) self.Productions.append(p) self.Prodmap[map] = p # Add to the global productions list try: self.Prodnames[prodname].append(p) except KeyError: self.Prodnames[prodname] = [p] # ----------------------------------------------------------------------------- # set_start() # # Sets the starting symbol and creates the augmented grammar. Production # rule 0 is S' -> start where start is the start symbol. # ----------------------------------------------------------------------------- def set_start(self, start=None): if not start: start = self.Productions[1].name if start not in self.Nonterminals: raise GrammarError('start symbol %s undefined' % start) self.Productions[0] = Production(0, "S'", [start]) self.Nonterminals[start].append(0) self.Start = start # ----------------------------------------------------------------------------- # find_unreachable() # # Find all of the nonterminal symbols that can't be reached from the starting # symbol. Returns a list of nonterminals that can't be reached. # ----------------------------------------------------------------------------- def find_unreachable(self): # Mark all symbols that are reachable from a symbol s def mark_reachable_from(s): if s in reachable: return reachable.add(s) for p in self.Prodnames.get(s, []): for r in p.prod: mark_reachable_from(r) reachable = set() mark_reachable_from(self.Productions[0].prod[0]) return [s for s in self.Nonterminals if s not in reachable] # ----------------------------------------------------------------------------- # infinite_cycles() # # This function looks at the various parsing rules and tries to detect # infinite recursion cycles (grammar rules where there is no possible way # to derive a string of only terminals). # ----------------------------------------------------------------------------- def infinite_cycles(self): terminates = {} # Terminals: for t in self.Terminals: terminates[t] = True terminates['$end'] = True # Nonterminals: # Initialize to false: for n in self.Nonterminals: terminates[n] = False # Then propagate termination until no change: while True: some_change = False for (n, pl) in self.Prodnames.items(): # Nonterminal n terminates iff any of its productions terminates. for p in pl: # Production p terminates iff all of its rhs symbols terminate. for s in p.prod: if not terminates[s]: # The symbol s does not terminate, # so production p does not terminate. p_terminates = False break else: # didn't break from the loop, # so every symbol s terminates # so production p terminates. p_terminates = True if p_terminates: # symbol n terminates! if not terminates[n]: terminates[n] = True some_change = True # Don't need to consider any more productions for this n. break if not some_change: break infinite = [] for (s, term) in terminates.items(): if not term: if s not in self.Prodnames and s not in self.Terminals and s != 'error': # s is used-but-not-defined, and we've already warned of that, # so it would be overkill to say that it's also non-terminating. pass else: infinite.append(s) return infinite # ----------------------------------------------------------------------------- # undefined_symbols() # # Find all symbols that were used the grammar, but not defined as tokens or # grammar rules. Returns a list of tuples (sym, prod) where sym in the symbol # and prod is the production where the symbol was used. # ----------------------------------------------------------------------------- def undefined_symbols(self): result = [] for p in self.Productions: if not p: continue for s in p.prod: if s not in self.Prodnames and s not in self.Terminals and s != 'error': result.append((s, p)) return result # ----------------------------------------------------------------------------- # unused_terminals() # # Find all terminals that were defined, but not used by the grammar. Returns # a list of all symbols. # ----------------------------------------------------------------------------- def unused_terminals(self): unused_tok = [] for s, v in self.Terminals.items(): if s != 'error' and not v: unused_tok.append(s) return unused_tok # ------------------------------------------------------------------------------ # unused_rules() # # Find all grammar rules that were defined, but not used (maybe not reachable) # Returns a list of productions. # ------------------------------------------------------------------------------ def unused_rules(self): unused_prod = [] for s, v in self.Nonterminals.items(): if not v: p = self.Prodnames[s][0] unused_prod.append(p) return unused_prod # ----------------------------------------------------------------------------- # unused_precedence() # # Returns a list of tuples (term,precedence) corresponding to precedence # rules that were never used by the grammar. term is the name of the terminal # on which precedence was applied and precedence is a string such as 'left' or # 'right' corresponding to the type of precedence. # ----------------------------------------------------------------------------- def unused_precedence(self): unused = [] for termname in self.Precedence: if not (termname in self.Terminals or termname in self.UsedPrecedence): unused.append((termname, self.Precedence[termname][0])) return unused # ------------------------------------------------------------------------- # _first() # # Compute the value of FIRST1(beta) where beta is a tuple of symbols. # # During execution of compute_first1, the result may be incomplete. # Afterward (e.g., when called from compute_follow()), it will be complete. # ------------------------------------------------------------------------- def _first(self, beta): # We are computing First(x1,x2,x3,...,xn) result = [] for x in beta: x_produces_empty = False # Add all the non- symbols of First[x] to the result. for f in self.First[x]: if f == '': x_produces_empty = True else: if f not in result: result.append(f) if x_produces_empty: # We have to consider the next x in beta, # i.e. stay in the loop. pass else: # We don't have to consider any further symbols in beta. break else: # There was no 'break' from the loop, # so x_produces_empty was true for all x in beta, # so beta produces empty as well. result.append('') return result # ------------------------------------------------------------------------- # compute_first() # # Compute the value of FIRST1(X) for all symbols # ------------------------------------------------------------------------- def compute_first(self): if self.First: return self.First # Terminals: for t in self.Terminals: self.First[t] = [t] self.First['$end'] = ['$end'] # Nonterminals: # Initialize to the empty set: for n in self.Nonterminals: self.First[n] = [] # Then propagate symbols until no change: while True: some_change = False for n in self.Nonterminals: for p in self.Prodnames[n]: for f in self._first(p.prod): if f not in self.First[n]: self.First[n].append(f) some_change = True if not some_change: break return self.First # --------------------------------------------------------------------- # compute_follow() # # Computes all of the follow sets for every non-terminal symbol. The # follow set is the set of all symbols that might follow a given # non-terminal. See the Dragon book, 2nd Ed. p. 189. # --------------------------------------------------------------------- def compute_follow(self, start=None): # If already computed, return the result if self.Follow: return self.Follow # If first sets not computed yet, do that first. if not self.First: self.compute_first() # Add '$end' to the follow list of the start symbol for k in self.Nonterminals: self.Follow[k] = [] if not start: start = self.Productions[1].name self.Follow[start] = ['$end'] while True: didadd = False for p in self.Productions[1:]: # Here is the production set for i, B in enumerate(p.prod): if B in self.Nonterminals: # Okay. We got a non-terminal in a production fst = self._first(p.prod[i+1:]) hasempty = False for f in fst: if f != '' and f not in self.Follow[B]: self.Follow[B].append(f) didadd = True if f == '': hasempty = True if hasempty or i == (len(p.prod)-1): # Add elements of follow(a) to follow(b) for f in self.Follow[p.name]: if f not in self.Follow[B]: self.Follow[B].append(f) didadd = True if not didadd: break return self.Follow # ----------------------------------------------------------------------------- # build_lritems() # # This function walks the list of productions and builds a complete set of the # LR items. The LR items are stored in two ways: First, they are uniquely # numbered and placed in the list _lritems. Second, a linked list of LR items # is built for each production. For example: # # E -> E PLUS E # # Creates the list # # [E -> . E PLUS E, E -> E . PLUS E, E -> E PLUS . E, E -> E PLUS E . ] # ----------------------------------------------------------------------------- def build_lritems(self): for p in self.Productions: lastlri = p i = 0 lr_items = [] while True: if i > len(p): lri = None else: lri = LRItem(p, i) # Precompute the list of productions immediately following try: lri.lr_after = self.Prodnames[lri.prod[i+1]] except (IndexError, KeyError): lri.lr_after = [] try: lri.lr_before = lri.prod[i-1] except IndexError: lri.lr_before = None lastlri.lr_next = lri if not lri: break lr_items.append(lri) lastlri = lri i += 1 p.lr_items = lr_items # ----------------------------------------------------------------------------- # == Class LRTable == # # This basic class represents a basic table of LR parsing information. # Methods for generating the tables are not defined here. They are defined # in the derived class LRGeneratedTable. # ----------------------------------------------------------------------------- class VersionError(YaccError): pass class LRTable(object): def __init__(self): self.lr_action = None self.lr_goto = None self.lr_productions = None self.lr_method = None def read_table(self, module): if isinstance(module, types.ModuleType): parsetab = module else: exec('import %s' % module) parsetab = sys.modules[module] if parsetab._tabversion != __tabversion__: raise VersionError('yacc table file version is out of date') self.lr_action = parsetab._lr_action self.lr_goto = parsetab._lr_goto self.lr_productions = [] for p in parsetab._lr_productions: self.lr_productions.append(MiniProduction(*p)) self.lr_method = parsetab._lr_method return parsetab._lr_signature def read_pickle(self, filename): try: import cPickle as pickle except ImportError: import pickle if not os.path.exists(filename): raise ImportError in_f = open(filename, 'rb') tabversion = pickle.load(in_f) if tabversion != __tabversion__: raise VersionError('yacc table file version is out of date') self.lr_method = pickle.load(in_f) signature = pickle.load(in_f) self.lr_action = pickle.load(in_f) self.lr_goto = pickle.load(in_f) productions = pickle.load(in_f) self.lr_productions = [] for p in productions: self.lr_productions.append(MiniProduction(*p)) in_f.close() return signature # Bind all production function names to callable objects in pdict def bind_callables(self, pdict): for p in self.lr_productions: p.bind(pdict) # ----------------------------------------------------------------------------- # === LR Generator === # # The following classes and functions are used to generate LR parsing tables on # a grammar. # ----------------------------------------------------------------------------- # ----------------------------------------------------------------------------- # digraph() # traverse() # # The following two functions are used to compute set valued functions # of the form: # # F(x) = F'(x) U U{F(y) | x R y} # # This is used to compute the values of Read() sets as well as FOLLOW sets # in LALR(1) generation. # # Inputs: X - An input set # R - A relation # FP - Set-valued function # ------------------------------------------------------------------------------ def digraph(X, R, FP): N = {} for x in X: N[x] = 0 stack = [] F = {} for x in X: if N[x] == 0: traverse(x, N, stack, F, X, R, FP) return F def traverse(x, N, stack, F, X, R, FP): stack.append(x) d = len(stack) N[x] = d F[x] = FP(x) # F(X) <- F'(x) rel = R(x) # Get y's related to x for y in rel: if N[y] == 0: traverse(y, N, stack, F, X, R, FP) N[x] = min(N[x], N[y]) for a in F.get(y, []): if a not in F[x]: F[x].append(a) if N[x] == d: N[stack[-1]] = MAXINT F[stack[-1]] = F[x] element = stack.pop() while element != x: N[stack[-1]] = MAXINT F[stack[-1]] = F[x] element = stack.pop() class LALRError(YaccError): pass # ----------------------------------------------------------------------------- # == LRGeneratedTable == # # This class implements the LR table generation algorithm. There are no # public methods except for write() # ----------------------------------------------------------------------------- class LRGeneratedTable(LRTable): def __init__(self, grammar, method='LALR', log=None): if method not in ['SLR', 'LALR']: raise LALRError('Unsupported method %s' % method) self.grammar = grammar self.lr_method = method # Set up the logger if not log: log = NullLogger() self.log = log # Internal attributes self.lr_action = {} # Action table self.lr_goto = {} # Goto table self.lr_productions = grammar.Productions # Copy of grammar Production array self.lr_goto_cache = {} # Cache of computed gotos self.lr0_cidhash = {} # Cache of closures self._add_count = 0 # Internal counter used to detect cycles # Diagonistic information filled in by the table generator self.sr_conflict = 0 self.rr_conflict = 0 self.conflicts = [] # List of conflicts self.sr_conflicts = [] self.rr_conflicts = [] # Build the tables self.grammar.build_lritems() self.grammar.compute_first() self.grammar.compute_follow() self.lr_parse_table() # Compute the LR(0) closure operation on I, where I is a set of LR(0) items. def lr0_closure(self, I): self._add_count += 1 # Add everything in I to J J = I[:] didadd = True while didadd: didadd = False for j in J: for x in j.lr_after: if getattr(x, 'lr0_added', 0) == self._add_count: continue # Add B --> .G to J J.append(x.lr_next) x.lr0_added = self._add_count didadd = True return J # Compute the LR(0) goto function goto(I,X) where I is a set # of LR(0) items and X is a grammar symbol. This function is written # in a way that guarantees uniqueness of the generated goto sets # (i.e. the same goto set will never be returned as two different Python # objects). With uniqueness, we can later do fast set comparisons using # id(obj) instead of element-wise comparison. def lr0_goto(self, I, x): # First we look for a previously cached entry g = self.lr_goto_cache.get((id(I), x)) if g: return g # Now we generate the goto set in a way that guarantees uniqueness # of the result s = self.lr_goto_cache.get(x) if not s: s = {} self.lr_goto_cache[x] = s gs = [] for p in I: n = p.lr_next if n and n.lr_before == x: s1 = s.get(id(n)) if not s1: s1 = {} s[id(n)] = s1 gs.append(n) s = s1 g = s.get('$end') if not g: if gs: g = self.lr0_closure(gs) s['$end'] = g else: s['$end'] = gs self.lr_goto_cache[(id(I), x)] = g return g # Compute the LR(0) sets of item function def lr0_items(self): C = [self.lr0_closure([self.grammar.Productions[0].lr_next])] i = 0 for I in C: self.lr0_cidhash[id(I)] = i i += 1 # Loop over the items in C and each grammar symbols i = 0 while i < len(C): I = C[i] i += 1 # Collect all of the symbols that could possibly be in the goto(I,X) sets asyms = {} for ii in I: for s in ii.usyms: asyms[s] = None for x in asyms: g = self.lr0_goto(I, x) if not g or id(g) in self.lr0_cidhash: continue self.lr0_cidhash[id(g)] = len(C) C.append(g) return C # ----------------------------------------------------------------------------- # ==== LALR(1) Parsing ==== # # LALR(1) parsing is almost exactly the same as SLR except that instead of # relying upon Follow() sets when performing reductions, a more selective # lookahead set that incorporates the state of the LR(0) machine is utilized. # Thus, we mainly just have to focus on calculating the lookahead sets. # # The method used here is due to DeRemer and Pennelo (1982). # # DeRemer, F. L., and T. J. Pennelo: "Efficient Computation of LALR(1) # Lookahead Sets", ACM Transactions on Programming Languages and Systems, # Vol. 4, No. 4, Oct. 1982, pp. 615-649 # # Further details can also be found in: # # J. Tremblay and P. Sorenson, "The Theory and Practice of Compiler Writing", # McGraw-Hill Book Company, (1985). # # ----------------------------------------------------------------------------- # ----------------------------------------------------------------------------- # compute_nullable_nonterminals() # # Creates a dictionary containing all of the non-terminals that might produce # an empty production. # ----------------------------------------------------------------------------- def compute_nullable_nonterminals(self): nullable = set() num_nullable = 0 while True: for p in self.grammar.Productions[1:]: if p.len == 0: nullable.add(p.name) continue for t in p.prod: if t not in nullable: break else: nullable.add(p.name) if len(nullable) == num_nullable: break num_nullable = len(nullable) return nullable # ----------------------------------------------------------------------------- # find_nonterminal_trans(C) # # Given a set of LR(0) items, this functions finds all of the non-terminal # transitions. These are transitions in which a dot appears immediately before # a non-terminal. Returns a list of tuples of the form (state,N) where state # is the state number and N is the nonterminal symbol. # # The input C is the set of LR(0) items. # ----------------------------------------------------------------------------- def find_nonterminal_transitions(self, C): trans = [] for stateno, state in enumerate(C): for p in state: if p.lr_index < p.len - 1: t = (stateno, p.prod[p.lr_index+1]) if t[1] in self.grammar.Nonterminals: if t not in trans: trans.append(t) return trans # ----------------------------------------------------------------------------- # dr_relation() # # Computes the DR(p,A) relationships for non-terminal transitions. The input # is a tuple (state,N) where state is a number and N is a nonterminal symbol. # # Returns a list of terminals. # ----------------------------------------------------------------------------- def dr_relation(self, C, trans, nullable): state, N = trans terms = [] g = self.lr0_goto(C[state], N) for p in g: if p.lr_index < p.len - 1: a = p.prod[p.lr_index+1] if a in self.grammar.Terminals: if a not in terms: terms.append(a) # This extra bit is to handle the start state if state == 0 and N == self.grammar.Productions[0].prod[0]: terms.append('$end') return terms # ----------------------------------------------------------------------------- # reads_relation() # # Computes the READS() relation (p,A) READS (t,C). # ----------------------------------------------------------------------------- def reads_relation(self, C, trans, empty): # Look for empty transitions rel = [] state, N = trans g = self.lr0_goto(C[state], N) j = self.lr0_cidhash.get(id(g), -1) for p in g: if p.lr_index < p.len - 1: a = p.prod[p.lr_index + 1] if a in empty: rel.append((j, a)) return rel # ----------------------------------------------------------------------------- # compute_lookback_includes() # # Determines the lookback and includes relations # # LOOKBACK: # # This relation is determined by running the LR(0) state machine forward. # For example, starting with a production "N : . A B C", we run it forward # to obtain "N : A B C ." We then build a relationship between this final # state and the starting state. These relationships are stored in a dictionary # lookdict. # # INCLUDES: # # Computes the INCLUDE() relation (p,A) INCLUDES (p',B). # # This relation is used to determine non-terminal transitions that occur # inside of other non-terminal transition states. (p,A) INCLUDES (p', B) # if the following holds: # # B -> LAT, where T -> epsilon and p' -L-> p # # L is essentially a prefix (which may be empty), T is a suffix that must be # able to derive an empty string. State p' must lead to state p with the string L. # # ----------------------------------------------------------------------------- def compute_lookback_includes(self, C, trans, nullable): lookdict = {} # Dictionary of lookback relations includedict = {} # Dictionary of include relations # Make a dictionary of non-terminal transitions dtrans = {} for t in trans: dtrans[t] = 1 # Loop over all transitions and compute lookbacks and includes for state, N in trans: lookb = [] includes = [] for p in C[state]: if p.name != N: continue # Okay, we have a name match. We now follow the production all the way # through the state machine until we get the . on the right hand side lr_index = p.lr_index j = state while lr_index < p.len - 1: lr_index = lr_index + 1 t = p.prod[lr_index] # Check to see if this symbol and state are a non-terminal transition if (j, t) in dtrans: # Yes. Okay, there is some chance that this is an includes relation # the only way to know for certain is whether the rest of the # production derives empty li = lr_index + 1 while li < p.len: if p.prod[li] in self.grammar.Terminals: break # No forget it if p.prod[li] not in nullable: break li = li + 1 else: # Appears to be a relation between (j,t) and (state,N) includes.append((j, t)) g = self.lr0_goto(C[j], t) # Go to next set j = self.lr0_cidhash.get(id(g), -1) # Go to next state # When we get here, j is the final state, now we have to locate the production for r in C[j]: if r.name != p.name: continue if r.len != p.len: continue i = 0 # This look is comparing a production ". A B C" with "A B C ." while i < r.lr_index: if r.prod[i] != p.prod[i+1]: break i = i + 1 else: lookb.append((j, r)) for i in includes: if i not in includedict: includedict[i] = [] includedict[i].append((state, N)) lookdict[(state, N)] = lookb return lookdict, includedict # ----------------------------------------------------------------------------- # compute_read_sets() # # Given a set of LR(0) items, this function computes the read sets. # # Inputs: C = Set of LR(0) items # ntrans = Set of nonterminal transitions # nullable = Set of empty transitions # # Returns a set containing the read sets # ----------------------------------------------------------------------------- def compute_read_sets(self, C, ntrans, nullable): FP = lambda x: self.dr_relation(C, x, nullable) R = lambda x: self.reads_relation(C, x, nullable) F = digraph(ntrans, R, FP) return F # ----------------------------------------------------------------------------- # compute_follow_sets() # # Given a set of LR(0) items, a set of non-terminal transitions, a readset, # and an include set, this function computes the follow sets # # Follow(p,A) = Read(p,A) U U {Follow(p',B) | (p,A) INCLUDES (p',B)} # # Inputs: # ntrans = Set of nonterminal transitions # readsets = Readset (previously computed) # inclsets = Include sets (previously computed) # # Returns a set containing the follow sets # ----------------------------------------------------------------------------- def compute_follow_sets(self, ntrans, readsets, inclsets): FP = lambda x: readsets[x] R = lambda x: inclsets.get(x, []) F = digraph(ntrans, R, FP) return F # ----------------------------------------------------------------------------- # add_lookaheads() # # Attaches the lookahead symbols to grammar rules. # # Inputs: lookbacks - Set of lookback relations # followset - Computed follow set # # This function directly attaches the lookaheads to productions contained # in the lookbacks set # ----------------------------------------------------------------------------- def add_lookaheads(self, lookbacks, followset): for trans, lb in lookbacks.items(): # Loop over productions in lookback for state, p in lb: if state not in p.lookaheads: p.lookaheads[state] = [] f = followset.get(trans, []) for a in f: if a not in p.lookaheads[state]: p.lookaheads[state].append(a) # ----------------------------------------------------------------------------- # add_lalr_lookaheads() # # This function does all of the work of adding lookahead information for use # with LALR parsing # ----------------------------------------------------------------------------- def add_lalr_lookaheads(self, C): # Determine all of the nullable nonterminals nullable = self.compute_nullable_nonterminals() # Find all non-terminal transitions trans = self.find_nonterminal_transitions(C) # Compute read sets readsets = self.compute_read_sets(C, trans, nullable) # Compute lookback/includes relations lookd, included = self.compute_lookback_includes(C, trans, nullable) # Compute LALR FOLLOW sets followsets = self.compute_follow_sets(trans, readsets, included) # Add all of the lookaheads self.add_lookaheads(lookd, followsets) # ----------------------------------------------------------------------------- # lr_parse_table() # # This function constructs the parse tables for SLR or LALR # ----------------------------------------------------------------------------- def lr_parse_table(self): Productions = self.grammar.Productions Precedence = self.grammar.Precedence goto = self.lr_goto # Goto array action = self.lr_action # Action array log = self.log # Logger for output actionp = {} # Action production array (temporary) log.info('Parsing method: %s', self.lr_method) # Step 1: Construct C = { I0, I1, ... IN}, collection of LR(0) items # This determines the number of states C = self.lr0_items() if self.lr_method == 'LALR': self.add_lalr_lookaheads(C) # Build the parser table, state by state st = 0 for I in C: # Loop over each production in I actlist = [] # List of actions st_action = {} st_actionp = {} st_goto = {} log.info('') log.info('state %d', st) log.info('') for p in I: log.info(' (%d) %s', p.number, p) log.info('') for p in I: if p.len == p.lr_index + 1: if p.name == "S'": # Start symbol. Accept! st_action['$end'] = 0 st_actionp['$end'] = p else: # We are at the end of a production. Reduce! if self.lr_method == 'LALR': laheads = p.lookaheads[st] else: laheads = self.grammar.Follow[p.name] for a in laheads: actlist.append((a, p, 'reduce using rule %d (%s)' % (p.number, p))) r = st_action.get(a) if r is not None: # Whoa. Have a shift/reduce or reduce/reduce conflict if r > 0: # Need to decide on shift or reduce here # By default we favor shifting. Need to add # some precedence rules here. # Shift precedence comes from the token sprec, slevel = Precedence.get(a, ('right', 0)) # Reduce precedence comes from rule being reduced (p) rprec, rlevel = Productions[p.number].prec if (slevel < rlevel) or ((slevel == rlevel) and (rprec == 'left')): # We really need to reduce here. st_action[a] = -p.number st_actionp[a] = p if not slevel and not rlevel: log.info(' ! shift/reduce conflict for %s resolved as reduce', a) self.sr_conflicts.append((st, a, 'reduce')) Productions[p.number].reduced += 1 elif (slevel == rlevel) and (rprec == 'nonassoc'): st_action[a] = None else: # Hmmm. Guess we'll keep the shift if not rlevel: log.info(' ! shift/reduce conflict for %s resolved as shift', a) self.sr_conflicts.append((st, a, 'shift')) elif r < 0: # Reduce/reduce conflict. In this case, we favor the rule # that was defined first in the grammar file oldp = Productions[-r] pp = Productions[p.number] if oldp.line > pp.line: st_action[a] = -p.number st_actionp[a] = p chosenp, rejectp = pp, oldp Productions[p.number].reduced += 1 Productions[oldp.number].reduced -= 1 else: chosenp, rejectp = oldp, pp self.rr_conflicts.append((st, chosenp, rejectp)) log.info(' ! reduce/reduce conflict for %s resolved using rule %d (%s)', a, st_actionp[a].number, st_actionp[a]) else: raise LALRError('Unknown conflict in state %d' % st) else: st_action[a] = -p.number st_actionp[a] = p Productions[p.number].reduced += 1 else: i = p.lr_index a = p.prod[i+1] # Get symbol right after the "." if a in self.grammar.Terminals: g = self.lr0_goto(I, a) j = self.lr0_cidhash.get(id(g), -1) if j >= 0: # We are in a shift state actlist.append((a, p, 'shift and go to state %d' % j)) r = st_action.get(a) if r is not None: # Whoa have a shift/reduce or shift/shift conflict if r > 0: if r != j: raise LALRError('Shift/shift conflict in state %d' % st) elif r < 0: # Do a precedence check. # - if precedence of reduce rule is higher, we reduce. # - if precedence of reduce is same and left assoc, we reduce. # - otherwise we shift # Shift precedence comes from the token sprec, slevel = Precedence.get(a, ('right', 0)) # Reduce precedence comes from the rule that could have been reduced rprec, rlevel = Productions[st_actionp[a].number].prec if (slevel > rlevel) or ((slevel == rlevel) and (rprec == 'right')): # We decide to shift here... highest precedence to shift Productions[st_actionp[a].number].reduced -= 1 st_action[a] = j st_actionp[a] = p if not rlevel: log.info(' ! shift/reduce conflict for %s resolved as shift', a) self.sr_conflicts.append((st, a, 'shift')) elif (slevel == rlevel) and (rprec == 'nonassoc'): st_action[a] = None else: # Hmmm. Guess we'll keep the reduce if not slevel and not rlevel: log.info(' ! shift/reduce conflict for %s resolved as reduce', a) self.sr_conflicts.append((st, a, 'reduce')) else: raise LALRError('Unknown conflict in state %d' % st) else: st_action[a] = j st_actionp[a] = p # Print the actions associated with each terminal _actprint = {} for a, p, m in actlist: if a in st_action: if p is st_actionp[a]: log.info(' %-15s %s', a, m) _actprint[(a, m)] = 1 log.info('') # Print the actions that were not used. (debugging) not_used = 0 for a, p, m in actlist: if a in st_action: if p is not st_actionp[a]: if not (a, m) in _actprint: log.debug(' ! %-15s [ %s ]', a, m) not_used = 1 _actprint[(a, m)] = 1 if not_used: log.debug('') # Construct the goto table for this state nkeys = {} for ii in I: for s in ii.usyms: if s in self.grammar.Nonterminals: nkeys[s] = None for n in nkeys: g = self.lr0_goto(I, n) j = self.lr0_cidhash.get(id(g), -1) if j >= 0: st_goto[n] = j log.info(' %-30s shift and go to state %d', n, j) action[st] = st_action actionp[st] = st_actionp goto[st] = st_goto st += 1 # ----------------------------------------------------------------------------- # write() # # This function writes the LR parsing tables to a file # ----------------------------------------------------------------------------- def write_table(self, tabmodule, outputdir='', signature=''): if isinstance(tabmodule, types.ModuleType): raise IOError("Won't overwrite existing tabmodule") basemodulename = tabmodule.split('.')[-1] filename = os.path.join(outputdir, basemodulename) + '.py' try: f = open(filename, 'w') f.write(''' # %s # This file is automatically generated. Do not edit. # pylint: disable=W,C,R _tabversion = %r _lr_method = %r _lr_signature = %r ''' % (os.path.basename(filename), __tabversion__, self.lr_method, signature)) # Change smaller to 0 to go back to original tables smaller = 1 # Factor out names to try and make smaller if smaller: items = {} for s, nd in self.lr_action.items(): for name, v in nd.items(): i = items.get(name) if not i: i = ([], []) items[name] = i i[0].append(s) i[1].append(v) f.write('\n_lr_action_items = {') for k, v in items.items(): f.write('%r:([' % k) for i in v[0]: f.write('%r,' % i) f.write('],[') for i in v[1]: f.write('%r,' % i) f.write(']),') f.write('}\n') f.write(''' _lr_action = {} for _k, _v in _lr_action_items.items(): for _x,_y in zip(_v[0],_v[1]): if not _x in _lr_action: _lr_action[_x] = {} _lr_action[_x][_k] = _y del _lr_action_items ''') else: f.write('\n_lr_action = { ') for k, v in self.lr_action.items(): f.write('(%r,%r):%r,' % (k[0], k[1], v)) f.write('}\n') if smaller: # Factor out names to try and make smaller items = {} for s, nd in self.lr_goto.items(): for name, v in nd.items(): i = items.get(name) if not i: i = ([], []) items[name] = i i[0].append(s) i[1].append(v) f.write('\n_lr_goto_items = {') for k, v in items.items(): f.write('%r:([' % k) for i in v[0]: f.write('%r,' % i) f.write('],[') for i in v[1]: f.write('%r,' % i) f.write(']),') f.write('}\n') f.write(''' _lr_goto = {} for _k, _v in _lr_goto_items.items(): for _x, _y in zip(_v[0], _v[1]): if not _x in _lr_goto: _lr_goto[_x] = {} _lr_goto[_x][_k] = _y del _lr_goto_items ''') else: f.write('\n_lr_goto = { ') for k, v in self.lr_goto.items(): f.write('(%r,%r):%r,' % (k[0], k[1], v)) f.write('}\n') # Write production table f.write('_lr_productions = [\n') for p in self.lr_productions: if p.func: f.write(' (%r,%r,%d,%r,%r,%d),\n' % (p.str, p.name, p.len, p.func, os.path.basename(p.file), p.line)) else: f.write(' (%r,%r,%d,None,None,None),\n' % (str(p), p.name, p.len)) f.write(']\n') f.close() except IOError as e: raise # ----------------------------------------------------------------------------- # pickle_table() # # This function pickles the LR parsing tables to a supplied file object # ----------------------------------------------------------------------------- def pickle_table(self, filename, signature=''): try: import cPickle as pickle except ImportError: import pickle with open(filename, 'wb') as outf: pickle.dump(__tabversion__, outf, pickle_protocol) pickle.dump(self.lr_method, outf, pickle_protocol) pickle.dump(signature, outf, pickle_protocol) pickle.dump(self.lr_action, outf, pickle_protocol) pickle.dump(self.lr_goto, outf, pickle_protocol) outp = [] for p in self.lr_productions: if p.func: outp.append((p.str, p.name, p.len, p.func, os.path.basename(p.file), p.line)) else: outp.append((str(p), p.name, p.len, None, None, None)) pickle.dump(outp, outf, pickle_protocol) # ----------------------------------------------------------------------------- # === INTROSPECTION === # # The following functions and classes are used to implement the PLY # introspection features followed by the yacc() function itself. # ----------------------------------------------------------------------------- # ----------------------------------------------------------------------------- # get_caller_module_dict() # # This function returns a dictionary containing all of the symbols defined within # a caller further down the call stack. This is used to get the environment # associated with the yacc() call if none was provided. # ----------------------------------------------------------------------------- def get_caller_module_dict(levels): f = sys._getframe(levels) ldict = f.f_globals.copy() if f.f_globals != f.f_locals: ldict.update(f.f_locals) return ldict # ----------------------------------------------------------------------------- # parse_grammar() # # This takes a raw grammar rule string and parses it into production data # ----------------------------------------------------------------------------- def parse_grammar(doc, file, line): grammar = [] # Split the doc string into lines pstrings = doc.splitlines() lastp = None dline = line for ps in pstrings: dline += 1 p = ps.split() if not p: continue try: if p[0] == '|': # This is a continuation of a previous rule if not lastp: raise SyntaxError("%s:%d: Misplaced '|'" % (file, dline)) prodname = lastp syms = p[1:] else: prodname = p[0] lastp = prodname syms = p[2:] assign = p[1] if assign != ':' and assign != '::=': raise SyntaxError("%s:%d: Syntax error. Expected ':'" % (file, dline)) grammar.append((file, dline, prodname, syms)) except SyntaxError: raise except Exception: raise SyntaxError('%s:%d: Syntax error in rule %r' % (file, dline, ps.strip())) return grammar # ----------------------------------------------------------------------------- # ParserReflect() # # This class represents information extracted for building a parser including # start symbol, error function, tokens, precedence list, action functions, # etc. # ----------------------------------------------------------------------------- class ParserReflect(object): def __init__(self, pdict, log=None): self.pdict = pdict self.start = None self.error_func = None self.tokens = None self.modules = set() self.grammar = [] self.error = False if log is None: self.log = PlyLogger(sys.stderr) else: self.log = log # Get all of the basic information def get_all(self): self.get_start() self.get_error_func() self.get_tokens() self.get_precedence() self.get_pfunctions() # Validate all of the information def validate_all(self): self.validate_start() self.validate_error_func() self.validate_tokens() self.validate_precedence() self.validate_pfunctions() self.validate_modules() return self.error # Compute a signature over the grammar def signature(self): parts = [] try: if self.start: parts.append(self.start) if self.prec: parts.append(''.join([''.join(p) for p in self.prec])) if self.tokens: parts.append(' '.join(self.tokens)) for f in self.pfuncs: if f[3]: parts.append(f[3]) except (TypeError, ValueError): pass return ''.join(parts) # ----------------------------------------------------------------------------- # validate_modules() # # This method checks to see if there are duplicated p_rulename() functions # in the parser module file. Without this function, it is really easy for # users to make mistakes by cutting and pasting code fragments (and it's a real # bugger to try and figure out why the resulting parser doesn't work). Therefore, # we just do a little regular expression pattern matching of def statements # to try and detect duplicates. # ----------------------------------------------------------------------------- def validate_modules(self): # Match def p_funcname( fre = re.compile(r'\s*def\s+(p_[a-zA-Z_0-9]*)\(') for module in self.modules: try: lines, linen = inspect.getsourcelines(module) except IOError: continue counthash = {} for linen, line in enumerate(lines): linen += 1 m = fre.match(line) if m: name = m.group(1) prev = counthash.get(name) if not prev: counthash[name] = linen else: filename = inspect.getsourcefile(module) self.log.warning('%s:%d: Function %s redefined. Previously defined on line %d', filename, linen, name, prev) # Get the start symbol def get_start(self): self.start = self.pdict.get('start') # Validate the start symbol def validate_start(self): if self.start is not None: if not isinstance(self.start, string_types): self.log.error("'start' must be a string") # Look for error handler def get_error_func(self): self.error_func = self.pdict.get('p_error') # Validate the error function def validate_error_func(self): if self.error_func: if isinstance(self.error_func, types.FunctionType): ismethod = 0 elif isinstance(self.error_func, types.MethodType): ismethod = 1 else: self.log.error("'p_error' defined, but is not a function or method") self.error = True return eline = self.error_func.__code__.co_firstlineno efile = self.error_func.__code__.co_filename module = inspect.getmodule(self.error_func) self.modules.add(module) argcount = self.error_func.__code__.co_argcount - ismethod if argcount != 1: self.log.error('%s:%d: p_error() requires 1 argument', efile, eline) self.error = True # Get the tokens map def get_tokens(self): tokens = self.pdict.get('tokens') if not tokens: self.log.error('No token list is defined') self.error = True return if not isinstance(tokens, (list, tuple)): self.log.error('tokens must be a list or tuple') self.error = True return if not tokens: self.log.error('tokens is empty') self.error = True return self.tokens = sorted(tokens) # Validate the tokens def validate_tokens(self): # Validate the tokens. if 'error' in self.tokens: self.log.error("Illegal token name 'error'. Is a reserved word") self.error = True return terminals = set() for n in self.tokens: if n in terminals: self.log.warning('Token %r multiply defined', n) terminals.add(n) # Get the precedence map (if any) def get_precedence(self): self.prec = self.pdict.get('precedence') # Validate and parse the precedence map def validate_precedence(self): preclist = [] if self.prec: if not isinstance(self.prec, (list, tuple)): self.log.error('precedence must be a list or tuple') self.error = True return for level, p in enumerate(self.prec): if not isinstance(p, (list, tuple)): self.log.error('Bad precedence table') self.error = True return if len(p) < 2: self.log.error('Malformed precedence entry %s. Must be (assoc, term, ..., term)', p) self.error = True return assoc = p[0] if not isinstance(assoc, string_types): self.log.error('precedence associativity must be a string') self.error = True return for term in p[1:]: if not isinstance(term, string_types): self.log.error('precedence items must be strings') self.error = True return preclist.append((term, assoc, level+1)) self.preclist = preclist # Get all p_functions from the grammar def get_pfunctions(self): p_functions = [] for name, item in self.pdict.items(): if not name.startswith('p_') or name == 'p_error': continue if isinstance(item, (types.FunctionType, types.MethodType)): line = getattr(item, 'co_firstlineno', item.__code__.co_firstlineno) module = inspect.getmodule(item) p_functions.append((line, module, name, item.__doc__)) # Sort all of the actions by line number; make sure to stringify # modules to make them sortable, since `line` may not uniquely sort all # p functions p_functions.sort(key=lambda p_function: ( p_function[0], str(p_function[1]), p_function[2], p_function[3])) self.pfuncs = p_functions # Validate all of the p_functions def validate_pfunctions(self): grammar = [] # Check for non-empty symbols if len(self.pfuncs) == 0: self.log.error('no rules of the form p_rulename are defined') self.error = True return for line, module, name, doc in self.pfuncs: file = inspect.getsourcefile(module) func = self.pdict[name] if isinstance(func, types.MethodType): reqargs = 2 else: reqargs = 1 if func.__code__.co_argcount > reqargs: self.log.error('%s:%d: Rule %r has too many arguments', file, line, func.__name__) self.error = True elif func.__code__.co_argcount < reqargs: self.log.error('%s:%d: Rule %r requires an argument', file, line, func.__name__) self.error = True elif not func.__doc__: self.log.warning('%s:%d: No documentation string specified in function %r (ignored)', file, line, func.__name__) else: try: parsed_g = parse_grammar(doc, file, line) for g in parsed_g: grammar.append((name, g)) except SyntaxError as e: self.log.error(str(e)) self.error = True # Looks like a valid grammar rule # Mark the file in which defined. self.modules.add(module) # Secondary validation step that looks for p_ definitions that are not functions # or functions that look like they might be grammar rules. for n, v in self.pdict.items(): if n.startswith('p_') and isinstance(v, (types.FunctionType, types.MethodType)): continue if n.startswith('t_'): continue if n.startswith('p_') and n != 'p_error': self.log.warning('%r not defined as a function', n) if ((isinstance(v, types.FunctionType) and v.__code__.co_argcount == 1) or (isinstance(v, types.MethodType) and v.__func__.__code__.co_argcount == 2)): if v.__doc__: try: doc = v.__doc__.split(' ') if doc[1] == ':': self.log.warning('%s:%d: Possible grammar rule %r defined without p_ prefix', v.__code__.co_filename, v.__code__.co_firstlineno, n) except IndexError: pass self.grammar = grammar # ----------------------------------------------------------------------------- # yacc(module) # # Build a parser # ----------------------------------------------------------------------------- def yacc(method='LALR', debug=yaccdebug, module=None, tabmodule=tab_module, start=None, check_recursion=True, optimize=False, write_tables=True, debugfile=debug_file, outputdir=None, debuglog=None, errorlog=None, picklefile=None): if tabmodule is None: tabmodule = tab_module # Reference to the parsing method of the last built parser global parse # If pickling is enabled, table files are not created if picklefile: write_tables = 0 if errorlog is None: errorlog = PlyLogger(sys.stderr) # Get the module dictionary used for the parser if module: _items = [(k, getattr(module, k)) for k in dir(module)] pdict = dict(_items) # If no __file__ or __package__ attributes are available, try to obtain them # from the __module__ instead if '__file__' not in pdict: pdict['__file__'] = sys.modules[pdict['__module__']].__file__ if '__package__' not in pdict and '__module__' in pdict: if hasattr(sys.modules[pdict['__module__']], '__package__'): pdict['__package__'] = sys.modules[pdict['__module__']].__package__ else: pdict = get_caller_module_dict(2) if outputdir is None: # If no output directory is set, the location of the output files # is determined according to the following rules: # - If tabmodule specifies a package, files go into that package directory # - Otherwise, files go in the same directory as the specifying module if isinstance(tabmodule, types.ModuleType): srcfile = tabmodule.__file__ else: if '.' not in tabmodule: srcfile = pdict['__file__'] else: parts = tabmodule.split('.') pkgname = '.'.join(parts[:-1]) exec('import %s' % pkgname) srcfile = getattr(sys.modules[pkgname], '__file__', '') outputdir = os.path.dirname(srcfile) # Determine if the module is package of a package or not. # If so, fix the tabmodule setting so that tables load correctly pkg = pdict.get('__package__') if pkg and isinstance(tabmodule, str): if '.' not in tabmodule: tabmodule = pkg + '.' + tabmodule # Set start symbol if it's specified directly using an argument if start is not None: pdict['start'] = start # Collect parser information from the dictionary pinfo = ParserReflect(pdict, log=errorlog) pinfo.get_all() if pinfo.error: raise YaccError('Unable to build parser') # Check signature against table files (if any) signature = pinfo.signature() # Read the tables try: lr = LRTable() if picklefile: read_signature = lr.read_pickle(picklefile) else: read_signature = lr.read_table(tabmodule) if optimize or (read_signature == signature): try: lr.bind_callables(pinfo.pdict) parser = LRParser(lr, pinfo.error_func) parse = parser.parse return parser except Exception as e: errorlog.warning('There was a problem loading the table file: %r', e) except VersionError as e: errorlog.warning(str(e)) except ImportError: pass if debuglog is None: if debug: try: debuglog = PlyLogger(open(os.path.join(outputdir, debugfile), 'w')) except IOError as e: errorlog.warning("Couldn't open %r. %s" % (debugfile, e)) debuglog = NullLogger() else: debuglog = NullLogger() debuglog.info('Created by PLY version %s (http://www.dabeaz.com/ply)', __version__) errors = False # Validate the parser information if pinfo.validate_all(): raise YaccError('Unable to build parser') if not pinfo.error_func: errorlog.warning('no p_error() function is defined') # Create a grammar object grammar = Grammar(pinfo.tokens) # Set precedence level for terminals for term, assoc, level in pinfo.preclist: try: grammar.set_precedence(term, assoc, level) except GrammarError as e: errorlog.warning('%s', e) # Add productions to the grammar for funcname, gram in pinfo.grammar: file, line, prodname, syms = gram try: grammar.add_production(prodname, syms, funcname, file, line) except GrammarError as e: errorlog.error('%s', e) errors = True # Set the grammar start symbols try: if start is None: grammar.set_start(pinfo.start) else: grammar.set_start(start) except GrammarError as e: errorlog.error(str(e)) errors = True if errors: raise YaccError('Unable to build parser') # Verify the grammar structure undefined_symbols = grammar.undefined_symbols() for sym, prod in undefined_symbols: errorlog.error('%s:%d: Symbol %r used, but not defined as a token or a rule', prod.file, prod.line, sym) errors = True unused_terminals = grammar.unused_terminals() if unused_terminals: debuglog.info('') debuglog.info('Unused terminals:') debuglog.info('') for term in unused_terminals: errorlog.warning('Token %r defined, but not used', term) debuglog.info(' %s', term) # Print out all productions to the debug log if debug: debuglog.info('') debuglog.info('Grammar') debuglog.info('') for n, p in enumerate(grammar.Productions): debuglog.info('Rule %-5d %s', n, p) # Find unused non-terminals unused_rules = grammar.unused_rules() for prod in unused_rules: errorlog.warning('%s:%d: Rule %r defined, but not used', prod.file, prod.line, prod.name) if len(unused_terminals) == 1: errorlog.warning('There is 1 unused token') if len(unused_terminals) > 1: errorlog.warning('There are %d unused tokens', len(unused_terminals)) if len(unused_rules) == 1: errorlog.warning('There is 1 unused rule') if len(unused_rules) > 1: errorlog.warning('There are %d unused rules', len(unused_rules)) if debug: debuglog.info('') debuglog.info('Terminals, with rules where they appear') debuglog.info('') terms = list(grammar.Terminals) terms.sort() for term in terms: debuglog.info('%-20s : %s', term, ' '.join([str(s) for s in grammar.Terminals[term]])) debuglog.info('') debuglog.info('Nonterminals, with rules where they appear') debuglog.info('') nonterms = list(grammar.Nonterminals) nonterms.sort() for nonterm in nonterms: debuglog.info('%-20s : %s', nonterm, ' '.join([str(s) for s in grammar.Nonterminals[nonterm]])) debuglog.info('') if check_recursion: unreachable = grammar.find_unreachable() for u in unreachable: errorlog.warning('Symbol %r is unreachable', u) infinite = grammar.infinite_cycles() for inf in infinite: errorlog.error('Infinite recursion detected for symbol %r', inf) errors = True unused_prec = grammar.unused_precedence() for term, assoc in unused_prec: errorlog.error('Precedence rule %r defined for unknown symbol %r', assoc, term) errors = True if errors: raise YaccError('Unable to build parser') # Run the LRGeneratedTable on the grammar if debug: errorlog.debug('Generating %s tables', method) lr = LRGeneratedTable(grammar, method, debuglog) if debug: num_sr = len(lr.sr_conflicts) # Report shift/reduce and reduce/reduce conflicts if num_sr == 1: errorlog.warning('1 shift/reduce conflict') elif num_sr > 1: errorlog.warning('%d shift/reduce conflicts', num_sr) num_rr = len(lr.rr_conflicts) if num_rr == 1: errorlog.warning('1 reduce/reduce conflict') elif num_rr > 1: errorlog.warning('%d reduce/reduce conflicts', num_rr) # Write out conflicts to the output file if debug and (lr.sr_conflicts or lr.rr_conflicts): debuglog.warning('') debuglog.warning('Conflicts:') debuglog.warning('') for state, tok, resolution in lr.sr_conflicts: debuglog.warning('shift/reduce conflict for %s in state %d resolved as %s', tok, state, resolution) already_reported = set() for state, rule, rejected in lr.rr_conflicts: if (state, id(rule), id(rejected)) in already_reported: continue debuglog.warning('reduce/reduce conflict in state %d resolved using rule (%s)', state, rule) debuglog.warning('rejected rule (%s) in state %d', rejected, state) errorlog.warning('reduce/reduce conflict in state %d resolved using rule (%s)', state, rule) errorlog.warning('rejected rule (%s) in state %d', rejected, state) already_reported.add((state, id(rule), id(rejected))) warned_never = [] for state, rule, rejected in lr.rr_conflicts: if not rejected.reduced and (rejected not in warned_never): debuglog.warning('Rule (%s) is never reduced', rejected) errorlog.warning('Rule (%s) is never reduced', rejected) warned_never.append(rejected) # Write the table file if requested if write_tables: try: lr.write_table(tabmodule, outputdir, signature) if tabmodule in sys.modules: del sys.modules[tabmodule] except IOError as e: errorlog.warning("Couldn't create %r. %s" % (tabmodule, e)) # Write a pickled version of the tables if picklefile: try: lr.pickle_table(picklefile, signature) except IOError as e: errorlog.warning("Couldn't create %r. %s" % (picklefile, e)) # Build the parser lr.bind_callables(pinfo.pdict) parser = LRParser(lr, pinfo.error_func) parse = parser.parse return parser ================================================ FILE: xonsh/ply/ply/ygen.py ================================================ # ply: ygen.py # # This is a support program that auto-generates different versions of the YACC parsing # function with different features removed for the purposes of performance. # # Users should edit the method LRParser.parsedebug() in yacc.py. The source code # for that method is then used to create the other methods. See the comments in # yacc.py for further details. import os.path import shutil def get_source_range(lines, tag): srclines = enumerate(lines) start_tag = '#--! %s-start' % tag end_tag = '#--! %s-end' % tag for start_index, line in srclines: if line.strip().startswith(start_tag): break for end_index, line in srclines: if line.strip().endswith(end_tag): break return (start_index + 1, end_index) def filter_section(lines, tag): filtered_lines = [] include = True tag_text = '#--! %s' % tag for line in lines: if line.strip().startswith(tag_text): include = not include elif include: filtered_lines.append(line) return filtered_lines def main(): dirname = os.path.dirname(__file__) shutil.copy2(os.path.join(dirname, 'yacc.py'), os.path.join(dirname, 'yacc.py.bak')) with open(os.path.join(dirname, 'yacc.py'), 'r') as f: lines = f.readlines() parse_start, parse_end = get_source_range(lines, 'parsedebug') parseopt_start, parseopt_end = get_source_range(lines, 'parseopt') parseopt_notrack_start, parseopt_notrack_end = get_source_range(lines, 'parseopt-notrack') # Get the original source orig_lines = lines[parse_start:parse_end] # Filter the DEBUG sections out parseopt_lines = filter_section(orig_lines, 'DEBUG') # Filter the TRACKING sections out parseopt_notrack_lines = filter_section(parseopt_lines, 'TRACKING') # Replace the parser source sections with updated versions lines[parseopt_notrack_start:parseopt_notrack_end] = parseopt_notrack_lines lines[parseopt_start:parseopt_end] = parseopt_lines lines = [line.rstrip()+'\n' for line in lines] with open(os.path.join(dirname, 'yacc.py'), 'w') as f: f.writelines(lines) print('Updated yacc.py') if __name__ == '__main__': main() ================================================ FILE: xonsh/ply/setup.md ================================================ # Maintained, No Package Releases PLY is maintained software, but no longer produces package releases. There is no `setup.py` file. It is not something that you install with `pip` or a similar tool. You must COPY the necessary code from PLY into your project and take ownership of it. Why this policy? PLY is a highly specialized tool for expert-level programmers who are writing parsers and compilers. If you are writing a compiler, there's a good chance that it's part of a substantially larger project. Managing external dependencies (such as PLY) in such projects is an ongoing challenge. However, the truth of the matter is that PLY just isn't that big. All of the core functionality is contained in just two files. PLY has no external dependencies of its own. It changes very rarely. Plus, there are various customizations that you might want to apply to how it works. So, all things equal, it's probably better for you to copy it. But what about getting all of the latest improvements and bug fixes? What improvements? PLY is implementing a 1970s-era parsing algorithm. It's not cutting edge. As for bug fixes, you'll know pretty rapidly if PLY works for your project or not. If it's working, there's literally no reason to ever upgrade it. Keep using the version of code that you copied. If you think you've found a bug, check back with the repository to see if it's been fixed. Or submit it as an issue so that it can be looked at. ================================================ FILE: xonsh/ply/test/README ================================================ This directory mostly contains tests for various types of error conditions. To run: $ python testlex.py $ python testyacc.py $ python testcpp.py The script 'cleanup.sh' cleans up this directory to its original state. ================================================ FILE: xonsh/ply/test/calclex.py ================================================ # ----------------------------------------------------------------------------- # calclex.py # ----------------------------------------------------------------------------- import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.lex as lex tokens = ( 'NAME','NUMBER', 'PLUS','MINUS','TIMES','DIVIDE','EQUALS', 'LPAREN','RPAREN', ) # Tokens t_PLUS = r'\+' t_MINUS = r'-' t_TIMES = r'\*' t_DIVIDE = r'/' t_EQUALS = r'=' t_LPAREN = r'\(' t_RPAREN = r'\)' t_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*' def t_NUMBER(t): r'\d+' try: t.value = int(t.value) except ValueError: print("Integer value too large %s" % t.value) t.value = 0 return t t_ignore = " \t" def t_newline(t): r'\n+' t.lexer.lineno += t.value.count("\n") def t_error(t): print("Illegal character '%s'" % t.value[0]) t.lexer.skip(1) # Build the lexer lexer = lex.lex() ================================================ FILE: xonsh/ply/test/cleanup.sh ================================================ #!/bin/sh rm -rf *~ *.pyc *.pyo *.dif *.out __pycache__ ================================================ FILE: xonsh/ply/test/lex_closure.py ================================================ # ----------------------------------------------------------------------------- # lex_closure.py # ----------------------------------------------------------------------------- import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.lex as lex tokens = ( 'NAME','NUMBER', 'PLUS','MINUS','TIMES','DIVIDE','EQUALS', 'LPAREN','RPAREN', ) def make_calc(): # Tokens t_PLUS = r'\+' t_MINUS = r'-' t_TIMES = r'\*' t_DIVIDE = r'/' t_EQUALS = r'=' t_LPAREN = r'\(' t_RPAREN = r'\)' t_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*' def t_NUMBER(t): r'\d+' try: t.value = int(t.value) except ValueError: print("Integer value too large %s" % t.value) t.value = 0 return t t_ignore = " \t" def t_newline(t): r'\n+' t.lineno += t.value.count("\n") def t_error(t): print("Illegal character '%s'" % t.value[0]) t.lexer.skip(1) # Build the lexer return lex.lex() make_calc() lex.runmain(data="3+4") ================================================ FILE: xonsh/ply/test/lex_doc1.py ================================================ # lex_doc1.py # # Missing documentation string import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.lex as lex tokens = [ "PLUS", "MINUS", "NUMBER", ] t_PLUS = r'\+' t_MINUS = r'-' def t_NUMBER(t): pass def t_error(t): pass lex.lex() ================================================ FILE: xonsh/ply/test/lex_dup1.py ================================================ # lex_dup1.py # # Duplicated rule specifiers import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.lex as lex tokens = [ "PLUS", "MINUS", "NUMBER", ] t_PLUS = r'\+' t_MINUS = r'-' t_NUMBER = r'\d+' t_NUMBER = r'\d+' def t_error(t): pass lex.lex() ================================================ FILE: xonsh/ply/test/lex_dup2.py ================================================ # lex_dup2.py # # Duplicated rule specifiers import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.lex as lex tokens = [ "PLUS", "MINUS", "NUMBER", ] t_PLUS = r'\+' t_MINUS = r'-' def t_NUMBER(t): r'\d+' pass def t_NUMBER(t): r'\d+' pass def t_error(t): pass lex.lex() ================================================ FILE: xonsh/ply/test/lex_dup3.py ================================================ # lex_dup3.py # # Duplicated rule specifiers import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.lex as lex tokens = [ "PLUS", "MINUS", "NUMBER", ] t_PLUS = r'\+' t_MINUS = r'-' t_NUMBER = r'\d+' def t_NUMBER(t): r'\d+' pass def t_error(t): pass lex.lex() ================================================ FILE: xonsh/ply/test/lex_empty.py ================================================ # lex_empty.py # # No rules defined import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.lex as lex tokens = [ "PLUS", "MINUS", "NUMBER", ] lex.lex() ================================================ FILE: xonsh/ply/test/lex_error1.py ================================================ # lex_error1.py # # Missing t_error() rule import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.lex as lex tokens = [ "PLUS", "MINUS", "NUMBER", ] t_PLUS = r'\+' t_MINUS = r'-' t_NUMBER = r'\d+' lex.lex() ================================================ FILE: xonsh/ply/test/lex_error2.py ================================================ # lex_error2.py # # t_error defined, but not function import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.lex as lex tokens = [ "PLUS", "MINUS", "NUMBER", ] t_PLUS = r'\+' t_MINUS = r'-' t_NUMBER = r'\d+' t_error = "foo" lex.lex() ================================================ FILE: xonsh/ply/test/lex_error3.py ================================================ # lex_error3.py # # t_error defined as function, but with wrong # args import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.lex as lex tokens = [ "PLUS", "MINUS", "NUMBER", ] t_PLUS = r'\+' t_MINUS = r'-' t_NUMBER = r'\d+' def t_error(): pass lex.lex() ================================================ FILE: xonsh/ply/test/lex_error4.py ================================================ # lex_error4.py # # t_error defined as function, but too many args import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.lex as lex tokens = [ "PLUS", "MINUS", "NUMBER", ] t_PLUS = r'\+' t_MINUS = r'-' t_NUMBER = r'\d+' def t_error(t,s): pass lex.lex() ================================================ FILE: xonsh/ply/test/lex_hedit.py ================================================ # ----------------------------------------------------------------------------- # hedit.py # # Paring of Fortran H Edit descriptions (Contributed by Pearu Peterson) # # These tokens can't be easily tokenized because they are of the following # form: # # nHc1...cn # # where n is a positive integer and c1 ... cn are characters. # # This example shows how to modify the state of the lexer to parse # such tokens # ----------------------------------------------------------------------------- import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.lex as lex tokens = ( 'H_EDIT_DESCRIPTOR', ) # Tokens t_ignore = " \t\n" def t_H_EDIT_DESCRIPTOR(t): r"\d+H.*" # This grabs all of the remaining text i = t.value.index('H') n = eval(t.value[:i]) # Adjust the tokenizing position t.lexer.lexpos -= len(t.value) - (i+1+n) t.value = t.value[i+1:i+1+n] return t def t_error(t): print("Illegal character '%s'" % t.value[0]) t.lexer.skip(1) # Build the lexer lex.lex() lex.runmain(data="3Habc 10Habcdefghij 2Hxy") ================================================ FILE: xonsh/ply/test/lex_ignore.py ================================================ # lex_ignore.py # # Improperly specific ignore declaration import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.lex as lex tokens = [ "PLUS", "MINUS", "NUMBER", ] t_PLUS = r'\+' t_MINUS = r'-' t_NUMBER = r'\d+' def t_ignore(t): ' \t' pass def t_error(t): pass import sys lex.lex() ================================================ FILE: xonsh/ply/test/lex_ignore2.py ================================================ # lex_ignore2.py # # ignore declaration as a raw string import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.lex as lex tokens = [ "PLUS", "MINUS", "NUMBER", ] t_PLUS = r'\+' t_MINUS = r'-' t_NUMBER = r'\d+' t_ignore = r' \t' def t_error(t): pass lex.lex() ================================================ FILE: xonsh/ply/test/lex_literal1.py ================================================ # lex_literal1.py # # Bad literal specification import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.lex as lex tokens = [ "NUMBER", ] literals = ["+","-","**"] def t_NUMBER(t): r'\d+' return t def t_error(t): pass lex.lex() ================================================ FILE: xonsh/ply/test/lex_literal2.py ================================================ # lex_literal2.py # # Bad literal specification import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.lex as lex tokens = [ "NUMBER", ] literals = 23 def t_NUMBER(t): r'\d+' return t def t_error(t): pass lex.lex() ================================================ FILE: xonsh/ply/test/lex_literal3.py ================================================ # lex_literal3.py # # An empty literal specification given as a list # Issue 8 : Literals empty list causes IndexError import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.lex as lex tokens = [ "NUMBER", ] literals = [] def t_NUMBER(t): r'\d+' return t def t_error(t): pass lex.lex() ================================================ FILE: xonsh/ply/test/lex_many_tokens.py ================================================ # lex_many_tokens.py # # Test lex's ability to handle a large number of tokens (beyond the # 100-group limit of the re module) import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.lex as lex tokens = ["TOK%d" % i for i in range(1000)] for tok in tokens: if sys.version_info[0] < 3: exec("t_%s = '%s:'" % (tok,tok)) else: exec("t_%s = '%s:'" % (tok,tok), globals()) t_ignore = " \t" def t_error(t): pass lex.lex(optimize=1,lextab="manytab") lex.runmain(data="TOK34: TOK143: TOK269: TOK372: TOK452: TOK561: TOK999:") ================================================ FILE: xonsh/ply/test/lex_module.py ================================================ # lex_module.py # import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.lex as lex import lex_module_import lex.lex(module=lex_module_import) lex.runmain(data="3+4") ================================================ FILE: xonsh/ply/test/lex_module_import.py ================================================ # ----------------------------------------------------------------------------- # lex_module_import.py # # A lexer defined in a module, but built in lex_module.py # ----------------------------------------------------------------------------- tokens = ( 'NAME','NUMBER', 'PLUS','MINUS','TIMES','DIVIDE','EQUALS', 'LPAREN','RPAREN', ) # Tokens t_PLUS = r'\+' t_MINUS = r'-' t_TIMES = r'\*' t_DIVIDE = r'/' t_EQUALS = r'=' t_LPAREN = r'\(' t_RPAREN = r'\)' t_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*' def t_NUMBER(t): r'\d+' try: t.value = int(t.value) except ValueError: print("Integer value too large %s" % t.value) t.value = 0 return t t_ignore = " \t" def t_newline(t): r'\n+' t.lineno += t.value.count("\n") def t_error(t): print("Illegal character '%s'" % t.value[0]) t.lexer.skip(1) ================================================ FILE: xonsh/ply/test/lex_object.py ================================================ # ----------------------------------------------------------------------------- # lex_object.py # ----------------------------------------------------------------------------- import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.lex as lex class CalcLexer: tokens = ( 'NAME','NUMBER', 'PLUS','MINUS','TIMES','DIVIDE','EQUALS', 'LPAREN','RPAREN', ) # Tokens t_PLUS = r'\+' t_MINUS = r'-' t_TIMES = r'\*' t_DIVIDE = r'/' t_EQUALS = r'=' t_LPAREN = r'\(' t_RPAREN = r'\)' t_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*' def t_NUMBER(self,t): r'\d+' try: t.value = int(t.value) except ValueError: print("Integer value too large %s" % t.value) t.value = 0 return t t_ignore = " \t" def t_newline(self,t): r'\n+' t.lineno += t.value.count("\n") def t_error(self,t): print("Illegal character '%s'" % t.value[0]) t.lexer.skip(1) calc = CalcLexer() # Build the lexer lex.lex(object=calc) lex.runmain(data="3+4") ================================================ FILE: xonsh/ply/test/lex_opt_alias.py ================================================ # ----------------------------------------------------------------------------- # lex_opt_alias.py # # Tests ability to match up functions with states, aliases, and # lexing tables. # ----------------------------------------------------------------------------- import sys if ".." not in sys.path: sys.path.insert(0,"..") tokens = ( 'NAME','NUMBER', ) states = (('instdef','inclusive'),('spam','exclusive')) literals = ['=','+','-','*','/', '(',')'] # Tokens def t_instdef_spam_BITS(t): r'[01-]+' return t t_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*' def NUMBER(t): r'\d+' try: t.value = int(t.value) except ValueError: print("Integer value too large %s" % t.value) t.value = 0 return t t_ANY_NUMBER = NUMBER t_ignore = " \t" t_spam_ignore = t_ignore def t_newline(t): r'\n+' t.lexer.lineno += t.value.count("\n") def t_error(t): print("Illegal character '%s'" % t.value[0]) t.lexer.skip(1) t_spam_error = t_error # Build the lexer import ply.lex as lex lex.lex(optimize=1,lextab="aliastab") lex.runmain(data="3+4") ================================================ FILE: xonsh/ply/test/lex_optimize.py ================================================ # ----------------------------------------------------------------------------- # lex_optimize.py # ----------------------------------------------------------------------------- import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.lex as lex tokens = ( 'NAME','NUMBER', 'PLUS','MINUS','TIMES','DIVIDE','EQUALS', 'LPAREN','RPAREN', ) # Tokens t_PLUS = r'\+' t_MINUS = r'-' t_TIMES = r'\*' t_DIVIDE = r'/' t_EQUALS = r'=' t_LPAREN = r'\(' t_RPAREN = r'\)' t_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*' def t_NUMBER(t): r'\d+' try: t.value = int(t.value) except ValueError: print("Integer value too large %s" % t.value) t.value = 0 return t t_ignore = " \t" def t_newline(t): r'\n+' t.lineno += t.value.count("\n") def t_error(t): print("Illegal character '%s'" % t.value[0]) t.lexer.skip(1) # Build the lexer lex.lex(optimize=1) lex.runmain(data="3+4") ================================================ FILE: xonsh/ply/test/lex_optimize2.py ================================================ # ----------------------------------------------------------------------------- # lex_optimize2.py # ----------------------------------------------------------------------------- import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.lex as lex tokens = ( 'NAME','NUMBER', 'PLUS','MINUS','TIMES','DIVIDE','EQUALS', 'LPAREN','RPAREN', ) # Tokens t_PLUS = r'\+' t_MINUS = r'-' t_TIMES = r'\*' t_DIVIDE = r'/' t_EQUALS = r'=' t_LPAREN = r'\(' t_RPAREN = r'\)' t_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*' def t_NUMBER(t): r'\d+' try: t.value = int(t.value) except ValueError: print("Integer value too large %s" % t.value) t.value = 0 return t t_ignore = " \t" def t_newline(t): r'\n+' t.lineno += t.value.count("\n") def t_error(t): print("Illegal character '%s'" % t.value[0]) t.lexer.skip(1) # Build the lexer lex.lex(optimize=1,lextab="opt2tab") lex.runmain(data="3+4") ================================================ FILE: xonsh/ply/test/lex_optimize3.py ================================================ # ----------------------------------------------------------------------------- # lex_optimize3.py # # Writes table in a subdirectory structure. # ----------------------------------------------------------------------------- import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.lex as lex tokens = ( 'NAME','NUMBER', 'PLUS','MINUS','TIMES','DIVIDE','EQUALS', 'LPAREN','RPAREN', ) # Tokens t_PLUS = r'\+' t_MINUS = r'-' t_TIMES = r'\*' t_DIVIDE = r'/' t_EQUALS = r'=' t_LPAREN = r'\(' t_RPAREN = r'\)' t_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*' def t_NUMBER(t): r'\d+' try: t.value = int(t.value) except ValueError: print("Integer value too large %s" % t.value) t.value = 0 return t t_ignore = " \t" def t_newline(t): r'\n+' t.lineno += t.value.count("\n") def t_error(t): print("Illegal character '%s'" % t.value[0]) t.lexer.skip(1) # Build the lexer lex.lex(optimize=1,lextab="lexdir.sub.calctab" ,outputdir="lexdir/sub") lex.runmain(data="3+4") ================================================ FILE: xonsh/ply/test/lex_optimize4.py ================================================ # ----------------------------------------------------------------------------- # lex_optimize4.py # ----------------------------------------------------------------------------- import re import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.lex as lex tokens = [ "PLUS", "MINUS", "NUMBER", ] t_PLUS = r'\+?' t_MINUS = r'-' t_NUMBER = r'(\d+)' def t_error(t): pass # Build the lexer lex.lex(optimize=True, lextab="opt4tab", reflags=re.UNICODE) lex.runmain(data="3+4") ================================================ FILE: xonsh/ply/test/lex_re1.py ================================================ # lex_re1.py # # Bad regular expression in a string import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.lex as lex tokens = [ "PLUS", "MINUS", "NUMBER", ] t_PLUS = r'\+' t_MINUS = r'-' t_NUMBER = r'(\d+' def t_error(t): pass lex.lex() ================================================ FILE: xonsh/ply/test/lex_re2.py ================================================ # lex_re2.py # # Regular expression rule matches empty string import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.lex as lex tokens = [ "PLUS", "MINUS", "NUMBER", ] t_PLUS = r'\+?' t_MINUS = r'-' t_NUMBER = r'(\d+)' def t_error(t): pass lex.lex() ================================================ FILE: xonsh/ply/test/lex_re3.py ================================================ # lex_re3.py # # Regular expression rule matches empty string import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.lex as lex tokens = [ "PLUS", "MINUS", "NUMBER", "POUND", ] t_PLUS = r'\+' t_MINUS = r'-' t_NUMBER = r'(\d+)' t_POUND = r'#' def t_error(t): pass lex.lex() ================================================ FILE: xonsh/ply/test/lex_rule1.py ================================================ # lex_rule1.py # # Rule function with incorrect number of arguments import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.lex as lex tokens = [ "PLUS", "MINUS", "NUMBER", ] t_PLUS = r'\+' t_MINUS = r'-' t_NUMBER = 1 def t_error(t): pass lex.lex() ================================================ FILE: xonsh/ply/test/lex_rule2.py ================================================ # lex_rule2.py # # Rule function with incorrect number of arguments import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.lex as lex tokens = [ "PLUS", "MINUS", "NUMBER", ] t_PLUS = r'\+' t_MINUS = r'-' def t_NUMBER(): r'\d+' return t def t_error(t): pass lex.lex() ================================================ FILE: xonsh/ply/test/lex_rule3.py ================================================ # lex_rule3.py # # Rule function with incorrect number of arguments import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.lex as lex tokens = [ "PLUS", "MINUS", "NUMBER", ] t_PLUS = r'\+' t_MINUS = r'-' def t_NUMBER(t,s): r'\d+' return t def t_error(t): pass lex.lex() ================================================ FILE: xonsh/ply/test/lex_state1.py ================================================ # lex_state1.py # # Bad state declaration import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.lex as lex tokens = [ "PLUS", "MINUS", "NUMBER", ] states = 'comment' t_PLUS = r'\+' t_MINUS = r'-' t_NUMBER = r'\d+' # Comments def t_comment(t): r'/\*' t.lexer.begin('comment') print("Entering comment state") def t_comment_body_part(t): r'(.|\n)*\*/' print("comment body %s" % t) t.lexer.begin('INITIAL') def t_error(t): pass lex.lex() ================================================ FILE: xonsh/ply/test/lex_state2.py ================================================ # lex_state2.py # # Bad state declaration import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.lex as lex tokens = [ "PLUS", "MINUS", "NUMBER", ] states = ('comment','example') t_PLUS = r'\+' t_MINUS = r'-' t_NUMBER = r'\d+' # Comments def t_comment(t): r'/\*' t.lexer.begin('comment') print("Entering comment state") def t_comment_body_part(t): r'(.|\n)*\*/' print("comment body %s" % t) t.lexer.begin('INITIAL') def t_error(t): pass lex.lex() ================================================ FILE: xonsh/ply/test/lex_state3.py ================================================ # lex_state3.py # # Bad state declaration import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.lex as lex tokens = [ "PLUS", "MINUS", "NUMBER", ] comment = 1 states = ((comment, 'inclusive'), ('example', 'exclusive')) t_PLUS = r'\+' t_MINUS = r'-' t_NUMBER = r'\d+' # Comments def t_comment(t): r'/\*' t.lexer.begin('comment') print("Entering comment state") def t_comment_body_part(t): r'(.|\n)*\*/' print("comment body %s" % t) t.lexer.begin('INITIAL') def t_error(t): pass lex.lex() ================================================ FILE: xonsh/ply/test/lex_state4.py ================================================ # lex_state4.py # # Bad state declaration import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.lex as lex tokens = [ "PLUS", "MINUS", "NUMBER", ] states = (('comment', 'exclsive'),) t_PLUS = r'\+' t_MINUS = r'-' t_NUMBER = r'\d+' # Comments def t_comment(t): r'/\*' t.lexer.begin('comment') print("Entering comment state") def t_comment_body_part(t): r'(.|\n)*\*/' print("comment body %s" % t) t.lexer.begin('INITIAL') def t_error(t): pass lex.lex() ================================================ FILE: xonsh/ply/test/lex_state5.py ================================================ # lex_state5.py # # Bad state declaration import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.lex as lex tokens = [ "PLUS", "MINUS", "NUMBER", ] states = (('comment', 'exclusive'), ('comment', 'exclusive')) t_PLUS = r'\+' t_MINUS = r'-' t_NUMBER = r'\d+' # Comments def t_comment(t): r'/\*' t.lexer.begin('comment') print("Entering comment state") def t_comment_body_part(t): r'(.|\n)*\*/' print("comment body %s" % t) t.lexer.begin('INITIAL') def t_error(t): pass lex.lex() ================================================ FILE: xonsh/ply/test/lex_state_noerror.py ================================================ # lex_state_noerror.py # # Declaration of a state for which no rules are defined import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.lex as lex tokens = [ "PLUS", "MINUS", "NUMBER", ] states = (('comment', 'exclusive'),) t_PLUS = r'\+' t_MINUS = r'-' t_NUMBER = r'\d+' # Comments def t_comment(t): r'/\*' t.lexer.begin('comment') print("Entering comment state") def t_comment_body_part(t): r'(.|\n)*\*/' print("comment body %s" % t) t.lexer.begin('INITIAL') def t_error(t): pass lex.lex() ================================================ FILE: xonsh/ply/test/lex_state_norule.py ================================================ # lex_state_norule.py # # Declaration of a state for which no rules are defined import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.lex as lex tokens = [ "PLUS", "MINUS", "NUMBER", ] states = (('comment', 'exclusive'), ('example', 'exclusive')) t_PLUS = r'\+' t_MINUS = r'-' t_NUMBER = r'\d+' # Comments def t_comment(t): r'/\*' t.lexer.begin('comment') print("Entering comment state") def t_comment_body_part(t): r'(.|\n)*\*/' print("comment body %s" % t) t.lexer.begin('INITIAL') def t_error(t): pass lex.lex() ================================================ FILE: xonsh/ply/test/lex_state_try.py ================================================ # lex_state_try.py # # Declaration of a state for which no rules are defined import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.lex as lex tokens = [ "PLUS", "MINUS", "NUMBER", ] states = (('comment', 'exclusive'),) t_PLUS = r'\+' t_MINUS = r'-' t_NUMBER = r'\d+' t_ignore = " \t" # Comments def t_comment(t): r'/\*' t.lexer.begin('comment') print("Entering comment state") def t_comment_body_part(t): r'(.|\n)*\*/' print("comment body %s" % t) t.lexer.begin('INITIAL') def t_error(t): pass t_comment_error = t_error t_comment_ignore = t_ignore lex.lex() data = "3 + 4 /* This is a comment */ + 10" lex.runmain(data=data) ================================================ FILE: xonsh/ply/test/lex_token1.py ================================================ # lex_token1.py # # Tests for absence of tokens variable import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.lex as lex t_PLUS = r'\+' t_MINUS = r'-' t_NUMBER = r'\d+' def t_error(t): pass lex.lex() ================================================ FILE: xonsh/ply/test/lex_token2.py ================================================ # lex_token2.py # # Tests for tokens of wrong type import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.lex as lex tokens = "PLUS MINUS NUMBER" t_PLUS = r'\+' t_MINUS = r'-' t_NUMBER = r'\d+' def t_error(t): pass lex.lex() ================================================ FILE: xonsh/ply/test/lex_token3.py ================================================ # lex_token3.py # # tokens is right type, but is missing a token for one rule import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.lex as lex tokens = [ "PLUS", "NUMBER", ] t_PLUS = r'\+' t_MINUS = r'-' t_NUMBER = r'\d+' def t_error(t): pass lex.lex() ================================================ FILE: xonsh/ply/test/lex_token4.py ================================================ # lex_token4.py # # Bad token name import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.lex as lex tokens = [ "PLUS", "MINUS", "-", "NUMBER", ] t_PLUS = r'\+' t_MINUS = r'-' t_NUMBER = r'\d+' def t_error(t): pass lex.lex() ================================================ FILE: xonsh/ply/test/lex_token5.py ================================================ # lex_token5.py # # Return a bad token name import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.lex as lex tokens = [ "PLUS", "MINUS", "NUMBER", ] t_PLUS = r'\+' t_MINUS = r'-' def t_NUMBER(t): r'\d+' t.type = "NUM" return t def t_error(t): pass lex.lex() lex.input("1234") t = lex.token() ================================================ FILE: xonsh/ply/test/lex_token_dup.py ================================================ # lex_token_dup.py # # Duplicate token name in tokens import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.lex as lex tokens = [ "PLUS", "MINUS", "NUMBER", "MINUS" ] t_PLUS = r'\+' t_MINUS = r'-' def t_NUMBER(t): r'\d+' return t def t_error(t): pass lex.lex() ================================================ FILE: xonsh/ply/test/pkg_test1/__init__.py ================================================ # Tests proper handling of lextab and parsetab files in package structures # Here for testing purposes import sys if '..' not in sys.path: sys.path.insert(0, '..') from .parsing.calcparse import parser ================================================ FILE: xonsh/ply/test/pkg_test1/parsing/__init__.py ================================================ ================================================ FILE: xonsh/ply/test/pkg_test1/parsing/calclex.py ================================================ # ----------------------------------------------------------------------------- # calclex.py # ----------------------------------------------------------------------------- import ply.lex as lex tokens = ( 'NAME','NUMBER', 'PLUS','MINUS','TIMES','DIVIDE','EQUALS', 'LPAREN','RPAREN', ) # Tokens t_PLUS = r'\+' t_MINUS = r'-' t_TIMES = r'\*' t_DIVIDE = r'/' t_EQUALS = r'=' t_LPAREN = r'\(' t_RPAREN = r'\)' t_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*' def t_NUMBER(t): r'\d+' try: t.value = int(t.value) except ValueError: print("Integer value too large %s" % t.value) t.value = 0 return t t_ignore = " \t" def t_newline(t): r'\n+' t.lexer.lineno += t.value.count("\n") def t_error(t): print("Illegal character '%s'" % t.value[0]) t.lexer.skip(1) # Build the lexer lexer = lex.lex(optimize=True) ================================================ FILE: xonsh/ply/test/pkg_test1/parsing/calcparse.py ================================================ # ----------------------------------------------------------------------------- # yacc_simple.py # # A simple, properly specifier grammar # ----------------------------------------------------------------------------- from .calclex import tokens from ply import yacc # Parsing rules precedence = ( ('left','PLUS','MINUS'), ('left','TIMES','DIVIDE'), ('right','UMINUS'), ) # dictionary of names names = { } def p_statement_assign(t): 'statement : NAME EQUALS expression' names[t[1]] = t[3] def p_statement_expr(t): 'statement : expression' t[0] = t[1] def p_expression_binop(t): '''expression : expression PLUS expression | expression MINUS expression | expression TIMES expression | expression DIVIDE expression''' if t[2] == '+' : t[0] = t[1] + t[3] elif t[2] == '-': t[0] = t[1] - t[3] elif t[2] == '*': t[0] = t[1] * t[3] elif t[2] == '/': t[0] = t[1] / t[3] def p_expression_uminus(t): 'expression : MINUS expression %prec UMINUS' t[0] = -t[2] def p_expression_group(t): 'expression : LPAREN expression RPAREN' t[0] = t[2] def p_expression_number(t): 'expression : NUMBER' t[0] = t[1] def p_expression_name(t): 'expression : NAME' try: t[0] = names[t[1]] except LookupError: print("Undefined name '%s'" % t[1]) t[0] = 0 def p_error(t): print("Syntax error at '%s'" % t.value) parser = yacc.yacc() ================================================ FILE: xonsh/ply/test/pkg_test2/__init__.py ================================================ # Tests proper handling of lextab and parsetab files in package structures # Here for testing purposes import sys if '..' not in sys.path: sys.path.insert(0, '..') from .parsing.calcparse import parser ================================================ FILE: xonsh/ply/test/pkg_test2/parsing/__init__.py ================================================ ================================================ FILE: xonsh/ply/test/pkg_test2/parsing/calclex.py ================================================ # ----------------------------------------------------------------------------- # calclex.py # ----------------------------------------------------------------------------- import ply.lex as lex tokens = ( 'NAME','NUMBER', 'PLUS','MINUS','TIMES','DIVIDE','EQUALS', 'LPAREN','RPAREN', ) # Tokens t_PLUS = r'\+' t_MINUS = r'-' t_TIMES = r'\*' t_DIVIDE = r'/' t_EQUALS = r'=' t_LPAREN = r'\(' t_RPAREN = r'\)' t_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*' def t_NUMBER(t): r'\d+' try: t.value = int(t.value) except ValueError: print("Integer value too large %s" % t.value) t.value = 0 return t t_ignore = " \t" def t_newline(t): r'\n+' t.lexer.lineno += t.value.count("\n") def t_error(t): print("Illegal character '%s'" % t.value[0]) t.lexer.skip(1) # Build the lexer lexer = lex.lex(optimize=True, lextab='calclextab') ================================================ FILE: xonsh/ply/test/pkg_test2/parsing/calcparse.py ================================================ # ----------------------------------------------------------------------------- # yacc_simple.py # # A simple, properly specifier grammar # ----------------------------------------------------------------------------- from .calclex import tokens from ply import yacc # Parsing rules precedence = ( ('left','PLUS','MINUS'), ('left','TIMES','DIVIDE'), ('right','UMINUS'), ) # dictionary of names names = { } def p_statement_assign(t): 'statement : NAME EQUALS expression' names[t[1]] = t[3] def p_statement_expr(t): 'statement : expression' t[0] = t[1] def p_expression_binop(t): '''expression : expression PLUS expression | expression MINUS expression | expression TIMES expression | expression DIVIDE expression''' if t[2] == '+' : t[0] = t[1] + t[3] elif t[2] == '-': t[0] = t[1] - t[3] elif t[2] == '*': t[0] = t[1] * t[3] elif t[2] == '/': t[0] = t[1] / t[3] def p_expression_uminus(t): 'expression : MINUS expression %prec UMINUS' t[0] = -t[2] def p_expression_group(t): 'expression : LPAREN expression RPAREN' t[0] = t[2] def p_expression_number(t): 'expression : NUMBER' t[0] = t[1] def p_expression_name(t): 'expression : NAME' try: t[0] = names[t[1]] except LookupError: print("Undefined name '%s'" % t[1]) t[0] = 0 def p_error(t): print("Syntax error at '%s'" % t.value) parser = yacc.yacc(tabmodule='calcparsetab') ================================================ FILE: xonsh/ply/test/pkg_test3/__init__.py ================================================ # Tests proper handling of lextab and parsetab files in package structures # Here for testing purposes import sys if '..' not in sys.path: sys.path.insert(0, '..') from .parsing.calcparse import parser ================================================ FILE: xonsh/ply/test/pkg_test3/generated/__init__.py ================================================ ================================================ FILE: xonsh/ply/test/pkg_test3/parsing/__init__.py ================================================ ================================================ FILE: xonsh/ply/test/pkg_test3/parsing/calclex.py ================================================ # ----------------------------------------------------------------------------- # calclex.py # ----------------------------------------------------------------------------- import ply.lex as lex tokens = ( 'NAME','NUMBER', 'PLUS','MINUS','TIMES','DIVIDE','EQUALS', 'LPAREN','RPAREN', ) # Tokens t_PLUS = r'\+' t_MINUS = r'-' t_TIMES = r'\*' t_DIVIDE = r'/' t_EQUALS = r'=' t_LPAREN = r'\(' t_RPAREN = r'\)' t_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*' def t_NUMBER(t): r'\d+' try: t.value = int(t.value) except ValueError: print("Integer value too large %s" % t.value) t.value = 0 return t t_ignore = " \t" def t_newline(t): r'\n+' t.lexer.lineno += t.value.count("\n") def t_error(t): print("Illegal character '%s'" % t.value[0]) t.lexer.skip(1) # Build the lexer lexer = lex.lex(optimize=True, lextab='pkg_test3.generated.lextab') ================================================ FILE: xonsh/ply/test/pkg_test3/parsing/calcparse.py ================================================ # ----------------------------------------------------------------------------- # yacc_simple.py # # A simple, properly specifier grammar # ----------------------------------------------------------------------------- from .calclex import tokens from ply import yacc # Parsing rules precedence = ( ('left','PLUS','MINUS'), ('left','TIMES','DIVIDE'), ('right','UMINUS'), ) # dictionary of names names = { } def p_statement_assign(t): 'statement : NAME EQUALS expression' names[t[1]] = t[3] def p_statement_expr(t): 'statement : expression' t[0] = t[1] def p_expression_binop(t): '''expression : expression PLUS expression | expression MINUS expression | expression TIMES expression | expression DIVIDE expression''' if t[2] == '+' : t[0] = t[1] + t[3] elif t[2] == '-': t[0] = t[1] - t[3] elif t[2] == '*': t[0] = t[1] * t[3] elif t[2] == '/': t[0] = t[1] / t[3] def p_expression_uminus(t): 'expression : MINUS expression %prec UMINUS' t[0] = -t[2] def p_expression_group(t): 'expression : LPAREN expression RPAREN' t[0] = t[2] def p_expression_number(t): 'expression : NUMBER' t[0] = t[1] def p_expression_name(t): 'expression : NAME' try: t[0] = names[t[1]] except LookupError: print("Undefined name '%s'" % t[1]) t[0] = 0 def p_error(t): print("Syntax error at '%s'" % t.value) parser = yacc.yacc(tabmodule='pkg_test3.generated.parsetab') ================================================ FILE: xonsh/ply/test/pkg_test4/__init__.py ================================================ # Tests proper handling of lextab and parsetab files in package structures # Check of warning messages when files aren't writable # Here for testing purposes import sys if '..' not in sys.path: sys.path.insert(0, '..') import ply.lex import ply.yacc def patched_open(filename, mode): if 'w' in mode: raise IOError("Permission denied %r" % filename) return open(filename, mode) ply.lex.open = patched_open ply.yacc.open = patched_open try: from .parsing.calcparse import parser finally: del ply.lex.open del ply.yacc.open ================================================ FILE: xonsh/ply/test/pkg_test4/parsing/__init__.py ================================================ ================================================ FILE: xonsh/ply/test/pkg_test4/parsing/calclex.py ================================================ # ----------------------------------------------------------------------------- # calclex.py # ----------------------------------------------------------------------------- import ply.lex as lex tokens = ( 'NAME','NUMBER', 'PLUS','MINUS','TIMES','DIVIDE','EQUALS', 'LPAREN','RPAREN', ) # Tokens t_PLUS = r'\+' t_MINUS = r'-' t_TIMES = r'\*' t_DIVIDE = r'/' t_EQUALS = r'=' t_LPAREN = r'\(' t_RPAREN = r'\)' t_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*' def t_NUMBER(t): r'\d+' try: t.value = int(t.value) except ValueError: print("Integer value too large %s" % t.value) t.value = 0 return t t_ignore = " \t" def t_newline(t): r'\n+' t.lexer.lineno += t.value.count("\n") def t_error(t): print("Illegal character '%s'" % t.value[0]) t.lexer.skip(1) # Build the lexer lexer = lex.lex(optimize=True) ================================================ FILE: xonsh/ply/test/pkg_test4/parsing/calcparse.py ================================================ # ----------------------------------------------------------------------------- # yacc_simple.py # # A simple, properly specifier grammar # ----------------------------------------------------------------------------- from .calclex import tokens from ply import yacc # Parsing rules precedence = ( ('left','PLUS','MINUS'), ('left','TIMES','DIVIDE'), ('right','UMINUS'), ) # dictionary of names names = { } def p_statement_assign(t): 'statement : NAME EQUALS expression' names[t[1]] = t[3] def p_statement_expr(t): 'statement : expression' t[0] = t[1] def p_expression_binop(t): '''expression : expression PLUS expression | expression MINUS expression | expression TIMES expression | expression DIVIDE expression''' if t[2] == '+' : t[0] = t[1] + t[3] elif t[2] == '-': t[0] = t[1] - t[3] elif t[2] == '*': t[0] = t[1] * t[3] elif t[2] == '/': t[0] = t[1] / t[3] def p_expression_uminus(t): 'expression : MINUS expression %prec UMINUS' t[0] = -t[2] def p_expression_group(t): 'expression : LPAREN expression RPAREN' t[0] = t[2] def p_expression_number(t): 'expression : NUMBER' t[0] = t[1] def p_expression_name(t): 'expression : NAME' try: t[0] = names[t[1]] except LookupError: print("Undefined name '%s'" % t[1]) t[0] = 0 def p_error(t): print("Syntax error at '%s'" % t.value) parser = yacc.yacc() ================================================ FILE: xonsh/ply/test/pkg_test5/__init__.py ================================================ # Tests proper handling of lextab and parsetab files in package structures # Here for testing purposes import sys if '..' not in sys.path: sys.path.insert(0, '..') from .parsing.calcparse import parser ================================================ FILE: xonsh/ply/test/pkg_test5/parsing/__init__.py ================================================ ================================================ FILE: xonsh/ply/test/pkg_test5/parsing/calclex.py ================================================ # ----------------------------------------------------------------------------- # calclex.py # ----------------------------------------------------------------------------- import ply.lex as lex tokens = ( 'NAME','NUMBER', 'PLUS','MINUS','TIMES','DIVIDE','EQUALS', 'LPAREN','RPAREN', ) # Tokens t_PLUS = r'\+' t_MINUS = r'-' t_TIMES = r'\*' t_DIVIDE = r'/' t_EQUALS = r'=' t_LPAREN = r'\(' t_RPAREN = r'\)' t_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*' def t_NUMBER(t): r'\d+' try: t.value = int(t.value) except ValueError: print("Integer value too large %s" % t.value) t.value = 0 return t t_ignore = " \t" def t_newline(t): r'\n+' t.lexer.lineno += t.value.count("\n") def t_error(t): print("Illegal character '%s'" % t.value[0]) t.lexer.skip(1) # Build the lexer import os.path lexer = lex.lex(optimize=True, outputdir=os.path.dirname(__file__)) ================================================ FILE: xonsh/ply/test/pkg_test5/parsing/calcparse.py ================================================ # ----------------------------------------------------------------------------- # yacc_simple.py # # A simple, properly specifier grammar # ----------------------------------------------------------------------------- from .calclex import tokens from ply import yacc # Parsing rules precedence = ( ('left','PLUS','MINUS'), ('left','TIMES','DIVIDE'), ('right','UMINUS'), ) # dictionary of names names = { } def p_statement_assign(t): 'statement : NAME EQUALS expression' names[t[1]] = t[3] def p_statement_expr(t): 'statement : expression' t[0] = t[1] def p_expression_binop(t): '''expression : expression PLUS expression | expression MINUS expression | expression TIMES expression | expression DIVIDE expression''' if t[2] == '+' : t[0] = t[1] + t[3] elif t[2] == '-': t[0] = t[1] - t[3] elif t[2] == '*': t[0] = t[1] * t[3] elif t[2] == '/': t[0] = t[1] / t[3] def p_expression_uminus(t): 'expression : MINUS expression %prec UMINUS' t[0] = -t[2] def p_expression_group(t): 'expression : LPAREN expression RPAREN' t[0] = t[2] def p_expression_number(t): 'expression : NUMBER' t[0] = t[1] def p_expression_name(t): 'expression : NAME' try: t[0] = names[t[1]] except LookupError: print("Undefined name '%s'" % t[1]) t[0] = 0 def p_error(t): print("Syntax error at '%s'" % t.value) import os.path parser = yacc.yacc(outputdir=os.path.dirname(__file__)) ================================================ FILE: xonsh/ply/test/pkg_test6/__init__.py ================================================ # Tests proper sorting of modules in yacc.ParserReflect.get_pfunctions # Here for testing purposes import sys if '..' not in sys.path: sys.path.insert(0, '..') from .parsing.calcparse import parser ================================================ FILE: xonsh/ply/test/pkg_test6/parsing/__init__.py ================================================ ================================================ FILE: xonsh/ply/test/pkg_test6/parsing/calclex.py ================================================ # ----------------------------------------------------------------------------- # calclex.py # ----------------------------------------------------------------------------- import ply.lex as lex tokens = ( 'NAME','NUMBER', 'PLUS','MINUS','TIMES','DIVIDE','EQUALS', 'LPAREN','RPAREN', ) # Tokens t_PLUS = r'\+' t_MINUS = r'-' t_TIMES = r'\*' t_DIVIDE = r'/' t_EQUALS = r'=' t_LPAREN = r'\(' t_RPAREN = r'\)' t_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*' def t_NUMBER(t): r'\d+' try: t.value = int(t.value) except ValueError: print("Integer value too large %s" % t.value) t.value = 0 return t t_ignore = " \t" def t_newline(t): r'\n+' t.lexer.lineno += t.value.count("\n") def t_error(t): print("Illegal character '%s'" % t.value[0]) t.lexer.skip(1) # Build the lexer import os.path lexer = lex.lex(optimize=True, outputdir=os.path.dirname(__file__)) ================================================ FILE: xonsh/ply/test/pkg_test6/parsing/calcparse.py ================================================ # ----------------------------------------------------------------------------- # yacc_simple.py # # A simple, properly specifier grammar # ----------------------------------------------------------------------------- from .calclex import tokens from ply import yacc # Parsing rules precedence = ( ('left','PLUS','MINUS'), ('left','TIMES','DIVIDE'), ('right','UMINUS'), ) # dictionary of names names = { } from .statement import * from .expression import * def p_error(t): print("Syntax error at '%s'" % t.value) import os.path parser = yacc.yacc(outputdir=os.path.dirname(__file__)) ================================================ FILE: xonsh/ply/test/pkg_test6/parsing/expression.py ================================================ # This file contains definitions of expression grammar def p_expression_binop(t): '''expression : expression PLUS expression | expression MINUS expression | expression TIMES expression | expression DIVIDE expression''' if t[2] == '+' : t[0] = t[1] + t[3] elif t[2] == '-': t[0] = t[1] - t[3] elif t[2] == '*': t[0] = t[1] * t[3] elif t[2] == '/': t[0] = t[1] / t[3] def p_expression_uminus(t): 'expression : MINUS expression %prec UMINUS' t[0] = -t[2] def p_expression_group(t): 'expression : LPAREN expression RPAREN' t[0] = t[2] def p_expression_number(t): 'expression : NUMBER' t[0] = t[1] def p_expression_name(t): 'expression : NAME' try: t[0] = names[t[1]] except LookupError: print("Undefined name '%s'" % t[1]) t[0] = 0 ================================================ FILE: xonsh/ply/test/pkg_test6/parsing/statement.py ================================================ # This file contains definitions of statement grammar def p_statement_assign(t): 'statement : NAME EQUALS expression' names[t[1]] = t[3] def p_statement_expr(t): 'statement : expression' t[0] = t[1] ================================================ FILE: xonsh/ply/test/test_cpp_nonascii.c ================================================ /* ë */ #define x 1 ================================================ FILE: xonsh/ply/test/testcpp.py ================================================ from unittest import TestCase, main from multiprocessing import Process, Queue from six.moves.queue import Empty import sys import locale if ".." not in sys.path: sys.path.insert(0, "..") from ply.lex import lex from ply.cpp import * def preprocessing(in_, out_queue): out = None try: p = Preprocessor(lex()) p.parse(in_) tokens = [t.value for t in p.parser] out = "".join(tokens) finally: out_queue.put(out) class CPPTests(TestCase): "Tests related to ANSI-C style lexical preprocessor." def __test_preprocessing(self, in_, expected, time_limit = 1.0): out_queue = Queue() preprocessor = Process( name = "PLY`s C preprocessor", target = preprocessing, args = (in_, out_queue) ) preprocessor.start() try: out = out_queue.get(timeout = time_limit) except Empty: preprocessor.terminate() raise RuntimeError("Time limit exceeded!") else: self.assertMultiLineEqual(out, expected) def test_infinite_argument_expansion(self): # CPP does not drags set of currently expanded macros through macro # arguments expansion. If there is a match between an argument value # and name of an already expanded macro then CPP falls into infinite # recursion. self.__test_preprocessing("""\ #define a(x) x #define b a(b) b """ , """\ b""" ) def test_concatenation(self): self.__test_preprocessing("""\ #define a(x) x##_ #define b(x) _##x #define c(x) _##x##_ #define d(x,y) _##x##y##_ a(i) b(j) c(k) d(q,s)""" , """\ i_ _j _k_ _qs_""" ) def test_deadloop_macro(self): # If there is a word which equals to name of a parametrized macro, then # attempt to expand such word as a macro manages the parser to fall # into an infinite loop. self.__test_preprocessing("""\ #define a(x) x a;""" , """\ a;""" ) def test_index_error(self): # If there are no tokens after a word ("a") which equals to name of # a parameterized macro, then attempt to expand this word leads to # IndexError. self.__test_preprocessing("""\ #define a(x) x a""" , """\ a""" ) def test_evalexpr(self): # #if 1 != 2 is not processed correctly; undefined values are converted # to 0L instead of 0 (issue #195) # self.__test_preprocessing("""\ #if (1!=0) && (!x || (!(1==2))) a; #else b; #endif """ , """\ a; """ ) def test_include_nonascii(self): # Issue #196: #included files are read using the current locale's # getdefaultencoding. if a #included file contains non-ascii characters, # while default encoding is e.g. US_ASCII, this causes an error locale.setlocale(locale.LC_ALL, 'C') self.__test_preprocessing("""\ #include "test_cpp_nonascii.c" x; """ , """\ 1; """ ) main() ================================================ FILE: xonsh/ply/test/testlex.py ================================================ # testlex.py import unittest try: import StringIO except ImportError: import io as StringIO import sys import os import warnings import platform sys.path.insert(0,"..") sys.tracebacklimit = 0 import ply.lex try: from importlib.util import cache_from_source except ImportError: # Python 2.7, but we don't care. cache_from_source = None def make_pymodule_path(filename, optimization=None): path = os.path.dirname(filename) file = os.path.basename(filename) mod, ext = os.path.splitext(file) if sys.hexversion >= 0x3050000: fullpath = cache_from_source(filename, optimization=optimization) elif sys.hexversion >= 0x3040000: fullpath = cache_from_source(filename, ext=='.pyc') elif sys.hexversion >= 0x3020000: import imp modname = mod+"."+imp.get_tag()+ext fullpath = os.path.join(path,'__pycache__',modname) else: fullpath = filename return fullpath def pymodule_out_exists(filename, optimization=None): return os.path.exists(make_pymodule_path(filename, optimization=optimization)) def pymodule_out_remove(filename, optimization=None): os.remove(make_pymodule_path(filename, optimization=optimization)) def implementation(): if platform.system().startswith("Java"): return "Jython" elif hasattr(sys, "pypy_version_info"): return "PyPy" else: return "CPython" test_pyo = (implementation() == 'CPython') def check_expected(result, expected, contains=False): if sys.version_info[0] >= 3: if isinstance(result,str): result = result.encode('ascii') if isinstance(expected,str): expected = expected.encode('ascii') resultlines = result.splitlines() expectedlines = expected.splitlines() if len(resultlines) != len(expectedlines): return False for rline,eline in zip(resultlines,expectedlines): if contains: if eline not in rline: return False else: if not rline.endswith(eline): return False return True def run_import(module): code = "import "+module exec(code) del sys.modules[module] # Tests related to errors and warnings when building lexers class LexErrorWarningTests(unittest.TestCase): def setUp(self): sys.stderr = StringIO.StringIO() sys.stdout = StringIO.StringIO() if sys.hexversion >= 0x3020000: warnings.filterwarnings('ignore',category=ResourceWarning) def tearDown(self): sys.stderr = sys.__stderr__ sys.stdout = sys.__stdout__ def test_lex_doc1(self): self.assertRaises(SyntaxError,run_import,"lex_doc1") result = sys.stderr.getvalue() self.assert_(check_expected(result, "lex_doc1.py:18: No regular expression defined for rule 't_NUMBER'\n")) def test_lex_dup1(self): self.assertRaises(SyntaxError,run_import,"lex_dup1") result = sys.stderr.getvalue() self.assert_(check_expected(result, "lex_dup1.py:20: Rule t_NUMBER redefined. Previously defined on line 18\n" )) def test_lex_dup2(self): self.assertRaises(SyntaxError,run_import,"lex_dup2") result = sys.stderr.getvalue() self.assert_(check_expected(result, "lex_dup2.py:22: Rule t_NUMBER redefined. Previously defined on line 18\n" )) def test_lex_dup3(self): self.assertRaises(SyntaxError,run_import,"lex_dup3") result = sys.stderr.getvalue() self.assert_(check_expected(result, "lex_dup3.py:20: Rule t_NUMBER redefined. Previously defined on line 18\n" )) def test_lex_empty(self): self.assertRaises(SyntaxError,run_import,"lex_empty") result = sys.stderr.getvalue() self.assert_(check_expected(result, "No rules of the form t_rulename are defined\n" "No rules defined for state 'INITIAL'\n")) def test_lex_error1(self): run_import("lex_error1") result = sys.stderr.getvalue() self.assert_(check_expected(result, "No t_error rule is defined\n")) def test_lex_error2(self): self.assertRaises(SyntaxError,run_import,"lex_error2") result = sys.stderr.getvalue() self.assert_(check_expected(result, "Rule 't_error' must be defined as a function\n") ) def test_lex_error3(self): self.assertRaises(SyntaxError,run_import,"lex_error3") result = sys.stderr.getvalue() self.assert_(check_expected(result, "lex_error3.py:20: Rule 't_error' requires an argument\n")) def test_lex_error4(self): self.assertRaises(SyntaxError,run_import,"lex_error4") result = sys.stderr.getvalue() self.assert_(check_expected(result, "lex_error4.py:20: Rule 't_error' has too many arguments\n")) def test_lex_ignore(self): self.assertRaises(SyntaxError,run_import,"lex_ignore") result = sys.stderr.getvalue() self.assert_(check_expected(result, "lex_ignore.py:20: Rule 't_ignore' must be defined as a string\n")) def test_lex_ignore2(self): run_import("lex_ignore2") result = sys.stderr.getvalue() self.assert_(check_expected(result, "t_ignore contains a literal backslash '\\'\n")) def test_lex_re1(self): self.assertRaises(SyntaxError,run_import,"lex_re1") result = sys.stderr.getvalue() if sys.hexversion < 0x3050000: msg = "Invalid regular expression for rule 't_NUMBER'. unbalanced parenthesis\n" else: msg = "Invalid regular expression for rule 't_NUMBER'. missing ), unterminated subpattern at position 0" self.assert_(check_expected(result, msg, contains=True)) def test_lex_re2(self): self.assertRaises(SyntaxError,run_import,"lex_re2") result = sys.stderr.getvalue() self.assert_(check_expected(result, "Regular expression for rule 't_PLUS' matches empty string\n")) def test_lex_re3(self): self.assertRaises(SyntaxError,run_import,"lex_re3") result = sys.stderr.getvalue() # self.assert_(check_expected(result, # "Invalid regular expression for rule 't_POUND'. unbalanced parenthesis\n" # "Make sure '#' in rule 't_POUND' is escaped with '\\#'\n")) if sys.hexversion < 0x3050000: msg = ("Invalid regular expression for rule 't_POUND'. unbalanced parenthesis\n" "Make sure '#' in rule 't_POUND' is escaped with '\\#'\n") else: msg = ("Invalid regular expression for rule 't_POUND'. missing ), unterminated subpattern at position 0\n" "ERROR: Make sure '#' in rule 't_POUND' is escaped with '\#'") self.assert_(check_expected(result, msg, contains=True), result) def test_lex_rule1(self): self.assertRaises(SyntaxError,run_import,"lex_rule1") result = sys.stderr.getvalue() self.assert_(check_expected(result, "t_NUMBER not defined as a function or string\n")) def test_lex_rule2(self): self.assertRaises(SyntaxError,run_import,"lex_rule2") result = sys.stderr.getvalue() self.assert_(check_expected(result, "lex_rule2.py:18: Rule 't_NUMBER' requires an argument\n")) def test_lex_rule3(self): self.assertRaises(SyntaxError,run_import,"lex_rule3") result = sys.stderr.getvalue() self.assert_(check_expected(result, "lex_rule3.py:18: Rule 't_NUMBER' has too many arguments\n")) def test_lex_state1(self): self.assertRaises(SyntaxError,run_import,"lex_state1") result = sys.stderr.getvalue() self.assert_(check_expected(result, "states must be defined as a tuple or list\n")) def test_lex_state2(self): self.assertRaises(SyntaxError,run_import,"lex_state2") result = sys.stderr.getvalue() self.assert_(check_expected(result, "Invalid state specifier 'comment'. Must be a tuple (statename,'exclusive|inclusive')\n" "Invalid state specifier 'example'. Must be a tuple (statename,'exclusive|inclusive')\n")) def test_lex_state3(self): self.assertRaises(SyntaxError,run_import,"lex_state3") result = sys.stderr.getvalue() self.assert_(check_expected(result, "State name 1 must be a string\n" "No rules defined for state 'example'\n")) def test_lex_state4(self): self.assertRaises(SyntaxError,run_import,"lex_state4") result = sys.stderr.getvalue() self.assert_(check_expected(result, "State type for state comment must be 'inclusive' or 'exclusive'\n")) def test_lex_state5(self): self.assertRaises(SyntaxError,run_import,"lex_state5") result = sys.stderr.getvalue() self.assert_(check_expected(result, "State 'comment' already defined\n")) def test_lex_state_noerror(self): run_import("lex_state_noerror") result = sys.stderr.getvalue() self.assert_(check_expected(result, "No error rule is defined for exclusive state 'comment'\n")) def test_lex_state_norule(self): self.assertRaises(SyntaxError,run_import,"lex_state_norule") result = sys.stderr.getvalue() self.assert_(check_expected(result, "No rules defined for state 'example'\n")) def test_lex_token1(self): self.assertRaises(SyntaxError,run_import,"lex_token1") result = sys.stderr.getvalue() self.assert_(check_expected(result, "No token list is defined\n" "Rule 't_NUMBER' defined for an unspecified token NUMBER\n" "Rule 't_PLUS' defined for an unspecified token PLUS\n" "Rule 't_MINUS' defined for an unspecified token MINUS\n" )) def test_lex_token2(self): self.assertRaises(SyntaxError,run_import,"lex_token2") result = sys.stderr.getvalue() self.assert_(check_expected(result, "tokens must be a list or tuple\n" "Rule 't_NUMBER' defined for an unspecified token NUMBER\n" "Rule 't_PLUS' defined for an unspecified token PLUS\n" "Rule 't_MINUS' defined for an unspecified token MINUS\n" )) def test_lex_token3(self): self.assertRaises(SyntaxError,run_import,"lex_token3") result = sys.stderr.getvalue() self.assert_(check_expected(result, "Rule 't_MINUS' defined for an unspecified token MINUS\n")) def test_lex_token4(self): self.assertRaises(SyntaxError,run_import,"lex_token4") result = sys.stderr.getvalue() self.assert_(check_expected(result, "Bad token name '-'\n")) def test_lex_token5(self): try: run_import("lex_token5") except ply.lex.LexError: e = sys.exc_info()[1] self.assert_(check_expected(str(e),"lex_token5.py:19: Rule 't_NUMBER' returned an unknown token type 'NUM'")) def test_lex_token_dup(self): run_import("lex_token_dup") result = sys.stderr.getvalue() self.assert_(check_expected(result, "Token 'MINUS' multiply defined\n")) def test_lex_literal1(self): self.assertRaises(SyntaxError,run_import,"lex_literal1") result = sys.stderr.getvalue() self.assert_(check_expected(result, "Invalid literal '**'. Must be a single character\n")) def test_lex_literal2(self): self.assertRaises(SyntaxError,run_import,"lex_literal2") result = sys.stderr.getvalue() self.assert_(check_expected(result, "Invalid literals specification. literals must be a sequence of characters\n")) import os import subprocess import shutil # Tests related to various build options associated with lexers class LexBuildOptionTests(unittest.TestCase): def setUp(self): sys.stderr = StringIO.StringIO() sys.stdout = StringIO.StringIO() def tearDown(self): sys.stderr = sys.__stderr__ sys.stdout = sys.__stdout__ try: shutil.rmtree("lexdir") except OSError: pass def test_lex_module(self): run_import("lex_module") result = sys.stdout.getvalue() self.assert_(check_expected(result, "(NUMBER,3,1,0)\n" "(PLUS,'+',1,1)\n" "(NUMBER,4,1,2)\n")) def test_lex_object(self): run_import("lex_object") result = sys.stdout.getvalue() self.assert_(check_expected(result, "(NUMBER,3,1,0)\n" "(PLUS,'+',1,1)\n" "(NUMBER,4,1,2)\n")) def test_lex_closure(self): run_import("lex_closure") result = sys.stdout.getvalue() self.assert_(check_expected(result, "(NUMBER,3,1,0)\n" "(PLUS,'+',1,1)\n" "(NUMBER,4,1,2)\n")) def test_lex_optimize(self): try: os.remove("lextab.py") except OSError: pass try: os.remove("lextab.pyc") except OSError: pass try: os.remove("lextab.pyo") except OSError: pass run_import("lex_optimize") result = sys.stdout.getvalue() self.assert_(check_expected(result, "(NUMBER,3,1,0)\n" "(PLUS,'+',1,1)\n" "(NUMBER,4,1,2)\n")) self.assert_(os.path.exists("lextab.py")) p = subprocess.Popen([sys.executable,'-O','lex_optimize.py'], stdout=subprocess.PIPE) result = p.stdout.read() self.assert_(check_expected(result, "(NUMBER,3,1,0)\n" "(PLUS,'+',1,1)\n" "(NUMBER,4,1,2)\n")) if test_pyo: self.assert_(pymodule_out_exists("lextab.pyo", 1)) pymodule_out_remove("lextab.pyo", 1) p = subprocess.Popen([sys.executable,'-OO','lex_optimize.py'], stdout=subprocess.PIPE) result = p.stdout.read() self.assert_(check_expected(result, "(NUMBER,3,1,0)\n" "(PLUS,'+',1,1)\n" "(NUMBER,4,1,2)\n")) if test_pyo: self.assert_(pymodule_out_exists("lextab.pyo", 2)) try: os.remove("lextab.py") except OSError: pass try: pymodule_out_remove("lextab.pyc") except OSError: pass try: pymodule_out_remove("lextab.pyo", 2) except OSError: pass def test_lex_optimize2(self): try: os.remove("opt2tab.py") except OSError: pass try: os.remove("opt2tab.pyc") except OSError: pass try: os.remove("opt2tab.pyo") except OSError: pass run_import("lex_optimize2") result = sys.stdout.getvalue() self.assert_(check_expected(result, "(NUMBER,3,1,0)\n" "(PLUS,'+',1,1)\n" "(NUMBER,4,1,2)\n")) self.assert_(os.path.exists("opt2tab.py")) p = subprocess.Popen([sys.executable,'-O','lex_optimize2.py'], stdout=subprocess.PIPE) result = p.stdout.read() self.assert_(check_expected(result, "(NUMBER,3,1,0)\n" "(PLUS,'+',1,1)\n" "(NUMBER,4,1,2)\n")) if test_pyo: self.assert_(pymodule_out_exists("opt2tab.pyo", 1)) pymodule_out_remove("opt2tab.pyo", 1) p = subprocess.Popen([sys.executable,'-OO','lex_optimize2.py'], stdout=subprocess.PIPE) result = p.stdout.read() self.assert_(check_expected(result, "(NUMBER,3,1,0)\n" "(PLUS,'+',1,1)\n" "(NUMBER,4,1,2)\n")) if test_pyo: self.assert_(pymodule_out_exists("opt2tab.pyo", 2)) try: os.remove("opt2tab.py") except OSError: pass try: pymodule_out_remove("opt2tab.pyc") except OSError: pass try: pymodule_out_remove("opt2tab.pyo", 2) except OSError: pass def test_lex_optimize3(self): try: shutil.rmtree("lexdir") except OSError: pass os.mkdir("lexdir") os.mkdir("lexdir/sub") with open("lexdir/__init__.py","w") as f: f.write("") with open("lexdir/sub/__init__.py","w") as f: f.write("") run_import("lex_optimize3") result = sys.stdout.getvalue() self.assert_(check_expected(result, "(NUMBER,3,1,0)\n" "(PLUS,'+',1,1)\n" "(NUMBER,4,1,2)\n")) self.assert_(os.path.exists("lexdir/sub/calctab.py")) p = subprocess.Popen([sys.executable,'-O','lex_optimize3.py'], stdout=subprocess.PIPE) result = p.stdout.read() self.assert_(check_expected(result, "(NUMBER,3,1,0)\n" "(PLUS,'+',1,1)\n" "(NUMBER,4,1,2)\n")) if test_pyo: self.assert_(pymodule_out_exists("lexdir/sub/calctab.pyo", 1)) pymodule_out_remove("lexdir/sub/calctab.pyo", 1) p = subprocess.Popen([sys.executable,'-OO','lex_optimize3.py'], stdout=subprocess.PIPE) result = p.stdout.read() self.assert_(check_expected(result, "(NUMBER,3,1,0)\n" "(PLUS,'+',1,1)\n" "(NUMBER,4,1,2)\n")) if test_pyo: self.assert_(pymodule_out_exists("lexdir/sub/calctab.pyo", 2)) try: shutil.rmtree("lexdir") except OSError: pass def test_lex_optimize4(self): # Regression test to make sure that reflags works correctly # on Python 3. for extension in ['py', 'pyc']: try: os.remove("opt4tab.{0}".format(extension)) except OSError: pass run_import("lex_optimize4") run_import("lex_optimize4") for extension in ['py', 'pyc']: try: os.remove("opt4tab.{0}".format(extension)) except OSError: pass def test_lex_opt_alias(self): try: os.remove("aliastab.py") except OSError: pass try: os.remove("aliastab.pyc") except OSError: pass try: os.remove("aliastab.pyo") except OSError: pass run_import("lex_opt_alias") result = sys.stdout.getvalue() self.assert_(check_expected(result, "(NUMBER,3,1,0)\n" "(+,'+',1,1)\n" "(NUMBER,4,1,2)\n")) self.assert_(os.path.exists("aliastab.py")) p = subprocess.Popen([sys.executable,'-O','lex_opt_alias.py'], stdout=subprocess.PIPE) result = p.stdout.read() self.assert_(check_expected(result, "(NUMBER,3,1,0)\n" "(+,'+',1,1)\n" "(NUMBER,4,1,2)\n")) if test_pyo: self.assert_(pymodule_out_exists("aliastab.pyo", 1)) pymodule_out_remove("aliastab.pyo", 1) p = subprocess.Popen([sys.executable,'-OO','lex_opt_alias.py'], stdout=subprocess.PIPE) result = p.stdout.read() self.assert_(check_expected(result, "(NUMBER,3,1,0)\n" "(+,'+',1,1)\n" "(NUMBER,4,1,2)\n")) if test_pyo: self.assert_(pymodule_out_exists("aliastab.pyo", 2)) try: os.remove("aliastab.py") except OSError: pass try: pymodule_out_remove("aliastab.pyc") except OSError: pass try: pymodule_out_remove("aliastab.pyo", 2) except OSError: pass def test_lex_many_tokens(self): try: os.remove("manytab.py") except OSError: pass try: os.remove("manytab.pyc") except OSError: pass try: os.remove("manytab.pyo") except OSError: pass run_import("lex_many_tokens") result = sys.stdout.getvalue() self.assert_(check_expected(result, "(TOK34,'TOK34:',1,0)\n" "(TOK143,'TOK143:',1,7)\n" "(TOK269,'TOK269:',1,15)\n" "(TOK372,'TOK372:',1,23)\n" "(TOK452,'TOK452:',1,31)\n" "(TOK561,'TOK561:',1,39)\n" "(TOK999,'TOK999:',1,47)\n" )) self.assert_(os.path.exists("manytab.py")) if implementation() == 'CPython': p = subprocess.Popen([sys.executable,'-O','lex_many_tokens.py'], stdout=subprocess.PIPE) result = p.stdout.read() self.assert_(check_expected(result, "(TOK34,'TOK34:',1,0)\n" "(TOK143,'TOK143:',1,7)\n" "(TOK269,'TOK269:',1,15)\n" "(TOK372,'TOK372:',1,23)\n" "(TOK452,'TOK452:',1,31)\n" "(TOK561,'TOK561:',1,39)\n" "(TOK999,'TOK999:',1,47)\n" )) self.assert_(pymodule_out_exists("manytab.pyo", 1)) pymodule_out_remove("manytab.pyo", 1) try: os.remove("manytab.py") except OSError: pass try: os.remove("manytab.pyc") except OSError: pass try: os.remove("manytab.pyo") except OSError: pass # Tests related to run-time behavior of lexers class LexRunTests(unittest.TestCase): def setUp(self): sys.stderr = StringIO.StringIO() sys.stdout = StringIO.StringIO() def tearDown(self): sys.stderr = sys.__stderr__ sys.stdout = sys.__stdout__ def test_lex_hedit(self): run_import("lex_hedit") result = sys.stdout.getvalue() self.assert_(check_expected(result, "(H_EDIT_DESCRIPTOR,'abc',1,0)\n" "(H_EDIT_DESCRIPTOR,'abcdefghij',1,6)\n" "(H_EDIT_DESCRIPTOR,'xy',1,20)\n")) def test_lex_state_try(self): run_import("lex_state_try") result = sys.stdout.getvalue() self.assert_(check_expected(result, "(NUMBER,'3',1,0)\n" "(PLUS,'+',1,2)\n" "(NUMBER,'4',1,4)\n" "Entering comment state\n" "comment body LexToken(body_part,'This is a comment */',1,9)\n" "(PLUS,'+',1,30)\n" "(NUMBER,'10',1,32)\n" )) unittest.main() ================================================ FILE: xonsh/ply/test/testyacc.py ================================================ # testyacc.py import unittest try: import StringIO except ImportError: import io as StringIO import sys import os import warnings import re import platform sys.path.insert(0,"..") sys.tracebacklimit = 0 import ply.yacc def make_pymodule_path(filename): path = os.path.dirname(filename) file = os.path.basename(filename) mod, ext = os.path.splitext(file) if sys.hexversion >= 0x3040000: import importlib.util fullpath = importlib.util.cache_from_source(filename, ext=='.pyc') elif sys.hexversion >= 0x3020000: import imp modname = mod+"."+imp.get_tag()+ext fullpath = os.path.join(path,'__pycache__',modname) else: fullpath = filename return fullpath def pymodule_out_exists(filename): return os.path.exists(make_pymodule_path(filename)) def pymodule_out_remove(filename): os.remove(make_pymodule_path(filename)) def implementation(): if platform.system().startswith("Java"): return "Jython" elif hasattr(sys, "pypy_version_info"): return "PyPy" else: return "CPython" # Check the output to see if it contains all of a set of expected output lines. # This alternate implementation looks weird, but is needed to properly handle # some variations in error message order that occurs due to dict hash table # randomization that was introduced in Python 3.3 def check_expected(result, expected): # Normalize 'state n' text to account for randomization effects in Python 3.3 expected = re.sub(r' state \d+', 'state ', expected) result = re.sub(r' state \d+', 'state ', result) resultlines = set() for line in result.splitlines(): if line.startswith("WARNING: "): line = line[9:] elif line.startswith("ERROR: "): line = line[7:] resultlines.add(line) # Selectively remove expected lines from the output for eline in expected.splitlines(): resultlines = set(line for line in resultlines if not line.endswith(eline)) # Return True if no result lines remain return not bool(resultlines) def run_import(module): code = "import "+module exec(code) del sys.modules[module] # Tests related to errors and warnings when building parsers class YaccErrorWarningTests(unittest.TestCase): def setUp(self): sys.stderr = StringIO.StringIO() sys.stdout = StringIO.StringIO() try: os.remove("parsetab.py") pymodule_out_remove("parsetab.pyc") except OSError: pass if sys.hexversion >= 0x3020000: warnings.filterwarnings('ignore', category=ResourceWarning) warnings.filterwarnings('ignore', category=DeprecationWarning) def tearDown(self): sys.stderr = sys.__stderr__ sys.stdout = sys.__stdout__ def test_yacc_badargs(self): self.assertRaises(ply.yacc.YaccError,run_import,"yacc_badargs") result = sys.stderr.getvalue() self.assert_(check_expected(result, "yacc_badargs.py:23: Rule 'p_statement_assign' has too many arguments\n" "yacc_badargs.py:27: Rule 'p_statement_expr' requires an argument\n" )) def test_yacc_badid(self): self.assertRaises(ply.yacc.YaccError,run_import,"yacc_badid") result = sys.stderr.getvalue() self.assert_(check_expected(result, "yacc_badid.py:32: Illegal name 'bad&rule' in rule 'statement'\n" "yacc_badid.py:36: Illegal rule name 'bad&rule'\n" )) def test_yacc_badprec(self): try: run_import("yacc_badprec") except ply.yacc.YaccError: result = sys.stderr.getvalue() self.assert_(check_expected(result, "precedence must be a list or tuple\n" )) def test_yacc_badprec2(self): self.assertRaises(ply.yacc.YaccError,run_import,"yacc_badprec2") result = sys.stderr.getvalue() self.assert_(check_expected(result, "Bad precedence table\n" )) def test_yacc_badprec3(self): run_import("yacc_badprec3") result = sys.stderr.getvalue() self.assert_(check_expected(result, "Precedence already specified for terminal 'MINUS'\n" "Generating LALR tables\n" )) def test_yacc_badrule(self): self.assertRaises(ply.yacc.YaccError,run_import,"yacc_badrule") result = sys.stderr.getvalue() self.assert_(check_expected(result, "yacc_badrule.py:24: Syntax error. Expected ':'\n" "yacc_badrule.py:28: Syntax error in rule 'statement'\n" "yacc_badrule.py:33: Syntax error. Expected ':'\n" "yacc_badrule.py:42: Syntax error. Expected ':'\n" )) def test_yacc_badtok(self): try: run_import("yacc_badtok") except ply.yacc.YaccError: result = sys.stderr.getvalue() self.assert_(check_expected(result, "tokens must be a list or tuple\n")) def test_yacc_dup(self): run_import("yacc_dup") result = sys.stderr.getvalue() self.assert_(check_expected(result, "yacc_dup.py:27: Function p_statement redefined. Previously defined on line 23\n" "Token 'EQUALS' defined, but not used\n" "There is 1 unused token\n" "Generating LALR tables\n" )) def test_yacc_error1(self): try: run_import("yacc_error1") except ply.yacc.YaccError: result = sys.stderr.getvalue() self.assert_(check_expected(result, "yacc_error1.py:61: p_error() requires 1 argument\n")) def test_yacc_error2(self): try: run_import("yacc_error2") except ply.yacc.YaccError: result = sys.stderr.getvalue() self.assert_(check_expected(result, "yacc_error2.py:61: p_error() requires 1 argument\n")) def test_yacc_error3(self): try: run_import("yacc_error3") except ply.yacc.YaccError: e = sys.exc_info()[1] result = sys.stderr.getvalue() self.assert_(check_expected(result, "'p_error' defined, but is not a function or method\n")) def test_yacc_error4(self): self.assertRaises(ply.yacc.YaccError,run_import,"yacc_error4") result = sys.stderr.getvalue() self.assert_(check_expected(result, "yacc_error4.py:62: Illegal rule name 'error'. Already defined as a token\n" )) def test_yacc_error5(self): run_import("yacc_error5") result = sys.stdout.getvalue() self.assert_(check_expected(result, "Group at 3:10 to 3:12\n" "Undefined name 'a'\n" "Syntax error at 'b'\n" "Syntax error at 4:18 to 4:22\n" "Assignment Error at 2:5 to 5:27\n" "13\n" )) def test_yacc_error6(self): run_import("yacc_error6") result = sys.stdout.getvalue() self.assert_(check_expected(result, "a=7\n" "Line 3: Syntax error at '*'\n" "c=21\n" )) def test_yacc_error7(self): run_import("yacc_error7") result = sys.stdout.getvalue() self.assert_(check_expected(result, "a=7\n" "Line 3: Syntax error at '*'\n" "c=21\n" )) def test_yacc_inf(self): self.assertRaises(ply.yacc.YaccError,run_import,"yacc_inf") result = sys.stderr.getvalue() self.assert_(check_expected(result, "Token 'NUMBER' defined, but not used\n" "There is 1 unused token\n" "Infinite recursion detected for symbol 'statement'\n" "Infinite recursion detected for symbol 'expression'\n" )) def test_yacc_literal(self): self.assertRaises(ply.yacc.YaccError,run_import,"yacc_literal") result = sys.stderr.getvalue() self.assert_(check_expected(result, "yacc_literal.py:36: Literal token '**' in rule 'expression' may only be a single character\n" )) def test_yacc_misplaced(self): self.assertRaises(ply.yacc.YaccError,run_import,"yacc_misplaced") result = sys.stderr.getvalue() self.assert_(check_expected(result, "yacc_misplaced.py:32: Misplaced '|'\n" )) def test_yacc_missing1(self): self.assertRaises(ply.yacc.YaccError,run_import,"yacc_missing1") result = sys.stderr.getvalue() self.assert_(check_expected(result, "yacc_missing1.py:24: Symbol 'location' used, but not defined as a token or a rule\n" )) def test_yacc_nested(self): run_import("yacc_nested") result = sys.stdout.getvalue() self.assert_(check_expected(result, "A\n" "A\n" "A\n", )) def test_yacc_nodoc(self): run_import("yacc_nodoc") result = sys.stderr.getvalue() self.assert_(check_expected(result, "yacc_nodoc.py:27: No documentation string specified in function 'p_statement_expr' (ignored)\n" "Generating LALR tables\n" )) def test_yacc_noerror(self): run_import("yacc_noerror") result = sys.stderr.getvalue() self.assert_(check_expected(result, "no p_error() function is defined\n" "Generating LALR tables\n" )) def test_yacc_nop(self): run_import("yacc_nop") result = sys.stderr.getvalue() self.assert_(check_expected(result, "yacc_nop.py:27: Possible grammar rule 'statement_expr' defined without p_ prefix\n" "Generating LALR tables\n" )) def test_yacc_notfunc(self): run_import("yacc_notfunc") result = sys.stderr.getvalue() self.assert_(check_expected(result, "'p_statement_assign' not defined as a function\n" "Token 'EQUALS' defined, but not used\n" "There is 1 unused token\n" "Generating LALR tables\n" )) def test_yacc_notok(self): try: run_import("yacc_notok") except ply.yacc.YaccError: result = sys.stderr.getvalue() self.assert_(check_expected(result, "No token list is defined\n")) def test_yacc_rr(self): run_import("yacc_rr") result = sys.stderr.getvalue() self.assert_(check_expected(result, "Generating LALR tables\n" "1 reduce/reduce conflict\n" "reduce/reduce conflict in state 15 resolved using rule (statement -> NAME EQUALS NUMBER)\n" "rejected rule (expression -> NUMBER) in state 15\n" )) def test_yacc_rr_unused(self): run_import("yacc_rr_unused") result = sys.stderr.getvalue() self.assert_(check_expected(result, "no p_error() function is defined\n" "Generating LALR tables\n" "3 reduce/reduce conflicts\n" "reduce/reduce conflict in state 1 resolved using rule (rule3 -> A)\n" "rejected rule (rule4 -> A) in state 1\n" "reduce/reduce conflict in state 1 resolved using rule (rule3 -> A)\n" "rejected rule (rule5 -> A) in state 1\n" "reduce/reduce conflict in state 1 resolved using rule (rule4 -> A)\n" "rejected rule (rule5 -> A) in state 1\n" "Rule (rule5 -> A) is never reduced\n" )) def test_yacc_simple(self): run_import("yacc_simple") result = sys.stderr.getvalue() self.assert_(check_expected(result, "Generating LALR tables\n" )) def test_yacc_sr(self): run_import("yacc_sr") result = sys.stderr.getvalue() self.assert_(check_expected(result, "Generating LALR tables\n" "20 shift/reduce conflicts\n" )) def test_yacc_term1(self): self.assertRaises(ply.yacc.YaccError,run_import,"yacc_term1") result = sys.stderr.getvalue() self.assert_(check_expected(result, "yacc_term1.py:24: Illegal rule name 'NUMBER'. Already defined as a token\n" )) def test_yacc_unicode_literals(self): run_import("yacc_unicode_literals") result = sys.stderr.getvalue() self.assert_(check_expected(result, "Generating LALR tables\n" )) def test_yacc_unused(self): self.assertRaises(ply.yacc.YaccError,run_import,"yacc_unused") result = sys.stderr.getvalue() self.assert_(check_expected(result, "yacc_unused.py:62: Symbol 'COMMA' used, but not defined as a token or a rule\n" "Symbol 'COMMA' is unreachable\n" "Symbol 'exprlist' is unreachable\n" )) def test_yacc_unused_rule(self): run_import("yacc_unused_rule") result = sys.stderr.getvalue() self.assert_(check_expected(result, "yacc_unused_rule.py:62: Rule 'integer' defined, but not used\n" "There is 1 unused rule\n" "Symbol 'integer' is unreachable\n" "Generating LALR tables\n" )) def test_yacc_uprec(self): self.assertRaises(ply.yacc.YaccError,run_import,"yacc_uprec") result = sys.stderr.getvalue() self.assert_(check_expected(result, "yacc_uprec.py:37: Nothing known about the precedence of 'UMINUS'\n" )) def test_yacc_uprec2(self): self.assertRaises(ply.yacc.YaccError,run_import,"yacc_uprec2") result = sys.stderr.getvalue() self.assert_(check_expected(result, "yacc_uprec2.py:37: Syntax error. Nothing follows %prec\n" )) def test_yacc_prec1(self): self.assertRaises(ply.yacc.YaccError,run_import,"yacc_prec1") result = sys.stderr.getvalue() self.assert_(check_expected(result, "Precedence rule 'left' defined for unknown symbol '+'\n" "Precedence rule 'left' defined for unknown symbol '*'\n" "Precedence rule 'left' defined for unknown symbol '-'\n" "Precedence rule 'left' defined for unknown symbol '/'\n" )) def test_pkg_test1(self): from pkg_test1 import parser self.assertTrue(os.path.exists('pkg_test1/parsing/parsetab.py')) self.assertTrue(os.path.exists('pkg_test1/parsing/lextab.py')) self.assertTrue(os.path.exists('pkg_test1/parsing/parser.out')) r = parser.parse('3+4+5') self.assertEqual(r, 12) def test_pkg_test2(self): from pkg_test2 import parser self.assertTrue(os.path.exists('pkg_test2/parsing/calcparsetab.py')) self.assertTrue(os.path.exists('pkg_test2/parsing/calclextab.py')) self.assertTrue(os.path.exists('pkg_test2/parsing/parser.out')) r = parser.parse('3+4+5') self.assertEqual(r, 12) def test_pkg_test3(self): from pkg_test3 import parser self.assertTrue(os.path.exists('pkg_test3/generated/parsetab.py')) self.assertTrue(os.path.exists('pkg_test3/generated/lextab.py')) self.assertTrue(os.path.exists('pkg_test3/generated/parser.out')) r = parser.parse('3+4+5') self.assertEqual(r, 12) def test_pkg_test4(self): from pkg_test4 import parser self.assertFalse(os.path.exists('pkg_test4/parsing/parsetab.py')) self.assertFalse(os.path.exists('pkg_test4/parsing/lextab.py')) self.assertFalse(os.path.exists('pkg_test4/parsing/parser.out')) r = parser.parse('3+4+5') self.assertEqual(r, 12) def test_pkg_test5(self): from pkg_test5 import parser self.assertTrue(os.path.exists('pkg_test5/parsing/parsetab.py')) self.assertTrue(os.path.exists('pkg_test5/parsing/lextab.py')) self.assertTrue(os.path.exists('pkg_test5/parsing/parser.out')) r = parser.parse('3+4+5') self.assertEqual(r, 12) def test_pkg_test6(self): from pkg_test6 import parser self.assertTrue(os.path.exists('pkg_test6/parsing/parsetab.py')) self.assertTrue(os.path.exists('pkg_test6/parsing/lextab.py')) self.assertTrue(os.path.exists('pkg_test6/parsing/parser.out')) r = parser.parse('3+4+5') self.assertEqual(r, 12) unittest.main() ================================================ FILE: xonsh/ply/test/yacc_badargs.py ================================================ # ----------------------------------------------------------------------------- # yacc_badargs.py # # Rules with wrong # args # ----------------------------------------------------------------------------- import sys sys.tracebacklimit = 0 sys.path.insert(0,"..") import ply.yacc as yacc from calclex import tokens # Parsing rules precedence = ( ('left','PLUS','MINUS'), ('left','TIMES','DIVIDE'), ('right','UMINUS'), ) # dictionary of names names = { } def p_statement_assign(t,s): 'statement : NAME EQUALS expression' names[t[1]] = t[3] def p_statement_expr(): 'statement : expression' print(t[1]) def p_expression_binop(t): '''expression : expression PLUS expression | expression MINUS expression | expression TIMES expression | expression DIVIDE expression''' if t[2] == '+' : t[0] = t[1] + t[3] elif t[2] == '-': t[0] = t[1] - t[3] elif t[2] == '*': t[0] = t[1] * t[3] elif t[2] == '/': t[0] = t[1] / t[3] def p_expression_uminus(t): 'expression : MINUS expression %prec UMINUS' t[0] = -t[2] def p_expression_group(t): 'expression : LPAREN expression RPAREN' t[0] = t[2] def p_expression_number(t): 'expression : NUMBER' t[0] = t[1] def p_expression_name(t): 'expression : NAME' try: t[0] = names[t[1]] except LookupError: print("Undefined name '%s'" % t[1]) t[0] = 0 def p_error(t): print("Syntax error at '%s'" % t.value) yacc.yacc() ================================================ FILE: xonsh/ply/test/yacc_badid.py ================================================ # ----------------------------------------------------------------------------- # yacc_badid.py # # Attempt to define a rule with a bad-identifier name # ----------------------------------------------------------------------------- import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.yacc as yacc from calclex import tokens # Parsing rules precedence = ( ('left','PLUS','MINUS'), ('left','TIMES','DIVIDE'), ('right','UMINUS'), ) # dictionary of names names = { } def p_statement_assign(t): 'statement : NAME EQUALS expression' names[t[1]] = t[3] def p_statement_expr(t): 'statement : expression' print(t[1]) def p_statement_expr2(t): 'statement : bad&rule' pass def p_badrule(t): 'bad&rule : expression' pass def p_expression_binop(t): '''expression : expression PLUS expression | expression MINUS expression | expression TIMES expression | expression DIVIDE expression''' if t[2] == '+' : t[0] = t[1] + t[3] elif t[2] == '-': t[0] = t[1] - t[3] elif t[2] == '*': t[0] = t[1] * t[3] elif t[2] == '/': t[0] = t[1] / t[3] def p_expression_uminus(t): 'expression : MINUS expression %prec UMINUS' t[0] = -t[2] def p_expression_group(t): 'expression : LPAREN expression RPAREN' t[0] = t[2] def p_expression_number(t): 'expression : NUMBER' t[0] = t[1] def p_expression_name(t): 'expression : NAME' try: t[0] = names[t[1]] except LookupError: print("Undefined name '%s'" % t[1]) t[0] = 0 def p_error(t): pass yacc.yacc() ================================================ FILE: xonsh/ply/test/yacc_badprec.py ================================================ # ----------------------------------------------------------------------------- # yacc_badprec.py # # Bad precedence specifier # ----------------------------------------------------------------------------- import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.yacc as yacc from calclex import tokens # Parsing rules precedence = "blah" # dictionary of names names = { } def p_statement_assign(t): 'statement : NAME EQUALS expression' names[t[1]] = t[3] def p_statement_expr(t): 'statement : expression' print(t[1]) def p_expression_binop(t): '''expression : expression PLUS expression | expression MINUS expression | expression TIMES expression | expression DIVIDE expression''' if t[2] == '+' : t[0] = t[1] + t[3] elif t[2] == '-': t[0] = t[1] - t[3] elif t[2] == '*': t[0] = t[1] * t[3] elif t[2] == '/': t[0] = t[1] / t[3] def p_expression_uminus(t): 'expression : MINUS expression %prec UMINUS' t[0] = -t[2] def p_expression_group(t): 'expression : LPAREN expression RPAREN' t[0] = t[2] def p_expression_number(t): 'expression : NUMBER' t[0] = t[1] def p_expression_name(t): 'expression : NAME' try: t[0] = names[t[1]] except LookupError: print("Undefined name '%s'" % t[1]) t[0] = 0 def p_error(t): print("Syntax error at '%s'" % t.value) yacc.yacc() ================================================ FILE: xonsh/ply/test/yacc_badprec2.py ================================================ # ----------------------------------------------------------------------------- # yacc_badprec2.py # # Bad precedence # ----------------------------------------------------------------------------- import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.yacc as yacc from calclex import tokens # Parsing rules precedence = ( 42, ('left','TIMES','DIVIDE'), ('right','UMINUS'), ) # dictionary of names names = { } def p_statement_assign(t): 'statement : NAME EQUALS expression' names[t[1]] = t[3] def p_statement_expr(t): 'statement : expression' print(t[1]) def p_expression_binop(t): '''expression : expression PLUS expression | expression MINUS expression | expression TIMES expression | expression DIVIDE expression''' if t[2] == '+' : t[0] = t[1] + t[3] elif t[2] == '-': t[0] = t[1] - t[3] elif t[2] == '*': t[0] = t[1] * t[3] elif t[2] == '/': t[0] = t[1] / t[3] def p_expression_uminus(t): 'expression : MINUS expression %prec UMINUS' t[0] = -t[2] def p_expression_group(t): 'expression : LPAREN expression RPAREN' t[0] = t[2] def p_expression_number(t): 'expression : NUMBER' t[0] = t[1] def p_expression_name(t): 'expression : NAME' try: t[0] = names[t[1]] except LookupError: print("Undefined name '%s'" % t[1]) t[0] = 0 def p_error(t): print("Syntax error at '%s'" % t.value) yacc.yacc() ================================================ FILE: xonsh/ply/test/yacc_badprec3.py ================================================ # ----------------------------------------------------------------------------- # yacc_badprec3.py # # Bad precedence # ----------------------------------------------------------------------------- import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.yacc as yacc from calclex import tokens # Parsing rules precedence = ( ('left','PLUS','MINUS'), ('left','TIMES','DIVIDE','MINUS'), ('right','UMINUS'), ) # dictionary of names names = { } def p_statement_assign(t): 'statement : NAME EQUALS expression' names[t[1]] = t[3] def p_statement_expr(t): 'statement : expression' print(t[1]) def p_expression_binop(t): '''expression : expression PLUS expression | expression MINUS expression | expression TIMES expression | expression DIVIDE expression''' if t[2] == '+' : t[0] = t[1] + t[3] elif t[2] == '-': t[0] = t[1] - t[3] elif t[2] == '*': t[0] = t[1] * t[3] elif t[3] == '/': t[0] = t[1] / t[3] def p_expression_uminus(t): 'expression : MINUS expression %prec UMINUS' t[0] = -t[2] def p_expression_group(t): 'expression : LPAREN expression RPAREN' t[0] = t[2] def p_expression_number(t): 'expression : NUMBER' t[0] = t[1] def p_expression_name(t): 'expression : NAME' try: t[0] = names[t[1]] except LookupError: print("Undefined name '%s'" % t[1]) t[0] = 0 def p_error(t): print("Syntax error at '%s'" % t.value) yacc.yacc() ================================================ FILE: xonsh/ply/test/yacc_badrule.py ================================================ # ----------------------------------------------------------------------------- # yacc_badrule.py # # Syntax problems in the rule strings # ----------------------------------------------------------------------------- import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.yacc as yacc from calclex import tokens # Parsing rules precedence = ( ('left','PLUS','MINUS'), ('left','TIMES','DIVIDE'), ('right','UMINUS'), ) # dictionary of names names = { } def p_statement_assign(t): 'statement NAME EQUALS expression' names[t[1]] = t[3] def p_statement_expr(t): 'statement' print(t[1]) def p_expression_binop(t): '''expression : expression PLUS expression expression MINUS expression | expression TIMES expression | expression DIVIDE expression''' if t[2] == '+' : t[0] = t[1] + t[3] elif t[2] == '-': t[0] = t[1] - t[3] elif t[2] == '*': t[0] = t[1] * t[3] elif t[2] == '/': t[0] = t[1] / t[3] def p_expression_uminus(t): 'expression: MINUS expression %prec UMINUS' t[0] = -t[2] def p_expression_group(t): 'expression : LPAREN expression RPAREN' t[0] = t[2] def p_expression_number(t): 'expression : NUMBER' t[0] = t[1] def p_expression_name(t): 'expression : NAME' try: t[0] = names[t[1]] except LookupError: print("Undefined name '%s'" % t[1]) t[0] = 0 def p_error(t): print("Syntax error at '%s'" % t.value) yacc.yacc() ================================================ FILE: xonsh/ply/test/yacc_badtok.py ================================================ # ----------------------------------------------------------------------------- # yacc_badtok.py # # A grammar, but tokens is a bad datatype # ----------------------------------------------------------------------------- import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.yacc as yacc tokens = "Hello" # Parsing rules precedence = ( ('left','PLUS','MINUS'), ('left','TIMES','DIVIDE'), ('right','UMINUS'), ) # dictionary of names names = { } def p_statement_assign(t): 'statement : NAME EQUALS expression' names[t[1]] = t[3] def p_statement_expr(t): 'statement : expression' print(t[1]) def p_expression_binop(t): '''expression : expression PLUS expression | expression MINUS expression | expression TIMES expression | expression DIVIDE expression''' if t[2] == '+' : t[0] = t[1] + t[3] elif t[2] == '-': t[0] = t[1] - t[3] elif t[2] == '*': t[0] = t[1] * t[3] elif t[2] == '/': t[0] = t[1] / t[3] def p_expression_uminus(t): 'expression : MINUS expression %prec UMINUS' t[0] = -t[2] def p_expression_group(t): 'expression : LPAREN expression RPAREN' t[0] = t[2] def p_expression_number(t): 'expression : NUMBER' t[0] = t[1] def p_expression_name(t): 'expression : NAME' try: t[0] = names[t[1]] except LookupError: print("Undefined name '%s'" % t[1]) t[0] = 0 def p_error(t): print("Syntax error at '%s'" % t.value) yacc.yacc() ================================================ FILE: xonsh/ply/test/yacc_dup.py ================================================ # ----------------------------------------------------------------------------- # yacc_dup.py # # Duplicated rule name # ----------------------------------------------------------------------------- import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.yacc as yacc from calclex import tokens # Parsing rules precedence = ( ('left','PLUS','MINUS'), ('left','TIMES','DIVIDE'), ('right','UMINUS'), ) # dictionary of names names = { } def p_statement(t): 'statement : NAME EQUALS expression' names[t[1]] = t[3] def p_statement(t): 'statement : expression' print(t[1]) def p_expression_binop(t): '''expression : expression PLUS expression | expression MINUS expression | expression TIMES expression | expression DIVIDE expression''' if t[2] == '+' : t[0] = t[1] + t[3] elif t[2] == '-': t[0] = t[1] - t[3] elif t[2] == '*': t[0] = t[1] * t[3] elif t[2] == '/': t[0] = t[1] / t[3] def p_expression_uminus(t): 'expression : MINUS expression %prec UMINUS' t[0] = -t[2] def p_expression_group(t): 'expression : LPAREN expression RPAREN' t[0] = t[2] def p_expression_number(t): 'expression : NUMBER' t[0] = t[1] def p_expression_name(t): 'expression : NAME' try: t[0] = names[t[1]] except LookupError: print("Undefined name '%s'" % t[1]) t[0] = 0 def p_error(t): print("Syntax error at '%s'" % t.value) yacc.yacc() ================================================ FILE: xonsh/ply/test/yacc_error1.py ================================================ # ----------------------------------------------------------------------------- # yacc_error1.py # # Bad p_error() function # ----------------------------------------------------------------------------- import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.yacc as yacc from calclex import tokens # Parsing rules precedence = ( ('left','PLUS','MINUS'), ('left','TIMES','DIVIDE'), ('right','UMINUS'), ) # dictionary of names names = { } def p_statement_assign(t): 'statement : NAME EQUALS expression' names[t[1]] = t[3] def p_statement_expr(t): 'statement : expression' print(t[1]) def p_expression_binop(t): '''expression : expression PLUS expression | expression MINUS expression | expression TIMES expression | expression DIVIDE expression''' if t[2] == '+' : t[0] = t[1] + t[3] elif t[2] == '-': t[0] = t[1] - t[3] elif t[2] == '*': t[0] = t[1] * t[3] elif t[2] == '/': t[0] = t[1] / t[3] def p_expression_uminus(t): 'expression : MINUS expression %prec UMINUS' t[0] = -t[2] def p_expression_group(t): 'expression : LPAREN expression RPAREN' t[0] = t[2] def p_expression_number(t): 'expression : NUMBER' t[0] = t[1] def p_expression_name(t): 'expression : NAME' try: t[0] = names[t[1]] except LookupError: print("Undefined name '%s'" % t[1]) t[0] = 0 def p_error(t,s): print("Syntax error at '%s'" % t.value) yacc.yacc() ================================================ FILE: xonsh/ply/test/yacc_error2.py ================================================ # ----------------------------------------------------------------------------- # yacc_error2.py # # Bad p_error() function # ----------------------------------------------------------------------------- import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.yacc as yacc from calclex import tokens # Parsing rules precedence = ( ('left','PLUS','MINUS'), ('left','TIMES','DIVIDE'), ('right','UMINUS'), ) # dictionary of names names = { } def p_statement_assign(t): 'statement : NAME EQUALS expression' names[t[1]] = t[3] def p_statement_expr(t): 'statement : expression' print(t[1]) def p_expression_binop(t): '''expression : expression PLUS expression | expression MINUS expression | expression TIMES expression | expression DIVIDE expression''' if t[2] == '+' : t[0] = t[1] + t[3] elif t[2] == '-': t[0] = t[1] - t[3] elif t[2] == '*': t[0] = t[1] * t[3] elif t[2] == '/': t[0] = t[1] / t[3] def p_expression_uminus(t): 'expression : MINUS expression %prec UMINUS' t[0] = -t[2] def p_expression_group(t): 'expression : LPAREN expression RPAREN' t[0] = t[2] def p_expression_number(t): 'expression : NUMBER' t[0] = t[1] def p_expression_name(t): 'expression : NAME' try: t[0] = names[t[1]] except LookupError: print("Undefined name '%s'" % t[1]) t[0] = 0 def p_error(): print("Syntax error at '%s'" % t.value) yacc.yacc() ================================================ FILE: xonsh/ply/test/yacc_error3.py ================================================ # ----------------------------------------------------------------------------- # yacc_error3.py # # Bad p_error() function # ----------------------------------------------------------------------------- import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.yacc as yacc from calclex import tokens # Parsing rules precedence = ( ('left','PLUS','MINUS'), ('left','TIMES','DIVIDE'), ('right','UMINUS'), ) # dictionary of names names = { } def p_statement_assign(t): 'statement : NAME EQUALS expression' names[t[1]] = t[3] def p_statement_expr(t): 'statement : expression' print(t[1]) def p_expression_binop(t): '''expression : expression PLUS expression | expression MINUS expression | expression TIMES expression | expression DIVIDE expression''' if t[2] == '+' : t[0] = t[1] + t[3] elif t[2] == '-': t[0] = t[1] - t[3] elif t[2] == '*': t[0] = t[1] * t[3] elif t[2] == '/': t[0] = t[1] / t[3] def p_expression_uminus(t): 'expression : MINUS expression %prec UMINUS' t[0] = -t[2] def p_expression_group(t): 'expression : LPAREN expression RPAREN' t[0] = t[2] def p_expression_number(t): 'expression : NUMBER' t[0] = t[1] def p_expression_name(t): 'expression : NAME' try: t[0] = names[t[1]] except LookupError: print("Undefined name '%s'" % t[1]) t[0] = 0 p_error = "blah" yacc.yacc() ================================================ FILE: xonsh/ply/test/yacc_error4.py ================================================ # ----------------------------------------------------------------------------- # yacc_error4.py # # Attempt to define a rule named 'error' # ----------------------------------------------------------------------------- import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.yacc as yacc from calclex import tokens # Parsing rules precedence = ( ('left','PLUS','MINUS'), ('left','TIMES','DIVIDE'), ('right','UMINUS'), ) # dictionary of names names = { } def p_statement_assign(t): 'statement : NAME EQUALS expression' names[t[1]] = t[3] def p_statement_expr(t): 'statement : expression' print(t[1]) def p_expression_binop(t): '''expression : expression PLUS expression | expression MINUS expression | expression TIMES expression | expression DIVIDE expression''' if t[2] == '+' : t[0] = t[1] + t[3] elif t[2] == '-': t[0] = t[1] - t[3] elif t[2] == '*': t[0] = t[1] * t[3] elif t[2] == '/': t[0] = t[1] / t[3] def p_expression_uminus(t): 'expression : MINUS expression %prec UMINUS' t[0] = -t[2] def p_expression_group(t): 'expression : LPAREN expression RPAREN' t[0] = t[2] def p_expression_number(t): 'expression : NUMBER' t[0] = t[1] def p_expression_name(t): 'expression : NAME' try: t[0] = names[t[1]] except LookupError: print("Undefined name '%s'" % t[1]) t[0] = 0 def p_error_handler(t): 'error : NAME' pass def p_error(t): pass yacc.yacc() ================================================ FILE: xonsh/ply/test/yacc_error5.py ================================================ # ----------------------------------------------------------------------------- # yacc_error5.py # # Lineno and position tracking with error tokens # ----------------------------------------------------------------------------- import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.yacc as yacc from calclex import tokens # Parsing rules precedence = ( ('left','PLUS','MINUS'), ('left','TIMES','DIVIDE'), ('right','UMINUS'), ) # dictionary of names names = { } def p_statement_assign(t): 'statement : NAME EQUALS expression' names[t[1]] = t[3] def p_statement_assign_error(t): 'statement : NAME EQUALS error' line_start, line_end = t.linespan(3) pos_start, pos_end = t.lexspan(3) print("Assignment Error at %d:%d to %d:%d" % (line_start,pos_start,line_end,pos_end)) def p_statement_expr(t): 'statement : expression' print(t[1]) def p_expression_binop(t): '''expression : expression PLUS expression | expression MINUS expression | expression TIMES expression | expression DIVIDE expression''' if t[2] == '+' : t[0] = t[1] + t[3] elif t[2] == '-': t[0] = t[1] - t[3] elif t[2] == '*': t[0] = t[1] * t[3] elif t[2] == '/': t[0] = t[1] / t[3] def p_expression_uminus(t): 'expression : MINUS expression %prec UMINUS' t[0] = -t[2] def p_expression_group(t): 'expression : LPAREN expression RPAREN' line_start, line_end = t.linespan(2) pos_start, pos_end = t.lexspan(2) print("Group at %d:%d to %d:%d" % (line_start,pos_start, line_end, pos_end)) t[0] = t[2] def p_expression_group_error(t): 'expression : LPAREN error RPAREN' line_start, line_end = t.linespan(2) pos_start, pos_end = t.lexspan(2) print("Syntax error at %d:%d to %d:%d" % (line_start,pos_start, line_end, pos_end)) t[0] = 0 def p_expression_number(t): 'expression : NUMBER' t[0] = t[1] def p_expression_name(t): 'expression : NAME' try: t[0] = names[t[1]] except LookupError: print("Undefined name '%s'" % t[1]) t[0] = 0 def p_error(t): print("Syntax error at '%s'" % t.value) parser = yacc.yacc() import calclex calclex.lexer.lineno=1 parser.parse(""" a = 3 + (4*5) + (a b c) + + 6 + 7 """, tracking=True) ================================================ FILE: xonsh/ply/test/yacc_error6.py ================================================ # ----------------------------------------------------------------------------- # yacc_error6.py # # Panic mode recovery test # ----------------------------------------------------------------------------- import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.yacc as yacc from calclex import tokens # Parsing rules precedence = ( ('left','PLUS','MINUS'), ('left','TIMES','DIVIDE'), ('right','UMINUS'), ) def p_statements(t): 'statements : statements statement' pass def p_statements_1(t): 'statements : statement' pass def p_statement_assign(p): 'statement : LPAREN NAME EQUALS expression RPAREN' print("%s=%s" % (p[2],p[4])) def p_statement_expr(t): 'statement : LPAREN expression RPAREN' print(t[1]) def p_expression_binop(t): '''expression : expression PLUS expression | expression MINUS expression | expression TIMES expression | expression DIVIDE expression''' if t[2] == '+' : t[0] = t[1] + t[3] elif t[2] == '-': t[0] = t[1] - t[3] elif t[2] == '*': t[0] = t[1] * t[3] elif t[2] == '/': t[0] = t[1] / t[3] def p_expression_uminus(t): 'expression : MINUS expression %prec UMINUS' t[0] = -t[2] def p_expression_number(t): 'expression : NUMBER' t[0] = t[1] def p_error(p): if p: print("Line %d: Syntax error at '%s'" % (p.lineno, p.value)) # Scan ahead looking for a name token while True: tok = parser.token() if not tok or tok.type == 'RPAREN': break if tok: parser.restart() return None parser = yacc.yacc() import calclex calclex.lexer.lineno=1 parser.parse(""" (a = 3 + 4) (b = 4 + * 5 - 6 + *) (c = 10 + 11) """) ================================================ FILE: xonsh/ply/test/yacc_error7.py ================================================ # ----------------------------------------------------------------------------- # yacc_error7.py # # Panic mode recovery test using deprecated functionality # ----------------------------------------------------------------------------- import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.yacc as yacc from calclex import tokens # Parsing rules precedence = ( ('left','PLUS','MINUS'), ('left','TIMES','DIVIDE'), ('right','UMINUS'), ) def p_statements(t): 'statements : statements statement' pass def p_statements_1(t): 'statements : statement' pass def p_statement_assign(p): 'statement : LPAREN NAME EQUALS expression RPAREN' print("%s=%s" % (p[2],p[4])) def p_statement_expr(t): 'statement : LPAREN expression RPAREN' print(t[1]) def p_expression_binop(t): '''expression : expression PLUS expression | expression MINUS expression | expression TIMES expression | expression DIVIDE expression''' if t[2] == '+' : t[0] = t[1] + t[3] elif t[2] == '-': t[0] = t[1] - t[3] elif t[2] == '*': t[0] = t[1] * t[3] elif t[2] == '/': t[0] = t[1] / t[3] def p_expression_uminus(t): 'expression : MINUS expression %prec UMINUS' t[0] = -t[2] def p_expression_number(t): 'expression : NUMBER' t[0] = t[1] def p_error(p): if p: print("Line %d: Syntax error at '%s'" % (p.lineno, p.value)) # Scan ahead looking for a name token while True: tok = yacc.token() if not tok or tok.type == 'RPAREN': break if tok: yacc.restart() return None parser = yacc.yacc() import calclex calclex.lexer.lineno=1 parser.parse(""" (a = 3 + 4) (b = 4 + * 5 - 6 + *) (c = 10 + 11) """) ================================================ FILE: xonsh/ply/test/yacc_inf.py ================================================ # ----------------------------------------------------------------------------- # yacc_inf.py # # Infinite recursion # ----------------------------------------------------------------------------- import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.yacc as yacc from calclex import tokens # Parsing rules precedence = ( ('left','PLUS','MINUS'), ('left','TIMES','DIVIDE'), ('right','UMINUS'), ) # dictionary of names names = { } def p_statement_assign(t): 'statement : NAME EQUALS expression' names[t[1]] = t[3] def p_statement_expr(t): 'statement : expression' print(t[1]) def p_expression_binop(t): '''expression : expression PLUS expression | expression MINUS expression | expression TIMES expression | expression DIVIDE expression''' if t[2] == '+' : t[0] = t[1] + t[3] elif t[2] == '-': t[0] = t[1] - t[3] elif t[2] == '*': t[0] = t[1] * t[3] elif t[2] == '/': t[0] = t[1] / t[3] def p_expression_uminus(t): 'expression : MINUS expression %prec UMINUS' t[0] = -t[2] def p_expression_group(t): 'expression : LPAREN expression RPAREN' t[0] = t[2] def p_error(t): print("Syntax error at '%s'" % t.value) yacc.yacc() ================================================ FILE: xonsh/ply/test/yacc_literal.py ================================================ # ----------------------------------------------------------------------------- # yacc_literal.py # # Grammar with bad literal characters # ----------------------------------------------------------------------------- import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.yacc as yacc from calclex import tokens # Parsing rules precedence = ( ('left','+','-'), ('left','*','/'), ('right','UMINUS'), ) # dictionary of names names = { } def p_statement_assign(t): 'statement : NAME EQUALS expression' names[t[1]] = t[3] def p_statement_expr(t): 'statement : expression' print(t[1]) def p_expression_binop(t): '''expression : expression '+' expression | expression '-' expression | expression '*' expression | expression '/' expression | expression '**' expression ''' if t[2] == '+' : t[0] = t[1] + t[3] elif t[2] == '-': t[0] = t[1] - t[3] elif t[2] == '*': t[0] = t[1] * t[3] elif t[2] == '/': t[0] = t[1] / t[3] def p_expression_uminus(t): 'expression : MINUS expression %prec UMINUS' t[0] = -t[2] def p_expression_group(t): 'expression : LPAREN expression RPAREN' t[0] = t[2] def p_expression_number(t): 'expression : NUMBER' t[0] = t[1] def p_expression_name(t): 'expression : NAME' try: t[0] = names[t[1]] except LookupError: print("Undefined name '%s'" % t[1]) t[0] = 0 def p_error(t): print("Syntax error at '%s'" % t.value) yacc.yacc() ================================================ FILE: xonsh/ply/test/yacc_misplaced.py ================================================ # ----------------------------------------------------------------------------- # yacc_misplaced.py # # A misplaced | in grammar rules # ----------------------------------------------------------------------------- import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.yacc as yacc from calclex import tokens # Parsing rules precedence = ( ('left','PLUS','MINUS'), ('left','TIMES','DIVIDE'), ('right','UMINUS'), ) # dictionary of names names = { } def p_statement_assign(t): 'statement : NAME EQUALS expression' names[t[1]] = t[3] def p_statement_expr(t): 'statement : expression' print(t[1]) def p_expression_binop(t): ''' | expression PLUS expression | expression MINUS expression | expression TIMES expression | expression DIVIDE expression''' if t[2] == '+' : t[0] = t[1] + t[3] elif t[2] == '-': t[0] = t[1] - t[3] elif t[2] == '*': t[0] = t[1] * t[3] elif t[2] == '/': t[0] = t[1] / t[3] def p_expression_uminus(t): 'expression : MINUS expression %prec UMINUS' t[0] = -t[2] def p_expression_group(t): 'expression : LPAREN expression RPAREN' t[0] = t[2] def p_expression_number(t): 'expression : NUMBER' t[0] = t[1] def p_expression_name(t): 'expression : NAME' try: t[0] = names[t[1]] except LookupError: print("Undefined name '%s'" % t[1]) t[0] = 0 def p_error(t): print("Syntax error at '%s'" % t.value) yacc.yacc() ================================================ FILE: xonsh/ply/test/yacc_missing1.py ================================================ # ----------------------------------------------------------------------------- # yacc_missing1.py # # Grammar with a missing rule # ----------------------------------------------------------------------------- import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.yacc as yacc from calclex import tokens # Parsing rules precedence = ( ('left','PLUS','MINUS'), ('left','TIMES','DIVIDE'), ('right','UMINUS'), ) # dictionary of names names = { } def p_statement_assign(t): 'statement : location EQUALS expression' names[t[1]] = t[3] def p_statement_expr(t): 'statement : expression' print(t[1]) def p_expression_binop(t): '''expression : expression PLUS expression | expression MINUS expression | expression TIMES expression | expression DIVIDE expression''' if t[2] == '+' : t[0] = t[1] + t[3] elif t[2] == '-': t[0] = t[1] - t[3] elif t[2] == '*': t[0] = t[1] * t[3] elif t[2] == '/': t[0] = t[1] / t[3] def p_expression_uminus(t): 'expression : MINUS expression %prec UMINUS' t[0] = -t[2] def p_expression_group(t): 'expression : LPAREN expression RPAREN' t[0] = t[2] def p_expression_number(t): 'expression : NUMBER' t[0] = t[1] def p_expression_name(t): 'expression : NAME' try: t[0] = names[t[1]] except LookupError: print("Undefined name '%s'" % t[1]) t[0] = 0 def p_error(t): print("Syntax error at '%s'" % t.value) yacc.yacc() ================================================ FILE: xonsh/ply/test/yacc_nested.py ================================================ import sys if ".." not in sys.path: sys.path.insert(0,"..") from ply import lex, yacc t_A = 'A' t_B = 'B' t_C = 'C' tokens = ('A', 'B', 'C') the_lexer = lex.lex() def t_error(t): pass def p_error(p): pass def p_start(t): '''start : A nest C''' pass def p_nest(t): '''nest : B''' print(t[-1]) the_parser = yacc.yacc(debug = False, write_tables = False) the_parser.parse('ABC', the_lexer) the_parser.parse('ABC', the_lexer, tracking=True) the_parser.parse('ABC', the_lexer, tracking=True, debug=1) ================================================ FILE: xonsh/ply/test/yacc_nodoc.py ================================================ # ----------------------------------------------------------------------------- # yacc_nodoc.py # # Rule with a missing doc-string # ----------------------------------------------------------------------------- import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.yacc as yacc from calclex import tokens # Parsing rules precedence = ( ('left','PLUS','MINUS'), ('left','TIMES','DIVIDE'), ('right','UMINUS'), ) # dictionary of names names = { } def p_statement_assign(t): 'statement : NAME EQUALS expression' names[t[1]] = t[3] def p_statement_expr(t): print(t[1]) def p_expression_binop(t): '''expression : expression PLUS expression | expression MINUS expression | expression TIMES expression | expression DIVIDE expression''' if t[2] == '+' : t[0] = t[1] + t[3] elif t[2] == '-': t[0] = t[1] - t[3] elif t[2] == '*': t[0] = t[1] * t[3] elif t[2] == '/': t[0] = t[1] / t[3] def p_expression_uminus(t): 'expression : MINUS expression %prec UMINUS' t[0] = -t[2] def p_expression_group(t): 'expression : LPAREN expression RPAREN' t[0] = t[2] def p_expression_number(t): 'expression : NUMBER' t[0] = t[1] def p_expression_name(t): 'expression : NAME' try: t[0] = names[t[1]] except LookupError: print("Undefined name '%s'" % t[1]) t[0] = 0 def p_error(t): print("Syntax error at '%s'" % t.value) yacc.yacc() ================================================ FILE: xonsh/ply/test/yacc_noerror.py ================================================ # ----------------------------------------------------------------------------- # yacc_noerror.py # # No p_error() rule defined. # ----------------------------------------------------------------------------- import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.yacc as yacc from calclex import tokens # Parsing rules precedence = ( ('left','PLUS','MINUS'), ('left','TIMES','DIVIDE'), ('right','UMINUS'), ) # dictionary of names names = { } def p_statement_assign(t): 'statement : NAME EQUALS expression' names[t[1]] = t[3] def p_statement_expr(t): 'statement : expression' print(t[1]) def p_expression_binop(t): '''expression : expression PLUS expression | expression MINUS expression | expression TIMES expression | expression DIVIDE expression''' if t[2] == '+' : t[0] = t[1] + t[3] elif t[2] == '-': t[0] = t[1] - t[3] elif t[2] == '*': t[0] = t[1] * t[3] elif t[2] == '/': t[0] = t[1] / t[3] def p_expression_uminus(t): 'expression : MINUS expression %prec UMINUS' t[0] = -t[2] def p_expression_group(t): 'expression : LPAREN expression RPAREN' t[0] = t[2] def p_expression_number(t): 'expression : NUMBER' t[0] = t[1] def p_expression_name(t): 'expression : NAME' try: t[0] = names[t[1]] except LookupError: print("Undefined name '%s'" % t[1]) t[0] = 0 yacc.yacc() ================================================ FILE: xonsh/ply/test/yacc_nop.py ================================================ # ----------------------------------------------------------------------------- # yacc_nop.py # # Possible grammar rule defined without p_ prefix # ----------------------------------------------------------------------------- import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.yacc as yacc from calclex import tokens # Parsing rules precedence = ( ('left','PLUS','MINUS'), ('left','TIMES','DIVIDE'), ('right','UMINUS'), ) # dictionary of names names = { } def p_statement_assign(t): 'statement : NAME EQUALS expression' names[t[1]] = t[3] def statement_expr(t): 'statement : expression' print(t[1]) def p_expression_binop(t): '''expression : expression PLUS expression | expression MINUS expression | expression TIMES expression | expression DIVIDE expression''' if t[2] == '+' : t[0] = t[1] + t[3] elif t[2] == '-': t[0] = t[1] - t[3] elif t[2] == '*': t[0] = t[1] * t[3] elif t[2] == '/': t[0] = t[1] / t[3] def p_expression_uminus(t): 'expression : MINUS expression %prec UMINUS' t[0] = -t[2] def p_expression_group(t): 'expression : LPAREN expression RPAREN' t[0] = t[2] def p_expression_number(t): 'expression : NUMBER' t[0] = t[1] def p_expression_name(t): 'expression : NAME' try: t[0] = names[t[1]] except LookupError: print("Undefined name '%s'" % t[1]) t[0] = 0 def p_error(t): print("Syntax error at '%s'" % t.value) yacc.yacc() ================================================ FILE: xonsh/ply/test/yacc_notfunc.py ================================================ # ----------------------------------------------------------------------------- # yacc_notfunc.py # # p_rule not defined as a function # ----------------------------------------------------------------------------- import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.yacc as yacc from calclex import tokens # Parsing rules precedence = ( ('left','PLUS','MINUS'), ('left','TIMES','DIVIDE'), ('right','UMINUS'), ) # dictionary of names names = { } p_statement_assign = "Blah" def p_statement_expr(t): 'statement : expression' print(t[1]) def p_expression_binop(t): '''expression : expression PLUS expression | expression MINUS expression | expression TIMES expression | expression DIVIDE expression''' if t[2] == '+' : t[0] = t[1] + t[3] elif t[2] == '-': t[0] = t[1] - t[3] elif t[2] == '*': t[0] = t[1] * t[3] elif t[2] == '/': t[0] = t[1] / t[3] def p_expression_uminus(t): 'expression : MINUS expression %prec UMINUS' t[0] = -t[2] def p_expression_group(t): 'expression : LPAREN expression RPAREN' t[0] = t[2] def p_expression_number(t): 'expression : NUMBER' t[0] = t[1] def p_expression_name(t): 'expression : NAME' try: t[0] = names[t[1]] except LookupError: print("Undefined name '%s'" % t[1]) t[0] = 0 def p_error(t): print("Syntax error at '%s'" % t.value) yacc.yacc() ================================================ FILE: xonsh/ply/test/yacc_notok.py ================================================ # ----------------------------------------------------------------------------- # yacc_notok.py # # A grammar, but we forgot to import the tokens list # ----------------------------------------------------------------------------- import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.yacc as yacc # Parsing rules precedence = ( ('left','PLUS','MINUS'), ('left','TIMES','DIVIDE'), ('right','UMINUS'), ) # dictionary of names names = { } def p_statement_assign(t): 'statement : NAME EQUALS expression' names[t[1]] = t[3] def p_statement_expr(t): 'statement : expression' print(t[1]) def p_expression_binop(t): '''expression : expression PLUS expression | expression MINUS expression | expression TIMES expression | expression DIVIDE expression''' if t[2] == '+' : t[0] = t[1] + t[3] elif t[2] == '-': t[0] = t[1] - t[3] elif t[2] == '*': t[0] = t[1] * t[3] elif t[2] == '/': t[0] = t[1] / t[3] def p_expression_uminus(t): 'expression : MINUS expression %prec UMINUS' t[0] = -t[2] def p_expression_group(t): 'expression : LPAREN expression RPAREN' t[0] = t[2] def p_expression_number(t): 'expression : NUMBER' t[0] = t[1] def p_expression_name(t): 'expression : NAME' try: t[0] = names[t[1]] except LookupError: print("Undefined name '%s'" % t[1]) t[0] = 0 def p_error(t): print("Syntax error at '%s'" % t.value) yacc.yacc() ================================================ FILE: xonsh/ply/test/yacc_prec1.py ================================================ # ----------------------------------------------------------------------------- # yacc_prec1.py # # Tests case where precedence specifier doesn't match up to terminals # ----------------------------------------------------------------------------- import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.yacc as yacc from calclex import tokens # Parsing rules precedence = ( ('left', '+', '-'), ('left', '*', '/'), ('right','UMINUS'), ) # dictionary of names names = { } def p_statement_assign(t): 'statement : NAME EQUALS expression' names[t[1]] = t[3] def p_statement_expr(t): 'statement : expression' print(t[1]) def p_expression_binop(t): '''expression : expression PLUS expression | expression MINUS expression | expression TIMES expression | expression DIVIDE expression''' if t[2] == '+' : t[0] = t[1] + t[3] elif t[2] == '-': t[0] = t[1] - t[3] elif t[2] == '*': t[0] = t[1] * t[3] elif t[2] == '/': t[0] = t[1] / t[3] def p_expression_uminus(t): 'expression : MINUS expression %prec UMINUS' t[0] = -t[2] def p_expression_group(t): 'expression : LPAREN expression RPAREN' t[0] = t[2] def p_expression_number(t): 'expression : NUMBER' t[0] = t[1] def p_expression_name(t): 'expression : NAME' try: t[0] = names[t[1]] except LookupError: print("Undefined name '%s'" % t[1]) t[0] = 0 def p_error(t): print("Syntax error at '%s'" % t.value) yacc.yacc() ================================================ FILE: xonsh/ply/test/yacc_rr.py ================================================ # ----------------------------------------------------------------------------- # yacc_rr.py # # A grammar with a reduce/reduce conflict # ----------------------------------------------------------------------------- import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.yacc as yacc from calclex import tokens # Parsing rules precedence = ( ('left','PLUS','MINUS'), ('left','TIMES','DIVIDE'), ('right','UMINUS'), ) # dictionary of names names = { } def p_statement_assign(t): 'statement : NAME EQUALS expression' names[t[1]] = t[3] def p_statement_assign_2(t): 'statement : NAME EQUALS NUMBER' names[t[1]] = t[3] def p_statement_expr(t): 'statement : expression' print(t[1]) def p_expression_binop(t): '''expression : expression PLUS expression | expression MINUS expression | expression TIMES expression | expression DIVIDE expression''' if t[2] == '+' : t[0] = t[1] + t[3] elif t[2] == '-': t[0] = t[1] - t[3] elif t[2] == '*': t[0] = t[1] * t[3] elif t[2] == '/': t[0] = t[1] / t[3] def p_expression_uminus(t): 'expression : MINUS expression %prec UMINUS' t[0] = -t[2] def p_expression_group(t): 'expression : LPAREN expression RPAREN' t[0] = t[2] def p_expression_number(t): 'expression : NUMBER' t[0] = t[1] def p_expression_name(t): 'expression : NAME' try: t[0] = names[t[1]] except LookupError: print("Undefined name '%s'" % t[1]) t[0] = 0 def p_error(t): print("Syntax error at '%s'" % t.value) yacc.yacc() ================================================ FILE: xonsh/ply/test/yacc_rr_unused.py ================================================ # ----------------------------------------------------------------------------- # yacc_rr_unused.py # # A grammar with reduce/reduce conflicts and a rule that never # gets reduced. # ----------------------------------------------------------------------------- import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.yacc as yacc tokens = ('A', 'B', 'C') def p_grammar(p): ''' rule1 : rule2 B | rule2 C rule2 : rule3 B | rule4 | rule5 rule3 : A rule4 : A rule5 : A ''' yacc.yacc() ================================================ FILE: xonsh/ply/test/yacc_simple.py ================================================ # ----------------------------------------------------------------------------- # yacc_simple.py # # A simple, properly specifier grammar # ----------------------------------------------------------------------------- import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.yacc as yacc from calclex import tokens # Parsing rules precedence = ( ('left','PLUS','MINUS'), ('left','TIMES','DIVIDE'), ('right','UMINUS'), ) # dictionary of names names = { } def p_statement_assign(t): 'statement : NAME EQUALS expression' names[t[1]] = t[3] def p_statement_expr(t): 'statement : expression' print(t[1]) def p_expression_binop(t): '''expression : expression PLUS expression | expression MINUS expression | expression TIMES expression | expression DIVIDE expression''' if t[2] == '+' : t[0] = t[1] + t[3] elif t[2] == '-': t[0] = t[1] - t[3] elif t[2] == '*': t[0] = t[1] * t[3] elif t[2] == '/': t[0] = t[1] / t[3] def p_expression_uminus(t): 'expression : MINUS expression %prec UMINUS' t[0] = -t[2] def p_expression_group(t): 'expression : LPAREN expression RPAREN' t[0] = t[2] def p_expression_number(t): 'expression : NUMBER' t[0] = t[1] def p_expression_name(t): 'expression : NAME' try: t[0] = names[t[1]] except LookupError: print("Undefined name '%s'" % t[1]) t[0] = 0 def p_error(t): print("Syntax error at '%s'" % t.value) yacc.yacc() ================================================ FILE: xonsh/ply/test/yacc_sr.py ================================================ # ----------------------------------------------------------------------------- # yacc_sr.py # # A grammar with shift-reduce conflicts # ----------------------------------------------------------------------------- import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.yacc as yacc from calclex import tokens # Parsing rules # dictionary of names names = { } def p_statement_assign(t): 'statement : NAME EQUALS expression' names[t[1]] = t[3] def p_statement_expr(t): 'statement : expression' print(t[1]) def p_expression_binop(t): '''expression : expression PLUS expression | expression MINUS expression | expression TIMES expression | expression DIVIDE expression''' if t[2] == '+' : t[0] = t[1] + t[3] elif t[2] == '-': t[0] = t[1] - t[3] elif t[2] == '*': t[0] = t[1] * t[3] elif t[2] == '/': t[0] = t[1] / t[3] def p_expression_uminus(t): 'expression : MINUS expression' t[0] = -t[2] def p_expression_group(t): 'expression : LPAREN expression RPAREN' t[0] = t[2] def p_expression_number(t): 'expression : NUMBER' t[0] = t[1] def p_expression_name(t): 'expression : NAME' try: t[0] = names[t[1]] except LookupError: print("Undefined name '%s'" % t[1]) t[0] = 0 def p_error(t): print("Syntax error at '%s'" % t.value) yacc.yacc() ================================================ FILE: xonsh/ply/test/yacc_term1.py ================================================ # ----------------------------------------------------------------------------- # yacc_term1.py # # Terminal used on the left-hand-side # ----------------------------------------------------------------------------- import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.yacc as yacc from calclex import tokens # Parsing rules precedence = ( ('left','PLUS','MINUS'), ('left','TIMES','DIVIDE'), ('right','UMINUS'), ) # dictionary of names names = { } def p_statement_assign(t): 'NUMBER : NAME EQUALS expression' names[t[1]] = t[3] def p_statement_expr(t): 'statement : expression' print(t[1]) def p_expression_binop(t): '''expression : expression PLUS expression | expression MINUS expression | expression TIMES expression | expression DIVIDE expression''' if t[2] == '+' : t[0] = t[1] + t[3] elif t[2] == '-': t[0] = t[1] - t[3] elif t[2] == '*': t[0] = t[1] * t[3] elif t[2] == '/': t[0] = t[1] / t[3] def p_expression_uminus(t): 'expression : MINUS expression %prec UMINUS' t[0] = -t[2] def p_expression_group(t): 'expression : LPAREN expression RPAREN' t[0] = t[2] def p_expression_number(t): 'expression : NUMBER' t[0] = t[1] def p_expression_name(t): 'expression : NAME' try: t[0] = names[t[1]] except LookupError: print("Undefined name '%s'" % t[1]) t[0] = 0 def p_error(t): print("Syntax error at '%s'" % t.value) yacc.yacc() ================================================ FILE: xonsh/ply/test/yacc_unicode_literals.py ================================================ # ----------------------------------------------------------------------------- # yacc_unicode_literals # # Test for unicode literals on Python 2.x # ----------------------------------------------------------------------------- from __future__ import unicode_literals import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.yacc as yacc from calclex import tokens # Parsing rules precedence = ( ('left','PLUS','MINUS'), ('left','TIMES','DIVIDE'), ('right','UMINUS'), ) # dictionary of names names = { } def p_statement_assign(t): 'statement : NAME EQUALS expression' names[t[1]] = t[3] def p_statement_expr(t): 'statement : expression' print(t[1]) def p_expression_binop(t): '''expression : expression PLUS expression | expression MINUS expression | expression TIMES expression | expression DIVIDE expression''' if t[2] == '+' : t[0] = t[1] + t[3] elif t[2] == '-': t[0] = t[1] - t[3] elif t[2] == '*': t[0] = t[1] * t[3] elif t[2] == '/': t[0] = t[1] / t[3] def p_expression_uminus(t): 'expression : MINUS expression %prec UMINUS' t[0] = -t[2] def p_expression_group(t): 'expression : LPAREN expression RPAREN' t[0] = t[2] def p_expression_number(t): 'expression : NUMBER' t[0] = t[1] def p_expression_name(t): 'expression : NAME' try: t[0] = names[t[1]] except LookupError: print("Undefined name '%s'" % t[1]) t[0] = 0 def p_error(t): print("Syntax error at '%s'" % t.value) yacc.yacc() ================================================ FILE: xonsh/ply/test/yacc_unused.py ================================================ # ----------------------------------------------------------------------------- # yacc_unused.py # # A grammar with an unused rule # ----------------------------------------------------------------------------- import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.yacc as yacc from calclex import tokens # Parsing rules precedence = ( ('left','PLUS','MINUS'), ('left','TIMES','DIVIDE'), ('right','UMINUS'), ) # dictionary of names names = { } def p_statement_assign(t): 'statement : NAME EQUALS expression' names[t[1]] = t[3] def p_statement_expr(t): 'statement : expression' print(t[1]) def p_expression_binop(t): '''expression : expression PLUS expression | expression MINUS expression | expression TIMES expression | expression DIVIDE expression''' if t[2] == '+' : t[0] = t[1] + t[3] elif t[2] == '-': t[0] = t[1] - t[3] elif t[2] == '*': t[0] = t[1] * t[3] elif t[2] == '/': t[0] = t[1] / t[3] def p_expression_uminus(t): 'expression : MINUS expression %prec UMINUS' t[0] = -t[2] def p_expression_group(t): 'expression : LPAREN expression RPAREN' t[0] = t[2] def p_expression_number(t): 'expression : NUMBER' t[0] = t[1] def p_expression_name(t): 'expression : NAME' try: t[0] = names[t[1]] except LookupError: print("Undefined name '%s'" % t[1]) t[0] = 0 def p_expr_list(t): 'exprlist : exprlist COMMA expression' pass def p_expr_list_2(t): 'exprlist : expression' pass def p_error(t): print("Syntax error at '%s'" % t.value) yacc.yacc() ================================================ FILE: xonsh/ply/test/yacc_unused_rule.py ================================================ # ----------------------------------------------------------------------------- # yacc_unused_rule.py # # Grammar with an unused rule # ----------------------------------------------------------------------------- import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.yacc as yacc from calclex import tokens # Parsing rules precedence = ( ('left','PLUS','MINUS'), ('left','TIMES','DIVIDE'), ('right','UMINUS'), ) # dictionary of names names = { } def p_statement_assign(t): 'statement : NAME EQUALS expression' names[t[1]] = t[3] def p_statement_expr(t): 'statement : expression' print(t[1]) def p_expression_binop(t): '''expression : expression PLUS expression | expression MINUS expression | expression TIMES expression | expression DIVIDE expression''' if t[2] == '+' : t[0] = t[1] + t[3] elif t[2] == '-': t[0] = t[1] - t[3] elif t[2] == '*': t[0] = t[1] * t[3] elif t[2] == '/': t[0] = t[1] / t[3] def p_expression_uminus(t): 'expression : MINUS expression %prec UMINUS' t[0] = -t[2] def p_expression_group(t): 'expression : LPAREN expression RPAREN' t[0] = t[2] def p_expression_number(t): 'expression : NUMBER' t[0] = t[1] def p_expression_name(t): 'expression : NAME' try: t[0] = names[t[1]] except LookupError: print("Undefined name '%s'" % t[1]) t[0] = 0 def p_integer(t): 'integer : NUMBER' t[0] = t[1] def p_error(t): print("Syntax error at '%s'" % t.value) yacc.yacc() ================================================ FILE: xonsh/ply/test/yacc_uprec.py ================================================ # ----------------------------------------------------------------------------- # yacc_uprec.py # # A grammar with a bad %prec specifier # ----------------------------------------------------------------------------- import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.yacc as yacc from calclex import tokens # Parsing rules # dictionary of names names = { } def p_statement_assign(t): 'statement : NAME EQUALS expression' names[t[1]] = t[3] def p_statement_expr(t): 'statement : expression' print(t[1]) def p_expression_binop(t): '''expression : expression PLUS expression | expression MINUS expression | expression TIMES expression | expression DIVIDE expression''' if t[2] == '+' : t[0] = t[1] + t[3] elif t[2] == '-': t[0] = t[1] - t[3] elif t[2] == '*': t[0] = t[1] * t[3] elif t[2] == '/': t[0] = t[1] / t[3] def p_expression_uminus(t): 'expression : MINUS expression %prec UMINUS' t[0] = -t[2] def p_expression_group(t): 'expression : LPAREN expression RPAREN' t[0] = t[2] def p_expression_number(t): 'expression : NUMBER' t[0] = t[1] def p_expression_name(t): 'expression : NAME' try: t[0] = names[t[1]] except LookupError: print("Undefined name '%s'" % t[1]) t[0] = 0 def p_error(t): print("Syntax error at '%s'" % t.value) yacc.yacc() ================================================ FILE: xonsh/ply/test/yacc_uprec2.py ================================================ # ----------------------------------------------------------------------------- # yacc_uprec2.py # # A grammar with a bad %prec specifier # ----------------------------------------------------------------------------- import sys if ".." not in sys.path: sys.path.insert(0,"..") import ply.yacc as yacc from calclex import tokens # Parsing rules # dictionary of names names = { } def p_statement_assign(t): 'statement : NAME EQUALS expression' names[t[1]] = t[3] def p_statement_expr(t): 'statement : expression' print(t[1]) def p_expression_binop(t): '''expression : expression PLUS expression | expression MINUS expression | expression TIMES expression | expression DIVIDE expression''' if t[2] == '+' : t[0] = t[1] + t[3] elif t[2] == '-': t[0] = t[1] - t[3] elif t[2] == '*': t[0] = t[1] * t[3] elif t[2] == '/': t[0] = t[1] / t[3] def p_expression_uminus(t): 'expression : MINUS expression %prec' t[0] = -t[2] def p_expression_group(t): 'expression : LPAREN expression RPAREN' t[0] = t[2] def p_expression_number(t): 'expression : NUMBER' t[0] = t[1] def p_expression_name(t): 'expression : NAME' try: t[0] = names[t[1]] except LookupError: print("Undefined name '%s'" % t[1]) t[0] = 0 def p_error(t): print("Syntax error at '%s'" % t.value) yacc.yacc() ================================================ FILE: xonsh/pretty.py ================================================ # -*- coding: utf-8 -*- """ Python advanced pretty printer. This pretty printer is intended to replace the old `pprint` python module which does not allow developers to provide their own pretty print callbacks. This module is based on ruby's `prettyprint.rb` library by `Tanaka Akira`. The following implementations were forked from the IPython project: * Copyright (c) 2008-2014, IPython Development Team * Copyright (C) 2001-2007 Fernando Perez * Copyright (c) 2001, Janko Hauser * Copyright (c) 2001, Nathaniel Gray Example Usage ------------- To directly print the representation of an object use `pprint`:: from pretty import pretty_print pretty_pprint(complex_object) To get a string of the output use `pretty`:: from pretty import pretty string = pretty(complex_object) Extending --------- The pretty library allows developers to add pretty printing rules for their own objects. This process is straightforward. All you have to do is to add a `_repr_pretty_` method to your object and call the methods on the pretty printer passed:: class MyObject(object): def _repr_pretty_(self, p, cycle): ... Here is an example implementation of a `_repr_pretty_` method for a list subclass:: class MyList(list): def _repr_pretty_(self, p, cycle): if cycle: p.text('MyList(...)') else: with p.group(8, 'MyList([', '])'): for idx, item in enumerate(self): if idx: p.text(',') p.breakable() p.pretty(item) The `cycle` parameter is `True` if pretty detected a cycle. You *have* to react to that or the result is an infinite loop. `p.text()` just adds non breaking text to the output, `p.breakable()` either adds a whitespace or breaks here. If you pass it an argument it's used instead of the default space. `p.pretty` prettyprints another object using the pretty print method. The first parameter to the `group` function specifies the extra indentation of the next line. In this example the next item will either be on the same line (if the items are short enough) or aligned with the right edge of the opening bracket of `MyList`. If you just want to indent something you can use the group function without open / close parameters. You can also use this code:: with p.indent(2): ... :copyright: 2007 by Armin Ronacher. Portions (c) 2009 by Robert Kern. :license: BSD License. """ import io import re import sys import types import datetime import contextlib import collections from xonsh.lazyasd import LazyObject, lazyobject __all__ = [ "pretty", "pretty_print", "PrettyPrinter", "RepresentationPrinter", "for_type", "for_type_by_name", ] MAX_SEQ_LENGTH = 1000 def _safe_getattr(obj, attr, default=None): """Safe version of getattr. Same as getattr, but will return ``default`` on any Exception, rather than raising. """ try: return getattr(obj, attr, default) except Exception: return default CUnicodeIO = io.StringIO def pretty( obj, verbose=False, max_width=79, newline="\n", max_seq_length=MAX_SEQ_LENGTH ): """ Pretty print the object's representation. """ if hasattr(obj, "xonsh_display"): return obj.xonsh_display() stream = CUnicodeIO() printer = RepresentationPrinter( stream, verbose, max_width, newline, max_seq_length=max_seq_length ) printer.pretty(obj) printer.flush() return stream.getvalue() def pretty_print( obj, verbose=False, max_width=79, newline="\n", max_seq_length=MAX_SEQ_LENGTH ): """ Like pretty() but print to stdout. """ printer = RepresentationPrinter( sys.stdout, verbose, max_width, newline, max_seq_length=max_seq_length ) printer.pretty(obj) printer.flush() sys.stdout.write(newline) sys.stdout.flush() class _PrettyPrinterBase(object): @contextlib.contextmanager def indent(self, indent): """with statement support for indenting/dedenting.""" self.indentation += indent try: yield finally: self.indentation -= indent @contextlib.contextmanager def group(self, indent=0, open="", close=""): """like begin_group / end_group but for the with statement.""" self.begin_group(indent, open) try: yield finally: self.end_group(indent, close) class PrettyPrinter(_PrettyPrinterBase): """ Baseclass for the `RepresentationPrinter` prettyprinter that is used to generate pretty reprs of objects. Contrary to the `RepresentationPrinter` this printer knows nothing about the default pprinters or the `_repr_pretty_` callback method. """ def __init__( self, output, max_width=79, newline="\n", max_seq_length=MAX_SEQ_LENGTH ): self.output = output self.max_width = max_width self.newline = newline self.max_seq_length = max_seq_length self.output_width = 0 self.buffer_width = 0 self.buffer = collections.deque() root_group = Group(0) self.group_stack = [root_group] self.group_queue = GroupQueue(root_group) self.indentation = 0 def _break_outer_groups(self): while self.max_width < self.output_width + self.buffer_width: group = self.group_queue.deq() if not group: return while group.breakables: x = self.buffer.popleft() self.output_width = x.output(self.output, self.output_width) self.buffer_width -= x.width while self.buffer and isinstance(self.buffer[0], Text): x = self.buffer.popleft() self.output_width = x.output(self.output, self.output_width) self.buffer_width -= x.width def text(self, obj): """Add literal text to the output.""" width = len(obj) if self.buffer: text = self.buffer[-1] if not isinstance(text, Text): text = Text() self.buffer.append(text) text.add(obj, width) self.buffer_width += width self._break_outer_groups() else: self.output.write(obj) self.output_width += width def breakable(self, sep=" "): """ Add a breakable separator to the output. This does not mean that it will automatically break here. If no breaking on this position takes place the `sep` is inserted which default to one space. """ width = len(sep) group = self.group_stack[-1] if group.want_break: self.flush() self.output.write(self.newline) self.output.write(" " * self.indentation) self.output_width = self.indentation self.buffer_width = 0 else: self.buffer.append(Breakable(sep, width, self)) self.buffer_width += width self._break_outer_groups() def break_(self): """ Explicitly insert a newline into the output, maintaining correct indentation. """ self.flush() self.output.write(self.newline) self.output.write(" " * self.indentation) self.output_width = self.indentation self.buffer_width = 0 def begin_group(self, indent=0, open=""): """ Begin a group. If you want support for python < 2.5 which doesn't has the with statement this is the preferred way: p.begin_group(1, '{') ... p.end_group(1, '}') The python 2.5 expression would be this: with p.group(1, '{', '}'): ... The first parameter specifies the indentation for the next line (usually the width of the opening text), the second the opening text. All parameters are optional. """ if open: self.text(open) group = Group(self.group_stack[-1].depth + 1) self.group_stack.append(group) self.group_queue.enq(group) self.indentation += indent def _enumerate(self, seq): """like enumerate, but with an upper limit on the number of items""" for idx, x in enumerate(seq): if self.max_seq_length and idx >= self.max_seq_length: self.text(",") self.breakable() self.text("...") return yield idx, x def end_group(self, dedent=0, close=""): """End a group. See `begin_group` for more details.""" self.indentation -= dedent group = self.group_stack.pop() if not group.breakables: self.group_queue.remove(group) if close: self.text(close) def flush(self): """Flush data that is left in the buffer.""" for data in self.buffer: self.output_width += data.output(self.output, self.output_width) self.buffer.clear() self.buffer_width = 0 def _get_mro(obj_class): """ Get a reasonable method resolution order of a class and its superclasses for both old-style and new-style classes. """ if not hasattr(obj_class, "__mro__"): # Old-style class. Mix in object to make a fake new-style class. try: obj_class = type(obj_class.__name__, (obj_class, object), {}) except TypeError: # Old-style extension type that does not descend from object. # FIXME: try to construct a more thorough MRO. mro = [obj_class] else: mro = obj_class.__mro__[1:-1] else: mro = obj_class.__mro__ return mro class RepresentationPrinter(PrettyPrinter): """ Special pretty printer that has a `pretty` method that calls the pretty printer for a python object. This class stores processing data on `self` so you must *never* use this class in a threaded environment. Always lock it or reinstantiate it. Instances also have a verbose flag callbacks can access to control their output. For example the default instance repr prints all attributes and methods that are not prefixed by an underscore if the printer is in verbose mode. """ def __init__( self, output, verbose=False, max_width=79, newline="\n", singleton_pprinters=None, type_pprinters=None, deferred_pprinters=None, max_seq_length=MAX_SEQ_LENGTH, ): PrettyPrinter.__init__( self, output, max_width, newline, max_seq_length=max_seq_length ) self.verbose = verbose self.stack = [] if singleton_pprinters is None: singleton_pprinters = _singleton_pprinters.copy() self.singleton_pprinters = singleton_pprinters if type_pprinters is None: type_pprinters = _type_pprinters.copy() self.type_pprinters = type_pprinters if deferred_pprinters is None: deferred_pprinters = _deferred_type_pprinters.copy() self.deferred_pprinters = deferred_pprinters def pretty(self, obj): """Pretty print the given object.""" obj_id = id(obj) cycle = obj_id in self.stack self.stack.append(obj_id) self.begin_group() try: obj_class = _safe_getattr(obj, "__class__", None) or type(obj) # First try to find registered singleton printers for the type. try: printer = self.singleton_pprinters[obj_id] except (TypeError, KeyError): pass else: return printer(obj, self, cycle) # Next walk the mro and check for either: # 1) a registered printer # 2) a _repr_pretty_ method for cls in _get_mro(obj_class): if cls in self.type_pprinters: # printer registered in self.type_pprinters return self.type_pprinters[cls](obj, self, cycle) else: # deferred printer printer = self._in_deferred_types(cls) if printer is not None: return printer(obj, self, cycle) else: # Finally look for special method names. # Some objects automatically create any requested # attribute. Try to ignore most of them by checking for # callability. if "_repr_pretty_" in cls.__dict__: meth = cls._repr_pretty_ if callable(meth): return meth(obj, self, cycle) return _default_pprint(obj, self, cycle) finally: self.end_group() self.stack.pop() def _in_deferred_types(self, cls): """ Check if the given class is specified in the deferred type registry. Returns the printer from the registry if it exists, and None if the class is not in the registry. Successful matches will be moved to the regular type registry for future use. """ mod = _safe_getattr(cls, "__module__", None) name = _safe_getattr(cls, "__name__", None) key = (mod, name) printer = None if key in self.deferred_pprinters: # Move the printer over to the regular registry. printer = self.deferred_pprinters.pop(key) self.type_pprinters[cls] = printer return printer class Printable(object): def output(self, stream, output_width): return output_width class Text(Printable): def __init__(self): self.objs = [] self.width = 0 def output(self, stream, output_width): for obj in self.objs: stream.write(obj) return output_width + self.width def add(self, obj, width): self.objs.append(obj) self.width += width class Breakable(Printable): def __init__(self, seq, width, pretty): self.obj = seq self.width = width self.pretty = pretty self.indentation = pretty.indentation self.group = pretty.group_stack[-1] self.group.breakables.append(self) def output(self, stream, output_width): self.group.breakables.popleft() if self.group.want_break: stream.write(self.pretty.newline) stream.write(" " * self.indentation) return self.indentation if not self.group.breakables: self.pretty.group_queue.remove(self.group) stream.write(self.obj) return output_width + self.width class Group(Printable): def __init__(self, depth): self.depth = depth self.breakables = collections.deque() self.want_break = False class GroupQueue(object): def __init__(self, *groups): self.queue = [] for group in groups: self.enq(group) def enq(self, group): depth = group.depth while depth > len(self.queue) - 1: self.queue.append([]) self.queue[depth].append(group) def deq(self): for stack in self.queue: for idx, group in enumerate(reversed(stack)): if group.breakables: del stack[idx] group.want_break = True return group for group in stack: group.want_break = True del stack[:] def remove(self, group): try: self.queue[group.depth].remove(group) except ValueError: pass @lazyobject def _baseclass_reprs(): try: br = (object.__repr__, types.InstanceType.__repr__) except AttributeError: # Python 3 br = (object.__repr__,) return br def _default_pprint(obj, p, cycle): """ The default print function. Used if an object does not provide one and it's none of the builtin objects. """ klass = _safe_getattr(obj, "__class__", None) or type(obj) if _safe_getattr(klass, "__repr__", None) not in _baseclass_reprs: # A user-provided repr. Find newlines and replace them with p.break_() _repr_pprint(obj, p, cycle) return p.begin_group(1, "<") p.pretty(klass) p.text(" at 0x%x" % id(obj)) if cycle: p.text(" ...") elif p.verbose: first = True for key in dir(obj): if not key.startswith("_"): try: value = getattr(obj, key) except AttributeError: continue if isinstance(value, types.MethodType): continue if not first: p.text(",") p.breakable() p.text(key) p.text("=") step = len(key) + 1 p.indentation += step p.pretty(value) p.indentation -= step first = False p.end_group(1, ">") def _seq_pprinter_factory(start, end, basetype): """ Factory that returns a pprint function useful for sequences. Used by the default pprint for tuples, dicts, and lists. """ def inner(obj, p, cycle): typ = type(obj) if ( basetype is not None and typ is not basetype and typ.__repr__ != basetype.__repr__ ): # If the subclass provides its own repr, use it instead. return p.text(typ.__repr__(obj)) if cycle: return p.text(start + "..." + end) step = len(start) p.begin_group(step, start) for idx, x in p._enumerate(obj): if idx: p.text(",") p.breakable() p.pretty(x) if len(obj) == 1 and type(obj) is tuple: # Special case for 1-item tuples. p.text(",") p.end_group(step, end) return inner def _set_pprinter_factory(start, end, basetype): """ Factory that returns a pprint function useful for sets and frozensets. """ def inner(obj, p, cycle): typ = type(obj) if ( basetype is not None and typ is not basetype and typ.__repr__ != basetype.__repr__ ): # If the subclass provides its own repr, use it instead. return p.text(typ.__repr__(obj)) if cycle: return p.text(start + "..." + end) if len(obj) == 0: # Special case. p.text(basetype.__name__ + "()") else: step = len(start) p.begin_group(step, start) # Like dictionary keys, we will try to sort the items if there aren't too many items = obj if not (p.max_seq_length and len(obj) >= p.max_seq_length): try: items = sorted(obj) except Exception: # Sometimes the items don't sort. pass for idx, x in p._enumerate(items): if idx: p.text(",") p.breakable() p.pretty(x) p.end_group(step, end) return inner def _dict_pprinter_factory(start, end, basetype=None): """ Factory that returns a pprint function used by the default pprint of dicts and dict proxies. """ def inner(obj, p, cycle): typ = type(obj) if ( basetype is not None and typ is not basetype and typ.__repr__ != basetype.__repr__ ): # If the subclass provides its own repr, use it instead. return p.text(typ.__repr__(obj)) if cycle: return p.text("{...}") p.begin_group(1, start) keys = obj.keys() # if dict isn't large enough to be truncated, sort keys before displaying if not (p.max_seq_length and len(obj) >= p.max_seq_length): try: keys = sorted(keys) except Exception: # Sometimes the keys don't sort. pass for idx, key in p._enumerate(keys): if idx: p.text(",") p.breakable() p.pretty(key) p.text(": ") p.pretty(obj[key]) p.end_group(1, end) return inner def _super_pprint(obj, p, cycle): """The pprint for the super type.""" p.begin_group(8, "") def _re_pattern_pprint(obj, p, cycle): """The pprint function for regular expression patterns.""" p.text("re.compile(") pattern = repr(obj.pattern) if pattern[:1] in "uU": pattern = pattern[1:] prefix = "ur" else: prefix = "r" pattern = prefix + pattern.replace("\\\\", "\\") p.text(pattern) if obj.flags: p.text(",") p.breakable() done_one = False for flag in ( "TEMPLATE", "IGNORECASE", "LOCALE", "MULTILINE", "DOTALL", "UNICODE", "VERBOSE", "DEBUG", ): if obj.flags & getattr(re, flag): if done_one: p.text("|") p.text("re." + flag) done_one = True p.text(")") def _type_pprint(obj, p, cycle): """The pprint for classes and types.""" # Heap allocated types might not have the module attribute, # and others may set it to None. # Checks for a __repr__ override in the metaclass if type(obj).__repr__ is not type.__repr__: _repr_pprint(obj, p, cycle) return mod = _safe_getattr(obj, "__module__", None) try: name = obj.__qualname__ if not isinstance(name, str): # This can happen if the type implements __qualname__ as a property # or other descriptor in Python 2. raise Exception("Try __name__") except Exception: name = obj.__name__ if not isinstance(name, str): name = "" if mod in (None, "__builtin__", "builtins", "exceptions"): p.text(name) else: p.text(mod + "." + name) def _repr_pprint(obj, p, cycle): """A pprint that just redirects to the normal repr function.""" # Find newlines and replace them with p.break_() output = repr(obj) for idx, output_line in enumerate(output.splitlines()): if idx: p.break_() p.text(output_line) def _function_pprint(obj, p, cycle): """Base pprint for all functions and builtin functions.""" name = _safe_getattr(obj, "__qualname__", obj.__name__) mod = obj.__module__ if mod and mod not in ("__builtin__", "builtins", "exceptions"): name = mod + "." + name p.text("" % name) def _exception_pprint(obj, p, cycle): """Base pprint for all exceptions.""" name = getattr(obj.__class__, "__qualname__", obj.__class__.__name__) if obj.__class__.__module__ not in ("exceptions", "builtins"): name = "%s.%s" % (obj.__class__.__module__, name) step = len(name) + 1 p.begin_group(step, name + "(") for idx, arg in enumerate(getattr(obj, "args", ())): if idx: p.text(",") p.breakable() p.pretty(arg) p.end_group(step, ")") @lazyobject def _type_pprinters(): #: printers for builtin types tp = { int: _repr_pprint, float: _repr_pprint, str: _repr_pprint, tuple: _seq_pprinter_factory("(", ")", tuple), list: _seq_pprinter_factory("[", "]", list), dict: _dict_pprinter_factory("{", "}", dict), set: _set_pprinter_factory("{", "}", set), frozenset: _set_pprinter_factory("frozenset({", "})", frozenset), super: _super_pprint, type(re.compile("")): _re_pattern_pprint, type: _type_pprint, types.FunctionType: _function_pprint, types.BuiltinFunctionType: _function_pprint, types.MethodType: _repr_pprint, datetime.datetime: _repr_pprint, datetime.timedelta: _repr_pprint, } #: the exception base try: _exception_base = BaseException except NameError: _exception_base = Exception tp[_exception_base] = _exception_pprint try: tp[types.DictProxyType] = _dict_pprinter_factory("") tp[types.ClassType] = _type_pprint tp[types.SliceType] = _repr_pprint except AttributeError: # Python 3 tp[slice] = _repr_pprint try: tp[xrange] = _repr_pprint tp[long] = _repr_pprint tp[unicode] = _repr_pprint except NameError: tp[range] = _repr_pprint tp[bytes] = _repr_pprint return tp #: printers for types specified by name @lazyobject def _deferred_type_pprinters(): dtp = {} for_type_by_name("collections", "defaultdict", _defaultdict_pprint, dtp=dtp) for_type_by_name("collections", "OrderedDict", _ordereddict_pprint, dtp=dtp) for_type_by_name("collections", "deque", _deque_pprint, dtp=dtp) for_type_by_name("collections", "Counter", _counter_pprint, dtp=dtp) return dtp def for_type(typ, func): """ Add a pretty printer for a given type. """ oldfunc = _type_pprinters.get(typ, None) if func is not None: # To support easy restoration of old pprinters, we need to ignore Nones. _type_pprinters[typ] = func return oldfunc def for_type_by_name(type_module, type_name, func, dtp=None): """ Add a pretty printer for a type specified by the module and name of a type rather than the type object itself. """ if dtp is None: dtp = _deferred_type_pprinters key = (type_module, type_name) oldfunc = dtp.get(key, None) if func is not None: # To support easy restoration of old pprinters, we need to ignore Nones. dtp[key] = func return oldfunc #: printers for the default singletons _singleton_pprinters = LazyObject( lambda: dict.fromkeys( map(id, [None, True, False, Ellipsis, NotImplemented]), _repr_pprint ), globals(), "_singleton_pprinters", ) def _defaultdict_pprint(obj, p, cycle): name = obj.__class__.__name__ with p.group(len(name) + 1, name + "(", ")"): if cycle: p.text("...") else: p.pretty(obj.default_factory) p.text(",") p.breakable() p.pretty(dict(obj)) def _ordereddict_pprint(obj, p, cycle): name = obj.__class__.__name__ with p.group(len(name) + 1, name + "(", ")"): if cycle: p.text("...") elif len(obj): p.pretty(list(obj.items())) def _deque_pprint(obj, p, cycle): name = obj.__class__.__name__ with p.group(len(name) + 1, name + "(", ")"): if cycle: p.text("...") else: p.pretty(list(obj)) def _counter_pprint(obj, p, cycle): name = obj.__class__.__name__ with p.group(len(name) + 1, name + "(", ")"): if cycle: p.text("...") elif len(obj): p.pretty(dict(obj)) ================================================ FILE: xonsh/proc.py ================================================ # -*- coding: utf-8 -*- """Interface for running Python functions as subprocess-mode commands. Code for several helper methods in the `ProcProxy` class have been reproduced without modification from `subprocess.py` in the Python 3.4.2 standard library. The contents of `subprocess.py` (and, thus, the reproduced methods) are Copyright (c) 2003-2005 by Peter Astrand and were licensed to the Python Software foundation under a Contributor Agreement. """ import io import os import re import sys import time import queue import array import ctypes import signal import inspect import builtins import functools import threading import subprocess import collections.abc as cabc from xonsh.platform import ( ON_WINDOWS, ON_POSIX, ON_MSYS, ON_CYGWIN, CAN_RESIZE_WINDOW, LFLAG, CC, ) from xonsh.tools import ( redirect_stdout, redirect_stderr, print_exception, XonshCalledProcessError, findfirst, on_main_thread, XonshError, format_std_prepost, ALIAS_KWARG_NAMES, ) from xonsh.lazyasd import lazyobject, LazyObject from xonsh.jobs import wait_for_active_job, give_terminal_to, _continue from xonsh.lazyimps import fcntl, termios, _winapi, msvcrt, winutils # these decorators are imported for users back-compatible from xonsh.tools import unthreadable, uncapturable # NOQA # foreground has be deprecated foreground = unthreadable @lazyobject def STDOUT_CAPTURE_KINDS(): return frozenset(["stdout", "object"]) # The following escape codes are xterm codes. # See http://rtfm.etla.org/xterm/ctlseq.html for more. MODE_NUMS = ("1049", "47", "1047") START_ALTERNATE_MODE = LazyObject( lambda: frozenset("\x1b[?{0}h".format(i).encode() for i in MODE_NUMS), globals(), "START_ALTERNATE_MODE", ) END_ALTERNATE_MODE = LazyObject( lambda: frozenset("\x1b[?{0}l".format(i).encode() for i in MODE_NUMS), globals(), "END_ALTERNATE_MODE", ) ALTERNATE_MODE_FLAGS = LazyObject( lambda: tuple(START_ALTERNATE_MODE) + tuple(END_ALTERNATE_MODE), globals(), "ALTERNATE_MODE_FLAGS", ) RE_HIDDEN_BYTES = LazyObject( lambda: re.compile(b"(\001.*?\002)"), globals(), "RE_HIDDEN" ) @lazyobject def RE_VT100_ESCAPE(): return re.compile(b"(\x9B|\x1B\\[)[0-?]*[ -\\/]*[@-~]") @lazyobject def RE_HIDE_ESCAPE(): return re.compile( b"(" + RE_HIDDEN_BYTES.pattern + b"|" + RE_VT100_ESCAPE.pattern + b")" ) class QueueReader: """Provides a file-like interface to reading from a queue.""" def __init__(self, fd, timeout=None): """ Parameters ---------- fd : int A file descriptor timeout : float or None, optional The queue reading timeout. """ self.fd = fd self.timeout = timeout self.closed = False self.queue = queue.Queue() self.thread = None def close(self): """close the reader""" self.closed = True def is_fully_read(self): """Returns whether or not the queue is fully read and the reader is closed. """ return ( self.closed and (self.thread is None or not self.thread.is_alive()) and self.queue.empty() ) def read_queue(self): """Reads a single chunk from the queue. This is blocking if the timeout is None and non-blocking otherwise. """ try: return self.queue.get(block=True, timeout=self.timeout) except queue.Empty: return b"" def read(self, size=-1): """Reads bytes from the file.""" i = 0 buf = b"" while size < 0 or i != size: line = self.read_queue() if line: buf += line else: break i += len(line) return buf def readline(self, size=-1): """Reads a line, or a partial line from the file descriptor.""" i = 0 nl = b"\n" buf = b"" while size < 0 or i != size: line = self.read_queue() if line: buf += line if line.endswith(nl): break else: break i += len(line) return buf def _read_all_lines(self): """This reads all remaining lines in a blocking fashion.""" lines = [] while not self.is_fully_read(): chunk = self.read_queue() lines.extend(chunk.splitlines(keepends=True)) return lines def readlines(self, hint=-1): """Reads lines from the file descriptor. This is blocking for negative hints (i.e. read all the remaining lines) and non-blocking otherwise. """ if hint == -1: return self._read_all_lines() lines = [] while len(lines) != hint: chunk = self.read_queue() if not chunk: break lines.extend(chunk.splitlines(keepends=True)) return lines def fileno(self): """Returns the file descriptor number.""" return self.fd @staticmethod def readable(): """Returns true, because this object is always readable.""" return True def iterqueue(self): """Iterates through all remaining chunks in a blocking fashion.""" while not self.is_fully_read(): chunk = self.read_queue() if not chunk: continue yield chunk def populate_fd_queue(reader, fd, queue): """Reads 1 kb of data from a file descriptor into a queue. If this ends or fails, it flags the calling reader object as closed. """ while True: try: c = os.read(fd, 1024) except OSError: reader.closed = True break if c: queue.put(c) else: reader.closed = True break class NonBlockingFDReader(QueueReader): """A class for reading characters from a file descriptor on a background thread. This has the advantages that the calling thread can close the file and that the reading does not block the calling thread. """ def __init__(self, fd, timeout=None): """ Parameters ---------- fd : int A file descriptor timeout : float or None, optional The queue reading timeout. """ super().__init__(fd, timeout=timeout) # start reading from stream self.thread = threading.Thread( target=populate_fd_queue, args=(self, self.fd, self.queue) ) self.thread.daemon = True self.thread.start() def populate_buffer(reader, fd, buffer, chunksize): """Reads bytes from the file descriptor and copies them into a buffer. The reads happen in parallel using the pread() syscall; which is only available on POSIX systems. If the read fails for any reason, the reader is flagged as closed. """ offset = 0 while True: try: buf = os.pread(fd, chunksize, offset) except OSError: reader.closed = True break if buf: buffer.write(buf) offset += len(buf) else: reader.closed = True break class BufferedFDParallelReader: """Buffered, parallel background thread reader.""" def __init__(self, fd, buffer=None, chunksize=1024): """ Parameters ---------- fd : int File descriptor from which to read. buffer : binary file-like or None, optional A buffer to write bytes into. If None, a new BytesIO object is created. chunksize : int, optional The max size of the parallel reads, default 1 kb. """ self.fd = fd self.buffer = io.BytesIO() if buffer is None else buffer self.chunksize = chunksize self.closed = False # start reading from stream self.thread = threading.Thread( target=populate_buffer, args=(self, fd, self.buffer, chunksize) ) self.thread.daemon = True self.thread.start() def _expand_console_buffer(cols, max_offset, expandsize, orig_posize, fd): # if we are getting close to the end of the console buffer, # expand it so that we can read from it successfully. if cols == 0: return orig_posize[-1], max_offset, orig_posize rows = ((max_offset + expandsize) // cols) + 1 winutils.set_console_screen_buffer_size(cols, rows, fd=fd) orig_posize = orig_posize[:3] + (rows,) max_offset = (rows - 1) * cols return rows, max_offset, orig_posize def populate_console(reader, fd, buffer, chunksize, queue, expandsize=None): """Reads bytes from the file descriptor and puts lines into the queue. The reads happened in parallel, using xonsh.winutils.read_console_output_character(), and is thus only available on windows. If the read fails for any reason, the reader is flagged as closed. """ # OK, so this function is super annoying because Windows stores its # buffers as a 2D regular, dense array -- without trailing newlines. # Meanwhile, we want to add *lines* to the queue. Also, as is typical # with parallel reads, the entire buffer that you ask for may not be # filled. Thus we have to deal with the full generality. # 1. reads may end in the middle of a line # 2. excess whitespace at the end of a line may not be real, unless # 3. you haven't read to the end of the line yet! # So there are alignment issues everywhere. Also, Windows will automatically # read past the current cursor position, even though there is presumably # nothing to see there. # # These chunked reads basically need to happen like this because, # a. The default buffer size is HUGE for the console (90k lines x 120 cols) # as so we can't just read in everything at the end and see what we # care about without a noticeable performance hit. # b. Even with this huge size, it is still possible to write more lines than # this, so we should scroll along with the console. # Unfortunately, because we do not have control over the terminal emulator, # It is not possible to compute how far back we should set the beginning # read position because we don't know how many characters have been popped # off the top of the buffer. If we did somehow know this number we could do # something like the following: # # new_offset = (y*cols) + x # if new_offset == max_offset: # new_offset -= scrolled_offset # x = new_offset%cols # y = new_offset//cols # continue # # So this method is imperfect and only works as long as the screen has # room to expand to. Thus the trick here is to expand the screen size # when we get close enough to the end of the screen. There remain some # async issues related to not being able to set the cursor position. # but they just affect the alignment / capture of the output of the # first command run after a screen resize. if expandsize is None: expandsize = 100 * chunksize x, y, cols, rows = posize = winutils.get_position_size(fd) pre_x = pre_y = -1 orig_posize = posize offset = (cols * y) + x max_offset = (rows - 1) * cols # I believe that there is a bug in PTK that if we reset the # cursor position, the cursor on the next prompt is accidentally on # the next line. If this is fixed, uncomment the following line. # if max_offset < offset + expandsize: # rows, max_offset, orig_posize = _expand_console_buffer( # cols, max_offset, expandsize, # orig_posize, fd) # winutils.set_console_cursor_position(x, y, fd=fd) while True: posize = winutils.get_position_size(fd) offset = (cols * y) + x if ((posize[1], posize[0]) <= (y, x) and posize[2:] == (cols, rows)) or ( pre_x == x and pre_y == y ): # already at or ahead of the current cursor position. if reader.closed: break else: time.sleep(reader.timeout) continue elif max_offset <= offset + expandsize: ecb = _expand_console_buffer(cols, max_offset, expandsize, orig_posize, fd) rows, max_offset, orig_posize = ecb continue elif posize[2:] == (cols, rows): # cursor updated but screen size is the same. pass else: # screen size changed, which is offset preserving orig_posize = posize cols, rows = posize[2:] x = offset % cols y = offset // cols pre_x = pre_y = -1 max_offset = (rows - 1) * cols continue try: buf = winutils.read_console_output_character( x=x, y=y, fd=fd, buf=buffer, bufsize=chunksize, raw=True ) except (OSError, IOError): reader.closed = True break # cursor position and offset if not reader.closed: buf = buf.rstrip() nread = len(buf) if nread == 0: time.sleep(reader.timeout) continue cur_x, cur_y = posize[0], posize[1] cur_offset = (cols * cur_y) + cur_x beg_offset = (cols * y) + x end_offset = beg_offset + nread if end_offset > cur_offset and cur_offset != max_offset: buf = buf[: cur_offset - end_offset] # convert to lines xshift = cols - x yshift = (nread // cols) + (1 if nread % cols > 0 else 0) lines = [buf[:xshift]] lines += [ buf[l * cols + xshift : (l + 1) * cols + xshift] for l in range(yshift) ] lines = [line for line in lines if line] if not lines: time.sleep(reader.timeout) continue # put lines in the queue nl = b"\n" for line in lines[:-1]: queue.put(line.rstrip() + nl) if len(lines[-1]) == xshift: queue.put(lines[-1].rstrip() + nl) else: queue.put(lines[-1]) # update x and y locations if (beg_offset + len(buf)) % cols == 0: new_offset = beg_offset + len(buf) else: new_offset = beg_offset + len(buf.rstrip()) pre_x = x pre_y = y x = new_offset % cols y = new_offset // cols time.sleep(reader.timeout) class ConsoleParallelReader(QueueReader): """Parallel reader for consoles that runs in a background thread. This is only needed, available, and useful on Windows. """ def __init__(self, fd, buffer=None, chunksize=1024, timeout=None): """ Parameters ---------- fd : int Standard buffer file descriptor, 0 for stdin, 1 for stdout (default), and 2 for stderr. buffer : ctypes.c_wchar_p, optional An existing buffer to (re-)use. chunksize : int, optional The max size of the parallel reads, default 1 kb. timeout : float, optional The queue reading timeout. """ timeout = timeout or builtins.__xonsh__.env.get("XONSH_PROC_FREQUENCY") super().__init__(fd, timeout=timeout) self._buffer = buffer # this cannot be public if buffer is None: self._buffer = ctypes.c_char_p(b" " * chunksize) self.chunksize = chunksize # start reading from stream self.thread = threading.Thread( target=populate_console, args=(self, fd, self._buffer, chunksize, self.queue), ) self.thread.daemon = True self.thread.start() def safe_fdclose(handle, cache=None): """Closes a file handle in the safest way possible, and potentially storing the result. """ if cache is not None and cache.get(handle, False): return status = True if handle is None: pass elif isinstance(handle, int): if handle >= 3: # don't close stdin, stdout, stderr, -1 try: os.close(handle) except OSError: status = False elif handle is sys.stdin or handle is sys.stdout or handle is sys.stderr: # don't close stdin, stdout, or stderr pass else: try: handle.close() except OSError: status = False if cache is not None: cache[handle] = status def safe_flush(handle): """Attempts to safely flush a file handle, returns success bool.""" status = True try: handle.flush() except OSError: status = False return status def still_writable(fd): """Determines whether a file descriptor is still writable by trying to write an empty string and seeing if it fails. """ try: os.write(fd, b"") status = True except OSError: status = False return status class PopenThread(threading.Thread): """A thread for running and managing subprocess. This allows reading from the stdin, stdout, and stderr streams in a non-blocking fashion. This takes the same arguments and keyword arguments as regular Popen. This requires that the captured_stdout and captured_stderr attributes to be set following instantiation. """ def __init__(self, *args, stdin=None, stdout=None, stderr=None, **kwargs): super().__init__() self.lock = threading.RLock() env = builtins.__xonsh__.env # stdin setup self.orig_stdin = stdin if stdin is None: self.stdin_fd = 0 elif isinstance(stdin, int): self.stdin_fd = stdin else: self.stdin_fd = stdin.fileno() self.store_stdin = env.get("XONSH_STORE_STDIN") self.timeout = env.get("XONSH_PROC_FREQUENCY") self.in_alt_mode = False self.stdin_mode = None # stdout setup self.orig_stdout = stdout self.stdout_fd = 1 if stdout is None else stdout.fileno() self._set_pty_size() # stderr setup self.orig_stderr = stderr # Set some signal handles, if we can. Must come before process # is started to prevent deadlock on windows self.proc = None # has to be here for closure for handles self.old_int_handler = self.old_winch_handler = None self.old_tstp_handler = self.old_quit_handler = None if on_main_thread(): self.old_int_handler = signal.signal(signal.SIGINT, self._signal_int) if ON_POSIX: self.old_tstp_handler = signal.signal(signal.SIGTSTP, self._signal_tstp) self.old_quit_handler = signal.signal(signal.SIGQUIT, self._signal_quit) if CAN_RESIZE_WINDOW: self.old_winch_handler = signal.signal( signal.SIGWINCH, self._signal_winch ) # start up process if ON_WINDOWS and stdout is not None: os.set_inheritable(stdout.fileno(), False) try: self.proc = proc = subprocess.Popen( *args, stdin=stdin, stdout=stdout, stderr=stderr, **kwargs ) except Exception: self._clean_up() raise self.pid = proc.pid self.universal_newlines = uninew = proc.universal_newlines if uninew: self.encoding = enc = env.get("XONSH_ENCODING") self.encoding_errors = err = env.get("XONSH_ENCODING_ERRORS") self.stdin = io.BytesIO() # stdin is always bytes! self.stdout = io.TextIOWrapper(io.BytesIO(), encoding=enc, errors=err) self.stderr = io.TextIOWrapper(io.BytesIO(), encoding=enc, errors=err) else: self.encoding = self.encoding_errors = None self.stdin = io.BytesIO() self.stdout = io.BytesIO() self.stderr = io.BytesIO() self.suspended = False self.prevs_are_closed = False self.start() def run(self): """Runs the subprocess by performing a parallel read on stdin if allowed, and copying bytes from captured_stdout to stdout and bytes from captured_stderr to stderr. """ proc = self.proc spec = self._wait_and_getattr("spec") # get stdin and apply parallel reader if needed. stdin = self.stdin if self.orig_stdin is None: origin = None elif ON_POSIX and self.store_stdin: origin = self.orig_stdin origfd = origin if isinstance(origin, int) else origin.fileno() origin = BufferedFDParallelReader(origfd, buffer=stdin) else: origin = None # get non-blocking stdout stdout = self.stdout.buffer if self.universal_newlines else self.stdout capout = spec.captured_stdout if capout is None: procout = None else: procout = NonBlockingFDReader(capout.fileno(), timeout=self.timeout) # get non-blocking stderr stderr = self.stderr.buffer if self.universal_newlines else self.stderr caperr = spec.captured_stderr if caperr is None: procerr = None else: procerr = NonBlockingFDReader(caperr.fileno(), timeout=self.timeout) # initial read from buffer self._read_write(procout, stdout, sys.__stdout__) self._read_write(procerr, stderr, sys.__stderr__) # loop over reads while process is running. i = j = cnt = 1 while proc.poll() is None: # this is here for CPU performance reasons. if i + j == 0: cnt = min(cnt + 1, 1000) tout = self.timeout * cnt if procout is not None: procout.timeout = tout if procerr is not None: procerr.timeout = tout elif cnt == 1: pass else: cnt = 1 if procout is not None: procout.timeout = self.timeout if procerr is not None: procerr.timeout = self.timeout # redirect some output! i = self._read_write(procout, stdout, sys.__stdout__) j = self._read_write(procerr, stderr, sys.__stderr__) if self.suspended: break if self.suspended: return # close files to send EOF to non-blocking reader. # capout & caperr seem to be needed only by Windows, while # orig_stdout & orig_stderr are need by posix and Windows. # Also, order seems to matter here, # with orig_* needed to be closed before cap* safe_fdclose(self.orig_stdout) safe_fdclose(self.orig_stderr) if ON_WINDOWS: safe_fdclose(capout) safe_fdclose(caperr) # read in the remaining data in a blocking fashion. while (procout is not None and not procout.is_fully_read()) or ( procerr is not None and not procerr.is_fully_read() ): self._read_write(procout, stdout, sys.__stdout__) self._read_write(procerr, stderr, sys.__stderr__) # kill the process if it is still alive. Happens when piping. if proc.poll() is None: proc.terminate() def _wait_and_getattr(self, name): """make sure the instance has a certain attr, and return it.""" while not hasattr(self, name): time.sleep(1e-7) return getattr(self, name) def _read_write(self, reader, writer, stdbuf): """Reads a chunk of bytes from a buffer and write into memory or back down to the standard buffer, as appropriate. Returns the number of successful reads. """ if reader is None: return 0 i = -1 for i, chunk in enumerate(iter(reader.read_queue, b"")): self._alt_mode_switch(chunk, writer, stdbuf) if i >= 0: writer.flush() stdbuf.flush() return i + 1 def _alt_mode_switch(self, chunk, membuf, stdbuf): """Enables recursively switching between normal capturing mode and 'alt' mode, which passes through values to the standard buffer. Pagers, text editors, curses applications, etc. use alternate mode. """ i, flag = findfirst(chunk, ALTERNATE_MODE_FLAGS) if flag is None: self._alt_mode_writer(chunk, membuf, stdbuf) else: # This code is executed when the child process switches the # terminal into or out of alternate mode. The line below assumes # that the user has opened vim, less, or similar, and writes writes # to stdin. j = i + len(flag) # write the first part of the chunk in the current mode. self._alt_mode_writer(chunk[:i], membuf, stdbuf) # switch modes # write the flag itself the current mode where alt mode is on # so that it is streamed to the terminal ASAP. # this is needed for terminal emulators to find the correct # positions before and after alt mode. alt_mode = flag in START_ALTERNATE_MODE if alt_mode: self.in_alt_mode = alt_mode self._alt_mode_writer(flag, membuf, stdbuf) self._enable_cbreak_stdin() else: self._alt_mode_writer(flag, membuf, stdbuf) self.in_alt_mode = alt_mode self._disable_cbreak_stdin() # recurse this function, but without the current flag. self._alt_mode_switch(chunk[j:], membuf, stdbuf) def _alt_mode_writer(self, chunk, membuf, stdbuf): """Write bytes to the standard buffer if in alt mode or otherwise to the in-memory buffer. """ if not chunk: pass # don't write empty values elif self.in_alt_mode: stdbuf.buffer.write(chunk) else: with self.lock: p = membuf.tell() membuf.seek(0, io.SEEK_END) membuf.write(chunk) membuf.seek(p) # # Window resize handlers # def _signal_winch(self, signum, frame): """Signal handler for SIGWINCH - window size has changed.""" self.send_signal(signal.SIGWINCH) self._set_pty_size() def _set_pty_size(self): """Sets the window size of the child pty based on the window size of our own controlling terminal. """ if ON_WINDOWS or not os.isatty(self.stdout_fd): return # Get the terminal size of the real terminal, set it on the # pseudoterminal. buf = array.array("h", [0, 0, 0, 0]) # 1 = stdout here try: fcntl.ioctl(1, termios.TIOCGWINSZ, buf, True) fcntl.ioctl(self.stdout_fd, termios.TIOCSWINSZ, buf) except OSError: pass # # SIGINT handler # def _signal_int(self, signum, frame): """Signal handler for SIGINT - Ctrl+C may have been pressed.""" self.send_signal(signum) if self.proc is not None and self.proc.poll() is not None: self._restore_sigint(frame=frame) if on_main_thread(): signal.pthread_kill(threading.get_ident(), signal.SIGINT) def _restore_sigint(self, frame=None): old = self.old_int_handler if old is not None: if on_main_thread(): signal.signal(signal.SIGINT, old) self.old_int_handler = None if frame is not None: self._disable_cbreak_stdin() if old is not None and old is not self._signal_int: old(signal.SIGINT, frame) # # SIGTSTP handler # def _signal_tstp(self, signum, frame): """Signal handler for suspending SIGTSTP - Ctrl+Z may have been pressed. """ self.suspended = True self.send_signal(signum) self._restore_sigtstp(frame=frame) def _restore_sigtstp(self, frame=None): old = self.old_tstp_handler if old is not None: if on_main_thread(): signal.signal(signal.SIGTSTP, old) self.old_tstp_handler = None if frame is not None: self._disable_cbreak_stdin() # # SIGQUIT handler # def _signal_quit(self, signum, frame): r"""Signal handler for quiting SIGQUIT - Ctrl+\ may have been pressed. """ self.send_signal(signum) self._restore_sigquit(frame=frame) def _restore_sigquit(self, frame=None): old = self.old_quit_handler if old is not None: if on_main_thread(): signal.signal(signal.SIGQUIT, old) self.old_quit_handler = None if frame is not None: self._disable_cbreak_stdin() # # cbreak mode handlers # def _enable_cbreak_stdin(self): if not ON_POSIX: return try: self.stdin_mode = termios.tcgetattr(self.stdin_fd)[:] except termios.error: # this can happen for cases where another process is controlling # xonsh's tty device, such as in testing. self.stdin_mode = None return new = self.stdin_mode[:] new[LFLAG] &= ~(termios.ECHO | termios.ICANON) new[CC][termios.VMIN] = 1 new[CC][termios.VTIME] = 0 try: # termios.TCSAFLUSH may be less reliable than termios.TCSANOW termios.tcsetattr(self.stdin_fd, termios.TCSANOW, new) except termios.error: self._disable_cbreak_stdin() def _disable_cbreak_stdin(self): if not ON_POSIX or self.stdin_mode is None: return new = self.stdin_mode[:] new[LFLAG] |= termios.ECHO | termios.ICANON new[CC][termios.VMIN] = 1 new[CC][termios.VTIME] = 0 try: termios.tcsetattr(self.stdin_fd, termios.TCSANOW, new) except termios.error: pass # # Dispatch methods # def poll(self): """Dispatches to Popen.returncode.""" return self.proc.returncode def wait(self, timeout=None): """Dispatches to Popen.wait(), but also does process cleanup such as joining this thread and replacing the original window size signal handler. """ self._disable_cbreak_stdin() rtn = self.proc.wait(timeout=timeout) self.join() # need to replace the old signal handlers somewhere... if self.old_winch_handler is not None and on_main_thread(): signal.signal(signal.SIGWINCH, self.old_winch_handler) self.old_winch_handler = None self._clean_up() return rtn def _clean_up(self): self._restore_sigint() self._restore_sigtstp() self._restore_sigquit() @property def returncode(self): """Process return code.""" return self.proc.returncode @returncode.setter def returncode(self, value): """Process return code.""" self.proc.returncode = value @property def signal(self): """Process signal, or None.""" s = getattr(self.proc, "signal", None) if s is None: rtn = self.returncode if rtn is not None and rtn != 0: s = (-1 * rtn, rtn < 0 if ON_WINDOWS else os.WCOREDUMP(rtn)) return s @signal.setter def signal(self, value): """Process signal, or None.""" self.proc.signal = value def send_signal(self, signal): """Dispatches to Popen.send_signal().""" dt = 0.0 while self.proc is None and dt < self.timeout: time.sleep(1e-7) dt += 1e-7 if self.proc is None: return try: rtn = self.proc.send_signal(signal) except ProcessLookupError: # This can happen in the case of !(cmd) when the command has ended rtn = None return rtn def terminate(self): """Dispatches to Popen.terminate().""" return self.proc.terminate() def kill(self): """Dispatches to Popen.kill().""" return self.proc.kill() class Handle(int): closed = False def Close(self, CloseHandle=None): CloseHandle = CloseHandle or _winapi.CloseHandle if not self.closed: self.closed = True CloseHandle(self) def Detach(self): if not self.closed: self.closed = True return int(self) raise ValueError("already closed") def __repr__(self): return "Handle(%d)" % int(self) __del__ = Close __str__ = __repr__ class FileThreadDispatcher: """Dispatches to different file handles depending on the current thread. Useful if you want file operation to go to different places for different threads. """ def __init__(self, default=None): """ Parameters ---------- default : file-like or None, optional The file handle to write to if a thread cannot be found in the registry. If None, a new in-memory instance. Attributes ---------- registry : dict Maps thread idents to file handles. """ if default is None: default = io.TextIOWrapper(io.BytesIO()) self.default = default self.registry = {} def register(self, handle): """Registers a file handle for the current thread. Returns self so that this method can be used in a with-statement. """ if handle is self: # prevent weird recurssion errors return self self.registry[threading.get_ident()] = handle return self def deregister(self): """Removes the current thread from the registry.""" ident = threading.get_ident() if ident in self.registry: # don't remove if we have already been deregistered del self.registry[threading.get_ident()] @property def available(self): """True if the thread is available in the registry.""" return threading.get_ident() in self.registry @property def handle(self): """Gets the current handle for the thread.""" return self.registry.get(threading.get_ident(), self.default) def __enter__(self): pass def __exit__(self, ex_type, ex_value, ex_traceback): self.deregister() # # io.TextIOBase interface # @property def encoding(self): """Gets the encoding for this thread's handle.""" return self.handle.encoding @property def errors(self): """Gets the errors for this thread's handle.""" return self.handle.errors @property def newlines(self): """Gets the newlines for this thread's handle.""" return self.handle.newlines @property def buffer(self): """Gets the buffer for this thread's handle.""" return self.handle.buffer def detach(self): """Detaches the buffer for the current thread.""" return self.handle.detach() def read(self, size=None): """Reads from the handle for the current thread.""" return self.handle.read(size) def readline(self, size=-1): """Reads a line from the handle for the current thread.""" return self.handle.readline(size) def readlines(self, hint=-1): """Reads lines from the handle for the current thread.""" return self.handle.readlines(hint) def seek(self, offset, whence=io.SEEK_SET): """Seeks the current file.""" return self.handle.seek(offset, whence) def tell(self): """Reports the current position in the handle for the current thread.""" return self.handle.tell() def write(self, s): """Writes to this thread's handle. This also flushes, just to be extra sure the string was written. """ h = self.handle try: r = h.write(s) h.flush() except OSError: r = None return r @property def line_buffering(self): """Gets if line buffering for this thread's handle enabled.""" return self.handle.line_buffering # # io.IOBase interface # def close(self): """Closes the current thread's handle.""" return self.handle.close() @property def closed(self): """Is the thread's handle closed.""" return self.handle.closed def fileno(self): """Returns the file descriptor for the current thread.""" return self.handle.fileno() def flush(self): """Flushes the file descriptor for the current thread.""" return safe_flush(self.handle) def isatty(self): """Returns if the file descriptor for the current thread is a tty.""" return self.handle.isatty() def readable(self): """Returns if file descriptor for the current thread is readable.""" return self.handle.readable() def seekable(self): """Returns if file descriptor for the current thread is seekable.""" return self.handle.seekable() def truncate(self, size=None): """Truncates the file for for the current thread.""" return self.handle.truncate() def writable(self, size=None): """Returns if file descriptor for the current thread is writable.""" return self.handle.writable(size) def writelines(self): """Writes lines for the file descriptor for the current thread.""" return self.handle.writelines() # These should NOT be lazy since they *need* to get the true stdout from the # main thread. Also their creation time should be negligible. STDOUT_DISPATCHER = FileThreadDispatcher(default=sys.stdout) STDERR_DISPATCHER = FileThreadDispatcher(default=sys.stderr) def parse_proxy_return(r, stdout, stderr): """Proxies may return a variety of outputs. This handles them generally. Parameters ---------- r : tuple, str, int, or None Return from proxy function stdout : file-like Current stdout stream stdout : file-like Current stderr stream Returns ------- cmd_result : int The return code of the proxy """ cmd_result = 0 if isinstance(r, str): stdout.write(r) stdout.flush() elif isinstance(r, int): cmd_result = r elif isinstance(r, cabc.Sequence): rlen = len(r) if rlen > 0 and r[0] is not None: stdout.write(r[0]) stdout.flush() if rlen > 1 and r[1] is not None: stderr.write(r[1]) stderr.flush() if rlen > 2 and r[2] is not None: cmd_result = r[2] elif r is not None: # for the random object... stdout.write(str(r)) stdout.flush() return cmd_result def proxy_zero(f, args, stdin, stdout, stderr, spec, stack): """Calls a proxy function which takes no parameters.""" return f() def proxy_one(f, args, stdin, stdout, stderr, spec, stack): """Calls a proxy function which takes one parameter: args""" return f(args) def proxy_two(f, args, stdin, stdout, stderr, spec, stack): """Calls a proxy function which takes two parameter: args and stdin.""" return f(args, stdin) def proxy_three(f, args, stdin, stdout, stderr, spec, stack): """Calls a proxy function which takes three parameter: args, stdin, stdout. """ return f(args, stdin, stdout) def proxy_four(f, args, stdin, stdout, stderr, spec, stack): """Calls a proxy function which takes four parameter: args, stdin, stdout, and stderr. """ return f(args, stdin, stdout, stderr) def proxy_five(f, args, stdin, stdout, stderr, spec, stack): """Calls a proxy function which takes four parameter: args, stdin, stdout, stderr, and spec. """ return f(args, stdin, stdout, stderr, spec) PROXIES = (proxy_zero, proxy_one, proxy_two, proxy_three, proxy_four, proxy_five) def partial_proxy(f): """Dispatches the appropriate proxy function based on the number of args.""" numargs = 0 for name, param in inspect.signature(f).parameters.items(): if ( param.kind == param.POSITIONAL_ONLY or param.kind == param.POSITIONAL_OR_KEYWORD ): numargs += 1 elif name in ALIAS_KWARG_NAMES and param.kind == param.KEYWORD_ONLY: numargs += 1 if numargs < 6: return functools.partial(PROXIES[numargs], f) elif numargs == 6: # don't need to partial. return f else: e = "Expected proxy with 6 or fewer arguments for {}, not {}" raise XonshError(e.format(", ".join(ALIAS_KWARG_NAMES), numargs)) class ProcProxyThread(threading.Thread): """ Class representing a function to be run as a subprocess-mode command. """ def __init__( self, f, args, stdin=None, stdout=None, stderr=None, universal_newlines=False, close_fds=False, env=None, ): """Parameters ---------- f : function The function to be executed. args : list A (possibly empty) list containing the arguments that were given on the command line stdin : file-like, optional A file-like object representing stdin (input can be read from here). If `stdin` is not provided or if it is explicitly set to `None`, then an instance of `io.StringIO` representing an empty file is used. stdout : file-like, optional A file-like object representing stdout (normal output can be written here). If `stdout` is not provided or if it is explicitly set to `None`, then `sys.stdout` is used. stderr : file-like, optional A file-like object representing stderr (error output can be written here). If `stderr` is not provided or if it is explicitly set to `None`, then `sys.stderr` is used. universal_newlines : bool, optional Whether or not to use universal newlines. close_fds : bool, optional Whether or not to close file descriptors. This is here for Popen compatability and currently does nothing. env : Mapping, optional Environment mapping. """ self.orig_f = f self.f = partial_proxy(f) self.args = args self.pid = None self.returncode = None self._closed_handle_cache = {} handles = self._get_handles(stdin, stdout, stderr) ( self.p2cread, self.p2cwrite, self.c2pread, self.c2pwrite, self.errread, self.errwrite, ) = handles # default values self.stdin = stdin self.stdout = stdout self.stderr = stderr self.close_fds = close_fds self.env = env or builtins.__xonsh__.env self._interrupted = False if ON_WINDOWS: if self.p2cwrite != -1: self.p2cwrite = msvcrt.open_osfhandle(self.p2cwrite.Detach(), 0) if self.c2pread != -1: self.c2pread = msvcrt.open_osfhandle(self.c2pread.Detach(), 0) if self.errread != -1: self.errread = msvcrt.open_osfhandle(self.errread.Detach(), 0) if self.p2cwrite != -1: self.stdin = io.open(self.p2cwrite, "wb", -1) if universal_newlines: self.stdin = io.TextIOWrapper( self.stdin, write_through=True, line_buffering=False ) elif isinstance(stdin, int) and stdin != 0: self.stdin = io.open(stdin, "wb", -1) if self.c2pread != -1: self.stdout = io.open(self.c2pread, "rb", -1) if universal_newlines: self.stdout = io.TextIOWrapper(self.stdout) if self.errread != -1: self.stderr = io.open(self.errread, "rb", -1) if universal_newlines: self.stderr = io.TextIOWrapper(self.stderr) # Set some signal handles, if we can. Must come before process # is started to prevent deadlock on windows self.old_int_handler = None if on_main_thread(): self.old_int_handler = signal.signal(signal.SIGINT, self._signal_int) # start up the proc super().__init__() self.start() def __del__(self): self._restore_sigint() def run(self): """Set up input/output streams and execute the child function in a new thread. This is part of the `threading.Thread` interface and should not be called directly. """ if self.f is None: return spec = self._wait_and_getattr("spec") last_in_pipeline = spec.last_in_pipeline if last_in_pipeline: capout = spec.captured_stdout # NOQA caperr = spec.captured_stderr # NOQA env = builtins.__xonsh__.env enc = env.get("XONSH_ENCODING") err = env.get("XONSH_ENCODING_ERRORS") if ON_WINDOWS: if self.p2cread != -1: self.p2cread = msvcrt.open_osfhandle(self.p2cread.Detach(), 0) if self.c2pwrite != -1: self.c2pwrite = msvcrt.open_osfhandle(self.c2pwrite.Detach(), 0) if self.errwrite != -1: self.errwrite = msvcrt.open_osfhandle(self.errwrite.Detach(), 0) # get stdin if self.stdin is None: sp_stdin = None elif self.p2cread != -1: sp_stdin = io.TextIOWrapper( io.open(self.p2cread, "rb", -1), encoding=enc, errors=err ) else: sp_stdin = sys.stdin # stdout if self.c2pwrite != -1: sp_stdout = io.TextIOWrapper( io.open(self.c2pwrite, "wb", -1), encoding=enc, errors=err ) else: sp_stdout = sys.stdout # stderr if self.errwrite == self.c2pwrite: sp_stderr = sp_stdout elif self.errwrite != -1: sp_stderr = io.TextIOWrapper( io.open(self.errwrite, "wb", -1), encoding=enc, errors=err ) else: sp_stderr = sys.stderr # run the function itself try: with STDOUT_DISPATCHER.register(sp_stdout), STDERR_DISPATCHER.register( sp_stderr ), redirect_stdout(STDOUT_DISPATCHER), redirect_stderr(STDERR_DISPATCHER): r = self.f(self.args, sp_stdin, sp_stdout, sp_stderr, spec, spec.stack) except SystemExit as e: r = e.code if isinstance(e.code, int) else int(bool(e.code)) except OSError: status = still_writable(self.c2pwrite) and still_writable(self.errwrite) if status: # stdout and stderr are still writable, so error must # come from function itself. print_exception() r = 1 else: # stdout and stderr are no longer writable, so error must # come from the fact that the next process in the pipeline # has closed the other side of the pipe. The function then # attempted to write to this side of the pipe anyway. This # is not truly an error and we should exit gracefully. r = 0 except Exception: print_exception() r = 1 safe_flush(sp_stdout) safe_flush(sp_stderr) self.returncode = parse_proxy_return(r, sp_stdout, sp_stderr) if not last_in_pipeline and not ON_WINDOWS: # mac requires us *not to* close the handles here while # windows requires us *to* close the handles here return # clean up # scopz: not sure why this is needed, but stdin cannot go here # and stdout & stderr must. handles = [self.stdout, self.stderr] for handle in handles: safe_fdclose(handle, cache=self._closed_handle_cache) def _wait_and_getattr(self, name): """make sure the instance has a certain attr, and return it.""" while not hasattr(self, name): time.sleep(1e-7) return getattr(self, name) def poll(self): """Check if the function has completed. Returns ------- None if the function is still executing, and the returncode otherwise """ return self.returncode def wait(self, timeout=None): """Waits for the process to finish and returns the return code.""" self.join() self._restore_sigint() return self.returncode # # SIGINT handler # def _signal_int(self, signum, frame): """Signal handler for SIGINT - Ctrl+C may have been pressed.""" # Check if we have already been interrupted. This should prevent # the possibility of infinite recursion. if self._interrupted: return self._interrupted = True # close file handles here to stop an processes piped to us. handles = ( self.p2cread, self.p2cwrite, self.c2pread, self.c2pwrite, self.errread, self.errwrite, ) for handle in handles: safe_fdclose(handle) if self.poll() is not None: self._restore_sigint(frame=frame) if on_main_thread(): signal.pthread_kill(threading.get_ident(), signal.SIGINT) def _restore_sigint(self, frame=None): old = self.old_int_handler if old is not None: if on_main_thread(): signal.signal(signal.SIGINT, old) self.old_int_handler = None if frame is not None: if old is not None and old is not self._signal_int: old(signal.SIGINT, frame) if self._interrupted: self.returncode = 1 # The code below (_get_devnull, _get_handles, and _make_inheritable) comes # from subprocess.py in the Python 3.4.2 Standard Library def _get_devnull(self): if not hasattr(self, "_devnull"): self._devnull = os.open(os.devnull, os.O_RDWR) return self._devnull if ON_WINDOWS: def _make_inheritable(self, handle): """Return a duplicate of handle, which is inheritable""" h = _winapi.DuplicateHandle( _winapi.GetCurrentProcess(), handle, _winapi.GetCurrentProcess(), 0, 1, _winapi.DUPLICATE_SAME_ACCESS, ) return Handle(h) def _get_handles(self, stdin, stdout, stderr): """Construct and return tuple with IO objects: p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite """ if stdin is None and stdout is None and stderr is None: return (-1, -1, -1, -1, -1, -1) p2cread, p2cwrite = -1, -1 c2pread, c2pwrite = -1, -1 errread, errwrite = -1, -1 if stdin is None: p2cread = _winapi.GetStdHandle(_winapi.STD_INPUT_HANDLE) if p2cread is None: p2cread, _ = _winapi.CreatePipe(None, 0) p2cread = Handle(p2cread) _winapi.CloseHandle(_) elif stdin == subprocess.PIPE: p2cread, p2cwrite = Handle(p2cread), Handle(p2cwrite) elif stdin == subprocess.DEVNULL: p2cread = msvcrt.get_osfhandle(self._get_devnull()) elif isinstance(stdin, int): p2cread = msvcrt.get_osfhandle(stdin) else: # Assuming file-like object p2cread = msvcrt.get_osfhandle(stdin.fileno()) p2cread = self._make_inheritable(p2cread) if stdout is None: c2pwrite = _winapi.GetStdHandle(_winapi.STD_OUTPUT_HANDLE) if c2pwrite is None: _, c2pwrite = _winapi.CreatePipe(None, 0) c2pwrite = Handle(c2pwrite) _winapi.CloseHandle(_) elif stdout == subprocess.PIPE: c2pread, c2pwrite = _winapi.CreatePipe(None, 0) c2pread, c2pwrite = Handle(c2pread), Handle(c2pwrite) elif stdout == subprocess.DEVNULL: c2pwrite = msvcrt.get_osfhandle(self._get_devnull()) elif isinstance(stdout, int): c2pwrite = msvcrt.get_osfhandle(stdout) else: # Assuming file-like object c2pwrite = msvcrt.get_osfhandle(stdout.fileno()) c2pwrite = self._make_inheritable(c2pwrite) if stderr is None: errwrite = _winapi.GetStdHandle(_winapi.STD_ERROR_HANDLE) if errwrite is None: _, errwrite = _winapi.CreatePipe(None, 0) errwrite = Handle(errwrite) _winapi.CloseHandle(_) elif stderr == subprocess.PIPE: errread, errwrite = _winapi.CreatePipe(None, 0) errread, errwrite = Handle(errread), Handle(errwrite) elif stderr == subprocess.STDOUT: errwrite = c2pwrite elif stderr == subprocess.DEVNULL: errwrite = msvcrt.get_osfhandle(self._get_devnull()) elif isinstance(stderr, int): errwrite = msvcrt.get_osfhandle(stderr) else: # Assuming file-like object errwrite = msvcrt.get_osfhandle(stderr.fileno()) errwrite = self._make_inheritable(errwrite) return (p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite) else: # POSIX versions def _get_handles(self, stdin, stdout, stderr): """Construct and return tuple with IO objects: p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite """ p2cread, p2cwrite = -1, -1 c2pread, c2pwrite = -1, -1 errread, errwrite = -1, -1 if stdin is None: pass elif stdin == subprocess.PIPE: p2cread, p2cwrite = os.pipe() elif stdin == subprocess.DEVNULL: p2cread = self._get_devnull() elif isinstance(stdin, int): p2cread = stdin else: # Assuming file-like object p2cread = stdin.fileno() if stdout is None: pass elif stdout == subprocess.PIPE: c2pread, c2pwrite = os.pipe() elif stdout == subprocess.DEVNULL: c2pwrite = self._get_devnull() elif isinstance(stdout, int): c2pwrite = stdout else: # Assuming file-like object c2pwrite = stdout.fileno() if stderr is None: pass elif stderr == subprocess.PIPE: errread, errwrite = os.pipe() elif stderr == subprocess.STDOUT: errwrite = c2pwrite elif stderr == subprocess.DEVNULL: errwrite = self._get_devnull() elif isinstance(stderr, int): errwrite = stderr else: # Assuming file-like object errwrite = stderr.fileno() return (p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite) # # Foreground Thread Process Proxies # class ProcProxy(object): """This is process proxy class that runs its alias functions on the same thread that it was called from, which is typically the main thread. This prevents the process from running on a background thread, but enables debugger and profiler tools (functions) be run on the same thread that they are attempting to debug. """ def __init__( self, f, args, stdin=None, stdout=None, stderr=None, universal_newlines=False, close_fds=False, env=None, ): self.orig_f = f self.f = partial_proxy(f) self.args = args self.pid = os.getpid() self.returncode = None self.stdin = stdin self.stdout = stdout self.stderr = stderr self.universal_newlines = universal_newlines self.close_fds = close_fds self.env = env def poll(self): """Check if the function has completed via the returncode or None. """ return self.returncode def wait(self, timeout=None): """Runs the function and returns the result. Timeout argument only present for API compatibility. """ if self.f is None: return 0 env = builtins.__xonsh__.env enc = env.get("XONSH_ENCODING") err = env.get("XONSH_ENCODING_ERRORS") spec = self._wait_and_getattr("spec") # set file handles if self.stdin is None: stdin = None else: if isinstance(self.stdin, int): inbuf = io.open(self.stdin, "rb", -1) else: inbuf = self.stdin stdin = io.TextIOWrapper(inbuf, encoding=enc, errors=err) stdout = self._pick_buf(self.stdout, sys.stdout, enc, err) stderr = self._pick_buf(self.stderr, sys.stderr, enc, err) # run the actual function try: r = self.f(self.args, stdin, stdout, stderr, spec, spec.stack) except Exception: print_exception() r = 1 self.returncode = parse_proxy_return(r, stdout, stderr) safe_flush(stdout) safe_flush(stderr) return self.returncode @staticmethod def _pick_buf(handle, sysbuf, enc, err): if handle is None or handle is sysbuf: buf = sysbuf elif isinstance(handle, int): if handle < 3: buf = sysbuf else: buf = io.TextIOWrapper( io.open(handle, "wb", -1), encoding=enc, errors=err ) elif hasattr(handle, "encoding"): # must be a text stream, no need to wrap. buf = handle else: # must be a binary stream, should wrap it. buf = io.TextIOWrapper(handle, encoding=enc, errors=err) return buf def _wait_and_getattr(self, name): """make sure the instance has a certain attr, and return it.""" while not hasattr(self, name): time.sleep(1e-7) return getattr(self, name) @lazyobject def SIGNAL_MESSAGES(): sm = { signal.SIGABRT: "Aborted", signal.SIGFPE: "Floating point exception", signal.SIGILL: "Illegal instructions", signal.SIGTERM: "Terminated", signal.SIGSEGV: "Segmentation fault", } if ON_POSIX: sm.update( {signal.SIGQUIT: "Quit", signal.SIGHUP: "Hangup", signal.SIGKILL: "Killed"} ) return sm def safe_readlines(handle, hint=-1): """Attempts to read lines without throwing an error.""" try: lines = handle.readlines(hint) except OSError: lines = [] return lines def safe_readable(handle): """Attempts to find if the handle is readable without throwing an error.""" try: status = handle.readable() except (OSError, ValueError): status = False return status def update_fg_process_group(pipeline_group, background): if background: return False if not ON_POSIX: return False env = builtins.__xonsh__.env if not env.get("XONSH_INTERACTIVE"): return False return give_terminal_to(pipeline_group) class CommandPipeline: """Represents a subprocess-mode command pipeline.""" attrnames = ( "stdin", "stdout", "stderr", "pid", "returncode", "args", "alias", "stdin_redirect", "stdout_redirect", "stderr_redirect", "timestamps", "executed_cmd", "input", "output", "errors", ) nonblocking = (io.BytesIO, NonBlockingFDReader, ConsoleParallelReader) def __init__(self, specs): """ Parameters ---------- specs : list of SubprocSpec Process specifications Attributes ---------- spec : SubprocSpec The last specification in specs proc : Popen-like The process in procs ended : bool Boolean for if the command has stopped executing. input : str A string of the standard input. output : str A string of the standard output. errors : str A string of the standard error. lines : list of str The output lines starttime : floats or None Pipeline start timestamp. """ self.starttime = None self.ended = False self.procs = [] self.specs = specs self.spec = specs[-1] self.captured = specs[-1].captured self.input = self._output = self.errors = self.endtime = None self._closed_handle_cache = {} self.lines = [] self._stderr_prefix = self._stderr_postfix = None self.term_pgid = None background = self.spec.background pipeline_group = None for spec in specs: if self.starttime is None: self.starttime = time.time() try: proc = spec.run(pipeline_group=pipeline_group) except Exception: print_exception() self._return_terminal() self.proc = None return if ( proc.pid and pipeline_group is None and not spec.is_proxy and self.captured != "object" ): pipeline_group = proc.pid if update_fg_process_group(pipeline_group, background): self.term_pgid = pipeline_group self.procs.append(proc) self.proc = self.procs[-1] def __repr__(self): s = self.__class__.__name__ + "(" s += ", ".join(a + "=" + str(getattr(self, a)) for a in self.attrnames) s += ")" return s def __bool__(self): return self.returncode == 0 def __len__(self): return len(self.procs) def __iter__(self): """Iterates through stdout and returns the lines, converting to strings and universal newlines if needed. """ if self.ended: yield from iter(self.lines) else: yield from self.tee_stdout() def iterraw(self): """Iterates through the last stdout, and returns the lines exactly as found. """ # get appropriate handles spec = self.spec proc = self.proc if proc is None: return timeout = builtins.__xonsh__.env.get("XONSH_PROC_FREQUENCY") # get the correct stdout stdout = proc.stdout if ( stdout is None or spec.stdout is None or not safe_readable(stdout) ) and spec.captured_stdout is not None: stdout = spec.captured_stdout if hasattr(stdout, "buffer"): stdout = stdout.buffer if stdout is not None and not isinstance(stdout, self.nonblocking): stdout = NonBlockingFDReader(stdout.fileno(), timeout=timeout) if ( not stdout or self.captured == "stdout" or not safe_readable(stdout) or not spec.threadable ): # we get here if the process is not threadable or the # class is the real Popen PrevProcCloser(pipeline=self) task = wait_for_active_job() if task is None or task["status"] != "stopped": proc.wait() self._endtime() if self.captured == "object": self.end(tee_output=False) elif self.captured == "hiddenobject" and stdout: b = stdout.read() lines = b.splitlines(keepends=True) yield from lines self.end(tee_output=False) elif self.captured == "stdout": b = stdout.read() s = self._decode_uninew(b, universal_newlines=True) self.lines = s.splitlines(keepends=True) return # get the correct stderr stderr = proc.stderr if ( stderr is None or spec.stderr is None or not safe_readable(stderr) ) and spec.captured_stderr is not None: stderr = spec.captured_stderr if hasattr(stderr, "buffer"): stderr = stderr.buffer if stderr is not None and not isinstance(stderr, self.nonblocking): stderr = NonBlockingFDReader(stderr.fileno(), timeout=timeout) # read from process while it is running check_prev_done = len(self.procs) == 1 prev_end_time = None i = j = cnt = 1 while proc.poll() is None: if getattr(proc, "suspended", False): return elif getattr(proc, "in_alt_mode", False): time.sleep(0.1) # probably not leaving any time soon continue elif not check_prev_done: # In the case of pipelines with more than one command # we should give the commands a little time # to start up fully. This is particularly true for # GNU Parallel, which has a long startup time. pass elif self._prev_procs_done(): self._close_prev_procs() proc.prevs_are_closed = True break stdout_lines = safe_readlines(stdout, 1024) i = len(stdout_lines) if i != 0: yield from stdout_lines stderr_lines = safe_readlines(stderr, 1024) j = len(stderr_lines) if j != 0: self.stream_stderr(stderr_lines) if not check_prev_done: # if we are piping... if stdout_lines or stderr_lines: # see if we have some output. check_prev_done = True elif prev_end_time is None: # or see if we already know that the next-to-last # proc in the pipeline has ended. if self._prev_procs_done(): # if it has, record the time prev_end_time = time.time() elif time.time() - prev_end_time >= 0.1: # if we still don't have any output, even though the # next-to-last proc has finished, wait a bit to make # sure we have fully started up, etc. check_prev_done = True # this is for CPU usage if i + j == 0: cnt = min(cnt + 1, 1000) else: cnt = 1 time.sleep(timeout * cnt) # read from process now that it is over yield from safe_readlines(stdout) self.stream_stderr(safe_readlines(stderr)) proc.wait() self._endtime() yield from safe_readlines(stdout) self.stream_stderr(safe_readlines(stderr)) if self.captured == "object": self.end(tee_output=False) def itercheck(self): """Iterates through the command lines and throws an error if the returncode is non-zero. """ yield from self if self.returncode: # I included self, as providing access to stderr and other details # useful when instance isn't assigned to a variable in the shell. raise XonshCalledProcessError( self.returncode, self.executed_cmd, self.stdout, self.stderr, self ) def tee_stdout(self): """Writes the process stdout to the output variable, line-by-line, and yields each line. This may optionally accept lines (in bytes) to iterate over, in which case it does not call iterraw(). """ env = builtins.__xonsh__.env enc = env.get("XONSH_ENCODING") err = env.get("XONSH_ENCODING_ERRORS") lines = self.lines stream = self.captured not in STDOUT_CAPTURE_KINDS if stream and not self.spec.stdout: stream = False stdout_has_buffer = hasattr(sys.stdout, "buffer") nl = b"\n" cr = b"\r" crnl = b"\r\n" for line in self.iterraw(): # write to stdout line ASAP, if needed if stream: if stdout_has_buffer: sys.stdout.buffer.write(line) else: sys.stdout.write(line.decode(encoding=enc, errors=err)) sys.stdout.flush() # do some munging of the line before we return it if line.endswith(crnl): line = line[:-2] + nl elif line.endswith(cr): line = line[:-1] + nl line = RE_HIDE_ESCAPE.sub(b"", line) line = line.decode(encoding=enc, errors=err) # tee it up! lines.append(line) yield line def stream_stderr(self, lines): """Streams lines to sys.stderr and the errors attribute.""" if not lines: return env = builtins.__xonsh__.env enc = env.get("XONSH_ENCODING") err = env.get("XONSH_ENCODING_ERRORS") b = b"".join(lines) if self.stderr_prefix: b = self.stderr_prefix + b if self.stderr_postfix: b += self.stderr_postfix stderr_has_buffer = hasattr(sys.stderr, "buffer") # write bytes to std stream if stderr_has_buffer: sys.stderr.buffer.write(b) else: sys.stderr.write(b.decode(encoding=enc, errors=err)) sys.stderr.flush() # do some munging of the line before we save it to the attr b = b.replace(b"\r\n", b"\n").replace(b"\r", b"\n") b = RE_HIDE_ESCAPE.sub(b"", b) env = builtins.__xonsh__.env s = b.decode( encoding=env.get("XONSH_ENCODING"), errors=env.get("XONSH_ENCODING_ERRORS") ) # set the errors if self.errors is None: self.errors = s else: self.errors += s def _decode_uninew(self, b, universal_newlines=None): """Decode bytes into a str and apply universal newlines as needed.""" if not b: return "" if isinstance(b, bytes): env = builtins.__xonsh__.env s = b.decode( encoding=env.get("XONSH_ENCODING"), errors=env.get("XONSH_ENCODING_ERRORS"), ) else: s = b if universal_newlines or self.spec.universal_newlines: s = s.replace("\r\n", "\n").replace("\r", "\n") return s # # Ending methods # def end(self, tee_output=True): """ End the pipeline, return the controlling terminal if needed. Main things done in self._end(). """ if self.ended: return self._end(tee_output=tee_output) self._return_terminal() def _end(self, tee_output): """Waits for the command to complete and then runs any closing and cleanup procedures that need to be run. """ if tee_output: for _ in self.tee_stdout(): pass self._endtime() # since we are driven by getting output, input may not be available # until the command has completed. self._set_input() self._close_prev_procs() self._close_proc() self._check_signal() self._apply_to_history() self.ended = True self._raise_subproc_error() def _return_terminal(self): if ON_WINDOWS or not ON_POSIX: return pgid = os.getpgid(0) if self.term_pgid is None or pgid == self.term_pgid: return if give_terminal_to(pgid): # if gave term succeed self.term_pgid = pgid if builtins.__xonsh__.shell is not None: # restoring sanity could probably be called whenever we return # control to the shell. But it only seems to matter after a # ^Z event. This *has* to be called after we give the terminal # back to the shell. builtins.__xonsh__.shell.shell.restore_tty_sanity() def resume(self, job, tee_output=True): self.ended = False if give_terminal_to(job["pgrp"]): self.term_pgid = job["pgrp"] _continue(job) self.end(tee_output=tee_output) def _endtime(self): """Sets the closing timestamp if it hasn't been already.""" if self.endtime is None: self.endtime = time.time() def _safe_close(self, handle): safe_fdclose(handle, cache=self._closed_handle_cache) def _prev_procs_done(self): """Boolean for if all previous processes have completed. If there is only a single process in the pipeline, this returns False. """ any_running = False for s, p in zip(self.specs[:-1], self.procs[:-1]): if p.poll() is None: any_running = True continue self._safe_close(s.stdin) self._safe_close(s.stdout) self._safe_close(s.stderr) if p is None: continue self._safe_close(p.stdin) self._safe_close(p.stdout) self._safe_close(p.stderr) return False if any_running else (len(self) > 1) def _close_prev_procs(self): """Closes all but the last proc's stdout.""" for s, p in zip(self.specs[:-1], self.procs[:-1]): self._safe_close(s.stdin) self._safe_close(s.stdout) self._safe_close(s.stderr) if p is None: continue self._safe_close(p.stdin) self._safe_close(p.stdout) self._safe_close(p.stderr) def _close_proc(self): """Closes last proc's stdout.""" s = self.spec p = self.proc self._safe_close(s.stdin) self._safe_close(s.stdout) self._safe_close(s.stderr) self._safe_close(s.captured_stdout) self._safe_close(s.captured_stderr) if p is None: return self._safe_close(p.stdin) self._safe_close(p.stdout) self._safe_close(p.stderr) def _set_input(self): """Sets the input variable.""" if self.proc is None: return stdin = self.proc.stdin if ( stdin is None or isinstance(stdin, int) or stdin.closed or not stdin.seekable() or not safe_readable(stdin) ): input = b"" else: stdin.seek(0) input = stdin.read() self.input = self._decode_uninew(input) def _check_signal(self): """Checks if a signal was received and issues a message.""" proc_signal = getattr(self.proc, "signal", None) if proc_signal is None: return sig, core = proc_signal sig_str = SIGNAL_MESSAGES.get(sig) if sig_str: if core: sig_str += " (core dumped)" print(sig_str, file=sys.stderr) if self.errors is not None: self.errors += sig_str + "\n" def _apply_to_history(self): """Applies the results to the current history object.""" hist = builtins.__xonsh__.history if hist is not None: hist.last_cmd_rtn = 1 if self.proc is None else self.proc.returncode def _raise_subproc_error(self): """Raises a subprocess error, if we are supposed to.""" spec = self.spec rtn = self.returncode if ( rtn is not None and rtn > 0 and builtins.__xonsh__.env.get("RAISE_SUBPROC_ERROR") ): try: raise subprocess.CalledProcessError(rtn, spec.args, output=self.output) finally: # this is need to get a working terminal in interactive mode self._return_terminal() # # Properties # @property def stdin(self): """Process stdin.""" return self.proc.stdin @property def stdout(self): """Process stdout.""" return self.proc.stdout @property def stderr(self): """Process stderr.""" return self.proc.stderr @property def inp(self): """Creates normalized input string from args.""" return " ".join(self.args) @property def output(self): """Non-blocking, lazy access to output""" if self.ended: if self._output is None: self._output = "".join(self.lines) return self._output else: return "".join(self.lines) @property def out(self): """Output value as a str.""" self.end() return self.output @property def err(self): """Error messages as a string.""" self.end() return self.errors @property def pid(self): """Process identifier.""" return self.proc.pid @property def returncode(self): """Process return code, waits until command is completed.""" self.end() if self.proc is None: return 1 return self.proc.returncode rtn = returncode @property def args(self): """Arguments to the process.""" return self.spec.args @property def rtn(self): """Alias to return code.""" return self.returncode @property def alias(self): """Alias the process used.""" return self.spec.alias @property def stdin_redirect(self): """Redirection used for stdin.""" stdin = self.spec.stdin name = getattr(stdin, "name", "") mode = getattr(stdin, "mode", "r") return [name, mode] @property def stdout_redirect(self): """Redirection used for stdout.""" stdout = self.spec.stdout name = getattr(stdout, "name", "") mode = getattr(stdout, "mode", "a") return [name, mode] @property def stderr_redirect(self): """Redirection used for stderr.""" stderr = self.spec.stderr name = getattr(stderr, "name", "") mode = getattr(stderr, "mode", "r") return [name, mode] @property def timestamps(self): """The start and end time stamps.""" return [self.starttime, self.endtime] @property def executed_cmd(self): """The resolve and executed command.""" return self.spec.cmd @property def stderr_prefix(self): """Prefix to print in front of stderr, as bytes.""" p = self._stderr_prefix if p is None: env = builtins.__xonsh__.env t = env.get("XONSH_STDERR_PREFIX") s = format_std_prepost(t, env=env) p = s.encode( encoding=env.get("XONSH_ENCODING"), errors=env.get("XONSH_ENCODING_ERRORS"), ) self._stderr_prefix = p return p @property def stderr_postfix(self): """Postfix to print after stderr, as bytes.""" p = self._stderr_postfix if p is None: env = builtins.__xonsh__.env t = env.get("XONSH_STDERR_POSTFIX") s = format_std_prepost(t, env=env) p = s.encode( encoding=env.get("XONSH_ENCODING"), errors=env.get("XONSH_ENCODING_ERRORS"), ) self._stderr_postfix = p return p class HiddenCommandPipeline(CommandPipeline): def __repr__(self): return "" def pause_call_resume(p, f, *args, **kwargs): """For a process p, this will call a function f with the remaining args and and kwargs. If the process cannot accept signals, the function will be called. Parameters ---------- p : Popen object or similar f : callable args : remaining arguments kwargs : keyword arguments """ can_send_signal = ( hasattr(p, "send_signal") and ON_POSIX and not ON_MSYS and not ON_CYGWIN ) if can_send_signal: try: p.send_signal(signal.SIGSTOP) except PermissionError: pass try: f(*args, **kwargs) except Exception: pass if can_send_signal: p.send_signal(signal.SIGCONT) class PrevProcCloser(threading.Thread): """Previous process closer thread for pipelines whose last command is itself unthreadable. This makes sure that the pipeline is driven forward and does not deadlock. """ def __init__(self, pipeline): """ Parameters ---------- pipeline : CommandPipeline The pipeline whose prev procs we should close. """ self.pipeline = pipeline super().__init__() self.daemon = True self.start() def run(self): """Runs the closing algorithm.""" pipeline = self.pipeline check_prev_done = len(pipeline.procs) == 1 if check_prev_done: return proc = pipeline.proc prev_end_time = None timeout = builtins.__xonsh__.env.get("XONSH_PROC_FREQUENCY") sleeptime = min(timeout * 1000, 0.1) while proc.poll() is None: if not check_prev_done: # In the case of pipelines with more than one command # we should give the commands a little time # to start up fully. This is particularly true for # GNU Parallel, which has a long startup time. pass elif pipeline._prev_procs_done(): pipeline._close_prev_procs() proc.prevs_are_closed = True break if not check_prev_done: # if we are piping... if prev_end_time is None: # or see if we already know that the next-to-last # proc in the pipeline has ended. if pipeline._prev_procs_done(): # if it has, record the time prev_end_time = time.time() elif time.time() - prev_end_time >= 0.1: # if we still don't have any output, even though the # next-to-last proc has finished, wait a bit to make # sure we have fully started up, etc. check_prev_done = True # this is for CPU usage time.sleep(sleeptime) ================================================ FILE: xonsh/prompt/__init__.py ================================================ # amalgamate exclude import os as _os if _os.getenv("XONSH_DEBUG", ""): pass else: import sys as _sys try: from xonsh.prompt import __amalgam__ cwd = __amalgam__ _sys.modules["xonsh.prompt.cwd"] = __amalgam__ env = __amalgam__ _sys.modules["xonsh.prompt.env"] = __amalgam__ gitstatus = __amalgam__ _sys.modules["xonsh.prompt.gitstatus"] = __amalgam__ job = __amalgam__ _sys.modules["xonsh.prompt.job"] = __amalgam__ vc = __amalgam__ _sys.modules["xonsh.prompt.vc"] = __amalgam__ base = __amalgam__ _sys.modules["xonsh.prompt.base"] = __amalgam__ del __amalgam__ except ImportError: pass del _sys del _os # amalgamate end ================================================ FILE: xonsh/prompt/base.py ================================================ # -*- coding: utf-8 -*- """Base prompt, provides PROMPT_FIELDS and prompt related functions""" import builtins import itertools import os import re import socket import sys import xonsh.lazyasd as xl import xonsh.tools as xt import xonsh.platform as xp from xonsh.prompt.cwd import ( _collapsed_pwd, _replace_home_cwd, _dynamically_collapsed_pwd, ) from xonsh.prompt.job import _current_job from xonsh.prompt.env import env_name, vte_new_tab_cwd from xonsh.prompt.vc import current_branch, branch_color, branch_bg_color from xonsh.prompt.gitstatus import gitstatus_prompt @xt.lazyobject def DEFAULT_PROMPT(): return default_prompt() class PromptFormatter: """Class that holds all the related prompt formatting methods, uses the ``PROMPT_FIELDS`` envvar (no color formatting). """ def __init__(self): self.cache = {} def __call__(self, template=DEFAULT_PROMPT, fields=None): """Formats a xonsh prompt template string.""" if fields is None: self.fields = builtins.__xonsh__.env.get("PROMPT_FIELDS", PROMPT_FIELDS) else: self.fields = fields try: prompt = self._format_prompt(template=template) except Exception: return _failover_template_format(template) # keep cache only during building prompt self.cache.clear() return prompt def _format_prompt(self, template=DEFAULT_PROMPT): template = template() if callable(template) else template toks = [] for literal, field, spec, conv in xt.FORMATTER.parse(template): toks.append(literal) entry = self._format_field(field, spec, conv) if entry is not None: toks.append(entry) return "".join(toks) def _format_field(self, field, spec, conv): if field is None: return elif field.startswith("$"): val = builtins.__xonsh__.env[field[1:]] return _format_value(val, spec, conv) elif field in self.fields: val = self._get_field_value(field) return _format_value(val, spec, conv) else: # color or unknown field, return as is return "{" + field + "}" def _get_field_value(self, field): field_value = self.fields[field] if field_value in self.cache: return self.cache[field_value] try: value = field_value() if callable(field_value) else field_value self.cache[field_value] = value except Exception: print("prompt: error: on field {!r}" "".format(field), file=sys.stderr) xt.print_exception() value = "(ERROR:{})".format(field) return value @xl.lazyobject def PROMPT_FIELDS(): return dict( user=xp.os_environ.get("USERNAME" if xp.ON_WINDOWS else "USER", ""), prompt_end="#" if xt.is_superuser() else "$", hostname=socket.gethostname().split(".", 1)[0], cwd=_dynamically_collapsed_pwd, cwd_dir=lambda: os.path.dirname(_replace_home_cwd()), cwd_base=lambda: os.path.basename(_replace_home_cwd()), short_cwd=_collapsed_pwd, curr_branch=current_branch, branch_color=branch_color, branch_bg_color=branch_bg_color, current_job=_current_job, env_name=env_name, env_prefix="(", env_postfix=") ", vte_new_tab_cwd=vte_new_tab_cwd, gitstatus=gitstatus_prompt, ) def default_prompt(): return ('{RED}{user} ' '{BOLD_WHITE}at ' '{YELLOW}{hostname} ' '{BOLD_WHITE}in ' '{GREEN}{cwd} ' '{BOLD_WHITE}on ' '{branch_color}{curr_branch} ' '{BOLD_WHITE}\n' '${NO_COLOR} ') def _failover_template_format(template): if callable(template): try: # Exceptions raises from function of producing $PROMPT # in user's xonshrc should not crash xonsh return template() except Exception: xt.print_exception() return "$ " return template @xt.lazyobject def RE_HIDDEN(): return re.compile("\001.*?\002") def multiline_prompt(curr=""): """Returns the filler text for the prompt in multiline scenarios.""" line = curr.rsplit("\n", 1)[1] if "\n" in curr else curr line = RE_HIDDEN.sub("", line) # gets rid of colors # most prompts end in whitespace, head is the part before that. head = line.rstrip() headlen = len(head) # tail is the trailing whitespace tail = line if headlen == 0 else line.rsplit(head[-1], 1)[1] # now to construct the actual string dots = builtins.__xonsh__.env.get("MULTILINE_PROMPT") dots = dots() if callable(dots) else dots if dots is None or len(dots) == 0: return "" tokstr = xt.format_color(dots, hide=True) baselen = 0 basetoks = [] for x in tokstr.split("\001"): pre, sep, post = x.partition("\002") if len(sep) == 0: basetoks.append(("", pre)) baselen += len(pre) else: basetoks.append(("\001" + pre + "\002", post)) baselen += len(post) if baselen == 0: return xt.format_color("{NO_COLOR}" + tail, hide=True) toks = basetoks * (headlen // baselen) n = headlen % baselen count = 0 for tok in basetoks: slen = len(tok[1]) newcount = slen + count if slen == 0: continue elif newcount <= n: toks.append(tok) else: toks.append((tok[0], tok[1][: n - count])) count = newcount if n <= count: break toks.append((xt.format_color("{NO_COLOR}", hide=True), tail)) rtn = "".join(itertools.chain.from_iterable(toks)) return rtn def is_template_string(template, PROMPT_FIELDS=None): """Returns whether or not the string is a valid template.""" template = template() if callable(template) else template try: included_names = set(i[1] for i in xt.FORMATTER.parse(template)) except ValueError: return False included_names.discard(None) if PROMPT_FIELDS is None: fmtter = builtins.__xonsh__.env.get("PROMPT_FIELDS", PROMPT_FIELDS) else: fmtter = PROMPT_FIELDS known_names = set(fmtter.keys()) return included_names <= known_names def _format_value(val, spec, conv): """Formats a value from a template string {val!conv:spec}. The spec is applied as a format string itself, but if the value is None, the result will be empty. The purpose of this is to allow optional parts in a prompt string. For example, if the prompt contains '{current_job:{} | }', and 'current_job' returns 'sleep', the result is 'sleep | ', and if 'current_job' returns None, the result is ''. """ if val is None: return "" val = xt.FORMATTER.convert_field(val, conv) if spec: val = xt.FORMATTER.format(spec, val) if not isinstance(val, str): val = str(val) return val ================================================ FILE: xonsh/prompt/cwd.py ================================================ # -*- coding: utf-8 -*- """CWD related prompt formatter""" import os import shutil import builtins import xonsh.tools as xt import xonsh.platform as xp def _replace_home(x): if xp.ON_WINDOWS: home = ( builtins.__xonsh__.env["HOMEDRIVE"] + builtins.__xonsh__.env["HOMEPATH"][0] ) if x.startswith(home): x = x.replace(home, "~", 1) if builtins.__xonsh__.env.get("FORCE_POSIX_PATHS"): x = x.replace(os.sep, os.altsep) return x else: home = builtins.__xonsh__.env["HOME"] if x.startswith(home): x = x.replace(home, "~", 1) return x def _replace_home_cwd(): return _replace_home(builtins.__xonsh__.env["PWD"]) def _collapsed_pwd(): sep = xt.get_sep() pwd = _replace_home_cwd().split(sep) l = len(pwd) leader = sep if l > 0 and len(pwd[0]) == 0 else "" base = [ i[0] if ix != l - 1 and i[0] != "." else i[0:2] if ix != l - 1 else i for ix, i in enumerate(pwd) if len(i) > 0 ] return leader + sep.join(base) def _dynamically_collapsed_pwd(): """Return the compact current working directory. It respects the environment variable DYNAMIC_CWD_WIDTH. """ original_path = _replace_home_cwd() target_width, units = builtins.__xonsh__.env["DYNAMIC_CWD_WIDTH"] elision_char = builtins.__xonsh__.env["DYNAMIC_CWD_ELISION_CHAR"] if target_width == float("inf"): return original_path if units == "%": cols, _ = shutil.get_terminal_size() target_width = (cols * target_width) // 100 sep = xt.get_sep() pwd = original_path.split(sep) last = pwd.pop() remaining_space = target_width - len(last) # Reserve space for separators remaining_space_for_text = remaining_space - len(pwd) parts = [] for i in range(len(pwd)): part = pwd[i] part_len = int( min(len(part), max(1, remaining_space_for_text // (len(pwd) - i))) ) remaining_space_for_text -= part_len if len(part) > part_len: reduced_part = part[0 : part_len - len(elision_char)] + elision_char parts.append(reduced_part) else: parts.append(part) parts.append(last) full = sep.join(parts) truncature_char = elision_char if elision_char else "..." # If even if displaying one letter per dir we are too long if len(full) > target_width: # We truncate the left most part full = truncature_char + full[int(-target_width) + len(truncature_char) :] # if there is not even a single separator we still # want to display at least the beginning of the directory if full.find(sep) == -1: full = (truncature_char + sep + last)[ 0 : int(target_width) - len(truncature_char) ] + truncature_char return full ================================================ FILE: xonsh/prompt/env.py ================================================ # -*- coding: utf-8 -*- """Prompt formatter for virtualenv and others""" import os import builtins import xonsh.platform as xp def find_env_name(): """Finds the current environment name from $VIRTUAL_ENV or $CONDA_DEFAULT_ENV if that is set. """ env_path = builtins.__xonsh__.env.get("VIRTUAL_ENV", "") if len(env_path) == 0 and xp.ON_ANACONDA: env_path = builtins.__xonsh__.env.get("CONDA_DEFAULT_ENV", "") env_name = os.path.basename(env_path) return env_name def env_name(): """Returns the current env_name if it non-empty, surrounded by the ``{env_prefix}`` and ``{env_postfix}`` fields. """ env_name = find_env_name() if ( builtins.__xonsh__.env.get("VIRTUAL_ENV_DISABLE_PROMPT") or not env_name ): # env name prompt printing disabled, or no environment; just return return venv_prompt = builtins.__xonsh__.env.get("VIRTUAL_ENV_PROMPT") if venv_prompt is not None: return venv_prompt else: pf = builtins.__xonsh__.shell.prompt_formatter pre = pf._get_field_value("env_prefix") post = pf._get_field_value("env_postfix") return pre + env_name + post def vte_new_tab_cwd(): """This prints an escape sequence that tells VTE terminals the hostname and pwd. This should not be needed in most cases, but sometimes is for certain Linux terminals that do not read the PWD from the environment on startup. Note that this does not return a string, it simply prints and flushes the escape sequence to stdout directly. """ env = builtins.__xonsh__.env t = "\033]7;file://{}{}\007" s = t.format(env.get("HOSTNAME"), env.get("PWD")) print(s, end="", flush=True) ================================================ FILE: xonsh/prompt/gitstatus.py ================================================ # -*- coding: utf-8 -*- """Informative git status prompt formatter""" import builtins import collections import os import subprocess import xonsh.lazyasd as xl GitStatus = collections.namedtuple( "GitStatus", [ "branch", "num_ahead", "num_behind", "untracked", "changed", "conflicts", "staged", "stashed", "operations", ], ) def _check_output(*args, **kwargs): kwargs.update( dict( env=builtins.__xonsh__.env.detype(), stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, universal_newlines=True, ) ) timeout = builtins.__xonsh__.env["VC_BRANCH_TIMEOUT"] # See https://docs.python.org/3/library/subprocess.html#subprocess.Popen.communicate with subprocess.Popen(*args, **kwargs) as proc: try: out, err = proc.communicate(timeout=timeout) if proc.returncode != 0: raise subprocess.CalledProcessError( proc.returncode, proc.args, output=out, stderr=err ) # note err will always be empty as we redirect stderr to DEVNULL abvoe return out except subprocess.TimeoutExpired: # We use `.terminate()` (SIGTERM) instead of `.kill()` (SIGKILL) here # because otherwise we guarantee that a `.git/index.lock` file will be # left over, and subsequent git operations will fail. # We don't want that. # As a result, we must rely on git to exit properly on SIGTERM. proc.terminate() # We wait() to ensure that git has finished before the next # `gitstatus` prompt is rendered (otherwise `index.lock` still exists, # and it will fail). # We don't technically have to call `wait()` here as the # `with subprocess.Popen()` context manager above would do that # for us, but we do it to be explicit that waiting is being done. proc.wait() # we ignore what git says after we sent it SIGTERM raise @xl.lazyobject def _DEFS(): DEFS = { "HASH": ":", "BRANCH": "{CYAN}", "OPERATION": "{CYAN}", "STAGED": "{RED}●", "CONFLICTS": "{RED}×", "CHANGED": "{BLUE}+", "UNTRACKED": "…", "STASHED": "⚑", "CLEAN": "{BOLD_GREEN}✓", "AHEAD": "↑·", "BEHIND": "↓·", } return DEFS def _get_def(key): def_ = builtins.__xonsh__.env.get("XONSH_GITSTATUS_" + key) return def_ if def_ is not None else _DEFS[key] def _get_tag_or_hash(): tag_or_hash = _check_output(["git", "describe", "--always"]).strip() hash_ = _check_output(["git", "rev-parse", "--short", "HEAD"]).strip() have_tag_name = tag_or_hash != hash_ return tag_or_hash if have_tag_name else _get_def("HASH") + hash_ def _get_stash(gitdir): try: with open(os.path.join(gitdir, "logs/refs/stash")) as f: return sum(1 for _ in f) except IOError: return 0 def _gitoperation(gitdir): files = ( ("rebase-merge", "REBASE"), ("rebase-apply", "AM/REBASE"), ("MERGE_HEAD", "MERGING"), ("CHERRY_PICK_HEAD", "CHERRY-PICKING"), ("REVERT_HEAD", "REVERTING"), ("BISECT_LOG", "BISECTING"), ) return [f[1] for f in files if os.path.exists(os.path.join(gitdir, f[0]))] def gitstatus(): """Return namedtuple with fields: branch name, number of ahead commit, number of behind commit, untracked number, changed number, conflicts number, staged number, stashed number, operation.""" status = _check_output(["git", "status", "--porcelain", "--branch"]) branch = "" num_ahead, num_behind = 0, 0 untracked, changed, conflicts, staged = 0, 0, 0, 0 for line in status.splitlines(): if line.startswith("##"): line = line[2:].strip() if "Initial commit on" in line: branch = line.split()[-1] elif "no branch" in line: branch = _get_tag_or_hash() elif "..." not in line: branch = line else: branch, rest = line.split("...") if " " in rest: divergence = rest.split(" ", 1)[-1] divergence = divergence.strip("[]") for div in divergence.split(", "): if "ahead" in div: num_ahead = int(div[len("ahead ") :].strip()) elif "behind" in div: num_behind = int(div[len("behind ") :].strip()) elif line.startswith("??"): untracked += 1 else: if len(line) > 1 and line[1] == "M": changed += 1 if len(line) > 0 and line[0] == "U": conflicts += 1 elif len(line) > 0 and line[0] != " ": staged += 1 gitdir = _check_output(["git", "rev-parse", "--git-dir"]).strip() stashed = _get_stash(gitdir) operations = _gitoperation(gitdir) return GitStatus( branch, num_ahead, num_behind, untracked, changed, conflicts, staged, stashed, operations, ) def gitstatus_prompt(): """Return str `BRANCH|OPERATOR|numbers`""" try: s = gitstatus() except subprocess.SubprocessError: return None ret = _get_def("BRANCH") + s.branch if s.num_ahead > 0: ret += _get_def("AHEAD") + str(s.num_ahead) if s.num_behind > 0: ret += _get_def("BEHIND") + str(s.num_behind) if s.operations: ret += _get_def("OPERATION") + "|" + "|".join(s.operations) ret += "|" if s.staged > 0: ret += _get_def("STAGED") + str(s.staged) + "{NO_COLOR}" if s.conflicts > 0: ret += _get_def("CONFLICTS") + str(s.conflicts) + "{NO_COLOR}" if s.changed > 0: ret += _get_def("CHANGED") + str(s.changed) + "{NO_COLOR}" if s.untracked > 0: ret += _get_def("UNTRACKED") + str(s.untracked) + "{NO_COLOR}" if s.stashed > 0: ret += _get_def("STASHED") + str(s.stashed) + "{NO_COLOR}" if s.staged + s.conflicts + s.changed + s.untracked + s.stashed == 0: ret += _get_def("CLEAN") + "{NO_COLOR}" ret += "{NO_COLOR}" return ret ================================================ FILE: xonsh/prompt/job.py ================================================ # -*- coding: utf-8 -*- """Prompt formatter for current jobs""" import xonsh.jobs as xj def _current_job(): j = xj.get_next_task() if j is not None: if not j["bg"]: cmd = j["cmds"][-1] s = cmd[0] if s == "sudo" and len(cmd) > 1: s = cmd[1] return s ================================================ FILE: xonsh/prompt/vc.py ================================================ # -*- coding: utf-8 -*- """Prompt formatter for simple version control branches""" # pylint:disable=no-member, invalid-name import os import sys import queue import builtins import threading import subprocess import xonsh.tools as xt def _get_git_branch(q): denv = builtins.__xonsh__.env.detype() try: branches = xt.decode_bytes( subprocess.check_output( ["git", "branch"], env=denv, stderr=subprocess.DEVNULL ) ).splitlines() except (subprocess.CalledProcessError, OSError, FileNotFoundError): q.put(None) else: for branch in branches: if not branch.startswith("* "): continue elif branch.endswith(")"): branch = branch.split()[-1][:-1] else: branch = branch.split()[-1] q.put(branch) break else: q.put(None) def get_git_branch(): """Attempts to find the current git branch. If this could not be determined (timeout, not in a git repo, etc.) then this returns None. """ branch = None timeout = builtins.__xonsh__.env.get("VC_BRANCH_TIMEOUT") q = queue.Queue() t = threading.Thread(target=_get_git_branch, args=(q,)) t.start() t.join(timeout=timeout) try: branch = q.get_nowait() except queue.Empty: branch = None return branch def _get_hg_root(q): _curpwd = builtins.__xonsh__.env["PWD"] while True: if not os.path.isdir(_curpwd): return False if any([b.name == ".hg" for b in xt.scandir(_curpwd)]): q.put(_curpwd) break else: _oldpwd = _curpwd _curpwd = os.path.split(_curpwd)[0] if _oldpwd == _curpwd: return False def get_hg_branch(root=None): """Try to get the mercurial branch of the current directory, return None if not in a repo or subprocess.TimeoutExpired if timed out. """ env = builtins.__xonsh__.env timeout = env["VC_BRANCH_TIMEOUT"] q = queue.Queue() t = threading.Thread(target=_get_hg_root, args=(q,)) t.start() t.join(timeout=timeout) try: root = q.get_nowait() except queue.Empty: return None if env.get("VC_HG_SHOW_BRANCH"): # get branch name branch_path = os.path.sep.join([root, ".hg", "branch"]) if os.path.exists(branch_path): with open(branch_path, "r") as branch_file: branch = branch_file.read() else: branch = "default" else: branch = "" # add bookmark, if we can bookmark_path = os.path.sep.join([root, ".hg", "bookmarks.current"]) if os.path.exists(bookmark_path): with open(bookmark_path, "r") as bookmark_file: active_bookmark = bookmark_file.read() if env.get("VC_HG_SHOW_BRANCH") is True: branch = "{0}, {1}".format( *(b.strip(os.linesep) for b in (branch, active_bookmark)) ) else: branch = active_bookmark.strip(os.linesep) else: branch = branch.strip(os.linesep) return branch _FIRST_BRANCH_TIMEOUT = True def _first_branch_timeout_message(): global _FIRST_BRANCH_TIMEOUT sbtm = builtins.__xonsh__.env["SUPPRESS_BRANCH_TIMEOUT_MESSAGE"] if not _FIRST_BRANCH_TIMEOUT or sbtm: return _FIRST_BRANCH_TIMEOUT = False print( "xonsh: branch timeout: computing the branch name, color, or both " "timed out while formatting the prompt. You may avoid this by " "increasing the value of $VC_BRANCH_TIMEOUT or by removing branch " "fields, like {curr_branch}, from your $PROMPT. See the FAQ " "for more details. This message will be suppressed for the remainder " "of this session. To suppress this message permanently, set " "$SUPPRESS_BRANCH_TIMEOUT_MESSAGE = True in your xonshrc file.", file=sys.stderr, ) def current_branch(): """Gets the branch for a current working directory. Returns an empty string if the cwd is not a repository. This currently only works for git and hg and should be extended in the future. If a timeout occurred, the string '' is returned. """ branch = None cmds = builtins.__xonsh__.commands_cache # check for binary only once if cmds.is_empty(): has_git = bool(cmds.locate_binary("git", ignore_alias=True)) has_hg = bool(cmds.locate_binary("hg", ignore_alias=True)) else: has_git = bool(cmds.lazy_locate_binary("git", ignore_alias=True)) has_hg = bool(cmds.lazy_locate_binary("hg", ignore_alias=True)) if has_git: branch = get_git_branch() if not branch and has_hg: branch = get_hg_branch() if isinstance(branch, subprocess.TimeoutExpired): branch = "" _first_branch_timeout_message() return branch or None def _git_dirty_working_directory(q, include_untracked): status = None denv = builtins.__xonsh__.env.detype() try: cmd = ["git", "status", "--porcelain"] if include_untracked: cmd.append("--untracked-files=normal") else: cmd.append("--untracked-files=no") status = subprocess.check_output(cmd, stderr=subprocess.DEVNULL, env=denv) except (subprocess.CalledProcessError, OSError, FileNotFoundError): q.put(None) if status is not None: return q.put(bool(status)) def git_dirty_working_directory(include_untracked=False): """Returns whether or not the git directory is dirty. If this could not be determined (timeout, file not found, etc.) then this returns None. """ timeout = builtins.__xonsh__.env.get("VC_BRANCH_TIMEOUT") q = queue.Queue() t = threading.Thread( target=_git_dirty_working_directory, args=(q, include_untracked) ) t.start() t.join(timeout=timeout) try: return q.get_nowait() except queue.Empty: return None def hg_dirty_working_directory(): """Computes whether or not the mercurial working directory is dirty or not. If this cannot be determined, None is returned. """ env = builtins.__xonsh__.env cwd = env["PWD"] denv = env.detype() vcbt = env["VC_BRANCH_TIMEOUT"] # Override user configurations settings and aliases denv["HGRCPATH"] = "" try: s = subprocess.check_output( ["hg", "identify", "--id"], stderr=subprocess.PIPE, cwd=cwd, timeout=vcbt, universal_newlines=True, env=denv, ) return s.strip(os.linesep).endswith("+") except ( subprocess.CalledProcessError, subprocess.TimeoutExpired, FileNotFoundError, ): return None def dirty_working_directory(): """Returns a boolean as to whether there are uncommitted files in version control repository we are inside. If this cannot be determined, returns None. Currently supports git and hg. """ dwd = None cmds = builtins.__xonsh__.commands_cache if cmds.lazy_locate_binary("git", ignore_alias=True): dwd = git_dirty_working_directory() if cmds.lazy_locate_binary("hg", ignore_alias=True) and dwd is None: dwd = hg_dirty_working_directory() return dwd def branch_color(): """Return red if the current branch is dirty, yellow if the dirtiness can not be determined, and green if it clean. These are bold, intense colors for the foreground. """ dwd = dirty_working_directory() if dwd is None: color = "{YELLOW}" elif dwd: color = "{RED}" else: color = "{GREEN}" return color def branch_bg_color(): """Return red if the current branch is dirty, yellow if the dirtiness can not be determined, and green if it clean. These are background colors. """ dwd = dirty_working_directory() if dwd is None: color = "{BACKGROUND_YELLOW}" elif dwd: color = "{BACKGROUND_RED}" else: color = "{BACKGROUND_GREEN}" return color ================================================ FILE: xonsh/ptk/__init__.py ================================================ ================================================ FILE: xonsh/ptk/completer.py ================================================ # -*- coding: utf-8 -*- """Completer implementation to use with prompt_toolkit.""" import os import builtins from prompt_toolkit.layout.dimension import LayoutDimension from prompt_toolkit.completion import Completer, Completion from prompt_toolkit.auto_suggest import AutoSuggestFromHistory class PromptToolkitCompleter(Completer): """Simple prompt_toolkit Completer object. It just redirects requests to normal Xonsh completer. """ def __init__(self, completer, ctx, shell): """Takes instance of xonsh.completer.Completer, the xonsh execution context, and the shell instance itself. """ self.completer = completer self.ctx = ctx self.shell = shell self.hist_suggester = AutoSuggestFromHistory() def get_completions(self, document, complete_event): """Returns a generator for list of completions.""" env = builtins.__xonsh__.env should_complete = complete_event.completion_requested or env.get( "UPDATE_COMPLETIONS_ON_KEYPRESS" ) # Only generate completions when the user hits tab. if not should_complete or self.completer is None: return # generate actual completions line = document.current_line.lstrip() line_ex = builtins.aliases.expand_alias(line) endidx = document.cursor_position_col begidx = line[:endidx].rfind(" ") + 1 if line[:endidx].rfind(" ") >= 0 else 0 prefix = line[begidx:endidx] expand_offset = len(line_ex) - len(line) # get normal completions completions, l = self.completer.complete( prefix, line_ex, begidx + expand_offset, endidx + expand_offset, self.ctx ) # completions from auto suggest sug_comp = None if env.get("AUTO_SUGGEST") and env.get("AUTO_SUGGEST_IN_COMPLETIONS"): sug_comp = self.suggestion_completion(document, line) if sug_comp is None: pass elif len(completions) == 0: completions = (sug_comp,) else: completions = set(completions) completions.discard(sug_comp) completions = (sug_comp,) + tuple(sorted(completions)) # reserve space, if needed. if len(completions) <= 1: pass elif len(os.path.commonprefix(completions)) <= len(prefix): self.reserve_space() # Find common prefix (strip quoting) c_prefix = os.path.commonprefix([a.strip("'\"") for a in completions]) # Find last split symbol, do not trim the last part while c_prefix: if c_prefix[-1] in r"/\.:@,": break c_prefix = c_prefix[:-1] # yield completions if sug_comp is None: pre = min(document.cursor_position_col - begidx, len(c_prefix)) else: pre = len(c_prefix) for comp in completions: # do not display quote disp = comp[pre:].strip("'\"") yield Completion(comp, -l, display=disp) def suggestion_completion(self, document, line): """Provides a completion based on the current auto-suggestion.""" cli = self.shell.prompter.cli sug = self.hist_suggester.get_suggestion(cli, cli.current_buffer, document) if sug is None: return None comp, _, _ = sug.text.partition(" ") _, _, prev = line.rpartition(" ") return prev + comp def reserve_space(self): cli = builtins.__xonsh__.shell.shell.prompter.cli window = cli.application.layout.children[0].content.children[1] if window and window.render_info: h = window.render_info.content_height r = builtins.__xonsh__.env.get("COMPLETIONS_MENU_ROWS") size = h + r def comp_height(cli): # If there is an autocompletion menu to be shown, make sure that o # layout has at least a minimal height in order to display it. if not cli.is_done: return LayoutDimension(min=size) else: return LayoutDimension() window._height = comp_height ================================================ FILE: xonsh/ptk/history.py ================================================ # -*- coding: utf-8 -*- """History object for use with prompt_toolkit.""" import builtins from threading import Thread import prompt_toolkit.history class PromptToolkitHistory(prompt_toolkit.history.History): """History class that implements the prompt-toolkit history interface with the xonsh backend. """ def __init__(self, load_prev=True, wait_for_gc=True, *args, **kwargs): """Initialize history object.""" super().__init__() self.strings = [] if load_prev: PromptToolkitHistoryAdder(self, wait_for_gc=wait_for_gc) def append(self, entry): """Append new entry to the history.""" self.strings.append(entry) def __getitem__(self, index): return self.strings[index] def __len__(self): return len(self.strings) def __iter__(self): return iter(self.strings) class PromptToolkitHistoryAdder(Thread): def __init__(self, ptkhist, wait_for_gc=True, *args, **kwargs): """Thread responsible for adding inputs from history to the current prompt-toolkit history instance. May wait for the history garbage collector to finish. """ super(PromptToolkitHistoryAdder, self).__init__(*args, **kwargs) self.daemon = True self.ptkhist = ptkhist self.wait_for_gc = wait_for_gc self.start() def run(self): hist = builtins.__xonsh__.history if hist is None: return buf = None ptkhist = self.ptkhist for cmd in hist.all_items(): line = cmd["inp"].rstrip() if len(ptkhist) == 0 or line != ptkhist[-1]: ptkhist.append(line) if buf is None: buf = self._buf() if buf is None: continue buf.reset(initial_document=buf.document) def _buf(self): # Thread-safe version of # buf = builtins.__xonsh__.shell.shell.prompter.cli.application.buffer path = [ "__xonsh__", "shell", "shell", "prompter", "cli", "application", "buffer", ] buf = builtins for a in path: buf = getattr(buf, a, None) if buf is None: break return buf ================================================ FILE: xonsh/ptk/key_bindings.py ================================================ # -*- coding: utf-8 -*- """Key bindings for prompt_toolkit xonsh shell.""" import builtins from prompt_toolkit.enums import DEFAULT_BUFFER from prompt_toolkit.filters import ( Condition, IsMultiline, HasSelection, EmacsInsertMode, ViInsertMode, ) from prompt_toolkit.keys import Keys from xonsh.aliases import xonsh_exit from xonsh.tools import check_for_partial_string, get_line_continuation from xonsh.shell import transform_command env = builtins.__xonsh__.env DEDENT_TOKENS = frozenset(["raise", "return", "pass", "break", "continue"]) def carriage_return(b, cli, *, autoindent=True): """Preliminary parser to determine if 'Enter' key should send command to the xonsh parser for execution or should insert a newline for continued input. Current 'triggers' for inserting a newline are: - Not on first line of buffer and line is non-empty - Previous character is a colon (covers if, for, etc...) - User is in an open paren-block - Line ends with backslash - Any text exists below cursor position (relevant when editing previous multiline blocks) """ doc = b.document at_end_of_line = _is_blank(doc.current_line_after_cursor) current_line_blank = _is_blank(doc.current_line) indent = env.get("INDENT") if autoindent else "" partial_string_info = check_for_partial_string(doc.text) in_partial_string = ( partial_string_info[0] is not None and partial_string_info[1] is None ) # indent after a colon if doc.current_line_before_cursor.strip().endswith(":") and at_end_of_line: b.newline(copy_margin=autoindent) b.insert_text(indent, fire_event=False) # if current line isn't blank, check dedent tokens elif ( not current_line_blank and doc.current_line.split(maxsplit=1)[0] in DEDENT_TOKENS and doc.line_count > 1 ): b.newline(copy_margin=autoindent) b.delete_before_cursor(count=len(indent)) elif not doc.on_first_line and not current_line_blank: b.newline(copy_margin=autoindent) elif doc.current_line.endswith(get_line_continuation()): b.newline(copy_margin=autoindent) elif doc.find_next_word_beginning() is not None and ( any(not _is_blank(i) for i in doc.lines_from_current[1:]) ): b.newline(copy_margin=autoindent) elif not current_line_blank and not can_compile(doc.text): b.newline(copy_margin=autoindent) elif current_line_blank and in_partial_string: b.newline(copy_margin=autoindent) else: b.accept_action.validate_and_handle(cli, b) def _is_blank(l): return len(l.strip()) == 0 def can_compile(src): """Returns whether the code can be compiled, i.e. it is valid xonsh.""" src = src if src.endswith("\n") else src + "\n" src = transform_command(src, show_diff=False) src = src.lstrip() try: builtins.__xonsh__.execer.compile( src, mode="single", glbs=None, locs=builtins.__xonsh__.ctx ) rtn = True except SyntaxError: rtn = False except Exception: rtn = True return rtn @Condition def tab_insert_indent(cli): """Check if should insert indent instead of starting autocompletion. Checks if there are only whitespaces before the cursor - if so indent should be inserted, otherwise autocompletion. """ before_cursor = cli.current_buffer.document.current_line_before_cursor return bool(before_cursor.isspace()) @Condition def beginning_of_line(cli): """Check if cursor is at beginning of a line other than the first line in a multiline document """ before_cursor = cli.current_buffer.document.current_line_before_cursor return bool( len(before_cursor) == 0 and not cli.current_buffer.document.on_first_line ) @Condition def end_of_line(cli): """Check if cursor is at the end of a line other than the last line in a multiline document """ d = cli.current_buffer.document at_end = d.is_cursor_at_the_end_of_line last_line = d.is_cursor_at_the_end return bool(at_end and not last_line) @Condition def should_confirm_completion(cli): """Check if completion needs confirmation""" return ( builtins.__xonsh__.env.get("COMPLETIONS_CONFIRM") and cli.current_buffer.complete_state ) # Copied from prompt-toolkit's key_binding/bindings/basic.py @Condition def ctrl_d_condition(cli): """Ctrl-D binding is only active when the default buffer is selected and empty. """ if builtins.__xonsh__.env.get("IGNOREEOF"): raise EOFError else: return cli.current_buffer_name == DEFAULT_BUFFER and not cli.current_buffer.text @Condition def autopair_condition(cli): """Check if XONSH_AUTOPAIR is set""" return builtins.__xonsh__.env.get("XONSH_AUTOPAIR", False) @Condition def whitespace_or_bracket_before(cli): """Check if there is whitespace or an opening bracket to the left of the cursor""" d = cli.current_buffer.document return bool( d.cursor_position == 0 or d.char_before_cursor.isspace() or d.char_before_cursor in "([{" ) @Condition def whitespace_or_bracket_after(cli): """Check if there is whitespace or a closing bracket to the right of the cursor""" d = cli.current_buffer.document return bool( d.is_cursor_at_the_end_of_line or d.current_char.isspace() or d.current_char in ")]}" ) def load_xonsh_bindings(key_bindings_manager): """ Load custom key bindings. """ handle = key_bindings_manager.registry.add_binding has_selection = HasSelection() insert_mode = ViInsertMode() | EmacsInsertMode() @handle(Keys.Tab, filter=tab_insert_indent) def insert_indent(event): """ If there are only whitespaces before current cursor position insert indent instead of autocompleting. """ event.cli.current_buffer.insert_text(env.get("INDENT")) @handle(Keys.ControlX, Keys.ControlE, filter=~has_selection) def open_editor(event): """ Open current buffer in editor """ event.current_buffer.open_in_editor(event.cli) @handle(Keys.BackTab, filter=insert_mode) def insert_literal_tab(event): """ Insert literal tab on Shift+Tab instead of autocompleting """ b = event.current_buffer if b.complete_state: b.complete_previous() else: event.cli.current_buffer.insert_text(env.get("INDENT")) @handle("(", filter=autopair_condition & whitespace_or_bracket_after) def insert_right_parens(event): event.cli.current_buffer.insert_text("(") event.cli.current_buffer.insert_text(")", move_cursor=False) @handle(")", filter=autopair_condition) def overwrite_right_parens(event): buffer = event.cli.current_buffer if buffer.document.current_char == ")": buffer.cursor_position += 1 else: buffer.insert_text(")") @handle("[", filter=autopair_condition & whitespace_or_bracket_after) def insert_right_bracket(event): event.cli.current_buffer.insert_text("[") event.cli.current_buffer.insert_text("]", move_cursor=False) @handle("]", filter=autopair_condition) def overwrite_right_bracket(event): buffer = event.cli.current_buffer if buffer.document.current_char == "]": buffer.cursor_position += 1 else: buffer.insert_text("]") @handle("{", filter=autopair_condition & whitespace_or_bracket_after) def insert_right_brace(event): event.cli.current_buffer.insert_text("{") event.cli.current_buffer.insert_text("}", move_cursor=False) @handle("}", filter=autopair_condition) def overwrite_right_brace(event): buffer = event.cli.current_buffer if buffer.document.current_char == "}": buffer.cursor_position += 1 else: buffer.insert_text("}") @handle("'", filter=autopair_condition) def insert_right_quote(event): buffer = event.cli.current_buffer if buffer.document.current_char == "'": buffer.cursor_position += 1 elif whitespace_or_bracket_before(event.cli) and whitespace_or_bracket_after( event.cli ): buffer.insert_text("'") buffer.insert_text("'", move_cursor=False) else: buffer.insert_text("'") @handle('"', filter=autopair_condition) def insert_right_double_quote(event): buffer = event.cli.current_buffer if buffer.document.current_char == '"': buffer.cursor_position += 1 elif whitespace_or_bracket_before(event.cli) and whitespace_or_bracket_after( event.cli ): buffer.insert_text('"') buffer.insert_text('"', move_cursor=False) else: buffer.insert_text('"') @handle(Keys.Backspace, filter=autopair_condition) def delete_brackets_or_quotes(event): """Delete empty pair of brackets or quotes""" buffer = event.cli.current_buffer before = buffer.document.char_before_cursor after = buffer.document.current_char if any( [before == b and after == a for (b, a) in ["()", "[]", "{}", "''", '""']] ): buffer.delete(1) buffer.delete_before_cursor(1) @handle(Keys.ControlD, filter=ctrl_d_condition) def call_exit_alias(event): """Use xonsh exit function""" b = event.cli.current_buffer b.accept_action.validate_and_handle(event.cli, b) xonsh_exit([]) @handle(Keys.ControlJ, filter=IsMultiline()) def multiline_carriage_return(event): """ Wrapper around carriage_return multiline parser """ b = event.cli.current_buffer carriage_return(b, event.cli) @handle(Keys.ControlJ, filter=should_confirm_completion) def enter_confirm_completion(event): """Ignore (confirm completion)""" event.current_buffer.complete_state = None @handle(Keys.Escape, filter=should_confirm_completion) def esc_cancel_completion(event): """Use to cancel completion""" event.cli.current_buffer.cancel_completion() @handle(Keys.Escape, Keys.ControlJ) def execute_block_now(event): """Execute a block of text irrespective of cursor position""" b = event.cli.current_buffer b.accept_action.validate_and_handle(event.cli, b) @handle(Keys.Left, filter=beginning_of_line) def wrap_cursor_back(event): """Move cursor to end of previous line unless at beginning of document """ b = event.cli.current_buffer b.cursor_up(count=1) relative_end_index = b.document.get_end_of_line_position() b.cursor_right(count=relative_end_index) @handle(Keys.Right, filter=end_of_line) def wrap_cursor_forward(event): """Move cursor to beginning of next line unless at end of document""" b = event.cli.current_buffer relative_begin_index = b.document.get_start_of_line_position() b.cursor_left(count=abs(relative_begin_index)) b.cursor_down(count=1) @handle(Keys.ControlI, filter=insert_mode) def generate_completions(event): """ Tab-completion: where the first tab completes the common suffix and the second tab lists all the completions. Notes ----- This method was forked from the mainline prompt-toolkit repo. Copyright (c) 2014, Jonathan Slenders, All rights reserved. """ b = event.current_buffer def second_tab(): if b.complete_state: b.complete_next() else: event.cli.start_completion(select_first=False) # On the second tab-press, or when already navigating through # completions. if event.is_repeat or b.complete_state: second_tab() else: event.cli.start_completion(insert_common_part=True, select_first=False) ================================================ FILE: xonsh/ptk/shell.py ================================================ # -*- coding: utf-8 -*- """The prompt_toolkit based xonsh shell.""" import sys import builtins from prompt_toolkit.key_binding.manager import KeyBindingManager from prompt_toolkit.auto_suggest import AutoSuggestFromHistory from prompt_toolkit.layout.lexers import PygmentsLexer from prompt_toolkit.shortcuts import print_tokens from prompt_toolkit.styles import PygmentsStyle, style_from_dict from xonsh.base_shell import BaseShell from xonsh.tools import print_exception, carriage_return, ansicolors_to_ptk1_names from xonsh.ptk.completer import PromptToolkitCompleter from xonsh.ptk.history import PromptToolkitHistory from xonsh.ptk.key_bindings import load_xonsh_bindings from xonsh.ptk.shortcuts import Prompter from xonsh.events import events from xonsh.shell import transform_command from xonsh.platform import HAS_PYGMENTS, ON_WINDOWS from xonsh.style_tools import ( partial_color_tokenize, _TokenType, DEFAULT_STYLE_DICT as _DEFAULT_STYLE_DICT, ) from xonsh.lazyimps import pygments, pyghooks, winutils from xonsh.pygments_cache import get_all_styles from xonsh.lazyasd import LazyObject Token = _TokenType() events.transmogrify("on_ptk_create", "LoadEvent") events.doc( "on_ptk_create", """ on_ptk_create(prompter: Prompter, history: PromptToolkitHistory, completer: PromptToolkitCompleter, bindings: KeyBindingManager) -> Fired after prompt toolkit has been initialized """, ) # Convert new ansicolor names to names # understood by PTK1 DEFAULT_STYLE_DICT = LazyObject( lambda: ansicolors_to_ptk1_names(_DEFAULT_STYLE_DICT), globals(), "DEFAULT_STYLE_DICT", ) class PromptToolkitShell(BaseShell): """The xonsh shell.""" def __init__(self, **kwargs): super().__init__(**kwargs) if ON_WINDOWS: winutils.enable_virtual_terminal_processing() self._first_prompt = True self.prompter = Prompter() self.history = PromptToolkitHistory() self.pt_completer = PromptToolkitCompleter(self.completer, self.ctx, self) key_bindings_manager_args = { "enable_auto_suggest_bindings": True, "enable_search": True, "enable_abort_and_exit_bindings": True, } self.key_bindings_manager = KeyBindingManager(**key_bindings_manager_args) load_xonsh_bindings(self.key_bindings_manager) # This assumes that PromptToolkitShell is a singleton events.on_ptk_create.fire( prompter=self.prompter, history=self.history, completer=self.pt_completer, bindings=self.key_bindings_manager, ) def singleline( self, store_in_history=True, auto_suggest=None, enable_history_search=True, multiline=True, **kwargs ): """Reads a single line of input from the shell. The store_in_history kwarg flags whether the input should be stored in PTK's in-memory history. """ events.on_pre_prompt.fire() env = builtins.__xonsh__.env mouse_support = env.get("MOUSE_SUPPORT") if store_in_history: history = self.history else: history = None enable_history_search = False auto_suggest = auto_suggest if env.get("AUTO_SUGGEST") else None completions_display = env.get("COMPLETIONS_DISPLAY") multicolumn = completions_display == "multi" complete_while_typing = env.get("UPDATE_COMPLETIONS_ON_KEYPRESS") if complete_while_typing: # PTK requires history search to be none when completing while typing enable_history_search = False if HAS_PYGMENTS: self.styler.style_name = env.get("XONSH_COLOR_STYLE") completer = None if completions_display == "none" else self.pt_completer if not env.get("UPDATE_PROMPT_ON_KEYPRESS"): prompt_tokens_cached = self.prompt_tokens(None) get_prompt_tokens = lambda cli: prompt_tokens_cached rprompt_tokens_cached = self.rprompt_tokens(None) get_rprompt_tokens = lambda cli: rprompt_tokens_cached bottom_toolbar_tokens_cached = self.bottom_toolbar_tokens(None) get_bottom_toolbar_tokens = lambda cli: bottom_toolbar_tokens_cached else: get_prompt_tokens = self.prompt_tokens get_rprompt_tokens = self.rprompt_tokens get_bottom_toolbar_tokens = self.bottom_toolbar_tokens with self.prompter: prompt_args = { "mouse_support": mouse_support, "auto_suggest": auto_suggest, "get_prompt_tokens": get_prompt_tokens, "get_rprompt_tokens": get_rprompt_tokens, "get_bottom_toolbar_tokens": get_bottom_toolbar_tokens, "completer": completer, "multiline": multiline, "get_continuation_tokens": self.continuation_tokens, "history": history, "enable_history_search": enable_history_search, "reserve_space_for_menu": 0, "key_bindings_registry": self.key_bindings_manager.registry, "display_completions_in_columns": multicolumn, "complete_while_typing": complete_while_typing, } if builtins.__xonsh__.env.get("COLOR_INPUT"): if HAS_PYGMENTS: prompt_args["lexer"] = PygmentsLexer(pyghooks.XonshLexer) prompt_args["style"] = PygmentsStyle( pyghooks.xonsh_style_proxy(self.styler) ) else: prompt_args["style"] = style_from_dict(DEFAULT_STYLE_DICT) line = self.prompter.prompt(**prompt_args) events.on_post_prompt.fire() return line def _push(self, line): """Pushes a line onto the buffer and compiles the code in a way that enables multiline input. """ code = None self.buffer.append(line) if self.need_more_lines: return None, code src = "".join(self.buffer) src = transform_command(src) try: code = self.execer.compile(src, mode="single", glbs=self.ctx, locs=None) self.reset_buffer() except Exception: # pylint: disable=broad-except self.reset_buffer() print_exception() return src, None return src, code def cmdloop(self, intro=None): """Enters a loop that reads and execute input from user.""" if intro: print(intro) auto_suggest = AutoSuggestFromHistory() self.push = self._push while not builtins.__xonsh__.exit: try: line = self.singleline(auto_suggest=auto_suggest) if not line: self.emptyline() else: line = self.precmd(line) self.default(line) except (KeyboardInterrupt, SystemExit): self.reset_buffer() except EOFError: if builtins.__xonsh__.env.get("IGNOREEOF"): print('Use "exit" to leave the shell.', file=sys.stderr) else: break def prompt_tokens(self, cli): """Returns a list of (token, str) tuples for the current prompt.""" p = builtins.__xonsh__.env.get("PROMPT") try: p = self.prompt_formatter(p) except Exception: # pylint: disable=broad-except print_exception() toks = partial_color_tokenize(p) if self._first_prompt: carriage_return() self._first_prompt = False self.settitle() return toks def rprompt_tokens(self, cli): """Returns a list of (token, str) tuples for the current right prompt. """ p = builtins.__xonsh__.env.get("RIGHT_PROMPT") # self.prompt_formatter does handle empty strings properly, # but this avoids descending into it in the common case of # $RIGHT_PROMPT == ''. if isinstance(p, str) and len(p) == 0: return [] try: p = self.prompt_formatter(p) except Exception: # pylint: disable=broad-except print_exception() toks = partial_color_tokenize(p) return toks def bottom_toolbar_tokens(self, cli): """Returns a list of (token, str) tuples for the current bottom toolbar. """ p = builtins.__xonsh__.env.get("BOTTOM_TOOLBAR") # self.prompt_formatter does handle empty strings properly, # but this avoids descending into it in the common case of # $TOOLBAR == ''. if isinstance(p, str) and len(p) == 0: return [] try: p = self.prompt_formatter(p) except Exception: # pylint: disable=broad-except print_exception() toks = partial_color_tokenize(p) return toks def continuation_tokens(self, cli, width): """Displays dots in multiline prompt""" width = width - 1 dots = builtins.__xonsh__.env.get("MULTILINE_PROMPT") dots = dots() if callable(dots) else dots if dots is None: return [(Token, " " * (width + 1))] basetoks = self.format_color(dots) baselen = sum(len(t[1]) for t in basetoks) if baselen == 0: return [(Token, " " * (width + 1))] toks = basetoks * (width // baselen) n = width % baselen count = 0 for tok in basetoks: slen = len(tok[1]) newcount = slen + count if slen == 0: continue elif newcount <= n: toks.append(tok) else: toks.append((tok[0], tok[1][: n - count])) count = newcount if n <= count: break toks.append((Token, " ")) # final space return toks def format_color(self, string, hide=False, force_string=False, **kwargs): """Formats a color string using Pygments. This, therefore, returns a list of (Token, str) tuples. If force_string is set to true, though, this will return a color formatted string. """ tokens = partial_color_tokenize(string) if force_string and HAS_PYGMENTS: env = builtins.__xonsh__.env self.styler.style_name = env.get("XONSH_COLOR_STYLE") proxy_style = pyghooks.xonsh_style_proxy(self.styler) formatter = pyghooks.XonshTerminal256Formatter(style=proxy_style) s = pygments.format(tokens, formatter) return s elif force_string: print("To force colorization of string, install Pygments") return tokens else: return tokens def print_color(self, string, end="\n", **kwargs): """Prints a color string using prompt-toolkit color management.""" if isinstance(string, str): tokens = partial_color_tokenize(string + end) else: # assume this is a list of (Token, str) tuples and just print tokens = string if HAS_PYGMENTS: env = builtins.__xonsh__.env self.styler.style_name = env.get("XONSH_COLOR_STYLE") proxy_style = PygmentsStyle(pyghooks.xonsh_style_proxy(self.styler)) else: proxy_style = style_from_dict(DEFAULT_STYLE_DICT) print_tokens(tokens, style=proxy_style) def color_style_names(self): """Returns an iterable of all available style names.""" if not HAS_PYGMENTS: return ["For other xonsh styles, please install pygments"] return get_all_styles() def color_style(self): """Returns the current color map.""" if not HAS_PYGMENTS: return DEFAULT_STYLE_DICT env = builtins.__xonsh__.env self.styler.style_name = env.get("XONSH_COLOR_STYLE") return self.styler.styles def restore_tty_sanity(self): """An interface for resetting the TTY stdin mode. This is highly dependent on the shell backend. Also it is mostly optional since it only affects ^Z backgrounding behaviour. """ # PTK does not seem to need any specialization here. However, # if it does for some reason in the future... # The following writes an ANSI escape sequence that sends the cursor # to the end of the line. This has the effect of restoring ECHO mode. # See http://unix.stackexchange.com/a/108014/129048 for more details. # This line can also be replaced by os.system("stty sane"), as per # http://stackoverflow.com/questions/19777129/interactive-python-interpreter-run-in-background#comment29421919_19778355 # However, it is important to note that not termios-based solution # seems to work. My guess is that this is because termios restoration # needs to be performed by the subprocess itself. This fix is important # when subprocesses don't properly restore the terminal attributes, # like Python in interactive mode. Also note that the sequences "\033M" # and "\033E" seem to work too, but these are technically VT100 codes. # I used the more primitive ANSI sequence to maximize compatibility. # -scopatz 2017-01-28 # if not ON_POSIX: # return # sys.stdout.write('\033[9999999C\n') ================================================ FILE: xonsh/ptk/shortcuts.py ================================================ """A prompt-toolkit inspired shortcut collection.""" import builtins import textwrap from prompt_toolkit.interface import CommandLineInterface from prompt_toolkit.enums import EditingMode from prompt_toolkit.utils import DummyContext from prompt_toolkit.shortcuts import ( create_prompt_application, create_eventloop, create_asyncio_eventloop, create_output, ) from xonsh.platform import ptk_version_info import xonsh.tools as xt class Prompter(object): def __init__(self, cli=None, *args, **kwargs): """Implements a prompt that statefully holds a command-line interface. When used as a context manager, it will return itself on entry and reset itself on exit. Parameters ---------- cli : CommandLineInterface or None, optional If this is not a CommandLineInterface object, such an object will be created when the prompt() method is called. """ self.cli = cli self.major_minor = ptk_version_info()[:2] def __enter__(self): self.reset() return self def __exit__(self, exc_type, exc_value, traceback): pass def prompt(self, message="", **kwargs): """Get input from the user and return it. This is a wrapper around a lot of prompt_toolkit functionality and can be a replacement for raw_input. (or GNU readline.) If you want to keep your history across several calls, create one `~prompt_toolkit.history.History instance and pass it every time. This function accepts many keyword arguments. Except for the following. they are a proxy to the arguments of create_prompt_application(). Parameters ---------- patch_stdout : file-like, optional Replace ``sys.stdout`` by a proxy that ensures that print statements from other threads won't destroy the prompt. (They will be printed above the prompt instead.) return_asyncio_coroutine : bool, optional When True, return a asyncio coroutine. (Python >3.3) Notes ----- This method was forked from the mainline prompt-toolkit repo. Copyright (c) 2014, Jonathan Slenders, All rights reserved. """ patch_stdout = kwargs.pop("patch_stdout", False) return_asyncio_coroutine = kwargs.pop("return_asyncio_coroutine", False) if return_asyncio_coroutine: eventloop = create_asyncio_eventloop() else: eventloop = kwargs.pop("eventloop", None) or create_eventloop() # Create CommandLineInterface. if self.cli is None: if builtins.__xonsh__.env.get("VI_MODE"): editing_mode = EditingMode.VI else: editing_mode = EditingMode.EMACS kwargs["editing_mode"] = editing_mode cli = CommandLineInterface( application=create_prompt_application(message, **kwargs), eventloop=eventloop, output=create_output(), ) self.cli = cli else: cli = self.cli # Replace stdout. patch_context = cli.patch_stdout_context() if patch_stdout else DummyContext() # Read input and return it. if return_asyncio_coroutine: # Create an asyncio coroutine and call it. exec_context = {"patch_context": patch_context, "cli": cli} exec( textwrap.dedent( """ import asyncio @asyncio.coroutine def prompt_coro(): with patch_context: document = yield from cli.run_async(reset_current_buffer=False) if document: return document.text """ ), exec_context, ) return exec_context["prompt_coro"]() else: # Note: We pass `reset_current_buffer=False`, because that way # it's easy to give DEFAULT_BUFFER a default value, without it # getting erased. We don't have to reset anyway, because this is # the first and only time that this CommandLineInterface will run. try: with patch_context: document = cli.run(reset_current_buffer=False) if document: return document.text except Exception: xt.print_exception() # return something to prevent xonsh crash when any # exceptions raise return "" finally: eventloop.close() def reset(self): """Resets the prompt and cli to a pristine state on this object.""" self.cli = None ================================================ FILE: xonsh/ptk2/__init__.py ================================================ # must come before ptk / pygments imports from xonsh.lazyasd import load_module_in_background load_module_in_background( "pkg_resources", debug="XONSH_DEBUG", replacements={"pygments.plugin": "pkg_resources"}, ) ================================================ FILE: xonsh/ptk2/completer.py ================================================ # -*- coding: utf-8 -*- """Completer implementation to use with prompt_toolkit.""" from prompt_toolkit.completion import Completer from gitsome.completer import CompleterGitsome class PromptToolkitCompleter(Completer): """Simple prompt_toolkit Completer object. It just redirects requests to normal Xonsh completer. """ def __init__(self, completer, ctx, shell): """Takes instance of xonsh.completer.Completer, the xonsh execution context, and the shell instance itself. """ self.completer = completer self.ctx = ctx self.shell = shell self.completer_gitsome = CompleterGitsome() def get_completions(self, document, complete_event): """Returns a generator for list of completions.""" line = document.current_line.lstrip() endidx = document.cursor_position_col space_pos = document.find_backwards(' ') if space_pos is None: begidx = 0 else: begidx = space_pos + endidx + 1 prefix = line[begidx:endidx] completions, dummy = self.completer.complete(prefix, line, begidx, endidx, self.ctx) completions_gitsome = \ self.completer_gitsome.get_completions(document, complete_event) completions_with_meta = \ self.completer_gitsome.build_completions_with_meta(line, prefix, completions) completions_gitsome.extend(completions_with_meta) for comp in completions_gitsome: yield comp ================================================ FILE: xonsh/ptk2/history.py ================================================ # -*- coding: utf-8 -*- """History object for use with prompt_toolkit.""" import builtins import prompt_toolkit.history class PromptToolkitHistory(prompt_toolkit.history.History): """History class that implements the prompt-toolkit history interface with the xonsh backend. """ def __init__(self, load_prev=True, *args, **kwargs): """Initialize history object.""" super().__init__() self.load_prev = load_prev def store_string(self, entry): pass def load_history_strings(self): """Loads synchronous history strings""" if not self.load_prev: return hist = builtins.__xonsh__.history if hist is None: return for cmd in hist.all_items(newest_first=True): line = cmd["inp"].rstrip() strs = self.get_strings() if len(strs) == 0 or line != strs[-1]: yield line def __getitem__(self, index): return self.get_strings()[index] def __len__(self): return len(self.get_strings()) def __iter__(self): return iter(self.get_strings()) def _cust_history_matches(self, i): """Custom history search method for prompt_toolkit that matches previous commands anywhere on a line, not just at the start. This gets monkeypatched into the prompt_toolkit prompter if ``XONSH_HISTORY_MATCH_ANYWHERE=True``""" return ( self.history_search_text is None or self.history_search_text in self._working_lines[i] ) ================================================ FILE: xonsh/ptk2/key_bindings.py ================================================ # -*- coding: utf-8 -*- """Key bindings for prompt_toolkit xonsh shell.""" import builtins from prompt_toolkit import search from prompt_toolkit.enums import DEFAULT_BUFFER from prompt_toolkit.filters import ( Condition, IsMultiline, HasSelection, EmacsInsertMode, ViInsertMode, IsSearching, ) from prompt_toolkit.keys import Keys from prompt_toolkit.application.current import get_app from xonsh.aliases import xonsh_exit from xonsh.tools import check_for_partial_string, get_line_continuation from xonsh.shell import transform_command DEDENT_TOKENS = frozenset(["raise", "return", "pass", "break", "continue"]) def carriage_return(b, cli, *, autoindent=True): """Preliminary parser to determine if 'Enter' key should send command to the xonsh parser for execution or should insert a newline for continued input. Current 'triggers' for inserting a newline are: - Not on first line of buffer and line is non-empty - Previous character is a colon (covers if, for, etc...) - User is in an open paren-block - Line ends with backslash - Any text exists below cursor position (relevant when editing previous multiline blocks) """ doc = b.document at_end_of_line = _is_blank(doc.current_line_after_cursor) current_line_blank = _is_blank(doc.current_line) env = builtins.__xonsh__.env indent = env.get("INDENT") if autoindent else "" partial_string_info = check_for_partial_string(doc.text) in_partial_string = ( partial_string_info[0] is not None and partial_string_info[1] is None ) # indent after a colon if doc.current_line_before_cursor.strip().endswith(":") and at_end_of_line: b.newline(copy_margin=autoindent) b.insert_text(indent, fire_event=False) # if current line isn't blank, check dedent tokens elif ( not current_line_blank and doc.current_line.split(maxsplit=1)[0] in DEDENT_TOKENS and doc.line_count > 1 ): b.newline(copy_margin=autoindent) b.delete_before_cursor(count=len(indent)) elif not doc.on_first_line and not current_line_blank: b.newline(copy_margin=autoindent) elif doc.current_line.endswith(get_line_continuation()): b.newline(copy_margin=autoindent) elif doc.find_next_word_beginning() is not None and ( any(not _is_blank(i) for i in doc.lines_from_current[1:]) ): b.newline(copy_margin=autoindent) elif not current_line_blank and not can_compile(doc.text): b.newline(copy_margin=autoindent) elif current_line_blank and in_partial_string: b.newline(copy_margin=autoindent) else: b.validate_and_handle() def _is_blank(l): return len(l.strip()) == 0 def can_compile(src): """Returns whether the code can be compiled, i.e. it is valid xonsh.""" src = src if src.endswith("\n") else src + "\n" src = transform_command(src, show_diff=False) src = src.lstrip() try: builtins.__xonsh__.execer.compile( src, mode="single", glbs=None, locs=builtins.__xonsh__.ctx ) rtn = True except SyntaxError: rtn = False except Exception: rtn = True return rtn @Condition def tab_insert_indent(): """Check if should insert indent instead of starting autocompletion. Checks if there are only whitespaces before the cursor - if so indent should be inserted, otherwise autocompletion. """ before_cursor = get_app().current_buffer.document.current_line_before_cursor return bool(before_cursor.isspace()) @Condition def beginning_of_line(): """Check if cursor is at beginning of a line other than the first line in a multiline document """ app = get_app() before_cursor = app.current_buffer.document.current_line_before_cursor return bool( len(before_cursor) == 0 and not app.current_buffer.document.on_first_line ) @Condition def end_of_line(): """Check if cursor is at the end of a line other than the last line in a multiline document """ d = get_app().current_buffer.document at_end = d.is_cursor_at_the_end_of_line last_line = d.is_cursor_at_the_end return bool(at_end and not last_line) @Condition def should_confirm_completion(): """Check if completion needs confirmation""" return ( builtins.__xonsh__.env.get("COMPLETIONS_CONFIRM") and get_app().current_buffer.complete_state ) # Copied from prompt-toolkit's key_binding/bindings/basic.py @Condition def ctrl_d_condition(): """Ctrl-D binding is only active when the default buffer is selected and empty. """ if builtins.__xonsh__.env.get("IGNOREEOF"): return False else: app = get_app() buffer_name = app.current_buffer.name return buffer_name == DEFAULT_BUFFER and not app.current_buffer.text @Condition def autopair_condition(): """Check if XONSH_AUTOPAIR is set""" return builtins.__xonsh__.env.get("XONSH_AUTOPAIR", False) @Condition def whitespace_or_bracket_before(): """Check if there is whitespace or an opening bracket to the left of the cursor""" d = get_app().current_buffer.document return bool( d.cursor_position == 0 or d.char_before_cursor.isspace() or d.char_before_cursor in "([{" ) @Condition def whitespace_or_bracket_after(): """Check if there is whitespace or a closing bracket to the right of the cursor""" d = get_app().current_buffer.document return bool( d.is_cursor_at_the_end_of_line or d.current_char.isspace() or d.current_char in ")]}" ) def load_xonsh_bindings(key_bindings): """ Load custom key bindings. """ handle = key_bindings.add has_selection = HasSelection() insert_mode = ViInsertMode() | EmacsInsertMode() @handle(Keys.Tab, filter=tab_insert_indent) def insert_indent(event): """ If there are only whitespaces before current cursor position insert indent instead of autocompleting. """ env = builtins.__xonsh__.env event.cli.current_buffer.insert_text(env.get("INDENT")) @handle(Keys.ControlX, Keys.ControlE, filter=~has_selection) def open_editor(event): """ Open current buffer in editor """ event.current_buffer.open_in_editor(event.cli) @handle(Keys.BackTab, filter=insert_mode) def insert_literal_tab(event): """ Insert literal tab on Shift+Tab instead of autocompleting """ b = event.current_buffer if b.complete_state: b.complete_previous() else: env = builtins.__xonsh__.env event.cli.current_buffer.insert_text(env.get("INDENT")) @handle("(", filter=autopair_condition & whitespace_or_bracket_after) def insert_right_parens(event): event.cli.current_buffer.insert_text("(") event.cli.current_buffer.insert_text(")", move_cursor=False) @handle(")", filter=autopair_condition) def overwrite_right_parens(event): buffer = event.cli.current_buffer if buffer.document.current_char == ")": buffer.cursor_position += 1 else: buffer.insert_text(")") @handle("[", filter=autopair_condition & whitespace_or_bracket_after) def insert_right_bracket(event): event.cli.current_buffer.insert_text("[") event.cli.current_buffer.insert_text("]", move_cursor=False) @handle("]", filter=autopair_condition) def overwrite_right_bracket(event): buffer = event.cli.current_buffer if buffer.document.current_char == "]": buffer.cursor_position += 1 else: buffer.insert_text("]") @handle("{", filter=autopair_condition & whitespace_or_bracket_after) def insert_right_brace(event): event.cli.current_buffer.insert_text("{") event.cli.current_buffer.insert_text("}", move_cursor=False) @handle("}", filter=autopair_condition) def overwrite_right_brace(event): buffer = event.cli.current_buffer if buffer.document.current_char == "}": buffer.cursor_position += 1 else: buffer.insert_text("}") @handle("'", filter=autopair_condition) def insert_right_quote(event): buffer = event.cli.current_buffer if buffer.document.current_char == "'": buffer.cursor_position += 1 elif whitespace_or_bracket_before() and whitespace_or_bracket_after(): buffer.insert_text("'") buffer.insert_text("'", move_cursor=False) else: buffer.insert_text("'") @handle('"', filter=autopair_condition) def insert_right_double_quote(event): buffer = event.cli.current_buffer if buffer.document.current_char == '"': buffer.cursor_position += 1 elif whitespace_or_bracket_before() and whitespace_or_bracket_after(): buffer.insert_text('"') buffer.insert_text('"', move_cursor=False) else: buffer.insert_text('"') @handle(Keys.Backspace, filter=autopair_condition) def delete_brackets_or_quotes(event): """Delete empty pair of brackets or quotes""" buffer = event.cli.current_buffer before = buffer.document.char_before_cursor after = buffer.document.current_char if any( [before == b and after == a for (b, a) in ["()", "[]", "{}", "''", '""']] ): buffer.delete(1) buffer.delete_before_cursor(1) @handle(Keys.ControlD, filter=ctrl_d_condition) def call_exit_alias(event): """Use xonsh exit function""" b = event.cli.current_buffer b.validate_and_handle() xonsh_exit([]) @handle(Keys.ControlJ, filter=IsMultiline()) @handle(Keys.ControlM, filter=IsMultiline()) def multiline_carriage_return(event): """ Wrapper around carriage_return multiline parser """ b = event.cli.current_buffer carriage_return(b, event.cli) @handle(Keys.ControlJ, filter=should_confirm_completion) @handle(Keys.ControlM, filter=should_confirm_completion) def enter_confirm_completion(event): """Ignore (confirm completion)""" event.current_buffer.complete_state = None @handle(Keys.Escape, filter=should_confirm_completion) def esc_cancel_completion(event): """Use to cancel completion""" event.cli.current_buffer.cancel_completion() @handle(Keys.Escape, Keys.ControlJ) def execute_block_now(event): """Execute a block of text irrespective of cursor position""" b = event.cli.current_buffer b.validate_and_handle() @handle(Keys.Left, filter=beginning_of_line) def wrap_cursor_back(event): """Move cursor to end of previous line unless at beginning of document """ b = event.cli.current_buffer b.cursor_up(count=1) relative_end_index = b.document.get_end_of_line_position() b.cursor_right(count=relative_end_index) @handle(Keys.Right, filter=end_of_line) def wrap_cursor_forward(event): """Move cursor to beginning of next line unless at end of document""" b = event.cli.current_buffer relative_begin_index = b.document.get_start_of_line_position() b.cursor_left(count=abs(relative_begin_index)) b.cursor_down(count=1) @handle(Keys.ControlM, filter=IsSearching()) @handle(Keys.ControlJ, filter=IsSearching()) def accept_search(event): search.accept_search() ================================================ FILE: xonsh/ptk2/shell.py ================================================ # -*- coding: utf-8 -*- """The prompt_toolkit based xonsh shell.""" import sys import builtins from types import MethodType from xonsh.events import events from xonsh.base_shell import BaseShell from xonsh.shell import transform_command from xonsh.tools import print_exception, carriage_return from xonsh.platform import HAS_PYGMENTS, ON_WINDOWS from xonsh.style_tools import partial_color_tokenize, _TokenType, DEFAULT_STYLE_DICT from xonsh.lazyimps import pygments, pyghooks, winutils from xonsh.pygments_cache import get_all_styles from xonsh.ptk2.history import PromptToolkitHistory, _cust_history_matches from xonsh.ptk2.completer import PromptToolkitCompleter from xonsh.ptk2.key_bindings import load_xonsh_bindings from prompt_toolkit.auto_suggest import AutoSuggestFromHistory from prompt_toolkit.lexers import PygmentsLexer from prompt_toolkit.enums import EditingMode from prompt_toolkit.key_binding import KeyBindings from prompt_toolkit.history import ThreadedHistory from prompt_toolkit.shortcuts import print_formatted_text as ptk_print from prompt_toolkit.shortcuts import CompleteStyle from prompt_toolkit.shortcuts.prompt import PromptSession from prompt_toolkit.formatted_text import PygmentsTokens from prompt_toolkit.styles import merge_styles, Style from prompt_toolkit.styles.pygments import ( style_from_pygments_cls, style_from_pygments_dict, ) Token = _TokenType() events.transmogrify("on_ptk_create", "LoadEvent") events.doc( "on_ptk_create", """ on_ptk_create(prompter: PromptSession, history: PromptToolkitHistory, completer: PromptToolkitCompleter, bindings: KeyBindings) -> Fired after prompt toolkit has been initialized """, ) class PromptToolkit2Shell(BaseShell): """The xonsh shell for prompt_toolkit v2.""" completion_displays_to_styles = { "multi": CompleteStyle.MULTI_COLUMN, "single": CompleteStyle.COLUMN, "readline": CompleteStyle.READLINE_LIKE, "none": None, } def __init__(self, **kwargs): super().__init__(**kwargs) if ON_WINDOWS: winutils.enable_virtual_terminal_processing() self._first_prompt = True self.history = ThreadedHistory(PromptToolkitHistory()) self.prompter = PromptSession(history=self.history) self.pt_completer = PromptToolkitCompleter(self.completer, self.ctx, self) self.key_bindings = KeyBindings() load_xonsh_bindings(self.key_bindings) # Store original `_history_matches` in case we need to restore it self._history_matches_orig = self.prompter.default_buffer._history_matches # This assumes that PromptToolkit2Shell is a singleton events.on_ptk_create.fire( prompter=self.prompter, history=self.history, completer=self.pt_completer, bindings=self.key_bindings, ) def singleline( self, auto_suggest=None, enable_history_search=True, multiline=True, **kwargs ): """Reads a single line of input from the shell. The store_in_history kwarg flags whether the input should be stored in PTK's in-memory history. """ events.on_pre_prompt.fire() env = builtins.__xonsh__.env mouse_support = env.get("MOUSE_SUPPORT") auto_suggest = auto_suggest if env.get("AUTO_SUGGEST") else None completions_display = env.get("COMPLETIONS_DISPLAY") complete_style = self.completion_displays_to_styles[completions_display] complete_while_typing = env.get("UPDATE_COMPLETIONS_ON_KEYPRESS") if complete_while_typing: # PTK requires history search to be none when completing while typing enable_history_search = False if HAS_PYGMENTS: self.styler.style_name = env.get("XONSH_COLOR_STYLE") completer = None if completions_display == "none" else self.pt_completer if env.get("UPDATE_PROMPT_ON_KEYPRESS"): get_prompt_tokens = self.prompt_tokens get_rprompt_tokens = self.rprompt_tokens get_bottom_toolbar_tokens = self.bottom_toolbar_tokens else: get_prompt_tokens = self.prompt_tokens() get_rprompt_tokens = self.rprompt_tokens() get_bottom_toolbar_tokens = self.bottom_toolbar_tokens() if env.get("VI_MODE"): editing_mode = EditingMode.VI else: editing_mode = EditingMode.EMACS if env.get("XONSH_HISTORY_MATCH_ANYWHERE"): self.prompter.default_buffer._history_matches = MethodType( _cust_history_matches, self.prompter.default_buffer ) elif ( self.prompter.default_buffer._history_matches is not self._history_matches_orig ): self.prompter.default_buffer._history_matches = self._history_matches_orig prompt_args = { "mouse_support": mouse_support, "auto_suggest": auto_suggest, "message": get_prompt_tokens, "rprompt": get_rprompt_tokens, "bottom_toolbar": get_bottom_toolbar_tokens, "completer": completer, "multiline": multiline, "editing_mode": editing_mode, "prompt_continuation": self.continuation_tokens, "enable_history_search": enable_history_search, "reserve_space_for_menu": 10, "key_bindings": self.key_bindings, "complete_style": complete_style, "complete_while_typing": complete_while_typing, "include_default_pygments_style": False, } if builtins.__xonsh__.env.get("COLOR_INPUT"): style = style_from_pygments_dict(DEFAULT_STYLE_DICT) prompt_args["style"] = style style_overrides_env = env.get("PTK_STYLE_OVERRIDES") if style_overrides_env: try: style_overrides = Style.from_dict(style_overrides_env) prompt_args["style"] = merge_styles([style, style_overrides]) except (AttributeError, TypeError, ValueError): print_exception() line = self.prompter.prompt(**prompt_args) events.on_post_prompt.fire() return line def _push(self, line): """Pushes a line onto the buffer and compiles the code in a way that enables multiline input. """ code = None self.buffer.append(line) if self.need_more_lines: return None, code src = "".join(self.buffer) src = transform_command(src) try: code = self.execer.compile(src, mode="single", glbs=self.ctx, locs=None) self.reset_buffer() except Exception: # pylint: disable=broad-except self.reset_buffer() print_exception() return src, None return src, code def cmdloop(self, intro=None): """Enters a loop that reads and execute input from user.""" if intro: print(intro) auto_suggest = AutoSuggestFromHistory() self.push = self._push while not builtins.__xonsh__.exit: try: line = self.singleline(auto_suggest=auto_suggest) if not line: self.emptyline() else: line = self.precmd(line) self.default(line) except (KeyboardInterrupt, SystemExit): self.reset_buffer() except EOFError: if builtins.__xonsh__.env.get("IGNOREEOF"): print('Use "exit" to leave the shell.', file=sys.stderr) else: break def prompt_tokens(self): """Returns a list of (token, str) tuples for the current prompt.""" p = builtins.__xonsh__.env.get("PROMPT") try: p = self.prompt_formatter(p) except Exception: # pylint: disable=broad-except print_exception() toks = partial_color_tokenize(p) if self._first_prompt: carriage_return() self._first_prompt = False self.settitle() return PygmentsTokens(toks) def rprompt_tokens(self): """Returns a list of (token, str) tuples for the current right prompt. """ p = builtins.__xonsh__.env.get("RIGHT_PROMPT") # self.prompt_formatter does handle empty strings properly, # but this avoids descending into it in the common case of # $RIGHT_PROMPT == ''. if isinstance(p, str) and len(p) == 0: return [] try: p = self.prompt_formatter(p) except Exception: # pylint: disable=broad-except print_exception() toks = partial_color_tokenize(p) return PygmentsTokens(toks) def bottom_toolbar_tokens(self): """Returns a list of (token, str) tuples for the current bottom toolbar. """ p = builtins.__xonsh__.env.get("BOTTOM_TOOLBAR") if not p: return try: p = self.prompt_formatter(p) except Exception: # pylint: disable=broad-except print_exception() toks = partial_color_tokenize(p) return PygmentsTokens(toks) def continuation_tokens(self, width, line_number, is_soft_wrap=False): """Displays dots in multiline prompt""" if is_soft_wrap: return "" width = width - 1 dots = builtins.__xonsh__.env.get("MULTILINE_PROMPT") dots = dots() if callable(dots) else dots if dots is None: return [(Token, " " * (width + 1))] basetoks = self.format_color(dots) baselen = sum(len(t[1]) for t in basetoks) if baselen == 0: return [(Token, " " * (width + 1))] toks = basetoks * (width // baselen) n = width % baselen count = 0 for tok in basetoks: slen = len(tok[1]) newcount = slen + count if slen == 0: continue elif newcount <= n: toks.append(tok) else: toks.append((tok[0], tok[1][: n - count])) count = newcount if n <= count: break toks.append((Token, " ")) # final space return PygmentsTokens(toks) def format_color(self, string, hide=False, force_string=False, **kwargs): """Formats a color string using Pygments. This, therefore, returns a list of (Token, str) tuples. If force_string is set to true, though, this will return a color formatted string. """ tokens = partial_color_tokenize(string) if force_string and HAS_PYGMENTS: env = builtins.__xonsh__.env self.styler.style_name = env.get("XONSH_COLOR_STYLE") proxy_style = pyghooks.xonsh_style_proxy(self.styler) formatter = pyghooks.XonshTerminal256Formatter(style=proxy_style) s = pygments.format(tokens, formatter) return s elif force_string: print("To force colorization of string, install Pygments") return tokens else: return tokens def print_color(self, string, end="\n", **kwargs): """Prints a color string using prompt-toolkit color management.""" if isinstance(string, str): tokens = partial_color_tokenize(string) else: # assume this is a list of (Token, str) tuples and just print tokens = string tokens = PygmentsTokens(tokens) if HAS_PYGMENTS: env = builtins.__xonsh__.env self.styler.style_name = env.get("XONSH_COLOR_STYLE") proxy_style = style_from_pygments_cls( pyghooks.xonsh_style_proxy(self.styler) ) else: proxy_style = style_from_pygments_dict(DEFAULT_STYLE_DICT) ptk_print( tokens, style=proxy_style, end=end, include_default_pygments_style=False ) def color_style_names(self): """Returns an iterable of all available style names.""" if not HAS_PYGMENTS: return ["For other xonsh styles, please install pygments"] return get_all_styles() def color_style(self): """Returns the current color map.""" if not HAS_PYGMENTS: return DEFAULT_STYLE_DICT env = builtins.__xonsh__.env self.styler.style_name = env.get("XONSH_COLOR_STYLE") return self.styler.styles def restore_tty_sanity(self): """An interface for resetting the TTY stdin mode. This is highly dependent on the shell backend. Also it is mostly optional since it only affects ^Z backgrounding behaviour. """ # PTK does not seem to need any specialization here. However, # if it does for some reason in the future... # The following writes an ANSI escape sequence that sends the cursor # to the end of the line. This has the effect of restoring ECHO mode. # See http://unix.stackexchange.com/a/108014/129048 for more details. # This line can also be replaced by os.system("stty sane"), as per # http://stackoverflow.com/questions/19777129/interactive-python-interpreter-run-in-background#comment29421919_19778355 # However, it is important to note that not termios-based solution # seems to work. My guess is that this is because termios restoration # needs to be performed by the subprocess itself. This fix is important # when subprocesses don't properly restore the terminal attributes, # like Python in interactive mode. Also note that the sequences "\033M" # and "\033E" seem to work too, but these are technically VT100 codes. # I used the more primitive ANSI sequence to maximize compatibility. # -scopatz 2017-01-28 # if not ON_POSIX: # return # sys.stdout.write('\033[9999999C\n') ================================================ FILE: xonsh/pyghooks.py ================================================ # -*- coding: utf-8 -*- """Hooks for pygments syntax highlighting.""" import os import re import sys import builtins from collections import ChainMap from collections.abc import MutableMapping from pygments.lexer import inherit, bygroups, include from pygments.lexers.agile import PythonLexer from pygments.token import ( Keyword, Name, Comment, String, Error, Number, Operator, Generic, Whitespace, Token, Punctuation, Text, ) from pygments.style import Style import pygments.util from xonsh.commands_cache import CommandsCache from xonsh.lazyasd import LazyObject, LazyDict, lazyobject from xonsh.tools import ( ON_WINDOWS, intensify_colors_for_cmd_exe, ansicolors_to_ptk1_names, ANSICOLOR_NAMES_MAP, PTK_NEW_OLD_COLOR_MAP, hardcode_colors_for_win10, FORMATTER, ) from xonsh.color_tools import ( RE_BACKGROUND, BASE_XONSH_COLORS, make_palette, find_closest_color, ) from xonsh.style_tools import norm_name from xonsh.lazyimps import terminal256 from xonsh.platform import ( os_environ, win_ansi_support, ptk_version_info, pygments_version_info, ) from xonsh.pygments_cache import get_style_by_name def _command_is_valid(cmd): try: cmd_abspath = os.path.abspath(os.path.expanduser(cmd)) except (FileNotFoundError, OSError): return False return cmd in builtins.__xonsh__.commands_cache or ( os.path.isfile(cmd_abspath) and os.access(cmd_abspath, os.X_OK) ) def _command_is_autocd(cmd): if not builtins.__xonsh__.env.get("AUTO_CD", False): return False try: cmd_abspath = os.path.abspath(os.path.expanduser(cmd)) except (FileNotFoundError, OSError): return False return os.path.isdir(cmd_abspath) def subproc_cmd_callback(_, match): """Yield Builtin token if match contains valid command, otherwise fallback to fallback lexer. """ cmd = match.group() yield match.start(), Name.Builtin if _command_is_valid(cmd) else Error, cmd def subproc_arg_callback(_, match): """Check if match contains valid path""" text = match.group() try: ispath = os.path.exists(os.path.expanduser(text)) except (FileNotFoundError, OSError): ispath = False yield (match.start(), Name.Constant if ispath else Text, text) COMMAND_TOKEN_RE = r'[^=\s\[\]{}()$"\'`<&|;!]+(?=\s|$|\)|\]|\}|!)' class XonshLexer(PythonLexer): """Xonsh console lexer for pygments.""" name = "Xonsh lexer" aliases = ["xonsh", "xsh"] filenames = ["*.xsh", "*xonshrc"] def __init__(self, *args, **kwargs): # If the lexer is loaded as a pygment plugin, we have to mock # __xonsh__.env and __xonsh__.commands_cache if not hasattr(builtins, "__xonsh__"): from argparse import Namespace setattr(builtins, "__xonsh__", Namespace()) if not hasattr(builtins.__xonsh__, "env"): setattr(builtins.__xonsh__, "env", {}) if ON_WINDOWS: pathext = os_environ.get("PATHEXT", [".EXE", ".BAT", ".CMD"]) builtins.__xonsh__.env["PATHEXT"] = pathext.split(os.pathsep) if not hasattr(builtins.__xonsh__, "commands_cache"): setattr(builtins.__xonsh__, "commands_cache", CommandsCache()) _ = builtins.__xonsh__.commands_cache.all_commands # NOQA super().__init__(*args, **kwargs) tokens = { "mode_switch_brackets": [ (r"(\$)(\{)", bygroups(Keyword, Punctuation), "py_curly_bracket"), (r"(@)(\()", bygroups(Keyword, Punctuation), "py_bracket"), ( r"([\!\$])(\()", bygroups(Keyword, Punctuation), ("subproc_bracket", "subproc_start"), ), ( r"(@\$)(\()", bygroups(Keyword, Punctuation), ("subproc_bracket", "subproc_start"), ), ( r"([\!\$])(\[)", bygroups(Keyword, Punctuation), ("subproc_square_bracket", "subproc_start"), ), (r"(g?)(`)", bygroups(String.Affix, String.Backtick), "backtick_re"), ], "subproc_bracket": [(r"\)", Punctuation, "#pop"), include("subproc")], "subproc_square_bracket": [(r"\]", Punctuation, "#pop"), include("subproc")], "py_bracket": [(r"\)", Punctuation, "#pop"), include("root")], "py_curly_bracket": [(r"\}", Punctuation, "#pop"), include("root")], "backtick_re": [ (r"[\.\^\$\*\+\?\[\]\|]", String.Regex), (r"({[0-9]+}|{[0-9]+,[0-9]+})\??", String.Regex), (r"\\([0-9]+|[AbBdDsSwWZabfnrtuUvx\\])", String.Escape), (r"`", String.Backtick, "#pop"), (r"[^`\.\^\$\*\+\?\[\]\|]+", String.Backtick), ], "root": [ (r"\?", Keyword), (r"(?<=\w)!", Keyword), (r"\$\w+", Name.Variable), (r"\(", Punctuation, "py_bracket"), (r"\{", Punctuation, "py_curly_bracket"), include("mode_switch_brackets"), inherit, ], "subproc_start": [ (r"\s+", Whitespace), (COMMAND_TOKEN_RE, subproc_cmd_callback, "#pop"), (r"", Whitespace, "#pop"), ], "subproc": [ include("mode_switch_brackets"), (r"&&|\|\|", Operator, "subproc_start"), (r'"(\\\\|\\[0-7]+|\\.|[^"\\])*"', String.Double), (r"'(\\\\|\\[0-7]+|\\.|[^'\\])*'", String.Single), (r"(?<=\w|\s)!", Keyword, "subproc_macro"), (r"^!", Keyword, "subproc_macro"), (r";", Punctuation, "subproc_start"), (r"&|=", Punctuation), (r"\|", Punctuation, "subproc_start"), (r"\s+", Text), (r'[^=\s\[\]{}()$"\'`<&|;]+', subproc_arg_callback), (r"<", Text), (r"\$\w+", Name.Variable), ], "subproc_macro": [ (r"(\s*)([^\n]+)", bygroups(Whitespace, String)), (r"", Whitespace, "#pop"), ], } def get_tokens_unprocessed(self, text): """Check first command, then call super.get_tokens_unprocessed with root or subproc state""" start = 0 state = ("root",) m = re.match(r"(\s*)({})".format(COMMAND_TOKEN_RE), text) if m is not None: yield m.start(1), Whitespace, m.group(1) cmd = m.group(2) cmd_is_valid = _command_is_valid(cmd) cmd_is_autocd = _command_is_autocd(cmd) if cmd_is_valid or cmd_is_autocd: yield (m.start(2), Name.Builtin if cmd_is_valid else Name.Constant, cmd) start = m.end(2) state = ("subproc",) for i, t, v in super().get_tokens_unprocessed(text[start:], state): yield i + start, t, v class XonshConsoleLexer(XonshLexer): """Xonsh console lexer for pygments.""" name = "Xonsh console lexer" aliases = ["xonshcon"] filenames = [] tokens = { "root": [ (r"^(>>>|\.\.\.) ", Generic.Prompt), (r"\n(>>>|\.\.\.)", Generic.Prompt), (r"\n(?![>.][>.][>.] )([^\n]*)", Generic.Output), (r"\n(?![>.][>.][>.] )(.*?)$", Generic.Output), inherit, ] } # # Colors and Styles # Color = Token.Color # alias to new color token namespace def color_by_name(name, fg=None, bg=None): """Converts a color name to a color token, foreground name, and background name. Will take into consideration current foreground and background colors, if provided. Parameters ---------- name : str Color name. fg : str, optional Foreground color name. bg : str, optional Background color name. Returns ------- tok : Token Pygments Token.Color subclass fg : str or None New computed foreground color name. bg : str or None New computed background color name. """ name = name.upper() if name == "NO_COLOR": return Color.NO_COLOR, None, None m = RE_BACKGROUND.search(name) if m is None: # must be foreground color fg = norm_name(name) else: bg = norm_name(name) # assemble token if fg is None and bg is None: tokname = "NO_COLOR" elif fg is None: tokname = bg elif bg is None: tokname = fg else: tokname = fg + "__" + bg tok = getattr(Color, tokname) return tok, fg, bg def code_by_name(name, styles): """Converts a token name into a pygments-style color code. Parameters ---------- name : str Color token name. styles : Mapping Mapping for looking up non-hex colors Returns ------- code : str Pygments style color code. """ fg, _, bg = name.lower().partition("__") if fg.startswith("background_"): fg, bg = bg, fg codes = [] # foreground color if len(fg) == 0: pass elif "hex" in fg: for p in fg.split("_"): codes.append("#" + p[3:] if p.startswith("hex") else p) else: fgtok = getattr(Color, fg.upper()) if fgtok in styles: codes.append(styles[fgtok]) else: codes += fg.split("_") # background color if len(bg) == 0: pass elif bg.startswith("background_hex"): codes.append("bg:#" + bg[14:]) else: bgtok = getattr(Color, bg.upper()) if bgtok in styles: codes.append(styles[bgtok]) else: codes.append(bg.replace("background_", "bg:")) code = " ".join(codes) return code def partial_color_tokenize(template): """Tokenizes a template string containing colors. Will return a list of tuples mapping the token to the string which has that color. These sub-strings maybe templates themselves. """ if builtins.__xonsh__.shell is not None: styles = __xonsh__.shell.shell.styler.styles else: styles = None color = Color.NO_COLOR try: toks, color = _partial_color_tokenize_main(template, styles) except Exception: toks = [(Color.NO_COLOR, template)] if styles is not None: styles[color] # ensure color is available return toks def _partial_color_tokenize_main(template, styles): bopen = "{" bclose = "}" colon = ":" expl = "!" color = Color.NO_COLOR fg = bg = None value = "" toks = [] for literal, field, spec, conv in FORMATTER.parse(template): if field is None: value += literal elif field in KNOWN_COLORS or "#" in field: value += literal next_color, fg, bg = color_by_name(field, fg, bg) if next_color is not color: if len(value) > 0: toks.append((color, value)) if styles is not None: styles[color] # ensure color is available color = next_color value = "" elif field is not None: parts = [literal, bopen, field] if conv is not None and len(conv) > 0: parts.append(expl) parts.append(conv) if spec is not None and len(spec) > 0: parts.append(colon) parts.append(spec) parts.append(bclose) value += "".join(parts) else: value += literal toks.append((color, value)) return toks, color class CompoundColorMap(MutableMapping): """Looks up color tokens by name, potentially generating the value from the lookup. """ def __init__(self, styles, *args, **kwargs): self.styles = styles self.colors = dict(*args, **kwargs) def __getitem__(self, key): if key in self.colors: return self.colors[key] if key in self.styles: value = self.styles[key] self[key] = value return value if key is Color: raise KeyError pre, _, name = str(key).rpartition(".") if pre != "Token.Color": raise KeyError value = code_by_name(name, self.styles) self[key] = value return value def __setitem__(self, key, value): self.colors[key] = value def __delitem__(self, key): del self.colors[key] def __iter__(self): yield from self.colors.keys() def __len__(self): return len(self.colors) class XonshStyle(Style): """A xonsh pygments style that will dispatch to the correct color map by using a ChainMap. The style_name property may be used to reset the current style. """ def __init__(self, style_name="default"): """ Parameters ---------- style_name : str, optional The style name to initialize with. """ self.trap = {} # for trapping custom colors set by user self._smap = {} self._style_name = "" self.style_name = style_name super().__init__() @property def style_name(self): return self._style_name @style_name.setter def style_name(self, value): if self._style_name == value: return if value not in STYLES: try: # loading style dynamically pygments_style_by_name(value) except Exception: print( "Could not find style {0!r}, using default".format(value), file=sys.stderr, ) value = "default" builtins.__xonsh__.env["XONSH_COLOR_STYLE"] = value cmap = STYLES[value] if value == "default": self._smap = XONSH_BASE_STYLE.copy() else: try: self._smap = get_style_by_name(value)().styles.copy() except (ImportError, pygments.util.ClassNotFound): self._smap = XONSH_BASE_STYLE.copy() compound = CompoundColorMap(ChainMap(self.trap, cmap, PTK_STYLE, self._smap)) self.styles = ChainMap(self.trap, cmap, PTK_STYLE, self._smap, compound) self._style_name = value # Convert new ansicolor names to old PTK1 names # Can be remvoed when PTK1 support is dropped. if builtins.__xonsh__.shell.shell_type != "prompt_toolkit2": for smap in [self.trap, cmap, PTK_STYLE, self._smap]: smap.update(ansicolors_to_ptk1_names(smap)) if ON_WINDOWS and "prompt_toolkit" in builtins.__xonsh__.shell.shell_type: self.enhance_colors_for_cmd_exe() @style_name.deleter def style_name(self): self._style_name = "" def enhance_colors_for_cmd_exe(self): """ Enhance colors when using cmd.exe on windows. When using the default style all blue and dark red colors are changed to CYAN and intense red. """ env = builtins.__xonsh__.env # Ensure we are not using ConEmu or Visual Stuio Code if "CONEMUANSI" in env or "VSCODE_PID" in env: return if env.get("INTENSIFY_COLORS_ON_WIN", False): if win_ansi_support(): newcolors = hardcode_colors_for_win10(self.styles) else: newcolors = intensify_colors_for_cmd_exe(self.styles) self.trap.update(newcolors) def xonsh_style_proxy(styler): """Factory for a proxy class to a xonsh style.""" # Monky patch pygments' list of known ansi colors # with the new ansi color names used by PTK2 # Can be removed once pygment names get fixed. pygments.style.ansicolors.update(ANSICOLOR_NAMES_MAP) class XonshStyleProxy(Style): """Simple proxy class to fool prompt toolkit.""" target = styler styles = styler.styles def __new__(cls, *args, **kwargs): return cls.target return XonshStyleProxy PTK_STYLE = LazyObject( lambda: { Token.Menu.Completions: "bg:ansigray ansiblack", Token.Menu.Completions.Completion: "", Token.Menu.Completions.Completion.Current: "bg:ansibrightblack ansiwhite", Token.Scrollbar: "bg:ansibrightblack", Token.Scrollbar.Button: "bg:ansiblack", Token.Scrollbar.Arrow: "bg:ansiblack ansiwhite bold", Token.AutoSuggestion: "ansibrightblack", Token.Aborted: "ansibrightblack", }, globals(), "PTK_STYLE", ) XONSH_BASE_STYLE = LazyObject( lambda: { Whitespace: "ansigray", Comment: "underline ansicyan", Comment.Preproc: "underline ansiyellow", Keyword: "bold ansigreen", Keyword.Pseudo: "nobold", Keyword.Type: "nobold ansired", Operator: "ansibrightblack", Operator.Word: "bold ansimagenta", Name.Builtin: "ansigreen", Name.Function: "ansibrightblue", Name.Class: "bold ansibrightblue", Name.Namespace: "bold ansibrightblue", Name.Exception: "bold ansibrightred", Name.Variable: "ansiblue", Name.Constant: "ansired", Name.Label: "ansibrightyellow", Name.Entity: "bold ansigray", Name.Attribute: "ansibrightyellow", Name.Tag: "bold ansigreen", Name.Decorator: "ansibrightmagenta", String: "ansibrightred", String.Doc: "underline", String.Interpol: "bold ansimagenta", String.Escape: "bold ansiyellow", String.Regex: "ansimagenta", String.Symbol: "ansiyellow", String.Other: "ansigreen", Number: "ansibrightblack", Generic.Heading: "bold ansiblue", Generic.Subheading: "bold ansimagenta", Generic.Deleted: "ansired", Generic.Inserted: "ansibrightgreen", Generic.Error: "bold ansibrightred", Generic.Emph: "underline", Generic.Prompt: "bold ansiblue", Generic.Output: "ansiblue", Generic.Traceback: "ansiblue", Error: "ansibrightred", }, globals(), "XONSH_BASE_STYLE", ) KNOWN_COLORS = LazyObject( lambda: frozenset( [ "BACKGROUND_BLACK", "BACKGROUND_BLUE", "BACKGROUND_CYAN", "BACKGROUND_GREEN", "BACKGROUND_INTENSE_BLACK", "BACKGROUND_INTENSE_BLUE", "BACKGROUND_INTENSE_CYAN", "BACKGROUND_INTENSE_GREEN", "BACKGROUND_INTENSE_PURPLE", "BACKGROUND_INTENSE_RED", "BACKGROUND_INTENSE_WHITE", "BACKGROUND_INTENSE_YELLOW", "BACKGROUND_PURPLE", "BACKGROUND_RED", "BACKGROUND_WHITE", "BACKGROUND_YELLOW", "BLACK", "BLUE", "BOLD_BLACK", "BOLD_BLUE", "BOLD_CYAN", "BOLD_GREEN", "BOLD_INTENSE_BLACK", "BOLD_INTENSE_BLUE", "BOLD_INTENSE_CYAN", "BOLD_INTENSE_GREEN", "BOLD_INTENSE_PURPLE", "BOLD_INTENSE_RED", "BOLD_INTENSE_WHITE", "BOLD_INTENSE_YELLOW", "BOLD_PURPLE", "BOLD_RED", "BOLD_UNDERLINE_BLACK", "BOLD_UNDERLINE_BLUE", "BOLD_UNDERLINE_CYAN", "BOLD_UNDERLINE_GREEN", "BOLD_UNDERLINE_INTENSE_BLACK", "BOLD_UNDERLINE_INTENSE_BLUE", "BOLD_UNDERLINE_INTENSE_CYAN", "BOLD_UNDERLINE_INTENSE_GREEN", "BOLD_UNDERLINE_INTENSE_PURPLE", "BOLD_UNDERLINE_INTENSE_RED", "BOLD_UNDERLINE_INTENSE_WHITE", "BOLD_UNDERLINE_INTENSE_YELLOW", "BOLD_UNDERLINE_PURPLE", "BOLD_UNDERLINE_RED", "BOLD_UNDERLINE_WHITE", "BOLD_UNDERLINE_YELLOW", "BOLD_WHITE", "BOLD_YELLOW", "CYAN", "GREEN", "INTENSE_BLACK", "INTENSE_BLUE", "INTENSE_CYAN", "INTENSE_GREEN", "INTENSE_PURPLE", "INTENSE_RED", "INTENSE_WHITE", "INTENSE_YELLOW", "NO_COLOR", "PURPLE", "RED", "UNDERLINE_BLACK", "UNDERLINE_BLUE", "UNDERLINE_CYAN", "UNDERLINE_GREEN", "UNDERLINE_INTENSE_BLACK", "UNDERLINE_INTENSE_BLUE", "UNDERLINE_INTENSE_CYAN", "UNDERLINE_INTENSE_GREEN", "UNDERLINE_INTENSE_PURPLE", "UNDERLINE_INTENSE_RED", "UNDERLINE_INTENSE_WHITE", "UNDERLINE_INTENSE_YELLOW", "UNDERLINE_PURPLE", "UNDERLINE_RED", "UNDERLINE_WHITE", "UNDERLINE_YELLOW", "WHITE", "YELLOW", ] ), globals(), "KNOWN_COLORS", ) def _expand_style(cmap): """Expands a style in order to more quickly make color map changes.""" for key, val in list(cmap.items()): if key is Color.NO_COLOR: continue _, _, key = str(key).rpartition(".") cmap[getattr(Color, "BOLD_" + key)] = "bold " + val cmap[getattr(Color, "UNDERLINE_" + key)] = "underline " + val cmap[getattr(Color, "BOLD_UNDERLINE_" + key)] = "bold underline " + val if val == "noinherit": cmap[getattr(Color, "BACKGROUND_" + key)] = val else: cmap[getattr(Color, "BACKGROUND_" + key)] = "bg:" + val def _bw_style(): style = { Color.BLACK: "noinherit", Color.BLUE: "noinherit", Color.CYAN: "noinherit", Color.GREEN: "noinherit", Color.INTENSE_BLACK: "noinherit", Color.INTENSE_BLUE: "noinherit", Color.INTENSE_CYAN: "noinherit", Color.INTENSE_GREEN: "noinherit", Color.INTENSE_PURPLE: "noinherit", Color.INTENSE_RED: "noinherit", Color.INTENSE_WHITE: "noinherit", Color.INTENSE_YELLOW: "noinherit", Color.NO_COLOR: "noinherit", Color.PURPLE: "noinherit", Color.RED: "noinherit", Color.WHITE: "noinherit", Color.YELLOW: "noinherit", } _expand_style(style) return style def _default_style(): style = { Color.BLACK: "ansiblack", Color.BLUE: "ansiblue", Color.CYAN: "ansicyan", Color.GREEN: "ansigreen", Color.INTENSE_BLACK: "ansibrightblack", Color.INTENSE_BLUE: "ansibrightblue", Color.INTENSE_CYAN: "ansibrightcyan", Color.INTENSE_GREEN: "ansibrightgreen", Color.INTENSE_PURPLE: "ansibrightmagenta", Color.INTENSE_RED: "ansibrightred", Color.INTENSE_WHITE: "ansiwhite", Color.INTENSE_YELLOW: "ansibrightyellow", Color.NO_COLOR: "noinherit", Color.PURPLE: "ansimagenta", Color.RED: "ansired", Color.WHITE: "ansigray", Color.YELLOW: "ansiyellow", } _expand_style(style) return style def _monokai_style(): style = { Color.BLACK: "#1e0010", Color.BLUE: "#6666ef", Color.CYAN: "#66d9ef", Color.GREEN: "#2ee22e", Color.INTENSE_BLACK: "#5e5e5e", Color.INTENSE_BLUE: "#2626d7", Color.INTENSE_CYAN: "#2ed9d9", Color.INTENSE_GREEN: "#a6e22e", Color.INTENSE_PURPLE: "#ae81ff", Color.INTENSE_RED: "#f92672", Color.INTENSE_WHITE: "#f8f8f2", Color.INTENSE_YELLOW: "#e6db74", Color.NO_COLOR: "noinherit", Color.PURPLE: "#960050", Color.RED: "#AF0000", Color.WHITE: "#d7d7d7", Color.YELLOW: "#e2e22e", } _expand_style(style) return style ###################################### # Auto-generated below this line # ###################################### def _algol_style(): style = { Color.BLACK: "#666", Color.BLUE: "#666", Color.CYAN: "#666", Color.GREEN: "#666", Color.INTENSE_BLACK: "#666", Color.INTENSE_BLUE: "#888", Color.INTENSE_CYAN: "#888", Color.INTENSE_GREEN: "#888", Color.INTENSE_PURPLE: "#888", Color.INTENSE_RED: "#FF0000", Color.INTENSE_WHITE: "#888", Color.INTENSE_YELLOW: "#888", Color.NO_COLOR: "noinherit", Color.PURPLE: "#666", Color.RED: "#FF0000", Color.WHITE: "#888", Color.YELLOW: "#FF0000", } _expand_style(style) return style def _algol_nu_style(): style = { Color.BLACK: "#666", Color.BLUE: "#666", Color.CYAN: "#666", Color.GREEN: "#666", Color.INTENSE_BLACK: "#666", Color.INTENSE_BLUE: "#888", Color.INTENSE_CYAN: "#888", Color.INTENSE_GREEN: "#888", Color.INTENSE_PURPLE: "#888", Color.INTENSE_RED: "#FF0000", Color.INTENSE_WHITE: "#888", Color.INTENSE_YELLOW: "#888", Color.NO_COLOR: "noinherit", Color.PURPLE: "#666", Color.RED: "#FF0000", Color.WHITE: "#888", Color.YELLOW: "#FF0000", } _expand_style(style) return style def _autumn_style(): style = { Color.BLACK: "#000080", Color.BLUE: "#0000aa", Color.CYAN: "#00aaaa", Color.GREEN: "#00aa00", Color.INTENSE_BLACK: "#555555", Color.INTENSE_BLUE: "#1e90ff", Color.INTENSE_CYAN: "#1e90ff", Color.INTENSE_GREEN: "#4c8317", Color.INTENSE_PURPLE: "#FAA", Color.INTENSE_RED: "#aa5500", Color.INTENSE_WHITE: "#bbbbbb", Color.INTENSE_YELLOW: "#FAA", Color.NO_COLOR: "noinherit", Color.PURPLE: "#800080", Color.RED: "#aa0000", Color.WHITE: "#aaaaaa", Color.YELLOW: "#aa5500", } _expand_style(style) return style def _borland_style(): style = { Color.BLACK: "#000000", Color.BLUE: "#000080", Color.CYAN: "#008080", Color.GREEN: "#008800", Color.INTENSE_BLACK: "#555555", Color.INTENSE_BLUE: "#0000FF", Color.INTENSE_CYAN: "#ddffdd", Color.INTENSE_GREEN: "#888888", Color.INTENSE_PURPLE: "#e3d2d2", Color.INTENSE_RED: "#FF0000", Color.INTENSE_WHITE: "#ffdddd", Color.INTENSE_YELLOW: "#e3d2d2", Color.NO_COLOR: "noinherit", Color.PURPLE: "#800080", Color.RED: "#aa0000", Color.WHITE: "#aaaaaa", Color.YELLOW: "#a61717", } _expand_style(style) return style def _colorful_style(): style = { Color.BLACK: "#000", Color.BLUE: "#00C", Color.CYAN: "#0e84b5", Color.GREEN: "#00A000", Color.INTENSE_BLACK: "#555", Color.INTENSE_BLUE: "#33B", Color.INTENSE_CYAN: "#bbbbbb", Color.INTENSE_GREEN: "#888", Color.INTENSE_PURPLE: "#FAA", Color.INTENSE_RED: "#D42", Color.INTENSE_WHITE: "#fff0ff", Color.INTENSE_YELLOW: "#FAA", Color.NO_COLOR: "noinherit", Color.PURPLE: "#800080", Color.RED: "#A00000", Color.WHITE: "#bbbbbb", Color.YELLOW: "#A60", } _expand_style(style) return style def _emacs_style(): style = { Color.BLACK: "#008000", Color.BLUE: "#000080", Color.CYAN: "#04D", Color.GREEN: "#00A000", Color.INTENSE_BLACK: "#666666", Color.INTENSE_BLUE: "#04D", Color.INTENSE_CYAN: "#bbbbbb", Color.INTENSE_GREEN: "#00BB00", Color.INTENSE_PURPLE: "#AA22FF", Color.INTENSE_RED: "#D2413A", Color.INTENSE_WHITE: "#bbbbbb", Color.INTENSE_YELLOW: "#bbbbbb", Color.NO_COLOR: "noinherit", Color.PURPLE: "#800080", Color.RED: "#A00000", Color.WHITE: "#bbbbbb", Color.YELLOW: "#BB6622", } _expand_style(style) return style def _friendly_style(): style = { Color.BLACK: "#007020", Color.BLUE: "#000080", Color.CYAN: "#0e84b5", Color.GREEN: "#00A000", Color.INTENSE_BLACK: "#555555", Color.INTENSE_BLUE: "#70a0d0", Color.INTENSE_CYAN: "#60add5", Color.INTENSE_GREEN: "#40a070", Color.INTENSE_PURPLE: "#bb60d5", Color.INTENSE_RED: "#d55537", Color.INTENSE_WHITE: "#fff0f0", Color.INTENSE_YELLOW: "#bbbbbb", Color.NO_COLOR: "noinherit", Color.PURPLE: "#800080", Color.RED: "#A00000", Color.WHITE: "#bbbbbb", Color.YELLOW: "#c65d09", } _expand_style(style) return style def _fruity_style(): style = { Color.BLACK: "#0f140f", Color.BLUE: "#0086d2", Color.CYAN: "#0086d2", Color.GREEN: "#008800", Color.INTENSE_BLACK: "#444444", Color.INTENSE_BLUE: "#0086f7", Color.INTENSE_CYAN: "#0086f7", Color.INTENSE_GREEN: "#888888", Color.INTENSE_PURPLE: "#ff0086", Color.INTENSE_RED: "#fb660a", Color.INTENSE_WHITE: "#ffffff", Color.INTENSE_YELLOW: "#cdcaa9", Color.NO_COLOR: "noinherit", Color.PURPLE: "#ff0086", Color.RED: "#ff0007", Color.WHITE: "#cdcaa9", Color.YELLOW: "#fb660a", } _expand_style(style) return style def _igor_style(): style = { Color.BLACK: "#009C00", Color.BLUE: "#0000FF", Color.CYAN: "#007575", Color.GREEN: "#009C00", Color.INTENSE_BLACK: "#007575", Color.INTENSE_BLUE: "#0000FF", Color.INTENSE_CYAN: "#007575", Color.INTENSE_GREEN: "#009C00", Color.INTENSE_PURPLE: "#CC00A3", Color.INTENSE_RED: "#C34E00", Color.INTENSE_WHITE: "#CC00A3", Color.INTENSE_YELLOW: "#C34E00", Color.NO_COLOR: "noinherit", Color.PURPLE: "#CC00A3", Color.RED: "#C34E00", Color.WHITE: "#CC00A3", Color.YELLOW: "#C34E00", } _expand_style(style) return style def _lovelace_style(): style = { Color.BLACK: "#444444", Color.BLUE: "#2838b0", Color.CYAN: "#289870", Color.GREEN: "#388038", Color.INTENSE_BLACK: "#666666", Color.INTENSE_BLUE: "#2838b0", Color.INTENSE_CYAN: "#888888", Color.INTENSE_GREEN: "#289870", Color.INTENSE_PURPLE: "#a848a8", Color.INTENSE_RED: "#b83838", Color.INTENSE_WHITE: "#888888", Color.INTENSE_YELLOW: "#a89028", Color.NO_COLOR: "noinherit", Color.PURPLE: "#a848a8", Color.RED: "#c02828", Color.WHITE: "#888888", Color.YELLOW: "#b85820", } _expand_style(style) return style def _manni_style(): style = { Color.BLACK: "#000000", Color.BLUE: "#000099", Color.CYAN: "#009999", Color.GREEN: "#00CC00", Color.INTENSE_BLACK: "#555555", Color.INTENSE_BLUE: "#9999FF", Color.INTENSE_CYAN: "#00CCFF", Color.INTENSE_GREEN: "#99CC66", Color.INTENSE_PURPLE: "#CC00FF", Color.INTENSE_RED: "#FF6600", Color.INTENSE_WHITE: "#FFCCCC", Color.INTENSE_YELLOW: "#FFCC33", Color.NO_COLOR: "noinherit", Color.PURPLE: "#CC00FF", Color.RED: "#AA0000", Color.WHITE: "#AAAAAA", Color.YELLOW: "#CC3300", } _expand_style(style) return style def _murphy_style(): style = { Color.BLACK: "#000", Color.BLUE: "#000080", Color.CYAN: "#0e84b5", Color.GREEN: "#00A000", Color.INTENSE_BLACK: "#555", Color.INTENSE_BLUE: "#66f", Color.INTENSE_CYAN: "#5ed", Color.INTENSE_GREEN: "#5ed", Color.INTENSE_PURPLE: "#e9e", Color.INTENSE_RED: "#f84", Color.INTENSE_WHITE: "#eee", Color.INTENSE_YELLOW: "#fc8", Color.NO_COLOR: "noinherit", Color.PURPLE: "#800080", Color.RED: "#A00000", Color.WHITE: "#bbbbbb", Color.YELLOW: "#c65d09", } _expand_style(style) return style def _native_style(): style = { Color.BLACK: "#520000", Color.BLUE: "#3677a9", Color.CYAN: "#24909d", Color.GREEN: "#589819", Color.INTENSE_BLACK: "#666666", Color.INTENSE_BLUE: "#447fcf", Color.INTENSE_CYAN: "#40ffff", Color.INTENSE_GREEN: "#6ab825", Color.INTENSE_PURPLE: "#e3d2d2", Color.INTENSE_RED: "#cd2828", Color.INTENSE_WHITE: "#ffffff", Color.INTENSE_YELLOW: "#ed9d13", Color.NO_COLOR: "noinherit", Color.PURPLE: "#666666", Color.RED: "#a61717", Color.WHITE: "#aaaaaa", Color.YELLOW: "#a61717", } _expand_style(style) return style def _paraiso_dark_style(): style = { Color.BLACK: "#776e71", Color.BLUE: "#815ba4", Color.CYAN: "#06b6ef", Color.GREEN: "#48b685", Color.INTENSE_BLACK: "#776e71", Color.INTENSE_BLUE: "#815ba4", Color.INTENSE_CYAN: "#5bc4bf", Color.INTENSE_GREEN: "#48b685", Color.INTENSE_PURPLE: "#e7e9db", Color.INTENSE_RED: "#ef6155", Color.INTENSE_WHITE: "#e7e9db", Color.INTENSE_YELLOW: "#fec418", Color.NO_COLOR: "noinherit", Color.PURPLE: "#815ba4", Color.RED: "#ef6155", Color.WHITE: "#5bc4bf", Color.YELLOW: "#f99b15", } _expand_style(style) return style def _paraiso_light_style(): style = { Color.BLACK: "#2f1e2e", Color.BLUE: "#2f1e2e", Color.CYAN: "#06b6ef", Color.GREEN: "#48b685", Color.INTENSE_BLACK: "#2f1e2e", Color.INTENSE_BLUE: "#815ba4", Color.INTENSE_CYAN: "#5bc4bf", Color.INTENSE_GREEN: "#48b685", Color.INTENSE_PURPLE: "#815ba4", Color.INTENSE_RED: "#ef6155", Color.INTENSE_WHITE: "#5bc4bf", Color.INTENSE_YELLOW: "#fec418", Color.NO_COLOR: "noinherit", Color.PURPLE: "#815ba4", Color.RED: "#2f1e2e", Color.WHITE: "#8d8687", Color.YELLOW: "#f99b15", } _expand_style(style) return style def _pastie_style(): style = { Color.BLACK: "#000000", Color.BLUE: "#0000DD", Color.CYAN: "#0066bb", Color.GREEN: "#008800", Color.INTENSE_BLACK: "#555555", Color.INTENSE_BLUE: "#3333bb", Color.INTENSE_CYAN: "#ddffdd", Color.INTENSE_GREEN: "#22bb22", Color.INTENSE_PURPLE: "#e3d2d2", Color.INTENSE_RED: "#dd7700", Color.INTENSE_WHITE: "#fff0ff", Color.INTENSE_YELLOW: "#e3d2d2", Color.NO_COLOR: "noinherit", Color.PURPLE: "#bb0066", Color.RED: "#aa0000", Color.WHITE: "#bbbbbb", Color.YELLOW: "#aa6600", } _expand_style(style) return style def _perldoc_style(): style = { Color.BLACK: "#000080", Color.BLUE: "#000080", Color.CYAN: "#1e889b", Color.GREEN: "#00aa00", Color.INTENSE_BLACK: "#555555", Color.INTENSE_BLUE: "#B452CD", Color.INTENSE_CYAN: "#bbbbbb", Color.INTENSE_GREEN: "#228B22", Color.INTENSE_PURPLE: "#B452CD", Color.INTENSE_RED: "#CD5555", Color.INTENSE_WHITE: "#e3d2d2", Color.INTENSE_YELLOW: "#e3d2d2", Color.NO_COLOR: "noinherit", Color.PURPLE: "#8B008B", Color.RED: "#aa0000", Color.WHITE: "#a7a7a7", Color.YELLOW: "#cb6c20", } _expand_style(style) return style def _rrt_style(): style = { Color.BLACK: "#ff0000", Color.BLUE: "#87ceeb", Color.CYAN: "#87ceeb", Color.GREEN: "#00ff00", Color.INTENSE_BLACK: "#87ceeb", Color.INTENSE_BLUE: "#87ceeb", Color.INTENSE_CYAN: "#7fffd4", Color.INTENSE_GREEN: "#00ff00", Color.INTENSE_PURPLE: "#ee82ee", Color.INTENSE_RED: "#ff0000", Color.INTENSE_WHITE: "#e5e5e5", Color.INTENSE_YELLOW: "#eedd82", Color.NO_COLOR: "noinherit", Color.PURPLE: "#ee82ee", Color.RED: "#ff0000", Color.WHITE: "#87ceeb", Color.YELLOW: "#ff0000", } _expand_style(style) return style def _tango_style(): style = { Color.BLACK: "#000000", Color.BLUE: "#0000cf", Color.CYAN: "#3465a4", Color.GREEN: "#00A000", Color.INTENSE_BLACK: "#204a87", Color.INTENSE_BLUE: "#5c35cc", Color.INTENSE_CYAN: "#f8f8f8", Color.INTENSE_GREEN: "#4e9a06", Color.INTENSE_PURPLE: "#f8f8f8", Color.INTENSE_RED: "#ef2929", Color.INTENSE_WHITE: "#f8f8f8", Color.INTENSE_YELLOW: "#c4a000", Color.NO_COLOR: "noinherit", Color.PURPLE: "#800080", Color.RED: "#a40000", Color.WHITE: "#f8f8f8", Color.YELLOW: "#8f5902", } _expand_style(style) return style def _trac_style(): style = { Color.BLACK: "#000000", Color.BLUE: "#000080", Color.CYAN: "#009999", Color.GREEN: "#808000", Color.INTENSE_BLACK: "#555555", Color.INTENSE_BLUE: "#445588", Color.INTENSE_CYAN: "#ddffdd", Color.INTENSE_GREEN: "#999988", Color.INTENSE_PURPLE: "#e3d2d2", Color.INTENSE_RED: "#bb8844", Color.INTENSE_WHITE: "#ffdddd", Color.INTENSE_YELLOW: "#e3d2d2", Color.NO_COLOR: "noinherit", Color.PURPLE: "#800080", Color.RED: "#aa0000", Color.WHITE: "#aaaaaa", Color.YELLOW: "#808000", } _expand_style(style) return style def _vim_style(): style = { Color.BLACK: "#000080", Color.BLUE: "#000080", Color.CYAN: "#00cdcd", Color.GREEN: "#00cd00", Color.INTENSE_BLACK: "#666699", Color.INTENSE_BLUE: "#3399cc", Color.INTENSE_CYAN: "#00cdcd", Color.INTENSE_GREEN: "#00cd00", Color.INTENSE_PURPLE: "#cd00cd", Color.INTENSE_RED: "#FF0000", Color.INTENSE_WHITE: "#cccccc", Color.INTENSE_YELLOW: "#cdcd00", Color.NO_COLOR: "noinherit", Color.PURPLE: "#cd00cd", Color.RED: "#cd0000", Color.WHITE: "#cccccc", Color.YELLOW: "#cd0000", } _expand_style(style) return style def _vs_style(): style = { Color.BLACK: "#008000", Color.BLUE: "#0000ff", Color.CYAN: "#2b91af", Color.GREEN: "#008000", Color.INTENSE_BLACK: "#2b91af", Color.INTENSE_BLUE: "#2b91af", Color.INTENSE_CYAN: "#2b91af", Color.INTENSE_GREEN: "#2b91af", Color.INTENSE_PURPLE: "#2b91af", Color.INTENSE_RED: "#FF0000", Color.INTENSE_WHITE: "#2b91af", Color.INTENSE_YELLOW: "#2b91af", Color.NO_COLOR: "noinherit", Color.PURPLE: "#a31515", Color.RED: "#a31515", Color.WHITE: "#2b91af", Color.YELLOW: "#a31515", } _expand_style(style) return style def _xcode_style(): style = { Color.BLACK: "#000000", Color.BLUE: "#1C01CE", Color.CYAN: "#3F6E75", Color.GREEN: "#177500", Color.INTENSE_BLACK: "#3F6E75", Color.INTENSE_BLUE: "#2300CE", Color.INTENSE_CYAN: "#3F6E75", Color.INTENSE_GREEN: "#3F6E75", Color.INTENSE_PURPLE: "#A90D91", Color.INTENSE_RED: "#C41A16", Color.INTENSE_WHITE: "#3F6E75", Color.INTENSE_YELLOW: "#836C28", Color.NO_COLOR: "noinherit", Color.PURPLE: "#A90D91", Color.RED: "#C41A16", Color.WHITE: "#3F6E75", Color.YELLOW: "#836C28", } _expand_style(style) return style STYLES = LazyDict( { "algol": _algol_style, "algol_nu": _algol_nu_style, "autumn": _autumn_style, "borland": _borland_style, "bw": _bw_style, "colorful": _colorful_style, "default": _default_style, "emacs": _emacs_style, "friendly": _friendly_style, "fruity": _fruity_style, "igor": _igor_style, "lovelace": _lovelace_style, "manni": _manni_style, "monokai": _monokai_style, "murphy": _murphy_style, "native": _native_style, "paraiso-dark": _paraiso_dark_style, "paraiso-light": _paraiso_light_style, "pastie": _pastie_style, "perldoc": _perldoc_style, "rrt": _rrt_style, "tango": _tango_style, "trac": _trac_style, "vim": _vim_style, "vs": _vs_style, "xcode": _xcode_style, }, globals(), "STYLES", ) del ( _algol_style, _algol_nu_style, _autumn_style, _borland_style, _bw_style, _colorful_style, _default_style, _emacs_style, _friendly_style, _fruity_style, _igor_style, _lovelace_style, _manni_style, _monokai_style, _murphy_style, _native_style, _paraiso_dark_style, _paraiso_light_style, _pastie_style, _perldoc_style, _rrt_style, _tango_style, _trac_style, _vim_style, _vs_style, _xcode_style, ) # dynamic styles def make_pygments_style(palette): """Makes a pygments style based on a color palette.""" global Color style = {getattr(Color, "NO_COLOR"): "noinherit"} for name, t in BASE_XONSH_COLORS.items(): color = find_closest_color(t, palette) style[getattr(Color, name)] = "#" + color style[getattr(Color, "BOLD_" + name)] = "bold #" + color style[getattr(Color, "UNDERLINE_" + name)] = "underline #" + color style[getattr(Color, "BOLD_UNDERLINE_" + name)] = "bold underline #" + color style[getattr(Color, "BACKGROUND_" + name)] = "bg:#" + color return style def pygments_style_by_name(name): """Gets or makes a pygments color style by its name.""" if name in STYLES: return STYLES[name] pstyle = get_style_by_name(name) palette = make_palette(pstyle.styles.values()) astyle = make_pygments_style(palette) STYLES[name] = astyle return astyle def _monkey_patch_pygments_codes(): """ Monky patch pygments' dict of console codes, with new color names """ import pygments.console if "brightblack" in pygments.console.codes: # Assume that colors are already fixed in pygments # for example when using pygments from source return if not getattr(pygments.console, "_xonsh_patched", False): patched_codes = {} for new, old in PTK_NEW_OLD_COLOR_MAP.items(): if old in pygments.console.codes: patched_codes[new[1:]] = pygments.console.codes[old] pygments.console.codes.update(patched_codes) pygments.console._xonsh_patched = True # # Formatter # @lazyobject def XonshTerminal256Formatter(): if ( ptk_version_info() and ptk_version_info() > (2, 0) and pygments_version_info() and (2, 2, 0) <= pygments_version_info() < (2, 4, 0) ): # Monky patch pygments' dict of console codes # with the new color names used by PTK2 # Can be removed once pygment names get fixed. _monkey_patch_pygments_codes() class XonshTerminal256FormatterProxy(terminal256.Terminal256Formatter): """Proxy class for xonsh terminal256 formatting that understands. xonsh color tokens. """ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) # just keep the opening token for colors. color_names = set(map(str, Color.subtypes)) for name, (opener, closer) in self.style_string.items(): if name in color_names: self.style_string[name] = (opener, "") # special case NO_COLOR, because it is special. self.style_string["Token.Color.NO_COLOR"] = ("\x1b[39m", "") return XonshTerminal256FormatterProxy ================================================ FILE: xonsh/pygments_cache.py ================================================ """A fast, drop-in replacement for pygments ``get_*()`` and ``guess_*()`` funtions. The following pygments API functions are currently supplied here:: from pygments_cache import get_lexer_for_filename, guess_lexer_for_filename from pygments_cache import get_formatter_for_filename, get_formatter_by_name from pygments_cache import get_style_by_name, get_all_styles from pygments_cache import get_filter_by_name The cache itself is stored at the location given by the ``$PYGMENTS_CACHE_FILE`` environment variable, or by default at ``~/.local/share/pygments-cache/cache.py``. The cache file is created on first use, if it does not already exist. """ import os import importlib # Global storage variables __version__ = "0.1.1" CACHE = None DEBUG = False def _print_duplicate_message(duplicates): import sys for filename, vals in sorted(duplicates.items()): msg = "for {0} ambiquity between:\n ".format(filename) vals = [m + ":" + c for m, c in vals] msg += "\n ".join(sorted(vals)) print(msg, file=sys.stderr) def _discover_lexers(): import inspect from pygments.lexers import get_all_lexers, find_lexer_class # maps file extension (and names) to (module, classname) tuples default_exts = { # C / C++ ".h": ("pygments.lexers.c_cpp", "CLexer"), ".hh": ("pygments.lexers.c_cpp", "CppLexer"), ".cp": ("pygments.lexers.c_cpp", "CppLexer"), # python ".py": ("pygments.lexers.python", "Python3Lexer"), ".pyw": ("pygments.lexers.python", "Python3Lexer"), ".sc": ("pygments.lexers.python", "Python3Lexer"), ".tac": ("pygments.lexers.python", "Python3Lexer"), "SConstruct": ("pygments.lexers.python", "Python3Lexer"), "SConscript": ("pygments.lexers.python", "Python3Lexer"), ".sage": ("pygments.lexers.python", "Python3Lexer"), ".pytb": ("pygments.lexers.python", "Python3TracebackLexer"), # perl ".t": ("pygments.lexers.perl", "Perl6Lexer"), ".pl": ("pygments.lexers.perl", "Perl6Lexer"), ".pm": ("pygments.lexers.perl", "Perl6Lexer"), # asm ".s": ("pygments.lexers.asm", "GasLexer"), ".S": ("pygments.lexers.asm", "GasLexer"), ".asm": ("pygments.lexers.asm", "NasmLexer"), ".ASM": ("pygments.lexers.asm", "NasmLexer"), # Antlr ".g": ("pygments.lexers.parsers", "AntlrCppLexer"), ".G": ("pygments.lexers.parsers", "AntlrCppLexer"), # XML ".xml": ("pygments.lexers.html", "XmlLexer"), ".xsl": ("pygments.lexers.html", "XsltLexer"), ".xslt": ("pygments.lexers.html", "XsltLexer"), # ASP ".axd": ("pygments.lexers.dotnet", "CSharpAspxLexer"), ".asax": ("pygments.lexers.dotnet", "CSharpAspxLexer"), ".ascx": ("pygments.lexers.dotnet", "CSharpAspxLexer"), ".ashx": ("pygments.lexers.dotnet", "CSharpAspxLexer"), ".asmx": ("pygments.lexers.dotnet", "CSharpAspxLexer"), ".aspx": ("pygments.lexers.dotnet", "CSharpAspxLexer"), # misc ".b": ("pygments.lexers.esoteric", "BrainfuckLexer"), ".j": ("pygments.lexers.jvm", "JasminLexer"), ".m": ("pygments.lexers.matlab", "MatlabLexer"), ".n": ("pygments.lexers.dotnet", "NemerleLexer"), ".p": ("pygments.lexers.pawn", "PawnLexer"), ".v": ("pygments.lexers.theorem", "CoqLexer"), ".as": ("pygments.lexers.actionscript", "ActionScript3Lexer"), ".fs": ("pygments.lexers.forth", "ForthLexer"), ".hy": ("pygments.lexers.lisp", "HyLexer"), ".ts": ("pygments.lexers.javascript", "TypeScriptLexer"), ".rl": ("pygments.lexers.parsers", "RagelCppLexer"), ".bas": ("pygments.lexers.basic", "QBasicLexer"), ".bug": ("pygments.lexers.modeling", "BugsLexer"), ".ecl": ("pygments.lexers.ecl", "ECLLexer"), ".inc": ("pygments.lexers.php", "PhpLexer"), ".inf": ("pygments.lexers.configs", "IniLexer"), ".pro": ("pygments.lexers.prolog", "PrologLexer"), ".sql": ("pygments.lexers.sql", "SqlLexer"), ".txt": ("pygments.lexers.special", "TextLexer"), ".html": ("pygments.lexers.html", "HtmlLexer"), } exts = {} lexers = {"exts": exts} if DEBUG: from collections import defaultdict duplicates = defaultdict(set) for longname, aliases, filenames, mimetypes in get_all_lexers(): cls = find_lexer_class(longname) mod = inspect.getmodule(cls) val = (mod.__name__, cls.__name__) for filename in filenames: if filename.startswith("*."): filename = filename[1:] if "*" in filename: continue if ( DEBUG and filename in exts and exts[filename] != val and filename not in default_exts ): duplicates[filename].add(val) duplicates[filename].add(exts[filename]) exts[filename] = val # remove some ambiquity exts.update(default_exts) # print duplicate message if DEBUG: _print_duplicate_message(duplicates) return lexers def _discover_formatters(): import inspect from pygments.formatters import get_all_formatters # maps file extension (and names) to (module, classname) tuples default_exts = {} exts = {} # maps formatter 'name' (not the class name) and alias to (module, classname) tuples default_names = {} names = {} formatters = {"exts": exts, "names": names} if DEBUG: from collections import defaultdict duplicates = defaultdict(set) for cls in get_all_formatters(): mod = inspect.getmodule(cls) val = (mod.__name__, cls.__name__) # add extentions for filename in cls.filenames: if filename.startswith("*."): filename = filename[1:] if "*" in filename: continue if ( DEBUG and filename in exts and exts[filename] != val and filename not in default_exts ): duplicates[filename].add(val) duplicates[filename].add(exts[filename]) exts[filename] = val # add names and aliases names[cls.name] = val for alias in cls.aliases: if ( DEBUG and alias in names and names[alias] != val and alias not in default_names ): duplicates[alias].add(val) duplicates[alias].add(names[alias]) names[alias] = val # remove some ambiquity exts.update(default_exts) names.update(default_names) # print dumplicate message if DEBUG: _print_duplicate_message(duplicates) return formatters def _discover_styles(): import inspect from pygments.styles import get_all_styles, get_style_by_name # maps style 'name' (not the class name) and aliases to (module, classname) tuples default_names = {} names = {} styles = {"names": names} if DEBUG: from collections import defaultdict duplicates = defaultdict(set) for name in get_all_styles(): cls = get_style_by_name(name) mod = inspect.getmodule(cls) val = (mod.__name__, cls.__name__) if DEBUG and name in names and names[name] != val and name not in default_names: duplicates[name].add(val) duplicates[name].add(names[name]) names[name] = val # remove some ambiquity names.update(default_names) # print dumplicate message if DEBUG: _print_duplicate_message(duplicates) return styles def _discover_filters(): import inspect from pygments.filters import get_all_filters, get_filter_by_name # maps filter 'name' (not the class name) to (module, classname) tuples default_names = {} names = {} filters = {"names": names} if DEBUG: from collections import defaultdict duplicates = defaultdict(set) for name in get_all_filters(): filter = get_filter_by_name(name) cls = type(filter) mod = inspect.getmodule(cls) val = (mod.__name__, cls.__name__) if DEBUG and name in names and names[name] != val and name not in default_names: duplicates[name].add(val) duplicates[name].add(names[name]) names[name] = val # remove some ambiquity names.update(default_names) # print dumplicate message if DEBUG: _print_duplicate_message(duplicates) return filters def build_cache(): """Does the hard work of building a cache from nothing.""" cache = {} cache["lexers"] = _discover_lexers() cache["formatters"] = _discover_formatters() cache["styles"] = _discover_styles() cache["filters"] = _discover_filters() return cache def cache_filename(): """Gets the name of the cache file to use.""" # Configuration variables read from the environment if "PYGMENTS_CACHE_FILE" in os.environ: return os.environ["PYGMENTS_CACHE_FILE"] else: return os.path.join( os.environ.get( "XDG_DATA_HOME", os.path.join(os.path.expanduser("~"), ".local", "share"), ), "pygments-cache", "cache.py", ) def load(filename): """Loads the cache from a filename.""" global CACHE with open(filename) as f: s = f.read() ctx = globals() CACHE = eval(s, ctx, ctx) return CACHE def write_cache(filename): """Writes the current cache to the file""" from pprint import pformat d = os.path.dirname(filename) os.makedirs(d, exist_ok=True) s = pformat(CACHE) with open(filename, "w") as f: f.write(s) def load_or_build(): """Loads the cache from disk. If the cache does not exist, this will build and write it out. """ global CACHE fname = cache_filename() if os.path.exists(fname): load(fname) else: import sys print("pygments cache not found, building...", file=sys.stderr) CACHE = build_cache() print("...writing cache to " + fname, file=sys.stderr) write_cache(fname) # # pygments interface # def get_lexer_for_filename(filename, text="", **options): """Gets a lexer from a filename (usually via the filename extension). This mimics the behavior of ``pygments.lexers.get_lexer_for_filename()`` and ``pygments.lexers.guess_lexer_for_filename()``. """ if CACHE is None: load_or_build() exts = CACHE["lexers"]["exts"] fname = os.path.basename(filename) key = fname if fname in exts else os.path.splitext(fname)[1] if key in exts: modname, clsname = exts[key] mod = importlib.import_module(modname) cls = getattr(mod, clsname) lexer = cls(**options) else: # couldn't find lexer in cache, fallback to the hard way import inspect from pygments.lexers import guess_lexer_for_filename lexer = guess_lexer_for_filename(filename, text, **options) # add this filename to the cache for future use cls = type(lexer) mod = inspect.getmodule(cls) exts[fname] = (mod.__name__, cls.__name__) write_cache(cache_filename()) return lexer guess_lexer_for_filename = get_lexer_for_filename def get_formatter_for_filename(fn, **options): """Gets a formatter instance from a filename (usually via the filename extension). This mimics the behavior of ``pygments.formatters.get_formatter_for_filename()``. """ if CACHE is None: load_or_build() exts = CACHE["formatters"]["exts"] fname = os.path.basename(fn) key = fname if fname in exts else os.path.splitext(fname)[1] if key in exts: modname, clsname = exts[key] mod = importlib.import_module(modname) cls = getattr(mod, clsname) formatter = cls(**options) else: # couldn't find formatter in cache, fallback to the hard way import inspect from pygments.formatters import get_formatter_for_filename formatter = get_formatter_for_filename(fn, **options) # add this filename to the cache for future use cls = type(formatter) mod = inspect.getmodule(cls) exts[fname] = (mod.__name__, cls.__name__) write_cache(cache_filename()) return formatter def get_formatter_by_name(alias, **options): """Gets a formatter instance from its name or alias. This mimics the behavior of ``pygments.formatters.get_formatter_by_name()``. """ if CACHE is None: load_or_build() names = CACHE["formatters"]["names"] if alias in names: modname, clsname = names[alias] mod = importlib.import_module(modname) cls = getattr(mod, clsname) formatter = cls(**options) else: # couldn't find formatter in cache, fallback to the hard way import inspect from pygments.formatters import get_formatter_by_name formatter = get_formatter_by_name(alias, **options) # add this filename to the cache for future use cls = type(formatter) mod = inspect.getmodule(cls) names[alias] = (mod.__name__, cls.__name__) write_cache(cache_filename()) return formatter def get_style_by_name(name): """Gets a style class from its name or alias. This mimics the behavior of ``pygments.styles.get_style_by_name()``. """ if CACHE is None: load_or_build() names = CACHE["styles"]["names"] if name in names: modname, clsname = names[name] mod = importlib.import_module(modname) style = getattr(mod, clsname) else: # couldn't find style in cache, fallback to the hard way import inspect from pygments.styles import get_style_by_name style = get_style_by_name(name) # add this style to the cache for future use mod = inspect.getmodule(style) names[name] = (mod.__name__, style.__name__) write_cache(cache_filename()) return style def get_all_styles(): """Iterable through all known style names. This mimics the behavior of ``pygments.styles.get_all_styles``. """ if CACHE is None: load_or_build() yield from CACHE["styles"]["names"] def get_filter_by_name(filtername, **options): """Gets a filter instance from its name. This mimics the behavior of ``pygments.filters.get_filtere_by_name()``. """ if CACHE is None: load_or_build() names = CACHE["filters"]["names"] if filtername in names: modname, clsname = names[filtername] mod = importlib.import_module(modname) cls = getattr(mod, clsname) filter = cls(**options) else: # couldn't find style in cache, fallback to the hard way import inspect from pygments.filters import get_filter_by_name filter = get_filter_by_name(filtername, **options) # add this filter to the cache for future use cls = type(filter) mod = inspect.getmodule(cls) names[filtername] = (mod.__name__, cls.__name__) write_cache(cache_filename()) return filter ================================================ FILE: xonsh/pytest_plugin.py ================================================ # -*- coding: utf-8 -*- """Pytest plugin for testing xsh files.""" import sys import importlib from traceback import format_list, extract_tb import pytest from xonsh.main import setup def pytest_configure(config): setup() def pytest_collection_modifyitems(items): items.sort(key=lambda x: 0 if isinstance(x, XshFunction) else 1) def _limited_traceback(excinfo): """ Return a formatted traceback with all the stack from this frame (i.e __file__) up removed """ tb = extract_tb(excinfo.tb) try: idx = [__file__ in e for e in tb].index(True) return format_list(tb[idx + 1 :]) except ValueError: return format_list(tb) def pytest_collect_file(parent, path): if path.ext.lower() == ".xsh" and path.basename.startswith("test_"): return XshFile(path, parent) class XshFile(pytest.File): def collect(self): sys.path.append(self.fspath.dirname) mod = importlib.import_module(self.fspath.purebasename) sys.path.pop(0) tests = [t for t in dir(mod) if t.startswith("test_")] for test_name in tests: obj = getattr(mod, test_name) if hasattr(obj, "__call__"): yield XshFunction( name=test_name, parent=self, test_func=obj, test_module=mod ) class XshFunction(pytest.Item): def __init__(self, name, parent, test_func, test_module): super().__init__(name, parent) self._test_func = test_func self._test_module = test_module def runtest(self, *args, **kwargs): self._test_func(*args, **kwargs) def repr_failure(self, excinfo): """ called when self.runtest() raises an exception. """ formatted_tb = _limited_traceback(excinfo) formatted_tb.insert(0, "xonsh execution failed\n") formatted_tb.append("{}: {}".format(excinfo.type.__name__, excinfo.value)) return "".join(formatted_tb) def reportinfo(self): return self.fspath, 0, "xonsh test: {}".format(self.name) ================================================ FILE: xonsh/readline_shell.py ================================================ # -*- coding: utf-8 -*- """The readline based xonsh shell. Portions of this code related to initializing the readline library are included from the IPython project. The IPython project is: * Copyright (c) 2008-2014, IPython Development Team * Copyright (c) 2001-2007, Fernando Perez * Copyright (c) 2001, Janko Hauser * Copyright (c) 2001, Nathaniel Gray """ import os import sys import cmd import select import shutil import builtins import importlib import threading import subprocess import collections from xonsh.lazyasd import LazyObject, lazyobject from xonsh.base_shell import BaseShell from xonsh.ansi_colors import ( ansi_partial_color_format, ansi_color_style_names, ansi_color_style, ) from xonsh.prompt.base import multiline_prompt from xonsh.tools import ( print_exception, check_for_partial_string, to_bool, columnize, carriage_return, ) from xonsh.platform import ( ON_WINDOWS, ON_CYGWIN, ON_MSYS, ON_DARWIN, ON_POSIX, os_environ, ) from xonsh.lazyimps import pygments, pyghooks, winutils from xonsh.events import events readline = None RL_COMPLETION_SUPPRESS_APPEND = RL_LIB = RL_STATE = None RL_COMPLETION_QUERY_ITEMS = None RL_CAN_RESIZE = False RL_DONE = None RL_VARIABLE_VALUE = None _RL_STATE_DONE = 0x1000000 _RL_STATE_ISEARCH = 0x0000080 _RL_PREV_CASE_SENSITIVE_COMPLETIONS = "to-be-set" def setup_readline(): """Sets up the readline module and completion suppression, if available.""" global RL_COMPLETION_SUPPRESS_APPEND, RL_LIB, RL_CAN_RESIZE, RL_STATE, readline, RL_COMPLETION_QUERY_ITEMS if RL_COMPLETION_SUPPRESS_APPEND is not None: return for _rlmod_name in ("gnureadline", "readline"): try: readline = importlib.import_module(_rlmod_name) sys.modules["readline"] = readline except ImportError: pass else: break if readline is None: print( """Skipping setup. Because no `readline` implementation available. Please install a backend (`readline`, `prompt-toolkit`, etc) to use `xonsh` interactively. See https://github.com/xonsh/xonsh/issues/1170""" ) return import ctypes import ctypes.util uses_libedit = readline.__doc__ and "libedit" in readline.__doc__ readline.set_completer_delims(" \t\n") # Cygwin seems to hang indefinitely when querying the readline lib if (not ON_CYGWIN) and (not ON_MSYS) and (not readline.__file__.endswith(".py")): RL_LIB = lib = ctypes.cdll.LoadLibrary(readline.__file__) try: RL_COMPLETION_SUPPRESS_APPEND = ctypes.c_int.in_dll( lib, "rl_completion_suppress_append" ) except ValueError: # not all versions of readline have this symbol, ie Macs sometimes RL_COMPLETION_SUPPRESS_APPEND = None try: RL_COMPLETION_QUERY_ITEMS = ctypes.c_int.in_dll( lib, "rl_completion_query_items" ) except ValueError: # not all versions of readline have this symbol, ie Macs sometimes RL_COMPLETION_QUERY_ITEMS = None try: RL_STATE = ctypes.c_int.in_dll(lib, "rl_readline_state") except Exception: pass RL_CAN_RESIZE = hasattr(lib, "rl_reset_screen_size") env = builtins.__xonsh__.env # reads in history readline.set_history_length(-1) ReadlineHistoryAdder() # sets up IPython-like history matching with up and down readline.parse_and_bind('"\\e[B": history-search-forward') readline.parse_and_bind('"\\e[A": history-search-backward') # Setup Shift-Tab to indent readline.parse_and_bind('"\\e[Z": "{0}"'.format(env.get("INDENT"))) # handle tab completion differences found in libedit readline compatibility # as discussed at http://stackoverflow.com/a/7116997 if uses_libedit and ON_DARWIN: readline.parse_and_bind("bind ^I rl_complete") print( "\n".join( [ "", "*" * 78, "libedit detected - readline will not be well behaved, including but not limited to:", " * crashes on tab completion", " * incorrect history navigation", " * corrupting long-lines", " * failure to wrap or indent lines properly", "", "It is highly recommended that you install gnureadline, which is installable with:", " xpip install gnureadline", "*" * 78, ] ), file=sys.stderr, ) else: readline.parse_and_bind("tab: complete") # try to load custom user settings inputrc_name = os_environ.get("INPUTRC") if inputrc_name is None: if uses_libedit: inputrc_name = ".editrc" else: inputrc_name = ".inputrc" inputrc_name = os.path.join(os.path.expanduser("~"), inputrc_name) if (not ON_WINDOWS) and (not os.path.isfile(inputrc_name)): inputrc_name = "/etc/inputrc" if ON_WINDOWS: winutils.enable_virtual_terminal_processing() if os.path.isfile(inputrc_name): try: readline.read_init_file(inputrc_name) except Exception: # this seems to fail with libedit print_exception("xonsh: could not load readline default init file.") # properly reset input typed before the first prompt readline.set_startup_hook(carriage_return) def teardown_readline(): """Tears down up the readline module, if available.""" try: import readline except (ImportError, TypeError): return def _rebind_case_sensitive_completions(): # handle case sensitive, see Github issue #1342 for details global _RL_PREV_CASE_SENSITIVE_COMPLETIONS env = builtins.__xonsh__.env case_sensitive = env.get("CASE_SENSITIVE_COMPLETIONS") if case_sensitive is _RL_PREV_CASE_SENSITIVE_COMPLETIONS: return if case_sensitive: readline.parse_and_bind("set completion-ignore-case off") else: readline.parse_and_bind("set completion-ignore-case on") _RL_PREV_CASE_SENSITIVE_COMPLETIONS = case_sensitive def fix_readline_state_after_ctrl_c(): """ Fix to allow Ctrl-C to exit reverse-i-search. Based on code from: http://bugs.python.org/file39467/raw_input__workaround_demo.py """ if ON_WINDOWS: # hack to make pyreadline mimic the desired behavior try: _q = readline.rl.mode.process_keyevent_queue if len(_q) > 1: _q.pop() except Exception: pass if RL_STATE is None: return if RL_STATE.value & _RL_STATE_ISEARCH: RL_STATE.value &= ~_RL_STATE_ISEARCH if not RL_STATE.value & _RL_STATE_DONE: RL_STATE.value |= _RL_STATE_DONE def rl_completion_suppress_append(val=1): """Sets the rl_completion_suppress_append variable, if possible. A value of 1 (default) means to suppress, a value of 0 means to enable. """ if RL_COMPLETION_SUPPRESS_APPEND is None: return RL_COMPLETION_SUPPRESS_APPEND.value = val def rl_completion_query_items(val=None): """Sets the rl_completion_query_items variable, if possible. A None value will set this to $COMPLETION_QUERY_LIMIT, otherwise any integer is accepted. """ if RL_COMPLETION_QUERY_ITEMS is None: return if val is None: val = builtins.__xonsh__.env.get("COMPLETION_QUERY_LIMIT") RL_COMPLETION_QUERY_ITEMS.value = val def rl_variable_dumper(readable=True): """Dumps the currently set readline variables. If readable is True, then this output may be used in an inputrc file. """ RL_LIB.rl_variable_dumper(int(readable)) def rl_variable_value(variable): """Returns the currently set value for a readline configuration variable.""" global RL_VARIABLE_VALUE if RL_VARIABLE_VALUE is None: import ctypes RL_VARIABLE_VALUE = RL_LIB.rl_variable_value RL_VARIABLE_VALUE.restype = ctypes.c_char_p env = builtins.__xonsh__.env enc, errors = env.get("XONSH_ENCODING"), env.get("XONSH_ENCODING_ERRORS") if isinstance(variable, str): variable = variable.encode(encoding=enc, errors=errors) rtn = RL_VARIABLE_VALUE(variable) return rtn.decode(encoding=enc, errors=errors) @lazyobject def rl_on_new_line(): """Grabs one of a few possible redisplay functions in readline.""" names = ["rl_on_new_line", "rl_forced_update_display", "rl_redisplay"] for name in names: func = getattr(RL_LIB, name, None) if func is not None: break else: def print_for_newline(): print() func = print_for_newline return func def _insert_text_func(s, readline): """Creates a function to insert text via readline.""" def inserter(): readline.insert_text(s) readline.redisplay() return inserter DEDENT_TOKENS = LazyObject( lambda: frozenset(["raise", "return", "pass", "break", "continue"]), globals(), "DEDENT_TOKENS", ) class ReadlineShell(BaseShell, cmd.Cmd): """The readline based xonsh shell.""" def __init__(self, completekey="tab", stdin=None, stdout=None, **kwargs): super().__init__(completekey=completekey, stdin=stdin, stdout=stdout, **kwargs) setup_readline() self._current_indent = "" self._current_prompt = "" self._force_hide = None self._complete_only_last_table = { # Truth table for completions, keys are: # (prefix_begs_quote, prefix_ends_quote, i_ends_quote, # last_starts_with_prefix, i_has_space) (True, True, True, True, True): True, (True, True, True, True, False): True, (True, True, True, False, True): False, (True, True, True, False, False): True, (True, True, False, True, True): False, (True, True, False, True, False): False, (True, True, False, False, True): False, (True, True, False, False, False): False, (True, False, True, True, True): True, (True, False, True, True, False): False, (True, False, True, False, True): False, (True, False, True, False, False): True, (True, False, False, True, True): False, (True, False, False, True, False): False, (True, False, False, False, True): False, (True, False, False, False, False): False, (False, True, True, True, True): True, (False, True, True, True, False): True, (False, True, True, False, True): True, (False, True, True, False, False): True, (False, True, False, True, True): False, (False, True, False, True, False): False, (False, True, False, False, True): False, (False, True, False, False, False): False, (False, False, True, True, True): False, (False, False, True, True, False): False, (False, False, True, False, True): False, (False, False, True, False, False): True, (False, False, False, True, True): True, (False, False, False, True, False): False, (False, False, False, False, True): False, (False, False, False, False, False): False, } self.cmdqueue = collections.deque() def __del__(self): teardown_readline() def singleline(self, store_in_history=True, **kwargs): """Reads a single line of input. The store_in_history kwarg flags whether the input should be stored in readline's in-memory history. """ if not store_in_history: # store current position to remove it later try: import readline except ImportError: store_in_history = True pos = readline.get_current_history_length() - 1 events.on_pre_prompt.fire() rtn = input(self.prompt) events.on_post_prompt.fire() if not store_in_history and pos >= 0: readline.remove_history_item(pos) return rtn def parseline(self, line): """Overridden to no-op.""" return "", line, line def _querycompletions(self, completions, loc): """Returns whether or not we should show completions. 0 means that prefixes should not be shown, 1 means that there is a common prefix among all completions and they should be shown, while 2 means that there is no common prefix but we are under the query limit and they should be shown. """ if os.path.commonprefix([c[loc:] for c in completions]): return 1 elif len(completions) <= builtins.__xonsh__.env.get("COMPLETION_QUERY_LIMIT"): return 2 msg = "\nDisplay all {} possibilities? ".format(len(completions)) msg += "({GREEN}y{NO_COLOR} or {RED}n{NO_COLOR})" self.print_color(msg, end="", flush=True, file=sys.stderr) yn = "x" while yn not in "yn": yn = sys.stdin.read(1) show_completions = to_bool(yn) print() if not show_completions: rl_on_new_line() return 0 w, h = shutil.get_terminal_size() lines = columnize(completions, width=w) more_msg = self.format_color( "{YELLOW}==={NO_COLOR} more or " "{PURPLE}({NO_COLOR}q{PURPLE}){NO_COLOR}uit " "{YELLOW}==={NO_COLOR}" ) while len(lines) > h - 1: print("".join(lines[: h - 1]), end="", flush=True, file=sys.stderr) lines = lines[h - 1 :] print(more_msg, end="", flush=True, file=sys.stderr) q = sys.stdin.read(1).lower() print(flush=True, file=sys.stderr) if q == "q": rl_on_new_line() return 0 print("".join(lines), end="", flush=True, file=sys.stderr) rl_on_new_line() return 0 def completedefault(self, prefix, line, begidx, endidx): """Implements tab-completion for text.""" if self.completer is None: return [] rl_completion_suppress_append() # this needs to be called each time _rebind_case_sensitive_completions() rl_completion_query_items(val=999999999) completions, l = self.completer.complete( prefix, line, begidx, endidx, ctx=self.ctx ) chopped = prefix[:-l] if chopped: rtn_completions = [chopped + i for i in completions] else: rtn_completions = completions rtn = [] prefix_begs_quote = prefix.startswith("'") or prefix.startswith('"') prefix_ends_quote = prefix.endswith("'") or prefix.endswith('"') for i in rtn_completions: i_ends_quote = i.endswith("'") or i.endswith('"') last = i.rsplit(" ", 1)[-1] last_starts_prefix = last.startswith(prefix) i_has_space = " " in i key = ( prefix_begs_quote, prefix_ends_quote, i_ends_quote, last_starts_prefix, i_has_space, ) rtn.append(last if self._complete_only_last_table[key] else i) # return based on show completions show_completions = self._querycompletions(completions, endidx - begidx) if show_completions == 0: return [] elif show_completions == 1: return rtn elif show_completions == 2: return completions else: raise ValueError("query completions flag not understood.") # tab complete on first index too completenames = completedefault def _load_remaining_input_into_queue(self): buf = b"" while True: r, w, x = select.select([self.stdin], [], [], 1e-6) if len(r) == 0: break buf += os.read(self.stdin.fileno(), 1024) if len(buf) > 0: buf = buf.decode().replace("\r\n", "\n").replace("\r", "\n") self.cmdqueue.extend(buf.splitlines(keepends=True)) def postcmd(self, stop, line): """Called just before execution of line. For readline, this handles the automatic indentation of code blocks. """ try: import readline except ImportError: return stop if self.need_more_lines: if len(line.strip()) == 0: readline.set_pre_input_hook(None) self._current_indent = "" elif line.rstrip()[-1] == ":": ind = line[: len(line) - len(line.lstrip())] ind += builtins.__xonsh__.env.get("INDENT") readline.set_pre_input_hook(_insert_text_func(ind, readline)) self._current_indent = ind elif line.split(maxsplit=1)[0] in DEDENT_TOKENS: env = builtins.__xonsh__.env ind = self._current_indent[: -len(env.get("INDENT"))] readline.set_pre_input_hook(_insert_text_func(ind, readline)) self._current_indent = ind else: ind = line[: len(line) - len(line.lstrip())] if ind != self._current_indent: insert_func = _insert_text_func(ind, readline) readline.set_pre_input_hook(insert_func) self._current_indent = ind else: readline.set_pre_input_hook(None) return stop def _cmdloop(self, intro=None): """Repeatedly issue a prompt, accept input, parse an initial prefix off the received input, and dispatch to action methods, passing them the remainder of the line as argument. This was forked from Lib/cmd.py from the Python standard library v3.4.3, (C) Python Software Foundation, 2015. """ self.preloop() if self.use_rawinput and self.completekey: try: import readline self.old_completer = readline.get_completer() readline.set_completer(self.complete) readline.parse_and_bind(self.completekey + ": complete") have_readline = True except ImportError: have_readline = False try: if intro is not None: self.intro = intro if self.intro: self.stdout.write(str(self.intro) + "\n") stop = None while not stop: line = None exec_now = False if len(self.cmdqueue) > 0: line = self.cmdqueue.popleft() exec_now = line.endswith("\n") if self.use_rawinput and not exec_now: inserter = ( None if line is None else _insert_text_func(line, readline) ) if inserter is not None: readline.set_pre_input_hook(inserter) try: line = self.singleline() except EOFError: if builtins.__xonsh__.env.get("IGNOREEOF"): self.stdout.write('Use "exit" to leave the shell.' "\n") line = "" else: line = "EOF" if inserter is not None: readline.set_pre_input_hook(None) else: self.print_color(self.prompt, file=self.stdout) if line is not None: os.write(self.stdin.fileno(), line.encode()) if not exec_now: line = self.stdin.readline() if len(line) == 0: line = "EOF" else: line = line.rstrip("\r\n") if have_readline and line != "EOF": readline.add_history(line) if not ON_WINDOWS: # select() is not fully functional on windows self._load_remaining_input_into_queue() line = self.precmd(line) stop = self.onecmd(line) stop = self.postcmd(stop, line) if ON_WINDOWS: winutils.enable_virtual_terminal_processing() self.postloop() finally: if self.use_rawinput and self.completekey: try: import readline readline.set_completer(self.old_completer) except ImportError: pass def cmdloop(self, intro=None): while not builtins.__xonsh__.exit: try: self._cmdloop(intro=intro) except (KeyboardInterrupt, SystemExit): print() # Gives a newline fix_readline_state_after_ctrl_c() self.reset_buffer() intro = None @property def prompt(self): """Obtains the current prompt string.""" global RL_LIB, RL_CAN_RESIZE if RL_CAN_RESIZE: # This is needed to support some system where line-wrapping doesn't # work. This is a bug in upstream Python, or possibly readline. RL_LIB.rl_reset_screen_size() if self.need_more_lines: if self.mlprompt is None: try: self.mlprompt = multiline_prompt(curr=self._current_prompt) except Exception: # pylint: disable=broad-except print_exception() self.mlprompt = " " return self.mlprompt env = builtins.__xonsh__.env # pylint: disable=no-member p = env.get("PROMPT") try: p = self.prompt_formatter(p) except Exception: # pylint: disable=broad-except print_exception() hide = True if self._force_hide is None else self._force_hide p = ansi_partial_color_format(p, style=env.get("XONSH_COLOR_STYLE"), hide=hide) self._current_prompt = p self.settitle() return p def format_color(self, string, hide=False, force_string=False, **kwargs): """Readline implementation of color formatting. This uses ANSI color codes. """ hide = hide if self._force_hide is None else self._force_hide style = builtins.__xonsh__.env.get("XONSH_COLOR_STYLE") return ansi_partial_color_format(string, hide=hide, style=style) def print_color(self, string, hide=False, **kwargs): if isinstance(string, str): s = self.format_color(string, hide=hide) else: # assume this is a list of (Token, str) tuples and format it env = builtins.__xonsh__.env self.styler.style_name = env.get("XONSH_COLOR_STYLE") style_proxy = pyghooks.xonsh_style_proxy(self.styler) formatter = pyghooks.XonshTerminal256Formatter(style=style_proxy) s = pygments.format(string, formatter).rstrip() print(s, **kwargs) def color_style_names(self): """Returns an iterable of all available style names.""" return ansi_color_style_names() def color_style(self): """Returns the current color map.""" style = style = builtins.__xonsh__.env.get("XONSH_COLOR_STYLE") return ansi_color_style(style=style) def restore_tty_sanity(self): """An interface for resetting the TTY stdin mode. This is highly dependent on the shell backend. Also it is mostly optional since it only affects ^Z backgrounding behaviour. """ if not ON_POSIX: return stty, _ = builtins.__xonsh__.commands_cache.lazyget("stty", None) if stty is None: return # If available, we should just call the stty utility. This call should # not throw even if stty fails. It should also be noted that subprocess # calls, like the following, seem to be ineffective: # subprocess.call([stty, 'sane'], shell=True) # My guess is that this is because Popen does some crazy redirecting # under the covers. This effectively hides the true TTY stdin handle # from stty. To get around this we have to use the lower level # os.system() function. os.system(stty + " sane") class ReadlineHistoryAdder(threading.Thread): def __init__(self, wait_for_gc=True, *args, **kwargs): """Thread responsible for adding inputs from history to the current readline instance. May wait for the history garbage collector to finish. """ super(ReadlineHistoryAdder, self).__init__(*args, **kwargs) self.daemon = True self.wait_for_gc = wait_for_gc self.start() def run(self): try: import readline except ImportError: return hist = builtins.__xonsh__.history if hist is None: return i = 1 for h in hist.all_items(): line = h["inp"].rstrip() if i == 1: pass elif line == readline.get_history_item(i - 1): continue readline.add_history(line) if RL_LIB is not None: RL_LIB.history_set_pos(i) i += 1 ================================================ FILE: xonsh/replay.py ================================================ # -*- coding: utf-8 -*- """Tools to replay xonsh history files.""" import json import time import builtins import collections.abc as cabc from xonsh.tools import swap from xonsh.lazyjson import LazyJSON from xonsh.environ import Env import xonsh.history.main as xhm DEFAULT_MERGE_ENVS = ("replay", "native") class Replayer(object): """Replays a xonsh history file.""" def __init__(self, f, reopen=True): """ Parameters ---------- f : file handle or str Path to xonsh history file. reopen : bool, optional Whether new file handle should be opened for each load, passed directly into LazyJSON class. """ self._lj = LazyJSON(f, reopen=reopen) def __del__(self): self._lj.close() def replay(self, merge_envs=DEFAULT_MERGE_ENVS, target=None): """Replays the history specified, returns the history object where the code was executed. Parameters ---------- merge_env : tuple of str or Mappings, optional Describes how to merge the environments, in order of increasing precedence. Available strings are 'replay' and 'native'. The 'replay' env comes from the history file that we are replaying. The 'native' env comes from what this instance of xonsh was started up with. Instead of a string, a dict or other mapping may be passed in as well. Defaults to ('replay', 'native'). target : str, optional Path to new history file. """ shell = builtins.__xonsh__.shell re_env = self._lj["env"].load() new_env = self._merge_envs(merge_envs, re_env) new_hist = xhm.construct_history( env=new_env.detype(), locked=True, ts=[time.time(), None], gc=False, filename=target, ) with swap(builtins.__xonsh__, "env", new_env), swap( builtins.__xonsh__, "history", new_hist ): for cmd in self._lj["cmds"]: inp = cmd["inp"] shell.default(inp) if builtins.__xonsh__.exit: # prevent premature exit builtins.__xonsh__.exit = False new_hist.flush(at_exit=True) return new_hist def _merge_envs(self, merge_envs, re_env): new_env = {} for e in merge_envs: if e == "replay": new_env.update(re_env) elif e == "native": new_env.update(builtins.__xonsh__.env) elif isinstance(e, cabc.Mapping): new_env.update(e) else: raise TypeError("Type of env not understood: {0!r}".format(e)) new_env = Env(**new_env) return new_env _REPLAY_PARSER = None def replay_create_parser(p=None): global _REPLAY_PARSER p_was_none = p is None if _REPLAY_PARSER is not None and p_was_none: return _REPLAY_PARSER if p_was_none: from argparse import ArgumentParser p = ArgumentParser("replay", description="replays a xonsh history file") p.add_argument( "--merge-envs", dest="merge_envs", default=DEFAULT_MERGE_ENVS, nargs="+", help="Describes how to merge the environments, in order of " "increasing precedence. Available strings are 'replay' and " "'native'. The 'replay' env comes from the history file that we " "are replaying. The 'native' env comes from what this instance " "of xonsh was started up with. One or more of these options may " "be passed in. Defaults to '--merge-envs replay native'.", ) p.add_argument( "--json", dest="json", default=False, action="store_true", help="print history info in JSON format", ) p.add_argument( "-o", "--target", dest="target", default=None, help="path to new history file" ) p.add_argument("path", help="path to replay history file") if p_was_none: _REPLAY_PARSER = p return p def replay_main_action(h, ns, stdout=None, stderr=None): replayer = Replayer(ns.path) hist = replayer.replay(merge_envs=ns.merge_envs, target=ns.target) print("----------------------------------------------------------------") print("Just replayed history, new history has the following information") print("----------------------------------------------------------------") data = hist.info() if ns.json: s = json.dumps(data) print(s, file=stdout) else: lines = ["{0}: {1}".format(k, v) for k, v in data.items()] print("\n".join(lines), file=stdout) def replay_main(args, stdin=None): """Acts as main function for replaying a xonsh history file.""" parser = replay_create_parser() ns = parser.parse_args(args) replay_main_action(ns) ================================================ FILE: xonsh/shell.py ================================================ # -*- coding: utf-8 -*- """The xonsh shell""" import sys import random import time import difflib import builtins import warnings from xonsh.platform import ( best_shell_type, has_prompt_toolkit, ptk_above_min_supported, ptk_shell_type, ) from xonsh.tools import XonshError, print_exception from xonsh.events import events import xonsh.history.main as xhm events.doc( "on_transform_command", """ on_transform_command(cmd: str) -> str Fired to request xontribs to transform a command line. Return the transformed command, or the same command if no transformation occurs. Only done for interactive sessions. This may be fired multiple times per command, with other transformers input or output, so design any handlers for this carefully. """, ) events.doc( "on_precommand", """ on_precommand(cmd: str) -> None Fires just before a command is executed. """, ) events.doc( "on_postcommand", """ on_postcommand(cmd: str, rtn: int, out: str or None, ts: list) -> None Fires just after a command is executed. The arguments are the same as history. Parameters: * ``cmd``: The command that was executed (after transformation) * ``rtn``: The result of the command executed (``0`` for success) * ``out``: If xonsh stores command output, this is the output * ``ts``: Timestamps, in the order of ``[starting, ending]`` """, ) events.doc( "on_pre_prompt", """ on_pre_prompt() -> None Fires just before the prompt is shown """, ) events.doc( "on_post_prompt", """ on_post_prompt() -> None Fires just after the prompt returns """, ) def transform_command(src, show_diff=True): """Returns the results of firing the precommand handles.""" i = 0 limit = sys.getrecursionlimit() lst = "" raw = src while src != lst: lst = src srcs = events.on_transform_command.fire(cmd=src) for s in srcs: if s != lst: src = s break i += 1 if i == limit: print_exception( "Modifications to source input took more than " "the recursion limit number of iterations to " "converge." ) debug_level = builtins.__xonsh__.env.get("XONSH_DEBUG") if show_diff and debug_level > 1 and src != raw: sys.stderr.writelines( difflib.unified_diff( raw.splitlines(keepends=True), src.splitlines(keepends=True), fromfile="before precommand event", tofile="after precommand event", ) ) return src class Shell(object): """Main xonsh shell. Initializes execution environment and decides if prompt_toolkit or readline version of shell should be used. """ shell_type_aliases = { "b": "best", "best": "best", "d": "dumb", "dumb": "dumb", "ptk": "prompt_toolkit", "ptk1": "prompt_toolkit1", "ptk2": "prompt_toolkit2", "prompt-toolkit": "prompt_toolkit", "prompt_toolkit": "prompt_toolkit", "prompt-toolkit1": "prompt_toolkit1", "prompt-toolkit2": "prompt_toolkit2", "rand": "random", "random": "random", "rl": "readline", "readline": "readline", } def __init__(self, execer, ctx=None, shell_type=None, **kwargs): """ Parameters ---------- execer : Execer An execer instance capable of running xonsh code. ctx : Mapping, optional The execution context for the shell (e.g. the globals namespace). If none, this is computed by loading the rc files. If not None, this no additional context is computed and this is used directly. shell_type : str, optional The shell type to start, such as 'readline', 'prompt_toolkit1', or 'random'. """ self.execer = execer self.ctx = {} if ctx is None else ctx env = builtins.__xonsh__.env # build history backend before creating shell builtins.__xonsh__.history = hist = xhm.construct_history( env=env.detype(), ts=[time.time(), None], locked=True ) # pick a valid shell -- if no shell is specified by the user, # shell type is pulled from env if shell_type is None: shell_type = env.get("SHELL_TYPE") if shell_type == "none": # This bricks interactive xonsh # Can happen from the use of .xinitrc, .xsession, etc shell_type = "best" shell_type = self.shell_type_aliases.get(shell_type, shell_type) if shell_type == "best" or shell_type is None: shell_type = best_shell_type() elif env.get("TERM", "") == "dumb": shell_type = "dumb" elif shell_type == "random": shell_type = random.choice(("readline", "prompt_toolkit")) if shell_type == "prompt_toolkit": if not has_prompt_toolkit(): warnings.warn( "prompt_toolkit is not available, using " "readline instead." ) shell_type = "readline" elif not ptk_above_min_supported(): warnings.warn( "prompt-toolkit version < v1.0.0 is not " "supported. Please update prompt-toolkit. Using " "readline instead." ) shell_type = "readline" else: shell_type = ptk_shell_type() self.shell_type = env["SHELL_TYPE"] = shell_type # actually make the shell if shell_type == "none": from xonsh.base_shell import BaseShell as shell_class elif shell_type == "prompt_toolkit2": from xonsh.ptk2.shell import PromptToolkit2Shell as shell_class elif shell_type == "prompt_toolkit1": from xonsh.ptk.shell import PromptToolkitShell as shell_class elif shell_type == "readline": from xonsh.readline_shell import ReadlineShell as shell_class elif shell_type == "jupyter": from xonsh.jupyter_shell import JupyterShell as shell_class elif shell_type == "dumb": from xonsh.dumb_shell import DumbShell as shell_class else: raise XonshError("{} is not recognized as a shell type".format(shell_type)) self.shell = shell_class(execer=self.execer, ctx=self.ctx, **kwargs) # allows history garbage collector to start running if hist.gc is not None: hist.gc.wait_for_shell = False def __getattr__(self, attr): """Delegates calls to appropriate shell instance.""" return getattr(self.shell, attr) ================================================ FILE: xonsh/style_tools.py ================================================ """Xonsh color styling tools that simulate pygments, when it is unavailable.""" import builtins from collections import defaultdict from xonsh.platform import HAS_PYGMENTS from xonsh.lazyasd import LazyObject from xonsh.color_tools import RE_BACKGROUND from xonsh.tools import FORMATTER class _TokenType(tuple): """ Forked from the pygments project https://bitbucket.org/birkenfeld/pygments-main Copyright (c) 2006-2017 by the respective authors, All rights reserved. See https://bitbucket.org/birkenfeld/pygments-main/raw/05818a4ef9891d9ac22c851f7b3ea4b4fce460ab/AUTHORS """ parent = None def split(self): buf = [] node = self while node is not None: buf.append(node) node = node.parent buf.reverse() return buf def __init__(self, *args): # no need to call super.__init__ self.subtypes = set() def __contains__(self, val): return self is val or (type(val) is self.__class__ and val[: len(self)] == self) def __getattr__(self, val): if not val or not val[0].isupper(): return tuple.__getattribute__(self, val) new = _TokenType(self + (val,)) setattr(self, val, new) self.subtypes.add(new) new.parent = self return new def __repr__(self): return "Token" + (self and "." or "") + ".".join(self) def __copy__(self): # These instances are supposed to be singletons return self def __deepcopy__(self, memo): # These instances are supposed to be singletons return self Token = _TokenType() Color = Token.Color def partial_color_tokenize(template): """Tokenizes a template string containing colors. Will return a list of tuples mapping the token to the string which has that color. These sub-strings maybe templates themselves. """ if HAS_PYGMENTS and builtins.__xonsh__.shell is not None: styles = __xonsh__.shell.shell.styler.styles elif builtins.__xonsh__.shell is not None: styles = DEFAULT_STYLE_DICT else: styles = None color = Color.NO_COLOR try: toks, color = _partial_color_tokenize_main(template, styles) except Exception: toks = [(Color.NO_COLOR, template)] if styles is not None: styles[color] # ensure color is available return toks def _partial_color_tokenize_main(template, styles): bopen = "{" bclose = "}" colon = ":" expl = "!" color = Color.NO_COLOR fg = bg = None value = "" toks = [] for literal, field, spec, conv in FORMATTER.parse(template): if field is None: value += literal elif field in KNOWN_COLORS or "#" in field: value += literal next_color, fg, bg = color_by_name(field, fg, bg) if next_color is not color: if len(value) > 0: toks.append((color, value)) if styles is not None: styles[color] # ensure color is available color = next_color value = "" elif field is not None: parts = [literal, bopen, field] if conv is not None and len(conv) > 0: parts.append(expl) parts.append(conv) if spec is not None and len(spec) > 0: parts.append(colon) parts.append(spec) parts.append(bclose) value += "".join(parts) else: value += literal toks.append((color, value)) return toks, color def color_by_name(name, fg=None, bg=None): """Converts a color name to a color token, foreground name, and background name. Will take into consideration current foreground and background colors, if provided. Parameters ---------- name : str Color name. fg : str, optional Foreground color name. bg : str, optional Background color name. Returns ------- tok : Token Pygments Token.Color subclass fg : str or None New computed foreground color name. bg : str or None New computed background color name. """ name = name.upper() if name == "NO_COLOR": return Color.NO_COLOR, None, None m = RE_BACKGROUND.search(name) if m is None: # must be foreground color fg = norm_name(name) else: bg = norm_name(name) # assemble token if fg is None and bg is None: tokname = "NO_COLOR" elif fg is None: tokname = bg elif bg is None: tokname = fg else: tokname = fg + "__" + bg tok = getattr(Color, tokname) return tok, fg, bg def norm_name(name): """Normalizes a color name.""" return name.replace("#", "HEX").replace("BGHEX", "BACKGROUND_HEX") KNOWN_COLORS = LazyObject( lambda: frozenset( [ "BACKGROUND_BLACK", "BACKGROUND_BLUE", "BACKGROUND_CYAN", "BACKGROUND_GREEN", "BACKGROUND_INTENSE_BLACK", "BACKGROUND_INTENSE_BLUE", "BACKGROUND_INTENSE_CYAN", "BACKGROUND_INTENSE_GREEN", "BACKGROUND_INTENSE_PURPLE", "BACKGROUND_INTENSE_RED", "BACKGROUND_INTENSE_WHITE", "BACKGROUND_INTENSE_YELLOW", "BACKGROUND_PURPLE", "BACKGROUND_RED", "BACKGROUND_WHITE", "BACKGROUND_YELLOW", "BLACK", "BLUE", "BOLD_BLACK", "BOLD_BLUE", "BOLD_CYAN", "BOLD_GREEN", "BOLD_INTENSE_BLACK", "BOLD_INTENSE_BLUE", "BOLD_INTENSE_CYAN", "BOLD_INTENSE_GREEN", "BOLD_INTENSE_PURPLE", "BOLD_INTENSE_RED", "BOLD_INTENSE_WHITE", "BOLD_INTENSE_YELLOW", "BOLD_PURPLE", "BOLD_RED", "BOLD_UNDERLINE_BLACK", "BOLD_UNDERLINE_BLUE", "BOLD_UNDERLINE_CYAN", "BOLD_UNDERLINE_GREEN", "BOLD_UNDERLINE_INTENSE_BLACK", "BOLD_UNDERLINE_INTENSE_BLUE", "BOLD_UNDERLINE_INTENSE_CYAN", "BOLD_UNDERLINE_INTENSE_GREEN", "BOLD_UNDERLINE_INTENSE_PURPLE", "BOLD_UNDERLINE_INTENSE_RED", "BOLD_UNDERLINE_INTENSE_WHITE", "BOLD_UNDERLINE_INTENSE_YELLOW", "BOLD_UNDERLINE_PURPLE", "BOLD_UNDERLINE_RED", "BOLD_UNDERLINE_WHITE", "BOLD_UNDERLINE_YELLOW", "BOLD_WHITE", "BOLD_YELLOW", "CYAN", "GREEN", "INTENSE_BLACK", "INTENSE_BLUE", "INTENSE_CYAN", "INTENSE_GREEN", "INTENSE_PURPLE", "INTENSE_RED", "INTENSE_WHITE", "INTENSE_YELLOW", "NO_COLOR", "PURPLE", "RED", "UNDERLINE_BLACK", "UNDERLINE_BLUE", "UNDERLINE_CYAN", "UNDERLINE_GREEN", "UNDERLINE_INTENSE_BLACK", "UNDERLINE_INTENSE_BLUE", "UNDERLINE_INTENSE_CYAN", "UNDERLINE_INTENSE_GREEN", "UNDERLINE_INTENSE_PURPLE", "UNDERLINE_INTENSE_RED", "UNDERLINE_INTENSE_WHITE", "UNDERLINE_INTENSE_YELLOW", "UNDERLINE_PURPLE", "UNDERLINE_RED", "UNDERLINE_WHITE", "UNDERLINE_YELLOW", "WHITE", "YELLOW", ] ), globals(), "KNOWN_COLORS", ) DEFAULT_STYLE_DICT = LazyObject( lambda: defaultdict( lambda: "", { Token: "", Token.Aborted: "ansibrightblack", Token.AutoSuggestion: "ansibrightblack", Token.Color.BACKGROUND_BLACK: "bg:ansiblack", Token.Color.BACKGROUND_BLUE: "bg:ansiblue", Token.Color.BACKGROUND_CYAN: "bg:ansicyan", Token.Color.BACKGROUND_GREEN: "bg:ansigreen", Token.Color.BACKGROUND_INTENSE_BLACK: "bg:ansibrightblack", Token.Color.BACKGROUND_INTENSE_BLUE: "bg:ansibrightblue", Token.Color.BACKGROUND_INTENSE_CYAN: "bg:ansibrightcyan", Token.Color.BACKGROUND_INTENSE_GREEN: "bg:ansibrightgreen", Token.Color.BACKGROUND_INTENSE_PURPLE: "bg:ansibrightmagenta", Token.Color.BACKGROUND_INTENSE_RED: "bg:ansibrightred", Token.Color.BACKGROUND_INTENSE_WHITE: "bg:ansiwhite", Token.Color.BACKGROUND_INTENSE_YELLOW: "bg:ansibrightyellow", Token.Color.BACKGROUND_PURPLE: "bg:ansimagenta", Token.Color.BACKGROUND_RED: "bg:ansired", Token.Color.BACKGROUND_WHITE: "bg:ansigray", Token.Color.BACKGROUND_YELLOW: "bg:ansiyellow", Token.Color.BLACK: "ansiblack", Token.Color.BLUE: "ansiblue", Token.Color.BOLD_BLACK: "bold ansiblack", Token.Color.BOLD_BLUE: "bold ansiblue", Token.Color.BOLD_CYAN: "bold ansicyan", Token.Color.BOLD_GREEN: "bold ansigreen", Token.Color.BOLD_INTENSE_BLACK: "bold ansibrightblack", Token.Color.BOLD_INTENSE_BLUE: "bold ansibrightblue", Token.Color.BOLD_INTENSE_CYAN: "bold ansibrightcyan", Token.Color.BOLD_INTENSE_GREEN: "bold ansibrightgreen", Token.Color.BOLD_INTENSE_PURPLE: "bold ansibrightmagenta", Token.Color.BOLD_INTENSE_RED: "bold ansibrightred", Token.Color.BOLD_INTENSE_WHITE: "bold ansiwhite", Token.Color.BOLD_INTENSE_YELLOW: "bold ansibrightyellow", Token.Color.BOLD_PURPLE: "bold ansimagenta", Token.Color.BOLD_RED: "bold ansired", Token.Color.BOLD_UNDERLINE_BLACK: "bold underline ansiblack", Token.Color.BOLD_UNDERLINE_BLUE: "bold underline ansiblue", Token.Color.BOLD_UNDERLINE_CYAN: "bold underline ansicyan", Token.Color.BOLD_UNDERLINE_GREEN: "bold underline ansigreen", Token.Color.BOLD_UNDERLINE_INTENSE_BLACK: "bold underline ansibrightblack", Token.Color.BOLD_UNDERLINE_INTENSE_BLUE: "bold underline ansibrightblue", Token.Color.BOLD_UNDERLINE_INTENSE_CYAN: "bold underline ansibrightcyan", Token.Color.BOLD_UNDERLINE_INTENSE_GREEN: "bold underline ansibrightgreen", Token.Color.BOLD_UNDERLINE_INTENSE_PURPLE: "bold underline ansibrightmagenta", Token.Color.BOLD_UNDERLINE_INTENSE_RED: "bold underline ansibrightred", Token.Color.BOLD_UNDERLINE_INTENSE_WHITE: "bold underline ansiwhite", Token.Color.BOLD_UNDERLINE_INTENSE_YELLOW: "bold underline ansibrightyellow", Token.Color.BOLD_UNDERLINE_PURPLE: "bold underline ansimagenta", Token.Color.BOLD_UNDERLINE_RED: "bold underline ansired", Token.Color.BOLD_UNDERLINE_WHITE: "bold underline ansigray", Token.Color.BOLD_UNDERLINE_YELLOW: "bold underline ansiyellow", Token.Color.BOLD_WHITE: "bold ansigray", Token.Color.BOLD_YELLOW: "bold ansiyellow", Token.Color.CYAN: "ansicyan", Token.Color.GREEN: "ansigreen", Token.Color.INTENSE_BLACK: "ansibrightblack", Token.Color.INTENSE_BLUE: "ansibrightblue", Token.Color.INTENSE_CYAN: "ansibrightcyan", Token.Color.INTENSE_GREEN: "ansibrightgreen", Token.Color.INTENSE_PURPLE: "ansibrightmagenta", Token.Color.INTENSE_RED: "ansibrightred", Token.Color.INTENSE_WHITE: "ansiwhite", Token.Color.INTENSE_YELLOW: "ansibrightyellow", Token.Color.NO_COLOR: "noinherit", Token.Color.PURPLE: "ansimagenta", Token.Color.RED: "ansired", Token.Color.UNDERLINE_BLACK: "underline ansiblack", Token.Color.UNDERLINE_BLUE: "underline ansiblue", Token.Color.UNDERLINE_CYAN: "underline ansicyan", Token.Color.UNDERLINE_GREEN: "underline ansigreen", Token.Color.UNDERLINE_INTENSE_BLACK: "underline ansibrightblack", Token.Color.UNDERLINE_INTENSE_BLUE: "underline ansibrightblue", Token.Color.UNDERLINE_INTENSE_CYAN: "underline ansibrightcyan", Token.Color.UNDERLINE_INTENSE_GREEN: "underline ansibrightgreen", Token.Color.UNDERLINE_INTENSE_PURPLE: "underline ansibrightmagenta", Token.Color.UNDERLINE_INTENSE_RED: "underline ansibrightred", Token.Color.UNDERLINE_INTENSE_WHITE: "underline ansiwhite", Token.Color.UNDERLINE_INTENSE_YELLOW: "underline ansibrightyellow", Token.Color.UNDERLINE_PURPLE: "underline ansimagenta", Token.Color.UNDERLINE_RED: "underline ansired", Token.Color.UNDERLINE_WHITE: "underline ansigray", Token.Color.UNDERLINE_YELLOW: "underline ansiyellow", Token.Color.WHITE: "ansigray", Token.Color.YELLOW: "ansiyellow", Token.Comment: "underline ansicyan", Token.Comment.Hashbang: "", Token.Comment.Multiline: "", Token.Comment.Preproc: "underline ansiyellow", Token.Comment.PreprocFile: "", Token.Comment.Single: "", Token.Comment.Special: "", Token.Error: "ansibrightred", Token.Escape: "", Token.Generic: "", Token.Generic.Deleted: "ansired", Token.Generic.Emph: "underline", Token.Generic.Error: "bold ansibrightred", Token.Generic.Heading: "bold ansiblue", Token.Generic.Inserted: "ansibrightgreen", Token.Generic.Output: "ansiblue", Token.Generic.Prompt: "bold ansiblue", Token.Generic.Strong: "", Token.Generic.Subheading: "bold ansimagenta", Token.Generic.Traceback: "ansiblue", Token.Keyword: "bold ansigreen", Token.Keyword.Constant: "", Token.Keyword.Declaration: "", Token.Keyword.Namespace: "", Token.Keyword.Pseudo: "nobold", Token.Keyword.Reserved: "", Token.Keyword.Type: "nobold ansired", Token.Literal: "", Token.Literal.Date: "", Token.Literal.Number: "ansibrightblack", Token.Literal.Number.Bin: "", Token.Literal.Number.Float: "", Token.Literal.Number.Hex: "", Token.Literal.Number.Integer: "", Token.Literal.Number.Integer.Long: "", Token.Literal.Number.Oct: "", Token.Literal.String: "ansibrightred", Token.Literal.String.Affix: "", Token.Literal.String.Backtick: "", Token.Literal.String.Char: "", Token.Literal.String.Delimiter: "", Token.Literal.String.Doc: "underline", Token.Literal.String.Double: "", Token.Literal.String.Escape: "bold ansiyellow", Token.Literal.String.Heredoc: "", Token.Literal.String.Interpol: "bold ansimagenta", Token.Literal.String.Other: "ansigreen", Token.Literal.String.Regex: "ansimagenta", Token.Literal.String.Single: "", Token.Literal.String.Symbol: "ansiyellow", Token.Menu.Completions: "bg:ansigray ansiblack", Token.Menu.Completions.Completion: "", Token.Menu.Completions.Completion.Current: "bg:ansibrightblack ansiwhite", Token.Name: "", Token.Name.Attribute: "ansibrightyellow", Token.Name.Builtin: "ansigreen", Token.Name.Builtin.Pseudo: "", Token.Name.Class: "bold ansibrightblue", Token.Name.Constant: "ansired", Token.Name.Decorator: "ansibrightmagenta", Token.Name.Entity: "bold ansigray", Token.Name.Exception: "bold ansibrightred", Token.Name.Function: "ansibrightblue", Token.Name.Function.Magic: "", Token.Name.Label: "ansibrightyellow", Token.Name.Namespace: "bold ansibrightblue", Token.Name.Other: "", Token.Name.Property: "", Token.Name.Tag: "bold ansigreen", Token.Name.Variable: "ansiblue", Token.Name.Variable.Class: "", Token.Name.Variable.Global: "", Token.Name.Variable.Instance: "", Token.Name.Variable.Magic: "", Token.Operator: "ansibrightblack", Token.Operator.Word: "bold ansimagenta", Token.Other: "", Token.Punctuation: "", Token.Scrollbar: "bg:ansibrightblack", Token.Scrollbar.Arrow: "bg:ansiblack ansiwhite bold", Token.Scrollbar.Button: "bg:ansiblack", Token.Text: "", Token.Text.Whitespace: "ansigray", }, ), globals(), "DEFAULT_STYLE_DICT", ) PTK2_STYLE = { "completion-menu": "bg:ansicyan ansiwhite", "completion-menu.completion": "bg:#008888 #ffffff", "completion-menu.completion.current": "bg:ansibrightblack ansiwhite", "completion-menu.meta.completion": "bg:#00aaaa #ffffff", "completion-menu.meta.completion.current": "bg:#00aaaa #000000", "scrollbar.background": "bg:ansibrightblack", "scrollbar.arrow": "bg:ansiblack ansiwhite bold", "scrollbar.button": "bg:ansiblack", } ================================================ FILE: xonsh/timings.py ================================================ # -*- coding: utf-8 -*- """Timing related functionality for the xonsh shell. The following time_it alias and Timer was forked from the IPython project: * Copyright (c) 2008-2014, IPython Development Team * Copyright (C) 2001-2007 Fernando Perez * Copyright (c) 2001, Janko Hauser * Copyright (c) 2001, Nathaniel Gray """ import os import gc import sys import math import time import timeit import builtins import itertools from xonsh.lazyasd import lazyobject, lazybool from xonsh.events import events from xonsh.platform import ON_WINDOWS @lazybool def _HAVE_RESOURCE(): try: import resource as r have = True except ImportError: # There is no distinction of user/system time under windows, so we # just use time.perf_counter() for everything... have = False return have @lazyobject def resource(): import resource as r return r @lazyobject def clocku(): if _HAVE_RESOURCE: def clocku(): """clocku() -> floating point number Return the *USER* CPU time in seconds since the start of the process.""" return resource.getrusage(resource.RUSAGE_SELF)[0] else: clocku = time.perf_counter return clocku @lazyobject def clocks(): if _HAVE_RESOURCE: def clocks(): """clocks() -> floating point number Return the *SYSTEM* CPU time in seconds since the start of the process.""" return resource.getrusage(resource.RUSAGE_SELF)[1] else: clocks = time.perf_counter return clocks @lazyobject def clock(): if _HAVE_RESOURCE: def clock(): """clock() -> floating point number Return the *TOTAL USER+SYSTEM* CPU time in seconds since the start of the process.""" u, s = resource.getrusage(resource.RUSAGE_SELF)[:2] return u + s else: clock = time.perf_counter return clock @lazyobject def clock2(): if _HAVE_RESOURCE: def clock2(): """clock2() -> (t_user,t_system) Similar to clock(), but return a tuple of user/system times.""" return resource.getrusage(resource.RUSAGE_SELF)[:2] else: def clock2(): """Under windows, system CPU time can't be measured. This just returns perf_counter() and zero.""" return time.perf_counter(), 0.0 return clock2 def format_time(timespan, precision=3): """Formats the timespan in a human readable form""" if timespan >= 60.0: # we have more than a minute, format that in a human readable form parts = [("d", 60 * 60 * 24), ("h", 60 * 60), ("min", 60), ("s", 1)] time = [] leftover = timespan for suffix, length in parts: value = int(leftover / length) if value > 0: leftover = leftover % length time.append("{0}{1}".format(str(value), suffix)) if leftover < 1: break return " ".join(time) # Unfortunately the unicode 'micro' symbol can cause problems in # certain terminals. # See bug: https://bugs.launchpad.net/ipython/+bug/348466 # Try to prevent crashes by being more secure than it needs to # E.g. eclipse is able to print a mu, but has no sys.stdout.encoding set. units = ["s", "ms", "us", "ns"] # the save value if hasattr(sys.stdout, "encoding") and sys.stdout.encoding: try: "\xb5".encode(sys.stdout.encoding) units = ["s", "ms", "\xb5s", "ns"] except Exception: pass scaling = [1, 1e3, 1e6, 1e9] if timespan > 0.0: order = min(-int(math.floor(math.log10(timespan)) // 3), 3) else: order = 3 return "{1:.{0}g} {2}".format(precision, timespan * scaling[order], units[order]) class Timer(timeit.Timer): """Timer class that explicitly uses self.inner which is an undocumented implementation detail of CPython, not shared by PyPy. """ # Timer.timeit copied from CPython 3.4.2 def timeit(self, number=timeit.default_number): """Time 'number' executions of the main statement. To be precise, this executes the setup statement once, and then returns the time it takes to execute the main statement a number of times, as a float measured in seconds. The argument is the number of times through the loop, defaulting to one million. The main statement, the setup statement and the timer function to be used are passed to the constructor. """ it = itertools.repeat(None, number) gcold = gc.isenabled() gc.disable() try: timing = self.inner(it, self.timer) finally: if gcold: gc.enable() return timing INNER_TEMPLATE = """ def inner(_it, _timer): #setup _t0 = _timer() for _i in _it: {stmt} _t1 = _timer() return _t1 - _t0 """ def timeit_alias(args, stdin=None): """Runs timing study on arguments.""" # some real args number = 0 quiet = False repeat = 3 precision = 3 # setup ctx = builtins.__xonsh__.ctx timer = Timer(timer=clock) stmt = " ".join(args) innerstr = INNER_TEMPLATE.format(stmt=stmt) # Track compilation time so it can be reported if too long # Minimum time above which compilation time will be reported tc_min = 0.1 t0 = clock() innercode = builtins.compilex( innerstr, filename="", mode="exec", glbs=ctx ) tc = clock() - t0 # get inner func ns = {} builtins.execx(innercode, glbs=ctx, locs=ns, mode="exec") timer.inner = ns["inner"] # Check if there is a huge difference between the best and worst timings. worst_tuning = 0 if number == 0: # determine number so that 0.2 <= total time < 2.0 number = 1 for _ in range(1, 10): time_number = timer.timeit(number) worst_tuning = max(worst_tuning, time_number / number) if time_number >= 0.2: break number *= 10 all_runs = timer.repeat(repeat, number) best = min(all_runs) / number # print some debug info if not quiet: worst = max(all_runs) / number if worst_tuning: worst = max(worst, worst_tuning) # Check best timing is greater than zero to avoid a # ZeroDivisionError. # In cases where the slowest timing is less than 10 microseconds # we assume that it does not really matter if the fastest # timing is 4 times faster than the slowest timing or not. if worst > 4 * best and best > 0 and worst > 1e-5: print( ( "The slowest run took {0:0.2f} times longer than the " "fastest. This could mean that an intermediate result " "is being cached." ).format(worst / best) ) print( "{0} loops, best of {1}: {2} per loop".format( number, repeat, format_time(best, precision) ) ) if tc > tc_min: print("Compiler time: {0:.2f} s".format(tc)) return _timings = {"start": clock()} def setup_timings(argv): global _timings if "--timings" in argv: events.doc( "on_timingprobe", """ on_timingprobe(name: str) -> None Fired to insert some timings into the startuptime list """, ) @events.on_timingprobe def timing_on_timingprobe(name, **kw): global _timings _timings[name] = clock() @events.on_post_cmdloop def timing_on_post_cmdloop(**kw): global _timings _timings["on_post_cmdloop"] = clock() @events.on_post_init def timing_on_post_init(**kw): global _timings _timings["on_post_init"] = clock() @events.on_post_rc def timing_on_post_rc(**kw): global _timings _timings["on_post_rc"] = clock() @events.on_postcommand def timing_on_postcommand(**kw): global _timings _timings["on_postcommand"] = clock() @events.on_pre_cmdloop def timing_on_pre_cmdloop(**kw): global _timings _timings["on_pre_cmdloop"] = clock() @events.on_pre_rc def timing_on_pre_rc(**kw): global _timings _timings["on_pre_rc"] = clock() @events.on_precommand def timing_on_precommand(**kw): global _timings _timings["on_precommand"] = clock() @events.on_ptk_create def timing_on_ptk_create(**kw): global _timings _timings["on_ptk_create"] = clock() @events.on_chdir def timing_on_chdir(**kw): global _timings _timings["on_chdir"] = clock() @events.on_post_prompt def timing_on_post_prompt(**kw): global _timings _timings = {"on_post_prompt": clock()} @events.on_pre_prompt def timing_on_pre_prompt(**kw): global _timings _timings["on_pre_prompt"] = clock() times = list(_timings.items()) times = sorted(times, key=lambda x: x[1]) width = max(len(s) for s, _ in times) + 2 header_format = "|{{:<{}}}|{{:^11}}|{{:^11}}|".format(width) entry_format = "|{{:<{}}}|{{:^11.3f}}|{{:^11.3f}}|".format(width) sepline = "|{}|{}|{}|".format("-" * width, "-" * 11, "-" * 11) # Print result table print(" Debug level: {}".format(os.getenv("XONSH_DEBUG", "Off"))) print(sepline) print(header_format.format("Event name", "Time (s)", "Delta (s)")) print(sepline) prevtime = tstart = times[0][1] for name, ts in times: print(entry_format.format(name, ts - tstart, ts - prevtime)) prevtime = ts print(sepline) ================================================ FILE: xonsh/tokenize.py ================================================ """Tokenization help for xonsh programs. This file is a modified version of tokenize.py form the Python 3.4 and 3.5 standard libraries (licensed under the Python Software Foundation License, version 2), which provides tokenization help for Python programs. It is modified to properly tokenize xonsh code, including backtick regex path and several xonsh-specific operators. A few pieces of this file are specific to the version of Python being used. To find these pieces, search the PY35. Original file credits: __author__ = 'Ka-Ping Yee ' __credits__ = ('GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, ' 'Skip Montanaro, Raymond Hettinger, Trent Nelson, ' 'Michael Foord') """ import re import io import sys import codecs import builtins import itertools import collections import token from token import ( AMPER, AMPEREQUAL, AT, CIRCUMFLEX, CIRCUMFLEXEQUAL, COLON, COMMA, DEDENT, DOT, DOUBLESLASH, DOUBLESLASHEQUAL, DOUBLESTAR, DOUBLESTAREQUAL, ENDMARKER, EQEQUAL, EQUAL, ERRORTOKEN, GREATER, GREATEREQUAL, INDENT, LBRACE, LEFTSHIFT, LEFTSHIFTEQUAL, LESS, LESSEQUAL, LPAR, LSQB, MINEQUAL, MINUS, NAME, NEWLINE, NOTEQUAL, NUMBER, N_TOKENS, OP, PERCENT, PERCENTEQUAL, PLUS, PLUSEQUAL, RBRACE, RIGHTSHIFT, RIGHTSHIFTEQUAL, RPAR, RSQB, SEMI, SLASH, SLASHEQUAL, STAR, STAREQUAL, STRING, TILDE, VBAR, VBAREQUAL, tok_name, ) from xonsh.lazyasd import LazyObject from xonsh.platform import PYTHON_VERSION_INFO cookie_re = LazyObject( lambda: re.compile(r"^[ \t\f]*#.*coding[:=][ \t]*([-\w.]+)", re.ASCII), globals(), "cookie_re", ) blank_re = LazyObject( lambda: re.compile(br"^[ \t\f]*(?:[#\r\n]|$)", re.ASCII), globals(), "blank_re" ) # # token modifications # tok_name = tok_name.copy() __all__ = token.__all__ + [ "COMMENT", "tokenize", "detect_encoding", "NL", "untokenize", "ENCODING", "TokenInfo", "TokenError", "SEARCHPATH", "ATDOLLAR", "ATEQUAL", "DOLLARNAME", "IOREDIRECT", ] HAS_ASYNC = (3, 5, 0) <= PYTHON_VERSION_INFO < (3, 7, 0) if HAS_ASYNC: ASYNC = token.ASYNC AWAIT = token.AWAIT ADDSPACE_TOKS = (NAME, NUMBER, ASYNC, AWAIT) else: ADDSPACE_TOKS = (NAME, NUMBER) del token # must clean up token PY35 = (3, 5, 0) <= PYTHON_VERSION_INFO AUGASSIGN_OPS = r"[+\-*/%&@|^=<>]=?" if not PY35: AUGASSIGN_OPS = AUGASSIGN_OPS.replace("@", "") COMMENT = N_TOKENS tok_name[COMMENT] = "COMMENT" NL = N_TOKENS + 1 tok_name[NL] = "NL" ENCODING = N_TOKENS + 2 tok_name[ENCODING] = "ENCODING" N_TOKENS += 3 SEARCHPATH = N_TOKENS tok_name[N_TOKENS] = "SEARCHPATH" N_TOKENS += 1 IOREDIRECT = N_TOKENS tok_name[N_TOKENS] = "IOREDIRECT" N_TOKENS += 1 DOLLARNAME = N_TOKENS tok_name[N_TOKENS] = "DOLLARNAME" N_TOKENS += 1 ATDOLLAR = N_TOKENS tok_name[N_TOKENS] = "ATDOLLAR" N_TOKENS += 1 ATEQUAL = N_TOKENS tok_name[N_TOKENS] = "ATEQUAL" N_TOKENS += 1 _xonsh_tokens = { "?": "QUESTION", "@=": "ATEQUAL", "@$": "ATDOLLAR", "||": "DOUBLEPIPE", "&&": "DOUBLEAMPER", "@(": "ATLPAREN", "!(": "BANGLPAREN", "![": "BANGLBRACKET", "$(": "DOLLARLPAREN", "$[": "DOLLARLBRACKET", "${": "DOLLARLBRACE", "??": "DOUBLEQUESTION", "@$(": "ATDOLLARLPAREN", } additional_parenlevs = frozenset({"@(", "!(", "![", "$(", "$[", "${", "@$("}) _glbs = globals() for v in _xonsh_tokens.values(): _glbs[v] = N_TOKENS tok_name[N_TOKENS] = v N_TOKENS += 1 __all__.append(v) del _glbs, v EXACT_TOKEN_TYPES = { "(": LPAR, ")": RPAR, "[": LSQB, "]": RSQB, ":": COLON, ",": COMMA, ";": SEMI, "+": PLUS, "-": MINUS, "*": STAR, "/": SLASH, "|": VBAR, "&": AMPER, "<": LESS, ">": GREATER, "=": EQUAL, ".": DOT, "%": PERCENT, "{": LBRACE, "}": RBRACE, "==": EQEQUAL, "!=": NOTEQUAL, "<=": LESSEQUAL, ">=": GREATEREQUAL, "~": TILDE, "^": CIRCUMFLEX, "<<": LEFTSHIFT, ">>": RIGHTSHIFT, "**": DOUBLESTAR, "+=": PLUSEQUAL, "-=": MINEQUAL, "*=": STAREQUAL, "/=": SLASHEQUAL, "%=": PERCENTEQUAL, "&=": AMPEREQUAL, "|=": VBAREQUAL, "^=": CIRCUMFLEXEQUAL, "<<=": LEFTSHIFTEQUAL, ">>=": RIGHTSHIFTEQUAL, "**=": DOUBLESTAREQUAL, "//": DOUBLESLASH, "//=": DOUBLESLASHEQUAL, "@": AT, } EXACT_TOKEN_TYPES.update(_xonsh_tokens) class TokenInfo(collections.namedtuple("TokenInfo", "type string start end line")): def __repr__(self): annotated_type = "%d (%s)" % (self.type, tok_name[self.type]) return ( "TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r)" % self._replace(type=annotated_type) ) @property def exact_type(self): if self.type == OP and self.string in EXACT_TOKEN_TYPES: return EXACT_TOKEN_TYPES[self.string] else: return self.type def group(*choices): return "(" + "|".join(choices) + ")" def tokany(*choices): return group(*choices) + "*" def maybe(*choices): return group(*choices) + "?" # Note: we use unicode matching for names ("\w") but ascii matching for # number literals. Whitespace = r"[ \f\t]*" Comment = r"#[^\r\n]*" Ignore = Whitespace + tokany(r"\\\r?\n" + Whitespace) + maybe(Comment) Name_RE = r"\$?\w+" Hexnumber = r"0[xX](?:_?[0-9a-fA-F])+" Binnumber = r"0[bB](?:_?[01])+" Octnumber = r"0[oO](?:_?[0-7])+" Decnumber = r"(?:0(?:_?0)*|[1-9](?:_?[0-9])*)" Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber) Exponent = r"[eE][-+]?[0-9](?:_?[0-9])*" Pointfloat = group( r"[0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?", r"\.[0-9](?:_?[0-9])*" ) + maybe(Exponent) Expfloat = r"[0-9](?:_?[0-9])*" + Exponent Floatnumber = group(Pointfloat, Expfloat) Imagnumber = group(r"[0-9](?:_?[0-9])*[jJ]", Floatnumber + r"[jJ]") Number = group(Imagnumber, Floatnumber, Intnumber) StringPrefix = r"(?:[bB][rR]?|[p][fFrR]?|[rR][bBpfF]?|[uU]|[fF][rR]?[p]?)?" # Tail end of ' string. Single = r"[^'\\]*(?:\\.[^'\\]*)*'" # Tail end of " string. Double = r'[^"\\]*(?:\\.[^"\\]*)*"' # Tail end of ''' string. Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''" # Tail end of """ string. Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""' Triple = group(StringPrefix + "'''", StringPrefix + '"""') # Single-line ' or " string. String = group( StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*'", StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*"', ) # Xonsh-specific Syntax SearchPath = r"((?:[rgp]+|@\w*)?)`([^\n`\\]*(?:\\.[^\n`\\]*)*)`" # Because of leftmost-then-longest match semantics, be sure to put the # longest operators first (e.g., if = came before ==, == would get # recognized as two instances of =). _redir_names = ("out", "all", "err", "e", "2", "a", "&", "1", "o") _redir_map = ( # stderr to stdout "err>out", "err>&1", "2>out", "err>o", "err>1", "e>out", "e>&1", "2>&1", "e>o", "2>o", "e>1", "2>1", # stdout to stderr "out>err", "out>&2", "1>err", "out>e", "out>2", "o>err", "o>&2", "1>&2", "o>e", "1>e", "o>2", "1>2", ) IORedirect = group(group(*_redir_map), "{}>>?".format(group(*_redir_names))) _redir_check = set(_redir_map) _redir_check = {"{}>".format(i) for i in _redir_names}.union(_redir_check) _redir_check = {"{}>>".format(i) for i in _redir_names}.union(_redir_check) _redir_check = frozenset(_redir_check) Operator = group( r"\*\*=?", r">>=?", r"<<=?", r"!=", r"//=?", r"->", r"@\$\(?", r"\|\|", "&&", r"@\(", r"!\(", r"!\[", r"\$\(", r"\$\[", r"\${", r"\?\?", r"\?", AUGASSIGN_OPS, r"~", ) Bracket = "[][(){}]" Special = group(r"\r?\n", r"\.\.\.", r"[:;.,@]") Funny = group(Operator, Bracket, Special) PlainToken = group(IORedirect, Number, Funny, String, Name_RE, SearchPath) # First (or only) line of ' or " string. ContStr = group( StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*" + group("'", r"\\\r?\n"), StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*' + group('"', r"\\\r?\n"), ) PseudoExtras = group(r"\\\r?\n|\Z", Comment, Triple, SearchPath) PseudoToken = Whitespace + group( PseudoExtras, IORedirect, Number, Funny, ContStr, Name_RE ) def _compile(expr): return re.compile(expr, re.UNICODE) endpats = { "'": Single, '"': Double, "'''": Single3, '"""': Double3, "r'''": Single3, 'r"""': Double3, "b'''": Single3, 'b"""': Double3, "f'''": Single3, 'f"""': Double3, "R'''": Single3, 'R"""': Double3, "B'''": Single3, 'B"""': Double3, "F'''": Single3, 'F"""': Double3, "br'''": Single3, 'br"""': Double3, "fr'''": Single3, 'fr"""': Double3, "fp'''": Single3, 'fp"""': Double3, "bR'''": Single3, 'bR"""': Double3, "Br'''": Single3, 'Br"""': Double3, "BR'''": Single3, 'BR"""': Double3, "rb'''": Single3, 'rb"""': Double3, "rf'''": Single3, 'rf"""': Double3, "Rb'''": Single3, 'Rb"""': Double3, "Fr'''": Single3, 'Fr"""': Double3, "Fp'''": Single3, 'Fp"""': Double3, "rB'''": Single3, 'rB"""': Double3, "rF'''": Single3, 'rF"""': Double3, "RB'''": Single3, 'RB"""': Double3, "RF'''": Single3, 'RF"""': Double3, "u'''": Single3, 'u"""': Double3, "U'''": Single3, 'U"""': Double3, "p'''": Single3, 'p"""': Double3, "pr'''": Single3, 'pr"""': Double3, "pf'''": Single3, 'pf"""': Double3, "pF'''": Single3, 'pF"""': Double3, "pR'''": Single3, 'pR"""': Double3, "rp'''": Single3, 'rp"""': Double3, "Rp'''": Single3, 'Rp"""': Double3, "r": None, "R": None, "b": None, "B": None, "u": None, "U": None, "p": None, "f": None, "F": None, } triple_quoted = {} for t in ( "'''", '"""', "r'''", 'r"""', "R'''", 'R"""', "b'''", 'b"""', "B'''", 'B"""', "f'''", 'f"""', "F'''", 'F"""', "br'''", 'br"""', "Br'''", 'Br"""', "bR'''", 'bR"""', "BR'''", 'BR"""', "rb'''", 'rb"""', "rB'''", 'rB"""', "Rb'''", 'Rb"""', "RB'''", 'RB"""', "fr'''", 'fr"""', "Fr'''", 'Fr"""', "fR'''", 'fR"""', "FR'''", 'FR"""', "rf'''", 'rf"""', "rF'''", 'rF"""', "Rf'''", 'Rf"""', "RF'''", 'RF"""', "u'''", 'u"""', "U'''", 'U"""', "p'''", 'p""""', "pr'''", 'pr""""', "pR'''", 'pR""""', "rp'''", 'rp""""', "Rp'''", 'Rp""""', "pf'''", 'pf""""', "pF'''", 'pF""""', "fp'''", 'fp""""', "Fp'''", 'Fp""""', ): triple_quoted[t] = t single_quoted = {} for t in ( "'", '"', "r'", 'r"', "R'", 'R"', "b'", 'b"', "B'", 'B"', "f'", 'f"', "F'", 'F"', "br'", 'br"', "Br'", 'Br"', "bR'", 'bR"', "BR'", 'BR"', "rb'", 'rb"', "rB'", 'rB"', "Rb'", 'Rb"', "RB'", 'RB"', "fr'", 'fr"', "Fr'", 'Fr"', "fR'", 'fR"', "FR'", 'FR"', "rf'", 'rf"', "rF'", 'rF"', "Rf'", 'Rf"', "RF'", 'RF"', "u'", 'u"', "U'", 'U"', "p'", 'p"', "pr'", 'pr"', "pR'", 'pR"', "rp'", 'rp"', "Rp'", 'Rp"', "pf'", 'pf"', "pF'", 'pF"', "fp'", 'fp"', "Fp'", 'Fp"', ): single_quoted[t] = t tabsize = 8 class TokenError(Exception): pass class StopTokenizing(Exception): pass class Untokenizer: def __init__(self): self.tokens = [] self.prev_row = 1 self.prev_col = 0 self.encoding = None def add_whitespace(self, start): row, col = start if row < self.prev_row or row == self.prev_row and col < self.prev_col: raise ValueError( "start ({},{}) precedes previous end ({},{})".format( row, col, self.prev_row, self.prev_col ) ) row_offset = row - self.prev_row if row_offset: self.tokens.append("\\\n" * row_offset) self.prev_col = 0 col_offset = col - self.prev_col if col_offset: self.tokens.append(" " * col_offset) def untokenize(self, iterable): it = iter(iterable) indents = [] startline = False for t in it: if len(t) == 2: self.compat(t, it) break tok_type, token, start, end, line = t if tok_type == ENCODING: self.encoding = token continue if tok_type == ENDMARKER: break if tok_type == INDENT: indents.append(token) continue elif tok_type == DEDENT: indents.pop() self.prev_row, self.prev_col = end continue elif tok_type in (NEWLINE, NL): startline = True elif startline and indents: indent = indents[-1] if start[1] >= len(indent): self.tokens.append(indent) self.prev_col = len(indent) startline = False self.add_whitespace(start) self.tokens.append(token) self.prev_row, self.prev_col = end if tok_type in (NEWLINE, NL): self.prev_row += 1 self.prev_col = 0 return "".join(self.tokens) def compat(self, token, iterable): indents = [] toks_append = self.tokens.append startline = token[0] in (NEWLINE, NL) prevstring = False for tok in itertools.chain([token], iterable): toknum, tokval = tok[:2] if toknum == ENCODING: self.encoding = tokval continue if toknum in ADDSPACE_TOKS: tokval += " " # Insert a space between two consecutive strings if toknum == STRING: if prevstring: tokval = " " + tokval prevstring = True else: prevstring = False if toknum == INDENT: indents.append(tokval) continue elif toknum == DEDENT: indents.pop() continue elif toknum in (NEWLINE, NL): startline = True elif startline and indents: toks_append(indents[-1]) startline = False toks_append(tokval) def untokenize(iterable): """Transform tokens back into Python source code. It returns a bytes object, encoded using the ENCODING token, which is the first token sequence output by tokenize. Each element returned by the iterable must be a token sequence with at least two elements, a token number and token value. If only two tokens are passed, the resulting output is poor. Round-trip invariant for full input: Untokenized source will match input source exactly Round-trip invariant for limited intput: # Output bytes will tokenize the back to the input t1 = [tok[:2] for tok in tokenize(f.readline)] newcode = untokenize(t1) readline = BytesIO(newcode).readline t2 = [tok[:2] for tok in tokenize(readline)] assert t1 == t2 """ ut = Untokenizer() out = ut.untokenize(iterable) if ut.encoding is not None: out = out.encode(ut.encoding) return out def _get_normal_name(orig_enc): """Imitates get_normal_name in tokenizer.c.""" # Only care about the first 12 characters. enc = orig_enc[:12].lower().replace("_", "-") if enc == "utf-8" or enc.startswith("utf-8-"): return "utf-8" if enc in ("latin-1", "iso-8859-1", "iso-latin-1") or enc.startswith( ("latin-1-", "iso-8859-1-", "iso-latin-1-") ): return "iso-8859-1" return orig_enc def detect_encoding(readline): """ The detect_encoding() function is used to detect the encoding that should be used to decode a Python source file. It requires one argument, readline, in the same way as the tokenize() generator. It will call readline a maximum of twice, and return the encoding used (as a string) and a list of any lines (left as bytes) it has read in. It detects the encoding from the presence of a utf-8 bom or an encoding cookie as specified in pep-0263. If both a bom and a cookie are present, but disagree, a SyntaxError will be raised. If the encoding cookie is an invalid charset, raise a SyntaxError. Note that if a utf-8 bom is found, 'utf-8-sig' is returned. If no encoding is specified, then the default of 'utf-8' will be returned. """ try: filename = readline.__self__.name except AttributeError: filename = None bom_found = False encoding = None default = "utf-8" def read_or_stop(): try: return readline() except StopIteration: return b"" def find_cookie(line): try: # Decode as UTF-8. Either the line is an encoding declaration, # in which case it should be pure ASCII, or it must be UTF-8 # per default encoding. line_string = line.decode("utf-8") except UnicodeDecodeError: msg = "invalid or missing encoding declaration" if filename is not None: msg = "{} for {!r}".format(msg, filename) raise SyntaxError(msg) match = cookie_re.match(line_string) if not match: return None encoding = _get_normal_name(match.group(1)) try: codecs.lookup(encoding) except LookupError: # This behaviour mimics the Python interpreter if filename is None: msg = "unknown encoding: " + encoding else: msg = "unknown encoding for {!r}: {}".format(filename, encoding) raise SyntaxError(msg) if bom_found: if encoding != "utf-8": # This behaviour mimics the Python interpreter if filename is None: msg = "encoding problem: utf-8" else: msg = "encoding problem for {!r}: utf-8".format(filename) raise SyntaxError(msg) encoding += "-sig" return encoding first = read_or_stop() if first.startswith(codecs.BOM_UTF8): bom_found = True first = first[3:] default = "utf-8-sig" if not first: return default, [] encoding = find_cookie(first) if encoding: return encoding, [first] if not blank_re.match(first): return default, [first] second = read_or_stop() if not second: return default, [first] encoding = find_cookie(second) if encoding: return encoding, [first, second] return default, [first, second] def tokopen(filename): """Open a file in read only mode using the encoding detected by detect_encoding(). """ buffer = builtins.open(filename, "rb") try: encoding, lines = detect_encoding(buffer.readline) buffer.seek(0) text = io.TextIOWrapper(buffer, encoding, line_buffering=True) text.mode = "r" return text except Exception: buffer.close() raise def _tokenize(readline, encoding): lnum = parenlev = continued = 0 numchars = "0123456789" contstr, needcont = "", 0 contline = None indents = [0] # 'stashed' and 'async_*' are used for async/await parsing stashed = None async_def = False async_def_indent = 0 async_def_nl = False if encoding is not None: if encoding == "utf-8-sig": # BOM will already have been stripped. encoding = "utf-8" yield TokenInfo(ENCODING, encoding, (0, 0), (0, 0), "") while True: # loop over lines in stream try: line = readline() except StopIteration: line = b"" if encoding is not None: line = line.decode(encoding) lnum += 1 pos, max = 0, len(line) if contstr: # continued string if not line: raise TokenError("EOF in multi-line string", strstart) endmatch = endprog.match(line) if endmatch: pos = end = endmatch.end(0) yield TokenInfo( STRING, contstr + line[:end], strstart, (lnum, end), contline + line ) contstr, needcont = "", 0 contline = None elif needcont and line[-2:] != "\\\n" and line[-3:] != "\\\r\n": yield TokenInfo( ERRORTOKEN, contstr + line, strstart, (lnum, len(line)), contline ) contstr = "" contline = None continue else: contstr = contstr + line contline = contline + line continue elif parenlev == 0 and not continued: # new statement if not line: break column = 0 while pos < max: # measure leading whitespace if line[pos] == " ": column += 1 elif line[pos] == "\t": column = (column // tabsize + 1) * tabsize elif line[pos] == "\f": column = 0 else: break pos += 1 if pos == max: break if line[pos] in "#\r\n": # skip comments or blank lines if line[pos] == "#": comment_token = line[pos:].rstrip("\r\n") nl_pos = pos + len(comment_token) yield TokenInfo( COMMENT, comment_token, (lnum, pos), (lnum, pos + len(comment_token)), line, ) yield TokenInfo( NL, line[nl_pos:], (lnum, nl_pos), (lnum, len(line)), line ) else: yield TokenInfo( (NL, COMMENT)[line[pos] == "#"], line[pos:], (lnum, pos), (lnum, len(line)), line, ) continue if column > indents[-1]: # count indents or dedents indents.append(column) yield TokenInfo(INDENT, line[:pos], (lnum, 0), (lnum, pos), line) while column < indents[-1]: if column not in indents: raise IndentationError( "unindent does not match any outer indentation level", ("", lnum, pos, line), ) indents = indents[:-1] if async_def and async_def_indent >= indents[-1]: async_def = False async_def_nl = False async_def_indent = 0 yield TokenInfo(DEDENT, "", (lnum, pos), (lnum, pos), line) if async_def and async_def_nl and async_def_indent >= indents[-1]: async_def = False async_def_nl = False async_def_indent = 0 else: # continued statement if not line: raise TokenError("EOF in multi-line statement", (lnum, 0)) continued = 0 while pos < max: pseudomatch = _compile(PseudoToken).match(line, pos) if pseudomatch: # scan for tokens start, end = pseudomatch.span(1) spos, epos, pos = (lnum, start), (lnum, end), end if start == end: continue token, initial = line[start:end], line[start] if token in _redir_check: yield TokenInfo(IOREDIRECT, token, spos, epos, line) elif initial in numchars or ( # ordinary number initial == "." and token != "." and token != "..." ): yield TokenInfo(NUMBER, token, spos, epos, line) elif initial in "\r\n": if stashed: yield stashed stashed = None if parenlev > 0: yield TokenInfo(NL, token, spos, epos, line) else: yield TokenInfo(NEWLINE, token, spos, epos, line) if async_def: async_def_nl = True elif initial == "#": assert not token.endswith("\n") if stashed: yield stashed stashed = None yield TokenInfo(COMMENT, token, spos, epos, line) # Xonsh-specific Regex Globbing elif re.match(SearchPath, token): yield TokenInfo(SEARCHPATH, token, spos, epos, line) elif token in triple_quoted: endprog = _compile(endpats[token]) endmatch = endprog.match(line, pos) if endmatch: # all on one line pos = endmatch.end(0) token = line[start:pos] yield TokenInfo(STRING, token, spos, (lnum, pos), line) else: strstart = (lnum, start) # multiple lines contstr = line[start:] contline = line break elif ( initial in single_quoted or token[:2] in single_quoted or token[:3] in single_quoted ): if token[-1] == "\n": # continued string strstart = (lnum, start) endprog = _compile( endpats[initial] or endpats[token[1]] or endpats[token[2]] ) contstr, needcont = line[start:], 1 contline = line break else: # ordinary string yield TokenInfo(STRING, token, spos, epos, line) elif token.startswith("$") and token[1:].isidentifier(): yield TokenInfo(DOLLARNAME, token, spos, epos, line) elif initial.isidentifier(): # ordinary name if token in ("async", "await"): if async_def: yield TokenInfo( ASYNC if token == "async" else AWAIT, token, spos, epos, line, ) continue tok = TokenInfo(NAME, token, spos, epos, line) if token == "async" and not stashed: stashed = tok continue if ( HAS_ASYNC and token == "def" and ( stashed and stashed.type == NAME and stashed.string == "async" ) ): async_def = True async_def_indent = indents[-1] yield TokenInfo( ASYNC, stashed.string, stashed.start, stashed.end, stashed.line, ) stashed = None if stashed: yield stashed stashed = None yield tok elif token == "\\\n" or token == "\\\r\n": # continued stmt continued = 1 yield TokenInfo(ERRORTOKEN, token, spos, epos, line) elif initial == "\\": # continued stmt # for cases like C:\\path\\to\\file continued = 1 else: if initial in "([{": parenlev += 1 elif initial in ")]}": parenlev -= 1 elif token in additional_parenlevs: parenlev += 1 if stashed: yield stashed stashed = None yield TokenInfo(OP, token, spos, epos, line) else: yield TokenInfo( ERRORTOKEN, line[pos], (lnum, pos), (lnum, pos + 1), line ) pos += 1 if stashed: yield stashed stashed = None for indent in indents[1:]: # pop remaining indent levels yield TokenInfo(DEDENT, "", (lnum, 0), (lnum, 0), "") yield TokenInfo(ENDMARKER, "", (lnum, 0), (lnum, 0), "") def tokenize(readline): """ The tokenize() generator requires one argument, readline, which must be a callable object which provides the same interface as the readline() method of built-in file objects. Each call to the function should return one line of input as bytes. Alternately, readline can be a callable function terminating with StopIteration: readline = open(myfile, 'rb').__next__ # Example of alternate readline The generator produces 5-tuples with these members: the token type; the token string; a 2-tuple (srow, scol) of ints specifying the row and column where the token begins in the source; a 2-tuple (erow, ecol) of ints specifying the row and column where the token ends in the source; and the line on which the token was found. The line passed is the logical line; continuation lines are included. The first token sequence will always be an ENCODING token which tells you which encoding was used to decode the bytes stream. """ encoding, consumed = detect_encoding(readline) rl_gen = iter(readline, b"") empty = itertools.repeat(b"") return _tokenize(itertools.chain(consumed, rl_gen, empty).__next__, encoding) # An undocumented, backwards compatible, API for all the places in the standard # library that expect to be able to use tokenize with strings def generate_tokens(readline): return _tokenize(readline, None) def tokenize_main(): import argparse # Helper error handling routines def perror(message): print(message, file=sys.stderr) def error(message, filename=None, location=None): if location: args = (filename,) + location + (message,) perror("%s:%d:%d: error: %s" % args) elif filename: perror("%s: error: %s" % (filename, message)) else: perror("error: %s" % message) sys.exit(1) # Parse the arguments and options parser = argparse.ArgumentParser(prog="python -m tokenize") parser.add_argument( dest="filename", nargs="?", metavar="filename.py", help="the file to tokenize; defaults to stdin", ) parser.add_argument( "-e", "--exact", dest="exact", action="store_true", help="display token names using the exact type", ) args = parser.parse_args() try: # Tokenize the input if args.filename: filename = args.filename with builtins.open(filename, "rb") as f: tokens = list(tokenize(f.readline)) else: filename = "" tokens = _tokenize(sys.stdin.readline, None) # Output the tokenization for token in tokens: token_type = token.type if args.exact: token_type = token.exact_type token_range = "%d,%d-%d,%d:" % (token.start + token.end) print("%-20s%-15s%-15r" % (token_range, tok_name[token_type], token.string)) except IndentationError as err: line, column = err.args[1][1:3] error(err.args[0], filename, (line, column)) except TokenError as err: line, column = err.args[1] error(err.args[0], filename, (line, column)) except SyntaxError as err: error(err, filename) except OSError as err: error(err) except KeyboardInterrupt: print("interrupted\n") except Exception as err: perror("unexpected error: %s" % err) raise ================================================ FILE: xonsh/tools.py ================================================ # -*- coding: utf-8 -*- """Misc. xonsh tools. The following implementations were forked from the IPython project: * Copyright (c) 2008-2014, IPython Development Team * Copyright (C) 2001-2007 Fernando Perez * Copyright (c) 2001, Janko Hauser * Copyright (c) 2001, Nathaniel Gray Implementations: * decode() * encode() * cast_unicode() * safe_hasattr() * indent() """ import builtins import collections import collections.abc as cabc import contextlib import ctypes import datetime from distutils.version import LooseVersion import functools import glob import itertools import os import pathlib import re import subprocess import sys import threading import traceback import warnings import operator import ast import string # adding imports from further xonsh modules is discouraged to avoid circular # dependencies from xonsh import __version__ from xonsh.lazyasd import LazyObject, LazyDict, lazyobject from xonsh.platform import ( scandir, DEFAULT_ENCODING, ON_LINUX, ON_WINDOWS, PYTHON_VERSION_INFO, expanduser, os_environ, ) @functools.lru_cache(1) def is_superuser(): if ON_WINDOWS: rtn = ctypes.windll.shell32.IsUserAnAdmin() != 0 else: rtn = os.getuid() == 0 return rtn class XonshError(Exception): pass class XonshCalledProcessError(XonshError, subprocess.CalledProcessError): """Raised when there's an error with a called process Inherits from XonshError and subprocess.CalledProcessError, catching either will also catch this error. Raised *after* iterating over stdout of a captured command, if the returncode of the command is nonzero. Example: try: for line in !(ls): print(line) except subprocess.CalledProcessError as error: print("Error in process: {}.format(error.completed_command.pid)) This also handles differences between Python3.4 and 3.5 where CalledProcessError is concerned. """ def __init__( self, returncode, command, output=None, stderr=None, completed_command=None ): super().__init__(returncode, command, output) self.stderr = stderr self.completed_command = completed_command def expand_path(s, expand_user=True): """Takes a string path and expands ~ to home if expand_user is set and environment vars if EXPAND_ENV_VARS is set.""" session = getattr(builtins, "__xonsh__", None) env = os_environ if session is None else getattr(session, "env", os_environ) if env.get("EXPAND_ENV_VARS", False): s = expandvars(s) if expand_user: # expand ~ according to Bash unquoted rules "Each variable assignment is # checked for unquoted tilde-prefixes immediately following a ':' or the # first '='". See the following for more details. # https://www.gnu.org/software/bash/manual/html_node/Tilde-Expansion.html pre, char, post = s.partition("=") if char: s = expanduser(pre) + char s += os.pathsep.join(map(expanduser, post.split(os.pathsep))) else: s = expanduser(s) return s def _expandpath(path): """Performs environment variable / user expansion on a given path if EXPAND_ENV_VARS is set. """ session = getattr(builtins, "__xonsh__", None) env = os_environ if session is None else getattr(session, "env", os_environ) expand_user = env.get("EXPAND_ENV_VARS", False) return expand_path(path, expand_user=expand_user) def decode_bytes(b): """Tries to decode the bytes using XONSH_ENCODING if available, otherwise using sys.getdefaultencoding(). """ session = getattr(builtins, "__xonsh__", None) env = os_environ if session is None else getattr(session, "env", os_environ) enc = env.get("XONSH_ENCODING") or DEFAULT_ENCODING err = env.get("XONSH_ENCODING_ERRORS") or "strict" return b.decode(encoding=enc, errors=err) def findfirst(s, substrs): """Finds whichever of the given substrings occurs first in the given string and returns that substring, or returns None if no such strings occur. """ i = len(s) result = None for substr in substrs: pos = s.find(substr) if -1 < pos < i: i = pos result = substr return i, result class EnvPath(cabc.MutableSequence): """A class that implements an environment path, which is a list of strings. Provides a custom method that expands all paths if the relevant env variable has been set. """ def __init__(self, args=None): if not args: self._l = [] else: if isinstance(args, str): self._l = args.split(os.pathsep) elif isinstance(args, pathlib.Path): self._l = [args] elif isinstance(args, bytes): # decode bytes to a string and then split based on # the default path separator self._l = decode_bytes(args).split(os.pathsep) elif isinstance(args, cabc.Iterable): # put everything in a list -before- performing the type check # in order to be able to retrieve it later, for cases such as # when a generator expression was passed as an argument args = list(args) if not all(isinstance(i, (str, bytes, pathlib.Path)) for i in args): # make TypeError's message as informative as possible # when given an invalid initialization sequence raise TypeError( "EnvPath's initialization sequence should only " "contain str, bytes and pathlib.Path entries" ) self._l = args else: raise TypeError( "EnvPath cannot be initialized with items " "of type %s" % type(args) ) def __getitem__(self, item): # handle slices separately if isinstance(item, slice): return [_expandpath(i) for i in self._l[item]] else: return _expandpath(self._l[item]) def __setitem__(self, index, item): self._l.__setitem__(index, item) def __len__(self): return len(self._l) def __delitem__(self, key): self._l.__delitem__(key) def insert(self, index, value): self._l.insert(index, value) @property def paths(self): """ Returns the list of directories that this EnvPath contains. """ return list(self) def __repr__(self): return repr(self._l) def __eq__(self, other): if len(self) != len(other): return False return all(map(operator.eq, self, other)) def _repr_pretty_(self, p, cycle): """ Pretty print path list """ if cycle: p.text("EnvPath(...)") else: with p.group(1, "EnvPath(\n[", "]\n)"): for idx, item in enumerate(self): if idx: p.text(",") p.breakable() p.pretty(item) def __add__(self, other): if isinstance(other, EnvPath): other = other._l return EnvPath(self._l + other) def __radd__(self, other): if isinstance(other, EnvPath): other = other._l return EnvPath(other + self._l) def add(self, data, front=False, replace=False): """Add a value to this EnvPath, path.add(data, front=bool, replace=bool) -> ensures that path contains data, with position determined by kwargs Parameters ---------- data : string or bytes or pathlib.Path value to be added front : bool whether the value should be added to the front, will be ignored if the data already exists in this EnvPath and replace is False Default : False replace : bool If True, the value will be removed and added to the start or end(depending on the value of front) Default : False Returns ------- None """ if data not in self._l: self._l.insert(0 if front else len(self._l), data) elif replace: self._l.remove(data) self._l.insert(0 if front else len(self._l), data) @lazyobject def FORMATTER(): return string.Formatter() class DefaultNotGivenType(object): """Singleton for representing when no default value is given.""" __inst = None def __new__(cls): if DefaultNotGivenType.__inst is None: DefaultNotGivenType.__inst = object.__new__(cls) return DefaultNotGivenType.__inst DefaultNotGiven = DefaultNotGivenType() BEG_TOK_SKIPS = LazyObject( lambda: frozenset(["WS", "INDENT", "NOT", "LPAREN"]), globals(), "BEG_TOK_SKIPS" ) END_TOK_TYPES = LazyObject( lambda: frozenset(["SEMI", "AND", "OR", "RPAREN"]), globals(), "END_TOK_TYPES" ) RE_END_TOKS = LazyObject( lambda: re.compile(r"(;|and|\&\&|or|\|\||\))"), globals(), "RE_END_TOKS" ) LPARENS = LazyObject( lambda: frozenset( ["LPAREN", "AT_LPAREN", "BANG_LPAREN", "DOLLAR_LPAREN", "ATDOLLAR_LPAREN"] ), globals(), "LPARENS", ) def _is_not_lparen_and_rparen(lparens, rtok): """Tests if an RPAREN token is matched with something other than a plain old LPAREN type. """ # note that any([]) is False, so this covers len(lparens) == 0 return rtok.type == "RPAREN" and any(x != "LPAREN" for x in lparens) def balanced_parens(line, mincol=0, maxcol=None, lexer=None): """Determines if parentheses are balanced in an expression.""" line = line[mincol:maxcol] if lexer is None: lexer = builtins.__xonsh__.execer.parser.lexer if "(" not in line and ")" not in line: return True cnt = 0 lexer.input(line) for tok in lexer: if tok.type in LPARENS: cnt += 1 elif tok.type == "RPAREN": cnt -= 1 elif tok.type == "ERRORTOKEN" and ")" in tok.value: cnt -= 1 return cnt == 0 def find_next_break(line, mincol=0, lexer=None): """Returns the column number of the next logical break in subproc mode. This function may be useful in finding the maxcol argument of subproc_toks(). """ if mincol >= 1: line = line[mincol:] if lexer is None: lexer = builtins.__xonsh__.execer.parser.lexer if RE_END_TOKS.search(line) is None: return None maxcol = None lparens = [] lexer.input(line) for tok in lexer: if tok.type in LPARENS: lparens.append(tok.type) elif tok.type in END_TOK_TYPES: if _is_not_lparen_and_rparen(lparens, tok): lparens.pop() else: maxcol = tok.lexpos + mincol + 1 break elif tok.type == "ERRORTOKEN" and ")" in tok.value: maxcol = tok.lexpos + mincol + 1 break elif tok.type == "BANG": maxcol = mincol + len(line) + 1 break return maxcol def _offset_from_prev_lines(line, last): lines = line.splitlines(keepends=True)[:last] return sum(map(len, lines)) def subproc_toks( line, mincol=-1, maxcol=None, lexer=None, returnline=False, greedy=False ): """Encapsulates tokens in a source code line in a uncaptured subprocess ![] starting at a minimum column. If there are no tokens (ie in a comment line) this returns None. If greedy is True, it will encapsulate normal parentheses. Greedy is False by default. """ if lexer is None: lexer = builtins.__xonsh__.execer.parser.lexer if maxcol is None: maxcol = len(line) + 1 lexer.reset() lexer.input(line) toks = [] lparens = [] saw_macro = False end_offset = 0 for tok in lexer: pos = tok.lexpos if tok.type not in END_TOK_TYPES and pos >= maxcol: break if tok.type == "BANG": saw_macro = True if saw_macro and tok.type not in ("NEWLINE", "DEDENT"): toks.append(tok) continue if tok.type in LPARENS: lparens.append(tok.type) if greedy and len(lparens) > 0 and "LPAREN" in lparens: toks.append(tok) if tok.type == "RPAREN": lparens.pop() continue if len(toks) == 0 and tok.type in BEG_TOK_SKIPS: continue # handle indentation elif len(toks) > 0 and toks[-1].type in END_TOK_TYPES: if _is_not_lparen_and_rparen(lparens, toks[-1]): lparens.pop() # don't continue or break elif pos < maxcol and tok.type not in ("NEWLINE", "DEDENT", "WS"): if not greedy: toks.clear() if tok.type in BEG_TOK_SKIPS: continue else: break if pos < mincol: continue toks.append(tok) if tok.type == "WS" and tok.value == "\\": pass # line continuation elif tok.type == "NEWLINE": break elif tok.type == "DEDENT": # fake a newline when dedenting without a newline tok.type = "NEWLINE" tok.value = "\n" tok.lineno -= 1 if len(toks) >= 2: prev_tok_end = toks[-2].lexpos + len(toks[-2].value) else: prev_tok_end = len(line) if "#" in line[prev_tok_end:]: tok.lexpos = prev_tok_end # prevents wrapping comments else: tok.lexpos = len(line) break elif check_bad_str_token(tok): return else: if len(toks) > 0 and toks[-1].type in END_TOK_TYPES: if _is_not_lparen_and_rparen(lparens, toks[-1]): pass elif greedy and toks[-1].type == "RPAREN": pass else: toks.pop() if len(toks) == 0: return # handle comment lines tok = toks[-1] pos = tok.lexpos if isinstance(tok.value, str): end_offset = len(tok.value.rstrip()) else: el = line[pos:].split("#")[0].rstrip() end_offset = len(el) if len(toks) == 0: return # handle comment lines elif saw_macro or greedy: end_offset = len(toks[-1].value.rstrip()) + 1 if toks[0].lineno != toks[-1].lineno: # handle multiline cases end_offset += _offset_from_prev_lines(line, toks[-1].lineno) beg, end = toks[0].lexpos, (toks[-1].lexpos + end_offset) end = len(line[:end].rstrip()) rtn = "![" + line[beg:end] + "]" if returnline: rtn = line[:beg] + rtn + line[end:] return rtn def check_bad_str_token(tok): """Checks if a token is a bad string.""" if tok.type == "ERRORTOKEN" and tok.value == "EOF in multi-line string": return True elif isinstance(tok.value, str) and not check_quotes(tok.value): return True else: return False def check_quotes(s): """Checks a string to make sure that if it starts with quotes, it also ends with quotes. """ starts_as_str = RE_BEGIN_STRING.match(s) is not None ends_as_str = s.endswith('"') or s.endswith("'") if not starts_as_str and not ends_as_str: ok = True elif starts_as_str and not ends_as_str: ok = False elif not starts_as_str and ends_as_str: ok = False else: m = RE_COMPLETE_STRING.match(s) ok = m is not None return ok def _have_open_triple_quotes(s): if s.count('"""') % 2 == 1: open_triple = '"""' elif s.count("'''") % 2 == 1: open_triple = "'''" else: open_triple = False return open_triple def get_line_continuation(): """ The line continuation characters used in subproc mode. In interactive mode on Windows the backslash must be preceded by a space. This is because paths on Windows may end in a backslash. """ if ( ON_WINDOWS and hasattr(builtins.__xonsh__, "env") and builtins.__xonsh__.env.get("XONSH_INTERACTIVE", False) ): return " \\" else: return "\\" def get_logical_line(lines, idx): """Returns a single logical line (i.e. one without line continuations) from a list of lines. This line should begin at index idx. This also returns the number of physical lines the logical line spans. The lines should not contain newlines """ n = 1 nlines = len(lines) linecont = get_line_continuation() while idx > 0 and lines[idx - 1].endswith(linecont): idx -= 1 start = idx line = lines[idx] open_triple = _have_open_triple_quotes(line) while (line.endswith(linecont) or open_triple) and idx < nlines - 1: n += 1 idx += 1 if line.endswith(linecont): line = line[:-1] + lines[idx] else: line = line + "\n" + lines[idx] open_triple = _have_open_triple_quotes(line) return line, n, start def replace_logical_line(lines, logical, idx, n): """Replaces lines at idx that may end in line continuation with a logical line that spans n lines. """ linecont = get_line_continuation() if n == 1: lines[idx] = logical return space = " " for i in range(idx, idx + n - 1): a = len(lines[i]) b = logical.find(space, a - 1) if b < 0: # no space found lines[i] = logical logical = "" else: # found space to split on lines[i] = logical[:b] + linecont logical = logical[b:] lines[idx + n - 1] = logical def is_balanced(expr, ltok, rtok): """Determines whether an expression has unbalanced opening and closing tokens.""" lcnt = expr.count(ltok) if lcnt == 0: return True rcnt = expr.count(rtok) if lcnt == rcnt: return True else: return False def subexpr_from_unbalanced(expr, ltok, rtok): """Attempts to pull out a valid subexpression for unbalanced grouping, based on opening tokens, eg. '(', and closing tokens, eg. ')'. This does not do full tokenization, but should be good enough for tab completion. """ if is_balanced(expr, ltok, rtok): return expr subexpr = expr.rsplit(ltok, 1)[-1] subexpr = subexpr.rsplit(",", 1)[-1] subexpr = subexpr.rsplit(":", 1)[-1] return subexpr def subexpr_before_unbalanced(expr, ltok, rtok): """Obtains the expression prior to last unbalanced left token.""" subexpr, _, post = expr.rpartition(ltok) nrtoks_in_post = post.count(rtok) while nrtoks_in_post != 0: for i in range(nrtoks_in_post): subexpr, _, post = subexpr.rpartition(ltok) nrtoks_in_post = post.count(rtok) _, _, subexpr = subexpr.rpartition(rtok) _, _, subexpr = subexpr.rpartition(ltok) return subexpr @lazyobject def STARTING_WHITESPACE_RE(): return re.compile(r"^(\s*)") def starting_whitespace(s): """Returns the whitespace at the start of a string""" return STARTING_WHITESPACE_RE.match(s).group(1) def decode(s, encoding=None): encoding = encoding or DEFAULT_ENCODING return s.decode(encoding, "replace") def encode(u, encoding=None): encoding = encoding or DEFAULT_ENCODING return u.encode(encoding, "replace") def cast_unicode(s, encoding=None): if isinstance(s, bytes): return decode(s, encoding) return s def safe_hasattr(obj, attr): """In recent versions of Python, hasattr() only catches AttributeError. This catches all errors. """ try: getattr(obj, attr) return True except Exception: # pylint:disable=bare-except return False def indent(instr, nspaces=4, ntabs=0, flatten=False): """Indent a string a given number of spaces or tabstops. indent(str,nspaces=4,ntabs=0) -> indent str by ntabs+nspaces. Parameters ---------- instr : basestring The string to be indented. nspaces : int (default: 4) The number of spaces to be indented. ntabs : int (default: 0) The number of tabs to be indented. flatten : bool (default: False) Whether to scrub existing indentation. If True, all lines will be aligned to the same indentation. If False, existing indentation will be strictly increased. Returns ------- outstr : string indented by ntabs and nspaces. """ if instr is None: return ind = "\t" * ntabs + " " * nspaces if flatten: pat = re.compile(r"^\s*", re.MULTILINE) else: pat = re.compile(r"^", re.MULTILINE) outstr = re.sub(pat, ind, instr) if outstr.endswith(os.linesep + ind): return outstr[: -len(ind)] else: return outstr def get_sep(): """ Returns the appropriate filepath separator char depending on OS and xonsh options set """ if ON_WINDOWS and builtins.__xonsh__.env.get("FORCE_POSIX_PATHS"): return os.altsep else: return os.sep def fallback(cond, backup): """Decorator for returning the object if cond is true and a backup if cond is false. """ def dec(obj): return obj if cond else backup return dec # The following redirect classes were taken directly from Python 3.5's source # code (from the contextlib module). This can be removed when 3.5 is released, # although redirect_stdout exists in 3.4, redirect_stderr does not. # See the Python software license: https://docs.python.org/3/license.html # Copyright (c) Python Software Foundation. All rights reserved. class _RedirectStream: _stream = None def __init__(self, new_target): self._new_target = new_target # We use a list of old targets to make this CM re-entrant self._old_targets = [] def __enter__(self): self._old_targets.append(getattr(sys, self._stream)) setattr(sys, self._stream, self._new_target) return self._new_target def __exit__(self, exctype, excinst, exctb): setattr(sys, self._stream, self._old_targets.pop()) class redirect_stdout(_RedirectStream): """Context manager for temporarily redirecting stdout to another file:: # How to send help() to stderr with redirect_stdout(sys.stderr): help(dir) # How to write help() to a file with open('help.txt', 'w') as f: with redirect_stdout(f): help(pow) Mostly for backwards compatibility. """ _stream = "stdout" class redirect_stderr(_RedirectStream): """Context manager for temporarily redirecting stderr to another file.""" _stream = "stderr" def _yield_accessible_unix_file_names(path): """yield file names of executable files in path.""" if not os.path.exists(path): return for file_ in scandir(path): try: if file_.is_file() and os.access(file_.path, os.X_OK): yield file_.name except OSError: # broken Symlink are neither dir not files pass def _executables_in_posix(path): if not os.path.exists(path): return elif PYTHON_VERSION_INFO < (3, 5, 0): for fname in os.listdir(path): fpath = os.path.join(path, fname) if ( os.path.exists(fpath) and os.access(fpath, os.X_OK) and (not os.path.isdir(fpath)) ): yield fname else: yield from _yield_accessible_unix_file_names(path) def _executables_in_windows(path): if not os.path.isdir(path): return extensions = builtins.__xonsh__.env["PATHEXT"] if PYTHON_VERSION_INFO < (3, 5, 0): for fname in os.listdir(path): fpath = os.path.join(path, fname) if os.path.exists(fpath) and not os.path.isdir(fpath): base_name, ext = os.path.splitext(fname) if ext.upper() in extensions: yield fname else: for x in scandir(path): try: is_file = x.is_file() except OSError: continue if is_file: fname = x.name else: continue base_name, ext = os.path.splitext(fname) if ext.upper() in extensions: yield fname def executables_in(path): """Returns a generator of files in path that the user could execute. """ if ON_WINDOWS: func = _executables_in_windows else: func = _executables_in_posix try: yield from func(path) except PermissionError: return def command_not_found(cmd): """Uses the debian/ubuntu command-not-found utility to suggest packages for a command that cannot currently be found. """ if not ON_LINUX: return "" elif not os.path.isfile("/usr/lib/command-not-found"): # utility is not on PATH return "" c = "/usr/lib/command-not-found {0}; exit 0" s = subprocess.check_output( c.format(cmd), universal_newlines=True, stderr=subprocess.STDOUT, shell=True ) s = "\n".join(s.rstrip().splitlines()).strip() return s def suggest_commands(cmd, env, aliases): """Suggests alternative commands given an environment and aliases.""" if not env.get("SUGGEST_COMMANDS"): return "" thresh = env.get("SUGGEST_THRESHOLD") max_sugg = env.get("SUGGEST_MAX_NUM") if max_sugg < 0: max_sugg = float("inf") cmd = cmd.lower() suggested = {} for alias in builtins.aliases: if alias not in suggested: if levenshtein(alias.lower(), cmd, thresh) < thresh: suggested[alias] = "Alias" for path in filter(os.path.isdir, env.get("PATH")): for _file in executables_in(path): if ( _file not in suggested and levenshtein(_file.lower(), cmd, thresh) < thresh ): suggested[_file] = "Command ({0})".format(os.path.join(path, _file)) suggested = collections.OrderedDict( sorted( suggested.items(), key=lambda x: suggestion_sort_helper(x[0].lower(), cmd) ) ) num = min(len(suggested), max_sugg) if num == 0: rtn = command_not_found(cmd) else: oneof = "" if num == 1 else "one of " tips = "Did you mean {}the following?".format(oneof) items = list(suggested.popitem(False) for _ in range(num)) length = max(len(key) for key, _ in items) + 2 alternatives = "\n".join( " {: <{}} {}".format(key + ":", length, val) for key, val in items ) rtn = "{}\n{}".format(tips, alternatives) c = command_not_found(cmd) rtn += ("\n\n" + c) if len(c) > 0 else "" return rtn def print_exception(msg=None): """Print exceptions with/without traceback.""" env = getattr(builtins.__xonsh__, "env", None) # flags indicating whether the traceback options have been manually set if env is None: env = os_environ manually_set_trace = "XONSH_SHOW_TRACEBACK" in env manually_set_logfile = "XONSH_TRACEBACK_LOGFILE" in env else: manually_set_trace = env.is_manually_set("XONSH_SHOW_TRACEBACK") manually_set_logfile = env.is_manually_set("XONSH_TRACEBACK_LOGFILE") if (not manually_set_trace) and (not manually_set_logfile): # Notify about the traceback output possibility if neither of # the two options have been manually set sys.stderr.write( "xonsh: For full traceback set: " "$XONSH_SHOW_TRACEBACK = True\n" ) # get env option for traceback and convert it if necessary show_trace = env.get("XONSH_SHOW_TRACEBACK", False) if not is_bool(show_trace): show_trace = to_bool(show_trace) # if the trace option has been set, print all traceback info to stderr if show_trace: # notify user about XONSH_TRACEBACK_LOGFILE if it has # not been set manually if not manually_set_logfile: sys.stderr.write( "xonsh: To log full traceback to a file set: " "$XONSH_TRACEBACK_LOGFILE = \n" ) traceback.print_exc() # additionally, check if a file for traceback logging has been # specified and convert to a proper option if needed log_file = env.get("XONSH_TRACEBACK_LOGFILE", None) log_file = to_logfile_opt(log_file) if log_file: # if log_file <> '' or log_file <> None, append # traceback log there as well with open(os.path.abspath(log_file), "a") as f: traceback.print_exc(file=f) if not show_trace: # if traceback output is disabled, print the exception's # error message on stderr. display_error_message() if msg: msg = msg if msg.endswith("\n") else msg + "\n" sys.stderr.write(msg) def display_error_message(strip_xonsh_error_types=True): """ Prints the error message of the current exception on stderr. """ exc_type, exc_value, exc_traceback = sys.exc_info() exception_only = traceback.format_exception_only(exc_type, exc_value) if exc_type is XonshError and strip_xonsh_error_types: exception_only[0] = exception_only[0].partition(": ")[-1] sys.stderr.write("".join(exception_only)) def is_writable_file(filepath): """ Checks if a filepath is valid for writing. """ filepath = expand_path(filepath) # convert to absolute path if needed if not os.path.isabs(filepath): filepath = os.path.abspath(filepath) # cannot write to directories if os.path.isdir(filepath): return False # if the file exists and is writable, we're fine if os.path.exists(filepath): return True if os.access(filepath, os.W_OK) else False # if the path doesn't exist, isolate its directory component # and ensure that directory is writable instead return os.access(os.path.dirname(filepath), os.W_OK) # Modified from Public Domain code, by Magnus Lie Hetland # from http://hetland.org/coding/python/levenshtein.py def levenshtein(a, b, max_dist=float("inf")): """Calculates the Levenshtein distance between a and b.""" n, m = len(a), len(b) if abs(n - m) > max_dist: return float("inf") if n > m: # Make sure n <= m, to use O(min(n,m)) space a, b = b, a n, m = m, n current = range(n + 1) for i in range(1, m + 1): previous, current = current, [i] + [0] * n for j in range(1, n + 1): add, delete = previous[j] + 1, current[j - 1] + 1 change = previous[j - 1] if a[j - 1] != b[i - 1]: change = change + 1 current[j] = min(add, delete, change) return current[n] def suggestion_sort_helper(x, y): """Returns a score (lower is better) for x based on how similar it is to y. Used to rank suggestions.""" x = x.lower() y = y.lower() lendiff = len(x) + len(y) inx = len([i for i in x if i not in y]) iny = len([i for i in y if i not in x]) return lendiff + inx + iny def escape_windows_cmd_string(s): """Returns a string that is usable by the Windows cmd.exe. The escaping is based on details here and empirical testing: http://www.robvanderwoude.com/escapechars.php """ for c in '^()%!<>&|"': s = s.replace(c, "^" + c) return s def argvquote(arg, force=False): """ Returns an argument quoted in such a way that that CommandLineToArgvW on Windows will return the argument string unchanged. This is the same thing Popen does when supplied with an list of arguments. Arguments in a command line should be separated by spaces; this function does not add these spaces. This implementation follows the suggestions outlined here: https://blogs.msdn.microsoft.com/twistylittlepassagesallalike/2011/04/23/everyone-quotes-command-line-arguments-the-wrong-way/ """ if not force and len(arg) != 0 and not any([c in arg for c in ' \t\n\v"']): return arg else: n_backslashes = 0 cmdline = '"' for c in arg: if c == "\\": # first count the number of current backslashes n_backslashes += 1 continue if c == '"': # Escape all backslashes and the following double quotation mark cmdline += (n_backslashes * 2 + 1) * "\\" else: # backslashes are not special here cmdline += n_backslashes * "\\" n_backslashes = 0 cmdline += c # Escape all backslashes, but let the terminating # double quotation mark we add below be interpreted # as a metacharacter cmdline += +n_backslashes * 2 * "\\" + '"' return cmdline def on_main_thread(): """Checks if we are on the main thread or not.""" return threading.current_thread() is threading.main_thread() _DEFAULT_SENTINEL = object() @contextlib.contextmanager def swap(namespace, name, value, default=_DEFAULT_SENTINEL): """Swaps a current variable name in a namespace for another value, and then replaces it when the context is exited. """ old = getattr(namespace, name, default) setattr(namespace, name, value) yield value if old is default: delattr(namespace, name) else: setattr(namespace, name, old) @contextlib.contextmanager def swap_values(d, updates, default=_DEFAULT_SENTINEL): """Updates a dictionary (or other mapping) with values from another mapping, and then restores the original mapping when the context is exited. """ old = {k: d.get(k, default) for k in updates} d.update(updates) yield for k, v in old.items(): if v is default and k in d: del d[k] else: d[k] = v # # Validators and converters # def detype(x): """This assumes that the object has a detype method, and calls that.""" return x.detype() def is_int(x): """Tests if something is an integer""" return isinstance(x, int) def is_float(x): """Tests if something is a float""" return isinstance(x, float) def is_string(x): """Tests if something is a string""" return isinstance(x, str) def is_slice(x): """Tests if something is a slice""" return isinstance(x, slice) def is_callable(x): """Tests if something is callable""" return callable(x) def is_string_or_callable(x): """Tests if something is a string or callable""" return is_string(x) or is_callable(x) def is_class(x): """Tests if something is a class""" return isinstance(x, type) def always_true(x): """Returns True""" return True def always_false(x): """Returns False""" return False def always_none(x): """Returns None""" return None def ensure_string(x): """Returns a string if x is not a string, and x if it already is.""" return str(x) def is_env_path(x): """This tests if something is an environment path, ie a list of strings.""" return isinstance(x, EnvPath) def str_to_env_path(x): """Converts a string to an environment path, ie a list of strings, splitting on the OS separator. """ # splitting will be done implicitly in EnvPath's __init__ return EnvPath(x) def env_path_to_str(x): """Converts an environment path to a string by joining on the OS separator. """ return os.pathsep.join(x) def is_bool(x): """Tests if something is a boolean.""" return isinstance(x, bool) def is_logfile_opt(x): """ Checks if x is a valid $XONSH_TRACEBACK_LOGFILE option. Returns False if x is not a writable/creatable file or an empty string or None. """ if x is None: return True if not isinstance(x, str): return False else: return is_writable_file(x) or x == "" def to_logfile_opt(x): """ Converts a $XONSH_TRACEBACK_LOGFILE option to either a str containing the filepath if it is a writable file or None if the filepath is not valid, informing the user on stderr about the invalid choice. """ superclass = pathlib.PurePath if PYTHON_VERSION_INFO >= (3, 6, 0): superclass = os.PathLike if isinstance(x, superclass): x = str(x) if is_logfile_opt(x): return x else: # if option is not valid, return a proper # option and inform the user on stderr sys.stderr.write( "xonsh: $XONSH_TRACEBACK_LOGFILE must be a " "filepath pointing to a file that either exists " "and is writable or that can be created.\n" ) return None def logfile_opt_to_str(x): """ Detypes a $XONSH_TRACEBACK_LOGFILE option. """ if x is None: # None should not be detyped to 'None', as 'None' constitutes # a perfectly valid filename and retyping it would introduce # ambiguity. Detype to the empty string instead. return "" return str(x) _FALSES = LazyObject( lambda: frozenset(["", "0", "n", "f", "no", "none", "false", "off"]), globals(), "_FALSES", ) def to_bool(x): """"Converts to a boolean in a semantically meaningful way.""" if isinstance(x, bool): return x elif isinstance(x, str): return False if x.lower() in _FALSES else True else: return bool(x) def to_itself(x): """No conversion, returns itself.""" return x def bool_to_str(x): """Converts a bool to an empty string if False and the string '1' if True. """ return "1" if x else "" _BREAKS = LazyObject( lambda: frozenset(["b", "break", "s", "skip", "q", "quit"]), globals(), "_BREAKS" ) def to_bool_or_break(x): if isinstance(x, str) and x.lower() in _BREAKS: return "break" else: return to_bool(x) def is_bool_or_int(x): """Returns whether a value is a boolean or integer.""" return is_bool(x) or is_int(x) def to_bool_or_int(x): """Converts a value to a boolean or an integer.""" if isinstance(x, str): return int(x) if x.isdigit() else to_bool(x) elif is_int(x): # bools are ints too! return x else: return bool(x) def bool_or_int_to_str(x): """Converts a boolean or integer to a string.""" return bool_to_str(x) if is_bool(x) else str(x) @lazyobject def SLICE_REG(): return re.compile( r"(?P(?:-\d)?\d*):(?P(?:-\d)?\d*):?(?P(?:-\d)?\d*)" ) def ensure_slice(x): """Try to convert an object into a slice, complain on failure""" if not x and x != 0: return slice(None) elif is_slice(x): return x try: x = int(x) if x != -1: s = slice(x, x + 1) else: s = slice(-1, None, None) except ValueError: x = x.strip("[]()") m = SLICE_REG.fullmatch(x) if m: groups = (int(i) if i else None for i in m.groups()) s = slice(*groups) else: raise ValueError("cannot convert {!r} to slice".format(x)) except TypeError: try: s = slice(*(int(i) for i in x)) except (TypeError, ValueError): raise ValueError("cannot convert {!r} to slice".format(x)) return s def get_portions(it, slices): """Yield from portions of an iterable. Parameters ---------- it: iterable slices: a slice or a list of slice objects """ if is_slice(slices): slices = [slices] if len(slices) == 1: s = slices[0] try: yield from itertools.islice(it, s.start, s.stop, s.step) return except ValueError: # islice failed pass it = list(it) for s in slices: yield from it[s] def is_slice_as_str(x): """ Test if string x is a slice. If not a string return False. """ try: x = x.strip("[]()") m = SLICE_REG.fullmatch(x) if m: return True except AttributeError: pass return False def is_int_as_str(x): """ Test if string x is an integer. If not a string return False. """ try: return x.isdecimal() except AttributeError: return False def is_string_set(x): """Tests if something is a set of strings""" return isinstance(x, cabc.Set) and all(isinstance(a, str) for a in x) def csv_to_set(x): """Convert a comma-separated list of strings to a set of strings.""" if not x: return set() else: return set(x.split(",")) def set_to_csv(x): """Convert a set of strings to a comma-separated list of strings.""" return ",".join(x) def pathsep_to_set(x): """Converts a os.pathsep separated string to a set of strings.""" if not x: return set() else: return set(x.split(os.pathsep)) def set_to_pathsep(x, sort=False): """Converts a set to an os.pathsep separated string. The sort kwarg specifies whether to sort the set prior to str conversion. """ if sort: x = sorted(x) return os.pathsep.join(x) def is_string_seq(x): """Tests if something is a sequence of strings""" return isinstance(x, cabc.Sequence) and all(isinstance(a, str) for a in x) def is_nonstring_seq_of_strings(x): """Tests if something is a sequence of strings, where the top-level sequence is not a string itself. """ return ( isinstance(x, cabc.Sequence) and not isinstance(x, str) and all(isinstance(a, str) for a in x) ) def pathsep_to_seq(x): """Converts a os.pathsep separated string to a sequence of strings.""" if not x: return [] else: return x.split(os.pathsep) def seq_to_pathsep(x): """Converts a sequence to an os.pathsep separated string.""" return os.pathsep.join(x) def pathsep_to_upper_seq(x): """Converts a os.pathsep separated string to a sequence of uppercase strings. """ if not x: return [] else: return x.upper().split(os.pathsep) def seq_to_upper_pathsep(x): """Converts a sequence to an uppercase os.pathsep separated string.""" return os.pathsep.join(x).upper() def is_bool_seq(x): """Tests if an object is a sequence of bools.""" return isinstance(x, cabc.Sequence) and all(isinstance(y, bool) for y in x) def csv_to_bool_seq(x): """Takes a comma-separated string and converts it into a list of bools.""" return [to_bool(y) for y in csv_to_set(x)] def bool_seq_to_csv(x): """Converts a sequence of bools to a comma-separated string.""" return ",".join(map(str, x)) def ptk2_color_depth_setter(x): """ Setter function for $PROMPT_TOOLKIT_COLOR_DEPTH. Also updates os.environ so prompt toolkit can pickup the value. """ x = str(x) if x in { "DEPTH_1_BIT", "MONOCHROME", "DEPTH_4_BIT", "ANSI_COLORS_ONLY", "DEPTH_8_BIT", "DEFAULT", "DEPTH_24_BIT", "TRUE_COLOR", }: pass elif x in {"", None}: x = "" else: msg = '"{}" is not a valid value for $PROMPT_TOOLKIT_COLOR_DEPTH. '.format(x) warnings.warn(msg, RuntimeWarning) x = "" if x == "" and "PROMPT_TOOLKIT_COLOR_DEPTH" in os_environ: del os_environ["PROMPT_TOOLKIT_COLOR_DEPTH"] else: os_environ["PROMPT_TOOLKIT_COLOR_DEPTH"] = x return x def is_completions_display_value(x): return x in {"none", "single", "multi"} def to_completions_display_value(x): x = str(x).lower() if x in {"none", "false"}: x = "none" elif x in {"multi", "true"}: x = "multi" elif x in {"single", "readline"}: pass else: msg = '"{}" is not a valid value for $COMPLETIONS_DISPLAY. '.format(x) msg += 'Using "multi".' warnings.warn(msg, RuntimeWarning) x = "multi" return x def is_str_str_dict(x): """Tests if something is a str:str dictionary""" return isinstance(x, dict) and all( isinstance(k, str) and isinstance(v, str) for k, v in x.items() ) def to_dict(x): """Converts a string to a dictionary""" if isinstance(x, dict): return x try: x = ast.literal_eval(x) except (ValueError, SyntaxError): msg = '"{}" can not be converted to Python dictionary.'.format(x) warnings.warn(msg, RuntimeWarning) x = dict() return x def to_str_str_dict(x): """Converts a string to str:str dictionary""" if is_str_str_dict(x): return x x = to_dict(x) if not is_str_str_dict(x): msg = '"{}" can not be converted to str:str dictionary.'.format(x) warnings.warn(msg, RuntimeWarning) x = dict() return x def dict_to_str(x): """Converts a dictionary to a string""" if not x or len(x) == 0: return "" return str(x) def setup_win_unicode_console(enable): """"Enables or disables unicode display on windows.""" try: import win_unicode_console except ImportError: win_unicode_console = False enable = to_bool(enable) if ON_WINDOWS and win_unicode_console: if enable: win_unicode_console.enable() else: win_unicode_console.disable() return enable # history validation _min_to_sec = lambda x: 60.0 * float(x) _hour_to_sec = lambda x: 60.0 * _min_to_sec(x) _day_to_sec = lambda x: 24.0 * _hour_to_sec(x) _month_to_sec = lambda x: 30.4375 * _day_to_sec(x) _year_to_sec = lambda x: 365.25 * _day_to_sec(x) _kb_to_b = lambda x: 1024 * int(x) _mb_to_b = lambda x: 1024 * _kb_to_b(x) _gb_to_b = lambda x: 1024 * _mb_to_b(x) _tb_to_b = lambda x: 1024 * _tb_to_b(x) CANON_HISTORY_UNITS = LazyObject( lambda: frozenset(["commands", "files", "s", "b"]), globals(), "CANON_HISTORY_UNITS" ) HISTORY_UNITS = LazyObject( lambda: { "": ("commands", int), "c": ("commands", int), "cmd": ("commands", int), "cmds": ("commands", int), "command": ("commands", int), "commands": ("commands", int), "f": ("files", int), "files": ("files", int), "s": ("s", float), "sec": ("s", float), "second": ("s", float), "seconds": ("s", float), "m": ("s", _min_to_sec), "min": ("s", _min_to_sec), "mins": ("s", _min_to_sec), "h": ("s", _hour_to_sec), "hr": ("s", _hour_to_sec), "hour": ("s", _hour_to_sec), "hours": ("s", _hour_to_sec), "d": ("s", _day_to_sec), "day": ("s", _day_to_sec), "days": ("s", _day_to_sec), "mon": ("s", _month_to_sec), "month": ("s", _month_to_sec), "months": ("s", _month_to_sec), "y": ("s", _year_to_sec), "yr": ("s", _year_to_sec), "yrs": ("s", _year_to_sec), "year": ("s", _year_to_sec), "years": ("s", _year_to_sec), "b": ("b", int), "byte": ("b", int), "bytes": ("b", int), "kb": ("b", _kb_to_b), "kilobyte": ("b", _kb_to_b), "kilobytes": ("b", _kb_to_b), "mb": ("b", _mb_to_b), "meg": ("b", _mb_to_b), "megs": ("b", _mb_to_b), "megabyte": ("b", _mb_to_b), "megabytes": ("b", _mb_to_b), "gb": ("b", _gb_to_b), "gig": ("b", _gb_to_b), "gigs": ("b", _gb_to_b), "gigabyte": ("b", _gb_to_b), "gigabytes": ("b", _gb_to_b), "tb": ("b", _tb_to_b), "terabyte": ("b", _tb_to_b), "terabytes": ("b", _tb_to_b), }, globals(), "HISTORY_UNITS", ) """Maps lowercase unit names to canonical name and conversion utilities.""" def is_history_tuple(x): """Tests if something is a proper history value, units tuple.""" if ( isinstance(x, cabc.Sequence) and len(x) == 2 and isinstance(x[0], (int, float)) and x[1].lower() in CANON_HISTORY_UNITS ): return True return False def is_history_backend(x): """Tests if something is a valid history backend.""" return is_string(x) or is_class(x) or isinstance(x, object) def is_dynamic_cwd_width(x): """ Determine if the input is a valid input for the DYNAMIC_CWD_WIDTH environment variable. """ return ( isinstance(x, tuple) and len(x) == 2 and isinstance(x[0], float) and x[1] in set("c%") ) def to_dynamic_cwd_tuple(x): """Convert to a canonical cwd_width tuple.""" unit = "c" if isinstance(x, str): if x[-1] == "%": x = x[:-1] unit = "%" else: unit = "c" return (float(x), unit) else: return (float(x[0]), x[1]) def dynamic_cwd_tuple_to_str(x): """Convert a canonical cwd_width tuple to a string.""" if x[1] == "%": return str(x[0]) + "%" else: return str(x[0]) RE_HISTORY_TUPLE = LazyObject( lambda: re.compile(r"([-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?)\s*([A-Za-z]*)"), globals(), "RE_HISTORY_TUPLE", ) def to_history_tuple(x): """Converts to a canonical history tuple.""" if not isinstance(x, (cabc.Sequence, float, int)): raise ValueError("history size must be given as a sequence or number") if isinstance(x, str): m = RE_HISTORY_TUPLE.match(x.strip().lower()) return to_history_tuple((m.group(1), m.group(3))) elif isinstance(x, (float, int)): return to_history_tuple((x, "commands")) units, converter = HISTORY_UNITS[x[1]] value = converter(x[0]) return (value, units) def history_tuple_to_str(x): """Converts a valid history tuple to a canonical string.""" return "{0} {1}".format(*x) def format_color(string, **kwargs): """Formats strings that may contain colors. This simply dispatches to the shell instances method of the same name. The results of this function should be directly usable by print_color(). """ return builtins.__xonsh__.shell.shell.format_color(string, **kwargs) def print_color(string, **kwargs): """Prints a string that may contain colors. This dispatched to the shell method of the same name. Colors will be formatted if they have not already been. """ builtins.__xonsh__.shell.shell.print_color(string, **kwargs) def color_style_names(): """Returns an iterable of all available style names.""" return builtins.__xonsh__.shell.shell.color_style_names() def color_style(): """Returns the current color map.""" return builtins.__xonsh__.shell.shell.color_style() def _token_attr_from_stylemap(stylemap): """yields tokens attr, and index from a stylemap """ import prompt_toolkit as ptk if builtins.__xonsh__.shell.shell_type == "prompt_toolkit1": style = ptk.styles.style_from_dict(stylemap) for token in stylemap: yield token, style.token_to_attrs[token] else: style = ptk.styles.style_from_pygments_dict(stylemap) for token in stylemap: style_str = "class:{}".format( ptk.styles.pygments.pygments_token_to_classname(token) ) yield (token, style.get_attrs_for_style_str(style_str)) def _get_color_lookup_table(): """Returns the prompt_toolkit win32 ColorLookupTable """ if builtins.__xonsh__.shell.shell_type == "prompt_toolkit1": from prompt_toolkit.terminal.win32_output import ColorLookupTable else: from prompt_toolkit.output.win32 import ColorLookupTable return ColorLookupTable() def _get_color_indexes(style_map): """Generates the color and windows color index for a style """ table = _get_color_lookup_table() for token, attr in _token_attr_from_stylemap(style_map): if attr.color: index = table.lookup_fg_color(attr.color) try: rgb = ( int(attr.color[0:2], 16), int(attr.color[2:4], 16), int(attr.color[4:6], 16), ) except Exception: rgb = None yield token, index, rgb # Map of new PTK2 color names to PTK1 variants PTK_NEW_OLD_COLOR_MAP = LazyObject( lambda: { "black": "black", "red": "darkred", "green": "darkgreen", "yellow": "brown", "blue": "darkblue", "magenta": "purple", "cyan": "teal", "gray": "lightgray", "brightblack": "darkgray", "brightred": "red", "brightgreen": "green", "brightyellow": "yellow", "brightblue": "blue", "brightmagenta": "fuchsia", "brightcyan": "turquoise", "white": "white", }, globals(), "PTK_NEW_OLD_COLOR_MAP", ) # Map of new ansicolor names to old PTK1 names ANSICOLOR_NAMES_MAP = LazyObject( lambda: {"ansi" + k: "#ansi" + v for k, v in PTK_NEW_OLD_COLOR_MAP.items()}, globals(), "ANSICOLOR_NAMES_MAP", ) def _win10_color_map(): cmap = { "ansiblack": (12, 12, 12), "ansiblue": (0, 55, 218), "ansigreen": (19, 161, 14), "ansicyan": (58, 150, 221), "ansired": (197, 15, 31), "ansimagenta": (136, 23, 152), "ansiyellow": (193, 156, 0), "ansigray": (204, 204, 204), "ansibrightblack": (118, 118, 118), "ansibrightblue": (59, 120, 255), "ansibrightgreen": (22, 198, 12), "ansibrightcyan": (97, 214, 214), "ansibrightred": (231, 72, 86), "ansibrightmagenta": (180, 0, 158), "ansibrightyellow": (249, 241, 165), "ansiwhite": (242, 242, 242), } return { k: "#{0:02x}{1:02x}{2:02x}".format(r, g, b) for k, (r, g, b) in cmap.items() } WIN10_COLOR_MAP = LazyObject(_win10_color_map, globals(), "WIN10_COLOR_MAP") def _win_bold_color_map(): """ Map dark ansi colors to lighter version. """ return { "ansiblack": "ansibrightblack", "ansiblue": "ansibrightblue", "ansigreen": "ansibrightgreen", "ansicyan": "ansibrightcyan", "ansired": "ansibrightred", "ansimagenta": "ansibrightmagenta", "ansiyellow": "ansibrightyellow", "ansigray": "ansiwhite", } WIN_BOLD_COLOR_MAP = LazyObject(_win_bold_color_map, globals(), "WIN_BOLD_COLOR_MAP") def hardcode_colors_for_win10(style_map): """Replace all ansi colors with hardcoded colors to avoid unreadable defaults in conhost.exe """ modified_style = {} if not builtins.__xonsh__.env["PROMPT_TOOLKIT_COLOR_DEPTH"]: builtins.__xonsh__.env["PROMPT_TOOLKIT_COLOR_DEPTH"] = "DEPTH_24_BIT" # Replace all ansi colors with hardcoded colors to avoid unreadable defaults # in conhost.exe for token, style_str in style_map.items(): for ansicolor in WIN10_COLOR_MAP: if ansicolor in style_str: if "bold" in style_str and "nobold" not in style_str: # Win10 doesn't yet handle bold colors. Instead dark # colors are mapped to their lighter version. We simulate # the same here. style_str.replace("bold", "") hexcolor = WIN10_COLOR_MAP[ WIN_BOLD_COLOR_MAP.get(ansicolor, ansicolor) ] else: hexcolor = WIN10_COLOR_MAP[ansicolor] style_str = style_str.replace(ansicolor, hexcolor) modified_style[token] = style_str return modified_style def ansicolors_to_ptk1_names(stylemap): """Converts ansicolor names in a stylemap to old PTK1 color names """ modified_stylemap = {} for token, style_str in stylemap.items(): for color, ptk1_color in ANSICOLOR_NAMES_MAP.items(): if "#" + color not in style_str: style_str = style_str.replace(color, ptk1_color) modified_stylemap[token] = style_str return modified_stylemap def intensify_colors_for_cmd_exe(style_map): """Returns a modified style to where colors that maps to dark colors are replaced with brighter versions. """ modified_style = {} replace_colors = { 1: "ansibrightcyan", # subst blue with bright cyan 2: "ansibrightgreen", # subst green with bright green 4: "ansibrightred", # subst red with bright red 5: "ansibrightmagenta", # subst magenta with bright magenta 6: "ansibrightyellow", # subst yellow with bright yellow 9: "ansicyan", # subst intense blue with dark cyan (more readable) } if builtins.__xonsh__.shell.shell_type == "prompt_toolkit1": replace_colors = ansicolors_to_ptk1_names(replace_colors) for token, idx, _ in _get_color_indexes(style_map): if idx in replace_colors: modified_style[token] = replace_colors[idx] return modified_style def intensify_colors_on_win_setter(enable): """Resets the style when setting the INTENSIFY_COLORS_ON_WIN environment variable. """ enable = to_bool(enable) if builtins.__xonsh__.shell is not None: if hasattr(builtins.__xonsh__.shell.shell.styler, "style_name"): delattr(builtins.__xonsh__.shell.shell.styler, "style_name") return enable def format_std_prepost(template, env=None): """Formats a template prefix/postfix string for a standard buffer. Returns a string suitable for prepending or appending. """ if not template: return "" env = builtins.__xonsh__.env if env is None else env shell = builtins.__xonsh__.shell.shell try: s = shell.prompt_formatter(template) except Exception: print_exception() # \001\002 is there to fool pygments into not returning an empty string # for potentially empty input. This happens when the template is just a # color code with no visible text. invis = "\001\002" s = shell.format_color(invis + s + invis, force_string=True) s = s.replace(invis, "") return s _RE_STRING_START = "[bBprRuUf]*" _RE_STRING_TRIPLE_DOUBLE = '"""' _RE_STRING_TRIPLE_SINGLE = "'''" _RE_STRING_DOUBLE = '"' _RE_STRING_SINGLE = "'" _STRINGS = ( _RE_STRING_TRIPLE_DOUBLE, _RE_STRING_TRIPLE_SINGLE, _RE_STRING_DOUBLE, _RE_STRING_SINGLE, ) RE_BEGIN_STRING = LazyObject( lambda: re.compile("(" + _RE_STRING_START + "(" + "|".join(_STRINGS) + "))"), globals(), "RE_BEGIN_STRING", ) """Regular expression matching the start of a string, including quotes and leading characters (r, b, or u)""" RE_STRING_START = LazyObject( lambda: re.compile(_RE_STRING_START), globals(), "RE_STRING_START" ) """Regular expression matching the characters before the quotes when starting a string (r, b, or u, case insensitive)""" RE_STRING_CONT = LazyDict( { '"': lambda: re.compile(r'((\\(.|\n))|([^"\\]))*'), "'": lambda: re.compile(r"((\\(.|\n))|([^'\\]))*"), '"""': lambda: re.compile(r'((\\(.|\n))|([^"\\])|("(?!""))|\n)*'), "'''": lambda: re.compile(r"((\\(.|\n))|([^'\\])|('(?!''))|\n)*"), }, globals(), "RE_STRING_CONT", ) """Dictionary mapping starting quote sequences to regular expressions that match the contents of a string beginning with those quotes (not including the terminating quotes)""" @lazyobject def RE_COMPLETE_STRING(): ptrn = ( "^" + _RE_STRING_START + "(?P" + "|".join(_STRINGS) + ")" + ".*?(?P=quote)$" ) return re.compile(ptrn, re.DOTALL) def strip_simple_quotes(s): """Gets rid of single quotes, double quotes, single triple quotes, and single double quotes from a string, if present front and back of a string. Otherwiswe, does nothing. """ starts_single = s.startswith("'") starts_double = s.startswith('"') if not starts_single and not starts_double: return s elif starts_single: ends_single = s.endswith("'") if not ends_single: return s elif s.startswith("'''") and s.endswith("'''") and len(s) >= 6: return s[3:-3] elif len(s) >= 2: return s[1:-1] else: return s else: # starts double ends_double = s.endswith('"') if not ends_double: return s elif s.startswith('"""') and s.endswith('"""') and len(s) >= 6: return s[3:-3] elif len(s) >= 2: return s[1:-1] else: return s def check_for_partial_string(x): """Returns the starting index (inclusive), ending index (exclusive), and starting quote string of the most recent Python string found in the input. check_for_partial_string(x) -> (startix, endix, quote) Parameters ---------- x : str The string to be checked (representing a line of terminal input) Returns ------- startix : int (or None) The index where the most recent Python string found started (inclusive), or None if no strings exist in the input endix : int (or None) The index where the most recent Python string found ended (exclusive), or None if no strings exist in the input OR if the input ended in the middle of a Python string quote : str (or None) A string containing the quote used to start the string (e.g., b", ", '''), or None if no string was found. """ string_indices = [] starting_quote = [] current_index = 0 match = re.search(RE_BEGIN_STRING, x) while match is not None: # add the start in start = match.start() quote = match.group(0) lenquote = len(quote) current_index += start # store the starting index of the string, as well as the # characters in the starting quotes (e.g., ", ', """, r", etc) string_indices.append(current_index) starting_quote.append(quote) # determine the string that should terminate this string ender = re.sub(RE_STRING_START, "", quote) x = x[start + lenquote :] current_index += lenquote # figure out what is inside the string continuer = RE_STRING_CONT[ender] contents = re.match(continuer, x) inside = contents.group(0) leninside = len(inside) current_index += contents.start() + leninside + len(ender) # if we are not at the end of the input string, add the ending index of # the string to string_indices if contents.end() < len(x): string_indices.append(current_index) x = x[leninside + len(ender) :] # find the next match match = re.search(RE_BEGIN_STRING, x) numquotes = len(string_indices) if numquotes == 0: return (None, None, None) elif numquotes % 2: return (string_indices[-1], None, starting_quote[-1]) else: return (string_indices[-2], string_indices[-1], starting_quote[-1]) # regular expressions for matching environment variables # i.e $FOO, ${'FOO'} @lazyobject def POSIX_ENVVAR_REGEX(): pat = r"""\$({(?P['"])|)(?P\w+)((?P=quote)}|(?:\1\b))""" return re.compile(pat) def expandvars(path): """Expand shell variables of the forms $var, ${var} and %var%. Unknown variables are left unchanged.""" env = builtins.__xonsh__.env if isinstance(path, bytes): path = path.decode( encoding=env.get("XONSH_ENCODING"), errors=env.get("XONSH_ENCODING_ERRORS") ) elif isinstance(path, pathlib.Path): # get the path's string representation path = str(path) if "$" in path: for match in POSIX_ENVVAR_REGEX.finditer(path): name = match.group("envvar") if name in env: ensurer = env.get_ensurer(name) val = env[name] value = str(val) if ensurer.detype is None else ensurer.detype(val) value = str(val) if value is None else value path = POSIX_ENVVAR_REGEX.sub(value, path, count=1) return path # # File handling tools # def backup_file(fname): """Moves an existing file to a new name that has the current time right before the extension. """ # lazy imports import shutil from datetime import datetime base, ext = os.path.splitext(fname) timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M-%S-%f") newfname = "%s.%s%s" % (base, timestamp, ext) shutil.move(fname, newfname) def normabspath(p): """Returns as normalized absolute path, namely, normcase(abspath(p))""" return os.path.normcase(os.path.abspath(p)) def expanduser_abs_path(inp): """ Provides user expanded absolute path """ return os.path.abspath(expanduser(inp)) WINDOWS_DRIVE_MATCHER = LazyObject( lambda: re.compile(r"^\w:"), globals(), "WINDOWS_DRIVE_MATCHER" ) def expand_case_matching(s): """Expands a string to a case insensitive globable string.""" t = [] openers = {"[", "{"} closers = {"]", "}"} nesting = 0 drive_part = WINDOWS_DRIVE_MATCHER.match(s) if ON_WINDOWS else None if drive_part: drive_part = drive_part.group(0) t.append(drive_part) s = s[len(drive_part) :] for c in s: if c in openers: nesting += 1 elif c in closers: nesting -= 1 elif nesting > 0: pass elif c.isalpha(): folded = c.casefold() if len(folded) == 1: c = "[{0}{1}]".format(c.upper(), c.lower()) else: newc = ["[{0}{1}]?".format(f.upper(), f.lower()) for f in folded[:-1]] newc = "".join(newc) newc += "[{0}{1}{2}]".format(folded[-1].upper(), folded[-1].lower(), c) c = newc t.append(c) return "".join(t) def globpath( s, ignore_case=False, return_empty=False, sort_result=None, include_dotfiles=None ): """Simple wrapper around glob that also expands home and env vars.""" o, s = _iglobpath( s, ignore_case=ignore_case, sort_result=sort_result, include_dotfiles=include_dotfiles, ) o = list(o) no_match = [] if return_empty else [s] return o if len(o) != 0 else no_match def _dotglobstr(s): modified = False dotted_s = s if "/*" in dotted_s: dotted_s = dotted_s.replace("/*", "/.*") dotted_s = dotted_s.replace("/.**/.*", "/**/.*") modified = True if dotted_s.startswith("*") and not dotted_s.startswith("**"): dotted_s = "." + dotted_s modified = True return dotted_s, modified def _iglobpath(s, ignore_case=False, sort_result=None, include_dotfiles=None): s = builtins.__xonsh__.expand_path(s) if sort_result is None: sort_result = builtins.__xonsh__.env.get("GLOB_SORTED") if include_dotfiles is None: include_dotfiles = builtins.__xonsh__.env.get("DOTGLOB") if ignore_case: s = expand_case_matching(s) if sys.version_info > (3, 5): if "**" in s and "**/*" not in s: s = s.replace("**", "**/*") if include_dotfiles: dotted_s, dotmodified = _dotglobstr(s) # `recursive` is only a 3.5+ kwarg. if sort_result: paths = glob.glob(s, recursive=True) if include_dotfiles and dotmodified: paths.extend(glob.iglob(dotted_s, recursive=True)) paths.sort() paths = iter(paths) else: paths = glob.iglob(s, recursive=True) if include_dotfiles and dotmodified: paths = itertools.chain(glob.iglob(dotted_s, recursive=True), paths) return paths, s else: if include_dotfiles: dotted_s, dotmodified = _dotglobstr(s) if sort_result: paths = glob.glob(s) if include_dotfiles and dotmodified: paths.extend(glob.iglob(dotted_s)) paths.sort() paths = iter(paths) else: paths = glob.iglob(s) if include_dotfiles and dotmodified: paths = itertools.chain(glob.iglob(dotted_s), paths) return paths, s def iglobpath(s, ignore_case=False, sort_result=None, include_dotfiles=None): """Simple wrapper around iglob that also expands home and env vars.""" try: return _iglobpath( s, ignore_case=ignore_case, sort_result=sort_result, include_dotfiles=include_dotfiles, )[0] except IndexError: # something went wrong in the actual iglob() call return iter(()) def ensure_timestamp(t, datetime_format=None): if isinstance(t, (int, float)): return t try: return float(t) except (ValueError, TypeError): pass if datetime_format is None: datetime_format = builtins.__xonsh__.env["XONSH_DATETIME_FORMAT"] if isinstance(t, datetime.datetime): t = t.timestamp() else: t = datetime.datetime.strptime(t, datetime_format).timestamp() return t def format_datetime(dt): """Format datetime object to string base on $XONSH_DATETIME_FORMAT Env.""" format_ = builtins.__xonsh__.env["XONSH_DATETIME_FORMAT"] return dt.strftime(format_) def columnize(elems, width=80, newline="\n"): """Takes an iterable of strings and returns a list of lines with the elements placed in columns. Each line will be at most *width* columns. The newline character will be appended to the end of each line. """ sizes = [len(e) + 1 for e in elems] total = sum(sizes) nelem = len(elems) if total - 1 <= width: ncols = len(sizes) nrows = 1 columns = [sizes] last_longest_row = total enter_loop = False else: ncols = 1 nrows = len(sizes) columns = [sizes] last_longest_row = max(sizes) enter_loop = True while enter_loop: longest_row = sum(map(max, columns)) if longest_row - 1 <= width: # we might be able to fit another column. ncols += 1 nrows = nelem // ncols columns = [sizes[i * nrows : (i + 1) * nrows] for i in range(ncols)] last_longest_row = longest_row else: # we can't fit another column ncols -= 1 nrows = nelem // ncols break pad = (width - last_longest_row + ncols) // ncols pad = pad if pad > 1 else 1 data = [elems[i * nrows : (i + 1) * nrows] for i in range(ncols)] colwidths = [max(map(len, d)) + pad for d in data] colwidths[-1] -= pad row_t = "".join(["{{row[{i}]: <{{w[{i}]}}}}".format(i=i) for i in range(ncols)]) row_t += newline lines = [ row_t.format(row=row, w=colwidths) for row in itertools.zip_longest(*data, fillvalue="") ] return lines ALIAS_KWARG_NAMES = frozenset(["args", "stdin", "stdout", "stderr", "spec", "stack"]) def unthreadable(f): """Decorator that specifies that a callable alias should be run only on the main thread process. This is often needed for debuggers and profilers. """ f.__xonsh_threadable__ = False return f def uncapturable(f): """Decorator that specifies that a callable alias should not be run with any capturing. This is often needed if the alias call interactive subprocess, like pagers and text editors. """ f.__xonsh_capturable__ = False return f def carriage_return(): """Writes a carriage return to stdout, and nothing else.""" print("\r", flush=True, end="") def deprecated(deprecated_in=None, removed_in=None): """Parametrized decorator that deprecates a function in a graceful manner. Updates the decorated function's docstring to mention the version that deprecation occurred in and the version it will be removed in if both of these values are passed. When removed_in is not a release equal to or less than the current release, call ``warnings.warn`` with details, while raising ``DeprecationWarning``. When removed_in is a release equal to or less than the current release, raise an ``AssertionError``. Parameters ---------- deprecated_in : str The version number that deprecated this function. removed_in : str The version number that this function will be removed in. """ message_suffix = _deprecated_message_suffix(deprecated_in, removed_in) if not message_suffix: message_suffix = "" def decorated(func): warning_message = "{} has been deprecated".format(func.__name__) warning_message += message_suffix @functools.wraps(func) def wrapped(*args, **kwargs): _deprecated_error_on_expiration(func.__name__, removed_in) func(*args, **kwargs) warnings.warn(warning_message, DeprecationWarning) wrapped.__doc__ = ( "{}\n\n{}".format(wrapped.__doc__, warning_message) if wrapped.__doc__ else warning_message ) return wrapped return decorated def _deprecated_message_suffix(deprecated_in, removed_in): if deprecated_in and removed_in: message_suffix = " in version {} and will be removed in version {}".format( deprecated_in, removed_in ) elif deprecated_in and not removed_in: message_suffix = " in version {}".format(deprecated_in) elif not deprecated_in and removed_in: message_suffix = " and will be removed in version {}".format(removed_in) else: message_suffix = None return message_suffix def _deprecated_error_on_expiration(name, removed_in): if not removed_in: return elif LooseVersion(__version__) >= LooseVersion(removed_in): raise AssertionError( "{} has passed its version {} expiry date!".format(name, removed_in) ) ================================================ FILE: xonsh/tracer.py ================================================ """Implements a xonsh tracer.""" import os import re import sys import inspect import argparse import linecache import importlib import functools from xonsh.lazyasd import LazyObject from xonsh.platform import HAS_PYGMENTS from xonsh.tools import DefaultNotGiven, print_color, normabspath, to_bool from xonsh.inspectors import find_file, getouterframes from xonsh.lazyimps import pygments, pyghooks from xonsh.proc import STDOUT_CAPTURE_KINDS import xonsh.prompt.cwd as prompt terminal = LazyObject( lambda: importlib.import_module("pygments.formatters.terminal"), globals(), "terminal", ) class TracerType(object): """Represents a xonsh tracer object, which keeps track of all tracing state. This is a singleton. """ _inst = None valid_events = frozenset(["line", "call"]) def __new__(cls, *args, **kwargs): if cls._inst is None: cls._inst = super(TracerType, cls).__new__(cls, *args, **kwargs) return cls._inst def __init__(self): self.prev_tracer = DefaultNotGiven self.files = set() self.usecolor = True self.lexer = pyghooks.XonshLexer() self.formatter = terminal.TerminalFormatter() self._last = ("", -1) # filename, lineno tuple def __del__(self): for f in set(self.files): self.stop(f) def color_output(self, usecolor): """Specify whether or not the tracer output should be colored.""" # we have to use a function to set usecolor because of the way that # lazyasd works. Namely, it cannot dispatch setattr to the target # object without being unable to access its own __dict__. This makes # setting an attr look like getting a function. self.usecolor = usecolor def start(self, filename): """Starts tracing a file.""" files = self.files if len(files) == 0: self.prev_tracer = sys.gettrace() files.add(normabspath(filename)) sys.settrace(self.trace) curr = inspect.currentframe() for frame, fname, *_ in getouterframes(curr, context=0): if normabspath(fname) in files: frame.f_trace = self.trace def stop(self, filename): """Stops tracing a file.""" filename = normabspath(filename) self.files.discard(filename) if len(self.files) == 0: sys.settrace(self.prev_tracer) curr = inspect.currentframe() for frame, fname, *_ in getouterframes(curr, context=0): if normabspath(fname) == filename: frame.f_trace = self.prev_tracer self.prev_tracer = DefaultNotGiven def trace(self, frame, event, arg): """Implements a line tracing function.""" if event not in self.valid_events: return self.trace fname = find_file(frame) if fname in self.files: lineno = frame.f_lineno curr = (fname, lineno) if curr != self._last: line = linecache.getline(fname, lineno).rstrip() s = tracer_format_line( fname, lineno, line, color=self.usecolor, lexer=self.lexer, formatter=self.formatter, ) print_color(s) self._last = curr return self.trace tracer = LazyObject(TracerType, globals(), "tracer") COLORLESS_LINE = "{fname}:{lineno}:{line}" COLOR_LINE = "{{PURPLE}}{fname}{{BLUE}}:" "{{GREEN}}{lineno}{{BLUE}}:" "{{NO_COLOR}}" def tracer_format_line(fname, lineno, line, color=True, lexer=None, formatter=None): """Formats a trace line suitable for printing.""" fname = min(fname, prompt._replace_home(fname), os.path.relpath(fname), key=len) if not color: return COLORLESS_LINE.format(fname=fname, lineno=lineno, line=line) cline = COLOR_LINE.format(fname=fname, lineno=lineno) if not HAS_PYGMENTS: return cline + line # OK, so we have pygments tokens = pyghooks.partial_color_tokenize(cline) lexer = lexer or pyghooks.XonshLexer() tokens += pygments.lex(line, lexer=lexer) if tokens[-1][1] == "\n": del tokens[-1] elif tokens[-1][1].endswith("\n"): tokens[-1] = (tokens[-1][0], tokens[-1][1].rstrip()) return tokens # # Command line interface # def _find_caller(args): """Somewhat hacky method of finding the __file__ based on the line executed.""" re_line = re.compile(r"[^;\s|&<>]+\s+" + r"\s+".join(args)) curr = inspect.currentframe() for _, fname, lineno, _, lines, _ in getouterframes(curr, context=1)[3:]: if lines is not None and re_line.search(lines[0]) is not None: return fname elif ( lineno == 1 and re_line.search(linecache.getline(fname, lineno)) is not None ): # There is a bug in CPython such that getouterframes(curr, context=1) # will actually return the 2nd line in the code_context field, even though # line number is itself correct. We manually fix that in this branch. return fname else: msg = ( "xonsh: warning: __file__ name could not be found. You may be " "trying to trace interactively. Please pass in the file names " "you want to trace explicitly." ) print(msg, file=sys.stderr) def _on(ns, args): """Turns on tracing for files.""" for f in ns.files: if f == "__file__": f = _find_caller(args) if f is None: continue tracer.start(f) def _off(ns, args): """Turns off tracing for files.""" for f in ns.files: if f == "__file__": f = _find_caller(args) if f is None: continue tracer.stop(f) def _color(ns, args): """Manages color action for tracer CLI.""" tracer.color_output(ns.toggle) @functools.lru_cache(1) def _tracer_create_parser(): """Creates tracer argument parser""" p = argparse.ArgumentParser( prog="trace", description="tool for tracing xonsh code as it runs." ) subp = p.add_subparsers(title="action", dest="action") onp = subp.add_parser( "on", aliases=["start", "add"], help="begins tracing selected files." ) onp.add_argument( "files", nargs="*", default=["__file__"], help=( 'file paths to watch, use "__file__" (default) to select ' "the current file." ), ) off = subp.add_parser( "off", aliases=["stop", "del", "rm"], help="removes selected files fom tracing." ) off.add_argument( "files", nargs="*", default=["__file__"], help=( 'file paths to stop watching, use "__file__" (default) to ' "select the current file." ), ) col = subp.add_parser("color", help="output color management for tracer.") col.add_argument( "toggle", type=to_bool, help="true/false, y/n, etc. to toggle color usage." ) return p _TRACER_MAIN_ACTIONS = { "on": _on, "add": _on, "start": _on, "rm": _off, "off": _off, "del": _off, "stop": _off, "color": _color, } def tracermain(args=None, stdin=None, stdout=None, stderr=None, spec=None): """Main function for tracer command-line interface.""" parser = _tracer_create_parser() ns = parser.parse_args(args) usecolor = (spec.captured not in STDOUT_CAPTURE_KINDS) and sys.stdout.isatty() tracer.color_output(usecolor) return _TRACER_MAIN_ACTIONS[ns.action](ns, args) ================================================ FILE: xonsh/winutils.py ================================================ """ This file is based on the code from https://github.com/JustAMan/pyWinClobber/blob/master/win32elevate.py Copyright (c) 2013 by JustAMan at GitHub Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ import os import ctypes import subprocess from ctypes import c_ulong, c_char_p, c_int, c_void_p, POINTER, byref from ctypes.wintypes import ( HANDLE, BOOL, DWORD, HWND, HINSTANCE, HKEY, LPDWORD, SHORT, LPCWSTR, WORD, SMALL_RECT, LPCSTR, ) from xonsh.lazyasd import lazyobject from xonsh import lazyimps # we aren't amalgamated in this module. from xonsh import platform __all__ = ("sudo",) @lazyobject def CloseHandle(): ch = ctypes.windll.kernel32.CloseHandle ch.argtypes = (HANDLE,) ch.restype = BOOL return ch @lazyobject def GetActiveWindow(): gaw = ctypes.windll.user32.GetActiveWindow gaw.argtypes = () gaw.restype = HANDLE return gaw TOKEN_READ = 0x20008 class ShellExecuteInfo(ctypes.Structure): _fields_ = [ ("cbSize", DWORD), ("fMask", c_ulong), ("hwnd", HWND), ("lpVerb", c_char_p), ("lpFile", c_char_p), ("lpParameters", c_char_p), ("lpDirectory", c_char_p), ("nShow", c_int), ("hInstApp", HINSTANCE), ("lpIDList", c_void_p), ("lpClass", c_char_p), ("hKeyClass", HKEY), ("dwHotKey", DWORD), ("hIcon", HANDLE), ("hProcess", HANDLE), ] def __init__(self, **kw): ctypes.Structure.__init__(self) self.cbSize = ctypes.sizeof(self) for field_name, field_value in kw.items(): setattr(self, field_name, field_value) @lazyobject def ShellExecuteEx(): see = ctypes.windll.Shell32.ShellExecuteExA PShellExecuteInfo = ctypes.POINTER(ShellExecuteInfo) see.argtypes = (PShellExecuteInfo,) see.restype = BOOL return see @lazyobject def WaitForSingleObject(): wfso = ctypes.windll.kernel32.WaitForSingleObject wfso.argtypes = (HANDLE, DWORD) wfso.restype = DWORD return wfso # SW_HIDE = 0 SW_SHOW = 5 SEE_MASK_NOCLOSEPROCESS = 0x00000040 SEE_MASK_NO_CONSOLE = 0x00008000 INFINITE = -1 def wait_and_close_handle(process_handle): """ Waits till spawned process finishes and closes the handle for it Parameters ---------- process_handle : HANDLE The Windows handle for the process """ WaitForSingleObject(process_handle, INFINITE) CloseHandle(process_handle) def sudo(executable, args=None): """ This will re-run current Python script requesting to elevate administrative rights. Parameters ---------- param executable : str The path/name of the executable args : list of str The arguments to be passed to the executable """ if not args: args = [] execute_info = ShellExecuteInfo( fMask=SEE_MASK_NOCLOSEPROCESS | SEE_MASK_NO_CONSOLE, hwnd=GetActiveWindow(), lpVerb=b"runas", lpFile=executable.encode("utf-8"), lpParameters=subprocess.list2cmdline(args).encode("utf-8"), lpDirectory=None, nShow=SW_SHOW, ) if not ShellExecuteEx(byref(execute_info)): raise ctypes.WinError() wait_and_close_handle(execute_info.hProcess) # # The following has been refactored from # http://stackoverflow.com/a/37505496/2312428 # # input flags ENABLE_PROCESSED_INPUT = 0x0001 ENABLE_LINE_INPUT = 0x0002 ENABLE_ECHO_INPUT = 0x0004 ENABLE_WINDOW_INPUT = 0x0008 ENABLE_MOUSE_INPUT = 0x0010 ENABLE_INSERT_MODE = 0x0020 ENABLE_QUICK_EDIT_MODE = 0x0040 # output flags ENABLE_PROCESSED_OUTPUT = 0x0001 ENABLE_WRAP_AT_EOL_OUTPUT = 0x0002 ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x0004 # VT100 (Win 10) def check_zero(result, func, args): if not result: err = ctypes.get_last_error() if err: raise ctypes.WinError(err) return args @lazyobject def GetStdHandle(): return lazyimps._winapi.GetStdHandle @lazyobject def STDHANDLES(): """Tuple of the Windows handles for (stdin, stdout, stderr).""" hs = [ lazyimps._winapi.STD_INPUT_HANDLE, lazyimps._winapi.STD_OUTPUT_HANDLE, lazyimps._winapi.STD_ERROR_HANDLE, ] hcons = [] for h in hs: hcon = GetStdHandle(int(h)) hcons.append(hcon) return tuple(hcons) @lazyobject def GetConsoleMode(): gcm = ctypes.windll.kernel32.GetConsoleMode gcm.errcheck = check_zero gcm.argtypes = (HANDLE, LPDWORD) # _In_ hConsoleHandle # _Out_ lpMode return gcm def get_console_mode(fd=1): """Get the mode of the active console input, output, or error buffer. Note that if the process isn't attached to a console, this function raises an EBADF IOError. Parameters ---------- fd : int Standard buffer file descriptor, 0 for stdin, 1 for stdout (default), and 2 for stderr """ mode = DWORD() hcon = STDHANDLES[fd] GetConsoleMode(hcon, byref(mode)) return mode.value @lazyobject def SetConsoleMode(): scm = ctypes.windll.kernel32.SetConsoleMode scm.errcheck = check_zero scm.argtypes = (HANDLE, DWORD) # _In_ hConsoleHandle # _Out_ lpMode return scm def set_console_mode(mode, fd=1): """Set the mode of the active console input, output, or error buffer. Note that if the process isn't attached to a console, this function raises an EBADF IOError. Parameters ---------- mode : int Mode flags to set on the handle. fd : int, optional Standard buffer file descriptor, 0 for stdin, 1 for stdout (default), and 2 for stderr """ hcon = STDHANDLES[fd] SetConsoleMode(hcon, mode) def enable_virtual_terminal_processing(): """Enables virtual terminal processing on Windows. This includes ANSI escape sequence interpretation. See http://stackoverflow.com/a/36760881/2312428 """ SetConsoleMode(GetStdHandle(-11), 7) @lazyobject def COORD(): if platform.has_prompt_toolkit(): # turns out that PTK has a separate ctype wrapper # for this struct and also wraps similar function calls # we need to use the same struct to prevent clashes. import prompt_toolkit.win32_types return prompt_toolkit.win32_types.COORD class _COORD(ctypes.Structure): """Struct from the winapi, representing coordinates in the console. Attributes ---------- X : int Column position Y : int Row position """ _fields_ = [("X", SHORT), ("Y", SHORT)] return _COORD @lazyobject def ReadConsoleOutputCharacterA(): rcoc = ctypes.windll.kernel32.ReadConsoleOutputCharacterA rcoc.errcheck = check_zero rcoc.argtypes = ( HANDLE, # _In_ hConsoleOutput LPCSTR, # _Out_ LPTSTR lpMode DWORD, # _In_ nLength COORD, # _In_ dwReadCoord, LPDWORD, ) # _Out_ lpNumberOfCharsRead rcoc.restype = BOOL return rcoc @lazyobject def ReadConsoleOutputCharacterW(): rcoc = ctypes.windll.kernel32.ReadConsoleOutputCharacterW rcoc.errcheck = check_zero rcoc.argtypes = ( HANDLE, # _In_ hConsoleOutput LPCWSTR, # _Out_ LPTSTR lpMode DWORD, # _In_ nLength COORD, # _In_ dwReadCoord, LPDWORD, ) # _Out_ lpNumberOfCharsRead rcoc.restype = BOOL return rcoc def read_console_output_character(x=0, y=0, fd=1, buf=None, bufsize=1024, raw=False): """Reads characters from the console buffer. Parameters ---------- x : int, optional Starting column. y : int, optional Starting row. fd : int, optional Standard buffer file descriptor, 0 for stdin, 1 for stdout (default), and 2 for stderr. buf : ctypes.c_wchar_p if raw else ctypes.c_wchar_p, optional An existing buffer to (re-)use. bufsize : int, optional The maximum read size. raw : bool, optional Whether to read in and return as bytes (True) or as a unicode string (False, default). Returns ------- value : str Result of what was read, may be shorter than bufsize. """ hcon = STDHANDLES[fd] if buf is None: if raw: buf = ctypes.c_char_p(b" " * bufsize) else: buf = ctypes.c_wchar_p(" " * bufsize) coord = COORD(x, y) n = DWORD() if raw: ReadConsoleOutputCharacterA(hcon, buf, bufsize, coord, byref(n)) else: ReadConsoleOutputCharacterW(hcon, buf, bufsize, coord, byref(n)) return buf.value[: n.value] def pread_console(fd, buffersize, offset, buf=None): """This is a console-based implementation of os.pread() for windows. that uses read_console_output_character(). """ cols, rows = os.get_terminal_size(fd=fd) x = offset % cols y = offset // cols return read_console_output_character( x=x, y=y, fd=fd, buf=buf, bufsize=buffersize, raw=True ) # # The following piece has been forked from colorama.win32 # Copyright Jonathan Hartley 2013. BSD 3-Clause license, see LICENSE file. # @lazyobject def CONSOLE_SCREEN_BUFFER_INFO(): if platform.has_prompt_toolkit(): # turns out that PTK has a separate ctype wrapper # for this struct and also wraps kernel32.GetConsoleScreenBufferInfo # we need to use the same struct to prevent clashes. import prompt_toolkit.win32_types return prompt_toolkit.win32_types.CONSOLE_SCREEN_BUFFER_INFO # Otherwise we should wrap it ourselves COORD() # force COORD to load class _CONSOLE_SCREEN_BUFFER_INFO(ctypes.Structure): """Struct from in wincon.h. See Windows API docs for more details. Attributes ---------- dwSize : COORD Size of dwCursorPosition : COORD Current cursor location. wAttributes : WORD Flags for screen buffer. srWindow : SMALL_RECT Actual size of screen dwMaximumWindowSize : COORD Maximum window scrollback size. """ _fields_ = [ ("dwSize", COORD), ("dwCursorPosition", COORD), ("wAttributes", WORD), ("srWindow", SMALL_RECT), ("dwMaximumWindowSize", COORD), ] return _CONSOLE_SCREEN_BUFFER_INFO @lazyobject def GetConsoleScreenBufferInfo(): """Returns the windows version of the get screen buffer.""" gcsbi = ctypes.windll.kernel32.GetConsoleScreenBufferInfo gcsbi.errcheck = check_zero gcsbi.argtypes = (HANDLE, POINTER(CONSOLE_SCREEN_BUFFER_INFO)) gcsbi.restype = BOOL return gcsbi def get_console_screen_buffer_info(fd=1): """Returns an screen buffer info object for the relevant stdbuf. Parameters ---------- fd : int, optional Standard buffer file descriptor, 0 for stdin, 1 for stdout (default), and 2 for stderr. Returns ------- csbi : CONSOLE_SCREEN_BUFFER_INFO Information about the console screen buffer. """ hcon = STDHANDLES[fd] csbi = CONSOLE_SCREEN_BUFFER_INFO() GetConsoleScreenBufferInfo(hcon, byref(csbi)) return csbi # # end colorama forked section # def get_cursor_position(fd=1): """Gets the current cursor position as an (x, y) tuple.""" csbi = get_console_screen_buffer_info(fd=fd) coord = csbi.dwCursorPosition return (coord.X, coord.Y) def get_cursor_offset(fd=1): """Gets the current cursor position as a total offset value.""" csbi = get_console_screen_buffer_info(fd=fd) pos = csbi.dwCursorPosition size = csbi.dwSize return (pos.Y * size.X) + pos.X def get_position_size(fd=1): """Gets the current cursor position and screen size tuple: (x, y, columns, lines). """ info = get_console_screen_buffer_info(fd) return ( info.dwCursorPosition.X, info.dwCursorPosition.Y, info.dwSize.X, info.dwSize.Y, ) @lazyobject def SetConsoleScreenBufferSize(): """Set screen buffer dimensions.""" scsbs = ctypes.windll.kernel32.SetConsoleScreenBufferSize scsbs.errcheck = check_zero scsbs.argtypes = (HANDLE, COORD) # _In_ HANDLE hConsoleOutput # _In_ COORD dwSize scsbs.restype = BOOL return scsbs def set_console_screen_buffer_size(x, y, fd=1): """Sets the console size for a standard buffer. Parameters ---------- x : int Number of columns. y : int Number of rows. fd : int, optional Standard buffer file descriptor, 0 for stdin, 1 for stdout (default), and 2 for stderr. """ coord = COORD() coord.X = x coord.Y = y hcon = STDHANDLES[fd] rtn = SetConsoleScreenBufferSize(hcon, coord) return rtn @lazyobject def SetConsoleCursorPosition(): """Set cursor position in console.""" sccp = ctypes.windll.kernel32.SetConsoleCursorPosition sccp.errcheck = check_zero sccp.argtypes = ( HANDLE, # _In_ HANDLE hConsoleOutput COORD, # _In_ COORD dwCursorPosition ) sccp.restype = BOOL return sccp def set_console_cursor_position(x, y, fd=1): """Sets the console cursor position for a standard buffer. Parameters ---------- x : int Number of columns. y : int Number of rows. fd : int, optional Standard buffer file descriptor, 0 for stdin, 1 for stdout (default), and 2 for stderr. """ coord = COORD() coord.X = x coord.Y = y hcon = STDHANDLES[fd] rtn = SetConsoleCursorPosition(hcon, coord) return rtn ================================================ FILE: xonsh/wizard.py ================================================ """Tools for creating command-line and web-based wizards from a tree of nodes. """ import os import re import ast import json import pprint import fnmatch import builtins import textwrap import collections.abc as cabc from xonsh.tools import to_bool, to_bool_or_break, backup_file, print_color from xonsh.jsonutils import serialize_xonsh_json # # Nodes themselves # class Node(object): """Base type of all nodes.""" attrs = () def __str__(self): return PrettyFormatter(self).visit() def __repr__(self): return str(self).replace("\n", "") class Wizard(Node): """Top-level node in the tree.""" attrs = ("children", "path") def __init__(self, children, path=None): self.children = children self.path = path class Pass(Node): """Simple do-nothing node""" class Message(Node): """Contains a simple message to report to the user.""" attrs = "message" def __init__(self, message): self.message = message class Question(Node): """Asks a question and then chooses the next node based on the response. """ attrs = ("question", "responses", "converter", "path") def __init__(self, question, responses, converter=None, path=None): """ Parameters ---------- question : str The question itself. responses : dict with str keys and Node values Mapping from user-input responses to nodes. converter : callable, optional Converts the string the user typed into another object that serves as a key to the responses dict. path : str or sequence of str, optional A path within the storage object. """ self.question = question self.responses = responses self.converter = converter self.path = path class Input(Node): """Gets input from the user.""" attrs = ("prompt", "converter", "show_conversion", "confirm", "path") def __init__( self, prompt=">>> ", converter=None, show_conversion=False, confirm=False, retry=False, path=None, ): """ Parameters ---------- prompt : str, optional Prompt string prior to input converter : callable, optional Converts the string the user typed into another object prior to storage. show_conversion : bool, optional Flag for whether or not to show the results of the conversion function if the conversion function was meaningfully executed. Default False. confirm : bool, optional Whether the input should be confirmed until true or broken, default False. retry : bool, optional In the event that the conversion operation fails, should users be re-prompted until they provide valid input. Default False. path : str or sequence of str, optional A path within the storage object. """ self.prompt = prompt self.converter = converter self.show_conversion = show_conversion self.confirm = confirm self.retry = retry self.path = path class While(Node): """Computes a body while a condition function evaluates to true. The condition function has the form ``cond(visitor=None, node=None)`` and must return an object that responds to the Python magic method ``__bool__``. The beg attribute specifies the number to start the loop iteration at. """ attrs = ("cond", "body", "idxname", "beg", "path") def __init__(self, cond, body, idxname="idx", beg=0, path=None): """ Parameters ---------- cond : callable Function that determines if the next loop iteration should be executed. body : sequence of nodes A list of node to execute on each iteration. The condition function has the form ``cond(visitor=None, node=None)`` and must return an object that responds to the Python magic method ``__bool__``. idxname : str, optional The variable name for the index. beg : int, optional The first index value when evaluating path format strings. path : str or sequence of str, optional A path within the storage object. """ self.cond = cond self.body = body self.idxname = idxname self.beg = beg self.path = path # # Helper nodes # class YesNo(Question): """Represents a simple yes/no question.""" def __init__(self, question, yes, no, path=None): """ Parameters ---------- question : str The question itself. yes : Node Node to execute if the response is True. no : Node Node to execute if the response is False. path : str or sequence of str, optional A path within the storage object. """ responses = {True: yes, False: no} super().__init__(question, responses, converter=to_bool, path=path) class TrueFalse(Input): """Input node the returns a True or False value.""" def __init__(self, prompt="yes or no [default: no]? ", path=None): super().__init__( prompt=prompt, converter=to_bool, show_conversion=False, confirm=False, path=path, ) class TrueFalseBreak(Input): """Input node the returns a True, False, or 'break' value.""" def __init__(self, prompt="yes, no, or break [default: no]? ", path=None): super().__init__( prompt=prompt, converter=to_bool_or_break, show_conversion=False, confirm=False, path=path, ) class StoreNonEmpty(Input): """Stores the user input only if the input was not an empty string. This works by wrapping the converter function. """ def __init__( self, prompt=">>> ", converter=None, show_conversion=False, confirm=False, retry=False, path=None, store_raw=False, ): def nonempty_converter(x): """Converts non-empty values and converts empty inputs to Unstorable. """ if len(x) == 0: x = Unstorable elif converter is None: pass elif store_raw: converter(x) # make sure str is valid, even if storing raw else: x = converter(x) return x super().__init__( prompt=prompt, converter=nonempty_converter, show_conversion=show_conversion, confirm=confirm, path=path, retry=retry, ) class StateFile(Input): """Node for representing the state as a file under a default or user given file name. This node type is likely not useful on its own. """ attrs = ("default_file", "check", "ask_filename") def __init__(self, default_file=None, check=True, ask_filename=True): """ Parameters ---------- default_file : str, optional The default filename to save the file as. check : bool, optional Whether to print the current state and ask if it should be saved/loaded prior to asking for the file name and saving the file, default=True. ask_filename : bool, optional Whether to ask for the filename (if ``False``, always use the default filename) """ self._df = None super().__init__(prompt="filename: ", converter=None, confirm=False, path=None) self.ask_filename = ask_filename self.default_file = default_file self.check = check @property def default_file(self): return self._df @default_file.setter def default_file(self, val): self._df = val if val is None: self.prompt = "filename: " else: self.prompt = "filename [default={0!r}]: ".format(val) class SaveJSON(StateFile): """Node for saving the state as a JSON file under a default or user given file name. """ class LoadJSON(StateFile): """Node for loading the state as a JSON file under a default or user given file name. """ class FileInserter(StateFile): """Node for inserting the state into a file in between a prefix and suffix. The state is converted according to some dumper rules. """ attrs = ("prefix", "suffix", "dump_rules", "default_file", "check", "ask_filename") def __init__( self, prefix, suffix, dump_rules, default_file=None, check=True, ask_filename=True, ): """ Parameters ---------- prefix : str Starting unique string in file to find and begin the insertion at, e.g. '# XONSH WIZARD START\n' suffix : str Ending unique string to find in the file and end the replacement at, e.g. '\n# XONSH WIZARD END' dump_rules : dict of strs to functions This is a dictionary that maps the path-like match strings to functions that take the flat path and the value as arguments and convert the state value at a path to a string. The keys here may use wildcards (as seen in the standard library fnmatch module). For example:: dump_rules = { '/path/to/exact': lambda path, x: str(x), '/otherpath/*': lambda path, x: x, '*ending': lambda path x: repr(x), '/': None, } If a wildcard is not used in a path, then that rule will be used used on an exact match. If wildcards are used, the deepest and longest match is used. If None is given instead of a the function, it means to skip generating that key. default_file : str, optional The default filename to save the file as. check : bool, optional Whether to print the current state and ask if it should be saved/loaded prior to asking for the file name and saving the file, default=True. ask_filename : bool, optional Whether to ask for the filename (if ``False``, always use the default filename) """ self._dr = None super().__init__( default_file=default_file, check=check, ask_filename=ask_filename ) self.prefix = prefix self.suffix = suffix self.dump_rules = self.string_rules = dump_rules @property def dump_rules(self): return self._dr @dump_rules.setter def dump_rules(self, value): dr = {} for key, func in value.items(): key_trans = fnmatch.translate(key) r = re.compile(key_trans) dr[r] = func self._dr = dr @staticmethod def _find_rule_key(x): """Key function for sorting regular expression rules""" return (x[0], len(x[1].pattern)) def find_rule(self, path): """For a path, find the key and conversion function that should be used to dump a value. """ if path in self.string_rules: return path, self.string_rules[path] len_funcs = [] for rule, func in self.dump_rules.items(): m = rule.match(path) if m is None: continue i, j = m.span() len_funcs.append((j - i, rule, func)) if len(len_funcs) == 0: # No dump rule function for path return path, None len_funcs.sort(reverse=True, key=self._find_rule_key) _, rule, func = len_funcs[0] return rule, func def dumps(self, flat): """Dumps a flat mapping of (string path keys, values) pairs and returns a formatted string. """ lines = [self.prefix] for path, value in sorted(flat.items()): rule, func = self.find_rule(path) if func is None: continue line = func(path, value) lines.append(line) lines.append(self.suffix) new = "\n".join(lines) + "\n" return new def create_truefalse_cond(prompt="yes or no [default: no]? ", path=None): """This creates a basic condition function for use with nodes like While or other conditions. The condition function creates and visits a TrueFalse node and returns the result. This TrueFalse node takes the prompt and path that is passed in here. """ def truefalse_cond(visitor, node=None): """Prompts the user for a true/false condition.""" tf = TrueFalse(prompt=prompt, path=path) rtn = visitor.visit(tf) return rtn return truefalse_cond # # Tools for trees of nodes. # def _lowername(cls): return cls.__name__.lower() class Visitor(object): """Super-class for all classes that should walk over a tree of nodes. This implements the visit() method. """ def __init__(self, tree=None): self.tree = tree def visit(self, node=None): """Walks over a node. If no node is provided, the tree is used.""" if node is None: node = self.tree if node is None: raise RuntimeError("no node or tree given!") for clsname in map(_lowername, type.mro(node.__class__)): meth = getattr(self, "visit_" + clsname, None) if callable(meth): rtn = meth(node) break else: msg = "could not find valid visitor method for {0} on {1}" nodename = node.__class__.__name__ selfname = self.__class__.__name__ raise AttributeError(msg.format(nodename, selfname)) return rtn class PrettyFormatter(Visitor): """Formats a tree of nodes into a pretty string""" def __init__(self, tree=None, indent=" "): super().__init__(tree=tree) self.level = 0 self.indent = indent def visit_node(self, node): s = node.__class__.__name__ + "(" if len(node.attrs) == 0: return s + ")" s += "\n" self.level += 1 t = [] for aname in node.attrs: a = getattr(node, aname) t.append(self.visit(a) if isinstance(a, Node) else pprint.pformat(a)) t = ["{0}={1}".format(n, x) for n, x in zip(node.attrs, t)] s += textwrap.indent(",\n".join(t), self.indent) self.level -= 1 s += "\n)" return s def visit_wizard(self, node): s = "Wizard(children=[" if len(node.children) == 0: if node.path is None: return s + "])" else: return s + "], path={0!r})".format(node.path) s += "\n" self.level += 1 s += textwrap.indent(",\n".join(map(self.visit, node.children)), self.indent) self.level -= 1 if node.path is None: s += "\n])" else: s += "{0}],\n{0}path={1!r}\n)".format(self.indent, node.path) return s def visit_message(self, node): return "Message({0!r})".format(node.message) def visit_question(self, node): s = node.__class__.__name__ + "(\n" self.level += 1 s += self.indent + "question={0!r},\n".format(node.question) s += self.indent + "responses={" if len(node.responses) == 0: s += "}" else: s += "\n" t = sorted(node.responses.items()) t = ["{0!r}: {1}".format(k, self.visit(v)) for k, v in t] s += textwrap.indent(",\n".join(t), 2 * self.indent) s += "\n" + self.indent + "}" if node.converter is not None: s += ",\n" + self.indent + "converter={0!r}".format(node.converter) if node.path is not None: s += ",\n" + self.indent + "path={0!r}".format(node.path) self.level -= 1 s += "\n)" return s def visit_input(self, node): s = "{0}(prompt={1!r}".format(node.__class__.__name__, node.prompt) if node.converter is None and node.path is None: return s + "\n)" if node.converter is not None: s += ",\n" + self.indent + "converter={0!r}".format(node.converter) s += ",\n" + self.indent + "show_conversion={0!r}".format(node.show_conversion) s += ",\n" + self.indent + "confirm={0!r}".format(node.confirm) s += ",\n" + self.indent + "retry={0!r}".format(node.retry) if node.path is not None: s += ",\n" + self.indent + "path={0!r}".format(node.path) s += "\n)" return s def visit_statefile(self, node): s = "{0}(default_file={1!r}, check={2}, ask_filename={3})" s = s.format( node.__class__.__name__, node.default_file, node.check, node.ask_filename ) return s def visit_while(self, node): s = "{0}(cond={1!r}".format(node.__class__.__name__, node.cond) s += ",\n" + self.indent + "body=[" if len(node.body) > 0: s += "\n" self.level += 1 s += textwrap.indent(",\n".join(map(self.visit, node.body)), self.indent) self.level -= 1 s += "\n" + self.indent s += "]" s += ",\n" + self.indent + "idxname={0!r}".format(node.idxname) s += ",\n" + self.indent + "beg={0!r}".format(node.beg) if node.path is not None: s += ",\n" + self.indent + "path={0!r}".format(node.path) s += "\n)" return s def ensure_str_or_int(x): """Creates a string or int.""" if isinstance(x, int): return x x = x if isinstance(x, str) else str(x) try: x = ast.literal_eval(x) except (ValueError, SyntaxError): pass if not isinstance(x, (int, str)): msg = "{0!r} could not be converted to int or str".format(x) raise ValueError(msg) return x def canon_path(path, indices=None): """Returns the canonical form of a path, which is a tuple of str or ints. Indices may be optionally passed in. """ if not isinstance(path, str): return tuple(map(ensure_str_or_int, path)) if indices is not None: path = path.format(**indices) path = path[1:] if path.startswith("/") else path path = path[:-1] if path.endswith("/") else path if len(path) == 0: return () return tuple(map(ensure_str_or_int, path.split("/"))) class UnstorableType(object): """Represents an unstorable return value for when no input was given or such input was skipped. Typically represented by the Unstorable singleton. """ _inst = None def __new__(cls, *args, **kwargs): if cls._inst is None: cls._inst = super(UnstorableType, cls).__new__(cls, *args, **kwargs) return cls._inst Unstorable = UnstorableType() class StateVisitor(Visitor): """This class visits the nodes and stores the results in a top-level dict of data according to the state path of the node. The the node does not have a path or the path does not exist, the storage is skipped. This class can be optionally initialized with an existing state. """ def __init__(self, tree=None, state=None, indices=None): super().__init__(tree=tree) self.state = {} if state is None else state self.indices = {} if indices is None else indices def visit(self, node=None): if node is None: node = self.tree if node is None: raise RuntimeError("no node or tree given!") rtn = super().visit(node) path = getattr(node, "path", None) if callable(path): path = path(visitor=self, node=node, val=rtn) if path is not None and rtn is not Unstorable: self.store(path, rtn, indices=self.indices) return rtn def store(self, path, val, indices=None): """Stores a value at the path location.""" path = canon_path(path, indices=indices) loc = self.state for p, n in zip(path[:-1], path[1:]): if isinstance(p, str) and p not in loc: loc[p] = {} if isinstance(n, str) else [] elif isinstance(p, int) and abs(p) + (p >= 0) > len(loc): i = abs(p) + (p >= 0) - len(loc) if isinstance(n, str): ex = [{} for _ in range(i)] else: ex = [[] for _ in range(i)] loc.extend(ex) loc = loc[p] p = path[-1] if isinstance(p, int) and abs(p) + (p >= 0) > len(loc): i = abs(p) + (p >= 0) - len(loc) ex = [None] * i loc.extend(ex) loc[p] = val def flatten(self, path="/", value=None, flat=None): """Returns a dict version of the store whose keys are paths. Note that list and dict entries will always end in '/', allowing disambiquation in dump_rules. """ value = self.state if value is None else value flat = {} if flat is None else flat if isinstance(value, cabc.Mapping): path = path if path.endswith("/") else path + "/" flat[path] = value for k, v in value.items(): p = path + k self.flatten(path=p, value=v, flat=flat) elif isinstance(value, (str, bytes)): flat[path] = value elif isinstance(value, cabc.Sequence): path = path if path.endswith("/") else path + "/" flat[path] = value for i, v in enumerate(value): p = path + str(i) self.flatten(path=p, value=v, flat=flat) else: flat[path] = value return flat YN = "{GREEN}yes{NO_COLOR} or {RED}no{NO_COLOR} [default: no]? " YNB = ( "{GREEN}yes{NO_COLOR}, {RED}no{NO_COLOR}, or " "{YELLOW}break{NO_COLOR} [default: no]? " ) class PromptVisitor(StateVisitor): """Visits the nodes in the tree via the a command-line prompt.""" def __init__(self, tree=None, state=None, **kwargs): """ Parameters ---------- tree : Node, optional Tree of nodes to start visitor with. state : dict, optional Initial state to begin with. kwargs : optional Options that are passed through to the prompt via the shell's singleline() method. See BaseShell for mor details. """ super().__init__(tree=tree, state=state) self.env = builtins.__xonsh__.env self.shell = builtins.__xonsh__.shell.shell self.shell_kwargs = kwargs def visit_wizard(self, node): for child in node.children: self.visit(child) def visit_pass(self, node): pass def visit_message(self, node): print_color(node.message) def visit_question(self, node): self.env["PROMPT"] = node.question r = self.shell.singleline(**self.shell_kwargs) if callable(node.converter): r = node.converter(r) self.visit(node.responses[r]) return r def visit_input(self, node): need_input = True while need_input: self.env["PROMPT"] = node.prompt raw = self.shell.singleline(**self.shell_kwargs) if callable(node.converter): try: x = node.converter(raw) except KeyboardInterrupt: raise except Exception: if node.retry: msg = ( "{{BOLD_RED}}Invalid{{NO_COLOR}} input {0!r}, " "please retry." ) print_color(msg.format(raw)) continue else: raise if node.show_conversion and x is not Unstorable and str(x) != raw: msg = "{{BOLD_PURPLE}}Converted{{NO_COLOR}} input {0!r} to {1!r}." print_color(msg.format(raw, x)) else: x = raw if node.confirm: msg = "Would you like to keep the input: {0}" print(msg.format(pprint.pformat(x))) confirmer = TrueFalseBreak(prompt=YNB) status = self.visit(confirmer) if isinstance(status, str) and status == "break": x = Unstorable break else: need_input = not status else: need_input = False return x def visit_while(self, node): rtns = [] origidx = self.indices.get(node.idxname, None) self.indices[node.idxname] = idx = node.beg while node.cond(visitor=self, node=node): rtn = list(map(self.visit, node.body)) rtns.append(rtn) idx += 1 self.indices[node.idxname] = idx if origidx is None: del self.indices[node.idxname] else: self.indices[node.idxname] = origidx return rtns def visit_savejson(self, node): jstate = json.dumps( self.state, indent=1, sort_keys=True, default=serialize_xonsh_json ) if node.check: msg = "The current state is:\n\n{0}\n" print(msg.format(textwrap.indent(jstate, " "))) ap = "Would you like to save this state, " + YN asker = TrueFalse(prompt=ap) do_save = self.visit(asker) if not do_save: return Unstorable fname = None if node.ask_filename: fname = self.visit_input(node) if fname is None or len(fname) == 0: fname = node.default_file if os.path.isfile(fname): backup_file(fname) else: os.makedirs(os.path.dirname(fname), exist_ok=True) with open(fname, "w") as f: f.write(jstate) return fname def visit_loadjson(self, node): if node.check: ap = "Would you like to load an existing file, " + YN asker = TrueFalse(prompt=ap) do_load = self.visit(asker) if not do_load: return Unstorable fname = self.visit_input(node) if fname is None or len(fname) == 0: fname = node.default_file if os.path.isfile(fname): with open(fname, "r") as f: self.state = json.load(f) print_color("{{GREEN}}{0!r} loaded.{{NO_COLOR}}".format(fname)) else: print_color( ("{{RED}}{0!r} could not be found, " "continuing.{{NO_COLOR}}").format( fname ) ) return fname def visit_fileinserter(self, node): # perform the dumping operation. new = node.dumps(self.flatten()) # check if we should write this out if node.check: msg = "The current state to insert is:\n\n{0}\n" print(msg.format(textwrap.indent(new, " "))) ap = "Would you like to write out the current state, " + YN asker = TrueFalse(prompt=ap) do_save = self.visit(asker) if not do_save: return Unstorable # get and backup the file. fname = None if node.ask_filename: fname = self.visit_input(node) if fname is None or len(fname) == 0: fname = node.default_file if os.path.isfile(fname): with open(fname, "r") as f: s = f.read() before, _, s = s.partition(node.prefix) _, _, after = s.partition(node.suffix) backup_file(fname) else: before = after = "" dname = os.path.dirname(fname) if dname: os.makedirs(dname, exist_ok=True) # write out the file with open(fname, "w") as f: f.write(before + new + after) return fname ================================================ FILE: xonsh/xonfig.py ================================================ """The xonsh configuration (xonfig) utility.""" import os import re import ast import json import shutil import random import pprint import textwrap import builtins import argparse import functools import itertools import contextlib import collections from xonsh.ply import ply import xonsh.wizard as wiz from xonsh import __version__ as XONSH_VERSION from xonsh.prompt.base import is_template_string from xonsh.platform import ( is_readline_available, ptk_version, PYTHON_VERSION_INFO, pygments_version, ON_POSIX, ON_LINUX, linux_distro, ON_DARWIN, ON_WINDOWS, ON_CYGWIN, DEFAULT_ENCODING, ON_MSYS, githash, ) from xonsh.tools import ( to_bool, is_string, print_exception, is_superuser, color_style_names, print_color, color_style, ) from xonsh.foreign_shells import CANON_SHELL_NAMES from xonsh.xontribs import xontrib_metadata, find_xontrib from xonsh.lazyasd import lazyobject HR = "'`-.,_,.-*'`-.,_,.-*'`-.,_,.-*'`-.,_,.-*'`-.,_,.-*'`-.,_,.-*'`-.,_,.-*'" WIZARD_HEAD = """ {{BOLD_WHITE}}Welcome to the xonsh configuration wizard!{{NO_COLOR}} {{YELLOW}}------------------------------------------{{NO_COLOR}} This will present a guided tour through setting up the xonsh static config file. Xonsh will automatically ask you if you want to run this wizard if the configuration file does not exist. However, you can always rerun this wizard with the xonfig command: $ xonfig wizard This wizard will load an existing configuration, if it is available. Also never fear when this wizard saves its results! It will create a backup of any existing configuration automatically. This wizard has two main phases: foreign shell setup and environment variable setup. Each phase may be skipped in its entirety. For the configuration to take effect, you will need to restart xonsh. {hr} """.format( hr=HR ) WIZARD_FS = """ {hr} {{BOLD_WHITE}}Foreign Shell Setup{{NO_COLOR}} {{YELLOW}}-------------------{{NO_COLOR}} The xonsh shell has the ability to interface with foreign shells such as Bash, or zsh (fish not yet implemented). For configuration, this means that xonsh can load the environment, aliases, and functions specified in the config files of these shells. Naturally, these shells must be available on the system to work. Being able to share configuration (and source) from foreign shells makes it easier to transition to and from xonsh. """.format( hr=HR ) WIZARD_ENV = """ {hr} {{BOLD_WHITE}}Environment Variable Setup{{NO_COLOR}} {{YELLOW}}--------------------------{{NO_COLOR}} The xonsh shell also allows you to setup environment variables from the static configuration file. Any variables set in this way are superseded by the definitions in the xonshrc or on the command line. Still, setting environment variables in this way can help define options that are global to the system or user. The following lists the environment variable name, its documentation, the default value, and the current value. The default and current values are presented as pretty repr strings of their Python types. {{BOLD_GREEN}}Note:{{NO_COLOR}} Simply hitting enter for any environment variable will accept the default value for that entry. """.format( hr=HR ) WIZARD_ENV_QUESTION = "Would you like to set env vars now, " + wiz.YN WIZARD_XONTRIB = """ {hr} {{BOLD_WHITE}}Xontribs{{NO_COLOR}} {{YELLOW}}--------{{NO_COLOR}} No shell is complete without extensions, and xonsh is no exception. Xonsh extensions are called {{BOLD_GREEN}}xontribs{{NO_COLOR}}, or xonsh contributions. Xontribs are dynamically loadable, either by importing them directly or by using the 'xontrib' command. However, you can also configure xonsh to load xontribs automatically on startup prior to loading the run control files. This allows the xontrib to be used immediately in your xonshrc files. The following describes all xontribs that have been registered with xonsh. These come from users, 3rd party developers, or xonsh itself! """.format( hr=HR ) WIZARD_XONTRIB_QUESTION = "Would you like to enable xontribs now, " + wiz.YN WIZARD_TAIL = """ Thanks for using the xonsh configuration wizard!""" _XONFIG_SOURCE_FOREIGN_SHELL_COMMAND = collections.defaultdict( lambda: "source-foreign", bash="source-bash", cmd="source-cmd", zsh="source-zsh" ) def _dump_xonfig_foreign_shell(path, value): shell = value["shell"] shell = CANON_SHELL_NAMES.get(shell, shell) cmd = [_XONFIG_SOURCE_FOREIGN_SHELL_COMMAND[shell]] interactive = value.get("interactive", None) if interactive is not None: cmd.extend(["--interactive", str(interactive)]) login = value.get("login", None) if login is not None: cmd.extend(["--login", str(login)]) envcmd = value.get("envcmd", None) if envcmd is not None: cmd.extend(["--envcmd", envcmd]) aliascmd = value.get("aliasmd", None) if aliascmd is not None: cmd.extend(["--aliascmd", aliascmd]) extra_args = value.get("extra_args", None) if extra_args: cmd.extend(["--extra-args", repr(" ".join(extra_args))]) safe = value.get("safe", None) if safe is not None: cmd.extend(["--safe", str(safe)]) prevcmd = value.get("prevcmd", "") if prevcmd: cmd.extend(["--prevcmd", repr(prevcmd)]) postcmd = value.get("postcmd", "") if postcmd: cmd.extend(["--postcmd", repr(postcmd)]) funcscmd = value.get("funcscmd", None) if funcscmd: cmd.extend(["--funcscmd", repr(funcscmd)]) sourcer = value.get("sourcer", None) if sourcer: cmd.extend(["--sourcer", sourcer]) if cmd[0] == "source-foreign": cmd.append(shell) cmd.append('"echo loading xonsh foreign shell"') return " ".join(cmd) def _dump_xonfig_env(path, value): name = os.path.basename(path.rstrip("/")) ensurer = builtins.__xonsh__.env.get_ensurer(name) dval = str(value) if ensurer.detype is None else ensurer.detype(value) dval = str(value) if dval is None else dval return "${name} = {val!r}".format(name=name, val=dval) def _dump_xonfig_xontribs(path, value): return "xontrib load {0}".format(" ".join(value)) @lazyobject def XONFIG_DUMP_RULES(): return { "/": None, "/env/": None, "/foreign_shells/*/": _dump_xonfig_foreign_shell, "/env/*": _dump_xonfig_env, "/env/*/[0-9]*": None, "/xontribs/": _dump_xonfig_xontribs, } def make_fs_wiz(): """Makes the foreign shell part of the wizard.""" cond = wiz.create_truefalse_cond(prompt="Add a new foreign shell, " + wiz.YN) fs = wiz.While( cond=cond, body=[ wiz.Input("shell name (e.g. bash): ", path="/foreign_shells/{idx}/shell"), wiz.StoreNonEmpty( "interactive shell [bool, default=True]: ", converter=to_bool, show_conversion=True, path="/foreign_shells/{idx}/interactive", ), wiz.StoreNonEmpty( "login shell [bool, default=False]: ", converter=to_bool, show_conversion=True, path="/foreign_shells/{idx}/login", ), wiz.StoreNonEmpty( "env command [str, default='env']: ", path="/foreign_shells/{idx}/envcmd", ), wiz.StoreNonEmpty( "alias command [str, default='alias']: ", path="/foreign_shells/{idx}/aliascmd", ), wiz.StoreNonEmpty( ("extra command line arguments [list of str, " "default=[]]: "), converter=ast.literal_eval, show_conversion=True, path="/foreign_shells/{idx}/extra_args", ), wiz.StoreNonEmpty( "safely handle exceptions [bool, default=True]: ", converter=to_bool, show_conversion=True, path="/foreign_shells/{idx}/safe", ), wiz.StoreNonEmpty( "pre-command [str, default='']: ", path="/foreign_shells/{idx}/prevcmd" ), wiz.StoreNonEmpty( "post-command [str, default='']: ", path="/foreign_shells/{idx}/postcmd" ), wiz.StoreNonEmpty( "foreign function command [str, default=None]: ", path="/foreign_shells/{idx}/funcscmd", ), wiz.StoreNonEmpty( "source command [str, default=None]: ", path="/foreign_shells/{idx}/sourcer", ), wiz.Message(message="Foreign shell added.\n"), ], ) return fs def _wrap_paragraphs(text, width=70, **kwargs): """Wraps paragraphs instead.""" pars = text.split("\n") pars = ["\n".join(textwrap.wrap(p, width=width, **kwargs)) for p in pars] s = "\n".join(pars) return s ENVVAR_MESSAGE = """ {{BOLD_CYAN}}${name}{{NO_COLOR}} {docstr} {{RED}}default value:{{NO_COLOR}} {default} {{RED}}current value:{{NO_COLOR}} {current}""" ENVVAR_PROMPT = "{BOLD_GREEN}>>>{NO_COLOR} " def make_exit_message(): """Creates a message for how to exit the wizard.""" shell_type = builtins.__xonsh__.shell.shell_type keyseq = "Ctrl-D" if shell_type == "readline" else "Ctrl-C" msg = "To exit the wizard at any time, press {BOLD_UNDERLINE_CYAN}" msg += keyseq + "{NO_COLOR}.\n" m = wiz.Message(message=msg) return m def make_envvar(name): """Makes a StoreNonEmpty node for an environment variable.""" env = builtins.__xonsh__.env vd = env.get_docs(name) if not vd.configurable: return default = vd.default if "\n" in default: default = "\n" + _wrap_paragraphs(default, width=69) curr = env.get(name) if is_string(curr) and is_template_string(curr): curr = curr.replace("{", "{{").replace("}", "}}") curr = pprint.pformat(curr, width=69) if "\n" in curr: curr = "\n" + curr msg = ENVVAR_MESSAGE.format( name=name, default=default, current=curr, docstr=_wrap_paragraphs(vd.docstr, width=69), ) mnode = wiz.Message(message=msg) ens = env.get_ensurer(name) path = "/env/" + name pnode = wiz.StoreNonEmpty( ENVVAR_PROMPT, converter=ens.convert, show_conversion=True, path=path, retry=True, store_raw=vd.store_as_str, ) return mnode, pnode def _make_flat_wiz(kidfunc, *args): kids = map(kidfunc, *args) flatkids = [] for k in kids: if k is None: continue flatkids.extend(k) wizard = wiz.Wizard(children=flatkids) return wizard def make_env_wiz(): """Makes an environment variable wizard.""" w = _make_flat_wiz(make_envvar, sorted(builtins.__xonsh__.env._docs.keys())) return w XONTRIB_PROMPT = "{BOLD_GREEN}Add this xontrib{NO_COLOR}, " + wiz.YN def _xontrib_path(visitor=None, node=None, val=None): # need this to append only based on user-selected size return ("xontribs", len(visitor.state.get("xontribs", ()))) def make_xontrib(xontrib, package): """Makes a message and StoreNonEmpty node for a xontrib.""" name = xontrib.get("name", "") msg = "\n{BOLD_CYAN}" + name + "{NO_COLOR}\n" if "url" in xontrib: msg += "{RED}url:{NO_COLOR} " + xontrib["url"] + "\n" if "package" in xontrib: msg += "{RED}package:{NO_COLOR} " + xontrib["package"] + "\n" if "url" in package: if "url" in xontrib and package["url"] != xontrib["url"]: msg += "{RED}package-url:{NO_COLOR} " + package["url"] + "\n" if "license" in package: msg += "{RED}license:{NO_COLOR} " + package["license"] + "\n" msg += "{PURPLE}installed?{NO_COLOR} " msg += ("no" if find_xontrib(name) is None else "yes") + "\n" desc = xontrib.get("description", "") if not isinstance(desc, str): desc = "".join(desc) msg += _wrap_paragraphs(desc, width=69) if msg.endswith("\n"): msg = msg[:-1] mnode = wiz.Message(message=msg) convert = lambda x: name if to_bool(x) else wiz.Unstorable pnode = wiz.StoreNonEmpty(XONTRIB_PROMPT, converter=convert, path=_xontrib_path) return mnode, pnode def make_xontribs_wiz(): """Makes a xontrib wizard.""" md = xontrib_metadata() pkgs = [md["packages"].get(d.get("package", None), {}) for d in md["xontribs"]] w = _make_flat_wiz(make_xontrib, md["xontribs"], pkgs) return w def make_xonfig_wizard(default_file=None, confirm=False, no_wizard_file=None): """Makes a configuration wizard for xonsh config file. Parameters ---------- default_file : str, optional Default filename to save and load to. User will still be prompted. confirm : bool, optional Confirm that the main part of the wizard should be run. no_wizard_file : str, optional Filename for that will flag to future runs that the wizard should not be run again. If None (default), this defaults to default_file. """ w = wiz.Wizard( children=[ wiz.Message(message=WIZARD_HEAD), make_exit_message(), wiz.Message(message=WIZARD_FS), make_fs_wiz(), wiz.Message(message=WIZARD_ENV), wiz.YesNo(question=WIZARD_ENV_QUESTION, yes=make_env_wiz(), no=wiz.Pass()), wiz.Message(message=WIZARD_XONTRIB), wiz.YesNo( question=WIZARD_XONTRIB_QUESTION, yes=make_xontribs_wiz(), no=wiz.Pass() ), wiz.Message(message="\n" + HR + "\n"), wiz.FileInserter( prefix="# XONSH WIZARD START", suffix="# XONSH WIZARD END", dump_rules=XONFIG_DUMP_RULES, default_file=default_file, check=True, ), wiz.Message(message=WIZARD_TAIL), ] ) if confirm: q = ( "Would you like to run the xonsh configuration wizard now?\n\n" "1. Yes (You can abort at any time)\n" "2. No, but ask me next time.\n" "3. No, and don't ask me again.\n\n" "1, 2, or 3 [default: 2]? " ) no_wizard_file = default_file if no_wizard_file is None else no_wizard_file passer = wiz.Pass() saver = wiz.SaveJSON( check=False, ask_filename=False, default_file=no_wizard_file ) w = wiz.Question( q, {1: w, 2: passer, 3: saver}, converter=lambda x: int(x) if x != "" else 2 ) return w def _wizard(ns): env = builtins.__xonsh__.env shell = builtins.__xonsh__.shell.shell fname = env.get("XONSHRC")[-1] if ns.file is None else ns.file no_wiz = os.path.join(env.get("XONSH_CONFIG_DIR"), "no-wizard") w = make_xonfig_wizard( default_file=fname, confirm=ns.confirm, no_wizard_file=no_wiz ) tempenv = {"PROMPT": "", "XONSH_STORE_STDOUT": False} pv = wiz.PromptVisitor(w, store_in_history=False, multiline=False) @contextlib.contextmanager def force_hide(): if env.get("XONSH_STORE_STDOUT") and hasattr(shell, "_force_hide"): orig, shell._force_hide = shell._force_hide, False yield shell._force_hide = orig else: yield with force_hide(), env.swap(tempenv): try: pv.visit() except (KeyboardInterrupt, Exception): print() print_exception() def _xonfig_format_human(data): wcol1 = wcol2 = 0 for key, val in data: wcol1 = max(wcol1, len(key)) wcol2 = max(wcol2, len(str(val))) hr = "+" + ("-" * (wcol1 + 2)) + "+" + ("-" * (wcol2 + 2)) + "+\n" row = "| {key!s:<{wcol1}} | {val!s:<{wcol2}} |\n" s = hr for key, val in data: s += row.format(key=key, wcol1=wcol1, val=val, wcol2=wcol2) s += hr return s def _xonfig_format_json(data): data = {k.replace(" ", "_"): v for k, v in data} s = json.dumps(data, sort_keys=True, indent=1) + "\n" return s def _info(ns): env = builtins.__xonsh__.env data = [("xonsh", XONSH_VERSION)] hash_, date_ = githash() if hash_: data.append(("Git SHA", hash_)) data.append(("Commit Date", date_)) data.extend( [ ("Python", "{}.{}.{}".format(*PYTHON_VERSION_INFO)), ("PLY", ply.__version__), ("have readline", is_readline_available()), ("prompt toolkit", ptk_version() or None), ("shell type", env.get("SHELL_TYPE")), ("pygments", pygments_version()), ("on posix", bool(ON_POSIX)), ("on linux", bool(ON_LINUX)), ] ) if ON_LINUX: data.append(("distro", linux_distro())) data.extend( [ ("on darwin", ON_DARWIN), ("on windows", ON_WINDOWS), ("on cygwin", ON_CYGWIN), ("on msys2", ON_MSYS), ("is superuser", is_superuser()), ("default encoding", DEFAULT_ENCODING), ("xonsh encoding", env.get("XONSH_ENCODING")), ("encoding errors", env.get("XONSH_ENCODING_ERRORS")), ] ) formatter = _xonfig_format_json if ns.json else _xonfig_format_human s = formatter(data) return s def _styles(ns): env = builtins.__xonsh__.env curr = env.get("XONSH_COLOR_STYLE") styles = sorted(color_style_names()) if ns.json: s = json.dumps(styles, sort_keys=True, indent=1) print(s) return lines = [] for style in styles: if style == curr: lines.append("* {GREEN}" + style + "{NO_COLOR}") else: lines.append(" " + style) s = "\n".join(lines) print_color(s) def _str_colors(cmap, cols): color_names = sorted(cmap.keys(), key=(lambda s: (len(s), s))) grper = lambda s: min(cols // (len(s) + 1), 8) lines = [] for n, group in itertools.groupby(color_names, key=grper): width = cols // n line = "" for i, name in enumerate(group): buf = " " * (width - len(name)) line += "{" + name + "}" + name + "{NO_COLOR}" + buf if (i + 1) % n == 0: lines.append(line) line = "" if len(line) != 0: lines.append(line) return "\n".join(lines) def _tok_colors(cmap, cols): from xonsh.style_tools import Color nc = Color.NO_COLOR names_toks = {} for t in cmap.keys(): name = str(t) if name.startswith("Token.Color."): _, _, name = name.rpartition(".") names_toks[name] = t color_names = sorted(names_toks.keys(), key=(lambda s: (len(s), s))) grper = lambda s: min(cols // (len(s) + 1), 8) toks = [] for n, group in itertools.groupby(color_names, key=grper): width = cols // n for i, name in enumerate(group): toks.append((names_toks[name], name)) buf = " " * (width - len(name)) if (i + 1) % n == 0: buf += "\n" toks.append((nc, buf)) if not toks[-1][1].endswith("\n"): toks[-1] = (nc, toks[-1][1] + "\n") return toks def _colors(args): columns, _ = shutil.get_terminal_size() columns -= int(ON_WINDOWS) style_stash = builtins.__xonsh__.env["XONSH_COLOR_STYLE"] if args.style is not None: if args.style not in color_style_names(): print("Invalid style: {}".format(args.style)) return builtins.__xonsh__.env["XONSH_COLOR_STYLE"] = args.style color_map = color_style() akey = next(iter(color_map)) if isinstance(akey, str): s = _str_colors(color_map, columns) else: s = _tok_colors(color_map, columns) print_color(s) builtins.__xonsh__.env["XONSH_COLOR_STYLE"] = style_stash def _tutorial(args): import webbrowser webbrowser.open("http://xon.sh/tutorial.html") @functools.lru_cache(1) def _xonfig_create_parser(): p = argparse.ArgumentParser( prog="xonfig", description="Manages xonsh configuration." ) subp = p.add_subparsers(title="action", dest="action") info = subp.add_parser( "info", help=("displays configuration information, " "default action") ) info.add_argument( "--json", action="store_true", default=False, help="reports results as json" ) wiz = subp.add_parser("wizard", help="displays configuration information") wiz.add_argument( "--file", default=None, help="config file location, default=$XONSHRC" ) wiz.add_argument( "--confirm", action="store_true", default=False, help="confirm that the wizard should be run.", ) sty = subp.add_parser("styles", help="prints available xonsh color styles") sty.add_argument( "--json", action="store_true", default=False, help="reports results as json" ) colors = subp.add_parser("colors", help="preview color style") colors.add_argument( "style", nargs="?", default=None, help="style to preview, default: " ) subp.add_parser("tutorial", help="Launch tutorial in browser.") return p _XONFIG_MAIN_ACTIONS = { "info": _info, "wizard": _wizard, "styles": _styles, "colors": _colors, "tutorial": _tutorial, } def xonfig_main(args=None): """Main xonfig entry point.""" if not args or ( args[0] not in _XONFIG_MAIN_ACTIONS and args[0] not in {"-h", "--help"} ): args.insert(0, "info") parser = _xonfig_create_parser() ns = parser.parse_args(args) if ns.action is None: # apply default action ns = parser.parse_args(["info"] + args) return _XONFIG_MAIN_ACTIONS[ns.action](ns) @lazyobject def STRIP_COLOR_RE(): return re.compile("{.*?}") def _align_string(string, align="<", fill=" ", width=80): """ Align and pad a color formatted string """ linelen = len(STRIP_COLOR_RE.sub("", string)) padlen = max(width - linelen, 0) if align == "^": return fill * (padlen // 2) + string + fill * (padlen // 2 + padlen % 2) elif align == ">": return fill * padlen + string elif align == "<": return string + fill * padlen else: return string @lazyobject def TAGLINES(): return [ "Exofrills in the shell", "No frills in the shell", "Become the Lord of the Files", "Break out of your shell", "The only shell that is also a shell", "All that is and all that shell be", "It cannot be that hard", "Pass the xonsh, Piggy", "Piggy glanced nervously into hell and cradled the xonsh", "The xonsh is a symbol", "It is pronounced conch", "The shell, bourne again", "Snailed it", "Starfish loves you", "Come snail away", "This is Major Tom to Ground Xonshtrol", "Sally sells csh and keeps xonsh to herself", "Nice indeed. Everything's accounted for, except your old shell.", "I wanna thank you for putting me back in my snail shell", "Crustaceanly Yours", "With great shell comes great reproducibility", "None shell pass", "You shell not pass!", "The x-on shell", "Ever wonder why there isn't a Taco Shell? Because it is a corny idea.", "The carcolh will catch you!", "People xonshtantly mispronounce these things", "WHAT...is your favorite shell?", "Conches for the xonsh god!", "Python-powered, cross-platform, Unix-gazing shell", "Tab completion in Alderaan places", "This fix was trickier than expected", "The unholy cross of Bash/Python", ] # list of strings or tuples (string, align, fill) WELCOME_MSG = [ "", ("{{INTENSE_WHITE}}Welcome to the xonsh shell ({version}){{NO_COLOR}}", "^", " "), "", ("{{INTENSE_RED}}~{{NO_COLOR}} {tagline} {{INTENSE_RED}}~{{NO_COLOR}}", "^", " "), "", ("{{INTENSE_BLACK}}", "<", "-"), "{{GREEN}}xonfig{{NO_COLOR}} tutorial {{INTENSE_WHITE}}-> Launch the tutorial in " "the browser{{NO_COLOR}}", "{{GREEN}}xonfig{{NO_COLOR}} wizard {{INTENSE_WHITE}}-> Run the configuration " "wizard and claim your shell {{NO_COLOR}}", "{{INTENSE_BLACK}}(Note: Run the Wizard or create a {{RED}}~/.xonshrc{{INTENSE_BLACK}} file " "to suppress the welcome screen)", "", ] def print_welcome_screen(): subst = dict(tagline=random.choice(list(TAGLINES)), version=XONSH_VERSION) for elem in WELCOME_MSG: if isinstance(elem, str): elem = (elem, "", "") line = elem[0].format(**subst) termwidth = os.get_terminal_size().columns line = _align_string(line, elem[1], elem[2], width=termwidth) print_color(line) ================================================ FILE: xonsh/xonshrc ================================================ # adjust some paths #$BASH_COMPLETIONS.append('/usr/local/etc/bash_completion.d/git-completion.bash') #$PATH.append('/some/path/bin') #$LD_LIBRARY_PATH = ['/some/path1/lib', '/some/path2/lib', ''] aliases['hi'] = echo hello world # some customization options #$MULTILINE_PROMPT = '`·.,¸,.·*¯`·.,¸,.·*¯' ================================================ FILE: xonsh/xontribs.json ================================================ {"xontribs": [ {"name": "apt_tabcomplete", "package": "xonsh-apt-tabcomplete", "url": "https://github.com/DangerOnTheRanger/xonsh-apt-tabcomplete", "description": ["Adds tabcomplete functionality to apt-get/apt-cache inside of xonsh."] }, {"name": "autojump", "package": "xontrib-autojump", "url": "https://github.com/gsaga/autojump-xonsh", "description": ["autojump support for xonsh"] }, {"name": "autoxsh", "package": "xonsh-autoxsh", "url": "https://github.com/Granitas/xonsh-autoxsh", "description": ["Adds automatic execution of xonsh script files called", "``.autoxsh`` when enterting a directory with ``cd`` function"] }, {"name": "bashisms", "package": "xonsh", "url": "http://xon.sh", "description": [ "Enables additional Bash-like syntax while at the command prompt. For ", "example, the ``!!`` syntax for running the previous command is now usable.", "Note that these features are implemented as precommand events and these ", "additions do not affect the xonsh language when run as script. That said, ", "you might find them useful if you have strong muscle memory.\n\n", "**Warning:** This xontrib may modify user command line input to implement ", "its behavior. To see the modifications as they are applied (in unified diff", "format), please set ``$XONSH_DEBUG`` to ``2`` or higher."] }, {"name": "base16_shell", "package": "xontrib-base16-shell", "url": "https://github.com/ErickTucto/xontrib-base16-shell", "description": ["Change base16 shell themes"] }, {"name": "coreutils", "package": "xonsh", "url": "http://xon.sh", "description": [ "Additional core utilities that are implemented in xonsh. The current list ", "includes:\n", "\n", "* cat\n", "* echo\n", "* pwd\n", "* tee\n", "* tty", "* yes\n", "\n", "In many cases, these may have a lower performance overhead than the ", "posix command line utility with the same name. This is because these ", "tools avoid the need for a full subprocess call. Additionally, these ", "tools are cross-platform."] }, {"name": "direnv", "package": "xonsh-direnv", "url": "https://github.com/74th/xonsh-direnv", "description": ["Supports direnv."] }, {"name": "distributed", "package": "xonsh", "url": "http://xon.sh", "description": [ "The distributed parallel computing library hooks for xonsh. ", "Importantly this provides a substitute 'dworker' command which enables ", "distributed workers to have access to xonsh builtins.\n\n", "Furthermore, this xontrib adds a 'DSubmitter' context manager for ", "executing a block remotely. Moreover, this also adds a convenience ", "function 'dsubmit()' for creating DSubmitter and Executor instances ", "at the same time. Thus users may submit distributed jobs with::\n\n", " with dsubmit('127.0.0.1:8786', rtn='x') as dsub:\n", " x = $(echo I am elsewhere)\n\n", " res = dsub.future.result()\n", " print(res)\n\n", "This is useful for long running or non-blocking jobs."] }, {"name": "docker_tabcomplete", "package": "xonsh-docker-tabcomplete", "url": "https://github.com/xsteadfastx/xonsh-docker-tabcomplete", "description": ["Adds tabcomplete functionality to docker inside of xonsh."] }, {"name": "jedi", "package": "xonsh", "url": "http://xon.sh", "description": ["Jedi tab completion hooks for xonsh."] }, {"name": "mpl", "package": "xonsh", "url": "http://xon.sh", "description": ["Matplotlib hooks for xonsh, including the new 'mpl' alias ", "that displays the current figure on the screen."] }, {"name": "prompt_ret_code", "package": "xonsh", "url": "http://xon.sh", "description": ["Adds return code info to the prompt"] }, {"name": "free_cwd", "package": "xonsh", "url": "http://xon.sh", "description": [ "Windows only xontrib, to release the lock on the current directory ", "whenever the prompt is shown. Enabling this will allow the other ", "programs or Windows Explorer to delete or rename the current or parent ", "directories. Internally, it is accomplished by temporarily resetting ", "CWD to the root drive folder while waiting at the prompt. This only ", "works with the prompt_toolkit backend and can cause cause issues ", "if any extensions are enabled that hook the prompt and relies on ", "``os.getcwd()``"] }, {"name": "whole_word_jumping", "package": "xonsh", "url": "http://xon.sh", "description": [ "Jumping across whole words (non-whitespace) with Ctrl+Left/Right.", "Alt+Left/Right remains unmodified to jump over smaller word segments."] }, {"name": "scrapy_tabcomplete", "package": "xonsh-scrapy-tabcomplete", "url": "https://github.com/Granitas/xonsh-scrapy-tabcomplete", "description": ["Adds tabcomplete functionality to scrapy inside of xonsh."] }, {"name": "vox", "package": "xonsh", "url": "http://xon.sh", "description": ["Python virtual environment manager for xonsh."] }, {"name": "vox_tabcomplete", "package": "xonsh-vox-tabcomplete", "url": "https://github.com/Granitosaurus/xonsh-vox-tabcomplete", "description": ["Adds tabcomplete functionality to vox inside of xonsh."] }, {"name": "xo", "package": "exofrills", "url": "https://github.com/scopatz/xo", "description": ["Adds an 'xo' alias to run the exofrills text editor in the ", "current Python interpreter session. This shaves off a ", "bit of the startup time when running your favorite, minimal ", "text editor."] }, {"name": "xonda", "package": "xonda", "url": "https://github.com/gforsyth/xonda", "description": ["A thin wrapper around conda with tab completion"] }, {"name": "avox", "package": "xontrib-avox", "url": "https://github.com/astronouth7303/xontrib-avox", "description": ["Automatic (de)activation of virtual environments as you cd around"] }, {"name": "z", "package": "xontrib-z", "url": "https://github.com/astronouth7303/xontrib-z", "description": ["Tracks your most used directories, based on 'frecency'."] }, {"name": "powerline", "package": "xontrib-powerline", "url": "https://github.com/santagada/xontrib-powerline", "description": ["Powerline for Xonsh shell"] }, {"name": "prompt_vi_mode", "package": "xontrib-prompt-vi-mode", "url": "https://github.com/t184256/xontrib-prompt-vi-mode", "description": ["vi-mode status formatter for xonsh prompt"] }, {"name": "click_tabcomplete", "package": "xonsh-click-tabcomplete", "url": "https://github.com/Granitosaurus/xonsh-click-tabcomplete", "description": ["Adds tabcomplete functionality to click based python applications inside of xonsh."] }, {"name": "fzf-widgets", "package": "xontrib-fzf-widgets", "url": "https://github.com/shahinism/xontrib-fzf-widgets", "description": ["Adds some fzf widgets to your xonsh shell."] }, {"name": "schedule", "package": "xontrib-schedule", "url": "https://github.com/astronouth7303/xontrib-schedule", "description": ["Xonsh Task Scheduling"] } ], "packages": { "exofrills": { "license": "WTFPL", "url": "http://exofrills.org", "install": { "conda": "conda install -c conda-forge xo", "pip": "xpip install exofrills"} }, "xonsh": { "license": "BSD 3-clause", "url": "http://xon.sh", "install": { "conda": "conda install -c conda-forge xonsh", "pip": "xpip install xonsh", "aura": "sudo aura -A xonsh", "yaourt": "yaourt -Sa xonsh"} }, "xontrib-prompt-ret-code": { "license": "MIT", "url": "https://github.com/Siecje/xontrib-prompt-ret-code", "install": { "pip": "xpip install xontrib-prompt-ret-code" } }, "xonsh-apt-tabcomplete": { "license": "BSD 2-clause", "url": "https://github.com/DangerOnTheRanger/xonsh-apt-tabcomplete", "install": { "pip": "xpip install xonsh-apt-tabcomplete" } }, "xonsh-direnv": { "license": "MIT", "url": "https://github.com/74th/xonsh-direnv", "install": { "pip": "xpip install xonsh-direnv" } }, "xonsh-docker-tabcomplete": { "license": "MIT", "url": "https://github.com/xsteadfastx/xonsh-docker-tabcomplete", "install": { "pip": "xpip install xonsh-docker-tabcomplete" } }, "xonsh-scrapy-tabcomplete": { "license": "GPLv3", "url": "https://github.com/Granitas/xonsh-scrapy-tabcomplete", "install": { "pip": "xpip install xonsh-scrapy-tabcomplete" } }, "xonsh-vox-tabcomplete": { "license": "GPLv3", "url": "https://github.com/Granitosaurus/xonsh-vox-tabcomplete", "install": { "pip": "xpip install xonsh-vox-tabcomplete" } }, "xonsh-click-tabcomplete": { "license": "GPLv3", "url": "https://github.com/Granitosaurus/xonsh-click-tabcomplete", "install": { "pip": "xpip install xonsh-click-tabcomplete" } }, "xonsh-autoxsh": { "license": "GPLv3", "url": "https://github.com/Granitas/xonsh-autoxsh", "install": { "pip": "xpip install xonsh-autoxsh" } }, "xonda": { "license": "MIT", "url": "https://github.com/gforsyth/xonda", "install": { "pip": "xpip install xonda" } }, "xontrib-avox": { "license": "GPLv3", "url": "https://github.com/astronouth7303/xontrib-avox", "install": { "pip": "xpip install xontrib-avox" } }, "xontrib-z": { "license": "GPLv3", "url": "https://github.com/astronouth7303/xontrib-z", "install": { "pip": "xpip install xontrib-z" } }, "xontrib-powerline": { "license": "MIT", "url": "https://github.com/santagada/xontrib-powerline", "install": { "pip": "xpip install xontrib-powerline" } }, "xontrib-thefuck": { "license": "MIT", "url": "https://github.com/meatballs/xontrib-thefuck", "install": { "pip": "xpip install xontrib-thefuck" } }, "xontrib-prompt-vi-mode": { "license": "MIT", "url": "https://github.com/t184256/xontrib-prompt-vi-mode", "install": { "pip": "xpip install xontrib-prompt-vi-mode" } }, "xontrib-fzf-widgets": { "license": "GPLv3", "url": "https://github.com/shahinism/xontrib-fzf-widgets", "install": { "pip": "xpip install xontrib-fzf-widgets" } }, "xontrib-schedule": { "license": "MIT", "url": "https://github.com/astronouth7303/xontrib-schedule", "install": { "pip": "xpip install xontrib-schedule" } } } } ================================================ FILE: xonsh/xontribs.py ================================================ """Tools for helping manage xontributions.""" import os import sys import json import builtins import argparse import functools import importlib import importlib.util from xonsh.tools import print_color, unthreadable @functools.lru_cache(1) def xontribs_json(): return os.path.join(os.path.dirname(__file__), "xontribs.json") def find_xontrib(name): """Finds a xontribution from its name.""" if name.startswith("."): spec = importlib.util.find_spec(name, package="xontrib") else: spec = importlib.util.find_spec("." + name, package="xontrib") return spec or importlib.util.find_spec(name) def xontrib_context(name): """Return a context dictionary for a xontrib of a given name.""" spec = find_xontrib(name) if spec is None: return None m = importlib.import_module(spec.name) pubnames = getattr(m, "__all__", None) if pubnames is not None: ctx = {k: getattr(m, k) for k in pubnames} else: ctx = {k: getattr(m, k) for k in dir(m) if not k.startswith("_")} return ctx def prompt_xontrib_install(names): """Returns a formatted string with name of xontrib package to prompt user""" md = xontrib_metadata() packages = [] for name in names: for xontrib in md["xontribs"]: if xontrib["name"] == name: packages.append(xontrib["package"]) print( "The following xontribs are enabled but not installed: \n" " {xontribs}\n" "To install them run \n" " xpip install {packages}".format( xontribs=" ".join(names), packages=" ".join(packages) ) ) def update_context(name, ctx=None): """Updates a context in place from a xontrib. If ctx is not provided, then __xonsh__.ctx is updated. """ if ctx is None: ctx = builtins.__xonsh__.ctx if not hasattr(update_context, "bad_imports"): update_context.bad_imports = [] modctx = xontrib_context(name) if modctx is None: update_context.bad_imports.append(name) return ctx return ctx.update(modctx) @functools.lru_cache() def xontrib_metadata(): """Loads and returns the xontribs.json file.""" with open(xontribs_json(), "r") as f: md = json.load(f) return md def xontribs_load(names, verbose=False): """Load xontribs from a list of names""" ctx = builtins.__xonsh__.ctx for name in names: if verbose: print("loading xontrib {0!r}".format(name)) update_context(name, ctx=ctx) if update_context.bad_imports: prompt_xontrib_install(update_context.bad_imports) del update_context.bad_imports def _load(ns): """load xontribs""" xontribs_load(ns.names, verbose=ns.verbose) def _list(ns): """Lists xontribs.""" meta = xontrib_metadata() data = [] nname = 6 # ensures some buffer space. names = None if len(ns.names) == 0 else set(ns.names) for md in meta["xontribs"]: name = md["name"] if names is not None and md["name"] not in names: continue nname = max(nname, len(name)) spec = find_xontrib(name) if spec is None: installed = loaded = False else: installed = True loaded = spec.name in sys.modules d = {"name": name, "installed": installed, "loaded": loaded} data.append(d) if ns.json: jdata = {d.pop("name"): d for d in data} s = json.dumps(jdata) print(s) else: s = "" for d in data: name = d["name"] lname = len(name) s += "{PURPLE}" + name + "{NO_COLOR} " + " " * (nname - lname) if d["installed"]: s += "{GREEN}installed{NO_COLOR} " else: s += "{RED}not-installed{NO_COLOR} " if d["loaded"]: s += "{GREEN}loaded{NO_COLOR}" else: s += "{RED}not-loaded{NO_COLOR}" s += "\n" print_color(s[:-1]) @functools.lru_cache() def _create_xontrib_parser(): # parse command line args parser = argparse.ArgumentParser( prog="xontrib", description="Manages xonsh extensions" ) subp = parser.add_subparsers(title="action", dest="action") load = subp.add_parser("load", help="loads xontribs") load.add_argument( "-v", "--verbose", action="store_true", default=False, dest="verbose" ) load.add_argument("names", nargs="+", default=(), help="names of xontribs") lyst = subp.add_parser( "list", help=("list xontribs, whether they are " "installed, and loaded.") ) lyst.add_argument( "--json", action="store_true", default=False, help="reports results as json" ) lyst.add_argument("names", nargs="*", default=(), help="names of xontribs") return parser _MAIN_XONTRIB_ACTIONS = {"load": _load, "list": _list} @unthreadable def xontribs_main(args=None, stdin=None): """Alias that loads xontribs""" if not args or ( args[0] not in _MAIN_XONTRIB_ACTIONS and args[0] not in {"-h", "--help"} ): args.insert(0, "load") parser = _create_xontrib_parser() ns = parser.parse_args(args) if ns.action is None: # apply default action ns = parser.parse_args(["load"] + args) return _MAIN_XONTRIB_ACTIONS[ns.action](ns) ================================================ FILE: xonsh/xoreutils/__init__.py ================================================ # amalgamate # amalgamate end ================================================ FILE: xonsh/xoreutils/_which.py ================================================ #!/usr/bin/env python # Copyright (c) 2002-2007 ActiveState Software Inc. # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. # IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # # Author: # Trent Mick (TrentM@ActiveState.com) # Home: # http://trentm.com/projects/which/ import os import sys import stat import getopt import builtins import collections.abc as cabc r"""Find the full path to commands. which(command, path=None, verbose=0, exts=None) Return the full path to the first match of the given command on the path. whichall(command, path=None, verbose=0, exts=None) Return a list of full paths to all matches of the given command on the path. whichgen(command, path=None, verbose=0, exts=None) Return a generator which will yield full paths to all matches of the given command on the path. By default the PATH environment variable is searched (as well as, on Windows, the AppPaths key in the registry), but a specific 'path' list to search may be specified as well. On Windows, the PATHEXT environment variable is applied as appropriate. If "verbose" is true then a tuple of the form (, ) is returned for each match. The latter element is a textual description of where the match was found. For example: from PATH element 0 from HKLM\SOFTWARE\...\perl.exe """ _cmdlnUsage = """ Show the full path of commands. Usage: which [...] [...] Options: -h, --help Print this help and exit. -V, --version Print the version info and exit. -a, --all Print *all* matching paths. -v, --verbose Print out how matches were located and show near misses on stderr. -q, --quiet Just print out matches. I.e., do not print out near misses. -p , --path= An alternative path (list of directories) may be specified for searching. -e , --exts= Specify a list of extensions to consider instead of the usual list (';'-separate list, Windows only). Show the full path to the program that would be run for each given command name, if any. Which, like GNU's which, returns the number of failed arguments, or -1 when no was given. Near misses include duplicates, non-regular files and (on Un*x) files without executable access. """ __version_info__ = (1, 2, 0) __version__ = ".".join(map(str, __version_info__)) __all__ = ["which", "whichall", "whichgen", "WhichError"] class WhichError(Exception): pass # internal support stuff def _getRegisteredExecutable(exeName): """Windows allow application paths to be registered in the registry.""" registered = None if sys.platform.startswith("win"): if os.path.splitext(exeName)[1].lower() != ".exe": exeName += ".exe" try: import winreg as _winreg except ImportError: import _winreg try: key = "SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\App Paths\\" + exeName value = _winreg.QueryValue(_winreg.HKEY_LOCAL_MACHINE, key) registered = (value, "from HKLM\\" + key) except _winreg.error: pass if registered and not os.path.exists(registered[0]): registered = None return registered def _samefile(fname1, fname2): if sys.platform.startswith("win"): return os.path.normpath(os.path.normcase(fname1)) == os.path.normpath( os.path.normcase(fname2) ) else: return os.path.samefile(fname1, fname2) def _cull(potential, matches, verbose=0): """Cull inappropriate matches. Possible reasons: - a duplicate of a previous match - not a disk file - not executable (non-Windows) If 'potential' is approved it is returned and added to 'matches'. Otherwise, None is returned. """ for match in matches: # don't yield duplicates if _samefile(potential[0], match[0]): if verbose: sys.stderr.write("duplicate: %s (%s)\n" % potential) return None else: if not stat.S_ISREG(os.stat(potential[0]).st_mode): if verbose: sys.stderr.write("not a regular file: %s (%s)\n" % potential) elif sys.platform != "win32" and not os.access(potential[0], os.X_OK): if verbose: sys.stderr.write("no executable access: %s (%s)\n" % potential) else: matches.append(potential) return potential # module API def whichgen(command, path=None, verbose=0, exts=None): """Return a generator of full paths to the given command. "command" is a the name of the executable to search for. "path" is an optional alternate path list to search. The default it to use the PATH environment variable. "verbose", if true, will cause a 2-tuple to be returned for each match. The second element is a textual description of where the match was found. "exts" optionally allows one to specify a list of extensions to use instead of the standard list for this system. This can effectively be used as an optimization to, for example, avoid stat's of "foo.vbs" when searching for "foo" and you know it is not a VisualBasic script but ".vbs" is on PATHEXT. This option is only supported on Windows. This method returns a generator which yields tuples of the form (, ). """ matches = [] if path is None: usingGivenPath = 0 path = os.environ.get("PATH", "").split(os.pathsep) if sys.platform.startswith("win"): path.insert(0, os.curdir) # implied by Windows shell else: usingGivenPath = 1 # Windows has the concept of a list of extensions (PATHEXT env var). if sys.platform.startswith("win"): if exts is None: exts = builtins.__xonsh__.env["PATHEXT"] # If '.exe' is not in exts then obviously this is Win9x and # or a bogus PATHEXT, then use a reasonable default. for ext in exts: if ext.lower() == ".exe": break else: exts = [".COM", ".EXE", ".BAT", ".CMD"] elif not isinstance(exts, cabc.Sequence): raise TypeError("'exts' argument must be a sequence or None") else: if exts is not None: raise WhichError( "'exts' argument is not supported on " "platform '%s'" % sys.platform ) exts = [] # File name cannot have path separators because PATH lookup does not # work that way. if os.sep in command or os.altsep and os.altsep in command: if os.path.exists(command): match = _cull((command, "explicit path given"), matches, verbose) yield match else: for i in range(len(path)): dirName = path[i] # On windows the dirName *could* be quoted, drop the quotes if ( sys.platform.startswith("win") and len(dirName) >= 2 and dirName[0] == '"' and dirName[-1] == '"' ): dirName = dirName[1:-1] for ext in [""] + exts: absName = os.path.abspath( os.path.normpath(os.path.join(dirName, command + ext)) ) if os.path.isfile(absName): if usingGivenPath: fromWhere = "from given path element %d" % i elif not sys.platform.startswith("win"): fromWhere = "from PATH element %d" % i elif i == 0: fromWhere = "from current directory" else: fromWhere = "from PATH element %d" % (i - 1) match = _cull((absName, fromWhere), matches, verbose) if match: yield match match = _getRegisteredExecutable(command) if match is not None: match = _cull(match, matches, verbose) if match: yield match def which(command, path=None, verbose=0, exts=None): """Return the full path to the first match of the given command on the path. "command" is a the name of the executable to search for. "path" is an optional alternate path list to search. The default it to use the PATH environment variable. "verbose", if true, will cause a 2-tuple to be returned. The second element is a textual description of where the match was found. "exts" optionally allows one to specify a list of extensions to use instead of the standard list for this system. This can effectively be used as an optimization to, for example, avoid stat's of "foo.vbs" when searching for "foo" and you know it is not a VisualBasic script but ".vbs" is on PATHEXT. This option is only supported on Windows. If no match is found for the command, a WhichError is raised. """ try: absName, fromWhere = next(whichgen(command, path, verbose, exts)) except StopIteration: raise WhichError("Could not find '%s' on the path." % command) if verbose: return absName, fromWhere else: return absName def whichall(command, path=None, verbose=0, exts=None): """Return a list of full paths to all matches of the given command on the path. "command" is a the name of the executable to search for. "path" is an optional alternate path list to search. The default it to use the PATH environment variable. "verbose", if true, will cause a 2-tuple to be returned for each match. The second element is a textual description of where the match was found. "exts" optionally allows one to specify a list of extensions to use instead of the standard list for this system. This can effectively be used as an optimization to, for example, avoid stat's of "foo.vbs" when searching for "foo" and you know it is not a VisualBasic script but ".vbs" is on PATHEXT. This option is only supported on Windows. """ if verbose: return list(whichgen(command, path, verbose, exts)) else: return list(absName for absName, _ in whichgen(command, path, verbose, exts)) # mainline def main(argv): all = 0 verbose = 0 altpath = None exts = None try: optlist, args = getopt.getopt( argv[1:], "haVvqp:e:", ["help", "all", "version", "verbose", "quiet", "path=", "exts="], ) except getopt.GetoptErrsor as msg: sys.stderr.write("which: error: %s. Your invocation was: %s\n" % (msg, argv)) sys.stderr.write("Try 'which --help'.\n") return 1 for opt, optarg in optlist: if opt in ("-h", "--help"): print(_cmdlnUsage) return 0 elif opt in ("-V", "--version"): print("which %s" % __version__) return 0 elif opt in ("-a", "--all"): all = 1 elif opt in ("-v", "--verbose"): verbose = 1 elif opt in ("-q", "--quiet"): verbose = 0 elif opt in ("-p", "--path"): if optarg: altpath = optarg.split(os.pathsep) else: altpath = [] elif opt in ("-e", "--exts"): if optarg: exts = optarg.split(os.pathsep) else: exts = [] if len(args) == 0: return -1 failures = 0 for arg in args: # print "debug: search for %r" % arg nmatches = 0 for absName, fromWhere in whichgen( arg, path=altpath, verbose=verbose, exts=exts ): if verbose: print("%s (%s)" % (absName, fromWhere)) else: print(absName) nmatches += 1 if not all: break if not nmatches: failures += 1 return failures if __name__ == "__main__": sys.exit(main(sys.argv)) ================================================ FILE: xonsh/xoreutils/cat.py ================================================ """Implements a cat command for xonsh.""" import os import time import builtins import xonsh.proc as xproc from xonsh.xoreutils.util import arg_handler def _cat_line( f, sep, last_was_blank, line_count, opts, out, enc, enc_errors, read_size ): _r = r = f.readline(size=80) if isinstance(_r, str): _r = r = _r.encode(enc, enc_errors) if r == b"": last_was_blank, line_count, read_size, True if r.endswith(sep): _r = _r[: -len(sep)] this_one_blank = _r == b"" if last_was_blank and this_one_blank and opts["squeeze_blank"]: last_was_blank, line_count, read_size, False last_was_blank = this_one_blank if opts["number_all"] or (opts["number_nonblank"] and not this_one_blank): start = ("%6d " % line_count).encode(enc, enc_errors) _r = start + _r line_count += 1 if opts["show_ends"]: _r = _r + b"$" out.buffer.write(_r) out.flush() read_size += len(r) return last_was_blank, line_count, read_size, False def _cat_single_file(opts, fname, stdin, out, err, line_count=1): env = builtins.__xonsh__.env enc = env.get("XONSH_ENCODING") enc_errors = env.get("XONSH_ENCODING_ERRORS") read_size = 0 file_size = fobj = None if fname == "-": f = stdin elif os.path.isdir(fname): print("cat: {}: Is a directory.".format(fname), file=err) return True, line_count elif not os.path.exists(fname): print("cat: No such file or directory: {}".format(fname), file=err) return True, line_count else: file_size = os.stat(fname).st_size if file_size == 0: file_size = None fobj = open(fname, "rb") f = xproc.NonBlockingFDReader(fobj.fileno(), timeout=0.1) sep = os.linesep.encode(enc, enc_errors) last_was_blank = False while file_size is None or read_size < file_size: try: last_was_blank, line_count, read_size, endnow = _cat_line( f, sep, last_was_blank, line_count, opts, out, enc, enc_errors, read_size, ) if endnow: break if last_was_blank: time.sleep(1e-3) except KeyboardInterrupt: print("got except", flush=True, file=out) break except Exception as e: print("xonsh:", e, flush=True, file=out) pass if fobj is not None: fobj.close() return False, line_count def cat(args, stdin, stdout, stderr): """A cat command for xonsh.""" opts = _cat_parse_args(args) if opts is None: print(CAT_HELP_STR, file=stdout) return 0 line_count = 1 errors = False if len(args) == 0: args = ["-"] for i in args: o = _cat_single_file(opts, i, stdin, stdout, stderr, line_count) if o is None: return -1 _e, line_count = o errors = _e or errors return int(errors) def _cat_parse_args(args): out = { "number_nonblank": False, "number_all": False, "squeeze_blank": False, "show_ends": False, } if "--help" in args: return arg_handler(args, out, "-b", "number_nonblank", True, "--number-nonblank") arg_handler(args, out, "-n", "number_all", True, "--number") arg_handler(args, out, "-E", "show_ends", True, "--show-ends") arg_handler(args, out, "-s", "squeeze_blank", True, "--squeeze-blank") arg_handler(args, out, "-T", "show_tabs", True, "--show-tabs") return out CAT_HELP_STR = """This version of cat was written in Python for the xonsh project: http://xon.sh Based on cat from GNU coreutils: http://www.gnu.org/software/coreutils/ Usage: cat [OPTION]... [FILE]... Concatenate FILE(s), or standard input, to standard output. -b, --number-nonblank number nonempty output lines, overrides -n -E, --show-ends display $ at end of each line -n, --number number all output lines -s, --squeeze-blank suppress repeated empty output lines -T, --show-tabs display TAB characters as ^I -u (ignored) --help display this help and exit With no FILE, or when FILE is -, read standard input. Examples: cat f - g Output f's contents, then standard input, then g's contents. cat Copy standard input to standard output.""" # NOT IMPLEMENTED: # -A, --show-all equivalent to -vET # -e equivalent to -vE # -t equivalent to -vT # -v, --show-nonprinting use ^ and M- notation, except for LFD and TAB # --version output version information and exit""" def cat_main(args=None): import sys from xonsh.main import setup setup() args = sys.argv if args is None else args cat(args, sys.stdin, sys.stdout, sys.stderr) if __name__ == "__main__": cat_main() ================================================ FILE: xonsh/xoreutils/echo.py ================================================ """Implements a simple echo command for xonsh.""" def echo(args, stdin, stdout, stderr): """A simple echo command.""" opts = _echo_parse_args(args) if opts is None: return if opts["help"]: print(ECHO_HELP, file=stdout) return 0 ender = opts["end"] args = map(str, args) if opts["escapes"]: args = map(lambda x: x.encode().decode("unicode_escape"), args) print(*args, end=ender, file=stdout) def _echo_parse_args(args): out = {"escapes": False, "end": "\n", "help": False} if "-e" in args: args.remove("-e") out["escapes"] = True if "-E" in args: args.remove("-E") out["escapes"] = False if "-n" in args: args.remove("-n") out["end"] = "" if "-h" in args or "--help" in args: out["help"] = True return out ECHO_HELP = """Usage: echo [OPTIONS]... [STRING]... Echo the STRING(s) to standard output. -n do not include the trailing newline -e enable interpretation of backslash escapes -E disable interpretation of backslash escapes (default) -h --help display this message and exit This version of echo was written in Python for the xonsh project: http://xon.sh Based on echo from GNU coreutils: http://www.gnu.org/software/coreutils/""" ================================================ FILE: xonsh/xoreutils/pwd.py ================================================ """A pwd implementation for xonsh.""" import os def pwd(args, stdin, stdout, stderr): """A pwd implementation""" e = __xonsh__.env["PWD"] if "-h" in args or "--help" in args: print(PWD_HELP, file=stdout) return 0 if "-P" in args: e = os.path.realpath(e) print(e, file=stdout) return 0 PWD_HELP = """Usage: pwd [OPTION]... Print the full filename of the current working directory. -P, --physical avoid all symlinks --help display this help and exit This version of pwd was written in Python for the xonsh project: http://xon.sh Based on pwd from GNU coreutils: http://www.gnu.org/software/coreutils/""" # Not Implemented # -L, --logical use PWD from environment, even if it contains symlinks ================================================ FILE: xonsh/xoreutils/tee.py ================================================ """A tee implementation for xonsh.""" def tee(args, stdin, stdout, stderr): """A tee command for xonsh.""" mode = "w" if "-a" in args: args.remove("-a") mode = "a" if "--append" in args: args.remove("--append") mode = "a" if "--help" in args: print(TEE_HELP, file=stdout) return 0 if stdin is None: msg = "tee was not piped stdin, must have input stream to read from." print(msg, file=stderr) return 1 errors = False files = [] for i in args: if i == "-": files.append(stdout) else: try: files.append(open(i, mode)) except: print("tee: failed to open {}".format(i), file=stderr) errors = True files.append(stdout) while True: r = stdin.read(1024) if r == "": break for i in files: i.write(r) for i in files: if i != stdout: i.close() return int(errors) TEE_HELP = """This version of tee was written in Python for the xonsh project: http://xon.sh Based on tee from GNU coreutils: http://www.gnu.org/software/coreutils/ Usage: tee [OPTION]... [FILE]... Copy standard input to each FILE, and also to standard output. -a, --append append to the given FILEs, do not overwrite --help display this help and exit If a FILE is -, copy again to standard output.""" # NOT IMPLEMENTED: # -i, --ignore-interrupts ignore interrupt signals ================================================ FILE: xonsh/xoreutils/tty.py ================================================ """A tty implementation for xonsh""" import os import sys def tty(args, stdin, stdout, stderr): """A tty command for xonsh.""" if "--help" in args: print(TTY_HELP, file=stdout) return 0 silent = False for i in ("-s", "--silent", "--quiet"): if i in args: silent = True args.remove(i) if len(args) > 0: if not silent: for i in args: print("tty: Invalid option: {}".format(i), file=stderr) print("Try 'tty --help' for more information", file=stderr) return 2 try: fd = stdin.fileno() except: fd = sys.stdin.fileno() if not os.isatty(fd): if not silent: print("not a tty", file=stdout) return 1 if not silent: try: print(os.ttyname(fd), file=stdout) except: return 3 return 0 TTY_HELP = """Usage: tty [OPTION]... Print the file name of the terminal connected to standard input. -s, --silent, --quiet print nothing, only return an exit status --help display this help and exit This version of tty was written in Python for the xonsh project: http://xon.sh Based on tty from GNU coreutils: http://www.gnu.org/software/coreutils/""" ================================================ FILE: xonsh/xoreutils/uptime.py ================================================ """ Provides a cross-platform way to figure out the system uptime. Should work on damned near any operating system you can realistically expect to be asked to write Python code for. If this module is invoked as a stand-alone script, it will print the current uptime in a human-readable format, or display an error message if it can't, to standard output. This file was forked from the uptime project: https://github.com/Cairnarvon/uptime Copyright (c) 2012, Koen Crolla, All rights reserved. """ import os import sys import time import ctypes import struct import xonsh.platform as xp import xonsh.lazyimps as xlimps import xonsh.lazyasd as xl _BOOTTIME = None def _uptime_osx(): """Returns the uptime on mac / darwin.""" global _BOOTTIME bt = xlimps.macutils.sysctlbyname(b"kern.boottime", return_str=False) if len(bt) == 4: bt = struct.unpack_from("@hh", bt) elif len(bt) == 8: bt = struct.unpack_from("@ii", bt) elif len(bt) == 16: bt = struct.unpack_from("@qq", bt) else: raise ValueError("length of boot time not understood: " + repr(bt)) bt = bt[0] + bt[1] * 1e-6 if bt == 0.0: return None _BOOTTIME = bt return time.time() - bt def _uptime_linux(): """Returns uptime in seconds or None, on Linux.""" # With procfs try: with open("/proc/uptime", "r") as f: up = float(f.readline().split()[0]) return up except (IOError, ValueError): pass buf = ctypes.create_string_buffer(128) # 64 suffices on 32-bit, whatever. if xp.LIBC.sysinfo(buf) < 0: return None up = struct.unpack_from("@l", buf.raw)[0] if up < 0: up = None return up def _boottime_linux(): """A way to figure out the boot time directly on Linux.""" global _BOOTTIME try: with open("/proc/stat", "r") as f: for line in f: if line.startswith("btime"): _BOOTTIME = float(line.split()[1]) return _BOOTTIME except (IOError, IndexError): return None def _uptime_amiga(): """Returns uptime in seconds or None, on AmigaOS.""" global _BOOTTIME try: _BOOTTIME = os.stat("RAM:").st_ctime return time.time() - _BOOTTIME except (NameError, OSError): return None def _uptime_beos(): """Returns uptime in seconds on None, on BeOS/Haiku.""" if not hasattr(xp.LIBC, "system_time"): return None xp.LIBC.system_time.restype = ctypes.c_int64 return xp.LIBC.system_time() / 1000000. def _uptime_bsd(): """Returns uptime in seconds or None, on BSD (including OS X).""" global _BOOTTIME if not hasattr(xp.LIBC, "sysctlbyname"): # Not BSD. return None # Determine how much space we need for the response. sz = ctypes.c_uint(0) xp.LIBC.sysctlbyname("kern.boottime", None, ctypes.byref(sz), None, 0) if sz.value != struct.calcsize("@LL"): # Unexpected, let's give up. return None # For real now. buf = ctypes.create_string_buffer(sz.value) xp.LIBC.sysctlbyname("kern.boottime", buf, ctypes.byref(sz), None, 0) sec, usec = struct.unpack_from("@LL", buf.raw) # OS X disagrees what that second value is. if usec > 1000000: usec = 0. _BOOTTIME = sec + usec / 1000000. up = time.time() - _BOOTTIME if up < 0: up = None return up def _uptime_minix(): """Returns uptime in seconds or None, on MINIX.""" try: with open("/proc/uptime", "r") as f: up = float(f.read()) return up except (IOError, ValueError): return None def _uptime_plan9(): """Returns uptime in seconds or None, on Plan 9.""" # Apparently Plan 9 only has Python 2.2, which I'm not prepared to # support. Maybe some Linuxes implement /dev/time, though, someone was # talking about it somewhere. try: # The time file holds one 32-bit number representing the sec- # onds since start of epoch and three 64-bit numbers, repre- # senting nanoseconds since start of epoch, clock ticks, and # clock frequency. # -- cons(3) with open("/dev/time", "r") as f: s, ns, ct, cf = f.read().split() return float(ct) / float(cf) except (IOError, ValueError): return None def _uptime_solaris(): """Returns uptime in seconds or None, on Solaris.""" global _BOOTTIME try: kstat = ctypes.CDLL("libkstat.so") except (AttributeError, OSError): return None # kstat doesn't have uptime, but it does have boot time. # Unfortunately, getting at it isn't perfectly straightforward. # First, let's pretend to be kstat.h # Constant KSTAT_STRLEN = 31 # According to every kstat.h I could find. # Data structures class anon_union(ctypes.Union): # The ``value'' union in kstat_named_t actually has a bunch more # members, but we're only using it for boot_time, so we only need # the padding and the one we're actually using. _fields_ = [("c", ctypes.c_char * 16), ("time", ctypes.c_int)] class kstat_named_t(ctypes.Structure): _fields_ = [ ("name", ctypes.c_char * KSTAT_STRLEN), ("data_type", ctypes.c_char), ("value", anon_union), ] # Function signatures kstat.kstat_open.restype = ctypes.c_void_p kstat.kstat_lookup.restype = ctypes.c_void_p kstat.kstat_lookup.argtypes = [ ctypes.c_void_p, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ] kstat.kstat_read.restype = ctypes.c_int kstat.kstat_read.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p] kstat.kstat_data_lookup.restype = ctypes.POINTER(kstat_named_t) kstat.kstat_data_lookup.argtypes = [ctypes.c_void_p, ctypes.c_char_p] # Now, let's do something useful. # Initialise kstat control structure. kc = kstat.kstat_open() if not kc: return None # We're looking for unix:0:system_misc:boot_time. ksp = kstat.kstat_lookup(kc, "unix", 0, "system_misc") if ksp and kstat.kstat_read(kc, ksp, None) != -1: data = kstat.kstat_data_lookup(ksp, "boot_time") if data: _BOOTTIME = data.contents.value.time # Clean-up. kstat.kstat_close(kc) if _BOOTTIME is not None: return time.time() - _BOOTTIME return None def _uptime_syllable(): """Returns uptime in seconds or None, on Syllable.""" global _BOOTTIME try: _BOOTTIME = os.stat("/dev/pty/mst/pty0").st_mtime return time.time() - _BOOTTIME except (NameError, OSError): return None def _uptime_windows(): """ Returns uptime in seconds or None, on Windows. Warning: may return incorrect answers after 49.7 days on versions older than Vista. """ if hasattr(xp.LIBC, "GetTickCount64"): # Vista/Server 2008 or later. xp.LIBC.GetTickCount64.restype = ctypes.c_uint64 return xp.LIBC.GetTickCount64() / 1000. if hasattr(xp.LIBC, "GetTickCount"): # WinCE and Win2k or later; gives wrong answers after 49.7 days. xp.LIBC.GetTickCount.restype = ctypes.c_uint32 return xp.LIBC.GetTickCount() / 1000. return None @xl.lazyobject def _UPTIME_FUNCS(): return { "amiga": _uptime_amiga, "aros12": _uptime_amiga, "beos5": _uptime_beos, "cygwin": _uptime_linux, "darwin": _uptime_osx, "haiku1": _uptime_beos, "linux": _uptime_linux, "linux-armv71": _uptime_linux, "linux2": _uptime_linux, "minix3": _uptime_minix, "sunos5": _uptime_solaris, "syllable": _uptime_syllable, "win32": _uptime_windows, "wince": _uptime_windows, } def uptime(): """Returns uptime in seconds if even remotely possible, or None if not.""" if _BOOTTIME is not None: return time.time() - _BOOTTIME up = _UPTIME_FUNCS.get(sys.platform, _uptime_bsd)() if up is None: up = ( _uptime_bsd() or _uptime_plan9() or _uptime_linux() or _uptime_windows() or _uptime_solaris() or _uptime_beos() or _uptime_amiga() or _uptime_syllable() or _uptime_osx() ) return up def boottime(): """Returns boot time if remotely possible, or None if not.""" global _BOOTTIME if _BOOTTIME is None: up = uptime() if up is None: return None _BOOTTIME = time.time() - up return _BOOTTIME ================================================ FILE: xonsh/xoreutils/util.py ================================================ """Assorted utilities for xonsh core utils.""" def arg_handler(args, out, short, key, val, long=None): """A simple argument handler for xoreutils.""" if short in args: args.remove(short) if isinstance(key, (list, tuple)): for k in key: out[k] = val else: out[key] = val if long is not None and long in args: args.remove(long) if isinstance(key, (list, tuple)): for k in key: out[k] = val else: out[key] = val ================================================ FILE: xonsh/xoreutils/which.py ================================================ """Implements the which xoreutil.""" import os import argparse import builtins import functools from xonsh.xoreutils import _which import xonsh.platform as xp import xonsh.proc as xproc @functools.lru_cache() def _which_create_parser(): desc = "Parses arguments to which wrapper" parser = argparse.ArgumentParser("which", description=desc) parser.add_argument( "args", type=str, nargs="+", help="The executables or aliases to search for" ) parser.add_argument( "-a", "--all", action="store_true", dest="all", help="Show all matches in globals, xonsh.aliases, $PATH", ) parser.add_argument( "-s", "--skip-alias", action="store_true", help="Do not search inxonsh.aliases", dest="skip", ) parser.add_argument( "-V", "--version", action="version", version="{}".format(_which.__version__), help="Display the version of the python which module " "used by xonsh", ) parser.add_argument( "-v", "--verbose", action="store_true", dest="verbose", help="Print out how matches were located and show " "near misses on stderr", ) parser.add_argument( "-p", "--plain", action="store_true", dest="plain", help="Do not display alias expansions or location of " "where binaries are found. This is the " "default behavior, but the option can be used to " "override the --verbose option", ) parser.add_argument("--very-small-rocks", action=AWitchAWitch) if xp.ON_WINDOWS: parser.add_argument( "-e", "--exts", nargs="*", type=str, help="Specify a list of extensions to use instead " "of the standard list for this system. This can " "effectively be used as an optimization to, for " 'example, avoid stat\'s of "foo.vbs" when ' 'searching for "foo" and you know it is not a ' 'VisualBasic script but ".vbs" is on PATHEXT. ' "This option is only supported on Windows", dest="exts", ) return parser def print_global_object(arg, stdout): """Print the object.""" obj = builtins.__xonsh__.ctx.get(arg) print("global object of {}".format(type(obj)), file=stdout) def print_path(abs_name, from_where, stdout, verbose=False, captured=False): """Print the name and path of the command.""" if xp.ON_WINDOWS: # Use list dir to get correct case for the filename # i.e. windows is case insensitive but case preserving p, f = os.path.split(abs_name) f = next(s.name for s in xp.scandir(p) if s.name.lower() == f.lower()) abs_name = os.path.join(p, f) if builtins.__xonsh__.env.get("FORCE_POSIX_PATHS", False): abs_name.replace(os.sep, os.altsep) if verbose: print("{} ({})".format(abs_name, from_where), file=stdout) else: end = "" if captured else "\n" print(abs_name, end=end, file=stdout) def print_alias(arg, stdout, verbose=False): """Print the alias.""" if not verbose: if not callable(builtins.aliases[arg]): print(" ".join(builtins.aliases[arg]), file=stdout) else: print(arg, file=stdout) else: print("aliases['{}'] = {}".format(arg, builtins.aliases[arg]), file=stdout) if callable(builtins.aliases[arg]): builtins.__xonsh__.superhelp(builtins.aliases[arg]) def which(args, stdin=None, stdout=None, stderr=None, spec=None): """ Checks if each arguments is a xonsh aliases, then if it's an executable, then finally return an error code equal to the number of misses. If '-a' flag is passed, run both to return both `xonsh` match and `which` match. """ parser = _which_create_parser() if len(args) == 0: parser.print_usage(file=stderr) return -1 pargs = parser.parse_args(args) verbose = pargs.verbose or pargs.all if spec is not None: captured = spec.captured in xproc.STDOUT_CAPTURE_KINDS else: captured = False if pargs.plain: verbose = False if xp.ON_WINDOWS: if pargs.exts: exts = pargs.exts else: exts = builtins.__xonsh__.env["PATHEXT"] else: exts = None failures = [] for arg in pargs.args: nmatches = 0 if pargs.all and arg in builtins.__xonsh__.ctx: print_global_object(arg, stdout) nmatches += 1 if arg in builtins.aliases and not pargs.skip: print_alias(arg, stdout, verbose) nmatches += 1 if not pargs.all: continue # which.whichgen gives the nicest 'verbose' output if PATH is taken # from os.environ so we temporarily override it with # __xosnh_env__['PATH'] original_os_path = xp.os_environ["PATH"] xp.os_environ["PATH"] = builtins.__xonsh__.env.detype()["PATH"] matches = _which.whichgen(arg, exts=exts, verbose=verbose) for abs_name, from_where in matches: print_path(abs_name, from_where, stdout, verbose, captured) nmatches += 1 if not pargs.all: break xp.os_environ["PATH"] = original_os_path if not nmatches: failures.append(arg) if len(failures) == 0: return 0 else: print("{} not in ".format(", ".join(failures)), file=stderr, end="") if pargs.all: print("globals or ", file=stderr, end="") print("$PATH", file=stderr, end="") if not pargs.skip: print(" or xonsh.builtins.aliases", file=stderr, end="") print("", file=stderr, end="\n") return len(failures) class AWitchAWitch(argparse.Action): """The Ducstring, the mother of all ducs.""" SUPPRESS = "==SUPPRESS==" def __init__( self, option_strings, version=None, dest=SUPPRESS, default=SUPPRESS, **kwargs ): super().__init__( option_strings=option_strings, dest=dest, default=default, nargs=0, **kwargs ) def __call__(self, parser, namespace, values, option_string=None): import webbrowser webbrowser.open("https://github.com/xonsh/xonsh/commit/f49b400") parser.exit() ================================================ FILE: xonsh/xoreutils/yes.py ================================================ """An implementation of yes for xonsh.""" def yes(args, stdin, stdout, stderr): """A yes command.""" if "--help" in args: print(YES_HELP, file=stdout) return 0 to_print = ["y"] if len(args) == 0 else [str(i) for i in args] while True: print(*to_print, file=stdout) return 0 YES_HELP = """Usage: yes [STRING]... or: yes OPTION Repeatedly output a line with all specified STRING(s), or 'y'. --help display this help and exit This version of yes was written in Python for the xonsh project: http://xon.sh Based on yes from GNU coreutils: http://www.gnu.org/software/coreutils/"""